From 6b8d70ecb717b512d8c53da54f531517ee4f5580 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Fri, 29 Aug 2025 17:58:04 +0800 Subject: [PATCH 01/14] modify test indicator --- ACL_PyTorch/built-in/audio/whisper/README.md | 25 ++++---------------- ACL_PyTorch/built-in/audio/whisper/infer.py | 9 +++++++ 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/ACL_PyTorch/built-in/audio/whisper/README.md b/ACL_PyTorch/built-in/audio/whisper/README.md index 0e2b164d03..b5187cd746 100644 --- a/ACL_PyTorch/built-in/audio/whisper/README.md +++ b/ACL_PyTorch/built-in/audio/whisper/README.md @@ -95,27 +95,10 @@ infer.py推理参数： warmup结束之后，开始推理librispeech_asr_dummy数据集，推理过程中会打屏输出E2E性能，推理结束后会输出WER精度得分。 -**如果你想推理过程中打印encode和decode的耗时，你可以执行以下命令：** -```SHELL -# 1. 找到当前的环境路径（简称${location}），Location后面的那一串就是当前环境路径 -pip show openai-whisper | grep Location -# 2. 记录当前whisper库decoding.py的文件路径 -${decoding_path} = ${location}/whisper/decoding.py -# 3. 执行patch文件 -patch -p1 < whisper_decoding.patch -# 可能会提示你 -# cant find file to patch at input line 3 -# ... -# File to patch: -# 这时候需要你手动指定文件路径，输入之前得到的 -${decoding_path} -# 按回车，提示 patching file ${decoding_path} 即成功 -``` - ## 性能数据 在librispeech_asr_dummy/clean数据集上的性能如下： - | 模型 | 芯片 | 平均encode | 平均decode |平均E2E | - |---------|------------|----------|-----------------|---------| - | whisper | 800I A2 | 0.90ms | 3.25ms | 67.32ms | - 注：平均decode 指在decode阶段，生成单个token的平均耗时。 \ No newline at end of file + | 模型 | 芯片 | RTF | + |---------|------------|----------| + | whisper | 800I A2 | 0.0236 | + 注：RTF表示转录一段音频所需的时间与音频实际长度的比值，多次运行取平均 \ No newline at end of file diff --git a/ACL_PyTorch/built-in/audio/whisper/infer.py b/ACL_PyTorch/built-in/audio/whisper/infer.py index ba5da6fa13..0617aa7eb0 100644 --- a/ACL_PyTorch/built-in/audio/whisper/infer.py +++ b/ACL_PyTorch/built-in/audio/whisper/infer.py @@ -17,6 +17,7 @@ import jiwer import numpy as np import pandas as pd from datasets import load_dataset +import librosa import torch from torch import nn, Tensor @@ -279,6 +280,12 @@ if __name__ == '__main__': npu_backend = tng.get_npu_backend(compiler_config=config) dataset = LibriSpeechDataset(wsp_args.speech_path, device=device) + audios = load_dataset(wsp_args.speech_path, split="validation") + duration_seconds = 0 + for audio in audios: + y, audio_sr = audio["audio"]["array"], audio["audio"]["sampling_rate"] + duration_seconds += librosa.get_duration(y=y, sr=audio_sr) + loader = torch.utils.data.DataLoader(dataset, batch_size=wsp_args.batch_size) options = whisper.DecodingOptions(language='en', without_timestamps=True, fp16=True) @@ -300,5 +307,7 @@ if __name__ == '__main__': print("{}/{} - {}".format(_step, wsp_args.warmup, result[bs].text)) print("LibriSpeech infer, English to English TRANSCRIBE ...") + start_time = time.time() p_wer = libri_speech_infer(wsp_model, options, loader) + print(f"RTF: {(time.time()-start_time)/duration_seconds:.4f}") print(f"LibriSpeech infer WER score = {p_wer * 100:.2f} %") -- Gitee From eb280416a48b54c90a7623f0ed2cc2820ad597c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Fri, 29 Aug 2025 18:19:45 +0800 Subject: [PATCH 02/14] modify test indicator --- ACL_PyTorch/built-in/audio/whisper/README.md | 25 ++++---------------- ACL_PyTorch/built-in/audio/whisper/infer.py | 9 +++++++ 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/ACL_PyTorch/built-in/audio/whisper/README.md b/ACL_PyTorch/built-in/audio/whisper/README.md index 0e2b164d03..135d9f87b9 100644 --- a/ACL_PyTorch/built-in/audio/whisper/README.md +++ b/ACL_PyTorch/built-in/audio/whisper/README.md @@ -95,27 +95,10 @@ infer.py推理参数： warmup结束之后，开始推理librispeech_asr_dummy数据集，推理过程中会打屏输出E2E性能，推理结束后会输出WER精度得分。 -**如果你想推理过程中打印encode和decode的耗时，你可以执行以下命令：** -```SHELL -# 1. 找到当前的环境路径（简称${location}），Location后面的那一串就是当前环境路径 -pip show openai-whisper | grep Location -# 2. 记录当前whisper库decoding.py的文件路径 -${decoding_path} = ${location}/whisper/decoding.py -# 3. 执行patch文件 -patch -p1 < whisper_decoding.patch -# 可能会提示你 -# cant find file to patch at input line 3 -# ... -# File to patch: -# 这时候需要你手动指定文件路径，输入之前得到的 -${decoding_path} -# 按回车，提示 patching file ${decoding_path} 即成功 -``` - ## 性能数据 在librispeech_asr_dummy/clean数据集上的性能如下： - | 模型 | 芯片 | 平均encode | 平均decode |平均E2E | - |---------|------------|----------|-----------------|---------| - | whisper | 800I A2 | 0.90ms | 3.25ms | 67.32ms | - 注：平均decode 指在decode阶段，生成单个token的平均耗时。 \ No newline at end of file + | 模型 | 芯片 | 转录倍率QPS | + |---------|------------|----------| + | whisper | 800I A2 | 42.34 | + 注：QPS表示音频实际长度与转录音频所需的时间的比值，多次运行取平均 \ No newline at end of file diff --git a/ACL_PyTorch/built-in/audio/whisper/infer.py b/ACL_PyTorch/built-in/audio/whisper/infer.py index ba5da6fa13..066a46d3fd 100644 --- a/ACL_PyTorch/built-in/audio/whisper/infer.py +++ b/ACL_PyTorch/built-in/audio/whisper/infer.py @@ -17,6 +17,7 @@ import jiwer import numpy as np import pandas as pd from datasets import load_dataset +import librosa import torch from torch import nn, Tensor @@ -279,6 +280,12 @@ if __name__ == '__main__': npu_backend = tng.get_npu_backend(compiler_config=config) dataset = LibriSpeechDataset(wsp_args.speech_path, device=device) + audios = load_dataset(wsp_args.speech_path, split="validation") + duration_seconds = 0 + for audio in audios: + y, audio_sr = audio["audio"]["array"], audio["audio"]["sampling_rate"] + duration_seconds += librosa.get_duration(y=y, sr=audio_sr) + loader = torch.utils.data.DataLoader(dataset, batch_size=wsp_args.batch_size) options = whisper.DecodingOptions(language='en', without_timestamps=True, fp16=True) @@ -300,5 +307,7 @@ if __name__ == '__main__': print("{}/{} - {}".format(_step, wsp_args.warmup, result[bs].text)) print("LibriSpeech infer, English to English TRANSCRIBE ...") + start_time = time.time() p_wer = libri_speech_infer(wsp_model, options, loader) + print(f"QPS: {duration_seconds/(time.time()-start_time):.2f}") print(f"LibriSpeech infer WER score = {p_wer * 100:.2f} %") -- Gitee From 46c4501320deb7b846c081b92b206dab773b7453 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Fri, 29 Aug 2025 18:38:01 +0800 Subject: [PATCH 03/14] modify test indicator --- ACL_PyTorch/built-in/audio/whisper/README.md | 2 +- .../audio/whisper/whisper_decoding.patch | 34 ------------------- 2 files changed, 1 insertion(+), 35 deletions(-) delete mode 100644 ACL_PyTorch/built-in/audio/whisper/whisper_decoding.patch diff --git a/ACL_PyTorch/built-in/audio/whisper/README.md b/ACL_PyTorch/built-in/audio/whisper/README.md index 4ed36a7f30..548db00008 100644 --- a/ACL_PyTorch/built-in/audio/whisper/README.md +++ b/ACL_PyTorch/built-in/audio/whisper/README.md @@ -100,5 +100,5 @@ warmup结束之后，开始推理librispeech_asr_dummy数据集，推理过程 | 模型 | 芯片 | 转录倍率QPS | |---------|------------|----------| - | whisper | 800I A2 | 42.34 | + | whisper | 800I A2 64G | 42.34 | 注：QPS表示音频实际长度与转录音频所需的时间的比值，多次运行取平均 diff --git a/ACL_PyTorch/built-in/audio/whisper/whisper_decoding.patch b/ACL_PyTorch/built-in/audio/whisper/whisper_decoding.patch deleted file mode 100644 index 871e972c2f..0000000000 --- a/ACL_PyTorch/built-in/audio/whisper/whisper_decoding.patch +++ /dev/null @@ -1,34 +0,0 @@ -+++ decoding.py -@@ -652,7 +652,10 @@ - # encoded audio features are given; skip audio encoding - audio_features = mel - else: -+ import time -+ time1 = time.time() - audio_features = self.model.encoder(mel) -+ print(f"encode time = {(time.time() - time1) * 1000:.2f} ms") - - if audio_features.dtype != ( - torch.float16 if self.options.fp16 else torch.float32 -@@ -683,6 +686,8 @@ - no_speech_probs = [np.nan] * n_batch - - try: -+ import time -+ time1 = time.time() - for i in range(self.sample_len): - logits = self.inference.logits(tokens, audio_features) - -@@ -703,6 +708,8 @@ - tokens, completed = self.decoder.update(tokens, logits, sum_logprobs) - - if completed or tokens.shape[-1] > self.n_ctx: -+ avg_time = (time.time() - time1) / i * 1000 -+ print(f"avg decode time = {avg_time:.2f} ms") - break - finally: - self.inference.cleanup_caching() -@@ -824,3 +831,4 @@ - result = DecodingTask(model, options).run(mel) - - return result[0] if single else result -- Gitee From ca522f30b19251671ecc9f5362a847cffcf5abd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Wed, 10 Sep 2025 10:11:27 +0800 Subject: [PATCH 04/14] add mineru torchair adaptation --- ACL_PyTorch/built-in/ocr/MinerU/README.md | 194 ++++++++++ .../built-in/ocr/MinerU/doclayout_yolo.patch | 50 +++ ACL_PyTorch/built-in/ocr/MinerU/infer.py | 346 ++++++++++++++++++ .../built-in/ocr/MinerU/requirements.txt | 34 ++ .../built-in/ocr/MinerU/ultralytics.patch | 77 ++++ 5 files changed, 701 insertions(+) create mode 100644 ACL_PyTorch/built-in/ocr/MinerU/README.md create mode 100644 ACL_PyTorch/built-in/ocr/MinerU/doclayout_yolo.patch create mode 100644 ACL_PyTorch/built-in/ocr/MinerU/infer.py create mode 100644 ACL_PyTorch/built-in/ocr/MinerU/requirements.txt create mode 100644 ACL_PyTorch/built-in/ocr/MinerU/ultralytics.patch diff --git a/ACL_PyTorch/built-in/ocr/MinerU/README.md b/ACL_PyTorch/built-in/ocr/MinerU/README.md new file mode 100644 index 0000000000..19f28fbb03 --- /dev/null +++ b/ACL_PyTorch/built-in/ocr/MinerU/README.md @@ -0,0 +1,194 @@ +# MinerU(TorchAir)-推理指导 + +- [MinerU(TorchAir)-推理指导](#MinerU(TorchAir)-推理指导) +- [概述](#概述) +- [推理环境准备](#推理环境准备) +- [快速上手](#快速上手) + - [获取源码](#获取源码) + - [获取权重](#获取权重) + - [获取数据集](#获取数据集) + - [执行推理](#执行推理) + - [精度测试](#精度测试) + +****** + +# 概述 +MinerU是由上海人工智能实验室OpenDataLab团队开发的开源文档解析工具，致力于解决大模型（LLM）训练和RAG（检索增强生成）应用中高质量结构化数据的提取难题。其核心价值在于将复杂文档（如PDF、网页、电子书）转换为机器可读的Markdown、JSON格式，同时保留原始文档的语义逻辑与多模态元素。 + +- 版本说明： + + ``` + url=https://github.com/opendatalab/MinerU.git + commit_id=de41fa58590263e43b783fe224b6d07cae290a33 + model_name=MinerU + ``` + +# 推理环境准备 + +- 该模型需要以下插件与驱动 + **表 1** 版本配套表 + + | 配套 | 版本 | 环境准备指导 | + | ------------------------------------------------------- | ----------- | --------------------------------------------------------------------------------------------- | + | 固件与驱动 | 25.0.RC1 | [Pytorch框架推理环境准备](https://www.hiascend.com/document/detail/zh/ModelZoo/pytorchframework/pies) | + | CANN | 8.3.0 | - | + | Python | 3.11 | - | + | PyTorch | 2.6.0 | - | + | Ascend Extension PyTorch | 2.6.0 | - | + | 说明：Atlas 800I A2/Atlas 300I Pro 推理卡请以CANN版本选择实际固件与驱动版本。 | \ | \ | + +# 快速上手 + +## 获取源码 + +1. 获取`Pytorch`源码 + + ``` + git clone https://github.com/opendatalab/MinerU.git + cd MinerU + git reset --hard de41fa58590263e43b783fe224b6d07cae290a33 + pip3 install -e . + cd .. + ``` + +2. 安装依赖 + + ``` + pip3 install -r requirements.txt + ### 此外，还需安装 Torchvision Adapter + git clone https://gitee.com/ascend/vision.git vision_npu + cd vision_npu + git checkout v0.21.0-7.1.0 + pip3 install -r requirement.txt + source /usr/local/Ascend/ascend-toolkit/set_env.sh # Default path, change it if needed. + python setup.py bdist_wheel + cd dist + pip install torchvision_npu-0.21.0+git22ca6b2-cp311-cp311-linux_aarch64.whl + cd ../../ + ``` + + +3. 修改第三方库 +进入第三方库安装路径，默认为`source_path = /usr/local/lib/python3.11/site-packages`,通过工作目录`workdir`(自定义)中的`ultralytics.patch`和`doclayout_yolo.patch`进行修改 + ``` + source_path=/usr/local/lib/python3.11/site-packages + cd ${source_path}/ultralytics + patch -p2 < ${workdir}/ultralytics.patch + cd ${source_path}/doclayout_yolo + patch -p2 < ${workdir}/doclayout_yolo.patch + ``` +修改完成后回到工作目录`workdir` + +## 获取权重 + +运行以下指令，下载权重文件[Model weights](https://www.modelscope.cn/models/OpenDataLab/PDF-Extract-Kit-1.0/summary)，默认存放为`/root/.cache/modelscope/hub/models/OpenDataLab/PDF-Extract-Kit-1___0` + +``` +mineru-models-download --source modelscope --model_type pipeline +``` +下载完成后，默认在根目录生成`mineru.json`文件，移动数据集时，需修改`/root/mineru.json`文件中"models-dir": "pipeline"为修改后权重存放路径 + +权重目录大致结构为： +```text +📁 models +├── 📁 Layout +│   └── 📁 YOLO +│   └── doclayout_yolo_docstructbench_imgsz1280_2501.pt +├── 📁 MFD +│   └── 📁 YOLO +│   └── yolo_v8_ft.pt +├── 📁 MFR +│   └── 📁 unimernet_hf_small_2503 +│   ├── model.safetensors +│   ├── …… +│   └── tokenizer_config.json +├── 📁 OCR +│   └── 📁 paddleocr_torch +│   ├── Multilingual_PP-OCRv3_det_infer.pth +│   ├── arabic_PP-OCRv3_rec_infer.pth +│   ├── …… +│   ├── …… +│   └── th_PP-OCRv5_rec_infer.pth +├── 📁 ReadingOrder +│   └── 📁 layout_reader +│   ├── config.json +│   └── model.safetensors +└── 📁 TabRec + └── 📁 SlanetPlus + └── slanet-plus.onnx +``` + + +## 获取数据集 + +创建数据集目录`OmniDocBench_dataset`，下载多样性文档解析评测集`OmniDocBench`数据集的[pdfs和标注](https://opendatalab.com/OpenDataLab/OmniDocBench)，解压并放置在`OmniDocBench_dataset`目录下 +文件目录格式大致如下： + ``` + 📁 workdir + ├── infer.py + ├── …… + ├── 📁 MinerU + └── 📁 OmniDocBench_dataset +   ├── OmniDocBench.json +   └── 📁 pdfs + └── ***.pdf + ``` + +## 执行推理 + +运行推理脚本infer.py + +``` +python3 infer.py --data_path=OmniDocBench_dataset --model_source=local +``` + +- 参数说明 + - data_path: 数据集路径 + - model_source: 模型源类型，local表示使用本地文件，modelscope/huggingface表示在线拉取权重 + +推理执行完成后，解析结果存放于`OmniDocBench_dataset/output/`目录，结果除了输出主要的 markdown 文件外，还会生成多个辅助文件用于调试、质检和进一步处理。 + +## 精度测试 + +使用`OmniDocBench`数据集配套评测代码测试精度。 + +1. 推理结果整理 +将解析结果文件夹中的markdown文件整理放置于同一目录，本例将所有markdown文件存放于OmniDocBench_dataset目录下的results_md文件夹 + ``` + cp OmniDocBench_dataset/output/*/auto/*.md OmniDocBench_dataset/results_md/ + ``` + +2. 获取测评源码并构建环境 + + ``` + git clone https://github.com/opendatalab/OmniDocBench.git + cd OmniDocBench + conda create -n omnidocbench python=3.10 + conda activate omnidocbench + pip install -r requirements.txt + ``` + +3. 测评配置修改 +修改`OmniDocBench`测评代码中的config文件，具体来说，我们使用端到端测评配置，修改configs/end2end.yaml文件中的ground_truth的data_path为下载的OmniDocBench.json路径，修改prediction的data_path中提供整理的推理结果的文件夹路径，如下： + ``` + # -----以下是需要修改的部分 ----- + dataset: + dataset_name: end2end_dataset + ground_truth: + data_path: ../OmniDocBench_dataset/OmniDocBench.json + prediction: + data_path: ../OmniDocBench_dataset/result_md + ``` + +4. 精度测量结果 +配置好config文件后，只需要将config文件作为参数传入，运行以下代码即可进行评测： + ``` + python pdf_validation.py --config ./configs/end2end.yaml + ``` + + 在`OmniDocBench`数据集上的精度为： + |模型|芯片|overall_EN|overall_CH| + |------|------|------|------| + |MinerU|300I DUO|0.1588|0.2527| + |MinerU|800I A2 64G|0.1580|0.2510| + diff --git a/ACL_PyTorch/built-in/ocr/MinerU/doclayout_yolo.patch b/ACL_PyTorch/built-in/ocr/MinerU/doclayout_yolo.patch new file mode 100644 index 0000000000..7cf22c0b32 --- /dev/null +++ b/ACL_PyTorch/built-in/ocr/MinerU/doclayout_yolo.patch @@ -0,0 +1,50 @@ +diff -ruN doclayout_yolo-0.0.4/doclayout_yolo/engine/predictor.py doclayout_yolo-0.0.4_fix/doclayout_yolo/engine/predictor.py +--- doclayout_yolo-0.0.4/doclayout_yolo/engine/predictor.py 2025-02-11 15:49:31.000000000 +0800 ++++ doclayout_yolo-0.0.4_fix/doclayout_yolo/engine/predictor.py 2025-09-09 16:05:20.011737230 +0800 +@@ -152,7 +152,8 @@ + (list): A list of transformed images. + """ + same_shapes = len({x.shape for x in im}) == 1 +- letterbox = LetterBox(self.imgsz, auto=same_shapes and self.model.pt, stride=self.model.stride) ++ letterbox = LetterBox(self.imgsz, auto=False, stride=self.model.stride) ++ # letterbox = LetterBox(self.imgsz, auto=same_shapes and self.model.pt, stride=self.model.stride) + return [letterbox(image=x) for x in im] + + def postprocess(self, preds, img, orig_imgs): +@@ -225,7 +226,8 @@ + + # Warmup model + if not self.done_warmup: +- self.model.warmup(imgsz=(1 if self.model.pt or self.model.triton else self.dataset.bs, 3, *self.imgsz)) ++ # self.model.warmup(imgsz=(1 if self.model.pt or self.model.triton else self.dataset.bs, 3, *self.imgsz)) ++ self.model.warmup(imgsz=(self.dataset.bs, 3, *self.imgsz)) + self.done_warmup = True + + self.seen, self.windows, self.batch = 0, [], None + +diff -ruN doclayout_yolo-0.0.4/doclayout_yolo/nn/modules/block.py doclayout_yolo-0.0.4_fix/doclayout_yolo/nn/modules/block.py +--- doclayout_yolo-0.0.4/doclayout_yolo/nn/modules/block.py 2025-02-11 15:49:31.000000000 +0800 ++++ doclayout_yolo-0.0.4_fix/doclayout_yolo/nn/modules/block.py 2025-09-09 16:05:20.019737230 +0800 +@@ -230,7 +230,9 @@ + def forward(self, x): + """Forward pass through C2f layer.""" + y = list(self.cv1(x).chunk(2, 1)) +- y.extend(m(y[-1]) for m in self.m) ++ # y.extend(m(y[-1]) for m in self.m) ++ for m in self.m: ++ y.append(m(y[-1])) + return self.cv2(torch.cat(y, 1)) + + def forward_split(self, x): + +diff -ruN doclayout_yolo-0.0.4/doclayout_yolo/utils/tal.py doclayout_yolo-0.0.4_fix/doclayout_yolo/utils/tal.py +--- doclayout_yolo-0.0.4/doclayout_yolo/utils/tal.py 2025-02-11 15:49:31.000000000 +0800 ++++ doclayout_yolo-0.0.4_fix/doclayout_yolo/utils/tal.py 2025-09-09 16:05:20.023737230 +0800 +@@ -328,7 +328,8 @@ + sy = torch.arange(end=h, device=device, dtype=dtype) + grid_cell_offset # shift y + sy, sx = torch.meshgrid(sy, sx, indexing="ij") if TORCH_1_10 else torch.meshgrid(sy, sx) + anchor_points.append(torch.stack((sx, sy), -1).view(-1, 2)) +- stride_tensor.append(torch.full((h * w, 1), stride, dtype=dtype, device=device)) ++ # stride_tensor.append(torch.full((h * w, 1), stride, dtype=dtype, device=device)) ++ stride_tensor.append(torch.ones((h * w, 1), dtype=dtype, device=device)*stride) + return torch.cat(anchor_points), torch.cat(stride_tensor) \ No newline at end of file diff --git a/ACL_PyTorch/built-in/ocr/MinerU/infer.py b/ACL_PyTorch/built-in/ocr/MinerU/infer.py new file mode 100644 index 0000000000..933917f639 --- /dev/null +++ b/ACL_PyTorch/built-in/ocr/MinerU/infer.py @@ -0,0 +1,346 @@ +# Copyright (c) Opendatalab. All rights reserved. +import os +from pathlib import Path +import inspect +import argparse +from loguru import logger +import math +from typing import Optional, Tuple, Union +import pypdfium2 as pdfium +import time + +import torch +import torch_npu +import torch.nn as nn +import torchvision +import torchvision_npu +import torchair as tng +from torchair.configs.compiler_config import CompilerConfig + +# import logging +# from torchair import logger +# logger.setLevel(logging.DEBUG) + +from mineru.backend.pipeline.model_list import AtomicModel +from mineru.model.mfr.unimernet.unimernet_hf.unimer_swin.modeling_unimer_swin import UnimerSwinSelfAttention +from mineru.backend.pipeline.model_init import ( + AtomModelSingleton, + table_model_init, + mfd_model_init, + mfr_model_init, + doclayout_yolo_model_init, + ocr_model_init, + ) +from mineru.utils.model_utils import get_vram +from mineru.backend.pipeline.batch_analyze import ( + YOLO_LAYOUT_BASE_BATCH_SIZE, + MFD_BASE_BATCH_SIZE, + MFR_BASE_BATCH_SIZE, + ) + +from transformers.generation.utils import GenerationMixin +from demo.demo import parse_doc + + +def parse_args(): + parser = argparse.ArgumentParser("MinerU infer") + parser.add_argument("--model_source", type=str, default="local", help="model checkpoint source") + parser.add_argument("--data_path", type=str, default="/home/z00939677/OmniDocBench_dataset") + parser.add_argument("--warmup", type=int, default=2, help="Warm up times") + parser.add_argument("--warmup_data_path", type=str, default="../OmniDocBench_dataset/pdfs/jiaocai_71434495.pdf_0.pdf") + args = parser.parse_args() + return args + +def atom_model_init_compile(model_name: str, **kwargs): + atom_model = None + if model_name == AtomicModel.Layout: + atom_model = doclayout_yolo_model_init( + kwargs.get('doclayout_yolo_weights'), + kwargs.get('device') + ) + atom_model.model.model = compile_model(atom_model.model.model, False, True) + npu_input = torch.zeros((batch_candidate[AtomicModel.Layout][0], 3, atom_model.imgsz, atom_model.imgsz)) + tng.inference.set_dim_gears(npu_input, {0: batch_candidate[AtomicModel.Layout]}) + + elif model_name == AtomicModel.MFD: + atom_model = mfd_model_init( + kwargs.get('mfd_weights'), + kwargs.get('device') + ) + atom_model.model.model = compile_model(atom_model.model.model, False, True) + npu_input = torch.zeros((batch_candidate[AtomicModel.MFD][0], 3, atom_model.imgsz, atom_model.imgsz)) + tng.inference.set_dim_gears(npu_input, {0: batch_candidate[AtomicModel.MFD]}) + + elif model_name == AtomicModel.MFR: + atom_model = mfr_model_init( + kwargs.get('mfr_weight_dir'), + kwargs.get('device') + ) + + modify_mfr_model(atom_model.model) + + print(atom_model.model.encoder.__class__) + + atom_model.model.encoder = compile_model(atom_model.model.encoder, False, True) + # npu_input = torch.zeros((batch_candidate[AtomicModel.MFR], 3, *atom_model.model.transform.input_size), dtype=torch.half, device=atom_model.device) + # tng.inference.set_dim_gears(npu_input, {0: batch_candidate[AtomicModel.MFR]}) + atom_model.model.decoder = compile_model(atom_model.model.decoder, True, True) + + elif model_name == AtomicModel.OCR: + atom_model = ocr_model_init( + kwargs.get('det_db_box_thresh'), + kwargs.get('lang'), + kwargs.get('det_limit_side_len'), + ) + + elif model_name == AtomicModel.Table: + atom_model = table_model_init( + kwargs.get('lang'), + ) + + else: + logger.error('model name not allow') + exit(1) + + if atom_model is None: + logger.error('model init failed') + exit(1) + else: + return atom_model + + + +def rewrite_mfr_encoder_multi_head_attention_forward(model): + wq = model.query.weight + wk = model.key.weight + wv = model.value.weight + model.qkv = nn.Linear(in_features=wk.shape[1], out_features= wq.shape[0] + wk.shape[0] + wv.shape[0]) + model.qkv.weight = nn.Parameter(torch.concat([wq, wk, wv], dim=0), requires_grad=False) + wq_bias = model.query.bias if model.query.bias is not None else torch.zeros(wq.shape[0]) + wk_bias = model.key.bias if model.key.bias is not None else torch.zeros(wk.shape[0]) + wv_bias = model.key.bias if model.value.bias is not None else torch.zeros(wv.shape[0]) + model.qkv.bias = nn.Parameter(torch.concat([wq_bias, wk_bias, wv_bias], dim=0), requires_grad=False) + + def attn_forward( + self, + hidden_states: torch.Tensor, + attention_mask: Optional[torch.FloatTensor] = None, + head_mask: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = False + ) -> Tuple[torch.Tensor]: + + # """融合qk为大矩阵，由于加入相对位置编码，PFA接口用不了，暂时只修改矩阵乘法""" + batch_size, dim, num_channels = hidden_states.shape + qkv = model.qkv(hidden_states) + q, k, v = qkv.chunk(3, dim=-1) + + query_layer = q.view(*q.shape[:2], self.num_attention_heads, -1).permute(0, 2, 1, 3) + key_layer = k.view(*k.shape[:2], self.num_attention_heads, -1).permute(0, 2, 1, 3) + value_layer = v.view(*v.shape[:2], self.num_attention_heads, -1).permute(0, 2, 1, 3) + + + relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)] + relative_position_bias = relative_position_bias.view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1 + ) + relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() + + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + attention_scores = attention_scores / math.sqrt(self.attention_head_size) + attention_scores = attention_scores + relative_position_bias.unsqueeze(0) + + if attention_mask is not None: + # Apply the attention mask is (precomputed for all layers in UnimerSwinModel forward() function) + mask_shape = attention_mask.shape[0] + attention_scores = attention_scores.view( + batch_size // mask_shape, mask_shape, self.num_attention_heads, dim, dim + ) + attention_scores = attention_scores + attention_mask.unsqueeze(1).unsqueeze(0) + attention_scores = attention_scores.view(-1, self.num_attention_heads, dim, dim) + + # Normalize the attention scores to probabilities. + attention_probs = nn.functional.softmax(attention_scores, dim=-1) + attention_probs = self.dropout(attention_probs) + + # Mask heads if we want to + if head_mask is not None: + attention_probs = attention_probs * head_mask + + context_layer = torch.matmul(attention_probs, value_layer) + context_layer = context_layer.permute(0, 2, 1, 3).contiguous() + new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) + context_layer = context_layer.view(new_context_layer_shape) + + outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) + return outputs + model.forward = attn_forward.__get__(model) + + +def modify_mfr_model(model): + # 修改encoder的attention forward + for name, module in model.encoder.named_modules(): + if isinstance(module, UnimerSwinSelfAttention): + rewrite_mfr_encoder_multi_head_attention_forward(module) + rewrite_mfr_encoder_forward() + +def compile_model(model, dynamic, fullgraph): + config = CompilerConfig() + config.experimental_config.frozen_parameter = True + config.experimental_config.tiling_schedule_optimize = True + npu_backend = tng.get_npu_backend(compiler_config=config) + compiled_model = torch.compile(model, dynamic=dynamic, fullgraph=fullgraph, backend=npu_backend) + # tng.use_internal_format_weight(compiled_model) + return compiled_model + + +def rewrite_model_init(): + def _patched_getmodel(self, atom_model_name: str, **kwargs): + lang = kwargs.get('lang', None) + table_model_name = kwargs.get('table_model_name', None) + + if atom_model_name in [AtomicModel.OCR]: + key = (atom_model_name, lang) + elif atom_model_name in [AtomicModel.Table]: + key = (atom_model_name, table_model_name, lang) + else: + key = atom_model_name + + if key not in self._models: + self._models[key] = atom_model_init_compile(model_name=atom_model_name, **kwargs) + return self._models[key] + AtomModelSingleton.get_atom_model = _patched_getmodel + +def rewrite_mfr_encoder_forward(): + def _patched_prepare_encoder_decoder_kwargs_for_generation(self, + inputs_tensor: torch.Tensor, + model_kwargs, + model_input_name: Optional[str], + generation_config, + ): + # 1. get encoder + encoder = self.get_encoder() + + # 2. Prepare encoder args and encoder kwargs from model kwargs and generation config. + irrelevant_prefix = ["decoder_", "cross_attn", "use_cache"] + encoder_kwargs = { + argument: value + for argument, value in model_kwargs.items() + if not any(argument.startswith(p) for p in irrelevant_prefix) + } + encoder_signature = set(inspect.signature(encoder.forward).parameters) + encoder_accepts_wildcard = "kwargs" in encoder_signature or "model_kwargs" in encoder_signature + if not encoder_accepts_wildcard: + encoder_kwargs = { + argument: value for argument, value in encoder_kwargs.items() if argument in encoder_signature + } + encoder_kwargs["output_attentions"] = generation_config.output_attentions + encoder_kwargs["output_hidden_states"] = generation_config.output_hidden_states + + # 3. make sure that encoder returns `ModelOutput` + model_input_name = model_input_name if model_input_name is not None else self.main_input_name + encoder_kwargs["return_dict"] = True + + ####### 固定input_tensor形状 + pad_count = 0 + if batch_candidate[AtomicModel.MFR] != inputs_tensor.shape[0]: + pad_count = batch_candidate[AtomicModel.MFR]-inputs_tensor.shape[0] + padding_tensor = torch.zeros(pad_count, *inputs_tensor.shape[1:], dtype=inputs_tensor.dtype, device=inputs_tensor.device) + inputs_tensor = torch.cat((inputs_tensor, padding_tensor), dim=0) + + encoder_kwargs[model_input_name] = inputs_tensor + output = encoder(**encoder_kwargs)# type: ignore + if pad_count != 0: + output.last_hidden_state = output.last_hidden_state[:-pad_count] + output.pooler_output = output.pooler_output[:-pad_count] + model_kwargs["encoder_outputs"] = output + return model_kwargs + + GenerationMixin._prepare_encoder_decoder_kwargs_for_generation = _patched_prepare_encoder_decoder_kwargs_for_generation + +def warmup(data_path, warmup_iters): + data_path = Path(data_path) + + output_dir = Path(data_path).parent + output_dir = os.path.join(output_dir, "warmup_res") + pdf_suffixes = [".pdf"] + image_suffixes = [".png", ".jpeg", ".jpg"] + supported_suffixes = pdf_suffixes + image_suffixes + + if data_path.suffix.lower() not in supported_suffixes: + raise ValueError( + f"Unsupported file type: '{data_path.suffix}'. " + f"Supported types: {supported_suffixes}" + ) + + doc_path_list = [data_path] * sum(batch_candidate[AtomicModel.Layout]) + for _ in range(warmup_iters): + parse_doc(doc_path_list, output_dir, backend="pipeline") + + +def get_batch_ratio(device="npu"): + batch_ratio = 1 + if str(device).startswith('npu') or str(device).startswith('cuda'): + vram = get_vram(device) + if vram is not None: + gpu_memory = int(os.getenv('MINERU_VIRTUAL_VRAM_SIZE', round(vram))) + if gpu_memory >= 16: + batch_ratio = 16 + elif gpu_memory >= 12: + batch_ratio = 8 + elif gpu_memory >= 8: + batch_ratio = 4 + elif gpu_memory >= 6: + batch_ratio = 2 + else: + batch_ratio = 1 + logger.info(f'gpu_memory: {gpu_memory} GB, batch_ratio: {batch_ratio}') + else: + # Default batch_ratio when VRAM can't be determined + batch_ratio = 1 + logger.info(f'Could not determine GPU memory, using default batch_ratio: {batch_ratio}') + return batch_ratio + + +def get_pdf_page_count(pdf_path): + pdf = pdfium.PdfDocument(pdf_path) + try: + return len(pdf) + finally: + pdf.close() + +if __name__ == '__main__': + args = parse_args() + os.environ['MINERU_MODEL_SOURCE'] = args.model_source + + __dir__ = args.data_path + pdf_files_dir = os.path.join(__dir__, "pdfs") + output_dir = os.path.join(__dir__, "output") + pdf_suffixes = [".pdf"] + image_suffixes = [".png", ".jpeg", ".jpg"] + + + print(pdf_files_dir) + batch_ratio = get_batch_ratio() + # warmup(args.warmup, batch_ratio, ) + + rewrite_model_init() + + doc_path_list = [] + pdfs_page_count = 0 + for doc_path in Path(pdf_files_dir).glob('*'): + if doc_path.suffix in pdf_suffixes + image_suffixes: + doc_path_list.append(doc_path) + pdfs_page_count += get_pdf_page_count(doc_path) + + batch_candidate = { + AtomicModel.Layout: [YOLO_LAYOUT_BASE_BATCH_SIZE, pdfs_page_count % YOLO_LAYOUT_BASE_BATCH_SIZE], + AtomicModel.MFD: [MFD_BASE_BATCH_SIZE, pdfs_page_count % MFD_BASE_BATCH_SIZE], + AtomicModel.MFR: batch_ratio*MFR_BASE_BATCH_SIZE, + } + print(len(doc_path_list), batch_candidate) + warmup(args.warmup_data_path, args.warmup) + + print("******** 精度测试 **********") + start_time = time.time() + parse_doc(doc_path_list, output_dir, backend="pipeline") + print(f"per page process time: {(time.time()-start_time)/pdfs_page_count:.2f}s") diff --git a/ACL_PyTorch/built-in/ocr/MinerU/requirements.txt b/ACL_PyTorch/built-in/ocr/MinerU/requirements.txt new file mode 100644 index 0000000000..b98527828a --- /dev/null +++ b/ACL_PyTorch/built-in/ocr/MinerU/requirements.txt @@ -0,0 +1,34 @@ +boto3==1.40.24 +click==8.2.1 +loguru==0.7.3 +numpy==2.2.6 +pandas==2.3.2 +pdfminer.six==20250506 +tqdm==4.67.1 +requests +httpx +pillow==11.3.0 +pypdfium2==4.30.0 +pypdf==6.0.0 +reportlab==4.4.3 +pdftext==0.6.3 +modelscope==1.29.2 +huggingface-hub==0.34.4 +json-repair==0.50.0 +opencv-python==4.12.0.88 +fast-langdetect==0.2.5 +matplotlib==3.10.6 +ultralytics==8.3.193 +doclayout_yolo==0.0.4 +dill==0.3.8 +rapid_table==1.0.5 +PyYAML==6.0.2 +ftfy==6.3.1 +openai==1.106.1 +shapely==2.1.1 +pyclipper==1.3.0.post6 +omegaconf==2.3.0 +torch==2.6.0 +torch_npu==2.6.0 +torchvision==0.21.0 +transformers==4.56.1 \ No newline at end of file diff --git a/ACL_PyTorch/built-in/ocr/MinerU/ultralytics.patch b/ACL_PyTorch/built-in/ocr/MinerU/ultralytics.patch new file mode 100644 index 0000000000..4fab87d605 --- /dev/null +++ b/ACL_PyTorch/built-in/ocr/MinerU/ultralytics.patch @@ -0,0 +1,77 @@ +diff -ruN ultralytics-8.3.193/ultralytics/engine/predictor.py ultralytics_/ultralytics/engine/predictor.py +--- ultralytics-8.3.193/ultralytics/engine/predictor.py 2025-09-04 19:51:11.000000000 +0800 ++++ ultralytics_/ultralytics/engine/predictor.py 2025-09-09 14:56:14.535737230 +0800 +@@ -196,9 +196,10 @@ + same_shapes = len({x.shape for x in im}) == 1 + letterbox = LetterBox( + self.imgsz, +- auto=same_shapes +- and self.args.rect +- and (self.model.pt or (getattr(self.model, "dynamic", False) and not self.model.imx)), ++ # auto=same_shapes ++ # and self.args.rect ++ # and (self.model.pt or (getattr(self.model, "dynamic", False) and not self.model.imx)), ++ auto=False, + stride=self.model.stride, + ) + return [letterbox(image=x) for x in im] +@@ -311,8 +312,11 @@ + + # Warmup model + if not self.done_warmup: ++ # self.model.warmup( ++ # imgsz=(1 if self.models.pt or self.model.triton else self.dataset.bs, self.model.ch, *self.imgsz) ++ # ) + self.model.warmup( +- imgsz=(1 if self.model.pt or self.model.triton else self.dataset.bs, self.model.ch, *self.imgsz) ++ imgsz=(self.dataset.bs, self.model.ch, *self.imgsz) + ) + self.done_warmup = True + +@@ -400,7 +404,8 @@ + dnn=self.args.dnn, + data=self.args.data, + fp16=self.args.half, +- fuse=True, ++ # fuse=True, ++ fuse=False, + verbose=verbose, + ) + +diff -ruN ultralytics-8.3.193/ultralytics/nn/modules/block.py ultralytics_/ultralytics/nn/modules/block.py +--- ultralytics-8.3.193/ultralytics/nn/modules/block.py 2025-09-04 19:51:11.000000000 +0800 ++++ ultralytics_/ultralytics/nn/modules/block.py 2025-09-09 14:56:14.543737230 +0800 +@@ -237,7 +237,9 @@ + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Apply sequential pooling operations to input and return concatenated feature maps.""" + y = [self.cv1(x)] +- y.extend(self.m(y[-1]) for _ in range(3)) ++ # y.extend(self.m(y[-1]) for _ in range(3)) ++ for _ in range(3): ++ y.append(self.m(y[-1])) + return self.cv2(torch.cat(y, 1)) + + +@@ -315,7 +317,9 @@ + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward pass through C2f layer.""" + y = list(self.cv1(x).chunk(2, 1)) +- y.extend(m(y[-1]) for m in self.m) ++ # y.extend(m(y[-1]) for m in self.m) ++ for m in self.m: ++ y.append(m(y[-1])) + return self.cv2(torch.cat(y, 1)) + + def forward_split(self, x: torch.Tensor) -> torch.Tensor: + +diff -ruN ultralytics-8.3.193/ultralytics/utils/tal.py ultralytics_/ultralytics/utils/tal.py +--- ultralytics-8.3.193/ultralytics/utils/tal.py 2025-09-04 19:51:11.000000000 +0800 ++++ ultralytics_/ultralytics/utils/tal.py 2025-09-09 14:56:14.551737230 +0800 +@@ -375,7 +375,8 @@ + sy = torch.arange(end=h, device=device, dtype=dtype) + grid_cell_offset # shift y + sy, sx = torch.meshgrid(sy, sx, indexing="ij") if TORCH_1_10 else torch.meshgrid(sy, sx) + anchor_points.append(torch.stack((sx, sy), -1).view(-1, 2)) +- stride_tensor.append(torch.full((h * w, 1), stride, dtype=dtype, device=device)) ++ # stride_tensor.append(torch.full((h * w, 1), stride, dtype=dtype, device=device)) ++ stride_tensor.append(torch.ones((h * w, 1), dtype=dtype, device=device)*stride) + return torch.cat(anchor_points), torch.cat(stride_tensor) \ No newline at end of file -- Gitee From 786d49c07aada86b1bab99ab3c09c84a94554a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Wed, 10 Sep 2025 11:21:26 +0800 Subject: [PATCH 05/14] add mineru torchair adaptation --- ACL_PyTorch/built-in/ocr/MinerU/infer.py | 56 +++++++++++++++--------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/ACL_PyTorch/built-in/ocr/MinerU/infer.py b/ACL_PyTorch/built-in/ocr/MinerU/infer.py index 933917f639..b3947b91e8 100644 --- a/ACL_PyTorch/built-in/ocr/MinerU/infer.py +++ b/ACL_PyTorch/built-in/ocr/MinerU/infer.py @@ -1,13 +1,28 @@ -# Copyright (c) Opendatalab. All rights reserved. +# Copyright 2025 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os -from pathlib import Path +import math +import time +import sys +from typing import Optional, Tuple, Union import inspect + +from pathlib import Path import argparse from loguru import logger -import math -from typing import Optional, Tuple, Union import pypdfium2 as pdfium -import time import torch import torch_npu @@ -17,10 +32,6 @@ import torchvision_npu import torchair as tng from torchair.configs.compiler_config import CompilerConfig -# import logging -# from torchair import logger -# logger.setLevel(logging.DEBUG) - from mineru.backend.pipeline.model_list import AtomicModel from mineru.model.mfr.unimernet.unimernet_hf.unimer_swin.modeling_unimer_swin import UnimerSwinSelfAttention from mineru.backend.pipeline.model_init import ( @@ -51,6 +62,7 @@ def parse_args(): args = parser.parse_args() return args + def atom_model_init_compile(model_name: str, **kwargs): atom_model = None if model_name == AtomicModel.Layout: @@ -82,8 +94,6 @@ def atom_model_init_compile(model_name: str, **kwargs): print(atom_model.model.encoder.__class__) atom_model.model.encoder = compile_model(atom_model.model.encoder, False, True) - # npu_input = torch.zeros((batch_candidate[AtomicModel.MFR], 3, *atom_model.model.transform.input_size), dtype=torch.half, device=atom_model.device) - # tng.inference.set_dim_gears(npu_input, {0: batch_candidate[AtomicModel.MFR]}) atom_model.model.decoder = compile_model(atom_model.model.decoder, True, True) elif model_name == AtomicModel.OCR: @@ -100,21 +110,20 @@ def atom_model_init_compile(model_name: str, **kwargs): else: logger.error('model name not allow') - exit(1) + sys.exit(1) if atom_model is None: logger.error('model init failed') - exit(1) - else: - return atom_model - + sys.exit(1) + + return atom_model def rewrite_mfr_encoder_multi_head_attention_forward(model): wq = model.query.weight wk = model.key.weight wv = model.value.weight - model.qkv = nn.Linear(in_features=wk.shape[1], out_features= wq.shape[0] + wk.shape[0] + wv.shape[0]) + model.qkv = nn.Linear(in_feature = wk.shape[1], out_features = wq.shape[0] + wk.shape[0] + wv.shape[0]) model.qkv.weight = nn.Parameter(torch.concat([wq, wk, wv], dim=0), requires_grad=False) wq_bias = model.query.bias if model.query.bias is not None else torch.zeros(wq.shape[0]) wk_bias = model.key.bias if model.key.bias is not None else torch.zeros(wk.shape[0]) @@ -178,18 +187,18 @@ def rewrite_mfr_encoder_multi_head_attention_forward(model): def modify_mfr_model(model): # 修改encoder的attention forward - for name, module in model.encoder.named_modules(): + for _, module in model.encoder.named_modules(): if isinstance(module, UnimerSwinSelfAttention): rewrite_mfr_encoder_multi_head_attention_forward(module) rewrite_mfr_encoder_forward() + def compile_model(model, dynamic, fullgraph): config = CompilerConfig() config.experimental_config.frozen_parameter = True config.experimental_config.tiling_schedule_optimize = True npu_backend = tng.get_npu_backend(compiler_config=config) compiled_model = torch.compile(model, dynamic=dynamic, fullgraph=fullgraph, backend=npu_backend) - # tng.use_internal_format_weight(compiled_model) return compiled_model @@ -210,6 +219,7 @@ def rewrite_model_init(): return self._models[key] AtomModelSingleton.get_atom_model = _patched_getmodel + def rewrite_mfr_encoder_forward(): def _patched_prepare_encoder_decoder_kwargs_for_generation(self, inputs_tensor: torch.Tensor, @@ -231,7 +241,9 @@ def rewrite_mfr_encoder_forward(): encoder_accepts_wildcard = "kwargs" in encoder_signature or "model_kwargs" in encoder_signature if not encoder_accepts_wildcard: encoder_kwargs = { - argument: value for argument, value in encoder_kwargs.items() if argument in encoder_signature + argument: value + for argument, value in encoder_kwargs.items() + if argument in encoder_signature } encoder_kwargs["output_attentions"] = generation_config.output_attentions encoder_kwargs["output_hidden_states"] = generation_config.output_hidden_states @@ -243,7 +255,7 @@ def rewrite_mfr_encoder_forward(): ####### 固定input_tensor形状 pad_count = 0 if batch_candidate[AtomicModel.MFR] != inputs_tensor.shape[0]: - pad_count = batch_candidate[AtomicModel.MFR]-inputs_tensor.shape[0] + pad_count = batch_candidate[AtomicModel.MFR] - inputs_tensor.shape[0] padding_tensor = torch.zeros(pad_count, *inputs_tensor.shape[1:], dtype=inputs_tensor.dtype, device=inputs_tensor.device) inputs_tensor = torch.cat((inputs_tensor, padding_tensor), dim=0) @@ -257,6 +269,7 @@ def rewrite_mfr_encoder_forward(): GenerationMixin._prepare_encoder_decoder_kwargs_for_generation = _patched_prepare_encoder_decoder_kwargs_for_generation + def warmup(data_path, warmup_iters): data_path = Path(data_path) @@ -308,6 +321,7 @@ def get_pdf_page_count(pdf_path): finally: pdf.close() + if __name__ == '__main__': args = parse_args() os.environ['MINERU_MODEL_SOURCE'] = args.model_source -- Gitee From daf9c2a2bf11f9cc9fdce7af41ea0c3bf0dd6cf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Wed, 10 Sep 2025 11:51:29 +0800 Subject: [PATCH 06/14] add MinerU torchair adaptation --- ACL_PyTorch/built-in/ocr/MinerU/infer.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ACL_PyTorch/built-in/ocr/MinerU/infer.py b/ACL_PyTorch/built-in/ocr/MinerU/infer.py index b3947b91e8..6ce7c54565 100644 --- a/ACL_PyTorch/built-in/ocr/MinerU/infer.py +++ b/ACL_PyTorch/built-in/ocr/MinerU/infer.py @@ -110,12 +110,12 @@ def atom_model_init_compile(model_name: str, **kwargs): else: logger.error('model name not allow') - sys.exit(1) + raise ValueError("model name not allow") if atom_model is None: logger.error('model init failed') - sys.exit(1) - + raise RuntimeError("model init failed") + return atom_model @@ -123,7 +123,8 @@ def rewrite_mfr_encoder_multi_head_attention_forward(model): wq = model.query.weight wk = model.key.weight wv = model.value.weight - model.qkv = nn.Linear(in_feature = wk.shape[1], out_features = wq.shape[0] + wk.shape[0] + wv.shape[0]) + model.qkv = nn.Linear(in_feature = wk.shape[1], + out_features = wq.shape[0] + wk.shape[0] + wv.shape[0]) model.qkv.weight = nn.Parameter(torch.concat([wq, wk, wv], dim=0), requires_grad=False) wq_bias = model.query.bias if model.query.bias is not None else torch.zeros(wq.shape[0]) wk_bias = model.key.bias if model.key.bias is not None else torch.zeros(wk.shape[0]) @@ -335,7 +336,6 @@ if __name__ == '__main__': print(pdf_files_dir) batch_ratio = get_batch_ratio() - # warmup(args.warmup, batch_ratio, ) rewrite_model_init() @@ -349,7 +349,7 @@ if __name__ == '__main__': batch_candidate = { AtomicModel.Layout: [YOLO_LAYOUT_BASE_BATCH_SIZE, pdfs_page_count % YOLO_LAYOUT_BASE_BATCH_SIZE], AtomicModel.MFD: [MFD_BASE_BATCH_SIZE, pdfs_page_count % MFD_BASE_BATCH_SIZE], - AtomicModel.MFR: batch_ratio*MFR_BASE_BATCH_SIZE, + AtomicModel.MFR: batch_ratio * MFR_BASE_BATCH_SIZE, } print(len(doc_path_list), batch_candidate) warmup(args.warmup_data_path, args.warmup) -- Gitee From 0eb4a2b5ec105dee509b5cd7a60c45848ff1ca66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Wed, 10 Sep 2025 11:58:28 +0800 Subject: [PATCH 07/14] add MinerU torchair adaptation --- ACL_PyTorch/built-in/ocr/MinerU/infer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ACL_PyTorch/built-in/ocr/MinerU/infer.py b/ACL_PyTorch/built-in/ocr/MinerU/infer.py index 6ce7c54565..70bbdb91cb 100644 --- a/ACL_PyTorch/built-in/ocr/MinerU/infer.py +++ b/ACL_PyTorch/built-in/ocr/MinerU/infer.py @@ -13,11 +13,11 @@ # limitations under the License. import os +import sys import math import time -import sys -from typing import Optional, Tuple, Union import inspect +from typing import Optional, Tuple, Union from pathlib import Path import argparse @@ -56,9 +56,9 @@ from demo.demo import parse_doc def parse_args(): parser = argparse.ArgumentParser("MinerU infer") parser.add_argument("--model_source", type=str, default="local", help="model checkpoint source") - parser.add_argument("--data_path", type=str, default="/home/z00939677/OmniDocBench_dataset") + parser.add_argument("--data_path", type=str, default="OmniDocBench_dataset") parser.add_argument("--warmup", type=int, default=2, help="Warm up times") - parser.add_argument("--warmup_data_path", type=str, default="../OmniDocBench_dataset/pdfs/jiaocai_71434495.pdf_0.pdf") + parser.add_argument("--warmup_data_path", type=str, default="OmniDocBench_dataset/pdfs/jiaocai_71434495.pdf_0.pdf") args = parser.parse_args() return args -- Gitee From 3d67c9c0b020447745c4905576a17cb6196f5979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Wed, 10 Sep 2025 12:55:02 +0800 Subject: [PATCH 08/14] add MinerU torchair adaptation --- ACL_PyTorch/built-in/ocr/MinerU/infer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ACL_PyTorch/built-in/ocr/MinerU/infer.py b/ACL_PyTorch/built-in/ocr/MinerU/infer.py index 70bbdb91cb..dca505247e 100644 --- a/ACL_PyTorch/built-in/ocr/MinerU/infer.py +++ b/ACL_PyTorch/built-in/ocr/MinerU/infer.py @@ -123,8 +123,7 @@ def rewrite_mfr_encoder_multi_head_attention_forward(model): wq = model.query.weight wk = model.key.weight wv = model.value.weight - model.qkv = nn.Linear(in_feature = wk.shape[1], - out_features = wq.shape[0] + wk.shape[0] + wv.shape[0]) + model.qkv = nn.Linear(in_feature=wk.shape[1], out_features=wq.shape[0]+wk.shape[0]+wv.shape[0]) model.qkv.weight = nn.Parameter(torch.concat([wq, wk, wv], dim=0), requires_grad=False) wq_bias = model.query.bias if model.query.bias is not None else torch.zeros(wq.shape[0]) wk_bias = model.key.bias if model.key.bias is not None else torch.zeros(wk.shape[0]) -- Gitee From 8d055490924c0a0f4456d97402d4a866dbb82874 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Wed, 10 Sep 2025 13:36:54 +0800 Subject: [PATCH 09/14] add MinerU torchair adaptation --- ACL_PyTorch/built-in/ocr/MinerU/infer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ACL_PyTorch/built-in/ocr/MinerU/infer.py b/ACL_PyTorch/built-in/ocr/MinerU/infer.py index dca505247e..cf53dbd1ac 100644 --- a/ACL_PyTorch/built-in/ocr/MinerU/infer.py +++ b/ACL_PyTorch/built-in/ocr/MinerU/infer.py @@ -1,4 +1,5 @@ # Copyright 2025 Huawei Technologies Co., Ltd +# Copyright 2023 The LAION-AI Team and The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -91,8 +92,6 @@ def atom_model_init_compile(model_name: str, **kwargs): modify_mfr_model(atom_model.model) - print(atom_model.model.encoder.__class__) - atom_model.model.encoder = compile_model(atom_model.model.encoder, False, True) atom_model.model.decoder = compile_model(atom_model.model.decoder, True, True) @@ -130,6 +129,7 @@ def rewrite_mfr_encoder_multi_head_attention_forward(model): wv_bias = model.key.bias if model.value.bias is not None else torch.zeros(wv.shape[0]) model.qkv.bias = nn.Parameter(torch.concat([wq_bias, wk_bias, wv_bias], dim=0), requires_grad=False) + # Adapted from: transformers.models.swin.modeling_swin.SwinSelfAttention def attn_forward( self, hidden_states: torch.Tensor, @@ -147,7 +147,6 @@ def rewrite_mfr_encoder_multi_head_attention_forward(model): key_layer = k.view(*k.shape[:2], self.num_attention_heads, -1).permute(0, 2, 1, 3) value_layer = v.view(*v.shape[:2], self.num_attention_heads, -1).permute(0, 2, 1, 3) - relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)] relative_position_bias = relative_position_bias.view( self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1 @@ -221,6 +220,7 @@ def rewrite_model_init(): def rewrite_mfr_encoder_forward(): + # Adapted from: transformers.generation.utils.GenerationMixin def _patched_prepare_encoder_decoder_kwargs_for_generation(self, inputs_tensor: torch.Tensor, model_kwargs, -- Gitee From f877e1140da154bf48f8a8f12fa88c53e7d237ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Wed, 10 Sep 2025 14:09:43 +0800 Subject: [PATCH 10/14] add MinerU torchair adaptation --- ACL_PyTorch/built-in/ocr/MinerU/README.md | 2 + ACL_PyTorch/built-in/ocr/MinerU/infer.py | 58 +------------------ .../ocr/MinerU/mfr_encoder_mhsa.patch | 23 ++++++++ 3 files changed, 26 insertions(+), 57 deletions(-) create mode 100644 ACL_PyTorch/built-in/ocr/MinerU/mfr_encoder_mhsa.patch diff --git a/ACL_PyTorch/built-in/ocr/MinerU/README.md b/ACL_PyTorch/built-in/ocr/MinerU/README.md index 19f28fbb03..20069b5bbd 100644 --- a/ACL_PyTorch/built-in/ocr/MinerU/README.md +++ b/ACL_PyTorch/built-in/ocr/MinerU/README.md @@ -76,6 +76,8 @@ MinerU是由上海人工智能实验室OpenDataLab团队开发的开源文档解 patch -p2 < ${workdir}/ultralytics.patch cd ${source_path}/doclayout_yolo patch -p2 < ${workdir}/doclayout_yolo.patch + cd ${workdir} + patch -p0 < mfr_encoder_mhsa.patch ``` 修改完成后回到工作目录`workdir` diff --git a/ACL_PyTorch/built-in/ocr/MinerU/infer.py b/ACL_PyTorch/built-in/ocr/MinerU/infer.py index cf53dbd1ac..91b53a045c 100644 --- a/ACL_PyTorch/built-in/ocr/MinerU/infer.py +++ b/ACL_PyTorch/built-in/ocr/MinerU/infer.py @@ -1,5 +1,4 @@ # Copyright 2025 Huawei Technologies Co., Ltd -# Copyright 2023 The LAION-AI Team and The HuggingFace Team. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -122,67 +121,13 @@ def rewrite_mfr_encoder_multi_head_attention_forward(model): wq = model.query.weight wk = model.key.weight wv = model.value.weight - model.qkv = nn.Linear(in_feature=wk.shape[1], out_features=wq.shape[0]+wk.shape[0]+wv.shape[0]) + model.qkv = nn.Linear(in_features=wk.shape[1], out_features=wq.shape[0]+wk.shape[0]+wv.shape[0]) model.qkv.weight = nn.Parameter(torch.concat([wq, wk, wv], dim=0), requires_grad=False) wq_bias = model.query.bias if model.query.bias is not None else torch.zeros(wq.shape[0]) wk_bias = model.key.bias if model.key.bias is not None else torch.zeros(wk.shape[0]) wv_bias = model.key.bias if model.value.bias is not None else torch.zeros(wv.shape[0]) model.qkv.bias = nn.Parameter(torch.concat([wq_bias, wk_bias, wv_bias], dim=0), requires_grad=False) - # Adapted from: transformers.models.swin.modeling_swin.SwinSelfAttention - def attn_forward( - self, - hidden_states: torch.Tensor, - attention_mask: Optional[torch.FloatTensor] = None, - head_mask: Optional[torch.FloatTensor] = None, - output_attentions: Optional[bool] = False - ) -> Tuple[torch.Tensor]: - - # """融合qk为大矩阵，由于加入相对位置编码，PFA接口用不了，暂时只修改矩阵乘法""" - batch_size, dim, num_channels = hidden_states.shape - qkv = model.qkv(hidden_states) - q, k, v = qkv.chunk(3, dim=-1) - - query_layer = q.view(*q.shape[:2], self.num_attention_heads, -1).permute(0, 2, 1, 3) - key_layer = k.view(*k.shape[:2], self.num_attention_heads, -1).permute(0, 2, 1, 3) - value_layer = v.view(*v.shape[:2], self.num_attention_heads, -1).permute(0, 2, 1, 3) - - relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)] - relative_position_bias = relative_position_bias.view( - self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1 - ) - relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() - - attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) - attention_scores = attention_scores / math.sqrt(self.attention_head_size) - attention_scores = attention_scores + relative_position_bias.unsqueeze(0) - - if attention_mask is not None: - # Apply the attention mask is (precomputed for all layers in UnimerSwinModel forward() function) - mask_shape = attention_mask.shape[0] - attention_scores = attention_scores.view( - batch_size // mask_shape, mask_shape, self.num_attention_heads, dim, dim - ) - attention_scores = attention_scores + attention_mask.unsqueeze(1).unsqueeze(0) - attention_scores = attention_scores.view(-1, self.num_attention_heads, dim, dim) - - # Normalize the attention scores to probabilities. - attention_probs = nn.functional.softmax(attention_scores, dim=-1) - attention_probs = self.dropout(attention_probs) - - # Mask heads if we want to - if head_mask is not None: - attention_probs = attention_probs * head_mask - - context_layer = torch.matmul(attention_probs, value_layer) - context_layer = context_layer.permute(0, 2, 1, 3).contiguous() - new_context_layer_shape = context_layer.size()[:-2] + (self.all_head_size,) - context_layer = context_layer.view(new_context_layer_shape) - - outputs = (context_layer, attention_probs) if output_attentions else (context_layer,) - return outputs - model.forward = attn_forward.__get__(model) - def modify_mfr_model(model): # 修改encoder的attention forward @@ -220,7 +165,6 @@ def rewrite_model_init(): def rewrite_mfr_encoder_forward(): - # Adapted from: transformers.generation.utils.GenerationMixin def _patched_prepare_encoder_decoder_kwargs_for_generation(self, inputs_tensor: torch.Tensor, model_kwargs, diff --git a/ACL_PyTorch/built-in/ocr/MinerU/mfr_encoder_mhsa.patch b/ACL_PyTorch/built-in/ocr/MinerU/mfr_encoder_mhsa.patch new file mode 100644 index 0000000000..1fe80a05cb --- /dev/null +++ b/ACL_PyTorch/built-in/ocr/MinerU/mfr_encoder_mhsa.patch @@ -0,0 +1,23 @@ +--- MinerU/mineru/model/mfr/unimernet/unimernet_hf/unimer_swin/modeling_unimer_swin.py 2025-09-02 17:58:15.032000000 +0800 ++++ copy_mfr.py 2025-09-10 13:58:36.616000000 +0800 +@@ -465,11 +465,15 @@ + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.Tensor]: + batch_size, dim, num_channels = hidden_states.shape +- mixed_query_layer = self.query(hidden_states) + +- key_layer = self.transpose_for_scores(self.key(hidden_states)) +- value_layer = self.transpose_for_scores(self.value(hidden_states)) +- query_layer = self.transpose_for_scores(mixed_query_layer) ++ # """融合qk为大矩阵，由于加入相对位置编码，PFA接口用不了，暂时只修改矩阵乘法""" ++ batch_size, dim, num_channels = hidden_states.shape ++ qkv = self.qkv(hidden_states) ++ q, k, v = qkv.chunk(3, dim=-1) ++ ++ query_layer = q.view(*q.shape[:2], self.num_attention_heads, -1).permute(0, 2, 1, 3) ++ key_layer = k.view(*k.shape[:2], self.num_attention_heads, -1).permute(0, 2, 1, 3) ++ value_layer = v.view(*v.shape[:2], self.num_attention_heads, -1).permute(0, 2, 1, 3) + + # Take the dot product between "query" and "key" to get the raw attention scores. + attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2)) + -- Gitee From dc435a0323dc66a4661089f80a4bacf8053f4ec6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Wed, 10 Sep 2025 14:27:53 +0800 Subject: [PATCH 11/14] add MinerU torchair adaptation --- ACL_PyTorch/built-in/ocr/MinerU/infer.py | 28 ++---------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/ACL_PyTorch/built-in/ocr/MinerU/infer.py b/ACL_PyTorch/built-in/ocr/MinerU/infer.py index 91b53a045c..7780e89949 100644 --- a/ACL_PyTorch/built-in/ocr/MinerU/infer.py +++ b/ACL_PyTorch/built-in/ocr/MinerU/infer.py @@ -121,7 +121,7 @@ def rewrite_mfr_encoder_multi_head_attention_forward(model): wq = model.query.weight wk = model.key.weight wv = model.value.weight - model.qkv = nn.Linear(in_features=wk.shape[1], out_features=wq.shape[0]+wk.shape[0]+wv.shape[0]) + model.qkv = nn.Linear(in_features=wk.shape[1], out_features=wq.shape[0] + wk.shape[0] + wv.shape[0]) model.qkv.weight = nn.Parameter(torch.concat([wq, wk, wv], dim=0), requires_grad=False) wq_bias = model.query.bias if model.query.bias is not None else torch.zeros(wq.shape[0]) wk_bias = model.key.bias if model.key.bias is not None else torch.zeros(wk.shape[0]) @@ -234,30 +234,6 @@ def warmup(data_path, warmup_iters): parse_doc(doc_path_list, output_dir, backend="pipeline") -def get_batch_ratio(device="npu"): - batch_ratio = 1 - if str(device).startswith('npu') or str(device).startswith('cuda'): - vram = get_vram(device) - if vram is not None: - gpu_memory = int(os.getenv('MINERU_VIRTUAL_VRAM_SIZE', round(vram))) - if gpu_memory >= 16: - batch_ratio = 16 - elif gpu_memory >= 12: - batch_ratio = 8 - elif gpu_memory >= 8: - batch_ratio = 4 - elif gpu_memory >= 6: - batch_ratio = 2 - else: - batch_ratio = 1 - logger.info(f'gpu_memory: {gpu_memory} GB, batch_ratio: {batch_ratio}') - else: - # Default batch_ratio when VRAM can't be determined - batch_ratio = 1 - logger.info(f'Could not determine GPU memory, using default batch_ratio: {batch_ratio}') - return batch_ratio - - def get_pdf_page_count(pdf_path): pdf = pdfium.PdfDocument(pdf_path) try: @@ -278,7 +254,7 @@ if __name__ == '__main__': print(pdf_files_dir) - batch_ratio = get_batch_ratio() + batch_ratio = 16 rewrite_model_init() -- Gitee From 08c450b0a436dd82f1aff6e9fefc8b15e0380072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Wed, 10 Sep 2025 15:39:34 +0800 Subject: [PATCH 12/14] add MinerU torchair adaptation --- ACL_PyTorch/built-in/ocr/MinerU/infer.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ACL_PyTorch/built-in/ocr/MinerU/infer.py b/ACL_PyTorch/built-in/ocr/MinerU/infer.py index 7780e89949..93779d6622 100644 --- a/ACL_PyTorch/built-in/ocr/MinerU/infer.py +++ b/ACL_PyTorch/built-in/ocr/MinerU/infer.py @@ -17,7 +17,6 @@ import sys import math import time import inspect -from typing import Optional, Tuple, Union from pathlib import Path import argparse @@ -168,7 +167,7 @@ def rewrite_mfr_encoder_forward(): def _patched_prepare_encoder_decoder_kwargs_for_generation(self, inputs_tensor: torch.Tensor, model_kwargs, - model_input_name: Optional[str], + model_input_name, generation_config, ): # 1. get encoder -- Gitee From eaec3391c1ba20fcd17e992ef09d51d78993aff8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Wed, 10 Sep 2025 17:09:30 +0800 Subject: [PATCH 13/14] add MinerU torchair adaptation --- ACL_PyTorch/built-in/ocr/MinerU/README.md | 3 +-- ACL_PyTorch/built-in/ocr/MinerU/infer.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ACL_PyTorch/built-in/ocr/MinerU/README.md b/ACL_PyTorch/built-in/ocr/MinerU/README.md index 20069b5bbd..cdf01eb0ea 100644 --- a/ACL_PyTorch/built-in/ocr/MinerU/README.md +++ b/ACL_PyTorch/built-in/ocr/MinerU/README.md @@ -30,7 +30,7 @@ MinerU是由上海人工智能实验室OpenDataLab团队开发的开源文档解 | 配套 | 版本 | 环境准备指导 | | ------------------------------------------------------- | ----------- | --------------------------------------------------------------------------------------------- | - | 固件与驱动 | 25.0.RC1 | [Pytorch框架推理环境准备](https://www.hiascend.com/document/detail/zh/ModelZoo/pytorchframework/pies) | + | 固件与驱动 | 25.2.RC1 | [Pytorch框架推理环境准备](https://www.hiascend.com/document/detail/zh/ModelZoo/pytorchframework/pies) | | CANN | 8.3.0 | - | | Python | 3.11 | - | | PyTorch | 2.6.0 | - | @@ -79,7 +79,6 @@ MinerU是由上海人工智能实验室OpenDataLab团队开发的开源文档解 cd ${workdir} patch -p0 < mfr_encoder_mhsa.patch ``` -修改完成后回到工作目录`workdir` ## 获取权重 diff --git a/ACL_PyTorch/built-in/ocr/MinerU/infer.py b/ACL_PyTorch/built-in/ocr/MinerU/infer.py index 93779d6622..b04545b7fc 100644 --- a/ACL_PyTorch/built-in/ocr/MinerU/infer.py +++ b/ACL_PyTorch/built-in/ocr/MinerU/infer.py @@ -49,7 +49,7 @@ from mineru.backend.pipeline.batch_analyze import ( ) from transformers.generation.utils import GenerationMixin -from demo.demo import parse_doc +from MinerU.demo.demo import parse_doc def parse_args(): -- Gitee From 5b494f8ea15e3d9ba70fe3d5c9adb50dfaa276ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B9=E4=BA=A6=E8=88=9F?= Date: Wed, 17 Sep 2025 14:44:34 +0800 Subject: [PATCH 14/14] Updating the Third-Party Library Version --- ACL_PyTorch/built-in/cv/SAM/README.md | 12 ++++++------ ACL_PyTorch/built-in/cv/SAM/requirements.txt | 13 +++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/ACL_PyTorch/built-in/cv/SAM/README.md b/ACL_PyTorch/built-in/cv/SAM/README.md index 7fee7963ca..4ca3afa1f5 100644 --- a/ACL_PyTorch/built-in/cv/SAM/README.md +++ b/ACL_PyTorch/built-in/cv/SAM/README.md @@ -61,11 +61,11 @@ SAM 首先会自动分割图像中的所有内容，但是如果你需要分割 | 配套 | 版本 | 环境准备指导 | | ---- | ---- | ---- | -| 固件与驱动 | 24.0.T1.B010 | [Pytorch框架推理环境准备](https://www.hiascend.com/document/detail/zh/ModelZoo/pytorchframework/pies) | -| CANN | 8.0.RC1.B060 | - | -| MindIE | 1.0.RC1.B060 | - | -| Python | 3.10.13（MindIE 要求 Python 3.10） | - | -| PyTorch | 1.13.1 | - | +| 固件与驱动 | 25.2.RC1 | [Pytorch框架推理环境准备](https://www.hiascend.com/document/detail/zh/ModelZoo/pytorchframework/pies) | +| CANN | 8.2.RC1 | - | +| MindIE | 2.1.RC1 | - | +| Python | 3.11.10 | - | +| PyTorch | 2.1.0 | - | | 说明：Atlas 300I Duo 推理卡请以CANN版本选择实际固件与驱动版本。 | \ | \ | ## 3. 快速上手 @@ -78,7 +78,7 @@ cd ModelZoo-PyTorch/ACL_PyTorch/built-in/cv/SAM git clone https://github.com/facebookresearch/segment-anything.git cd segment-anything git reset --hard 6fdee8f2727f4506cfbbe553e23b895e27956588 -patch -p2 < segment_anything_diff.patch +patch -p2 < ../segment_anything_diff.patch pip3 install -e . cd .. ``` diff --git a/ACL_PyTorch/built-in/cv/SAM/requirements.txt b/ACL_PyTorch/built-in/cv/SAM/requirements.txt index 22d75b25e8..6722adb464 100644 --- a/ACL_PyTorch/built-in/cv/SAM/requirements.txt +++ b/ACL_PyTorch/built-in/cv/SAM/requirements.txt @@ -1,14 +1,15 @@ -torch==1.13.1 -torchvision==0.14.1 -torchaudio==0.13.1 +torch==2.1.0 +torch_npu==2.1.0.post17.dev20250905 +torchvision==0.16.0 +torchaudio==2.1.0 decorator scipy attrs psutil -numpy -opencv-python +numpy==1.26.0 +opencv-python==4.11.0.86 pycocotools matplotlib +onnx==1.16.1 onnxruntime -onnx onnxsim==0.4.17 \ No newline at end of file -- Gitee