训练lenet报错EH9999: Inner Error!

一、问题现象（附报错日志上下文）：
Exception has occurred: RuntimeError
The Inner error is reported as above.
 Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, pleace set the environment variable ASCEND_LAUNCH_BLOCKING=1.
[ERROR] 2024-07-11-22:15:02 (PID:1466203, Device:0, RankID:-1) ERR00005 PTA internal error
  File "/home/liwb/code/LeNet-5/LeNet_5_train.py", line 65, in train
    loss = loss_fn(pred, y)
  File "/home/liwb/code/LeNet-5/LeNet_5_train.py", line 106, in <module>
    loss = train(train_dataloader, model_lenet5, loss_fn, optimizer)
RuntimeError: The Inner error is reported as above.
 Since the operator is called asynchronously, the stacktrace may be inaccurate. If you want to get the accurate stacktrace, pleace set the environment variable ASCEND_LAUNCH_BLOCKING=1.
[ERROR] 2024-07-11-22:15:02 (PID:1466203, Device:0, RankID:-1) ERR00005 PTA internal error

![报错截图](https://foruda.gitee.com/images/1720707337308618969/672947c8_1302447.png "屏幕截图")

二、软件版本:
-- CANN 版本 (e.g., CANN 3.0.x，5.x.x):   CANN 7.0.0
--Tensorflow/Pytorch/MindSpore 版本:Pytorch 2.1.0
--Python 版本 (e.g., Python 3.7.5):3.10.14
-- MindStudio版本 (e.g., MindStudio 2.0.0 (beta3)): /
--操作系统版本 (e.g., Ubuntu 18.04): Ubuntu 22.04

三、测试步骤：

lenet训练分了两个.py文件，具体如下：

### LeNet_5.py:

```

import torch
import torch_npu
from torch import nn
from torch.utils.data import DataLoader
import torch_npu.npu
from torchvision import datasets
from torchvision.transforms import ToTensor

class LeNet5(nn.Module):
    def __init__(self, num_classes):
        super(LeNet5, self).__init__()
  
        self.conv1=nn.Sequential(                   
            nn.Conv2d(1,6,kernel_size=5,stride=1,padding=2,dtype=torch.float32),
            nn.ReLU(),
        )
        self.pooling1=nn.AvgPool2d(kernel_size=2,stride=2)

self.conv2=nn.Sequential(                   
            nn.Conv2d(6,16,kernel_size=5,stride=1,dtype=torch.float32),
            nn.ReLU(),
        )

self.pooling2=nn.AvgPool2d(kernel_size=2,stride=2)
        
        self.fc1 =nn.Sequential(  
            nn.Linear(400, 120,dtype=torch.float32),
            nn.ReLU(),
        )
        self.fc2 =nn.Sequential(  
            nn.Linear(120, 84),
            nn.ReLU(),
        )
        self.fc3= nn.Linear(84, num_classes)

def forward(self, x, num_classes):
        out = self.conv1(x)
        out = self.pooling1(out)
        out = self.conv2(out)
        out = self.pooling2(out)
        out = torch.flatten(out,1)
        nn.Linear(400, 120,dtype=torch.float32)
        nn.ReLU()
        nn.Linear(120, 84)
        nn.ReLU()
        nn.Linear(84, num_classes)
   
        return out

```

### LeNet_5_train.py

```

import torch
import torch_npu
import torch_npu.contrib
import torch_npu.contrib.transfer_to_npu
from torch_npu.npu import amp # 导入AMP模块
from torch_npu.contrib import transfer_to_npu    # 使能自动迁移

from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets
from torchvision.transforms import ToTensor
from LeNet_5 import LeNet5

training_data = datasets.MNIST(
    root="MNIST_data",
    train=True,
    download=True,
    transform=ToTensor()
)

test_data = datasets.MNIST(
    root="MNIST_data",
    train=False,
    download=True,
    transform=ToTensor()
)

batch_size = 15
train_dataloader = DataLoader(training_data, batch_size=batch_size)
test_dataloader = DataLoader(test_data, batch_size=batch_size)

device = (
    "npu"
    if torch_npu.npu.is_available()
    else "cpu"
)

print(f"Using {device} device")

model_lenet5 = LeNet5(num_classes=10).to(device)
print(model_lenet5)

loss_fn = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model_lenet5.parameters(), lr=1e-3)
scaler = amp.GradScaler()    # 在模型、优化器定义之后，定义GradScaler

def train(dataloader, model, loss_fn, optimizer):
    size = len(dataloader.dataset)
    losses_record = []
    model.train()
    for batch, (X, y) in enumerate(dataloader):
        X, y = X.to(device), y.to(device)

with amp.autocast():
            pred = model(X,10) 
            loss = loss_fn(pred, y)

scaler.scale(loss).backward()    # loss缩放并反向转播
        scaler.step(optimizer)    # 更新参数（自动unscaling）
        scaler.update()    # 基于动态Loss Scale更新loss_scaling系数
        optimizer.zero_grad()

if batch % 100 == 0:
            loss, current = loss.item(), (batch + 1) * len(X)
            losses_record.append(loss)
            print(f"loss: {loss:>7f}  [{current:>5d}/{size:>5d}]")

return losses_record

def test(dataloader, model, loss_fn):
    size = len(dataloader.dataset)
    num_batches = len(dataloader)
    model.eval()
    test_loss, correct = 0, 0
    with torch.no_grad():
        for X, y in dataloader:
            X, y = X.to(device), y.to(device)
            pred = model(X)
            test_loss += loss_fn(pred, y).item()
            correct += (pred.argmax(1) == y).type(torch.float).sum().item()
    test_loss /= num_batches
    correct /= size
    print(f"Test Error: \n Accuracy: {(100*correct):>0.1f}%, Avg loss: {test_loss:>8f} \n")

print(f"Using {device} device")
epochs = 30
losses=[]
for t in range(epochs):
    print(f"Epoch {t+1}\n-------------------------------")
    loss = train(train_dataloader, model_lenet5, loss_fn, optimizer)
    losses = losses + loss
    test(test_dataloader, model_lenet5, loss_fn)
print("Done!")

torch.save(model_lenet5.state_dict(), "model.pth")
print("Saved PyTorch Model State to model.pth")

```

四、日志信息:

```

cd /home/liwb/code/LeNet-5 ; /usr/bin/env /home/liwb/anaconda3/envs/torch21/bin/python /home/liwb/.vscode-server/extensions/ms-python.debugpy-2024.8.0-linux-arm64/bundled/libs/debugpy/adapter/../../debugpy/launcher 41273 -- /home/liwb/code/LeNet-5/LeNet_5_train.py 
/home/liwb/anaconda3/envs/torch21/lib/python3.10/site-packages/torch_npu/utils/path_manager.py:79: UserWarning: Warning: The /usr/local/Ascend/ascend-toolkit/latest owner does not match the current user.
  warnings.warn(f"Warning: The {path} owner does not match the current user.")
/home/liwb/anaconda3/envs/torch21/lib/python3.10/site-packages/torch_npu/utils/path_manager.py:79: UserWarning: Warning: The /usr/local/Ascend/ascend-toolkit/7.0.0/aarch64-linux/ascend_toolkit_install.info owner does not match the current user.
  warnings.warn(f"Warning: The {path} owner does not match the current user.")
/home/liwb/anaconda3/envs/torch21/lib/python3.10/site-packages/torch_npu/contrib/transfer_to_npu.py:211: ImportWarning: 
    *************************************************************************************************************
    The torch.Tensor.cuda and torch.nn.Module.cuda are replaced with torch.Tensor.npu and torch.nn.Module.npu now..
    The torch.cuda.DoubleTensor is replaced with torch.npu.FloatTensor cause the double type is not supported now..
    The backend in torch.distributed.init_process_group set to hccl now..
    The torch.cuda.* and torch.cuda.amp.* are replaced with torch.npu.* and torch.npu.amp.* now..
    The device parameters have been replaced with npu in the function below:
    torch.logspace, torch.randint, torch.hann_window, torch.rand, torch.full_like, torch.ones_like, torch.rand_like, torch.randperm, torch.arange, torch.frombuffer, torch.normal, torch._empty_per_channel_affine_quantized, torch.empty_strided, torch.empty_like, torch.scalar_tensor, torch.tril_indices, torch.bartlett_window, torch.ones, torch.sparse_coo_tensor, torch.randn, torch.kaiser_window, torch.tensor, torch.triu_indices, torch.as_tensor, torch.zeros, torch.randint_like, torch.full, torch.eye, torch._sparse_csr_tensor_unsafe, torch.empty, torch._sparse_coo_tensor_unsafe, torch.blackman_window, torch.zeros_like, torch.range, torch.sparse_csr_tensor, torch.randn_like, torch.from_file, torch._cudnn_init_dropout_state, torch._empty_affine_quantized, torch.linspace, torch.hamming_window, torch.empty_quantized, torch._pin_memory, torch.autocast, torch.load, torch.Generator, torch.Tensor.new_empty, torch.Tensor.new_empty_strided, torch.Tensor.new_full, torch.Tensor.new_ones, torch.Tensor.new_tensor, torch.Tensor.new_zeros, torch.Tensor.to, torch.nn.Module.to, torch.nn.Module.to_empty
    *************************************************************************************************************
    
  warnings.warn(msg, ImportWarning)
/home/liwb/anaconda3/envs/torch21/lib/python3.10/site-packages/torchvision/io/image.py:13: UserWarning: Failed to load image Python extension: '/home/liwb/anaconda3/envs/torch21/lib/python3.10/site-packages/torchvision/image.so: undefined symbol: _ZN5torch3jit17parseSchemaOrNameERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE'If you don't plan on using image functionality from `torchvision.io`, you can ignore this warning. Otherwise, there might be something wrong with your environment. Did you have `libjpeg` or `libpng` installed before building `torchvision` from source?
  warn(
Using npu device
/usr/local/Ascend/ascend-toolkit/7.0.0/python/site-packages/tbe/tvm/contrib/ccec.py:769: DeprecationWarning: invalid escape sequence '\L'
  if not dirpath.find("AppData\Local\Temp"):
LeNet5(
  (conv1): Sequential(
    (0): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))
    (1): ReLU()
  )
  (pooling1): AvgPool2d(kernel_size=2, stride=2, padding=0)
  (conv2): Sequential(
    (0): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))
    (1): ReLU()
  )
  (pooling2): AvgPool2d(kernel_size=2, stride=2, padding=0)
  (fc1): Sequential(
    (0): Linear(in_features=400, out_features=120, bias=True)
    (1): ReLU()
  )
  (fc2): Sequential(
    (0): Linear(in_features=120, out_features=84, bias=True)
    (1): ReLU()
  )
  (fc3): Linear(in_features=84, out_features=10, bias=True)
)
Using npu device
Epoch 1
-------------------------------
EH9999: Inner Error!
EH9999  [Init][Env]init env failed![FUNC:ReportInnerError][FILE:log_inner.cpp][LINE:145]
        TraceBack (most recent call last):
        build op model failed, result = 500001[FUNC:ReportInnerError][FILE:log_inner.cpp][LINE:145]

Warning: Device do not support double dtype now, dtype cast repalce with float.

```

请根据自己的运行环境参考以下方式搜集日志信息，如果涉及到算子开发相关的问题，建议也提供UT/ST测试和单算子集成测试相关的日志。

日志提供方式:
将日志打包后作为附件上传。若日志大小超出附件限制，则可上传至外部网盘后提供链接。

获取方法请参考wiki：
https://gitee.com/ascend/modelzoo/wikis/%E5%A6%82%E4%BD%95%E8%8E%B7%E5%8F%96%E6%97%A5%E5%BF%97%E5%92%8C%E8%AE%A1%E7%AE%97%E5%9B%BE?sort_id=4097825

Ascend/pytorch
暂停

内容风险标识

评论 (2)

Ascend/pytorch暂停 .gitee-modal { width: 500px !important; }

内容风险标识