Pangu_alpha模型数据转换报错

环境信息：Atlas800-9000 arm Euler2.8 
CANN：CANN6.0.RC1 
Python3.7.5

[root@4a03d154aabc Pangu_alpha]#  python3 -c "import mindspore as ms;ms.run_check()"
MindSpore version:  1.9.0
The result of multiplication calculation is correct, MindSpore has been installed successfull
![输入图片说明](https://foruda.gitee.com/images/1683798054950925958/2efd7ea7_8021201.png "屏幕截图")

模型与代码：https://gitee.com/mindspore/models/tree/master/official/nlp/Pangu_alpha

数据目录
[root@4a03d154aabc Pangu_alpha]#  ll data/
total 54172
-rw------- 1 root root  4152914 May 11 09:28 153331.txt
-rw------- 1 root root 15301163 May 11 09:28 22298.txt
-rw------- 1 root root 12125907 May 11 09:28 260878.txt
-rw------- 1 root root 23886830 May 11 09:28 2.txt

执行命令：
python3 -m src.preprocess --input_glob  'data/*.txt' --tokenizer gpt --eot 50256 --data_column_name input_ids --seq_length 1025

root@4a03d154aabc Pangu_alpha]# python3 -m src.preprocess --input_glob  'data/*.txt' --tokenizer gpt --eot 50256 --data_column_name input_ids --seq_length 1025
[WARNING] ME(8821:281473787678720,MainProcess):2023-05-11-09:28:49.481.980 [mindspore/mindrecord/filewriter.py:278] This interface will be deleted or invisible in the future.
[ERROR] CORE(8892,ffffb920d000,python3):2023-05-11-09:34:20.005.200 [mindspore/core/utils/file_utils.cc:253] GetRealPath] Get realpath failed, path[/home/wukong/models/official/nlp/Pangu_alpha/output/transfered_mindrecord_Pages]
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
  File "/usr/local/python37/lib/python3.7/multiprocessing/pool.py", line 121, in worker
    result = (True, func(*args, **kwds))
  File "/usr/local/python37/lib/python3.7/multiprocessing/pool.py", line 44, in mapstar
    return list(map(*args))
  File "/home/wukong/models/official/nlp/Pangu_alpha/src/preprocess.py", line 154, in task_unit
    writer.write_raw_data(data_batch, parallel_writer=parallel_writer)
  File "/usr/local/python37/lib/python3.7/site-packages/mindspore/mindrecord/filewriter.py", line 323, in write_raw_data
    return self._writer.write_raw_data(raw_data, True, parallel_writer)
  File "/usr/local/python37/lib/python3.7/site-packages/mindspore/mindrecord/shardwriter.py", line 178, in write_raw_data
    ret = self._writer.write_raw_data(raw_data, blob_data, validate, parallel_writer)
RuntimeError: Unexpected error. [Internal ERROR] Error raised in FileToPages function.
Line of code : 531
File         : mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc

"""

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/usr/local/python37/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/usr/local/python37/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/home/wukong/models/official/nlp/Pangu_alpha/src/preprocess.py", line 220, in <module>
    pool.map(map_func, package_file(file_iter, args.file_batch_size))
  File "/usr/local/python37/lib/python3.7/multiprocessing/pool.py", line 268, in map
    return self._map_async(func, iterable, mapstar, chunksize).get()
  File "/usr/local/python37/lib/python3.7/multiprocessing/pool.py", line 657, in get
    raise self._value
RuntimeError: Unexpected error. [Internal ERROR] Error raised in FileToPages function.
Line of code : 531
File         : mindspore/ccsrc/minddata/mindrecord/io/shard_writer.cc
![输入图片说明](https://foruda.gitee.com/images/1683798080533030749/65a00f39_8021201.png "屏幕截图")

MindSpore/models

内容风险标识

评论 (5)

MindSpore/models .gitee-modal { width: 500px !important; }

内容风险标识