310P3 npu启动qwen2-vl-7b推理服务失败

一、问题现象（附报错日志上下文）：
310P3 npu启动qwen2-vl-7b推理服务失败
二、软件版本:
使用官方mindie镜像，镜像版本为1.0.0-300I-Duo-py311-openeuler24.03-lts
-- CANN 版本: CANN 8.0.0
--Tensorflow/Pytorch/MindSpore 版本:
--Python 版本 (e.g., Python 3.7.5):Python 3.11.6
-- MindStudio版本: MindStudio 2.0.0 (beta3)
--操作系统版本:openEuler 24.03

三、测试步骤：
按照[官网文档](https://www.hiascend.com/software/modelzoo/models/detail/f2fdfa4188184630a246e0468692612a)配置环境进行服务化推理，使用./mindieservice_daemon命令启动服务化，开始正常打印日志，然后就卡在[2025-04-24 11:00:01,932] [319] [281466620866912] [llm] [INFO] [logging.py-331] : >>>>>>id of kcache is 281468909085968 id of vcache is 281468909086064不动，无法正常拉起服务，/usr/local/Ascend/mindie/latest/mindie-service/conf/config.json配置文件信息如下：

```
{
    "Version" : "1.1.0",
    "LogConfig" :
    {
        "logLevel" : "Info",
        "logFileSize" : 20,
        "logFileNum" : 20,
        "logPath" : "logs/mindservice.log"
    },

"ServerConfig" :
    {
        "ipAddress" : "127.0.0.1",
        "managementIpAddress" : "127.0.0.2",
        "port" : 9200,
        "managementPort" : 9201,
        "metricsPort" : 9202,
        "allowAllZeroIpListening" : false,
        "maxLinkNum" : 1000,
        "httpsEnabled" : false,
        "fullTextEnabled" : false,
        "tlsCaPath" : "security/ca/",
        "tlsCaFile" : ["ca.pem"],
        "tlsCert" : "security/certs/server.pem",
        "tlsPk" : "security/keys/server.key.pem",
        "tlsPkPwd" : "security/pass/key_pwd.txt",
        "tlsCrlPath" : "security/certs/",
        "tlsCrlFiles" : ["server_crl.pem"],
        "managementTlsCaFile" : ["management_ca.pem"],
        "managementTlsCert" : "security/certs/management/server.pem",
        "managementTlsPk" : "security/keys/management/server.key.pem",
        "managementTlsPkPwd" : "security/pass/management/key_pwd.txt",
        "managementTlsCrlPath" : "security/management/certs/",
        "managementTlsCrlFiles" : ["server_crl.pem"],
        "kmcKsfMaster" : "tools/pmt/master/ksfa",
        "kmcKsfStandby" : "tools/pmt/standby/ksfb",
        "inferMode" : "standard",
        "interCommTLSEnabled" : true,
        "interCommPort" : 1121,
        "interCommTlsCaPath" : "security/grpc/ca/",
        "interCommTlsCaFiles" : ["ca.pem"],
        "interCommTlsCert" : "security/grpc/certs/server.pem",
        "interCommPk" : "security/grpc/keys/server.key.pem",
        "interCommPkPwd" : "security/grpc/pass/key_pwd.txt",
        "interCommTlsCrlPath" : "security/grpc/certs/",
        "interCommTlsCrlFiles" : ["server_crl.pem"],
        "openAiSupport" : "vllm"
    },

"BackendConfig" : {
        "backendName" : "mindieservice_llm_engine",
        "modelInstanceNumber" : 1,
        "npuDeviceIds" : [[0,1,2,3]],
        "tokenizerProcessNumber" : 8,
        "multiNodesInferEnabled" : false,
        "multiNodesInferPort" : 1120,
        "interNodeTLSEnabled" : true,
        "interNodeTlsCaPath" : "security/grpc/ca/",
        "interNodeTlsCaFiles" : ["ca.pem"],
        "interNodeTlsCert" : "security/grpc/certs/server.pem",
        "interNodeTlsPk" : "security/grpc/keys/server.key.pem",
        "interNodeTlsPkPwd" : "security/grpc/pass/mindie_server_key_pwd.txt",
        "interNodeTlsCrlPath" : "security/grpc/certs/",
        "interNodeTlsCrlFiles" : ["server_crl.pem"],
        "interNodeKmcKsfMaster" : "tools/pmt/master/ksfa",
        "interNodeKmcKsfStandby" : "tools/pmt/standby/ksfb",
        "ModelDeployConfig" :
        {
            "maxSeqLen" : 2560,
            "maxInputTokenLen" : 2048,
            "truncation" : false,
            "ModelConfig" : [
                {
                    "modelInstanceType" : "Standard",
                    "modelName" : "qwen2_vl",
                    "modelWeightPath" : "/mnt/Qwen2-VL-7B-Instruct",
                    "worldSize" : 4,
                    "cpuMemSize" : 5,
                    "npuMemSize" : 8,
                    "backendType" : "atb",
                    "trustRemoteCode" : false
                }
            ]
        },

"ScheduleConfig" :
        {
            "templateType" : "Standard",
            "templateName" : "Standard_LLM",
            "cacheBlockSize" : 128,

"maxPrefillBatchSize" : 50,
            "maxPrefillTokens" : 8192,
            "prefillTimeMsPerReq" : 150,
            "prefillPolicyType" : 0,

"decodeTimeMsPerReq" : 50,
            "decodePolicyType" : 0,

"maxBatchSize" : 200,
            "maxIterTimes" : 512,
            "maxPreemptCount" : 0,
            "supportSelectBatch" : false,
            "maxQueueDelayMicroseconds" : 5000
        }
    }
}
```

四、日志信息:

```
[root@9cb33e04f2ed bin]# ./mindieservice_daemon
2025-04-24 13:19:34,206 [INFO] config.py:30 - model_config {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '2', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/mnt/Qwen2-VL-7B-Instruct', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'qwen2_vl', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '2', 'npu_device_ids': '0,1,2,3', 'npu_mem': '8', 'pp': '0', 'rank': '2', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '4'}
2025-04-24 13:19:34,206 [INFO] config.py:35 - do not init dmi config
2025-04-24 13:19:34,206 [INFO] standard_model.py:145 - rank id 2 get model config: {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '2', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/mnt/Qwen2-VL-7B-Instruct', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'qwen2_vl', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '2', 'npu_device_ids': '0,1,2,3', 'npu_mem': '8', 'pp': '0', 'rank': '2', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '4'}
2025-04-24 13:19:34,294 [INFO] config.py:30 - model_config {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '1', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/mnt/Qwen2-VL-7B-Instruct', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'qwen2_vl', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '1', 'npu_device_ids': '0,1,2,3', 'npu_mem': '8', 'pp': '0', 'rank': '1', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '4'}
2025-04-24 13:19:34,294 [INFO] config.py:35 - do not init dmi config
2025-04-24 13:19:34,294 [INFO] standard_model.py:145 - rank id 1 get model config: {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '1', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/mnt/Qwen2-VL-7B-Instruct', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'qwen2_vl', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '1', 'npu_device_ids': '0,1,2,3', 'npu_mem': '8', 'pp': '0', 'rank': '1', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '4'}
2025-04-24 13:19:34,298 [INFO] config.py:30 - model_config {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '0', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/mnt/Qwen2-VL-7B-Instruct', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'qwen2_vl', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '0', 'npu_device_ids': '0,1,2,3', 'npu_mem': '8', 'pp': '0', 'rank': '0', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '4'}
2025-04-24 13:19:34,299 [INFO] config.py:35 - do not init dmi config
2025-04-24 13:19:34,299 [INFO] standard_model.py:145 - rank id 0 get model config: {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '0', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/mnt/Qwen2-VL-7B-Instruct', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'qwen2_vl', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '0', 'npu_device_ids': '0,1,2,3', 'npu_mem': '8', 'pp': '0', 'rank': '0', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '4'}
2025-04-24 13:19:34,371 [INFO] config.py:30 - model_config {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '3', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/mnt/Qwen2-VL-7B-Instruct', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'qwen2_vl', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '3', 'npu_device_ids': '0,1,2,3', 'npu_mem': '8', 'pp': '0', 'rank': '3', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '4'}
2025-04-24 13:19:34,371 [INFO] config.py:35 - do not init dmi config
2025-04-24 13:19:34,371 [INFO] standard_model.py:145 - rank id 3 get model config: {'backend_bin_path': '/usr/local/Ascend/mindie/1.0.0/mindie-llm/bin/', 'backend_log_file': '/usr/local/Ascend/mindie/1.0.0/mindie-service/logs/mindservice.log', 'backend_modelInstance_id': '0', 'backend_type': 'atb', 'block_size': '128', 'cpu_mem': '5', 'deploy_type': 'INTER_PROCESS', 'dp': '0', 'ep': '0', 'executor_type': 'LLM_EXECUTOR_PYTHON', 'globalRankIds': '', 'globalWorldSize': '0', 'interNodeKmcKsfMaster': 'tools/pmt/master/ksfa', 'interNodeKmcKsfStandby': 'tools/pmt/standby/ksfb', 'interNodeTLSEnabled': '1', 'interNodeTlsCaFiles': 'ca.pem,', 'interNodeTlsCaPath': 'security/grpc/ca/', 'interNodeTlsCert': 'security/grpc/certs/server.pem', 'interNodeTlsCrlFiles': 'server_crl.pem,', 'interNodeTlsCrlPath': 'security/grpc/certs/', 'interNodeTlsPk': 'security/grpc/keys/server.key.pem', 'interNodeTlsPkPwd': 'security/grpc/pass/mindie_server_key_pwd.txt', 'isMaster': '0', 'localIP': '', 'local_rank': '3', 'log_error': '1', 'log_file_num': '20', 'log_file_size': '20', 'log_info': '1', 'log_verbose': '0', 'log_warning': '1', 'masterIP': '', 'max_input_len': '2048', 'max_iter_times': '512', 'max_prefill_tokens': '8192', 'max_seq_len': '2560', 'model_cut_policy': 'standard', 'model_id': '/mnt/Qwen2-VL-7B-Instruct', 'model_instance_number': '1', 'model_instance_type': 'Standard', 'model_name': 'qwen2_vl', 'multiNodesInferEnabled': '0', 'multiNodesInferPort': '1120', 'npu_device_id': '3', 'npu_device_ids': '0,1,2,3', 'npu_mem': '8', 'pp': '0', 'rank': '3', 'slaveIPs': '', 'speculation_gamma': '0', 'tp': '0', 'trust_remote_code': '0', 'world_size': '4'}
[2025-04-24 13:19:38,972] [668] [281465156596064] [llm] [INFO] [cpu_binding.py-212] : rank_id: 2, device_id: 2, numa_id: 1, shard_devices: [2, 3], cpus: [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]
[2025-04-24 13:19:38,973] [668] [281465156596064] [llm] [INFO] [cpu_binding.py-238] : process 668, new_affinity is [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35], cpu count 12
[2025-04-24 13:19:39,086] [666] [281465063666016] [llm] [INFO] [cpu_binding.py-212] : rank_id: 1, device_id: 1, numa_id: 0, shard_devices: [0, 1], cpus: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
[2025-04-24 13:19:39,088] [666] [281465063666016] [llm] [INFO] [cpu_binding.py-238] : process 666, new_affinity is [12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], cpu count 12
[2025-04-24 13:19:39,274] [664] [281451871400288] [llm] [INFO] [cpu_binding.py-212] : rank_id: 0, device_id: 0, numa_id: 0, shard_devices: [0, 1], cpus: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23]
[2025-04-24 13:19:39,276] [664] [281451871400288] [llm] [INFO] [cpu_binding.py-238] : process 664, new_affinity is [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11], cpu count 12
[2025-04-24 13:19:39,350] [670] [281459638661472] [llm] [INFO] [cpu_binding.py-212] : rank_id: 3, device_id: 3, numa_id: 1, shard_devices: [2, 3], cpus: [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47]
[2025-04-24 13:19:39,353] [670] [281459638661472] [llm] [INFO] [cpu_binding.py-238] : process 670, new_affinity is [36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47], cpu count 12
[2025-04-24 13:19:39,910] [664] [281451871400288] [llm] [INFO] [logging.py-331] : model_runner.quantize: None, model_runner.kv_quant_type: None, model_runner.fa_quant_type: None, model_runner.dtype: torch.float16
[2025-04-24 13:19:44,669] [666] [281465063666016] [llm] [INFO] [dist.py-77] : initialize_distributed has been Set
[2025-04-24 13:19:44,679] [666] [281465063666016] [llm] [INFO] [dist.py-98] : ProcessGroupHCCL has been Set
[W socket.cpp:663] [c10d] The client socket has failed to connect to [localhost]:7896 (errno: 99 - Cannot assign requested address).
[2025-04-24 13:19:45,202] [664] [281451871400288] [llm] [INFO] [dist.py-77] : initialize_distributed has been Set
[2025-04-24 13:19:45,202] [664] [281451871400288] [llm] [INFO] [logging.py-331] : init tokenizer done: Qwen2TokenizerFast(name_or_path='/mnt/Qwen2-VL-7B-Instruct', vocab_size=151643, model_max_length=32768, is_fast=True, padding_side='left', truncation_side='right', special_tokens={'eos_token': '<|im_end|>', 'pad_token': '<|endoftext|>', 'additional_special_tokens': ['<|im_start|>', '<|im_end|>', '<|object_ref_start|>', '<|object_ref_end|>', '<|box_start|>', '<|box_end|>', '<|quad_start|>', '<|quad_end|>', '<|vision_start|>', '<|vision_end|>', '<|vision_pad|>', '<|image_pad|>', '<|video_pad|>']}, clean_up_tokenization_spaces=False),  added_tokens_decoder={
        151643: AddedToken("<|endoftext|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151644: AddedToken("<|im_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151645: AddedToken("<|im_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151646: AddedToken("<|object_ref_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151647: AddedToken("<|object_ref_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151648: AddedToken("<|box_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151649: AddedToken("<|box_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151650: AddedToken("<|quad_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151651: AddedToken("<|quad_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151652: AddedToken("<|vision_start|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151653: AddedToken("<|vision_end|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151654: AddedToken("<|vision_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151655: AddedToken("<|image_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
        151656: AddedToken("<|video_pad|>", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),
}
[2025-04-24 13:19:45,213] [664] [281451871400288] [llm] [INFO] [dist.py-98] : ProcessGroupHCCL has been Set
[W compiler_depend.ts:714] Warning: The HCCL execution timeout 1800000ms is bigger than watchdog timeout 300000ms which is set by init_process_group! The plog may not be recorded. (function ProcessGroupHCCL)
[2025-04-24 13:19:45,681] [666] [281465063666016] [llm] [INFO] [dist.py-112] : rank 1 init True, init_process_group has been activated
[2025-04-24 13:19:49,825] [666] [281465063666016] [llm] [INFO] [flash_causal_qwen2.py-115] : >>>> qwen_QwenDecoderModel is called.
[2025-04-24 13:20:04,305] [666] [281465063666016] [llm] [INFO] [flash_causal_qwen2.py-394] : <<<<<<<after transdata k_caches[0].shape=torch.Size([4681, 8, 128, 16])
.[W compiler_depend.ts:714] Warning: The HCCL execution timeout 1800000ms is bigger than watchdog timeout 300000ms which is set by init_process_group! The plog may not be recorded. (function ProcessGroupHCCL)
[2025-04-24 13:24:46,217] [664] [281451871400288] [llm] [INFO] [dist.py-112] : rank 0 init True, init_process_group has been activated
[2025-04-24 13:24:46,220] [664] [281451871400288] [llm] [INFO] [logging.py-331] : NPUSocInfo(soc_name='', soc_version=202, need_nz=True, matmul_nd_nz=False)
[2025-04-24 13:24:48,489] [664] [281451871400288] [llm] [INFO] [logging.py-331] : NPUSocInfo(soc_name='', soc_version=202, need_nz=True, matmul_nd_nz=False)
[2025-04-24 13:24:50,020] [664] [281451871400288] [llm] [INFO] [flash_causal_qwen2.py-115] : >>>> qwen_QwenDecoderModel is called.
[2025-04-24 13:25:01,000] [664] [281451871400288] [llm] [INFO] [logging.py-331] : model:
 FlashQwen2vlForCausalLM(
  (rotary_embedding): PositionRotaryEmbedding()
  (attn_mask): AttentionMask()
  (vision_tower): Qwen2VisionTransformerPretrainedModelATB(
    (patch_embed): PatchEmbed(
      (proj): Conv3d(3, 1280, kernel_size=(2, 14, 14), stride=(2, 14, 14), bias=False)
    )
    (rotary_pos_emb): VisionRotaryEmbedding()
    (encoder): Qwen2VLVisionEncoder(
      (layers): ModuleList(
        (0-31): 32 x Qwen2VLVisionBlock(
          (norm1): LayerNormATB()
          (attn): VisionAttention(
            (qkv): TensorParallelColumnLinear(
              (linear): FastLinear()
            )
            (proj): TensorParallelRowLinear(
              (linear): FastLinear()
            )
          )
          (norm2): LayerNormATB()
          (mlp): VisionMlp(
            (fc1): TensorParallelColumnLinear(
              (linear): FastLinear()
            )
            (fc2): TensorParallelRowLinear(
              (linear): FastLinear()
            )
          )
        )
      )
    )
    (merger): PatchMerger(
      (ln_q): LayerNorm((1280,), eps=1e-06, elementwise_affine=True)
      (mlp): Sequential(
        (0): Linear(in_features=5120, out_features=5120, bias=True)
        (1): GELU(approximate='none')
        (2): Linear(in_features=5120, out_features=3584, bias=True)
      )
    )
  )
  (language_model): FlashQwen2UsingMROPEForCausalLM(
    (rotary_embedding): PositionRotaryEmbedding()
    (attn_mask): AttentionMask()
    (transformer): FlashQwenModel(
      (wte): TensorEmbeddingWithoutChecking()
      (h): ModuleList(
        (0-27): 28 x FlashQwenLayer(
          (attn): FlashQwenAttention(
            (rotary_emb): PositionRotaryEmbedding()
            (c_attn): TensorParallelColumnLinear(
              (linear): FastLinear()
            )
            (c_proj): TensorParallelRowLinear(
              (linear): FastLinear()
            )
          )
          (mlp): QwenMLP(
            (act): SiLU()
            (w2_w1): TensorParallelColumnLinear(
              (linear): FastLinear()
            )
            (c_proj): TensorParallelRowLinear(
              (linear): FastLinear()
            )
          )
          (ln_1): QwenRMSNorm()
          (ln_2): QwenRMSNorm()
        )
      )
      (ln_f): QwenRMSNorm()
    )
    (lm_head): TensorParallelHead(
      (linear): FastLinear()
    )
  )
  (normalizer): Conv1d(3, 3, kernel_size=(1,), stride=(1,), groups=3)
)
[2025-04-24 13:25:01,139] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,139] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,140] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,141] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,141] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,142] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,144] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,144] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,145] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,145] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,145] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,146] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,146] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,147] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,147] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,147] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,148] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,148] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,149] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,149] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,150] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,150] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,150] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,151] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,151] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,152] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,152] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,153] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,153] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,154] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,154] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,154] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,155] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,155] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,156] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,156] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,157] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,157] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,158] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,158] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,158] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,159] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,159] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,159] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,160] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,160] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,161] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,161] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,162] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,162] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,163] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,163] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,163] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,164] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,164] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,165] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,165] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,165] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,166] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,166] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,167] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,167] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,167] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,168] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,168] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,169] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,169] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,169] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,170] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,170] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,171] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,171] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,172] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,172] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,173] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,173] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,174] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,174] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,174] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,175] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,175] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,176] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,176] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,177] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,177] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,178] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,178] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,178] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,179] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,179] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,179] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,180] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,180] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,181] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,181] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,181] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,182] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,182] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,183] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,183] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,184] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,184] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,184] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,185] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,185] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,186] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,186] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,186] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,187] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,187] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,188] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,188] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,193] [664] [281451871400288] [llm] [INFO] [logging.py-331] : trans to 29
[2025-04-24 13:25:01,747] [664] [281451871400288] [llm] [INFO] [logging.py-331] : <<<<<<< ori k_caches[0].shape=torch.Size([4681, 8, 128, 16])
[2025-04-24 13:25:01,748] [664] [281451871400288] [llm] [INFO] [flash_causal_qwen2.py-394] : <<<<<<<after transdata k_caches[0].shape=torch.Size([4681, 8, 128, 16])
[2025-04-24 13:25:01,749] [664] [281451871400288] [llm] [INFO] [logging.py-331] : >>>>>>id of kcache is 281451563161872 id of vcache is 281451563161968
.[2025-04-24 13:35:03.12988] [error] [1085] [hccl_runner.cpp:224] AllReduceHcclRunner:0 barrier fail, check all ready timeout
[2025-04-24 13:35:03.12986] [error] [1084] [hccl_runner.cpp:224] AllReduceHcclRunner:0 barrier fail, check all ready timeout
[2025-04-24 13:35:03.13140] [error] [1085] [comm_pool.h:43] CommPool commCreateFunc fail
[2025-04-24 13:35:03.13160] [error] [1085] [hccl_runner.cpp:72] AllReduceHcclRunner:0 get hccl comm fail by rank:0
[2025-04-24 13:35:03.13208] [error] [1084] [comm_pool.h:43] CommPool commCreateFunc fail
[2025-04-24 13:35:03.13224] [error] [1084] [hccl_runner.cpp:72] AllReduceHcclRunner:0 get hccl comm fail by rank:1
[2025-04-24 13:45:04.12876] [error] [1085] [hccl_runner.cpp:224] AllReduceHcclRunner:0 barrier fail, check all ready timeout
[2025-04-24 13:45:04.12877] [error] [1084] [hccl_runner.cpp:224] AllReduceHcclRunner:0 barrier fail, check all ready timeout
[2025-04-24 13:45:04.13036] [error] [1085] [comm_pool.h:43] CommPool commCreateFunc fail
[2025-04-24 13:45:04.13051] [error] [1085] [hccl_runner.cpp:72] AllReduceHcclRunner:0 get hccl comm fail by rank:0
[2025-04-24 13:45:04.13096] [error] [1084] [comm_pool.h:43] CommPool commCreateFunc fail
[2025-04-24 13:45:04.13133] [error] [1084] [hccl_runner.cpp:72] AllReduceHcclRunner:0 get hccl comm fail by rank:1
[2025-04-24 13:45:04.35232] [error] [2211] [all_reduce_hccl_runner.cpp:38] hcclComm is null, rank: 1
[2025-04-24 13:45:04.35274] [error] [2211] [runner.cpp:115] AllReduceHcclRunner_2_0_4_1:1 Execute Failed. st: 3
[2025-04-24 13:45:04.35289] [error] [2211] [graph_runner.cpp:906] LinearRowParallelNoAddRunner_2_0_4:0  node[1] execute fail, runner name:AllReduceHcclRunner
[2025-04-24 13:45:04.35299] [error] [2211] [runner.cpp:115] LinearRowParallelNoAddRunner_2_0_4:1 Execute Failed. st: 3
[2025-04-24 13:45:04.35321] [error] [2211] [graph_runner.cpp:906] AttentionRunner_2_0:0  node[4] execute fail, runner name:LinearRowParallelNoAddRunner
[2025-04-24 13:45:04.35330] [error] [2211] [runner.cpp:115] AttentionRunner_2_0:1 Execute Failed. st: 3
[2025-04-24 13:45:04.35336] [error] [2211] [graph_runner.cpp:906] Prefill_layerRunner_2:0  node[0] execute fail, runner name:AttentionRunner
[2025-04-24 13:45:04.35342] [error] [2211] [runner.cpp:115] Prefill_layerRunner_2:1 Execute Failed. st: 3
[2025-04-24 13:45:04.35348] [error] [2211] [operation_base.cpp:726] Prefill_layer_2 execute Prefill_layerRunner fail
[2025-04-24 13:45:04.35354] [error] [2211] [operation_base.cpp:764] Prefill_layer_2 execute fail, error code: 3
[2025-04-24 13:45:04.38231] [error] [2859] [all_reduce_hccl_runner.cpp:38] hcclComm is null, rank: 0
[2025-04-24 13:45:04.38261] [error] [2859] [runner.cpp:115] AllReduceHcclRunner_2_0_4_1:1 Execute Failed. st: 3
[2025-04-24 13:45:04.38272] [error] [2859] [graph_runner.cpp:906] LinearRowParallelNoAddRunner_2_0_4:0  node[1] execute fail, runner name:AllReduceHcclRunner
[2025-04-24 13:45:04.38282] [error] [2859] [runner.cpp:115] LinearRowParallelNoAddRunner_2_0_4:1 Execute Failed. st: 3
[2025-04-24 13:45:04.38289] [error] [2859] [graph_runner.cpp:906] AttentionRunner_2_0:0  node[4] execute fail, runner name:LinearRowParallelNoAddRunner
[2025-04-24 13:45:04.38306] [error] [2859] [runner.cpp:115] AttentionRunner_2_0:1 Execute Failed. st: 3
[2025-04-24 13:45:04.38312] [error] [2859] [graph_runner.cpp:906] Prefill_layerRunner_2:0  node[0] execute fail, runner name:AttentionRunner
[2025-04-24 13:45:04.38318] [error] [2859] [runner.cpp:115] Prefill_layerRunner_2:1 Execute Failed. st: 3
[2025-04-24 13:45:04.38325] [error] [2859] [operation_base.cpp:726] Prefill_layer_2 execute Prefill_layerRunner fail
[2025-04-24 13:45:04.38331] [error] [2859] [operation_base.cpp:764] Prefill_layer_2 execute fail, error code: 3
[2025-04-24 13:55:05.12880] [error] [1085] [hccl_runner.cpp:224] AllReduceHcclRunner:0 barrier fail, check all ready timeout
[2025-04-24 13:55:05.12878] [error] [1084] [hccl_runner.cpp:224] AllReduceHcclRunner:0 barrier fail, check all ready timeout
[2025-04-24 13:55:05.13041] [error] [1085] [comm_pool.h:43] CommPool commCreateFunc fail
[2025-04-24 13:55:05.13054] [error] [1085] [hccl_runner.cpp:72] AllReduceHcclRunner:0 get hccl comm fail by rank:0
[2025-04-24 13:55:05.13095] [error] [1084] [comm_pool.h:43] CommPool commCreateFunc fail
[2025-04-24 13:55:05.13108] [error] [1084] [hccl_runner.cpp:72] AllReduceHcclRunner:0 get hccl comm fail by rank:1
[2025-04-24 14:05:06.12875] [error] [1085] [hccl_runner.cpp:224] AllReduceHcclRunner:0 barrier fail, check all ready timeout
[2025-04-24 14:05:06.12886] [error] [1084] [hccl_runner.cpp:224] AllReduceHcclRunner:0 barrier fail, check all ready timeout
[2025-04-24 14:05:06.13034] [error] [1085] [comm_pool.h:43] CommPool commCreateFunc fail
[2025-04-24 14:05:06.13047] [error] [1085] [hccl_runner.cpp:72] AllReduceHcclRunner:0 get hccl comm fail by rank:0
[2025-04-24 14:05:06.13081] [error] [1084] [comm_pool.h:43] CommPool commCreateFunc fail
[2025-04-24 14:05:06.13093] [error] [1084] [hccl_runner.cpp:72] AllReduceHcclRunner:0 get hccl comm fail by rank:1
[2025-04-24 14:05:06.14973] [error] [2211] [all_reduce_hccl_runner.cpp:38] hcclComm is null, rank: 1
[2025-04-24 14:05:06.14999] [error] [2211] [runner.cpp:115] AllReduceHcclRunner_3_0_4_1:1 Execute Failed. st: 3
[2025-04-24 14:05:06.15017] [error] [2211] [graph_runner.cpp:906] LinearRowParallelNoAddRunner_3_0_4:0  node[1] execute fail, runner name:AllReduceHcclRunner
[2025-04-24 14:05:06.15027] [error] [2211] [runner.cpp:115] LinearRowParallelNoAddRunner_3_0_4:1 Execute Failed. st: 3
[2025-04-24 14:05:06.15034] [error] [2211] [graph_runner.cpp:906] AttentionRunner_3_0:0  node[4] execute fail, runner name:LinearRowParallelNoAddRunner
[2025-04-24 14:05:06.15019] [error] [2859] [all_reduce_hccl_runner.cpp:38] hcclComm is null, rank: 0
[2025-04-24 14:05:06.15043] [error] [2211] [runner.cpp:115] AttentionRunner_3_0:1 Execute Failed. st: 3
[2025-04-24 14:05:06.15062] [error] [2211] [graph_runner.cpp:906] Prefill_layerRunner_3:0  node[0] execute fail, runner name:AttentionRunner
[2025-04-24 14:05:06.15055] [error] [2859] [runner.cpp:115] AllReduceHcclRunner_3_0_4_1:1 Execute Failed. st: 3
[2025-04-24 14:05:06.15083] [error] [2211] [runner.cpp:115] Prefill_layerRunner_3:1 Execute Failed. st: 3
[2025-04-24 14:05:06.15091] [error] [2211] [operation_base.cpp:726] Prefill_layer_3 execute Prefill_layerRunner fail
[2025-04-24 14:05:06.15090] [error] [2859] [graph_runner.cpp:906] LinearRowParallelNoAddRunner_3_0_4:0  node[1] execute fail, runner name:AllReduceHcclRunner
[2025-04-24 14:05:06.15096] [error] [2211] [operation_base.cpp:764] Prefill_layer_3 execute fail, error code: 3
[2025-04-24 14:05:06.15105] [error] [2859] [runner.cpp:115] LinearRowParallelNoAddRunner_3_0_4:1 Execute Failed. st: 3
[2025-04-24 14:05:06.15130] [error] [2859] [graph_runner.cpp:906] AttentionRunner_3_0:0  node[4] execute fail, runner name:LinearRowParallelNoAddRunner
[2025-04-24 14:05:06.15137] [error] [2859] [runner.cpp:115] AttentionRunner_3_0:1 Execute Failed. st: 3
[2025-04-24 14:05:06.15143] [error] [2859] [graph_runner.cpp:906] Prefill_layerRunner_3:0  node[0] execute fail, runner name:AttentionRunner
[2025-04-24 14:05:06.15149] [error] [2859] [runner.cpp:115] Prefill_layerRunner_3:1 Execute Failed. st: 3
[2025-04-24 14:05:06.15155] [error] [2859] [operation_base.cpp:726] Prefill_layer_3 execute Prefill_layerRunner fail
[2025-04-24 14:05:06.15161] [error] [2859] [operation_base.cpp:764] Prefill_layer_3 execute fail, error code: 3
2025-04-24 14:05:06,015 [ERROR] model.py:39 - [Model]   >>> Exception:Execute fail, enable log: export ASDOPS_LOG_LEVEL=ERROR, export ASDOPS_LOG_TO_STDOUT=1 to findthe first error. For more details, see the MindIE official document. 
Traceback (most recent call last):
  File "/usr/local/lib/python3.11/site-packages/model_wrapper/model.py", line 37, in initialize
    return self.python_model.initialize(config)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/model_wrapper/standard_model.py", line 146, in initialize
    self.generator = Generator(
                     ^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 119, in __init__
    self.warm_up(max_prefill_tokens, max_seq_len, max_input_len, max_iter_times, inference_mode)
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 356, in warm_up
    raise e
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 348, in warm_up
    self._generate_inputs_warm_up_backend(input_metadata, inference_mode)
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 378, in _generate_inputs_warm_up_backend
    self.generator_backend.warm_up(model_inputs, inference_mode=inference_mode)
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 198, in warm_up
    super().warm_up(model_inputs)
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_backend.py", line 170, in warm_up
    _ = self.forward(model_inputs, **kwargs)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/utils/decorators/time_decorator.py", line 38, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 153, in forward
    logits = self.model_wrapper.forward(model_inputs, self.cache_pool.npu_cache, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 89, in forward
    logits = self.forward_tensor(
             ^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 116, in forward_tensor
    logits = self.model_runner.forward(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Ascend/atb-models/atb_llm/runner/model_runner.py", line 193, in forward
    return self.model.forward(**kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2_vl/flash_causal_qwen2_vl.py", line 203, in forward
    return self.language_model.forward(inputs_embeds,
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2_vl/flash_causal_qwen2_using_mrope.py", line 142, in forward
    logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2/flash_causal_qwen2.py", line 374, in execute_ascend_operator
    acl_model_out = model_operation.execute(acl_inputs, acl_param)
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: Execute fail, enable log: export ASDOPS_LOG_LEVEL=ERROR, export ASDOPS_LOG_TO_STDOUT=1 to findthe first error. For more details, see the MindIE official document.

2025-04-24 14:05:06,015 [ERROR] model.py:39 - [Model]   >>> Exception:Execute fail, enable log: export ASDOPS_LOG_LEVEL=ERROR, export ASDOPS_LOG_TO_STDOUT=1 to findthe first error. For more details, see the MindIE official document. 
Traceback (most recent call last):
  File "/usr/local/lib/python3.11/site-packages/model_wrapper/model.py", line 37, in initialize
    return self.python_model.initialize(config)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/model_wrapper/standard_model.py", line 146, in initialize
    self.generator = Generator(
                     ^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 119, in __init__
    self.warm_up(max_prefill_tokens, max_seq_len, max_input_len, max_iter_times, inference_mode)
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 356, in warm_up
    raise e
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 348, in warm_up
    self._generate_inputs_warm_up_backend(input_metadata, inference_mode)
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/generator.py", line 378, in _generate_inputs_warm_up_backend
    self.generator_backend.warm_up(model_inputs, inference_mode=inference_mode)
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 198, in warm_up
    super().warm_up(model_inputs)
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_backend.py", line 170, in warm_up
    _ = self.forward(model_inputs, **kwargs)
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/utils/decorators/time_decorator.py", line 38, in wrapper
    return func(*args, **kwargs)
           ^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/text_generator/adapter/generator_torch.py", line 153, in forward
    logits = self.model_wrapper.forward(model_inputs, self.cache_pool.npu_cache, **kwargs)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 89, in forward
    logits = self.forward_tensor(
             ^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/lib/python3.11/site-packages/mindie_llm/modeling/model_wrapper/atb/atb_model_wrapper.py", line 116, in forward_tensor
    logits = self.model_runner.forward(
             ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Ascend/atb-models/atb_llm/runner/model_runner.py", line 193, in forward
    return self.model.forward(**kwargs)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2_vl/flash_causal_qwen2_vl.py", line 203, in forward
    return self.language_model.forward(inputs_embeds,
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2_vl/flash_causal_qwen2_using_mrope.py", line 142, in forward
    logits = self.execute_ascend_operator(acl_inputs, acl_param, is_prefill)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/usr/local/Ascend/atb-models/atb_llm/models/qwen2/flash_causal_qwen2.py", line 374, in execute_ascend_operator
    acl_model_out = model_operation.execute(acl_inputs, acl_param)
                    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
RuntimeError: Execute fail, enable log: export ASDOPS_LOG_LEVEL=ERROR, export ASDOPS_LOG_TO_STDOUT=1 to findthe first error. For more details, see the MindIE official document.

2025-04-24 14:05:06,019 [ERROR] model.py:42 - [Model]   >>> return initialize error result: {'status': 'error', 'npuBlockNum': '0', 'cpuBlockNum': '0'}
2025-04-24 14:05:06,019 [ERROR] model.py:42 - [Model]   >>> return initialize error result: {'status': 'error', 'npuBlockNum': '0', 'cpuBlockNum': '0'}
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
[ERROR] TBE Subprocess[task_distribute] raise error[], main process disappeared!
/usr/lib64/python3.11/multiprocessing/resource_tracker.py:254: UserWarning: resource_tracker: There appear to be 30 leaked semaphore objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '
/usr/lib64/python3.11/multiprocessing/resource_tracker.py:254: UserWarning: resource_tracker: There appear to be 30 leaked semaphore objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '
Daemon is killing...
Killed
```

Ascend/ModelZoo-PyTorch

内容风险标识

评论 (1)

Ascend/ModelZoo-PyTorch .gitee-modal { width: 500px !important; }

内容风险标识