diff --git a/mindformers/core/context/build_context.py b/mindformers/core/context/build_context.py index fd33c029ceda4de966d8dfce9d761a610e4f8282..16fdcc2b64e6a164fc7e70e565629d51ba183a06 100644 --- a/mindformers/core/context/build_context.py +++ b/mindformers/core/context/build_context.py @@ -240,13 +240,23 @@ class MFContextOperator(MFContextConfig): custom_matmul_shuffle = 'on' lccl_deterministic = '0' + ms_alloc_conf = os.environ.get('MS_ALLOC_CONF', 'enable_vmm:False') + cpu_affinity = os.environ.get('CPU_AFFINITY', 'True') + ms_internal_disable_custom_kernel_list = "" + mode = self.config.get_value('context.mode', 'GRAPH_MODE') + if MODE.get(mode) == MODE.get('GRAPH_MODE'): + ms_internal_disable_custom_kernel_list = os.environ.get('MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST', + 'PagedAttention') env = { 'HCCL_DETERMINISTIC': hccl_deterministic, 'ASCEND_LAUNCH_BLOCKING': ascend_launch_blocking, 'TE_PARALLEL_COMPILER': te_parallel_compiler, 'CUSTOM_MATMUL_SHUFFLE': custom_matmul_shuffle, 'LCCL_DETERMINISTIC': lccl_deterministic, - 'MS_ENABLE_GRACEFUL_EXIT': '1' if self.use_graceful_exit else '0' + 'MS_ENABLE_GRACEFUL_EXIT': '1' if self.use_graceful_exit else '0', + 'MS_ALLOC_CONF': ms_alloc_conf, + 'CPU_AFFINITY': cpu_affinity, + 'MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST': ms_internal_disable_custom_kernel_list } run_mode = ( @@ -257,13 +267,7 @@ class MFContextOperator(MFContextConfig): and RunMode(run_mode) in [RunMode.PREDICT, RunMode.EVAL] and use_past ): - env['MS_ALLOC_CONF'] = 'enable_vmm:False' env['RUN_MODE'] = run_mode - env['CPU_AFFINITY'] = 'True' - mode = self.config.get_value('context.mode', 'GRAPH_MODE') - if MODE.get(mode) == MODE.get('GRAPH_MODE'): - env['MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST'] = 'PagedAttention' - if ( self.enable_mindio_ttp_save_ckpt and self.config.runner_config.sink_size == 1 @@ -272,6 +276,7 @@ class MFContextOperator(MFContextConfig): env['MINDIO_FOR_MINDSPORE'] = '1' os.environ.update(env) + logger.info(f"env: {env}") def set_context(self, **kwargs): """Set mf context value according to the input key words.""" diff --git a/research/telechat/run_telechat.py b/research/telechat/run_telechat.py index 737a70d85c6119129896b750ab62096078510eef..7cf5549d1b95904b7afc1c30fe731b6816640269 100644 --- a/research/telechat/run_telechat.py +++ b/research/telechat/run_telechat.py @@ -53,11 +53,6 @@ def main(): if args.remote_save_url is not None: config.remote_save_url = args.remote_save_url - if config.trainer.model_name == "telechat_52b": - os.environ['MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST'] = None - else: - os.environ['MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST'] = 'InferenceMatmulSplit' - # init context build_context(config) diff --git a/research/telechat/run_telechat_predict.py b/research/telechat/run_telechat_predict.py index 294e5b5f58683628414996ff943199703c50acc4..e189f5e227870abe24a00871b6e9d372e337230a 100644 --- a/research/telechat/run_telechat_predict.py +++ b/research/telechat/run_telechat_predict.py @@ -13,7 +13,6 @@ # limitations under the License. # ============================================================================ """Telechat predict scripts.""" -import os import argparse import json import copy @@ -128,11 +127,6 @@ def main(): # set model config config = MindFormerConfig(args.yaml_file) - if config.trainer.model_name == "telechat_52b": - os.environ['MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST'] = 'PagedAttention' - else: - os.environ['MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST'] = 'InferenceMatmulSplit,PagedAttention' - if args.device_id is not None: config.context.device_id = args.device_id if args.checkpoint_path is not None: diff --git a/research/telechat/telechat.md b/research/telechat/telechat.md index 04378cf9ae49c61f6875f675ab003fad0ae6a074..81f45870a609495cd0c2d850268e0a165051ffe0 100644 --- a/research/telechat/telechat.md +++ b/research/telechat/telechat.md @@ -132,20 +132,6 @@ mp: 目标切分个数,比如指定为8时,得到的完整权重适用于mp= 涉及到ckpt的单卡,多卡转换,详细教程请参考特性文档模型[权重切分与合并](../../docs/feature_cards/Transform_Ckpt.md)。 -## 注意事项 - -启动微调/推理任务前,由于7b/12b模型网络结构差异,需要关闭`MatmulSplit`融合算子 - -```bash -export MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST=InferenceMatmulSplit -``` - -52b需要开启`MatmulSplit`融合算子 - -```bash -unset MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST -``` - ## 微调 ### 数据集准备 diff --git a/research/telechat2/README.md b/research/telechat2/README.md index 3e0eb483eee19e3c45d23846bee882f65c5da071..5b54519ece62c7aed5eb9f75d6daa2421456f14e 100644 --- a/research/telechat2/README.md +++ b/research/telechat2/README.md @@ -237,7 +237,6 @@ export ENABLE_CELL_REUSE=1 #编译加速 export MS_DEV_SIDE_EFFECT_LOAD_ELIM=3 # 去除TensorMove export MS_MEMORY_POOL_RECYCLE=1 # 内存优化 export GE_NOT_CUT=1 # 内存优化 -export MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST=InferenceMatmulSplit,PagedAttention ``` - step 4. 执行运行脚本。 @@ -278,10 +277,6 @@ register_path: 外部模型注册路径 ### 参数配置 -- 设置环境变量: - -export MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST=InferenceMatmulSplit,PagedAttention - - 7b模型支持单机**单卡推理** 在`predict_telechat_7b.yaml`中填写`vocab_file`字段 diff --git a/research/telechat2/run_telechat.py b/research/telechat2/run_telechat.py index e8205ad2bbe44b15ab40ba74a95273dc93504f1b..18d959a7c7975ae013e6cebd943ce8e2c9e83a9e 100644 --- a/research/telechat2/run_telechat.py +++ b/research/telechat2/run_telechat.py @@ -53,11 +53,6 @@ def main(): config.auto_trans_ckpt = args.auto_trans_ckpt if args.remote_save_url is None: config.remote_save_url = args.remote_save_url - os.environ["MS_INTERNAL_ENABLE_CUSTOM_KERNEL_LIST"] = "MatMulElemwise" - if config.model.model_config.qkv_concat: - os.environ['MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST'] = 'PagedAttention,FlashAttentionScore' - else: - os.environ['MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST'] = 'InferenceMatmulSplit,PagedAttention,FlashAttentionScore' # init context build_context(config) diff --git a/research/telechat2/run_telechat_predict.py b/research/telechat2/run_telechat_predict.py index 87a95f8a80631417cfbe757e08d657f171520e61..ec61cfae1200b75c58b7773947b1756450c3b0a8 100644 --- a/research/telechat2/run_telechat_predict.py +++ b/research/telechat2/run_telechat_predict.py @@ -13,7 +13,6 @@ # limitations under the License. # ============================================================================ """Telechat predict scripts.""" -import os import argparse import mindspore as ms from mindspore import Model, Tensor @@ -37,8 +36,6 @@ def main(): # set config config = MindFormerConfig(args.yaml_file) - os.environ['MS_INTERNAL_DISABLE_CUSTOM_KERNEL_LIST'] = 'InferenceMatmulSplit,PagedAttention' - if args.device_id is not None: config.context.device_id = args.device_id if args.checkpoint_path is not None: