diff --git a/mindformers/pipeline/multi_modal_to_text_generation_pipeline.py b/mindformers/pipeline/multi_modal_to_text_generation_pipeline.py index ff041197fff93f1ad65606b984ebc422b2f8a71e..a86d6eedf706d61e4cbaf01ced936b116870d808 100644 --- a/mindformers/pipeline/multi_modal_to_text_generation_pipeline.py +++ b/mindformers/pipeline/multi_modal_to_text_generation_pipeline.py @@ -62,9 +62,9 @@ class MultiModalToTextPipeline(Pipeline): >>> config = MindFormerConfig(config_path) >>> build_context(config) >>> model_config = AutoConfig.from_pretrained(config_path) - >>> tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, use_fast=True) + >>> tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True) >>> model = AutoModel.from_config(model_config) - >>> processor = AutoProcessor.from_pretrained(config_path, trust_remote_code=True, use_fast=True) + >>> processor = AutoProcessor.from_pretrained(config_path, use_fast=True) >>> param_dict = ms.load_checkpoint("/path/to/cogvlm2image.ckpt") >>> _, not_load = ms.load_param_into_net(model, param_dict) >>> text_generation_pipeline = pipeline(task="multi_modal_to_text_generation", diff --git a/research/deepseek3/run_predict_deepseek.py b/research/deepseek3/run_predict_deepseek.py index 69fe16140793b818e7284142921fa7461ab61ec3..79a286e97c35975a7eea2cad55e59f3c00d433be 100644 --- a/research/deepseek3/run_predict_deepseek.py +++ b/research/deepseek3/run_predict_deepseek.py @@ -53,7 +53,7 @@ def run_predict(args): unk_token=config.processor.tokenizer.unk_token, bos_token=config.processor.tokenizer.bos_token, eos_token=config.processor.tokenizer.eos_token, - fast_tokenizer=True, trust_remote_code=True) + fast_tokenizer=True) tokenizer.pad_token = tokenizer.eos_token # build model from config diff --git a/research/telechat2/infer/run_telechat_predict_parallel.py b/research/telechat2/infer/run_telechat_predict_parallel.py index 139dc43675a7085a223b71d1ef76d240c32071ec..6dc4ec8b36d801dde246747355c63ce889a29399 100644 --- a/research/telechat2/infer/run_telechat_predict_parallel.py +++ b/research/telechat2/infer/run_telechat_predict_parallel.py @@ -55,8 +55,8 @@ def main(): # build tokenizer chat_template = "{%- if tools %}\n {%- if messages[0]['role'] == 'system' %}\n {{-'<_system>'+messages[0]['content'] }}\n {%- else %}\n {{- '<_system>'+'你是中国电信星辰语义大模型，英文名是TeleChat，你是由中电信人工智能科技有限公司和中国电信人工智能研究院（TeleAI）研发的人工智能助手。' }}\n {%- endif %}\n {{- '\\n\\n# 可用工具\\n你可以调用标签中包含的一个或多个工具来辅助你回答问题,以下是可用工具详情：\\n\\n' }}\n {%- for tool in tools %}\n {{- tool | tojson }}\n {{-'\\n'}}\n {%- endfor %}\n {{- '\\n\\n# 调用方法\\n你需要遵循工具的要求，使用json格式返回工具名称及参数，并用包含。下方是一个调用模板：\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n\\n\\n' }}\n{%- else %}\n {%- if messages[0]['role'] == 'system' %}\n {{- '<_system>' + messages[0]['content'] + '\\n' }}\n {%- else %}\n {{- '<_system>'+'你是中国电信星辰语义大模型，英文名是TeleChat，你是由中电信人工智能科技有限公司和中国电信人工智能研究院（TeleAI）研发的人工智能助手。\\n' }}\n {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n {%- if (message.role == 'user') %}\n {{- '<_user>' + message.content }}\n {%- elif message.role == 'bot' or message.role == 'assistant' %}\n {{- '<_bot>' }}\n {%- if message.content %}\n {{- message.content }}\n {%- endif %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.index0 == 0 %}\n {{-''}}\n {%- else %}\n {{-'\\n'}}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}{{ tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {{- tool_call.arguments | tojson }}\n {{- '}\\n' }}\n {%- endfor %}\n {{- '<_end>\\n' }}\n {%- elif message.role == 'tool' %}\n {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != 'tool') %}\n {{- '<_user>'+'\\n' }}\n {%- else %}\n {{- '\\n\\n' }}\n {%- endif %}\n {{- message.content }}\n {{- '\\n' }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<_bot>' }}\n{%- endif %}" - tokenizer = TelechatTokenizer(config.processor.tokenizer.vocab_file, \ - chat_template=chat_template, fast_tokenizer=True, trust_remote_code=True) + tokenizer = TelechatTokenizer(config.processor.tokenizer.vocab_file, + chat_template=chat_template, fast_tokenizer=True) # build model config model_config = config.model.model_config diff --git a/toolkit/benchmarks/eval_with_harness.py b/toolkit/benchmarks/eval_with_harness.py index c297ca22a7d9a0f01a8a17c3df085944a3d7a309..c1baed04157478a8ca0481ecb104e3b46e8a8c63 100644 --- a/toolkit/benchmarks/eval_with_harness.py +++ b/toolkit/benchmarks/eval_with_harness.py @@ -212,7 +212,7 @@ class MFLM(TemplateLM): def _create_tokenizer(self, pretrained: str) -> None: """Initialize Tokenizer""" - self.tokenizer = AutoTokenizer.from_pretrained(pretrained, trust_remote_code=True) + self.tokenizer = AutoTokenizer.from_pretrained(pretrained) eval_logger.info(f"Build tokenizer finished.") def tok_encode( diff --git a/toolkit/benchmarks/vlmevalkit_models/multimodal_models.py b/toolkit/benchmarks/vlmevalkit_models/multimodal_models.py index 545d5dd4d1987de6e31109b912782986709b977b..fed8df9482197000b14529ab6a5079a8666d495b 100644 --- a/toolkit/benchmarks/vlmevalkit_models/multimodal_models.py +++ b/toolkit/benchmarks/vlmevalkit_models/multimodal_models.py @@ -52,13 +52,13 @@ def init_model(model_path): generation_config = GenerationConfig.from_model_config(model_config) - tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True, use_fast=True) + tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=True) logger.info(f"Build tokenizer finished.") model = AutoModel.from_config(model_config) logger.info(f"Build model finished.") - processor = AutoProcessor.from_pretrained(config_path[0], trust_remote_code=True, use_fast=True) + processor = AutoProcessor.from_pretrained(config_path[0], use_fast=True) logger.info(f"Build processor finished.") batch_size = 1