diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index 6197fcd1b44cb301496c37e641134c723acf49f3..3f3775cde4d31994d03103ed85ac84baa341296a 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -211,21 +211,21 @@ def make_dump_path_if_not_exists(dump_path): print_error_log('{} already exists and is not a directory.'.format(dump_path)) -def _print_log(level, msg): +def _print_log(level, msg, end='\n'): current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))) pid = os.getgid() - print(current_time + "(" + str(pid) + ")-[" + level + "]" + msg) + print(current_time + "(" + str(pid) + ")-[" + level + "]" + msg, end=end) sys.stdout.flush() -def print_info_log(info_msg): +def print_info_log(info_msg, end='\n'): """ Function Description: print info log. Parameter: info_msg: the info message. """ - _print_log("INFO", info_msg) + _print_log("INFO", info_msg, end=end) def print_error_log(error_msg): diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py index 53001c1571416a53a007a378be6d10b3fbb7f2a4..8c20b15e58b078b876d3265b9edf4d50e57f8f93 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py @@ -137,6 +137,7 @@ def dump_data(dump_file_name, dump_step, prefix, data_info): if not DumpUtil.summary_only: np.save(output_path, data_info.save_data) api_list.append([prefix, dump_step, [], data_info.dtype, data_info.shape, data_info.summary_data]) + print_info_log(f"ptdbg is dumping rank{rank} api: {prefix}" + " " * 10, end='\r') except Exception as e: print_warn_log("Dump data failed, error: {}".format(e)) finally: diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/hook_module/register_hook.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/hook_module/register_hook.py index 0d69d465d134688d0ac1777972750faa15869393..3ed4d51c64b9e7b285ee679a394af486cbf429ab 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/hook_module/register_hook.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/hook_module/register_hook.py @@ -122,8 +122,6 @@ def register_hook_core(hook, **kwargs): # In NPU scene, clear the overflow flag before overflow detection if need_clear: HOOKModule.__init__ = add_clear_overflow(HOOKModule.__init__, pid) - elif "acc_cmp_dump" in hook_name: - remove_dropout() print_info_log("Start mounting the {} hook function to the model.".format(hook_name)) hook = functools.partial(hook, dump_step=0, pid=pid) @@ -131,6 +129,9 @@ def register_hook_core(hook, **kwargs): initialize_hook(hook) + if "acc_cmp_dump" in hook_name: + remove_dropout() + def init_dump_config(kwargs): dump_mode = kwargs.get('dump_mode', "api") diff --git a/debug/accuracy_tools/ptdbg_ascend/test/ut/overflow/test_info_dump.py b/debug/accuracy_tools/ptdbg_ascend/test/ut/overflow/test_info_dump.py index 96f3abd0ef66c2ac184beef191f34d9f92ae8d9a..e0206382171448806de486aa5d89df5faec66dac 100644 --- a/debug/accuracy_tools/ptdbg_ascend/test/ut/overflow/test_info_dump.py +++ b/debug/accuracy_tools/ptdbg_ascend/test/ut/overflow/test_info_dump.py @@ -79,10 +79,14 @@ class TestInfoDump(unittest.TestCase): def test_write_api_info_json(self): info_dump.write_api_info_json(self.forward_api_info) self.assertTrue(os.path.exists(os.path.join(self.dump_path, f'forward_info_{self.forward_api_info.rank}.json'))) + if os.path.exists(os.path.join(self.dump_path, f'forward_info_{self.forward_api_info.rank}.json')): + os.remove(os.path.join(self.dump_path, f'forward_info_{self.forward_api_info.rank}.json')) def test_write_json(self): info_dump.write_json(self.json_file_path, {'test': 'data'}) self.assertTrue(os.path.exists(self.json_file_path)) + if os.path.exists(self.json_file_path): + os.remove(self.json_file_path) def test_initialize_output_json(self): try: diff --git a/debug/accuracy_tools/ptdbg_ascend/test/ut/parse_tool/test_visualization.py b/debug/accuracy_tools/ptdbg_ascend/test/ut/parse_tool/test_visualization.py index b537cb076f5340eae7f853402f79359f563c1784..308f0e02164fb8fca6ddae3b3058e0e5177856c9 100644 --- a/debug/accuracy_tools/ptdbg_ascend/test/ut/parse_tool/test_visualization.py +++ b/debug/accuracy_tools/ptdbg_ascend/test/ut/parse_tool/test_visualization.py @@ -1,4 +1,5 @@ import unittest +import os import numpy as np from ptdbg_ascend.parse_tool.lib.visualization import Visualization @@ -28,3 +29,8 @@ class TestVisualization(unittest.TestCase): except Exception as e: self.fail(f"parse_pkl raised exception {e}") + def tearDown(self): + if os.path.exists('test.npy'): + os.remove('test.npy') + if os.path.exists('test.pkl'): + os.remove('test.pkl') \ No newline at end of file