From ac0b569141ce9dff053d22f67d3231ef26640565 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Thu, 12 Oct 2023 10:34:20 +0800 Subject: [PATCH 1/4] fix dropout --- .../src/python/ptdbg_ascend/hook_module/register_hook.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/hook_module/register_hook.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/hook_module/register_hook.py index 0d69d465d..3ed4d51c6 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/hook_module/register_hook.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/hook_module/register_hook.py @@ -122,8 +122,6 @@ def register_hook_core(hook, **kwargs): # In NPU scene, clear the overflow flag before overflow detection if need_clear: HOOKModule.__init__ = add_clear_overflow(HOOKModule.__init__, pid) - elif "acc_cmp_dump" in hook_name: - remove_dropout() print_info_log("Start mounting the {} hook function to the model.".format(hook_name)) hook = functools.partial(hook, dump_step=0, pid=pid) @@ -131,6 +129,9 @@ def register_hook_core(hook, **kwargs): initialize_hook(hook) + if "acc_cmp_dump" in hook_name: + remove_dropout() + def init_dump_config(kwargs): dump_mode = kwargs.get('dump_mode', "api") -- Gitee From e085b888e3ff895651e8ef1fee0dd86a0bbd3366 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Thu, 12 Oct 2023 10:34:39 +0800 Subject: [PATCH 2/4] remove UT file --- .../ptdbg_ascend/test/ut/overflow/test_info_dump.py | 4 ++++ .../ptdbg_ascend/test/ut/parse_tool/test_visualization.py | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/debug/accuracy_tools/ptdbg_ascend/test/ut/overflow/test_info_dump.py b/debug/accuracy_tools/ptdbg_ascend/test/ut/overflow/test_info_dump.py index 96f3abd0e..e02063821 100644 --- a/debug/accuracy_tools/ptdbg_ascend/test/ut/overflow/test_info_dump.py +++ b/debug/accuracy_tools/ptdbg_ascend/test/ut/overflow/test_info_dump.py @@ -79,10 +79,14 @@ class TestInfoDump(unittest.TestCase): def test_write_api_info_json(self): info_dump.write_api_info_json(self.forward_api_info) self.assertTrue(os.path.exists(os.path.join(self.dump_path, f'forward_info_{self.forward_api_info.rank}.json'))) + if os.path.exists(os.path.join(self.dump_path, f'forward_info_{self.forward_api_info.rank}.json')): + os.remove(os.path.join(self.dump_path, f'forward_info_{self.forward_api_info.rank}.json')) def test_write_json(self): info_dump.write_json(self.json_file_path, {'test': 'data'}) self.assertTrue(os.path.exists(self.json_file_path)) + if os.path.exists(self.json_file_path): + os.remove(self.json_file_path) def test_initialize_output_json(self): try: diff --git a/debug/accuracy_tools/ptdbg_ascend/test/ut/parse_tool/test_visualization.py b/debug/accuracy_tools/ptdbg_ascend/test/ut/parse_tool/test_visualization.py index b537cb076..308f0e021 100644 --- a/debug/accuracy_tools/ptdbg_ascend/test/ut/parse_tool/test_visualization.py +++ b/debug/accuracy_tools/ptdbg_ascend/test/ut/parse_tool/test_visualization.py @@ -1,4 +1,5 @@ import unittest +import os import numpy as np from ptdbg_ascend.parse_tool.lib.visualization import Visualization @@ -28,3 +29,8 @@ class TestVisualization(unittest.TestCase): except Exception as e: self.fail(f"parse_pkl raised exception {e}") + def tearDown(self): + if os.path.exists('test.npy'): + os.remove('test.npy') + if os.path.exists('test.pkl'): + os.remove('test.pkl') \ No newline at end of file -- Gitee From b7633222b556feffc1b89dd26c8410948e3e2b72 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Thu, 12 Oct 2023 10:34:52 +0800 Subject: [PATCH 3/4] add print --- .../ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py | 8 ++++---- .../ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py index 6197fcd1b..3f3775cde 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/common/utils.py @@ -211,21 +211,21 @@ def make_dump_path_if_not_exists(dump_path): print_error_log('{} already exists and is not a directory.'.format(dump_path)) -def _print_log(level, msg): +def _print_log(level, msg, end='\n'): current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))) pid = os.getgid() - print(current_time + "(" + str(pid) + ")-[" + level + "]" + msg) + print(current_time + "(" + str(pid) + ")-[" + level + "]" + msg, end=end) sys.stdout.flush() -def print_info_log(info_msg): +def print_info_log(info_msg, end='\n'): """ Function Description: print info log. Parameter: info_msg: the info message. """ - _print_log("INFO", info_msg) + _print_log("INFO", info_msg, end=end) def print_error_log(error_msg): diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py index 53001c157..7771e5f13 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py @@ -137,6 +137,7 @@ def dump_data(dump_file_name, dump_step, prefix, data_info): if not DumpUtil.summary_only: np.save(output_path, data_info.save_data) api_list.append([prefix, dump_step, [], data_info.dtype, data_info.shape, data_info.summary_data]) + print_info_log(f"Dumping api: {prefix}" + " " * 20, end='\r') except Exception as e: print_warn_log("Dump data failed, error: {}".format(e)) finally: -- Gitee From df209812f843821ff85e2cfc7c381f2dec7f1638 Mon Sep 17 00:00:00 2001 From: s30048155 Date: Thu, 12 Oct 2023 11:28:27 +0800 Subject: [PATCH 4/4] update print --- .../ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py index 7771e5f13..8c20b15e5 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/dump.py @@ -137,7 +137,7 @@ def dump_data(dump_file_name, dump_step, prefix, data_info): if not DumpUtil.summary_only: np.save(output_path, data_info.save_data) api_list.append([prefix, dump_step, [], data_info.dtype, data_info.shape, data_info.summary_data]) - print_info_log(f"Dumping api: {prefix}" + " " * 20, end='\r') + print_info_log(f"ptdbg is dumping rank{rank} api: {prefix}" + " " * 10, end='\r') except Exception as e: print_warn_log("Dump data failed, error: {}".format(e)) finally: -- Gitee