diff --git a/debug/accuracy_tools/api_accuracy_checker/test/resources/forward.json b/debug/accuracy_tools/api_accuracy_checker/test/resources/forward.json new file mode 100644 index 0000000000000000000000000000000000000000..fe5212ebe3313673508a9b5a2a0739cec3dbc88d --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/resources/forward.json @@ -0,0 +1,3 @@ +{ + "Functional*silu*0": {"args": [{"type": "torch.Tensor", "dtype": "torch.float16", "shape": [2, 2560, 24, 24], "Max": 5.7421875, "Min": -5.125, "requires_grad": true}], "kwargs" :{"inplace": {"type": "bool", "value": false}}} +} \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/test/run_test.sh b/debug/accuracy_tools/api_accuracy_checker/test/run_test.sh new file mode 100644 index 0000000000000000000000000000000000000000..fdd00c6021c9827a68e005616b1b4d916e63e995 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/run_test.sh @@ -0,0 +1,31 @@ +#!/bin/bash +CUR_DIR=$(dirname $(readlink -f $0)) +TOP_DIR=${CUR_DIR}/.. +TEST_DIR=${TOP_DIR}/"test" +SRC_DIR=${TOP_DIR}/../ + +clean() { + cd ${TEST_DIR} + + if [ -e ${TEST_DIR}/"report" ]; then + rm -r ${TEST_DIR}/"report" + echo "remove last ut_report successfully." + fi + +} + +run_ut() { + export PYTHONPATH=${SRC_DIR}:${PYTHONPATH} + python3 run_ut.py +} + +main() { + clean + if [ "$1"x == "clean"x ]; then + return 0 + fi + + cd ${TEST_DIR} && run_ut +} + +main $@ diff --git a/debug/accuracy_tools/api_accuracy_checker/test/run_ut.py b/debug/accuracy_tools/api_accuracy_checker/test/run_ut.py new file mode 100644 index 0000000000000000000000000000000000000000..c73949697941d84782c4983aa484c06b1a7cbcc2 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/run_ut.py @@ -0,0 +1,41 @@ +import os +import shutil +import subprocess +import sys + +def run_ut(): + cur_dir = os.path.realpath(os.path.dirname(__file__)) + top_dir = os.path.realpath(os.path.dirname(cur_dir)) + ut_path = os.path.join(cur_dir, "ut/") + src_dir = top_dir + report_dir = os.path.join(cur_dir, "report") + + if os.path.exists(report_dir): + shutil.rmtree(report_dir) + + os.makedirs(report_dir) + + cmd = ["python3", "-m", "pytest", ut_path, "--junitxml=" + report_dir + "/final.xml", + "--cov=" + src_dir, "--cov-branch", "--cov-report=xml:" + report_dir + "/coverage.xml"] + + result_ut = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + while result_ut.poll() is None: + line = result_ut.stdout.readline().strip() + if line: + print(line) + + ut_flag = False + if result_ut.returncode == 0: + ut_flag = True + print("run ut successfully.") + else: + print("run ut failed.") + + return ut_flag + +if __name__=="__main__": + if run_ut(): + sys.exit(0) + else: + sys.exit(1) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/hook_module/test_wrap_functional.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/hook_module/test_wrap_functional.py new file mode 100644 index 0000000000000000000000000000000000000000..37058e77fd87e697b7dd7fde5e94b78d01a2cb89 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/hook_module/test_wrap_functional.py @@ -0,0 +1,15 @@ +# coding=utf-8 +import unittest +import torch +from api_accuracy_checker.hook_module import wrap_functional as wf + +class TestWrapFunctional(unittest.TestCase): + + def test_get_functional_ops(self): + expected_ops = {'relu', 'sigmoid', 'softmax'} + actual_ops = wf.get_functional_ops() + self.assertTrue(expected_ops.issubset(actual_ops)) + + def test_wrap_functional_ops_and_bind(self): + wf.wrap_functional_ops_and_bind(None) + self.assertTrue(hasattr(wf.HOOKFunctionalOP, 'wrap_relu')) \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/hook_module/test_wrap_tensor.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/hook_module/test_wrap_tensor.py new file mode 100644 index 0000000000000000000000000000000000000000..bfae3c72771510b141abf9204723bfe48bfa8de3 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/hook_module/test_wrap_tensor.py @@ -0,0 +1,29 @@ +# coding=utf-8 +import unittest +import torch +import yaml +from api_accuracy_checker.hook_module.wrap_tensor import get_tensor_ops, HOOKTensor, TensorOPTemplate, wrap_tensor_op, wrap_tensor_ops_and_bind + +class TestWrapTensor(unittest.TestCase): + def hook(self, a, b): + return + + def test_get_tensor_ops(self): + result = get_tensor_ops() + self.assertIsInstance(result, set) + + def test_HOOKTensor(self): + hook_tensor = HOOKTensor() + self.assertIsInstance(hook_tensor, HOOKTensor) + + def test_TensorOPTemplate(self): + tensor_op_template = TensorOPTemplate('add', self.hook) + self.assertEqual(tensor_op_template.op_name_, 'add') + + def test_wrap_tensor_op(self): + wrapped_op = wrap_tensor_op('add', self.hook) + self.assertTrue(callable(wrapped_op)) + + def test_wrap_tensor_ops_and_bind(self): + wrap_tensor_ops_and_bind(self.hook) + self.assertTrue(hasattr(HOOKTensor, 'wrap_add')) \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/hook_module/test_wrap_torch.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/hook_module/test_wrap_torch.py new file mode 100644 index 0000000000000000000000000000000000000000..40cef939adfd06158eb543c07b3d682e29d6cdab --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/hook_module/test_wrap_torch.py @@ -0,0 +1,37 @@ +# coding=utf-8 +import unittest +import torch +import yaml +from api_accuracy_checker.hook_module.wrap_torch import * + +class TestWrapTorch(unittest.TestCase): + + def setUp(self): + self.op_name = 'add' + self.torch_op = wrap_torch_op(self.op_name, self.hook) + + def hook(self, a, b): + return + + def test_get_torch_ops(self): + ops = get_torch_ops() + self.assertIsInstance(ops, set) + self.assertIn(self.op_name, ops) + + def test_TorchOPTemplate(self): + template = TorchOPTemplate(self.op_name, self.hook) + self.assertEqual(template.op_name_, self.op_name) + self.assertEqual(template.prefix_op_name_, "Torch*" + str(self.op_name) + "*") + + def test_input_param_need_adapt(self): + template = TorchOPTemplate(self.op_name, self.hook) + self.assertFalse(template.input_param_need_adapt()) + + def test_forward(self): + template = TorchOPTemplate(self.op_name, self.hook) + result = template.forward(torch.tensor([1, 2, 3]), torch.tensor([4, 5, 6])) + torch.testing.assert_allclose(result, torch.tensor([5, 7, 9])) + + def test_wrap_torch_ops_and_bind(self): + wrap_torch_ops_and_bind(self.hook) + self.assertTrue(hasattr(HOOKTorchOP, "wrap_" + self.op_name)) \ No newline at end of file diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_data_generate.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_data_generate.py new file mode 100644 index 0000000000000000000000000000000000000000..7cb9a8504d53520af3ccc21cdddae3f2927ee967 --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_data_generate.py @@ -0,0 +1,105 @@ +# coding=utf-8 +import unittest +import numpy as np +import os +import copy +from api_accuracy_checker.run_ut.data_generate import * +from api_accuracy_checker.common.utils import get_json_contents + +base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +forward_file = os.path.join(base_dir, "../resources/forward.json") +forward_content = get_json_contents(forward_file) +for api_full_name, api_info_dict in forward_content.items(): + api_full_name = api_full_name + api_info_dict = api_info_dict + +max_value = 5.7421875 +min_value = -5.125 + +class TestDataGenerateMethods(unittest.TestCase): + def test_gen_api_params(self): + api_info = copy.deepcopy(api_info_dict) + args_params, kwargs_params = gen_api_params(api_info, True, None) + max_diff = abs(args_params[0].max() - max_value) + min_diff = abs(args_params[0].min() - min_value) + self.assertEqual(len(args_params), 1) + self.assertEqual(args_params[0].dtype, torch.float16) + self.assertLessEqual(max_diff, 0.001) + self.assertLessEqual(min_diff, 0.001) + self.assertEqual(args_params[0].shape, torch.Size([2, 2560, 24, 24])) + self.assertEqual(kwargs_params, {'inplace': False}) + + def test_gen_args(self): + args_result = gen_args(api_info_dict.get('args')) + max_diff = abs(args_result[0].max() - max_value) + min_diff = abs(args_result[0].min() - min_value) + self.assertEqual(len(args_result), 1) + self.assertEqual(args_result[0].dtype, torch.float16) + self.assertLessEqual(max_diff, 0.001) + self.assertLessEqual(min_diff, 0.001) + self.assertEqual(args_result[0].shape, torch.Size([2, 2560, 24, 24])) + + def test_gen_data(self): + data = gen_data(api_info_dict.get('args')[0], True, None) + max_diff = abs(data.max() - max_value) + min_diff = abs(data.min() - min_value) + self.assertEqual(data.dtype, torch.float16) + self.assertEqual(data.requires_grad, True) + self.assertLessEqual(max_diff, 0.001) + self.assertLessEqual(min_diff, 0.001) + self.assertEqual(data.shape, torch.Size([2, 2560, 24, 24])) + + def test_gen_kwargs(self): + api_info = copy.deepcopy(api_info_dict) + kwargs_params = gen_kwargs(api_info, None) + self.assertEqual(kwargs_params, {'inplace': False}) + + def test_gen_kwargs_device(self): + k_dict = {"kwargs": {"device": {"type": "torch.device", "value": "npu:0"}}} + kwargs_params = gen_kwargs(k_dict, None) + self.assertEqual(str(kwargs_params), "{'device': device(type='npu', index=0)}") + + def test_gen_kwargs_1(self): + k_dict = {"device": {"type": "torch.device", "value": "npu:0"}} + for key, value in k_dict.items(): + gen_torch_kwargs(k_dict, key, value) + self.assertEqual(str(k_dict), "{'device': device(type='npu', index=0)}") + + def test_gen_kwargs_2(self): + k_dict = {"inplace": {"type": "bool", "value": "False"}} + for key, value in k_dict.items(): + gen_torch_kwargs(k_dict, key, value) + self.assertEqual(k_dict, {'inplace': False}) + + def test_gen_random_tensor(self): + data = gen_random_tensor(api_info_dict.get('args')[0], None) + max_diff = abs(data.max() - max_value) + min_diff = abs(data.min() - min_value) + self.assertEqual(data.dtype, torch.float16) + self.assertEqual(data.requires_grad, False) + self.assertLessEqual(max_diff, 0.001) + self.assertLessEqual(min_diff, 0.001) + self.assertEqual(data.shape, torch.Size([2, 2560, 24, 24])) + + def test_gen_common_tensor(self): + info = api_info_dict.get('args')[0] + low, high = info.get('Min'), info.get('Max') + data_dtype = info.get('dtype') + shape = tuple(info.get('shape')) + data = gen_common_tensor(low, high, shape, data_dtype, None) + max_diff = abs(data.max() - max_value) + min_diff = abs(data.min() - min_value) + self.assertEqual(data.dtype, torch.float16) + self.assertEqual(data.requires_grad, False) + self.assertLessEqual(max_diff, 0.001) + self.assertLessEqual(min_diff, 0.001) + self.assertEqual(data.shape, torch.Size([2, 2560, 24, 24])) + + def test_gen_bool_tensor(self): + info = {"type": "torch.Tensor", "dtype": "torch.bool", "shape": [1, 1, 160, 256], \ + "Max": 1, "Min": 0, "requires_grad": False} + low, high = info.get("Min"), info.get("Max") + shape = tuple(info.get("shape")) + data = gen_bool_tensor(low, high, shape) + self.assertEqual(data.shape, torch.Size([1, 1, 160, 256])) + self.assertEqual(data.dtype, torch.bool) diff --git a/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_run_ut.py b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_run_ut.py new file mode 100644 index 0000000000000000000000000000000000000000..58e0da18c0438c8d1d8c343727565aa4d9f230bd --- /dev/null +++ b/debug/accuracy_tools/api_accuracy_checker/test/ut/run_ut/test_run_ut.py @@ -0,0 +1,48 @@ +# coding=utf-8 +from api_accuracy_checker.run_ut.run_ut import generate_cpu_params, get_api_info +import unittest +import numpy as np +import os +import copy +from api_accuracy_checker.run_ut.run_ut import * +from api_accuracy_checker.common.utils import get_json_contents + +base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +forward_file = os.path.join(base_dir, "../resources/forward.json") +forward_content = get_json_contents(forward_file) +for api_full_name, api_info_dict in forward_content.items(): + api_full_name = api_full_name + api_info_dict = api_info_dict + +class TestRunUtMethods(unittest.TestCase): + def test_exec_api(self): + api_info = copy.deepcopy(api_info_dict) + [api_type, api_name, _] = api_full_name.split("*") + args, kwargs, need_grad = get_api_info(api_info, api_name) + cpu_args, cpu_kwargs = generate_cpu_params(args, kwargs, True) + out = exec_api(api_type, api_name, cpu_args, cpu_kwargs) + self.assertEqual(out.dtype, torch.float32) + self.assertEqual(out.requires_grad, True) + self.assertEqual(out.shape, torch.Size([2, 2560, 24, 24])) + + def test_generate_npu_params(self): + api_info = copy.deepcopy(api_info_dict) + [api_type, api_name, _] = api_full_name.split("*") + args, kwargs, need_grad = get_api_info(api_info, api_name) + npu_args, npu_kwargs = generate_npu_params(args, kwargs, True) + self.assertEqual(len(npu_args), 1) + self.assertEqual(npu_args[0].dtype, torch.float16) + self.assertEqual(npu_args[0].requires_grad, True) + self.assertEqual(npu_args[0].shape, torch.Size([2, 2560, 24, 24])) + self.assertEqual(npu_kwargs, {'inplace': False}) + + def test_generate_cpu_params(self): + api_info = copy.deepcopy(api_info_dict) + [api_type, api_name, _] = api_full_name.split("*") + args, kwargs, need_grad = get_api_info(api_info, api_name) + cpu_args, cpu_kwargs = generate_cpu_params(args, kwargs, True) + self.assertEqual(len(cpu_args), 1) + self.assertEqual(cpu_args[0].dtype, torch.float32) + self.assertEqual(cpu_args[0].requires_grad, True) + self.assertEqual(cpu_args[0].shape, torch.Size([2, 2560, 24, 24])) + self.assertEqual(cpu_kwargs, {'inplace': False}) \ No newline at end of file