diff --git a/debug/accuracy_tools/api_accuracy_checker/generate_op_script/op_generator.py b/debug/accuracy_tools/api_accuracy_checker/generate_op_script/op_generator.py index 7d3e2b226bde101c5aeba46f5353cbf3c53549d0..324b5e30da7314bd3e1b2f10800ec08acd5edb09 100644 --- a/debug/accuracy_tools/api_accuracy_checker/generate_op_script/op_generator.py +++ b/debug/accuracy_tools/api_accuracy_checker/generate_op_script/op_generator.py @@ -17,47 +17,113 @@ import argparse import json import os +import re import math import numpy as np import torch + try: import torch_npu except ImportError: pass from api_accuracy_checker.compare.compare_utils import BinaryStandardApi, AbsoluteStandardApi, ULPStandardApi - +from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import check_file_or_directory_path +from msprobe.core.common.file_check import create_directory +from msprobe.core.common.const import Const +from msprobe.core.common.log import logger TENSOR_DATA_LIST = ["torch.Tensor"] TORCH_BOOL_TYPE = ["torch.bool"] TORCH_INT_TYPE = ["torch.uint8", "torch.int8", "torch.int16", "torch.short", "torch.int32", "torch.int", "torch.int64", "torch.long"] -TORCH_FLOAT_TYPE = ["torch.float16", "torch.half", "torch.bfloat16", "torch.float32", "torch.float", +TORCH_FLOAT_TYPE = ["torch.float16", "torch.half", "torch.bfloat16", "torch.float32", "torch.float", "torch.float64", "torch.double"] TORCH_COMPLEX_TYPE = ["torch.complex32", "torch.chalf", "torch.complex64", "torch.cfloat", "torch.complex128", "torch.cdouble"] - - -def check_json(json_path): - json_file = os.path.realpath(json_path) - with open(json_file) as f: - json_content = json.load(f) +OPERATOR_TYPE = ("Functional", "Tensor", "Torch") + +API_INFO = 2 +FOUR_SEGMENT = 4 +FIVE_SEGMENT = 5 +DATA_NAME = 'data_name' +API_MAX_LENGTH = 30 +class APIResult: + def __init__(self, api_full_name, api_info_dict, backward_info=None): + self.api_full_name = api_full_name + self.api_info_dict = api_info_dict + self.backward_info = backward_info + + @property + def api_type(self): + return self.api_full_name.split(Const.SEP, -1)[0] + + @classmethod + def from_json(cls, json_content, propagation): + forward_name, forward_dict = list(json_content.items())[0] + forward_info = cls(api_full_name=forward_name, api_info_dict=forward_dict) + + if propagation == Const.BACKWARD: + backward_name, backward_dict = list(json_content.items())[1] + backward_info = cls(api_full_name=backward_name, api_info_dict=backward_dict) + forward_info.backward_info = backward_info + + if not forward_info.is_supported_type(): + raise ValueError(f"type {forward_info.api_type} of API is not supported!") + + return forward_info + + def is_supported_type(self): + return self.api_type in OPERATOR_TYPE + +def check_path_pattern_valid(path_pattern): + pattern = re.compile(r'(\.|/|:|_|-|\s|[~0-9a-zA-Z])+') + if not pattern.fullmatch(path_pattern): + raise ValueError('Only the following characters are allowed in the path: A-Z a-z 0-9 - _ . / :') + if len(path_pattern) > API_MAX_LENGTH: + raise ValueError(f'API name {path_pattern} is too long!') + +def check_json_content(json_content, propagation): + # ensure json_content is of type dict if not isinstance(json_content, dict): - raise ValueError("content of json file is not a dictionary!") - if len(list(json_content.items())) > 1: - raise ValueError("json file has more than one API, only one API is allowed!") - (api_full_name, api_info_dict) = list(json_content.items())[0] - (api_type, api_name, ordinal_number) = api_full_name.split(".", -1) - if api_type not in ("Functional", "Tensor", "Torch"): - raise ValueError("type {0} of API is not supported!".format(api_type)) - return (api_full_name, api_info_dict) + raise ValueError(f'content of json file is not a dict!') + # ensure the dict is not empty + if not json_content: + raise ValueError(f'json file is empty!') + # ensure the length of json_content is within allowed limits + if len(json_content) > API_INFO: + raise ValueError(f'json file has more than one API, the API only contains forward and backward info') + # Retrieve the first API name and dictionary + forward_item = next(iter(json_content.items()), None) + if not forward_item or not isinstance(forward_item[1], dict): + raise ValueError(f'Invalid forward API data in json_content!') + + # if propagation is backward, ensure json file contains forward and backward info + if propagation == Const.BACKWARD and len(json_content) < API_INFO: + raise ValueError(f'Backward propagation requires contains forward and backward info!') + + # if propagation is backward, ensure it has valid data + if propagation == Const.BACKWARD: + backward_item = list(json_content.items())[1] + if not isinstance(backward_item[1], dict): + raise ValueError(f'Invalid backward API data in json_content!') def check_user_settings(cmd_args): iter_t = cmd_args.iter_times if iter_t <= 0: raise ValueError("iter_times should be an integer bigger than zero!") - (api_full_name, api_info_dict) = check_json(cmd_args.forward_json_path) - return api_full_name, api_info_dict + + json_file = cmd_args.output_file + propagation = cmd_args.propagation + + with FileOpen(json_file, 'r') as f: + json_content = json.load(f) + + check_json_content(json_content, propagation) + api_result = APIResult.from_json(json_content, propagation) + + return api_result def get_compare_standard(api_name): @@ -69,9 +135,29 @@ def get_compare_standard(api_name): return "CompareStandard.ULP_ERROR_STANDARD" return "CompareStandard.BENCHMARK_STANDARD" - -def get_settings(cmd_args): - ''' +def extract_detailed_api_segments(full_api_name): + """ + Function Description: + Extract the name of the API. + Parameter: + full_api_name_with_direction_status: Full name of the API. Example: torch.matmul.0.forward.output.0 + Return: + api_name: Name of api. Example: matmul, mul, etc. + full_api_name: Full name of api. Example: torch.matmul.0 + direction_status: Direction status of api. Example: forward, backward, etc. + """ + api_parts = full_api_name.split(Const.SEP) + api_parts_length = len(api_parts) + api_type, api_name, api_order = None, None, None + if api_parts_length == FOUR_SEGMENT: + api_type, api_name, api_order, _ = api_parts + elif api_parts_length == FIVE_SEGMENT: + api_type, prefix, api_name, api_order, _ = api_parts + api_name = Const.SEP.join([prefix, api_name]) + return api_type, api_name, api_order + +def get_settings(cmd_args, api_full_name, args_info_forward, kwargs_info_forward, args_info_backward): + ''' internal_settings contain all information needed for the operator program. keys: api_full_name: api_type.api_name.ordinal_number @@ -87,14 +173,13 @@ def get_settings(cmd_args): kwargs_value_assignment: code for kwargs assignment kwargs_dict_generator_device: code for generate kwargs dict on device kwargs_dict_generator_bench: code for generate kwargs dict on bench - ''' - api_full_name, api_info_dict = check_user_settings(cmd_args) - args_info = api_info_dict.get("args") - kwargs_info = api_info_dict.get("kwargs") - + ''' + # Generate an internal setting dictionary based on user settings + # including API name, type, comparison standard, random seed, number of iterations and other information internal_settings = {} + internal_settings["propagation"] = cmd_args.propagation internal_settings["api_full_name"] = api_full_name - (api_type, api_name, ordinal_number) = api_full_name.split(".", -1) + api_type, api_name, ordinal_number = extract_detailed_api_segments(api_full_name) if api_type == "Functional": internal_settings["api_type"] = "torch.nn.functional" elif api_type == "Tensor": @@ -104,18 +189,27 @@ def get_settings(cmd_args): internal_settings["api_name"] = api_name internal_settings["compare_standard"] = get_compare_standard(api_name) internal_settings["ordinal_number"] = ordinal_number - internal_settings["direction_status"] = "forward" + internal_settings["direction_status"] = cmd_args.propagation internal_settings["random_seed"] = cmd_args.random_seed if cmd_args.mode == "real_data": internal_settings["iter_times"] = 1 else: internal_settings["iter_times"] = cmd_args.iter_times - internal_settings["args_element_assignment"] = generate_args_element_assignment_code(args_info) - internal_settings["args_list_generator_device"] = generate_args_list_device(args_info) - internal_settings["args_list_generator_bench"] = generate_args_list_bench(args_info) - internal_settings["kwargs_value_assignment"] = generate_kwargs_value_assignment_code(kwargs_info) - internal_settings["kwargs_dict_generator_device"] = generate_kwargs_dict_device(kwargs_info) - internal_settings["kwargs_dict_generator_bench"] = generate_kwargs_dict_bench(kwargs_info) + internal_settings["args_element_assignment"] = generate_args_element_assignment_code(args_info_forward) + internal_settings["args_list_generator_device"] = generate_args_list_device(args_info_forward) + internal_settings["args_list_generator_bench"] = generate_args_list_bench(args_info_forward) + internal_settings["kwargs_value_assignment"] = generate_kwargs_value_assignment_code(kwargs_info_forward) + internal_settings["kwargs_dict_generator_device"] = generate_kwargs_dict_device(kwargs_info_forward) + internal_settings["kwargs_dict_generator_bench"] = generate_kwargs_dict_bench(kwargs_info_forward) + if cmd_args.propagation == Const.BACKWARD: + internal_settings["args_element_assignment_backward"] = generate_args_element_assignment_code(args_info_backward) + internal_settings["args_list_generator_device_backward"] = generate_args_list_device(args_info_backward) + internal_settings["args_list_generator_bench_backward"] = generate_args_list_bench(args_info_backward) + else: + internal_settings["args_element_assignment_backward"] = '' + internal_settings["args_list_generator_device_backward"] = '' + internal_settings["args_list_generator_bench_backward"] = '' + return internal_settings @@ -226,19 +320,69 @@ def generate_kwargs_dict_bench(kwargs_info): kwargs_dict_generator_bench += recursive_kwargs_dict(value, flag_bench=True) + ", " return kwargs_dict_generator_bench +def update_data_name(data, dump_data_dir): + if isinstance(data, list): + for item in data: + update_data_name(item, dump_data_dir) + elif DATA_NAME in data: + data[DATA_NAME] = os.path.join(dump_data_dir, data[DATA_NAME]) + +def load_real_data_path(value, dump_data_dir): + parameters = [Const.INPUT_ARGS, Const.GRAD_INPUT, Const.INPUT, Const.OUTPUT, Const.GRAD_OUTPUT] + for parameter in parameters: + for v in value.get(parameter, []): + if v is not None: + update_data_name(v, dump_data_dir) + return value + +def extract_op(args): + check_file_or_directory_path(args.dump_json_path) + check_path_pattern_valid(args.api_name) + with FileOpen(args.dump_json_path, 'r') as file: + data = json.load(file) + new_data = {} + extract_key_pattern = re.compile(f"^{re.escape(args.api_name)}\..+") + real_data_path = data.get('dump_data_dir', '') + for key, value in data.get('data', {}).items(): + if extract_key_pattern.match(key): + if real_data_path: + value = load_real_data_path(value, real_data_path) + new_data[key] = value + if not new_data: + logger.error(f"Error: The api '{args.api_name}' does not exist in the file.") + else: + with FileOpen(args.output_file, 'w') as file: + json.dump(new_data, file, indent=4) + logger.info(f"The api '{args.api_name}' has been successfully extracted and saved in: {args.output_file}") def op_generator_parser(parser): - parser.add_argument("-forward", "--forward_json_path", dest="forward_json_path", type=str, - help=" Path of forward API json file.", - required=True) - parser.add_argument("-m", "--mode", dest="mode", type=str, choices=("random_data", "real_data"), - help=" Execute mode, should be random_data or real_data.", + parser.add_argument("-dump", "--dump_json_path", dest="dump_json_path", default='', type=str, + help=" Path of dump json file, if the API has already been extracted," + "don't need to set it up.", + required=False) + parser.add_argument("-o", "--output_file", dest="output_file", type=str, + help=" Path of extract api_name.json.", required=True) + parser.add_argument("-a", "--api_name", dest="api_name", type=str, + help=" extract api_name, if the API has already been extracted," + "don't need to set it up.", + required=False) + parser.add_argument("-p", "--propagation", dest="propagation", default='forward', type=str, + choices=("forward", "backward"), + help=" forward or backward, the default value is forward.", + required=False) + parser.add_argument("-m", "--mode", dest="mode", type=str, default="random_data", + choices=("random_data", "real_data"), + help=" Execute mode, should be random_data or real_data," + "the default value is random_data.", + required=False) parser.add_argument("-rs", "--random_seed", dest = "random_seed", type=int, default=1234, - help=" If mode is random_data, it is random seed.", + help=" If mode is random_data, it is random seed, " + "the default value is 1234.", required=False) - parser.add_argument("-it", "--iter_times", dest="iter_times", type=int, default=5, - help=" If mode is random_data, generate iter_times group of data.", + parser.add_argument("-it", "--iter_times", dest="iter_times", type=int, default=1, + help=" If mode is random_data, generate iter_times group of data," + "the default value is 1.", required=False) @@ -246,19 +390,51 @@ def main(): parser = argparse.ArgumentParser() op_generator_parser(parser) cmd_args = parser.parse_args() - internal_settings = get_settings(cmd_args) + + output_path = os.path.dirname(cmd_args.output_file) + if not os.path.exists(output_path): + create_directory(output_path) + logger.info(f"Directory '{output_path}' created.") + else: + logger.info(f"Directory '{output_path}' already exists.") + check_file_or_directory_path(output_path, isdir=True) + if cmd_args.dump_json_path: + extract_op(cmd_args) + check_file_or_directory_path(cmd_args.output_file) + + if cmd_args.propagation == Const.BACKWARD: + # read and check json + api_result = check_user_settings(cmd_args) + api_full_name_forward, api_info_dict_forward = api_result.api_full_name, api_result.api_info_dict + api_full_name_backward, api_info_dict_backward = (api_result.backward_info.api_full_name, + api_result.backward_info.api_info_dict) + args_info_forward = api_info_dict_forward.get(Const.INPUT_ARGS) + kwargs_info_forward = api_info_dict_forward.get(Const.INPUT_KWARGS) + if Const.GRAD_INPUT in api_info_dict_backward: + args_info_backward = api_info_dict_backward.get(Const.GRAD_INPUT) + elif Const.INPUT in api_info_dict_backward: + args_info_backward = api_info_dict_backward.get(Const.INPUT) + internal_settings = get_settings(cmd_args, api_full_name_backward, args_info_forward, kwargs_info_forward, + args_info_backward) + else: + # read and check json + api_result = check_user_settings(cmd_args) + api_full_name_forward, api_info_dict_forward = api_result.api_full_name, api_result.api_info_dict + args_info_forward = api_info_dict_forward.get(Const.INPUT_ARGS) + kwargs_info_forward = api_info_dict_forward.get(Const.INPUT_KWARGS) + internal_settings = get_settings(cmd_args, api_full_name_forward, args_info_forward, kwargs_info_forward,None) template_path = os.path.join(os.path.dirname(__file__), "operator_replication.template") operator_script_path = os.path.join(os.path.dirname(__file__), "{0}.py".format(internal_settings.get("api_full_name"))) try: - with open(template_path, 'r') as ftemp, open(operator_script_path, 'w') as fout: + with FileOpen(template_path, 'r') as ftemp, FileOpen(operator_script_path, 'w') as fout: code_template = ftemp.read() fout.write(code_template.format(**internal_settings)) except OSError: - print(f"Failed to open file. Please check file {template_path} or {operator_script_path}.") + logger.error(f"Failed to open file. Please check file {template_path} or {operator_script_path}.") - print(f"Generate operator script successfully and the name is {operator_script_path}.") + logger.info(f"Generate operator script successfully and the name is {operator_script_path}.") if __name__ == "__main__": diff --git a/debug/accuracy_tools/api_accuracy_checker/generate_op_script/operator_replication.template b/debug/accuracy_tools/api_accuracy_checker/generate_op_script/operator_replication.template index 7630839aa937c6d0419629b5e93c34b51b71f295..2a555dc43c21782109a44483b127d71fd754af4c 100644 --- a/debug/accuracy_tools/api_accuracy_checker/generate_op_script/operator_replication.template +++ b/debug/accuracy_tools/api_accuracy_checker/generate_op_script/operator_replication.template @@ -7,13 +7,16 @@ try: import torch_npu except ImportError: pass - +from tabulate import tabulate +from msprobe.core.common.utils import check_file_or_directory_path +from msprobe.core.common.log import logger +from msprobe.core.common.const import Const TENSOR_DATA_LIST = ["torch.Tensor", "torch.nn.parameter.Parameter"] TORCH_BOOL_TYPE = ["torch.bool"] TORCH_INT_TYPE = ["torch.uint8", "torch.int8", "torch.int16", "torch.short", "torch.int32", "torch.int", "torch.int64", "torch.long"] -TORCH_FLOAT_TYPE = ["torch.float16", "torch.half", "torch.bfloat16", "torch.float32", "torch.float", +TORCH_FLOAT_TYPE = ["torch.float16", "torch.half", "torch.bfloat16", "torch.float32", "torch.float", "torch.float64", "torch.double"] TORCH_COMPLEX_TYPE = ["torch.complex32", "torch.chalf", "torch.complex64", "torch.cfloat", "torch.complex128", "torch.cdouble"] RAISE_PRECISION = {{ @@ -23,14 +26,26 @@ RAISE_PRECISION = {{ "torch.float32": torch.float64, "torch.float": torch.float64 }} - +THOUSANDTH_THRESHOLDING = 0.001 class CompareStandard(Enum): BINARY_EQUALITY_STANDARD = auto() ABSOLUTE_THRESHOLD_STANDARD = auto() ULP_ERROR_STANDARD = auto() BENCHMARK_STANDARD = auto() - + THOUSANDTH_STANDARD = auto() + +def load_pt(pt_path, to_cpu=False): + pt_path = os.path.realpath(pt_path) + check_file_or_directory_path(pt_path) + try: + if to_cpu: + pt = torch.load(pt_path, map_location=torch.device("cpu")) + else: + pt = torch.load(pt_path) + except Exception as e: + raise RuntimeError(f"load pt file {{pt_path}} failed") from e + return pt def get_device(): if torch.cuda.is_available(): @@ -80,14 +95,16 @@ def generate_random_tensor(info): def generate_real_tensor(data_path): + check_file_or_directory_path(data_path) data_path = os.path.realpath(data_path) - data = torch.load(data_path) + data = load_pt(data_path, to_cpu = True) return data def generate_data(info): data_type = info.get("type") - data_path = info.get("datapath") + data_path = info.get("data_name") + data_grad = info.get("requires_grad") if data_type in TENSOR_DATA_LIST: if data_path: data = generate_real_tensor(data_path) @@ -95,26 +112,44 @@ def generate_data(info): data = generate_random_tensor(info) else: data = info.get("value") + if data_grad == True: + data.requires_grad_(True) return data -def get_input(): +def get_input(propagation): {args_element_assignment} args_device = [{args_list_generator_device}] args_bench = [{args_list_generator_bench}] {kwargs_value_assignment} kwargs_device = {{{kwargs_dict_generator_device}}} kwargs_bench = {{{kwargs_dict_generator_bench}}} +{args_element_assignment_backward} + args_device_backward = [{args_list_generator_device_backward}] + args_bench_backward = [{args_list_generator_bench_backward}] + if propagation == Const.BACKWARD: + return args_device, kwargs_device, args_bench, kwargs_bench, args_device_backward, args_bench_backward return args_device, kwargs_device, args_bench, kwargs_bench - -def exec_api_device(args, kwargs): +def exec_api_device(args, kwargs, args_grad_input, propagation): output_device = {api_type}.{api_name}(*args, **kwargs) + if propagation == Const.BACKWARD: + args_input_tensor = [tensor for tensor in args if isinstance(tensor, torch.Tensor) and tensor.requires_grad] + args_input_tensor.extend( + [value for value in kwargs.values() if isinstance(value, torch.Tensor) and value.requires_grad]) + output_device_new = torch.autograd.grad(outputs=output_device, inputs=args_input_tensor, grad_outputs=args_grad_input) + return output_device_new return output_device -def exec_api_bench(args, kwargs): +def exec_api_bench(args, kwargs, args_grad_input, propagation): output_bench = {api_type}.{api_name}(*args, **kwargs) + if propagation == Const.BACKWARD: + args_input_tensor = [tensor for tensor in args if isinstance(tensor, torch.Tensor) and tensor.requires_grad] + args_input_tensor.extend( + [value for value in kwargs.values() if isinstance(value, torch.Tensor) and value.requires_grad]) + output_bench_new = torch.autograd.grad(outputs=output_bench, inputs=args_input_tensor, grad_outputs=args_grad_input) + return output_bench_new return output_bench @@ -148,32 +183,38 @@ def compute_rmse(abs_err, normal_value_mask): def compute_error_balance(out_device, out_bench): larger_count = torch.sum(torch.greater(out_device - out_bench.to(out_device.dtype), 0)) smaller_count = torch.sum(torch.less(out_device - out_bench.to(out_device.dtype), 0)) - total_count = torch.numel(out_bench) - error_balance = abs(larger_count - smaller_count) / total_count + if torch.numel(out_bench) == 0: + raise ZeroDivisionError(f"ERROR: please check torch.numel out_bench, its value is {{torch.numel(out_bench)}}") + error_balance = abs(larger_count - smaller_count) / torch.numel(out_bench) return error_balance def compare_tensor(out_device, out_bench, api_name): if out_device.shape != out_bench.shape: - print("ERROR: shape of out_device and out_bench is not equal!") + logger.error("ERROR: shape of out_device and out_bench is not equal!") return None if torch.numel(out_bench) == 0: - print("Both out_device and out_bench have zero elements.") + logger.error("Both out_device and out_bench have zero elements.") return None - print(f"shape is {{out_bench.shape}}") - print(f"dtype of out_device is {{out_device.dtype}}") - print(f"dtype of out_bench is {{out_bench.dtype}}") dtype_device = out_device.dtype dtype_bench = out_bench.dtype + headers = ["Metric", "Value"] + table = [ + ["Shape", out_bench.shape], + ["Dtype of out_device", out_device.dtype], + ["Dtype of out_bench", out_bench.dtype] + ] if str(dtype_device) in TORCH_FLOAT_TYPE and str(dtype_bench) in TORCH_FLOAT_TYPE \ or str(dtype_device) in TORCH_INT_TYPE and str(dtype_bench) in TORCH_INT_TYPE \ or str(dtype_device) in TORCH_BOOL_TYPE and str(dtype_bench) in TORCH_BOOL_TYPE: out_device = out_device.to(torch.device("cpu")) if str(dtype_device) in TORCH_BOOL_TYPE or str(dtype_device) in TORCH_INT_TYPE or compare_standard == CompareStandard.BINARY_EQUALITY_STANDARD: - print("compare standard: binary equality standard:") error_number = torch.sum(out_device != out_bench).item() + if torch.numel(out_bench) == 0: + raise ZeroDivisionError(f"ERROR: please check torch.numel out_bench, its value is {{torch.numel(out_bench)}}") error_rate = error_number / torch.numel(out_bench) - print(f"error rate is {{error_rate}}.") + table.append(["Compare Standard", "Binary Equality Standard"]) + table.append(["Error Rate", error_rate]) else: abs_err = torch.abs(out_device - out_bench) abs_bench = torch.abs(out_bench) @@ -210,9 +251,9 @@ def compare_tensor(out_device, out_bench, api_name): abs_err_proportion = 0 else: abs_err_proportion = torch.sum(abs_err_mask) / torch.sum(small_value_mask) - print("compare standard: absolute threshold standard") - print(f"relative error ratio is {{rel_err_proportion}}") - print(f"absolute error ratio is {{abs_err_proportion}}") + table.append(["Compare Standard", "Absolute Threshold Standard"]) + table.append(["Relative Error Ratio", rel_err_proportion]) + table.append(["Absolute Error Ratio", abs_err_proportion]) elif compare_standard == CompareStandard.ULP_ERROR_STANDARD: if dtype_device == torch.float16: min_eb, exponent_num = -14, 10 @@ -229,14 +270,25 @@ def compare_tensor(out_device, out_bench, api_name): ulp_err = torch.abs(ulp_err) max_ulp_err = torch.max(ulp_err) mean_ulp_err = torch.mean(ulp_err) + if torch.numel(out_bench) == 0: + raise ZeroDivisionError(f"ERROR: please check torch.numel out_bench, its value is {{torch.numel(out_bench)}}") if dtype_device == torch.float32: ulp_err_proportion = torch.sum(ulp_err > 32) / torch.numel(out_bench) else: ulp_err_proportion = torch.sum(ulp_err > 1) / torch.numel(out_bench) - print("compare standard: ulp error standard") - print(f"maximum ulp error is {{max_ulp_err}}") - print(f"mean ulp error is {{mean_ulp_err}}") - print(f"ulp error proportion is {{ulp_err_proportion}}") + table.append(["Compare Standard", "ULP error Standard"]) + table.append(["Maximum ULP Error", max_ulp_err]) + table.append(["Mean ULP Error", mean_ulp_err]) + table.append(["ULP Error Proportion", ulp_err_proportion]) + elif compare_standard == CompareStandard.THOUSANDTH_STANDARD: + rel_err_origin = torch.abs(abs_err / abs_bench_with_eps) + if torch.numel(rel_err_origin) == 0: + thousand_res = 1 + else: + thousand_res = torch.divide(torch.sum(rel_err < THOUSANDTH_THRESHOLDING), torch.numel(rel_err_origin)) + thousand_status = thousand_res > (1 - THOUSANDTH_THRESHOLDING) + table.append(["Compare Standard", "Thousandth Standard"]) + table.append(["Thousandth ratio", thousand_res]) else: if dtype_device == torch.float16: small_value, small_value_atol = 1.0e-3, 1.0e-5 @@ -264,62 +316,68 @@ def compare_tensor(out_device, out_bench, api_name): mean_rel_err = torch.sum(torch.clamp(rel_err, min=0)) / torch.sum(normal_value_mask) rmse = compute_rmse(abs_err, normal_value_mask) error_balance = compute_error_balance(out_device, out_bench) - print("compare standard: benchmark standard") - print(f"small value error proportion is {{small_value_err_proportion}}") - print(f"maximum relative error is {{max_rel_err}}") - print(f"mean relative error is {{mean_rel_err}}") - print(f"root mean squared error is {{rmse}}") - print(f"error balance is {{error_balance}}") + table.append(["Compare Standard", "Benchmark Standard"]) + table.append(["Small Value Error Proportion", small_value_err_proportion]) + table.append(["Maximum Relative Error", max_rel_err]) + table.append(["Mean Relative Error", mean_rel_err]) + table.append(["Root Mean Squared Error", rmse]) + table.append(["Error Balance", error_balance]) else: - print(f"ERROR: out_device dtype is {{dtype_device}}, out_bench dtype is {{dtype_bench}}, not comparable.") + logger.error(f"ERROR: out_device dtype is {{dtype_device}}, out_bench dtype is {{dtype_bench}}, not comparable.") + return None + print(tabulate(table, headers, tablefmt='grid')) return None def compare_element(out_device, out_bench, api_name): if type(out_device) != type(out_bench): - print("ERROR: out_device and out_bench is not the same type!") + logger.error("ERROR: out_device and out_bench is not the same type!") return None if isinstance(out_bench, torch.Tensor): - print(f"data type: {{type(out_bench)}}") compare_tensor(out_device, out_bench, api_name) elif isinstance(out_bench, (bool, int, float, str)): - print(f"data type: {{type(out_bench)}}") if out_device == out_bench: - print("PASS: out_device and out_bench equals.") + logger.info("PASS: out_device and out_bench equals.") else: - print("ERROR: out_device and out_bench is not equal!") + logger.error("ERROR: out_device and out_bench is not equal!") else: - print(f"ERROR: comparison of type {{type(out_bench)}} is not supported.") + logger.error(f"ERROR: comparison of type {{type(out_bench)}} is not supported.") return None def compare(out_device, out_bench, api_name): - print("Compare result:") + logger.info("Compare result:") if type(out_device) != type(out_bench): - print("ERROR: out_device and out_bench is not the same type!") - print("Compare finished.") + logger.error("ERROR: out_device and out_bench is not the same type!") + logger.info("Compare finished.") return None if isinstance(out_bench, (list, tuple)): - print(f"data type: {{type(out_bench)}}") if len(out_device) != len(out_bench): - print("ERROR: len of out_device and out_bench is different!") - print("Compare finished.") + logger.error("ERROR: len of out_device and out_bench is different!") + logger.info("Compare finished.") return None for index, _ in enumerate(out_bench): - print(f"index {{index}}:") + logger.info(f"index {{index}}:") compare_element(out_device[index], out_bench[index], api_name) else: compare_element(out_device, out_bench, api_name) - print("Compare finished.") + logger.info("Compare finished.") device = get_device() api_name = "{api_name}" +propagation = "{propagation}" compare_standard = {compare_standard} torch.manual_seed({random_seed}) for i in range({iter_times}): - print(f"iter: {{i}}:") - args_device, kwargs_device, args_bench, kwargs_bench = get_input() - output_device = exec_api_device(args_device, kwargs_device) - output_bench = exec_api_bench(args_bench, kwargs_bench) - compare(output_device, output_bench, api_name) + logger.info(f"iter: {{i}}:") + if propagation == Const.BACKWARD: + args_device, kwargs_device, args_bench, kwargs_bench, args_device_backward, args_bench_backward = get_input(propagation) + output_device = exec_api_device(args_device, kwargs_device, args_device_backward, propagation) + output_bench = exec_api_bench(args_bench, kwargs_bench, args_bench_backward, propagation) + compare(output_device, output_bench, api_name) + else: + args_device, kwargs_device, args_bench, kwargs_bench = get_input(propagation) + output_device = exec_api_device(args_device, kwargs_device, None, propagation) + output_bench = exec_api_bench(args_bench, kwargs_bench, None, propagation) + compare(output_device, output_bench, api_name)