diff --git a/ais-bench_workload/experimental_tools/benchmark/ais_bench/benchmark/datasets/synthetic.py b/ais-bench_workload/experimental_tools/benchmark/ais_bench/benchmark/datasets/synthetic.py index 81a0c6f6e0cf269d5feb54f0cf1469d5f3e95943..2e58a74176b09655e802af2f49c1681d29c54d68 100644 --- a/ais-bench_workload/experimental_tools/benchmark/ais_bench/benchmark/datasets/synthetic.py +++ b/ais-bench_workload/experimental_tools/benchmark/ais_bench/benchmark/datasets/synthetic.py @@ -56,6 +56,16 @@ def check_range(name: str, value: Any, param: NumberRange): if gt or ge: raise ValueError(f"Parameter {name} is {value}, not within the required range {interval_str}") +def get_config(path): + path = get_data_path(path) + path = os.path.join(path, "synthetic_config.json") + try: + with open(path, mode="r", encoding="utf-8") as file: + config = json.load(file) + return config + except (FileNotFoundError, json.JSONDecodeError) as e: + raise ValueError("Failed to load JSON config from `SyntheticConfigPath` file.") from e + return None @LOAD_DATASET.register_module() class SyntheticDataset(BaseDataset): @@ -158,14 +168,8 @@ class SyntheticDataset(BaseDataset): return data + str(num_expect_generated_tokens) def load(self, path, **kwargs): - path = get_data_path(path) - path = os.path.join(path, "synthetic_config.json") + config = get_config(path) dataset = [] - try: - with open(path, mode="r", encoding="utf-8") as file: - config = json.load(file) - except (FileNotFoundError, json.JSONDecodeError) as e: - raise ValueError("Failed to load JSON config from `SyntheticConfigPath` file.") from e self._check_config_json(config) request_count = config.get("RequestCount") input_method = config["Input"]["Method"] diff --git a/ais-bench_workload/experimental_tools/benchmark/tests/ST/test_performance.py b/ais-bench_workload/experimental_tools/benchmark/tests/ST/test_performance.py index 878862d2e4196cc0861d85cc5c0312b08cde04a1..7d6e51ba7231354c5262ce2ce5bc3850dde05e24 100644 --- a/ais-bench_workload/experimental_tools/benchmark/tests/ST/test_performance.py +++ b/ais-bench_workload/experimental_tools/benchmark/tests/ST/test_performance.py @@ -4,6 +4,7 @@ import shutil import sys import logging import pytest +import pandas as pd from ais_bench.benchmark.cli.main import main DATASETS_CONFIGS_LIST = [ @@ -37,6 +38,16 @@ class TestClass: shutil.rmtree(self.test_data_path) os.makedirs(self.test_data_path) self._set_datasets_config_path(self) + self.perf_json_keys = ['Benchmark Duration', 'Total Requests', 'Failed Requests', 'Success Requests', + 'Concurrency', 'Max Concurrency', 'Request Throughput', 'Total Input Tokens', + 'Prefill Token Throughput', 'Total Output Tokens', 'Input Token Throughput', + 'Output Token Throughput', 'Total Token Throughput'] + self.perf_csv_headers = ['Performance Parameters', 'Average', 'Min', 'Max', + 'Median', 'P75', 'P90', 'P99', 'N'] + self.perf_csv_params = ['Latency', 'TTFT', 'TPOT', 'InputTokens', 'OutputTokens', + 'PrefillTokenThroughput', 'OutputTokenThroughput', + 'Tokenizer', 'Detokenizer'] + def _set_datasets_config_path(self): dataset_configs_base_dir = os.path.abspath(os.path.join(self.cur_dir, "../../ais_bench/benchmark/configs/datasets")) @@ -77,6 +88,130 @@ class TestClass: assert os.path.exists(vis_txt_path) vis_md_path = os.path.join(self.test_data_path, f"{fake_time_str}/summary/summary_{fake_time_str}.md") assert os.path.exists(vis_md_path) + + def test_perf_mindie_stream_api_qwen2_7b_synthetic_save_result(self, monkeypatch): + fake_prediction = [{'id': '0', 'input_data': 'A A', 'input_token_id': [32, 362, 362], + 'output': ' A A A', 'output_token_id': [362, 362, 362], + 'prefill_latency': 56.9, 'prefill_throughput': 333.6, + 'decode_token_latencies': [26.4, 28.4], 'last_decode_latency': 28.4, + 'decode_max_token_latency': 28.4, 'seq_latency': 2700.04, + 'input_tokens_len': 2, 'generate_tokens_len': 3, + 'generate_tokens_speed': 37.03, 'input_characters_len': 3, + 'generate_characters_len': 6, 'tokenizer_time': 11.4, + 'detokenizer_time': [0.02, 0.02], 'characters_per_token': 2.0, + 'prefill_batch_size': 0, 'decode_batch_size': [], 'queue_wait_time': [], + 'request_id': '591c69416c694a6ab3194a06d6e1ed17', + 'start_time': 1742952029.5993671, 'end_time': 1742952032.299417, + 'is_success': True, 'is_empty': False}] + fake_time_str = "perf_synthetic_save_result" + datasets_script_name = "synthetic_gen" + + monkeypatch.setattr('sys.argv', + ["ais_bench", "--models", "mindie_stream_api_general", "--datasets", datasets_script_name, + "--mode", "perf", "-w", self.test_data_path]) + monkeypatch.setattr("ais_bench.benchmark.models.performance_api.PerformanceAPIModel.get_perforamce_data", lambda *arg: fake_prediction) + monkeypatch.setattr("ais_bench.benchmark.cli.main.get_current_time_str", lambda *arg: fake_time_str) + main() + + # check perf json + infer_outputs_json_path = os.path.join(self.test_data_path, f"{fake_time_str}/performances/mindie-stream-api/synthetic/synthetic.json") + assert os.path.exists(infer_outputs_json_path) + with open(infer_outputs_json_path, 'r') as file: + data = json.load(file) + assert isinstance(data, dict) + for key in self.perf_json_keys: + assert key in data + assert data['Total'] == len(fake_prediction) + + #check perf csv + infer_outputs_csv_path = os.path.join(self.test_data_path, f"{fake_time_str}/performances/mindie-stream-api/synthetic/synthetic.csv") + assert os.path.exists(infer_outputs_csv_path) + + data = pd.read_csv(infer_outputs_csv_path) + for header in self.perf_csv_headers: + assert header in data.columns + first_column = data.iloc[:,0] + for param in self.perf_csv_params: + assert param in first_column.values + + assert data.loc[data['Performance Parameters'] == 'MaxDecodeTime', 'Max'].values[0] == str(fake_prediction[0]['decode_max_token_latency']) + ' ms' + assert data.loc[data['Performance Parameters'] == 'GeneratedTokenSpeed', 'Average'].values[0] == str(fake_prediction[0]['generate_tokens_speed']) + ' token/s' + + def test_perf_mindie_stream_api_qwen2_7b_synthetic_in80_out110_req3(self, monkeypatch): + fake_time_str = "perf_synthetic_config1" + datasets_script_name = "synthetic_gen" + + path = os.path.abspath(os.path.join(self.cur_dir, "../datasets/synthetic/synthetic_config1.json")) + with open(path, mode="r", encoding="utf-8") as file: + fake_config = json.load(file) + + monkeypatch.setattr('sys.argv', + ["ais_bench", "--models", "mindie_stream_api_general", "--datasets", datasets_script_name, + "--mode", "perf", "-w", self.test_data_path]) + monkeypatch.setattr("ais_bench.benchmark.datasets.synthetic.get_config", lambda *arg: fake_config) + monkeypatch.setattr("ais_bench.benchmark.cli.main.get_current_time_str", lambda *arg: fake_time_str) + main() + + # check perf json + infer_outputs_json_path = os.path.join(self.test_data_path, f"{fake_time_str}/performances/mindie-stream-api/synthetic/synthetic.json") + assert os.path.exists(infer_outputs_json_path) + with open(infer_outputs_json_path, 'r', encoding='utf-8') as file: + data = json.load(file) + assert isinstance(data, dict) + for key in self.perf_json_keys: + assert key in data + assert data['Total'] == fake_config['RequestCount'] + + #check perf csv + infer_outputs_csv_path = os.path.join(self.test_data_path, f"{fake_time_str}/performances/mindie-stream-api/synthetic/synthetic.csv") + assert os.path.exists(infer_outputs_csv_path) + data = pd.read_csv(infer_outputs_csv_path) + for header in self.perf_csv_headers: + assert header in data.columns + first_column = data.iloc[:,0] + for param in self.perf_csv_params: + assert param in first_column.values + + assert float(data.loc[data['Performance Parameters'] == 'InputTokens', 'Max'].values[0])==float(fake_config['Input']['Params']['MinValue']) + assert float(data.loc[data['Performance Parameters'] == 'GeneratedTokens', 'Max'].values[0])==float(fake_config['Output']['Params']['MinValue']) + + def test_perf_mindie_stream_api_qwen2_7b_synthetic_in130_out200_req5(self, monkeypatch): + fake_time_str = "perf_synthetic_config2" + datasets_script_name = "synthetic_gen" + + path = os.path.abspath(os.path.join(self.cur_dir, "../datasets/synthetic/synthetic_config2.json")) + with open(path, mode="r", encoding="utf-8") as file: + fake_config = json.load(file) + + monkeypatch.setattr('sys.argv', + ["ais_bench", "--models", "mindie_stream_api_general", "--datasets", datasets_script_name, + "--mode", "perf", "-w", self.test_data_path]) + monkeypatch.setattr("ais_bench.benchmark.datasets.synthetic.get_config", lambda *arg: fake_config) + monkeypatch.setattr("ais_bench.benchmark.cli.main.get_current_time_str", lambda *arg: fake_time_str) + main() + + # check perf json + infer_outputs_json_path = os.path.join(self.test_data_path, f"{fake_time_str}/performances/mindie-stream-api/synthetic/synthetic.json") + assert os.path.exists(infer_outputs_json_path) + with open(infer_outputs_json_path, 'r', encoding='utf-8') as file: + data = json.load(file) + assert isinstance(data, dict) + for key in self.perf_json_keys: + assert key in data + assert data['Total'] == fake_config['RequestCount'] + + #check perf csv + infer_outputs_csv_path = os.path.join(self.test_data_path, f"{fake_time_str}/performances/mindie-stream-api/synthetic/synthetic.csv") + assert os.path.exists(infer_outputs_csv_path) + data = pd.read_csv(infer_outputs_csv_path) + for header in self.perf_csv_headers: + assert header in data.columns + first_column = data.iloc[:,0] + for param in self.perf_csv_params: + assert param in first_column.values + + assert float(data.loc[data['Performance Parameters'] == 'InputTokens', 'Max'].values[0])==float(fake_config['Input']['Params']['MinValue']) + assert float(data.loc[data['Performance Parameters'] == 'GeneratedTokens', 'Max'].values[0])==float(fake_config['Output']['Params']['MinValue']) diff --git a/ais-bench_workload/experimental_tools/benchmark/tests/datasets/synthetic/synthetic_config1.json b/ais-bench_workload/experimental_tools/benchmark/tests/datasets/synthetic/synthetic_config1.json new file mode 100644 index 0000000000000000000000000000000000000000..bccad55e7e3b64478198833f602fc583d262ace6 --- /dev/null +++ b/ais-bench_workload/experimental_tools/benchmark/tests/datasets/synthetic/synthetic_config1.json @@ -0,0 +1,11 @@ +{ + "Input":{ + "Method": "uniform", + "Params": {"MinValue": 80, "MaxValue": 80} + }, + "Output": { + "Method": "gaussian", + "Params": {"Mean": 100, "Var": 200, "MinValue": 110, "MaxValue": 110} + }, + "RequestCount": 3 +} \ No newline at end of file diff --git a/ais-bench_workload/experimental_tools/benchmark/tests/datasets/synthetic/synthetic_config2.json b/ais-bench_workload/experimental_tools/benchmark/tests/datasets/synthetic/synthetic_config2.json new file mode 100644 index 0000000000000000000000000000000000000000..45315bcf92c53f098d3745e10ad4a7cbc1501eab --- /dev/null +++ b/ais-bench_workload/experimental_tools/benchmark/tests/datasets/synthetic/synthetic_config2.json @@ -0,0 +1,11 @@ +{ + "Input":{ + "Method": "uniform", + "Params": {"MinValue": 130, "MaxValue": 130} + }, + "Output": { + "Method": "gaussian", + "Params": {"Mean": 100, "Var": 200, "MinValue": 200, "MaxValue": 200} + }, + "RequestCount": 5 +} \ No newline at end of file