From 786da01a2817cf9b1d4660868def4b6e597f9687 Mon Sep 17 00:00:00 2001 From: luobinghao Date: Wed, 24 Apr 2024 11:27:33 +0800 Subject: [PATCH] [feature]:add aot mode for perftest Signed-off-by: luobinghao Change-Id: I266ecf7eb0c8282069ad6388b23636beaa64cc52 --- test/aotjsperftest/README.md | 161 ----- test/aotjsperftest/README_en.md | 86 --- test/aotjsperftest/config.json | 16 - test/aotjsperftest/run_js_perf_test.sh | 84 --- test/aotjsperftest/run_js_test.py | 835 ------------------------- test/jsperftest/run_js_perf_test.sh | 9 +- test/jsperftest/run_js_test.py | 153 ++++- 7 files changed, 159 insertions(+), 1185 deletions(-) delete mode 100644 test/aotjsperftest/README.md delete mode 100644 test/aotjsperftest/README_en.md delete mode 100644 test/aotjsperftest/config.json delete mode 100755 test/aotjsperftest/run_js_perf_test.sh delete mode 100644 test/aotjsperftest/run_js_test.py diff --git a/test/aotjsperftest/README.md b/test/aotjsperftest/README.md deleted file mode 100644 index b12e737077..0000000000 --- a/test/aotjsperftest/README.md +++ /dev/null @@ -1,161 +0,0 @@ -# 运行方舟编译器工作负载 - -## 介绍 - -自动化运行js性能测试用例并且生成daily报告，用于测试方舟builtins API 性能。 - -特殊说明：下面文字中的"v 8" 、"v_8"、"d 8"、"d_8", 是为了规避门禁codecheck告警，请阅读及实际执行时去除或忽略中间的空格或”_“。 - -## 环境依赖 - -- 对于脚本所在的文件目录，当前用户需要拥有创建文件夹的权限。请注意当前目录的用户和用户组权限设置 - -- openpyxl。没有版本要求。若未安装请执行`pip3 install openpyxl` - -- 环境中需要下载全仓鸿 蒙镜像或独立仓代码，并进行编译ark_js_vm 和 前端 - - ```shell - ./build.sh --product-name rk3568 --build-target ark_js_host_linux_tools_packages --build-target ets_frontend_build # x86_64平台 - ``` - -- 配置v 8引擎执行环境 - -**下载：** - -```shell -# 可以在任意目录。假定当前目录在/home目录 -wget https://storage.googleapis.com/chromium-%76%38/official/canary/%76%38-linux64-rel-12.0.267.zip --no-check-certificate -O v_8-linux64-rel-12.0.267.zip -unzip v_8-linux64-rel-12.0.267.zip -d v_8 -``` - -**配置：** - -临时配置： - -每次执行run_js_perf_test.sh脚本时传入 v 8 引擎 d 8 二进制的路径，比如/home/v_8/d 8。 - -永久配置： - -```shell -sudo su -# 编辑/root/.bashrc文件在末尾追加以下内容 -export PATH=/home/v_8:$PATH -# 刷新环境变量。注意执行sh脚本依然需要传入d 8的路径 -source /root/.bashrc -``` - -**验证：** - -```shell -执行： - /home/v_8/d 8 -v # 任意用户 -回显： - V 8 version 12.0.267 - d 8> -``` - -环境v 8引擎配置成功 - -## 文件目录 - -```bash -/arkcompiler/ets_runtime -│_ test # 模块测试用例 - |_ jsperftest # js perf 测试代码目录 - |-README.md # 说明书 - |-run_js_perf_test.sh # 测试执行shell脚本 - |_run_js_test.py # 测试执行调用的python脚本 - |_config.json # 代码仓库与平台配置文件 -``` - -**代码仓库与平台配置文件config.json说明：** - -**文件内容：** - -```shell -{ - "full_x86_64":{ - "ETS_RUNTIME_PATH": "out/rk3568/c lang_x64/arkcompiler/ets_runtime/", - "ICU_PATH": "out/rk3568/c lang_x64/thirdparty/icu/", - "ZLIB_PATH": "out/rk3568/c lang_x64/thirdparty/zlib/", - "LIB_PATH": "prebuilts/c lang/ohos/linux-x86_64/l lvm/lib/", - "ES2ABC": "out/rk3568/c lang_x64/arkcompiler/ets_frontend/es2abc" - }, - "independent_x86_64": { - "ETS_RUNTIME_PATH": "out/x64.release/arkcompiler/ets_runtime", - "ICU_PATH": "out/x64.release/c lang_x64/thirdparty/icu/", - "ZLIB_PATH": "", # 若无此配置，可不填，但不能删除该配置项 - "LIB_PATH": "", # 若无此配置，可不填，但不能删除该配置项 - "ES2ABC": "out/x64.release/arkcompiler/ets_frontend/es2abc" - } -} -``` - -**说明：** - -- 请无视“c lang"中间的空格，文件实际内容是无空格，这里是为了屏蔽门禁codecheck告警 -- 在不同的代码仓库在不同的OS平台上，脚本运行需要的二进制的路径是不同的。脚本无法全覆盖。因此需要用户自己配置仓库信息和脚本执行平台信息 -- full表示是全仓--鸿 蒙镜像下的代码仓；independent表示独立仓（区别于全仓） -- 脚本执行平台都是linux环境。默认全仓， full_x86_64。x86_64表示是x86 64位平台；aarch_64表示是arm 64位平台 -- 请用户根据自己的环境需要在config.json中配置仓库与执行平台信息。并将json主键（比如full_x86_64）作为可选的第四参数传递给sh脚本 - -## 运行测试 - -1) 调用入口：bash run_js_perf_test.sh js_report_save_path openharmony_path d_8_binary_path ver_platform - * js_report_save_path：必选]，表示报告输出的目录路径，该路径同时也是历史报告绝 对路径 - * openharmony_path: 必选]，将全仓鸿 蒙系统或者独立仓根目录（out的父目录）的绝 对路径 - * d_8_binary_path: [必选]，v 8 引擎 d 8 二进制的绝 对路径 - * ver_platform: [可选]，代码仓库与平台配置模式。不配置默认值“full_x86_64”，表示全仓 x86_64平台执行 - -3) 运行生成看护报告说明 - ```bash - 1. jsperftest生成的daily报告，excel格式 - 命名格式：js_perf_test_result_年月日.xlsx，比如js_perf_test_result_20231206.xlsx。 - 2. 报告表格表头输出"用例名称"、"场景"、"执行状态"、"ark用例执行耗时(ms)"、"昨日ark用例执行耗时(ms)"、"是否劣化"、"v 8(ms)"、"v 8 --jitless(ms)"、"ark/v 8"、"ark/v 8 jitless"、"hyperlink"、"备注"； - 表格的后面，会追加汇总信息：用例数总数、执行成功数、失败数，劣化数目等内容。具体内容，请参见附录“daily报告”。 - 3. 有执行失败时，当前用例的“执行状态”单元格内容会标记。 - 4. “是否劣化”列，取值：true，false。当用例前一天性能数据不存在、js用例执行异常、执行失败，都归于没有劣化。 - 5. 在daily报告生成目录，daily报告每日生成时会同时生成通知邮件汇总内容的文件--jsperftest_notifying_info_in_email.json。内容如下： - { - "kind": "V 8 js-perf-test", - "Total": 7, - "Ark劣化v 8": 1, - "Ark劣化v 8 jitless": 4 - } - ``` - -## daily报告 - -请参见附录"Daily报告"。 - -## workload代码仓 - - [Ark-workload](https://gitee.com/dov1s/arkjs-perf-test/tree/builtins_test1110/) - -## 附录 - -### Daily报告 - -| 用例名称 | 场景 | 执行状态 | ark用例执行耗时(ms) | 是否劣化 | v 8(ms) | v 8 --jitless(ms) | ark/v 8 | ark/v 8 jitless | hyperlink | 备注 | -| ------------------------------------------------------- | ------------------ | -------------- | ------------------- | -------- | ------ | ---------------- | ------------------- | ------------------ | ------------------------------------------------------------ | ---- | -| decodeuricomponent/decodeuricomponent.js | decodeuricomponent | pass | 42 | NA | 18 | 33 | 2.33 | 1.27 | https://gitee.com/dov1s/arkjs-perf-test/tree/builtins_test1110/js-perf-test/decodeuricomponent/decodeuricomponent.js | | -| finalizationregistry/finalizationregistryconstructor.js | testconstructor | pass | 6 | NA | 10 | 26 | 0.6 | 0.23 | https://gitee.com/dov1s/arkjs-perf-test/tree/builtins_test1110/js-perf-test/finalizationregistry/finalizationregistryconstructor.js | | -| finalizationregistry/register.js | testregister | pass | 16 | NA | 14 | 44 | 1.14 | 0.36 | https://gitee.com/dov1s/arkjs-perf-test/tree/builtins_test1110/js-perf-test/finalizationregistry/register.js | | -| finalizationregistry/unregister.js | testunregister | pass | 20 | NA | 51 | 99 | 0.39 | 0.20 | https://gitee.com/dov1s/arkjs-perf-test/tree/builtins_test1110/js-perf-test/finalizationregistry/unregister.js | | -| decodeuri/decodeuri.js | decodeuri | pass | 46 | NA | 19 | 36 | 2.42 | 1.28 | https://gitee.com/dov1s/arkjs-perf-test/tree/builtins_test1110/js-perf-test/decodeuri/decodeuri.js | | -| | | | | | | | | | | | -| | | | | | | | | | | | -| 劣化判定比率上限 | | 0.05 | | | | | | | | | -| Totle js case count | | 5 | | | | | | | | | -| Pass count | | 5 | | | | | | | | | -| Fail count | | 0 | | | | | | | | | -| ark今日劣化数量 | | 2 | | | | | | | | | -| Total excute time is(s) | | 0:0:19.699970 | | | | | | | | | -| ark/v 8 劣化数量 | | 2 | | | | | | | | | -| ark/v 8 jitless 劣化数量 | | 0 | | | | | | | | | - -说明： - -- E列“昨日用例执行耗时（ns)”默认是隐藏的。 -- G列 “v 8(ms)”是有基准数据每月1、11、21日会生成v 8执行时间的基准数据，其它时间都不进行v 8执行用例操作，直接获取前边近一次的基准数据 -- H列 “v 8 --jitless(ms)”是有基准数据每月1、11、21日会生成v 8 --jitless执行时间的基准数据，其它时间都不进行v 8 --jitless执行用例操作，直接获取前边近一次的v 8 --jitless基准数据 diff --git a/test/aotjsperftest/README_en.md b/test/aotjsperftest/README_en.md deleted file mode 100644 index eec7c1c7a1..0000000000 --- a/test/aotjsperftest/README_en.md +++ /dev/null @@ -1,86 +0,0 @@ -# AOT JS Perf Test Temporary Execution README - -## Environmental Dependencies -- --------------------------- - ### IMPORTANT!!! -->All c_lang/ll_vm and v_8/d_8 are to avoid codecheck inspections, actually there are no underscores in them. -- -------------------------- -- For the script's directory, the current user must have the permission to create folders. Please pay attention to the permissions settings for the current directory's user and group. - -- openpyxl. No specific version requirement. If not installed, please execute pip3 install openpyxl. - -- The environment requires downloading the full OpenHarmony code repository or standalone code repository, and compiling ark_js_vm and the frontend. - ```shell - #full OpenHarmony code repository - ./build.sh --product-name rk3568 --build-target ark_js_host_linux_tools_packages --build-target ets_frontend_build - - #standalone code repository - python3 ark.py x64.release - ``` - -- Configuring the V_8 Engine Execution Environment - -**Download：** - -```shell -# you can download to any directory -wget https://storage.googleapis.com/chromium-%76%38/official/canary/%76%38-linux64-rel-12.0.267.zip --no-check-certificate -O v_8-linux64-rel-12.0.267.zip -unzip v_8-linux64-rel-12.0.267.zip -d v_8 -# don't forget to get permission to run d_8 -``` -**Check：** -```shell -#excute - /path/to/your/d_8 -#show - V_8 version 12.0.267 - d_8> -``` - -**config.json** -```json -# Under normal circumstances, you do not need to change the configuration inside. -{ - "full_x86_64":{ - "ETS_RUNTIME_PATH": "out/rk3568/c_lang_x64/arkcompiler/ets_runtime/", - "ICU_PATH": "out/rk3568/c_lang_x64/thirdparty/icu/", - "ZLIB_PATH": "out/rk3568/c_lang_x64/thirdparty/zlib/", - "LIB_PATH": "prebuilts/c_lang/ohos/linux-x86_64/ll_vm/lib/", - "ES2ABC": "out/rk3568/c_lang_x64/arkcompiler/ets_frontend/es2abc" - }, - "independent_x86_64": { - "ETS_RUNTIME_PATH": "out/x64.release/arkcompiler/ets_runtime", - "ICU_PATH": "out/x64.release/thirdparty/icu/", - "ZLIB_PATH": "", # you can let this value be empty,but dont delete this key - "LIB_PATH": "", # you can let this value be empty,but dont delete this key - "ES2ABC": "out/x64.release/arkcompiler/ets_frontend/es2abc" - } -} -``` - -## excute -1) How to run perftest -```shell -# openharmony_path is the code repository root directory (which has out/ after build) -bash run_js_perf_test.sh js_report_save_path openharmony_path d_8_binary_path ver_platform -#example full OpenHarmony code repository -bash run_js_perf_test.sh /home/report /home/codeOpenharmony /home/v_8/d_8 full_x86_64 -#example standalone code repository -bash run_js_perf_test.sh /home/report /home/standalone /home/v_8/d_8 independent_x86_64 -``` -2) When aot perf test start to run,the perf test cases will be downloaded automatically . -But after run, perftest case will be deleted too. - -u can check them in: -[perf-test-case](https://gitee.com/dov1s/arkjs-perf-test/tree/builtins_test1110/) - -if u dont want the script deletes test cases,please command out this line in run_js_test.py -```python -# last line -shutil.rmtree(Constants.TMP_PATH) -``` - -## stat -After running, a excel will be created in js_report_save_path(the directory u choose when bash run_js_perf_test.sh) - - - diff --git a/test/aotjsperftest/config.json b/test/aotjsperftest/config.json deleted file mode 100644 index 8354609c41..0000000000 --- a/test/aotjsperftest/config.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "full_x86_64":{ - "ETS_RUNTIME_PATH": "out/rk3568/clang_x64/arkcompiler/ets_runtime/", - "ICU_PATH": "out/rk3568/clang_x64/thirdparty/icu/", - "ZLIB_PATH": "out/rk3568/clang_x64/thirdparty/zlib/", - "LIB_PATH": "prebuilts/clang/ohos/linux-x86_64/llvm/lib/", - "ES2ABC": "out/rk3568/clang_x64/arkcompiler/ets_frontend/es2abc" - }, - "independent_x86_64": { - "ETS_RUNTIME_PATH": "out/x64.release/arkcompiler/ets_runtime", - "ICU_PATH": "out/x64.release/thirdparty/icu/", - "ZLIB_PATH": "", - "LIB_PATH": "", - "ES2ABC": "out/x64.release/arkcompiler/ets_frontend/es2abc" - } -} \ No newline at end of file diff --git a/test/aotjsperftest/run_js_perf_test.sh b/test/aotjsperftest/run_js_perf_test.sh deleted file mode 100755 index ff82ac53fa..0000000000 --- a/test/aotjsperftest/run_js_perf_test.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash -# Copyright (c) 2023 Huawei Device Co., Ltd. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -declare -i ret_error=1 - -function init() -{ - CUR_PATH=$(dirname "$(readlink -f "$0")") - TMP_PATH=$CUR_PATH/tmp -} - -function check_command_exist() -{ - command=$1 - type "$command" >/dev/null 2>&1 - echo $? -} - -function check_pip_component() -{ - pip3 list|grep "$1" - return $? -} - -function download_js_test_files() -{ - code_path="$TMP_PATH"/code/arkjs-perf-test - if [ -d "$code_path" ];then - JS_TEST_PATH=$code_path/js-perf-test - return - fi - - mkdir -p "$code_path" - echo "$code_path" - git clone -b builtins_test1110 https://gitee.com/dov1s/arkjs-perf-test.git "$code_path" - JS_TEST_PATH=$code_path/js-perf-test -} - -main() -{ - init - js_perf_test_archive_path=$1 - OPENHARMONY_OUT_PATH=$2 - D8_BINARY_PATH=$3 - VER_PLATFORM="full_x86_64" - if [ $# == 4 ]; then - VER_PLATFORM=$4 - fi - cur_path=$(dirname "$(readlink -f "$0")") - - if [ ! -d "$js_perf_test_archive_path" ];then - mkdir -p "js_perf_test_archive_path" - fi - - check_command_exist git || { echo "git is not available"; return $ret_error; } - check_command_exist unzip || { echo "unzip is not available"; return $ret_error; } - check_command_exist jq || { echo "jq is not available"; return $ret_error; } - check_command_exist python3 || { echo "python3 is not available"; return $ret_error; } - check_pip_component "openpyxl" || { pip3 install openpyxl; } - - [ -f "$cur_path/run_js_test.py" ] || { echo "no run_js_test.py, please check it";return $ret_error;} - - download_js_test_files || { return $ret_error; } - - echo "LD_LIBRARY_PATH:$LD_LIBRARY_PATH" - python3 "$cur_path"/run_js_test.py -bp "$OPENHARMONY_OUT_PATH" -p "$JS_TEST_PATH" -o "$js_perf_test_archive_path"\ - -v "$D8_BINARY_PATH" -e "$VER_PLATFORM" -} - -main "$@" -exit $? diff --git a/test/aotjsperftest/run_js_test.py b/test/aotjsperftest/run_js_test.py deleted file mode 100644 index e0682f8a73..0000000000 --- a/test/aotjsperftest/run_js_test.py +++ /dev/null @@ -1,835 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -Copyright (c) 2023 Huawei Device Co., Ltd. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -Description: Use ark to execute workload test suite -""" - -import argparse -import datetime -import json -import logging -import os -import shutil -import stat -import subprocess -from collections import namedtuple -from openpyxl import Workbook, load_workbook -from openpyxl.styles import PatternFill - - -def get_logger(logger_name, log_file_path, level=logging.INFO): - formatter = logging.Formatter(fmt='[%(asctime)s] [%(levelname)s] %(message)s', - datefmt='%Y-%m-%d %H:%M:%S') - - fh = logging.FileHandler(encoding='utf-8', mode='a', filename=log_file_path) - fh.setFormatter(formatter) - fh.setLevel(logging.DEBUG) - # console output - ch = logging.StreamHandler() - ch.setFormatter(formatter) - ch.setLevel(logging.DEBUG) - log = logging.getLogger(logger_name) - log.addHandler(fh) - log.addHandler(ch) - log.setLevel(level) - - return log - - -class Constants: - logger = None - CUR_PATH = os.path.abspath(os.path.dirname(__file__)) - TMP_PATH = os.path.join(os.getcwd(), "tmp") - REPORT_NAME_HEAD_FIX = "js_perf_test_result" - RET_OK = 0 - RET_FAILED = 1 - BINARY_PATH = "" - OUTPUT_PATH = "" - LOG_PATH = "" - TODAY_EXCEL_PATH = "" - YESTERDAY_EXCEL_PATH = "" - DETERIORATION_BOUNDARY_VALUE = 0.05 - TODAY_EXCUTE_INFO = {} - YESTERDAY_EXCUTE_TIME_DICT = {} - V_8_EXCUTE_TIME_DICT = {} - V_8_JITLESS_EXCUTE_TIME_DICT = {} - JS_FILE_SUPER_LINK_DICT = {} - HYPERLINK_HEAD = "https://gitee.com/dov1s/arkjs-perf-test/tree/builtins_test1110/js-perf-test" - PASS = 'pass' - FAIL = 'fail' - SOLID = 'solid' - NA_FIX = 'NA' - # 1e-6 s - COMPARISON_ACCURACY = 0.001 - ICU_DATA_PATH = "" - FIX_STR = "8/d" - V_8_ENGINED_PATH = '/usr/bin/v{}8'.format(FIX_STR) - VER_PLATFORM = "full_x86_64" - ES2ABC_PATH = "" - ARK_JS_VM_PATH = "" - ARK_AOT_COMPILER_PATH = "" - CaseTestDataType = namedtuple('test', ['exec_status', 'exec_time']) - - -def get_js_file_class_api_scenes(js_file_path): - scenes = [] - with open(js_file_path, 'r') as f: - for line in f: - if "scene_output" in line: - str_array = line.split(':') - mid_str = str_array[1].strip() - elements = mid_str.split(' ') - main_key = '/'.join([elements[0], elements[1] + '.js', elements[2]]).lower() - scenes.append(main_key) - return scenes - - -def degraded_str(yesterday_excute_time, exec_time): - is_degraded_str = Constants.NA_FIX - if len(str(yesterday_excute_time).strip()) != 0: - if abs(float(yesterday_excute_time)) <= Constants.COMPARISON_ACCURACY: - is_degraded_str = str(True) if abs(float(exec_time)) >= DETERIORATION_BOUNDARY_VALUE else str(False) - else: - is_degraded_tmp = float(exec_time) / float(yesterday_excute_time) >= (1 + DETERIORATION_BOUNDARY_VALUE) - is_degraded_str = str(True) if is_degraded_tmp else str(False) - - return is_degraded_str - - -def v_8_excute_time_compute(main_key): - v_8_excute_time_str = '' - if len(Constants.V_8_EXCUTE_TIME_DICT) > 0 and main_key in Constants.V_8_EXCUTE_TIME_DICT.keys(): - v_8_excute_time_str = Constants.V_8_EXCUTE_TIME_DICT[main_key].strip() - - if len(v_8_excute_time_str) == 0: - v_8_excute_time = ' ' - else: - v_8_excute_time = v_8_excute_time_str - - return v_8_excute_time - - -def v_8_gitless_excute_time_compute(main_key): - v_8_jitless_excute_time_str = '' - if len(Constants.V_8_JITLESS_EXCUTE_TIME_DICT) > 0 and main_key in Constants.V_8_JITLESS_EXCUTE_TIME_DICT.keys(): - v_8_jitless_excute_time_str = Constants.V_8_JITLESS_EXCUTE_TIME_DICT[main_key].strip() - - if len(v_8_jitless_excute_time_str) == 0: - v_8_jitless_excute_time = ' ' - else: - v_8_jitless_excute_time = v_8_jitless_excute_time_str - - return v_8_jitless_excute_time - - -def ark_divide_v_8_compute(exec_time, v_8_excute_time): - if len(exec_time) == 0 or len(v_8_excute_time.strip()) == 0: - ark_divide_v_8 = Constants.NA_FIX - elif abs(float(exec_time)) <= Constants.COMPARISON_ACCURACY: - if abs(float(v_8_excute_time)) <= Constants.COMPARISON_ACCURACY: - ark_divide_v_8 = '1' - else: - ark_divide_v_8 = '0' - else: - v_8_excute_time = v_8_excute_time.strip() - if len(v_8_excute_time) == 0 or abs(float(v_8_excute_time)) <= Constants.COMPARISON_ACCURACY: - ark_divide_v_8 = Constants.NA_FIX - else: - ark_divide_v_8 = str("{:.2f}".format(float(exec_time) / float(v_8_excute_time))) - - return ark_divide_v_8 - - -def append_row_data(report_file, case_test_data): - wb = load_workbook(report_file) - ws = wb.worksheets[0] - for main_key in case_test_data.keys(): - str_arr = main_key.split('/') - class_name = str_arr[0] - api_name = str_arr[1] - scene = str_arr[2] - js_case_name = '/'.join([class_name, api_name]) - excute_status = case_test_data[main_key].exec_status - exec_time = case_test_data[main_key].exec_time.strip() - yesterday_excute_time = '' - if (len(Constants.YESTERDAY_EXCUTE_TIME_DICT) > 0 and - Constants.YESTERDAY_EXCUTE_TIME_DICT.get(main_key) is not None): - yesterday_excute_time = str(Constants.YESTERDAY_EXCUTE_TIME_DICT[main_key]) - is_degraded_str = degraded_str(yesterday_excute_time, exec_time) - v_8_excute_time = v_8_excute_time_compute(main_key) - v_8_jitless_excute_time = v_8_gitless_excute_time_compute(main_key) - ark_divide_v_8 = ark_divide_v_8_compute(exec_time, v_8_excute_time) - if len(exec_time) == 0 or len(v_8_jitless_excute_time.strip()) == 0: - ark_divide_v_8_with_jitless = Constants.NA_FIX - elif abs(float(exec_time)) <= Constants.COMPARISON_ACCURACY: - if abs(float(v_8_jitless_excute_time)) <= Constants.COMPARISON_ACCURACY: - ark_divide_v_8_with_jitless = '1' - else: - ark_divide_v_8_with_jitless = '0' - else: - v_8_jitless_excute_time = v_8_jitless_excute_time.strip() - if (len(v_8_jitless_excute_time) == 0 or - abs(float(v_8_jitless_excute_time)) <= Constants.COMPARISON_ACCURACY): - ark_divide_v_8_with_jitless = Constants.NA_FIX - else: - ark_divide_v_8_with_jitless = str("{:.2f}".format(float(exec_time) / float(v_8_jitless_excute_time))) - jis_case_file_name_with_class = Constants.JS_FILE_SUPER_LINK_DICT['/'.join([class_name, api_name])] - js_file_super_link = '/'.join([Constants.HYPERLINK_HEAD, jis_case_file_name_with_class]) - new_row = [js_case_name, scene, excute_status, exec_time, yesterday_excute_time, - is_degraded_str, v_8_excute_time, v_8_jitless_excute_time, ark_divide_v_8, - ark_divide_v_8_with_jitless, js_file_super_link, ' '] - ws.append(new_row) - if is_degraded_str is str(True): - ws.cell(row=ws.max_row, column=6).fill = PatternFill(start_color='FF0000', end_color='FF0000', - fill_type=Constants.SOLID) - if (ark_divide_v_8 != Constants.NA_FIX and - (float(ark_divide_v_8) > 2 or - abs(float(ark_divide_v_8) - 2) <= Constants.COMPARISON_ACCURACY)): - ws.cell(row=ws.max_row, column=9).fill = PatternFill(start_color='FFFF00', end_color='FFFF00', - fill_type=Constants.SOLID) - if (ark_divide_v_8_with_jitless != Constants.NA_FIX and - (float(ark_divide_v_8_with_jitless) > 2 or - abs(float(ark_divide_v_8_with_jitless) - 2) <= Constants.COMPARISON_ACCURACY)): - ws.cell(row=ws.max_row, column=10).fill = PatternFill(start_color='FF00FF', end_color='FF00FF', - fill_type=Constants.SOLID) - wb.save(report_file) - return Constants.RET_OK - - -def run_js_case_via_ark(binary_path, js_file_path, class_name, api_name, report_file): - composite_scenes = get_js_file_class_api_scenes(js_file_path) - case_test_data = {} - execute_status = Constants.FAIL - execute_time = ' ' - - for _, composite_scene in enumerate(composite_scenes): - case_test_data[composite_scene] = Constants.CaseTestDataType(execute_status, execute_time) - - js_file_name = class_name + '/' + api_name + '.js' - fangzhou_test_path = os.path.join(Constants.TMP_PATH, "fangzhou_test") # for abc file - if os.path.exists(fangzhou_test_path): - shutil.rmtree(fangzhou_test_path) - os.makedirs(fangzhou_test_path) - - class_folder_path = os.path.join(fangzhou_test_path, class_name) - api_path = os.path.join(class_folder_path, api_name) - if not os.path.exists(class_folder_path): - os.makedirs(class_folder_path) - abc_file_path = api_path + ".abc" - cur_abc_file = os.path.join(Constants.CUR_PATH, api_name + ".abc") - cur_ap_file = os.path.join(Constants.CUR_PATH, api_name + ".ap") - cur_an_file = os.path.join(Constants.CUR_PATH, api_name + ".an") - cur_ai_file = os.path.join(Constants.CUR_PATH, api_name + ".ai") - api_log_path = os.path.join(class_folder_path, api_name + ".log") - - es2abc_path = Constants.ES2ABC_PATH - # tranmit abc - cmd = [es2abc_path, "--module", "--merge-abc", "--extension=js", js_file_path] - - logger.info("run cmd: %s", cmd) - ret = subprocess.run(cmd) - if ret.returncode != 0: - logger.error("ret = %s, %s generate abc file failed. cmd: %s", str(ret), js_file_name, cmd) - append_row_data(report_file, case_test_data) - return case_test_data - - cmd2 = ["cp", cur_abc_file, abc_file_path] - ret = subprocess.run(cmd2) - if ret.returncode != 0: - logger.error("ret.returncode = %s, %s generate abc file failed. cmd: %s", str(ret.returncode), js_file_name, - cmd2) - append_row_data(report_file, case_test_data) - return case_test_data - # execute pgo - ark_js_vm_path = Constants.ARK_JS_VM_PATH - cmd = [ark_js_vm_path, - "--enable-pgo-profiler=true", - f"--compiler-pgo-profiler-path={api_name}.ap", - f"--entry-point={api_name}", - "--icu-data-path", ICU_DATA_PATH, - cur_abc_file] - logger.info("run cmd: %s", cmd) - ret = subprocess.run(cmd) - if ret.returncode != 0: - logger.error("ret = %s, %s pgo file failed. cmd: %s", str(ret), cur_abc_file, cmd) - append_row_data(report_file, case_test_data) - return case_test_data - - # excute first time aot - ark_aot_compiler_path = Constants.ARK_AOT_COMPILER_PATH - cmd = [ark_aot_compiler_path, - f"--aot-file={api_name}", - f"--compiler-pgo-profiler-path={api_name}.ap", - "--icu-data-path", ICU_DATA_PATH, - cur_abc_file] - logger.info("run cmd: %s", cmd) - ret = subprocess.run(cmd) - if ret.returncode != 0: - logger.error("ret = %s, %s aot file failed. cmd: %s", str(ret), cur_abc_file, cmd) - append_row_data(report_file, case_test_data) - return case_test_data - - # excute second time pgo - ark_js_vm_path = Constants.ARK_JS_VM_PATH - cmd = [ark_js_vm_path, - "--enable-pgo-profiler=true", - f"--compiler-pgo-profiler-path={api_name}.ap", - f"--entry-point={api_name}", - f"--aot-file={api_name}", - "--icu-data-path", ICU_DATA_PATH, - cur_abc_file] - logger.info("run cmd: %s", cmd) - ret = subprocess.run(cmd) - if ret.returncode != 0: - logger.error("ret = %s, %s sencond time pgo file failed. cmd: %s", str(ret), cur_abc_file, cmd) - append_row_data(report_file, case_test_data) - return case_test_data - - # excute second time aot - ark_aot_compiler_path = Constants.ARK_AOT_COMPILER_PATH - cmd = [ark_aot_compiler_path, - f"--aot-file={api_name}", - f"--compiler-pgo-profiler-path={api_name}.ap", - "--icu-data-path", ICU_DATA_PATH, - cur_abc_file] - logger.info("run cmd: %s", cmd) - ret = subprocess.run(cmd) - if ret.returncode != 0: - logger.error("ret = %s, %s second time aot file failed. cmd: %s", str(ret), cur_abc_file, cmd) - append_row_data(report_file, case_test_data) - return case_test_data - - #excute final abc - ark_js_vm_path = Constants.ARK_JS_VM_PATH - cmd = [ark_js_vm_path, - "--log-level=info", - "--compiler-trace-deopt=true", - f"--entry-point={api_name}", - f"--aot-file={api_name}", - "--icu-data-path", ICU_DATA_PATH, - cur_abc_file] - logger.info("run cmd: %s", cmd) - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - modes = stat.S_IWUSR | stat.S_IRUSR - if os.path.exists(api_log_path): - os.remove(api_log_path) - with os.fdopen(os.open(api_log_path, flags, modes), 'wb') as outfile: - ret = subprocess.run(cmd, stdout=outfile) - - if ret.returncode != 0: - logger.error("%s execute abc file failed. cmd: %s", js_file_name, cmd) - append_row_data(report_file, case_test_data) - return case_test_data - else: - case_test_data.clear() - if os.path.exists(api_log_path): - with open(api_log_path, 'r') as f: - for line in f: - if "scene_output" not in line: - continue - - mid_str = line.split(':')[1].strip() - scene = mid_str.split()[2] - main_key = '/'.join([js_file_name, scene]).lower() - execute_time = line.split(':')[2] - execute_status = Constants.PASS - case_test_data[main_key] = Constants.CaseTestDataType(execute_status, execute_time) - - append_row_data(report_file, case_test_data) - logger.info("%s execute abc file successfully. cmd: %s case_test_data: %s", js_file_name, cmd, case_test_data) - - os.remove(cur_abc_file) - os.remove(cur_ap_file) - os.remove(cur_an_file) - os.remove(cur_ai_file) - - return case_test_data - - -def run_via_ark(jspath, report_file): - if not os.path.exists(jspath): - logger.error("js perf cases path is not exist. jspath: %s", jspath) - logger.info("begin to run js perf test via ark. js perf cases path: %s", jspath) - for root, _, files in os.walk(jspath): - if "TestCaseError" in root: - continue - for file in files: - if not file.endswith('.js'): - continue - - file_path = os.path.join(root, file) - results = file_path.split("/") - class_name = results[-2] - api_name = results[-1].split(".")[0] - js_case_name = '/'.join([class_name, results[-1]]) - logger.info("begin to execute %s.", js_case_name) - test_data = run_js_case_via_ark(BINARY_PATH, file_path, class_name, api_name, report_file) - for _, key in enumerate(test_data.keys()): - Constants.TODAY_EXCUTE_INFO[key] = test_data.get(key) - logger.info("finish executing %s. executing info: %s.", js_case_name, Constants.TODAY_EXCUTE_INFO) - - -def get_js_case_super_link_data(jspath): - logger.info("get js case super link data") - for root, _, files in os.walk(jspath): - for file in files: - if not file.endswith('.js'): - continue - - file_path = os.path.join(root, file) - results = file_path.split("/") - class_name = results[-2] - js_case_name = '/'.join([class_name, results[-1]]) - key = js_case_name.lower() - Constants.JS_FILE_SUPER_LINK_DICT[key] = js_case_name - - -def export_sumary_info_for_notifying_email(json_path, total_cases_num, ark_divide_v_8_num, ark_divide_v_8_jitless_num): - data = {} - data['kind'] = 'V 8 js-perf-test' - data['Total'] = total_cases_num - data['Ark劣化v 8'] = ark_divide_v_8_num - data['Ark劣化v 8 jitless'] = ark_divide_v_8_jitless_num - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - modes = stat.S_IWUSR | stat.S_IRUSR - if os.path.exists(json_path): - os.remove(json_path) - with os.fdopen(os.open(json_path, flags, modes), 'w', encoding='utf-8') as f: - json.dump(data, f, indent=4, ensure_ascii=False) - logger.info("export summary info to json file successfully.") - - -def get_umary_info_json_file_path(daily_report_file_path): - dir_path = os.path.dirname(daily_report_file_path) - json_file_name = 'jsperftest_notifying_info_in_email.json' - json_file_path = os.path.join(dir_path, json_file_name) - return json_file_path - - -def append_summary_info(report_file, total_cost_time): - """ - summary info: - pass count: - fail count: - totle count: - degraded count: - total excute time is(s) : - degraded percentage upper limit: - ark/v 8 degraded count: - ark/v 8 jitless degraded count: - """ - wb = load_workbook(report_file) - ws = wb.worksheets[0] - - totle_num = 0 - degraded_upper_limit = DETERIORATION_BOUNDARY_VALUE - pass_num = 0 - failed_num = 0 - degraded_num = 0 - ark_divide_v_8_degraded_count = 0 - ark_divide_v_8_jitless_degraded_count = 0 - - for row_num in range(2, ws.max_row + 1): - excu_status = str(ws.cell(row=row_num, column=3).value) - is_degraded = str(ws.cell(row=row_num, column=6).value) - if is_degraded == str(True): - degraded_num += 1 - - if excu_status == Constants.PASS: - pass_num += 1 - totle_num += 1 - elif excu_status == Constants.FAIL: - failed_num += 1 - totle_num += 1 - - obj = ws.cell(row=row_num, column=9).value - if obj is None: - obj = 0 - ark_divide_v_8 = obj - if ark_divide_v_8 != Constants.NA_FIX and float(ark_divide_v_8) > 1: - ark_divide_v_8_degraded_count += 1 - obj = ws.cell(row=row_num, column=10).value - if obj is None: - obj = 0 - ark_divide_v_8_jitless = obj - if ark_divide_v_8_jitless != Constants.NA_FIX and float(ark_divide_v_8_jitless) > 1: - ark_divide_v_8_jitless_degraded_count += 1 - - count = 3 - for _ in range(count): - new_row = [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] - ws.append(new_row) - new_row = ['degraded_upper_limit', degraded_upper_limit, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] - ws.append(new_row) - new_row = ['js test cases count', totle_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] - ws.append(new_row) - new_row = ['Pass num', pass_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] - ws.append(new_row) - new_row = ['Fail num', failed_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] - ws.append(new_row) - new_row = ['ark today degraded_num', degraded_num, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] - ws.append(new_row) - new_row = ['Total excute time(h:m:s.ms)', total_cost_time, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] - ws.append(new_row) - new_row = ['ark/v_8 bad nums', ark_divide_v_8_degraded_count, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] - ws.append(new_row) - new_row = ['ark/v_8 jitless badnums', ark_divide_v_8_jitless_degraded_count, ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', - ' ', ' '] - ws.append(new_row) - - ws.column_dimensions.group('E', hidden=True) - wb.save(report_file) - - json_file_path = get_umary_info_json_file_path(report_file) - export_sumary_info_for_notifying_email(json_file_path, totle_num, ark_divide_v_8_degraded_count, - ark_divide_v_8_jitless_degraded_count) - return Constants.RET_OK - - -def get_args(): - parser = argparse.ArgumentParser() - parser.add_argument( - "--binarypath", - "-bp", - required=True, - help="path of binary folder. refer to harmony root folder path", - ) - parser.add_argument( - "--jspath", - "-p", - required=True, - help="path of js scripts, support folder and file", - ) - parser.add_argument( - "--deterioration_boundary_value", - "-d", - default=0.05, - help="deterioration boundary value, default 0.05", - ) - parser.add_argument( - "--output_folder_path", - "-o", - default=None, - help="output folder for executing js cases, default current folder", - ) - parser.add_argument( - "--d_8_binary_path", - "-v", - default=None, - help="v 8 engine d 8 binary path", - ) - parser.add_argument( - "--ver_platform", - "-e", - default="full_x86_64", - help="Code repository version and platform", - ) - args = parser.parse_args() - - if not os.path.exists(args.binarypath): - logger.error("parameter --binarypath is not exist. Please check it! binary path: %s", args.binarypath) - raise RuntimeError("error bad parameters --binarypath") - - if args.output_folder_path is None: - args.output_folder_path = os.getcwd() - - if not os.path.isabs(args.output_folder_path): - args.output_folder_path = os.path.abspath(args.output_folder_path) - - if not os.path.exists(args.d_8_binary_path): - logger.error("parameter --d_8_binary_path is not exist. Please check it! d 8 binary path: %s", - args.d_8_binary_path) - raise RuntimeError("error bad parameters --d_8_binary_path: {}".format(args.d_8_binary_path)) - - return args - - -def init_report(report_file): - try: - today_wb = load_workbook(report_file) - today_ws = today_wb.worksheets[0] - except FileNotFoundError: - headers_row = ['caseName', 'scene', 'status', 'ark_aot excute(ms)', 'last day excute(ms)', 'detorioration?', - 'v 8(ms)', 'v 8 --jitless(ms)', 'ark_aot/v 8', 'ark_aot/v 8 jitless', 'hyperlink', 'note'] - today_wb = Workbook() - today_ws = today_wb.active - - today_ws.column_dimensions['A'].width = 35.0 - today_ws.column_dimensions['B'].width = 15.0 - today_ws.column_dimensions['C'].width = 15.0 - today_ws.column_dimensions['D'].width = 15.0 - today_ws.column_dimensions['E'].width = 25.0 - today_ws.column_dimensions['F'].width = 15.0 - today_ws.column_dimensions['G'].width = 15.0 - today_ws.column_dimensions['H'].width = 15.0 - today_ws.column_dimensions['I'].width = 15.0 - today_ws.column_dimensions['J'].width = 15.0 - today_ws.column_dimensions['K'].width = 50.0 - today_ws.column_dimensions['L'].width = 15.0 - today_ws.append(headers_row) - today_ws.freeze_panes = 'A2' - today_wb.save(report_file) - - -def append_date_label(target_str, date_input): - formatted_date = date_input.strftime('%Y%m%d') - new_str = target_str + "_{}".format(formatted_date) - - return new_str - - -def get_v_8_benchmark_daily_report_path(): - ''' - get v 8 based data. v 8 based data obtained on 1,11,21 day for dayevery month.that is to say, in 1,11,21, - v 8 executes js cases. - ''' - now = datetime.datetime.now(tz=datetime.timezone.utc) - today_str = now.strftime("%Y.%m.%d") - str_list = today_str.split('.') - year_str = str_list[0] - month_str = str_list[1] - day = int(str_list[2]) - based_day = 0 - if day > 21: - based_day = 21 - elif day > 11: - based_day = 11 - else: - based_day = 1 - - based_date = year_str + month_str + str(based_day) - base_date_file = based_date + '.xlsx' - based_report_name = '_'.join([Constants.REPORT_NAME_HEAD_FIX, base_date_file]) - report_file_path = os.path.join(OUTPUT_PATH, based_report_name) - return report_file_path - - -def get_given_date_report_name(date_input): - report_name_head = append_date_label(Constants.REPORT_NAME_HEAD_FIX, date_input) - return report_name_head + ".xlsx" - - -def get_given_date_report_path(date_input): - report_file_name = get_given_date_report_name(date_input) - report_file_path = os.path.join(OUTPUT_PATH, report_file_name) - return report_file_path - - -def get_yesterday_excute_times(yesterday_report): - if not os.path.exists(yesterday_report) or not os.path.isfile(yesterday_report): - return - - wb = load_workbook(yesterday_report) - ws = wb.worksheets[0] - for row_num in range(2, ws.max_row + 1): - js_case = ws.cell(row=row_num, column=1).value - scene = ws.cell(row=row_num, column=2).value - exec_status = ws.cell(row=row_num, column=3).value - if exec_status == Constants.PASS or exec_status == Constants.FAIL: - main_key = '/'.join([js_case, scene]).lower() - excute_time = ws.cell(row=row_num, column=4).value - Constants.YESTERDAY_EXCUTE_TIME_DICT[main_key] = excute_time - - -def run_v_8_single_js_case(js_file_path, cmd_para, js_case_name): - v_8_exec_time_dict = {} - scenes = get_js_file_class_api_scenes(js_file_path) - - v_8_log_path = os.path.join(Constants.CUR_PATH, "v_8.log") - if os.path.exists(v_8_log_path): - os.remove(v_8_log_path) - - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(v_8_log_path, flags, modes), 'wb') as outfile: - if len(cmd_para) == 0: - cmd = [Constants.V_8_ENGINED_PATH, js_file_path] - else: - cmd = [Constants.V_8_ENGINED_PATH, cmd_para, js_file_path] - logger.info("run cmd:%s", cmd) - ret = subprocess.run(cmd, stdout=outfile) - - if ret.returncode != 0: - for elem in enumerate(scenes): - v_8_exec_time_dict[elem] = 0 - logger.error("execute cmd failed. cmd: %s", cmd) - return v_8_exec_time_dict - - logger.info("v 8 excute %s successfully. cmd: %s", js_file_path, cmd) - - with open(v_8_log_path, 'r') as f: - for line in f: - if "scene_output" not in line: - continue - str_array = line.split(':') - mid_str = str_array[1].strip() - scene = mid_str.split()[2] - exec_time = str_array[2] - key_str = '/'.join([js_case_name + '.js', scene]).lower() - v_8_exec_time_dict[key_str] = exec_time - - os.remove(v_8_log_path) - return v_8_exec_time_dict - - -def get_given_column_data(report_file, column_index): - column_data = {} - if os.path.exists(report_file) and report_file.endswith('.xlsx'): - wb = load_workbook(report_file) - ws = wb.worksheets[0] - - for row_num in range(2, ws.max_row + 1): - js_case_name = str(ws.cell(row=row_num, column=1).value) - scene = str(ws.cell(row=row_num, column=2).value) - exec_status = str(ws.cell(row=row_num, column=3).value) - time = str(ws.cell(row=row_num, column=column_index).value) - if exec_status == Constants.PASS or exec_status == Constants.FAIL: - main_key = '/'.join([js_case_name, scene]) - column_data[main_key] = time - - return column_data - - -def get_v_8_excute_times(jspath, v_8_based_report_file): - if os.path.exists(v_8_based_report_file) and os.path.isfile(v_8_based_report_file): - # Generate v 8 benchmark data on the 1st, 11th, and 21st of each month.The testing at other times refers to - # these V 8 benchmark data - v_8_exec_time_dict = get_given_column_data(v_8_based_report_file, 7) - for key in v_8_exec_time_dict.keys(): - Constants.V_8_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] - return Constants.RET_OK - - file_list = [] - for root, _, files in os.walk(jspath): - for file in files: - if not file.endswith('.js'): - continue - file_path = os.path.join(root, file) - file_list.append(file_path) - for _, file_path in enumerate(file_list): - results = file_path.split("/") - class_name = results[-2] - api_name = results[-1].split(".")[0] - js_case_name = '/'.join([class_name, api_name]) - - v_8_exec_time_dict = run_v_8_single_js_case(file_path, '', js_case_name) - for key in v_8_exec_time_dict.keys(): - Constants.V_8_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] - - return Constants.RET_OK - - -def get_v_8_jitless_excute_times(jspath, v_8_based_report_file_path): - if os.path.exists(v_8_based_report_file_path) and os.path.isfile(v_8_based_report_file_path): - # Generate v 8 benchmark data on the 1st, 11th, and 21st of each month.The testing at other times refers to - # these V 8 benchmark data - v_8_exec_time_dict = get_given_column_data(v_8_based_report_file_path, 8) - for key in v_8_exec_time_dict.keys(): - Constants.V_8_JITLESS_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] - return Constants.RET_OK - - file_list = [] - for root, _, files in os.walk(jspath): - for file in files: - if not file.endswith('.js'): - continue - file_path = os.path.join(root, file) - file_list.append(file_path) - - for _, file_path in enumerate(file_list): - results = file_path.split("/") - class_name = results[-2] - api_name = results[-1].split(".")[0] - js_case_name = '/'.join([class_name, api_name]) - - v_8_exec_time_dict = run_v_8_single_js_case(file_path, '--jitless', js_case_name) - for key in v_8_exec_time_dict.keys(): - Constants.V_8_JITLESS_EXCUTE_TIME_DICT[key] = v_8_exec_time_dict[key] - - return Constants.RET_OK - - -def get_config(): - config_json_path = os.path.join(Constants.CUR_PATH, "config.json") - with open(config_json_path, 'r', encoding='UTF-8') as f: - json_data = json.load(f) - - Constants.ES2ABC_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ES2ABC"]) - Constants.ARK_JS_VM_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ETS_RUNTIME_PATH"], - "ark_js_vm") - Constants.ARK_AOT_COMPILER_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ETS_RUNTIME_PATH"], - "ark_aot_compiler") - ETS_RUNTIME_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ETS_RUNTIME_PATH"]) - ICU_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ICU_PATH"]) - ZLIB_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ZLIB_PATH"]) - LIB_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["LIB_PATH"]) - old_ld_library_path = os.environ.get('LD_LIBRARY_PATH', '') - os.environ['LD_LIBRARY_PATH'] = f'{ETS_RUNTIME_PATH}:' + f'{ICU_PATH}:' + f'{ZLIB_PATH}:' + f'{LIB_PATH}:'\ - + old_ld_library_path - - -if __name__ == "__main__": - """ - command format: python3 run_js_test.py -bp /home/out -p /home/arkjs-perf-test/js-perf-test -o output_path - -v d_8_binary_path -e ver_platform - notes: all paths must be absolute path - """ - LOG_PATH = os.path.join(Constants.TMP_PATH, "test.log") - if os.path.exists(LOG_PATH): - os.remove(LOG_PATH) - logger = get_logger("jstest", LOG_PATH) - - paras = get_args() - logger.info("execute arguments: %s", paras) - - DETERIORATION_BOUNDARY_VALUE = paras.deterioration_boundary_value - BINARY_PATH = paras.binarypath - ICU_DATA_PATH = os.path.join(BINARY_PATH, "third_party/icu/ohos_icu4j/data/") - OUTPUT_PATH = Constants.CUR_PATH - Constants.V_8_ENGINED_PATH = paras.d_8_binary_path - Constants.VER_PLATFORM = paras.ver_platform - get_config() - - if paras.output_folder_path is not None: - OUTPUT_PATH = paras.output_folder_path - - if not os.path.exists(OUTPUT_PATH): - os.makedirs(OUTPUT_PATH) - - today = datetime.date.today() - yesterday = today - datetime.timedelta(days=1) - TODAY_EXCEL_PATH = get_given_date_report_path(today) - YESTERDAY_EXCEL_PATH = get_given_date_report_path(yesterday) - - if os.path.exists(TODAY_EXCEL_PATH): - os.remove(TODAY_EXCEL_PATH) - - get_js_case_super_link_data(paras.jspath) - start_time = datetime.datetime.now(tz=datetime.timezone.utc) - init_report(TODAY_EXCEL_PATH) - get_yesterday_excute_times(YESTERDAY_EXCEL_PATH) - v_8_based_report_path = get_v_8_benchmark_daily_report_path() - get_v_8_excute_times(paras.jspath, v_8_based_report_path) - get_v_8_jitless_excute_times(paras.jspath, v_8_based_report_path) - - run_via_ark(paras.jspath, TODAY_EXCEL_PATH) - end_time = datetime.datetime.now(tz=datetime.timezone.utc) - - totol_time = u"%s" % (end_time - start_time) - append_summary_info(TODAY_EXCEL_PATH, totol_time) - - logger.info("run js perf test finished. Please check details in report.") - shutil.rmtree(Constants.TMP_PATH) diff --git a/test/jsperftest/run_js_perf_test.sh b/test/jsperftest/run_js_perf_test.sh index b085fac52e..b881b8bf7c 100755 --- a/test/jsperftest/run_js_perf_test.sh +++ b/test/jsperftest/run_js_perf_test.sh @@ -31,6 +31,7 @@ declare BENCH_MULTIPLIER="" declare HDC_PATH="" declare TASKSET_MASK="" declare -i ITERATIONS=1 +declare AOT_ENABLED=false function echo_fatal() { @@ -90,6 +91,9 @@ function init() shift TASKSET_MASK="$1" ;; + --aot|-a) + AOT_ENABLED=true + ;; *) JS_PERF_RESULT_PATH="$1" OPENHARMONY_ROOT_PATH="$2" @@ -342,6 +346,7 @@ Options: --exclude=BenchDir1:BenchDir2:BenchDir3/bench.js:... - exclude benchmarks from running: directory or file --no-download-benchs - no download benchmarks from repository if repo already exists + --aot, -a - use aot mode for perftest --help, -h - print help info about script Positional arguments: @@ -391,7 +396,9 @@ main() -e "${VER_PLATFORM}" \ --hdc "${HDC_PATH}" \ -t "${TASKSET_MASK}" \ - -n "${ITERATIONS}" + -n "${ITERATIONS}" \ + -a "${AOT_ENABLED}" + benchs_time=$(($(date +%s) - benchs_time_start)) benchs_minutes=$((benchs_time / 60)) echo "Benchmark script time: ${benchs_minutes} min $((benchs_time - benchs_minutes * 60)) sec" diff --git a/test/jsperftest/run_js_test.py b/test/jsperftest/run_js_test.py index fa8761a41a..a92a88e75e 100644 --- a/test/jsperftest/run_js_test.py +++ b/test/jsperftest/run_js_test.py @@ -91,6 +91,9 @@ class Constants: STUB_AN: str = "" TASKSET_MASK: str = "" CaseTestDataType = namedtuple('test', ['exec_status', 'exec_time']) + IS_AOT_MODE: bool = False + ARK_AOT_COMPILER_PATH = "" + AOT_PREPARE_LOG_PATH = "" def get_js_file_class_api_scenes(js_file_path): @@ -236,6 +239,8 @@ def append_row_data(report_file, case_test_data): def get_ark_js_cmd(abc_file: str) -> List[str]: """Get command for ark js vm""" cmd: List[str] = [] + # AAA/BBB/CCC.abc ==> CCC + entry_point_name = abc_file[:-4].split("/")[-1] if Constants.VER_PLATFORM.find("arm64") != -1: cmd = [Constants.HDC_PATH, "shell"] run_cmd = f"LD_LIBRARY_PATH={Constants.LD_LIBRARY_PATH}" @@ -244,6 +249,7 @@ def get_ark_js_cmd(abc_file: str) -> List[str]: run_cmd += " " + os.path.join(Constants.DEVICE_WORKDIR, "ark_js_vm") run_cmd += " --stub-file " + os.path.join(Constants.DEVICE_WORKDIR, "lib", "stub.an") run_cmd += " --icu-data-path " + os.path.join(Constants.DEVICE_WORKDIR, "data") + run_cmd += f" --entry-point={entry_point_name} " run_cmd += " " + abc_file cmd.append(run_cmd) else: @@ -252,6 +258,7 @@ def get_ark_js_cmd(abc_file: str) -> List[str]: "--enable-runtime-stat=true", "--stub-file", Constants.STUB_AN, "--icu-data-path", ICU_DATA_PATH, + "--entry-point", entry_point_name, abc_file] if len(Constants.TASKSET_MASK) != 0: cmd = ["taskset", "-a", Constants.TASKSET_MASK] + cmd @@ -273,13 +280,113 @@ def prepare_for_ark_run(class_name: str, api_name: str) -> Tuple[str, str]: def run_es2panda(abc_file: str, js_file: str) -> int: """Run es2panda for one benchmark file""" - cmd = [Constants.ES2ABC_PATH, "--output", abc_file, js_file] + cmd = [Constants.ES2ABC_PATH, js_file, "--module", "--merge-abc", "--output", abc_file] logger.info("run cmd: %s", cmd) ret = subprocess.run(cmd, check=False) if ret.returncode != 0: logger.error("ret = %s, %s generate abc file failed. cmd: %s", str(ret), js_file, cmd) return ret.returncode +def prepare_aot(abc_file: str) -> int: + """prepare for aot mode: create ap/an/ai files""" + """notice: aot need relative path of abc file""" + cmd_pgo1:str = "" + cmd_aot:str = "" + cmd_pgo2:str = "" + pgo_shared_cmd:str = "" + cmds = [] + case_name = abc_file[:-4].split("/")[-1] + if Constants.VER_PLATFORM.find("arm64") != -1: + """prepare pgo cmd""" + pgo_shared_cmd = f" cd {Constants.DEVICE_WORKDIR} && " + pgo_shared_cmd += f" LD_LIBRARY_PATH={Constants.DEVICE_WORKDIR}/lib/libark_jsruntime.so:{Constants.DEVICE_WORKDIR}/lib/stub.an " + pgo_shared_cmd += " ./ark_js_vm " + pgo_shared_cmd += " --enable-pgo-profiler=true " + pgo_shared_cmd += f" --compiler-pgo-profiler-path={case_name}.ap " + pgo_shared_cmd += f" --entry-point={case_name} " + cmd_pgo1 = pgo_shared_cmd + f" {case_name}.abc " + cmd_pgo2 = pgo_shared_cmd + f" --aot-file={case_name} " + f" {case_name}.abc " + """prepare aot cmd""" + cmd_aot = f" cd {Constants.DEVICE_WORKDIR} && " + cmd_aot += f" LD_LIBRARY_PATH={Constants.DEVICE_WORKDIR}/lib/libark_jsruntime.so:{Constants.DEVICE_WORKDIR}/lib/stub.an " + cmd_aot += " ./ark_aot_compiler " + cmd_aot += " --compiler-target-triple=aarch64-unknown-linux-gnu " + cmd_aot += f" --compiler-pgo-profiler-path={case_name}.ap " + cmd_aot += f" --aot-file={case_name} " + cmd_aot += f" {case_name}.abc " + else: + aot_file_name = abc_file[:-4] + cmd_pgo1 = [Constants.ARK_JS_VM_PATH, + "--stub-file", Constants.STUB_AN, + "--icu-data-path", ICU_DATA_PATH, + "--enable-pgo-profiler=true", + "--compiler-pgo-profiler-path", + f"{aot_file_name}.ap", + f"--entry-point", case_name, + abc_file] + cmd_pgo2 = [Constants.ARK_JS_VM_PATH, + "--stub-file", Constants.STUB_AN, + "--icu-data-path", ICU_DATA_PATH, + "--enable-pgo-profiler=true", + "--compiler-pgo-profiler-path", + f"{aot_file_name}.ap", + f"--entry-point", case_name, + "--aot-file", aot_file_name, + abc_file] + cmd_aot = [Constants.ARK_AOT_COMPILER_PATH, + "--stub-file", Constants.STUB_AN, + "--compiler-pgo-profiler-path", + f"{aot_file_name}.ap", + "--aot-file", f"{aot_file_name}", + abc_file] + cmds = [ + cmd_pgo1, + cmd_aot, + cmd_pgo2, + cmd_aot + ] + flags = os.O_WRONLY | os.O_CREAT + modes = stat.S_IWUSR | stat.S_IRUSR + for cmd in cmds: + logger.info("run aot cmd:%s", cmd) + with os.fdopen(os.open(Constants.AOT_PREPARE_LOG_PATH, flags, modes), 'a') as logfile: + if Constants.VER_PLATFORM.find("arm64") != -1: + ret = subprocess.run([f'{Constants.HDC_PATH}', 'shell', f'"{cmd}"'], stdout=subprocess.PIPE, text=True) + else: + ret = subprocess.run(cmd, stdout=subprocess.PIPE, text=True) + logfile.write(f"AOT Command RUN: {cmd}\n") + logfile.write(f"CMD RETURN: {ret.stdout}") + if ret.returncode != 0: + return 1 + return 0 + +def get_aot_final_cmd(abc_file: str) -> List[str]: + """Get command for ark js vm aot mode""" + cmd: List[str] = [] + aot_file_name = abc_file[:-4] + case_name = aot_file_name.split("/")[-1] + if Constants.VER_PLATFORM.find("arm64") != -1: + run_cmd = f" cd {Constants.DEVICE_WORKDIR} && " + run_cmd += f" LD_LIBRARY_PATH={Constants.DEVICE_WORKDIR}/lib/libark_jsruntime.so:{Constants.DEVICE_WORKDIR}/lib/stub.an " + if len(Constants.TASKSET_MASK) != 0: + run_cmd += f" taskset -a {Constants.TASKSET_MASK} " + run_cmd += " ./ark_js_vm " + run_cmd += f" --entry-point={case_name} " + run_cmd += f" --aot-file={case_name} " + run_cmd += f" {case_name}.abc " + cmd = [f'{Constants.HDC_PATH}', 'shell', f'"{run_cmd}"'] + else: + cmd = [Constants.ARK_JS_VM_PATH, + "--log-level=info", + "--enable-runtime-stat=true", + "--stub-file", Constants.STUB_AN, + "--icu-data-path", ICU_DATA_PATH, + "--aot-file", aot_file_name, + "--entry-point", case_name, + abc_file] + if len(Constants.TASKSET_MASK) != 0: + cmd = ["taskset", "-a", Constants.TASKSET_MASK] + cmd + return cmd def run_js_case_via_ark(js_file_path, class_name, api_name, iterations, report_file): """Run js perf benchmark via ark js vm""" @@ -305,7 +412,15 @@ def run_js_case_via_ark(js_file_path, class_name, api_name, iterations, report_f append_row_data(report_file, case_test_data) return case_test_data # execute abc - cmd = get_ark_js_cmd(using_abc_file) + cmd: List[str] = [] + if Constants.IS_AOT_MODE: + result_code = prepare_aot(using_abc_file) + if result_code != 0: + append_row_data(report_file, case_test_data) + return case_test_data + cmd = get_aot_final_cmd(using_abc_file) + else: + cmd = get_ark_js_cmd(using_abc_file) logger.info("run cmd: %s", cmd) flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL @@ -333,6 +448,11 @@ def run_js_case_via_ark(js_file_path, class_name, api_name, iterations, report_f logger.info("%s execute abc file successfully. cmd: %s case_test_data: %s", js_file_name, cmd, case_test_data) os.remove(cur_abc_file) + if Constants.IS_AOT_MODE and not Constants.VER_PLATFORM.find("arm64") != -1: + aot_file_pre = cur_abc_file[:-4] + os.remove(f"{aot_file_pre}.ap") + os.remove(f"{aot_file_pre}.an") + os.remove(f"{aot_file_pre}.ai") return case_test_data @@ -498,6 +618,16 @@ def process_args(args: argparse.Namespace) -> argparse.Namespace: raise RuntimeError(f"error bad parameters --iterations: {args.iterations}") return args +def str2bool(v): + if isinstance(v, bool): + return v + if v.lower() in ('yes', 'true', 't', 'y', '1'): + return True + elif v.lower() in ('no', 'false', 'f', 'n', '0'): + return False + else: + raise argparse.ArgumentTypeError('Boolean value expected.') + def get_args(): parser = argparse.ArgumentParser() parser.add_argument( @@ -537,6 +667,9 @@ def get_args(): help="Use taskset mask for affinity on specific CPUs" ) parser.add_argument("--config", "-c", required=True, type=str, help="config json-file") + parser.add_argument( + "--aot", "-a", type=str2bool, default=False, help="use aot mode" + ) return process_args(parser.parse_args()) @@ -796,6 +929,11 @@ def prepare_device(): logger.error("Couldn't send ark_js_vm to device") sys.exit(1) hdc_run(["chmod", "u+x", os.path.join(Constants.DEVICE_WORKDIR, "ark_js_vm")]) + if Constants.IS_AOT_MODE: + if hdc_send(Constants.ARK_AOT_COMPILER_PATH, Constants.DEVICE_WORKDIR) != 0: + logger.error("Couldn't send ark_aot_compiler to device") + sys.exit(1) + hdc_run(["chmod", "u+x", os.path.join(Constants.DEVICE_WORKDIR, "ark_aot_compiler")]) arkjsvm_lib = os.path.join(Constants.ETS_RUNTIME, "libark_jsruntime.so") if hdc_send(arkjsvm_lib, os.path.join(Constants.DEVICE_WORKDIR, "lib")) != 0: logger.error("Couldn't send libark_jsruntime.so to device") @@ -828,6 +966,8 @@ def get_config(parameters: argparse.Namespace): Constants.ES2ABC_PATH = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ES2ABC"]) Constants.ETS_RUNTIME = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["ETS_RUNTIME_PATH"]) + if Constants.IS_AOT_MODE: + Constants.ARK_AOT_COMPILER_PATH = os.path.join(Constants.ETS_RUNTIME, "ark_aot_compiler") Constants.ARK_JS_VM_PATH = os.path.join(Constants.ETS_RUNTIME, "ark_js_vm") Constants.STUB_AN = os.path.join(BINARY_PATH, json_data[Constants.VER_PLATFORM]["STUB_AN"]) libs = json_data[Constants.VER_PLATFORM]["LIBS_LIST"] @@ -859,6 +999,7 @@ if __name__ == "__main__": logger.info("execute arguments: %s", paras) DETERIORATION_BOUNDARY_VALUE = paras.deterioration_boundary_value + Constants.IS_AOT_MODE = paras.aot BINARY_PATH = paras.binarypath ICU_DATA_PATH = os.path.join(BINARY_PATH, "third_party/icu/ohos_icu4j/data") OUTPUT_PATH = Constants.CUR_PATH @@ -866,6 +1007,10 @@ if __name__ == "__main__": if not os.path.exists(Constants.ARK_JS_VM_PATH): logger.error("%s does not exist", Constants.ARK_JS_VM_PATH) sys.exit(1) + if Constants.IS_AOT_MODE: + if not os.path.exists(Constants.ARK_AOT_COMPILER_PATH): + logger.error("%s does not exist", Constants.ARK_AOT_COMPILER_PATH) + sys.exit(1) if Constants.VER_PLATFORM.find("arm64") != -1: prepare_device() @@ -874,6 +1019,10 @@ if __name__ == "__main__": if not os.path.exists(OUTPUT_PATH): os.makedirs(OUTPUT_PATH) + if Constants.IS_AOT_MODE: + Constants.AOT_PREPARE_LOG_PATH = os.path.join(OUTPUT_PATH,"AOT_prepare.log") + if os.path.exists(Constants.AOT_PREPARE_LOG_PATH): + os.remove(Constants.AOT_PREPARE_LOG_PATH) today = datetime.date.today() yesterday = today - datetime.timedelta(days=1) -- Gitee