diff --git a/omniadvisor/script/auto_deploy.sh b/omniadvisor/script/auto_deploy.sh new file mode 100755 index 0000000000000000000000000000000000000000..2c18ce9b371ab07e3a3ed0f3f032ab6e697ef023 --- /dev/null +++ b/omniadvisor/script/auto_deploy.sh @@ -0,0 +1,64 @@ +prefix=$(dirname `dirname $(pwd)`) +core_dir=$prefix/BoostKit-omniadvisor_core_2.0 +frame_dir=$prefix/BoostKit-omniadvisor_2.0 + +# 合并核心代码与框架代码 +ls -l $frame_dir/src +ls -l $core_dir/src +cp -r $core_dir/src/algo $frame_dir/src/ + +if [ -z "${SPARK_HOME}" ]; then + # 如果 SPARK_HOME 未设置,打印错误信息并退出 + echo "错误: SPARK_HOME 环境变量未设置!" >&2 + echo "请先配置 SPARK_HOME 环境变量,指向您的 Spark 安装目录" >&2 + echo "例如: " >&2 + echo " export SPARK_HOME=/path/to/your/spark" >&2 + echo "然后将此行添加到您的 ~/.bashrc 或者 ~/.zshrc 文件中" >&2 + exit 1 # 以非零状态退出 表示错误 +fi + +# 替换spark-submit脚本 +spark_submit_modified=$frame_dir/script/spark-submit +spark_submit_link=$SPARK_HOME/bin/spark-submit +original_spark_submit_file=spark_submit_link + + +# 修改spark_submit_modified 中的hijack.pyc路径 +hijack_path=$frame_dir/src/hijack.pyc +sed -i "31s|hijack_path=\"\"|hijack_path=\"${hijack_path}\"|" "$spark_submit_modified" +sed -n '31p' "$spark_submit_modified" +chmod +x $spark_submit_modified + +if command -v dos2unix &> /dev/null; then + dos2unix $spark_submit_modified +fi + +# 如果spark-submit不存在则直接创建软链接 +if [ -L "$spark_submit_link" ]; then + echo -e "\n$spark_submit_link 是一个已存在的软链接" + ls -l $spark_submit_link + echo "即将对软链接进行刷新" + original_spark_submit_file=`readlink -f $spark_submit_link` + ln -snf $spark_submit_modified $spark_submit_link + echo "将$spark_submit_link链接到$spark_submit_modified" +# 如果spark-submit存在则备份并重置spark-submit的软连接 +elif [ -f "$spark_submit_link" ]; then + echo -e "\n$spark_submit_link 是一个普通文件 进行备份" + cp -r $spark_submit_link $SPARK_HOME/bin/spark-submit.oribackup + original_spark_submit_file="$SPARK_HOME/bin/spark-submit.oribackup" + echo "文件已备份到$original_spark_submit_file" + ln -snf $spark_submit_modified $spark_submit_link + echo "将$spark_submit_link链接到$spark_submit_modified" +else + echo -e "\n$spark_submit_link 不是软连接也不是文件 创建新的软链接" + ln -n $spark_submit_modified $spark_submit_link + echo "将$spark_submit_link链接到$spark_submit_modified" +fi + +# 展现修改的成果 +echo -e "\更新后$spark_submit_link的链接情况如下" +ls -l $spark_submit_link +echo -e "\n软链接刷新前原始spark_submit的位置如下" +echo $original_spark_submit_file +echo -e "\n修改后与原始文件的修改差异如下:" +diff $spark_submit_link $original_spark_submit_file diff --git a/omniadvisor/src/omniadvisor/repository/__init__.py b/omniadvisor/src/omniadvisor/repository/__init__.py index 23b2cfefac5c0c11c4bb70afbeb96909bd9c66e7..0e1c8e920da3bc34dfe6ee7cdeda75307b45eb94 100644 --- a/omniadvisor/src/omniadvisor/repository/__init__.py +++ b/omniadvisor/src/omniadvisor/repository/__init__.py @@ -8,8 +8,15 @@ if not settings.configured: # 启用内存数据库,确保不影响原来的数据库 DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': OA_CONF.server_base_dir / 'db.sqlite3', + 'ENGINE': 'django.db.backends.mysql', + 'NAME': 'omniadvisor_test', #数据库名 + 'USER': 'root', # MySQL 用户名 + 'PASSWORD': '123456', # MySQL 密码 + 'HOST': 'localhost', # 数据库主机 如果是本地就是localhost + 'PORT': '9999', # MySQL默认端口是3306 + 'OPTIONS': { + 'charset': 'utf8mb4' + } } } diff --git a/omniadvisor/src/omniadvisor/service/retest_service.py b/omniadvisor/src/omniadvisor/service/retest_service.py index efb641708bc06801113207d30c42118901247bd8..061319584679388c097b388f5b40120bfb0a4ec3 100644 --- a/omniadvisor/src/omniadvisor/service/retest_service.py +++ b/omniadvisor/src/omniadvisor/service/retest_service.py @@ -35,6 +35,6 @@ def retest(load: Load, config: dict): # 若是出现异常配置,也要退出 global_logger.warning('Retest failed in round %d. Exception source: Spark exception: %s.', i, spark_output) tuning_result = get_tuning_result(load, config) - if tuning_result.failed_count >= OA_CONF.config_fail_threshold: + if tuning_result.failed_times >= OA_CONF.config_fail_threshold: global_logger.info('The number of retest failures has reached the failure threshold.') return diff --git a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py index 9954120586a6f8b96d99f12d09fd1298f6f0fc65..233103825d05423a9c5b6e9cc2142170f6e46aee 100644 --- a/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py +++ b/omniadvisor/src/omniadvisor/service/tuning_result/tuning_result.py @@ -85,17 +85,13 @@ class TuningResult: return self._tuning_record.rounds @property - def exam_count(self): + def exam_times(self): return len(self._exam_records) @property - def failed_count(self): + def failed_times(self): return sum([exam_record.status == OA_CONF.ExecStatus.fail for exam_record in self._exam_records]) - @property - def completed_count(self): - return sum([exam_record.status != OA_CONF.ExecStatus.running for exam_record in self._exam_records]) - @property def status(self): """ @@ -105,9 +101,9 @@ class TuningResult: 复测中:复测次数小于指定复测次数,且失败次数没有超过阈值,复测流程尚在进行中 :return: """ - if self.failed_count >= OA_CONF.config_fail_threshold: + if self.failed_times >= OA_CONF.config_fail_threshold: overall_status = OA_CONF.ExecStatus.fail - elif self.completed_count >= OA_CONF.tuning_retest_times: + elif len(self._exam_records) >= OA_CONF.tuning_retest_times: overall_status = OA_CONF.ExecStatus.success else: overall_status = OA_CONF.ExecStatus.running @@ -115,9 +111,6 @@ class TuningResult: @property def runtime(self): - if self.status == OA_CONF.ExecStatus.fail: - return OA_CONF.exec_fail_return_runtime - # 计算执行状态为success的平均runtime success_runtimes = [ exam_record.runtime @@ -132,9 +125,6 @@ class TuningResult: @property def trace(self): - if self.status == OA_CONF.ExecStatus.fail: - return OA_CONF.exec_fail_return_trace - success_exam_results = [ exam_record for exam_record in self._exam_records if exam_record.status == OA_CONF.ExecStatus.success diff --git a/omniadvisor/src/server/app/admin.py b/omniadvisor/src/server/app/admin.py index d6d2c70bb59c5468072426c07d2b924e7a88c110..bb7ce5201e40d4572733d1164f29227ccab2213f 100644 --- a/omniadvisor/src/server/app/admin.py +++ b/omniadvisor/src/server/app/admin.py @@ -19,5 +19,5 @@ class TuningRecordAdmin(admin.ModelAdmin): @admin.register(DatabaseExamRecord) class ExamRecordAdmin(admin.ModelAdmin): - list_display = ('id', 'load', 'config', 'start_time', 'end_time', 'status', 'runtime', 'trace') + list_display = ('id', 'tuning_record', 'config', 'start_time', 'end_time', 'status', 'runtime', 'trace') list_filter = () diff --git a/omniadvisor/src/server/app/models.py b/omniadvisor/src/server/app/models.py index bc5279bef8da279adfe85d77713c5b755a3e0212..b9761f76e2fac1a6736e562b8ed9e5d11f2b8275 100644 --- a/omniadvisor/src/server/app/models.py +++ b/omniadvisor/src/server/app/models.py @@ -18,9 +18,9 @@ class DatabaseLoad(models.Model): tuning_needed = models.BooleanField(default=True) class Meta: - constraints = [ - models.UniqueConstraint(fields=['exec_attr', 'default_config'], name='load_unique') - ] + # constraints = [ + # models.UniqueConstraint(fields=['exec_attr', 'default_config'], name='load_unique') + # ] db_table = 'omniadvisor_load' # 自定义表名 @@ -46,9 +46,9 @@ class DatabaseTuningRecord(models.Model): rounds = models.IntegerField(null=True) class Meta: - constraints = [ - models.UniqueConstraint(fields=['load', 'config'], name='tuning_record_unique') - ] + # constraints = [ + # models.UniqueConstraint(fields=['load', 'config'], name='tuning_record_unique') + # ] db_table = 'omniadvisor_tuning_record' # 自定义表名 @@ -63,8 +63,8 @@ class DatabaseExamRecord(models.Model): ] id = models.AutoField(primary_key=True) - load = models.ForeignKey( - DatabaseLoad, + tuning_record = models.ForeignKey( + DatabaseTuningRecord, on_delete=models.DO_NOTHING, null=False ) diff --git a/omniadvisor/src/server/engine/__init__.py b/omniadvisor/src/server/engine/__init__.py index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..c45523b2f83d4ee6d8742e97d5c9f9775f01cae0 100644 --- a/omniadvisor/src/server/engine/__init__.py +++ b/omniadvisor/src/server/engine/__init__.py @@ -0,0 +1,2 @@ +import pymysql +pymysql.install_as_MySQLdb() \ No newline at end of file diff --git a/omniadvisor/src/server/engine/settings.py b/omniadvisor/src/server/engine/settings.py index e100d21c736eecbb80cc7a0b3bc0679aecbb2242..63d6c7471864effb066b806ee402b700bed946d4 100644 --- a/omniadvisor/src/server/engine/settings.py +++ b/omniadvisor/src/server/engine/settings.py @@ -76,8 +76,15 @@ WSGI_APPLICATION = 'server.engine.wsgi.application' DATABASES = { 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': BASE_DIR / 'db.sqlite3' + 'ENGINE': 'django.db.backends.mysql', + 'NAME': 'omniadvisor', + 'USER': 'root', + 'PASSWORD': '123456', + 'HOST': 'localhost', + 'PORT': '9999', + 'OPTIONS': { + 'charset': 'utf8mb4' + } } } diff --git a/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py b/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py index dd4d701b23118195a593a096ff85f6f061deac11..660db90490247e341be32210d7f83d13fcd29e9a 100644 --- a/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py +++ b/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py @@ -65,7 +65,7 @@ class TestTuning: mock_spark_run.return_value = mock_exam_record, spark_output mock_smac_tuning.return_value = self.tune_return_val mock_tuning_result = MagicMock() - mock_tuning_result.failed_count = OA_CONF.config_fail_threshold + mock_tuning_result.failed_times = OA_CONF.config_fail_threshold mock_get_tuning_result.return_value = mock_tuning_result tuning_result_history = MagicMock() tuning_result_history.best_config = {**self.load.default_config, **self.tune_return_val[0]} diff --git a/omniadvisor/tests/omniadvisor/service/tuning_result/test_tuning_result.py b/omniadvisor/tests/omniadvisor/service/tuning_result/test_tuning_result.py index ff50acfaa70269607a74fcd71d3fd37f2135b9a6..0d6759328764d824e04388754928374c4a038767 100644 --- a/omniadvisor/tests/omniadvisor/service/tuning_result/test_tuning_result.py +++ b/omniadvisor/tests/omniadvisor/service/tuning_result/test_tuning_result.py @@ -17,9 +17,7 @@ class TestTuningResult: # Mocking TuningRecord and ExamRecord tuning_record_mock = MagicMock() - exam_record_mock_1 = MagicMock() - exam_record_mock_2 = MagicMock() - exam_record_mock_3 = MagicMock() + task_mock = MagicMock() # Set up mock attributes for tuning record tuning_record_mock.config = {"param1": "value1", "param2": "value2"} @@ -28,28 +26,20 @@ class TestTuningResult: tuning_record_mock.rounds = 1 # Set up mock attributes for exam record - exam_record_mock_1.status = OA_CONF.ExecStatus.success - exam_record_mock_1.runtime = 10 - exam_record_mock_1.trace = "trace_info1" - exam_record_mock_2.status = OA_CONF.ExecStatus.success - exam_record_mock_2.runtime = 8 - exam_record_mock_2.trace = "trace_info2" - exam_record_mock_3.status = OA_CONF.ExecStatus.success - exam_record_mock_3.runtime = 12 - exam_record_mock_3.trace = "trace_info3" - - return load, config, tuning_record_mock, [exam_record_mock_1, exam_record_mock_2, exam_record_mock_3] + task_mock.status = OA_CONF.ExecStatus.success + task_mock.runtime = 10 + task_mock.trace = "trace_info" + + return load, config, tuning_record_mock, task_mock @patch('omniadvisor.service.tuning_result.tuning_result.TuningRecordRepository.query_by_load_and_config') @patch('omniadvisor.service.tuning_result.tuning_result.ExamRecordRepository.query_by_load_and_config') def test_get_tuning_result(self, mock_task_query, mock_tuning_record_query, setup_mocks): - # 强制设定复测次数为3,用于测试 - OA_CONF.tuning_retest_times = 3 + load, config, tuning_record_mock, task_mock = setup_mocks - # 准备测试数据 - load, config, tuning_record_mock, exam_record_mock_list = setup_mocks + # Set return values for mocks mock_tuning_record_query.return_value = [tuning_record_mock] - mock_task_query.return_value = exam_record_mock_list + mock_task_query.return_value = [task_mock] result = get_tuning_result(load, config) @@ -61,16 +51,28 @@ class TestTuningResult: assert result.runtime == 10 def test_tuning_result_property(self, setup_mocks): - # 强制设定复测次数为3,用于测试 - OA_CONF.tuning_retest_times = 3 + _, _, tuning_record_mock, _ = setup_mocks # 准备测试数据 - _, _, tuning_record_mock, exam_record_mock_list = setup_mocks - exam_record_mock_list[-1].status = OA_CONF.ExecStatus.running + task_mock1 = MagicMock() + task_mock1.status = OA_CONF.ExecStatus.success + task_mock1.runtime = 10 + task_mock1.trace = "trace_info1" + task_mock2 = MagicMock() + task_mock2.status = OA_CONF.ExecStatus.success + task_mock2.runtime = 15 + task_mock2.trace = "trace_info2" + task_mock3 = MagicMock() + task_mock3.status = OA_CONF.ExecStatus.fail + task_mock3.runtime = 10 + task_mock3.trace = "trace_info3" + # 强制设定复测次数为3,用于测试 + OA_CONF.tuning_retest_times = 3 + tasks = [task_mock1, task_mock2] tuning_result = TuningResult( tuning_record=tuning_record_mock, - exam_records=exam_record_mock_list + exam_records=tasks ) assert tuning_result.config == {"param1": "value1", "param2": "value2"} @@ -78,16 +80,15 @@ class TestTuningResult: assert tuning_result.method_extend == "method_extend1" assert tuning_result.rounds == 1 assert tuning_result.status == OA_CONF.ExecStatus.running - assert tuning_result.runtime == 9 - assert tuning_result.trace == "trace_info2" + assert tuning_result.runtime == 12.5 + assert tuning_result.trace == "trace_info1" - # 准备测试数据 - exam_record_mock_list[-1].status = OA_CONF.ExecStatus.fail + tasks = [task_mock1, task_mock2, task_mock3] tuning_result = TuningResult( tuning_record=tuning_record_mock, - exam_records=exam_record_mock_list + exam_records=tasks ) assert tuning_result.status == OA_CONF.ExecStatus.fail - assert tuning_result.runtime == OA_CONF.exec_fail_return_runtime - assert tuning_result.trace == OA_CONF.exec_fail_return_trace + assert tuning_result.runtime == 12.5 + assert tuning_result.trace == "trace_info1"