diff --git a/src/main/java/org/graphybench/driver/driver/Db.java b/src/main/java/org/graphybench/driver/driver/Db.java index 218efefc8ff9e3d10fc25431acac337b9c2ed097..da3c8fb65631b20002235ba30f7fdfaa0a049d95 100644 --- a/src/main/java/org/graphybench/driver/driver/Db.java +++ b/src/main/java/org/graphybench/driver/driver/Db.java @@ -1,5 +1,6 @@ /* * Copyright © 2022 Linked Data Benchmark Council (info@ldbcouncil.org) + * Copyright © 2025 BingTong@CreateLink (tongbing@chuanglintech.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,6 +13,9 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modifications: + * - Added automatic test mode (2025) by BingTong@CreateLink (tongbing@chuanglintech.com) */ package org.graphybench.driver.driver; @@ -85,6 +89,17 @@ public abstract class Db implements Closeable { ); } + public final synchronized void reInitAutomatic() throws DbException { + try { + ((PoolingOperationHandlerRunnerFactory) operationHandlerRunnableContextFactory).shutdownAutomatic(); + } catch (OperationException e) { + throw new DbException("Error shutting down operation handler runnable factory", e); + } + operationHandlerRunnableContextFactory = new PoolingOperationHandlerRunnerFactory( + new InstantiatingOperationHandlerRunnerFactory() + ); + } + /** * Called once to cleanup state for DB client */ diff --git a/src/main/java/org/graphybench/driver/driver/PoolingOperationHandlerRunnerFactory.java b/src/main/java/org/graphybench/driver/driver/PoolingOperationHandlerRunnerFactory.java index 9309bd9f5a15b5e6aef5903d3089a29dffc1e86e..2db34722e9382dcbeac2ed9871054e83c52ac82e 100644 --- a/src/main/java/org/graphybench/driver/driver/PoolingOperationHandlerRunnerFactory.java +++ b/src/main/java/org/graphybench/driver/driver/PoolingOperationHandlerRunnerFactory.java @@ -1,5 +1,6 @@ /* * Copyright © 2022 Linked Data Benchmark Council (info@ldbcouncil.org) + * Copyright © 2025 BingTong@CreateLink (tongbing@chuanglintech.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,6 +13,9 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modifications: + * - Added automatic test mode (2025) by BingTong@CreateLink (tongbing@chuanglintech.com) */ package org.graphybench.driver.driver; @@ -131,6 +135,11 @@ public class PoolingOperationHandlerRunnerFactory implements OperationHandlerRun } } + public void shutdownAutomatic() throws OperationException { + innerOperationHandlerRunnerFactory.shutdown(); + operationHandlerRunnerPool.shutdown(); + } + @Override public String toString() { return PoolingOperationHandlerRunnerFactory.class.getSimpleName() + "{" diff --git a/src/main/java/org/graphybench/driver/driver/Workload.java b/src/main/java/org/graphybench/driver/driver/Workload.java index bc95c1a5ba45b00dad7dbbe2255239c7c6032713..93129b29c11cc4c0a8cfb622719f16ef08e7705c 100644 --- a/src/main/java/org/graphybench/driver/driver/Workload.java +++ b/src/main/java/org/graphybench/driver/driver/Workload.java @@ -1,5 +1,6 @@ /* * Copyright © 2022 Linked Data Benchmark Council (info@ldbcouncil.org) + * Copyright © 2025 BingTong@CreateLink (tongbing@chuanglintech.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,6 +13,9 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modifications: + * - Added automatic test mode (2025) by BingTong@CreateLink (tongbing@chuanglintech.com) */ package org.graphybench.driver.driver; @@ -48,10 +52,33 @@ public abstract class Workload implements Closeable { boolean warmup ) { long excessiveDelayThresholdAsMilli = TimeUnit.SECONDS.toMillis(1); - double toleratedExcessiveDelayCountPercentage = 0.05; // 95% of the queries must run below delay threshold + // Specifies the fraction of the delay threshold that is allowed to be exceeded + double toleratedExcessiveDelayCountPercentage = configuration.timeoutRate(); long toleratedExcessiveDelayCount = // Total tolerated excessive delay count - (warmup) ? Math.round(configuration.warmupCount() * toleratedExcessiveDelayCountPercentage) - : Math.round(configuration.operationCount() * toleratedExcessiveDelayCountPercentage); + (warmup) ? Math.round(configuration.warmupCount() * toleratedExcessiveDelayCountPercentage) + : Math.round(configuration.operationCount() * toleratedExcessiveDelayCountPercentage); + return new ResultsLogValidationTolerances( + excessiveDelayThresholdAsMilli, + toleratedExcessiveDelayCount, + toleratedExcessiveDelayCountPercentage + ); + } + + /** + * resultsLogValidationTolerances Automated beta version of the method + * + * @param operationCount + * @return + */ + public ResultsLogValidationTolerances resultsLogValidationTolerancesAutomatic( + DriverConfiguration configuration, + long operationCount + ) { + long excessiveDelayThresholdAsMilli = TimeUnit.SECONDS.toMillis(1); + // Specifies the fraction of the delay threshold that is allowed to be exceeded + double toleratedExcessiveDelayCountPercentage = configuration.timeoutRate(); + // Total tolerated excessive delay count + long toleratedExcessiveDelayCount = Math.round(operationCount * toleratedExcessiveDelayCountPercentage); return new ResultsLogValidationTolerances( excessiveDelayThresholdAsMilli, toleratedExcessiveDelayCount, diff --git a/src/main/java/org/graphybench/driver/driver/control/ConsoleAndFileDriverConfiguration.java b/src/main/java/org/graphybench/driver/driver/control/ConsoleAndFileDriverConfiguration.java index c15f1157eb2e45e41f55422c9c71101a9957b902..ae92258b923b0168841c3c7368fda8ab31fd1a2c 100644 --- a/src/main/java/org/graphybench/driver/driver/control/ConsoleAndFileDriverConfiguration.java +++ b/src/main/java/org/graphybench/driver/driver/control/ConsoleAndFileDriverConfiguration.java @@ -1,5 +1,6 @@ /* * Copyright © 2022 Linked Data Benchmark Council (info@ldbcouncil.org) + * Copyright © 2025 BingTong@CreateLink (tongbing@chuanglintech.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,8 +13,12 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modifications: + * - Added automatic test mode (2025) by BingTong@CreateLink (tongbing@chuanglintech.com) */ + package org.graphybench.driver.driver.control; import static java.lang.String.format; @@ -120,6 +125,43 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { public static final String WARMUP_COUNT_ARG = "wu"; public static final long WARMUP_COUNT_DEFAULT = 0; public static final String WARMUP_COUNT_DEFAULT_STRING = Long.toString(WARMUP_COUNT_DEFAULT); + public static final String ESTIMATE_TEST_TIME_ARG = "ett"; + public static final long ESTIMATE_TEST_DEFAULT_TIME = 5 * 60 * 1000; + public static final String ESTIMATE_TEST_DEFAULT_TIME_STRING = Long.toString(ESTIMATE_TEST_DEFAULT_TIME); + public static final String ESTIMATE_TEST_TIME_DESCRIPTION = format( + "Quickly estimate the duration of each test in the phase (default: %s)." + + "if -1, the operation to complete the number of warmup is finished", + ESTIMATE_TEST_DEFAULT_TIME_STRING); + public static final String ACCURATE_TEST_TIME_ARG = "att"; + public static final long ACCURATE_TEST_DEFAULT_TIME = 7200 * 1000; + public static final String ACCURATE_TEST_DEFAULT_TIME_STRING = Long.toString(ACCURATE_TEST_DEFAULT_TIME); + public static final String ACCURATE_TEST_TIME_DESCRIPTION = format( + "The duration of each test in the precision tuning phase (default: %s)." + + "If -1, the operation on the number of operation_count is completed", + ACCURATE_TEST_DEFAULT_TIME_STRING); + public static final String DICHOTOMY_ERROR_RANGE_ARG = "der"; + public static final double DEFAULT_DICHOTOMY_ERROR_RANGE = 1E-5; + public static final String DEFAULT_DICHOTOMY_ERROR_RANGE_STRING = Double.toString(DEFAULT_DICHOTOMY_ERROR_RANGE); + public static final String DICHOTOMY_ERROR_RANGE_DESCRIPTION = format( + "Binary end condition, tolerance range, (default: %s)", DEFAULT_DICHOTOMY_ERROR_RANGE_STRING); + public static final String TCR_MIN_ARG = "tcrMin"; + public static final double DEFAULT_TCR_MIN = 1E-9; + public static final String DEFAULT_TCR_MIN_STRING = Double.toString(DEFAULT_TCR_MIN); + public static final String TCR_MIN_DESCRIPTION = format( + "Minimum time compression ratio limit (default: %s)", DEFAULT_TCR_MIN_STRING); + public static final String TCR_MAX_ARG = "tcrMax"; + public static final double DEFAULT_TCR_MAX = 1; + public static final String DEFAULT_TCR_MAX_STRING = Double.toString(DEFAULT_TCR_MAX); + public static final String TCR_MAX_DESCRIPTION = format( + "Maximum time compression ratio limit (default: %s)", DEFAULT_TCR_MAX_STRING); + public static final String TIMEOUT_RATE_ARG = "timeoutRate"; + public static final double DEFAULT_TIMEOUT_RATE = 0.05; // 95% of the queries must run below delay threshold + public static final String DEFAULT_TIMEOUT_RATE_STRING = Double.toString(DEFAULT_TIMEOUT_RATE); + public static final String TIMEOUT_RATE_DESCRIPTION = format( + "Specifies the fraction of the delay threshold that is allowed to be exceeded (default: %s)", + DEFAULT_TIMEOUT_RATE_STRING); + + public static final String PROPERTY_FILE_ARG = "P"; public static final String PROPERTY_ARG = "p"; private static final String THREADS_DESCRIPTION = @@ -180,6 +222,12 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { "sleep duration (ms) injected into busy wait loops (to reduce CPU consumption)"; private static final String SKIP_COUNT_ARG_LONG = "skip"; private static final String WARMUP_COUNT_ARG_LONG = "warmup"; + public static final String ESTIMATE_TEST_TIME_ARG_LONG = "estimate"; + public static final String ACCURATE_TEST_TIME_ARG_LONG = "accurate"; + public static final String DICHOTOMY_ERROR_RANGE_ARG_LONG = "error_range"; + public static final String TCR_MIN_ARG_LONG = "tcr_min"; + public static final String TCR_MAX_ARG_LONG = "tcr_max"; + public static final String TIMEOUT_RATE_ARG_LONG = "timeout_rate"; private static final String PROPERTY_FILE_DESCRIPTION = "load properties from file(s) - files will be loaded in the order provided\n" + "first files are highest priority; later values will not override earlier values"; @@ -192,12 +240,12 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { private final String name; private final String dbClassName; private final String workloadClassName; - private final long operationCount; + private long operationCount; private final int threadCount; private final int statusDisplayIntervalAsSeconds; private final TimeUnit timeUnit; private final String resultDirPath; - private final double timeCompressionRatio; + private double timeCompressionRatio; private final int validationParametersSize; private final boolean validationSerializationCheck; private final boolean recordDelayedOperations; @@ -205,19 +253,43 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { private final long spinnerSleepDurationAsMilli; private final boolean printHelp; private final boolean ignoreScheduledStartTimes; - private final long warmupCount; + private long warmupCount; private final long skipCount; private final boolean flushLog; - - public ConsoleAndFileDriverConfiguration(Map paramsMap, String mode, String name, - String dbClassName, String workloadClassName, long operationCount, - int threadCount, int statusDisplayIntervalAsSeconds, TimeUnit timeUnit, - String resultDirPath, double timeCompressionRatio, - int validationParametersSize, boolean validationSerializationCheck, - boolean recordDelayedOperations, String databaseValidationFilePath, - long spinnerSleepDurationAsMilli, boolean printHelp, - boolean ignoreScheduledStartTimes, long warmupCount, long skipCount, - boolean flushLog) { + private final long estimateTestTime; + private final long accurateTestTime; + private final double dichotomyErrorRange; + private final double tcrMin; + private final double tcrMax; + private final double timeoutRate; + + public ConsoleAndFileDriverConfiguration(Map paramsMap, + String mode, + String name, + String dbClassName, + String workloadClassName, + long operationCount, + int threadCount, + int statusDisplayIntervalAsSeconds, + TimeUnit timeUnit, + String resultDirPath, + double timeCompressionRatio, + int validationParametersSize, + boolean validationSerializationCheck, + boolean recordDelayedOperations, + String databaseValidationFilePath, + long spinnerSleepDurationAsMilli, + boolean printHelp, + boolean ignoreScheduledStartTimes, + long warmupCount, + long skipCount, + boolean flushLog, + long estimateTestTime, + long accurateTestTime, + double dichotomyErrorRange, + double tcrMin, + double tcrMax, + double timeoutRate) { if (null == paramsMap) { paramsMap = new HashMap<>(); } @@ -242,7 +314,12 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { this.warmupCount = warmupCount; this.skipCount = skipCount; this.flushLog = flushLog; - + this.estimateTestTime = estimateTestTime; + this.accurateTestTime = accurateTestTime; + this.dichotomyErrorRange = dichotomyErrorRange; + this.tcrMin = tcrMin; + this.tcrMax = tcrMax; + this.timeoutRate = timeoutRate; if (null != mode) { paramsMap.put(MODE_ARG, mode); } @@ -272,6 +349,12 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { paramsMap.put(WARMUP_COUNT_ARG, Long.toString(warmupCount)); paramsMap.put(SKIP_COUNT_ARG, Long.toString(skipCount)); paramsMap.put(FLUSH_LOG_ARG, Boolean.toString(flushLog)); + paramsMap.put(ESTIMATE_TEST_TIME_ARG, Long.toString(estimateTestTime)); + paramsMap.put(ACCURATE_TEST_TIME_ARG, Long.toString(accurateTestTime)); + paramsMap.put(DICHOTOMY_ERROR_RANGE_ARG, Double.toString(dichotomyErrorRange)); + paramsMap.put(TCR_MIN_ARG, Double.toString(tcrMin)); + paramsMap.put(TCR_MAX_ARG, Double.toString(tcrMax)); + paramsMap.put(TIMEOUT_RATE_ARG, Double.toString(timeoutRate)); // Validation specific if (null != databaseValidationFilePath) { paramsMap.put(DB_VALIDATION_FILE_PATH_ARG, databaseValidationFilePath); @@ -304,6 +387,12 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { defaultParamsMap.put(SPINNER_SLEEP_DURATION_ARG, SPINNER_SLEEP_DURATION_DEFAULT_STRING); defaultParamsMap.put(WARMUP_COUNT_ARG, WARMUP_COUNT_DEFAULT_STRING); defaultParamsMap.put(SKIP_COUNT_ARG, SKIP_COUNT_DEFAULT_STRING); + defaultParamsMap.put(ESTIMATE_TEST_TIME_ARG, ESTIMATE_TEST_DEFAULT_TIME_STRING); + defaultParamsMap.put(ACCURATE_TEST_TIME_ARG, ACCURATE_TEST_DEFAULT_TIME_STRING); + defaultParamsMap.put(DICHOTOMY_ERROR_RANGE_ARG, DEFAULT_DICHOTOMY_ERROR_RANGE_STRING); + defaultParamsMap.put(TCR_MIN_ARG, DEFAULT_TCR_MIN_STRING); + defaultParamsMap.put(TCR_MAX_ARG, DEFAULT_TCR_MAX_STRING); + defaultParamsMap.put(TIMEOUT_RATE_ARG, DEFAULT_TIMEOUT_RATE_STRING); return defaultParamsMap; } @@ -389,14 +478,22 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { long skipCount = Long.parseLong(paramsMap.get(SKIP_COUNT_ARG)); long warmupCount = Long.parseLong(paramsMap.get(WARMUP_COUNT_ARG)); boolean printHelp = Boolean.parseBoolean(paramsMap.get(HELP_ARG)); + boolean recordDelayedOperations = Boolean.parseBoolean(paramsMap.get(RECORD_DELAYED_OPERATIONS_ARG)); boolean ignoreScheduledStartTimes = Boolean.parseBoolean(paramsMap.get(IGNORE_SCHEDULED_START_TIMES_ARG)); + long estimateTestTime = Long.parseLong(paramsMap.get(ESTIMATE_TEST_TIME_ARG)); + long accurateTestTime = Long.parseLong(paramsMap.get(ACCURATE_TEST_TIME_ARG)); + double dichotomyErrorRange = Double.parseDouble(paramsMap.get(DICHOTOMY_ERROR_RANGE_ARG)); + double tcrMin = Double.parseDouble(paramsMap.get(TCR_MIN_ARG)); + double tcrMax = Double.parseDouble(paramsMap.get(TCR_MAX_ARG)); + double timeoutRate = Double.parseDouble(paramsMap.get(TIMEOUT_RATE_ARG)); boolean flushLog = Boolean.parseBoolean(paramsMap.get(FLUSH_LOG_ARG)); return new ConsoleAndFileDriverConfiguration(paramsMap, mode, name, dbClassName, workloadClassName, - operationCount, threadCount, statusDisplayIntervalAsSeconds, timeUnit, resultDirPath, - timeCompressionRatio, validationParametersSize, validationSerializationCheck, recordDelayedOperations, - databaseValidationFilePath, spinnerSleepDurationAsMilli, printHelp, ignoreScheduledStartTimes, - warmupCount, skipCount, flushLog); + operationCount, threadCount, statusDisplayIntervalAsSeconds, timeUnit, resultDirPath, + timeCompressionRatio, validationParametersSize, validationSerializationCheck, + recordDelayedOperations, databaseValidationFilePath, spinnerSleepDurationAsMilli, + printHelp, ignoreScheduledStartTimes, warmupCount, skipCount, flushLog, estimateTestTime, + accurateTestTime, dichotomyErrorRange, tcrMin, tcrMax, timeoutRate); } catch (DriverConfigurationException e) { throw new DriverConfigurationException(format("%s\n%s", e.getMessage(), commandlineHelpString()), e); } @@ -525,7 +622,6 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { cmdParams.put((String) cmdProperty.getKey(), (String) cmdProperty.getValue()); } } - boolean overwrite = true; return MapUtils.mergeMaps(convertComplexKeysToSimpleKeys(fileParams), convertComplexKeysToSimpleKeys(cmdParams), overwrite); @@ -548,6 +644,12 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { paramsMap = replaceKey(paramsMap, SPINNER_SLEEP_DURATION_ARG_LONG, SPINNER_SLEEP_DURATION_ARG); paramsMap = replaceKey(paramsMap, WARMUP_COUNT_ARG_LONG, WARMUP_COUNT_ARG); paramsMap = replaceKey(paramsMap, SKIP_COUNT_ARG_LONG, SKIP_COUNT_ARG); + paramsMap = replaceKey(paramsMap, ESTIMATE_TEST_TIME_ARG_LONG, ESTIMATE_TEST_TIME_ARG); + paramsMap = replaceKey(paramsMap, ACCURATE_TEST_TIME_ARG_LONG, ACCURATE_TEST_TIME_ARG); + paramsMap = replaceKey(paramsMap, DICHOTOMY_ERROR_RANGE_ARG_LONG, DICHOTOMY_ERROR_RANGE_ARG); + paramsMap = replaceKey(paramsMap, TCR_MIN_ARG_LONG, TCR_MIN_ARG); + paramsMap = replaceKey(paramsMap, TCR_MAX_ARG_LONG, TCR_MAX_ARG); + paramsMap = replaceKey(paramsMap, TIMEOUT_RATE_ARG_LONG, TIMEOUT_RATE_ARG); return paramsMap; } @@ -650,7 +752,50 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { .withLongOpt(SKIP_COUNT_ARG_LONG).create(SKIP_COUNT_ARG); options.addOption(skipCountOption); - Option printHelpOption = OptionBuilder.withDescription(HELP_DESCRIPTION).create(HELP_ARG); + Option estimateTestTimeOption = OptionBuilder.hasArgs(1) + .withArgName("milli") + .withDescription(ESTIMATE_TEST_TIME_DESCRIPTION) + .withLongOpt(ESTIMATE_TEST_TIME_ARG_LONG) + .create(ESTIMATE_TEST_TIME_ARG); + options.addOption(estimateTestTimeOption); + + Option accurateTestTimeOption = OptionBuilder.hasArgs(1) + .withArgName("milli") + .withDescription(ACCURATE_TEST_TIME_DESCRIPTION) + .withLongOpt(ACCURATE_TEST_TIME_ARG_LONG) + .create(ACCURATE_TEST_TIME_ARG); + options.addOption(accurateTestTimeOption); + + Option dichotomyErrorRangeOption = OptionBuilder.hasArgs(1) + .withArgName("error_range") + .withDescription(DICHOTOMY_ERROR_RANGE_DESCRIPTION) + .withLongOpt(DICHOTOMY_ERROR_RANGE_ARG_LONG) + .create(DICHOTOMY_ERROR_RANGE_ARG); + options.addOption(dichotomyErrorRangeOption); + + Option tcrMinOption = OptionBuilder.hasArgs(1) + .withArgName("min") + .withDescription(TCR_MIN_DESCRIPTION) + .withLongOpt(TCR_MIN_ARG_LONG) + .create(TCR_MIN_ARG); + options.addOption(tcrMinOption); + + Option tcrMaxOption = OptionBuilder.hasArgs(1) + .withArgName("max") + .withDescription(TCR_MAX_DESCRIPTION) + .withLongOpt(TCR_MAX_ARG_LONG) + .create(TCR_MAX_ARG); + options.addOption(tcrMaxOption); + + Option timeoutRateOption = OptionBuilder.hasArgs(1) + .withArgName("timeout") + .withDescription(TIMEOUT_RATE_DESCRIPTION) + .withLongOpt(TIMEOUT_RATE_ARG_LONG) + .create(TIMEOUT_RATE_ARG); + options.addOption(timeoutRateOption); + + Option printHelpOption = OptionBuilder.withDescription(HELP_DESCRIPTION) + .create(HELP_ARG); options.addOption(printHelpOption); @@ -764,6 +909,11 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { return timeCompressionRatio; } + @Override + public void setTimeCompressionRatio(double tcr) { + this.timeCompressionRatio = tcr; + } + @Override public boolean validationSerializationCheck() { return validationSerializationCheck; @@ -809,6 +959,16 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { return warmupCount; } + @Override + public void setWarmupCount(long warmupCount) { + this.warmupCount = warmupCount; + } + + @Override + public void setOperationCount(long operationCount) { + this.operationCount = operationCount; + } + @Override public long skipCount() { return skipCount; @@ -819,6 +979,36 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { return flushLog; } + @Override + public long estimateTestTime() { + return estimateTestTime; + } + + @Override + public long accurateTestTime() { + return accurateTestTime; + } + + @Override + public double dichotomyErrorRange() { + return dichotomyErrorRange; + } + + @Override + public double tcrMin() { + return tcrMin; + } + + @Override + public double tcrMax() { + return tcrMax; + } + + @Override + public double timeoutRate() { + return timeoutRate; + } + @Override public Map asMap() { return paramsMap; @@ -903,11 +1093,28 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { boolean newFlushLog = (newParamsMapWithSimpleKeys.containsKey(FLUSH_LOG_ARG)) ? Boolean.parseBoolean( newParamsMapWithSimpleKeys.get(FLUSH_LOG_ARG)) : flushLog; + long newEstimateTestTime = (newParamsMapWithSimpleKeys.containsKey(ESTIMATE_TEST_TIME_ARG)) + ? Long.parseLong(paramsMap.get(ESTIMATE_TEST_TIME_ARG)) : estimateTestTime; + long newAccurateTestTime = (newParamsMapWithSimpleKeys.containsKey(ACCURATE_TEST_TIME_ARG)) + ? Long.parseLong(paramsMap.get(ACCURATE_TEST_TIME_ARG)) : accurateTestTime; + double newDichotomyErrorRange = (newParamsMapWithSimpleKeys.containsKey(DICHOTOMY_ERROR_RANGE_ARG)) + ? Double.parseDouble(paramsMap.get(DICHOTOMY_ERROR_RANGE_ARG)) : dichotomyErrorRange; + double newTcrMin = + (newParamsMapWithSimpleKeys.containsKey(TCR_MIN_ARG)) ? Double.parseDouble(paramsMap.get(TCR_MIN_ARG)) + : tcrMin; + double newTcrMax = + (newParamsMapWithSimpleKeys.containsKey(TCR_MAX_ARG)) ? Double.parseDouble(paramsMap.get(TCR_MAX_ARG)) + : tcrMax; + double newTimeoutRate = (newParamsMapWithSimpleKeys.containsKey(TIMEOUT_RATE_ARG)) + ? Double.parseDouble(paramsMap.get(TIMEOUT_RATE_ARG)) + : timeoutRate; + return new ConsoleAndFileDriverConfiguration(newOtherParams, newMode, newName, newDbClassName, - newWorkloadClassName, newOperationCount, newThreadCount, newStatusDisplayIntervalAsSeconds, newTimeUnit, - newResultDirPath, newTimeCompressionRatio, newValidationParametersSize, newValidationSerializationCheck, - newRecordDelayedOperations, newDatabaseValidationFilePath, newSpinnerSleepDurationAsMilli, newPrintHelp, - newIgnoreScheduledStartTimes, newWarmupCount, newSkipCount, newFlushLog); + newWorkloadClassName, newOperationCount, newThreadCount, newStatusDisplayIntervalAsSeconds, newTimeUnit, + newResultDirPath, newTimeCompressionRatio, newValidationParametersSize, newValidationSerializationCheck, + newRecordDelayedOperations, newDatabaseValidationFilePath, newSpinnerSleepDurationAsMilli, newPrintHelp, + newIgnoreScheduledStartTimes, newWarmupCount, newSkipCount, newFlushLog, newEstimateTestTime, + newAccurateTestTime, newDichotomyErrorRange, newTcrMin, newTcrMax, newTimeoutRate); } /** @@ -952,7 +1159,7 @@ public class ConsoleAndFileDriverConfiguration implements DriverConfiguration { if (!validationSerializationCheck) { argsList.addAll(Lists.newArrayList("-" + VALIDATION_SERIALIZATION_CHECK_ARG, - Boolean.toString(validationSerializationCheck))); + Boolean.toString(validationSerializationCheck))); } if (null != databaseValidationFilePath) { argsList.addAll(Lists.newArrayList("-" + DB_VALIDATION_FILE_PATH_ARG, databaseValidationFilePath)); diff --git a/src/main/java/org/graphybench/driver/driver/control/DriverConfiguration.java b/src/main/java/org/graphybench/driver/driver/control/DriverConfiguration.java index a0198cf369c59e328d989ad1317d5566f26efaf0..d4d193c13f71420f788701c5457657917d74f64b 100644 --- a/src/main/java/org/graphybench/driver/driver/control/DriverConfiguration.java +++ b/src/main/java/org/graphybench/driver/driver/control/DriverConfiguration.java @@ -1,5 +1,6 @@ /* * Copyright © 2022 Linked Data Benchmark Council (info@ldbcouncil.org) + * Copyright © 2025 BingTong@CreateLink (tongbing@chuanglintech.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,8 +13,12 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modifications: + * - Added automatic test mode (2025) by BingTong@CreateLink (tongbing@chuanglintech.com) */ + package org.graphybench.driver.driver.control; import java.util.Map; @@ -62,6 +67,24 @@ public interface DriverConfiguration { boolean flushLog(); + void setTimeCompressionRatio(double tcr); + + void setWarmupCount(long warmupCount); + + void setOperationCount(long operationCount); + + long estimateTestTime(); + + long accurateTestTime(); + + double dichotomyErrorRange(); + + double tcrMin(); + + double tcrMax(); + + double timeoutRate(); + String toPropertiesString() throws DriverConfigurationException; Map asMap(); diff --git a/src/main/java/org/graphybench/driver/driver/control/OperationMode.java b/src/main/java/org/graphybench/driver/driver/control/OperationMode.java index 8436d127ed7c54415d5098c4dbd38b3fbf589727..8cea72ff6a18ca2ea0c67cd9d1e782367c56f8d6 100644 --- a/src/main/java/org/graphybench/driver/driver/control/OperationMode.java +++ b/src/main/java/org/graphybench/driver/driver/control/OperationMode.java @@ -1,5 +1,6 @@ /* * Copyright © 2022 Linked Data Benchmark Council (info@ldbcouncil.org) + * Copyright © 2025 BingTong@CreateLink (tongbing@chuanglintech.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,13 +13,18 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modifications: + * - Added automatic test mode (2025) by BingTong@CreateLink (tongbing@chuanglintech.com) */ + package org.graphybench.driver.driver.control; public enum OperationMode { CREATE_VALIDATION, VALIDATE_DATABASE, CREATE_STATISTICS, - EXECUTE_BENCHMARK + EXECUTE_BENCHMARK, + AUTOMATIC_TEST } diff --git a/src/main/java/org/graphybench/driver/driver/driver/AutomaticTestMode.java b/src/main/java/org/graphybench/driver/driver/driver/AutomaticTestMode.java new file mode 100644 index 0000000000000000000000000000000000000000..e1bc6ad82dac74763c5c67bc8e7e305ca55274ea --- /dev/null +++ b/src/main/java/org/graphybench/driver/driver/driver/AutomaticTestMode.java @@ -0,0 +1,598 @@ +/* + * Copyright © 2025 BingTong@CreateLink (tongbing@chuanglintech.com) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +package org.graphybench.driver.driver.driver; + +import static java.lang.String.format; + +import com.google.common.base.Charsets; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.TimeUnit; +import org.graphybench.driver.driver.Db; +import org.graphybench.driver.driver.DbException; +import org.graphybench.driver.driver.Workload; +import org.graphybench.driver.driver.WorkloadException; +import org.graphybench.driver.driver.WorkloadStreams; +import org.graphybench.driver.driver.control.ControlService; +import org.graphybench.driver.driver.generator.GeneratorFactory; +import org.graphybench.driver.driver.generator.RandomDataGeneratorFactory; +import org.graphybench.driver.driver.log.LoggingService; +import org.graphybench.driver.driver.runtime.ConcurrentErrorReporter; +import org.graphybench.driver.driver.runtime.DefaultQueues; +import org.graphybench.driver.driver.runtime.SimpleResultsLogWriter; +import org.graphybench.driver.driver.runtime.WorkloadRunner; +import org.graphybench.driver.driver.runtime.coordination.CompletionTimeException; +import org.graphybench.driver.driver.runtime.coordination.CompletionTimeService; +import org.graphybench.driver.driver.runtime.coordination.CompletionTimeServiceAssistant; +import org.graphybench.driver.driver.runtime.coordination.CompletionTimeWriter; +import org.graphybench.driver.driver.runtime.metrics.DisruptorSbeMetricsService; +import org.graphybench.driver.driver.runtime.metrics.JsonWorkloadMetricsFormatter; +import org.graphybench.driver.driver.runtime.metrics.MetricsCollectionException; +import org.graphybench.driver.driver.runtime.metrics.MetricsManager; +import org.graphybench.driver.driver.runtime.metrics.MetricsService; +import org.graphybench.driver.driver.runtime.metrics.NullResultsLogWriter; +import org.graphybench.driver.driver.runtime.metrics.ResultsLogWriter; +import org.graphybench.driver.driver.runtime.metrics.WorkloadResultsSnapshot; +import org.graphybench.driver.driver.runtime.metrics.WorkloadStatusSnapshot; +import org.graphybench.driver.driver.temporal.TemporalUtil; +import org.graphybench.driver.driver.temporal.TimeSource; +import org.graphybench.driver.driver.util.ClassLoaderHelper; +import org.graphybench.driver.driver.util.Tuple3; +import org.graphybench.driver.driver.validation.ResultsLogValidationResult; +import org.graphybench.driver.driver.validation.ResultsLogValidationSummary; +import org.graphybench.driver.driver.validation.ResultsLogValidationTolerances; +import org.graphybench.driver.driver.validation.ResultsLogValidator; + + +/** + * Automatic test mode, using dichotomous method to test the configuration parameters suitable for the current machine + */ +public class AutomaticTestMode implements DriverMode { + private final ControlService controlService; + private final TimeSource timeSource; + private final LoggingService loggingService; + private final long randomSeed; + private final TemporalUtil temporalUtil; + private final ResultsDirectory resultsDirectory; + + private Workload workload = null; + private Db database = null; + private MetricsService metricsService = null; + private CompletionTimeService completionTimeService = null; + private WorkloadRunner workloadRunner = null; + private ResultsLogWriter resultsLogWriter = null; + + public AutomaticTestMode( + ControlService controlService, + TimeSource timeSource, + long randomSeed) throws DriverException { + this.controlService = controlService; + this.timeSource = timeSource; + this.loggingService = controlService.loggingServiceFactory() + .loggingServiceFor(getClass().getSimpleName()); + this.randomSeed = randomSeed; + this.temporalUtil = new TemporalUtil(); + this.resultsDirectory = new ResultsDirectory(controlService.configuration()); + } + + public WorkloadStatusSnapshot status() throws MetricsCollectionException { + throw new UnsupportedOperationException("Not yet implemented"); + } + + @Override + public void init() throws DriverException { + loggingService.info("Driver Configuration"); + loggingService.info(controlService.toString()); + } + + @Override + public Object startExecutionAndAwaitCompletion() throws DriverException { + double l = controlService.configuration().tcrMin(); + double r = controlService.configuration().tcrMax(); + // Record the results of the current and last successful one + ResultsLogValidationResult successfulResult = new ResultsLogValidationResult(); + ResultsLogValidationResult currentResult; + + loggingService.info("--------------------------Rapid estimate phase--------------------------"); + // Assuming that tcr=1, the approximate number of basic operations required in a minute + // (initially set to be larger, and later change according to the machine situation) + long baseCnt = 1000; + // Stores the original total number of operations + long sourceOpCnt = controlService.configuration().operationCount(); + long sourceWaCnt = controlService.configuration().operationCount(); + // Gets the binary end condition, tolerance range + double range = controlService.configuration().dichotomyErrorRange(); + // Time compression ratio of this round + double tcr = 0; + int numberOfRounds = 1; + while (r - l >= range) { + // The first time you try to run with the tcr specified by the configuration + if (tcr != 0) { + tcr = l + (r - l) / 2; + controlService.configuration().setTimeCompressionRatio(tcr); + } else { + tcr = controlService.configuration().timeCompressionRatio(); + } + // Sets the number of preheat operations + computeRunOperationCount(baseCnt, sourceWaCnt, sourceOpCnt); + loggingService.info(String.format("--New round %d: Compression ratio: %f,\t warmup count: %d--", + numberOfRounds++, tcr, controlService.configuration().warmupCount() + )); + currentResult = validationTest(true); + + if (currentResult.isSuccessful()) { + r = tcr; + successfulResult = currentResult; + } else { + l = tcr; + } + // Based on the current throughput, determine the base operand per minute at tcr=1 + baseCnt = (long) (Math.ceil(currentResult.throughput()) * tcr * 60); + } + + loggingService.info("--------------------------Accurate adjust parameter phase--------------------------"); + // The theory is accurate tcr >= estimated tcr, + r = Math.min(l * 10, controlService.configuration().tcrMax()); + // to prevent accidents and give a little bit of room that may be smaller + l = controlService.configuration().tcrMin() + (l - controlService.configuration().tcrMin()) * 0.7; + // Ensure that at least one precision tuning phase is performed + tcr = 0; + numberOfRounds = 1; + do { + // The first attempt is to run with the results of the estimation phase + if (tcr != 0) { + tcr = l + (r - l) / 2; + controlService.configuration().setTimeCompressionRatio(tcr); + } else { + tcr = controlService.configuration().timeCompressionRatio(); + } + // Set the number of operations for the warm-up and formal phases + computeRunOperationCount(baseCnt, sourceWaCnt, sourceOpCnt); + loggingService.info(String.format( + "--New round %d: Compression ratio: %f,\t warmup count: %d,\t operation count: %d--", + numberOfRounds++, tcr, controlService.configuration().warmupCount(), + controlService.configuration().operationCount() + )); + + currentResult = validationTest(false); + if (currentResult.isSuccessful()) { + r = tcr; + successfulResult = currentResult; + } else { + l = tcr; + } + baseCnt = (long) (Math.ceil(currentResult.throughput()) * tcr * 60); + } while (r - l >= range); + + // If finding the last l is not successful, replace it with the last successful one + if (!currentResult.isSuccessful()) { + controlService.configuration().setTimeCompressionRatio(r); + // At this point, the state of the machine has slipped, and there is no need to repeat the test + } + try { + loggingService.info("Shutting down database connector..."); + Instant dbShutdownStart = Instant.now(); + database.close(); + Duration shutdownDuration = Duration.between(dbShutdownStart, Instant.now()); + loggingService.info("Database connector shutdown successfully in: " + shutdownDuration); + } catch (IOException e) { + throw new DriverException("Error shutting down database", e); + } + loggingService.info("Workload completed successfully"); + loggingService.info(String.format("\n" + + "--------------------------------------------------------------------------\n" + + "------- time compression ratio suitable for the machine: %-10f-------\n" + + "--------------------------------------------------------------------------", + controlService.configuration().timeCompressionRatio())); + return successfulResult; + } + + /** + * Set the number of operations + * + * @param baseCnt Assuming that tcr=1, the approximate number of basic operations required in a minute + * @param sourceWaCnt Boot specified in the configuration source warmup count + * @param sourceOpCnt Boot specified in the configuration source operation count + */ + public void computeRunOperationCount(long baseCnt, + long sourceWaCnt, + long sourceOpCnt) { + // Enlarge baseCnt by 30 times to prevent accidents and adjust the quantity according to tcr + double perMinute = baseCnt * 30 / controlService.configuration().timeCompressionRatio(); + if (controlService.configuration().estimateTestTime() != -1) { + // baseCnt is one minute, now converted to estimate time + controlService.configuration().setWarmupCount((long) Math.max( + perMinute * (controlService.configuration().estimateTestTime() / 60000.0), sourceWaCnt + )); + } + if (controlService.configuration().accurateTestTime() != -1) { + controlService.configuration().setOperationCount((long) Math.max( + perMinute * (controlService.configuration().accurateTestTime() / 60000.0), sourceOpCnt + )); + } + } + + /** + * Perform a configuration parameter test + * + * @return ResultsLogValidationResult test result + */ + private ResultsLogValidationResult validationTest(boolean warmup) throws DriverException { + ResultsLogValidationResult result = executionAndAwaitCompletion(warmup); + if (result == null) { + loggingService.info("Obtain result The test result failed"); + return new ResultsLogValidationResult(); + } + loggingService.info(String.format("\n" + + "---------------------------- This test check %-8s-------------------\n" + + "---------------------------- tcr: %-10f----------------------------\n" + + "---------------------------- throughput: %-10f---------------------\n" + + "---------------------------- onTimeRatio: %-10f--------------------", + result.isSuccessful() ? "PASS" : "FAILURE", + controlService.configuration().timeCompressionRatio(), result.throughput(), result.onTimeRatio())); + return result; + } + + public ResultsLogValidationResult executionAndAwaitCompletion(boolean warmup) + throws DriverException { + ResultsLogValidationResult result = null; + if (controlService.configuration().warmupCount() > 0) { + loggingService.info("\n" + + " --------------------\n" + + " --- Warmup Phase ---\n" + + " --------------------"); + doInit(true); + result = doExecute(true, controlService.configuration().estimateTestTime()); + try { + // TODO remove in future + // This is necessary to clear the runnable context pool + // As objects in the pool would otherwise hold references to services used during warmup + loggingService.info("reInit database..."); + database.reInitAutomatic(); + } catch (DbException e) { + throw new DriverException(format("Error reinitializing DB: %s", database.getClass() + .getName()), e); + } + } else { + loggingService.info("\n" + + " ---------------------------------\n" + + " --- No Warmup Phase Requested ---\n" + + " ---------------------------------"); + } + + if (!warmup) { + loggingService.info("\n" + + " -----------------\n" + + " --- Run Phase ---\n" + + " -----------------"); + doInit(false); + result = doExecute(false, controlService.configuration().accurateTestTime()); + try { + // TODO remove in future + // This is necessary to clear the runnable context pool + // As objects in the pool would otherwise hold references to services used during warmup + loggingService.info("reInit database..."); + database.reInitAutomatic(); + } catch (DbException e) { + throw new DriverException(format("Error reinitializing DB: %s", database.getClass() + .getName()), e); + } + } + return result; + } + + private void doInit(boolean warmup) throws DriverException { + ConcurrentErrorReporter errorReporter = new ConcurrentErrorReporter(); + GeneratorFactory gf = new GeneratorFactory(new RandomDataGeneratorFactory(randomSeed)); + + // ================================ + // === Results Log CSV Writer === + // ================================ + File resultsLog = resultsDirectory.getOrCreateResultsLogFile(warmup); + try { + resultsLogWriter = (null == resultsLog) + ? new NullResultsLogWriter() + : new SimpleResultsLogWriter( + resultsLog, + controlService.configuration().timeUnit(), + controlService.configuration().flushLog()); + } catch (IOException e) { + throw new DriverException( + format("Error creating results log writer for: %s", resultsLog.getAbsolutePath()), e); + } + + // ------------------ + // --- Workload --- + // ------------------ + loggingService.info("Scanning workload streams to calculate their limits..."); + + long offset = (warmup) + ? controlService.configuration().skipCount() + : controlService.configuration().skipCount() + controlService.configuration().warmupCount(); + long limit = (warmup) + ? controlService.configuration().warmupCount() + : controlService.configuration().operationCount(); + + WorkloadStreams workloadStreams; + long minimumTimeStamp; + try { + boolean returnStreamsWithDbConnector = true; + Tuple3 streamsAndWorkloadAndMinimumTimeStamp = + WorkloadStreams.createNewWorkloadWithOffsetAndLimitedWorkloadStreams( + controlService.configuration(), + gf, + returnStreamsWithDbConnector, + offset, + limit, + controlService.loggingServiceFactory() + ); + workloadStreams = streamsAndWorkloadAndMinimumTimeStamp._1(); + workload = streamsAndWorkloadAndMinimumTimeStamp._2(); + minimumTimeStamp = streamsAndWorkloadAndMinimumTimeStamp._3(); + } catch (Exception e) { + throw new DriverException(format("Error loading workload class: %s", + controlService.configuration().workloadClassName()), e); + } + loggingService.info(format("Loaded workload: %s", workload.getClass().getName())); + + loggingService.info(format("Retrieving workload stream: %s", workload.getClass().getSimpleName())); + controlService.setWorkloadStartTimeAsMilli(System.currentTimeMillis() + TimeUnit.SECONDS.toMillis(5)); + WorkloadStreams timeMappedWorkloadStreams; + try { + timeMappedWorkloadStreams = WorkloadStreams.timeOffsetAndCompressWorkloadStreams( + workloadStreams, + controlService.workloadStartTimeAsMilli(), + controlService.configuration().timeCompressionRatio(), + gf + ); + } catch (WorkloadException e) { + throw new DriverException("Error while retrieving operation stream for workload", e); + } + + // ---------------= + // ---== DB ---== + // ---------------= + if (null == database) { + try { + database = ClassLoaderHelper.loadDb(controlService.configuration().dbClassName()); + database.init( + controlService.configuration().asMap(), + controlService.loggingServiceFactory().loggingServiceFor(database.getClass().getSimpleName()), + workload.operationTypeToClassMapping() + ); + } catch (DbException e) { + throw new DriverException( + format("Error initializing DB: %s", controlService.configuration().dbClassName()), e); + } + loggingService.info(format("Loaded DB: %s", database.getClass().getName())); + } + + // ------------------------ + // --- Metrics Service == + // ------------------------ + try { + // TODO create metrics service factory so different ones can be easily created + metricsService = new DisruptorSbeMetricsService( + timeSource, + errorReporter, + controlService.configuration().timeUnit(), + DisruptorSbeMetricsService.DEFAULT_HIGHEST_EXPECTED_RUNTIME_DURATION_AS_NANO, + resultsLogWriter, + workload.operationTypeToClassMapping(), + controlService.loggingServiceFactory() + ); + } catch (MetricsCollectionException e) { + throw new DriverException("Error creating metrics service", e); + } + + // --------------------------------- + // --- Completion Time Service --- + // --------------------------------- + CompletionTimeServiceAssistant completionTimeServiceAssistant = new CompletionTimeServiceAssistant(); + try { + completionTimeService = + completionTimeServiceAssistant.newThreadedQueuedCompletionTimeService( + timeSource, + errorReporter + ); + } catch (CompletionTimeException e) { + throw new DriverException("Error instantiating Completion Time Service", e); + } + + // ------------------------ + // --- Workload Runner == + // ------------------------ + loggingService.info(format("Instantiating %s", WorkloadRunner.class.getSimpleName())); + try { + int operationHandlerExecutorsBoundedQueueSize = DefaultQueues.DEFAULT_BOUND_1000; + workloadRunner = new WorkloadRunner( + timeSource, + database, + timeMappedWorkloadStreams, + metricsService, + errorReporter, + completionTimeService, + controlService.loggingServiceFactory(), + controlService.configuration().threadCount(), + controlService.configuration().statusDisplayIntervalAsSeconds(), + controlService.configuration().spinnerSleepDurationAsMilli(), + controlService.configuration().ignoreScheduledStartTimes(), + operationHandlerExecutorsBoundedQueueSize); + } catch (Exception e) { + throw new DriverException(format("Error instantiating %s", WorkloadRunner.class.getSimpleName()), e); + } + + // ------------------------------------------= + // --- Initialize Completion Time Service == + // ------------------------------------------= + // TODO note, this MUST be done after creation of Workload Runner because Workload Runner creates the + // TODO "writers" for completion time service (refactor this mess at some stage) + try { + if (completionTimeService.getAllWriters().isEmpty()) { + // There are no completion time writers, CT would never advance or be non-null, + // set to max so nothing ever waits on it + long nearlyMaxPossibleTimeAsMilli = Long.MAX_VALUE - 1; + long maxPossibleTimeAsMilli = Long.MAX_VALUE; + // Create a writer to use for advancing CT + CompletionTimeWriter completionTimeWriter = completionTimeService.newCompletionTimeWriter(); + completionTimeWriter.submitInitiatedTime(nearlyMaxPossibleTimeAsMilli); + completionTimeWriter.submitCompletedTime(nearlyMaxPossibleTimeAsMilli); + completionTimeWriter.submitInitiatedTime(maxPossibleTimeAsMilli); + completionTimeWriter.submitCompletedTime(maxPossibleTimeAsMilli); + } else { + // There are some completion time writers, initialize them to lowest time stamp in workload + completionTimeServiceAssistant.writeInitiatedAndCompletedTimesToAllWriters( + completionTimeService, minimumTimeStamp - 1); + completionTimeServiceAssistant + .writeInitiatedAndCompletedTimesToAllWriters(completionTimeService, minimumTimeStamp); + boolean completionTimeAdvancedToDesiredTime = + completionTimeServiceAssistant.waitForCompletionTime( + timeSource, + minimumTimeStamp - 1, + TimeUnit.SECONDS.toMillis(5), + completionTimeService, + errorReporter + ); + long completionTimeWaitTimeoutDurationAsMilli = TimeUnit.SECONDS.toMillis(5); + if (!completionTimeAdvancedToDesiredTime) { + throw new DriverException( + format( + "Timed out [%s] while waiting for completion time to advance to workload " + + "start time\nCurrent CT: %s\nWaiting For CT: %s", + completionTimeWaitTimeoutDurationAsMilli, + completionTimeService.completionTimeAsMilli(), + controlService.workloadStartTimeAsMilli()) + ); + } + loggingService.info("CT: " + temporalUtil + .milliTimeToDateTimeString(completionTimeService.completionTimeAsMilli()) + " / " + + completionTimeService.completionTimeAsMilli()); + } + } catch (CompletionTimeException e) { + throw new DriverException( + "Error while writing initial initiated and completed times to Completion Time Service", e); + } + } + + private ResultsLogValidationResult doExecute(boolean warmup, + long milli) throws DriverException { + try { + ConcurrentErrorReporter errorReporter = null; + if (milli == -1) { + // To execute normally, follow the EXECUTE_BENCHMARK process + errorReporter = workloadRunner.getFuture().get(); + } else { + errorReporter = workloadRunner.getFuture(milli); + } + loggingService.info("Shutting down workload..."); + workload.close(); + if (errorReporter.errorEncountered()) { + throw new DriverException("Error running workload\n" + errorReporter.toString()); + } + } catch (Exception e) { + throw new DriverException("Error running workload", e); + } + + loggingService.info("Shutting down completion time service..."); + try { + completionTimeService.shutdown(); + } catch (CompletionTimeException e) { + throw new DriverException("Error during shutdown of completion time service", e); + } + + loggingService.info("Shutting down metrics collection service..."); + WorkloadResultsSnapshot workloadResults; + try { + workloadResults = metricsService.getWriter().results(); + metricsService.shutdown(); + } catch (MetricsCollectionException e) { + throw new DriverException("Error during shutdown of metrics collection service", e); + } + + try { + if (warmup) { + loggingService.summaryResult(workloadResults); + } else { + loggingService.detailedResult(workloadResults); + } + if (resultsDirectory.exists()) { + File resultsSummaryFile = resultsDirectory.getOrCreateResultsSummaryFile(warmup); + loggingService.info( + format("Exporting workload metrics to %s...", resultsSummaryFile.getAbsolutePath()) + ); + MetricsManager.export(workloadResults, + new JsonWorkloadMetricsFormatter(), + new FileOutputStream(resultsSummaryFile), + Charsets.UTF_8 + ); + File configurationFile = resultsDirectory.getOrCreateConfigurationFile(warmup); + Files.write( + configurationFile.toPath(), + controlService.configuration().toPropertiesString().getBytes(StandardCharsets.UTF_8) + ); + resultsLogWriter.close(); + if (!controlService.configuration().ignoreScheduledStartTimes()) { + loggingService.info("Validating workload results..."); + // TODO make this feature accessible directly + ResultsLogValidator resultsLogValidator = new ResultsLogValidator(); + ResultsLogValidationTolerances resultsLogValidationTolerances = + workload.resultsLogValidationTolerancesAutomatic(controlService.configuration(), + workloadResults.totalOperationCount()); + + ResultsLogValidationSummary resultsLogValidationSummary = resultsLogValidator.compute( + resultsDirectory.getOrCreateResultsLogFile(warmup), + resultsLogValidationTolerances.excessiveDelayThresholdAsMilli() + ); + File resultsValidationFile = resultsDirectory.getOrCreateResultsValidationFile(warmup); + loggingService.info( + format("Exporting workload results validation to: %s", + resultsValidationFile.getAbsolutePath()) + ); + // Files.write( + // resultsValidationFile.toPath(), + // resultsLogValidationSummary.toJson().getBytes(StandardCharsets.UTF_8) + // ); + // TODO export result + ResultsLogValidationResult validationResult = resultsLogValidator.validateAutomatic( + resultsLogValidationSummary, + resultsLogValidationTolerances, + workloadResults + ); + loggingService.info(validationResult.getScheduleAuditResult( + controlService.configuration().recordDelayedOperations() + )); + Files.write( + resultsValidationFile.toPath(), + resultsLogValidationSummary.toJson().getBytes(StandardCharsets.UTF_8) + ); + return validationResult; + } + } + } catch (Exception e) { + throw new DriverException("Could not export workload metrics", e); + } + return null; + } +} + diff --git a/src/main/java/org/graphybench/driver/driver/driver/Driver.java b/src/main/java/org/graphybench/driver/driver/driver/Driver.java index 4f66705c9d092c078ef00243cd3f763eab6b65f2..17681bcd92982361487ddf22503cdcff6416c941 100644 --- a/src/main/java/org/graphybench/driver/driver/driver/Driver.java +++ b/src/main/java/org/graphybench/driver/driver/driver/Driver.java @@ -1,5 +1,6 @@ /* * Copyright © 2022 Linked Data Benchmark Council (info@ldbcouncil.org) + * Copyright © 2025 BingTong@CreateLink (tongbing@chuanglintech.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,6 +13,9 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modifications: + * - Added automatic test mode (2025) by BingTong@CreateLink (tongbing@chuanglintech.com) */ package org.graphybench.driver.driver.driver; @@ -42,7 +46,6 @@ public class Driver { LoggingServiceFactory loggingServiceFactory = new Log4jLoggingServiceFactory(detailedStatus); LoggingService loggingService = loggingServiceFactory.loggingServiceFor(Driver.class.getSimpleName()); - // try { TimeSource systemTimeSource = new SystemTimeSource(); ConsoleAndFileDriverConfiguration configuration = ConsoleAndFileDriverConfiguration.fromArgs(args); @@ -98,6 +101,8 @@ public class Driver { return new CalculateWorkloadStatisticsMode(controlService, RANDOM_SEED); case VALIDATE_DATABASE: return new ValidateDatabaseMode(controlService); + case AUTOMATIC_TEST: + return new AutomaticTestMode(controlService, new SystemTimeSource(), RANDOM_SEED); case EXECUTE_BENCHMARK: default: // Execute benchmark is default behaviour return new ExecuteWorkloadMode(controlService, new SystemTimeSource(), RANDOM_SEED); diff --git a/src/main/java/org/graphybench/driver/driver/runtime/WorkloadRunner.java b/src/main/java/org/graphybench/driver/driver/runtime/WorkloadRunner.java index b8646c0dd837b9987f5da5e62c2cdabd9cfc5e67..11347be18a779dc2dcb80b25a6e8dab50322d859 100644 --- a/src/main/java/org/graphybench/driver/driver/runtime/WorkloadRunner.java +++ b/src/main/java/org/graphybench/driver/driver/runtime/WorkloadRunner.java @@ -1,5 +1,6 @@ /* * Copyright © 2022 Linked Data Benchmark Council (info@ldbcouncil.org) + * Copyright © 2025 BingTong@CreateLink (tongbing@chuanglintech.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,6 +13,9 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modifications: + * - Added automatic test mode (2025) by BingTong@CreateLink (tongbing@chuanglintech.com) */ package org.graphybench.driver.driver.runtime; @@ -20,6 +24,8 @@ import static java.lang.String.format; import java.util.ArrayList; import java.util.List; +import java.util.Timer; +import java.util.TimerTask; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -45,6 +51,7 @@ import org.graphybench.driver.driver.temporal.TimeSource; public class WorkloadRunner { static final long RUNNER_POLLING_INTERVAL_AS_MILLI = 100; + static final long RUNNER_POLLING_INTERVAL_AS_MILLI_AUTOMATIC = 500; private static final CompletionTimeWriter DUMMY_COMPLETION_TIME_WRITER = new DummyCompletionTimeWriter(); private final WorkloadRunnerFuture workloadRunnerFuture; @@ -83,6 +90,11 @@ public class WorkloadRunner { return workloadRunnerFuture; } + public ConcurrentErrorReporter getFuture(long milli) { + workloadRunnerFuture.startThread(milli); + return workloadRunnerFuture.errorReporter; + } + private enum WorkloadRunnerThreadState { NOT_STARTED, RUNNING, @@ -137,6 +149,39 @@ public class WorkloadRunner { } } + private void startThread(long milli) { + if (!workloadRunnerThread.state().equals(WorkloadRunnerThreadState.NOT_STARTED)) { + workloadRunnerThread.shutdownEverything(WorkloadRunnerThread.ShutdownType.FORCED, + new ConcurrentErrorReporter()); + } + workloadRunnerThread.start(); + // You cannot sleep(milli) directly, because if the execution finishes in the meantime, you will + // sleep for extra time + + AtomicBoolean expire = new AtomicBoolean(false); + Timer timer = new Timer(); + while (workloadRunnerThread.state().equals(WorkloadRunnerThreadState.NOT_STARTED)) { + Spinner.powerNap(RUNNER_POLLING_INTERVAL_AS_MILLI); + } + timer.schedule(new TimerTask() { + @Override + public void run() { + expire.set(true); + } + }, milli); + while (!expire.get() && workloadRunnerThread.state().equals(WorkloadRunnerThreadState.RUNNING)) { + Spinner.powerNap(RUNNER_POLLING_INTERVAL_AS_MILLI_AUTOMATIC); + } + timer.cancel(); + if (isCancelled || isDone) { + throw new IllegalStateException("Can not call method after future has been cancelled or completed"); + } + workloadRunnerThread.interrupt(); + isCancelled = true; + isDone = true; + workloadRunnerThread.stateRef.set(WorkloadRunnerThreadState.COMPLETED_SUCCEEDED); + } + @Override public boolean cancel(boolean mayInterruptIfRunning) { // After this method returns, subsequent calls to isDone will always return true diff --git a/src/main/java/org/graphybench/driver/driver/validation/ResultsLogValidationResult.java b/src/main/java/org/graphybench/driver/driver/validation/ResultsLogValidationResult.java index 624a727b821c78ece07a6ea7675a6c185a930962..71d71b73b41ef4a082517ec506ef6dd9de5a6315 100644 --- a/src/main/java/org/graphybench/driver/driver/validation/ResultsLogValidationResult.java +++ b/src/main/java/org/graphybench/driver/driver/validation/ResultsLogValidationResult.java @@ -1,5 +1,6 @@ /* * Copyright © 2022 Linked Data Benchmark Council (info@ldbcouncil.org) + * Copyright © 2025 BingTong@CreateLink (tongbing@chuanglintech.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,6 +13,9 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modifications: + * - Added automatic test mode (2025) by BingTong@CreateLink (tongbing@chuanglintech.com) */ package org.graphybench.driver.driver.validation; @@ -27,6 +31,34 @@ public class ResultsLogValidationResult { } private boolean aboveThreshold = false; + private Double onTimeRatio; + private Double throughput; + private Long operationCount; + + public void computeOnTimeRatio(Long excessiveDelayCount) { + onTimeRatio = (double) excessiveDelayCount / operationCount; + onTimeRatio = (1 - onTimeRatio) * 100; + } + + public Double onTimeRatio() { + return onTimeRatio; + } + + public void setThroughput(Double throughput) { + this.throughput = throughput; + } + + public Double throughput() { + return throughput; + } + + public void setOperationCount(Long operationCount) { + this.operationCount = operationCount; + } + + public Long operationCount() { + return operationCount; + } public static class ValidationError { private final ValidationErrorType errorType; diff --git a/src/main/java/org/graphybench/driver/driver/validation/ResultsLogValidator.java b/src/main/java/org/graphybench/driver/driver/validation/ResultsLogValidator.java index 8c2772425e1545bc17f556f8caff28dff2d9ee27..734251160c2b7bdd8d5f47ea686bf9aaaad1d344 100644 --- a/src/main/java/org/graphybench/driver/driver/validation/ResultsLogValidator.java +++ b/src/main/java/org/graphybench/driver/driver/validation/ResultsLogValidator.java @@ -1,5 +1,6 @@ /* * Copyright © 2022 Linked Data Benchmark Council (info@ldbcouncil.org) + * Copyright © 2025 BingTong@CreateLink (tongbing@chuanglintech.com) * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -12,6 +13,9 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * + * Modifications: + * - Added automatic test mode (2025) by BingTong@CreateLink (tongbing@chuanglintech.com) */ package org.graphybench.driver.driver.validation; @@ -59,12 +63,50 @@ public class ResultsLogValidator { operationCountPerTypeMap.put(metric.name(), metric.count()); } - for (String operationType : summary.excessiveDelayCountPerType().keySet()) { + for (String operationType : summary.excessiveDelayCountPerType() + .keySet()) { + Long cnt = operationCountPerTypeMap.get(operationType); long allowedLateOperations = Math.round( - operationCountPerTypeMap.get(operationType) * tolerances.toleratedExcessiveDelayCountPercentage()); + (cnt == null ? 0 : cnt) * tolerances.toleratedExcessiveDelayCountPercentage()); if (recordDelayedOperations && summary.excessiveDelayCountPerType().get(operationType) > allowedLateOperations) { result.aboveThreshold(); + result.addError( + ValidationErrorType.TOO_MANY_LATE_OPERATIONS, + format("Late Count for %s (%s) > (%s) Tolerated Late Count", + operationType, + summary.excessiveDelayCountPerType() + .get(operationType), + allowedLateOperations + ) + ); + } + } + return result; + } + + /** + * validate Method Automation beta + */ + public ResultsLogValidationResult validateAutomatic( + ResultsLogValidationSummary summary, + ResultsLogValidationTolerances tolerances, + WorkloadResultsSnapshot workloadResults) { + + ResultsLogValidationResult result = new ResultsLogValidationResult(); + + Map operationCountPerTypeMap = new HashMap<>(); + for (OperationMetricsSnapshot metric : workloadResults.allMetrics()) { + operationCountPerTypeMap.put(metric.name(), metric.count()); + } + + for (String operationType : summary.excessiveDelayCountPerType() + .keySet()) { + Long cnt = operationCountPerTypeMap.get(operationType); + long allowedLateOperations = Math.round( + (cnt == null ? 0 : cnt) * tolerances.toleratedExcessiveDelayCountPercentage()); + if (summary.excessiveDelayCountPerType() + .get(operationType) > allowedLateOperations) { result.addError( ValidationErrorType.TOO_MANY_LATE_OPERATIONS, format("Late Count for %s (%s) > (%s) Tolerated Late Count", @@ -75,6 +117,15 @@ public class ResultsLogValidator { ); } } + + result.setThroughput(workloadResults.throughput()); + result.setOperationCount(workloadResults.totalOperationCount()); + // Determine whether time out + if (summary.excessiveDelayCount() > tolerances.toleratedExcessiveDelayCount()) { + result.aboveThreshold(); + } + // Calculated timeliness + result.computeOnTimeRatio(summary.excessiveDelayCount()); return result; } diff --git a/src/main/resources/example/ldbc_finbench_automatic_test_dummy.properties b/src/main/resources/example/ldbc_finbench_automatic_test_dummy.properties new file mode 100644 index 0000000000000000000000000000000000000000..aff3eb61fafb196f28dabe5d45dd0a22f168a14d --- /dev/null +++ b/src/main/resources/example/ldbc_finbench_automatic_test_dummy.properties @@ -0,0 +1,107 @@ +############################################################ +# SUT defined configurations # +############################################################ +host=localhost +port=9091 +user=admin +pass=123456 +path=cypher/ +############################################################ +# Driver configurations # +############################################################ +# Quickly estimate the duration of each test in the phase (millisecond). +# if -1, the operation to complete the number of warmup is finished +estimate=300000 +# The duration of each test in the precise tuning phase (millisecond). +# if -1, the operation to complete the number of operation_count is finished +accurate=7200000 +# Binary end condition, tolerance range +error_range=1E-5 +# Minimum time compression ratio limit +tcr_min=1E-9 +# Maximum time compression ratio limit +tcr_max=1 +# Specifies the fraction of the delay threshold that is allowed to be exceeded +timeout_rate=0.05 +# The ratio on the first test +time_compression_ratio=0.1 +status=1 +thread_count=1 +name=LDBC-FinBench +# Modes available: 1.CREATE_VALIDATION 2.VALIDATE_DATABASE 3.EXECUTE_BENCHMARK 4.AUTOMATIC_TEST +mode=AUTOMATIC_TEST +results_log=true +time_unit=MICROSECONDS +peer_identifiers= +workload_statistics=false +spinner_wait_duration=1 +help=false +ignore_scheduled_start_times=false +workload=org.ldbcouncil.finbench.driver.workloads.transaction.LdbcFinBenchTransactionWorkload +db=org.ldbcouncil.finbench.impls.dummy.DummyDb +operation_count=100000 +validation_parameters_size=1000 +validate_workload=true +validate_database=validation_params.csv +warmup=5 +graphybench.transaction.queries.parameters_dir=src/main/resources/example/data/read_params +graphybench.transaction.queries.updates_dir=src/main/resources/example/data/incremental +# param and update files suffix, `csv` or `parquet`, default is `csv` +graphybench.transaction.queries.files_suffix=csv +graphybench.transaction.queries.simple_read_dissipation=0.2 +graphybench.transaction.queries.update_interleave=10000000 +graphybench.transaction.queries.scale_factor=1 +# Frequency of complex read queries +graphybench.transaction.queries.ComplexRead1_freq=1 +graphybench.transaction.queries.ComplexRead2_freq=1 +graphybench.transaction.queries.ComplexRead3_freq=1 +graphybench.transaction.queries.ComplexRead4_freq=1 +graphybench.transaction.queries.ComplexRead5_freq=1 +graphybench.transaction.queries.ComplexRead6_freq=1 +graphybench.transaction.queries.ComplexRead7_freq=1 +graphybench.transaction.queries.ComplexRead8_freq=1 +graphybench.transaction.queries.ComplexRead9_freq=1 +graphybench.transaction.queries.ComplexRead10_freq=1 +graphybench.transaction.queries.ComplexRead11_freq=1 +graphybench.transaction.queries.ComplexRead12_freq=1 +# For debugging purposes +graphybench.transaction.queries.ComplexRead1_enable=true +graphybench.transaction.queries.ComplexRead2_enable=true +graphybench.transaction.queries.ComplexRead3_enable=true +graphybench.transaction.queries.ComplexRead4_enable=true +graphybench.transaction.queries.ComplexRead5_enable=true +graphybench.transaction.queries.ComplexRead6_enable=true +graphybench.transaction.queries.ComplexRead7_enable=true +graphybench.transaction.queries.ComplexRead8_enable=true +graphybench.transaction.queries.ComplexRead9_enable=true +graphybench.transaction.queries.ComplexRead10_enable=true +graphybench.transaction.queries.ComplexRead11_enable=true +graphybench.transaction.queries.ComplexRead12_enable=true +graphybench.transaction.queries.SimpleRead1_enable=true +graphybench.transaction.queries.SimpleRead2_enable=true +graphybench.transaction.queries.SimpleRead3_enable=true +graphybench.transaction.queries.SimpleRead4_enable=true +graphybench.transaction.queries.SimpleRead5_enable=true +graphybench.transaction.queries.SimpleRead6_enable=true +graphybench.transaction.queries.Write1_enable=true +graphybench.transaction.queries.Write2_enable=true +graphybench.transaction.queries.Write3_enable=true +graphybench.transaction.queries.Write4_enable=true +graphybench.transaction.queries.Write5_enable=true +graphybench.transaction.queries.Write6_enable=true +graphybench.transaction.queries.Write7_enable=true +graphybench.transaction.queries.Write8_enable=true +graphybench.transaction.queries.Write9_enable=true +graphybench.transaction.queries.Write10_enable=true +graphybench.transaction.queries.Write11_enable=true +graphybench.transaction.queries.Write12_enable=true +graphybench.transaction.queries.Write13_enable=true +graphybench.transaction.queries.Write14_enable=true +graphybench.transaction.queries.Write15_enable=true +graphybench.transaction.queries.Write16_enable=true +graphybench.transaction.queries.Write17_enable=true +graphybench.transaction.queries.Write18_enable=true +graphybench.transaction.queries.Write19_enable=true +graphybench.transaction.queries.ReadWrite1_enable=true +graphybench.transaction.queries.ReadWrite2_enable=true +graphybench.transaction.queries.ReadWrite3_enable=true \ No newline at end of file diff --git a/src/main/resources/example/ldbc_finbench_driver_dummy.properties b/src/main/resources/example/ldbc_finbench_driver_dummy.properties index 520453996fbf6399de7b8e6a133a2e6c30bbb46e..50eb2ea63592738aa35b319c88748e6322d262ae 100644 --- a/src/main/resources/example/ldbc_finbench_driver_dummy.properties +++ b/src/main/resources/example/ldbc_finbench_driver_dummy.properties @@ -12,7 +12,7 @@ path=cypher/ status=1 thread_count=1 name=GraphyBench -# Modes available: 1.CREATE_VALIDATION 2.VALIDATE_DATABASE 3.EXECUTE_BENCHMARK +# Modes available: 1.CREATE_VALIDATION 2.VALIDATE_DATABASE 3.EXECUTE_BENCHMARK 4.AUTOMATIC_TEST mode=EXECUTE_BENCHMARK results_log=true time_unit=MICROSECONDS