diff --git a/CMakeLists.txt b/CMakeLists.txt index 3df1daea6786b014239fcb8f2ea546610c566e42..fb11c8383c303fb7ff9f8d167f792989e1ec135b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,15 @@ cmake_minimum_required(VERSION 3.14.1) +# When downloading and extracting files from a URL, the timestamp is handled by using the current system time as the timestamp for the extracted files. +if(POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) +endif() + +# Enable _ROOT variable support (e.g., ZLIB_ROOT, etc.) +if(POLICY CMP0074) + cmake_policy(SET CMP0074 NEW) +endif() + project(Datasystem DESCRIPTION "DataSystem for Distributed Data Cache" LANGUAGES CXX C) # Version specified diff --git a/build.sh b/build.sh index 1763f63b3034203b7f540d7698d47245d61e3aa5..7c3c0f368c354e017a1fc5c36525d90dfa064dc9 100755 --- a/build.sh +++ b/build.sh @@ -296,9 +296,7 @@ function build_example() { cmake "${DATASYSTEM_DIR}/example" \ -DCMAKE_PREFIX_PATH="${prefix_path}" \ - -DBUILD_HETERO="${BUILD_HETERO}" \ - -DBUILD_WITH_URMA="${BUILD_WITH_URMA}" \ - -DENABLE_PERF=${ENABLE_PERF} || go_die "---- build example CMake project failed!" + -DBUILD_HETERO="${BUILD_HETERO}" || go_die "---- build example CMake project failed!" make || go_die "---- example make failed!" echo -e "---- build example success!" @@ -489,12 +487,18 @@ function build_datasystem() "-DBUILD_THREAD_NUM:STRING=${BUILD_THREAD_NUM}" "-DENABLE_STRIP:BOOL=${ENABLE_STRIP}" "-DBUILD_HETERO:BOOL=${BUILD_HETERO}" - "-DBUILD_WITH_URMA:BOOL=${BUILD_WITH_URMA}" - "-DDOWNLOAD_UB:BOOL=${DOWNLOAD_UB}" - "-DUB_URL:STRING=${UB_URL}" - "-DUB_SHA256:STRING=${UB_SHA256}" "-DSUPPORT_JEPROF:BOOL=${SUPPORT_JEPROF}" + "-DBUILD_WITH_URMA:BOOL=${BUILD_WITH_URMA}" + ) + + if [[ "${BUILD_WITH_URMA}" == "on" ]]; then + cmake_options+=( + "-DDOWNLOAD_UB:BOOL=${DOWNLOAD_UB}" + "-DUB_URL:STRING=${UB_URL}" + "-DUB_SHA256:STRING=${UB_SHA256}" ) + fi + if is_on "${PACKAGE_PYTHON}" && [ -n "${PYTHON_ROOT_DIR}" ]; then echo -e "-- Specify python root path: ${PYTHON_ROOT_DIR}" cmake_options=("${cmake_options[@]}" "-DPython3_ROOT_DIR:PATH=${PYTHON_ROOT_DIR}") diff --git a/cmake/external_libs/absl.cmake b/cmake/external_libs/absl.cmake index 0fa982916b068328f574c81f9cb45c2a2f485194..9ed040cb4f0cd95dda63ff2840d0bcb0ca1582c6 100644 --- a/cmake/external_libs/absl.cmake +++ b/cmake/external_libs/absl.cmake @@ -10,7 +10,8 @@ set(absl_SHA256 "c1e391c517790669dfcbbfda1278a61053679c303c0fb05018bf2266197f054 set(absl_CMAKE_OPTIONS -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE - -DCMAKE_CXX_STANDARD=17) + -DCMAKE_CXX_STANDARD=17 + -DABSL_PROPAGATE_CXX_STD=ON) set(absl_CXX_FLAGS ${THIRDPARTY_SAFE_FLAGS}) diff --git a/cmake/external_libs/cjson.cmake b/cmake/external_libs/cjson.cmake index d5c9d9a2477dfd11ddee999001589b3b7b138b92..a3bd4e3ee0624bec16573f8bfe19f83891115e57 100644 --- a/cmake/external_libs/cjson.cmake +++ b/cmake/external_libs/cjson.cmake @@ -7,7 +7,6 @@ endif() set(cjson_SHA256 "6edd31f4faa373dc94b26e4ab6b4483d32fbfc003771a6d0e1a9106e0d9521f2") set(cjson_C_FLAGS ${THIRDPARTY_SAFE_FLAGS}) -set(cjson_CXX_FLAGS ${THIRDPARTY_SAFE_FLAGS}) set(cjson_LINK_FLAGS ${THIRDPARTY_SAFE_FLAGS}) if ("$ENV{DS_PACKAGE}" STREQUAL "") set(cjson_PATCHES ${CMAKE_SOURCE_DIR}/third_party/patches/cjson/1.7.17/Backport-CVE-2024-31755-Add-NULL-check-to-cJSON_SetValuestring-c.patch) @@ -19,7 +18,6 @@ add_thirdparty_lib(CJSON FAKE_SHA256 ${cjson_FAKE_SHA256} VERSION ${cjson_VERSION} C_FLAGS ${cjson_C_FLAGS} - CXX_FLAGS ${cjson_CXX_FLAGS} LINK_FLAGS ${cjson_LINK_FLAGS} PATCHES ${cjson_PATCHES}) diff --git a/cmake/external_libs/grpc.cmake b/cmake/external_libs/grpc.cmake index 7e6cc84ad89252a597261d4f440d8aa0e50fc89c..3bb6efe7ea69561d8ecdbd0f9acafb4b686c489f 100644 --- a/cmake/external_libs/grpc.cmake +++ b/cmake/external_libs/grpc.cmake @@ -58,7 +58,6 @@ set(gRPC_CMAKE_OPTIONS -DgRPC_DOWNLOAD_ARCHIVES:BOOL=OFF -DgRPC_BUILD_TESTS:BOOL=OFF -DgRPC_PROTOBUF_PROVIDER:STRING=package - -DgRPC_PROTOBUF_PACKAGE_TYPE:STRING=CONFIG -Dutf8_range_DIR:PATH=${utf8_range_PKG_PATH} -DProtobuf_DIR:PATH=${Protobuf_PKG_PATH} -DgRPC_ABSL_PROVIDER:STRING=package @@ -71,8 +70,7 @@ set(gRPC_CMAKE_OPTIONS -DOPENSSL_ROOT_DIR=${OpenSSL_ROOT} -DgRPC_ZLIB_PROVIDER:STRING=package -DZLIB_ROOT:PATH=${ZLIB_ROOT} - -DCMAKE_CXX_STANDARD=17 - -DCMAKE_POLICY_DEFAULT_CMP0074=NEW) + -DCMAKE_CXX_STANDARD=17) if (gRPC_VERSION STREQUAL "1.54.2") set(gRPC_CMAKE_OPTIONS diff --git a/cmake/external_libs/libcurl.cmake b/cmake/external_libs/libcurl.cmake index ec0f34c0f2408deee9b5ade4cb5d352770d9c8f2..3c1d1a3d4716bb97e00f84689ab3d646d7ae9b9b 100644 --- a/cmake/external_libs/libcurl.cmake +++ b/cmake/external_libs/libcurl.cmake @@ -10,8 +10,6 @@ set(curl_CMAKE_OPTIONS -DCMAKE_CXX_STANDARD=11 -DOPENSSL_ROOT_DIR:PATH=${OpenSSL_ROOT}) -set(curl_CXX_FLAGS ${THIRDPARTY_SAFE_FLAGS}) - set(curl_C_FLAGS ${THIRDPARTY_SAFE_FLAGS}) if (curl_VERSION STREQUAL "8.8.0") @@ -33,7 +31,6 @@ add_thirdparty_lib(CURL FAKE_SHA256 ${curl_FAKE_SHA256} VERSION ${curl_VERSION} CONF_OPTIONS ${curl_CMAKE_OPTIONS} - CXX_FLAGS ${curl_CXX_FLAGS} C_FLAGS ${curl_C_FLAGS} PATCHES ${curl_PATCHES}) diff --git a/cmake/external_libs/spdlog.cmake b/cmake/external_libs/spdlog.cmake index be44d40c9adba75c84593e3456f8bc13907cf3f6..5fe09c33c575eaec16d95af5bb9dc4f8e01c2e99 100644 --- a/cmake/external_libs/spdlog.cmake +++ b/cmake/external_libs/spdlog.cmake @@ -6,7 +6,6 @@ else() endif() set(spdlog_SHA256 "a0a22ed8e4932cf5f7abc354fc96d20217a8b45471e732d5e672cc01979fe588") -set(spdlog_C_FLAGS ${THIRDPARTY_SAFE_FLAGS}) set(spdlog_CXX_FLAGS ${THIRDPARTY_SAFE_FLAGS}) set(spdlog_LINK_FLAGS ${THIRDPARTY_SAFE_FLAGS}) @@ -21,7 +20,6 @@ add_thirdparty_lib(SPDLOG SHA256 ${spdlog_SHA256} FAKE_SHA256 ${spdlog_FAKE_SHA256} VERSION ${spdlog_VERSION} - C_FLAGS ${spdlog_C_FLAGS} CXX_FLAGS ${spdlog_CXX_FLAGS} LINK_FLAGS ${spdlog_LINK_FLAGS} CONF_OPTIONS ${spdlog_CMAKE_OPTIONS} diff --git a/cmake/scripts/PackageDatasystem.cmake.in b/cmake/scripts/PackageDatasystem.cmake.in index d70a5f8b8a4f694d00eb0ce6dc0666d1aaefc06b..4cdc31fd5a6cf098342c0ca9aafcc7a183f7ca83 100644 --- a/cmake/scripts/PackageDatasystem.cmake.in +++ b/cmake/scripts/PackageDatasystem.cmake.in @@ -24,9 +24,12 @@ foreach(FILE ${FILE_LIST}) endforeach() find_program(CMAKE_STRIP NAMES strip) -file(GLOB SO_FILES "${DATASYSTEM_WHEEL_PATH}/lib/*.so*") +file(GLOB SO_FILES "${DATASYSTEM_WHEEL_PATH}/lib/*.so*" EXCLUDE REGEX "libacl_plugin.so") +#libacl_plugin.so need check sha256, can not strip foreach(SO_FILE ${SO_FILES}) - execute_process(COMMAND ${CMAKE_STRIP} ${SO_FILE}) + if (NOT SO_FILE MATCHES "libacl_plugin.so$") + execute_process(COMMAND ${CMAKE_STRIP} ${SO_FILE}) + endif() endforeach() # Run python setup.py bdist_whell to generate origin wheel file. @@ -37,4 +40,4 @@ file(GLOB ORIGIN_WHEEL "dist/dist/*.whl") file(COPY ${ORIGIN_WHEEL} DESTINATION ${CMAKE_INSTALL_PATH}/) # Remove temporary package path -file(REMOVE_RECURSE ${DATASYSTEM_SETUP_PATH}) \ No newline at end of file +file(REMOVE_RECURSE ${DATASYSTEM_SETUP_PATH}) diff --git a/cmake/scripts/PackagePython.cmake.in b/cmake/scripts/PackagePython.cmake.in index 918c5f1a07516204b6eefdb3b8c9271e415251c5..06088a5e40b0643bc8ae2456aa755e167936b8f6 100644 --- a/cmake/scripts/PackagePython.cmake.in +++ b/cmake/scripts/PackagePython.cmake.in @@ -18,8 +18,11 @@ file(COPY ${PYTHON_LIBPATH}/ find_program(CMAKE_STRIP NAMES strip) file(GLOB SO_FILES "${PYTHON_PACKAGE_LIBPATH}/lib/*.so*") +#libacl_plugin.so need check sha256, can not strip foreach(SO_FILE ${SO_FILES}) - execute_process(COMMAND ${CMAKE_STRIP} ${SO_FILE}) + if (NOT SO_FILE MATCHES "libacl_plugin.so$") + execute_process(COMMAND ${CMAKE_STRIP} ${SO_FILE}) + endif() endforeach() file(RENAME ${PYTHON_PACKAGE_LIBPATH}/setup.py ${PYTHON_PACKAGE_PATH}/setup.py) @@ -30,4 +33,4 @@ file(GLOB ORIGIN_WHEEL "python_api/dist/*.whl") # Rename origin wheel file and move to install path. file(COPY ${ORIGIN_WHEEL} DESTINATION ${CMAKE_INSTALL_PATH}) # Remove temporary package path -file(REMOVE_RECURSE ${PYTHON_PACKAGE_PATH}) \ No newline at end of file +file(REMOVE_RECURSE ${PYTHON_PACKAGE_PATH}) diff --git a/cmake/util.cmake b/cmake/util.cmake index 2dcd2aebdb9a168284d48747325e2355229c34ec..ce9720615fbd7678582add8e9ebc535511366ebc 100644 --- a/cmake/util.cmake +++ b/cmake/util.cmake @@ -1,5 +1,7 @@ include(FetchContent) +set(DOWNLOAD_EXTRACT_TIMESTAMP TRUE) + set(THIRDPARTY_SAFE_FLAGS "-fPIC -D_FORTIFY_SOURCE=2 -O2 -fstack-protector-strong -ffunction-sections -fdata-sections -Wl,--gc-sections -Wl,--build-id=none -Wl,-z,relro,-z,noexecstack,-z,now ${EXT_FLAGS}") if (CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "aarch64" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "9.1.0") diff --git a/docs/source_zh_cn/appendix/log_guide.md b/docs/source_zh_cn/appendix/log_guide.md index 8e57579324c9fdc3080efedcc695f0d1aeb62abf..60c6dbbf479b92659054c4025f9ddfb3debe4080 100644 --- a/docs/source_zh_cn/appendix/log_guide.md +++ b/docs/source_zh_cn/appendix/log_guide.md @@ -51,19 +51,19 @@ openYuanrong datasystem 不同模块日志分类如下表所示： | datasize | 16 | 记录Publish请求接收到的Payload大小。 | | request param | 2560 | 记录该请求的关键请求参数，最大长度2048。请参考“关键请求参数”表格。 | | response param | 1024 | 记录该请求的响应信息。最大长度为1024 Byte，超出则截断。 | -| shm_info | 47 | 记录共享内存使用信息，单位为Byte，按照1T限制大小，每个长度 13 Byte，格式为：memoryUsage/physicalMemoryUsage/totalLimit/rate
1) memoryUsage 已分配的内存大小，是已缓存的对象大小总和。
2) physicalMemoryUsage 已分配的物理内存大小。
3) totalLimit 共享内存总大小。
4) Rate 共享内存使用率，memoryUsage/totalLimit, 保留3位有效数字，单位: %. | -| spill_disk_inf0 | 47 | 记录Spill磁盘使用信息。单位为Byte，按照1T限制大小，每个长度 13 Byte，格式为：spaceUsage/physicalSpaceUsage/totalLimit/rate
1) spaceUsage 已使用的磁盘大小，是已Spill的对象大小总和。
2) physicalSpaceUsage 已使用的物理磁盘大小。
3) totalLimit Spill磁盘总大小。
4) Rate Spill磁盘使用率，spaceUsage /totalLimit, 保留3位有效数字，单位: %. | -| client nums | 5 | 记录已和worker成功建立连接的Client数。最大值为10000. | -| object nums | 9 | 记录worker已缓存对象数。按照1亿对象限制数量。 | -| object total datasize | 13 | 记录worker已缓存对象的大小。按照1T限制大小，长度 13 Byte | -| WorkerOcService threadpool | 21 | WorkerOcService threadpool使用信息，线程数限制最大128；格式为：idleNum/currentTotalNum/MaxThreadNum/waitingTaskNum/rate
1) idelNum 空闲线程数；
2) currentTotalNum 当前正在运行任务的线程数；
3) MaxThreadNum threadpool最大可申请的线程数；
4) waitingTaskNum 正在等待的任务数。
5) rate 线程利用率，currentTotalNum/ MaxThreadNum,单位：%，保留3位有效数字。 | -| WorkerWorkerOcService threadpool | 21 | threadpool使用信息 | -| MasterWokrerOcService threadpool | 21 | threadpool使用信息 | +| shm_info | 47 | 记录共享内存使用信息，单位为Byte，按照1T限制大小，每个长度 13 Byte，格式为：memoryUsage/physicalMemoryUsage/totalLimit/rate
1) memoryUsage 已分配的内存大小，是已缓存的对象大小总和。
2) physicalMemoryUsage 已分配的物理内存大小。
3) totalLimit 共享内存总大小。
4) Rate 共享内存使用率，memoryUsage/totalLimit, 保留3位有效数字，单位: %. | +| spill_disk_info | 47 | 记录Spill磁盘使用信息。单位为Byte，按照1T限制大小，每个长度 13 Byte，格式为：spaceUsage/physicalSpaceUsage/totalLimit/rate
1) spaceUsage 已使用的磁盘大小，是已Spill的对象大小总和。
2) physicalSpaceUsage 已使用的物理磁盘大小。
3) totalLimit Spill磁盘总大小。
4) Rate Spill磁盘使用率，spaceUsage /totalLimit, 保留3位有效数字，单位: %. | +| client nums | 5 | 记录已和worker成功建立连接的Client数。最大值为10000. | +| object nums | 9 | 记录worker已缓存对象数。按照1亿对象限制数量。| +| object total datasize | 13 | 记录worker已缓存对象的大小。按照1T限制大小，长度 13 Byte | +| WorkerOcService threadpool | 21 | WorkerOcService threadpool使用信息，线程数限制最大128；格式为：idleNum/currentTotalNum/MaxThreadNum/waitingTaskNum/rate
1) idelNum 空闲线程数；
2) currentTotalNum 当前正在运行任务的线程数；
3) MaxThreadNum threadpool最大可申请的线程数；
4) waitingTaskNum 正在等待的任务数。
5) rate 线程利用率，currentTotalNum/ MaxThreadNum,单位：%，保留3位有效数字。| +| WorkerWorkerOcService threadpool | 21 | threadpool使用信息 | +| MasterWokrerOcService threadpool | 21 | threadpool使用信息 | | MasterOcService threadpool | 21 | threadpool使用信息 | -| write ETCD queue | 15 | 队列使用信息 | -| ETCDrequest success rate | 6 | 请求使用率，单位 %,保留3位有效数字 | -| OBSrequest success rate | 6 | 请求使用率，单位 %,保留3位有效数字 | -| Master AsyncTask threadpool | 21 | threadpool使用信息，格式为：idleNum/currentTotalNum/MaxThreadNum/waitingTaskNum/rate | +| write ETCD queue | 15 | 队列使用信息 | +| ETCDrequest success rate | 6 | 请求使用率，单位 %,保留3位有效数字 | +| OBSrequest success rate | 6 | 请求使用率，单位 %,保留3位有效数字 | +| Master AsyncTask threadpool | 21 | threadpool使用信息，格式为：idleNum/currentTotalNum/MaxThreadNum/waitingTaskNum/rate | #### SDK与worker访问日志关键请求参数 diff --git a/docs/source_zh_cn/development-guide/hetero.md b/docs/source_zh_cn/development-guide/hetero.md index 01398cb5b13a3c5133e7b00aa111aaddcede0b9c..7c58cf25baaffe5a1473fcf6f54ff00faceb8812 100644 --- a/docs/source_zh_cn/development-guide/hetero.md +++ b/docs/source_zh_cn/development-guide/hetero.md @@ -423,7 +423,7 @@ def hetero_mset_d2h_mget_h2d(): ```cpp -#include "datasystem/hetero_cache/hetero_client.h" +#include "datasystem/hetero_client.h" #include ConnectOptions connectOptions = { .host = "127.0.0.1", .port = 31501 }; diff --git a/example/src/kv_cache/kv_example.cpp b/example/src/kv_cache/kv_example.cpp index 9b2b7686d616e5dc1e5c6689121d69129680f25c..22e6deb30401b6e158cb2ad994b3e847fa1904be 100644 --- a/example/src/kv_cache/kv_example.cpp +++ b/example/src/kv_cache/kv_example.cpp @@ -20,7 +20,7 @@ #include -#include "datasystem/kv_cache.h" +#include "datasystem/kv_client.h" using datasystem::ConnectOptions; using datasystem::Context; diff --git a/example/src/object_cache/object_example.cpp b/example/src/object_cache/object_example.cpp index e81e438a81756d11b65c3b48a07458800c0d07f7..191481aa6d1cb42c7e3e5c57c675e6c5971d20a1 100644 --- a/example/src/object_cache/object_example.cpp +++ b/example/src/object_cache/object_example.cpp @@ -20,7 +20,7 @@ #include -#include "datasystem/object_cache.h" +#include "datasystem/object_client.h" using datasystem::Buffer; using datasystem::ConnectOptions; diff --git a/include/datasystem/datasystem.h b/include/datasystem/datasystem.h index 141df4061d59ba04ada6b0dee191bca920c4faa0..63a8b17516ebdbdc2b4ac1d40b7370c2a75b6d19 100644 --- a/include/datasystem/datasystem.h +++ b/include/datasystem/datasystem.h @@ -23,9 +23,9 @@ #include -#include "datasystem/hetero_cache/hetero_client.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/hetero_client.h" +#include "datasystem/object_client.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/status.h" namespace datasystem { diff --git a/include/datasystem/hetero_cache/device_common.h b/include/datasystem/hetero/device_common.h similarity index 92% rename from include/datasystem/hetero_cache/device_common.h rename to include/datasystem/hetero/device_common.h index c00b3edb6189695182d5b1258ac7260ff4a35701..147c530a1eacac44f58381c87f525617e3fe35bd 100644 --- a/include/datasystem/hetero_cache/device_common.h +++ b/include/datasystem/hetero/device_common.h @@ -19,8 +19,8 @@ * Description: Definition of heterogeneous device tensor description structure. */ -#ifndef DATASYSTEM_HETERO_CACHE_DEVICE_COMMON_H -#define DATASYSTEM_HETERO_CACHE_DEVICE_COMMON_H +#ifndef DATASYSTEM_HETERO_DEVICE_COMMON_H +#define DATASYSTEM_HETERO_DEVICE_COMMON_H #include #include diff --git a/include/datasystem/hetero_cache/future.h b/include/datasystem/hetero/future.h similarity index 95% rename from include/datasystem/hetero_cache/future.h rename to include/datasystem/hetero/future.h index a268e0abeeb6d4c6c8b7c7ff6d37c45156c56b37..3973616100f90cdf633928856af702039954d5ea 100644 --- a/include/datasystem/hetero_cache/future.h +++ b/include/datasystem/hetero/future.h @@ -19,8 +19,8 @@ * Description: Data system Future class implementation. */ -#ifndef DATASYSTEM_HETERO_CACHE_FUTURE_H -#define DATASYSTEM_HETERO_CACHE_FUTURE_H +#ifndef DATASYSTEM_HETERO_FUTURE_H +#define DATASYSTEM_HETERO_FUTURE_H #include #include diff --git a/include/datasystem/hetero_cache/hetero_client.h b/include/datasystem/hetero_client.h similarity index 98% rename from include/datasystem/hetero_cache/hetero_client.h rename to include/datasystem/hetero_client.h index ec626fe4a15b836fb40a85c8fdd661b0c4413d00..a169b18a4a8377a08c134275a45f2e2d03d47a4b 100644 --- a/include/datasystem/hetero_cache/hetero_client.h +++ b/include/datasystem/hetero_client.h @@ -18,15 +18,15 @@ * Description: Data system Hetero cache client management. */ -#ifndef DATASYSTEM_HETERO_CACHE_HETERO_CLIENT_H -#define DATASYSTEM_HETERO_CACHE_HETERO_CLIENT_H +#ifndef DATASYSTEM_HETERO_CLIENT_H +#define DATASYSTEM_HETERO_CLIENT_H #include #include -#include "datasystem/hetero_cache/device_common.h" -#include "datasystem/hetero_cache/future.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/hetero/device_common.h" +#include "datasystem/hetero/future.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/connection.h" #include "datasystem/utils/status.h" diff --git a/include/datasystem/kv_cache/read_only_buffer.h b/include/datasystem/kv/read_only_buffer.h similarity index 93% rename from include/datasystem/kv_cache/read_only_buffer.h rename to include/datasystem/kv/read_only_buffer.h index 4979f451108ddcdda224ea6215fd012701bbed4e..4e65d7176e2db1c014b565f4a0dbf21f5920f923 100644 --- a/include/datasystem/kv_cache/read_only_buffer.h +++ b/include/datasystem/kv/read_only_buffer.h @@ -18,13 +18,13 @@ * Description: This file is used to read data in the server. */ -#ifndef DATASYSTEM_KV_CACHE_READ_ONLY_BUFFER_H -#define DATASYSTEM_KV_CACHE_READ_ONLY_BUFFER_H +#ifndef DATASYSTEM_KV_READ_ONLY_BUFFER_H +#define DATASYSTEM_KV_READ_ONLY_BUFFER_H #include #include -#include "datasystem/object_cache/buffer.h" +#include "datasystem/object/buffer.h" #include "datasystem/utils/status.h" namespace datasystem { diff --git a/include/datasystem/kv_cache.h b/include/datasystem/kv_cache.h deleted file mode 100644 index 8b2a4d95345c0a9f36af81c120436263bd657e30..0000000000000000000000000000000000000000 --- a/include/datasystem/kv_cache.h +++ /dev/null @@ -1,25 +0,0 @@ -/** - * Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef DATASYSTEM_KV_CACHE_H -#define DATASYSTEM_KV_CACHE_H - -#include "datasystem/context/context.h" -#include "datasystem/kv_cache/read_only_buffer.h" -#include "datasystem/kv_cache/kv_client.h" -#include "datasystem/utils/status.h" - -#endif // DATASYSTEM_KV_CACHE_H \ No newline at end of file diff --git a/include/datasystem/kv_cache/kv_client.h b/include/datasystem/kv_client.h similarity index 98% rename from include/datasystem/kv_cache/kv_client.h rename to include/datasystem/kv_client.h index cb7896715d6d0ba6425c576660238d7c4b9d353a..b1fd14e9e1ac07765a085b9e53ff242359de3662 100644 --- a/include/datasystem/kv_cache/kv_client.h +++ b/include/datasystem/kv_client.h @@ -17,16 +17,17 @@ /** * Description: Data system state cache client management. */ -#ifndef DATASYSTEM_KV_CACHE_KV_CLIENT_H -#define DATASYSTEM_KV_CACHE_KV_CLIENT_H +#ifndef DATASYSTEM_KV_CLIENT_H +#define DATASYSTEM_KV_CLIENT_H #include #include #include -#include "datasystem/object_cache/buffer.h" -#include "datasystem/object_cache/object_enum.h" -#include "datasystem/kv_cache/read_only_buffer.h" +#include "datasystem/context/context.h" +#include "datasystem/object/buffer.h" +#include "datasystem/object/object_enum.h" +#include "datasystem/kv/read_only_buffer.h" #include "datasystem/utils/connection.h" #include "datasystem/utils/optional.h" #include "datasystem/utils/status.h" diff --git a/include/datasystem/object_cache/buffer.h b/include/datasystem/object/buffer.h similarity index 98% rename from include/datasystem/object_cache/buffer.h rename to include/datasystem/object/buffer.h index cee00c66999a3e8f37db75efade8bdf4b90edca5..951b52763f1b1d5a025b2c24dfb29d2cb2558aab 100644 --- a/include/datasystem/object_cache/buffer.h +++ b/include/datasystem/object/buffer.h @@ -18,8 +18,8 @@ * Description: This file is used to read and write data and publish data to the server. */ -#ifndef DATASYSTEM_OBJECT_CACHE_BUFFER_H -#define DATASYSTEM_OBJECT_CACHE_BUFFER_H +#ifndef DATASYSTEM_OBJECT_BUFFER_H +#define DATASYSTEM_OBJECT_BUFFER_H #include #include diff --git a/include/datasystem/object_cache/object_enum.h b/include/datasystem/object/object_enum.h similarity index 100% rename from include/datasystem/object_cache/object_enum.h rename to include/datasystem/object/object_enum.h diff --git a/include/datasystem/object_cache.h b/include/datasystem/object_cache.h deleted file mode 100644 index 523096dca04f4713951d14fce4c8d7c650db3963..0000000000000000000000000000000000000000 --- a/include/datasystem/object_cache.h +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef DATASYSTEM_OBJECT_CACHE_H -#define DATASYSTEM_OBJECT_CACHE_H - -#include "datasystem/context/context.h" -#include "datasystem/object_cache/buffer.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/object_cache/object_enum.h" -#include "datasystem/utils/connection.h" -#include "datasystem/utils/optional.h" -#include "datasystem/utils/status.h" -#include "datasystem/utils/string_view.h" - -#endif // DATASYSTEM_OBJECT_CACHE_H \ No newline at end of file diff --git a/include/datasystem/object_cache/object_client.h b/include/datasystem/object_client.h similarity index 97% rename from include/datasystem/object_cache/object_client.h rename to include/datasystem/object_client.h index ee186a107da4a8ba3b39fe0b156d1a4d8de3e77c..b15cff24c413ddff815ebd890d9f196d0dd92688 100644 --- a/include/datasystem/object_cache/object_client.h +++ b/include/datasystem/object_client.h @@ -18,8 +18,8 @@ * Description: Data system object cache client management. */ -#ifndef DATASYSTEM_OBJECT_CACHE_OBJECT_CLIENT_H -#define DATASYSTEM_OBJECT_CACHE_OBJECT_CLIENT_H +#ifndef DATASYSTEM_OBJECT_CLIENT_H +#define DATASYSTEM_OBJECT_CLIENT_H #include #include @@ -29,11 +29,13 @@ #include #include -#include "datasystem/object_cache/buffer.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/context/context.h" +#include "datasystem/object/buffer.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/connection.h" #include "datasystem/utils/optional.h" #include "datasystem/utils/status.h" +#include "datasystem/utils/string_view.h" namespace datasystem { namespace object_cache { diff --git a/include/datasystem/utils/connection.h b/include/datasystem/utils/connection.h index 4fb460de3bd5414e5dc9369cda2b80cc7452a751..03352c2583258a2376d2451f4c88f4c3791a510b 100644 --- a/include/datasystem/utils/connection.h +++ b/include/datasystem/utils/connection.h @@ -20,7 +20,7 @@ #include "datasystem/utils/sensitive_value.h" #include "datasystem/utils/string_view.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" namespace datasystem { struct ConnectOptions { diff --git a/k8s/helm_chart/datasystem/values.yaml b/k8s/helm_chart/datasystem/values.yaml index ac6c2f18c8bf2e48c0e9c99fffe516b7901cc5b3..0da7197edf643bc6634fd897d55187a48e8e328a 100644 --- a/k8s/helm_chart/datasystem/values.yaml +++ b/k8s/helm_chart/datasystem/values.yaml @@ -7,7 +7,7 @@ global: imageRegistry: "" images: # Image name and tag in 'name:tag' format - datasystem: "openyuanrong_datasystem:0.5.0" + datasystem: "yr_datasystem:v0.1" # Config ETCD table prefix, the value should only contain english alphabetics (a-zA-Z), numbers (0-9) only. azName: "AZ1" diff --git a/scripts/build_thirdparty.sh b/scripts/build_thirdparty.sh index a557cb5804f993890c25e84de7e427c6caf319db..465b747113462506a2d8feb8cd40b4c477891f5d 100644 --- a/scripts/build_thirdparty.sh +++ b/scripts/build_thirdparty.sh @@ -80,33 +80,109 @@ MULTI_BUILD_PATH="$(realpath "${BUILD_PATH}/multi_build")" BUILD_THREAD_NUM=$2 CMAKE_OPTIONS=("${@:3}") DEPENDENCY_DIR="${MULTI_BUILD_PATH}"/dependency +TIME_DIR="${MULTI_BUILD_PATH}"/timing +PROGRESS_DIR="${MULTI_BUILD_PATH}"/progress -on_exit() { - local exit_status=$? - if [ $exit_status -eq 0 ]; then - echo -e "---- Compilie ${cmake_file%%.*} successfully ----" - else - echo -e "---- Compilie ${cmake_file%%.*} failed, see ${tmp_compile_dir}/log for more details ----" - fi +# total libs number +TOTAL_LIBS=$((${#CMAKE_FILES_L0[@]} + ${#CMAKE_FILES_L1[@]})) + +function log() { + local progress=$(get_progress_percent) + printf "[%3d%%] %s\n" "$progress" "$1" +} + +function get_progress_percent() { + local completed_count=$(ls -1 "${PROGRESS_DIR}" 2>/dev/null | wc -l) + local percent=$((completed_count * 100 / TOTAL_LIBS)) + echo "$percent" +} + +function on_exit() { + local exit_status=$? + local lib_name="${cmake_file%%.*}" + local end_time=$(date +%s) + local total_duration=$((end_time - overall_start_time)) + + if [ $exit_status -eq 0 ]; then + echo "$total_duration" > "${TIME_DIR}/${lib_name}.time" + touch "${PROGRESS_DIR}/${lib_name}.complete" + log "$lib_name build success - ${total_duration}s" + else + log "$lib_name build failed!!! see ${tmp_compile_dir}/log for details" + fi +} + +function wait_for_dependencies() { + local lib_name=$1 + local dependencies=$2 + local lib_dependencies=() + + if [ -n "$dependencies" ]; then + lib_dependencies_str=$(echo "$dependencies" | tr ':' ' ') + IFS=' ' read -r -a lib_dependencies <<< "${lib_dependencies_str}" + + log "$lib_name is still waiting for: ${lib_dependencies_str}" + + local timeout=1800 + local wait_start_time=$(date +%s) + + for ((i=1; i<=timeout; i++)); do + all_files_created=true + missing_deps=() + for file in "${lib_dependencies[@]}"; do + if [ ! -f "${DEPENDENCY_DIR}/${file}.complete" ]; then + all_files_created=false + missing_deps+=("$file") + fi + done + + if [ "$all_files_created" = true ]; then + local current_time=$(date +%s) + local wait_duration=$((current_time - wait_start_time)) + log "$lib_name all dependencies ready (waited ${wait_duration}s)" + return 0 + fi + + # 每30秒显示一次等待状态 + if [ $((i % 30)) -eq 0 ]; then + local current_time=$(date +%s) + local elapsed=$((current_time - wait_start_time)) + log "$lib_name still waiting for: ${missing_deps[*]} (${elapsed}s elapsed)" + fi + sleep 1 + done + + log_failed "$lib_name timeout waiting for dependencies" + return 1 + fi + return 0 } function main() { - echo -e "---- Start compiling third-party libraries in parallel ----" + log "Start compiling thirdparty libraries in parallel" local start_time_s local end_time_s local sumTime start_time_s=$(date +%s) mkdir -p ${DEPENDENCY_DIR} + mkdir -p ${TIME_DIR} + mkdir -p ${PROGRESS_DIR} + + log "Total libraries to compile: $TOTAL_LIBS" + # 编译 L0 级别的库（无依赖） + log "Building independent libraries (L0)" for cmake_file in "${CMAKE_FILES_L0[@]}"; do tmp_compile_dir="${MULTI_BUILD_PATH}/${cmake_file%%.*}" mkdir -p "${tmp_compile_dir}" ( - echo -e "---- Start compiling ${cmake_file%%.*} ----" - trap on_exit EXIT - cd "${tmp_compile_dir}" - cat > "CMakeLists.txt" << EOF + lib_name="${cmake_file%%.*}" + overall_start_time=$(date +%s) + log "Building $lib_name..." + trap on_exit EXIT + cd "${tmp_compile_dir}" + cat > "CMakeLists.txt" << EOF cmake_minimum_required(VERSION 3.14.1) set(CMAKE_SOURCE_DIR ${DATASYSTEM_HOME}) set(CMAKE_BINARY_DIR ${BUILD_PATH}) @@ -115,55 +191,43 @@ include(${DATASYSTEM_HOME}/cmake/options.cmake) include(${DATASYSTEM_HOME}/cmake/util.cmake) include(${DATASYSTEM_HOME}/cmake/external_libs/${cmake_file}) EOF - mkdir -p build && cd build - cmake "${CMAKE_OPTIONS[@]}" .. >> "${tmp_compile_dir}/log" 2>&1 - cmake --build . >> "${tmp_compile_dir}/log" 2>&1 - touch ${DEPENDENCY_DIR}/${cmake_file%%.*}.complete + mkdir -p build && cd build + cmake "${CMAKE_OPTIONS[@]}" .. >> "${tmp_compile_dir}/log" 2>&1 + cmake --build . >> "${tmp_compile_dir}/log" 2>&1 + touch ${DEPENDENCY_DIR}/${lib_name}.complete ) & done touch ${MULTI_BUILD_PATH}/L1_ENV + + # 编译 L1 级别的库（有依赖） + log "Building dependent libraries (L1)" for cmake_file in "${CMAKE_FILES_L1[@]}"; do tmp_compile_dir="${MULTI_BUILD_PATH}/${cmake_file%%.*}" mkdir -p "${tmp_compile_dir}" - for dep in "${DEPENDENCIES[@]}"; - do - local lib_dependencies=() - dependencies=$(echo "${dep}" | grep "${cmake_file%%.*}:" | cut -d':' -f2-) - if [ -z "${dependencies}" ]; then - continue - else - break - fi - done + + # 查找依赖关系 + dependencies="" + for dep in "${DEPENDENCIES[@]}"; do + if [[ "$dep" == "${cmake_file%%.*}:"* ]]; then + dependencies=$(echo "${dep}" | cut -d':' -f2-) + break + fi + done + ( - lib_dependencies_str=$(echo ${dependencies} | tr ':' ' ') - IFS=' ' read -r -a lib_dependencies <<< "${lib_dependencies_str}" - timeout=1800 - - echo -e "---- Prepare compiling ${cmake_file%%.*} ----" - for ((i=1; i<=timeout; i++)); do - all_files_created=true - for file in "${lib_dependencies[@]}"; do - if [ ! -f "${DEPENDENCY_DIR}/${file}.complete" ]; then - echo "Waiting for file $file to be created... (Attempt $i of $timeout)" - all_files_created=false - break - fi - done - - if [ "$all_files_created" = true ]; then - echo "All files have been created." - break - fi + lib_name="${cmake_file%%.*}" + overall_start_time=$(date +%s) + # 等待依赖项完成 + if ! wait_for_dependencies "$lib_name" "$dependencies"; then + exit 1 + fi - sleep 1 # 每隔1秒检查一次 - done - echo -e "---- Start compiling ${cmake_file%%.*} ----" - trap on_exit EXIT - cd "${tmp_compile_dir}" - cat > "CMakeLists.txt" << EOF + log "Building $lib_name..." + trap on_exit EXIT + cd "${tmp_compile_dir}" + cat > "CMakeLists.txt" << EOF cmake_minimum_required(VERSION 3.14.1) set(CMAKE_SOURCE_DIR ${DATASYSTEM_HOME}) set(CMAKE_BINARY_DIR ${BUILD_PATH}) @@ -177,15 +241,16 @@ EOF mkdir -p build && cd build cmake "${CMAKE_OPTIONS[@]}" .. >> "${tmp_compile_dir}/log" 2>&1 cmake --build . -j "${BUILD_THREAD_NUM}" >> "${tmp_compile_dir}/log" 2>&1 - touch ${DEPENDENCY_DIR}/${cmake_file%%.*}.complete + touch ${DEPENDENCY_DIR}/${lib_name}.complete ) & done wait end_time_s=$(date +%s) sumTime=$(($end_time_s - $start_time_s)) - echo -e "---- Finish compiling third-party libraries in parallel. Total use time:$sumTime seconds ----" + + log "Compile thirdparty libraries success, total wall time: ${sumTime}s" exit 0 } -main "$@" +main "$@" \ No newline at end of file diff --git a/src/datasystem/client/hetero_cache/device_buffer.h b/src/datasystem/client/hetero_cache/device_buffer.h index a880cc1927569751e547b5649492ffa42e0b176e..f67b0f22f6594015cd2903644924332bdae1fb2b 100644 --- a/src/datasystem/client/hetero_cache/device_buffer.h +++ b/src/datasystem/client/hetero_cache/device_buffer.h @@ -31,8 +31,8 @@ #include #include "datasystem/client/hetero_cache/device_util.h" -#include "datasystem/hetero_cache/future.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/hetero/future.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/status.h" namespace datasystem { diff --git a/src/datasystem/client/hetero_cache/device_util.h b/src/datasystem/client/hetero_cache/device_util.h index a0b9b8b6925aaf3f5e1600882378aa19589fb499..46bce9a19e82ebaa22f7a963c2d4e860e06b8b40 100644 --- a/src/datasystem/client/hetero_cache/device_util.h +++ b/src/datasystem/client/hetero_cache/device_util.h @@ -24,7 +24,7 @@ #include #include "datasystem/common/constants.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" namespace datasystem { /** diff --git a/src/datasystem/client/hetero_cache/hetero_client.cpp b/src/datasystem/client/hetero_cache/hetero_client.cpp index 8fa5f0b9dcba5a553afdb111f0ffe376656fcafb..1c3b85ff08f8fae64058b13f31c68ffca6a76b68 100644 --- a/src/datasystem/client/hetero_cache/hetero_client.cpp +++ b/src/datasystem/client/hetero_cache/hetero_client.cpp @@ -18,7 +18,7 @@ * Description: Data system Object Cache Client management. */ -#include "datasystem/hetero_cache/hetero_client.h" +#include "datasystem/hetero_client.h" #include "datasystem/client/hetero_cache/device_util.h" #include "datasystem/client/object_cache/object_client_impl.h" diff --git a/src/datasystem/client/kv_cache/kv_client.cpp b/src/datasystem/client/kv_cache/kv_client.cpp index 0cd1f5788d67684dc14ce7596180379495965da4..690d7527a8a659e7121cb3739725a7b501a5b113 100644 --- a/src/datasystem/client/kv_cache/kv_client.cpp +++ b/src/datasystem/client/kv_cache/kv_client.cpp @@ -17,7 +17,7 @@ /** * Description: Data system State Cache Client management. */ -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include #include @@ -34,7 +34,7 @@ #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/uuid_generator.h" -#include "datasystem/kv_cache/read_only_buffer.h" +#include "datasystem/kv/read_only_buffer.h" #include "datasystem/utils/status.h" namespace datasystem { diff --git a/src/datasystem/client/kv_cache/read_only_buffer.cpp b/src/datasystem/client/kv_cache/read_only_buffer.cpp index c6d0979b3dc06e5caea4b7419917b08ad80d706d..b373cefb75012464a3b4c939f07b1be8ded6b950 100644 --- a/src/datasystem/client/kv_cache/read_only_buffer.cpp +++ b/src/datasystem/client/kv_cache/read_only_buffer.cpp @@ -20,7 +20,7 @@ #include "datasystem/common/util/status_helper.h" #include "datasystem/common/log/log.h" -#include "datasystem/kv_cache/read_only_buffer.h" +#include "datasystem/kv/read_only_buffer.h" namespace datasystem { int64_t ReadOnlyBuffer::GetSize() const diff --git a/src/datasystem/client/object_cache/client_worker_api.h b/src/datasystem/client/object_cache/client_worker_api.h index 9e19d3dd2e043e623f0fbd4feec79e5a83c83cfb..58c6f26595a9c1d9a2b59a886112471f77c7f593 100644 --- a/src/datasystem/client/object_cache/client_worker_api.h +++ b/src/datasystem/client/object_cache/client_worker_api.h @@ -36,12 +36,12 @@ #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/uuid_generator.h" #include "datasystem/common/util/status_helper.h" -#include "datasystem/object_cache/buffer.h" +#include "datasystem/object/buffer.h" #include "datasystem/client/hetero_cache/device_util.h" #include "datasystem/protos/master_object.pb.h" #include "datasystem/protos/object_posix.service.rpc.pb.h" #include "datasystem/protos/object_posix.stub.rpc.pb.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include "datasystem/protos/p2p_subscribe.pb.h" #include "datasystem/utils/sensitive_value.h" #include "datasystem/utils/status.h" diff --git a/src/datasystem/client/object_cache/device/client_device_object_manager.cpp b/src/datasystem/client/object_cache/device/client_device_object_manager.cpp index d7be67f02707a548028daf437e0a2a25241b2073..919fc11d3b4c9fbe802c1042f82c5782a2e280ee 100644 --- a/src/datasystem/client/object_cache/device/client_device_object_manager.cpp +++ b/src/datasystem/client/object_cache/device/client_device_object_manager.cpp @@ -36,7 +36,7 @@ #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/thread_pool.h" #include "datasystem/client/hetero_cache/device_buffer.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/status.h" namespace datasystem { diff --git a/src/datasystem/client/object_cache/device/client_device_object_manager.h b/src/datasystem/client/object_cache/device/client_device_object_manager.h index 8b0b62333ef12033c6a51612ae8e0be4d917426c..ed4f210ed4dcc212ad734aaaf652f9901a840cf0 100644 --- a/src/datasystem/client/object_cache/device/client_device_object_manager.h +++ b/src/datasystem/client/object_cache/device/client_device_object_manager.h @@ -31,7 +31,7 @@ #include "datasystem/client/object_cache/device/device_memory_unit.h" #include "datasystem/client/object_cache/device/p2p_subscribe.h" #include "datasystem/common/device/ascend/acl_device_manager.h" -#include "datasystem/object_cache/object_client.h" +#include "datasystem/object_client.h" #include "datasystem/utils/status.h" namespace datasystem { diff --git a/src/datasystem/client/object_cache/device/future.cpp b/src/datasystem/client/object_cache/device/future.cpp index 576643eb6e631637054c39532cd599652c5efd1b..de4491ddc32acaad52f4b4b252b013c64af2d25b 100644 --- a/src/datasystem/client/object_cache/device/future.cpp +++ b/src/datasystem/client/object_cache/device/future.cpp @@ -15,7 +15,7 @@ * limitations under the License. */ -#include "datasystem/hetero_cache/future.h" +#include "datasystem/hetero/future.h" #include "datasystem/common/device/ascend/acl_pointer_wrapper.h" #include "datasystem/common/rpc/timeout_duration.h" #include "datasystem/common/util/format.h" diff --git a/src/datasystem/client/object_cache/device/p2p_subscribe.cpp b/src/datasystem/client/object_cache/device/p2p_subscribe.cpp index 87bfa2091d1d765cd9c3431d863d5419ffb82a83..bd087b4ea35e13b2720232e772fbfaee398aafcf 100644 --- a/src/datasystem/client/object_cache/device/p2p_subscribe.cpp +++ b/src/datasystem/client/object_cache/device/p2p_subscribe.cpp @@ -34,8 +34,8 @@ #include "datasystem/common/util/raii.h" #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/strings_util.h" -#include "datasystem/hetero_cache/future.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/hetero/future.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/p2p_subscribe.pb.h" #include "datasystem/utils/status.h" diff --git a/src/datasystem/client/object_cache/device/p2p_subscribe.h b/src/datasystem/client/object_cache/device/p2p_subscribe.h index fa0b865b973eb25a6bea61a95e7737abd8cd4901..b95d8650d01199b9173577fdc3833331a1ce7d3d 100644 --- a/src/datasystem/client/object_cache/device/p2p_subscribe.h +++ b/src/datasystem/client/object_cache/device/p2p_subscribe.h @@ -44,8 +44,8 @@ #include "datasystem/common/util/queue/blocking_queue.h" #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/thread_pool.h" -#include "datasystem/hetero_cache/future.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/hetero/future.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/status.h" namespace datasystem { diff --git a/src/datasystem/client/object_cache/device/page_attn_utils.h b/src/datasystem/client/object_cache/device/page_attn_utils.h index 6684e2b7b65ee3f95a05af548dbc997005805664..15284a1cabcec42bdb329f450f661a0e83e175f4 100644 --- a/src/datasystem/client/object_cache/device/page_attn_utils.h +++ b/src/datasystem/client/object_cache/device/page_attn_utils.h @@ -22,7 +22,7 @@ #include -#include "datasystem/hetero_cache/device_common.h" +#include "datasystem/hetero/device_common.h" #include "datasystem/utils/status.h" namespace datasystem { diff --git a/src/datasystem/client/object_cache/object_client.cpp b/src/datasystem/client/object_cache/object_client.cpp index 0689461901e408f69498b7a7443b2a4fe2bdb9a1..e5b6d86e1593f70abe643f059159e87b2a53f20a 100644 --- a/src/datasystem/client/object_cache/object_client.cpp +++ b/src/datasystem/client/object_cache/object_client.cpp @@ -18,7 +18,7 @@ * Description: Data system Object Cache Client management. */ -#include "datasystem/object_cache/object_client.h" +#include "datasystem/object_client.h" #include "datasystem/client/object_cache/object_client_impl.h" #include "datasystem/common/log/access_recorder.h" diff --git a/src/datasystem/client/object_cache/object_client_impl.cpp b/src/datasystem/client/object_cache/object_client_impl.cpp index 82a25fc23dccd52a175965f747bf3b7150cfbb4d..1e5efe420c85abf8184ede54fec0feaa8b1578f8 100644 --- a/src/datasystem/client/object_cache/object_client_impl.cpp +++ b/src/datasystem/client/object_cache/object_client_impl.cpp @@ -60,7 +60,7 @@ #include "datasystem/common/util/uri.h" #include "datasystem/common/util/strings_util.h" #include "datasystem/client/hetero_cache/device_buffer.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/object_posix.stub.rpc.pb.h" #include "datasystem/utils/sensitive_value.h" #include "datasystem/utils/status.h" diff --git a/src/datasystem/client/object_cache/object_client_impl.h b/src/datasystem/client/object_cache/object_client_impl.h index 8e46ed96733625a0d882a43d0efd2e0a8e04dcf7..b4ddeebc9dc0a95bf495a59f6abbbb670059b8bb 100644 --- a/src/datasystem/client/object_cache/object_client_impl.h +++ b/src/datasystem/client/object_cache/object_client_impl.h @@ -48,11 +48,11 @@ #include "datasystem/common/util/validator.h" #include "datasystem/common/util/wait_post.h" #include "datasystem/client/hetero_cache/device_buffer.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/hetero_cache/hetero_client.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object_client.h" +#include "datasystem/hetero_client.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/object_posix.pb.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include "datasystem/common/rpc/rpc_auth_keys.h" #include "datasystem/utils/sensitive_value.h" #include "datasystem/utils/string_view.h" diff --git a/src/datasystem/common/immutable_string/ref_count_string.h b/src/datasystem/common/immutable_string/ref_count_string.h index 75294a15a8f3cba7b703c15b4574b438739eb33c..484d0216e47a5017df4154e9d28bd2eb69f430a7 100644 --- a/src/datasystem/common/immutable_string/ref_count_string.h +++ b/src/datasystem/common/immutable_string/ref_count_string.h @@ -94,8 +94,8 @@ private: * (Multi threads may check countRef_ is 0 and try to call erase) * It can be seen that a value is effectively erasable only when the delRef_ count goes down to 0. */ - mutable std::atomic_int32_t delRef_; - mutable std::atomic_int32_t countRef_; + mutable std::atomic_int32_t delRef_{ 0 }; + mutable std::atomic_int32_t countRef_{ 0 }; std::string value_; size_t hash_; }; diff --git a/src/datasystem/common/object_cache/buffer.cpp b/src/datasystem/common/object_cache/buffer.cpp index 7f1f623fb1cf6ae12b0ebe934023a00ea69db3e4..cdc93978cae6f535d22bf736f3bb60d402b0af7c 100644 --- a/src/datasystem/common/object_cache/buffer.cpp +++ b/src/datasystem/common/object_cache/buffer.cpp @@ -18,7 +18,7 @@ * Description: This file is used to read and write data and publish data to the server. */ -#include "datasystem/object_cache/buffer.h" +#include "datasystem/object/buffer.h" #include diff --git a/src/datasystem/common/object_cache/buffer_composer.h b/src/datasystem/common/object_cache/buffer_composer.h index dfadb4127a1329d5eb86488a53e6d564ed1a546a..12e397f27f6b251d370faa0797cb65f333872402 100644 --- a/src/datasystem/common/object_cache/buffer_composer.h +++ b/src/datasystem/common/object_cache/buffer_composer.h @@ -24,7 +24,7 @@ #include #include -#include "datasystem/object_cache/buffer.h" +#include "datasystem/object/buffer.h" #include "datasystem/client/hetero_cache/device_util.h" namespace datasystem { diff --git a/src/datasystem/common/object_cache/object_base.h b/src/datasystem/common/object_cache/object_base.h index 087f00bf8467b37c2d17fe1c87e1d3dfc815f093..7f69fd8118f53fc159655928632245c9eb5f7d2b 100644 --- a/src/datasystem/common/object_cache/object_base.h +++ b/src/datasystem/common/object_cache/object_base.h @@ -30,8 +30,8 @@ #include "datasystem/common/object_cache/object_bitmap.h" #include "datasystem/common/rpc/rpc_message.h" #include "datasystem/common/shared_memory/shm_unit.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object_client.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/optional.h" #include "datasystem/utils/status.h" diff --git a/src/datasystem/common/object_cache/object_bitmap.h b/src/datasystem/common/object_cache/object_bitmap.h index 8bbc3bf3a11b571ee3d7ed3fa055689ca6e31573..bd07612311807240cd1a1260b02a1fd91505876a 100644 --- a/src/datasystem/common/object_cache/object_bitmap.h +++ b/src/datasystem/common/object_cache/object_bitmap.h @@ -22,7 +22,7 @@ #define DATASYSTEM_OBJECT_CACHE_OBJECT_INTERFACE_BITMAP_H #include -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/common/util/bitmask_enum.h" namespace datasystem { diff --git a/src/datasystem/master/metadata_redirect_helper.h b/src/datasystem/master/metadata_redirect_helper.h index 39ceb2bf4ba0b7b1197ec7d554f93966616da9f1..c4716172428617445410e640b494b09b47b673ac 100644 --- a/src/datasystem/master/metadata_redirect_helper.h +++ b/src/datasystem/master/metadata_redirect_helper.h @@ -150,6 +150,13 @@ protected: VLOG(1) << "receive redirect object: " << id; return; } + std::string localAddr; + if (etcdCM_->CheckIsLocalNodeIsUpdate(localAddr)) { + response.set_meta_is_moving(true); + RedirectMetaInfo *info = response.mutable_info(); + info->set_redirect_meta_address(localAddr); + return; + } CheckNeedToRedirectOrNot(id, redirect, newMetaAddr); INJECT_POINT("redirect.create.update.copy.meta", [&newMetaAddr, &redirect](std::string addr) { newMetaAddr = addr; @@ -184,6 +191,13 @@ protected: if (!redirect) { return; } + std::string localAddr; + if (etcdCM_->CheckIsLocalNodeIsUpdate(localAddr)) { + rsp.set_meta_is_moving(true); + RedirectMetaInfo *info = rsp.add_info(); + info->set_redirect_meta_address(localAddr); + return; + } std::unordered_map> redirectQueryObjKeys; GroupRedirctObjectByNewMetaAddr(ids, redirectQueryObjKeys); bool isMoving = false; diff --git a/src/datasystem/master/object_cache/device/master_dev_oc_directory.h b/src/datasystem/master/object_cache/device/master_dev_oc_directory.h index cfd29a2347f96370c9a224b2dcb55d94724fe383..2335a200d3c3609eb9a392d985c3a3893a4d503b 100644 --- a/src/datasystem/master/object_cache/device/master_dev_oc_directory.h +++ b/src/datasystem/master/object_cache/device/master_dev_oc_directory.h @@ -33,7 +33,7 @@ #include "datasystem/common/immutable_string/immutable_string.h" #include "datasystem/common/util/request_table.h" #include "datasystem/client/hetero_cache/device_util.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/master_object.pb.h" #include "datasystem/protos/p2p_subscribe.pb.h" #include "datasystem/utils/status.h" diff --git a/src/datasystem/master/object_cache/device/master_dev_oc_manager.cpp b/src/datasystem/master/object_cache/device/master_dev_oc_manager.cpp index 2137c3fb55aebd2124371cab46f4809e64f0d69a..990b7d3045b728debfc5437fb6c6055a62a34309 100644 --- a/src/datasystem/master/object_cache/device/master_dev_oc_manager.cpp +++ b/src/datasystem/master/object_cache/device/master_dev_oc_manager.cpp @@ -32,7 +32,7 @@ #include "datasystem/master/object_cache/device/master_dev_hccl_rootinfo.h" #include "datasystem/master/object_cache/device/master_dev_npu_events.h" #include "datasystem/master/object_cache/device/master_dev_oc_directory.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/master_object.pb.h" #include "datasystem/protos/p2p_subscribe.pb.h" #include "datasystem/utils/status.h" diff --git a/src/datasystem/master/object_cache/oc_metadata_manager.cpp b/src/datasystem/master/object_cache/oc_metadata_manager.cpp index 7fdb92e77ae1746a25ab973483746501b498c565..9efd1180edbb3ef0809ef97d0196d85d3373df45 100644 --- a/src/datasystem/master/object_cache/oc_metadata_manager.cpp +++ b/src/datasystem/master/object_cache/oc_metadata_manager.cpp @@ -59,7 +59,7 @@ #include "datasystem/master/object_cache/oc_notify_worker_manager.h" #include "datasystem/master/object_cache/store/meta_async_queue.h" #include "datasystem/master/object_cache/store/object_meta_store.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/master_object.pb.h" #include "datasystem/protos/object_posix.pb.h" #include "datasystem/protos/worker_object.pb.h" diff --git a/src/datasystem/master/object_cache/oc_metadata_manager.h b/src/datasystem/master/object_cache/oc_metadata_manager.h index 3f65093d171f568078ae4d733ac98ff8b985f7ab..b6ffbbcde804b75f95eaec17a33af99682077574 100644 --- a/src/datasystem/master/object_cache/oc_metadata_manager.h +++ b/src/datasystem/master/object_cache/oc_metadata_manager.h @@ -39,7 +39,7 @@ #include "datasystem/common/l2cache/persistence_api.h" #include "datasystem/common/rpc/rpc_server_stream_base.h" #include "datasystem/master/object_cache/device/master_dev_oc_manager.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/worker/hash_ring/hash_ring.h" #include "datasystem/common/eventloop/timer_queue.h" #include "datasystem/common/immutable_string/immutable_string.h" diff --git a/src/datasystem/pybind_api/pybind_register_hetero.cpp b/src/datasystem/pybind_api/pybind_register_hetero.cpp index 0ee5d5a7de260407543fa6e1aa4d447d8bd02ca7..453d4ab90b0118da3863b6c53295226577f6a633 100644 --- a/src/datasystem/pybind_api/pybind_register_hetero.cpp +++ b/src/datasystem/pybind_api/pybind_register_hetero.cpp @@ -29,10 +29,10 @@ #include "datasystem/common/log/log.h" #include "datasystem/common/log/trace.h" #include "datasystem/common/util/raii.h" -#include "datasystem/hetero_cache/hetero_client.h" -#include "datasystem/hetero_cache/future.h" -#include "datasystem/kv_cache/kv_client.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/hetero_client.h" +#include "datasystem/hetero/future.h" +#include "datasystem/kv_client.h" +#include "datasystem/object/object_enum.h" #include "datasystem/pybind_api/pybind_register.h" using datasystem::ConnectOptions; diff --git a/src/datasystem/pybind_api/pybind_register_kv.cpp b/src/datasystem/pybind_api/pybind_register_kv.cpp index 0e2d91f9537b1fcb7799888ba9399b58551e31cd..771bc5b5fd40ff6e7347b40eea30676b68621dc7 100644 --- a/src/datasystem/pybind_api/pybind_register_kv.cpp +++ b/src/datasystem/pybind_api/pybind_register_kv.cpp @@ -22,7 +22,7 @@ #include -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include "datasystem/client/object_cache/object_client_impl.h" #include "datasystem/common/log/log.h" #include "datasystem/common/log/access_recorder.h" diff --git a/src/datasystem/pybind_api/pybind_register_object.cpp b/src/datasystem/pybind_api/pybind_register_object.cpp index 4304df2c93a60a5f3b4f82aa8ec14175fdf37c5e..ec23c77bfbc0afce3a45ed8a16784dd67163726d 100644 --- a/src/datasystem/pybind_api/pybind_register_object.cpp +++ b/src/datasystem/pybind_api/pybind_register_object.cpp @@ -31,9 +31,9 @@ #include "datasystem/common/log/log.h" #include "datasystem/common/log/trace.h" #include "datasystem/common/util/format.h" -#include "datasystem/object_cache/buffer.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/buffer.h" +#include "datasystem/object_client.h" +#include "datasystem/object/object_enum.h" #include "datasystem/pybind_api/pybind_register.h" #include "datasystem/utils/status.h" diff --git a/src/datasystem/worker/cluster_manager/etcd_cluster_manager.cpp b/src/datasystem/worker/cluster_manager/etcd_cluster_manager.cpp index eadad6bff76a874eda152db1fb7914eec3c79655..53816df6ceda17e63f2f79eb877ac4371591084b 100644 --- a/src/datasystem/worker/cluster_manager/etcd_cluster_manager.cpp +++ b/src/datasystem/worker/cluster_manager/etcd_cluster_manager.cpp @@ -1343,6 +1343,12 @@ Status EtcdClusterManager::GetPrimaryReplicaDbNames(const HostPort &address, std return replicaManager_->GetPrimaryReplicaDbNames(workerUuid, dbNames); } +bool EtcdClusterManager::CheckIsLocalNodeIsUpdate(std::string &localAddr) +{ + localAddr = workerAddress_.ToString(); + return hashRing_->CheckIsLocalNodeIsUpdate(); +} + std::set EtcdClusterManager::GetNodesInTable() { std::set nodes; diff --git a/src/datasystem/worker/cluster_manager/etcd_cluster_manager.h b/src/datasystem/worker/cluster_manager/etcd_cluster_manager.h index fb485e32913e2cd0940dc38e6e5836b0109a01aa..1fb0f37e29bf085925638ad90088869c2e9177f9 100644 --- a/src/datasystem/worker/cluster_manager/etcd_cluster_manager.h +++ b/src/datasystem/worker/cluster_manager/etcd_cluster_manager.h @@ -205,6 +205,12 @@ public: */ Status GetNodeAddrListFromEtcd(std::vector &nodeAddrs); + /** + * @brief Check local node is update node. + * @return true if local node is update node. + */ + bool CheckIsLocalNodeIsUpdate(std::string &localAddr); + /** * @brief Returns all of the currently tracked nodes * @param[out] nodeAddrs HostPort address of all the nodes diff --git a/src/datasystem/worker/hash_ring/hash_ring.cpp b/src/datasystem/worker/hash_ring/hash_ring.cpp index bc7349b2a8581df8ec94b23ad0f57367f3bdbd74..fb05422058278acf5dce692dac81e550e35ee507 100644 --- a/src/datasystem/worker/hash_ring/hash_ring.cpp +++ b/src/datasystem/worker/hash_ring/hash_ring.cpp @@ -262,6 +262,8 @@ Status HashRing::InitRing(const std::string &oldValue, std::unique_ptrinsert({ workerAddr_, workerPb }); @@ -1160,6 +1162,7 @@ void HashRing::UpdateLocalState(bool forceUpdate) auto stateInEtcd = iter->second.state(); if (stateInEtcd == WorkerPb::ACTIVE) { + isUpdateNode_ = false; ChangeStateTo(RUNNING); } else if (stateInEtcd == WorkerPb::LEAVING) { ChangeStateTo(PRE_LEAVING); @@ -1790,6 +1793,23 @@ Status HashRing::GetRelatedWorkerImpl(const std::string &currWorkerUuid, (getNextNode ? "next" : "prev"), currWorkerUuid)); } +bool HashRing::CheckIsLocalNodeIsUpdate() +{ + if (!isUpdateNode_.load()) { + return false; + } + std::shared_lock lock(mutex_); + auto worker = ringInfo_.workers().find(workerAddr_); + if (worker != ringInfo_.workers().end() + && ringInfo_.update_worker_map().find(workerAddr_) != ringInfo_.update_worker_map().end()) { + if ((worker->second.state() == WorkerPb::INITIAL && worker->second.worker_uuid().empty()) + || worker->second.state() == WorkerPb::JOINING) { + return true; + } + } + return false; +} + Status HashRing::GetNextWorker(const std::string &currWorkerUuid, std::string &nextWorkerUuid) { std::string nextWorkerAddr; diff --git a/src/datasystem/worker/hash_ring/hash_ring.h b/src/datasystem/worker/hash_ring/hash_ring.h index 62e375ff95162058c7d21c92a7abc140bf15f2d8..9894839b9d53d2dc36a9e3bcef104691d4391c19 100644 --- a/src/datasystem/worker/hash_ring/hash_ring.h +++ b/src/datasystem/worker/hash_ring/hash_ring.h @@ -108,6 +108,12 @@ public: */ bool IsLeaving(const std::string &workerAddr); + /** + * @brief Check local node is update node. + * @return true if local node is update node. + */ + bool CheckIsLocalNodeIsUpdate(); + /** * @brief Return true if the hash ring is in RUNNING or PRE_RUNNING or PRE_LEAVING state. * @return Return true if the hash ring is in RUNNING or PRE_RUNNING or PRE_LEAVING state. @@ -771,7 +777,7 @@ protected: std::atomic allWorkersVoluntaryScaleDown_{ false }; std::atomic baselineModRevisionOfRing_{ 0 }; std::atomic needForceJoin_{ false }; - + std::atomic isUpdateNode_{ false }; mutable std::shared_timed_mutex mutex_; // protect the following variables HashRingPb ringInfo_; using WorkerAddr = std::string; diff --git a/src/datasystem/worker/object_cache/migrate_data_handler.cpp b/src/datasystem/worker/object_cache/migrate_data_handler.cpp index 9e06f5175372953260a51a04d4ab5574850d4457..d8476205b219a4960b1fa8e007278483b2755c9e 100644 --- a/src/datasystem/worker/object_cache/migrate_data_handler.cpp +++ b/src/datasystem/worker/object_cache/migrate_data_handler.cpp @@ -40,7 +40,7 @@ #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/timer.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/worker_object.pb.h" #include "datasystem/utils/connection.h" #include "datasystem/utils/status.h" diff --git a/src/datasystem/worker/object_cache/migrate_data_handler.h b/src/datasystem/worker/object_cache/migrate_data_handler.h index 2a8c8f7c18d58da1f7ff222b1e403ad8e73e61a9..3e8cfa310e94348d3ba1b1cd0e0cfa63d7104d8f 100644 --- a/src/datasystem/worker/object_cache/migrate_data_handler.h +++ b/src/datasystem/worker/object_cache/migrate_data_handler.h @@ -35,7 +35,7 @@ #include "datasystem/common/rpc/rpc_message.h" #include "datasystem/common/shared_memory/shm_unit.h" #include "datasystem/common/util/thread.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/worker_object.pb.h" #include "datasystem/worker/object_cache/object_kv.h" #include "datasystem/worker/object_cache/worker_worker_oc_api.h" diff --git a/src/datasystem/worker/object_cache/object_kv.cpp b/src/datasystem/worker/object_cache/object_kv.cpp index f5444b111848a9906daeefc744ae3763142d7fc9..1e986c0f5b587ae7e9bd23540ddf5035b0944188 100644 --- a/src/datasystem/worker/object_cache/object_kv.cpp +++ b/src/datasystem/worker/object_cache/object_kv.cpp @@ -19,7 +19,7 @@ */ #include "datasystem/worker/object_cache/object_kv.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/worker/object_cache/obj_cache_shm_unit.h" namespace datasystem { diff --git a/src/datasystem/worker/object_cache/service/worker_oc_service_create_impl.h b/src/datasystem/worker/object_cache/service/worker_oc_service_create_impl.h index c9ea52d2265199df449ea1ce1b516f8142caa1d1..e228f0a870124e62ff4255db02bd06033c37ba6e 100644 --- a/src/datasystem/worker/object_cache/service/worker_oc_service_create_impl.h +++ b/src/datasystem/worker/object_cache/service/worker_oc_service_create_impl.h @@ -21,7 +21,7 @@ #ifndef DATASYSTEM_OBJECT_CACHE_WORKER_SERVICE_CREATE_IMPL_H #define DATASYSTEM_OBJECT_CACHE_WORKER_SERVICE_CREATE_IMPL_H -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/status.h" #include "datasystem/worker/object_cache/service/worker_oc_service_crud_common_api.h" diff --git a/src/datasystem/worker/object_cache/service/worker_oc_service_get_impl.cpp b/src/datasystem/worker/object_cache/service/worker_oc_service_get_impl.cpp index 50ecb623f7e0478eaf34b233ebe61a8f273d9ba6..2ceb0aded44443f70c889dcc3e935ae8d6da177d 100644 --- a/src/datasystem/worker/object_cache/service/worker_oc_service_get_impl.cpp +++ b/src/datasystem/worker/object_cache/service/worker_oc_service_get_impl.cpp @@ -31,7 +31,7 @@ #include "datasystem/common/log/log.h" #include "datasystem/common/perf/perf_manager.h" #include "datasystem/master/object_cache/master_worker_oc_api.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/common/rdma/urma_manager_wrapper.h" #include "datasystem/common/util/format.h" #include "datasystem/common/util/gflag/common_gflags.h" diff --git a/src/datasystem/worker/object_cache/service/worker_oc_service_migrate_impl.cpp b/src/datasystem/worker/object_cache/service/worker_oc_service_migrate_impl.cpp index 3f32ee35fb8b4e4f2d452c4667177d014e20ef82..be21dc689896ba54ad36db4f041bde59fc6f8f0d 100644 --- a/src/datasystem/worker/object_cache/service/worker_oc_service_migrate_impl.cpp +++ b/src/datasystem/worker/object_cache/service/worker_oc_service_migrate_impl.cpp @@ -39,7 +39,7 @@ #include "datasystem/common/util/rpc_util.h" #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/strings_util.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/master_object.pb.h" #include "datasystem/protos/master_object.service.rpc.pb.h" #include "datasystem/utils/status.h" diff --git a/src/datasystem/worker/object_cache/service/worker_oc_service_migrate_impl.h b/src/datasystem/worker/object_cache/service/worker_oc_service_migrate_impl.h index e0977938524ed3e00cad9d1907dc98763d5b6d11..df336808fd8e28cc8e34ed80e968066509422f59 100644 --- a/src/datasystem/worker/object_cache/service/worker_oc_service_migrate_impl.h +++ b/src/datasystem/worker/object_cache/service/worker_oc_service_migrate_impl.h @@ -36,7 +36,7 @@ #include "datasystem/common/rpc/rpc_message.h" #include "datasystem/common/util/net_util.h" #include "datasystem/common/util/timer.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/master_object.pb.h" #include "datasystem/protos/object_posix.pb.h" #include "datasystem/protos/worker_object.pb.h" diff --git a/src/datasystem/worker/object_cache/service/worker_oc_service_multi_publish_impl.h b/src/datasystem/worker/object_cache/service/worker_oc_service_multi_publish_impl.h index 8cc1b1dfad564556c6b58ff168110992e9d3181a..9eb55c3df523e8b479ae2f18db7dbecdb1c5cfc4 100644 --- a/src/datasystem/worker/object_cache/service/worker_oc_service_multi_publish_impl.h +++ b/src/datasystem/worker/object_cache/service/worker_oc_service_multi_publish_impl.h @@ -23,7 +23,7 @@ #include "datasystem/common/ak_sk/ak_sk_manager.h" #include "datasystem/common/rpc/rpc_message.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/master_object.pb.h" #include "datasystem/protos/object_posix.pb.h" #include "datasystem/utils/status.h" diff --git a/src/datasystem/worker/object_cache/worker_oc_eviction_manager.cpp b/src/datasystem/worker/object_cache/worker_oc_eviction_manager.cpp index a0fbadb532c3fbb4137904699332b3d53ca18eb2..283c8eadc6f36297f556b0e37e6b6d8c533bdc87 100644 --- a/src/datasystem/worker/object_cache/worker_oc_eviction_manager.cpp +++ b/src/datasystem/worker/object_cache/worker_oc_eviction_manager.cpp @@ -35,7 +35,7 @@ #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/timer.h" #include "datasystem/common/util/validator.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/master_object.pb.h" #include "datasystem/utils/status.h" #include "datasystem/worker/object_cache/object_kv.h" diff --git a/src/datasystem/worker/object_cache/worker_oc_eviction_manager.h b/src/datasystem/worker/object_cache/worker_oc_eviction_manager.h index c354e1c640a81d9104d203e38c59dceb01e65830..3acac43848bb7d823988870c082155716a9707eb 100644 --- a/src/datasystem/worker/object_cache/worker_oc_eviction_manager.h +++ b/src/datasystem/worker/object_cache/worker_oc_eviction_manager.h @@ -36,7 +36,7 @@ #include "datasystem/common/object_cache/safe_table.h" #include "datasystem/common/util/thread_pool.h" #include "datasystem/common/util/timer.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/worker/object_cache/eviction_list.h" #include "datasystem/worker/object_cache/object_kv.h" #include "datasystem/worker/object_cache/worker_master_oc_api.h" diff --git a/src/datasystem/worker/object_cache/worker_request_manager.cpp b/src/datasystem/worker/object_cache/worker_request_manager.cpp index 9301dbe4f172da9f6abbf95a57ec676dc5cb1814..e80642e9bf4fdd58e2cbddd3e88c50c38dce32f8 100644 --- a/src/datasystem/worker/object_cache/worker_request_manager.cpp +++ b/src/datasystem/worker/object_cache/worker_request_manager.cpp @@ -30,7 +30,7 @@ #include "datasystem/common/util/raii.h" #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/thread_local.h" -#include "datasystem/object_cache/buffer.h" +#include "datasystem/object/buffer.h" #include "datasystem/utils/status.h" #include "datasystem/worker/client_manager/client_manager.h" #include "datasystem/worker/object_cache/obj_cache_shm_unit.h" diff --git a/src/datasystem/worker/worker_liveness_check.cpp b/src/datasystem/worker/worker_liveness_check.cpp index 6ad4cd3128e9f0f92e2b38402c92b0b5c9958da5..e91a549d354934f8b7b56695d88f69e3049d7c21 100644 --- a/src/datasystem/worker/worker_liveness_check.cpp +++ b/src/datasystem/worker/worker_liveness_check.cpp @@ -36,7 +36,7 @@ #include "datasystem/common/util/thread_local.h" #include "datasystem/common/util/timer.h" #include "datasystem/common/util/uuid_generator.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/object_posix.pb.h" #include "datasystem/protos/share_memory.pb.h" #include "datasystem/common/rpc/rpc_auth_keys.h" diff --git a/tests/st/client/kv_cache/kv_cache_client_evict_test.cpp b/tests/st/client/kv_cache/kv_cache_client_evict_test.cpp index 42edc1763cf203af09fe480ed2a83055d0aa351d..b6be1c3d101d644cd3f4a81e8db9ec50610003e4 100644 --- a/tests/st/client/kv_cache/kv_cache_client_evict_test.cpp +++ b/tests/st/client/kv_cache/kv_cache_client_evict_test.cpp @@ -18,7 +18,7 @@ * Description: State client evict tests. */ -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include #include @@ -41,7 +41,7 @@ #include "datasystem/common/util/random_data.h" #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/timer.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/connection.h" #include "datasystem/utils/status.h" #include "datasystem/worker/hash_ring/hash_ring.h" diff --git a/tests/st/client/kv_cache/kv_cache_client_exist_test.cpp b/tests/st/client/kv_cache/kv_cache_client_exist_test.cpp index 68cf057aac8b03287b78823fbd2953419664ed34..0814e25cb130879896a0e8017dd3ff061e866cc1 100644 --- a/tests/st/client/kv_cache/kv_cache_client_exist_test.cpp +++ b/tests/st/client/kv_cache/kv_cache_client_exist_test.cpp @@ -29,8 +29,8 @@ #include "client/object_cache/oc_client_common.h" #include "common.h" -#include "datasystem/kv_cache/read_only_buffer.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv/read_only_buffer.h" +#include "datasystem/kv_client.h" #include "datasystem/common/flags/flags.h" #include "datasystem/common/log/log.h" diff --git a/tests/st/client/kv_cache/kv_cache_client_expire_test.cpp b/tests/st/client/kv_cache/kv_cache_client_expire_test.cpp index de710ec5c076b9daa2a2a60f014021f237226efb..27abc0b1abe194a7296be79cbd6fc33f447f05b5 100644 --- a/tests/st/client/kv_cache/kv_cache_client_expire_test.cpp +++ b/tests/st/client/kv_cache/kv_cache_client_expire_test.cpp @@ -29,8 +29,8 @@ #include "client/object_cache/oc_client_common.h" #include "common.h" -#include "datasystem/kv_cache/read_only_buffer.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv/read_only_buffer.h" +#include "datasystem/kv_client.h" #include "datasystem/common/flags/flags.h" #include "datasystem/common/log/log.h" diff --git a/tests/st/client/kv_cache/kv_cache_client_test.cpp b/tests/st/client/kv_cache/kv_cache_client_test.cpp index 560661a8ecd35c91474ac22e32d95e6eecf955fc..8e52ea44584f1563450d51e239200d3ecadde638 100644 --- a/tests/st/client/kv_cache/kv_cache_client_test.cpp +++ b/tests/st/client/kv_cache/kv_cache_client_test.cpp @@ -42,8 +42,8 @@ #include "datasystem/common/util/net_util.h" #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/thread_pool.h" -#include "datasystem/kv_cache/read_only_buffer.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv/read_only_buffer.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/connection.h" #include "datasystem/utils/status.h" #include "datasystem/common/flags/flags.h" diff --git a/tests/st/client/kv_cache/kv_client_cross_az_test.cpp b/tests/st/client/kv_cache/kv_client_cross_az_test.cpp index e57003bd5cfaef9d437a2ff2a2b2c5473fce8598..c1f1d047f87ff3ac13347e21e42ed5b22e5fbb35 100644 --- a/tests/st/client/kv_cache/kv_client_cross_az_test.cpp +++ b/tests/st/client/kv_cache/kv_client_cross_az_test.cpp @@ -44,8 +44,8 @@ #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/thread_pool.h" -#include "datasystem/object_cache/object_enum.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/object/object_enum.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/status.h" DS_DECLARE_string(etcd_address); diff --git a/tests/st/client/kv_cache/kv_client_dist_master_test.cpp b/tests/st/client/kv_cache/kv_client_dist_master_test.cpp index 814dd8e8459496767d1d786671055c05c8da64a7..ad8e21aa4da5f8560fe90363d27813e60cb964b9 100644 --- a/tests/st/client/kv_cache/kv_client_dist_master_test.cpp +++ b/tests/st/client/kv_cache/kv_client_dist_master_test.cpp @@ -19,7 +19,7 @@ */ #include "datasystem/common/util/random_data.h" #include "datasystem/common/util/timer.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include #include @@ -37,7 +37,7 @@ #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/file_util.h" #include "datasystem/common/log/log.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/connection.h" #include "datasystem/utils/status.h" #include "datasystem/worker/hash_ring/hash_ring.h" diff --git a/tests/st/client/kv_cache/kv_client_etcd_dfx_test.cpp b/tests/st/client/kv_cache/kv_client_etcd_dfx_test.cpp index ea4f81412171fe96694b026aa83ab3b2dfcc4d10..24be4bbe82f5b62367dc3ab30825ce983b9332ef 100644 --- a/tests/st/client/kv_cache/kv_client_etcd_dfx_test.cpp +++ b/tests/st/client/kv_cache/kv_client_etcd_dfx_test.cpp @@ -28,7 +28,7 @@ #include "datasystem/common/util/format.h" #include "datasystem/common/util/net_util.h" #include "datasystem/common/util/timer.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/status.h" namespace datasystem { diff --git a/tests/st/client/kv_cache/kv_client_log_monitor_test.cpp b/tests/st/client/kv_cache/kv_client_log_monitor_test.cpp index 7dce81642cfdcd3004a2b1affbd44bee573330aa..6e678765a6d4d27bc20dac27ac654bc4d2016a40 100644 --- a/tests/st/client/kv_cache/kv_client_log_monitor_test.cpp +++ b/tests/st/client/kv_cache/kv_client_log_monitor_test.cpp @@ -41,8 +41,8 @@ #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/thread_pool.h" #include "client/object_cache/oc_client_common.h" -#include "datasystem/object_cache/object_enum.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/object/object_enum.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/status.h" DS_DECLARE_string(etcd_address); diff --git a/tests/st/client/kv_cache/kv_client_mset_test.cpp b/tests/st/client/kv_cache/kv_client_mset_test.cpp index 99926988b841a8b5280a54a585252f91c448733e..a51482edd43f6eb30e057372499b089d740fe164 100644 --- a/tests/st/client/kv_cache/kv_client_mset_test.cpp +++ b/tests/st/client/kv_cache/kv_client_mset_test.cpp @@ -40,8 +40,8 @@ #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/thread_pool.h" #include "client/object_cache/oc_client_common.h" -#include "datasystem/object_cache/object_enum.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/object/object_enum.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/status.h" DS_DECLARE_string(etcd_address); diff --git a/tests/st/client/kv_cache/kv_client_offset_read_one_host_test.cpp b/tests/st/client/kv_cache/kv_client_offset_read_one_host_test.cpp index bee7cc4f92a50e9f7896e598482e2192d7ff7797..6af17bf6a733b0d38c35efdd3405556518b8c55e 100644 --- a/tests/st/client/kv_cache/kv_client_offset_read_one_host_test.cpp +++ b/tests/st/client/kv_cache/kv_client_offset_read_one_host_test.cpp @@ -19,7 +19,7 @@ */ #include "datasystem/common/util/uuid_generator.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include #include @@ -43,7 +43,7 @@ #include "datasystem/common/util/random_data.h" #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/timer.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/connection.h" #include "datasystem/utils/status.h" #include "datasystem/worker/hash_ring/hash_ring.h" diff --git a/tests/st/client/kv_cache/kv_client_replica_scale_test.cpp b/tests/st/client/kv_cache/kv_client_replica_scale_test.cpp index bc49372a0436e463122c7b6e183f62558185b5a6..73b01db4187d5f84fec612d12b88efa8db0ba332 100644 --- a/tests/st/client/kv_cache/kv_client_replica_scale_test.cpp +++ b/tests/st/client/kv_cache/kv_client_replica_scale_test.cpp @@ -36,9 +36,9 @@ #include "datasystem/common/util/thread.h" #include "datasystem/common/util/uuid_generator.h" #include "datasystem/common/log/log.h" -#include "datasystem/object_cache/object_enum.h" -#include "datasystem/kv_cache/read_only_buffer.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/object/object_enum.h" +#include "datasystem/kv/read_only_buffer.h" +#include "datasystem/kv_client.h" #include "datasystem/worker/hash_ring/hash_ring_allocator.h" DS_DECLARE_string(etcd_address); diff --git a/tests/st/client/kv_cache/kv_client_replica_test.cpp b/tests/st/client/kv_cache/kv_client_replica_test.cpp index d0bb2a976d8e5f7b23913d20ea537c86ed7d17e7..3418e1927def4031793dd4691d236592b4ba7842 100644 --- a/tests/st/client/kv_cache/kv_client_replica_test.cpp +++ b/tests/st/client/kv_cache/kv_client_replica_test.cpp @@ -34,8 +34,8 @@ #include "datasystem/common/util/net_util.h" #include "datasystem/common/util/strings_util.h" #include "datasystem/common/log/log.h" -#include "datasystem/object_cache/object_enum.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/object/object_enum.h" +#include "datasystem/kv_client.h" #include "datasystem/worker/hash_ring/hash_ring_allocator.h" DS_DECLARE_string(etcd_address); diff --git a/tests/st/client/kv_cache/kv_client_scale_test.cpp b/tests/st/client/kv_cache/kv_client_scale_test.cpp index 635b567b7c5d3b39c4c370033230856abda039de..776c01f6ee33819bcaadfd3c499377e778d917b1 100644 --- a/tests/st/client/kv_cache/kv_client_scale_test.cpp +++ b/tests/st/client/kv_cache/kv_client_scale_test.cpp @@ -33,7 +33,7 @@ #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/file_util.h" #include "datasystem/master/object_cache/store/object_meta_store.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/status.h" #include "datasystem/utils/string_view.h" #include "datasystem/worker/hash_ring/hash_ring.h" diff --git a/tests/st/client/kv_cache/kv_client_voluntary_scale_down_test.cpp b/tests/st/client/kv_cache/kv_client_voluntary_scale_down_test.cpp index 53345e3bc9dcaad80c9be963617eac35f90b41ba..dad0b717c5bb0bdcb7fb3bedcfbf385a749fe5c1 100644 --- a/tests/st/client/kv_cache/kv_client_voluntary_scale_down_test.cpp +++ b/tests/st/client/kv_cache/kv_client_voluntary_scale_down_test.cpp @@ -22,8 +22,8 @@ #include "datasystem/common/inject/inject_point.h" #include "datasystem/common/kvstore/etcd/etcd_constants.h" #include "datasystem/common/util/uuid_generator.h" -#include "datasystem/object_cache/object_enum.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/object/object_enum.h" +#include "datasystem/kv_client.h" #include #include #include @@ -374,6 +374,37 @@ TEST_F(KVClientVoluntaryScaleDownTest, NormalObjectConsecutiveVoluntaryScaleDown DS_ASSERT_OK(cluster_->StartNode(WORKER, 1, "")); } +TEST_F(KVClientVoluntaryScaleDownTest, StartKeyWithWorkerUuid) +{ + auto key = client0_->GenerateKey(); + std::string data = "aaaaaaaaaa"; + DS_ASSERT_OK(client2_->Set(key, data)); + for (size_t i = 1; i < DEFAULT_WORKER_NUM; i++) { + DS_ASSERT_OK(cluster_->SetInjectAction(WORKER, i, "MigrateByRanges.Delay", "sleep(3000)")); + } + VoluntaryScaleDownInject(0); + InitTestKVClient(1, client1_, 10000); // Init client1 to worker 1 with 10000ms timeout + bool stop = false; + std::thread t1([&stop, &key, &data, this] { + while (!stop) { + ReadParam readParam{ .key = key, .offset = 0, .size = data.size() - 1 }; + std::vector params = { readParam }; + std::vector> buffers; + DS_ASSERT_OK(client1_->Read(params, buffers)); + Optional buffer = buffers.back(); + std::string getValue(reinterpret_cast(buffer->ImmutableData()), buffer->GetSize()); + ASSERT_EQ(data.substr(0, data.size() - 1), getValue); + std::this_thread::sleep_for(std::chrono::milliseconds(100)); // wait for 100 ms + } + }); + WaitAllNodesJoinIntoHashRing(3); // The number of worker is 3 + AssertWorkerNum(3); // The number of worker is 3 + DS_ASSERT_OK(cluster_->StartNode(WORKER, 0, "")); + WaitAllNodesJoinIntoHashRing(DEFAULT_WORKER_NUM); + stop = true; + t1.join(); +} + TEST_F(KVClientVoluntaryScaleDownTest, LEVEL2_TestNotRemoveFailedWorkerWhenRestart) { SetWorkerHashInjection(); diff --git a/tests/st/client/kv_cache/kv_client_worker_timeout_storage.cpp b/tests/st/client/kv_cache/kv_client_worker_timeout_storage.cpp index abece670fbfca753a3f6f3fff67172f833082d7c..9abc557a09b9c1886cb061e922aabfc275ecb9e9 100644 --- a/tests/st/client/kv_cache/kv_client_worker_timeout_storage.cpp +++ b/tests/st/client/kv_cache/kv_client_worker_timeout_storage.cpp @@ -18,7 +18,7 @@ * Description: Test the behavior after worker timeout. */ -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include #include #include diff --git a/tests/st/client/object_cache/client_dfx_test.cpp b/tests/st/client/object_cache/client_dfx_test.cpp index a7a279f60860b60f319956c6caafbf72f4299bb7..9f1d00f8faafa4bbc218a88c5ba43faf1c4ffe49 100644 --- a/tests/st/client/object_cache/client_dfx_test.cpp +++ b/tests/st/client/object_cache/client_dfx_test.cpp @@ -32,7 +32,7 @@ #include "datasystem/common/util/net_util.h" #include "datasystem/common/util/random_data.h" #include "datasystem/common/util/status_helper.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/ut_object.stub.rpc.pb.h" #include "datasystem/utils/status.h" #include "datasystem/worker/object_cache/worker_master_oc_api.h" diff --git a/tests/st/client/object_cache/client_get_test.cpp b/tests/st/client/object_cache/client_get_test.cpp index e1e67d66447d2186f9bb7d0e2b9c305deba8045d..eb87abebe2e4aa2e7871cf2d570161408c7cd584 100644 --- a/tests/st/client/object_cache/client_get_test.cpp +++ b/tests/st/client/object_cache/client_get_test.cpp @@ -36,8 +36,8 @@ #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/timer.h" #include "datasystem/common/util/wait_post.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object_client.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/status.h" #include "oc_client_common.h" #include "datasystem/common/metrics/res_metric_collector.h" diff --git a/tests/st/client/object_cache/hetero_client_mock_test.cpp b/tests/st/client/object_cache/hetero_client_mock_test.cpp index 654bd36d31fc7b883a8016edfb0c3a17ebbc206e..45548bd4f6a62ad850d34cca17ca8cddb9ca2ebd 100644 --- a/tests/st/client/object_cache/hetero_client_mock_test.cpp +++ b/tests/st/client/object_cache/hetero_client_mock_test.cpp @@ -29,7 +29,7 @@ #include "datasystem/common/rpc/rpc_constants.h" #include "datasystem/common/util/uuid_generator.h" #include "datasystem/client/object_cache/client_worker_api.h" -#include "datasystem/hetero_cache/hetero_client.h" +#include "datasystem/hetero_client.h" #include "datasystem/client/hetero_cache/device_buffer.h" #include "datasystem/protos/p2p_subscribe.pb.h" diff --git a/tests/st/client/object_cache/object_client_replica_test.cpp b/tests/st/client/object_cache/object_client_replica_test.cpp index d9a665653eee331ae09d4f4864f6f744edd6d4e8..90ac85043d708faab3c821dd2a267d234114ba1d 100644 --- a/tests/st/client/object_cache/object_client_replica_test.cpp +++ b/tests/st/client/object_cache/object_client_replica_test.cpp @@ -36,8 +36,8 @@ #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/uuid_generator.h" #include "datasystem/common/log/log.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object_client.h" +#include "datasystem/object/object_enum.h" #include "datasystem/worker/hash_ring/hash_ring_allocator.h" DS_DECLARE_string(etcd_address); diff --git a/tests/st/client/object_cache/object_client_scale_test.cpp b/tests/st/client/object_cache/object_client_scale_test.cpp index 51e2c2b3adc7e6ffae26788e651b4a214c5f1df7..5cec6befce0cab863bc9ec509d3b7a4912422d1f 100644 --- a/tests/st/client/object_cache/object_client_scale_test.cpp +++ b/tests/st/client/object_cache/object_client_scale_test.cpp @@ -43,10 +43,10 @@ #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/timer.h" #include "datasystem/master/object_cache/store/object_meta_store.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object_client.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/master_object.pb.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/status.h" #include "datasystem/worker/hash_ring/hash_ring.h" #include "datasystem/worker/hash_ring/hash_ring_allocator.h" diff --git a/tests/st/client/object_cache/object_client_tenant_test.cpp b/tests/st/client/object_cache/object_client_tenant_test.cpp index 16bed7a4b8b9cd7c21bc7a0c7bb48d6ce26b3ebd..2d84673b2d8fe041c5f5d0a3feda4e44a458dc72 100644 --- a/tests/st/client/object_cache/object_client_tenant_test.cpp +++ b/tests/st/client/object_cache/object_client_tenant_test.cpp @@ -29,7 +29,7 @@ #include "datasystem/common/util/format.h" #include "datasystem/common/util/gflag/flags.h" #include "datasystem/context/context.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/common/log/log.h" #include "oc_client_common.h" diff --git a/tests/st/client/object_cache/oc_client_common.h b/tests/st/client/object_cache/oc_client_common.h index ac81ead4515ec1179cd7cb87b647125d1d0066b0..77971dd3cfde8e2a01f923c29aa2cc544bf3204e 100644 --- a/tests/st/client/object_cache/oc_client_common.h +++ b/tests/st/client/object_cache/oc_client_common.h @@ -29,11 +29,11 @@ #include "datasystem/common/util/random_data.h" #include "datasystem/common/util/thread_pool.h" #include "datasystem/datasystem.h" -#include "datasystem/hetero_cache/hetero_client.h" -#include "datasystem/object_cache/object_client.h" +#include "datasystem/hetero_client.h" +#include "datasystem/object_client.h" #include "datasystem/protos/hash_ring.pb.h" #include "datasystem/protos/master_object.pb.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/connection.h" #include "datasystem/common/kvstore/etcd/etcd_store.h" #include "datasystem/common/util/uuid_generator.h" diff --git a/tests/st/client/object_cache/oc_client_get_mem_alloc_optimize_test.cpp b/tests/st/client/object_cache/oc_client_get_mem_alloc_optimize_test.cpp index 63d17a9db7b09130df43aba558e03e7676bfc112..3bf86563d29fd6b988a3bd725155a13910b993b4 100644 --- a/tests/st/client/object_cache/oc_client_get_mem_alloc_optimize_test.cpp +++ b/tests/st/client/object_cache/oc_client_get_mem_alloc_optimize_test.cpp @@ -21,7 +21,7 @@ #include #include "common.h" #include "datasystem/common/util/wait_post.h" -#include "datasystem/object_cache/object_client.h" +#include "datasystem/object_client.h" #include "datasystem/utils/status.h" #include "oc_client_common.h" #include "datasystem/client/client_worker_common_api.h" diff --git a/tests/st/client/object_cache/oc_service_disable_test.cpp b/tests/st/client/object_cache/oc_service_disable_test.cpp index bb92a657d76ddb1ee9e22017d90dfa8a30b8f10f..090249960611e0a2f7c9f2b4ebe931b475051900 100644 --- a/tests/st/client/object_cache/oc_service_disable_test.cpp +++ b/tests/st/client/object_cache/oc_service_disable_test.cpp @@ -19,8 +19,8 @@ */ #include "common.h" #include "datasystem/common/util/random_data.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/object_client.h" +#include "datasystem/kv_client.h" #include #include diff --git a/tests/st/client/perf_client/perf_client_test.cpp b/tests/st/client/perf_client/perf_client_test.cpp index 23beff03cee5ef122d16770f90cca7d8d5f1b05c..19364f0f8e6a1d7a84012530dfb07c78bf6f2382 100644 --- a/tests/st/client/perf_client/perf_client_test.cpp +++ b/tests/st/client/perf_client/perf_client_test.cpp @@ -19,7 +19,7 @@ #include "common.h" #include "client/object_cache/oc_client_common.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include "datasystem/perf_client.h" namespace datasystem { diff --git a/tests/st/common/kvstore/etcd_store_test.cpp b/tests/st/common/kvstore/etcd_store_test.cpp index a08519dc896128a579d5f37b198ce49551b89b3f..c810465a6dc209cf7912598f30b57ede1da2576d 100644 --- a/tests/st/common/kvstore/etcd_store_test.cpp +++ b/tests/st/common/kvstore/etcd_store_test.cpp @@ -37,8 +37,8 @@ #include "datasystem/common/util/strings_util.h" #include "datasystem/common/log/log.h" #include "datasystem/common/util/random_data.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object_client.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/sensitive_value.h" #include "etcd/api/mvccpb/kv.pb.h" diff --git a/tests/st/common/log/kv_cache_log_performance_test.cpp b/tests/st/common/log/kv_cache_log_performance_test.cpp index 5aad70434f3abfb647f845b7bb928bebf1cfbf6c..72826940569d97f210df82d90f963b1ee1c67f3a 100644 --- a/tests/st/common/log/kv_cache_log_performance_test.cpp +++ b/tests/st/common/log/kv_cache_log_performance_test.cpp @@ -29,8 +29,8 @@ #include "common.h" #include "datasystem/common/flags/flags.h" #include "datasystem/common/log/log.h" -#include "datasystem/kv_cache/read_only_buffer.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv/read_only_buffer.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/status.h" DS_DECLARE_bool(alsologtostderr); diff --git a/tests/st/common/log/logging_free_test.cpp b/tests/st/common/log/logging_free_test.cpp index 562659c6bcefe0d4b57fc2df1e6e2814014a01ae..8decc2b51e3533f66d91d0bb3e33b6dc3a61c252 100644 --- a/tests/st/common/log/logging_free_test.cpp +++ b/tests/st/common/log/logging_free_test.cpp @@ -25,7 +25,7 @@ #include "client/object_cache/oc_client_common.h" #include "common.h" #include "datasystem/common/util/wait_post.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" namespace datasystem { namespace st { diff --git a/tests/st/device/dev_object_hetero_test.cpp b/tests/st/device/dev_object_hetero_test.cpp index a13b1019886a1a03ff7e5df4c35c76d602b39822..c8152340e91ec546a910d3aba54afd0dab4740f5 100644 --- a/tests/st/device/dev_object_hetero_test.cpp +++ b/tests/st/device/dev_object_hetero_test.cpp @@ -44,7 +44,7 @@ #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/uuid_generator.h" #include "datasystem/common/log/log.h" -#include "datasystem/hetero_cache/hetero_client.h" +#include "datasystem/hetero_client.h" #include "datasystem/client/hetero_cache/device_util.h" #include "device/dev_test_helper.h" diff --git a/tests/st/device/dev_object_scale_test.cpp b/tests/st/device/dev_object_scale_test.cpp index a067f2c32ea5eeaab518f1309dd072df0142282d..050efd5b0e10f4553c36ad8fa2174b56e85ccd7d 100644 --- a/tests/st/device/dev_object_scale_test.cpp +++ b/tests/st/device/dev_object_scale_test.cpp @@ -41,8 +41,8 @@ #include "datasystem/common/util/timer.h" #include "datasystem/common/util/uuid_generator.h" #include "datasystem/master/object_cache/store/object_meta_store.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object_client.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/optional.h" #include "datasystem/utils/status.h" #include "device/dev_test_helper.h" diff --git a/tests/st/device/dev_test_helper.h b/tests/st/device/dev_test_helper.h index 46ae1b3a05e34666afb2deafe25c7a198eb32e8d..6d66fbe5bc8ac82e53976421c8cb621a8b11bb37 100644 --- a/tests/st/device/dev_test_helper.h +++ b/tests/st/device/dev_test_helper.h @@ -41,11 +41,11 @@ #include "datasystem/common/util/random_data.h" #include "datasystem/common/util/thread_pool.h" #include "datasystem/common/util/uuid_generator.h" -#include "datasystem/object_cache/buffer.h" +#include "datasystem/object/buffer.h" #include "datasystem/client/hetero_cache/device_buffer.h" #include "datasystem/client/hetero_cache/device_util.h" -#include "datasystem/object_cache/object_client.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object_client.h" +#include "datasystem/object/object_enum.h" #include "mock/ascend_device_manager_mock.cpp" namespace datasystem { diff --git a/tests/st/device/edge_condition_dev_object_hetero_test.cpp b/tests/st/device/edge_condition_dev_object_hetero_test.cpp index 29cdda5d2e3232fff6c8249f9f660346dc2dd459..08093ce636f1534fa75003b7199d5b477b522849 100644 --- a/tests/st/device/edge_condition_dev_object_hetero_test.cpp +++ b/tests/st/device/edge_condition_dev_object_hetero_test.cpp @@ -36,7 +36,7 @@ #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/uuid_generator.h" -#include "datasystem/hetero_cache/hetero_client.h" +#include "datasystem/hetero_client.h" #include "datasystem/client/hetero_cache/device_util.h" #include "device/dev_test_helper.h" diff --git a/tests/st/device/hetero_get_meta_info_test.cpp b/tests/st/device/hetero_get_meta_info_test.cpp index e52d5a0bfd3ec34e7357ada701311d1cf14e2bc0..4796f7b88344bbc38004f4b1fe4359f0f6518d1c 100644 --- a/tests/st/device/hetero_get_meta_info_test.cpp +++ b/tests/st/device/hetero_get_meta_info_test.cpp @@ -44,7 +44,7 @@ #include "datasystem/common/util/status_helper.h" #include "datasystem/common/util/strings_util.h" #include "datasystem/common/util/uuid_generator.h" -#include "datasystem/hetero_cache/hetero_client.h" +#include "datasystem/hetero_client.h" #include "datasystem/client/hetero_cache/device_util.h" #include "device/dev_test_helper.h" diff --git a/tests/st/device/hetero_object_replica_test.cpp b/tests/st/device/hetero_object_replica_test.cpp index 1e2e9df9e87b6b5faba9dff9575e3aa8641fbedf..77c73871bb7df679feb4811fcecbcac4dd839281 100644 --- a/tests/st/device/hetero_object_replica_test.cpp +++ b/tests/st/device/hetero_object_replica_test.cpp @@ -20,8 +20,8 @@ #include #include "common.h" -#include "datasystem/kv_cache/kv_client.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/kv_client.h" +#include "datasystem/object/object_enum.h" #include "device/dev_test_helper.h" #include "gmock/gmock.h" diff --git a/tests/st/master/object_cache/oc_giveup_primary_test.cpp b/tests/st/master/object_cache/oc_giveup_primary_test.cpp index c1dd5c1f07058995e7ad1dd9b92b13fa6c97d8e7..b56596f596dc6ce157f0bbd3ee1e92aab165545b 100644 --- a/tests/st/master/object_cache/oc_giveup_primary_test.cpp +++ b/tests/st/master/object_cache/oc_giveup_primary_test.cpp @@ -39,11 +39,11 @@ #include "datasystem/common/util/uri.h" #include "datasystem/master/object_cache/oc_metadata_manager.h" #include "datasystem/master/object_cache/oc_nested_manager.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/protos/generic_service.pb.h" #include "datasystem/protos/master_object.pb.h" #include "datasystem/protos/object_posix.pb.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/status.h" #include "datasystem/worker/cluster_manager/etcd_cluster_manager.h" #include "datasystem/worker/hash_ring/hash_ring_allocator.h" diff --git a/tests/st/master/object_cache/oc_migrate_metadata_manager_test.cpp b/tests/st/master/object_cache/oc_migrate_metadata_manager_test.cpp index 9e61863821076aab63b18196cba7b2800670c04c..56c1daf37eb2bdf83b36cb1838b20088b8326698 100644 --- a/tests/st/master/object_cache/oc_migrate_metadata_manager_test.cpp +++ b/tests/st/master/object_cache/oc_migrate_metadata_manager_test.cpp @@ -41,7 +41,7 @@ #include "datasystem/protos/generic_service.pb.h" #include "datasystem/protos/master_object.pb.h" #include "datasystem/protos/object_posix.pb.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/kv_client.h" #include "datasystem/utils/status.h" #include "datasystem/worker/cluster_manager/etcd_cluster_manager.h" #include "datasystem/worker/hash_ring/hash_ring_allocator.h" diff --git a/tests/st/worker/object_cache/evict_mem_test.cpp b/tests/st/worker/object_cache/evict_mem_test.cpp index 04211374c4f48eb04ba3125c6c5a18bd701cea55..a652d9ee2935fdf1b136ce7341a09e35af876c1e 100644 --- a/tests/st/worker/object_cache/evict_mem_test.cpp +++ b/tests/st/worker/object_cache/evict_mem_test.cpp @@ -34,7 +34,7 @@ #include "datasystem/common/inject/inject_point.h" #include "datasystem/common/util/thread_pool.h" #include "datasystem/master/object_cache/store/object_meta_store.h" -#include "datasystem/object_cache/buffer.h" +#include "datasystem/object/buffer.h" #include "datasystem/worker/cluster_manager/etcd_cluster_manager.h" #include "datasystem/worker/object_cache/async_send_manager.h" #include "datasystem/worker/object_cache/obj_cache_shm_unit.h" diff --git a/tests/st/worker/object_cache/worker_oc_eviction_test.cpp b/tests/st/worker/object_cache/worker_oc_eviction_test.cpp index 6e9240306376b6d2666f97475dd50a7eba3a0156..64ee4347edf93fbb1f4031b8bf66809548f2fdd6 100644 --- a/tests/st/worker/object_cache/worker_oc_eviction_test.cpp +++ b/tests/st/worker/object_cache/worker_oc_eviction_test.cpp @@ -33,7 +33,7 @@ #include "datasystem/common/inject/inject_point.h" #include "datasystem/common/util/thread_pool.h" #include "datasystem/master/object_cache/store/object_meta_store.h" -#include "datasystem/object_cache/buffer.h" +#include "datasystem/object/buffer.h" #include "datasystem/worker/cluster_manager/etcd_cluster_manager.h" #include "datasystem/worker/object_cache/async_send_manager.h" #include "datasystem/worker/object_cache/obj_cache_shm_unit.h" diff --git a/tests/st/worker/object_cache/worker_oc_service_impl_test.cpp b/tests/st/worker/object_cache/worker_oc_service_impl_test.cpp index 637e81d31503ec4036077e4043a5d02288f445be..5f7bbbeefe12e5564f0e0131552bdce843f57b66 100644 --- a/tests/st/worker/object_cache/worker_oc_service_impl_test.cpp +++ b/tests/st/worker/object_cache/worker_oc_service_impl_test.cpp @@ -35,9 +35,9 @@ #include "datasystem/master/object_cache/store/object_meta_store.h" #include "datasystem/master/replica_manager.h" #include "datasystem/master/object_cache/master_oc_service_impl.h" -#include "datasystem/object_cache/buffer.h" -#include "datasystem/object_cache/object_enum.h" -#include "datasystem/kv_cache/kv_client.h" +#include "datasystem/object/buffer.h" +#include "datasystem/object/object_enum.h" +#include "datasystem/kv_client.h" #include "datasystem/worker/client_manager/client_manager.h" #include "datasystem/worker/cluster_manager/etcd_cluster_manager.h" #include "datasystem/worker/object_cache/worker_oc_service_impl.h" diff --git a/tests/ut/common.h b/tests/ut/common.h index 29bf382c3d32f1128530e49058b1186b417ee2c6..ddb885d86c5a417c27595cc8e8bf47b6122328d1 100644 --- a/tests/ut/common.h +++ b/tests/ut/common.h @@ -91,12 +91,13 @@ namespace ut { } \ } while (false) -inline bool IsArmArchitecture() { - #if defined(__arm__) || defined(__aarch64__) || defined(__ARM_ARCH) - return true; - #else - return false; - #endif +inline bool IsArmArchitecture() +{ +#if defined(__arm__) || defined(__aarch64__) || defined(__ARM_ARCH) + return true; +#else + return false; +#endif } class CommonTest : public testing::Test { diff --git a/tests/ut/worker/object_cache/eviction_manager_common.h b/tests/ut/worker/object_cache/eviction_manager_common.h index 42ed66d1dc1d5f341464536f366aa3d9015739b0..43041139bb062c14f832f32f9147e3512618a853 100644 --- a/tests/ut/worker/object_cache/eviction_manager_common.h +++ b/tests/ut/worker/object_cache/eviction_manager_common.h @@ -30,7 +30,7 @@ #include "datasystem/common/immutable_string/immutable_string.h" #include "datasystem/common/shared_memory/allocator.h" #include "datasystem/common/object_cache/safe_table.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/master/object_cache/store/object_meta_store.h" #include "datasystem/worker/object_cache/obj_cache_shm_unit.h" #include "datasystem/worker/object_cache/worker_oc_eviction_manager.h" diff --git a/tests/ut/worker/object_cache/migrate_data_handler_test.cpp b/tests/ut/worker/object_cache/migrate_data_handler_test.cpp index 27eb48b66e95788316c9b4e978d61a37c16783ee..74e8575fdb538b7b8fb5a11fd2248924cad644b7 100644 --- a/tests/ut/worker/object_cache/migrate_data_handler_test.cpp +++ b/tests/ut/worker/object_cache/migrate_data_handler_test.cpp @@ -35,7 +35,7 @@ #include "datasystem/common/util/net_util.h" #include "datasystem/common/util/timer.h" #include "datasystem/common/log/log.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/status.h" #include "datasystem/worker/object_cache/worker_oc_spill.h" #include "eviction_manager_common.h" diff --git a/tests/ut/worker/object_cache/worker_oc_eviction_test.cpp b/tests/ut/worker/object_cache/worker_oc_eviction_test.cpp index 9b9d85bdcdc0b740fdca92ff7609d56689355040..11111de820c962a6a7a9b48bb5ef3d21499a46c9 100644 --- a/tests/ut/worker/object_cache/worker_oc_eviction_test.cpp +++ b/tests/ut/worker/object_cache/worker_oc_eviction_test.cpp @@ -33,7 +33,7 @@ #include "datasystem/common/object_cache/safe_table.h" #include "datasystem/common/inject/inject_point.h" #include "datasystem/master/object_cache/store/object_meta_store.h" -#include "datasystem/object_cache/buffer.h" +#include "datasystem/object/buffer.h" #include "datasystem/worker/object_cache/async_send_manager.h" #include "datasystem/worker/object_cache/obj_cache_shm_unit.h" #include "datasystem/worker/object_cache/worker_master_oc_api.h" diff --git a/tests/ut/worker/object_cache/worker_oc_spill_eviction_test.cpp b/tests/ut/worker/object_cache/worker_oc_spill_eviction_test.cpp index 89f7d17c58f714c6181a1d3f20255739ee720f5d..6b411c0da9960637bd321bd73de3fb17ce8a5185 100644 --- a/tests/ut/worker/object_cache/worker_oc_spill_eviction_test.cpp +++ b/tests/ut/worker/object_cache/worker_oc_spill_eviction_test.cpp @@ -20,7 +20,7 @@ #include "datasystem/common/object_cache/object_ref_info.h" #include "datasystem/common/shared_memory/allocator.h" -#include "datasystem/object_cache/object_enum.h" +#include "datasystem/object/object_enum.h" #include "datasystem/utils/status.h" #include "datasystem/worker/object_cache/worker_oc_eviction_manager.h" #include "datasystem/worker/object_cache/worker_oc_spill.h"