From daf8178c4a1838a5c9f9e313b619701474d06272 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 22 Jul 2025 14:19:39 +0800 Subject: [PATCH 01/18] Add mpc middleware lib tools --- MPC/middleware/BUILD.bazel | 5 ++ MPC/middleware/CMakeLists.txt | 16 ++++++ MPC/middleware/kcal/CMakeLists.txt | 9 +++ MPC/middleware/kcal/api/kcal_api.h | 29 ++++++++++ MPC/middleware/kcal/core/context.cc | 48 ++++++++++++++++ MPC/middleware/kcal/core/context.h | 55 +++++++++++++++++++ MPC/middleware/kcal/enumeration/kcal_enum.h | 25 +++++++++ MPC/middleware/kcal/operator/kcal_psi.cc | 50 +++++++++++++++++ MPC/middleware/kcal/operator/kcal_psi.h | 40 ++++++++++++++ MPC/middleware/kcal/utils/node_info_helper.cc | 29 ++++++++++ MPC/middleware/kcal/utils/node_info_helper.h | 39 +++++++++++++ 11 files changed, 345 insertions(+) create mode 100644 MPC/middleware/BUILD.bazel create mode 100644 MPC/middleware/CMakeLists.txt create mode 100644 MPC/middleware/kcal/CMakeLists.txt create mode 100644 MPC/middleware/kcal/api/kcal_api.h create mode 100644 MPC/middleware/kcal/core/context.cc create mode 100644 MPC/middleware/kcal/core/context.h create mode 100644 MPC/middleware/kcal/enumeration/kcal_enum.h create mode 100644 MPC/middleware/kcal/operator/kcal_psi.cc create mode 100644 MPC/middleware/kcal/operator/kcal_psi.h create mode 100644 MPC/middleware/kcal/utils/node_info_helper.cc create mode 100644 MPC/middleware/kcal/utils/node_info_helper.h diff --git a/MPC/middleware/BUILD.bazel b/MPC/middleware/BUILD.bazel new file mode 100644 index 0000000..69e75e2 --- /dev/null +++ b/MPC/middleware/BUILD.bazel @@ -0,0 +1,5 @@ +filegroup( + name = "kcal_middleware_srcs", + srcs = glob(["**"]), + visibility = ["//visibility:public"], +) \ No newline at end of file diff --git a/MPC/middleware/CMakeLists.txt b/MPC/middleware/CMakeLists.txt new file mode 100644 index 0000000..4dc9988 --- /dev/null +++ b/MPC/middleware/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 3.16) + +project(kcal_middleware LANGUAGES C CXX) + +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +find_package(libkcal) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/kcal) + +install(DIRECTORY kcal + DESTINATION include + FILES_MATCHING PATTERN "*.h" +) diff --git a/MPC/middleware/kcal/CMakeLists.txt b/MPC/middleware/kcal/CMakeLists.txt new file mode 100644 index 0000000..a662972 --- /dev/null +++ b/MPC/middleware/kcal/CMakeLists.txt @@ -0,0 +1,9 @@ +file(GLOB_RECURSE COMMON_SRCS ${CMAKE_CURRENT_LIST_DIR}/*.cc) + +add_library(kcal_middle ${COMMON_SRCS}) +target_link_libraries(kcal_middle PUBLIC lib_kcal) + +install(TARGETS kcal_middle + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) diff --git a/MPC/middleware/kcal/api/kcal_api.h b/MPC/middleware/kcal/api/kcal_api.h new file mode 100644 index 0000000..627234a --- /dev/null +++ b/MPC/middleware/kcal/api/kcal_api.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * virtCCA_sdk is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#ifndef KCAL_API_H +#define KCAL_API_H + +#include "data_guard_anonymization.h" +#include "data_guard_callback.h" +#include "data_guard_cfs.h" +#include "data_guard_config.h" +#include "data_guard_di.h" +#include "data_guard_dp.h" +#include "data_guard_enum.h" +#include "data_guard_error_code.h" +#include "data_guard_mpc.h" +#include "data_guard_struct.h" +#include "data_guard_ucon.h" +#include "data_guard_watermark.h" + +#endif //KCAL_API_H diff --git a/MPC/middleware/kcal/core/context.cc b/MPC/middleware/kcal/core/context.cc new file mode 100644 index 0000000..0abdbcf --- /dev/null +++ b/MPC/middleware/kcal/core/context.cc @@ -0,0 +1,48 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * virtCCA_sdk is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#include "kcal/core/context.h" + +namespace kcal { + +Context::Context(KCAL_Config config, KCAL_AlgorithmsType type) : config_(config), type_(type) {} + +Context::~Context() +{ + DG_ReleaseConfig(&teeCfg_); + DG_ReleaseConfigOpts(&cfgOpts_); +} + +int Context::Init() +{ + int rv = DG_InitConfigOpts(DG_BusinessType::MPC, &cfgOpts_); + if (rv != DG_SUCCESS) { + return rv; + } + rv = cfgOpts_->init(&teeCfg_); + if (rv != DG_SUCCESS) { + return rv; + } + + cfgOpts_->setIntValue(teeCfg_, DG_CON_MPC_TEE_INT_NODEID, config_.nodeId); + if (type_ == KCAL_AlgorithmsType::PSI || type_ == KCAL_AlgorithmsType::PIR) { + cfgOpts_->setIntValue(teeCfg_, DG_CON_MPC_TEE_INT_FXP_BITS, 0); + } else { + cfgOpts_->setIntValue(teeCfg_, DG_CON_MPC_TEE_INT_FXP_BITS, config_.fixBits); + } + cfgOpts_->setIntValue(teeCfg_, DG_CON_MPC_TEE_INT_THREAD_COUNT, config_.threadCount); + + return DG_SUCCESS; +} + + +} diff --git a/MPC/middleware/kcal/core/context.h b/MPC/middleware/kcal/core/context.h new file mode 100644 index 0000000..5d2ef74 --- /dev/null +++ b/MPC/middleware/kcal/core/context.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * virtCCA_sdk is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#ifndef CONTEXT_H +#define CONTEXT_H + +#include "kcal/api/kcal_api.h" +#include "kcal/enumeration/kcal_enum.h" +#include + +namespace kcal { + +struct KCAL_Config { + int nodeId; + int fixBits; + int threadCount; + int worldSize; +}; + +class Context { +public: + Context() = default; + + explicit Context(KCAL_Config config, KCAL_AlgorithmsType type); + + ~Context(); + + int Init(); + + [[nodiscard]] int GetWorldSize() const { return config_.worldSize; } + + void *GetTeeConfig() { return teeCfg_; } + + void SetNetRes(TEE_NET_RES *teeNetRes); + +private: + KCAL_Config config_; + KCAL_AlgorithmsType type_; + + void *teeCfg_ = nullptr; + DG_ConfigOpts *cfgOpts_ = nullptr; +}; + +} + +#endif //CONTEXT_H diff --git a/MPC/middleware/kcal/enumeration/kcal_enum.h b/MPC/middleware/kcal/enumeration/kcal_enum.h new file mode 100644 index 0000000..45b1eed --- /dev/null +++ b/MPC/middleware/kcal/enumeration/kcal_enum.h @@ -0,0 +1,25 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * virtCCA_sdk is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#ifndef KCAL_ENUM_H +#define KCAL_ENUM_H + +namespace kcal { + +enum class KCAL_AlgorithmsType { + PSI, + PIR +}; + +} + +#endif //KCAL_ENUM_H diff --git a/MPC/middleware/kcal/operator/kcal_psi.cc b/MPC/middleware/kcal/operator/kcal_psi.cc new file mode 100644 index 0000000..d813ae7 --- /dev/null +++ b/MPC/middleware/kcal/operator/kcal_psi.cc @@ -0,0 +1,50 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * virtCCA_sdk is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#include "kcal/operator/kcal_psi.h" +#include "kcal/utils/node_info_helper.h" + +namespace kcal { + +Psi::Psi() +{ + opts_ = std::make_unique(); + *opts_ = DG_InitPsiOpts(); +} + +Psi::~Psi() { opts->releaseTeeCtx(&dgTeeCtx_); } + +int Psi::Init(std::shared_ptr ctx) +{ + utils::NodeInfoHelper nodeInfoHelper(ctx->GetWorldSize()); + + int rv = opts_->initTeeCtx(ctx->GetTeeConfig(), &dgTeeCtx_); + if (rv != 0) { + return rv; + } + + rv = opts_->setTeeNodeInfos(dgTeeCtx_, nodeInfoHelper.Get()); + if (rv != 0) { + return rv; + } + + baseCtx_ = std::move(ctx); + + return DG_SUCCESS; +} + +int Psi::Run(DG_TeeInput *input, DG_TeeOutput **output, DG_TeeMode outputMode) +{ + return opts_->calculate(dgTeeCtx_, PSI, input, output, outputMode); +} + +} diff --git a/MPC/middleware/kcal/operator/kcal_psi.h b/MPC/middleware/kcal/operator/kcal_psi.h new file mode 100644 index 0000000..4b8b028 --- /dev/null +++ b/MPC/middleware/kcal/operator/kcal_psi.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * virtCCA_sdk is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#ifndef KCAL_PSI_H +#define KCAL_PSI_H + +#include "kcal/api/kcal_api.h" +#include "kcal/core/context.h" +#include + +namespace kcal { + +class Psi { +public: + Psi(); + + ~Psi(); + + int Init(std::shared_ptr ctx); + + int Run(DG_TeeInput *input, DG_TeeOutput **output, DG_TeeMode outputMode); + +private: + DG_TeeCtx *dgTeeCtx_ = nullptr; + std::shared_ptr baseCtx_ = nullptr; + std::unique_ptr opts_; +}; + +} + +#endif //KCAL_PSI_H diff --git a/MPC/middleware/kcal/utils/node_info_helper.cc b/MPC/middleware/kcal/utils/node_info_helper.cc new file mode 100644 index 0000000..023e70c --- /dev/null +++ b/MPC/middleware/kcal/utils/node_info_helper.cc @@ -0,0 +1,29 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * virtCCA_sdk is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#include "node_info_helper.h" + +namespace kcal::utils { + +NodeInfoHelper::NodeInfoHelper(int worldSize) +{ + infos_.resize(worldSize); + for (int i = 0; i < worldSize; ++i) { + infos_[i].nodeId = i; + } + + nodeInfos_.nodeInfo = infos_.data(); + nodeInfos_.size = infos_.size(); +} + + +} diff --git a/MPC/middleware/kcal/utils/node_info_helper.h b/MPC/middleware/kcal/utils/node_info_helper.h new file mode 100644 index 0000000..a9ef9f7 --- /dev/null +++ b/MPC/middleware/kcal/utils/node_info_helper.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * virtCCA_sdk is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#ifndef NODE_INFO_HELPER_H +#define NODE_INFO_HELPER_H + +#include "kcal/api/kcal_api.h" +#include + +namespace kcal::utils { + +class NodeInfoHelper { +public: + explicit NodeInfoHelper(int worldSize); + + ~NodeInfoHelper() = default; + + NodeInfoHelper(const NodeInfoHelper &) = delete; + NodeInfoHelper &operator=(const NodeInfoHelper &) = delete; + + TeeNodeInfos *Get() { return &nodeInfos_; } + +private: + TeeNodeInfos nodeInfos_; + std::vector infos_; +}; + +} + +#endif //NODE_INFO_HELPER_H -- Gitee From 76d83f63f7d51b41d24604f676dc803a8549922a Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 22 Jul 2025 16:50:02 +0800 Subject: [PATCH 02/18] Delete BUILD.bazel --- MPC/middleware/BUILD.bazel | 5 ----- MPC/third_party_adaptor/secrerflow/psi/README.md | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) delete mode 100644 MPC/middleware/BUILD.bazel create mode 100644 MPC/third_party_adaptor/secrerflow/psi/README.md diff --git a/MPC/middleware/BUILD.bazel b/MPC/middleware/BUILD.bazel deleted file mode 100644 index 69e75e2..0000000 --- a/MPC/middleware/BUILD.bazel +++ /dev/null @@ -1,5 +0,0 @@ -filegroup( - name = "kcal_middleware_srcs", - srcs = glob(["**"]), - visibility = ["//visibility:public"], -) \ No newline at end of file diff --git a/MPC/third_party_adaptor/secrerflow/psi/README.md b/MPC/third_party_adaptor/secrerflow/psi/README.md new file mode 100644 index 0000000..77bfb5d --- /dev/null +++ b/MPC/third_party_adaptor/secrerflow/psi/README.md @@ -0,0 +1,5 @@ +# 下载代码 +```shell +git clone --branch "v0.6.0.dev250507" https://github.com/secretflow/psi.git +git switch -c kcal-on-v0.6.0 +``` \ No newline at end of file -- Gitee From 57e745a7a69d989d7927558465522b0f09b4ace1 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 22 Jul 2025 17:10:01 +0800 Subject: [PATCH 03/18] Delete unused --- MPC/middleware/kcal/api/kcal_api.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/MPC/middleware/kcal/api/kcal_api.h b/MPC/middleware/kcal/api/kcal_api.h index 627234a..4442e55 100644 --- a/MPC/middleware/kcal/api/kcal_api.h +++ b/MPC/middleware/kcal/api/kcal_api.h @@ -23,7 +23,5 @@ #include "data_guard_error_code.h" #include "data_guard_mpc.h" #include "data_guard_struct.h" -#include "data_guard_ucon.h" -#include "data_guard_watermark.h" #endif //KCAL_API_H -- Gitee From 49d5f7a6ae74ada348a2790eb24ddb4f4cfd39eb Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 22 Jul 2025 17:24:17 +0800 Subject: [PATCH 04/18] Add secretflow/psi kcal lib patch --- .../secrerflow/psi/patches/kcal.patch | 835 ++++++++++++++++++ 1 file changed, 835 insertions(+) create mode 100644 MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch diff --git a/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch new file mode 100644 index 0000000..4ef2ed0 --- /dev/null +++ b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch @@ -0,0 +1,835 @@ +diff --git a/.bazelrc b/.bazelrc +index 6406143..ca5d119 100644 +--- a/.bazelrc ++++ b/.bazelrc +@@ -26,6 +26,7 @@ build --incompatible_disallow_empty_glob=false + + build --cxxopt=-std=c++17 + build --host_cxxopt=-std=c++17 ++build --linkopt=-pthread + + # Binary safety flags + build --copt=-fPIC +diff --git a/MODULE.bazel b/MODULE.bazel +index 7170c4e..4292630 100644 +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -66,6 +66,12 @@ new_local_repository( + path = "/opt/homebrew/opt/libomp/", + ) + ++new_local_repository( ++ name = "kunpeng_kcal_middleware", ++ build_file = "//bazel:kunpengkcal.BUILD", ++ path = "third_party/kcal_middleware" ++) ++ + bazel_dep(name = "kuscia", version = "0.14.0b0") + + # test +diff --git a/bazel/kunpengkcal.BUILD b/bazel/kunpengkcal.BUILD +new file mode 100644 +index 0000000..4257c18 +--- /dev/null ++++ b/bazel/kunpengkcal.BUILD +@@ -0,0 +1,17 @@ ++load("@rules_foreign_cc//foreign_cc:def.bzl", "cmake") ++ ++package(default_visibility = ["//visibility:public"]) ++ ++filegroup( ++ name = "kcal_all", ++ srcs = glob(["**"]), ++) ++ ++cmake( ++ name = "kcal_middleware", ++ lib_source = ":kcal_all", ++ out_static_libs = ["libkcal_middle.a"], ++ deps = [ ++ "@//third_party:libkcal" ++ ] ++) +diff --git a/examples/psi/config/kcal_receiver.json b/examples/psi/config/kcal_receiver.json +new file mode 100644 +index 0000000..a42f107 +--- /dev/null ++++ b/examples/psi/config/kcal_receiver.json +@@ -0,0 +1,42 @@ ++{ ++ "psi_config": { ++ "protocol_config": { ++ "protocol": "PROTOCOL_KCAL", ++ "kcal_config": { ++ "thread_count": 16 ++ }, ++ "role": "ROLE_RECEIVER", ++ "broadcast_result": true ++ }, ++ "input_config": { ++ "type": "IO_TYPE_FILE_CSV", ++ "path": "/tmp/receiver_input.csv" ++ }, ++ "output_config": { ++ "type": "IO_TYPE_FILE_CSV", ++ "path": "/tmp/kcal_receiver_output.csv" ++ }, ++ "keys": ["id_0", "id_1"], ++ "debug_options": { ++ "trace_path": "/tmp/kcal_receiver.trace" ++ }, ++ "disable_alignment": true, ++ "recovery_config": { ++ "enabled": true, ++ "folder": "/tmp/kcal_receiver_cache" ++ } ++ }, ++ "link_config": { ++ "parties": [ ++ { ++ "id": "receiver", ++ "host": "127.0.0.1:5300" ++ }, ++ { ++ "id": "sender", ++ "host": "127.0.0.1:5400" ++ } ++ ] ++ }, ++ "self_link_party": "receiver" ++} +diff --git a/examples/psi/config/kcal_sender.json b/examples/psi/config/kcal_sender.json +new file mode 100644 +index 0000000..34abca9 +--- /dev/null ++++ b/examples/psi/config/kcal_sender.json +@@ -0,0 +1,42 @@ ++{ ++ "psi_config": { ++ "protocol_config": { ++ "protocol": "PROTOCOL_KCAL", ++ "kcal_config": { ++ "thread_count": 16 ++ }, ++ "role": "ROLE_SENDER", ++ "broadcast_result": true ++ }, ++ "input_config": { ++ "type": "IO_TYPE_FILE_CSV", ++ "path": "/tmp/sender_input.csv" ++ }, ++ "output_config": { ++ "type": "IO_TYPE_FILE_CSV", ++ "path": "/tmp/kcal_sender_output.csv" ++ }, ++ "keys": ["id_0", "id_1"], ++ "debug_options": { ++ "trace_path": "/tmp/kcal_sender.trace" ++ }, ++ "disable_alignment": true, ++ "recovery_config": { ++ "enabled": false, ++ "folder": "/tmp/kcal_sender_cache" ++ } ++ }, ++ "link_config": { ++ "parties": [ ++ { ++ "id": "receiver", ++ "host": "127.0.0.1:5300" ++ }, ++ { ++ "id": "sender", ++ "host": "127.0.0.1:5400" ++ } ++ ] ++ }, ++ "self_link_party": "sender" ++} +diff --git a/psi/algorithm/kcal/BUILD.bazel b/psi/algorithm/kcal/BUILD.bazel +new file mode 100644 +index 0000000..921797b +--- /dev/null ++++ b/psi/algorithm/kcal/BUILD.bazel +@@ -0,0 +1,56 @@ ++load("//bazel:psi.bzl", "psi_cc_binary", "psi_cc_library", "psi_cc_test") ++ ++package(default_visibility = ["//visibility:public"]) ++ ++psi_cc_library( ++ name = "global_manager", ++ hdrs = ["global_manager.h"], ++ srcs = ["global_manager.cc"], ++ deps = [ ++ "@yacl//yacl/link", ++ ] ++) ++ ++psi_cc_library( ++ name = "link_api", ++ hdrs = ["link_api.h"], ++ srcs = ["link_api.cc"], ++ deps = [ ++ ":global_manager", ++ "@kunpeng_kcal_middleware//:kcal_middleware", ++ ] ++) ++ ++psi_cc_library( ++ name = "data_adaptor", ++ hdrs = ["data_adaptor.h"], ++ srcs = ["data_adaptor.cc"], ++ deps = [ ++ "//psi/utils:batch_provider", ++ "@kunpeng_kcal_middleware//:kcal_middleware", ++ ] ++) ++ ++psi_cc_library( ++ name = "receiver", ++ srcs = ["receiver.cc"], ++ hdrs = ["receiver.h"], ++ deps = [ ++ ":link_api", ++ ":data_adaptor", ++ "//psi:interface", ++ "//psi/utils:arrow_csv_batch_provider", ++ ] ++) ++ ++psi_cc_library( ++ name = "sender", ++ srcs = ["sender.cc"], ++ hdrs = ["sender.h"], ++ deps = [ ++ ":link_api", ++ ":data_adaptor", ++ "//psi:interface", ++ "//psi/utils:arrow_csv_batch_provider", ++ ] ++) +diff --git a/psi/algorithm/kcal/data_adaptor.cc b/psi/algorithm/kcal/data_adaptor.cc +new file mode 100644 +index 0000000..27e856d +--- /dev/null ++++ b/psi/algorithm/kcal/data_adaptor.cc +@@ -0,0 +1,65 @@ ++#include "psi/algorithm/kcal/data_adaptor.h" ++ ++#include ++ ++#include "spdlog/spdlog.h" ++ ++namespace psi::kcal { ++ ++DG_TeeInput* DataAdaptor::TransData( ++ std::shared_ptr provider) { ++ std::vector rawData; ++ while (true) { ++ auto data = provider->ReadNextBatch(); ++ if (data.empty()) { ++ break; ++ } ++ rawData.insert(rawData.end(), data.begin(), data.end()); ++ } ++ ++ DG_String* strings = nullptr; ++ BuildDgString(rawData, &strings); ++ ++ input_ = new DG_TeeInput(); ++ input_->data.strings = strings; ++ input_->size = static_cast(rawData.size()); ++ input_->dataType = MPC_STRING; ++ ++ return input_; ++} ++ ++DataAdaptor::~DataAdaptor() { ReleaseOutput(&input_); } ++ ++void DataAdaptor::BuildDgString(std::vector& strings, ++ DG_String&& dg) { ++ auto* dgString = new DG_String[strings.size()]; ++ for (size_t i = 0; i < strings.size(); ++i) { ++ dgString[i].str = strdup(strings[i].c_str()); ++ dgString[i].size = strings[i].size() + 1; ++ } ++ *dg = dgString; ++} ++ ++void DataAdaptor::ReleaseOutput(DG_TeeOutput** output) { ++ if (output == nullptr || *output == nullptr) { ++ return; ++ } ++ if ((*output)->dataType == MPC_STRING && (*output)->data.strings != nullptr) { ++ for (int i = 0; i < (*output)->size; ++i) { ++ if ((*output)->data.strings[i].str != nullptr) { ++ delete[](*output)->data.strings[i].str; ++ } ++ } ++ delete[](*output)->data.strings; ++ } else if ((*output)->dataType == MPC_DOUBLE && ++ (*output)->data.doubleNumbers != nullptr) { ++ delete[](*output)->data.doubleNumbers; ++ } else if ((*output)->dataType == MPC_INT && ++ (*output)->data.u64Numbers != nullptr) { ++ delete[](*output)->data.u64Numbers; ++ } ++ delete *output; ++ *output = nullptr; ++} ++ ++} // namespace psi::kcal +\ No newline at end of file +diff --git a/psi/algorithm/kcal/data_adaptor.h b/psi/algorithm/kcal/data_adaptor.h +new file mode 100644 +index 0000000..6ea1a88 +--- /dev/null ++++ b/psi/algorithm/kcal/data_adaptor.h +@@ -0,0 +1,30 @@ ++#pragma once ++ ++#include ++#include ++ ++#include "kcal/api/kcal_api.h" ++ ++#include "psi/utils/batch_provider.h" ++ ++namespace psi::kcal { ++ ++class DataAdaptor { ++ public: ++ DataAdaptor() = default; ++ ++ ~DataAdaptor(); ++ ++ DG_TeeInput* TransData(std::shared_ptr provider); ++ ++ DG_TeeInput* GetInput() { return input_; } ++ ++ static void ReleaseOutput(DG_TeeOutput** output); ++ ++ private: ++ void BuildDgString(std::vector& strings, DG_String** dg); ++ ++ DG_TeeInput* input_; ++}; ++ ++} // namespace psi::kcal +diff --git a/psi/algorithm/kcal/global_manager.cc b/psi/algorithm/kcal/global_manager.cc +new file mode 100644 +index 0000000..061a1f5 +--- /dev/null ++++ b/psi/algorithm/kcal/global_manager.cc +@@ -0,0 +1,19 @@ ++#include "psi/algorithm/kcal/global_manager.h" ++ ++namespace psi::kcal { ++ ++GlobalManager& GlobalManager::GetInstance() { ++ static GlobalManager instance; ++ return instance; ++} ++ ++void GlobalManager::InitializeCtx( ++ const std::shared_ptr& lctx) { ++ lctx_ = lctx; ++} ++ ++std::shared_ptr& GlobalManager::GetLink() { return lctx_; } ++ ++void GlobalManager::CleanLink() { lctx_.reset(); } ++ ++} // namespace psi::kcal +\ No newline at end of file +diff --git a/psi/algorithm/kcal/global_manager.h b/psi/algorithm/kcal/global_manager.h +new file mode 100644 +index 0000000..c596b0f +--- /dev/null ++++ b/psi/algorithm/kcal/global_manager.h +@@ -0,0 +1,25 @@ ++#pragma once ++ ++#include "yacl/link/link.h" ++ ++namespace psi::kcal { ++ ++class GlobalManager { ++ public: ++ static GlobalManager& GetInstance(); ++ ++ ~GlobalManager() = default; ++ ++ void InitializeCtx(const std::shared_ptr& lctx); ++ ++ std::shared_ptr& GetLink(); ++ ++ void CleanLink(); ++ ++ private: ++ GlobalManager() = default; ++ ++ std::shared_ptr lctx_; ++}; ++ ++} // namespace psi::kcal +diff --git a/psi/algorithm/kcal/link_api.cc b/psi/algorithm/kcal/link_api.cc +new file mode 100644 +index 0000000..2d57536 +--- /dev/null ++++ b/psi/algorithm/kcal/link_api.cc +@@ -0,0 +1,26 @@ ++#include "psi/algorithm/kcal/link_api.h" ++ ++#include "psi/algorithm/kcal/global_manager.h" ++ ++namespace psi::kcal { ++ ++int GlobalSendWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, ++ unsigned long long len) { ++ yacl::Buffer msg(reinterpret_cast(buf), len); ++ std::string tag = "kcal_send"; ++ auto lctx = GlobalManager::GetInstance().GetLink(); ++ lctx->SendAsync(nodeInfo->nodeId, msg, tag); ++ return 0; ++} ++ ++int GlobalRecvWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, ++ unsigned long long* len) { ++ std::string tag = "kcal_recv"; ++ auto lctx = GlobalManager::GetInstance().GetLink(); ++ auto msg = lctx->Recv(nodeInfo->nodeId, tag); ++ memcpy(buf, msg.data(), msg.size()); ++ *len = msg.size(); ++ return 0; ++} ++ ++} // namespace psi::kcal +diff --git a/psi/algorithm/kcal/link_api.h b/psi/algorithm/kcal/link_api.h +new file mode 100644 +index 0000000..2dec56f +--- /dev/null ++++ b/psi/algorithm/kcal/link_api.h +@@ -0,0 +1,15 @@ ++#pragma once ++ ++#include ++ ++#include "kcal/api/kcal_api.h" ++ ++namespace psi::kcal { ++ ++int GlobalSendWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, ++ unsigned long long len); ++ ++int GlobalRecvWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, ++ unsigned long long* len); ++ ++} // namespace psi::kcal +\ No newline at end of file +diff --git a/psi/algorithm/kcal/receiver.cc b/psi/algorithm/kcal/receiver.cc +new file mode 100644 +index 0000000..a640411 +--- /dev/null ++++ b/psi/algorithm/kcal/receiver.cc +@@ -0,0 +1,53 @@ ++#include "psi/algorithm/kcal/receiver.h" ++ ++#include "kcal/operaotr/kcal_psi.h" ++ ++#include "psi/algorithm/kcal/data_adaptor.h" ++#include "psi/algorithm/kcal/global_manager.h" ++#include "psi/algorithm/kcal/link_api.h" ++ ++namespace psi::kcal { ++ ++KcalPsiReceiver::KcalPsiReceiver(const v2::PsiConfig& config, ++ std::shared_ptr lctx) ++ : AbstractPsiReceiver(config, std::move(lctx)) {} ++ ++void KcalPsiReceiver::Init() { ++ AbstractPsiReceiver::Init(); ++ // 全局网络资源管理 ++ GlobalManager::GetInstance().InitializeCtx(lctx_); ++ // 配置 ++ ::kcal::KCAL_Config kcal_cfg; ++ kcal_cfg.fixBits = 0; ++ kcal_cfg.nodeId = lctx_->Rank(); ++ kcal_cfg.threadCount = config_.protocol_config().kcal_config().thread_count(); ++ kcal_cfg.worldSize = lctx_.WorldSize(); ++ // 网络接口 ++ TEE_NET_RES net_res; ++ net_res.funcRecvData = GlobalRecvWrapper; ++ net_res.funcSendData = GlobalSendWrapper; ++ // 计算上下文 ++ kcal_ctx_ = ::kcal::CreateKCALContext(kcal_cfg, &net_res, ++ ::kcal::KCAL_AlgorithmsType::PSI); ++} ++ ++void KcalPsiReceiver::PreProcess() { ++ input_ = std::make_shared(); ++ input_->TransData(batch_provider_); ++} ++ ++void KcalPsiReceiver::Online() { ++ ::kcal::Psi psi; ++ psi.Init(kcal_ctx_); ++ psi.Run(input_->GetInput(), &output_, TEE_OUTPUT_INDEX); ++} ++ ++void KcalPsiReceiver::PostProcess() { ++ for (int i = 0; i < output_->size; ++i) { ++ intersection_indices_writer_->WriteCache(output_->data.u64Numbers[i]); ++ } ++ intersection_indices_writer_->Commit(); ++ DataAdaptor::ReleaseOutput(&output_); ++} ++ ++} // namespace psi::kcal +\ No newline at end of file +diff --git a/psi/algorithm/kcal/receiver.h b/psi/algorithm/kcal/receiver.h +new file mode 100644 +index 0000000..86f1bce +--- /dev/null ++++ b/psi/algorithm/kcal/receiver.h +@@ -0,0 +1,34 @@ ++#pragma once ++ ++#include "kcal/core/context.h" ++ ++#include "psi/algorithm/kcal/data_adaptor.h" ++#include "psi/interface.h" ++#include "psi/utils/arrow_csv_batch_provider.h" ++ ++#include "psi/proto/psi_v2.pb.h" ++ ++namespace psi::kcal { ++ ++class KcalPsiReceiver final : public AbstractPsiReceiver { ++ public: ++ explict KcalPsiReceiver(const v2::PsiConfig& config, ++ std::shared_ptr lctx = nullptr); ++ ++ ~KcalPsiReceiver() override = default; ++ ++ private: ++ void Init() override; ++ ++ void PreProcess() override; ++ ++ void Online() override; ++ ++ void PostProcess() override; ++ ++ std::shared_ptr<::kcal::Context> kcal_ctx_; ++ std::shared_ptr input_; ++ DG_TeeOutput* output_ = nullptr; ++}; ++ ++} // namespace psi::kcal +diff --git a/psi/algorithm/kcal/sender.cc b/psi/algorithm/kcal/sender.cc +new file mode 100644 +index 0000000..794aafd +--- /dev/null ++++ b/psi/algorithm/kcal/sender.cc +@@ -0,0 +1,53 @@ ++#include "psi/algorithm/kcal/sender.h" ++ ++#include "kcal/operaotr/kcal_psi.h" ++ ++#include "psi/algorithm/kcal/data_adaptor.h" ++#include "psi/algorithm/kcal/global_manager.h" ++#include "psi/algorithm/kcal/link_api.h" ++ ++namespace psi::kcal { ++ ++KcalPsiSender::KcalPsiSender(const v2::PsiConfig& config, ++ std::shared_ptr lctx) ++ : AbstractPsiSender(config, std::move(lctx)) {} ++ ++void KcalPsiSender::Init() { ++ AbstractPsiSender::Init(); ++ // 全局网络资源管理 ++ GlobalManager::GetInstance().InitializeCtx(lctx_); ++ // 配置 ++ ::kcal::KCAL_Config kcal_cfg; ++ kcal_cfg.fixBits = 0; ++ kcal_cfg.nodeId = lctx_->Rank(); ++ kcal_cfg.threadCount = config_.protocol_config().kcal_config().thread_count(); ++ kcal_cfg.worldSize = lctx_.WorldSize(); ++ // 网络接口 ++ TEE_NET_RES net_res; ++ net_res.funcRecvData = GlobalRecvWrapper; ++ net_res.funcSendData = GlobalSendWrapper; ++ // 计算上下文 ++ kcal_ctx_ = ::kcal::CreateKCALContext(kcal_cfg, &net_res, ++ ::kcal::KCAL_AlgorithmsType::PSI); ++} ++ ++void KcalPsiSender::PreProcess() { ++ input_ = std::make_shared(); ++ input_->TransData(batch_provider_); ++} ++ ++void KcalPsiSender::Online() { ++ ::kcal::Psi psi; ++ psi.Init(kcal_ctx_); ++ psi.Run(input_->GetInput(), &output_, TEE_OUTPUT_INDEX); ++} ++ ++void KcalPsiSender::PostProcess() { ++ for (int i = 0; i < output_->size; ++i) { ++ intersection_indices_writer_->WriteCache(output_->data.u64Numbers[i]); ++ } ++ intersection_indices_writer_->Commit(); ++ DataAdaptor::ReleaseOutput(&output_); ++} ++ ++} // namespace psi::kcal +\ No newline at end of file +diff --git a/psi/algorithm/kcal/sender.h b/psi/algorithm/kcal/sender.h +new file mode 100644 +index 0000000..ef63c7d +--- /dev/null ++++ b/psi/algorithm/kcal/sender.h +@@ -0,0 +1,34 @@ ++#pragma once ++ ++#include "kcal/core/context.h" ++ ++#include "psi/algorithm/kcal/data_adaptor.h" ++#include "psi/interface.h" ++#include "psi/utils/arrow_csv_batch_provider.h" ++ ++#include "psi/proto/psi_v2.pb.h" ++ ++namespace psi::kcal { ++ ++class KcalPsiSender final : public AbstractPsiSender { ++ public: ++ explict KcalPsiSender(const v2::PsiConfig& config, ++ std::shared_ptr lctx = nullptr); ++ ++ ~KcalPsiSender() override = default; ++ ++ private: ++ void Init() override; ++ ++ void PreProcess() override; ++ ++ void Online() override; ++ ++ void PostProcess() override; ++ ++ std::shared_ptr<::kcal::Context> kcal_ctx_; ++ std::shared_ptr input_; ++ DG_TeeOutput* output_ = nullptr; ++}; ++ ++} // namespace psi::kcal +diff --git a/psi/apps/psi_launcher/BUILD.bazel b/psi/apps/psi_launcher/BUILD.bazel +index ffa6a3f..794053a 100644 +--- a/psi/apps/psi_launcher/BUILD.bazel ++++ b/psi/apps/psi_launcher/BUILD.bazel +@@ -29,6 +29,8 @@ psi_cc_library( + "//psi/algorithm/kkrt:sender", + "//psi/algorithm/rr22:receiver", + "//psi/algorithm/rr22:sender", ++ "//psi/algorithm/kcal::receiver", ++ "//psi/algorithm/kcal::sender", + "@yacl//yacl/base:exception", + ], + ) +@@ -102,6 +104,7 @@ psi_cc_binary( + "//psi:version", + "//psi/proto:entry_cc_proto", + "//psi/utils:resource_manager", ++ "@kunpeng_kcal_middleware//:kcal_middleware", + "@gflags", + ], + ) +diff --git a/psi/apps/psi_launcher/factory.cc b/psi/apps/psi_launcher/factory.cc +index f57f1eb..97de4b3 100644 +--- a/psi/apps/psi_launcher/factory.cc ++++ b/psi/apps/psi_launcher/factory.cc +@@ -22,6 +22,8 @@ + #include "psi/algorithm/ecdh/sender.h" + #include "psi/algorithm/ecdh/ub_psi/client.h" + #include "psi/algorithm/ecdh/ub_psi/server.h" ++#include "psi/algorithm/kcal/receiver.h" ++#include "psi/algorithm/kcal/sender.h" + #include "psi/algorithm/kkrt/receiver.h" + #include "psi/algorithm/kkrt/sender.h" + #include "psi/algorithm/rr22/receiver.h" +@@ -62,6 +64,16 @@ std::unique_ptr createPsiParty( + YACL_THROW("Role is invalid."); + } + } ++ case v2::Protocol::PROTOCOL_KCAL: { ++ switch (config.protocol_config().role()) { ++ case v2::Role::ROLE_RECEIVER: ++ return std::make_unique(config, lctx); ++ case v2::Role::ROLE_SENDER: ++ return std::make_unique(config, lctx); ++ default: ++ YACL_THROW("Role is invalid."); ++ } ++ } + default: + YACL_THROW("Protocol is unspecified."); + } +diff --git a/psi/proto/psi_v2.proto b/psi/proto/psi_v2.proto +index 5682ef7..c56b575 100644 +--- a/psi/proto/psi_v2.proto ++++ b/psi/proto/psi_v2.proto +@@ -60,6 +60,8 @@ enum Protocol { + + // Blazing Fast PSI https://eprint.iacr.org/2022/320.pdf + PROTOCOL_RR22 = 3; ++ ++ PROTOCOL_KCAL = 4; + } + + // Configs for ECDH protocol. +@@ -92,6 +94,10 @@ message Rr22Config { + bool low_comm_mode = 2; + } + ++message KcalConfig { ++ uint32 thread_count = 1; ++} ++ + // Any items related to PSI protocols. + message ProtocolConfig { + Protocol protocol = 1; +@@ -109,6 +115,8 @@ message ProtocolConfig { + + // For RR22 protocol. + Rr22Config rr22_config = 6; ++ ++ KcalConfig kcal_config = 7; + } + + // TODO(junfeng): support more io types including oss, sql, etc. +diff --git a/third_party/BUILD.bazel b/third_party/BUILD.bazel +new file mode 100644 +index 0000000..a980827 +--- /dev/null ++++ b/third_party/BUILD.bazel +@@ -0,0 +1,17 @@ ++load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake") ++ ++package(default_visibility = ["//visibility:public"]) ++ ++cmake( ++ name = "libkcal", ++ generate_args = ["-GNinja"], ++ lib_source = "//third_party/kcal:kcal_srcs", ++ out_shared_libs = [ ++ "libdata_guard_common.so", ++ "libdata_guard.so", ++ "libhitls_bsl.so", ++ "libhitls_crypto.so", ++ "libmpc_tee.so", ++ "libsecurec.so", ++ ] ++) +diff --git a/third_party/kcal/BUILD.bazel b/third_party/kcal/BUILD.bazel +new file mode 100644 +index 0000000..88895f4 +--- /dev/null ++++ b/third_party/kcal/BUILD.bazel +@@ -0,0 +1,5 @@ ++filegroup( ++ name = "kcal_srcs", ++ srcs = glob(["**"]), ++ visibility = ["//visibility:public"], ++) +\ No newline at end of file +diff --git a/third_party/kcal/CMakeLists.txt b/third_party/kcal/CMakeLists.txt +new file mode 100644 +index 0000000..a4b424b +--- /dev/null ++++ b/third_party/kcal/CMakeLists.txt +@@ -0,0 +1,69 @@ ++cmake_minimum_required(VERSION 3.16) ++ ++project(lib_kcal) ++ ++add_library(lib_data_guard_common SHARED IMPORTED GLOBAL) ++set_target_properties(lib_data_guard_common PROPERTIES ++ IMPORTED_LOCATION "${CMAKE_CURRENT_LIST_DIR}/lib/libdata_guard_common.so" ++ INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/include" ++) ++ ++add_library(lib_data_guard SHARED IMPORTED GLOBAL) ++set_target_properties(lib_data_guard PROPERTIES ++ IMPORTED_LOCATION "${CMAKE_CURRENT_LIST_DIR}/lib/libdata_guard.so" ++ INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/include" ++) ++ ++add_library(lib_hitls_bsl SHARED IMPORTED GLOBAL) ++set_target_properties(lib_hitls_bsl PROPERTIES ++ IMPORTED_LOCATION "${CMAKE_CURRENT_LIST_DIR}/lib/libhitls_bsl.so" ++ INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/include" ++) ++ ++add_library(lib_hitls_crypto SHARED IMPORTED GLOBAL) ++set_target_properties(lib_hitls_crypto PROPERTIES ++ IMPORTED_LOCATION "${CMAKE_CURRENT_LIST_DIR}/lib/libhitls_crypto.so" ++ INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/include" ++) ++ ++add_library(lib_mpc_tee SHARED IMPORTED GLOBAL) ++set_target_properties(lib_mpc_tee PROPERTIES ++ IMPORTED_LOCATION "${CMAKE_CURRENT_LIST_DIR}/lib/libmpc_tee.so" ++ INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/include" ++) ++ ++add_library(lib_securec SHARED IMPORTED GLOBAL) ++set_target_properties(lib_securec PROPERTIES ++ IMPORTED_LOCATION "${CMAKE_CURRENT_LIST_DIR}/lib/libsecurec.so" ++ INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_CURRENT_LIST_DIR}/include" ++) ++ ++add_library(lib_kcal INTERFACE) ++target_include_directories(lib_kcal PROPERTIES ++ $ ++ $ ++) ++target_link_libraries(lib_kcal INTERFACE ++ lib_data_guard_common ++ lib_data_guard ++ lib_hitls_bsl ++ lib_hitls_crypto ++ lib_mpc_tee ++ lib_securec ++) ++ ++install(TARGETS lib_kcal ++ EXPORT lib_kcal_targets ++) ++ ++install(DIRECTORY include/ DESTINATION include ++ FILES_MACHING PATTERN "*.h" ++) ++ ++file(GLOB SO_FILES "${CMAKE_CURRENT_LIST_DIR}/lib/*.so") ++install(FILES ${SO_FILES} DESTINATION lib) ++ ++install(EXPORT lib_kcal_targets ++ DESTINATION lib/cmake/libkcal ++ FILE libkcal-config.cmake ++) +diff --git a/third_party/kcal_middleware/BUILD.bazel b/third_party/kcal_middleware/BUILD.bazel +new file mode 100644 +index 0000000..69e75e2 +--- /dev/null ++++ b/third_party/kcal_middleware/BUILD.bazel +@@ -0,0 +1,5 @@ ++filegroup( ++ name = "kcal_middleware_srcs", ++ srcs = glob(["**"]), ++ visibility = ["//visibility:public"], ++) +\ No newline at end of file -- Gitee From 26ca079da1b126eac703b9d3d69c89157b7a2286 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 22 Jul 2025 18:17:57 +0800 Subject: [PATCH 05/18] Add secretflow/psi kcal lib patch --- MPC/middleware/kcal/core/context.cc | 13 ++++++ MPC/middleware/kcal/core/context.h | 2 + MPC/middleware/kcal/operator/kcal_psi.cc | 2 +- .../secrerflow/psi/patches/kcal.patch | 44 +++++++++---------- 4 files changed, 38 insertions(+), 23 deletions(-) diff --git a/MPC/middleware/kcal/core/context.cc b/MPC/middleware/kcal/core/context.cc index 0abdbcf..c6bd7af 100644 --- a/MPC/middleware/kcal/core/context.cc +++ b/MPC/middleware/kcal/core/context.cc @@ -44,5 +44,18 @@ int Context::Init() return DG_SUCCESS; } +void Context::SetNetRes(TEE_NET_RES *teeNetRes) +{ + DG_Void netFunc = {.data = teeNetRes, .size = sizeof(TEE_NET_RES)}; + cfgOpts_->setVoidValue(teeCfg_, DG_CON_MPC_TEE_VOID_NET_API, &netFunc); +} + +std::shared_ptr CreateContext(KCAL_Config config, TEE_NET_RES *netRes, KCAL_AlgorithmsType type) +{ + auto context = std::make_shared(config, type); + context->Init(); + context->SetNetRes(netRes); + return context; +} } diff --git a/MPC/middleware/kcal/core/context.h b/MPC/middleware/kcal/core/context.h index 5d2ef74..4b7e0f2 100644 --- a/MPC/middleware/kcal/core/context.h +++ b/MPC/middleware/kcal/core/context.h @@ -50,6 +50,8 @@ private: DG_ConfigOpts *cfgOpts_ = nullptr; }; +std::shared_ptr CreateContext(KCAL_Config config, TEE_NET_RES *netRes, KCAL_AlgorithmsType type); + } #endif //CONTEXT_H diff --git a/MPC/middleware/kcal/operator/kcal_psi.cc b/MPC/middleware/kcal/operator/kcal_psi.cc index d813ae7..2bd5763 100644 --- a/MPC/middleware/kcal/operator/kcal_psi.cc +++ b/MPC/middleware/kcal/operator/kcal_psi.cc @@ -21,7 +21,7 @@ Psi::Psi() *opts_ = DG_InitPsiOpts(); } -Psi::~Psi() { opts->releaseTeeCtx(&dgTeeCtx_); } +Psi::~Psi() { opts_->releaseTeeCtx(&dgTeeCtx_); } int Psi::Init(std::shared_ptr ctx) { diff --git a/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch index 4ef2ed0..8be6237 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch +++ b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch @@ -29,11 +29,11 @@ index 7170c4e..4292630 100644 # test diff --git a/bazel/kunpengkcal.BUILD b/bazel/kunpengkcal.BUILD new file mode 100644 -index 0000000..4257c18 +index 0000000..8416212 --- /dev/null +++ b/bazel/kunpengkcal.BUILD @@ -0,0 +1,17 @@ -+load("@rules_foreign_cc//foreign_cc:def.bzl", "cmake") ++load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake") + +package(default_visibility = ["//visibility:public"]) + @@ -52,7 +52,7 @@ index 0000000..4257c18 +) diff --git a/examples/psi/config/kcal_receiver.json b/examples/psi/config/kcal_receiver.json new file mode 100644 -index 0000000..a42f107 +index 0000000..72e2694 --- /dev/null +++ b/examples/psi/config/kcal_receiver.json @@ -0,0 +1,42 @@ @@ -80,7 +80,7 @@ index 0000000..a42f107 + }, + "disable_alignment": true, + "recovery_config": { -+ "enabled": true, ++ "enabled": false, + "folder": "/tmp/kcal_receiver_cache" + } + }, @@ -210,7 +210,7 @@ index 0000000..921797b +) diff --git a/psi/algorithm/kcal/data_adaptor.cc b/psi/algorithm/kcal/data_adaptor.cc new file mode 100644 -index 0000000..27e856d +index 0000000..74332da --- /dev/null +++ b/psi/algorithm/kcal/data_adaptor.cc @@ -0,0 +1,65 @@ @@ -247,7 +247,7 @@ index 0000000..27e856d +DataAdaptor::~DataAdaptor() { ReleaseOutput(&input_); } + +void DataAdaptor::BuildDgString(std::vector& strings, -+ DG_String&& dg) { ++ DG_String** dg) { + auto* dgString = new DG_String[strings.size()]; + for (size_t i = 0; i < strings.size(); ++i) { + dgString[i].str = strdup(strings[i].c_str()); @@ -429,13 +429,13 @@ index 0000000..2dec56f \ No newline at end of file diff --git a/psi/algorithm/kcal/receiver.cc b/psi/algorithm/kcal/receiver.cc new file mode 100644 -index 0000000..a640411 +index 0000000..119a0b1 --- /dev/null +++ b/psi/algorithm/kcal/receiver.cc @@ -0,0 +1,53 @@ +#include "psi/algorithm/kcal/receiver.h" + -+#include "kcal/operaotr/kcal_psi.h" ++#include "kcal/operator/kcal_psi.h" + +#include "psi/algorithm/kcal/data_adaptor.h" +#include "psi/algorithm/kcal/global_manager.h" @@ -456,7 +456,7 @@ index 0000000..a640411 + kcal_cfg.fixBits = 0; + kcal_cfg.nodeId = lctx_->Rank(); + kcal_cfg.threadCount = config_.protocol_config().kcal_config().thread_count(); -+ kcal_cfg.worldSize = lctx_.WorldSize(); ++ kcal_cfg.worldSize = lctx_->WorldSize(); + // 网络接口 + TEE_NET_RES net_res; + net_res.funcRecvData = GlobalRecvWrapper; @@ -489,7 +489,7 @@ index 0000000..a640411 \ No newline at end of file diff --git a/psi/algorithm/kcal/receiver.h b/psi/algorithm/kcal/receiver.h new file mode 100644 -index 0000000..86f1bce +index 0000000..683de58 --- /dev/null +++ b/psi/algorithm/kcal/receiver.h @@ -0,0 +1,34 @@ @@ -507,7 +507,7 @@ index 0000000..86f1bce + +class KcalPsiReceiver final : public AbstractPsiReceiver { + public: -+ explict KcalPsiReceiver(const v2::PsiConfig& config, ++ explicit KcalPsiReceiver(const v2::PsiConfig& config, + std::shared_ptr lctx = nullptr); + + ~KcalPsiReceiver() override = default; @@ -529,13 +529,13 @@ index 0000000..86f1bce +} // namespace psi::kcal diff --git a/psi/algorithm/kcal/sender.cc b/psi/algorithm/kcal/sender.cc new file mode 100644 -index 0000000..794aafd +index 0000000..9102df8 --- /dev/null +++ b/psi/algorithm/kcal/sender.cc @@ -0,0 +1,53 @@ +#include "psi/algorithm/kcal/sender.h" + -+#include "kcal/operaotr/kcal_psi.h" ++#include "kcal/operator/kcal_psi.h" + +#include "psi/algorithm/kcal/data_adaptor.h" +#include "psi/algorithm/kcal/global_manager.h" @@ -556,7 +556,7 @@ index 0000000..794aafd + kcal_cfg.fixBits = 0; + kcal_cfg.nodeId = lctx_->Rank(); + kcal_cfg.threadCount = config_.protocol_config().kcal_config().thread_count(); -+ kcal_cfg.worldSize = lctx_.WorldSize(); ++ kcal_cfg.worldSize = lctx_->WorldSize(); + // 网络接口 + TEE_NET_RES net_res; + net_res.funcRecvData = GlobalRecvWrapper; @@ -589,7 +589,7 @@ index 0000000..794aafd \ No newline at end of file diff --git a/psi/algorithm/kcal/sender.h b/psi/algorithm/kcal/sender.h new file mode 100644 -index 0000000..ef63c7d +index 0000000..2f9e412 --- /dev/null +++ b/psi/algorithm/kcal/sender.h @@ -0,0 +1,34 @@ @@ -607,7 +607,7 @@ index 0000000..ef63c7d + +class KcalPsiSender final : public AbstractPsiSender { + public: -+ explict KcalPsiSender(const v2::PsiConfig& config, ++ explicit KcalPsiSender(const v2::PsiConfig& config, + std::shared_ptr lctx = nullptr); + + ~KcalPsiSender() override = default; @@ -628,15 +628,15 @@ index 0000000..ef63c7d + +} // namespace psi::kcal diff --git a/psi/apps/psi_launcher/BUILD.bazel b/psi/apps/psi_launcher/BUILD.bazel -index ffa6a3f..794053a 100644 +index ffa6a3f..c9f66f2 100644 --- a/psi/apps/psi_launcher/BUILD.bazel +++ b/psi/apps/psi_launcher/BUILD.bazel @@ -29,6 +29,8 @@ psi_cc_library( "//psi/algorithm/kkrt:sender", "//psi/algorithm/rr22:receiver", "//psi/algorithm/rr22:sender", -+ "//psi/algorithm/kcal::receiver", -+ "//psi/algorithm/kcal::sender", ++ "//psi/algorithm/kcal:receiver", ++ "//psi/algorithm/kcal:sender", "@yacl//yacl/base:exception", ], ) @@ -748,7 +748,7 @@ index 0000000..88895f4 \ No newline at end of file diff --git a/third_party/kcal/CMakeLists.txt b/third_party/kcal/CMakeLists.txt new file mode 100644 -index 0000000..a4b424b +index 0000000..972c809 --- /dev/null +++ b/third_party/kcal/CMakeLists.txt @@ -0,0 +1,69 @@ @@ -793,7 +793,7 @@ index 0000000..a4b424b +) + +add_library(lib_kcal INTERFACE) -+target_include_directories(lib_kcal PROPERTIES ++target_include_directories(lib_kcal INTERFACE + $ + $ +) @@ -811,7 +811,7 @@ index 0000000..a4b424b +) + +install(DIRECTORY include/ DESTINATION include -+ FILES_MACHING PATTERN "*.h" ++ FILES_MATCHING PATTERN "*.h" +) + +file(GLOB SO_FILES "${CMAKE_CURRENT_LIST_DIR}/lib/*.so") -- Gitee From 5ae349d781612a920e2d402787c5e429f5f02e40 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 22 Jul 2025 19:51:01 +0800 Subject: [PATCH 06/18] Modify create context func --- MPC/middleware/kcal/core/context.cc | 2 +- MPC/middleware/kcal/core/context.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MPC/middleware/kcal/core/context.cc b/MPC/middleware/kcal/core/context.cc index c6bd7af..cca6a88 100644 --- a/MPC/middleware/kcal/core/context.cc +++ b/MPC/middleware/kcal/core/context.cc @@ -50,7 +50,7 @@ void Context::SetNetRes(TEE_NET_RES *teeNetRes) cfgOpts_->setVoidValue(teeCfg_, DG_CON_MPC_TEE_VOID_NET_API, &netFunc); } -std::shared_ptr CreateContext(KCAL_Config config, TEE_NET_RES *netRes, KCAL_AlgorithmsType type) +std::shared_ptr CreateKCALContext(KCAL_Config config, TEE_NET_RES *netRes, KCAL_AlgorithmsType type) { auto context = std::make_shared(config, type); context->Init(); diff --git a/MPC/middleware/kcal/core/context.h b/MPC/middleware/kcal/core/context.h index 4b7e0f2..5684e05 100644 --- a/MPC/middleware/kcal/core/context.h +++ b/MPC/middleware/kcal/core/context.h @@ -50,7 +50,7 @@ private: DG_ConfigOpts *cfgOpts_ = nullptr; }; -std::shared_ptr CreateContext(KCAL_Config config, TEE_NET_RES *netRes, KCAL_AlgorithmsType type); +std::shared_ptr CreateKCALContext(KCAL_Config config, TEE_NET_RES *netRes, KCAL_AlgorithmsType type); } -- Gitee From 2252ef565e582acf6b8ff6094f1535b59f538e46 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Mon, 28 Jul 2025 15:48:35 +0800 Subject: [PATCH 07/18] Add pir impl --- MPC/middleware/kcal/core/context.cc | 14 +++--- MPC/middleware/kcal/core/context.h | 7 ++- MPC/middleware/kcal/operator/kcal_pir.cc | 64 ++++++++++++++++++++++++ MPC/middleware/kcal/operator/kcal_pir.h | 43 ++++++++++++++++ MPC/middleware/kcal/operator/kcal_psi.h | 5 +- 5 files changed, 122 insertions(+), 11 deletions(-) create mode 100644 MPC/middleware/kcal/operator/kcal_pir.cc create mode 100644 MPC/middleware/kcal/operator/kcal_pir.h diff --git a/MPC/middleware/kcal/core/context.cc b/MPC/middleware/kcal/core/context.cc index cca6a88..23ce3dc 100644 --- a/MPC/middleware/kcal/core/context.cc +++ b/MPC/middleware/kcal/core/context.cc @@ -44,13 +44,7 @@ int Context::Init() return DG_SUCCESS; } -void Context::SetNetRes(TEE_NET_RES *teeNetRes) -{ - DG_Void netFunc = {.data = teeNetRes, .size = sizeof(TEE_NET_RES)}; - cfgOpts_->setVoidValue(teeCfg_, DG_CON_MPC_TEE_VOID_NET_API, &netFunc); -} - -std::shared_ptr CreateKCALContext(KCAL_Config config, TEE_NET_RES *netRes, KCAL_AlgorithmsType type) +std::shared_ptr Context::Create(KCAL_Config config, TEE_NET_RES *netRes, KCAL_AlgorithmsType type) { auto context = std::make_shared(config, type); context->Init(); @@ -58,4 +52,10 @@ std::shared_ptr CreateKCALContext(KCAL_Config config, TEE_NET_RES *netR return context; } +void Context::SetNetRes(TEE_NET_RES *teeNetRes) +{ + DG_Void netFunc = {.data = teeNetRes, .size = sizeof(TEE_NET_RES)}; + cfgOpts_->setVoidValue(teeCfg_, DG_CON_MPC_TEE_VOID_NET_API, &netFunc); +} + } diff --git a/MPC/middleware/kcal/core/context.h b/MPC/middleware/kcal/core/context.h index 5684e05..a065578 100644 --- a/MPC/middleware/kcal/core/context.h +++ b/MPC/middleware/kcal/core/context.h @@ -32,8 +32,13 @@ public: explicit Context(KCAL_Config config, KCAL_AlgorithmsType type); + Context(const Context &) = delete; + Context &operator=(const Context &) = delete; + ~Context(); + static std::shared_ptr Create(KCAL_Config config, TEE_NET_RES *netRes, KCAL_AlgorithmsType type); + int Init(); [[nodiscard]] int GetWorldSize() const { return config_.worldSize; } @@ -50,8 +55,6 @@ private: DG_ConfigOpts *cfgOpts_ = nullptr; }; -std::shared_ptr CreateKCALContext(KCAL_Config config, TEE_NET_RES *netRes, KCAL_AlgorithmsType type); - } #endif //CONTEXT_H diff --git a/MPC/middleware/kcal/operator/kcal_pir.cc b/MPC/middleware/kcal/operator/kcal_pir.cc new file mode 100644 index 0000000..9c1a56b --- /dev/null +++ b/MPC/middleware/kcal/operator/kcal_pir.cc @@ -0,0 +1,64 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * virtCCA_sdk is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#include "kcal/operator/kcal_pir.h" +#include "kcal/utils/node_info_helper.h" + +namespace kcal { + +Pir::Pir() +{ + opts_ = std::make_unique(); + *opts_ = DG_InitPirOpts(); +} + +Pir::~Pir() +{ + opts_->releaseBucketMap(&bucketMap_); + opts_->releaseTeeCtx(&dgTeeCtx_); +} + +int Pir::Init(std::shared_ptr ctx) +{ + utils::NodeInfoHelper nodeInfoHelper(ctx->GetWorldSize()); + + int rv = opts_->initTeeCtx(ctx->GetTeeConfig(), &dgTeeCtx_); + if (rv != 0) { + return rv; + } + + rv = opts_->setTeeNodeInfos(dgTeeCtx_, nodeInfoHelper.Get()); + if (rv != 0) { + return rv; + } + + baseCtx_ = std::move(ctx); + + return DG_SUCCESS; +} + +int Pir::ServerPreProcess(DG_PairList *pairList) +{ + return opts_->offlineCalculate(dgTeeCtx_, pairList, &bucketMap_); +} + +int Pir::ClientQuery(DG_Input *input, DG_TeeOutput **output, DG_DummyMode dummyMode) +{ + return opts_->clientCalculate(dgTeeCtx_, dummyMode, input, output); +} + +int Pir::ServerAnswer() +{ + return opts_->serverCalculate(dgTeeCtx_, bucketMap_); +} + +} diff --git a/MPC/middleware/kcal/operator/kcal_pir.h b/MPC/middleware/kcal/operator/kcal_pir.h new file mode 100644 index 0000000..b0b6091 --- /dev/null +++ b/MPC/middleware/kcal/operator/kcal_pir.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. + * virtCCA_sdk is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +#ifndef KCAL_PIR_H +#define KCAL_PIR_H + +#include +#include "kcal/core/context.h" + +namespace kcal { + +class pir { +public: + Pir(); + ~Pir(); + + Pir(const Pir &) = delete; + Pir &operator=(const Pir &) = delete; + + int Init(std::shared_ptr ctx); + int ServerPreProcess(DG_PairList *pairList); + int ClientQuery(DG_Input *input, DG_TeeOutput **output, DG_DummyMode dummyMode); + int ServerAnswer(); + +private: + DG_TeeCtx *dgTeeCtx_ = nullptr; + std::shared_ptr baseCtx_; + std::unique_ptr opts_; + DG_BucketMap *bucketMap_ = nullptr; +}; + +} + +#endif //KCAL_PIR_H diff --git a/MPC/middleware/kcal/operator/kcal_psi.h b/MPC/middleware/kcal/operator/kcal_psi.h index 4b8b028..294d1be 100644 --- a/MPC/middleware/kcal/operator/kcal_psi.h +++ b/MPC/middleware/kcal/operator/kcal_psi.h @@ -22,11 +22,12 @@ namespace kcal { class Psi { public: Psi(); - ~Psi(); - int Init(std::shared_ptr ctx); + Psi(const Psi &) = delete; + Psi &operator=(const Psi &) = delete; + int Init(std::shared_ptr ctx); int Run(DG_TeeInput *input, DG_TeeOutput **output, DG_TeeMode outputMode); private: -- Gitee From 2d30bf85e39423155212d546c3b268b7a011655f Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 29 Jul 2025 10:14:17 +0800 Subject: [PATCH 08/18] Modify patch, add pir --- .../secrerflow/psi/patches/kcal.patch | 1227 +++++++++++++---- 1 file changed, 978 insertions(+), 249 deletions(-) diff --git a/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch index 8be6237..330159e 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch +++ b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch @@ -50,6 +50,98 @@ index 0000000..8416212 + "@//third_party:libkcal" + ] +) +diff --git a/examples/pir/config/apsi_receiver.json b/examples/pir/config/apsi_receiver.json +index 2c7f56b..f62177a 100644 +--- a/examples/pir/config/apsi_receiver.json ++++ b/examples/pir/config/apsi_receiver.json +@@ -3,7 +3,7 @@ + "threads": 1, + "query_file": "/tmp/query.csv", + "output_file": "/tmp/result.csv", +- "params_file": "/tmp/100K-1-16.json", ++ "params_file": "/tmp/1M-1024-cmp.json", + "log_level": "info" + }, + "link_config": { +diff --git a/examples/pir/config/apsi_sender_online.json b/examples/pir/config/apsi_sender_online.json +index b72e2e8..7444e5c 100644 +--- a/examples/pir/config/apsi_sender_online.json ++++ b/examples/pir/config/apsi_sender_online.json +@@ -1,6 +1,6 @@ + { + "apsi_sender_config": { +- "threads": 1, ++ "threads": 16, + "db_file": "/tmp/sdb", + "log_level": "info" + }, +diff --git a/examples/pir/config/apsi_sender_setup.json b/examples/pir/config/apsi_sender_setup.json +index e3e6991..c35b8da 100644 +--- a/examples/pir/config/apsi_sender_setup.json ++++ b/examples/pir/config/apsi_sender_setup.json +@@ -1,7 +1,7 @@ + { + "apsi_sender_config": { + "source_file": "/tmp/db.csv", +- "params_file": "/tmp/100K-1-16.json", ++ "params_file": "/tmp/1M-1024-cmp.json", + "sdb_out_file": "/tmp/sdb", + "save_db_only": true + } +diff --git a/examples/pir/config/kcal_pir_receiver.json b/examples/pir/config/kcal_pir_receiver.json +new file mode 100644 +index 0000000..14e97e6 +--- /dev/null ++++ b/examples/pir/config/kcal_pir_receiver.json +@@ -0,0 +1,21 @@ ++{ ++ "kcal_pir_receiver_config": { ++ "threads": 16, ++ "query_file": "/tmp/query.csv", ++ "output_file": "/tmp/result.csv", ++ "is_dummy_mode": true ++ }, ++ "link_config": { ++ "parties": [ ++ { ++ "id": "sender", ++ "host": "127.0.0.1:5300" ++ }, ++ { ++ "id": "receiver", ++ "host": "127.0.0.1:5400" ++ } ++ ] ++ }, ++ "self_link_party": "receiver" ++} +\ No newline at end of file +diff --git a/examples/pir/config/kcal_pir_sender.json b/examples/pir/config/kcal_pir_sender.json +new file mode 100644 +index 0000000..c2579cb +--- /dev/null ++++ b/examples/pir/config/kcal_pir_sender.json +@@ -0,0 +1,19 @@ ++{ ++ "kcal_pir_sender_config": { ++ "threads": 16, ++ "db_file": "/tmp/db.csv" ++ }, ++ "link_config": { ++ "parties": [ ++ { ++ "id": "sender", ++ "host": "127.0.0.1:5300" ++ }, ++ { ++ "id": "receiver", ++ "host": "127.0.0.1:5400" ++ } ++ ] ++ }, ++ "self_link_party": "sender" ++} +\ No newline at end of file diff --git a/examples/psi/config/kcal_receiver.json b/examples/psi/config/kcal_receiver.json new file mode 100644 index 0000000..72e2694 @@ -146,300 +238,511 @@ index 0000000..34abca9 + }, + "self_link_party": "sender" +} -diff --git a/psi/algorithm/kcal/BUILD.bazel b/psi/algorithm/kcal/BUILD.bazel +diff --git a/examples/psi/config/rr22_receiver_recovery.json b/examples/psi/config/rr22_receiver_recovery.json +index 9df3437..24acf48 100644 +--- a/examples/psi/config/rr22_receiver_recovery.json ++++ b/examples/psi/config/rr22_receiver_recovery.json +@@ -3,7 +3,10 @@ + "protocol_config": { + "protocol": "PROTOCOL_RR22", + "role": "ROLE_RECEIVER", +- "broadcast_result": true ++ "broadcast_result": true, ++ "rr22_config": { ++ "low_comm_mode": true ++ } + }, + "input_config": { + "type": "IO_TYPE_FILE_CSV", +diff --git a/examples/psi/config/rr22_sender_recovery.json b/examples/psi/config/rr22_sender_recovery.json +index d840dfd..0033330 100644 +--- a/examples/psi/config/rr22_sender_recovery.json ++++ b/examples/psi/config/rr22_sender_recovery.json +@@ -3,7 +3,10 @@ + "protocol_config": { + "protocol": "PROTOCOL_RR22", + "role": "ROLE_SENDER", +- "broadcast_result": true ++ "broadcast_result": true, ++ "rr22_config": { ++ "low_comm_mode": true ++ } + }, + "input_config": { + "type": "IO_TYPE_FILE_CSV", +diff --git a/psi/algorithm/kcal_pir/BUILD.bazel b/psi/algorithm/kcal_pir/BUILD.bazel new file mode 100644 -index 0000000..921797b +index 0000000..7bd55e4 --- /dev/null -+++ b/psi/algorithm/kcal/BUILD.bazel -@@ -0,0 +1,56 @@ ++++ b/psi/algorithm/kcal_pir/BUILD.bazel +@@ -0,0 +1,28 @@ +load("//bazel:psi.bzl", "psi_cc_binary", "psi_cc_library", "psi_cc_test") + +package(default_visibility = ["//visibility:public"]) + +psi_cc_library( -+ name = "global_manager", -+ hdrs = ["global_manager.h"], -+ srcs = ["global_manager.cc"], -+ deps = [ -+ "@yacl//yacl/link", -+ ] -+) -+ -+psi_cc_library( -+ name = "link_api", -+ hdrs = ["link_api.h"], -+ srcs = ["link_api.cc"], ++ name = "pir_receiver", ++ srcs = ["pir_receiver.cc"], ++ hdrs = ["pir_receiver.h"], + deps = [ -+ ":global_manager", -+ "@kunpeng_kcal_middleware//:kcal_middleware", -+ ] -+) -+ -+psi_cc_library( -+ name = "data_adaptor", -+ hdrs = ["data_adaptor.h"], -+ srcs = ["data_adaptor.cc"], -+ deps = [ -+ "//psi/utils:batch_provider", -+ "@kunpeng_kcal_middleware//:kcal_middleware", -+ ] -+) -+ -+psi_cc_library( -+ name = "receiver", -+ srcs = ["receiver.cc"], -+ hdrs = ["receiver.h"], -+ deps = [ -+ ":link_api", -+ ":data_adaptor", -+ "//psi:interface", ++ "//psi/kcal_adaptor:data_helper", ++ "//psi/kcal_adaptor:traffic_wrapper", ++ "//psi/kcal_adaptor:global_manager", + "//psi/utils:arrow_csv_batch_provider", ++ "@Kunpeng_kcal_middleware//:kcal_middleware", + ] +) + +psi_cc_library( -+ name = "sender", -+ srcs = ["sender.cc"], -+ hdrs = ["sender.h"], ++ name = "pir_sender", ++ srcs = ["pir_sender.cc"], ++ hdrs = ["pir_sender.h"], + deps = [ -+ ":link_api", -+ ":data_adaptor", -+ "//psi:interface", -+ "//psi/utils:arrow_csv_batch_provider", ++ "//psi/kcal_adaptor:traffic_wrapper", ++ "//psi/kcal_adaptor:global_manager", ++ "//psi/utils/arrow_csv_batch_provider", ++ "@Kunpeng_kcal_middleware//:kcal_middleware", + ] +) -diff --git a/psi/algorithm/kcal/data_adaptor.cc b/psi/algorithm/kcal/data_adaptor.cc +diff --git a/psi/algorithm/kcal_pir/pir_receiver.cc b/psi/algorithm/kcal_pir/pir_receiver.cc new file mode 100644 -index 0000000..74332da +index 0000000..92e1aec --- /dev/null -+++ b/psi/algorithm/kcal/data_adaptor.cc -@@ -0,0 +1,65 @@ -+#include "psi/algorithm/kcal/data_adaptor.h" ++++ b/psi/algorithm/kcal_pir/pir_receiver.cc +@@ -0,0 +1,108 @@ ++#include "psi/algorithm/kcal_pir/pir_receiver.h" + -+#include ++#include + -+#include "spdlog/spdlog.h" ++#include "psi/kcal_adaptor/data_helper.h" ++#include "psi/kcal_adaptor/global_manager.h" ++#include "psi/kcal_adaptor/traffic_wrapper.h" ++#include "psi/utils/arrow_csv_batch_provider.h" + -+namespace psi::kcal { ++namespace psi::kcal::pir { + -+DG_TeeInput* DataAdaptor::TransData( -+ std::shared_ptr provider) { -+ std::vector rawData; -+ while (true) { -+ auto data = provider->ReadNextBatch(); -+ if (data.empty()) { -+ break; -+ } -+ rawData.insert(rawData.end(), data.begin(), data.end()); -+ } ++KcalPirReceiver::KcalPirReceiver(const ReceiverOptions& options, ++ std::shared_ptr lctx) ++ : lctx_(std::move(lctx)) { ++ options_ = std::make_shared(); ++ *options_ = options; ++ pir_ = std::make_unique<::kcal::Pir>(); ++} + -+ DG_String* strings = nullptr; -+ BuildDgString(rawData, &strings); ++void KcalPirReceiver::Init() { ++ SPDLOG_INFO("Kcal pir receiver init."); ++ lctx_->ConnectToMesh(); ++ GlobalManager::GetInstance().InitializeCtx(lctx_); ++ ::kcal::KCAL_Config kcal_cfg; ++ kcal_cfg.fixBits = 1; ++ kcal_cfg.nodeId = lctx_->Rank(); ++ kcal_cfg.threadCount = options_->thread_count; ++ kcal_cfg.worldSize = lctx_->WorldSize(); + -+ input_ = new DG_TeeInput(); -+ input_->data.strings = strings; -+ input_->size = static_cast(rawData.size()); -+ input_->dataType = MPC_STRING; ++ TEE_NET_RES net_res; ++ net_res.funcRecvData = GlobalRecvWrapper; ++ net_res.funcSendData = GlobalSendWrapper; + -+ return input_; ++ kcal_ctx_ = ::kcal::Context::Create(kcal_cfg, &net_res, ++ ::kcal::KCAL_AlgorithmsType::PIR); ++ ++ pir_->Init(kcal_ctx_); +} + -+DataAdaptor::~DataAdaptor() { ReleaseOutput(&input_); } ++int KcalPirReceiver::PreProcess() { ++ SPDLOG_INFO("Kcal pir receiver preprocess."); ++ std::vector keys{"key"}; ++ size_t batch_size = 1 << 20; ++ auto key_data = ReadPirQuery(options_->query_file, keys, batch_size); + -+void DataAdaptor::BuildDgString(std::vector& strings, -+ DG_String** dg) { -+ auto* dgString = new DG_String[strings.size()]; -+ for (size_t i = 0; i < strings.size(); ++i) { -+ dgString[i].str = strdup(strings[i].c_str()); -+ dgString[i].size = strings[i].size() + 1; -+ } -+ *dg = dgString; ++ input_ = std::make_unique(); ++ input_->Fill(key_data); ++ SPDLOG_INFO("Kcal pir receiver input size = {}.", input_->Size()); ++ return 0; +} + -+void DataAdaptor::ReleaseOutput(DG_TeeOutput** output) { -+ if (output == nullptr || *output == nullptr) { -+ return; ++int KcalPirReceiver::Connect() { ++ yacl::link::Barrier(lctx_, "pir preprocess"); ++ return 0; ++} ++ ++int KcalPirReceiver::Online(DG_DummyMode dummyMode) { ++ SPDLOG_INFO("Kcal pir receiver online query."); ++ output_ = std::make_unique(); ++ int ret = pir_->ClientQuery(input_->Get(), output_->GetSecondaryPointer(), ++ dummyMode); ++ SPDLOG_INFO("Kcal pir receiver query ret = {}.", ret); ++ return ret; ++} ++ ++int KcalPirReceiver::PostProcess(int* match_cnt) { ++ SPDLOG_INFO("Kcal pir receiver postprocess."); ++ *match_cnt = output_->Size(); ++ SPDLOG_INFO("Kcal pir result size = {}.", *match_cnt); ++ ++ std::ofstream outFile(options_->result_file); ++ if (!outFile.is_open()) { ++ SPDLOG_INFO("Cannot open result file {}.", options_->result_file); ++ return -1; + } -+ if ((*output)->dataType == MPC_STRING && (*output)->data.strings != nullptr) { -+ for (int i = 0; i < (*output)->size; ++i) { -+ if ((*output)->data.strings[i].str != nullptr) { -+ delete[](*output)->data.strings[i].str; -+ } -+ } -+ delete[](*output)->data.strings; -+ } else if ((*output)->dataType == MPC_DOUBLE && -+ (*output)->data.doubleNumbers != nullptr) { -+ delete[](*output)->data.doubleNumbers; -+ } else if ((*output)->dataType == MPC_INT && -+ (*output)->data.u64Numbers != nullptr) { -+ delete[](*output)->data.u64Numbers; ++ outFile << "key,value\n"; ++ size_t str_nums = output_->Size(); ++ for (size_t i = 0; i < str_nums; ++i) { ++ outFile << input_->Get()->data.strings[i].str << ','; ++ outFile << output_->Get()->data.strings[i].str << '\n'; + } -+ delete *output; -+ *output = nullptr; ++ outFile.close(); ++ return 0; +} + -+} // namespace psi::kcal -\ No newline at end of file -diff --git a/psi/algorithm/kcal/data_adaptor.h b/psi/algorithm/kcal/data_adaptor.h ++int RunKcalReceiver(const ReceiverOptions& options, ++ const std::shared_ptr& lctx = nullptr, ++ int* match_cnt = nullptr) { ++ KcalPirReceiver pir_receiver(options, lctx); ++ pir_receiver.Init(); ++ pir_receiver.PreProcess(); ++ pir_receiver.Connect(); ++ auto start_time = std::chrono::steady_clock()::now(); ++ DG_DummyMode dummy_mode = NORMAL; ++ if (options.is_dummy_mode) { ++ dummy_mode = DUMMY; ++ } ++ pir_receiver.Online(dummy_mode); ++ auto end_time = std::chrono::steady_clock::now(); ++ auto dur = std::chrono::duraction_cast(end_time - ++ start_time) ++ .count(); ++ SPDLOG_INFO("Online query time {} ms", dur); ++ pir_receiver.PostProcess(); ++ return 0; ++} ++ ++} // namespace psi::kcal::pir +diff --git a/psi/algorithm/kcal_pir/pir_receiver.h b/psi/algorithm/kcal_pir/pir_receiver.h new file mode 100644 -index 0000000..6ea1a88 +index 0000000..98de787 --- /dev/null -+++ b/psi/algorithm/kcal/data_adaptor.h -@@ -0,0 +1,30 @@ ++++ b/psi/algorithm/kcal_pir/pir_receiver.h +@@ -0,0 +1,48 @@ +#pragma once + -+#include -+#include ++#include "kcal/core/context.h" ++#include "kcal/operator/kcal_pir.h" ++#include "yacl/link/link.h" + -+#include "kcal/api/kcal_api.h" ++#include "psi/kcal_adaptor/data_helper.h" + -+#include "psi/utils/batch_provider.h" ++namespace psi::kcal::pir { + -+namespace psi::kcal { ++struct ReceiverOptions { ++ int thread_count = 1; ++ std::string query_file; ++ std::string result_file; ++ bool is_dummy_mode = false; ++}; + -+class DataAdaptor { ++class KcalPirReceiver { + public: -+ DataAdaptor() = default; -+ -+ ~DataAdaptor(); -+ -+ DG_TeeInput* TransData(std::shared_ptr provider); ++ explicit KcalPirReceiver(const ReceiverOptions& options, ++ std::shared_ptr lctx = nullptr); + -+ DG_TeeInput* GetInput() { return input_; } ++ ~KcalPirReceiver() = default; + -+ static void ReleaseOutput(DG_TeeOutput** output); ++ void Init(); ++ int PreProcess(); ++ int Connect(); ++ int Online(DG_DummyMode dummyMode = NORMAL); ++ int PostProcess(int* match_cnt); + + private: -+ void BuildDgString(std::vector& strings, DG_String** dg); ++ static std::vector ReadPirQuery( ++ const std::string& file_path, const std::vector& keys, ++ size_t batch_size, char delimiter = ','); + -+ DG_TeeInput* input_; ++ std::shared_ptr lctx_; ++ std::shared_ptr<::kcal::Context> kcal_ctx_; ++ std::unique_ptr input_; ++ std::unique_ptr output_; ++ std::shared_ptr options_; ++ std::unique_ptr<::kcal::Pir> pir_; +}; + -+} // namespace psi::kcal -diff --git a/psi/algorithm/kcal/global_manager.cc b/psi/algorithm/kcal/global_manager.cc ++int RunKcalReceiver(const ReceiverOptions& options, ++ const std::shared_ptr& lctx = nullptr, ++ int* match_cnt = nullptr); ++ ++} // namespace psi::kcal::pir +diff --git a/psi/algorithm/kcal_pir/pir_sender.cc b/psi/algorithm/kcal_pir/pir_sender.cc new file mode 100644 -index 0000000..061a1f5 +index 0000000..6a2a0a8 --- /dev/null -+++ b/psi/algorithm/kcal/global_manager.cc -@@ -0,0 +1,19 @@ -+#include "psi/algorithm/kcal/global_manager.h" ++++ b/psi/algorithm/kcal_pir/pir_sender.cc +@@ -0,0 +1,156 @@ ++#include "psi/algorithm/kcal_pir/pir_sender.h" + -+namespace psi::kcal { ++#include "psi/kcal_adaptor/global_manager.h" ++#include "psi/kcal_adaptor/traffic_wrapper.h" ++#include "psi/utils/arrow_csv_batch_provider.h" + -+GlobalManager& GlobalManager::GetInstance() { -+ static GlobalManager instance; -+ return instance; -+} ++namespace psi::kcal::pir { + -+void GlobalManager::InitializeCtx( -+ const std::shared_ptr& lctx) { -+ lctx_ = lctx; ++KcalPirSender::KcalPirSender(const SenderOptions& options, ++ std::shared_ptr lctx) ++ : lctx_(std::move(lctx)) { ++ options_ = std::make_shared(); ++ *options_ = options; ++ pir_ = std::make_shared<::kcal::Pir>(); +} + -+std::shared_ptr& GlobalManager::GetLink() { return lctx_; } -+ -+void GlobalManager::CleanLink() { lctx_.reset(); } -+ -+} // namespace psi::kcal -\ No newline at end of file -diff --git a/psi/algorithm/kcal/global_manager.h b/psi/algorithm/kcal/global_manager.h -new file mode 100644 -index 0000000..c596b0f ---- /dev/null -+++ b/psi/algorithm/kcal/global_manager.h -@@ -0,0 +1,25 @@ -+#pragma once ++void KcalPirSender::Init() { ++ SPDLOG_INFO("Kcal pir sender init."); ++ lctx_->ConnectToMesh(); ++ GlobalManager::GetInstance().InitializeCtx(lctx_); + -+#include "yacl/link/link.h" ++ ::kcal::KCAL_Config kcal_cfg_{}; ++ kcal_cfg.fixBits = 1; ++ kcal_cfg.nodeId = lctx_->Rank(); ++ kcal_cfg.threadCount = options_->thread_count; ++ kcal_cfg.worldSize = lctx_->WorldSize(); + -+namespace psi::kcal { ++ TEE_NET_RES net_res; ++ net_res.funcRecvData = GlobalRecvWrapper; ++ net_res.funcSendData = GlobalSendWrapper; + -+class GlobalManager { -+ public: -+ static GlobalManager& GetInstance(); ++ kcal_ctx_ = ::kcal::Context::Create(kcal_cfg, &net_res, ++ ::kcal::KCAL_AlgorithmsType::PIR); ++ pir_->Init(kcal_ctx_); ++} + -+ ~GlobalManager() = default; ++int KcalPirSender::PreProcess() { ++ SPDLOG_INFO("Kcal pir sender preprocess."); ++ std::vector keys{"key"}; ++ std::vector labels{"value"}; ++ size_t batch_size = 1 << 20; ++ auto key_label = ReadPirDataSet(options_->db_file, keys, batch_size, labels); ++ ++ DG_PairList* pair_list = nullptr; ++ BuildDgPairList(key_label, &pair_list); ++ SPDLOG_INFO("Kcal pir sender db size = {}.", pair_list->size); ++ int ret = pir_->ServerPreProcess(pair_list); ++ ReleaseDgPairList(pair_list); ++ SPDLOG_INFO("Kcal pir sender preprocess ret = {}.", ret); ++ return ret; ++} + -+ void InitializeCtx(const std::shared_ptr& lctx); ++int KcalPirSender::Connect() { ++ yacl::link::Barrier(lctx_, "pir preprocess"); ++ return 0; ++} + -+ std::shared_ptr& GetLink(); ++int KcalPirSender::Online() { ++ SPDLOG_INFO("Kcal pir sender answer."); ++ int ret = pir_->ServerAnswer(); ++ SPDLOG_INFO("Kcal pir sender answer ret = {}.", ret); ++ return ret; ++} + -+ void CleanLink(); ++int KcalPirSender::PostProcess() { ++ SPDLOG_INFO("Kcal pir sender postprocess"); ++ return 0; ++} + -+ private: -+ GlobalManager() = default; ++std::pair, std::vector> ++KcalPirSender::ReadPirDataSet(const std::string& file_path, ++ const std::vector& keys, ++ size_t batch_size, ++ const std::vector& labels, ++ char delimiter) { ++ psi::ArrowCsvBatchProvider csv_batch_provider(file_path, keys, batch_size, ++ labels, delimiter); ++ std::vector key_data; ++ std::vector label_data; ++ bool is_reach_end = false; ++ while (!is_reach_end) { ++ auto batch = csv_batch_provider.ReadNextLabeledBatch(); ++ if (batch.first.size() < batch_size) { ++ is_reach_end = true; ++ } + -+ std::shared_ptr lctx_; -+}; ++ if (batch.first.size() > 0) { ++ key_data.insert(key_data.end(), batch.first.begin(), batch.first.end()); ++ label_data.insert(label_data.end(), batch.second.begin(), ++ batch.second.end()); ++ } ++ } ++ return {key_data, label_data}; ++} + -+} // namespace psi::kcal -diff --git a/psi/algorithm/kcal/link_api.cc b/psi/algorithm/kcal/link_api.cc -new file mode 100644 -index 0000000..2d57536 ---- /dev/null -+++ b/psi/algorithm/kcal/link_api.cc -@@ -0,0 +1,26 @@ -+#include "psi/algorithm/kcal/link_api.h" ++void KcalPirSender::ReleasePairString(DG_String* dgString) { ++ if (dgString != nullptr) { ++ delete[] dgString->str; ++ delete dgString; ++ } ++} + -+#include "psi/algorithm/kcal/global_manager.h" ++void KcalPirSender::ReleasePairList(DG_PairList* pairList) { ++ if (pairList != nullptr) { ++ for (unsigned long i = 0; i < pairList->size; ++i) { ++ ReleasePairString(pairList->dgPair[i].key); ++ ReleasePairString(pairList->dgPair[i].value); ++ } ++ } ++ delete[] pairList->dgPair; ++ delete pairList; ++} + -+namespace psi::kcal { ++DG_String* KcalPirSender::BuildPairString(const std::string& s) { ++ auto* dgString = new DG_string(); ++ dgString->size = s.size() + 1; ++ dgString->str = new char[dgString->size + 1]; ++ dgString->str[dgString->size] = '\0'; ++ strcpy(dgString->str, s.c_str()); ++ return dgString; ++} + -+int GlobalSendWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, -+ unsigned long long len) { -+ yacl::Buffer msg(reinterpret_cast(buf), len); -+ std::string tag = "kcal_send"; -+ auto lctx = GlobalManager::GetInstance().GetLink(); -+ lctx->SendAsync(nodeInfo->nodeId, msg, tag); -+ return 0; ++void KcalPirSender::BuildDgPairList( ++ const std::pair, std::vector>& ++ key_label, ++ DG_PairList** pair_list) { ++ *pair_list = new DG_PairList(); ++ size_t size = key_label.first.size(); ++ (*pair_list)->dgPair = new DG_Pair[size]; ++ ++ for (unsigned long i = 0; i < size; ++i) { ++ (*pair_list)->dgPair[i].key = BuildPairString(key_label.first[i]); ++ (*pair_list)->dgPair[i].value = BuildPairString(key_label.second[i]); ++ } ++ (*pair_list)->size = size; +} + -+int GlobalRecvWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, -+ unsigned long long* len) { -+ std::string tag = "kcal_recv"; -+ auto lctx = GlobalManager::GetInstance().GetLink(); -+ auto msg = lctx->Recv(nodeInfo->nodeId, tag); -+ memcpy(buf, msg.data(), msg.size()); -+ *len = msg.size(); ++int RunKcalSender(const SenderOptions& options, ++ const std::shared_ptr& lctx = nullptr) { ++ KcalPirSender pir_sender(options, lctx); ++ ++ pir_sender.Init(); ++ pir_sender.PreProcess(); ++ pir_sender.Connect(); ++ auto start_time = std::chrono::steady_clock::now(); ++ pir_sender.Online(); ++ auto end_time = std::chrono::steady_clock::now(); ++ auto dur = std::chrono::duraction_cast(end_time - ++ start_time) ++ .count(); ++ SPDLOG_INFO("Online query time {} ms", dur); ++ pir_sender.PostProcess(); + return 0; +} + -+} // namespace psi::kcal -diff --git a/psi/algorithm/kcal/link_api.h b/psi/algorithm/kcal/link_api.h ++} // namespace psi::kcal::pir +diff --git a/psi/algorithm/kcal_pir/pir_sender.h b/psi/algorithm/kcal_pir/pir_sender.h new file mode 100644 -index 0000000..2dec56f +index 0000000..efb9f9e --- /dev/null -+++ b/psi/algorithm/kcal/link_api.h -@@ -0,0 +1,15 @@ ++++ b/psi/algorithm/kcal_pir/pir_sender.h +@@ -0,0 +1,55 @@ +#pragma once + -+#include ++#include "kcal/core/context.h" ++#include "kcal/operator/kcal_pir.h" ++#include "yacl/link/link.h" + -+#include "kcal/api/kcal_api.h" ++namespace psi::kcal::pir { + -+namespace psi::kcal { ++struct SenderOptions { ++ int thread_count = 1; ++ std::string db_file; ++}; + -+int GlobalSendWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, -+ unsigned long long len); ++class KcalPirSender { ++ public: ++ explicit KcalPirSender(const SenderOptions& options, ++ std::shared_ptr lctx = nullptr); + -+int GlobalRecvWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, -+ unsigned long long* len); ++ ~KcalPirSender() = default; ++ ++ void Init(); ++ int PreProcess(); ++ int Connect(); ++ int Online(); ++ int PostProcess(); ++ ++ private: ++ static std::pair, std::vector> ++ ReadPirDataSet(const std::string& file_path, ++ const std::vector& keys, ++ size_t batch_size = 1 << 20, ++ const std::vector& labels = {}, ++ char delimiter = ','); ++ ++ static void ReleasePairString(DG_String* dgString); ++ ++ static void ReleasePairList(DG_PairList* pairList); ++ ++ static DG_String* BuildPairString(const std::string& s); ++ ++ static void BuildDgPairList( ++ const std::pair, std::vector>& ++ key_label, ++ DG_PairList** pair_list); ++ ++ std::shared_ptr lctx_; ++ std::shared_ptr<::kcal::Context> kcal_ctx_; ++ std::shared_ptr options_; ++ std::unique_ptr<::kcal::Pir> pir_; ++}; ++ ++int RunKcalSender(const SenderOptions& options, ++ const std::shared_ptr& lctx = nullptr); ++ ++} // namespace psi::kcal::pir +diff --git a/psi/algorithm/kcal_psi/BUILD.bazel b/psi/algorithm/kcal_psi/BUILD.bazel +new file mode 100644 +index 0000000..14dd8df +--- /dev/null ++++ b/psi/algorithm/kcal_psi/BUILD.bazel +@@ -0,0 +1,27 @@ ++load("//bazel:psi.bzl", "psi_cc_binary", "psi_cc_library", "psi_cc_test") ++ ++package(default_visibility = ["//visibility:public"]) ++ ++psi_cc_library( ++ name = "receiver", ++ srcs = ["receiver.cc"], ++ hdrs = ["receiver.h"], ++ deps = [ ++ "//psi/kcal_adaptor:data_helper", ++ "//psi/kcal_adaptor:traffic_wrapper", ++ "//psi:interface", ++ "//psi/utils:arrow_csv_batch_provider", ++ ] ++) + -+} // namespace psi::kcal -\ No newline at end of file -diff --git a/psi/algorithm/kcal/receiver.cc b/psi/algorithm/kcal/receiver.cc ++psi_cc_library( ++ name = "sender", ++ srcs = ["sender.cc"], ++ hdrs = ["sender.h"], ++ deps = [ ++ "//psi/kcal_adaptor:data_helper", ++ "//psi/kcal_adaptor:traffic_wrapper", ++ "//psi:interface", ++ "//psi/utils:arrow_csv_batch_provider", ++ ] ++) +diff --git a/psi/algorithm/kcal_psi/receiver.cc b/psi/algorithm/kcal_psi/receiver.cc new file mode 100644 -index 0000000..119a0b1 +index 0000000..035b52e --- /dev/null -+++ b/psi/algorithm/kcal/receiver.cc -@@ -0,0 +1,53 @@ ++++ b/psi/algorithm/kcal_psi/receiver.cc +@@ -0,0 +1,62 @@ +#include "psi/algorithm/kcal/receiver.h" + ++#include ++ +#include "kcal/operator/kcal_psi.h" + -+#include "psi/algorithm/kcal/data_adaptor.h" -+#include "psi/algorithm/kcal/global_manager.h" -+#include "psi/algorithm/kcal/link_api.h" ++#include "psi/kcal_adaptor/data_helper.h" ++#include "psi/kcal_adaptor/global_manager.h" ++#include "psi/kcal_adaptor/traffic_wrapper.h" + +namespace psi::kcal { + @@ -453,7 +756,7 @@ index 0000000..119a0b1 + GlobalManager::GetInstance().InitializeCtx(lctx_); + // 配置 + ::kcal::KCAL_Config kcal_cfg; -+ kcal_cfg.fixBits = 0; ++ kcal_cfg.fixBits = 1; + kcal_cfg.nodeId = lctx_->Rank(); + kcal_cfg.threadCount = config_.protocol_config().kcal_config().thread_count(); + kcal_cfg.worldSize = lctx_->WorldSize(); @@ -462,43 +765,50 @@ index 0000000..119a0b1 + net_res.funcRecvData = GlobalRecvWrapper; + net_res.funcSendData = GlobalSendWrapper; + // 计算上下文 -+ kcal_ctx_ = ::kcal::CreateKCALContext(kcal_cfg, &net_res, -+ ::kcal::KCAL_AlgorithmsType::PSI); ++ kcal_ctx_ = ::kcal::Context::Create(kcal_cfg, &net_res, ++ ::kcal::KCAL_AlgorithmsType::PSI); +} + +void KcalPsiReceiver::PreProcess() { -+ input_ = std::make_shared(); -+ input_->TransData(batch_provider_); ++ input_ = std::make_unique(); ++ input_->Fill(batch_provider_); +} + +void KcalPsiReceiver::Online() { ++ auto start_time = std::chrono::high_resolution_clock()::now(); + ::kcal::Psi psi; + psi.Init(kcal_ctx_); -+ psi.Run(input_->GetInput(), &output_, TEE_OUTPUT_INDEX); ++ output_ = std::make_unique(); ++ psi.Run(input_->Get(), output_->GetSecondaryPointer(), TEE_OUTPUT_INDEX); ++ auto end_time = std::chrono::high_resolution_clock()::now(); ++ auto dutation = std::chrono::dutation_cast( ++ end_time - start_time) ++ .count(); ++ SPDLOG_INFO("Online cost: {} ms", duration); +} + +void KcalPsiReceiver::PostProcess() { -+ for (int i = 0; i < output_->size; ++i) { -+ intersection_indices_writer_->WriteCache(output_->data.u64Numbers[i]); ++ for (int i = 0; i < output_->Size(); ++i) { ++ intersection_indices_writer_->WriteCache( ++ output_->Get()->data.u64Numbers[i]); + } + intersection_indices_writer_->Commit(); -+ DataAdaptor::ReleaseOutput(&output_); +} + +} // namespace psi::kcal \ No newline at end of file -diff --git a/psi/algorithm/kcal/receiver.h b/psi/algorithm/kcal/receiver.h +diff --git a/psi/algorithm/kcal_psi/receiver.h b/psi/algorithm/kcal_psi/receiver.h new file mode 100644 -index 0000000..683de58 +index 0000000..247e2cb --- /dev/null -+++ b/psi/algorithm/kcal/receiver.h ++++ b/psi/algorithm/kcal_psi/receiver.h @@ -0,0 +1,34 @@ +#pragma once + +#include "kcal/core/context.h" + -+#include "psi/algorithm/kcal/data_adaptor.h" +#include "psi/interface.h" ++#include "psi/kcal_adaptor/data_helper.h" +#include "psi/utils/arrow_csv_batch_provider.h" + +#include "psi/proto/psi_v2.pb.h" @@ -508,7 +818,7 @@ index 0000000..683de58 +class KcalPsiReceiver final : public AbstractPsiReceiver { + public: + explicit KcalPsiReceiver(const v2::PsiConfig& config, -+ std::shared_ptr lctx = nullptr); ++ std::shared_ptr lctx = nullptr); + + ~KcalPsiReceiver() override = default; + @@ -522,24 +832,24 @@ index 0000000..683de58 + void PostProcess() override; + + std::shared_ptr<::kcal::Context> kcal_ctx_; -+ std::shared_ptr input_; -+ DG_TeeOutput* output_ = nullptr; ++ std::unique_ptr input_; ++ std::unique_ptr output_; +}; + +} // namespace psi::kcal -diff --git a/psi/algorithm/kcal/sender.cc b/psi/algorithm/kcal/sender.cc +diff --git a/psi/algorithm/kcal_psi/sender.cc b/psi/algorithm/kcal_psi/sender.cc new file mode 100644 -index 0000000..9102df8 +index 0000000..e8dfafe --- /dev/null -+++ b/psi/algorithm/kcal/sender.cc -@@ -0,0 +1,53 @@ ++++ b/psi/algorithm/kcal_psi/sender.cc +@@ -0,0 +1,60 @@ +#include "psi/algorithm/kcal/sender.h" + +#include "kcal/operator/kcal_psi.h" + -+#include "psi/algorithm/kcal/data_adaptor.h" -+#include "psi/algorithm/kcal/global_manager.h" -+#include "psi/algorithm/kcal/link_api.h" ++#include "psi/kcal_adaptor/data_helper.h" ++#include "psi/kcal_adaptor/global_manager.h" ++#include "psi/kcal_adaptor/traffic_wrapper.h" + +namespace psi::kcal { + @@ -553,7 +863,7 @@ index 0000000..9102df8 + GlobalManager::GetInstance().InitializeCtx(lctx_); + // 配置 + ::kcal::KCAL_Config kcal_cfg; -+ kcal_cfg.fixBits = 0; ++ kcal_cfg.fixBits = 1; + kcal_cfg.nodeId = lctx_->Rank(); + kcal_cfg.threadCount = config_.protocol_config().kcal_config().thread_count(); + kcal_cfg.worldSize = lctx_->WorldSize(); @@ -562,43 +872,50 @@ index 0000000..9102df8 + net_res.funcRecvData = GlobalRecvWrapper; + net_res.funcSendData = GlobalSendWrapper; + // 计算上下文 -+ kcal_ctx_ = ::kcal::CreateKCALContext(kcal_cfg, &net_res, -+ ::kcal::KCAL_AlgorithmsType::PSI); ++ kcal_ctx_ = ::kcal::Context::Create(kcal_cfg, &net_res, ++ ::kcal::KCAL_AlgorithmsType::PSI); +} + +void KcalPsiSender::PreProcess() { -+ input_ = std::make_shared(); -+ input_->TransData(batch_provider_); ++ input_ = std::make_unique(); ++ input_->Fill(batch_provider_); +} + +void KcalPsiSender::Online() { ++ auto start_time = std::chrono::high_resolution_clock::now(); + ::kcal::Psi psi; + psi.Init(kcal_ctx_); -+ psi.Run(input_->GetInput(), &output_, TEE_OUTPUT_INDEX); ++ output_ = std::make_unique(); ++ psi.Run(input_->Get(), output_->GetSecondaryPointer(), TEE_OUTPUT_INDEX); ++ auto end_time = std::chrono::high_resolution_clock::now(); ++ auto duration = std::chrono::duration_cast( ++ end_time - start_time) ++ .count(); ++ SPDLOG_INFO("Online cost: {} ms", duration); +} + +void KcalPsiSender::PostProcess() { + for (int i = 0; i < output_->size; ++i) { -+ intersection_indices_writer_->WriteCache(output_->data.u64Numbers[i]); ++ intersection_indices_writer_->WriteCache( ++ output_->Get()->data.u64Numbers[i]); + } + intersection_indices_writer_->Commit(); -+ DataAdaptor::ReleaseOutput(&output_); +} + +} // namespace psi::kcal \ No newline at end of file -diff --git a/psi/algorithm/kcal/sender.h b/psi/algorithm/kcal/sender.h +diff --git a/psi/algorithm/kcal_psi/sender.h b/psi/algorithm/kcal_psi/sender.h new file mode 100644 -index 0000000..2f9e412 +index 0000000..b571632 --- /dev/null -+++ b/psi/algorithm/kcal/sender.h ++++ b/psi/algorithm/kcal_psi/sender.h @@ -0,0 +1,34 @@ +#pragma once + +#include "kcal/core/context.h" + -+#include "psi/algorithm/kcal/data_adaptor.h" +#include "psi/interface.h" ++#include "psi/kcal_adaptor/data_helper.h" +#include "psi/utils/arrow_csv_batch_provider.h" + +#include "psi/proto/psi_v2.pb.h" @@ -608,7 +925,7 @@ index 0000000..2f9e412 +class KcalPsiSender final : public AbstractPsiSender { + public: + explicit KcalPsiSender(const v2::PsiConfig& config, -+ std::shared_ptr lctx = nullptr); ++ std::shared_ptr lctx = nullptr); + + ~KcalPsiSender() override = default; + @@ -622,25 +939,34 @@ index 0000000..2f9e412 + void PostProcess() override; + + std::shared_ptr<::kcal::Context> kcal_ctx_; -+ std::shared_ptr input_; -+ DG_TeeOutput* output_ = nullptr; ++ std::unique_ptr input_; ++ std::unique_ptr output_; +}; + +} // namespace psi::kcal diff --git a/psi/apps/psi_launcher/BUILD.bazel b/psi/apps/psi_launcher/BUILD.bazel -index ffa6a3f..c9f66f2 100644 +index ffa6a3f..15752ff 100644 --- a/psi/apps/psi_launcher/BUILD.bazel +++ b/psi/apps/psi_launcher/BUILD.bazel @@ -29,6 +29,8 @@ psi_cc_library( "//psi/algorithm/kkrt:sender", "//psi/algorithm/rr22:receiver", "//psi/algorithm/rr22:sender", -+ "//psi/algorithm/kcal:receiver", -+ "//psi/algorithm/kcal:sender", ++ "//psi/algorithm/kcal_psi:receiver", ++ "//psi/algorithm/kcal_psi:sender", "@yacl//yacl/base:exception", ], ) -@@ -102,6 +104,7 @@ psi_cc_binary( +@@ -52,6 +54,8 @@ psi_cc_library( + "//psi/legacy:bucket_psi", + "//psi/proto:pir_cc_proto", + "//psi/wrapper/apsi/cli:entry", ++ "//psi/algorithm/kcal_pir:pir_sender", ++ "//psi/algorithm/kcal_pir:pir_receiver", + "@boost.algorithm//:boost.algorithm", + ], + ) +@@ -102,6 +106,7 @@ psi_cc_binary( "//psi:version", "//psi/proto:entry_cc_proto", "//psi/utils:resource_manager", @@ -649,15 +975,15 @@ index ffa6a3f..c9f66f2 100644 ], ) diff --git a/psi/apps/psi_launcher/factory.cc b/psi/apps/psi_launcher/factory.cc -index f57f1eb..97de4b3 100644 +index f57f1eb..a96a8ea 100644 --- a/psi/apps/psi_launcher/factory.cc +++ b/psi/apps/psi_launcher/factory.cc @@ -22,6 +22,8 @@ #include "psi/algorithm/ecdh/sender.h" #include "psi/algorithm/ecdh/ub_psi/client.h" #include "psi/algorithm/ecdh/ub_psi/server.h" -+#include "psi/algorithm/kcal/receiver.h" -+#include "psi/algorithm/kcal/sender.h" ++#include "psi/algorithm/kcal_psi/receiver.h" ++#include "psi/algorithm/kcal_psi/sender.h" #include "psi/algorithm/kkrt/receiver.h" #include "psi/algorithm/kkrt/sender.h" #include "psi/algorithm/rr22/receiver.h" @@ -678,6 +1004,409 @@ index f57f1eb..97de4b3 100644 default: YACL_THROW("Protocol is unspecified."); } +diff --git a/psi/apps/psi_launcher/launch.cc b/psi/apps/psi_launcher/launch.cc +index 31deab7..6c6780f 100644 +--- a/psi/apps/psi_launcher/launch.cc ++++ b/psi/apps/psi_launcher/launch.cc +@@ -320,6 +320,30 @@ PirResultReport RunDkPir(const DkPirSenderConfig& dk_pir_sender_config, + return PirResultReport(); + } + ++PirResultReport RunKcalPir( ++ const KcalPirReceiverConfig& kcal_pir_receiver_config, ++ const std::shared_ptr& lctx) { ++ kcal::pir::ReceiverOptions options; ++ options.thread_count = kcal_pir_receiver_config.threads(); ++ options.is_dummy_mode = kcal_pir_receiver_config.is_dummy_mode(); ++ options.query_file = kcal_pir_receiver_config.query_file(); ++ options.result_file = kcal_pir_receiver_config.output_file(); ++ int match_cnt = 0; ++ kcal::pir::RunKcalReceiver(options, lctx, &match_cnt); ++ PirResultReport report; ++ report.set_match_cnt(match_cnt); ++ return report; ++} ++ ++PirResultReport RunKcalPir(const KcalPirSenderConfig& kcal_pir_sender_config, ++ const std::shared_ptr& lctx) { ++ kcal::pir::SenderOptions options; ++ options.thread_count = kcal_pir_sender_config.threads(); ++ options.db_file = kcal_pir_sender_config.db_file(); ++ kcal::pir::RunKcalSender(options, lctx); ++ return {}; ++} ++ + namespace api { + + const std::map kV2ProtoclMap = { +diff --git a/psi/apps/psi_launcher/launch.h b/psi/apps/psi_launcher/launch.h +index 080f550..3e377ee 100644 +--- a/psi/apps/psi_launcher/launch.h ++++ b/psi/apps/psi_launcher/launch.h +@@ -19,6 +19,8 @@ + + #include "yacl/link/context.h" + ++#include "psi/algorithm/kcal_pir/pir_receiver.h" ++#include "psi/algorithm/kcal_pir/pir_sender.h" + #include "psi/apps/psi_launcher/report.h" + #include "psi/config/psi.h" + #include "psi/config/ub_psi.h" +@@ -53,6 +55,13 @@ PirResultReport RunDkPir(const DkPirReceiverConfig& dk_pir_receiver_config, + PirResultReport RunDkPir(const DkPirSenderConfig& dk_pir_sender_config, + const std::shared_ptr& lctx); + ++PirResultReport RunKcalPir( ++ const KcalPirReceiverConfig& kcal_pir_receiver_config, ++ const std::shared_ptr& lctx); ++ ++PirResultReport RunKcalPir(const KcalPirSenderConfig& kcal_pir_sender_config, ++ const std::shared_ptr& lctx); ++ + namespace api { + + namespace internal { +diff --git a/psi/apps/psi_launcher/main.cc b/psi/apps/psi_launcher/main.cc +index 3965006..c39c05d 100644 +--- a/psi/apps/psi_launcher/main.cc ++++ b/psi/apps/psi_launcher/main.cc +@@ -133,6 +133,18 @@ int main(int argc, char* argv[]) { + YACL_ENFORCE(google::protobuf::util::MessageToJsonString( + report, &report_json, json_print_options) + .ok()); ++ } else if (launch_config.has_kcal_pir_sender_config()) { ++ psi::PirResultReport report = ++ psi::RunKcalPir(launch_config.kcal_pir_sender_config(), lctx); ++ YACL_ENFORCE(google::protobuf::util::MessageToJsonString( ++ report, &report_json, json_print_options) ++ .ok()); ++ } else if (launch_config.has_kcal_pir_receiver_config()) { ++ psi::PirResultReport report = ++ psi::RunKcalPir(launch_config.kcal_pir_receiver_config(), lctx); ++ YACL_ENFORCE(google::protobuf::util::MessageToJsonString( ++ report, &report_json, json_print_options) ++ .ok()); + } else { + SPDLOG_WARN("No runtime config is provided."); + } +diff --git a/psi/kcal_adaptor/BUILD.bazel b/psi/kcal_adaptor/BUILD.bazel +new file mode 100644 +index 0000000..e20c134 +--- /dev/null ++++ b/psi/kcal_adaptor/BUILD.bazel +@@ -0,0 +1,32 @@ ++load("//bazel:psi.bzl", "psi_cc_binary", "psi_cc_library", "psi_cc_test") ++ ++package(default_visibility = ["//visibility:public"]) ++ ++psi_cc_library( ++ name = "data_helper", ++ srcs = ["data_helper.cc"], ++ hdrs = ["data_helper.h"], ++ deps = [ ++ "//psi/utils:batch_provider", ++ "@kunpeng_kcal_middleware//:kcal_middleware", ++ ] ++) ++ ++psi_cc_library( ++ name = "global_manager", ++ hdrs = ["global_manager.h"], ++ srcs = ["global_manager.cc"], ++ deps = [ ++ "@yacl//yacl/link", ++ ] ++) ++ ++psi_cc_library( ++ name = "traffic_wrapper", ++ hdrs = ["traffic_wrapper.h"], ++ srcs = ["traffic_wrapper.cc"], ++ deps = [ ++ ":global_manager", ++ "@kunpeng_kcal_middleware//:kcal_middleware", ++ ] ++) +diff --git a/psi/kcal_adaptor/data_helper.cc b/psi/kcal_adaptor/data_helper.cc +new file mode 100644 +index 0000000..ed9f5b9 +--- /dev/null ++++ b/psi/kcal_adaptor/data_helper.cc +@@ -0,0 +1,62 @@ ++#include "psi/kcal_adaptor/data_helper.h" ++ ++#include ++ ++namespace psi::kcal { ++ ++void DataHelper::BuildDgString(const std::vector& strings, ++ DG_String** dg) { ++ auto* dgString = new DG_String[strings.size()]; ++ for (size_t i = 0; i < strings.size(); ++i) { ++ dgString[i].str = strdup(strings[i].c_str()); ++ dgString[i].size = strings[i].size() + 1; ++ } ++ *dg = dgString; ++} ++ ++void DataHelper::ReleaseOutput(DG_TeeOutput** output) { ++ if (output == nullptr || *output == nullptr) { ++ return; ++ } ++ if ((*output)->dataType == MPC_STRING && (*output)->data.strings != nullptr) { ++ for (int i = 0; i < (*output)->size; ++i) { ++ if ((*output)->data.strings[i].str != nullptr) { ++ delete[](*output)->data.strings[i].str; ++ } ++ } ++ delete[](*output)->data.strings; ++ } else if ((*output)->dataType == MPC_DOUBLE && ++ (*output)->data.doubleNumbers != nullptr) { ++ delete[](*output)->data.doubleNumbers; ++ } else if ((*output)->dataType == MPC_INT && ++ (*output)->data.u64Numbers != nullptr) { ++ delete[](*output)->data.u64Numbers; ++ } ++ delete *output; ++ *output = nullptr; ++} ++ ++void KcalInput::Fill(const std::vector& data) { ++ DG_String* strings = nullptr; ++ DataHelper::BuildDgString(data, &strings); ++ ++ input_ = new DG_TeeInput(); ++ input_->data.strings = strings; ++ input_->size = static_cast(data.size()); ++ input_->dataType = MPC_STRING; ++} ++ ++void KcalInput::Fill(std::shared_ptr provider) { ++ std::vector rawData; ++ while (true) { ++ auto data = provider->ReadNextBatch(); ++ if (data.empty()) { ++ break; ++ } ++ rawData.insert(rawData.end(), data.begin(), data.end()); ++ } ++ ++ Fill(rawData); ++} ++ ++} // namespace psi::kcal +diff --git a/psi/kcal_adaptor/data_helper.h b/psi/kcal_adaptor/data_helper.h +new file mode 100644 +index 0000000..ef643c9 +--- /dev/null ++++ b/psi/kcal_adaptor/data_helper.h +@@ -0,0 +1,44 @@ ++#pragma once ++ ++#include ++#include ++#include ++ ++#include "kcal/api/kcal_api.h" ++ ++#include "psi/utils/batch_provider.h" ++ ++namespace psi::kcal { ++ ++class DataHelper { ++ public: ++ static void BuildDgString(const std::vector& strings, ++ DG_String** dg); ++ ++ static void ReleaseOutput(DG_TeeOutput** output); ++}; ++ ++class KcalInput { ++ public: ++ KcalInput() = default; ++ ++ ~KcalInput() { DataHelper::ReleaseOutput(&input_); } ++ ++ KcalInput(const KcalInput&) = delete; ++ KcalInput& operator=(const KcalInput&) = delete; ++ ++ void Fill(const std::vector& data); ++ void Fill(std::shared_ptr provider); ++ ++ DG_TeeInput* Get() { return input_; } ++ DG_TeeInput** GetSecondaryPointer() { return &input_; } ++ ++ int Size() { return input_->size; } ++ ++ private: ++ DG_TeeInput* input_ = nullptr; ++}; ++ ++using KcalOutput = KcalInput; ++ ++} // namespace psi::kcal +\ No newline at end of file +diff --git a/psi/kcal_adaptor/global_manager.cc b/psi/kcal_adaptor/global_manager.cc +new file mode 100644 +index 0000000..92ca35c +--- /dev/null ++++ b/psi/kcal_adaptor/global_manager.cc +@@ -0,0 +1,19 @@ ++#include "psi/kcal_adaptor/global_manager.h" ++ ++namespace psi::kcal { ++ ++GlobalManager& GlobalManager::GetInstance() { ++ static GlobalManager instance; ++ return instance; ++} ++ ++void GlobalManager::InitializeCtx( ++ const std::shared_ptr& lctx) { ++ lctx_ = lctx; ++} ++ ++std::shared_ptr& GlobalManager::GetLink() { return lctx_; } ++ ++void GlobalManager::CleanLink() { lctx_.reset(); } ++ ++} // namespace psi::kcal +\ No newline at end of file +diff --git a/psi/kcal_adaptor/global_manager.h b/psi/kcal_adaptor/global_manager.h +new file mode 100644 +index 0000000..c596b0f +--- /dev/null ++++ b/psi/kcal_adaptor/global_manager.h +@@ -0,0 +1,25 @@ ++#pragma once ++ ++#include "yacl/link/link.h" ++ ++namespace psi::kcal { ++ ++class GlobalManager { ++ public: ++ static GlobalManager& GetInstance(); ++ ++ ~GlobalManager() = default; ++ ++ void InitializeCtx(const std::shared_ptr& lctx); ++ ++ std::shared_ptr& GetLink(); ++ ++ void CleanLink(); ++ ++ private: ++ GlobalManager() = default; ++ ++ std::shared_ptr lctx_; ++}; ++ ++} // namespace psi::kcal +diff --git a/psi/kcal_adaptor/traffic_wrapper.cc b/psi/kcal_adaptor/traffic_wrapper.cc +new file mode 100644 +index 0000000..4ea541f +--- /dev/null ++++ b/psi/kcal_adaptor/traffic_wrapper.cc +@@ -0,0 +1,26 @@ ++#include "psi/kcal_adaptor/traffic_wrapper.h" ++ ++#include "psi/kcal_adaptor/global_manager.h" ++ ++namespace psi::kcal { ++ ++int GlobalSendWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, ++ unsigned long long len) { ++ yacl::Buffer msg(reinterpret_cast(buf), len); ++ std::string tag = "kcal_send"; ++ auto lctx = GlobalManager::GetInstance().GetLink(); ++ lctx->SendAsync(nodeInfo->nodeId, msg, tag); ++ return 0; ++} ++ ++int GlobalRecvWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, ++ unsigned long long* len) { ++ std::string tag = "kcal_recv"; ++ auto lctx = GlobalManager::GetInstance().GetLink(); ++ auto msg = lctx->Recv(nodeInfo->nodeId, tag); ++ memcpy(buf, msg.data(), msg.size()); ++ *len = msg.size(); ++ return 0; ++} ++ ++} // namespace psi::kcal +diff --git a/psi/kcal_adaptor/traffic_wrapper.h b/psi/kcal_adaptor/traffic_wrapper.h +new file mode 100644 +index 0000000..fc00fcc +--- /dev/null ++++ b/psi/kcal_adaptor/traffic_wrapper.h +@@ -0,0 +1,13 @@ ++#pragma once ++ ++#include "kcal/api/kcal_api.h" ++ ++namespace psi::kcal { ++ ++int GlobalSendWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, ++ unsigned long long len); ++ ++int GlobalRecvWrapper(struct TeeNodeInfo* nodeInfo, unsigned char* buf, ++ unsigned long long* len); ++ ++} // namespace psi::kcal +\ No newline at end of file +diff --git a/psi/proto/entry.proto b/psi/proto/entry.proto +index 265ebaa..76b2881 100644 +--- a/psi/proto/entry.proto ++++ b/psi/proto/entry.proto +@@ -47,5 +47,9 @@ message LaunchConfig { + DkPirSenderConfig dk_pir_sender_config = 8; + + DkPirReceiverConfig dk_pir_receiver_config = 9; ++ ++ KcalPirSenderConfig kcal_pir_sender_config = 10; ++ ++ KcalPirReceiverConfig kcal_pir_receiver_config = 11; + } + } +diff --git a/psi/proto/pir.proto b/psi/proto/pir.proto +index 094ecba..4fc74c8 100644 +--- a/psi/proto/pir.proto ++++ b/psi/proto/pir.proto +@@ -127,6 +127,33 @@ message ApsiReceiverConfig { + uint32 query_batch_size = 10; + } + ++message KcalPirSenderConfig { ++ // Number of threads to use ++ uint32 threads = 1; ++ ++ // Path to a CSV file describing the sender's dataset (an item-label pair on ++ // each row) or a file containing a serialized SenderDB; the CLI will first ++ // attempt to load the data as a serialized SenderDB, and - upon failure - ++ // will proceed to attempt to read it as a CSV file ++ // For CSV File: ++ // 1. the first col is processed as item while the second col as label. OTHER ++ // COLS ARE IGNORED. ++ // 2. NO HEADERS ARE ALLOWED ++ string db_file = 2; ++} ++ ++message KcalPirReceiverConfig { ++ // Number of threads to use ++ uint32 threads = 1; ++ ++ string query_file = 2; ++ ++ // Path to a file where intersection result will be written. ++ string output_file = 3; ++ ++ bool is_dummy_mode = 4; ++} ++ + message DkPirSenderConfig { + enum Mode { + MODE_UNSPECIFIED = 0; diff --git a/psi/proto/psi_v2.proto b/psi/proto/psi_v2.proto index 5682ef7..c56b575 100644 --- a/psi/proto/psi_v2.proto -- Gitee From e28973584859d4120538b58aafaa528b79be36a4 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 29 Jul 2025 11:17:55 +0800 Subject: [PATCH 09/18] Modify patch and pir --- MPC/middleware/kcal/operator/kcal_pir.cc | 2 +- MPC/middleware/kcal/operator/kcal_pir.h | 4 +- .../secrerflow/psi/patches/kcal.patch | 81 ++++++++++++------- 3 files changed, 54 insertions(+), 33 deletions(-) diff --git a/MPC/middleware/kcal/operator/kcal_pir.cc b/MPC/middleware/kcal/operator/kcal_pir.cc index 9c1a56b..7613777 100644 --- a/MPC/middleware/kcal/operator/kcal_pir.cc +++ b/MPC/middleware/kcal/operator/kcal_pir.cc @@ -51,7 +51,7 @@ int Pir::ServerPreProcess(DG_PairList *pairList) return opts_->offlineCalculate(dgTeeCtx_, pairList, &bucketMap_); } -int Pir::ClientQuery(DG_Input *input, DG_TeeOutput **output, DG_DummyMode dummyMode) +int Pir::ClientQuery(DG_TeeInput *input, DG_TeeOutput **output, DG_DummyMode dummyMode) { return opts_->clientCalculate(dgTeeCtx_, dummyMode, input, output); } diff --git a/MPC/middleware/kcal/operator/kcal_pir.h b/MPC/middleware/kcal/operator/kcal_pir.h index b0b6091..ceb2860 100644 --- a/MPC/middleware/kcal/operator/kcal_pir.h +++ b/MPC/middleware/kcal/operator/kcal_pir.h @@ -18,7 +18,7 @@ namespace kcal { -class pir { +class Pir { public: Pir(); ~Pir(); @@ -28,7 +28,7 @@ public: int Init(std::shared_ptr ctx); int ServerPreProcess(DG_PairList *pairList); - int ClientQuery(DG_Input *input, DG_TeeOutput **output, DG_DummyMode dummyMode); + int ClientQuery(DG_TeeInput *input, DG_TeeOutput **output, DG_DummyMode dummyMode); int ServerAnswer(); private: diff --git a/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch index 330159e..45d0e95 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch +++ b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch @@ -272,7 +272,7 @@ index d840dfd..0033330 100644 "type": "IO_TYPE_FILE_CSV", diff --git a/psi/algorithm/kcal_pir/BUILD.bazel b/psi/algorithm/kcal_pir/BUILD.bazel new file mode 100644 -index 0000000..7bd55e4 +index 0000000..72f2f14 --- /dev/null +++ b/psi/algorithm/kcal_pir/BUILD.bazel @@ -0,0 +1,28 @@ @@ -289,7 +289,7 @@ index 0000000..7bd55e4 + "//psi/kcal_adaptor:traffic_wrapper", + "//psi/kcal_adaptor:global_manager", + "//psi/utils:arrow_csv_batch_provider", -+ "@Kunpeng_kcal_middleware//:kcal_middleware", ++ "@kunpeng_kcal_middleware//:kcal_middleware", + ] +) + @@ -300,16 +300,16 @@ index 0000000..7bd55e4 + deps = [ + "//psi/kcal_adaptor:traffic_wrapper", + "//psi/kcal_adaptor:global_manager", -+ "//psi/utils/arrow_csv_batch_provider", -+ "@Kunpeng_kcal_middleware//:kcal_middleware", ++ "//psi/utils:arrow_csv_batch_provider", ++ "@kunpeng_kcal_middleware//:kcal_middleware", + ] +) diff --git a/psi/algorithm/kcal_pir/pir_receiver.cc b/psi/algorithm/kcal_pir/pir_receiver.cc new file mode 100644 -index 0000000..92e1aec +index 0000000..3761e18 --- /dev/null +++ b/psi/algorithm/kcal_pir/pir_receiver.cc -@@ -0,0 +1,108 @@ +@@ -0,0 +1,129 @@ +#include "psi/algorithm/kcal_pir/pir_receiver.h" + +#include @@ -395,25 +395,46 @@ index 0000000..92e1aec + return 0; +} + ++std::vector KcalPirReceiver::ReadPirQuery( ++ const std::string& file_path, const std::vector& keys, ++ size_t batch_size, char delimiter) { ++ const std::vector labels = {}; ++ psi::ArrowCsvBatchProvider csv_batch_provider(file_path, keys, batch_size, ++ labels, delimiter); ++ std::vector key_data; ++ bool is_reach_end = false; ++ while (!is_reach_end) { ++ auto batch = csv_batch_provider.ReadNextBatch(); ++ if (batch.size() < batch_size) { ++ is_reach_end = true; ++ } ++ ++ if (batch.size() > 0) { ++ key_data.insert(key_data.end(), batch.begin(), batch.end()); ++ } ++ } ++ return key_data; ++} ++ +int RunKcalReceiver(const ReceiverOptions& options, -+ const std::shared_ptr& lctx = nullptr, -+ int* match_cnt = nullptr) { ++ const std::shared_ptr& lctx, ++ int* match_cnt) { + KcalPirReceiver pir_receiver(options, lctx); + pir_receiver.Init(); + pir_receiver.PreProcess(); + pir_receiver.Connect(); -+ auto start_time = std::chrono::steady_clock()::now(); ++ auto start_time = std::chrono::steady_clock::now(); + DG_DummyMode dummy_mode = NORMAL; + if (options.is_dummy_mode) { + dummy_mode = DUMMY; + } + pir_receiver.Online(dummy_mode); + auto end_time = std::chrono::steady_clock::now(); -+ auto dur = std::chrono::duraction_cast(end_time - -+ start_time) ++ auto dur = std::chrono::duration_cast(end_time - ++ start_time) + .count(); + SPDLOG_INFO("Online query time {} ms", dur); -+ pir_receiver.PostProcess(); ++ pir_receiver.PostProcess(match_cnt); + return 0; +} + @@ -474,7 +495,7 @@ index 0000000..98de787 +} // namespace psi::kcal::pir diff --git a/psi/algorithm/kcal_pir/pir_sender.cc b/psi/algorithm/kcal_pir/pir_sender.cc new file mode 100644 -index 0000000..6a2a0a8 +index 0000000..3bdd810 --- /dev/null +++ b/psi/algorithm/kcal_pir/pir_sender.cc @@ -0,0 +1,156 @@ @@ -491,7 +512,7 @@ index 0000000..6a2a0a8 + : lctx_(std::move(lctx)) { + options_ = std::make_shared(); + *options_ = options; -+ pir_ = std::make_shared<::kcal::Pir>(); ++ pir_ = std::make_unique<::kcal::Pir>(); +} + +void KcalPirSender::Init() { @@ -499,7 +520,7 @@ index 0000000..6a2a0a8 + lctx_->ConnectToMesh(); + GlobalManager::GetInstance().InitializeCtx(lctx_); + -+ ::kcal::KCAL_Config kcal_cfg_{}; ++ ::kcal::KCAL_Config kcal_cfg{}; + kcal_cfg.fixBits = 1; + kcal_cfg.nodeId = lctx_->Rank(); + kcal_cfg.threadCount = options_->thread_count; @@ -580,7 +601,7 @@ index 0000000..6a2a0a8 + } +} + -+void KcalPirSender::ReleasePairList(DG_PairList* pairList) { ++void KcalPirSender::ReleaseDgPairList(DG_PairList* pairList) { + if (pairList != nullptr) { + for (unsigned long i = 0; i < pairList->size; ++i) { + ReleasePairString(pairList->dgPair[i].key); @@ -592,7 +613,7 @@ index 0000000..6a2a0a8 +} + +DG_String* KcalPirSender::BuildPairString(const std::string& s) { -+ auto* dgString = new DG_string(); ++ auto* dgString = new DG_String(); + dgString->size = s.size() + 1; + dgString->str = new char[dgString->size + 1]; + dgString->str[dgString->size] = '\0'; @@ -616,7 +637,7 @@ index 0000000..6a2a0a8 +} + +int RunKcalSender(const SenderOptions& options, -+ const std::shared_ptr& lctx = nullptr) { ++ const std::shared_ptr& lctx) { + KcalPirSender pir_sender(options, lctx); + + pir_sender.Init(); @@ -625,8 +646,8 @@ index 0000000..6a2a0a8 + auto start_time = std::chrono::steady_clock::now(); + pir_sender.Online(); + auto end_time = std::chrono::steady_clock::now(); -+ auto dur = std::chrono::duraction_cast(end_time - -+ start_time) ++ auto dur = std::chrono::duration_cast(end_time - ++ start_time) + .count(); + SPDLOG_INFO("Online query time {} ms", dur); + pir_sender.PostProcess(); @@ -636,7 +657,7 @@ index 0000000..6a2a0a8 +} // namespace psi::kcal::pir diff --git a/psi/algorithm/kcal_pir/pir_sender.h b/psi/algorithm/kcal_pir/pir_sender.h new file mode 100644 -index 0000000..efb9f9e +index 0000000..f44bd39 --- /dev/null +++ b/psi/algorithm/kcal_pir/pir_sender.h @@ -0,0 +1,55 @@ @@ -676,7 +697,7 @@ index 0000000..efb9f9e + + static void ReleasePairString(DG_String* dgString); + -+ static void ReleasePairList(DG_PairList* pairList); ++ static void ReleaseDgPairList(DG_PairList* pairList); + + static DG_String* BuildPairString(const std::string& s); + @@ -730,11 +751,11 @@ index 0000000..14dd8df +) diff --git a/psi/algorithm/kcal_psi/receiver.cc b/psi/algorithm/kcal_psi/receiver.cc new file mode 100644 -index 0000000..035b52e +index 0000000..5a6bd69 --- /dev/null +++ b/psi/algorithm/kcal_psi/receiver.cc @@ -0,0 +1,62 @@ -+#include "psi/algorithm/kcal/receiver.h" ++#include "psi/algorithm/kcal_psi/receiver.h" + +#include + @@ -775,13 +796,13 @@ index 0000000..035b52e +} + +void KcalPsiReceiver::Online() { -+ auto start_time = std::chrono::high_resolution_clock()::now(); ++ auto start_time = std::chrono::high_resolution_clock::now(); + ::kcal::Psi psi; + psi.Init(kcal_ctx_); + output_ = std::make_unique(); + psi.Run(input_->Get(), output_->GetSecondaryPointer(), TEE_OUTPUT_INDEX); -+ auto end_time = std::chrono::high_resolution_clock()::now(); -+ auto dutation = std::chrono::dutation_cast( ++ auto end_time = std::chrono::high_resolution_clock::now(); ++ auto duration = std::chrono::duration_cast( + end_time - start_time) + .count(); + SPDLOG_INFO("Online cost: {} ms", duration); @@ -839,11 +860,11 @@ index 0000000..247e2cb +} // namespace psi::kcal diff --git a/psi/algorithm/kcal_psi/sender.cc b/psi/algorithm/kcal_psi/sender.cc new file mode 100644 -index 0000000..e8dfafe +index 0000000..551b0ff --- /dev/null +++ b/psi/algorithm/kcal_psi/sender.cc @@ -0,0 +1,60 @@ -+#include "psi/algorithm/kcal/sender.h" ++#include "psi/algorithm/kcal_psi/sender.h" + +#include "kcal/operator/kcal_psi.h" + @@ -895,7 +916,7 @@ index 0000000..e8dfafe +} + +void KcalPsiSender::PostProcess() { -+ for (int i = 0; i < output_->size; ++i) { ++ for (int i = 0; i < output_->Size(); ++i) { + intersection_indices_writer_->WriteCache( + output_->Get()->data.u64Numbers[i]); + } -- Gitee From 48c84c987f0d3c05a1a986a2ceeff0b960f81c42 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 29 Jul 2025 14:09:40 +0800 Subject: [PATCH 10/18] Modify patch --- .../secrerflow/psi/patches/kcal.patch | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch index 45d0e95..e71d283 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch +++ b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch @@ -50,6 +50,39 @@ index 0000000..8416212 + "@//third_party:libkcal" + ] +) +diff --git a/examples/pir/README.md b/examples/pir/README.md +index b4bb342..3b0eddd 100644 +--- a/examples/pir/README.md ++++ b/examples/pir/README.md +@@ -6,11 +6,13 @@ + + ```bash + python examples/pir/apsi/test_data_creator.py --sender_size=100000 --receiver_size=1 --intersection_size=1 --label_byte_count=16 ++python examples/pir/apsi/test_data_creator.py --sender_size=10000000 --receiver_size=1000 --intersection_size=100 --label_byte_count=100 --item_byte_count=16 + + mv db.csv /tmp/db.csv + mv query.csv /tmp/query.csv + + cp examples/pir/apsi/parameters/100K-1-16.json /tmp/100K-1-16.json ++cp examples/pir/apsi/parameters/1M-1024-cmp.json /tmp/1M-1024-cmp.json + ``` + + ### NOTE +@@ -39,12 +41,14 @@ At sender terminal, run + + ```bash + ./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/pir/config/apsi_sender_online.json ++./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/pir/config/kcal_pir_sender.json + ``` + + At receiver terminal, run + + ```bash + ./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/pir/config/apsi_receiver.json ++./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/pir/config/kcal_pir_receiver.json + ``` + + ## Run Server with Full Mode (No Seperate Setup Stage) diff --git a/examples/pir/config/apsi_receiver.json b/examples/pir/config/apsi_receiver.json index 2c7f56b..f62177a 100644 --- a/examples/pir/config/apsi_receiver.json @@ -142,6 +175,25 @@ index 0000000..c2579cb + "self_link_party": "sender" +} \ No newline at end of file +diff --git a/examples/psi/README.md b/examples/psi/README.md +index 9327ec7..c2476ad 100644 +--- a/examples/psi/README.md ++++ b/examples/psi/README.md +@@ -36,12 +36,14 @@ + + ```bash + ./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/psi/config/ecdh_receiver_recovery.json ++ ./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/psi/config/kcal_receiver.json + ``` + + For **sender** terminal, + + ```bash + ./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/psi/config/ecdh_sender_recovery.json ++ ./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/psi/config/kcal_sender.json + ``` + + ## 2P UB PSI diff --git a/examples/psi/config/kcal_receiver.json b/examples/psi/config/kcal_receiver.json new file mode 100644 index 0000000..72e2694 -- Gitee From 598e6c980e942228367cd4d1cfa150513360bcf3 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 29 Jul 2025 14:17:03 +0800 Subject: [PATCH 11/18] Modify patch --- .../secrerflow/psi/patches/kcal.patch | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch index e71d283..b837a6e 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch +++ b/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch @@ -176,9 +176,18 @@ index 0000000..c2579cb +} \ No newline at end of file diff --git a/examples/psi/README.md b/examples/psi/README.md -index 9327ec7..c2476ad 100644 +index 9327ec7..42f2a83 100644 --- a/examples/psi/README.md +++ b/examples/psi/README.md +@@ -7,7 +7,7 @@ + 1. Compile the binary + + ```bash +- bazel build //psi:main -c opt ++ bazel build //... -c opt + ``` + + 2. Generate test data @@ -36,12 +36,14 @@ ```bash -- Gitee From d1284727ba1a38ac246e10255213b093d821e5d3 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 29 Jul 2025 15:36:47 +0800 Subject: [PATCH 12/18] Modify README.md --- .../secrerflow/psi/README.md | 283 +++++++++++++++++- 1 file changed, 281 insertions(+), 2 deletions(-) diff --git a/MPC/third_party_adaptor/secrerflow/psi/README.md b/MPC/third_party_adaptor/secrerflow/psi/README.md index 77bfb5d..608c780 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/README.md +++ b/MPC/third_party_adaptor/secrerflow/psi/README.md @@ -1,5 +1,284 @@ -# 下载代码 +# README.md + +# 前置条件 + +1. 需获取 `kcal` 包，含 `include` 和 `lib` 目录 +2. 需要 bazel 编译构建工具，编译环境依赖参考：[devtools/dockerfiles/release-ci-aarch64.DockerFile at main · secretflow/devtools](https://github.com/secretflow/devtools/blob/main/dockerfiles/release-ci-aarch64.DockerFile) +3. 运行环境为`virtCCA cvm`​ + +# 中间件在蚂蚁 psi 库的编译 + +## 创建工作目录 + +为方便进行演示，以下操作均以 `/home/admin/dev` 目录作为工作主目录 + +## clone virtCCA\_sdk 仓 + ```shell +cd /home/admin/dev + +git clone https://gitee.com/openeuler/virtCCA_sdk.git +``` + +## clone 蚂蚁 psi 仓，应用 patch + +注：目前已适配蚂蚁 `tag: psi-v0.6.0.dev250507` 版本的 `psi`库 + +```shell +cd /home/admin/dev + +# clone 仓库，并创建一个本地分支 git clone --branch "v0.6.0.dev250507" https://github.com/secretflow/psi.git git switch -c kcal-on-v0.6.0 -``` \ No newline at end of file + +# 应用 virtCCA_sdk 下面的 patch +git apply /home/admin/dev/virtCCA_sdk/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch +``` + +## 引入中间件和 kcal 库 + +### 引入 kcal 库 + +```shell +# 假设 kcal 库已下载在 /opt/kcal 目录下 +cp -r /opt/kcal/include /home/admin/dev/psi/third_party/kcal/ +cp -r /opt/kcal/lib /home/admin/dev/psi/third_party/kcal/ +``` + +### 引入中间件 + +```shell +cp -r /home/admin/dev/virtCCA_sdk/MPC/middleware/* /home/admin/dev/psi/third_party/kcal_middleware/ +``` + +## 执行编译 + +```shell +cd /home/admin/dev/psi + +bazel build //... -c opt +``` + +编译完成后，在`bazel-bin/psi/apps/psi_launcher`目录下生成`main`可执行文件，后续可通过执行`./bazel-bin/psi/apps/psi_launcher/main --config xxx.json`进行验证 + +## 将二进制部署至 virtCCA 内 + +可让`/home/admin/dev/psi`的目录结构与`virtCCA`内部保持一致，方便进行测试 + +# kcal psi 在蚂蚁 psi 库上的测试 + +编译完成后，进入`/home/admin/dev/psi`目录，步骤可参考：`examples/psi/README.md`​ + +## 生成 psi 测试数据 + +```bash +cd /home/admin/dev/psi + +python examples/psi/generate_psi_data.py \ + --receiver_item_cnt 1e6 \ + --sender_item_cnt 1e6 \ + --intersection_cnt 8e4 \ + --id_cnt 2 \ + --receiver_path /tmp/receiver_input.csv \ + --sender_path /tmp/sender_input.csv \ + --intersection_path /tmp/intersection.csv +``` + +> 说明： +> +> --receiver_item_cnt：receiver 方拥有的数据总量 +> +> --sender_item_cnt：sender 方拥有的数据总量 +> +> --intersection_cnt：约定两方产生交集部分的数据总量 +> +> --id_cnt：每个参与方的输入数据包含几个字段 +> +> --receiver_path：receiver 方输入数据的文件位置 +> +> --sender_path：sender 方输入数据的文件位置 +> +> --intersection_path：生成的交集数据的文件位置 + +## 配置文件说明 + +配置文件已在`patch`中提供，只需修改下列说明的部分进行测试 + +下面以`kcal_receiver.json`为例 + +```json +{ + "psi_config": { + "protocol_config": { + "protocol": "PROTOCOL_KCAL", + "kcal_config": { + "thread_count": 16 // 线程数按需修改，目前固定 16 线程 + }, + "role": "ROLE_SENDER", + "broadcast_result": true + }, + "input_config": { + "type": "IO_TYPE_FILE_CSV", + "path": "/tmp/sender_input.csv" // 当前参与方运行 psi 算法的数据输入文件位置，按需修改 + }, + "output_config": { + "type": "IO_TYPE_FILE_CSV", // 文件类型不需修改 + "path": "/tmp/kcal_sender_output.csv" // 两方运行完 psi 算法后，最终交集文件的输出位置，按需修改 + }, + "keys": ["id_0", "id_1"], // 有几个字段 + "debug_options": { // 无需修改 + "trace_path": "/tmp/kcal_sender.trace" + }, + "disable_alignment": true, + "recovery_config": { // 这个配置不需要修改，kcal 目前无 recovery 模式 + "enabled": false, + "folder": "/tmp/kcal_sender_cache" + } + }, + "link_config": { + "parties": [ // 两个参与方的通信 ip 和 端口，按需修改 + { + "id": "receiver", + "host": "127.0.0.1:5300" + }, + { + "id": "sender", + "host": "127.0.0.1:5400" + } + ] + }, + "self_link_party": "sender" // 当前参与方的标识 +} + +``` + +### kcal 两个配置文件 + +- examples/psi/config/kcal_receiver.json +- examples/psi/config/kcal_sender.json + +### 蚂蚁对比配置文件 + +- examples/psi/config/rr22_receiver_recovery.json +- examples/psi/config/rr22_sender_recovery.json + +## 测试 + +进入两个 cvm 分别执行以下命令 + +```bash +cd /home/admin/dev/psi + +# 参与方 receiver +./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/psi/config/kcal_receiver.json +# 参与方 sender +./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/psi/config/kcal_sender.json +``` + +运行完以上命令后，每个`cvm`内会在配置文件中指明的`output_config.path`路径中生成交集文件 + +## 结果对比 + +将`/tmp/kcal_sender_output.csv`、`/tmp/kcal_receiver_output.csv`的内容与一开始生成的交集文件`/tmp/intersection.csv`内容进行比对，结果保持一致 + +# kcal pir 在蚂蚁 psi 库上的测试 + +编译完成后，进入`/home/admin/dev/psi`目录，步骤可参考：`examples/pir/README.md`​ + +## 生成 pir 测试数据 + +```bash +cd /home/admin/dev/psi + +python examples/pir/apsi/test_data_creator.py \ + --sender_size=10000000 \ + --receiver_size=1000 \ + --intersection_size=100 \ + --label_byte_count=100 \ + --item_byte_count=16 + +mv ground_truth.csv /tmp/ground_truth.csv +mv db.csv /tmp/db.csv +mv query.csv /tmp/query.csv +``` + +> 说明： +> +> --sender_size：服务端数据库总体数据行数 +> +> --receiver_size：客户端要查询的 key 的数量 +> +> --intersection_size：实际上生成的数据里面，服务端只有 intersection_size 个包含客户端能够查到的键值 +> +> --label_byte_count：服务端数据库每个 value 所占的字节数 +> +> --item_byte_count：服务端数据库每个 key 所占的字节数 + +## 配置文件说明 + +配置文件已在`patch`中提供，只需修改下列说明的部分进行测试 + +```json +// 客户端配置文件 +{ + "kcal_pir_receiver_config": { + "threads": 16, // 多线程处理，按需修改 + "query_file": "/tmp/query.csv", // 要查询的 key 的集合文件位置，按需修改 + "output_file": "/tmp/result.csv", // 查询结果 value 的保存位置，按需修改 + "is_dummy_mode": true // 查询的 key 是否进行 dummy，按需修改 + }, + "link_config": { // 两个参与方的通信 ip 和 端口，按需修改 + "parties": [ + { + "id": "sender", + "host": "127.0.0.1:5300" + }, + { + "id": "receiver", + "host": "127.0.0.1:5400" + } + ] + }, + "self_link_party": "receiver" +} + +// 服务端配置文件 +{ + "kcal_pir_sender_config": { + "threads": 16, // 多线程处理，按需修改 + "db_file": "/tmp/db.csv" // 数据库文件位置 + }, + "link_config": { // 两个参与方的通信 ip 和 端口，按需修改 + "parties": [ + { + "id": "sender", + "host": "127.0.0.1:5300" + }, + { + "id": "receiver", + "host": "127.0.0.1:5400" + } + ] + }, + "self_link_party": "sender" +} +``` + +## 测试 + +进入两个 cvm 分别执行以下命令 + +```bash +cd /home/admin/dev/psi + +# 服务端 +./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/pir/config/kcal_pir_sender.json +# 客户端 +./bazel-bin/psi/apps/psi_launcher/main --config $(pwd)/examples/pir/config/kcal_pir_receiver.json +``` + +运行完以上命令后，客户端`cvm`内会在配置文件中指明的`kcal_pir_receiver_config.output_file`路径中生成查询结果文件 + +## 结果对比 + +将`/tmp/result.csv`的内容与一开始生成的交集文件`/tmp/ground_truth.csv`内容进行比对，结果保持一致 -- Gitee From 84eb9de93d6490b2d9f934f9452e4bf155b3c927 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 29 Jul 2025 15:40:02 +0800 Subject: [PATCH 13/18] Modify README.md --- MPC/third_party_adaptor/secrerflow/psi/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/MPC/third_party_adaptor/secrerflow/psi/README.md b/MPC/third_party_adaptor/secrerflow/psi/README.md index 608c780..12c23b4 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/README.md +++ b/MPC/third_party_adaptor/secrerflow/psi/README.md @@ -1,5 +1,3 @@ -# README.md - # 前置条件 1. 需获取 `kcal` 包，含 `include` 和 `lib` 目录 -- Gitee From a91c56ad02b686fd3b6e25cf782d178a1d9c8106 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 29 Jul 2025 15:52:00 +0800 Subject: [PATCH 14/18] Modify README.md --- MPC/third_party_adaptor/secrerflow/psi/README.md | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/MPC/third_party_adaptor/secrerflow/psi/README.md b/MPC/third_party_adaptor/secrerflow/psi/README.md index 12c23b4..06e98c9 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/README.md +++ b/MPC/third_party_adaptor/secrerflow/psi/README.md @@ -113,13 +113,13 @@ python examples/psi/generate_psi_data.py \ "thread_count": 16 // 线程数按需修改，目前固定 16 线程 }, "role": "ROLE_SENDER", - "broadcast_result": true + "broadcast_result": true }, "input_config": { "type": "IO_TYPE_FILE_CSV", "path": "/tmp/sender_input.csv" // 当前参与方运行 psi 算法的数据输入文件位置，按需修改 }, - "output_config": { + "output_config": { "type": "IO_TYPE_FILE_CSV", // 文件类型不需修改 "path": "/tmp/kcal_sender_output.csv" // 两方运行完 psi 算法后，最终交集文件的输出位置，按需修改 }, @@ -262,6 +262,17 @@ mv query.csv /tmp/query.csv } ``` +### kcal 两个配置文件 + +- examples/pir/config/kcal_pir_receiver.json +- examples/pir/config/kcal_pir_sender.json + +### 蚂蚁对比配置文件 + +- examples/pir/config/apsi_sender_setup.json +- examples/pir/config/apsi_sender_online.json +- examples/pir/config/apsi_receiver.json + ## 测试 进入两个 cvm 分别执行以下命令 -- Gitee From c705fb2aadc1b6ae2ae3dac002270fc40b66765b Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Wed, 30 Jul 2025 14:54:55 +0800 Subject: [PATCH 15/18] Code format --- MPC/middleware/kcal/api/kcal_api.h | 2 +- MPC/middleware/kcal/core/context.h | 4 ++-- MPC/middleware/kcal/enumeration/kcal_enum.h | 2 +- MPC/middleware/kcal/operator/kcal_pir.h | 2 +- MPC/middleware/kcal/operator/kcal_psi.h | 4 ++-- MPC/middleware/kcal/utils/node_info_helper.cc | 1 - MPC/middleware/kcal/utils/node_info_helper.h | 4 ++-- 7 files changed, 9 insertions(+), 10 deletions(-) diff --git a/MPC/middleware/kcal/api/kcal_api.h b/MPC/middleware/kcal/api/kcal_api.h index 4442e55..f125f9d 100644 --- a/MPC/middleware/kcal/api/kcal_api.h +++ b/MPC/middleware/kcal/api/kcal_api.h @@ -24,4 +24,4 @@ #include "data_guard_mpc.h" #include "data_guard_struct.h" -#endif //KCAL_API_H +#endif // KCAL_API_H diff --git a/MPC/middleware/kcal/core/context.h b/MPC/middleware/kcal/core/context.h index a065578..a30ad80 100644 --- a/MPC/middleware/kcal/core/context.h +++ b/MPC/middleware/kcal/core/context.h @@ -13,9 +13,9 @@ #ifndef CONTEXT_H #define CONTEXT_H +#include #include "kcal/api/kcal_api.h" #include "kcal/enumeration/kcal_enum.h" -#include namespace kcal { @@ -57,4 +57,4 @@ private: } -#endif //CONTEXT_H +#endif // CONTEXT_H diff --git a/MPC/middleware/kcal/enumeration/kcal_enum.h b/MPC/middleware/kcal/enumeration/kcal_enum.h index 45b1eed..e7597c4 100644 --- a/MPC/middleware/kcal/enumeration/kcal_enum.h +++ b/MPC/middleware/kcal/enumeration/kcal_enum.h @@ -22,4 +22,4 @@ enum class KCAL_AlgorithmsType { } -#endif //KCAL_ENUM_H +#endif // KCAL_ENUM_H diff --git a/MPC/middleware/kcal/operator/kcal_pir.h b/MPC/middleware/kcal/operator/kcal_pir.h index ceb2860..96a1bc8 100644 --- a/MPC/middleware/kcal/operator/kcal_pir.h +++ b/MPC/middleware/kcal/operator/kcal_pir.h @@ -40,4 +40,4 @@ private: } -#endif //KCAL_PIR_H +#endif // KCAL_PIR_H diff --git a/MPC/middleware/kcal/operator/kcal_psi.h b/MPC/middleware/kcal/operator/kcal_psi.h index 294d1be..de32d49 100644 --- a/MPC/middleware/kcal/operator/kcal_psi.h +++ b/MPC/middleware/kcal/operator/kcal_psi.h @@ -13,9 +13,9 @@ #ifndef KCAL_PSI_H #define KCAL_PSI_H +#include #include "kcal/api/kcal_api.h" #include "kcal/core/context.h" -#include namespace kcal { @@ -38,4 +38,4 @@ private: } -#endif //KCAL_PSI_H +#endif // KCAL_PSI_H diff --git a/MPC/middleware/kcal/utils/node_info_helper.cc b/MPC/middleware/kcal/utils/node_info_helper.cc index 023e70c..99742b1 100644 --- a/MPC/middleware/kcal/utils/node_info_helper.cc +++ b/MPC/middleware/kcal/utils/node_info_helper.cc @@ -25,5 +25,4 @@ NodeInfoHelper::NodeInfoHelper(int worldSize) nodeInfos_.size = infos_.size(); } - } diff --git a/MPC/middleware/kcal/utils/node_info_helper.h b/MPC/middleware/kcal/utils/node_info_helper.h index a9ef9f7..953d890 100644 --- a/MPC/middleware/kcal/utils/node_info_helper.h +++ b/MPC/middleware/kcal/utils/node_info_helper.h @@ -13,8 +13,8 @@ #ifndef NODE_INFO_HELPER_H #define NODE_INFO_HELPER_H -#include "kcal/api/kcal_api.h" #include +#include "kcal/api/kcal_api.h" namespace kcal::utils { @@ -36,4 +36,4 @@ private: } -#endif //NODE_INFO_HELPER_H +#endif // NODE_INFO_HELPER_H -- Gitee From 015b0a6d83a219626236bd3ad507e4805380a37b Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Thu, 31 Jul 2025 14:14:41 +0800 Subject: [PATCH 16/18] Modify README.md --- MPC/third_party_adaptor/secrerflow/psi/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/MPC/third_party_adaptor/secrerflow/psi/README.md b/MPC/third_party_adaptor/secrerflow/psi/README.md index 06e98c9..851710a 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/README.md +++ b/MPC/third_party_adaptor/secrerflow/psi/README.md @@ -30,6 +30,7 @@ git clone --branch "v0.6.0.dev250507" https://github.com/secretflow/psi.git git switch -c kcal-on-v0.6.0 # 应用 virtCCA_sdk 下面的 patch +cd /home/admin/dev/psi # 进到蚂蚁 psi 目录下 git apply /home/admin/dev/virtCCA_sdk/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch ``` -- Gitee From cced29bb4785ed47fc9b623e29a3d98a76870169 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 5 Aug 2025 18:03:54 +0800 Subject: [PATCH 17/18] Modify README.md --- .../secrerflow/psi/README.md | 265 ++++++++++-------- 1 file changed, 153 insertions(+), 112 deletions(-) diff --git a/MPC/third_party_adaptor/secrerflow/psi/README.md b/MPC/third_party_adaptor/secrerflow/psi/README.md index 851710a..cf5f23a 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/README.md +++ b/MPC/third_party_adaptor/secrerflow/psi/README.md @@ -1,105 +1,175 @@ -# 前置条件 +# KCAL 中间件适配蚂蚁 psi 库 -1. 需获取 `kcal` 包，含 `include` 和 `lib` 目录 -2. 需要 bazel 编译构建工具，编译环境依赖参考：[devtools/dockerfiles/release-ci-aarch64.DockerFile at main · secretflow/devtools](https://github.com/secretflow/devtools/blob/main/dockerfiles/release-ci-aarch64.DockerFile) -3. 运行环境为`virtCCA cvm`​ +本文档介绍 KCAL 中间件如何在 virtCCA 内部署已经适配了的蚂蚁 psi 库（`tag: psi-v0.6.0.dev250507` 版本） -# 中间件在蚂蚁 psi 库的编译 +## 前置条件 -## 创建工作目录 +1. 需获取 `kcal` 包，含 `include` 和 `lib` 目录，获取链接：[https://support.huawei.com/enterprise/zh/software/265814201-ESW2001490913](https://support.huawei.com/enterprise/zh/software/265814201-ESW2001490913) +2. 需要 bazel 编译构建工具，编译环境依赖参考：[devtools/dockerfiles/release-ci-aarch64.DockerFile at main · secretflow/devtools](https://github.com/secretflow/devtools/blob/main/dockerfiles/release-ci-aarch64.DockerFile) +3. 运行环境为`virtCCA cvm，前提是用户已经启动两个 virtCCA 的机密虚机（cvm1、cvm2）`​ -为方便进行演示，以下操作均以 `/home/admin/dev` 目录作为工作主目录 +## 中间件使能和部署步骤 -## clone virtCCA\_sdk 仓 +### 宿主机编译蚂蚁 psi 库 -```shell -cd /home/admin/dev +1. 创建工作目录 -git clone https://gitee.com/openeuler/virtCCA_sdk.git -``` + 为方便进行演示，以下操作均以 `/home/admin/dev` 目录作为工作主目录 -## clone 蚂蚁 psi 仓，应用 patch +2. clone virtCCA\_sdk 仓 -注：目前已适配蚂蚁 `tag: psi-v0.6.0.dev250507` 版本的 `psi`库 + ```bash + cd /home/admin/dev -```shell -cd /home/admin/dev + git clone https://gitee.com/openeuler/virtCCA_sdk.git + ``` -# clone 仓库，并创建一个本地分支 -git clone --branch "v0.6.0.dev250507" https://github.com/secretflow/psi.git -git switch -c kcal-on-v0.6.0 +3. clone 蚂蚁 psi 仓，应用 patch -# 应用 virtCCA_sdk 下面的 patch -cd /home/admin/dev/psi # 进到蚂蚁 psi 目录下 -git apply /home/admin/dev/virtCCA_sdk/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch -``` + ```bash + cd /home/admin/dev -## 引入中间件和 kcal 库 + # clone 仓库，并创建一个本地分支 + git clone --branch "v0.6.0.dev250507" https://github.com/secretflow/psi.git + git switch -c kcal-on-v0.6.0 -### 引入 kcal 库 + # 应用 virtCCA_sdk 下面的 patch + cd /home/admin/dev/psi # 进到蚂蚁 psi 目录下 + git apply /home/admin/dev/virtCCA_sdk/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch + ``` -```shell -# 假设 kcal 库已下载在 /opt/kcal 目录下 -cp -r /opt/kcal/include /home/admin/dev/psi/third_party/kcal/ -cp -r /opt/kcal/lib /home/admin/dev/psi/third_party/kcal/ -``` +4. 引入中间件和 kcal 库 -### 引入中间件 + ```bash + # 假设 kcal 库已下载在 /opt/kcal 目录下 + cp -r /opt/kcal/include /home/admin/dev/psi/third_party/kcal/ + cp -r /opt/kcal/lib /home/admin/dev/psi/third_party/kcal/ -```shell -cp -r /home/admin/dev/virtCCA_sdk/MPC/middleware/* /home/admin/dev/psi/third_party/kcal_middleware/ -``` + # 引入中间件 + cp -r /home/admin/dev/virtCCA_sdk/MPC/middleware/* /home/admin/dev/psi/third_party/kcal_middleware/ + ``` -## 执行编译 +5. 编译 -```shell -cd /home/admin/dev/psi + ```bash + cd /home/admin/dev/psi -bazel build //... -c opt -``` + bazel build //... -c opt -编译完成后，在`bazel-bin/psi/apps/psi_launcher`目录下生成`main`可执行文件，后续可通过执行`./bazel-bin/psi/apps/psi_launcher/main --config xxx.json`进行验证 + # 创建临时目录 bin，存放 main 可执行文件 + mkdir bin + cp ./bazel-bin/psi/apps/psi_launcher/main ./bin/main + ``` -## 将二进制部署至 virtCCA 内 + 编译完成后，在`bazel-bin/psi/apps/psi_launcher`目录下生成`main`可执行文件 + +### 部署至 virtCCA 内 可让`/home/admin/dev/psi`的目录结构与`virtCCA`内部保持一致，方便进行测试 -# kcal psi 在蚂蚁 psi 库上的测试 +```bash +# cvm 内创建目录 +virsh console cvm +mkdir -p /home/admin/dev + +# 宿主机内，拷贝蚂蚁 psi 编译后项目整体至 cvm 内 +cd /home/admin/dev +tar -czvf psi.tar.gz psi +scp -r /home/admin/dev/psi.tar.gz root@:/home/admin/dev/ -编译完成后，进入`/home/admin/dev/psi`目录，步骤可参考：`examples/psi/README.md`​ +# 进入 cvm 内，解压 +virsh console cvm +cd /home/admin/dev && tar -xzvf psi.tar.gz +``` -## 生成 psi 测试数据 +## KCAL 中间件适配蚂蚁 psi 库测试 + +### 测试数据准备 + +#### 宿主机侧数据生成 + +1. 进入`/home/admin/dev/psi`目录 +2. PSI 算法数据生成 + + 步骤可参考：`examples/psi/README.md`​ + + ```bash + python examples/psi/generate_psi_data.py \ + --receiver_item_cnt 1e6 \ + --sender_item_cnt 1e6 \ + --intersection_cnt 8e4 \ + --id_cnt 2 \ + --receiver_path /tmp/receiver_input.csv \ + --sender_path /tmp/sender_input.csv \ + --intersection_path /tmp/intersection.csv + ``` + + > 说明： + > + > --receiver_item_cnt：receiver 方拥有的数据总量 + > + > --sender_item_cnt：sender 方拥有的数据总量 + > + > --intersection_cnt：约定两方产生交集部分的数据总量 + > + > --id_cnt：每个参与方的输入数据包含几个字段 + > + > --receiver_path：receiver 方输入数据的文件位置 + > + > --sender_path：sender 方输入数据的文件位置 + > + > --intersection_path：生成的交集数据的文件位置 + > +3. PIR 算法数据生成 + + 步骤可参考：`examples/pir/README.md`​ + + ```bash + python examples/pir/apsi/test_data_creator.py \ + --sender_size=10000000 \ + --receiver_size=1000 \ + --intersection_size=100 \ + --label_byte_count=100 \ + --item_byte_count=16 + + # 生成的用来最后比对结果的交集集合 + mv ground_truth.csv /tmp/ground_truth.csv + # 数据库模拟数据 + mv db.csv /tmp/db.csv + # 查询数据 + mv query.csv /tmp/query.csv + ``` + + > 说明： + > + > --sender_size：服务端数据库总体数据行数 + > + > --receiver_size：客户端要查询的 key 的数量 + > + > --intersection_size：实际上生成的数据里面，服务端只有 intersection_size 个包含客户端能够查到的键值 + > + > --label_byte_count：服务端数据库每个 value 所占的字节数 + > + > --item_byte_count：服务端数据库每个 key 所占的字节数 + > + +#### 拷贝测试数据至 virtCCA 内 + +将 PSI PIR 的测试数据分别拷贝进两个 virtCCA 内 ```bash -cd /home/admin/dev/psi - -python examples/psi/generate_psi_data.py \ - --receiver_item_cnt 1e6 \ - --sender_item_cnt 1e6 \ - --intersection_cnt 8e4 \ - --id_cnt 2 \ - --receiver_path /tmp/receiver_input.csv \ - --sender_path /tmp/sender_input.csv \ - --intersection_path /tmp/intersection.csv +scp /tmp/receiver_input.csv \ + /tmp/sender_input.csv \ + /tmp/intersection.csv \ + /tmp/ground_truth.csv \ + /tmp/db.csv \ + /tmp/query.csv \ + root@:/tmp/ ``` -> 说明： -> -> --receiver_item_cnt：receiver 方拥有的数据总量 -> -> --sender_item_cnt：sender 方拥有的数据总量 -> -> --intersection_cnt：约定两方产生交集部分的数据总量 -> -> --id_cnt：每个参与方的输入数据包含几个字段 -> -> --receiver_path：receiver 方输入数据的文件位置 -> -> --sender_path：sender 方输入数据的文件位置 -> -> --intersection_path：生成的交集数据的文件位置 - -## 配置文件说明 +### KCAL PSI 测试 + +#### 配置文件说明 配置文件已在`patch`中提供，只需修改下列说明的部分进行测试 @@ -114,13 +184,13 @@ python examples/psi/generate_psi_data.py \ "thread_count": 16 // 线程数按需修改，目前固定 16 线程 }, "role": "ROLE_SENDER", - "broadcast_result": true + "broadcast_result": true }, "input_config": { "type": "IO_TYPE_FILE_CSV", "path": "/tmp/sender_input.csv" // 当前参与方运行 psi 算法的数据输入文件位置，按需修改 }, - "output_config": { + "output_config": { "type": "IO_TYPE_FILE_CSV", // 文件类型不需修改 "path": "/tmp/kcal_sender_output.csv" // 两方运行完 psi 算法后，最终交集文件的输出位置，按需修改 }, @@ -151,17 +221,17 @@ python examples/psi/generate_psi_data.py \ ``` -### kcal 两个配置文件 +#### kcal 两个配置文件 - examples/psi/config/kcal_receiver.json - examples/psi/config/kcal_sender.json -### 蚂蚁对比配置文件 +#### 蚂蚁对比配置文件 - examples/psi/config/rr22_receiver_recovery.json - examples/psi/config/rr22_sender_recovery.json -## 测试 +#### 测试 进入两个 cvm 分别执行以下命令 @@ -176,44 +246,15 @@ cd /home/admin/dev/psi 运行完以上命令后，每个`cvm`内会在配置文件中指明的`output_config.path`路径中生成交集文件 -## 结果对比 +#### 结果对比 将`/tmp/kcal_sender_output.csv`、`/tmp/kcal_receiver_output.csv`的内容与一开始生成的交集文件`/tmp/intersection.csv`内容进行比对，结果保持一致 -# kcal pir 在蚂蚁 psi 库上的测试 +### KCAL PIR 测试 编译完成后，进入`/home/admin/dev/psi`目录，步骤可参考：`examples/pir/README.md`​ -## 生成 pir 测试数据 - -```bash -cd /home/admin/dev/psi - -python examples/pir/apsi/test_data_creator.py \ - --sender_size=10000000 \ - --receiver_size=1000 \ - --intersection_size=100 \ - --label_byte_count=100 \ - --item_byte_count=16 - -mv ground_truth.csv /tmp/ground_truth.csv -mv db.csv /tmp/db.csv -mv query.csv /tmp/query.csv -``` - -> 说明： -> -> --sender_size：服务端数据库总体数据行数 -> -> --receiver_size：客户端要查询的 key 的数量 -> -> --intersection_size：实际上生成的数据里面，服务端只有 intersection_size 个包含客户端能够查到的键值 -> -> --label_byte_count：服务端数据库每个 value 所占的字节数 -> -> --item_byte_count：服务端数据库每个 key 所占的字节数 - -## 配置文件说明 +#### 配置文件说明 配置文件已在`patch`中提供，只需修改下列说明的部分进行测试 @@ -263,18 +304,18 @@ mv query.csv /tmp/query.csv } ``` -### kcal 两个配置文件 +#### kcal 两个配置文件 - examples/pir/config/kcal_pir_receiver.json - examples/pir/config/kcal_pir_sender.json -### 蚂蚁对比配置文件 +#### 蚂蚁对比配置文件 - examples/pir/config/apsi_sender_setup.json - examples/pir/config/apsi_sender_online.json - examples/pir/config/apsi_receiver.json -## 测试 +#### 测试 进入两个 cvm 分别执行以下命令 @@ -289,6 +330,6 @@ cd /home/admin/dev/psi 运行完以上命令后，客户端`cvm`内会在配置文件中指明的`kcal_pir_receiver_config.output_file`路径中生成查询结果文件 -## 结果对比 +#### 结果对比 将`/tmp/result.csv`的内容与一开始生成的交集文件`/tmp/ground_truth.csv`内容进行比对，结果保持一致 -- Gitee From 625ed03517c000bf2cdd21ca68f8b4c2fae46db5 Mon Sep 17 00:00:00 2001 From: cjs <1823782890@qq.com> Date: Tue, 5 Aug 2025 18:06:41 +0800 Subject: [PATCH 18/18] Modify README.md --- .../secrerflow/psi/README.md | 265 ++++++++++-------- 1 file changed, 153 insertions(+), 112 deletions(-) diff --git a/MPC/third_party_adaptor/secrerflow/psi/README.md b/MPC/third_party_adaptor/secrerflow/psi/README.md index 851710a..cf5f23a 100644 --- a/MPC/third_party_adaptor/secrerflow/psi/README.md +++ b/MPC/third_party_adaptor/secrerflow/psi/README.md @@ -1,105 +1,175 @@ -# 前置条件 +# KCAL 中间件适配蚂蚁 psi 库 -1. 需获取 `kcal` 包，含 `include` 和 `lib` 目录 -2. 需要 bazel 编译构建工具，编译环境依赖参考：[devtools/dockerfiles/release-ci-aarch64.DockerFile at main · secretflow/devtools](https://github.com/secretflow/devtools/blob/main/dockerfiles/release-ci-aarch64.DockerFile) -3. 运行环境为`virtCCA cvm`​ +本文档介绍 KCAL 中间件如何在 virtCCA 内部署已经适配了的蚂蚁 psi 库（`tag: psi-v0.6.0.dev250507` 版本） -# 中间件在蚂蚁 psi 库的编译 +## 前置条件 -## 创建工作目录 +1. 需获取 `kcal` 包，含 `include` 和 `lib` 目录，获取链接：[https://support.huawei.com/enterprise/zh/software/265814201-ESW2001490913](https://support.huawei.com/enterprise/zh/software/265814201-ESW2001490913) +2. 需要 bazel 编译构建工具，编译环境依赖参考：[devtools/dockerfiles/release-ci-aarch64.DockerFile at main · secretflow/devtools](https://github.com/secretflow/devtools/blob/main/dockerfiles/release-ci-aarch64.DockerFile) +3. 运行环境为`virtCCA cvm，前提是用户已经启动两个 virtCCA 的机密虚机（cvm1、cvm2）`​ -为方便进行演示，以下操作均以 `/home/admin/dev` 目录作为工作主目录 +## 中间件使能和部署步骤 -## clone virtCCA\_sdk 仓 +### 宿主机编译蚂蚁 psi 库 -```shell -cd /home/admin/dev +1. 创建工作目录 -git clone https://gitee.com/openeuler/virtCCA_sdk.git -``` + 为方便进行演示，以下操作均以 `/home/admin/dev` 目录作为工作主目录 -## clone 蚂蚁 psi 仓，应用 patch +2. clone virtCCA\_sdk 仓 -注：目前已适配蚂蚁 `tag: psi-v0.6.0.dev250507` 版本的 `psi`库 + ```bash + cd /home/admin/dev -```shell -cd /home/admin/dev + git clone https://gitee.com/openeuler/virtCCA_sdk.git + ``` -# clone 仓库，并创建一个本地分支 -git clone --branch "v0.6.0.dev250507" https://github.com/secretflow/psi.git -git switch -c kcal-on-v0.6.0 +3. clone 蚂蚁 psi 仓，应用 patch -# 应用 virtCCA_sdk 下面的 patch -cd /home/admin/dev/psi # 进到蚂蚁 psi 目录下 -git apply /home/admin/dev/virtCCA_sdk/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch -``` + ```bash + cd /home/admin/dev -## 引入中间件和 kcal 库 + # clone 仓库，并创建一个本地分支 + git clone --branch "v0.6.0.dev250507" https://github.com/secretflow/psi.git + git switch -c kcal-on-v0.6.0 -### 引入 kcal 库 + # 应用 virtCCA_sdk 下面的 patch + cd /home/admin/dev/psi # 进到蚂蚁 psi 目录下 + git apply /home/admin/dev/virtCCA_sdk/MPC/third_party_adaptor/secrerflow/psi/patches/kcal.patch + ``` -```shell -# 假设 kcal 库已下载在 /opt/kcal 目录下 -cp -r /opt/kcal/include /home/admin/dev/psi/third_party/kcal/ -cp -r /opt/kcal/lib /home/admin/dev/psi/third_party/kcal/ -``` +4. 引入中间件和 kcal 库 -### 引入中间件 + ```bash + # 假设 kcal 库已下载在 /opt/kcal 目录下 + cp -r /opt/kcal/include /home/admin/dev/psi/third_party/kcal/ + cp -r /opt/kcal/lib /home/admin/dev/psi/third_party/kcal/ -```shell -cp -r /home/admin/dev/virtCCA_sdk/MPC/middleware/* /home/admin/dev/psi/third_party/kcal_middleware/ -``` + # 引入中间件 + cp -r /home/admin/dev/virtCCA_sdk/MPC/middleware/* /home/admin/dev/psi/third_party/kcal_middleware/ + ``` -## 执行编译 +5. 编译 -```shell -cd /home/admin/dev/psi + ```bash + cd /home/admin/dev/psi -bazel build //... -c opt -``` + bazel build //... -c opt -编译完成后，在`bazel-bin/psi/apps/psi_launcher`目录下生成`main`可执行文件，后续可通过执行`./bazel-bin/psi/apps/psi_launcher/main --config xxx.json`进行验证 + # 创建临时目录 bin，存放 main 可执行文件 + mkdir bin + cp ./bazel-bin/psi/apps/psi_launcher/main ./bin/main + ``` -## 将二进制部署至 virtCCA 内 + 编译完成后，在`bazel-bin/psi/apps/psi_launcher`目录下生成`main`可执行文件 + +### 部署至 virtCCA 内 可让`/home/admin/dev/psi`的目录结构与`virtCCA`内部保持一致，方便进行测试 -# kcal psi 在蚂蚁 psi 库上的测试 +```bash +# cvm 内创建目录 +virsh console cvm +mkdir -p /home/admin/dev + +# 宿主机内，拷贝蚂蚁 psi 编译后项目整体至 cvm 内 +cd /home/admin/dev +tar -czvf psi.tar.gz psi +scp -r /home/admin/dev/psi.tar.gz root@:/home/admin/dev/ -编译完成后，进入`/home/admin/dev/psi`目录，步骤可参考：`examples/psi/README.md`​ +# 进入 cvm 内，解压 +virsh console cvm +cd /home/admin/dev && tar -xzvf psi.tar.gz +``` -## 生成 psi 测试数据 +## KCAL 中间件适配蚂蚁 psi 库测试 + +### 测试数据准备 + +#### 宿主机侧数据生成 + +1. 进入`/home/admin/dev/psi`目录 +2. PSI 算法数据生成 + + 步骤可参考：`examples/psi/README.md`​ + + ```bash + python examples/psi/generate_psi_data.py \ + --receiver_item_cnt 1e6 \ + --sender_item_cnt 1e6 \ + --intersection_cnt 8e4 \ + --id_cnt 2 \ + --receiver_path /tmp/receiver_input.csv \ + --sender_path /tmp/sender_input.csv \ + --intersection_path /tmp/intersection.csv + ``` + + > 说明： + > + > --receiver_item_cnt：receiver 方拥有的数据总量 + > + > --sender_item_cnt：sender 方拥有的数据总量 + > + > --intersection_cnt：约定两方产生交集部分的数据总量 + > + > --id_cnt：每个参与方的输入数据包含几个字段 + > + > --receiver_path：receiver 方输入数据的文件位置 + > + > --sender_path：sender 方输入数据的文件位置 + > + > --intersection_path：生成的交集数据的文件位置 + > +3. PIR 算法数据生成 + + 步骤可参考：`examples/pir/README.md`​ + + ```bash + python examples/pir/apsi/test_data_creator.py \ + --sender_size=10000000 \ + --receiver_size=1000 \ + --intersection_size=100 \ + --label_byte_count=100 \ + --item_byte_count=16 + + # 生成的用来最后比对结果的交集集合 + mv ground_truth.csv /tmp/ground_truth.csv + # 数据库模拟数据 + mv db.csv /tmp/db.csv + # 查询数据 + mv query.csv /tmp/query.csv + ``` + + > 说明： + > + > --sender_size：服务端数据库总体数据行数 + > + > --receiver_size：客户端要查询的 key 的数量 + > + > --intersection_size：实际上生成的数据里面，服务端只有 intersection_size 个包含客户端能够查到的键值 + > + > --label_byte_count：服务端数据库每个 value 所占的字节数 + > + > --item_byte_count：服务端数据库每个 key 所占的字节数 + > + +#### 拷贝测试数据至 virtCCA 内 + +将 PSI PIR 的测试数据分别拷贝进两个 virtCCA 内 ```bash -cd /home/admin/dev/psi - -python examples/psi/generate_psi_data.py \ - --receiver_item_cnt 1e6 \ - --sender_item_cnt 1e6 \ - --intersection_cnt 8e4 \ - --id_cnt 2 \ - --receiver_path /tmp/receiver_input.csv \ - --sender_path /tmp/sender_input.csv \ - --intersection_path /tmp/intersection.csv +scp /tmp/receiver_input.csv \ + /tmp/sender_input.csv \ + /tmp/intersection.csv \ + /tmp/ground_truth.csv \ + /tmp/db.csv \ + /tmp/query.csv \ + root@:/tmp/ ``` -> 说明： -> -> --receiver_item_cnt：receiver 方拥有的数据总量 -> -> --sender_item_cnt：sender 方拥有的数据总量 -> -> --intersection_cnt：约定两方产生交集部分的数据总量 -> -> --id_cnt：每个参与方的输入数据包含几个字段 -> -> --receiver_path：receiver 方输入数据的文件位置 -> -> --sender_path：sender 方输入数据的文件位置 -> -> --intersection_path：生成的交集数据的文件位置 - -## 配置文件说明 +### KCAL PSI 测试 + +#### 配置文件说明 配置文件已在`patch`中提供，只需修改下列说明的部分进行测试 @@ -114,13 +184,13 @@ python examples/psi/generate_psi_data.py \ "thread_count": 16 // 线程数按需修改，目前固定 16 线程 }, "role": "ROLE_SENDER", - "broadcast_result": true + "broadcast_result": true }, "input_config": { "type": "IO_TYPE_FILE_CSV", "path": "/tmp/sender_input.csv" // 当前参与方运行 psi 算法的数据输入文件位置，按需修改 }, - "output_config": { + "output_config": { "type": "IO_TYPE_FILE_CSV", // 文件类型不需修改 "path": "/tmp/kcal_sender_output.csv" // 两方运行完 psi 算法后，最终交集文件的输出位置，按需修改 }, @@ -151,17 +221,17 @@ python examples/psi/generate_psi_data.py \ ``` -### kcal 两个配置文件 +#### kcal 两个配置文件 - examples/psi/config/kcal_receiver.json - examples/psi/config/kcal_sender.json -### 蚂蚁对比配置文件 +#### 蚂蚁对比配置文件 - examples/psi/config/rr22_receiver_recovery.json - examples/psi/config/rr22_sender_recovery.json -## 测试 +#### 测试 进入两个 cvm 分别执行以下命令 @@ -176,44 +246,15 @@ cd /home/admin/dev/psi 运行完以上命令后，每个`cvm`内会在配置文件中指明的`output_config.path`路径中生成交集文件 -## 结果对比 +#### 结果对比 将`/tmp/kcal_sender_output.csv`、`/tmp/kcal_receiver_output.csv`的内容与一开始生成的交集文件`/tmp/intersection.csv`内容进行比对，结果保持一致 -# kcal pir 在蚂蚁 psi 库上的测试 +### KCAL PIR 测试 编译完成后，进入`/home/admin/dev/psi`目录，步骤可参考：`examples/pir/README.md`​ -## 生成 pir 测试数据 - -```bash -cd /home/admin/dev/psi - -python examples/pir/apsi/test_data_creator.py \ - --sender_size=10000000 \ - --receiver_size=1000 \ - --intersection_size=100 \ - --label_byte_count=100 \ - --item_byte_count=16 - -mv ground_truth.csv /tmp/ground_truth.csv -mv db.csv /tmp/db.csv -mv query.csv /tmp/query.csv -``` - -> 说明： -> -> --sender_size：服务端数据库总体数据行数 -> -> --receiver_size：客户端要查询的 key 的数量 -> -> --intersection_size：实际上生成的数据里面，服务端只有 intersection_size 个包含客户端能够查到的键值 -> -> --label_byte_count：服务端数据库每个 value 所占的字节数 -> -> --item_byte_count：服务端数据库每个 key 所占的字节数 - -## 配置文件说明 +#### 配置文件说明 配置文件已在`patch`中提供，只需修改下列说明的部分进行测试 @@ -263,18 +304,18 @@ mv query.csv /tmp/query.csv } ``` -### kcal 两个配置文件 +#### kcal 两个配置文件 - examples/pir/config/kcal_pir_receiver.json - examples/pir/config/kcal_pir_sender.json -### 蚂蚁对比配置文件 +#### 蚂蚁对比配置文件 - examples/pir/config/apsi_sender_setup.json - examples/pir/config/apsi_sender_online.json - examples/pir/config/apsi_receiver.json -## 测试 +#### 测试 进入两个 cvm 分别执行以下命令 @@ -289,6 +330,6 @@ cd /home/admin/dev/psi 运行完以上命令后，客户端`cvm`内会在配置文件中指明的`kcal_pir_receiver_config.output_file`路径中生成查询结果文件 -## 结果对比 +#### 结果对比 将`/tmp/result.csv`的内容与一开始生成的交集文件`/tmp/ground_truth.csv`内容进行比对，结果保持一致 -- Gitee