# group_secure_copy

**Repository Path**: wlf-darkmatter/gscp

## Basic Information

- **Project Name**: group_secure_copy
- **Description**: 一个用于服务器集群间高速传输文件的工具
- **Primary Language**: Unknown
- **License**: Not specified
- **Default Branch**: master
- **Homepage**: None
- **GVP Project**: No

## Statistics

- **Stars**: 0
- **Forks**: 0
- **Created**: 2024-11-06
- **Last Updated**: 2024-11-11

## Categories & Tags

**Categories**: Uncategorized

**Tags**: None

## README


# 项目说明

一个用于服务器集群间高速传输文件的工具

项目参考：

https://www.cnblogs.com/jackadam/p/8568833.html

https://github.com/Tribler/tribler

https://blog.csdn.net/liuxiao723846/article/details/82082083

docker镜像p2p分发：

https://singhwang.github.io/2019/05/27/p2p_distribute_images_000/

# 环境设置

项目一开始暂时设定通过python进行开发，之后为解决可能在离线环境下无法联网下载python依赖而编译生成二进制包（C++）

## 编译需要的依赖包

libtorrent 编译
这里通过 docker 进行编译

```bash
docker run -it -v ./:/workspace -v ~/.cache/pip:/root/.cache/pip --name torrent_builder harbor.x-contion.top:500/x-contion/base_ubuntu:zh

```

容器内配置 依赖
```bash

apt update
apt install -y libcrypto++-dev libssl-dev
apt install -y software-properties-common
add-apt-repository -y ppa:deadsnakes/ppa
apt_install_py(){
    py_version=$1
    apt install -y python${py_version}-full python${py_version}-dev libpython${py_version}-dev libpython${py_version}-dbg
}
apt install -y libc6-dev
#* 安装多个不同版本的 python
apt_install_py 3.7
apt_install_py 3.8
apt_install_py 3.9
apt_install_py 3.10
apt_install_py 3.11
apt_install_py 3.12



#* x86 下载 arm 交叉编译工具


#* arm 下载 x86 交叉编译工具

```

想编译python的支持，还是需要编译boost
```bash
version_num=86
version_num2=0
version_boost=boost-1.$version_num.$version_num2
edit_jam_path_python(){
    version_num_py=$1
    rm -f ~/user-config.jam
    echo "using gcc ;" >> ~/user-config.jam
    echo "using mpi ;" >> ~/user-config.jam
    # sed -i "/^using python.*/d" ~/user-config.jam
    path_include_python=$(python -c "from sysconfig import get_paths; print(get_paths()['include'])")
    path_lib_python=$(python -c "from sysconfig import get_paths; print(get_paths()['platstdlib'])")
    echo "using python : ${version_num_py} : $(which python${version_num_py}) : ${path_include_python} : ${path_lib_python} ;" >> ~/user-config.jam
    cat ~/user-config.jam
}
set_boost_pathversion(){
    bak_pwd=$(pwd)
    cd /workspace ; mkdir -p packages ; cd packages
    filename=$version_boost-b2-nodocs.tar.gz
    if [[ ! -f $filename ]];then
        wget https://mirror.ghproxy.com/https://github.com/boostorg/boost/releases/download/$version_boost/$filename
    fi
    if [[ ! -d $version_boost ]];then
        tar -zxvf $filename
    fi
    cd $version_boost
    ln -sf $(pwd)/b2 /usr/bin/b2
    if [[ -f ~/user-config.jam ]];then
        rm -f ~/user-config.jam
    fi
    cd $bak_pwd
}

activate_python(){
    py_version=$1
    #* 创建python venv虚拟环境
    mkdir -p /workspace/venv
    path_venv=/workspace/venv/${py_version}
    ${py_version} -m venv $path_venv
    . $path_venv/bin/activate
    ${py_version} -m pip install wheel
}

build_boost(){
    py_version=$1

    cd /workspace/packages/$version_boost
    rm -f project-config.jam*
    echo -e "\033[33m构建 boost with-python=$py_version 中...\033[0m"
    #* 构建boost 的 python包 --with-python=${py_version}

    ln -sf $(pwd)/b2 /usr/bin/b2

    #* 修改 tools/build/src/tools/gcc.jam
    sed -i 's@    compile-link-flags <link>.*@    compile-link-flags <link>shared/<link>static/<target-os>$(non-windows) : -fPIC ;@g' tools/build/src/tools/gcc.jam
    # tmp_str="toolset.flags gcc.compile"
    # sed -i "s@${tmp_str} USER_OPTIONS.*@${tmp_str} USER_OPTIONS <cflags> ;@g" tools/build/src/tools/gcc.jam
    # sed -i 's@    compile-link-flags <link>.*@    compile-link-flags <link>shared/<link>static/<target-os>$(non-windows) : -fPIC ;@g' tools/build/src/tools/gcc.jam

    cat tools/build/src/tools/gcc.jam | grep "compile-link-flags <link>"
    ./bootstrap.sh --with-python=/usr/bin/${py_version}
    export BOOST_PYTHON_PREFIX=/workspace/packages/boost_build_$py_version
    b2 --prefix=$BOOST_PYTHON_PREFIX cxxflags=-fPIC variant=release link=static runtime-link=static threading=multi headers install
}

build_libtorrent(){
    py_version=$1
    # 编译 libtorrent
    echo -e "\033[33m构建 libtorrent 中...\033[0m"

    cd /workspace/packages
    if [[ ! -d python-cmake-wheel ]];then
        git clone https://mirror.ghproxy.com/https://github.com/Klebert-Engineering/python-cmake-wheel.git
    fi

    cd /workspace/libtorrent;
    if [[ ! -f bindings/python/__init__.py ]];then
        touch bindings/python/__init__.py
    fi

    cd /workspace/libtorrent;
    echo "from .libtorrent import *" > bindings/python/__init__.py
    echo "
if (python-bindings)
	add_subdirectory(python)
endif()

set(WHEEL_DEPLOY_DIRECTORY \"\${CMAKE_BINARY_DIR}\")
list(APPEND CMAKE_MODULE_PATH \"/workspace/packages/python-cmake-wheel\")
include(python-wheel)

add_wheel(torrent-rasterbar
  NAME pytorrent
  AUTHOR \"wlf\"
  VERSION \${VER_MAJOR}.\${VER_MINOR}.\${VER_TINY}
  PYTHON_REQUIRES \">=3.8\"
  DESCRIPTION \"Binary Python libtorrent wheel.\"
  DEPLOY_FILES # 选择哪些拷贝到 pytorrent.wheel/pytorrent 目录下
	python/*.so
	${CMAKE_CURRENT_SOURCE_DIR}/python/*.py # 代码里的 bindings/python/*.py
)
" > bindings/CMakeLists.txt

    cd /workspace/libtorrent; mkdir -p bin/build_$py_version; cd bin/build_$py_version;

    rm -f CMakeCache.txt
    cmake -DCMAKE_BUILD_TYPE=Release \
        -DCMAKE_CXX_STANDARD=14 \
        -DBUILD_SHARED_LIBS=ON \
        -DCMAKE_FIND_ROOT_PATH=$BOOST_PYTHON_PREFIX/lib/cmake/ \
        -Dboost_python_DIR=$BOOST_PYTHON_PREFIX/lib/cmake/boost_python${version_boost#*boost} \
        -Dpython-install-system-dir=ON \
        -Dpython-bindings=ON \
        -Dstatic_runtime=ON \
        -Dpython-egg-info=ON \
        -DBoost_USE_STATIC_LIBS=ON \
        -DBoost_USE_STATIC_RUNTIME=ON \
        -DCMAKE_C_FLAGS="-fPIC -static-libgcc" \
        -DCMAKE_CXX_FLAGS="-fPIC -static-libstdc++" \
        -G Ninja ../..

    ninja -j12
    #* 临时解压
    tmp_name=$(ls | grep whl) ; rm *.zip -rf; cp $tmp_name ${tmp_name/?whl}.zip
    if [[ -d tmp ]];then rm -rf tmp; fi ; unzip ${tmp_name/?whl}.zip -d tmp; du -sh tmp/pytorrent/*

    #* 打包

    # du -sh dist/*
    echo -e "\033[33m构建 ${py_version} 完成\033[0m"
}




pack_wheel_libtorrent(){
    cd /workspace/libtorrent/bindings/python;
    if [[ -f setup.cfg ]];then
        mv -f setup.cfg setup.cfg.bak
    fi
    if [[ -f setup.py ]];then
        mv -f setup.py setup.py.bak
    fi
    cp -f /workspace/patch/libtorrent/setup.py setup.py
    echo "[metadata]" >> setup.cfg
    echo $(cat setup.cfg.bak| grep version) >> setup.cfg

    export PATH_LIBTORRENT_CMAKE_BUILD=../../build_$py_version

    ls | grep so | du -sh
    ${py_version} setup.py bdist_wheel


    if [[ -f setup.cfg.bak ]];then
        mv -f setup.cfg.bak setup.cfg
    fi
    if [[ -f setup.py.bak ]];then
        mv -f setup.py.bak setup.py
    fi
    rm -f ./*.so
    du -sh dist/*
}

build_all(){
    py_version=$1
    activate_python $py_version
    build_boost $py_version
    build_libtorrent $py_version
    pack_wheel_libtorrent $py_version
}

set_boost_pathversion
build_all python3.7
build_all python3.8
build_all python3.9
build_all python3.10
build_all python3.11
# build_libtorrent python3.12 #! 3.12 依旧会报错，就是 distutils 的锅，反正用不上，先不管了

echo -e "\033[32m生成的whl文件位于 ./bindings/python/dist \033[31m"
ls ./bindings/python/dist/*.whl

```

# 使用示例

## 生成临时调试配置文件

临时生成 docker-compose.yaml 的配置文件用于调试
```
python tmp/generate_config.py
```

会在`generate`下生成一个 `test-ipconfig.yaml` 文件，可以用于调试，且该文件符合配置文件的要求

> 如果要测试，可以把 `generate/test-ipconfig.yaml` 拖到 `test/test-ipconfig.yaml` 下

## 创建用于调试的容器

运行上一步的python的脚本后，会产生一个叫做`docker-compose.yaml`的文件，直接运行
```bash
docker-compose up -d
```
就可以启动对应数量的容器了

## 网络诊断命令

```bash
#* 查看路由表
iptables -t nat -nL
```