From 90f568992d68e75155c9f1b2b5df56b8cfe62d34 Mon Sep 17 00:00:00 2001 From: "zhongling.h" Date: Thu, 16 May 2024 16:05:22 +0800 Subject: [PATCH 1/6] add manylinux 8.8 dockerfile --- AI/manylinux/8.8/Dockerfile | 31 ++++++++++++++++++ AI/manylinux/8.8/finalize-python.sh | 44 ++++++++++++++++++++++++++ AI/manylinux/8.8/python-tag-abi-tag.py | 10 ++++++ AI/manylinux/buildspec.yml | 29 +++++++++++++++++ AI/manylinux/version.yml | 8 +++++ 5 files changed, 122 insertions(+) create mode 100644 AI/manylinux/8.8/Dockerfile create mode 100755 AI/manylinux/8.8/finalize-python.sh create mode 100644 AI/manylinux/8.8/python-tag-abi-tag.py create mode 100644 AI/manylinux/buildspec.yml create mode 100644 AI/manylinux/version.yml diff --git a/AI/manylinux/8.8/Dockerfile b/AI/manylinux/8.8/Dockerfile new file mode 100644 index 0000000..32546e3 --- /dev/null +++ b/AI/manylinux/8.8/Dockerfile @@ -0,0 +1,31 @@ +# default to latest supported policy, x86_64 +FROM registry.openanolis.cn/openanolis/anolisos:8.9 +ARG POLICY=manylinux_2_28 +ARG PLATFORM=x86_64 +ARG DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-12/root +ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst +ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin: + +ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} +ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 + +RUN sed -i 's#mirrors.openanolis.cn#mirrors.cloud.aliyuncs.com#g' /etc/yum.repos.d/* + +RUN yum install -y glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel \ + libICE-devel libSM-devel zlib-devel expat-devel zlib bzip2 expat ncurses readline gdbm libpcap xz openssl keyutils-libs \ + libkadm5 libcom_err libidn uuid libffi libdb tk git autoconf automake bison bzip2 diffutils \ + file make patch unzip curl glibc-locale-source glibc-langpack-en hardlink hostname libnsl libxcrypt which \ + gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ \ + openssl-devel rust cargo yasm autoconf automake libtool libxcrypt-devel git sqlite openssl-devel tcl-devel + +RUN git clone https://gitee.com/src-anolis-os/python3.git /tmp/python3 --branch=a23.0 --depth=1 && \ + pushd /tmp/python3 && tar -xaf Python-3.10.13.tar.xz && cd Python-3.10.13 && \ + ./configure --prefix="/opt/_internal/cpython-3.10.13" --disable-shared --with-ensurepip=no && \ + make -j && make install && rm -rf /tmp/python3 + +COPY python-tag-abi-tag.py finalize-python.sh /opt +RUN mkdir /opt/python && \ + for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do \ + /opt/finalize-python.sh $PREFIX ;\ + done + diff --git a/AI/manylinux/8.8/finalize-python.sh b/AI/manylinux/8.8/finalize-python.sh new file mode 100755 index 0000000..5c2abae --- /dev/null +++ b/AI/manylinux/8.8/finalize-python.sh @@ -0,0 +1,44 @@ +#!/bin/bash +set -exuo pipefail + +PREFIX=$1 + +function download_gitee_src () { + # arg1: pkg name + # arg2: target dir + cd ${2} + git clone https://gitee.com/src-anolis-os/python-${1}.git --branch=a23.0 --depth=1 && mv python-${1}/*.tar.gz ${2}/ + rm -rf python-${1} +} + +if [ -e ${PREFIX}/bin/python3 ] && [ ! -e ${PREFIX}/bin/python ]; then + ln -s python3 ${PREFIX}/bin/python +fi + +mkdir -p /tmp/python_srcs +${PREFIX}/bin/python -m ensurepip + +# download python basic package sources +# these are +for pkg in pip build packaging pyproject-hooks tomli wheel setuptools; do + download_gitee_src $pkg /tmp/python_srcs +done + +${PREFIX}/bin/python -m pip install -U /tmp/python_srcs/*.tar.gz \ + https://files.pythonhosted.org/packages/88/56/0f2d69ed9a0060da009f672ddec8a71c041d098a66f6b1d80264bf6bbdc0/pyelftools-0.31.tar.gz \ + https://files.pythonhosted.org/packages/62/6c/0a99a7df52487bd2204a1b8ba56cd90f6f651d23027dc355e0e21b7b0492/auditwheel-6.0.0.tar.gz && rm -rf /tmp/python_srcs + +if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then + ln -s pip3 ${PREFIX}/bin/pip +fi + +ABI_TAG=$(${PREFIX}/bin/python3 /opt/python-tag-abi-tag.py) +ln -s ${PREFIX} /opt/python/${ABI_TAG} + +# Make versioned python commands available directly in environment. +PY_VER=$(${PREFIX}/bin/python -c "import sys; print('.'.join(str(v) for v in sys.version_info[:2]))") +PY_IMPL=$(${PREFIX}/bin/python -c "import sys; print(sys.implementation.name)") +if [[ "${PY_IMPL}" == "cpython" ]]; then + ln -s ${PREFIX}/bin/python /usr/local/bin/python${PY_VER} +fi +ln -s ${PREFIX}/bin/python /usr/local/bin/${PY_IMPL}${PY_VER} diff --git a/AI/manylinux/8.8/python-tag-abi-tag.py b/AI/manylinux/8.8/python-tag-abi-tag.py new file mode 100644 index 0000000..c3ce1f1 --- /dev/null +++ b/AI/manylinux/8.8/python-tag-abi-tag.py @@ -0,0 +1,10 @@ +# Utility script to print the python tag + the abi tag for a Python +# See PEP 425 for exactly what these are, but an example would be: +# cp27-cp27mu + +from packaging.tags import sys_tags + + +# first tag is always the more specific tag +tag = next(sys_tags()) +print("{0}-{1}".format(tag.interpreter, tag.abi)) diff --git a/AI/manylinux/buildspec.yml b/AI/manylinux/buildspec.yml new file mode 100644 index 0000000..738223f --- /dev/null +++ b/AI/manylinux/buildspec.yml @@ -0,0 +1,29 @@ +name: &NAME +version: &VERSION +image_type: &IMAGE_TYPE +baseos_version: &BASEOS_VERSION + +# 定义镜像仓库信息 +repository_info: + acr_anolis: &ACR_ANOLIS anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis + +t-one: + workspace: &WORKSPACE container_ci_test + project: &PROJECT default_container_ci_test + test_suite: &TEST_SUITE image-ci-test + test_conf: &TEST_CONF group=language_container + test_case: &TEST_CASE golang_language_container + cloud_server_tag: &CLOUD_SERVER_TAG [anolis-container-ci-x86, anolis-container-ci-arm] + +images: + BuildPythonManylinuxDockerImage: + build: true + docker_file: + path: AI/manylinux/8.8/Dockerfile + variable: + scene: + args: [] + tags: [8.8] + registry: [*ACR_ANOLIS] + test_config: + - [*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''] \ No newline at end of file diff --git a/AI/manylinux/version.yml b/AI/manylinux/version.yml new file mode 100644 index 0000000..198aea6 --- /dev/null +++ b/AI/manylinux/version.yml @@ -0,0 +1,8 @@ +# 版本关系依赖表，默认继承 version-base.yml 配置，可重写覆盖 +BaseDependency: ../version-base.yml +Dependency: + name: python + image_type: language + versions: + 3.8: + baseos_version: [[ignored]] -- Gitee From c60d98923fb8e45a2ac9eee40e5f338762f9a147 Mon Sep 17 00:00:00 2001 From: "zhongling.h" Date: Wed, 22 May 2024 11:25:28 +0800 Subject: [PATCH 2/6] add auditwheel to PATH --- AI/manylinux/8.8/Dockerfile | 2 ++ AI/manylinux/8.8/finalize-python.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/AI/manylinux/8.8/Dockerfile b/AI/manylinux/8.8/Dockerfile index 32546e3..1f137d7 100644 --- a/AI/manylinux/8.8/Dockerfile +++ b/AI/manylinux/8.8/Dockerfile @@ -29,3 +29,5 @@ RUN mkdir /opt/python && \ /opt/finalize-python.sh $PREFIX ;\ done +RUN ln -s /opt/_internal/cpython-3.10.13/bin/auditwheel /usr/local/bin/auditwheel + diff --git a/AI/manylinux/8.8/finalize-python.sh b/AI/manylinux/8.8/finalize-python.sh index 5c2abae..247f382 100755 --- a/AI/manylinux/8.8/finalize-python.sh +++ b/AI/manylinux/8.8/finalize-python.sh @@ -19,7 +19,7 @@ mkdir -p /tmp/python_srcs ${PREFIX}/bin/python -m ensurepip # download python basic package sources -# these are +# these are temporary for pkg in pip build packaging pyproject-hooks tomli wheel setuptools; do download_gitee_src $pkg /tmp/python_srcs done -- Gitee From 5d57cd670a9a2e4fffe2f915b276b69b6bdce927 Mon Sep 17 00:00:00 2001 From: "zhongling.h" Date: Wed, 22 May 2024 13:59:13 +0800 Subject: [PATCH 3/6] add gcc g++ make --- AI/manylinux/8.8/Dockerfile | 6 ++++-- AI/manylinux/8.8/finalize-python.sh | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/AI/manylinux/8.8/Dockerfile b/AI/manylinux/8.8/Dockerfile index 1f137d7..41b17d6 100644 --- a/AI/manylinux/8.8/Dockerfile +++ b/AI/manylinux/8.8/Dockerfile @@ -16,7 +16,8 @@ RUN yum install -y glibc-devel libstdc++-devel glib2-devel libX11-devel libXext- libkadm5 libcom_err libidn uuid libffi libdb tk git autoconf automake bison bzip2 diffutils \ file make patch unzip curl glibc-locale-source glibc-langpack-en hardlink hostname libnsl libxcrypt which \ gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ \ - openssl-devel rust cargo yasm autoconf automake libtool libxcrypt-devel git sqlite openssl-devel tcl-devel + openssl-devel rust cargo yasm autoconf automake libtool libxcrypt-devel git sqlite openssl-devel tcl-devel \ + gcc gcc-c++ cmake RUN git clone https://gitee.com/src-anolis-os/python3.git /tmp/python3 --branch=a23.0 --depth=1 && \ pushd /tmp/python3 && tar -xaf Python-3.10.13.tar.xz && cd Python-3.10.13 && \ @@ -29,5 +30,6 @@ RUN mkdir /opt/python && \ /opt/finalize-python.sh $PREFIX ;\ done -RUN ln -s /opt/_internal/cpython-3.10.13/bin/auditwheel /usr/local/bin/auditwheel +RUN ln -s /opt/_internal/cpython-3.10.13/bin/auditwheel /usr/local/bin/auditwheel && \ + ln -s /opt/_internal/cpython-3.10.13/bin/patchelf /usr/local/bin/patchelf diff --git a/AI/manylinux/8.8/finalize-python.sh b/AI/manylinux/8.8/finalize-python.sh index 247f382..683a2a7 100755 --- a/AI/manylinux/8.8/finalize-python.sh +++ b/AI/manylinux/8.8/finalize-python.sh @@ -26,7 +26,8 @@ done ${PREFIX}/bin/python -m pip install -U /tmp/python_srcs/*.tar.gz \ https://files.pythonhosted.org/packages/88/56/0f2d69ed9a0060da009f672ddec8a71c041d098a66f6b1d80264bf6bbdc0/pyelftools-0.31.tar.gz \ - https://files.pythonhosted.org/packages/62/6c/0a99a7df52487bd2204a1b8ba56cd90f6f651d23027dc355e0e21b7b0492/auditwheel-6.0.0.tar.gz && rm -rf /tmp/python_srcs + https://files.pythonhosted.org/packages/62/6c/0a99a7df52487bd2204a1b8ba56cd90f6f651d23027dc355e0e21b7b0492/auditwheel-6.0.0.tar.gz \ + https://files.pythonhosted.org/packages/83/ec/ac383eb82792e092d8037649b382cf78a7b79c2ce4e5b861f61519b9b14e/patchelf-0.17.2.1.tar.gz && rm -rf /tmp/python_srcs if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then ln -s pip3 ${PREFIX}/bin/pip -- Gitee From 2dfffdd7795aa2327d3933c3eeabc8bb3ff31f19 Mon Sep 17 00:00:00 2001 From: "zhongling.h" Date: Wed, 22 May 2024 14:05:05 +0800 Subject: [PATCH 4/6] fix 8.9 --- AI/manylinux/{8.8 => 8.9}/Dockerfile | 0 AI/manylinux/{8.8 => 8.9}/finalize-python.sh | 0 AI/manylinux/{8.8 => 8.9}/python-tag-abi-tag.py | 0 AI/manylinux/buildspec.yml | 6 +++--- 4 files changed, 3 insertions(+), 3 deletions(-) rename AI/manylinux/{8.8 => 8.9}/Dockerfile (100%) rename AI/manylinux/{8.8 => 8.9}/finalize-python.sh (100%) rename AI/manylinux/{8.8 => 8.9}/python-tag-abi-tag.py (100%) diff --git a/AI/manylinux/8.8/Dockerfile b/AI/manylinux/8.9/Dockerfile similarity index 100% rename from AI/manylinux/8.8/Dockerfile rename to AI/manylinux/8.9/Dockerfile diff --git a/AI/manylinux/8.8/finalize-python.sh b/AI/manylinux/8.9/finalize-python.sh similarity index 100% rename from AI/manylinux/8.8/finalize-python.sh rename to AI/manylinux/8.9/finalize-python.sh diff --git a/AI/manylinux/8.8/python-tag-abi-tag.py b/AI/manylinux/8.9/python-tag-abi-tag.py similarity index 100% rename from AI/manylinux/8.8/python-tag-abi-tag.py rename to AI/manylinux/8.9/python-tag-abi-tag.py diff --git a/AI/manylinux/buildspec.yml b/AI/manylinux/buildspec.yml index 738223f..7a23941 100644 --- a/AI/manylinux/buildspec.yml +++ b/AI/manylinux/buildspec.yml @@ -19,11 +19,11 @@ images: BuildPythonManylinuxDockerImage: build: true docker_file: - path: AI/manylinux/8.8/Dockerfile + path: AI/manylinux/8.9/Dockerfile variable: scene: args: [] - tags: [8.8] + tags: [8.9] registry: [*ACR_ANOLIS] test_config: - - [*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''] \ No newline at end of file + - [*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''] -- Gitee From 3766e216c492b929e6937102d3e603279b801457 Mon Sep 17 00:00:00 2001 From: "zhongling.h" Date: Wed, 22 May 2024 15:36:32 +0800 Subject: [PATCH 5/6] limit arch to x64 --- AI/manylinux/buildspec.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/AI/manylinux/buildspec.yml b/AI/manylinux/buildspec.yml index 7a23941..b970ae7 100644 --- a/AI/manylinux/buildspec.yml +++ b/AI/manylinux/buildspec.yml @@ -24,6 +24,7 @@ images: scene: args: [] tags: [8.9] + platform: [linux/amd64] registry: [*ACR_ANOLIS] test_config: - [*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''] -- Gitee From 172c1a7f98f16dc758225d6b358e4cba0b0720f6 Mon Sep 17 00:00:00 2001 From: "zhongling.h" Date: Tue, 28 May 2024 17:45:17 +0800 Subject: [PATCH 6/6] update --- AI/manylinux/8.9/Dockerfile | 78 +++++++++++++++++++++++++---- AI/manylinux/8.9/finalize-python.sh | 8 +++ 2 files changed, 75 insertions(+), 11 deletions(-) diff --git a/AI/manylinux/8.9/Dockerfile b/AI/manylinux/8.9/Dockerfile index 41b17d6..5f668f9 100644 --- a/AI/manylinux/8.9/Dockerfile +++ b/AI/manylinux/8.9/Dockerfile @@ -9,15 +9,72 @@ ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin: ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 -RUN sed -i 's#mirrors.openanolis.cn#mirrors.cloud.aliyuncs.com#g' /etc/yum.repos.d/* - -RUN yum install -y glibc-devel libstdc++-devel glib2-devel libX11-devel libXext-devel libXrender-devel mesa-libGL-devel \ - libICE-devel libSM-devel zlib-devel expat-devel zlib bzip2 expat ncurses readline gdbm libpcap xz openssl keyutils-libs \ - libkadm5 libcom_err libidn uuid libffi libdb tk git autoconf automake bison bzip2 diffutils \ - file make patch unzip curl glibc-locale-source glibc-langpack-en hardlink hostname libnsl libxcrypt which \ - gcc-toolset-12-binutils gcc-toolset-12-gcc gcc-toolset-12-gcc-c++ \ - openssl-devel rust cargo yasm autoconf automake libtool libxcrypt-devel git sqlite openssl-devel tcl-devel \ - gcc gcc-c++ cmake +RUN yum install -y \ + autoconf \ + automake \ + bison \ + bluez-libs-devel \ + bzip2 bzip2-devel \ + cargo \ + cmake \ + curl \ + desktop-file-utils \ + diffutils \ + expat expat-devel \ + file \ + findutils \ + gcc-toolset-12-binutils \ + gcc-toolset-12-gcc \ + gcc-toolset-12-gcc-c++ \ + gdbm \ + git \ + glib2-devel \ + glibc-devel \ + glibc-langpack-en \ + glibc-locale-source \ + gmp-devel \ + gnupg2 \ + hardlink \ + hostname \ + keyutils-libs \ + libappstream-glib \ + libcom_err \ + libdb \ + libffi libffi-devel \ + libICE-devel \ + libidn \ + libGL-devel \ + libkadm5 \ + libnsl2-devel \ + libpcap \ + libSM-devel \ + libstdc++-devel \ + libtirpc-devel \ + libtool \ + libuuid-devel \ + libX11-devel \ + libxcrypt-devel \ + libXext-devel \ + libXrender-devel \ + make \ + mesa-libGL-devel \ + ncurses ncurses-devel \ + ninja-build \ + openssl openssl-devel \ + patch \ + pkgconfig \ + readline readline-devel \ + rust \ + sqlite-devel \ + tcl tcl-devel \ + tk tk-devel \ + tzdata \ + unzip \ + uuid \ + which \ + xz-devel \ + yasm \ + zlib-devel RUN git clone https://gitee.com/src-anolis-os/python3.git /tmp/python3 --branch=a23.0 --depth=1 && \ pushd /tmp/python3 && tar -xaf Python-3.10.13.tar.xz && cd Python-3.10.13 && \ @@ -25,11 +82,10 @@ RUN git clone https://gitee.com/src-anolis-os/python3.git /tmp/python3 --branch= make -j && make install && rm -rf /tmp/python3 COPY python-tag-abi-tag.py finalize-python.sh /opt -RUN mkdir /opt/python && \ +RUN source /opt/rh/gcc-toolset-12/enable && mkdir /opt/python && \ for PREFIX in $(find /opt/_internal/ -mindepth 1 -maxdepth 1 \( -name 'cpython*' -o -name 'pypy*' \)); do \ /opt/finalize-python.sh $PREFIX ;\ done RUN ln -s /opt/_internal/cpython-3.10.13/bin/auditwheel /usr/local/bin/auditwheel && \ ln -s /opt/_internal/cpython-3.10.13/bin/patchelf /usr/local/bin/patchelf - diff --git a/AI/manylinux/8.9/finalize-python.sh b/AI/manylinux/8.9/finalize-python.sh index 683a2a7..bd8b285 100755 --- a/AI/manylinux/8.9/finalize-python.sh +++ b/AI/manylinux/8.9/finalize-python.sh @@ -33,6 +33,14 @@ if [ -e ${PREFIX}/bin/pip3 ] && [ ! -e ${PREFIX}/bin/pip ]; then ln -s pip3 ${PREFIX}/bin/pip fi +cat << ! > /etc/pip.conf +[global] +index-url = https://mirrors.aliyun.com/pypi/simple/ + +[install] +trusted-host=mirrors.aliyun.com +! + ABI_TAG=$(${PREFIX}/bin/python3 /opt/python-tag-abi-tag.py) ln -s ${PREFIX} /opt/python/${ABI_TAG} -- Gitee