diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml index 2a5d87940f125ebcc7b26fc00397f473f13bcb7a..056a92289dcb8caa8129d32fee142cb77c5d9e25 100644 --- a/APPLICATION/spark/buildspec.yml +++ b/APPLICATION/spark/buildspec.yml @@ -22,30 +22,36 @@ images: docker_file: path: APPLICATION/spark/general/Dockerfile variable: - # - &SPARK33_JAVA8_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.6")] - # - &SPARK33_JAVA11_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.6")] - # - &SPARK33_JAVA8_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.8")] - # - &SPARK33_JAVA11_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.8")] - - &SPARK33_JAVA8_A3 [(BASE_OS, alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "3")] - - &SPARK33_JAVA11_A3 [(BASE_OS, alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "3")] - - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23")] - - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23")] + # - &SPARK33_JAVA8_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.6")] + # - &SPARK33_JAVA11_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.6")] + # - &SPARK33_JAVA8_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.8")] + # - &SPARK33_JAVA11_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.8")] + - &SPARK31_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.1.2"), (JAVA_VERSION, "0"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.1.2-bin-hadoop3.tgz")] + - &SPARK33_JAVA8_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:8-alinux), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "0"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] + - &SPARK33_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "0"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] + - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] + - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] + - &SPARK33_MRACC_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.1"), (JAVA_VERSION, "0"), (OS_SUFFIX, "3"), (SPARK_URL, "https://fastmr.oss-cn-shenzhen.aliyuncs.com/bigdata/arm/mracc-spark-3.3.tar.gz")] scene: args: [ # *SPARK33_JAVA8_A86, *SPARK33_JAVA11_A86, # *SPARK33_JAVA8_A88, *SPARK33_JAVA11_A88, + *SPARK31_JAVA11_A3, *SPARK33_JAVA8_A3, *SPARK33_JAVA11_A3, - *SPARK33_JAVA8_A23, *SPARK33_JAVA11_A23] + *SPARK33_JAVA8_A23, *SPARK33_JAVA11_A23, + *SPARK33_MRACC_JAVA11_A3] tags: - # - ["3.3.3-java8-8.6", "3.3-java8-8.6", "3-java8-8.6"] - # - ["3.3.3-java11-8.6", "3.3-java11-8.6", "3-java11-8.6", "3.3.3-8.6", "3.3-8.6", "3-8.6"] - # - ["3.3.3-java8-8.8", "3.3-java8-8.8", "3-java8-8.8"] - # - ["3.3.3-java11-8.8", "3.3-java11-8.8", "3-java11-8.8", "3.3.3-8.8", "3.3-8.8", "3-8.8"] - - ["3.3.3-java8-3", "3.3-java8-3", "3-java8-3"] - - ["3.3.3-java11-3", "3.3-java11-3", "3-java11-3", "3.3.3-3", "3.3-3", "3-3"] - - ["3.3.3-java8-23", "3.3-java8-23", "3-java8-23", "3.3.3-java8", "3.3-java8", "3-java8"] - - ["3.3.3-java11-23", "3.3-java11-23", "3-java11-23", "3.3.3-23", "3.3-23", "3-23", "3.3.3", "3.3", "3", "latest"] - registry: [*ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS] + # - ["3.3.4-java8-8.6", "3.3-java8-8.6", "3-java8-8.6"] + # - ["3.3.4-java11-8.6", "3.3-java11-8.6", "3-java11-8.6", "3.3.4-8.6", "3.3-8.6", "3-8.6"] + # - ["3.3.4-java8-8.8", "3.3-java8-8.8", "3-java8-8.8"] + # - ["3.3.4-java11-8.8", "3.3-java11-8.8", "3-java11-8.8", "3.3.4-8.8", "3.3-8.8", "3-8.8"] + - ["3.1.2-java11-3", "3.1-java11-3"] + - ["3.3.4-java8-3", "3.3-java8-3", "3-java8-3"] + - ["3.3.4-java11-3", "3.3-java11-3", "3-java11-3", "3.3.4-3", "3.3-3", "3-3"] + - ["3.3.4-java8-23", "3.3-java8-23", "3-java8-23", "3.3.4-java8", "3.3-java8", "3-java8"] + - ["3.3.4-java11-23", "3.3-java11-23", "3-java11-23", "3.3.4-23", "3.3-23", "3-23", "3.3.4", "3.3", "3", "latest"] + - ["3.3.1-mracc-java11-3", "3.3-mracc-java11-3", "3-mracc-java11", "3.3.1-mracc", "3.3-mracc", "3-mracc", "mracc"] + registry: [*ACR_ALINUX, *ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS, *ACR_ALINUX] test_config: # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] @@ -55,3 +61,5 @@ images: - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] + - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] + - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index b5152b080f1a46f4559379404d6c44fa6f1022bd..5ab7f17b1f0974ef50233904bfa724f13f44d26f 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -5,13 +5,23 @@ FROM registry.openanolis.cn/openanolis/anolisos:23 as downloader ARG SPARK_VERSION ARG SPARK_FILE=spark-${SPARK_VERSION}-bin-hadoop3 +# ARG SPARK_URL=https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/${SPARK_FILE}.tgz +ARG SPARK_URL RUN yum install -y tar gzip -RUN curl -O -m 600 https://mirrors.ustc.edu.cn/apache/spark/spark-${SPARK_VERSION}/${SPARK_FILE}.tgz -RUN gzip -d ${SPARK_FILE}.tgz -RUN tar xf ${SPARK_FILE}.tar -RUN mv ${SPARK_FILE} spark +RUN curl -o spark.tgz -m 600 ${SPARK_URL} +RUN gzip -d spark.tgz +RUN tar xf spark.tar +RUN rm spark.tar +RUN mv *spark-* spark + +RUN set -eux; \ + ( \ + pushd /spark/jars; \ + rm -rf zstd-jni*.jar; \ + curl -O -m 644 https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/zstd-jni-1.5.2-ali-ali.jar; \ + ) COPY tini-amd64 / COPY tini-arm64 / @@ -26,6 +36,8 @@ RUN set -eux; \ FROM ${BASE_OS} +CMD ["bash"] + ARG SPARK_VERSION ARG OS_SUFFIX ARG JAVA_VERSION @@ -35,11 +47,10 @@ LABEL org.opencontainers.image.title="Spark" \ org.opencontainers.image.version="${SPARK_VERSION}-${OS_SUFFIX}" ENV SPARK_VERSION ${SPARK_VERSION} -ENV JAVA_HOME /etc/alternatives/jre ENV PATH $JAVA_HOME/bin:$PATH RUN set -eux; \ - yum install -y net-tools iputils hostname; \ + yum install -y net-tools iputils hostname procps; \ case "${JAVA_VERSION}" in \ "8" ) \ yum install -y java-1.8.0-alibaba-dragonwell \ @@ -48,7 +59,7 @@ RUN set -eux; \ yum install -y java-11-alibaba-dragonwell \ ;; \ * ) \ - echo unsupported java version: ${JAVA_VERSION}; exit 1 \ + echo skip install java \ ;; \ esac; \ # (java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}' | xargs -I{} ln -s {} ${JAVA_HOME}); \