From 00199b90d88dd935e26d45b66702330cc8dd073c Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 22 Feb 2024 18:09:52 +0800 Subject: [PATCH 01/18] update --- APPLICATION/spark/buildspec.yml | 6 +++++- APPLICATION/spark/general/Dockerfile | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml index 2a5d879..f060790 100644 --- a/APPLICATION/spark/buildspec.yml +++ b/APPLICATION/spark/buildspec.yml @@ -26,6 +26,7 @@ images: # - &SPARK33_JAVA11_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.6")] # - &SPARK33_JAVA8_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.8")] # - &SPARK33_JAVA11_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.8")] + - &SPARK31_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.1.2"), (OS_SUFFIX, "3")] - &SPARK33_JAVA8_A3 [(BASE_OS, alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "3")] - &SPARK33_JAVA11_A3 [(BASE_OS, alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "3")] - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23")] @@ -34,6 +35,7 @@ images: args: [ # *SPARK33_JAVA8_A86, *SPARK33_JAVA11_A86, # *SPARK33_JAVA8_A88, *SPARK33_JAVA11_A88, + *SPARK31_JAVA11_A3, *SPARK33_JAVA8_A3, *SPARK33_JAVA11_A3, *SPARK33_JAVA8_A23, *SPARK33_JAVA11_A23] tags: @@ -41,11 +43,12 @@ images: # - ["3.3.3-java11-8.6", "3.3-java11-8.6", "3-java11-8.6", "3.3.3-8.6", "3.3-8.6", "3-8.6"] # - ["3.3.3-java8-8.8", "3.3-java8-8.8", "3-java8-8.8"] # - ["3.3.3-java11-8.8", "3.3-java11-8.8", "3-java11-8.8", "3.3.3-8.8", "3.3-8.8", "3-8.8"] + - ["3.1.2-java11-3", "3.1-java11-3"] - ["3.3.3-java8-3", "3.3-java8-3", "3-java8-3"] - ["3.3.3-java11-3", "3.3-java11-3", "3-java11-3", "3.3.3-3", "3.3-3", "3-3"] - ["3.3.3-java8-23", "3.3-java8-23", "3-java8-23", "3.3.3-java8", "3.3-java8", "3-java8"] - ["3.3.3-java11-23", "3.3-java11-23", "3-java11-23", "3.3.3-23", "3.3-23", "3-23", "3.3.3", "3.3", "3", "latest"] - registry: [*ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS] + registry: [*ACR_ALINUX, *ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS] test_config: # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] @@ -55,3 +58,4 @@ images: - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] + - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index b5152b0..0d58427 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -26,6 +26,8 @@ RUN set -eux; \ FROM ${BASE_OS} +CMD ["bash"] + ARG SPARK_VERSION ARG OS_SUFFIX ARG JAVA_VERSION @@ -48,7 +50,7 @@ RUN set -eux; \ yum install -y java-11-alibaba-dragonwell \ ;; \ * ) \ - echo unsupported java version: ${JAVA_VERSION}; exit 1 \ + echo skip install java \ ;; \ esac; \ # (java -XshowSettings:properties -version 2>&1 > /dev/null | grep 'java.home' | awk '{print $3}' | xargs -I{} ln -s {} ${JAVA_HOME}); \ -- Gitee From db2bfa8cdda36a9aaf15399362d10199a22fc7c1 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 22 Feb 2024 19:49:23 +0800 Subject: [PATCH 02/18] update --- APPLICATION/spark/buildspec.yml | 4 ++-- APPLICATION/spark/general/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml index f060790..db21eb5 100644 --- a/APPLICATION/spark/buildspec.yml +++ b/APPLICATION/spark/buildspec.yml @@ -27,8 +27,8 @@ images: # - &SPARK33_JAVA8_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.8")] # - &SPARK33_JAVA11_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.8")] - &SPARK31_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.1.2"), (OS_SUFFIX, "3")] - - &SPARK33_JAVA8_A3 [(BASE_OS, alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "3")] - - &SPARK33_JAVA11_A3 [(BASE_OS, alibaba-cloud-linux-3-registry.cn-hangzhou.cr.aliyuncs.com/alinux3/alinux3), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "3")] + - &SPARK33_JAVA8_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:8-alinux), (SPARK_VERSION, "3.3.3"), (OS_SUFFIX, "3")] + - &SPARK33_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.3"), (OS_SUFFIX, "3")] - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23")] - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23")] scene: diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index 0d58427..1381aae 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -8,7 +8,7 @@ ARG SPARK_FILE=spark-${SPARK_VERSION}-bin-hadoop3 RUN yum install -y tar gzip -RUN curl -O -m 600 https://mirrors.ustc.edu.cn/apache/spark/spark-${SPARK_VERSION}/${SPARK_FILE}.tgz +RUN curl -O -m 600 https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/${SPARK_FILE}.tgz RUN gzip -d ${SPARK_FILE}.tgz RUN tar xf ${SPARK_FILE}.tar RUN mv ${SPARK_FILE} spark -- Gitee From be0e4c80786646d00fe15a5f83e5753f14a43c9f Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 22 Feb 2024 19:50:16 +0800 Subject: [PATCH 03/18] fix --- APPLICATION/spark/general/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index 1381aae..5f37880 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -30,7 +30,7 @@ CMD ["bash"] ARG SPARK_VERSION ARG OS_SUFFIX -ARG JAVA_VERSION +ARG JAVA_VERSION= ARG spark_uid=185 LABEL org.opencontainers.image.title="Spark" \ -- Gitee From 3bff2cedd7892a79af5e27b24d331954e8ebfa14 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 22 Feb 2024 19:51:21 +0800 Subject: [PATCH 04/18] fix version --- APPLICATION/spark/buildspec.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml index db21eb5..484e223 100644 --- a/APPLICATION/spark/buildspec.yml +++ b/APPLICATION/spark/buildspec.yml @@ -22,15 +22,15 @@ images: docker_file: path: APPLICATION/spark/general/Dockerfile variable: - # - &SPARK33_JAVA8_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.6")] - # - &SPARK33_JAVA11_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.6")] - # - &SPARK33_JAVA8_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.8")] - # - &SPARK33_JAVA11_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.8")] + # - &SPARK33_JAVA8_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.6")] + # - &SPARK33_JAVA11_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.6")] + # - &SPARK33_JAVA8_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.8")] + # - &SPARK33_JAVA11_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.8")] - &SPARK31_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.1.2"), (OS_SUFFIX, "3")] - - &SPARK33_JAVA8_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:8-alinux), (SPARK_VERSION, "3.3.3"), (OS_SUFFIX, "3")] - - &SPARK33_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.3"), (OS_SUFFIX, "3")] - - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23")] - - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.3"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23")] + - &SPARK33_JAVA8_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:8-alinux), (SPARK_VERSION, "3.3.4"), (OS_SUFFIX, "3")] + - &SPARK33_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.4"), (OS_SUFFIX, "3")] + - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23")] + - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23")] scene: args: [ # *SPARK33_JAVA8_A86, *SPARK33_JAVA11_A86, @@ -39,15 +39,15 @@ images: *SPARK33_JAVA8_A3, *SPARK33_JAVA11_A3, *SPARK33_JAVA8_A23, *SPARK33_JAVA11_A23] tags: - # - ["3.3.3-java8-8.6", "3.3-java8-8.6", "3-java8-8.6"] - # - ["3.3.3-java11-8.6", "3.3-java11-8.6", "3-java11-8.6", "3.3.3-8.6", "3.3-8.6", "3-8.6"] - # - ["3.3.3-java8-8.8", "3.3-java8-8.8", "3-java8-8.8"] - # - ["3.3.3-java11-8.8", "3.3-java11-8.8", "3-java11-8.8", "3.3.3-8.8", "3.3-8.8", "3-8.8"] + # - ["3.3.4-java8-8.6", "3.3-java8-8.6", "3-java8-8.6"] + # - ["3.3.4-java11-8.6", "3.3-java11-8.6", "3-java11-8.6", "3.3.4-8.6", "3.3-8.6", "3-8.6"] + # - ["3.3.4-java8-8.8", "3.3-java8-8.8", "3-java8-8.8"] + # - ["3.3.4-java11-8.8", "3.3-java11-8.8", "3-java11-8.8", "3.3.4-8.8", "3.3-8.8", "3-8.8"] - ["3.1.2-java11-3", "3.1-java11-3"] - - ["3.3.3-java8-3", "3.3-java8-3", "3-java8-3"] - - ["3.3.3-java11-3", "3.3-java11-3", "3-java11-3", "3.3.3-3", "3.3-3", "3-3"] - - ["3.3.3-java8-23", "3.3-java8-23", "3-java8-23", "3.3.3-java8", "3.3-java8", "3-java8"] - - ["3.3.3-java11-23", "3.3-java11-23", "3-java11-23", "3.3.3-23", "3.3-23", "3-23", "3.3.3", "3.3", "3", "latest"] + - ["3.3.4-java8-3", "3.3-java8-3", "3-java8-3"] + - ["3.3.4-java11-3", "3.3-java11-3", "3-java11-3", "3.3.4-3", "3.3-3", "3-3"] + - ["3.3.4-java8-23", "3.3-java8-23", "3-java8-23", "3.3.4-java8", "3.3-java8", "3-java8"] + - ["3.3.4-java11-23", "3.3-java11-23", "3-java11-23", "3.3.4-23", "3.3-23", "3-23", "3.3.4", "3.3", "3", "latest"] registry: [*ACR_ALINUX, *ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS] test_config: # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] -- Gitee From ec327896cb1915d526dfc09babccd8e138c86690 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 22 Feb 2024 19:52:19 +0800 Subject: [PATCH 05/18] fix --- APPLICATION/spark/general/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index 5f37880..162db0f 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -11,7 +11,7 @@ RUN yum install -y tar gzip RUN curl -O -m 600 https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/${SPARK_FILE}.tgz RUN gzip -d ${SPARK_FILE}.tgz RUN tar xf ${SPARK_FILE}.tar -RUN mv ${SPARK_FILE} spark +RUN mv spark-* spark COPY tini-amd64 / COPY tini-arm64 / -- Gitee From 951a43f277d284f6852c88a45201d2718ad5d10b Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 22 Feb 2024 19:57:16 +0800 Subject: [PATCH 06/18] fix --- APPLICATION/spark/general/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index 162db0f..19eb742 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -11,6 +11,7 @@ RUN yum install -y tar gzip RUN curl -O -m 600 https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/${SPARK_FILE}.tgz RUN gzip -d ${SPARK_FILE}.tgz RUN tar xf ${SPARK_FILE}.tar +RUN rm ${SPARK_FILE}.tar RUN mv spark-* spark COPY tini-amd64 / -- Gitee From c7bfbcd0c49a8c7ca7ca088910cdcf06555a7fa4 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 22 Feb 2024 20:45:44 +0800 Subject: [PATCH 07/18] update --- APPLICATION/spark/buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml index 484e223..b96ce46 100644 --- a/APPLICATION/spark/buildspec.yml +++ b/APPLICATION/spark/buildspec.yml @@ -48,7 +48,7 @@ images: - ["3.3.4-java11-3", "3.3-java11-3", "3-java11-3", "3.3.4-3", "3.3-3", "3-3"] - ["3.3.4-java8-23", "3.3-java8-23", "3-java8-23", "3.3.4-java8", "3.3-java8", "3-java8"] - ["3.3.4-java11-23", "3.3-java11-23", "3-java11-23", "3.3.4-23", "3.3-23", "3-23", "3.3.4", "3.3", "3", "latest"] - registry: [*ACR_ALINUX, *ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS] + registry: [*ACR_ANOLIS, *ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS] test_config: # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] -- Gitee From 0f93aa924291d595b11270f2f657e4b44e99ce55 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 22 Feb 2024 20:46:04 +0800 Subject: [PATCH 08/18] update --- APPLICATION/spark/buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml index b96ce46..484e223 100644 --- a/APPLICATION/spark/buildspec.yml +++ b/APPLICATION/spark/buildspec.yml @@ -48,7 +48,7 @@ images: - ["3.3.4-java11-3", "3.3-java11-3", "3-java11-3", "3.3.4-3", "3.3-3", "3-3"] - ["3.3.4-java8-23", "3.3-java8-23", "3-java8-23", "3.3.4-java8", "3.3-java8", "3-java8"] - ["3.3.4-java11-23", "3.3-java11-23", "3-java11-23", "3.3.4-23", "3.3-23", "3-23", "3.3.4", "3.3", "3", "latest"] - registry: [*ACR_ANOLIS, *ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS] + registry: [*ACR_ALINUX, *ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS] test_config: # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] -- Gitee From e140a2fcef53de421622ba228f5fc98001b14951 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Thu, 22 Feb 2024 20:50:56 +0800 Subject: [PATCH 09/18] add zstd --- APPLICATION/spark/general/Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index 19eb742..8914c99 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -14,6 +14,13 @@ RUN tar xf ${SPARK_FILE}.tar RUN rm ${SPARK_FILE}.tar RUN mv spark-* spark +RUN set -eux; \ + ( \ + pushd /spark/jars; \ + rm -rf zstd-jni*.jar; \ + curl -O -m 644 https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/zstd-jni-1.5.2-ali-ali.jar; \ + ) + COPY tini-amd64 / COPY tini-arm64 / -- Gitee From 0094a803dceb18d9e42e5225792f2ffb13c0e3f3 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 27 Feb 2024 11:01:51 +0800 Subject: [PATCH 10/18] Add mracc image --- APPLICATION/spark/buildspec.yml | 9 +++++++-- APPLICATION/spark/general/Dockerfile | 9 +++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml index 484e223..2b067e0 100644 --- a/APPLICATION/spark/buildspec.yml +++ b/APPLICATION/spark/buildspec.yml @@ -31,13 +31,16 @@ images: - &SPARK33_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.4"), (OS_SUFFIX, "3")] - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23")] - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23")] + - &SPARK33_MRACC_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.1"), (OS_SUFFIX, "3"), + (SPARK_URL, "https://fastmr.oss-cn-shenzhen.aliyuncs.com/bigdata/arm/mracc-spark-3.3.tar.gz")] scene: args: [ # *SPARK33_JAVA8_A86, *SPARK33_JAVA11_A86, # *SPARK33_JAVA8_A88, *SPARK33_JAVA11_A88, *SPARK31_JAVA11_A3, *SPARK33_JAVA8_A3, *SPARK33_JAVA11_A3, - *SPARK33_JAVA8_A23, *SPARK33_JAVA11_A23] + *SPARK33_JAVA8_A23, *SPARK33_JAVA11_A23, + *SPARK33_MRACC_JAVA11_A3] tags: # - ["3.3.4-java8-8.6", "3.3-java8-8.6", "3-java8-8.6"] # - ["3.3.4-java11-8.6", "3.3-java11-8.6", "3-java11-8.6", "3.3.4-8.6", "3.3-8.6", "3-8.6"] @@ -48,7 +51,8 @@ images: - ["3.3.4-java11-3", "3.3-java11-3", "3-java11-3", "3.3.4-3", "3.3-3", "3-3"] - ["3.3.4-java8-23", "3.3-java8-23", "3-java8-23", "3.3.4-java8", "3.3-java8", "3-java8"] - ["3.3.4-java11-23", "3.3-java11-23", "3-java11-23", "3.3.4-23", "3.3-23", "3-23", "3.3.4", "3.3", "3", "latest"] - registry: [*ACR_ALINUX, *ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS] + - ["3.3.1-mracc-java11-3", "3.3-mracc-java11-3", "3-mracc-java11", "3.3.1-mracc", "3.3-mracc", "3-mracc", "mracc"] + registry: [*ACR_ALINUX, *ACR_ALINUX, *ACR_ALINUX, *ACR_ANOLIS, *ACR_ANOLIS, *ACR_ALINUX] test_config: # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] # - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] @@ -59,3 +63,4 @@ images: - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] + - [(*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[0], ''), (*WORKSPACE, *PROJECT, *TEST_SUITE, *TEST_CONF, *TEST_CASE, *CLOUD_SERVER_TAG[1], '')] diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index 8914c99..9591821 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -5,13 +5,14 @@ FROM registry.openanolis.cn/openanolis/anolisos:23 as downloader ARG SPARK_VERSION ARG SPARK_FILE=spark-${SPARK_VERSION}-bin-hadoop3 +ARG SPARK_URL=https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/${SPARK_FILE}.tgz RUN yum install -y tar gzip -RUN curl -O -m 600 https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/${SPARK_FILE}.tgz -RUN gzip -d ${SPARK_FILE}.tgz -RUN tar xf ${SPARK_FILE}.tar -RUN rm ${SPARK_FILE}.tar +RUN curl -O -m 600 ${SPARK_URL} +RUN gzip -d spark-*.tgz +RUN tar xf spark-*.tar +RUN rm spark-*.tar RUN mv spark-* spark RUN set -eux; \ -- Gitee From 526475a97c3a20962ebaa6c94c784ac91bc32e21 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 27 Feb 2024 11:03:53 +0800 Subject: [PATCH 11/18] fix format --- APPLICATION/spark/buildspec.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml index 2b067e0..e6d8ddc 100644 --- a/APPLICATION/spark/buildspec.yml +++ b/APPLICATION/spark/buildspec.yml @@ -31,8 +31,7 @@ images: - &SPARK33_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.4"), (OS_SUFFIX, "3")] - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23")] - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23")] - - &SPARK33_MRACC_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.1"), (OS_SUFFIX, "3"), - (SPARK_URL, "https://fastmr.oss-cn-shenzhen.aliyuncs.com/bigdata/arm/mracc-spark-3.3.tar.gz")] + - &SPARK33_MRACC_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.1"), (OS_SUFFIX, "3"), (SPARK_URL, "https://fastmr.oss-cn-shenzhen.aliyuncs.com/bigdata/arm/mracc-spark-3.3.tar.gz")] scene: args: [ # *SPARK33_JAVA8_A86, *SPARK33_JAVA11_A86, -- Gitee From efca3f57f17a071e30ad75bb47b66f8bcc0a614b Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 27 Feb 2024 11:13:18 +0800 Subject: [PATCH 12/18] fix --- APPLICATION/spark/general/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index 9591821..b78345d 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -1,5 +1,6 @@ ARG SPARK_VERSION ARG BASE_OS +ARG SPARK_URL= FROM registry.openanolis.cn/openanolis/anolisos:23 as downloader ARG SPARK_VERSION -- Gitee From 602eb6b8474ea7cf9e3d1287c69e0d4eac4026be Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 27 Feb 2024 11:16:17 +0800 Subject: [PATCH 13/18] update --- APPLICATION/spark/general/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index b78345d..9591821 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -1,6 +1,5 @@ ARG SPARK_VERSION ARG BASE_OS -ARG SPARK_URL= FROM registry.openanolis.cn/openanolis/anolisos:23 as downloader ARG SPARK_VERSION -- Gitee From dfed8b12690baac7ca37041e64af7e0a7d159609 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 27 Feb 2024 11:44:31 +0800 Subject: [PATCH 14/18] workaround --- APPLICATION/spark/buildspec.yml | 10 +++++----- APPLICATION/spark/general/Dockerfile | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml index e6d8ddc..685f909 100644 --- a/APPLICATION/spark/buildspec.yml +++ b/APPLICATION/spark/buildspec.yml @@ -26,11 +26,11 @@ images: # - &SPARK33_JAVA11_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.6")] # - &SPARK33_JAVA8_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.8")] # - &SPARK33_JAVA11_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.8")] - - &SPARK31_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.1.2"), (OS_SUFFIX, "3")] - - &SPARK33_JAVA8_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:8-alinux), (SPARK_VERSION, "3.3.4"), (OS_SUFFIX, "3")] - - &SPARK33_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.4"), (OS_SUFFIX, "3")] - - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23")] - - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23")] + - &SPARK31_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.1.2"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.1.2-bin-hadoop3.tgz")] + - &SPARK33_JAVA8_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:8-alinux), (SPARK_VERSION, "3.3.4"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] + - &SPARK33_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.4"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] + - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] + - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] - &SPARK33_MRACC_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.1"), (OS_SUFFIX, "3"), (SPARK_URL, "https://fastmr.oss-cn-shenzhen.aliyuncs.com/bigdata/arm/mracc-spark-3.3.tar.gz")] scene: args: [ diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index 9591821..55f3599 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -5,7 +5,8 @@ FROM registry.openanolis.cn/openanolis/anolisos:23 as downloader ARG SPARK_VERSION ARG SPARK_FILE=spark-${SPARK_VERSION}-bin-hadoop3 -ARG SPARK_URL=https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/${SPARK_FILE}.tgz +# ARG SPARK_URL=https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/${SPARK_FILE}.tgz +ARG SPARK_URL RUN yum install -y tar gzip -- Gitee From fb7984bf99d0e5d49e7688b3d940fdbc4e7683ce Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 27 Feb 2024 11:53:16 +0800 Subject: [PATCH 15/18] fix --- APPLICATION/spark/general/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index 55f3599..24c5569 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -10,11 +10,11 @@ ARG SPARK_URL RUN yum install -y tar gzip -RUN curl -O -m 600 ${SPARK_URL} -RUN gzip -d spark-*.tgz -RUN tar xf spark-*.tar -RUN rm spark-*.tar -RUN mv spark-* spark +RUN curl -o spark.tgz -m 600 ${SPARK_URL} +RUN gzip -d spark.tgz +RUN tar xf spark.tar +RUN rm spark.tar +RUN mv *spark-* spark RUN set -eux; \ ( \ -- Gitee From 74315c8944e73b1d4be24f58b82b06f8e6492dec Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 27 Feb 2024 16:31:57 +0800 Subject: [PATCH 16/18] update --- APPLICATION/spark/buildspec.yml | 6 +++--- APPLICATION/spark/general/Dockerfile | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml index 685f909..5d291d4 100644 --- a/APPLICATION/spark/buildspec.yml +++ b/APPLICATION/spark/buildspec.yml @@ -26,9 +26,9 @@ images: # - &SPARK33_JAVA11_A86 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.6), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.6")] # - &SPARK33_JAVA8_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "8.8")] # - &SPARK33_JAVA11_A88 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:8.8), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "8.8")] - - &SPARK31_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.1.2"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.1.2-bin-hadoop3.tgz")] - - &SPARK33_JAVA8_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:8-alinux), (SPARK_VERSION, "3.3.4"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] - - &SPARK33_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.4"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] + - &SPARK31_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.1.2"), (JAVA_VERSION, "0"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.1.2-bin-hadoop3.tgz")] + - &SPARK33_JAVA8_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:8-alinux), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "0"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] + - &SPARK33_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "0"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] - &SPARK33_MRACC_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.1"), (OS_SUFFIX, "3"), (SPARK_URL, "https://fastmr.oss-cn-shenzhen.aliyuncs.com/bigdata/arm/mracc-spark-3.3.tar.gz")] diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index 24c5569..c17b8c8 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -40,7 +40,7 @@ CMD ["bash"] ARG SPARK_VERSION ARG OS_SUFFIX -ARG JAVA_VERSION= +ARG JAVA_VERSION ARG spark_uid=185 LABEL org.opencontainers.image.title="Spark" \ -- Gitee From 4b58f00658c09b6dc3b413ae798c36bba0cb3949 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 27 Feb 2024 16:33:47 +0800 Subject: [PATCH 17/18] update --- APPLICATION/spark/buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/APPLICATION/spark/buildspec.yml b/APPLICATION/spark/buildspec.yml index 5d291d4..056a922 100644 --- a/APPLICATION/spark/buildspec.yml +++ b/APPLICATION/spark/buildspec.yml @@ -31,7 +31,7 @@ images: - &SPARK33_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "0"), (OS_SUFFIX, "3"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] - &SPARK33_JAVA8_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "8"), (OS_SUFFIX, "23"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] - &SPARK33_JAVA11_A23 [(BASE_OS, registry.openanolis.cn/openanolis/anolisos:23), (SPARK_VERSION, "3.3.4"), (JAVA_VERSION, "11"), (OS_SUFFIX, "23"), (SPARK_URL, "https://cr-images-pub.oss-cn-hangzhou.aliyuncs.com/root/big-data/spark-3.3.4-bin-hadoop3.tgz")] - - &SPARK33_MRACC_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.1"), (OS_SUFFIX, "3"), (SPARK_URL, "https://fastmr.oss-cn-shenzhen.aliyuncs.com/bigdata/arm/mracc-spark-3.3.tar.gz")] + - &SPARK33_MRACC_JAVA11_A3 [(BASE_OS, anolis-registry.cn-zhangjiakou.cr.aliyuncs.com/openanolis/dragonwell:11-alinux), (SPARK_VERSION, "3.3.1"), (JAVA_VERSION, "0"), (OS_SUFFIX, "3"), (SPARK_URL, "https://fastmr.oss-cn-shenzhen.aliyuncs.com/bigdata/arm/mracc-spark-3.3.tar.gz")] scene: args: [ # *SPARK33_JAVA8_A86, *SPARK33_JAVA11_A86, -- Gitee From 5001c22ca5755e3a7dfe8f3aeb96f384247048af Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 26 Mar 2024 11:18:37 +0800 Subject: [PATCH 18/18] fix --- APPLICATION/spark/general/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/APPLICATION/spark/general/Dockerfile b/APPLICATION/spark/general/Dockerfile index c17b8c8..5ab7f17 100644 --- a/APPLICATION/spark/general/Dockerfile +++ b/APPLICATION/spark/general/Dockerfile @@ -47,11 +47,10 @@ LABEL org.opencontainers.image.title="Spark" \ org.opencontainers.image.version="${SPARK_VERSION}-${OS_SUFFIX}" ENV SPARK_VERSION ${SPARK_VERSION} -ENV JAVA_HOME /etc/alternatives/jre ENV PATH $JAVA_HOME/bin:$PATH RUN set -eux; \ - yum install -y net-tools iputils hostname; \ + yum install -y net-tools iputils hostname procps; \ case "${JAVA_VERSION}" in \ "8" ) \ yum install -y java-1.8.0-alibaba-dragonwell \ -- Gitee