diff --git a/build/scripts/help.info b/build/scripts/help.info index b99ec057ae45806378aedef2683ef606c29d0331..213989f61ccb579960c7003db6b72070273121da 100644 --- a/build/scripts/help.info +++ b/build/scripts/help.info @@ -7,4 +7,11 @@ (eg: --install-type=A200IA2, when your product is A200I A2 or A200I DK A2) --ce= Only iSula need to specify the container engine(eg: --ce=isula) MUST use with --install or --uninstall - --version Query Ascend-docker-runtime version \ No newline at end of file + --version Query Ascend-docker-runtime version + --install-scene= Installation scenario, only docker or containerd + (eg: --install-scene=docker, default: docker) + --config-file-path Specifies the path of the Docker or containerd configuration file + (eg: --config-file-path=/etc/containerd/config.toml). + If this parameter is not specified, the default configuration file path + of docker or containerd is used. For docker, the path is /etc/docker/daemon.json. + For containerd, the path is /etc/containerd/config.toml. \ No newline at end of file diff --git a/build/scripts/run_main.sh b/build/scripts/run_main.sh index cc9f80b0858697a83e49b6ca49749dbe4916878f..216129dd235e670685c6f3819e666e6d31e9de3f 100644 --- a/build/scripts/run_main.sh +++ b/build/scripts/run_main.sh @@ -22,6 +22,9 @@ start_script=${start_arg#*--} ASCEND_RUNTIME_CONFIG_DIR=/etc/ascend-docker-runtime.d DOCKER_CONFIG_DIR=/etc/docker +CONTAINERD_CONFIG_DIR=/etc/containerd +CONFIG_FILE_PATH="" +INSTALL_SCENE=docker INSTALL_PATH=/usr/local/Ascend/Ascend-Docker-Runtime readonly INSTALL_LOG_DIR=/var/log/ascend-docker-runtime readonly INSTALL_LOG_PATH=${INSTALL_LOG_DIR}/installer.log @@ -38,7 +41,7 @@ function check_log { check_sub_path ${INSTALL_LOG_DIR} if [[ $? != 0 ]]; then - echo "[ERROR]: ${INSTALL_LOG_DIR} is invalid" + echo "[ERROR] ${INSTALL_LOG_DIR} is invalid" exit 1 fi @@ -69,11 +72,11 @@ function log { function check_path { local path="$1" if [[ ${#path} -gt 1024 ]] || [[ ${#path} -le 0 ]]; then - echo "[ERROR]: parameter is invalid, length not in 1~1024" + echo "[ERROR] parameter is invalid, length not in 1~1024" return 1 fi if [[ -n $(echo "${path}" | grep -Ev '^[a-zA-Z0-9./_-]*$') ]]; then - echo "[ERROR]: parameter is invalid, char not all in 'a-zA-Z0-9./_-'" + echo "[ERROR] parameter is invalid, char not all in 'a-zA-Z0-9./_-'" return 1 fi path=$(realpath -m -s "${path}") @@ -108,16 +111,16 @@ function check_sub_path { function check_path_permission { local path="$1" if [[ -L "${path}" ]]; then - echo "[ERROR]: ${path} is soft link" + echo "[ERROR] ${path} is soft link" return 1 fi if [[ $(stat -c %u "${path}") != 0 ]] || [[ "$(stat -c %g ${path})" != 0 ]]; then - echo "[ERROR]: user or group of ${path} is not root" + echo "[ERROR] user or group of ${path} is not root" return 1 fi local permission=$(stat -c %A "${path}") if [[ $(echo "${permission}" | cut -c6) == w ]] || [[ $(echo "${permission}" | cut -c9) == w ]]; then - echo "[ERROR]: group or other of ${path} has write permisson" + echo "[ERROR] group or other of ${path} has write permisson" return 1 fi } @@ -145,16 +148,22 @@ Options: --ce= Only iSula need to specify the container engine(eg: --ce=isula) MUST use with --install or --uninstall --version Query Ascend-docker-runtime version + --install-scene= Installation scenario, only docker or containerd(eg: --install-scene=docker, default: docker) + --config-file-path Specifies the path of the Docker or containerd configuration file + (eg: --config-file-path=/etc/containerd/config.toml). + If this parameter is not specified, the default configuration file path + of docker or containerd is used. For docker, the path is /etc/docker/daemon.json. + For containerd, the path is /etc/containerd/config.toml. " } function check_platform { plat="$(uname -m)" if [[ $start_script =~ $plat ]]; then - echo "[INFO]: platform($plat) matched!" + echo "[INFO] platform($plat) matched!" return 0 else - echo "[ERROR]: platform($plat) mismatch for $start_script, please check it" + echo "[ERROR] platform($plat) mismatch for $start_script, please check it" return 1 fi } @@ -171,6 +180,8 @@ function save_install_args() { echo -e "a200=${a200}" echo -e "a200isoc=${a200isoc}" echo -e "a200ia2=${a200ia2}" + echo -e "install-scene=${INSTALL_SCENE}" + echo -e "config-file-path=${CONFIG_FILE_PATH}" } > "${INSTALL_PATH}"/ascend_docker_runtime_install.info chmod 640 ${INSTALL_PATH}/ascend_docker_runtime_install.info } @@ -178,26 +189,26 @@ function save_install_args() { function add_so() { check_path "/etc/os-release" if [[ $? != 0 ]]; then - echo "[ERROR]: /etc/os-release is invalid" + echo "[ERROR] /etc/os-release is invalid" return 1 fi if grep -qi "ubuntu" "/etc/os-release"; then - echo "[info]: os is Ubuntu" + echo "[info] os is Ubuntu" echo -e "\n/usr/lib/aarch64-linux-gnu/libcrypto.so.1.1" >> ${ASCEND_RUNTIME_CONFIG_DIR}/base.list echo "/usr/lib/aarch64-linux-gnu/libyaml-0.so.2" >> ${ASCEND_RUNTIME_CONFIG_DIR}/base.list elif grep -qi "euler" "/etc/os-release"; then - echo "[info]: os is Euler/OpenEuler" + echo "[info] os is Euler/OpenEuler" echo -e "\n/usr/lib64/libcrypto.so.1.1" >> ${ASCEND_RUNTIME_CONFIG_DIR}/base.list echo "/usr/lib64/libyaml-0.so.2" >> ${ASCEND_RUNTIME_CONFIG_DIR}/base.list else - echo "[ERROR]: not support this os" + echo "[ERROR] not support this os" return 1 fi } function install() { - echo "[INFO]: installing ascend docker runtime" + echo "[INFO] installing ascend docker runtime" check_platform if [[ $? != 0 ]]; then log "[ERROR]" "install failed, run package and os not matched in arch" @@ -271,38 +282,66 @@ function install() fi chmod 440 ${ASCEND_RUNTIME_CONFIG_DIR}/base.list - echo "[INFO]: install executable files success" - - check_path ${DOCKER_CONFIG_DIR}/daemon.json - if [[ $? != 0 ]]; then - log "[ERROR]" "install failed, ${DOCKER_CONFIG_DIR}/daemon.json is invalid" - exit 1 + echo "[INFO] install executable files success" + + if [[ ${CONFIG_FILE_PATH} == "" ]]; then + if [[ "${INSTALL_SCENE}" == "docker" ]]; then + echo "[INFO] install scene is 'docker'." + check_path ${DOCKER_CONFIG_DIR}/daemon.json + if [[ $? != 0 ]]; then + log "[ERROR]" "install failed, ${DOCKER_CONFIG_DIR}/daemon.json is invalid" + exit 1 + fi + [[ ! -d ${DOCKER_CONFIG_DIR} ]] && mkdir -p -m 750 ${DOCKER_CONFIG_DIR} + + SRC="${DOCKER_CONFIG_DIR}/daemon.json.${PPID}" + DST="${DOCKER_CONFIG_DIR}/daemon.json" + elif [[ "${INSTALL_SCENE}" == "containerd" ]]; then + echo "[INFO] install scene is 'containerd'." + check_path ${CONTAINERD_CONFIG_DIR}/config.toml + if [[ $? != 0 ]]; then + log "[ERROR]" "install failed, ${CONTAINERD_CONFIG_DIR}/config.toml is invalid" + exit 1 + fi + [[ ! -d ${CONTAINERD_CONFIG_DIR} ]] && mkdir -p -m 750 ${CONTAINERD_CONFIG_DIR} + + SRC="${CONTAINERD_CONFIG_DIR}/config.toml.${PPID}" + DST="${CONTAINERD_CONFIG_DIR}/config.toml" + if [ ! -e ${DST} ]; then + echo "[INFO] containerd config file does not exist, default ${DST} will be created" + containerd config default > ${DST} + fi + else + log "[ERROR]" "install failed, invalid value '${INSTALL_SCENE}' of 'install-scene' " + exit 1 + fi + else + SRC="${CONFIG_FILE_PATH}.${PPID}" + DST="${CONFIG_FILE_PATH}" fi - [[ ! -d ${DOCKER_CONFIG_DIR} ]] && mkdir -p -m 750 ${DOCKER_CONFIG_DIR} - SRC="${DOCKER_CONFIG_DIR}/daemon.json.${PPID}" - DST="${DOCKER_CONFIG_DIR}/daemon.json" + + CGROUP_INFO=$(stat -fc %T /sys/fs/cgroup/) # exit when return code is not 0, if use 'set -e' - ./ascend-docker-plugin-install-helper add ${DST} ${SRC} ${INSTALL_PATH}/ascend-docker-runtime ${RESERVEDEFAULT} > /dev/null + ./ascend-docker-plugin-install-helper add ${DST} ${SRC} ${INSTALL_PATH}/ascend-docker-runtime ${RESERVEDEFAULT} ${INSTALL_SCENE} ${CGROUP_INFO} > /dev/null if [[ $? != 0 ]]; then - log "[ERROR]" "install failed, './ascend-docker-plugin-install-helper add ${DST} ${SRC} ${INSTALL_PATH}/ascend-docker-runtime ${RESERVEDEFAULT}' return non-zero" + log "[ERROR]" "install failed, './ascend-docker-plugin-install-helper add ${DST} ${SRC} ${INSTALL_PATH}/ascend-docker-runtime ${RESERVEDEFAULT} ${INSTALL_SCENE} ${CGROUP_INFO}' return non-zero" exit 1 fi - mv -f ${SRC} ${DST} log "[INFO]" "${DST} modify success" chmod 600 ${DST} save_install_args - echo "[INFO]: Ascend Docker Runtime has been installed in: ${INSTALL_PATH}" - echo "[INFO]: The version of Ascend Docker Runtime is: ${PACKAGE_VERSION}" - echo '[INFO]: please reboot daemon and container engine to take effect' + echo "[INFO] Ascend Docker Runtime has been installed in: ${INSTALL_PATH}" + echo "[INFO] the version of Ascend Docker Runtime is: ${PACKAGE_VERSION}" + echo '[INFO] please reboot daemon and container engine to take effect' log "[INFO]" "Ascend Docker Runtime install success" } function uninstall() { - echo "[INFO]: Uninstalling ascend docker runtime ${PACKAGE_VERSION}" + echo "[INFO] uninstalling ascend docker runtime ${PACKAGE_VERSION}" if [ ! -d "${INSTALL_PATH}" ]; then log "[WARNING]" "uninstall skipping, the specified install path does not exist" @@ -315,9 +354,9 @@ function uninstall() exit 1 fi - "${INSTALL_PATH}"/script/uninstall.sh ${ISULA} + "${INSTALL_PATH}"/script/uninstall.sh ${ISULA} ${INSTALL_SCENE} ${CONFIG_FILE_PATH} if [[ $? != 0 ]]; then - log "[ERROR]" "uninstall failed, '${INSTALL_PATH}/script/uninstall.sh ${ISULA}' return non-zero" + log "[ERROR]" "uninstall failed, '${INSTALL_PATH}/script/uninstall.sh ${ISULA} ${INSTALL_SCENE} ${CONFIG_FILE_PATH}' return non-zero" exit 1 fi @@ -326,7 +365,7 @@ function uninstall() function upgrade() { - echo "[INFO]: upgrading ascend docker runtime" + echo "[INFO] upgrading ascend docker runtime" check_platform if [[ $? != 0 ]]; then log "[ERROR]" "upgrade failed, run package and os not matched in arch" @@ -401,12 +440,13 @@ function upgrade() fi chmod 440 ${ASCEND_RUNTIME_CONFIG_DIR}/base.list - echo "[INFO]: Ascend Docker Runtime has been installed in: ${INSTALL_PATH}" - echo '[INFO]: upgrade ascend docker runtime success' - echo "[INFO]: The version of Ascend Docker Runtime is: v${PACKAGE_VERSION}" + echo "[INFO] Ascend Docker Runtime has been installed in: ${INSTALL_PATH}" + echo '[INFO] upgrade ascend docker runtime success' + echo "[INFO] the version of Ascend Docker Runtime is: v${PACKAGE_VERSION}" log "[INFO]" "Ascend Docker Runtime upgrade success" } - +INSTALL_SCENE_FLAG=n +CONFIG_FILE_PATH_FLAG=n INSTALL_FLAG=n INSTALL_PATH_FLAG=n UNINSTALL_FLAG=n @@ -431,9 +471,40 @@ fi while true do case "$3" in + --install-scene=*) + if [ "${INSTALL_SCENE_FLAG}" == "y" ]; then + log "[ERROR]" "failed, repeat parameter '--install-scene' !" + exit 1 + fi + need_help=n + INSTALL_SCENE_FLAG=y + if [ "$3" == "--install-scene=docker" ]; then + INSTALL_SCENE=docker + elif [ "$3" == "--install-scene=containerd" ]; then + INSTALL_SCENE=containerd + else + log "[ERROR]" "failed, please check the parameter of --install-scene=" + exit 1 + fi + shift + ;; + --config-file-path=*) + if [ "${CONFIG_FILE_PATH_FLAG}" == "y" ]; then + log "[ERROR]" "failed, repeat parameter '--config-file-path' !" + exit 1 + fi + need_help=n + CONFIG_FILE_PATH_FLAG=y + CONFIG_FILE_PATH=$(echo $3 | cut -d"=" -f2) + if [[ ! -e "$CONFIG_FILE_PATH" ]]; then + log "[ERROR]" "failed, file '$CONFIG_FILE_PATH' does not exist." + exit 1 + fi + shift + ;; --install) if [ "${INSTALL_FLAG}" == "y" ]; then - log "[ERROR]" "install failed, '--install' Repeat parameter!" + log "[ERROR]" "install failed, repeat parameter '--install' !" exit 1 fi need_help=n @@ -442,7 +513,7 @@ do ;; --uninstall) if [ "${UNINSTALL_FLAG}" == "y" ]; then - log "[ERROR]" "uninstall failed, '--uninstall' Repeat parameter!" + log "[ERROR]" "uninstall failed, repeat parameter '--uninstall' !" exit 1 fi need_help=n @@ -451,7 +522,7 @@ do ;; --install-path=*) if [ "${INSTALL_PATH_FLAG}" == "y" ]; then - log "[ERROR]" "failed, '--install-path' Repeat parameter!" + log "[ERROR]" "failed, repeat parameter '--install-path' !" exit 1 fi need_help=n @@ -462,7 +533,7 @@ do ;; --upgrade) if [ "${UPGRADE_FLAG}" == "y" ]; then - log "[ERROR]" "upgrade failed, '--upgrade' Repeat parameter!" + log "[ERROR]" "upgrade failed, repeat parameter '--upgrade' !" exit 1 fi need_help=n @@ -471,7 +542,7 @@ do ;; --ce=*) if [ "${ISULA}" == "isula" ]; then - log "[ERROR]" "failed, '--ce' Repeat parameter!" + log "[ERROR]" "failed, repeat parameter '--ce' !" exit 1 fi need_help=n @@ -480,7 +551,7 @@ do ISULA=isula RESERVEDEFAULT=yes else - log "[ERROR]" "failed, Please check the parameter of --ce=" + log "[ERROR]" "failed, please check the parameter of --ce=" exit 1 fi shift @@ -488,7 +559,7 @@ do --install-type=*) if [ "${a500}" == "y" ] || [ "${a200}" == "y" ] || [ "${a200isoc}" == "y" ] || [ "${a200ia2}" == "y" ] || [ "${a500a2}" == "y" ]; then - log "[ERROR]" "failed, '--install-type' Repeat parameter!" + log "[ERROR]" "failed, repeat parameter '--install-type' !" exit 1 fi need_help=n @@ -504,7 +575,7 @@ do elif [ "$3" == "--install-type=A200IA2" ]; then a200ia2=y else - log "[ERROR]" "failed, Please check the parameter of --install-type=" + log "[ERROR]" "failed, please check the parameter of --install-type=" exit 1 fi shift @@ -517,7 +588,7 @@ do ;; *) if [ "x$3" != "x" ]; then - log "[ERROR]" "failed, Unsupported parameters: $3" + log "[ERROR]" "failed, unsupported parameters: $3" print_help exit 1 fi diff --git a/build/scripts/uninstall.sh b/build/scripts/uninstall.sh index e6b113cb73172d707acad844ebf84bd84d6b0bb3..798eaf5a0a656bafbe66a4d632023092399eb9f3 100644 --- a/build/scripts/uninstall.sh +++ b/build/scripts/uninstall.sh @@ -29,7 +29,7 @@ function check_log { check_sub_path ${INSTALL_LOG_DIR} if [[ $? != 0 ]]; then - echo "[ERROR]: ${INSTALL_LOG_DIR} is invalid" + echo "[ERROR] ${INSTALL_LOG_DIR} is invalid" exit 1 fi @@ -60,11 +60,11 @@ function log { function check_path { local path="$1" if [[ ${#path} -gt 1024 ]] || [[ ${#path} -le 0 ]]; then - echo "[ERROR]: parameter is invalid, length not in 1~1024" + echo "[ERROR] parameter is invalid, length not in 1~1024" return 1 fi if [[ -n $(echo "${path}" | grep -Ev '^[a-zA-Z0-9./_-]*$') ]]; then - echo "[ERROR]: parameter is invalid, char not all in 'a-zA-Z0-9./_-'" + echo "[ERROR] parameter is invalid, char not all in 'a-zA-Z0-9./_-'" return 1 fi path=$(realpath -m -s "${path}") @@ -99,16 +99,16 @@ function check_sub_path { function check_path_permission { local path="$1" if [[ -L "${path}" ]]; then - echo "[ERROR]: ${path} is soft link" + echo "[ERROR] ${path} is soft link" return 1 fi if [[ $(stat -c %u "${path}") != 0 ]] || [[ "$(stat -c %g ${path})" != 0 ]]; then - echo "[ERROR]: user or group of ${path} is not root" + echo "[ERROR] user or group of ${path} is not root" return 1 fi local permission=$(stat -c %A "${path}") if [[ $(echo "${permission}" | cut -c6) == w ]] || [[ $(echo "${permission}" | cut -c9) == w ]]; then - echo "[ERROR]: group or other of ${path} has write permisson" + echo "[ERROR] group or other of ${path} has write permisson" return 1 fi } @@ -123,17 +123,23 @@ fi ROOT=$(cd $(dirname $0); pwd)/.. RESERVEDEFAULT=no -if [ "$*" == "isula" ] ; then +if [ "$1" == "isula" ] ; then DST='/etc/isulad/daemon.json' - echo "[INFO]: You will recover iSula's daemon" + echo "[INFO] You will recover iSula's daemon" RESERVEDEFAULT=yes else DST='/etc/docker/daemon.json' - echo "[INFO]: You will recover Docker's daemon" + echo "[INFO] You will recover Docker's daemon" +fi +INSTALL_SCENE=$2 +if [ ${INSTALL_SCENE} == containerd ] ; then + DST='/etc/containerd/config.toml' +fi +CONFIG_FILE_PATH=$3 +if [[ ${CONFIG_FILE_PATH} != "" ]]; then + DST=${CONFIG_FILE_PATH} fi - SRC="${DST}.${PPID}" - if [ ! -f "${DST}" ]; then log "[WARNING]" "uninstall skipping, ${DST} does not exist" exit 0 @@ -144,11 +150,11 @@ if [[ $? != 0 ]]; then log "[ERROR]" "uninstall failed, ${DST} is invalid" exit 1 fi - +CGROUP_INFO=$(stat -fc %T /sys/fs/cgroup/) # exit when return code is not 0, if use 'set -e' -${ROOT}/ascend-docker-plugin-install-helper rm ${DST} ${SRC} ${RESERVEDEFAULT} > /dev/null +${ROOT}/ascend-docker-plugin-install-helper rm ${DST} ${SRC} ${RESERVEDEFAULT} ${INSTALL_SCENE} ${CGROUP_INFO} > /dev/null if [[ $? != 0 ]]; then - log "[ERROR]" "uninstall failed, '${ROOT}/ascend-docker-plugin-install-helper rm ${DST} ${SRC} ${RESERVEDEFAULT}' return non-zero" + log "[ERROR]" "uninstall failed, '${ROOT}/ascend-docker-plugin-install-helper rm ${DST} ${SRC} ${RESERVEDEFAULT} ${INSTALL_SCENE} ${CGROUP_INFO}' return non-zero" exit 1 fi @@ -172,6 +178,6 @@ fi if test -d ${INSTALL_ROOT_PATH} then rm -rf ${INSTALL_ROOT_PATH} - echo "[INFO]: delete ${INSTALL_ROOT_PATH} successful" + echo "[INFO] delete ${INSTALL_ROOT_PATH} successful" fi log "[INFO]" "uninstall.sh exec success" diff --git a/cli/test/dt_go/build.sh b/cli/test/dt_go/build.sh index 8eb0feefe39bcf323b8188ae8e711d1e7fc97537..94976165e8c99c1eee75f6371f8e1c17d5aed1a0 100644 --- a/cli/test/dt_go/build.sh +++ b/cli/test/dt_go/build.sh @@ -18,8 +18,7 @@ set -e umask 077 CUR_DIR=$(dirname "$(readlink -f $0)") TOP_DIR=$(realpath "${CUR_DIR}"/../../..) -RUNTIME_DIR=${TOP_DIR}/CODE/mindxcheckutils -export GOPATH="${TOP_DIR}/CODE/opensource" +RUNTIME_DIR=${TOP_DIR}/ export PATH="${GOPATH}/bin/;$PATH" export GO111MODULE=on export GONOSUMDB="*" @@ -30,7 +29,7 @@ function execute_test() { go install github.com/axw/gocov/gocov@v1.0.0 go install github.com/matm/gocov-html@latest go install gotest.tools/gotestsum@latest - if ! (go test -mod=mod -gcflags=all=-l -v -race -coverprofile cov.out ${RUNTIME_DIR} >./$file_input); then + if ! (go test -mod=mod -gcflags=all=-l -v -race -coverprofile cov.out ${RUNTIME_DIR}/... >./$file_input); then echo '****** go test cases error! ******' exit 1 else diff --git a/go.mod b/go.mod index ffaf2ab67c2db0fa7c93f4439e15ddf1dd5959d0..354d854f8163127124a136ad456c464bb4d7d658 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/agiledragon/gomonkey/v2 v2.8.0 github.com/containerd/containerd v1.6.24 github.com/opencontainers/runtime-spec v1.0.3-0.20220718201635-a8106e99982b + github.com/pelletier/go-toml v1.9.5 github.com/prashantv/gostub v0.0.0-00010101000000-000000000000 github.com/stretchr/testify v1.8.2 huawei.com/npu-exporter/v5 v5.0.0 @@ -17,11 +18,14 @@ require ( github.com/containerd/cgroups v1.0.4 // indirect github.com/containerd/continuity v0.3.0 // indirect github.com/containerd/ttrpc v1.1.2 // indirect + github.com/containerd/typeurl v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/imdario/mergo v0.3.12 // indirect github.com/klauspost/compress v1.11.13 // indirect github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect diff --git a/go.sum b/go.sum index 3d095d5425d143212770d3eb1d8220be92afd2e3..f4f96aea0aedd74ad2bb0ce80e24f8f2c31cd4df 100644 --- a/go.sum +++ b/go.sum @@ -191,6 +191,7 @@ github.com/containerd/ttrpc v1.1.2/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Ev github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= +github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= @@ -245,6 +246,7 @@ github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= @@ -403,6 +405,7 @@ github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJ github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= @@ -528,6 +531,8 @@ github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3 github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1018,6 +1023,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/hook/main.go b/hook/main.go index 8355ca8d461f496892455f1d05257630f98b8d74..eecc9fc14dc551c7f8d86680b4342918a8a79b0b 100644 --- a/hook/main.go +++ b/hook/main.go @@ -12,371 +12,26 @@ limitations under the License. */ -// Package main +// Package main is the main entry. package main import ( - "bufio" + "ascend-docker-runtime/hook/process" "context" - "encoding/json" "fmt" "log" "os" - "path" - "path/filepath" "strings" - "syscall" - "github.com/opencontainers/runtime-spec/specs-go" "huawei.com/npu-exporter/v5/common-utils/hwlog" "ascend-docker-runtime/mindxcheckutils" ) const ( - loggingPrefix = "ascend-docker-hook" - runLogPath = "/var/log/ascend-docker-runtime/hook-run.log" - ascendRuntimeOptions = "ASCEND_RUNTIME_OPTIONS" - ascendRuntimeMounts = "ASCEND_RUNTIME_MOUNTS" - ascendVisibleDevices = "ASCEND_VISIBLE_DEVICES" - ascendAllowLink = "ASCEND_ALLOW_LINK" - ascendDockerCli = "ascend-docker-cli" - defaultAscendDockerCli = "/usr/local/bin/ascend-docker-cli" - configDir = "/etc/ascend-docker-runtime.d" - baseConfig = "base" - configFileSuffix = "list" - - kvPairSize = 2 - maxCommandLength = 65535 -) - -var ( - containerConfigInputStream = os.Stdin - doExec = syscall.Exec - ascendDockerCliName = ascendDockerCli - defaultAscendDockerCliName = defaultAscendDockerCli + loggingPrefix = "ascend-docker-hook" ) -var validRuntimeOptions = [...]string{ - "NODRV", - "VIRTUAL", -} - -type containerConfig struct { - Pid int - Rootfs string - Env []string -} - -func initLogModule(ctx context.Context) error { - const backups = 2 - const logMaxAge = 365 - const fileMaxSize = 2 - runLogConfig := hwlog.LogConfig{ - LogFileName: runLogPath, - LogLevel: 0, - MaxBackups: backups, - MaxAge: logMaxAge, - OnlyToFile: true, - FileMaxSize: fileMaxSize, - } - if err := hwlog.InitRunLogger(&runLogConfig, ctx); err != nil { - fmt.Printf("hwlog init failed, error is %v", err) - return err - } - return nil -} - -func parseMounts(mounts string) []string { - if mounts == "" { - return []string{baseConfig} - } - const maxMountLength = 128 - if len(mounts) > maxMountLength { - return []string{baseConfig} - } - - mountConfigs := make([]string, 0) - for _, m := range strings.Split(mounts, ",") { - m = strings.TrimSpace(m) - m = strings.ToLower(m) - mountConfigs = append(mountConfigs, m) - } - - return mountConfigs -} - -func isRuntimeOptionValid(option string) bool { - for _, validOption := range validRuntimeOptions { - if option == validOption { - return true - } - } - - return false -} - -func parseRuntimeOptions(runtimeOptions string) ([]string, error) { - parsedOptions := make([]string, 0) - - if runtimeOptions == "" { - return parsedOptions, nil - } - const maxLength = 128 - if len(runtimeOptions) > maxLength { - hwlog.RunLog.Errorf("length of ASCEND_RUNTIME_OPTIONS value is invalid, its length: %v", len(runtimeOptions)) - return nil, fmt.Errorf("invalid runtime option, the length exceeds 128 characters") - } - - for _, option := range strings.Split(runtimeOptions, ",") { - option = strings.TrimSpace(option) - if !isRuntimeOptionValid(option) { - hwlog.RunLog.Errorf("value of ASCEND_RUNTIME_OPTIONS is not in valid option list, value: %v", option) - return nil, fmt.Errorf("invalid runtime option of invalid input value") - } - - parsedOptions = append(parsedOptions, option) - } - - return parsedOptions, nil -} - -func parseSoftLinkMode(allowLink string) (string, error) { - if allowLink == "True" { - return "True", nil - } - if allowLink == "" || allowLink == "False" { - return "False", nil - } - - return "", fmt.Errorf("invalid soft link option") -} - -func parseOciSpecFile(file string) (*specs.Spec, error) { - f, err := os.Open(file) - if err != nil { - return nil, fmt.Errorf("failed to open the OCI config file: %s", file) - } - defer f.Close() - - spec := new(specs.Spec) - if err := json.NewDecoder(f).Decode(spec); err != nil { - return nil, fmt.Errorf("failed to parse OCI config file: %s, caused by: %v", file, err) - } - - if spec.Process == nil { - return nil, fmt.Errorf("invalid OCI spec for empty process") - } - - if spec.Root == nil { - return nil, fmt.Errorf("invalid OCI spec for empty root") - } - - return spec, nil -} - -var getContainerConfig = func() (*containerConfig, error) { - state := new(specs.State) - decoder := json.NewDecoder(containerConfigInputStream) - - if err := decoder.Decode(state); err != nil { - return nil, fmt.Errorf("failed to parse the container's state") - } - - configPath := path.Join(state.Bundle, "config.json") - if _, err := mindxcheckutils.RealFileChecker(configPath, true, true, mindxcheckutils.DefaultSize); err != nil { - return nil, err - } - - ociSpec, err := parseOciSpecFile(configPath) - if err != nil { - return nil, fmt.Errorf("failed to parse OCI spec: %v", err) - } - if len(ociSpec.Process.Env) > maxCommandLength { - return nil, fmt.Errorf("too many items in spec file") - } - // when use ctr->containerd. the rootfs in config.json is a relative path - rfs := ociSpec.Root.Path - if !filepath.IsAbs(rfs) { - rfs = path.Join(state.Bundle, ociSpec.Root.Path) - } - - ret := &containerConfig{ - Pid: state.Pid, - Rootfs: rfs, - Env: ociSpec.Process.Env, - } - - return ret, nil -} - -func getValueByKey(data []string, name string) string { - splitNumber := 2 - for _, s := range data { - p := strings.SplitN(s, "=", splitNumber) - if len(p) != kvPairSize { - hwlog.RunLog.Errorf("env is not key-value mode, env: %v", s) - log.Panicln("environment error") - } - - if p[0] == name && len(p) == kvPairSize { - return p[1] - } - } - - return "" -} - -func readMountConfig(dir string, name string) ([]string, []string, error) { - configFileName := fmt.Sprintf("%s.%s", name, configFileSuffix) - baseConfigFilePath, err := filepath.Abs(filepath.Join(dir, configFileName)) - if err != nil { - return nil, nil, fmt.Errorf("failed to assemble base config file path: %v", err) - } - - fileInfo, err := os.Stat(baseConfigFilePath) - if _, err := mindxcheckutils.RealFileChecker(baseConfigFilePath, true, false, - mindxcheckutils.DefaultSize); err != nil { - return nil, nil, err - } - if err != nil { - return nil, nil, fmt.Errorf("cannot stat base configuration file %s : %v", baseConfigFilePath, err) - } - - if !fileInfo.Mode().IsRegular() { - return nil, nil, fmt.Errorf("base configuration file damaged because is not a regular file") - } - - f, err := os.Open(baseConfigFilePath) - if err != nil { - return nil, nil, fmt.Errorf("failed to open base configuration file %s: %v", baseConfigFilePath, err) - } - defer f.Close() - - fileMountList, dirMountList := make([]string, 0), make([]string, 0) - const maxEntryNumber = 128 - entryCount := 0 - scanner := bufio.NewScanner(f) - for scanner.Scan() { - mountPath := scanner.Text() - entryCount = entryCount + 1 - if entryCount > maxEntryNumber { - return nil, nil, fmt.Errorf("mount list too long") - } - absMountPath, err := filepath.Abs(mountPath) - if err != nil { - continue // skipping files/dirs with any problems - } - mountPath = absMountPath - - stat, err := os.Stat(mountPath) - if err != nil { - continue // skipping files/dirs with any problems - } - - if stat.Mode().IsRegular() { - fileMountList = append(fileMountList, mountPath) - } else if stat.Mode().IsDir() { - dirMountList = append(dirMountList, mountPath) - } - } - - return fileMountList, dirMountList, nil -} - -func readConfigsOfDir(dir string, configs []string) ([]string, []string, error) { - fileInfo, err := os.Stat(dir) - if err != nil { - return nil, nil, fmt.Errorf("cannot stat configuration directory %s : %v", dir, err) - } - - if !fileInfo.Mode().IsDir() { - return nil, nil, fmt.Errorf("%s should be a dir for ascend docker runtime, but now it is not", dir) - } - - fileMountList := make([]string, 0) - dirMountList := make([]string, 0) - - for _, config := range configs { - fileList, dirList, err := readMountConfig(dir, config) - if err != nil { - return nil, nil, fmt.Errorf("failed to process config %s: %v", config, err) - } - - fileMountList = append(fileMountList, fileList...) - dirMountList = append(dirMountList, dirList...) - } - - return fileMountList, dirMountList, nil -} - -func getArgs(cliPath string, containerConfig *containerConfig, fileMountList []string, - dirMountList []string, allowLink string) []string { - args := append([]string{cliPath}, - "--allow-link", allowLink, "--pid", fmt.Sprintf("%d", containerConfig.Pid), - "--rootfs", containerConfig.Rootfs) - for _, filePath := range fileMountList { - args = append(args, "--mount-file", filePath) - } - for _, dirPath := range dirMountList { - args = append(args, "--mount-dir", dirPath) - } - return args -} - -func doPrestartHook() error { - containerConfig, err := getContainerConfig() - if err != nil { - return fmt.Errorf("failed to get container config: %#v", err) - } - - if visibleDevices := getValueByKey(containerConfig.Env, ascendVisibleDevices); visibleDevices == "" { - return nil - } - - mountConfigs := parseMounts(getValueByKey(containerConfig.Env, ascendRuntimeMounts)) - - fileMountList, dirMountList, err := readConfigsOfDir(configDir, mountConfigs) - if err != nil { - return fmt.Errorf("failed to read configuration from config directory: %#v", err) - } - - parsedOptions, err := parseRuntimeOptions(getValueByKey(containerConfig.Env, ascendRuntimeOptions)) - if err != nil { - return fmt.Errorf("failed to parse runtime options: %#v", err) - } - - allowLink, err := parseSoftLinkMode(getValueByKey(containerConfig.Env, ascendAllowLink)) - if err != nil { - return fmt.Errorf("failed to parse soft link mode: %#v", err) - } - - currentExecPath, err := os.Executable() - if err != nil { - return fmt.Errorf("cannot get the path of ascend-docker-hook: %#v", err) - } - - cliPath := path.Join(path.Dir(currentExecPath), ascendDockerCliName) - if _, err = os.Stat(cliPath); err != nil { - return fmt.Errorf("cannot find ascend-docker-cli executable file at %s: %#v", cliPath, err) - } - if _, err := mindxcheckutils.RealFileChecker(cliPath, true, false, mindxcheckutils.DefaultSize); err != nil { - return err - } - args := getArgs(cliPath, containerConfig, fileMountList, dirMountList, allowLink) - if len(parsedOptions) > 0 { - args = append(args, "--options", strings.Join(parsedOptions, ",")) - } - hwlog.RunLog.Info("ascend docker hook success, will start cli") - if err := mindxcheckutils.ChangeRuntimeLogMode("hook-run-"); err != nil { - return err - } - if err := doExec(cliPath, args, os.Environ()); err != nil { - return fmt.Errorf("failed to exec ascend-docker-cli %v: %v", args, err) - } - return nil -} - func main() { defer func() { if err := recover(); err != nil { @@ -386,7 +41,7 @@ func main() { log.SetPrefix(loggingPrefix) ctx, _ := context.WithCancel(context.Background()) - if err := initLogModule(ctx); err != nil { + if err := process.InitLogModule(ctx); err != nil { log.Fatal(err) } logPrefixWords, err := mindxcheckutils.GetLogPrefix() @@ -400,11 +55,11 @@ func main() { }() hwlog.RunLog.Infof("%v ascend docker hook starting, try to setup container", logPrefixWords) if !mindxcheckutils.StringChecker(strings.Join(os.Args, " "), 0, - maxCommandLength, mindxcheckutils.DefaultWhiteList+" ") { + process.MaxCommandLength, mindxcheckutils.DefaultWhiteList+" ") { hwlog.RunLog.Errorf("%v ascend docker hook failed", logPrefixWords) log.Fatal("command error") } - if err := doPrestartHook(); err != nil { + if err := process.DoPrestartHook(); err != nil { hwlog.RunLog.Errorf("%v ascend docker hook failed: %#v", logPrefixWords, err) log.Fatal(fmt.Errorf("failed in runtime.doProcess: %#v", err)) } diff --git a/hook/process/process.go b/hook/process/process.go new file mode 100644 index 0000000000000000000000000000000000000000..1f8a48c9ce7bfb04ef9920828c138dad6e99a8d0 --- /dev/null +++ b/hook/process/process.go @@ -0,0 +1,381 @@ +/* Copyright(C) 2022. Huawei Technologies Co.,Ltd. All rights reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package process arses the environment variables to obtain the files and directories to be mounted +// and transfers the files and directories to the CLI for mounting before the container is started. +package process + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "log" + "os" + "path" + "path/filepath" + "strings" + "syscall" + + "github.com/opencontainers/runtime-spec/specs-go" + "huawei.com/npu-exporter/v5/common-utils/hwlog" + + "ascend-docker-runtime/mindxcheckutils" +) + +const ( + runLogPath = "/var/log/ascend-docker-runtime/hook-run.log" + ascendRuntimeOptions = "ASCEND_RUNTIME_OPTIONS" + ascendRuntimeMounts = "ASCEND_RUNTIME_MOUNTS" + ascendVisibleDevices = "ASCEND_VISIBLE_DEVICES" + ascendAllowLink = "ASCEND_ALLOW_LINK" + ascendDockerCli = "ascend-docker-cli" + defaultAscendDockerCli = "/usr/local/bin/ascend-docker-cli" + configDir = "/etc/ascend-docker-runtime.d" + baseConfig = "base" + configFileSuffix = "list" + + kvPairSize = 2 + // MaxCommandLength is the max length of command. + MaxCommandLength = 65535 +) + +var ( + containerConfigInputStream = os.Stdin + doExec = syscall.Exec + ascendDockerCliName = ascendDockerCli + defaultAscendDockerCliName = defaultAscendDockerCli +) + +var validRuntimeOptions = [...]string{ + "NODRV", + "VIRTUAL", +} + +type containerConfig struct { + Pid int + Rootfs string + Env []string +} + +// InitLogModule initializes some logging configuration. +func InitLogModule(ctx context.Context) error { + const backups = 2 + const logMaxAge = 365 + const fileMaxSize = 2 + runLogConfig := hwlog.LogConfig{ + LogFileName: runLogPath, + LogLevel: 0, + MaxBackups: backups, + MaxAge: logMaxAge, + OnlyToFile: true, + FileMaxSize: fileMaxSize, + } + if err := hwlog.InitRunLogger(&runLogConfig, ctx); err != nil { + fmt.Printf("hwlog init failed, error is %v", err) + return err + } + return nil +} + +func parseMounts(mounts string) []string { + if mounts == "" { + return []string{baseConfig} + } + const maxMountLength = 128 + if len(mounts) > maxMountLength { + return []string{baseConfig} + } + + mountConfigs := make([]string, 0) + for _, m := range strings.Split(mounts, ",") { + m = strings.TrimSpace(m) + m = strings.ToLower(m) + mountConfigs = append(mountConfigs, m) + } + + return mountConfigs +} + +func isRuntimeOptionValid(option string) bool { + for _, validOption := range validRuntimeOptions { + if option == validOption { + return true + } + } + + return false +} + +func parseRuntimeOptions(runtimeOptions string) ([]string, error) { + parsedOptions := make([]string, 0) + + if runtimeOptions == "" { + return parsedOptions, nil + } + const maxLength = 128 + if len(runtimeOptions) > maxLength { + hwlog.RunLog.Errorf("length of ASCEND_RUNTIME_OPTIONS value is invalid, its length: %v", len(runtimeOptions)) + return nil, fmt.Errorf("invalid runtime option, the length exceeds 128 characters") + } + + for _, option := range strings.Split(runtimeOptions, ",") { + option = strings.TrimSpace(option) + if !isRuntimeOptionValid(option) { + hwlog.RunLog.Errorf("value of ASCEND_RUNTIME_OPTIONS is not in valid option list, value: %v", option) + return nil, fmt.Errorf("invalid runtime option of invalid input value") + } + + parsedOptions = append(parsedOptions, option) + } + + return parsedOptions, nil +} + +func parseSoftLinkMode(allowLink string) (string, error) { + if allowLink == "True" { + return "True", nil + } + if allowLink == "" || allowLink == "False" { + return "False", nil + } + + return "", fmt.Errorf("invalid soft link option") +} + +func parseOciSpecFile(file string) (*specs.Spec, error) { + f, err := os.Open(file) + if err != nil { + return nil, fmt.Errorf("failed to open the OCI config file: %s", file) + } + defer f.Close() + + spec := new(specs.Spec) + if err := json.NewDecoder(f).Decode(spec); err != nil { + return nil, fmt.Errorf("failed to parse OCI config file: %s, caused by: %v", file, err) + } + + if spec.Process == nil { + return nil, fmt.Errorf("invalid OCI spec for empty process") + } + + if spec.Root == nil { + return nil, fmt.Errorf("invalid OCI spec for empty root") + } + + return spec, nil +} + +var getContainerConfig = func() (*containerConfig, error) { + state := new(specs.State) + decoder := json.NewDecoder(containerConfigInputStream) + + if err := decoder.Decode(state); err != nil { + return nil, fmt.Errorf("failed to parse the container's state") + } + + configPath := path.Join(state.Bundle, "config.json") + if _, err := mindxcheckutils.RealFileChecker(configPath, true, true, mindxcheckutils.DefaultSize); err != nil { + return nil, err + } + + ociSpec, err := parseOciSpecFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to parse OCI spec: %v", err) + } + if len(ociSpec.Process.Env) > MaxCommandLength { + return nil, fmt.Errorf("too many items in spec file") + } + // when use ctr->containerd. the rootfs in config.json is a relative path + rfs := ociSpec.Root.Path + if !filepath.IsAbs(rfs) { + rfs = path.Join(state.Bundle, ociSpec.Root.Path) + } + + ret := &containerConfig{ + Pid: state.Pid, + Rootfs: rfs, + Env: ociSpec.Process.Env, + } + + return ret, nil +} + +func getValueByKey(data []string, name string) string { + splitNumber := 2 + for _, s := range data { + p := strings.SplitN(s, "=", splitNumber) + if len(p) != kvPairSize { + hwlog.RunLog.Errorf("env is not key-value mode, env: %v", s) + log.Panicln("environment error") + } + + if p[0] == name && len(p) == kvPairSize { + return p[1] + } + } + + return "" +} + +func readMountConfig(dir string, name string) ([]string, []string, error) { + configFileName := fmt.Sprintf("%s.%s", name, configFileSuffix) + baseConfigFilePath, err := filepath.Abs(filepath.Join(dir, configFileName)) + if err != nil { + return nil, nil, fmt.Errorf("failed to assemble base config file path: %v", err) + } + + fileInfo, err := os.Stat(baseConfigFilePath) + if _, err := mindxcheckutils.RealFileChecker(baseConfigFilePath, true, false, + mindxcheckutils.DefaultSize); err != nil { + return nil, nil, err + } + if err != nil { + return nil, nil, fmt.Errorf("cannot stat base configuration file %s : %v", baseConfigFilePath, err) + } + + if !fileInfo.Mode().IsRegular() { + return nil, nil, fmt.Errorf("base configuration file damaged because is not a regular file") + } + + f, err := os.Open(baseConfigFilePath) + if err != nil { + return nil, nil, fmt.Errorf("failed to open base configuration file %s: %v", baseConfigFilePath, err) + } + defer f.Close() + + fileMountList, dirMountList := make([]string, 0), make([]string, 0) + const maxEntryNumber = 128 + entryCount := 0 + scanner := bufio.NewScanner(f) + for scanner.Scan() { + mountPath := scanner.Text() + entryCount = entryCount + 1 + if entryCount > maxEntryNumber { + return nil, nil, fmt.Errorf("mount list too long") + } + absMountPath, err := filepath.Abs(mountPath) + if err != nil { + continue // skipping files/dirs with any problems + } + mountPath = absMountPath + + stat, err := os.Stat(mountPath) + if err != nil { + continue // skipping files/dirs with any problems + } + + if stat.Mode().IsRegular() { + fileMountList = append(fileMountList, mountPath) + } else if stat.Mode().IsDir() { + dirMountList = append(dirMountList, mountPath) + } + } + + return fileMountList, dirMountList, nil +} + +func readConfigsOfDir(dir string, configs []string) ([]string, []string, error) { + fileInfo, err := os.Stat(dir) + if err != nil { + return nil, nil, fmt.Errorf("cannot stat configuration directory %s : %v", dir, err) + } + + if !fileInfo.Mode().IsDir() { + return nil, nil, fmt.Errorf("%s should be a dir for ascend docker runtime, but now it is not", dir) + } + + fileMountList := make([]string, 0) + dirMountList := make([]string, 0) + + for _, config := range configs { + fileList, dirList, err := readMountConfig(dir, config) + if err != nil { + return nil, nil, fmt.Errorf("failed to process config %s: %v", config, err) + } + + fileMountList = append(fileMountList, fileList...) + dirMountList = append(dirMountList, dirList...) + } + + return fileMountList, dirMountList, nil +} + +func getArgs(cliPath string, containerConfig *containerConfig, fileMountList []string, + dirMountList []string, allowLink string) []string { + args := append([]string{cliPath}, + "--allow-link", allowLink, "--pid", fmt.Sprintf("%d", containerConfig.Pid), + "--rootfs", containerConfig.Rootfs) + for _, filePath := range fileMountList { + args = append(args, "--mount-file", filePath) + } + for _, dirPath := range dirMountList { + args = append(args, "--mount-dir", dirPath) + } + return args +} + +// DoPrestartHook parses the environment variables in the container to obtain the files and directories to be mounted. +func DoPrestartHook() error { + containerConfig, err := getContainerConfig() + if err != nil { + return fmt.Errorf("failed to get container config: %#v", err) + } + + if visibleDevices := getValueByKey(containerConfig.Env, ascendVisibleDevices); visibleDevices == "" { + return nil + } + + mountConfigs := parseMounts(getValueByKey(containerConfig.Env, ascendRuntimeMounts)) + + fileMountList, dirMountList, err := readConfigsOfDir(configDir, mountConfigs) + if err != nil { + return fmt.Errorf("failed to read configuration from config directory: %#v", err) + } + + parsedOptions, err := parseRuntimeOptions(getValueByKey(containerConfig.Env, ascendRuntimeOptions)) + if err != nil { + return fmt.Errorf("failed to parse runtime options: %#v", err) + } + + allowLink, err := parseSoftLinkMode(getValueByKey(containerConfig.Env, ascendAllowLink)) + if err != nil { + return fmt.Errorf("failed to parse soft link mode: %#v", err) + } + + currentExecPath, err := os.Executable() + if err != nil { + return fmt.Errorf("cannot get the path of ascend-docker-hook: %#v", err) + } + + cliPath := path.Join(path.Dir(currentExecPath), ascendDockerCliName) + if _, err = os.Stat(cliPath); err != nil { + return fmt.Errorf("cannot find ascend-docker-cli executable file at %s: %#v", cliPath, err) + } + if _, err := mindxcheckutils.RealFileChecker(cliPath, true, false, mindxcheckutils.DefaultSize); err != nil { + return err + } + args := getArgs(cliPath, containerConfig, fileMountList, dirMountList, allowLink) + if len(parsedOptions) > 0 { + args = append(args, "--options", strings.Join(parsedOptions, ",")) + } + hwlog.RunLog.Info("ascend docker hook success, will start cli") + if err := mindxcheckutils.ChangeRuntimeLogMode("hook-run-"); err != nil { + return err + } + if err := doExec(cliPath, args, os.Environ()); err != nil { + return fmt.Errorf("failed to exec ascend-docker-cli %v: %v", args, err) + } + return nil +} diff --git a/hook/main_test.go b/hook/process/process_test.go similarity index 82% rename from hook/main_test.go rename to hook/process/process_test.go index 8f52ca5db8a4b84d54f5d0a89d27a5a7eb53c2a7..aa2a824e08ab1100046691a91f6048b202d49c1b 100644 --- a/hook/main_test.go +++ b/hook/process/process_test.go @@ -13,22 +13,26 @@ */ // Package main -package main +package process import ( - "github.com/prashantv/gostub" + "context" "os" "os/exec" "testing" + + "github.com/prashantv/gostub" ) const ( - pidSample = 123 - fileMode0600 os.FileMode = 0600 + pidSample = 123 + fileMode0600 os.FileMode = 0600 + ascendVisibleDeviceTestStr = "ASCEND_VISIBLE_DEVICES=0-3,5,7" + configFile = "config.json" ) func TestDoPrestartHookCase1(t *testing.T) { - if err := doPrestartHook(); err != nil { + if err := DoPrestartHook(); err != nil { t.Log("failed") } } @@ -41,7 +45,7 @@ func TestDoPrestartHookCase2(t *testing.T) { } stub := gostub.StubFunc(&getContainerConfig, &conCfg, nil) defer stub.Reset() - if err := doPrestartHook(); err != nil { + if err := DoPrestartHook(); err != nil { t.Log("failed") } } @@ -54,7 +58,7 @@ func TestDoPrestartHookCase3(t *testing.T) { } stub := gostub.StubFunc(&getContainerConfig, &conCfg, nil) defer stub.Reset() - if err := doPrestartHook(); err != nil { + if err := DoPrestartHook(); err != nil { t.Log("failed") } } @@ -68,11 +72,15 @@ func TestDoPrestartHookCase4(t *testing.T) { "ASCEND_RUNTIME_OPTIONS=VIRTUAL,NODRV", }, } + err := InitLogModule(context.Background()) + if err != nil { + t.Log("failed") + } stub := gostub.StubFunc(&getContainerConfig, &conCfg, nil) defer stub.Reset() stub.Stub(&ascendDockerCliName, "") stub.StubFunc(&doExec, nil) - if err := doPrestartHook(); err != nil { + if err := DoPrestartHook(); err != nil { t.Log("failed") } } @@ -86,20 +94,20 @@ func TestDoPrestartHookCase5(t *testing.T) { conCfg := containerConfig{ Pid: pidSample, Rootfs: ".", - Env: []string{"ASCEND_VISIBLE_DEVICES=0-3,5,7"}, + Env: []string{ascendVisibleDeviceTestStr}, } stub := gostub.StubFunc(&getContainerConfig, &conCfg, nil) defer stub.Reset() stub.Stub(&ascendDockerCliName, "clii") stub.Stub(&defaultAscendDockerCliName, "clii") stub.StubFunc(&doExec, nil) - if err := doPrestartHook(); err != nil { + if err := DoPrestartHook(); err != nil { t.Log("failed") } } func TestGetValueByKeyCase1(t *testing.T) { - data := []string{"ASCEND_VISIBLE_DEVICES=0-3,5,7"} + data := []string{ascendVisibleDeviceTestStr} word := "ASCEND_VISIBLE_DEVICES" expectVal := "0-3,5,7" actualVal := getValueByKey(data, word) @@ -124,7 +132,7 @@ func TestGetValueByKeyCase2(t *testing.T) { } func TestGetValueByKeyCase3(t *testing.T) { - data := []string{"ASCEND_VISIBLE_DEVICES=0-3,5,7"} + data := []string{ascendVisibleDeviceTestStr} word := "ASCEND_VISIBLE_DEVICE" expectVal := "" actualVal := getValueByKey(data, word) @@ -160,20 +168,18 @@ func TestParseOciSpecFileCase2(t *testing.T) { } func TestParseOciSpecFileCase3(t *testing.T) { - file := "config.json" cmd := exec.Command("runc", "spec") if err := cmd.Run(); err != nil { t.Log("runc spec failed") } - defer os.Remove(file) - _, err := parseOciSpecFile(file) + defer os.Remove(configFile) + _, err := parseOciSpecFile(configFile) if err != nil { t.Fail() } } func TestGetContainerConfig(t *testing.T) { - file := "config.json" cmd := exec.Command("runc", "spec") if err := cmd.Run(); err != nil { t.Log("runc spec failed") @@ -183,8 +189,8 @@ func TestGetContainerConfig(t *testing.T) { t.Log("exception", err) } }() - defer os.Remove(file) - stateFile, err := os.Open("config.json") + defer os.Remove(configFile) + stateFile, err := os.Open(configFile) if err != nil { t.Log("open file failed") } diff --git a/install/main.go b/install/main.go index b14f1f2c296b7cf9383ab5d0ff0203a98c33dda7..dbd468ca5ed1dbc978116e3396b71ea67961a62f 100644 --- a/install/main.go +++ b/install/main.go @@ -17,17 +17,15 @@ package main import ( "context" - "encoding/json" "flag" "fmt" - "io/ioutil" "log" "os" - "path/filepath" "strings" "huawei.com/npu-exporter/v5/common-utils/hwlog" + "ascend-docker-runtime/install/process" "ascend-docker-runtime/mindxcheckutils" ) @@ -51,17 +49,9 @@ const noDefaultTemplate = `{ }` const ( - actionPosition = 0 - srcFilePosition = 1 - destFilePosition = 2 - runtimeFilePosition = 3 - rmCommandLength = 4 - addCommandLength = 5 - addCommand = "add" - maxCommandLength = 65535 - logPath = "/var/log/ascend-docker-runtime/install-helper-run.log" - rmCommand = "rm" - maxFileSize = 1024 * 1024 * 10 + maxCommandLength = 65535 + logPath = "/var/log/ascend-docker-runtime/install-helper-run.log" + installSceneIndexFromEnd = 2 ) var reserveDefaultRuntime = false @@ -83,7 +73,31 @@ func main() { log.Fatalf("command error, please check %s for detail", logPath) } - err, behavior := process() + const helpMessage = "\tadd " + + " \n" + + "\t rm " + + " \n" + "\t -h help command" + helpFlag := flag.Bool("h", false, helpMessage) + flag.Parse() + if *helpFlag { + _, err := fmt.Println(helpMessage) + log.Fatalf("need help, error: %v", err) + } + command := flag.Args() + if len(command) == 0 { + log.Fatalf("error param") + } + var behavior string + hwlog.RunLog.Infof("command: %v", command) + installScene := command[len(command)-installSceneIndexFromEnd] + if installScene == process.InstallSceneDocker { + err, behavior = process.DockerProcess(command) + } else if installScene == process.InstallSceneContainerd { + err, behavior = process.ContainerdProcess(command) + } else { + hwlog.RunLog.Errorf("error param: %v", command[len(command)-1]) + log.Fatalf("error param: %v", command[len(command)-1]) + } if err != nil { hwlog.RunLog.Errorf("%v run script failed: %v", logPrefixWords, err) log.Fatal(fmt.Errorf("error in installation")) @@ -109,195 +123,3 @@ func initLogModule(ctx context.Context) error { } return nil } - -func checkParamAndGetBehavior(action string, command []string) (bool, string) { - correctParam, behavior := false, "" - if action == addCommand && len(command) == addCommandLength { - correctParam = true - behavior = "install" - } - if action == rmCommand && len(command) == rmCommandLength { - correctParam = true - behavior = "uninstall" - } - return correctParam, behavior -} - -func process() (error, string) { - const helpMessage = "\tadd \n" + - "\t rm \n" + - "\t -h help command" - helpFlag := flag.Bool("h", false, helpMessage) - flag.Parse() - if *helpFlag { - _, err := fmt.Println(helpMessage) - return err, "" - } - command := flag.Args() - if len(command) == 0 { - return fmt.Errorf("error param"), "" - } - - action := command[actionPosition] - correctParam, behavior := checkParamAndGetBehavior(action, command) - if !correctParam { - return fmt.Errorf("error param"), "" - } - - srcFilePath := command[srcFilePosition] - if _, err := os.Stat(srcFilePath); os.IsNotExist(err) { - if _, err := mindxcheckutils.RealDirChecker(filepath.Dir(srcFilePath), true, false); err != nil { - return err, behavior - } - } else { - if _, err := mindxcheckutils.RealFileChecker(srcFilePath, true, false, mindxcheckutils.DefaultSize); err != nil { - return err, behavior - } - } - - destFilePath := command[destFilePosition] - if _, err := mindxcheckutils.RealDirChecker(filepath.Dir(destFilePath), true, false); err != nil { - return err, behavior - } - runtimeFilePath := "" - if len(command) == addCommandLength { - runtimeFilePath = command[runtimeFilePosition] - if _, err := mindxcheckutils.RealFileChecker(runtimeFilePath, true, false, mindxcheckutils.DefaultSize); err != nil { - return err, behavior - } - } - - setReserveDefaultRuntime(command) - - // check file permission - writeContent, err := createJsonString(srcFilePath, runtimeFilePath, action) - if err != nil { - return err, behavior - } - return writeJson(destFilePath, writeContent), behavior -} - -func createJsonString(srcFilePath, runtimeFilePath, action string) ([]byte, error) { - var writeContent []byte - if _, err := os.Stat(srcFilePath); err == nil { - daemon, err := modifyDaemon(srcFilePath, runtimeFilePath, action) - if err != nil { - return nil, err - } - writeContent, err = json.MarshalIndent(daemon, "", " ") - if err != nil { - return nil, err - } - } else if os.IsNotExist(err) { - // not existed - if !reserveDefaultRuntime { - writeContent = []byte(fmt.Sprintf(commonTemplate, runtimeFilePath)) - } else { - writeContent = []byte(fmt.Sprintf(noDefaultTemplate, runtimeFilePath)) - } - } else { - return nil, err - } - return writeContent, nil -} - -func writeJson(destFilePath string, writeContent []byte) error { - if _, err := os.Stat(destFilePath); os.IsNotExist(err) { - const perm = 0600 - file, err := os.OpenFile(destFilePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, perm) - if err != nil { - return fmt.Errorf("create target file failed") - } - _, err = file.Write(writeContent) - if err != nil { - closeErr := file.Close() - return fmt.Errorf("write target file failed with close err %v", closeErr) - } - err = file.Close() - if err != nil { - return fmt.Errorf("close target file failed") - } - return nil - } else { - return fmt.Errorf("target file already existed") - } -} - -func modifyDaemon(srcFilePath, runtimeFilePath, action string) (map[string]interface{}, error) { - // existed... - daemon, err := loadOriginJson(srcFilePath) - if err != nil { - return nil, err - } - - if _, ok := daemon["runtimes"]; !ok && action == addCommand { - daemon["runtimes"] = map[string]interface{}{} - } - runtimeValue := daemon["runtimes"] - runtimeConfig, runtimeConfigOk := runtimeValue.(map[string]interface{}) - if !runtimeConfigOk && action == addCommand { - return nil, fmt.Errorf("extract runtime failed") - } - if action == addCommand { - if _, ok := runtimeConfig["ascend"]; !ok { - runtimeConfig["ascend"] = map[string]interface{}{} - } - ascendConfig, ok := runtimeConfig["ascend"].(map[string]interface{}) - if !ok { - return nil, fmt.Errorf("extract ascend failed") - } - ascendConfig["path"] = runtimeFilePath - if _, ok := ascendConfig["runtimeArgs"]; !ok { - ascendConfig["runtimeArgs"] = []string{} - } - if !reserveDefaultRuntime { - daemon["default-runtime"] = "ascend" - } - } else if action == rmCommand { - if runtimeConfigOk { - delete(runtimeConfig, "ascend") - } - if value, ok := daemon["default-runtime"]; ok && value == "ascend" { - delete(daemon, "default-runtime") - } - } else { - return nil, fmt.Errorf("param error") - } - return daemon, nil -} - -func loadOriginJson(srcFilePath string) (map[string]interface{}, error) { - if fileInfo, err := os.Stat(srcFilePath); err != nil { - return nil, err - } else if fileInfo.Size() > maxFileSize { - return nil, fmt.Errorf("file size too large") - } - - file, err := os.Open(srcFilePath) - if err != nil { - return nil, fmt.Errorf("open daemon.json failed") - } - content, err := ioutil.ReadAll(file) - if err != nil { - closeErr := file.Close() - return nil, fmt.Errorf("read daemon.json failed, close file err is %v", closeErr) - } - err = file.Close() - if err != nil { - return nil, fmt.Errorf("close daemon.json failed") - } - - var daemon map[string]interface{} - err = json.Unmarshal(content, &daemon) - if err != nil { - return nil, fmt.Errorf("load daemon.json failed") - } - return daemon, nil -} - -func setReserveDefaultRuntime(command []string) { - reserveCmdPostion := len(command) - 1 - if command[reserveCmdPostion] == "yes" { - reserveDefaultRuntime = true - } -} diff --git a/install/process/common.go b/install/process/common.go new file mode 100644 index 0000000000000000000000000000000000000000..0cff12a3f325bff62a5acb347ac661b40335bfec --- /dev/null +++ b/install/process/common.go @@ -0,0 +1,28 @@ +/* Copyright(C) 2024. Huawei Technologies Co.,Ltd. All rights reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package process + +func checkParamAndGetBehavior(action string, command []string) (bool, string) { + correctParam, behavior := false, "" + if action == addCommand && len(command) == addCommandLength { + correctParam = true + behavior = "install" + } + if action == rmCommand && len(command) == rmCommandLength { + correctParam = true + behavior = "uninstall" + } + return correctParam, behavior +} diff --git a/install/process/constant.go b/install/process/constant.go new file mode 100644 index 0000000000000000000000000000000000000000..5f8991db6a88338057982cf3133b0c0388b4d28b --- /dev/null +++ b/install/process/constant.go @@ -0,0 +1,76 @@ +/* Copyright(C) 2024. Huawei Technologies Co.,Ltd. All rights reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package process + +const commonTemplate = `{ + "runtimes": { + "ascend": { + "path": "%s", + "runtimeArgs": [] + } + }, + "default-runtime": "ascend" +}` + +const noDefaultTemplate = `{ + "runtimes": { + "ascend": { + "path": "%s", + "runtimeArgs": [] + } + } +}` + +const ( + actionPosition = 0 + srcFilePosition = 1 + destFilePosition = 2 + runtimeFilePosition = 3 + rmCommandLength = 6 + addCommandLength = 7 + maxFileSize = 1024 * 1024 * 10 + cgroupInfoIndexFromEnd = 1 +) + +const ( + addCommand = "add" + rmCommand = "rm" + defaultRuntimeKey = "default-runtime" + // InstallSceneDocker is a 'docker' string of scene + InstallSceneDocker = "docker" + // InstallSceneContainerd is a 'containerd' string of scene + InstallSceneContainerd = "containerd" + v1NeedChangeKeyRuntime = "runtime" + v1NeedChangeKeyRuntimeType = "runtime_type" + v1RuntimeType = "io.containerd.runtime.v1.linux" + v1DefaultRuncRuntimeType = "io.containerd.runc.v2" + defaultRuntimeValue = "runc" + v1RuntimeTypeFisrtLevelPlugin = "io.containerd.grpc.v1.cri" + containerdKey = "containerd" + runtimesKey = "runtimes" + runcKey = "runc" + runcOptionsKey = "options" + binaryNameKey = "BinaryName" + cgroupV2InfoStr = "cgroup2fs" +) + +const ( + notFindPluginLogStr = "can not find plugin %v, plugins is: %+v" + notFindOluginErrorStr = "can not find plugin: %v" + convertConfigFailLogStr = "can not convert config %v, config is: %+v" + convertConfigFailErrorStr + convertTreeFailLogStr = "failed to convert map to tree, error: %v" + getMapFaileLogStr = "failed to get map, key: %v, error: %v" +) diff --git a/install/process/containerd_process.go b/install/process/containerd_process.go new file mode 100644 index 0000000000000000000000000000000000000000..fb0fee481151dd22aba07b9c39eb9a3366db9c27 --- /dev/null +++ b/install/process/containerd_process.go @@ -0,0 +1,256 @@ +/* Copyright(C) 2024. Huawei Technologies Co.,Ltd. All rights reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package process + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/containerd/containerd/services/server/config" + "github.com/pelletier/go-toml" + "huawei.com/npu-exporter/v5/common-utils/hwlog" + + "ascend-docker-runtime/mindxcheckutils" +) + +// ContainerdProcess modifies the containerd configuration file when installing or uninstalling the containerd scenario. +func ContainerdProcess(command []string) (error, string) { + action := command[actionPosition] + correctParam, behavior := checkParamAndGetBehavior(action, command) + if !correctParam { + return fmt.Errorf("error param"), "" + } + srcFilePath := command[srcFilePosition] + if _, err := os.Stat(srcFilePath); os.IsNotExist(err) { + if _, err := mindxcheckutils.RealDirChecker(filepath.Dir(srcFilePath), true, false); err != nil { + hwlog.RunLog.Errorf("check failed, error: %v", err) + return err, behavior + } + } else { + if _, err := mindxcheckutils.RealFileChecker(srcFilePath, true, false, mindxcheckutils.DefaultSize); err != nil { + hwlog.RunLog.Errorf("check failed, error: %v", err) + return err, behavior + } + } + destFilePath := command[destFilePosition] + if _, err := mindxcheckutils.RealDirChecker(filepath.Dir(destFilePath), true, false); err != nil { + return err, behavior + } + runtimeFilePath := "" + if len(command) == addCommandLength { + runtimeFilePath = command[runtimeFilePosition] + if _, err := mindxcheckutils.RealFileChecker(runtimeFilePath, true, false, mindxcheckutils.DefaultSize); err != nil { + hwlog.RunLog.Errorf("failed to check, error: %v", err) + return err, behavior + } + } + cgroupInfo := command[len(command)-cgroupInfoIndexFromEnd] + err := editContainerdConfig(srcFilePath, runtimeFilePath, destFilePath, action, cgroupInfo) + if err != nil { + hwlog.RunLog.Errorf("failed to edit containerd config, err: %v", err) + return err, behavior + } + return nil, behavior +} + +func isCgroupV2(cgroupInfo string) (bool, error) { + hwlog.RunLog.Infof("cgroup info: %v", cgroupInfo) + if strings.Contains(cgroupInfo, cgroupV2InfoStr) { + return true, nil + } + return false, nil +} + +func editContainerdConfig(srcFilePath, runtimeFilePath, destFilePath, action, cgroupInfo string) error { + cfg := config.Config{} + if err := config.LoadConfig(srcFilePath, &cfg); err != nil { + hwlog.RunLog.Errorf("failed to load configuration file: %v", err) + return err + } + isV2, err := isCgroupV2(cgroupInfo) + if err != nil { + hwlog.RunLog.Errorf("failed to check whether cgroup v2: %v", err) + return err + } + if isV2 { + hwlog.RunLog.Infof("it is cgroup v2") + binaryName := "" + if action == addCommand { + binaryName = runtimeFilePath + } + err = changeCgroupV2BinaryNameConfig(&cfg, binaryName) + if err != nil { + hwlog.RunLog.Errorf("failed to change cgroup v2 config, error: %v", err) + return err + } + } else { + hwlog.RunLog.Infof("it is cgroup v1") + runtimeValue := defaultRuntimeValue + runtimeType := v1DefaultRuncRuntimeType + if action == addCommand { + runtimeValue = runtimeFilePath + runtimeType = v1RuntimeType + } + err = changeCgroupV1Config(&cfg, runtimeValue, runtimeType) + if err != nil { + hwlog.RunLog.Errorf("failed to change cgroup v1 config, error: %v", err) + return err + } + } + err = writeContainerdConfigToFile(cfg, destFilePath) + if err != nil { + hwlog.RunLog.Errorf("failed to write configuration file: %v", err) + return err + } + return nil +} + +func changeCgroupV2BinaryNameConfig(cfg *config.Config, binaryName string) error { + value, ok := cfg.Plugins[v1RuntimeTypeFisrtLevelPlugin] + if !ok { + hwlog.RunLog.Errorf(notFindPluginLogStr, v1RuntimeTypeFisrtLevelPlugin, cfg.Plugins) + return fmt.Errorf(notFindOluginErrorStr, v1RuntimeTypeFisrtLevelPlugin) + } + valueMap := value.ToMap() + containerdConfig := valueMap[containerdKey] + runtimesConfig, err := getMap(containerdConfig, runtimesKey) + if err != nil { + hwlog.RunLog.Errorf(getMapFaileLogStr, runtimesKey, err) + return err + } + runcConfig, err := getMap(runtimesConfig, runcKey) + if err != nil { + hwlog.RunLog.Errorf(getMapFaileLogStr, runcKey, err) + return err + } + runcOptionsConfig, err := getMap(runcConfig, runcOptionsKey) + if err != nil { + hwlog.RunLog.Errorf(getMapFaileLogStr, runcOptionsKey, err) + return err + } + runcOptionsConfigMap, ok := runcOptionsConfig.(map[string]interface{}) + if !ok { + hwlog.RunLog.Errorf(convertConfigFailLogStr, runcOptionsKey, runcOptionsConfig) + return fmt.Errorf(convertConfigFailErrorStr, runcOptionsKey, runcOptionsConfig) + } + runcOptionsConfigMap[binaryNameKey] = binaryName + newTree, err := toml.TreeFromMap(valueMap) + if err != nil { + hwlog.RunLog.Errorf(convertTreeFailLogStr, err) + return err + } + cfg.Plugins[v1RuntimeTypeFisrtLevelPlugin] = *newTree + return nil +} + +func changeCgroupV1Config(cfg *config.Config, runtimeValue, runtimeType string) error { + err := changeCgroupV1RuntimeConfig(cfg, runtimeValue) + if err != nil { + hwlog.RunLog.Errorf("failed to change cgroup V1 runtime config, error: %v", err) + return err + } + return changeCgroupV1RuntimeTypeConfig(cfg, runtimeType) +} + +func changeCgroupV1RuntimeConfig(cfg *config.Config, runtimeValue string) error { + if value, ok := cfg.Plugins[v1RuntimeType]; ok { + valueMap := value.ToMap() + valueMap[v1NeedChangeKeyRuntime] = runtimeValue + newTree, err := toml.TreeFromMap(valueMap) + if err != nil { + hwlog.RunLog.Errorf(convertTreeFailLogStr, err) + return err + } + cfg.Plugins[v1RuntimeType] = *newTree + } else { + hwlog.RunLog.Errorf(notFindPluginLogStr, v1RuntimeType, cfg.Plugins) + return fmt.Errorf(notFindOluginErrorStr, v1RuntimeType) + } + return nil +} + +func changeCgroupV1RuntimeTypeConfig(cfg *config.Config, runtimeType string) error { + value, ok := cfg.Plugins[v1RuntimeTypeFisrtLevelPlugin] + if !ok { + hwlog.RunLog.Errorf(notFindPluginLogStr, v1RuntimeTypeFisrtLevelPlugin, cfg.Plugins) + return fmt.Errorf(notFindOluginErrorStr, v1RuntimeTypeFisrtLevelPlugin) + } + valueMap := value.ToMap() + containerdConfig := valueMap[containerdKey] + runtimesConfig, err := getMap(containerdConfig, runtimesKey) + if err != nil { + hwlog.RunLog.Errorf(getMapFaileLogStr, runtimesKey, err) + return err + } + runcConfig, err := getMap(runtimesConfig, runcKey) + if err != nil { + hwlog.RunLog.Errorf(getMapFaileLogStr, runcKey, err) + return err + } + runcConfigMap, ok := runcConfig.(map[string]interface{}) + if !ok { + hwlog.RunLog.Errorf(convertConfigFailLogStr, runcKey, runcConfig) + return fmt.Errorf(convertConfigFailErrorStr, runcKey, runcConfig) + } + runcConfigMap[v1NeedChangeKeyRuntimeType] = runtimeType + newTree, err := toml.TreeFromMap(valueMap) + if err != nil { + hwlog.RunLog.Errorf(convertTreeFailLogStr, err) + return err + } + cfg.Plugins[v1RuntimeTypeFisrtLevelPlugin] = *newTree + return nil +} + +func getMap(input interface{}, key string) (interface{}, error) { + inputMap, ok := input.(map[string]interface{}) + if !ok { + hwlog.RunLog.Errorf(convertConfigFailLogStr, key, input) + return nil, fmt.Errorf(convertConfigFailErrorStr, key, input) + } + output, ok := inputMap[key] + if !ok { + hwlog.RunLog.Errorf("can not find config %v, config is: %+v", key, inputMap) + return nil, fmt.Errorf("can not find config: %v", key) + } + return output, nil +} + +func writeContainerdConfigToFile(cfg config.Config, destFilePath string) error { + tomlString, err := toml.Marshal(cfg) + if err != nil { + hwlog.RunLog.Errorf("failed to marshall to toml, error: %v", err) + return err + } + file, err := os.Create(destFilePath) + if err != nil { + hwlog.RunLog.Errorf("failed to create file, error: %v", err) + return err + } + defer func() { + err := file.Close() + if err != nil { + hwlog.RunLog.Errorf("failed to close file, error: %v", err) + } + }() + _, err = file.Write(tomlString) + if err != nil { + hwlog.RunLog.Errorf("failed to write, error: %v", err) + return err + } + return nil +} diff --git a/install/process/docker_process.go b/install/process/docker_process.go new file mode 100644 index 0000000000000000000000000000000000000000..77b58712d972cc5d99043407c2828339adc62763 --- /dev/null +++ b/install/process/docker_process.go @@ -0,0 +1,207 @@ +/* Copyright(C) 2024. Huawei Technologies Co.,Ltd. All rights reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package process deal the docker or containerd scene installation +package process + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "ascend-docker-runtime/mindxcheckutils" +) + +var reserveDefaultRuntime = false + +// DockerProcess modifies the docker configuration file when installing or uninstalling the docker scenario. +func DockerProcess(command []string) (error, string) { + + action := command[actionPosition] + correctParam, behavior := checkParamAndGetBehavior(action, command) + if !correctParam { + return fmt.Errorf("error param"), "" + } + + srcFilePath := command[srcFilePosition] + if _, err := os.Stat(srcFilePath); os.IsNotExist(err) { + if _, err := mindxcheckutils.RealDirChecker(filepath.Dir(srcFilePath), true, false); err != nil { + return err, behavior + } + } else { + if _, err := mindxcheckutils.RealFileChecker(srcFilePath, true, false, mindxcheckutils.DefaultSize); err != nil { + return err, behavior + } + } + + destFilePath := command[destFilePosition] + if _, err := mindxcheckutils.RealDirChecker(filepath.Dir(destFilePath), true, false); err != nil { + return err, behavior + } + runtimeFilePath := "" + if len(command) == addCommandLength { + runtimeFilePath = command[runtimeFilePosition] + if _, err := mindxcheckutils.RealFileChecker(runtimeFilePath, true, false, mindxcheckutils.DefaultSize); err != nil { + return err, behavior + } + } + + setReserveDefaultRuntime(command) + + // check file permission + writeContent, err := createJsonString(srcFilePath, runtimeFilePath, action) + if err != nil { + return err, behavior + } + return writeJson(destFilePath, writeContent), behavior +} + +func createJsonString(srcFilePath, runtimeFilePath, action string) ([]byte, error) { + var writeContent []byte + if _, err := os.Stat(srcFilePath); err == nil { + daemon, err := modifyDaemon(srcFilePath, runtimeFilePath, action) + if err != nil { + return nil, err + } + writeContent, err = json.MarshalIndent(daemon, "", " ") + if err != nil { + return nil, err + } + } else if os.IsNotExist(err) { + // not existed + if !reserveDefaultRuntime { + writeContent = []byte(fmt.Sprintf(commonTemplate, runtimeFilePath)) + } else { + writeContent = []byte(fmt.Sprintf(noDefaultTemplate, runtimeFilePath)) + } + } else { + return nil, err + } + return writeContent, nil +} + +func writeJson(destFilePath string, writeContent []byte) error { + if _, err := os.Stat(destFilePath); os.IsNotExist(err) { + const perm = 0600 + file, err := os.OpenFile(destFilePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, perm) + if err != nil { + return fmt.Errorf("create target file failed") + } + _, err = file.Write(writeContent) + if err != nil { + closeErr := file.Close() + return fmt.Errorf("write target file failed with close err %v", closeErr) + } + err = file.Close() + if err != nil { + return fmt.Errorf("close target file failed") + } + return nil + } else { + return fmt.Errorf("target file already existed") + } +} + +func modifyDaemon(srcFilePath, runtimeFilePath, action string) (map[string]interface{}, error) { + // existed... + daemon, err := loadOriginJson(srcFilePath) + if err != nil { + return nil, err + } + + if _, ok := daemon["runtimes"]; !ok && action == addCommand { + daemon["runtimes"] = map[string]interface{}{} + } + runtimeValue := daemon["runtimes"] + runtimeConfig, runtimeConfigOk := runtimeValue.(map[string]interface{}) + if !runtimeConfigOk && action == addCommand { + return nil, fmt.Errorf("extract runtime failed") + } + if action == addCommand { + runtimeConfig, daemon, err = addDockerDaemon(runtimeConfig, daemon, runtimeFilePath) + } else if action == rmCommand { + runtimeConfig, daemon, err = rmDockerDaemon(runtimeConfig, daemon, runtimeConfigOk) + } else { + return nil, fmt.Errorf("param error") + } + return daemon, err +} + +func addDockerDaemon(runtimeConfig, daemon map[string]interface{}, runtimeFilePath string, +) (map[string]interface{}, map[string]interface{}, error) { + if _, ok := runtimeConfig["ascend"]; !ok { + runtimeConfig["ascend"] = map[string]interface{}{} + } + ascendConfig, ok := runtimeConfig["ascend"].(map[string]interface{}) + if !ok { + return nil, nil, fmt.Errorf("extract ascend failed") + } + ascendConfig["path"] = runtimeFilePath + if _, ok := ascendConfig["runtimeArgs"]; !ok { + ascendConfig["runtimeArgs"] = []string{} + } + if !reserveDefaultRuntime { + daemon[defaultRuntimeKey] = "ascend" + } + return runtimeConfig, daemon, nil +} + +func rmDockerDaemon(runtimeConfig, daemon map[string]interface{}, runtimeConfigOk bool, +) (map[string]interface{}, map[string]interface{}, error) { + if runtimeConfigOk { + delete(runtimeConfig, "ascend") + } + if value, ok := daemon[defaultRuntimeKey]; ok && value == "ascend" { + delete(daemon, defaultRuntimeKey) + } + return runtimeConfig, daemon, nil +} + +func loadOriginJson(srcFilePath string) (map[string]interface{}, error) { + if fileInfo, err := os.Stat(srcFilePath); err != nil { + return nil, err + } else if fileInfo.Size() > maxFileSize { + return nil, fmt.Errorf("file size too large") + } + + file, err := os.Open(srcFilePath) + if err != nil { + return nil, fmt.Errorf("open daemon.json failed") + } + content, err := ioutil.ReadAll(file) + if err != nil { + closeErr := file.Close() + return nil, fmt.Errorf("read daemon.json failed, close file err is %v", closeErr) + } + err = file.Close() + if err != nil { + return nil, fmt.Errorf("close daemon.json failed") + } + + var daemon map[string]interface{} + err = json.Unmarshal(content, &daemon) + if err != nil { + return nil, fmt.Errorf("load daemon.json failed") + } + return daemon, nil +} + +func setReserveDefaultRuntime(command []string) { + reserveCmdPostion := len(command) - 1 + if command[reserveCmdPostion] == "yes" { + reserveDefaultRuntime = true + } +} diff --git a/install/main_test.go b/install/process/docker_process_test.go similarity index 76% rename from install/main_test.go rename to install/process/docker_process_test.go index ebecd29ebfd95970f001f32aff9d55155a8a9b1a..2742b87fad0097877fc2bcd7d0ec310512a37486 100644 --- a/install/main_test.go +++ b/install/process/docker_process_test.go @@ -13,7 +13,7 @@ */ // Package main -package main +package process import ( "encoding/json" @@ -22,7 +22,8 @@ import ( "testing" ) -const oldString = `{ +const ( + oldString = `{ "runtimes": { "ascend": { "path": "/test/runtime", @@ -31,6 +32,12 @@ const oldString = `{ }, "default-runtime": "ascend" }` + defaultRuntime = `"default-runtime"` + oldJson = "old.json" + createOldFail = "create old failed %s" + updateFail = "update failed %s" + updateFailAndData = "update failed %s, %v" +) func jSONBytesEqual(a, b []byte) (bool, error) { var contentA, contentB interface{} @@ -56,16 +63,16 @@ func TestCreateJsonStringWholeNew(t *testing.T) { func TestCreateJsonStringUpdate(t *testing.T) { const perm = 0600 - if fid, err := os.OpenFile("old.json", os.O_CREATE|os.O_RDWR|os.O_TRUNC, perm); err == nil { + if fid, err := os.OpenFile(oldJson, os.O_CREATE|os.O_RDWR|os.O_TRUNC, perm); err == nil { _, err = fid.Write([]byte(oldString)) closeErr := fid.Close() if err != nil || closeErr != nil { - t.Fatalf("create old failed %s", err) + t.Fatalf(createOldFail, err) } } - data, err := createJsonString("old.json", "/test/runtime1", "add") + data, err := createJsonString(oldJson, "/test/runtime1", "add") if err != nil { - t.Fatalf("update failed %s", err) + t.Fatalf(updateFail, err) } expectString := `{ "runtimes": { @@ -74,10 +81,10 @@ func TestCreateJsonStringUpdate(t *testing.T) { "runtimeArgs": [] } }, - "default-runtime": "ascend" - }` + ` + defaultRuntime + `: "ascend" +}` if eq, err := jSONBytesEqual([]byte(expectString), data); err != nil || !eq { - t.Fatalf("update failed %s, %v", err, string(data)) + t.Fatalf(updateFailAndData, err, string(data)) } } @@ -94,18 +101,18 @@ func TestCreateJsonStringUpdateWithOtherParam(t *testing.T) { "runtimeArgs": [1,2,3] } }, - "default-runtime": "runc" - }` + ` + defaultRuntime + `: "runc" +}` if fid, err := os.OpenFile("old.json", os.O_CREATE|os.O_RDWR|os.O_TRUNC, perm); err == nil { _, err = fid.Write([]byte(oldStringWithParam)) closeErr := fid.Close() if err != nil || closeErr != nil { - t.Fatalf("create old failed %s", err) + t.Fatalf(createOldFail, err) } } - data, err := createJsonString("old.json", "/test/runtime1", "add") + data, err := createJsonString(oldJson, "/test/runtime1", "add") if err != nil { - t.Fatalf("update failed %s", err) + t.Fatalf(updateFail, err) } expectString := `{ "runtimes": { @@ -118,30 +125,30 @@ func TestCreateJsonStringUpdateWithOtherParam(t *testing.T) { "runtimeArgs": [1,2,3] } }, - "default-runtime": "ascend" - }` + ` + defaultRuntime + `: "ascend" +}` if eq, err := jSONBytesEqual([]byte(expectString), data); err != nil || !eq { - t.Fatalf("update failed %s, %v", err, string(data)) + t.Fatalf(updateFailAndData, err, string(data)) } } func TestCreateJsonStrinRm(t *testing.T) { const perm = 0600 - if fid, err := os.OpenFile("old.json", os.O_CREATE|os.O_RDWR|os.O_TRUNC, perm); err == nil { + if fid, err := os.OpenFile(oldJson, os.O_CREATE|os.O_RDWR|os.O_TRUNC, perm); err == nil { _, err = fid.Write([]byte(oldString)) closeErr := fid.Close() if err != nil || closeErr != nil { - t.Fatalf("create old failed %s", err) + t.Fatalf(createOldFail, err) } } - data, err := createJsonString("old.json", "", "rm") + data, err := createJsonString(oldJson, "", "rm") if err != nil { - t.Fatalf("update failed %s", err) + t.Fatalf(updateFail, err) } expectString := `{ "runtimes": {} }` if eq, err := jSONBytesEqual([]byte(expectString), data); err != nil || !eq { - t.Fatalf("update failed %s, %v", err, string(data)) + t.Fatalf(updateFailAndData, err, string(data)) } } diff --git a/runtime/dcmi/dcmi_api_test.go b/runtime/dcmi/dcmi_api_test.go index e609d6fceafc932ba68eec15979f0e08de981056..491da1e515a002e86a060cc3b241d5c1e03e015f 100644 --- a/runtime/dcmi/dcmi_api_test.go +++ b/runtime/dcmi/dcmi_api_test.go @@ -16,9 +16,11 @@ package dcmi import ( + "context" "testing" "github.com/opencontainers/runtime-spec/specs-go" + "huawei.com/npu-exporter/v5/common-utils/hwlog" ) const mockDeviceID = 100 @@ -37,7 +39,7 @@ func (w *mockWorker) ShutDown() { // CreateVDevice create v device func (w *mockWorker) CreateVDevice(_, _ int32, _ string) (int32, error) { - return int32(mockDeviceId), nil + return int32(mockDeviceID), nil } // DestroyVDevice destroy virtual device @@ -50,32 +52,57 @@ func (w *mockWorker) FindDevice(_ int32) (int32, int32, error) { return 0, 0, nil } +// GetProductType gets product type +func (w *mockWorker) GetProductType(cardID, deviceID int32) (string, error) { + return "", nil +} + +// GetChipInfo gets chip info +func (w *mockWorker) GetChipInfo(cardID, deviceID int32) (*ChipInfo, error) { + return &ChipInfo{}, nil +} + func TestCreateVDevice(t *testing.T) { t.Log("TestCreateVDevice start") process := specs.Process{} spec := specs.Spec{Process: &process} spec.Process.Env = []string{} + var deviceIdList []int + backups := 2 + logMaxAge := 365 + fileMaxSize := 2 + runLogConfig := hwlog.LogConfig{ + LogFileName: "./test/run.log", + LogLevel: 0, + MaxBackups: backups, + FileMaxSize: fileMaxSize, + MaxAge: logMaxAge, + } + if err := hwlog.InitRunLogger(&runLogConfig, context.Background()); err != nil { + t.Fatalf("hwlog init failed, error is %v", err) + } // no split, all ok - vdevice, err := CreateVDevice(&mockWorker{}, &spec) + vdevice, err := CreateVDevice(&mockWorker{}, &spec, deviceIdList) if err != nil { t.Fatalf("%v %v", vdevice, err) } // no npu assigin for split spec.Process.Env = []string{"ASCEND_VNPU_SPECS=vir04"} - vdevice, err = CreateVDevice(&mockWorker{}, &spec) + vdevice, err = CreateVDevice(&mockWorker{}, &spec, deviceIdList) if err == nil { t.Fatalf("%v %v", vdevice, err) } // split ok + deviceIdList = []int{0} spec.Process.Env = []string{"ASCEND_VNPU_SPECS=vir04", "ASCEND_VISIBLE_DEVICES=0"} - vdevice, err = CreateVDevice(&mockWorker{}, &spec) + vdevice, err = CreateVDevice(&mockWorker{}, &spec, deviceIdList) if err != nil { t.Fatalf("%v %v", vdevice, err) } - if vdevice.VdeviceID != mockDeviceId { + if vdevice.VdeviceID != mockDeviceID { t.Fatalf("%v %v", vdevice, err) } diff --git a/runtime/main.go b/runtime/main.go index f3dc0e7969daea81725d01cb1872df444e77e7d7..0f341fd678a05726f886579ae7e6b286fcbbfb28 100644 --- a/runtime/main.go +++ b/runtime/main.go @@ -17,664 +17,17 @@ package main import ( "context" - "encoding/json" "fmt" - "io/ioutil" "log" "os" - "os/exec" - "path" - "path/filepath" - "regexp" - "sort" - "strconv" "strings" - "syscall" - "github.com/containerd/containerd/oci" - "github.com/opencontainers/runtime-spec/specs-go" "huawei.com/npu-exporter/v5/common-utils/hwlog" "ascend-docker-runtime/mindxcheckutils" - "ascend-docker-runtime/runtime/dcmi" + "ascend-docker-runtime/runtime/process" ) -const ( - runLogPath = "/var/log/ascend-docker-runtime/runtime-run.log" - hookDefaultFilePath = "/usr/local/bin/ascend-docker-hook" - - maxCommandLength = 65535 - hookCli = "ascend-docker-hook" - destroyHookCli = "ascend-docker-destroy" - dockerRuncFile = "docker-runc" - runcFile = "runc" - envLength = 2 - kvPairSize = 2 - borderNum = 2 - - // ENV for device-plugin to identify ascend-docker-runtime - useAscendDocker = "ASCEND_DOCKER_RUNTIME=True" - devicePlugin = "ascend-device-plugin" - ascendVisibleDevices = "ASCEND_VISIBLE_DEVICES" - ascendRuntimeOptions = "ASCEND_RUNTIME_OPTIONS" - - // void indicates that the NPU card does not need to be mounted - void = "void" -) - -var ( - hookCliPath = hookCli - hookDefaultFile = hookDefaultFilePath - dockerRuncName = dockerRuncFile - runcName = runcFile -) - -const ( - // Atlas200ISoc Product name - Atlas200ISoc = "Atlas 200I SoC A1" - // Atlas200 Product name - Atlas200 = "Atlas 200 Model 3000" - // Ascend310 ascend 310 chip - Ascend310 = "Ascend310" - // Ascend310P ascend 310P chip - Ascend310P = "Ascend310P" - // Ascend310B ascend 310B chip - Ascend310B = "Ascend310B" - // Ascend910 ascend 910 chip - Ascend910 = "Ascend910" - ascend = "Ascend" - - devicePath = "/dev/" - davinciName = "davinci" - virtualDavinciName = "vdavinci" - davinciManager = "davinci_manager" - davinciManagerDocker = "davinci_manager_docker" - notRenameDeviceType = "" - devmmSvm = "devmm_svm" - hisiHdc = "hisi_hdc" - svm0 = "svm0" - tsAisle = "ts_aisle" - upgrade = "upgrade" - sys = "sys" - vdec = "vdec" - vpc = "vpc" - pngd = "pngd" - venc = "venc" - dvppCmdList = "dvpp_cmdlist" - logDrv = "log_drv" - acodec = "acodec" - ai = "ai" - ao = "ao" - vo = "vo" - hdmi = "hdmi" -) - -type args struct { - bundleDirPath string - cmd string -} - -// GetDeviceTypeByChipName get device type by chipName -func GetDeviceTypeByChipName(chipName string) string { - if strings.Contains(chipName, "310B") { - return Ascend310B - } - if strings.Contains(chipName, "310P") { - return Ascend310P - } - if strings.Contains(chipName, "310") { - return Ascend310 - } - if strings.Contains(chipName, "910") { - return Ascend910 - } - return "" -} - -func getArgs() (*args, error) { - args := &args{} - - for i, param := range os.Args { - if param == "--bundle" || param == "-b" { - if len(os.Args)-i <= 1 { - return nil, fmt.Errorf("bundle option needs an argument") - } - args.bundleDirPath = os.Args[i+1] - } else if param == "create" { - args.cmd = param - } - } - - return args, nil -} - -func initLogModule(ctx context.Context) error { - const backups = 2 - const logMaxAge = 365 - const fileMaxSize = 2 - runLogConfig := hwlog.LogConfig{ - LogFileName: runLogPath, - LogLevel: 0, - MaxBackups: backups, - MaxAge: logMaxAge, - OnlyToFile: true, - FileMaxSize: fileMaxSize, - } - if err := hwlog.InitRunLogger(&runLogConfig, ctx); err != nil { - fmt.Printf("hwlog init failed, error is %v", err) - return err - } - return nil -} - -var execRunc = func() error { - tempRuncPath, err := exec.LookPath(dockerRuncName) - if err != nil { - tempRuncPath, err = exec.LookPath(runcName) - if err != nil { - return fmt.Errorf("failed to find the path of runc: %v", err) - } - } - runcPath, err := filepath.EvalSymlinks(tempRuncPath) - if err != nil { - return fmt.Errorf("failed to find realpath of runc %v", err) - } - if _, err := mindxcheckutils.RealFileChecker(runcPath, true, false, mindxcheckutils.DefaultSize); err != nil { - return err - } - - if err := mindxcheckutils.ChangeRuntimeLogMode("runtime-run-"); err != nil { - return err - } - if err = syscall.Exec(runcPath, append([]string{runcPath}, os.Args[1:]...), os.Environ()); err != nil { - return fmt.Errorf("failed to exec runc: %v", err) - } - - return nil -} - -func addAscendDockerEnv(spec *specs.Spec) { - if spec == nil || spec.Process == nil || spec.Process.Env == nil { - return - } - spec.Process.Env = append(spec.Process.Env, useAscendDocker) -} - -func addHook(spec *specs.Spec, deviceIdList *[]int) error { - if deviceIdList == nil { - return nil - } - currentExecPath, err := os.Executable() - if err != nil { - return fmt.Errorf("cannot get the path of ascend-docker-runtime: %v", err) - } - - hookCliPath = path.Join(path.Dir(currentExecPath), hookCli) - if _, err := mindxcheckutils.RealFileChecker(hookCliPath, true, false, mindxcheckutils.DefaultSize); err != nil { - return err - } - if _, err = os.Stat(hookCliPath); err != nil { - return fmt.Errorf("cannot find ascend-docker-hook executable file at %s: %v", hookCliPath, err) - } - - if spec.Hooks == nil { - spec.Hooks = &specs.Hooks{} - } - - needUpdate := true - if len(spec.Hooks.Prestart) > maxCommandLength { - return fmt.Errorf("too many items in Prestart ") - } - for _, hook := range spec.Hooks.Prestart { - if strings.Contains(hook.Path, hookCli) { - needUpdate = false - break - } - } - if needUpdate { - spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{ - Path: hookCliPath, - Args: []string{hookCliPath}, - }) - } - - if len(spec.Process.Env) > maxCommandLength { - return fmt.Errorf("too many items in Env ") - } - - if strings.Contains(getValueByKey(spec.Process.Env, ascendRuntimeOptions), "VIRTUAL") { - return nil - } - - vdevice, err := dcmi.CreateVDevice(&dcmi.NpuWorker{}, spec, *deviceIdList) - if err != nil { - return err - } - hwlog.RunLog.Infof("vnpu split done: vdevice: %v", vdevice.VdeviceID) - - if vdevice.VdeviceID != -1 { - updateEnvAndPostHook(spec, vdevice, deviceIdList) - } - - return nil -} - -func removeDuplication(devices []int) []int { - list := make([]int, 0, len(devices)) - prev := -1 - - for _, device := range devices { - if device == prev { - continue - } - - list = append(list, device) - prev = device - } - - return list -} - -func parseDevices(visibleDevices string) ([]int, error) { - devices := make([]int, 0) - const maxDevice = 128 - - for _, d := range strings.Split(visibleDevices, ",") { - d = strings.TrimSpace(d) - if strings.Contains(d, "-") { - borders := strings.Split(d, "-") - if len(borders) != borderNum { - return nil, fmt.Errorf("invalid device range: %s", d) - } - - borders[0] = strings.TrimSpace(borders[0]) - borders[1] = strings.TrimSpace(borders[1]) - - left, err := strconv.Atoi(borders[0]) - if err != nil || left < 0 { - return nil, fmt.Errorf("invalid left boarder range parameter: %s", borders[0]) - } - - right, err := strconv.Atoi(borders[1]) - if err != nil || right > maxDevice { - return nil, fmt.Errorf("invalid right boarder range parameter: %s", borders[1]) - } - - if left > right { - return nil, fmt.Errorf("left boarder (%d) should not be larger than the right one(%d)", left, right) - } - - for n := left; n <= right; n++ { - devices = append(devices, n) - } - } else { - n, err := strconv.Atoi(d) - if err != nil { - return nil, fmt.Errorf("invalid single device parameter: %s", d) - } - - devices = append(devices, n) - } - } - - sort.Slice(devices, func(i, j int) bool { return i < j }) - return removeDuplication(devices), nil -} - -func parseAscendDevices(visibleDevices string) ([]int, error) { - devicesList := strings.Split(visibleDevices, ",") - devices := make([]int, 0, len(devicesList)) - chipType := "" - - for _, d := range devicesList { - matchGroups := regexp.MustCompile(`^Ascend(910|310|310B|310P)-(\d+)$`).FindStringSubmatch(strings.TrimSpace(d)) - if matchGroups == nil { - return nil, fmt.Errorf("invalid device format: %s", d) - } - n, err := strconv.Atoi(matchGroups[2]) - if err != nil { - return nil, fmt.Errorf("invalid device id: %s", d) - } - - if chipType == "" { - chipType = matchGroups[1] - } - if chipType != "" && chipType != matchGroups[1] { - return nil, fmt.Errorf("invalid device chip type: %s", d) - } - - devices = append(devices, n) - - } - chipName, err := dcmi.GetChipName() - if err != nil { - return nil, fmt.Errorf("get chip name error: %v", err) - } - if ascend+chipType != GetDeviceTypeByChipName(chipName) { - return nil, fmt.Errorf("chip type not match really: %s", chipType) - } - - sort.Slice(devices, func(i, j int) bool { return i < j }) - return removeDuplication(devices), nil -} - -func getValueByKey(data []string, name string) string { - for _, envLine := range data { - words := strings.SplitN(envLine, "=", kvPairSize) - if len(words) != kvPairSize { - hwlog.RunLog.Error("environment error") - return "" - } - - if words[0] == name { - return words[1] - } - } - - return "" -} - -func getValueByDeviceKey(data []string) string { - res := "" - for i := len(data) - 1; i >= 0; i-- { - words := strings.SplitN(data[i], "=", kvPairSize) - if len(words) != kvPairSize { - hwlog.RunLog.Error("environment error") - return "" - } - - if words[0] == ascendVisibleDevices { - res = words[1] - break - } - } - if res == "" { - hwlog.RunLog.Error("ASCEND_VISIBLE_DEVICES env variable is empty, will not mount any ascend device") - } - return res -} - -func addDeviceToSpec(spec *specs.Spec, dPath string, deviceType string) error { - device, err := oci.DeviceFromPath(dPath) - if err != nil { - return fmt.Errorf("failed to get %s info : %#v", dPath, err) - } - - switch deviceType { - case virtualDavinciName: - vDeviceNumber := regexp.MustCompile("[0-9]+").FindAllString(dPath, -1) - if len(vDeviceNumber) != 1 { - return fmt.Errorf("invalid vdavinci path: %s", dPath) - } - device.Path = devicePath + davinciName + vDeviceNumber[0] - case davinciManagerDocker: - device.Path = devicePath + davinciManager - default: // do nothing - - } - - spec.Linux.Devices = append(spec.Linux.Devices, *device) - newDeviceCgroup := specs.LinuxDeviceCgroup{ - Allow: true, - Type: device.Type, - Major: &device.Major, - Minor: &device.Minor, - Access: "rwm", - } - spec.Linux.Resources.Devices = append(spec.Linux.Resources.Devices, newDeviceCgroup) - return nil -} - -func addAscend310BManagerDevice(spec *specs.Spec) error { - var Ascend310BManageDevices = []string{ - svm0, - tsAisle, - upgrade, - sys, - vdec, - vpc, - pngd, - venc, - dvppCmdList, - logDrv, - acodec, - ai, - ao, - vo, - hdmi, - } - - for _, device := range Ascend310BManageDevices { - dPath := devicePath + device - if err := addDeviceToSpec(spec, dPath, notRenameDeviceType); err != nil { - hwlog.RunLog.Warnf("failed to add %s to spec : %#v", dPath, err) - } - } - - davinciManagerPath := devicePath + davinciManagerDocker - if _, err := os.Stat(davinciManagerPath); err != nil { - hwlog.RunLog.Warnf("failed to get davinci manager docker, err: %#v", err) - davinciManagerPath = devicePath + davinciManager - if _, err := os.Stat(davinciManagerPath); err != nil { - return fmt.Errorf("failed to get davinci manager, err: %#v", err) - } - } - return addDeviceToSpec(spec, davinciManagerPath, davinciManagerDocker) -} - -func addCommonManagerDevice(spec *specs.Spec) error { - var commonManagerDevices = []string{ - devmmSvm, - hisiHdc, - } - - for _, device := range commonManagerDevices { - dPath := devicePath + device - if err := addDeviceToSpec(spec, dPath, notRenameDeviceType); err != nil { - return fmt.Errorf("failed to add common manage device to spec : %#v", err) - } - } - - return nil -} - -func addManagerDevice(spec *specs.Spec) error { - chipName, err := dcmi.GetChipName() - if err != nil { - return fmt.Errorf("get chip name error: %#v", err) - } - devType := GetDeviceTypeByChipName(chipName) - hwlog.RunLog.Infof("device type is: %s", devType) - if devType == Ascend310B { - return addAscend310BManagerDevice(spec) - } - - if err := addDeviceToSpec(spec, devicePath+davinciManager, notRenameDeviceType); err != nil { - return fmt.Errorf("add davinci_manager to spec error: %#v", err) - } - - productType, err := dcmi.GetProductType(&dcmi.NpuWorker{}) - if err != nil { - return fmt.Errorf("parse product type error: %#v", err) - } - hwlog.RunLog.Infof("product type is %s", productType) - - switch productType { - // do nothing - case Atlas200ISoc, Atlas200: - default: - if err = addCommonManagerDevice(spec); err != nil { - return fmt.Errorf("add common manage device error: %#v", err) - } - } - - return nil -} - -func checkVisibleDevice(spec *specs.Spec) ([]int, error) { - visibleDevices := getValueByDeviceKey(spec.Process.Env) - if visibleDevices == "" || visibleDevices == void { - return nil, nil - } - - if strings.Contains(visibleDevices, ascend) { - devices, err := parseAscendDevices(visibleDevices) - if err != nil { - return nil, fmt.Errorf("failed to parse ascend device : %v", err) - } - hwlog.RunLog.Infof("ascend devices is: %v", devices) - return devices, err - } - devices, err := parseDevices(visibleDevices) - if err != nil { - return nil, fmt.Errorf("failed to parse device : %v", err) - } - hwlog.RunLog.Infof("devices is: %v", devices) - return devices, err -} - -func addDevice(spec *specs.Spec, deviceIdList []int) error { - deviceName := davinciName - if strings.Contains(getValueByKey(spec.Process.Env, ascendRuntimeOptions), "VIRTUAL") { - deviceName = virtualDavinciName - } - for _, deviceId := range deviceIdList { - dPath := devicePath + deviceName + strconv.Itoa(deviceId) - if err := addDeviceToSpec(spec, dPath, deviceName); err != nil { - return fmt.Errorf("failed to add davinci device to spec: %v", err) - } - } - - if err := addManagerDevice(spec); err != nil { - return fmt.Errorf("failed to add Manager device to spec: %v", err) - } - - return nil -} - -func updateEnvAndPostHook(spec *specs.Spec, vdevice dcmi.VDeviceInfo, deviceIdList *[]int) { - if deviceIdList == nil { - return - } - newEnv := make([]string, 0, len(spec.Process.Env)+1) - needAddVirtualFlag := true - *deviceIdList = []int{int(vdevice.VdeviceID)} - for _, line := range spec.Process.Env { - words := strings.Split(line, "=") - if len(words) == envLength && strings.TrimSpace(words[0]) == ascendRuntimeOptions { - needAddVirtualFlag = false - if strings.Contains(words[1], "VIRTUAL") { - newEnv = append(newEnv, line) - continue - } else { - newEnv = append(newEnv, strings.TrimSpace(line)+",VIRTUAL") - continue - } - } - newEnv = append(newEnv, line) - } - if needAddVirtualFlag { - newEnv = append(newEnv, fmt.Sprintf("ASCEND_RUNTIME_OPTIONS=VIRTUAL")) - } - spec.Process.Env = newEnv - if currentExecPath, err := os.Executable(); err == nil { - postHookCliPath := path.Join(path.Dir(currentExecPath), destroyHookCli) - spec.Hooks.Poststop = append(spec.Hooks.Poststop, specs.Hook{ - Path: postHookCliPath, - Args: []string{postHookCliPath, fmt.Sprintf("%d", vdevice.CardID), fmt.Sprintf("%d", vdevice.DeviceID), - fmt.Sprintf("%d", vdevice.VdeviceID)}, - }) - } -} - -func modifySpecFile(path string) error { - stat, err := os.Stat(path) - if err != nil { - return fmt.Errorf("spec file doesnt exist %s: %v", path, err) - } - if _, err = mindxcheckutils.RealFileChecker(path, true, true, mindxcheckutils.DefaultSize); err != nil { - return err - } - - jsonFile, err := os.OpenFile(path, os.O_RDWR, stat.Mode()) - if err != nil { - return fmt.Errorf("cannot open oci spec file %s: %v", path, err) - } - - defer jsonFile.Close() - - jsonContent, err := ioutil.ReadAll(jsonFile) - if err != nil { - return fmt.Errorf("failed to read oci spec file %s: %v", path, err) - } - - if err = jsonFile.Truncate(0); err != nil { - return fmt.Errorf("failed to truncate: %v", err) - } - if _, err = jsonFile.Seek(0, 0); err != nil { - return fmt.Errorf("failed to seek: %v", err) - } - - var spec specs.Spec - if err = json.Unmarshal(jsonContent, &spec); err != nil { - return fmt.Errorf("failed to unmarshal oci spec file %s: %v", path, err) - } - - devices, err := checkVisibleDevice(&spec) - if err != nil { - hwlog.RunLog.Errorf("failed to check ASCEND_VISIBLE_DEVICES parameter, err: %v", err) - return fmt.Errorf("failed to check ASCEND_VISIBLE_DEVICES parameter, err: %v", err) - } - if len(devices) != 0 { - if err = addHook(&spec, &devices); err != nil { - hwlog.RunLog.Errorf("failed to inject hook, err: %v", err) - return fmt.Errorf("failed to inject hook, err: %v", err) - } - if err = addDevice(&spec, devices); err != nil { - return fmt.Errorf("failed to add device to env: %v", err) - } - } - - addAscendDockerEnv(&spec) - - jsonOutput, err := json.Marshal(spec) - if err != nil { - return fmt.Errorf("failed to marshal OCI spec file: %v", err) - } - - if _, err = jsonFile.WriteAt(jsonOutput, 0); err != nil { - return fmt.Errorf("failed to write OCI spec file: %v", err) - } - - return nil -} - -func doProcess() error { - args, err := getArgs() - if err != nil { - return fmt.Errorf("failed to get args: %v", err) - } - - if args.cmd != "create" { - return execRunc() - } - - if args.bundleDirPath == "" { - args.bundleDirPath, err = os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current working dir: %v", err) - } - } - - specFilePath := args.bundleDirPath + "/config.json" - - if err = modifySpecFile(specFilePath); err != nil { - return fmt.Errorf("failed to modify spec file %s: %v", specFilePath, err) - } - - return execRunc() -} - func main() { defer func() { if err := recover(); err != nil { @@ -682,7 +35,7 @@ func main() { } }() ctx, _ := context.WithCancel(context.Background()) - if err := initLogModule(ctx); err != nil { + if err := process.InitLogModule(ctx); err != nil { log.Fatal(err) } logPrefixWords, err := mindxcheckutils.GetLogPrefix() @@ -695,11 +48,11 @@ func main() { } }() if !mindxcheckutils.StringChecker(strings.Join(os.Args, " "), 0, - maxCommandLength, mindxcheckutils.DefaultWhiteList+" ") { + process.MaxCommandLength, mindxcheckutils.DefaultWhiteList+" ") { hwlog.RunLog.Errorf("%v ascend docker runtime args check failed", logPrefixWords) log.Fatal("command error") } - if err = doProcess(); err != nil { + if err = process.DoProcess(); err != nil { hwlog.RunLog.Errorf("%v docker runtime failed: %v", logPrefixWords, err) log.Fatal(err) } diff --git a/runtime/process/process.go b/runtime/process/process.go new file mode 100644 index 0000000000000000000000000000000000000000..d38d7656c240c47bc567963bfa34fd9873572f26 --- /dev/null +++ b/runtime/process/process.go @@ -0,0 +1,688 @@ +/* Copyright(C) 2022. Huawei Technologies Co.,Ltd. All rights reserved. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package process does what ascend-docker-runtime is supposed to do before runc being executed. +package process + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "syscall" + + "github.com/containerd/containerd/oci" + "github.com/opencontainers/runtime-spec/specs-go" + "huawei.com/npu-exporter/v5/common-utils/hwlog" + + "ascend-docker-runtime/mindxcheckutils" + "ascend-docker-runtime/runtime/dcmi" +) + +const ( + runLogPath = "/var/log/ascend-docker-runtime/runtime-run.log" + hookDefaultFilePath = "/usr/local/bin/ascend-docker-hook" + // MaxCommandLength is the max length of command. + MaxCommandLength = 65535 + hookCli = "ascend-docker-hook" + destroyHookCli = "ascend-docker-destroy" + dockerRuncFile = "docker-runc" + runcFile = "runc" + envLength = 2 + kvPairSize = 2 + borderNum = 2 + + // ENV for device-plugin to identify ascend-docker-runtime + useAscendDocker = "ASCEND_DOCKER_RUNTIME=True" + devicePlugin = "ascend-device-plugin" + ascendVisibleDevices = "ASCEND_VISIBLE_DEVICES" + ascendRuntimeOptions = "ASCEND_RUNTIME_OPTIONS" + + // void indicates that the NPU card does not need to be mounted + void = "void" +) + +var ( + hookCliPath = hookCli + hookDefaultFile = hookDefaultFilePath + dockerRuncName = dockerRuncFile + runcName = runcFile +) + +const ( + // Atlas200ISoc Product name + Atlas200ISoc = "Atlas 200I SoC A1" + // Atlas200 Product name + Atlas200 = "Atlas 200 Model 3000" + // Ascend310 ascend 310 chip + Ascend310 = "Ascend310" + // Ascend310P ascend 310P chip + Ascend310P = "Ascend310P" + // Ascend310B ascend 310B chip + Ascend310B = "Ascend310B" + // Ascend910 ascend 910 chip + Ascend910 = "Ascend910" + ascend = "Ascend" + + devicePath = "/dev/" + davinciName = "davinci" + virtualDavinciName = "vdavinci" + davinciManager = "davinci_manager" + davinciManagerDocker = "davinci_manager_docker" + notRenameDeviceType = "" + devmmSvm = "devmm_svm" + hisiHdc = "hisi_hdc" + svm0 = "svm0" + tsAisle = "ts_aisle" + upgrade = "upgrade" + sys = "sys" + vdec = "vdec" + vpc = "vpc" + pngd = "pngd" + venc = "venc" + dvppCmdList = "dvpp_cmdlist" + logDrv = "log_drv" + acodec = "acodec" + ai = "ai" + ao = "ao" + vo = "vo" + hdmi = "hdmi" +) + +type args struct { + bundleDirPath string + cmd string +} + +// GetDeviceTypeByChipName get device type by chipName +func GetDeviceTypeByChipName(chipName string) string { + if strings.Contains(chipName, "310B") { + return Ascend310B + } + if strings.Contains(chipName, "310P") { + return Ascend310P + } + if strings.Contains(chipName, "310") { + return Ascend310 + } + if strings.Contains(chipName, "910") { + return Ascend910 + } + return "" +} + +func getArgs() (*args, error) { + args := &args{} + + for i, param := range os.Args { + if param == "--bundle" || param == "-b" { + if len(os.Args)-i <= 1 { + return nil, fmt.Errorf("bundle option needs an argument") + } + args.bundleDirPath = os.Args[i+1] + } else if param == "create" { + args.cmd = param + } + } + + return args, nil +} + +// InitLogModule initializes some logging configuration. +func InitLogModule(ctx context.Context) error { + const backups = 2 + const logMaxAge = 365 + const fileMaxSize = 2 + runLogConfig := hwlog.LogConfig{ + LogFileName: runLogPath, + LogLevel: 0, + MaxBackups: backups, + MaxAge: logMaxAge, + OnlyToFile: true, + FileMaxSize: fileMaxSize, + } + if err := hwlog.InitRunLogger(&runLogConfig, ctx); err != nil { + fmt.Printf("hwlog init failed, error is %v", err) + return err + } + return nil +} + +var execRunc = func() error { + tempRuncPath, err := exec.LookPath(dockerRuncName) + if err != nil { + tempRuncPath, err = exec.LookPath(runcName) + if err != nil { + return fmt.Errorf("failed to find the path of runc: %v", err) + } + } + runcPath, err := filepath.EvalSymlinks(tempRuncPath) + if err != nil { + return fmt.Errorf("failed to find realpath of runc %v", err) + } + if _, err := mindxcheckutils.RealFileChecker(runcPath, true, false, mindxcheckutils.DefaultSize); err != nil { + return err + } + + if err := mindxcheckutils.ChangeRuntimeLogMode("runtime-run-"); err != nil { + return err + } + if err = syscall.Exec(runcPath, append([]string{runcPath}, os.Args[1:]...), os.Environ()); err != nil { + return fmt.Errorf("failed to exec runc: %v", err) + } + + return nil +} + +func addAscendDockerEnv(spec *specs.Spec) { + if spec == nil || spec.Process == nil || spec.Process.Env == nil { + return + } + spec.Process.Env = append(spec.Process.Env, useAscendDocker) +} + +func addHook(spec *specs.Spec, deviceIdList *[]int) error { + if deviceIdList == nil { + return nil + } + currentExecPath, err := os.Executable() + if err != nil { + return fmt.Errorf("cannot get the path of ascend-docker-runtime: %v", err) + } + + hookCliPath = path.Join(path.Dir(currentExecPath), hookCli) + if _, err := mindxcheckutils.RealFileChecker(hookCliPath, true, false, mindxcheckutils.DefaultSize); err != nil { + return err + } + if _, err = os.Stat(hookCliPath); err != nil { + return fmt.Errorf("cannot find ascend-docker-hook executable file at %s: %v", hookCliPath, err) + } + + if spec.Hooks == nil { + spec.Hooks = &specs.Hooks{} + } + + needUpdate := true + if len(spec.Hooks.Prestart) > MaxCommandLength { + return fmt.Errorf("too many items in Prestart ") + } + for _, hook := range spec.Hooks.Prestart { + if strings.Contains(hook.Path, hookCli) { + needUpdate = false + break + } + } + if needUpdate { + spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{ + Path: hookCliPath, + Args: []string{hookCliPath}, + }) + } + + if len(spec.Process.Env) > MaxCommandLength { + return fmt.Errorf("too many items in Env ") + } + + if strings.Contains(getValueByKey(spec.Process.Env, ascendRuntimeOptions), "VIRTUAL") { + return nil + } + + vdevice, err := dcmi.CreateVDevice(&dcmi.NpuWorker{}, spec, *deviceIdList) + if err != nil { + return err + } + hwlog.RunLog.Infof("vnpu split done: vdevice: %v", vdevice.VdeviceID) + + if vdevice.VdeviceID != -1 { + updateEnvAndPostHook(spec, vdevice, deviceIdList) + } + + return nil +} + +func removeDuplication(devices []int) []int { + list := make([]int, 0, len(devices)) + prev := -1 + + for _, device := range devices { + if device == prev { + continue + } + + list = append(list, device) + prev = device + } + + return list +} + +func parseDevices(visibleDevices string) ([]int, error) { + devices := make([]int, 0) + + for _, value := range strings.Split(visibleDevices, ",") { + deviceFromValue, err := getDeviceListFromVisibleValue(value) + if err != nil { + hwlog.RunLog.Errorf("failed to get devices from ASCEND_VISIBLE_DEVICE value, error: %v", err) + return nil, err + } + devices = append(devices, deviceFromValue...) + } + + sort.Slice(devices, func(i, j int) bool { return i < j }) + return removeDuplication(devices), nil +} + +func getDeviceListFromVisibleValue(visibleValue string) ([]int, error) { + maxDevice := 128 + devices := make([]int, 0) + visibleValue = strings.TrimSpace(visibleValue) + if strings.Contains(visibleValue, "-") { + borders := strings.Split(visibleValue, "-") + if len(borders) != borderNum { + return nil, fmt.Errorf("invalid device range: %s", visibleValue) + } + + borders[0] = strings.TrimSpace(borders[0]) + borders[1] = strings.TrimSpace(borders[1]) + + left, err := strconv.Atoi(borders[0]) + if err != nil || left < 0 { + return nil, fmt.Errorf("invalid left boarder range parameter: %s", borders[0]) + } + + right, err := strconv.Atoi(borders[1]) + if err != nil || right > maxDevice { + return nil, fmt.Errorf("invalid right boarder range parameter: %s", borders[1]) + } + + if left > right { + return nil, fmt.Errorf("left boarder (%d) should not be larger than the right one(%d)", left, right) + } + + for n := left; n <= right; n++ { + devices = append(devices, n) + } + return devices, nil + } + n, err := strconv.Atoi(visibleValue) + if err != nil { + return nil, fmt.Errorf("invalid single device parameter: %s", visibleValue) + } + devices = append(devices, n) + + return devices, nil +} + +func parseAscendDevices(visibleDevices string) ([]int, error) { + devicesList := strings.Split(visibleDevices, ",") + devices := make([]int, 0, len(devicesList)) + chipType := "" + + for _, d := range devicesList { + matchGroups := regexp.MustCompile(`^Ascend(910|310|310B|310P)-(\d+)$`).FindStringSubmatch(strings.TrimSpace(d)) + if matchGroups == nil { + return nil, fmt.Errorf("invalid device format: %s", d) + } + n, err := strconv.Atoi(matchGroups[2]) + if err != nil { + return nil, fmt.Errorf("invalid device id: %s", d) + } + + if chipType == "" { + chipType = matchGroups[1] + } + if chipType != "" && chipType != matchGroups[1] { + return nil, fmt.Errorf("invalid device chip type: %s", d) + } + + devices = append(devices, n) + + } + chipName, err := dcmi.GetChipName() + if err != nil { + return nil, fmt.Errorf("get chip name error: %v", err) + } + if ascend+chipType != GetDeviceTypeByChipName(chipName) { + return nil, fmt.Errorf("chip type not match really: %s", chipType) + } + + sort.Slice(devices, func(i, j int) bool { return i < j }) + return removeDuplication(devices), nil +} + +func getValueByKey(data []string, name string) string { + for _, envLine := range data { + words := strings.SplitN(envLine, "=", kvPairSize) + if len(words) != kvPairSize { + hwlog.RunLog.Error("environment error") + return "" + } + + if words[0] == name { + return words[1] + } + } + + return "" +} + +func getValueByDeviceKey(data []string) string { + res := "" + for i := len(data) - 1; i >= 0; i-- { + words := strings.SplitN(data[i], "=", kvPairSize) + if len(words) != kvPairSize { + hwlog.RunLog.Error("environment error") + return "" + } + + if words[0] == ascendVisibleDevices { + res = words[1] + break + } + } + if res == "" { + hwlog.RunLog.Error("ASCEND_VISIBLE_DEVICES env variable is empty, will not mount any ascend device") + } + return res +} + +func addDeviceToSpec(spec *specs.Spec, dPath string, deviceType string) error { + device, err := oci.DeviceFromPath(dPath) + if err != nil { + return fmt.Errorf("failed to get %s info : %#v", dPath, err) + } + + switch deviceType { + case virtualDavinciName: + vDeviceNumber := regexp.MustCompile("[0-9]+").FindAllString(dPath, -1) + if len(vDeviceNumber) != 1 { + return fmt.Errorf("invalid vdavinci path: %s", dPath) + } + device.Path = devicePath + davinciName + vDeviceNumber[0] + case davinciManagerDocker: + device.Path = devicePath + davinciManager + default: // do nothing + + } + + spec.Linux.Devices = append(spec.Linux.Devices, *device) + newDeviceCgroup := specs.LinuxDeviceCgroup{ + Allow: true, + Type: device.Type, + Major: &device.Major, + Minor: &device.Minor, + Access: "rwm", + } + spec.Linux.Resources.Devices = append(spec.Linux.Resources.Devices, newDeviceCgroup) + return nil +} + +func addAscend310BManagerDevice(spec *specs.Spec) error { + var Ascend310BManageDevices = []string{ + svm0, + tsAisle, + upgrade, + sys, + vdec, + vpc, + pngd, + venc, + dvppCmdList, + logDrv, + acodec, + ai, + ao, + vo, + hdmi, + } + + for _, device := range Ascend310BManageDevices { + dPath := devicePath + device + if err := addDeviceToSpec(spec, dPath, notRenameDeviceType); err != nil { + hwlog.RunLog.Warnf("failed to add %s to spec : %#v", dPath, err) + } + } + + davinciManagerPath := devicePath + davinciManagerDocker + if _, err := os.Stat(davinciManagerPath); err != nil { + hwlog.RunLog.Warnf("failed to get davinci manager docker, err: %#v", err) + davinciManagerPath = devicePath + davinciManager + if _, err := os.Stat(davinciManagerPath); err != nil { + return fmt.Errorf("failed to get davinci manager, err: %#v", err) + } + } + return addDeviceToSpec(spec, davinciManagerPath, davinciManagerDocker) +} + +func addCommonManagerDevice(spec *specs.Spec) error { + var commonManagerDevices = []string{ + devmmSvm, + hisiHdc, + } + + for _, device := range commonManagerDevices { + dPath := devicePath + device + if err := addDeviceToSpec(spec, dPath, notRenameDeviceType); err != nil { + return fmt.Errorf("failed to add common manage device to spec : %#v", err) + } + } + + return nil +} + +func addManagerDevice(spec *specs.Spec) error { + chipName, err := dcmi.GetChipName() + if err != nil { + return fmt.Errorf("get chip name error: %#v", err) + } + devType := GetDeviceTypeByChipName(chipName) + hwlog.RunLog.Infof("device type is: %s", devType) + if devType == Ascend310B { + return addAscend310BManagerDevice(spec) + } + + if err := addDeviceToSpec(spec, devicePath+davinciManager, notRenameDeviceType); err != nil { + return fmt.Errorf("add davinci_manager to spec error: %#v", err) + } + + productType, err := dcmi.GetProductType(&dcmi.NpuWorker{}) + if err != nil { + return fmt.Errorf("parse product type error: %#v", err) + } + hwlog.RunLog.Infof("product type is %s", productType) + + switch productType { + // do nothing + case Atlas200ISoc, Atlas200: + default: + if err = addCommonManagerDevice(spec); err != nil { + return fmt.Errorf("add common manage device error: %#v", err) + } + } + + return nil +} + +func checkVisibleDevice(spec *specs.Spec) ([]int, error) { + visibleDevices := getValueByDeviceKey(spec.Process.Env) + if visibleDevices == "" || visibleDevices == void { + return nil, nil + } + + if strings.Contains(visibleDevices, ascend) { + devices, err := parseAscendDevices(visibleDevices) + if err != nil { + return nil, fmt.Errorf("failed to parse ascend device : %v", err) + } + hwlog.RunLog.Infof("ascend devices is: %v", devices) + return devices, err + } + devices, err := parseDevices(visibleDevices) + if err != nil { + return nil, fmt.Errorf("failed to parse device : %v", err) + } + hwlog.RunLog.Infof("devices is: %v", devices) + return devices, err +} + +func addDevice(spec *specs.Spec, deviceIdList []int) error { + deviceName := davinciName + if strings.Contains(getValueByKey(spec.Process.Env, ascendRuntimeOptions), "VIRTUAL") { + deviceName = virtualDavinciName + } + for _, deviceId := range deviceIdList { + dPath := devicePath + deviceName + strconv.Itoa(deviceId) + if err := addDeviceToSpec(spec, dPath, deviceName); err != nil { + return fmt.Errorf("failed to add davinci device to spec: %v", err) + } + } + + if err := addManagerDevice(spec); err != nil { + return fmt.Errorf("failed to add Manager device to spec: %v", err) + } + + return nil +} + +func updateEnvAndPostHook(spec *specs.Spec, vdevice dcmi.VDeviceInfo, deviceIdList *[]int) { + if deviceIdList == nil { + return + } + newEnv := make([]string, 0, len(spec.Process.Env)+1) + needAddVirtualFlag := true + *deviceIdList = []int{int(vdevice.VdeviceID)} + for _, line := range spec.Process.Env { + words := strings.Split(line, "=") + if len(words) == envLength && strings.TrimSpace(words[0]) == ascendRuntimeOptions { + needAddVirtualFlag = false + if strings.Contains(words[1], "VIRTUAL") { + newEnv = append(newEnv, line) + continue + } else { + newEnv = append(newEnv, strings.TrimSpace(line)+",VIRTUAL") + continue + } + } + newEnv = append(newEnv, line) + } + if needAddVirtualFlag { + newEnv = append(newEnv, fmt.Sprintf("ASCEND_RUNTIME_OPTIONS=VIRTUAL")) + } + spec.Process.Env = newEnv + if currentExecPath, err := os.Executable(); err == nil { + postHookCliPath := path.Join(path.Dir(currentExecPath), destroyHookCli) + spec.Hooks.Poststop = append(spec.Hooks.Poststop, specs.Hook{ + Path: postHookCliPath, + Args: []string{postHookCliPath, fmt.Sprintf("%d", vdevice.CardID), fmt.Sprintf("%d", vdevice.DeviceID), + fmt.Sprintf("%d", vdevice.VdeviceID)}, + }) + } +} + +func modifySpecFile(path string) error { + stat, err := os.Stat(path) + if err != nil { + return fmt.Errorf("spec file doesnt exist %s: %v", path, err) + } + if _, err = mindxcheckutils.RealFileChecker(path, true, true, mindxcheckutils.DefaultSize); err != nil { + return err + } + + jsonFile, err := os.OpenFile(path, os.O_RDWR, stat.Mode()) + if err != nil { + return fmt.Errorf("cannot open oci spec file %s: %v", path, err) + } + + defer jsonFile.Close() + + jsonContent, err := ioutil.ReadAll(jsonFile) + if err != nil { + return fmt.Errorf("failed to read oci spec file %s: %v", path, err) + } + + if err = jsonFile.Truncate(0); err != nil { + return fmt.Errorf("failed to truncate: %v", err) + } + if _, err = jsonFile.Seek(0, 0); err != nil { + return fmt.Errorf("failed to seek: %v", err) + } + + var spec specs.Spec + if err = json.Unmarshal(jsonContent, &spec); err != nil { + return fmt.Errorf("failed to unmarshal oci spec file %s: %v", path, err) + } + + devices, err := checkVisibleDevice(&spec) + if err != nil { + hwlog.RunLog.Errorf("failed to check ASCEND_VISIBLE_DEVICES parameter, err: %v", err) + return fmt.Errorf("failed to check ASCEND_VISIBLE_DEVICES parameter, err: %v", err) + } + if len(devices) != 0 { + if err = addHook(&spec, &devices); err != nil { + hwlog.RunLog.Errorf("failed to inject hook, err: %v", err) + return fmt.Errorf("failed to inject hook, err: %v", err) + } + if err = addDevice(&spec, devices); err != nil { + return fmt.Errorf("failed to add device to env: %v", err) + } + } + + addAscendDockerEnv(&spec) + + jsonOutput, err := json.Marshal(spec) + if err != nil { + return fmt.Errorf("failed to marshal OCI spec file: %v", err) + } + + if _, err = jsonFile.WriteAt(jsonOutput, 0); err != nil { + return fmt.Errorf("failed to write OCI spec file: %v", err) + } + + return nil +} + +// DoProcess does what ascend-docker-runtime is supposed to do before runc being executed. +func DoProcess() error { + args, err := getArgs() + if err != nil { + return fmt.Errorf("failed to get args: %v", err) + } + + if args.cmd != "create" { + return execRunc() + } + + if args.bundleDirPath == "" { + args.bundleDirPath, err = os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current working dir: %v", err) + } + } + + specFilePath := args.bundleDirPath + "/config.json" + + if err = modifySpecFile(specFilePath); err != nil { + return fmt.Errorf("failed to modify spec file %s: %v", specFilePath, err) + } + + return execRunc() +} diff --git a/runtime/main_test.go b/runtime/process/process_test.go similarity index 89% rename from runtime/main_test.go rename to runtime/process/process_test.go index 6721a4b7a2692ba0fd1f8e8a91eb94ca04c468da..a2d98a5b1a129d43fd5b309490e16e4fcf96ea7e 100644 --- a/runtime/main_test.go +++ b/runtime/process/process_test.go @@ -12,14 +12,14 @@ limitations under the License. */ -// Package main -package main +package process import ( "context" "fmt" "os" "reflect" + "strings" "testing" "github.com/agiledragon/gomonkey/v2" @@ -39,6 +39,10 @@ const ( fileMode0600 os.FileMode = 0600 fileMode0655 os.FileMode = 0655 needToMkdir = "./test" + fileExistErrorStr = "file exists" + bundleArgStr = "--bundle" + execStubLog = "execute stub" + configPath = "./test/config.json" ) var ( @@ -46,60 +50,52 @@ var ( ) func TestArgsIsCreate(t *testing.T) { - t.Log("进入测试用例") - - testArgs := []string{"create", "--bundle", "."} + testArgs := []string{"create", bundleArgStr, "."} stub := gomonkey.ApplyGlobalVar(&os.Args, testArgs) defer stub.Reset() stub.ApplyFunc(execRunc, func() error { - t.Log("execute stub") + t.Log(execStubLog) return nil }) - err := doProcess() + err := DoProcess() assert.NotNil(t, err) } func TestArgsIsCreateCase1(t *testing.T) { - t.Log("进入测试用例") - - testArgs := []string{"create", "--bundle"} + testArgs := []string{"create", bundleArgStr} stub := gomonkey.ApplyGlobalVar(&os.Args, testArgs) defer stub.Reset() stub.ApplyFunc(execRunc, func() error { - t.Log("execute stub") + t.Log(execStubLog) return nil }) - err := doProcess() + err := DoProcess() assert.NotNil(t, err) } func TestArgsIsCreateCase2(t *testing.T) { - t.Log("进入测试用例") - - testArgs := []string{"create", "--bundle", ""} + testArgs := []string{"create", bundleArgStr, ""} stub := gomonkey.ApplyGlobalVar(&os.Args, testArgs) defer stub.Reset() stub.ApplyFunc(execRunc, func() error { - t.Log("execute stub") + t.Log(execStubLog) return nil }) - err := doProcess() + err := DoProcess() assert.NotNil(t, err) } func TestArgsIsCreateCase3(t *testing.T) { - t.Log("进入测试用例") - - if err := os.Mkdir(needToMkdir, fileMode0655); err != nil { + if err := os.Mkdir(needToMkdir, fileMode0655); err != nil && !strings.Contains(err.Error(), fileExistErrorStr) { t.Fatalf("failed to create file, error: %v", err) } - f, err := os.Create("./test/config.json") + f, err := os.Create(configPath) defer f.Close() if err != nil { t.Logf("create file error: %v", err) @@ -108,26 +104,24 @@ func TestArgsIsCreateCase3(t *testing.T) { if err != nil { t.Logf("chmod file error: %v", err) } - testArgs := []string{"create", "--bundle", needToMkdir} + testArgs := []string{"create", bundleArgStr, needToMkdir} stub := gomonkey.ApplyGlobalVar(&os.Args, testArgs) defer stub.Reset() stub.ApplyFunc(execRunc, func() error { - t.Log("execute stub") + t.Log(execStubLog) return nil }) - err = doProcess() + err = DoProcess() assert.NotNil(t, err) } func TestArgsIsCreateCase4(t *testing.T) { - t.Log("进入测试用例") - - if err := os.Mkdir(needToMkdir, fileMode0655); err != nil { + if err := os.Mkdir(needToMkdir, fileMode0655); err != nil && !strings.Contains(err.Error(), fileExistErrorStr) { t.Fatalf("failed to create file, error: %v", err) } - f, err := os.Create("./test/config.json") + f, err := os.Create(configPath) defer f.Close() if err != nil { t.Logf("create file failed, error: %v", err) @@ -136,30 +130,34 @@ func TestArgsIsCreateCase4(t *testing.T) { if err != nil { t.Logf("chmod file failed, error: %v", err) } - testArgs := []string{"spec", "--bundle", needToMkdir} + testArgs := []string{"spec", bundleArgStr, needToMkdir} stub := gomonkey.ApplyGlobalVar(&os.Args, testArgs) defer stub.Reset() stub.ApplyFunc(execRunc, func() error { - t.Log("execute stub") + t.Log(execStubLog) return nil }) - err = doProcess() + err = DoProcess() assert.Nil(t, err) } func TestModifySpecFile(t *testing.T) { - err := modifySpecFile("./test/config.json") + err := modifySpecFile(configPath) assert.NotNil(t, err) } func TestModifySpecFileCase1(t *testing.T) { - if err := os.Mkdir(needToMkdir, fileMode0400); err != nil { + err := InitLogModule(context.Background()) + if err != nil { + t.Logf("init log failed, error: %v", err) + } + if err := os.Mkdir(needToMkdir, fileMode0400); err != nil && !strings.Contains(err.Error(), fileExistErrorStr) { t.Logf("mkdir error: %v", err) } - err := modifySpecFile(needToMkdir) + err = modifySpecFile(needToMkdir) assert.NotNil(t, err) if err := os.Remove(needToMkdir); err != nil { t.Logf("failed to remove dir, error: %v", err) @@ -181,7 +179,7 @@ func TestModifySpecFileCase2(t *testing.T) { t.Log("run modifySpecFile failed") } if err := os.Remove(file); err != nil { - + t.Logf("remove file(%v) failed, error: %v", file, err) } } @@ -405,7 +403,7 @@ func TestUpdateEnvAndPostHook(t *testing.T) { } updateEnvAndPostHook(&spec, vdvice, &deviceList) - assert.Contains(t, spec.Process.Env, "ASCEND_VISIBLE_DEVICES=100") + assert.Contains(t, spec.Process.Env, "ASCEND_VISIBLE_DEVICES=0") assert.Contains(t, spec.Process.Env, "ASCEND_RUNTIME_OPTIONS=VIRTUAL") assert.Contains(t, spec.Hooks.Poststop[0].Path, destroyHookCli) } @@ -504,7 +502,7 @@ func TestAddManagerDevice(t *testing.T) { }, } ctx, _ := context.WithCancel(context.Background()) - err := initLogModule(ctx) + err := InitLogModule(ctx) assert.Nil(t, err) err = addManagerDevice(&spec) assert.Nil(t, err) @@ -540,7 +538,7 @@ func TestAddDevice(t *testing.T) { } ctx, _ := context.WithCancel(context.Background()) - err := initLogModule(ctx) + err := InitLogModule(ctx) assert.Nil(t, err) err = addDevice(&spec, deviceList) assert.Nil(t, err)