From a8621d8b07ff764bed77433de5ee8934e7409d8d Mon Sep 17 00:00:00 2001 From: Damien Nguyen Date: Thu, 24 Nov 2022 15:32:42 +0100 Subject: [PATCH 1/8] [CI] Update minimum pybind11 version in some configuration files --- .pre-commit-config-gitee.yaml | 6 +++--- .pre-commit-config.yaml | 6 +++--- pyproject.toml | 2 +- scripts/build/python_virtualenv_update.bat | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config-gitee.yaml b/.pre-commit-config-gitee.yaml index 432ad3bad..fef4f91bc 100644 --- a/.pre-commit-config-gitee.yaml +++ b/.pre-commit-config-gitee.yaml @@ -143,8 +143,8 @@ repos: name: pylint-strict exclude: (test_.*\.py)$ args: [--score=n, --load-plugins=pylint_secure_coding_standard] - additional_dependencies: [pybind11>=2.6, wheel_filename, numpy, scipy, projectq, openfermion, sympy, matplotlib, - rich, pylint-secure-coding-standard] + additional_dependencies: [pybind11>=2.9.2, wheel_filename, numpy, scipy, projectq, openfermion, sympy, + matplotlib, rich, pylint-secure-coding-standard] - id: pylint name: pylint-test-files files: (test_.*\.py)$ @@ -156,7 +156,7 @@ repos: rev: '0.48' hooks: - id: check-manifest - additional_dependencies: ['pybind11>=2.6'] + additional_dependencies: ['pybind11>=2.9.2'] - repo: https://gitee.com/dnguyen/cmake-pre-commit-hooks-mirror rev: 'v1.5.3' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9b27c7930..a2473642a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -143,8 +143,8 @@ repos: name: pylint-strict exclude: (test_.*\.py)$ args: [--score=n, --load-plugins=pylint_secure_coding_standard] - additional_dependencies: [pybind11>=2.6, wheel_filename, numpy, scipy, projectq, openfermion, sympy, matplotlib, - rich, pylint-secure-coding-standard] + additional_dependencies: [pybind11>=2.9.2, wheel_filename, numpy, scipy, projectq, openfermion, sympy, + matplotlib, rich, pylint-secure-coding-standard] - id: pylint name: pylint-test-files files: (test_.*\.py)$ @@ -156,7 +156,7 @@ repos: rev: '0.48' hooks: - id: check-manifest - additional_dependencies: ['pybind11>=2.6'] + additional_dependencies: ['pybind11>=2.9.2'] - repo: https://github.com/Takishima/cmake-pre-commit-hooks/ rev: 'v1.5.3' diff --git a/pyproject.toml b/pyproject.toml index 2472612f9..77a5cb64e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = [ 'setuptools>=61,<=65.5.1;python_version<"3.12"', 'setuptools>=61;python_version>="3.12"', 'wheel', - 'pybind11>=2.9.0', + 'pybind11>=2.9.2', 'wheel-filename>1.2' ] build-backend = 'backend' diff --git a/scripts/build/python_virtualenv_update.bat b/scripts/build/python_virtualenv_update.bat index 1fb6677c4..acc73803c 100644 --- a/scripts/build/python_virtualenv_update.bat +++ b/scripts/build/python_virtualenv_update.bat @@ -40,7 +40,7 @@ goto :EOF :update_venv -set critical_pkgs=pip "setuptools>=61,<=65.5.1;python_version<'3.12'" "setuptools>=61;python_version>='3.12'" "wheel" "pybind11>=2.9.0" "wheel-filename>1.2" build +set critical_pkgs=pip "setuptools>=61,<=65.5.1;python_version<'3.12'" "setuptools>=61;python_version>='3.12'" "wheel" "pybind11>=2.9.2" "wheel-filename>1.2" build echo Updating critical Python packages: !PYTHON! -m pip install -U !critical_pkgs! call %BASEPATH%\dos\call_cmd.bat !PYTHON! -m pip install -U !critical_pkgs! -- Gitee From 3abfc6c6b3e3e9c5eabb9c377fdcf0a8e9c6afee Mon Sep 17 00:00:00 2001 From: Damien Nguyen Date: Thu, 24 Nov 2022 15:35:45 +0100 Subject: [PATCH 2/8] [PY] Add get_executable_in_path() function for Python build backend --- _build/backend.py | 13 ++++++++++--- _build/utils.py | 15 ++++++++++----- setup.py | 4 ++-- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/_build/backend.py b/_build/backend.py index 413cf837f..cf81cf5a1 100644 --- a/_build/backend.py +++ b/_build/backend.py @@ -25,7 +25,7 @@ import subprocess from pathlib import Path import setuptools.build_meta -from utils import fdopen, get_cmake_command +from utils import fdopen, get_executable_in_path from wheel_filename import parse_wheel_filename # ============================================================================== @@ -203,15 +203,22 @@ def get_requires_for_build_wheel(config_settings=None): """Identify packages required for building a wheel.""" requirements = setuptools.build_meta.get_requires_for_build_wheel(config_settings=config_settings) - if get_cmake_command() is None: - requirements.append('cmake') + executable_list = ['cmake'] + if int(os.environ.get('MQ_USE_NINJA', False)): + executable_list.append('ninja') delocate_wheel = int(os.environ.get('MQ_DELOCATE_WHEEL', False)) if delocate_wheel: if platform.system() == 'Linux': requirements.append('auditwheel') + executable_list.append('patchelf') elif platform.system() == 'Darwin': requirements.append('delocate') + + for exec_name in executable_list: + if get_executable_in_path(exec_name) is None: + requirements.append(exec_name) + return requirements diff --git a/_build/utils.py b/_build/utils.py index 2cbce9968..7d17191ff 100644 --- a/_build/utils.py +++ b/_build/utils.py @@ -138,10 +138,15 @@ def get_executable(exec_name): return None -def get_cmake_command(): - """Retrieve the path to the CMake executable.""" +def get_executable_in_path(exec_name): + """ + Retrieve the path to an executable. + + First lookup inside the PATH and then within a virtualenv. + """ try: - cmd = shutil.which('cmake') + logging.info('trying to locate %s inside the PATH', exec_name) + cmd = shutil.which(exec_name) if cmd is not None: with fdopen(os.devnull, 'w') as devnull: subprocess.check_call([cmd, '--version'], stdout=devnull, stderr=devnull) @@ -149,9 +154,9 @@ def get_cmake_command(): except (OSError, subprocess.CalledProcessError): pass - # CMake not in PATH, should have installed the Python CMake module + # Executable not in PATH, should have installed the Python CMake module # -> try to find out where it is - return get_executable('cmake') + return get_executable(exec_name) # ============================================================================== diff --git a/setup.py b/setup.py index 8333d85b5..d52734812 100644 --- a/setup.py +++ b/setup.py @@ -38,8 +38,8 @@ sys.path.append(str(Path(__file__).parent.resolve())) from _build.utils import ( # pylint: disable=wrong-import-position # noqa: E402 fdopen, - get_cmake_command, get_executable, + get_executable_in_path, modified_environ, parse_toml, remove_tree, @@ -165,7 +165,7 @@ class CMakeBuildExt(build_ext): # pylint: disable=too-many-instance-attributes logging.info('creating empty file at %s', dest_path) dest_path.write_text('', encoding='utf-8') return - cmake_cmd = get_cmake_command() + cmake_cmd = get_executable_in_path('cmake') if cmake_cmd is None: raise RuntimeError('Unable to locate the CMake command!') self.cmake_cmd = [cmake_cmd] -- Gitee From 79a6b32ec65cf91439b3bc415a74d609ee0f353c Mon Sep 17 00:00:00 2001 From: Damien Nguyen Date: Thu, 24 Nov 2022 15:56:48 +0100 Subject: [PATCH 3/8] [PY] Detect and proper handling of PEP517 build backend --- _build/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/_build/utils.py b/_build/utils.py index 7d17191ff..bc36e020b 100644 --- a/_build/utils.py +++ b/_build/utils.py @@ -99,6 +99,11 @@ def remove_tree(directory): def get_executable(exec_name): """Try to locate an executable in a Python virtual environment.""" try: + if os.environ.get('PEP517_BUILD_BACKEND', ''): + # We are being called from `python3 -m build` -> we actually need to ignore the VIRTUAL_ENV variable in + # order to get the proper path to the Python executable. + logging.info('Detected PEP517 build frontend') + raise KeyError('') root_path = os.environ['VIRTUAL_ENV'] python = os.path.basename(sys.executable) except KeyError: @@ -150,9 +155,11 @@ def get_executable_in_path(exec_name): if cmd is not None: with fdopen(os.devnull, 'w') as devnull: subprocess.check_call([cmd, '--version'], stdout=devnull, stderr=devnull) + logging.info(' -> found at %s', cmd) return cmd except (OSError, subprocess.CalledProcessError): pass + logging.info(' -> not found in PATH') # Executable not in PATH, should have installed the Python CMake module # -> try to find out where it is -- Gitee From 2c2491b599640e27a84486ae0c5c8ce374def1f0 Mon Sep 17 00:00:00 2001 From: Damien Nguyen Date: Fri, 25 Nov 2022 00:30:15 +0100 Subject: [PATCH 4/8] [CI][CMAKE][SCRIPT] Fix pybind11 compilation issues using CMake instead of GCC specs files The reason we abandon GCC specs files is that those only work with GCC and would not solve the issues we are having when using a different compiler. Additionally, we would actually need to generate the specs file within CMake as when building a wheel using build isolation (which is the default), we actually want the pybind11 installed inside the isolated environment to be used, which we cannot access while in the local build script (Bash, PowerShell or MS-DOS). Instead, we perform a quick compilation test after having added the third-party libraries and if the "normal" compilation fails, we simply make sure that the include path to pybind11 header appears first on the compilation command line. --- .cmake-format.yaml | 145 +++++++++++++++------------- CMakeLists.txt | 8 +- build.sh | 30 ------ cmake/macros_more.cmake | 9 ++ cmake/mindspore_ci.cmake | 141 +++++++++++++++++++++++++++ third_party/pybind11/pybind11.cmake | 2 + 6 files changed, 234 insertions(+), 101 deletions(-) create mode 100644 cmake/mindspore_ci.cmake diff --git a/.cmake-format.yaml b/.cmake-format.yaml index b97ac56ed..e9bef4014 100644 --- a/.cmake-format.yaml +++ b/.cmake-format.yaml @@ -25,6 +25,30 @@ format: require_valid_layout: false parse: additional_commands: + _apply_patch_file: + kwargs: + PATCH_ARGS: + + ERROR_VARIABLE: 1 + INPUT_FILE: 1 + OUTPUT_VARIABLE: 1 + RESULTS_VARIABLE: 1 + RESULT_VARIABLE: 1 + WORKING_DIRECTORY: 1 + __exec_cmd: + kwargs: + COMMAND: + + WORKING_DIRECTORY: 1 + __find_package: + pargs: + nargs: 1+ + kwargs: + SEARCH_NAME: 1 + define_compiler_linker_flags: + pargs: + nargs: 2 + kwargs: + COMPILER_FLAGS: 1 + LINKER_FLAGS: 1 file: pargs: flags: @@ -41,60 +65,11 @@ parse: - FLAGCHECK kwargs: LINKER_FLAGS: 1 - test_compile_option: - pargs: - nargs: 1+ - flags: - - FLAGCHECK - - NO_MQ_TARGET - - NO_TRYCOMPILE_FLAGCHECK_TARGET - - NO_TRYCOMPILE_TARGET - kwargs: - CMAKE_OPTION: 1 - FLAGS: + - GENEX: 1 - LANGS: + - LINKER_FLAGS: 1 check_link_flags: pargs: nargs: 2+ flags: - VERBATIM - test_linker_option: - pargs: - nargs: 1+ - flags: - - NO_MQ_TARGET - - VERBATIM - kwargs: - LANGS: + - FLAGS: + - CMAKE_OPTION: 1 - GENEX: 1 - _apply_patch_file: - kwargs: - PATCH_ARGS: + - ERROR_VARIABLE: 1 - INPUT_FILE: 1 - OUTPUT_VARIABLE: 1 - RESULTS_VARIABLE: 1 - RESULT_VARIABLE: 1 - WORKING_DIRECTORY: 1 - __find_package: - pargs: - nargs: 1+ - kwargs: - SEARCH_NAME: 1 - __exec_cmd: - kwargs: - COMMAND: + - WORKING_DIRECTORY: 1 - define_compiler_linker_flags: - pargs: - nargs: 2 - kwargs: - COMPILER_FLAGS: 1 - LINKER_FLAGS: 1 execute_process: pargs: flags: @@ -118,25 +93,6 @@ parse: RESULT_VARIABLE: 1 TIMEOUT: 1 WORKING_DIRECTORY: 1 - pybind11_add_module: - pargs: - nargs: 1+ - flags: - - MODULE - kwargs: - OUTPUT_HINT: 1 - mq_add_compile_definitions: - pargs: - nargs: + - flags: - - TRYCOMPILE - - TRYCOMPILE_FLAGCHECK - mq_link_libraries: - pargs: - nargs: + - flags: - - TRYCOMPILE - - TRYCOMPILE_FLAGCHECK mindquantum_add_pkg: pargs: nargs: 1 @@ -179,6 +135,59 @@ parse: PRE_CONFIGURE_COMMAND: + SYSTEM_EXTRA_DEFINES: + TARGET_ALIAS: + + mq_add_compile_definitions: + pargs: + nargs: + + flags: + - TRYCOMPILE + - TRYCOMPILE_FLAGCHECK + mq_link_libraries: + pargs: + nargs: + + flags: + - TRYCOMPILE + - TRYCOMPILE_FLAGCHECK + pybind11_add_module: + pargs: + nargs: 1+ + flags: + - MODULE + kwargs: + OUTPUT_HINT: 1 + test_compile_option: + pargs: + nargs: 1+ + flags: + - FLAGCHECK + - NO_MQ_TARGET + - NO_TRYCOMPILE_FLAGCHECK_TARGET + - NO_TRYCOMPILE_TARGET + kwargs: + CMAKE_OPTION: 1 + FLAGS: + + GENEX: 1 + LANGS: + + LINKER_FLAGS: 1 + test_linker_option: + pargs: + nargs: 1+ + flags: + - NO_MQ_TARGET + - VERBATIM + kwargs: + LANGS: + + FLAGS: + + CMAKE_OPTION: 1 + GENEX: 1 + try_compile_cxx_no_link: + pargs: + nargs: 2 + flags: + - QUIET + kwargs: + OUTPUT: 1 + INCLUDE_DIRECTORIES: + + LINK_LIBRARIES: + lint: argument_var_pattern: _?[a-z][a-z0-9_]+ local_var_pattern: _?([a-z][a-z0-9_]+|[A-Z][A-Z0-9_]+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 061c22317..bc8784125 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,14 +116,16 @@ if(ENABLE_CMAKE_DEBUG) list(POP_BACK CMAKE_MESSAGE_INDENT) endif() -# ============================================================================== -# Some more macro definitions - # ============================================================================== # First add third-party libraries add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/third_party) +# Include workarounds for mindspore CI if necessary +if(MINDSPORE_CI) + include(mindspore_ci) +endif() + # Then re-define some CMake macros/functions include(macros_more) diff --git a/build.sh b/build.sh index 34df4ef2d..c3c8f38d2 100755 --- a/build.sh +++ b/build.sh @@ -210,36 +210,6 @@ fi . "$ROOTDIR/scripts/build/python_virtualenv_update.sh" -# ------------------------------------------------------------------------------ - -if [ "$_IS_MINDSPORE_CI" -eq 1 ]; then - if [[ "$(uname)" == "Linux" ]]; then - echo '----------------------------------------' - echo 'Getting pybind11 include directory flag' - pybind11_include_flag=$("$PYTHON" -m pybind11 --includes | cut -d ' ' -f2) - echo " pybind11_include_flag = $pybind11_include_flag" - - - echo 'Generating GCC spec files' - gcc_specs="$ROOTDIR/gcc-gitee.specs" - - cc1_options="$(gcc -dumpspecs | sed -n '/*cc1_options:/,/^$/p' | grep -v 'cc1_options')" - - cat > "$gcc_specs" < +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== +# +# This file essentially contains some workarounds for building MindQuantum on the MindSpore CI which may have issues +# such as: +# - old system pybind11 version (2.5.0) +# +# ============================================================================== + +include(debug_print) + +# ============================================================================== + +# ~~~ +# Same as CMake try_compile but does not attempt to link the program +# +# try_compile_cxx_no_link( +# +# [QUIET] +# [OUTPUT ] +# [INCLUDE_DIRECTORIES [... ]] +# [LINK_LIBRARIES [... ]]) +# ~~~ +function(try_compile_cxx_no_link var source) + cmake_parse_arguments(PARSE_ARGV 1 TCCNL "QUIET" "OUTPUT" "INCLUDE_DIRECTORIES;LINK_LIBRARIES") + + set(_bindir "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp") + file(MAKE_DIRECTORY "${_bindir}") + set(_filename "${_bindir}/src.cpp") + file(WRITE "${_filename}" "${source}") + + if(NOT DEFINED ${var}) + if(NOT TCCNL_QUIET) + message(CHECK_START "Performing Test ${var}") + endif() + try_compile( + ${var} "${CMAKE_BINARY_DIR}" + "${_filename}" + OUTPUT_VARIABLE _output + LINK_LIBRARIES ${TCCNL_LINK_LIBRARIES} + CMAKE_FLAGS + "-DINCLUDE_DIRECTORIES:STRING=${TCCNL_INCLUDE_DIRECTORIES}" + "-DCMAKE_CXX_LINK_EXECUTABLE=${CMAKE_COMMAND} -E echo \"Not linking\"" CXX_STANDARD ${CMAKE_CXX_STANDARD}) + if(${var}) + if(NOT TCCNL_QUIET) + message(CHECK_PASS "Success") + endif() + set(${var} + 1 + CACHE INTERNAL "Test ${var}") + file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeOutput.log + "Performing C++ SOURCE FILE Test ${var} succeeded with the following output:\n" "${_output}\n" + "Source file was:\n${_source}\n") + else() + if(NOT TCCNL_QUIET) + message(CHECK_FAIL "Failed") + endif() + set(${var} + "" + CACHE INTERNAL "Test ${var}") + file(APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log + "Performing C++ SOURCE FILE Test ${var} failed with the following output:\n" "${_output}\n" + "Source file was:\n${_source}\n") + endif() + endif() + if(TCCNL_OUTPUT) + set(${TCCNL_OUTPUT} + "${_output}" + PARENT_SCOPE) + endif() +endfunction() + +# ============================================================================== +# Pybind11 workaround +# +# ~~~ +# Old pybind11 version installed in system. During compilation, we might have the system headers for python included +# *before* the ones from pybind11. +# +# For example: +# -isystem /usr/local/python/python375/include/python3.7m \ +# -isystem /.../venv/lib/python3.7/site-packages/pybind11/include +# +# The solution here is to first try to compile some code that does not work on pybind11 2.5.0 +# ~~~ +# ============================================================================== + +set(_pybind11_dependency_targets Python::Module pybind11::module) + +# First try to compile a simple pybind11 2.9.0+ code (pybind11::detail::const_name) using the same target list as would +# happen if we were to use pybind11_add_module() +if(NOT DEFINED compile_pybind11_normally_works) + try_compile_cxx_no_link( + compile_pybind11_normally_works + [[ +#include +int main() { + const auto name = pybind11::detail::const_name("Hello world").text; + return 0; +} +]] + LINK_LIBRARIES ${_pybind11_dependency_targets}) +endif() + +# If the above failed, we have a conflict between system pybind11 and the one located inside the virtual environment. +# Try prepending the pybind11::module target to link libraries. +if(NOT compile_pybind11_normally_works AND NOT DEFINED system_prepend_pybind11_include_works) + try_compile_cxx_no_link( + system_prepend_pybind11_include_works + [[ +#include +int main() { + const auto name = pybind11::detail::const_name("Hello world").text; + return 0; +} +]] + LINK_LIBRARIES pybind11::headers ${_pybind11_dependency_targets}) +endif() + +if(NOT compile_pybind11_normally_works AND system_prepend_pybind11_include_works) + set(_mq_pybind11_prepend_to_link_libraries + pybind11::headers + CACHE INTERNAL "") +elseif(NOT compile_pybind11_normally_works) + message(FATAL_ERROR "Unable to compile a recent pybind11 (2.9.0+) example and the workaround failed.") +endif() diff --git a/third_party/pybind11/pybind11.cmake b/third_party/pybind11/pybind11.cmake index 6c9c98f38..c0919d779 100644 --- a/third_party/pybind11/pybind11.cmake +++ b/third_party/pybind11/pybind11.cmake @@ -44,9 +44,11 @@ endif() # cmake-lint: disable=E1122 mindquantum_add_pkg( pybind11 + LIBS module headers pybind11 lto pybind11_headers python_headers VER ${VER} URL ${REQ_URL} MD5 ${MD5} + CMAKE_PKG_NO_COMPONENTS CMAKE_OPTION ${CMAKE_OPTION} LOCAL_EXTRA_DEFINES ${pybinbd11_LOCAL_EXTRA_DEFINES} SYSTEM_EXTRA_DEFINES ${pybinbd11_SYSTEM_EXTRA_DEFINES} -- Gitee From 37d04734f410ee007853f68dd93df5a2b6ea2201 Mon Sep 17 00:00:00 2001 From: Damien Nguyen Date: Fri, 25 Nov 2022 00:36:57 +0100 Subject: [PATCH 5/8] [SCRIPT][PY] Update Bash wheel building script - Add support for using MQ_USE_NINJA if ninja is selected - Fix cleaning of build/cache directories --- build.sh | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/build.sh b/build.sh index c3c8f38d2..1f3c12bf3 100755 --- a/build.sh +++ b/build.sh @@ -350,25 +350,40 @@ if [ "$build_isolation" -eq 0 ]; then fi # ------------------------------------------------------------------------------ -# Build the wheels +# Clean build directory if requested and possible +temp_build_dir=$("$PYTHON" -m mindquantum_config --tempdir) if [ "${_build_dir_was_set:-0}" -eq 1 ]; then + build_dir_for_cleaning="$build_dir" +elif [ -d "$temp_build_dir" ]; then + build_dir_for_cleaning="$temp_build_dir" +fi + +if [ -n "$build_dir_for_cleaning" ]; then if [ "$do_clean_build_dir" -eq 1 ]; then - echo "Deleting build folder: $build_dir" - call_cmd rm -rf "$build_dir" + echo "Deleting build folder: $build_dir_for_cleaning" + call_cmd rm -rf "$build_dir_for_cleaning" elif [ "$do_clean_cache" -eq 1 ]; then - echo "Removing CMake cache at: $build_dir/CMakeCache.txt" - call_cmd rm -f "$build_dir/CMakeCache.txt" - echo "Removing CMake files at: $build_dir/CMakeFiles" - call_cmd rm -rf "$build_dir/CMakeFiles" - echo "Removing CMake files at: $build_dir/cmake-ldtest*" - call_cmd rm -rf "$build_dir/cmake-ldtest*" + echo "Removing CMake cache at: $build_dir_for_cleaning/CMakeCache.txt" + call_cmd rm -f "$build_dir_for_cleaning/CMakeCache.txt" + echo "Removing CMake files at: $build_dir_for_cleaning/CMakeFiles" + call_cmd rm -rf "$build_dir_for_cleaning/CMakeFiles" + echo "Removing CMake files at: $build_dir_for_cleaning/cmake-ldtest*" + call_cmd rm -rf "$build_dir_for_cleaning/cmake-ldtest*" fi fi +# ------------------------------------------------------------------------------ +# Build the wheels + +env_vars=() +if [ "$cmake_generator" == "Ninja" ]; then + env_vars+=(MQ_USE_NINJA=1) +fi + if [ "$delocate_wheel" -eq 1 ]; then - env_vars=(MQ_DELOCATE_WHEEL=1 - "${LD_PATH_VAR}=${!LD_PATH_VAR}") + env_vars+=(MQ_DELOCATE_WHEEL=1 + "${LD_PATH_VAR}=${!LD_PATH_VAR}") if [[ "${_build_dir_was_set:-0}" -eq 1 || "${fast_build:-0}" -eq 1 ]]; then build_dir_for_env="$build_dir" @@ -382,11 +397,10 @@ if [ "$delocate_wheel" -eq 1 ]; then if [ -n "$platform_name" ]; then env_vars+=(MQ_DELOCATE_WHEEL_PLAT="$platform_name") fi - call_cmd env "${env_vars[@]}" "${PYTHON}" -m build "${args[@]}" "${fixed_args[@]}" "$@" -else - call_cmd "${PYTHON}" -m build "${args[@]}" "${fixed_args[@]}" "$@" fi +call_cmd env "${env_vars[@]}" "${PYTHON}" -m build "${args[@]}" "${fixed_args[@]}" "$@" + # ------------------------------------------------------------------------------ # Move the wheels to the output directory -- Gitee From c3a7e469b7b42d41aa77f8f0642e29fc41852f7b Mon Sep 17 00:00:00 2001 From: Damien Nguyen Date: Fri, 25 Nov 2022 00:47:21 +0100 Subject: [PATCH 6/8] [CMAKE][PY] Fix linter warnings --- _build/utils.py | 12 ++++++------ cmake/mindspore_ci.cmake | 2 ++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/_build/utils.py b/_build/utils.py index bc36e020b..5ba25aa58 100644 --- a/_build/utils.py +++ b/_build/utils.py @@ -243,8 +243,8 @@ except ImportError: start = lines.index('[build-system]') data = lines[start : _find_toml_section_end(lines, start)] idx = 0 - N = len(data) # noqa: N806 - while idx < N: + data_length = len(data) + while idx < data_length: line = data[idx] shift = 1 if line.startswith('requires'): @@ -257,8 +257,8 @@ except ImportError: start = lines.index('[project]') data = lines[start : _find_toml_section_end(lines, start)] idx = 0 - N = len(data) # noqa: N806 - while idx < N: + data_length = len(data) + while idx < data_length: line = data[idx] shift = 1 if _parse_string_value(result.setdefault('project', {}), 'name', line): @@ -275,8 +275,8 @@ except ImportError: start = lines.index('[project.optional-dependencies]') data = lines[start + 1 : _find_toml_section_end(lines, start)] idx = 0 - N = len(data) # noqa: N806 - while idx < N: + data_length = len(data) + while idx < data_length: (opt_name, opt_pkgs, shift) = _parse_list(data[idx:]) result.setdefault('project', {}).setdefault('optional-dependencies', {})[opt_name] = opt_pkgs idx += shift diff --git a/cmake/mindspore_ci.cmake b/cmake/mindspore_ci.cmake index 27d4e5adc..9dcc54f87 100644 --- a/cmake/mindspore_ci.cmake +++ b/cmake/mindspore_ci.cmake @@ -22,6 +22,8 @@ # # ============================================================================== +# lint_cmake: -whitespace/indent + include(debug_print) # ============================================================================== -- Gitee From 20b628170032cdbd08acde0e5f4086c5dd07be8d Mon Sep 17 00:00:00 2001 From: Damien Nguyen Date: Fri, 25 Nov 2022 01:34:19 +0100 Subject: [PATCH 7/8] [SCRIPT] Fix Windows wheel building scripts --- build.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.bat b/build.bat index 3f3524e14..bcaa4edd6 100644 --- a/build.bat +++ b/build.bat @@ -384,7 +384,7 @@ if !ninja! == 1 ( if NOT !n_jobs! == -1 ( set args=!args! -C--global-option=--var -C--global-option=JOBS -C--global-option=!n_jobs! - set args=!args! -C--global-option=build_ext -C--global-option=--jobs -C--global-option=!n_jobs! + set args=!args! -C--global-option=build_ext -C--global-option=--jobs=!n_jobs! ) if "!build_type!" == "Debug" set args=!args! -C--global-option=build -C--global-option=--debug -- Gitee From 21e58bb10f2f4da2c06f0df1fccc42a7ac6cb80e Mon Sep 17 00:00:00 2001 From: Damien Nguyen Date: Fri, 25 Nov 2022 01:47:14 +0100 Subject: [PATCH 8/8] [CI] Update list of pytest markers --- pyproject.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 77a5cb64e..9151014bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -169,8 +169,13 @@ addopts = "--ignore='tests/quick_test.py'" norecursedirs = ['third_party', 'mindquantum/ccsrc'] mock_use_standalone_module = true markers = [ - 'symengine: test using the mindquantum.symengine module', 'cxx_exp_projectq: tests involving the ProjectQ simulator', + 'env_onecard: test marker used on MindSpore CI for certain GPU runs', + 'level0: test marker used on MindSpore CI', + 'level1: test marker used on MindSpore CI', + 'platform_x86_cpu: test marker used on MindSpore CI for certain CPU runs', + 'platform_x86_gpu_training: test marker used on MindSpore CI for certain GPU runs', + 'symengine: test using the mindquantum.symengine module' ] filterwarnings = [ 'ignore:.*Please use `OptimizeResult`.*is deprecated:DeprecationWarning', -- Gitee