From 973b536bec1f577ac84c277b42ff8dc3bb89f2ee Mon Sep 17 00:00:00 2001 From: Xunhui Zhang <45024191+zhangxunhui@users.noreply.github.com> Date: Sat, 25 Mar 2023 11:44:37 +0800 Subject: [PATCH] init repository --- .gitignore | 113 +++++++++++++++++++++++++ .gitmodules | 36 ++++++++ CMakeLists.txt | 147 +++++++++++++++++++++++++++++++++ build.bat | 131 +++++++++++++++++++++++++++++ build.sh | 103 +++++++++++++++++++++++ setup.py | 219 +++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 749 insertions(+) create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 CMakeLists.txt create mode 100644 build.bat create mode 100644 build.sh create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e08ecf8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,113 @@ +# MindSpore +build/ +mindspore/lib +output +*.ir +st_tests +kernel_meta/ +somas_meta/ +trace_code_graph_* + +# mindspore lite java +mindspore/lite/java/.gradle/ +mindspore/lite/java/gradle +mindspore/lite/java/gradlew +mindspore/lite/java/gradlew.bat +mindspore/lite/java/java/.gradle +mindspore/lite/java/java/build +mindspore/lite/java/java/gradle +mindspore/lite/java/java/gradlew +mindspore/lite/java/java/gradlew.bat +mindspore/lite/java/java/app/build +mindspore/lite/java/java/app/libs + +# flatbuffer +mindspore/lite/tools/converter/parser/tflite/schema_generated.h +mindspore/lite/tools/converter/parser/caffe/caffe.pb.cc +mindspore/lite/tools/converter/parser/caffe/caffe.pb.h +mindspore/lite/tools/converter/parser/onnx/onnx.pb.h +mindspore/lite/tools/converter/parser/onnx/onnx.pb.h +mindspore/lite/tools/converter/schema/*.h +mindspore/lite/tools/converter/schema/inner +mindspore/lite/schema/*.h +mindspore/lite/schema/inner + +mindspore/lite/src/litert/kernel/opencl/cl/fp16/*.inc +mindspore/lite/src/litert/kernel/opencl/cl/fp32/*.inc + +# Cmake files +CMakeFiles/ +cmake_install.cmake +CMakeCache.txt +Makefile +cmake-build-debug + +# Dynamic libraries +*.so +*.so.* +*.dylib + +# Static libraries +*.la +*.lai +*.a +*.lib + +# Protocol buffers +*_pb2.py +*.pb.h +*.pb.cc +*.pb +*_grpc.py + +# Object files +*.o + +# Editor +.vscode +.idea/ + +# Cquery +.cquery_cached_index/ +compile_commands.json + +# Ctags and cscope +tags +TAGS +CTAGS +GTAGS +GRTAGS +GSYMS +GPATH +cscope.* + +# Python files +*__pycache__* +.pytest_cache + +# Mac files +*.DS_Store + +# Test results +test_temp_summary_event_file/ +*.dot +*.dat +*.svg +*.perf +*.info +*.ckpt +*.shp +*.pkl +*.pb +.clangd +mindspore/python/mindspore/version.py +mindspore/python/mindspore/default_config.py +mindspore/python/mindspore/device_target.py +mindspore/python/mindspore/package_name.py +mindspore/python/mindspore/.commit_id + +# lite test file +mindspore/lite/test/do_test/ + +# lite opencl compile file +*.cl.inc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..35de503 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,36 @@ +[submodule "akg"] + path = akg + url = https://gitee.com/mindspore/akg.git +[submodule "graphengine"] + path = graphengine + url = https://gitee.com/mindspore/graphengine.git +[submodule "tests/models"] + path = tests/models + url = https://gitee.com/mindspore/models.git +[submodule "MindSpore-first-experience"] + path = MindSpore-first-experience + url = ../MindSpore-first-experience.git +[submodule "MindSpore-install"] + path = MindSpore-install + url = ../MindSpore-install.git +[submodule "MindSpore-Application-practice"] + path = MindSpore-Application-practice + url = ../MindSpore-Application-practice.git +[submodule "MindSpore-Model-Development"] + path = MindSpore-Model-Development + url = ../MindSpore-Model-Development.git +[submodule "MindSpore-Data-preprocessing"] + path = MindSpore-Data-preprocessing + url = ../MindSpore-Data-preprocessing.git +[submodule "Mindspore-Data-storage-use"] + path = Mindspore-Data-storage-use + url = ../Mindspore-Data-storage-use.git +[submodule "MindSpore-Data-storage-kunpeng"] + path = MindSpore-Data-storage-kunpeng + url = ../MindSpore-Data-storage-kunpeng.git +[submodule "MindSpore-LeNet-jzx3"] + path = MindSpore-LeNet-jzx3 + url = ../MindSpore-LeNet-jzx3.git +[submodule "MindSpore-competition"] + path = MindSpore-competition + url = ../MindSpore-competition.git diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..2b44a83 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,147 @@ +cmake_minimum_required(VERSION 3.14.0) +project(MindSpore) + +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.3.0) + message(FATAL_ERROR "GCC version must be 7.3.0 and above, but found ${CMAKE_CXX_COMPILER_VERSION}") + elseif(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 11.3.0) + message(WARNING "GCC version ${CMAKE_CXX_COMPILER_VERSION} is greater than 11.3.0, may cause unknown problems.") + endif() +endif() + +include(${CMAKE_SOURCE_DIR}/cmake/options.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/check_requirements.cmake) +include(${CMAKE_SOURCE_DIR}/cmake/ascend_variables.cmake) + +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/modules/") +if(CMAKE_SYSTEM_NAME MATCHES "Linux") + if(NOT ENABLE_GLIBCXX) + add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0) + endif() +endif() + +if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(MACOSX_CXX_WARNING_FLAGS "-Wno-inconsistent-missing-override \ + -Wno-unused-lambda-capture -Wno-unneeded-internal-declaration -Wno-unused-variable \ + -Wno-return-std-move") + if("${CMAKE_CXX_COMPILER_ID}" MATCHES "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 13.1) + set(MACOSX_CXX_WARNING_FLAGS "${MACOSX_CXX_WARNING_FLAGS} -Wno-unused-but-set-variable") + endif() + set(CMAKE_CXX_FLAGS_RELEASE + "$ENV{CXXFLAGS} -O2 ${MACOSX_CXX_WARNING_FLAGS} -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") +elseif(ENABLE_SYM_FILE) + set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -g -ggdb -Wl,--allow-shlib-undefined \ + -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") +else() + if(NOT MSVC) + set(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O2 -Wl,--allow-shlib-undefined \ + -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2") + endif() +endif() + +if(ENABLE_PYTHON) + add_compile_definitions(ENABLE_PYTHON) +endif() + +if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer \ + -D_LIBCPP_INLINE_VISIBILITY='' -D_LIBCPP_DISABLE_EXTERN_TEMPLATE=1 -DHALF_ENABLE_CPP11_USER_LITERALS=0 \ + -D_FORTIFY_SOURCE=2 -Wno-cpp ${MACOSX_CXX_WARNING_FLAGS}") +else() + if(NOT MSVC) + set(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -g2 -ggdb -fno-inline-functions -fno-omit-frame-pointer \ + -Wl,--allow-shlib-undefined -D_LIBCPP_INLINE_VISIBILITY='' -D_LIBCPP_DISABLE_EXTERN_TEMPLATE=1 \ + -DHALF_ENABLE_CPP11_USER_LITERALS=0 -D_FORTIFY_SOURCE=2 -Wno-cpp") + endif() +endif() + +if(NOT MSVC) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -I/usr/local/include -std=c++17 \ + -Werror -Wall -Wno-deprecated-declarations -fPIC") +else() +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /std:c++17") +endif() +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +set(PYBIND11_CPP_STANDARD -std=c++17) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPTION_CXX_FLAGS}") + +if(ENABLE_AKG AND CMAKE_SYSTEM_NAME MATCHES "Linux") + add_subdirectory("${CMAKE_SOURCE_DIR}/akg") +endif() + +include(${CMAKE_SOURCE_DIR}/cmake/mind_expression.cmake) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/securec/include) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/flatbuffers/include) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/flatbuffers/include/flatbuffers) + +if(ENABLE_FAST_HASH_TABLE) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DENABLE_FAST_HASH_TABLE=1") +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/robin_hood/include) +endif() + +include(${CMAKE_SOURCE_DIR}/cmake/dependency_utils.cmake) +find_package(Python3 COMPONENTS Interpreter Development) +if(Python3_FOUND) + set(PYTHON_INCLUDE_DIRS "${Python3_INCLUDE_DIRS}") + set(PYTHON_LIBRARIES "${Python3_LIBRARIES}") + if(WIN32) + if(Python3_DIR) + message("Python3_DIR set already: " ${Python3_DIR}) + else() + string(LENGTH ${PYTHON_LIBRARIES} PYTHON_LIBRARIES_LEN) + string(LENGTH "libpythonxx.a" Python3_NAME_LEN) + math(EXPR Python3_DIR_LEN ${PYTHON_LIBRARIES_LEN}-${Python3_NAME_LEN}) + string(SUBSTRING ${Python3_LIBRARIES} 0 ${Python3_DIR_LEN} Python3_DIR) + message("Python3_DIR: " ${Python3_DIR}) + endif() + link_directories(${Python3_DIR}) + endif() +else() + find_python_package(py_inc py_lib) + set(PYTHON_INCLUDE_DIRS "${py_inc}") + set(PYTHON_LIBRARIES "${py_lib}") +endif() +message("PYTHON_INCLUDE_DIRS = ${PYTHON_INCLUDE_DIRS}") +message("PYTHON_LIBRARIES = ${PYTHON_LIBRARIES}") +include_directories(${PYTHON_INCLUDE_DIRS}) + +set(MS_CCSRC_PATH ${CMAKE_SOURCE_DIR}/mindspore/ccsrc) +set(MS_CCSRC_BUILD_PATH ${BUILD_PATH}/mindspore/mindspore/ccsrc) + +if(NOT MSVC) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden") +endif() + +include(${CMAKE_SOURCE_DIR}/cmake/init.cmake) +add_subdirectory(mindspore/core) +if(ENABLE_D OR ENABLE_ACL OR ENABLE_TESTCASES) + include(${CMAKE_SOURCE_DIR}/cmake/dependency_graphengine.cmake) +endif() + +add_subdirectory_with_faster_option(mindspore/ccsrc) + +if(ENABLE_TESTCASES OR ENABLE_CPP_ST) + add_subdirectory(tests) +endif() + +# packaging +file(READ ${CMAKE_SOURCE_DIR}/version.txt VERSION_NUMBER) +string(REPLACE "\n" "" VERSION_NUMBER ${VERSION_NUMBER}) +if(${VERSION_NUMBER} MATCHES ".*dev.*") + message("building dev mode") + set(BUILD_DEV_MODE ON) +endif() + +if(ONLY_BUILD_DEVICE_PLUGINS) + include(cmake/package_plugin.cmake) +elseif(MODE_ASCEND_ACL) + include(cmake/package_tar.cmake) +elseif(CMAKE_SYSTEM_NAME MATCHES "Windows") + include(cmake/package_win.cmake) +elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin") + include(cmake/package_mac.cmake) +else() + include(cmake/package.cmake) +endif() diff --git a/build.bat b/build.bat new file mode 100644 index 0000000..07f9676 --- /dev/null +++ b/build.bat @@ -0,0 +1,131 @@ +@rem Copyright 2020 Huawei Technologies Co., Ltd +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem http://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem ============================================================================ +@echo off +@title mindspore_build + +setlocal EnableDelayedExpansion + +@echo off +echo Start build at: %date% %time% + +SET BASE_PATH=%CD% +SET BUILD_PATH=%BASE_PATH%/build + +SET threads=8 +SET ENABLE_GITEE=OFF +SET ENABLE_MSVC=OFF +set BUILD_TYPE=Release +set VERSION_STR='' +for /f "tokens=1" %%a in (version.txt) do (set VERSION_STR=%%a) + +ECHO %2%|FINDSTR "^[0-9][0-9]*$" +IF %errorlevel% == 0 ( + SET threads=%2% +) + +IF "%FROM_GITEE%" == "1" ( + echo "DownLoad from gitee" + SET ENABLE_GITEE=ON +) + +ECHO %1%|FINDSTR "^ms_vs" +IF %errorlevel% == 0 ( + echo "use msvc compiler" + SET ENABLE_MSVC=ON +) else ( + echo "use mingw compiler" +) + +IF NOT EXIST "%BUILD_PATH%" ( + md "build" +) +cd %BUILD_PATH% +IF NOT EXIST "%BUILD_PATH%/mindspore" ( + md "mindspore" +) + +cd %BUILD_PATH%/mindspore +IF "%1%" == "lite" ( + echo "======Start building MindSpore Lite %VERSION_STR%======" + rd /s /q "%BASE_PATH%\output" + (git log -1 | findstr "^commit") > %BUILD_PATH%\.commit_id + IF defined VisualStudioVersion ( + cmake -DMSLITE_MINDDATA_IMPLEMENT=off -DMSLITE_ENABLE_TRAIN=off -DVERSION_STR=%VERSION_STR% ^ + -DCMAKE_BUILD_TYPE=Release -G "Ninja" "%BASE_PATH%/mindspore/lite" + ) ELSE ( + cmake -DMSLITE_MINDDATA_IMPLEMENT=off -DMSLITE_ENABLE_TRAIN=off -DVERSION_STR=%VERSION_STR% ^ + -DCMAKE_BUILD_TYPE=Release -G "CodeBlocks - MinGW Makefiles" "%BASE_PATH%/mindspore/lite" + ) +) ELSE ( + IF "%1%" == "ms_vs_gpu" ( + echo "======Start gen VS2019 Project for MS gpu ======" + cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_CPU=ON -DENABLE_GPU=ON -DGPU_BACKEND_CUDA=ON -DMS_REQUIRE_CUDA_VERSION=11.1 -DENABLE_MINDDATA=ON -DUSE_GLOG=ON -DENABLE_GITEE=%ENABLE_GITEE% ^ + -G "Visual Studio 16 2019" -A x64 ../.. + ) ELSE IF "%1%" == "ms_vs_cpu" ( + echo "======Start gen VS2019 Project for MS cpu ======" + for /f "delims=" %%i in ('powershell.exe -ExecutionPolicy Bypass -Command "Get-ChildItem HKLM:\SOFTWARE\Wow6432Node\Microsoft\Windows\CurrentVersion\Uninstall | foreach { Get-ItemProperty $_.PsPath } | where { $_.DisplayName -like '*Visual Studio*' -and $_.InstallLocation.Length -gt 0 } | sort InstallDate -Descending | foreach { Join-Path $_.InstallLocation 'VC\Auxiliary\Build'}"') do (call "%%i\vcvars64.bat") + where sccache + IF !errorlevel! == 0 ( + echo "use sccache to speed up compile" + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache -DENABLE_CPU=ON -DENABLE_MINDDATA=ON -DUSE_GLOG=ON -DENABLE_GITEE=%ENABLE_GITEE% ^ + -G Ninja ../.. + ) ELSE ( + echo "fail to find sccache" + cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_CPU=ON -DENABLE_MINDDATA=ON -DUSE_GLOG=ON -DENABLE_GITEE=%ENABLE_GITEE% ^ + -G Ninja ../.. + ) + ) ELSE IF "%1%" == "ms_vs_cpu_debug" ( + echo "======Start gen VS2019 Project for MS cpu debug======" + cmake -DCMAKE_BUILD_TYPE=Debug -DDEBUG_MODE=ON -DENABLE_CPU=ON -DENABLE_MINDDATA=ON -DUSE_GLOG=ON -DENABLE_GITEE=%ENABLE_GITEE% ^ + -G "Visual Studio 16 2019" -A x64 ../.. + set BUILD_TYPE=Debug + ) ELSE ( + echo "======Start gen MinGW64 Project for MS cpu ======" + cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_CPU=ON -DENABLE_MINDDATA=ON -DUSE_GLOG=ON -DENABLE_GITEE=%ENABLE_GITEE% ^ + -G "CodeBlocks - MinGW Makefiles" ../.. + ) +) +IF NOT %errorlevel% == 0 ( + echo "cmake fail." + call :clean + EXIT /b 1 +) + +IF ON == %ENABLE_MSVC% ( + cmake --build . --config %BUILD_TYPE% --target package +) ELSE ( + cmake --build . --target package -- -j%threads% +) + +IF NOT %errorlevel% == 0 ( + echo "build fail." + call :clean + EXIT /b 1 +) + +call :clean +EXIT /b 0 + +:clean + IF EXIST "%BASE_PATH%/output" ( + cd %BASE_PATH%/output + if EXIST "%BASE_PATH%/output/_CPack_Packages" ( + rd /s /q _CPack_Packages + ) + ) + cd %BASE_PATH% + +@echo off +echo End build at: %date% %time% \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..b812c7e --- /dev/null +++ b/build.sh @@ -0,0 +1,103 @@ +#!/bin/bash +# Copyright 2019-2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e +BASEPATH=$(cd "$(dirname $0)"; pwd) +export CUDA_PATH="" +export BUILD_PATH="${BASEPATH}/build/" + +source ./scripts/build/usage.sh +source ./scripts/build/default_options.sh +source ./scripts/build/option_proc_debug.sh +source ./scripts/build/option_proc_mindspore.sh +source ./scripts/build/option_proc_lite.sh +source ./scripts/build/process_options.sh +source ./scripts/build/parse_device.sh +source ./scripts/build/build_mindspore.sh + +# check value of input is 'on' or 'off' +# usage: check_on_off arg_value arg_name +check_on_off() +{ + if [[ "X$1" != "Xon" && "X$1" != "Xoff" ]]; then + echo "Invalid value $1 for option -$2" + usage + exit 1 + fi +} + +update_submodule() +{ + git submodule update --init graphengine + cd "${BASEPATH}/graphengine" + git submodule update --init metadef + cd "${BASEPATH}" + if [[ "X$ENABLE_AKG" = "Xon" ]]; then + if [[ "X$ENABLE_D" == "Xon" ]]; then + git submodule update --init akg + else + GIT_LFS_SKIP_SMUDGE=1 git submodule update --init akg + fi + fi +} + +build_exit() +{ + echo "$@" >&2 + stty echo + exit 1 +} + +make_clean() +{ + echo "enable make clean" + cd "${BUILD_PATH}/mindspore" + cmake --build . --target clean +} + +echo "---------------- MindSpore: build start ----------------" +init_default_options +process_options "$@" +parse_device + +if [[ "X$COMPILE_LITE" = "Xon" ]]; then + export COMPILE_MINDDATA_LITE + export ENABLE_VERBOSE + export LITE_PLATFORM + export LITE_ENABLE_AAR + source mindspore/lite/build_lite.sh +else + mkdir -pv "${BUILD_PATH}/package/mindspore/lib" + mkdir -pv "${BUILD_PATH}/package/mindspore/lib/plugin" + update_submodule + + build_mindspore + + if [[ "X$ENABLE_MAKE_CLEAN" = "Xon" ]]; then + make_clean + fi + if [[ "X$ENABLE_ACL" == "Xon" ]] && [[ "X$ENABLE_D" == "Xoff" ]]; then + echo "acl mode, skipping deploy phase" + rm -rf ${BASEPATH}/output/_CPack_Packages/ + elif [[ "X$FASTER_BUILD_FOR_PLUGINS" == "Xon" ]]; then + echo "plugin mode, skipping deploy phase" + rm -rf ${BASEPATH}/output/_CPack_Packages/ + else + cp -rf ${BUILD_PATH}/package/mindspore/lib ${BASEPATH}/mindspore/python/mindspore + cp -rf ${BUILD_PATH}/package/mindspore/*.so ${BASEPATH}/mindspore/python/mindspore + fi +fi +echo "---------------- MindSpore: build end ----------------" diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..4acc08f --- /dev/null +++ b/setup.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +# encoding: utf-8 +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""setup package.""" +import os +import stat +import platform + +from setuptools import setup, find_packages +from setuptools.command.egg_info import egg_info +from setuptools.command.build_py import build_py + +backend_policy = os.getenv('BACKEND_POLICY') +commit_id = os.getenv('COMMIT_ID').replace("\n", "") +package_name = os.getenv('MS_PACKAGE_NAME').replace("\n", "") +build_path = os.getenv('BUILD_PATH') + +pwd = os.path.dirname(os.path.realpath(__file__)) +pkg_dir = os.path.join(build_path, 'package') + + +def _read_file(filename): + with open(os.path.join(pwd, filename), encoding='UTF-8') as f: + return f.read() + + +version = _read_file('version.txt').replace("\n", "") +readme = _read_file('README.md') + + +def _write_version(file): + file.write("__version__ = '{}'\n".format(version)) + + +def _write_config(file): + file.write("__backend__ = '{}'\n".format(backend_policy)) + + +def _write_commit_file(file): + file.write("__commit_id__ = '{}'\n".format(commit_id)) + + +def _write_package_name(file): + file.write("__package_name__ = '{}'\n".format(package_name)) + + +def build_dependencies(): + """generate python file""" + version_file = os.path.join(pkg_dir, 'mindspore', 'version.py') + with open(version_file, 'w') as f: + _write_version(f) + + version_file = os.path.join(pwd, 'mindspore/python/mindspore', 'version.py') + with open(version_file, 'w') as f: + _write_version(f) + + config_file = os.path.join(pkg_dir, 'mindspore', 'default_config.py') + with open(config_file, 'w') as f: + _write_config(f) + + config_file = os.path.join(pwd, 'mindspore/python/mindspore', 'default_config.py') + with open(config_file, 'w') as f: + _write_config(f) + + package_info = os.path.join(pkg_dir, 'mindspore', 'default_config.py') + with open(package_info, 'a') as f: + _write_package_name(f) + + package_info = os.path.join(pwd, 'mindspore/python/mindspore', 'default_config.py') + with open(package_info, 'a') as f: + _write_package_name(f) + + commit_file = os.path.join(pkg_dir, 'mindspore', '.commit_id') + with open(commit_file, 'w') as f: + _write_commit_file(f) + + commit_file = os.path.join(pwd, 'mindspore/python/mindspore', '.commit_id') + with open(commit_file, 'w') as f: + _write_commit_file(f) + + +build_dependencies() + +required_package = [ + 'numpy >= 1.17.0', + 'protobuf >= 3.13.0', + 'asttokens >= 2.0.4', + 'pillow >= 6.2.0', + 'scipy >= 1.5.4', + 'packaging >= 20.0', + 'psutil >= 5.6.1', + 'astunparse >= 1.6.3' +] + +package_data = { + '': [ + '*.so*', + '*.pyd', + '*.dll', + '*.pdb', + 'bin/*', + 'lib/plugin/*', + 'lib/plugin/*/*', + 'lib/*.so*', + 'lib/*.a', + 'lib/*.dylib*', + '.commit_id', + 'config/*', + 'ops/bprop_mindir/*', + 'include/*', + 'include/*/*', + 'include/*/*/*', + 'include/*/*/*/*', + 'Third_Party_Open_Source_Software_Notice' + ] +} + + +def update_permissions(path): + """ + Update permissions. + + Args: + path (str): Target directory path. + """ + if platform.system() == "Windows": + return + + for dirpath, dirnames, filenames in os.walk(path): + for dirname in dirnames: + dir_fullpath = os.path.join(dirpath, dirname) + os.chmod(dir_fullpath, stat.S_IREAD | stat.S_IWRITE | + stat.S_IEXEC | stat.S_IRGRP | stat.S_IXGRP) + for filename in filenames: + file_fullpath = os.path.join(dirpath, filename) + os.chmod(file_fullpath, stat.S_IREAD) + + +class EggInfo(egg_info): + """Egg info.""" + + def run(self): + super().run() + egg_info_dir = os.path.join(pkg_dir, 'mindspore.egg-info') + update_permissions(egg_info_dir) + + +class BuildPy(build_py): + """BuildPy.""" + + def run(self): + super().run() + mindspore_dir = os.path.join(pkg_dir, 'build', 'lib', 'mindspore') + update_permissions(mindspore_dir) + mindspore_dir = os.path.join(pkg_dir, 'build', 'lib', 'mindspore', '_akg') + update_permissions(mindspore_dir) + + +setup( + name=package_name, + version=version, + author='The MindSpore Authors', + author_email='contact@mindspore.cn', + url='https://www.mindspore.cn', + download_url='https://github.com/mindspore-ai/mindspore/tags', + project_urls={ + 'Sources': 'https://github.com/mindspore-ai/mindspore', + 'Issue Tracker': 'https://github.com/mindspore-ai/mindspore/issues', + }, + description='MindSpore is a new open source deep learning training/inference ' + 'framework that could be used for mobile, edge and cloud scenarios.', + long_description=readme, + long_description_content_type="text/markdown", + packages=find_packages(), + package_data=package_data, + include_package_data=True, + cmdclass={ + 'egg_info': EggInfo, + 'build_py': BuildPy, + }, + entry_points={ + 'console_scripts': [ + 'cache_admin=mindspore.dataset.engine.cache_admin:main', + ], + }, + python_requires='>=3.7', + install_requires=required_package, + classifiers=[ + 'Development Status :: 4 - Beta', + 'Environment :: Console', + 'Intended Audience :: Science/Research', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 3 :: Only', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: C++', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + license='Apache 2.0', + keywords='mindspore machine learning', +)