wanjia9506
/
jittor
forked from maxjhandsome/jittor
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

add centos support

This commit is contained in:
Dun Liang 2021-05-13 21:25:13 +08:00
parent 0753f06713
commit ed159c8c79
8 changed files with 109 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -256,7 +256,7 @@ print(a.name())
### Operations
Jittor'op is similar with numpy. Let's try some operations. We create Var `a` and `b` via operation `jt.float32`, and add them. Printing those variables shows they have the same shape and dtype.
Jittor'op is simular with numpy. Let's try some operations. We create Var `a` and `b` via operation `jt.float32`, and add them. Printing those variables shows they have the same shape and dtype.
```python

2
python/jittor/__init__.py
View File

@ -9,7 +9,7 @@
# file 'LICENSE.txt', which is part of this source code package.
# ***************************************************************
__version__ = '1.2.2.73'
__version__ = '1.2.3.0'
from . import lock
with lock.lock_scope():
ori_int = int

8
python/jittor/compile_extern.py
View File

@ -24,12 +24,17 @@ def install_mkl(root_folder):
# url = "https://github.com/intel/mkl-dnn/releases/download/v1.0.2/mkldnn_lnx_1.0.2_cpu_gomp.tgz"
url = "https://cloud.tsinghua.edu.cn/f/da02bf62b55b4aa3b8ee/?dl=1"
filename = "mkldnn_lnx_1.0.2_cpu_gomp.tgz"
# newest version for oneDNN
# url = "https://github.com/oneapi-src/oneDNN/releases/download/v2.2/dnnl_lnx_2.2.0_cpu_gomp.tgz"
# filename = "dnnl_lnx_2.2.0_cpu_gomp.tgz"
fullname = os.path.join(root_folder, filename)
dirname = os.path.join(root_folder, filename.replace(".tgz",""))
if not os.path.isfile(os.path.join(dirname, "examples", "test")):
LOG.i("Downloading mkl...")
download_url_to_local(url, filename, root_folder, "47187284ede27ad3bd64b5f0e7d5e730")
# newest version for oneDNN
# download_url_to_local(url, filename, root_folder, "35bbbdf550a9d8ad54db798e372000f6")
import tarfile
with tarfile.open(fullname, "r") as tar:
@ -37,6 +42,9 @@ def install_mkl(root_folder):
assert 0 == os.system(f"cd {dirname}/examples && "
f"{cc_path} -std=c++14 cpu_cnn_inference_f32.cpp -Ofast -lmkldnn -I ../include -L ../lib -o test && LD_LIBRARY_PATH=../lib/ ./test")
# newest version for oneDNN
# assert 0 == os.system(f"cd {dirname}/examples && "
# f"{cc_path} -std=c++14 cnn_inference_f32.cpp -Ofast -lmkldnn -I ../include -L ../lib -o test && LD_LIBRARY_PATH=../lib/ ./test")
def setup_mkl():
global mkl_ops, use_mkl

15
python/jittor/compiler.py
View File

@ -987,15 +987,26 @@ libname = ctypes.util.find_library(libname)
assert libname is not None, "openmp library not found"
ctypes.CDLL(libname, os.RTLD_NOW | os.RTLD_GLOBAL)
# get os release
with open("/etc/os-release", "r", encoding='utf8') as f:
s = f.read().splitlines()
os_release = {}
for line in s:
a = line.split('=')
if len(a) != 2: continue
os_release[a[0]] = a[1].replace("\"", "")
version_file = os.path.join(jittor_path, "version")
if os.path.isfile(version_file) and not os.path.isdir(os.path.join(jittor_path, "src", "__data__")):
with open(version_file, 'r') as f:
version = f.read().strip()
# key = f"{version}-{cc_type}-{'cuda' if has_cuda else 'cpu'}.o"
key = f"{version}-g++-cpu.o"
key = f"{version}-g++-cpu"
if os_release["ID"] != 'ubuntu':
key += 'centos'
# TODO: open the website
extra_obj = os.path.join(cache_path, key)
url = os.path.join("https://cg.cs.tsinghua.edu.cn/jittor/assets/build/"+key)
url = os.path.join("https://cg.cs.tsinghua.edu.cn/jittor/assets/build/"+key+".o")
jit_utils.download(url, extra_obj)
files.append(extra_obj)

59
python/jittor/utils/polish.py
View File

@ -21,6 +21,7 @@ import jittor as jt
from jittor import LOG
from jittor.compiler import run_cmd
from jittor_utils import translator
from jittor.utils.polish_centos import run_in_centos
import sys
jittor_path = jt.flags.jittor_path
@ -51,33 +52,39 @@ from pathlib import Path
home = str(Path.home())
# for cc_type in ["g++", "clang"]:
# for device in ["cpu", "cuda"]:
for cc_type in ["g++"]:
for device in ["cpu"]:
key = f"{git_version}-{cc_type}-{device}"
env = f"cache_name=build/{cc_type}/{device} cc_path="
cname = "g++" if cc_type=="g++" else "clang-8"
env += cname
# use core2 arch, avoid using avx instructions
# TODO: support more archs, such as arm, or use ir(GIMPLE or LLVM)
env += " cc_flags='-march=core2' "
if device == "cpu":
env += "nvcc_path='' "
elif jt.flags.nvcc_path == "":
env = "unset nvcc_path && " + env
cmd = f"{env} {sys.executable} -c 'import jittor'"
LOG.i("run cmd:", cmd)
os.system(cmd)
LOG.i("run cmd:", cmd)
os.system(cmd)
for os_name in ['ubuntu', 'centos']:
for cc_type in ["g++"]:
for device in ["cpu"]:
key = f"{git_version}-{cc_type}-{device}"
env = f"cache_name=build/{cc_type}/{device} cc_path="
cname = "g++" if cc_type=="g++" else "clang-8"
env += cname
# use core2 arch, avoid using avx instructions
# TODO: support more archs, such as arm, or use ir(GIMPLE or LLVM)
env += " cc_flags='-march=core2' "
if device == "cpu":
env += "nvcc_path='' "
elif jt.flags.nvcc_path == "":
env = "unset nvcc_path && " + env
cmd = f"{env} {sys.executable} -c 'import jittor'"
if key != 'ubuntu': key += '-' + os_name
if os_name == 'centos':
run_in_centos(env)
obj_path = home + f"/.cache/centos/build/{cc_type}/{device}/{cname}/obj_files"
else:
LOG.i("run cmd:", cmd)
os.system(cmd)
LOG.i("run cmd:", cmd)
os.system(cmd)
obj_path = home + f"/.cache/jittor/build/{cc_type}/{device}/{cname}/obj_files"
obj_path = home + f"/.cache/jittor/build/{cc_type}/{device}/{cname}/obj_files"
obj_files = []
for name in data_files:
name = name.split("/")[-1]
fname = f"{obj_path}/{name}.o"
assert os.path.isfile(fname), fname
obj_files.append(fname)
run_cmd(f"ld -r {' '.join(obj_files)} -o {build_path}/{key}.o")
obj_files = []
for name in data_files:
name = name.split("/")[-1]
fname = f"{obj_path}/{name}.o"
assert os.path.isfile(fname), fname
obj_files.append(fname)
run_cmd(f"ld -r {' '.join(obj_files)} -o {build_path}/{key}.o")
# compress source
# tar -cvzf build/jittor.tgz . --exclude build --exclude .git --exclude .ipynb_checkpoints --exclude __pycache__

50
python/jittor/utils/polish_centos.py Normal file
View File

@ -0,0 +1,50 @@
#!/usr/bin/python3
# ***************************************************************
# Copyright (c) 2021 Jittor. All Rights Reserved.
# Maintainers: Dun Liang <randonlang@gmail.com>.
# This file is subject to the terms and conditions defined in
# file 'LICENSE.txt', which is part of this source code package.
# ***************************************************************
import jittor as jt
import os
from pathlib import Path
home_path = str(Path.home())
def run_cmd(cmd):
print("RUN CMD:", cmd)
assert os.system(cmd) == 0
def run_in_centos(env):
dockerfile_src = r"""
FROM centos:7
WORKDIR /root
# install python
RUN yum install gcc openssl-devel bzip2-devel libffi-devel zlib-devel wget -y
RUN wget https://www.python.org/ftp/python/3.8.3/Python-3.8.3.tgz
RUN tar xzf Python-3.8.3.tgz
RUN yum install make -y
RUN cd Python-3.8.3 && ./configure --enable-optimizations && make altinstall -j8
# install g++-7
RUN yum install centos-release-scl -y
RUN yum install devtoolset-7-gcc-c++ -y
RUN yum install which -y
RUN scl enable devtoolset-7 'g++ --version'
RUN python3.8 -m pip install numpy tqdm pillow astunparse
"""
with open("/tmp/centos_build_env", 'w') as f:
f.write(dockerfile_src)
centos_path = os.path.join(home_path, ".cache", "centos")
os.makedirs(centos_path+"/src/jittor", exist_ok=True)
os.makedirs(centos_path+"/src/jittor_utils", exist_ok=True)
os.system(f"cp -rL {jt.flags.jittor_path} {centos_path+'/src/'}")
os.system(f"cp -rL {jt.flags.jittor_path}/../jittor_utils {centos_path+'/src/'}")
run_cmd(f"sudo docker build --tag centos_build_env -f /tmp/centos_build_env .")
run_cmd(f"sudo docker run --rm -v {centos_path}:/root/.cache/jittor centos_build_env scl enable devtoolset-7 'PYTHONPATH=/root/.cache/jittor/src {env} python3.8 -m jittor.test.test_core'")
run_cmd(f"sudo docker run --rm -v {centos_path}:/root/.cache/jittor centos_build_env scl enable devtoolset-7 'PYTHONPATH=/root/.cache/jittor/src {env} python3.8 -m jittor.test.test_core'")

2
python/jittor/version
View File

@ -1 +1 @@
a83c2983e7581236ad125c5f92a1407e12fed4ac
5f0e1aa2f9891c12fc1e190d6cc6177fc6498302

3
setup.py
View File

@ -21,8 +21,9 @@ try:
m = {}
for line in s:
a = line.split('=')
if len(a) != 2: continue
m[a[0]] = a[1].replace("\"", "")
assert m["NAME"] == "Ubuntu" and float(m["VERSION_ID"])>16, error_msg
# assert m["NAME"] == "Ubuntu" and float(m["VERSION_ID"].split('.')[0])>=16, error_msg
except Exception as e:
print(e)
warn(error_msg)