Skip to content

Add python 3.9 support #5874

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 16 commits into from
Nov 30, 2020
14 changes: 7 additions & 7 deletions cgmanifests/cgmanifest.json
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@
"component": {
"type": "git",
"git": {
"commitHash": "f97fd8664fa824e5d081d776cf4a811dbf09bca4",
"commitHash": "bc8ce45b35ade9b8f08d88d4d9fdfd12b4f6f310",
"repositoryUrl": "https://github.com/pypa/manylinux"
},
"comments": "For building our CI build docker image"
Expand All @@ -363,20 +363,20 @@
"component": {
"type": "git",
"git": {
"commitHash": "439c93d51f45c50541fc755b597725168ecd939a",
"commitHash": "9cf6752276e6fcfd0c23fdb064ad27f448aaaf75",
"repositoryUrl": "https://github.com/python/cpython"
},
"comments": "Python 3.9.0rc1"
"comments": "Python 3.9.0"
}
},
{
"component": {
"type": "git",
"git": {
"commitHash": "580fbb018fd0844806119614d752b41fc69660f9",
"commitHash": "db455296be5f792b8c12b7cd7f3962b52e4f44ee",
"repositoryUrl": "https://github.com/python/cpython"
},
"comments": "Python 3.8.5"
"comments": "Python 3.8.6"
}
},
{
Expand All @@ -403,10 +403,10 @@
"component": {
"type": "git",
"git": {
"commitHash": "e5f6aba872e66bfd86eb592214696a519cded197",
"commitHash": "426b022776672fdf3d71ddd98d89af341c88080f",
"repositoryUrl": "https://github.com/python/cpython"
},
"comments": "Python 3.5.9"
"comments": "Python 3.5.10"
}
},
{
Expand Down
44 changes: 44 additions & 0 deletions cgmanifests/submodules/cgmanifest.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,50 @@
{
"Version": 1,
"Registrations": [
{
"Component": {
"Type": "other",
"other": {
"Name": "automake",
"Version": "1.16.2",
"DownloadUrl": "http://ftp.gnu.org/gnu/automake/automake-1.16.2.tar.gz"
},
"comments": "manylinux dependency"
}
},
{
"Component": {
"Type": "other",
"other": {
"Name": "libtool",
"Version": "2.4.6",
"DownloadUrl": "http://ftp.gnu.org/gnu/libtool/libtool-2.4.6.tar.gz"
},
"comments": "manylinux dependency"
}
},
{
"Component": {
"Type": "other",
"other": {
"Name": "sqlite",
"Version": "3330000",
"DownloadUrl": "https://www.sqlite.org/2020/sqlite-autoconf-3330000.tar.gz"
},
"comments": "manylinux dependency"
}
},
{
"Component": {
"Type": "other",
"other": {
"Name": "git",
"Version": "2.29.1",
"DownloadUrl": "https://www.kernel.org/pub/software/scm/git/git-2.29.1.tar.gz"
},
"comments": "manylinux dependency"
}
},
{
"component": {
"type": "git",
Expand Down
46 changes: 45 additions & 1 deletion cgmanifests/submodules/generate_submodule_cgmanifest.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,54 @@
import os
import subprocess
import sys
import re

SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", ".."))

package_name = None
package_filename = None
package_url = None

registrations = []

with open(os.path.join(REPO_DIR,'tools','ci_build','github','linux','docker','manylinux2014_build_scripts','build_env.sh'), "r") as f:
for line in f:
if not line.strip():
package_name = None
package_filename = None
package_url = None
if package_filename is None:
m = re.match("(.+?)_ROOT=(.*)$",line)
if m != None:
package_name = m.group(1)
package_filename = m.group(2)
else:
m = re.match("(.+?)_AUTOCONF_VERSION=(.*)$",line)
if m != None:
package_name = m.group(1)
package_filename = m.group(2)
elif package_url is None:
m = re.match("(.+?)_DOWNLOAD_URL=(.+)$",line)
if m != None:
package_url = m.group(2) + "/" + package_filename + ".tar.gz"
registration = {
"Component": {
"Type": "other",
"other": {
"Name": package_name.lower(),
"Version": package_filename.split("-")[-1],
"DownloadUrl": package_url,
},
"comments": "manylinux dependency"
}
}
registrations.append(registration)
package_name = None
package_filename = None
package_url = None


proc = subprocess.run(
["git", "submodule", "foreach", "--quiet", "--recursive", "{} {} $toplevel/$sm_path".format(
sys.executable, os.path.join(SCRIPT_DIR, "print_submodule_info.py"))],
Expand All @@ -17,7 +61,7 @@
stderr=subprocess.PIPE,
universal_newlines=True)

registrations = []

submodule_lines = proc.stdout.splitlines()
for submodule_line in submodule_lines:
(absolute_path, url, commit) = submodule_line.split(" ")
Expand Down
4 changes: 2 additions & 2 deletions cmake/external/pybind11.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ if(NOT TARGET pybind11::module)

set(pybind11_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/pybind11/src/pybind11/include)
set(pybind11_URL https://github.com/pybind/pybind11.git)
set(pybind11_TAG v2.4.0)
set(pybind11_TAG v2.6.1)

ExternalProject_Add(pybind11
PREFIX pybind11
Expand All @@ -21,4 +21,4 @@ if(NOT TARGET pybind11::module)
set(pybind11_dep pybind11)
else()
set(pybind11_lib pybind11::module)
endif()
endif()
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,6 @@ docker run --gpus all --rm \
-e "PackageName=$PackageName" \
-e "RunTestCsharp=$RunTestCsharp" \
-e "RunTestNative=$RunTestNative" \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch4 \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimegpubuild:ch4 \
/bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \
/home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,6 @@ docker run --rm \
-e "DisableMlOps=$DISABLEMLOPS" \
-e "RunTestCsharp=$RunTestCsharp" \
-e "RunTestNative=$RunTestNative" \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch3 \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:chr \
/bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \
/home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion
6 changes: 3 additions & 3 deletions onnxruntime/python/_pybind_state.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@
CUDNN_VERSION = "8"
cuda_env_variable = "CUDA_PATH_V" + CUDA_VERSION.replace(".", "_")
if cuda_env_variable not in os.environ:
raise ImportError(f"CUDA Toolkit {CUDA_VERSION} not installed on the machine.")
raise ImportError("CUDA Toolkit %s not installed on the machine." % CUDA_VERSION)
cuda_bin_dir = os.path.join(os.environ[cuda_env_variable], "bin")
if not os.path.isfile(os.path.join(cuda_bin_dir, f"cudnn64_{CUDNN_VERSION}.dll")):
raise ImportError(f"cuDNN {CUDNN_VERSION} not installed on the machine.")
if not os.path.isfile(os.path.join(cuda_bin_dir, "cudnn64_%s.dll" % CUDNN_VERSION)):
raise ImportError("cuDNN %s not installed on the machine." % CUDNN_VERSION)
os.add_dll_directory(cuda_bin_dir)

try:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ jobs:
script: |
mkdir -p $HOME/.onnx
docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch3 python3 \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:chr python3 \
/onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib --use_openmp
workingDirectory: $(Build.SourcesDirectory)
Expand Down Expand Up @@ -58,7 +58,7 @@ jobs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch4 \
--volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimegpubuild:chq \
python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2
workingDirectory: $(Build.SourcesDirectory)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch4 python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimegpubuild:chq python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2
workingDirectory: $(Build.SourcesDirectory)
- task: Docker@2
displayName: logout
Expand Down Expand Up @@ -258,7 +258,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch4 /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimegpubuild:chq /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build
workingDirectory: $(Build.BinariesDirectory)/final-jar

- task: Docker@2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ jobs:
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch3 /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --build_java --use_openmp --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:chr /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --build_java --use_openmp --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
workingDirectory: $(Build.SourcesDirectory)
displayName: 'Run build and test'
- task: Docker@2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch3 \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:chr \
python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug Release \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch3 \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:chr \
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh
workingDirectory: $(Build.SourcesDirectory)
- task: CmdLine@2
Expand All @@ -54,7 +54,7 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch3 \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:chr \
python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug\
Expand All @@ -76,7 +76,7 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch3 \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:chr \
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh
workingDirectory: $(Build.SourcesDirectory)
- task: CmdLine@2
Expand All @@ -97,7 +97,7 @@ jobs:
-e BUILD_ID=$(Build.BuildId) \
-e BUILD_REASON=$(Build.Reason) \
-e DASHBOARD_MYSQL_ORT_PASSWORD=$(dashboard-mysql-ort-password) \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch3 \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:chr \
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh
workingDirectory: $(Build.SourcesDirectory)
- task: Docker@2
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecpubuild:ch1 \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecpubuild:chp \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build \
--config Debug Release \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch4 \
python3 /onnxruntime_src/tools/ci_build/build.py \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimegpubuild:chq \
/opt/python/cp37-cp37m/bin/python /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Release \
--skip_submodule_sync \
Expand All @@ -38,7 +38,7 @@ jobs:
--build_wheel \
--enable_onnx_tests --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 \
--enable_pybind --build_java --build_nodejs \
--cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=52
--cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=52 PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so
workingDirectory: $(Build.SourcesDirectory)
- task: Docker@2
displayName: logout
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecuda11build:ch5 \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecuda11build:chs \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug Release \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch4 \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimegpubuild:chq \
python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug Release \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch3 \
onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:chr \
python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug Release \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ jobs:
inputs:
script: |
mkdir -p $HOME/.onnx && docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch3 /bin/bash -c "python3 \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:chr /bin/bash -c "python3 \
/onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --build_nodejs \
--skip_submodule_sync --parallel --build_shared_lib --use_openmp && cd /onnxruntime_src/nodejs && npm pack"
workingDirectory: $(Build.SourcesDirectory)
Expand Down
Loading