77from distutils import log as logger
88from distutils .command .build_ext import build_ext as _build_ext
99from glob import glob , iglob
10- from os import path , getcwd , environ , remove , listdir
10+ from os import path , getcwd , environ , remove , listdir , chdir , cpu_count
1111from shutil import copyfile , copytree , rmtree
1212import platform
1313import subprocess
1414import sys
1515import datetime
1616
1717from pathlib import Path
18- nightly_build = False
18+ import shlex
19+
20+ ONNXRUNTIME_BUILD_CMD = environ .get ('ONNXRUNTIME_BUILD_CMD' , None )
21+ if ONNXRUNTIME_BUILD_CMD is not None :
22+ cmd = [sys .executable , 'tools/ci_build/build.py' , '--config' , 'Release' , '--update' , '--skip_submodule_sync' ]
23+ cmd += shlex .split (ONNXRUNTIME_BUILD_CMD )
24+ cmd .append ("--build_dir=." )
25+ cmd .append ("--update" )
26+ subprocess .run (cmd , check = True )
27+ chdir ('Release' )
28+ subprocess .run (['make' , '-j%d' % cpu_count (), 'onnxruntime_pybind11_state' ], check = True )
29+
30+ nightly_build = environ .get ('NIGHTLY_BUILD' , None ) == '1'
1931featurizers_build = False
2032package_name = 'onnxruntime'
21- wheel_name_suffix = None
33+ wheel_name_suffix = environ .get ('WHEEL_NAME_SUFFIX' , None )
34+ cuda_version = environ .get ('ONNXRUNTIME_CUDA_VERSION' , None )
35+ rocm_version = environ .get ('ONNXRUNTIME_ROCM_VERSION' , None )
2236
2337
2438def parse_arg_remove_boolean (argv , arg_name ):
@@ -45,20 +59,18 @@ def parse_arg_remove_string(argv, arg_name_equal):
4559featurizers_build = parse_arg_remove_boolean (sys .argv , '--use_featurizers' )
4660
4761if parse_arg_remove_boolean (sys .argv , '--nightly_build' ):
48- package_name = 'ort-nightly'
4962 nightly_build = True
5063
51- wheel_name_suffix = parse_arg_remove_string (sys .argv , '--wheel_name_suffix=' )
64+ if wheel_name_suffix is None :
65+ wheel_name_suffix = parse_arg_remove_string (sys .argv , '--wheel_name_suffix=' )
5266
53- cuda_version = None
54- rocm_version = None
5567# The following arguments are mutually exclusive
5668if parse_arg_remove_boolean (sys .argv , '--use_tensorrt' ):
5769 package_name = 'onnxruntime-gpu-tensorrt' if not nightly_build else 'ort-trt-nightly'
58- elif wheel_name_suffix == 'gpu' :
70+ elif wheel_name_suffix == 'gpu' and cuda_version is None :
5971 # TODO: how to support multiple CUDA versions?
6072 cuda_version = parse_arg_remove_string (sys .argv , '--cuda_version=' )
61- elif parse_arg_remove_boolean (sys .argv , '--use_rocm' ):
73+ elif parse_arg_remove_boolean (sys .argv , '--use_rocm' ) and rocm_version is None :
6274 package_name = 'onnxruntime-rocm' if not nightly_build else 'ort-rocm-nightly'
6375 rocm_version = parse_arg_remove_string (sys .argv , '--rocm_version=' )
6476elif parse_arg_remove_boolean (sys .argv , '--use_openvino' ):
@@ -75,30 +87,7 @@ def parse_arg_remove_string(argv, arg_name_equal):
7587 package_name = 'onnxruntime-armnn'
7688
7789
78- # PEP 513 defined manylinux1_x86_64 and manylinux1_i686
79- # PEP 571 defined manylinux2010_x86_64 and manylinux2010_i686
80- # PEP 599 defines the following platform tags:
81- # manylinux2014_x86_64
82- # manylinux2014_i686
83- # manylinux2014_aarch64
84- # manylinux2014_armv7l
85- # manylinux2014_ppc64
86- # manylinux2014_ppc64le
87- # manylinux2014_s390x
88- manylinux_tags = [
89- 'manylinux1_x86_64' ,
90- 'manylinux1_i686' ,
91- 'manylinux2010_x86_64' ,
92- 'manylinux2010_i686' ,
93- 'manylinux2014_x86_64' ,
94- 'manylinux2014_i686' ,
95- 'manylinux2014_aarch64' ,
96- 'manylinux2014_armv7l' ,
97- 'manylinux2014_ppc64' ,
98- 'manylinux2014_ppc64le' ,
99- 'manylinux2014_s390x' ,
100- ]
101- is_manylinux = environ .get ('AUDITWHEEL_PLAT' , None ) in manylinux_tags
90+ is_manylinux = environ .get ('AUDITWHEEL_PLAT' , None ) is not None
10291
10392
10493class build_ext (_build_ext ):
@@ -177,7 +166,8 @@ def run(self):
177166 self ._rewrite_ld_preload (to_preload )
178167 self ._rewrite_ld_preload_cuda (to_preload_cuda )
179168 _bdist_wheel .run (self )
180- if is_manylinux :
169+ # Skip this part for CIBUILDWHEEL environment, because CIBUILDWHEEL will run auditwheel for us
170+ if is_manylinux and environ .get ('CIBUILDWHEEL' , None ) is None :
181171 file = glob (path .join (self .dist_dir , '*linux*.whl' ))[0 ]
182172 logger .info ('repairing %s for manylinux1' , file )
183173 try :
@@ -256,6 +246,9 @@ def run(self):
256246if not path .exists (README ):
257247 this = path .dirname (__file__ )
258248 README = path .join (this , "docs/python/README.rst" )
249+ if not path .exists (README ):
250+ this = path .dirname (__file__ )
251+ README = path .join (this , "onnxruntime/README.rst" )
259252if not path .exists (README ):
260253 raise FileNotFoundError ("Unable to find 'README.rst'" )
261254with open (README ) as f :
@@ -408,67 +401,70 @@ def run(self):
408401 version_number = f .readline ().strip ()
409402if nightly_build :
410403 # https://docs.microsoft.com/en-us/azure/devops/pipelines/build/variables
411- build_suffix = environ .get ('BUILD_BUILDNUMBER' )
404+ if enable_training :
405+ build_suffix = environ .get ('BUILD_BUILDNUMBER' )
406+ else :
407+ # A number that monotonically increasing
408+ build_suffix = environ .get ('BUILD_ID' )
412409 if build_suffix is None :
413410 # The following line is only for local testing
414411 build_suffix = str (datetime .datetime .now ().date ().strftime ("%Y%m%d" ))
415412 else :
416413 build_suffix = build_suffix .replace ('.' , '' )
417-
418- if len (build_suffix ) > 8 and len (build_suffix ) < 12 :
419- # we want to format the build_suffix to avoid (the 12th run on 20210630 vs the first run on 20210701):
420- # 2021063012 > 202107011
421- # in above 2021063012 is treated as the latest which is incorrect.
422- # we want to convert the format to:
423- # 20210630012 < 20210701001
424- # where the first 8 digits are date. the last 3 digits are run count.
425- # as long as there are less than 1000 runs per day, we will not have the problem.
426- # to test this code locally, run:
427- # NIGHTLY_BUILD=1 BUILD_BUILDNUMBER=202107011 python tools/ci_build/build.py --config RelWithDebInfo \
428- # --enable_training --use_cuda --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ \
429- # --nccl_home /usr/lib/x86_64-linux-gnu/ --build_dir build/Linux --build --build_wheel --skip_tests \
430- # --cuda_version 11.1
431- def check_date_format (date_str ):
432- try :
433- datetime .datetime .strptime (date_str , '%Y%m%d' )
434- return True
435- except : # noqa
436- return False
437-
438- def reformat_run_count (count_str ):
439- try :
440- count = int (count_str )
441- if count >= 0 and count < 1000 :
442- return "{:03}" .format (count )
443- elif count >= 1000 :
444- raise RuntimeError (f'Too many builds for the same day: { count } )
445- return ""
446- except : # noqa
447- return ""
448-
449- build_suffix_is_date_format = check_date_format (build_suffix [:8 ])
450- build_suffix_run_count = reformat_run_count (build_suffix [8 :])
451- if build_suffix_is_date_format and build_suffix_run_count :
452- build_suffix = build_suffix [:8 ] + build_suffix_run_count
453- elif len (build_suffix ) >= 12 :
454- raise RuntimeError (f'Incorrect build suffix: "{ build_suffix } )
455-
456414 if enable_training :
457- from packaging import version
458- from packaging .version import Version
459- # with training package, we need to bump up version minor number so that
460- # nightly releases take precedence over the latest release when --pre is used during pip install.
461- # eventually this shall be the behavior of all onnxruntime releases.
462- # alternatively we may bump up version number right after every release.
463- ort_version = version .parse (version_number )
464- if isinstance (ort_version , Version ):
465- # TODO: this is the last time we have to do this!!!
466- # We shall bump up release number right after release cut.
467- if ort_version .major == 1 and ort_version .minor == 8 and ort_version .micro == 0 :
468- version_number = '{major}.{minor}.{macro}' .format (
469- major = ort_version .major ,
470- minor = ort_version .minor + 1 ,
471- macro = ort_version .micro )
415+ if len (build_suffix ) > 8 and len (build_suffix ) < 12 :
416+ # we want to format the build_suffix to avoid (the 12th run on 20210630 vs the first run on 20210701):
417+ # 2021063012 > 202107011
418+ # in above 2021063012 is treated as the latest which is incorrect.
419+ # we want to convert the format to:
420+ # 20210630012 < 20210701001
421+ # where the first 8 digits are date. the last 3 digits are run count.
422+ # as long as there are less than 1000 runs per day, we will not have the problem.
423+ # to test this code locally, run:
424+ # NIGHTLY_BUILD=1 BUILD_BUILDNUMBER=202107011 python tools/ci_build/build.py --config RelWithDebInfo \
425+ # --enable_training --use_cuda --cuda_home /usr/local/cuda --cudnn_home /usr/lib/x86_64-linux-gnu/ \
426+ # --nccl_home /usr/lib/x86_64-linux-gnu/ --build_dir build/Linux --build --build_wheel --skip_tests \
427+ # --cuda_version 11.1
428+ def check_date_format (date_str ):
429+ try :
430+ datetime .datetime .strptime (date_str , '%Y%m%d' )
431+ return True
432+ except : # noqa
433+ return False
434+
435+ def reformat_run_count (count_str ):
436+ try :
437+ count = int (count_str )
438+ if count >= 0 and count < 1000 :
439+ return "{:03}" .format (count )
440+ elif count >= 1000 :
441+ raise RuntimeError (f'Too many builds for the same day: { count } )
442+ return ""
443+ except : # noqa
444+ return ""
445+
446+ build_suffix_is_date_format = check_date_format (build_suffix [:8 ])
447+ build_suffix_run_count = reformat_run_count (build_suffix [8 :])
448+ if build_suffix_is_date_format and build_suffix_run_count :
449+ build_suffix = build_suffix [:8 ] + build_suffix_run_count
450+ elif len (build_suffix ) >= 12 :
451+ raise RuntimeError (f'Incorrect build suffix: "{ build_suffix } )
452+
453+ from packaging import version
454+ from packaging .version import Version
455+ # with training package, we need to bump up version minor number so that
456+ # nightly releases take precedence over the latest release when --pre is used during pip install.
457+ # eventually this shall be the behavior of all onnxruntime releases.
458+ # alternatively we may bump up version number right after every release.
459+ ort_version = version .parse (version_number )
460+ if isinstance (ort_version , Version ):
461+ # TODO: this is the last time we have to do this!!!
462+ # We shall bump up release number right after release cut.
463+ if ort_version .major == 1 and ort_version .minor == 8 and ort_version .micro == 0 :
464+ version_number = '{major}.{minor}.{macro}' .format (
465+ major = ort_version .major ,
466+ minor = ort_version .minor + 1 ,
467+ macro = ort_version .micro )
472468
473469 version_number = version_number + ".dev" + build_suffix
474470
@@ -480,6 +476,7 @@ def reformat_run_count(count_str):
480476 # for training packages, local version is used to indicate device types
481477 package_name = "{}-{}" .format (package_name , wheel_name_suffix )
482478
479+
483480cmd_classes = {}
484481if bdist_wheel is not None :
485482 cmd_classes ['bdist_wheel' ] = bdist_wheel
0 commit comments