From bbb4b41efcda4de4abeff0868b6cc207c79578d5 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 2 Dec 2020 12:05:03 +0100 Subject: [PATCH 01/31] support python 3.9 --- .github/workflows/ci_dockers.yml | 4 ++-- .github/workflows/ci_pkg-install.yml | 2 +- .github/workflows/ci_test-full.yml | 2 +- .github/workflows/events-nightly.yml | 2 +- .github/workflows/release-docker.yml | 2 +- README.md | 6 +++--- setup.py | 1 + 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci_dockers.yml b/.github/workflows/ci_dockers.yml index 897e16a12d44f..b7bc93d1f58d0 100644 --- a/.github/workflows/ci_dockers.yml +++ b/.github/workflows/ci_dockers.yml @@ -75,8 +75,8 @@ jobs: pytorch_version: 1.6 - python_version: 3.8 pytorch_version: 1.7 - #- python_version: 3.9 - # pytorch_version: 1.8 + - python_version: 3.9 + pytorch_version: 1.7 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/ci_pkg-install.yml b/.github/workflows/ci_pkg-install.yml index 54c9f5c007c82..b4557e5ed75aa 100644 --- a/.github/workflows/ci_pkg-install.yml +++ b/.github/workflows/ci_pkg-install.yml @@ -17,7 +17,7 @@ jobs: matrix: # PyTorch 1.5 is failing on Win and bolts requires torchvision>=0.5 os: [ubuntu-20.04, macOS-10.15 , windows-2019] # - python-version: [3.6, 3.8] + python-version: [3.6, 3.9] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 5a3e23a37fd0b..97bb6c9cac19a 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -15,7 +15,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-18.04, windows-2019, macOS-10.15] - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] requires: ['minimal', 'latest'] # Timeout: https://stackoverflow.com/a/59076067/4521646 diff --git a/.github/workflows/events-nightly.yml b/.github/workflows/events-nightly.yml index 5ad4396a006f7..c14ab316cd8cc 100644 --- a/.github/workflows/events-nightly.yml +++ b/.github/workflows/events-nightly.yml @@ -79,7 +79,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: [3.6, 3.7, 3.8] + python_version: [3.6, 3.7, 3.8, 3.9] pytorch_version: [1.4, 1.5, 1.6, 1.7, 1.8] steps: diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 36ecbe229ac7c..69bac2ef2188a 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -13,7 +13,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: [3.6, 3.7, 3.8] + python_version: [3.6, 3.7, 3.8, 3.9] pytorch_version: [1.4, 1.5, 1.6, 1.7] steps: - name: Checkout diff --git a/README.md b/README.md index dd30bd94d78a6..cbe1341df2f55 100644 --- a/README.md +++ b/README.md @@ -79,9 +79,9 @@ Lightning is rigurously tested across multiple GPUs, TPUs CPUs and against major | Conda py3.7 [linux] | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | [![PyTorch & Conda](https://github.com/PyTorchLightning/pytorch-lightning/workflows/PyTorch%20&%20Conda/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22PyTorch+%26+Conda%22+branch%3Amaster) | | Linux py3.7 [GPUs**] | - | - | [![Build Status](https://dev.azure.com/PytorchLightning/pytorch-lightning/_apis/build/status/PyTorchLightning.pytorch-lightning?branchName=master)](https://dev.azure.com/PytorchLightning/pytorch-lightning/_build/latest?definitionId=2&branchName=master) | - | - | | Linux py3.{6,7} [TPUs***] | - | - | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | [![TPU tests](https://github.com/PyTorchLightning/pytorch-lightning/workflows/TPU%20tests/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22TPU+tests%22+branch%3Amaster) | - | Linux py3.{6,7} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | OSX py3.{6,7,8} | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | Windows py3.{6,7,8} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | + | Linux py3.{6,7,8,9} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | + | OSX py3.{6,7,8,9} | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | + | Windows py3.{6,7,8,9} | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - | [![CI complete testing](https://github.com/PyTorchLightning/pytorch-lightning/workflows/CI%20complete%20testing/badge.svg?branch=master&event=push)](https://github.com/PyTorchLightning/pytorch-lightning/actions?query=workflow%3A%22CI+testing%22) | - | - _\** tests run on two NVIDIA K80_ - _\*** tests run on Google GKE TPUv2/3_ diff --git a/setup.py b/setup.py index e53e24ebf0702..0979de8ce9065 100755 --- a/setup.py +++ b/setup.py @@ -110,5 +110,6 @@ 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', ], ) From 76e0e601c2f491ffe1b8dd56abb822ab15ad56a9 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Sun, 13 Dec 2020 10:35:38 +0100 Subject: [PATCH 02/31] update CI --- .github/workflows/ci_test-base.yml | 2 +- .github/workflows/ci_test-full.yml | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_test-base.yml b/.github/workflows/ci_test-base.yml index 77363992718af..cf92c64f1feec 100644 --- a/.github/workflows/ci_test-base.yml +++ b/.github/workflows/ci_test-base.yml @@ -16,7 +16,7 @@ jobs: # max-parallel: 6 matrix: os: [ubuntu-20.04, windows-2019, macOS-10.15] - python-version: [3.7] + python-version: [3.8] # Timeout: https://stackoverflow.com/a/59076067/4521646 timeout-minutes: 20 diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 97bb6c9cac19a..2675e4ee539f5 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -48,6 +48,20 @@ jobs: open(fname, 'w').writelines(lines) shell: python + # versions <= 1.3 may have issues on mac with some BLAS ops due to missing mkl (https://github.com/pytorch/pytorch/issues/18996) + - name: Adjust minimal for Python 3.9 + if: matrix.requires == 'minimal' && matrix.python-version == 3.9 + run: | + import re + def _req(fname, ptn, ver): + req = re.sub(ptn, ver, open(fname).read()) + open(fname, 'w').write(req) + + _req(requirements.txt', r'torch>=[\d\.]+', 'torch>=1.7.1') + _req('requirements/examples.txt', r'torchvision>=[\d\.]+', 'torchvision>=0.8.2') + _req('requirements/extra.txt', r'torchtext>=[\d\.]+', 'torchtext>=0.8.1') + shell: python + - name: Set min. dependencies if: matrix.requires == 'minimal' run: | From a1f16761c7e2971d19087bf079f1536824c73561 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Sun, 13 Dec 2020 11:13:54 +0100 Subject: [PATCH 03/31] onnxruntime --- .github/workflows/ci_test-full.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 2675e4ee539f5..c67f8a4f9fcb7 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -30,7 +30,8 @@ jobs: - name: Update Pip run: | # todo: unfreeze PIP after resolving minimal dependencies - pip install --quiet "pip==20.1" --upgrade --user # needed for get pip cacher folder + # pip install --quiet "pip==20.1" --upgrade --user # needed for get pip cacher folder + pip install "pip==20.3" -U # needed for later instalation onnxruntime # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646 - name: Setup macOS @@ -57,7 +58,7 @@ jobs: req = re.sub(ptn, ver, open(fname).read()) open(fname, 'w').write(req) - _req(requirements.txt', r'torch>=[\d\.]+', 'torch>=1.7.1') + _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.7.1') _req('requirements/examples.txt', r'torchvision>=[\d\.]+', 'torchvision>=0.8.2') _req('requirements/extra.txt', r'torchtext>=[\d\.]+', 'torchtext>=0.8.1') shell: python From 647bfdb405ef16036f654eab5b8a52631c7ce623 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Sun, 24 Jan 2021 23:06:47 +0100 Subject: [PATCH 04/31] . --- .github/workflows/ci_test-full.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index c67f8a4f9fcb7..17f63edc047aa 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -54,13 +54,9 @@ jobs: if: matrix.requires == 'minimal' && matrix.python-version == 3.9 run: | import re - def _req(fname, ptn, ver): - req = re.sub(ptn, ver, open(fname).read()) - open(fname, 'w').write(req) - - _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.7.1') - _req('requirements/examples.txt', r'torchvision>=[\d\.]+', 'torchvision>=0.8.2') - _req('requirements/extra.txt', r'torchtext>=[\d\.]+', 'torchtext>=0.8.1') + fname = 'requirements.txt' + req = re.sub(r'torch>=[\d\.]+', 'torch>=1.7.1', open(fname).read()) + open(fname, 'w').write(req) shell: python - name: Set min. dependencies From ab09e0e296d830ba0b9c1e69a87289bcb8b907b9 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Sun, 24 Jan 2021 23:22:34 +0100 Subject: [PATCH 05/31] . --- .github/workflows/ci_test-full.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 17f63edc047aa..f96aed0191667 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -30,8 +30,7 @@ jobs: - name: Update Pip run: | # todo: unfreeze PIP after resolving minimal dependencies - # pip install --quiet "pip==20.1" --upgrade --user # needed for get pip cacher folder - pip install "pip==20.3" -U # needed for later instalation onnxruntime + pip install --quiet "pip==20.1" --upgrade --user # needed for get pip cacher folder # Github Actions: Run step on specific OS: https://stackoverflow.com/a/57948488/4521646 - name: Setup macOS @@ -53,10 +52,8 @@ jobs: - name: Adjust minimal for Python 3.9 if: matrix.requires == 'minimal' && matrix.python-version == 3.9 run: | - import re - fname = 'requirements.txt' - req = re.sub(r'torch>=[\d\.]+', 'torch>=1.7.1', open(fname).read()) - open(fname, 'w').write(req) + pip install "pip==20.3" -U # needed for later instalation onnxruntime + python -c "import re ; fname='requirements.txt' ; req = re.sub(r'torch>=[\d\.]+', 'torch>=1.7.1', open(fname).read()) ; open(fname, 'w').write(req)" shell: python - name: Set min. dependencies From 584614d1abbefd7117de84d0fecac1d3567bfb07 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Tue, 26 Jan 2021 09:48:00 +0100 Subject: [PATCH 06/31] onnxruntime --- .github/workflows/ci_test-full.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index f96aed0191667..4b77242ee6bb9 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -48,13 +48,12 @@ jobs: open(fname, 'w').writelines(lines) shell: python - # versions <= 1.3 may have issues on mac with some BLAS ops due to missing mkl (https://github.com/pytorch/pytorch/issues/18996) - name: Adjust minimal for Python 3.9 if: matrix.requires == 'minimal' && matrix.python-version == 3.9 run: | pip install "pip==20.3" -U # needed for later instalation onnxruntime python -c "import re ; fname='requirements.txt' ; req = re.sub(r'torch>=[\d\.]+', 'torch>=1.7.1', open(fname).read()) ; open(fname, 'w').write(req)" - shell: python + shell: bash - name: Set min. dependencies if: matrix.requires == 'minimal' @@ -101,6 +100,21 @@ jobs: unzip -o checkpoints.zip ls -l checkpoints/ + - name: Build from source for Python 3.9 + if: matrix.python-version == 3.9 + run: | + # https://github.com/microsoft/onnxruntime/blob/master/BUILD.md + git clone --recursive https://github.com/Microsoft/onnxruntime + cd onnxruntime + if [[ runner.os == 'windows' ]]; then + .\build.bat --config RelWithDebInfo --build_shared_lib --parallel + elif [[ runner.os == 'macOS' ]]; then + ./build.sh --config RelWithDebInfo --build_shared_lib --parallel + else + ./build.sh --config RelWithDebInfo --build_shared_lib --parallel --use_xcode + fi + shell: bash + - name: Install dependencies env: # MAKEFLAGS: "-j2" From ff010d7b280794943d0fdcb88d0c67cb0ce552ab Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 27 Jan 2021 13:35:47 +0100 Subject: [PATCH 07/31] t 55 --- .github/workflows/ci_test-full.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 4b77242ee6bb9..3f6c61e215573 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -19,7 +19,10 @@ jobs: requires: ['minimal', 'latest'] # Timeout: https://stackoverflow.com/a/59076067/4521646 - timeout-minutes: 35 # TODO: the macOS is taking too long, probably caching did not work... + # TODO: the macOS is taking too long, probably caching did not work... + # todo: building own onnxruntime takes very long... + timeout-minutes: 55 + steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From 259619811219b9dd098ba83e96e83a88eb6ee841 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 27 Jan 2021 15:04:52 +0100 Subject: [PATCH 08/31] t 75 --- .github/workflows/ci_test-full.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 3f6c61e215573..dd82727e34e0d 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -21,7 +21,7 @@ jobs: # Timeout: https://stackoverflow.com/a/59076067/4521646 # TODO: the macOS is taking too long, probably caching did not work... # todo: building own onnxruntime takes very long... - timeout-minutes: 55 + timeout-minutes: 75 steps: - uses: actions/checkout@v2 @@ -106,6 +106,7 @@ jobs: - name: Build from source for Python 3.9 if: matrix.python-version == 3.9 run: | + python -c "import re ; fname='requirements/extra.txt' ; req = re.sub(r'onnxruntime>=[\d\.]+', '', open(fname).read()) ; open(fname, 'w').write(req)" # https://github.com/microsoft/onnxruntime/blob/master/BUILD.md git clone --recursive https://github.com/Microsoft/onnxruntime cd onnxruntime From 572f6aab51570fe6379ca4fc3e07ba68a50f0638 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 27 Jan 2021 16:46:44 +0100 Subject: [PATCH 09/31] add script --- .../{install_AMP.sh => install_Apex.sh} | 0 requirements/install_onnx.sh | 23 +++++++++++++++++++ tests/README.md | 2 +- 3 files changed, 24 insertions(+), 1 deletion(-) rename requirements/{install_AMP.sh => install_Apex.sh} (100%) create mode 100644 requirements/install_onnx.sh diff --git a/requirements/install_AMP.sh b/requirements/install_Apex.sh similarity index 100% rename from requirements/install_AMP.sh rename to requirements/install_Apex.sh diff --git a/requirements/install_onnx.sh b/requirements/install_onnx.sh new file mode 100644 index 0000000000000..c33b168e4e72d --- /dev/null +++ b/requirements/install_onnx.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +ROOT=$PWD + +git clone --recursive https://github.com/Microsoft/onnxruntime +cd onnxruntime + +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + ./build.sh --config RelWithDebInfo --build_shared_lib --parallel +elif [[ "$OSTYPE" == "darwin"* ]]; then + ./build.sh --config RelWithDebInfo --build_shared_lib --parallel --use_xcode +elif [[ "$OSTYPE" == "cygwin" ]]; then + # POSIX compatibility layer and Linux environment emulation for Windows +elif [[ "$OSTYPE" == "msys" ]]; then + # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) +elif [[ "$OSTYPE" == "win32" ]]; then + .\build.bat --config RelWithDebInfo --build_shared_lib --parallel +else + echo $OSTYPE # Unknown. +fi + +cd $ROOT +rm -rf onnxruntime diff --git a/tests/README.md b/tests/README.md index 7b857a1901fd7..0b0563a3ae540 100644 --- a/tests/README.md +++ b/tests/README.md @@ -17,7 +17,7 @@ git clone https://github.com/PyTorchLightning/pytorch-lightning cd pytorch-lightning # install AMP support -bash requirements/install_AMP.sh +bash requirements/install_Apex.sh # install dev deps pip install -r requirements/devel.txt From 3b0b161d8baeacc8a0bc8b885bafde304545db75 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 27 Jan 2021 16:50:03 +0100 Subject: [PATCH 10/31] use --- .github/workflows/ci_test-full.yml | 11 +---------- requirements/install_ONNX.sh | 27 +++++++++++++++++++++++++++ requirements/install_onnx.sh | 23 ----------------------- 3 files changed, 28 insertions(+), 33 deletions(-) create mode 100644 requirements/install_ONNX.sh delete mode 100644 requirements/install_onnx.sh diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index dd82727e34e0d..c479a49eba83b 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -107,16 +107,7 @@ jobs: if: matrix.python-version == 3.9 run: | python -c "import re ; fname='requirements/extra.txt' ; req = re.sub(r'onnxruntime>=[\d\.]+', '', open(fname).read()) ; open(fname, 'w').write(req)" - # https://github.com/microsoft/onnxruntime/blob/master/BUILD.md - git clone --recursive https://github.com/Microsoft/onnxruntime - cd onnxruntime - if [[ runner.os == 'windows' ]]; then - .\build.bat --config RelWithDebInfo --build_shared_lib --parallel - elif [[ runner.os == 'macOS' ]]; then - ./build.sh --config RelWithDebInfo --build_shared_lib --parallel - else - ./build.sh --config RelWithDebInfo --build_shared_lib --parallel --use_xcode - fi + bash requirements/install_ONNX.sh shell: bash - name: Install dependencies diff --git a/requirements/install_ONNX.sh b/requirements/install_ONNX.sh new file mode 100644 index 0000000000000..06d36faf186ac --- /dev/null +++ b/requirements/install_ONNX.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +ROOT=$PWD + +# https://github.com/microsoft/onnxruntime/blob/master/BUILD.md +git clone --recursive https://github.com/Microsoft/onnxruntime +cd onnxruntime + +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + ./build.sh --config RelWithDebInfo --build_shared_lib --parallel +elif [[ "$OSTYPE" == "darwin"* ]]; then + # Mac OSX + ./build.sh --config RelWithDebInfo --build_shared_lib --parallel --use_xcode +elif [[ "$OSTYPE" == "cygwin" ]]; then + # POSIX compatibility layer and Linux environment emulation for Windows + ./build.sh --config RelWithDebInfo --build_shared_lib --parallel +elif [[ "$OSTYPE" == "msys" ]]; then + # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) + .\build.bat --config RelWithDebInfo --build_shared_lib --parallel +elif [[ "$OSTYPE" == "win32" ]]; then + .\build.bat --config RelWithDebInfo --build_shared_lib --parallel +else + echo $OSTYPE # Unknown. +fi + +cd $ROOT +rm -rf onnxruntime diff --git a/requirements/install_onnx.sh b/requirements/install_onnx.sh deleted file mode 100644 index c33b168e4e72d..0000000000000 --- a/requirements/install_onnx.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -ROOT=$PWD - -git clone --recursive https://github.com/Microsoft/onnxruntime -cd onnxruntime - -if [[ "$OSTYPE" == "linux-gnu"* ]]; then - ./build.sh --config RelWithDebInfo --build_shared_lib --parallel -elif [[ "$OSTYPE" == "darwin"* ]]; then - ./build.sh --config RelWithDebInfo --build_shared_lib --parallel --use_xcode -elif [[ "$OSTYPE" == "cygwin" ]]; then - # POSIX compatibility layer and Linux environment emulation for Windows -elif [[ "$OSTYPE" == "msys" ]]; then - # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) -elif [[ "$OSTYPE" == "win32" ]]; then - .\build.bat --config RelWithDebInfo --build_shared_lib --parallel -else - echo $OSTYPE # Unknown. -fi - -cd $ROOT -rm -rf onnxruntime From 3294506f43af8f0c0a7b800e46d2738972642b20 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 27 Jan 2021 20:46:55 +0100 Subject: [PATCH 11/31] onnx --- .github/workflows/ci_test-full.yml | 1 + requirements/install_ONNX.sh | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index c479a49eba83b..0bdfd2df1e263 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -106,6 +106,7 @@ jobs: - name: Build from source for Python 3.9 if: matrix.python-version == 3.9 run: | + python -c "import re ; fname='requirements/extra.txt' ; req = re.sub(r'onnx>=[\d\.]+', '', open(fname).read()) ; open(fname, 'w').write(req)" python -c "import re ; fname='requirements/extra.txt' ; req = re.sub(r'onnxruntime>=[\d\.]+', '', open(fname).read()) ; open(fname, 'w').write(req)" bash requirements/install_ONNX.sh shell: bash diff --git a/requirements/install_ONNX.sh b/requirements/install_ONNX.sh index 06d36faf186ac..a724d4033da14 100644 --- a/requirements/install_ONNX.sh +++ b/requirements/install_ONNX.sh @@ -2,6 +2,15 @@ ROOT=$PWD +python -m pip install protobuf +git clone https://github.com/onnx/onnx.git +cd onnx +git submodule update --init --recursive +python setup.py install +cd $ROOT +rm -rf onnx + + # https://github.com/microsoft/onnxruntime/blob/master/BUILD.md git clone --recursive https://github.com/Microsoft/onnxruntime cd onnxruntime From a4a8b2f728ae2c21e57c7837bd2d62cb0d51a21c Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 27 Jan 2021 21:42:13 +0100 Subject: [PATCH 12/31] onnx --- requirements/install_ONNX.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements/install_ONNX.sh b/requirements/install_ONNX.sh index a724d4033da14..f5a594bdbe8fb 100644 --- a/requirements/install_ONNX.sh +++ b/requirements/install_ONNX.sh @@ -32,5 +32,9 @@ else echo $OSTYPE # Unknown. fi +export ONNX_ML=1 +python setup.py bdist_wheel +pip install --upgrade dist/*.whl + cd $ROOT rm -rf onnxruntime From c1df0126c9094040485b09f14737f2fba215c594 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Wed, 27 Jan 2021 22:34:37 +0100 Subject: [PATCH 13/31] onnx --- requirements/install_ONNX.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/requirements/install_ONNX.sh b/requirements/install_ONNX.sh index f5a594bdbe8fb..d29e47e4eea68 100644 --- a/requirements/install_ONNX.sh +++ b/requirements/install_ONNX.sh @@ -2,13 +2,13 @@ ROOT=$PWD -python -m pip install protobuf -git clone https://github.com/onnx/onnx.git -cd onnx -git submodule update --init --recursive -python setup.py install -cd $ROOT -rm -rf onnx +# python -m pip install protobuf +# git clone --recursive https://github.com/onnx/onnx.git +# cd onnx +# python setup.py bdist_wheel +# pip install --upgrade dist/*.whl +# cd $ROOT +# rm -rf onnx # https://github.com/microsoft/onnxruntime/blob/master/BUILD.md @@ -33,6 +33,7 @@ else fi export ONNX_ML=1 +pip install setuptools wheel python setup.py bdist_wheel pip install --upgrade dist/*.whl From cd1bd24af01bf6b5492dcde3673361081937aef2 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 28 Jan 2021 00:51:00 +0100 Subject: [PATCH 14/31] whl --- requirements/install_ONNX.sh | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/requirements/install_ONNX.sh b/requirements/install_ONNX.sh index d29e47e4eea68..a9fd24bf22a7c 100644 --- a/requirements/install_ONNX.sh +++ b/requirements/install_ONNX.sh @@ -14,28 +14,27 @@ ROOT=$PWD # https://github.com/microsoft/onnxruntime/blob/master/BUILD.md git clone --recursive https://github.com/Microsoft/onnxruntime cd onnxruntime +export ONNX_ML=1 +pip install setuptools wheel if [[ "$OSTYPE" == "linux-gnu"* ]]; then - ./build.sh --config RelWithDebInfo --build_shared_lib --parallel + ./build.sh --config RelWithDebInfo --build_shared_lib --build_wheel --parallel elif [[ "$OSTYPE" == "darwin"* ]]; then # Mac OSX - ./build.sh --config RelWithDebInfo --build_shared_lib --parallel --use_xcode + ./build.sh --config RelWithDebInfo --build_shared_lib --build_wheel --parallel --use_xcode elif [[ "$OSTYPE" == "cygwin" ]]; then # POSIX compatibility layer and Linux environment emulation for Windows - ./build.sh --config RelWithDebInfo --build_shared_lib --parallel + ./build.sh --config RelWithDebInfo --build_shared_lib --build_wheel --parallel elif [[ "$OSTYPE" == "msys" ]]; then # Lightweight shell and GNU utilities compiled for Windows (part of MinGW) - .\build.bat --config RelWithDebInfo --build_shared_lib --parallel + .\build.bat --config RelWithDebInfo --build_shared_lib --build_wheel --parallel elif [[ "$OSTYPE" == "win32" ]]; then - .\build.bat --config RelWithDebInfo --build_shared_lib --parallel + .\build.bat --config RelWithDebInfo --build_shared_lib --build_wheel --parallel else echo $OSTYPE # Unknown. fi -export ONNX_ML=1 -pip install setuptools wheel -python setup.py bdist_wheel -pip install --upgrade dist/*.whl +pip install --upgrade $(shell find . -name "*.whl") cd $ROOT rm -rf onnxruntime From 0cc4edd2585f4a152b672fbc0f7801cb2a73e6d5 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 28 Jan 2021 01:18:18 +0100 Subject: [PATCH 15/31] np --- requirements/install_ONNX.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/install_ONNX.sh b/requirements/install_ONNX.sh index a9fd24bf22a7c..61235caf06b25 100644 --- a/requirements/install_ONNX.sh +++ b/requirements/install_ONNX.sh @@ -15,7 +15,7 @@ ROOT=$PWD git clone --recursive https://github.com/Microsoft/onnxruntime cd onnxruntime export ONNX_ML=1 -pip install setuptools wheel +pip install setuptools wheel numpy if [[ "$OSTYPE" == "linux-gnu"* ]]; then ./build.sh --config RelWithDebInfo --build_shared_lib --build_wheel --parallel From 352aaa8f1a0d17435a89efa2016baaa8d2334967 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 28 Jan 2021 11:18:07 +0100 Subject: [PATCH 16/31] find --- requirements/install_ONNX.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements/install_ONNX.sh b/requirements/install_ONNX.sh index 61235caf06b25..d6784fa373d6f 100644 --- a/requirements/install_ONNX.sh +++ b/requirements/install_ONNX.sh @@ -34,7 +34,8 @@ else echo $OSTYPE # Unknown. fi -pip install --upgrade $(shell find . -name "*.whl") +find . -name "*.whl" +pip install --upgrade $(find . -name "*.whl") cd $ROOT rm -rf onnxruntime From 5cf81e031323ae2419075b9d91b36afd665ab0cb Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 28 Jan 2021 12:52:17 +0100 Subject: [PATCH 17/31] 21 --- .github/workflows/ci_test-full.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 0bdfd2df1e263..97d82d572497d 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -54,7 +54,6 @@ jobs: - name: Adjust minimal for Python 3.9 if: matrix.requires == 'minimal' && matrix.python-version == 3.9 run: | - pip install "pip==20.3" -U # needed for later instalation onnxruntime python -c "import re ; fname='requirements.txt' ; req = re.sub(r'torch>=[\d\.]+', 'torch>=1.7.1', open(fname).read()) ; open(fname, 'w').write(req)" shell: bash @@ -106,6 +105,7 @@ jobs: - name: Build from source for Python 3.9 if: matrix.python-version == 3.9 run: | + pip install "pip>=21.0" -U # needed for later instalation onnxruntime python -c "import re ; fname='requirements/extra.txt' ; req = re.sub(r'onnx>=[\d\.]+', '', open(fname).read()) ; open(fname, 'w').write(req)" python -c "import re ; fname='requirements/extra.txt' ; req = re.sub(r'onnxruntime>=[\d\.]+', '', open(fname).read()) ; open(fname, 'w').write(req)" bash requirements/install_ONNX.sh From b6193b60fb16662e94107d3ef10e2d5cdd74b324 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 4 Mar 2021 09:18:42 +0100 Subject: [PATCH 18/31] Apply suggestions from code review --- .github/workflows/ci_test-full.yml | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 97d82d572497d..59ccf1afb3759 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -20,8 +20,7 @@ jobs: # Timeout: https://stackoverflow.com/a/59076067/4521646 # TODO: the macOS is taking too long, probably caching did not work... - # todo: building own onnxruntime takes very long... - timeout-minutes: 75 + timeout-minutes: 40 steps: - uses: actions/checkout@v2 @@ -54,8 +53,11 @@ jobs: - name: Adjust minimal for Python 3.9 if: matrix.requires == 'minimal' && matrix.python-version == 3.9 run: | - python -c "import re ; fname='requirements.txt' ; req = re.sub(r'torch>=[\d\.]+', 'torch>=1.7.1', open(fname).read()) ; open(fname, 'w').write(req)" - shell: bash + import re + fname='requirements.txt' + req = re.sub(r'torch>=[\d\.]+', 'torch>=1.7.1', open(fname).read()) + open(fname, 'w').write(req)" + shell: python - name: Set min. dependencies if: matrix.requires == 'minimal' @@ -102,15 +104,6 @@ jobs: unzip -o checkpoints.zip ls -l checkpoints/ - - name: Build from source for Python 3.9 - if: matrix.python-version == 3.9 - run: | - pip install "pip>=21.0" -U # needed for later instalation onnxruntime - python -c "import re ; fname='requirements/extra.txt' ; req = re.sub(r'onnx>=[\d\.]+', '', open(fname).read()) ; open(fname, 'w').write(req)" - python -c "import re ; fname='requirements/extra.txt' ; req = re.sub(r'onnxruntime>=[\d\.]+', '', open(fname).read()) ; open(fname, 'w').write(req)" - bash requirements/install_ONNX.sh - shell: bash - - name: Install dependencies env: # MAKEFLAGS: "-j2" From 8d470c6b84e9f688a97f5d75c67dad57fccbd42b Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 15 Mar 2021 00:23:15 +0100 Subject: [PATCH 19/31] Apply suggestions from code review --- .github/workflows/ci_test-full.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 59ccf1afb3759..03463637f4fe7 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -56,7 +56,7 @@ jobs: import re fname='requirements.txt' req = re.sub(r'torch>=[\d\.]+', 'torch>=1.7.1', open(fname).read()) - open(fname, 'w').write(req)" + open(fname, 'w').write(req) shell: python - name: Set min. dependencies From 41c00dd1f62b574833b0139aad48e8fa8b4a644c Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 15 Mar 2021 00:53:05 +0100 Subject: [PATCH 20/31] onnx --- .github/workflows/ci_test-full.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 03463637f4fe7..d828b64dca0dc 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -54,9 +54,12 @@ jobs: if: matrix.requires == 'minimal' && matrix.python-version == 3.9 run: | import re - fname='requirements.txt' - req = re.sub(r'torch>=[\d\.]+', 'torch>=1.7.1', open(fname).read()) - open(fname, 'w').write(req) + def _req(fname, ptn, ver): + req = re.sub(ptn, ver, open(fname).read()) + open(fname, 'w').write(req) + + _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.7.1') + _req('requirements/extra.txt', r'onnxruntime>=[\d\.]+', 'onnxruntime>=1.7.0') shell: python - name: Set min. dependencies From bd7eb04583bb596e092a939f0503f19896b4f758 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 15 Mar 2021 00:57:07 +0100 Subject: [PATCH 21/31] CI --- .github/workflows/ci_dockers.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_dockers.yml b/.github/workflows/ci_dockers.yml index b7bc93d1f58d0..c1ff6ff6b2bdf 100644 --- a/.github/workflows/ci_dockers.yml +++ b/.github/workflows/ci_dockers.yml @@ -109,7 +109,9 @@ jobs: pytorch_version: 1.4 - python_version: 3.7 pytorch_version: 1.7 - - python_version: 3.7 + - python_version: 3.8 + pytorch_version: 1.8 + - python_version: 3.9 pytorch_version: 1.8 steps: - name: Checkout From c13681adc2ecca836456cd56a1060d7884b7ad53 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 15 Mar 2021 01:09:33 +0100 Subject: [PATCH 22/31] req --- requirements/test.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/test.txt b/requirements/test.txt index 259cc2e2d6442..3c81479e148c2 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -6,12 +6,12 @@ pytest>=6.0 flake8>=3.6 check-manifest twine==3.2 -scikit-learn>=0.22.2 -scikit-image>=0.17.2 isort>=5.6.4 mypy>=0.720, <0.800 pre-commit>=1.0 cloudpickle>=1.3 +scikit-learn>0.22.1 +scikit-image>0.17.1 nltk>=3.3 pandas # needed in benchmarks From bcc0b5da8a672545226a15a96be7f37eccbdde0c Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 15 Mar 2021 01:23:30 +0100 Subject: [PATCH 23/31] ~ dockers --- .github/workflows/ci_dockers.yml | 8 ++++---- .github/workflows/events-nightly.yml | 2 +- .github/workflows/release-docker.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_dockers.yml b/.github/workflows/ci_dockers.yml index c1ff6ff6b2bdf..a22be81b398fb 100644 --- a/.github/workflows/ci_dockers.yml +++ b/.github/workflows/ci_dockers.yml @@ -75,8 +75,8 @@ jobs: pytorch_version: 1.6 - python_version: 3.8 pytorch_version: 1.7 - - python_version: 3.9 - pytorch_version: 1.7 + # - python_version: 3.9 + # pytorch_version: 1.7 steps: - name: Checkout uses: actions/checkout@v2 @@ -111,8 +111,8 @@ jobs: pytorch_version: 1.7 - python_version: 3.8 pytorch_version: 1.8 - - python_version: 3.9 - pytorch_version: 1.8 + # - python_version: 3.9 + # pytorch_version: 1.8 steps: - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/events-nightly.yml b/.github/workflows/events-nightly.yml index c14ab316cd8cc..5ad4396a006f7 100644 --- a/.github/workflows/events-nightly.yml +++ b/.github/workflows/events-nightly.yml @@ -79,7 +79,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: [3.6, 3.7, 3.8, 3.9] + python_version: [3.6, 3.7, 3.8] pytorch_version: [1.4, 1.5, 1.6, 1.7, 1.8] steps: diff --git a/.github/workflows/release-docker.yml b/.github/workflows/release-docker.yml index 69bac2ef2188a..4838ac915bd47 100644 --- a/.github/workflows/release-docker.yml +++ b/.github/workflows/release-docker.yml @@ -13,8 +13,8 @@ jobs: strategy: fail-fast: false matrix: - python_version: [3.6, 3.7, 3.8, 3.9] - pytorch_version: [1.4, 1.5, 1.6, 1.7] + python_version: [3.6, 3.7, 3.8] + pytorch_version: [1.4, 1.5, 1.6, 1.7, 1.8] steps: - name: Checkout uses: actions/checkout@v2 From 2b94910bec0128100fbbbf4c4f483e47f5a575b2 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Mon, 15 Mar 2021 01:26:05 +0100 Subject: [PATCH 24/31] min --- .github/workflows/ci_test-full.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index d828b64dca0dc..c76345f56a036 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -17,6 +17,10 @@ jobs: os: [ubuntu-18.04, windows-2019, macOS-10.15] python-version: [3.6, 3.7, 3.8, 3.9] requires: ['minimal', 'latest'] + exclude: + - python-version: 3.9 + os: windows-2019 + requires: 'minimal' # Timeout: https://stackoverflow.com/a/59076067/4521646 # TODO: the macOS is taking too long, probably caching did not work... From 86e85adff6c2ad0b5c95f8d6ff27c942b2499ac6 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Thu, 25 Mar 2021 21:17:20 +0100 Subject: [PATCH 25/31] . --- .github/workflows/ci_test-full.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index c76345f56a036..496bd3f6da7fa 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -33,7 +33,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Update Pip + - name: Update pip run: | # todo: unfreeze PIP after resolving minimal dependencies pip install --quiet "pip==20.1" --upgrade --user # needed for get pip cacher folder @@ -62,7 +62,7 @@ jobs: req = re.sub(ptn, ver, open(fname).read()) open(fname, 'w').write(req) - _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.7.1') + _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.8.0') _req('requirements/extra.txt', r'onnxruntime>=[\d\.]+', 'onnxruntime>=1.7.0') shell: python From 7f9452c3c1834059ef79a8fa2aad1ed85c5b5d40 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 26 Mar 2021 08:38:23 +0100 Subject: [PATCH 26/31] drop horovod --- .github/workflows/ci_test-full.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 496bd3f6da7fa..d3094005b56ad 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -60,6 +60,8 @@ jobs: import re def _req(fname, ptn, ver): req = re.sub(ptn, ver, open(fname).read()) + # todo: temporary drop also Horovod + req = re.sub(r"horovod[<>=\d\.]+", ver, req) open(fname, 'w').write(req) _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.8.0') @@ -130,7 +132,8 @@ jobs: shell: bash - name: Reinstall Horovod if necessary - if: runner.os != 'windows' + # re-enable horovod on py3.9 when it will be supported + if: runner.os != 'windows' && matrix.python-version != 3.9 env: HOROVOD_BUILD_ARCH_FLAGS: "-mfma" run: | From 3a6c3390a99bd897366c8c929a8def8769abf21e Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 26 Mar 2021 09:06:45 +0100 Subject: [PATCH 27/31] drop horovod --- .github/workflows/ci_test-full.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index d3094005b56ad..9687a300986e7 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -54,14 +54,18 @@ jobs: open(fname, 'w').writelines(lines) shell: python + - name: Temp fix 3.9 - Horovod + if: matrix.python-version == 3.9 + run: | + pip uninstall -y horovod + python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)" + - name: Adjust minimal for Python 3.9 if: matrix.requires == 'minimal' && matrix.python-version == 3.9 run: | import re def _req(fname, ptn, ver): req = re.sub(ptn, ver, open(fname).read()) - # todo: temporary drop also Horovod - req = re.sub(r"horovod[<>=\d\.]+", ver, req) open(fname, 'w').write(req) _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.8.0') From 22c472b5826d7ae1836099a231ad8e856ce2ea06 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 26 Mar 2021 09:21:01 +0100 Subject: [PATCH 28/31] drop horovod --- .github/workflows/ci_test-full.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 9687a300986e7..ba8a52d749a5f 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -54,12 +54,6 @@ jobs: open(fname, 'w').writelines(lines) shell: python - - name: Temp fix 3.9 - Horovod - if: matrix.python-version == 3.9 - run: | - pip uninstall -y horovod - python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)" - - name: Adjust minimal for Python 3.9 if: matrix.requires == 'minimal' && matrix.python-version == 3.9 run: | @@ -117,6 +111,12 @@ jobs: unzip -o checkpoints.zip ls -l checkpoints/ + - name: py3.9 - temp skip Horovod + if: matrix.python-version == 3.9 + run: | + pip uninstall -y horovod + python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)" + - name: Install dependencies env: # MAKEFLAGS: "-j2" From 67d5cfa750ff9f6450e0331a2da58a7bf8651bc3 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 26 Mar 2021 09:58:29 +0100 Subject: [PATCH 29/31] fix --- .github/workflows/ci_test-full.yml | 2 +- tests/models/test_horovod.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index ba8a52d749a5f..faca235389803 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -114,7 +114,7 @@ jobs: - name: py3.9 - temp skip Horovod if: matrix.python-version == 3.9 run: | - pip uninstall -y horovod + # pip uninstall -y horovod python -c "fname = 'requirements/extra.txt' ; lines = [line for line in open(fname).readlines() if not line.startswith('horovod')] ; open(fname, 'w').writelines(lines)" - name: Install dependencies diff --git a/tests/models/test_horovod.py b/tests/models/test_horovod.py index 3c8c9b0f36041..5b4a700babd1d 100644 --- a/tests/models/test_horovod.py +++ b/tests/models/test_horovod.py @@ -63,7 +63,7 @@ def _run_horovod(trainer_options, on_gpu=False): assert exit_code == 0 -@RunIf(skip_windows=True) +@RunIf(skip_windows=True, horovod=True) def test_horovod_cpu(tmpdir): """Test Horovod running multi-process on CPU.""" trainer_options = dict( @@ -80,7 +80,7 @@ def test_horovod_cpu(tmpdir): _run_horovod(trainer_options) -@RunIf(skip_windows=True) +@RunIf(skip_windows=True, horovod=True) def test_horovod_cpu_implicit(tmpdir): """Test Horovod without specifying a backend, inferring from env set by `horovodrun`.""" trainer_options = dict( From 9b9468d9fe4db79b474acf89d6e4f1d28255dd08 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 26 Mar 2021 10:03:03 +0100 Subject: [PATCH 30/31] fix --- .github/workflows/ci_test-full.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index faca235389803..6b882eaeaf0ce 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -19,7 +19,6 @@ jobs: requires: ['minimal', 'latest'] exclude: - python-version: 3.9 - os: windows-2019 requires: 'minimal' # Timeout: https://stackoverflow.com/a/59076067/4521646 @@ -54,17 +53,18 @@ jobs: open(fname, 'w').writelines(lines) shell: python - - name: Adjust minimal for Python 3.9 - if: matrix.requires == 'minimal' && matrix.python-version == 3.9 - run: | - import re - def _req(fname, ptn, ver): - req = re.sub(ptn, ver, open(fname).read()) - open(fname, 'w').write(req) - - _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.8.0') - _req('requirements/extra.txt', r'onnxruntime>=[\d\.]+', 'onnxruntime>=1.7.0') - shell: python + # todo: re-enable when allow testing py 3.9 with min config, atm some Hydra issues + #- name: Adjust minimal for Python 3.9 + # if: matrix.requires == 'minimal' && matrix.python-version == 3.9 + # run: | + # import re + # def _req(fname, ptn, ver): + # req = re.sub(ptn, ver, open(fname).read()) + # open(fname, 'w').write(req) + # + # _req('requirements.txt', r'torch>=[\d\.]+', 'torch>=1.8.0') + # _req('requirements/extra.txt', r'onnxruntime>=[\d\.]+', 'onnxruntime>=1.7.0') + # shell: python - name: Set min. dependencies if: matrix.requires == 'minimal' @@ -102,7 +102,6 @@ jobs: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.requires }}- - name: Pull checkpoints from S3 - # todo: consider adding some caching, but ATM all models have less then 100KB run: | cd legacy # wget is simpler but does not work on Windows @@ -111,6 +110,7 @@ jobs: unzip -o checkpoints.zip ls -l checkpoints/ + # todo: re-enable testing with Horovod - name: py3.9 - temp skip Horovod if: matrix.python-version == 3.9 run: | From 30614d9c4b12060b4c184ba2b104c6ce617291d4 Mon Sep 17 00:00:00 2001 From: Jirka Borovec Date: Fri, 26 Mar 2021 10:04:35 +0100 Subject: [PATCH 31/31] . --- .github/workflows/ci_test-full.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_test-full.yml b/.github/workflows/ci_test-full.yml index 6b882eaeaf0ce..ec9e71c5b83b2 100644 --- a/.github/workflows/ci_test-full.yml +++ b/.github/workflows/ci_test-full.yml @@ -136,7 +136,7 @@ jobs: shell: bash - name: Reinstall Horovod if necessary - # re-enable horovod on py3.9 when it will be supported + # todo: re-enable horovod on py3.9 when it will be supported if: runner.os != 'windows' && matrix.python-version != 3.9 env: HOROVOD_BUILD_ARCH_FLAGS: "-mfma"