Merge branch 'master' into master

Lightning-Universe · Jun 29, 2023 · 9caa021 · 9caa021
2 parents a79ec7f + 10411c9
commit 9caa021
Show file tree

Hide file tree

Showing 80 changed files with 629 additions and 869 deletions.
diff --git a/.azure/gpu-tests.yml b/.azure/gpu-tests.yml
@@ -19,10 +19,17 @@ jobs:
           PYTHON_VERSION: '3.8'
           TORCH_VERSION: '1.13.1'
           CUDA_VERSION_MM: '117'
+          runner_image: "nvidia/cuda:11.7.0-cudnn8-runtime-ubuntu20.04"
         'torch v2.0 | py3.9':
           PYTHON_VERSION: '3.9'
           TORCH_VERSION: '2.0.1'
           CUDA_VERSION_MM: '117'
+          runner_image: "nvidia/cuda:11.7.0-cudnn8-runtime-ubuntu20.04"
+        'torch v2.0 | py3.10':
+          PYTHON_VERSION: '3.10'
+          TORCH_VERSION: '2.0.1'
+          CUDA_VERSION_MM: '121'
+          runner_image: "nvidia/cuda:12.1.1-cudnn8-runtime-ubuntu22.04"
     # how long to run the job before automatically cancelling
     timeoutInMinutes: "75"
     # how much time to give 'run always even if cancelled tasks' before stopping them
@@ -32,14 +39,17 @@ jobs:
       DEVICES: $( python -c 'name = "$(Agent.Name)" ; gpus = name.split("_")[-1] if "_" in name else "0,1"; print(gpus)' )
       DEBIAN_FRONTEND: "noninteractive"
       TZ: "Europe/Amsterdam"
+      PIP_CACHE_DIR: "/var/tmp/pip"
       TORCH_HOME: "/var/tmp/torch"
       TRANSFORMERS_CACHE: "/var/tmp/huggingface"
+      DATASETS_PATH: "/var/tmp/bolts_datasets"  # custom for running Bolts
       FREEZE_REQUIREMENTS: 1
 
     container:
       #image: "pytorchlightning/pytorch_lightning:base-cuda-py3.9-torch1.12-cuda11.6.1"
       #image: "pytorch/pytorch:1.13.1-cuda11.6-cudnn8-runtime"
-      image: "nvidia/cuda:11.7.0-cudnn8-runtime-ubuntu20.04"
+      #image: "nvidia/cuda:11.7.0-cudnn8-runtime-ubuntu20.04"
+      image: $(runner_image)
       #endpoint: azureContainerRegistryConnection
       options: "--gpus=all --shm-size=16g -v /usr/bin/docker:/tmp/docker:ro -v /var/tmp:/var/tmp"
     pool: "lit-rtx-3090"
@@ -111,7 +121,9 @@ jobs:
       displayName: Download ROMs
 
     - bash: |
-        python -m pytest tests/ -v --cov=pl_bolts --junitxml=$(Build.StagingDirectory)/test-results.xml --durations=30
+        # todo: optimize runtime, mainly because of retina example
+        python -m pytest tests/ -v --cov=pl_bolts --durations=30 \
+          --junitxml=$(Build.StagingDirectory)/test-results.xml
       displayName: 'Testing'
 
     - bash: |

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: bug / fix, help wanted
+labels: bug, help wanted
 assignees: ''
 ---
 

diff --git a/.github/mergify.yml b/.github/mergify.yml
@@ -45,6 +45,7 @@ pull_request_rules:
       - "#changes-requested-reviews-by=0"  # no requested changes
       - "#check-pending<=2"
       - "#check-failure<5"
+      - "#check-skipped<5"
     actions:
       label:
         add: [ "ready" ]
@@ -55,6 +56,7 @@ pull_request_rules:
         - "#approved-reviews-by=0"  # number of review approvals
         - "#changes-requested-reviews-by>=1"  # no requested changes
         - "#check-failure>=5"
+        - "#check-skipped>=5"
     actions:
       label:
         remove: [ "ready" ]
@@ -67,6 +69,7 @@ pull_request_rules:
       - -title~=(?i)wip # skip all PR that title contains “WIP” (ignoring case)
       - "#approved-reviews-by>=1" # number of review approvals
       - "#changes-requested-reviews-by=0" # no requested changes
+      - "#check-failure<5"
     actions:
       update: {}
 

diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
@@ -21,14 +21,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Something is wrong with windows tests, will need to debug separately.
-        os: [ubuntu-20.04, macOS-11]
-        # todo: seems there are some issues with py3.10 while install dependencies
-        python-version: ["3.8", "3.9"]
-        requires: ['oldest', 'latest']
+        os: ["ubuntu-20.04", "macOS-11"]
+        python-version: ["3.8", "3.9", "3.10"]
+        requires: ["latest"]  # + "oldest"
+        include:
+          - { os: 'windows-2022', python-version: "3.9" }
+          - { os: 'ubuntu-20.04', python-version: "3.8", requires: "oldest" }
+          - { os: 'macOS-11', python-version: "3.8", requires: "oldest" }
     env:
       FREEZE_REQUIREMENTS: 1
       TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
+      TRANSFORMERS_CACHE: _hf_cache
     # Timeout: https://stackoverflow.com/a/59076067/4521646
     # the reason for high number is MUCH slower tests on macOS and py3.8
     timeout-minutes: 50
@@ -50,9 +53,19 @@ jobs:
             open(fpath, 'w').write(req)
       shell: python
 
+    - name: Get pip cache dir
+      id: pip-cache
+      run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
+    - name: Restore pip cache
+      uses: actions/cache/restore@v3
+      id: restore-cache
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: pip-dependencies
+
     - name: Install package
       run: |
-        pip install "pip==22.2.1"  # todo: drop after resolving extras
+        python -m pip install "pip==22.3.1"  # todo: drop after resolving extras
         pip install -e . -U --prefer-binary -f ${TORCH_URL}
         pip list
 
@@ -66,42 +79,41 @@ jobs:
           --ignore=pl_bolts/datasets \
           --ignore=pl_bolts/models/rl
 
-    - name: Get pip cache dir
-      id: pip-cache
-      run: echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-
-    - name: Cache pip
-      uses: actions/cache@v3
-      with:
-        path: ${{ steps.pip-cache.outputs.dir }}
-        key: ${{ runner.os }}-pip-${{ matrix.requires }}-${{ hashFiles('requirements/*.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-${{ matrix.requires }}-
-
     - name: Install dependencies
       run: |
         pip install -r requirements/devel.txt -U -q -f ${TORCH_URL}
         pip list
 
+    - name: Save pip cache
+      if: github.ref == 'refs/heads/master'
+      uses: actions/cache/save@v3
+      with:
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: pip-dependencies
+
     - name: Cache datasets
       id: cache-datasets
       uses: actions/cache@v3
       with:
         path: ./_datasets
+        enableCrossOsArchive: true
         # bump this date if you need update cache
-        key: datasets-20230601
+        key: datasets-20230630
 
     - name: Setup macOS
       if: runner.os == 'macOS' && steps.cache-datasets.outputs.cache-hit != 'true'
       run: |
         brew update
         brew install rar
-
     - name: Setup Ubuntu
       if: runner.os == 'Linux'&& steps.cache-datasets.outputs.cache-hit != 'true'
       run: |
         sudo apt-get update
         sudo apt-get install -y unrar
+    - name: Setup Windows
+      if: runner.os == 'Windows'&& steps.cache-datasets.outputs.cache-hit != 'true'
+      run: |
+        choco install unrar
 
     - name: Download ROMs
       if: steps.cache-datasets.outputs.cache-hit != 'true'
@@ -113,11 +125,24 @@ jobs:
         rm Roms.rar
 
     - name: Init ROMs
-      working-directory: ./_datasets
+      working-directory: _datasets/
       run: python -m atari_py.import_roms ROMS
 
-    - name: Tests
-      run: python -m pytest tests -v --cov=pl_bolts
+    - name: Restore HF cache
+      uses: actions/cache/restore@v3
+      with:
+        path: ${{ env.TRANSFORMERS_CACHE }}
+        key: cache-transformers
+
+    - name: Testing
+      run: python -m pytest tests -v --cov=pl_bolts --timeout=200
+
+    - name: Save HF cache
+      if: github.ref == 'refs/heads/master'
+      uses: actions/cache/save@v3
+      with:
+        path: ${{ env.TRANSFORMERS_CACHE }}
+        key: cache-transformers
 
     - name: Statistics
       if: success()

diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml
@@ -10,79 +10,73 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }}
   cancel-in-progress: ${{ ! (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/heads/release/')) }}
 
+env:
+  FREEZE_REQUIREMENTS: 1
+  TORCH_URL: "https://download.pytorch.org/whl/cpu/torch_stable.html"
+  TRANSFORMERS_CACHE: _hf_cache
+
+defaults:
+  run:
+    shell: bash
+
 jobs:
+
   test-docs:
     runs-on: ubuntu-20.04
-
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
           python-version: 3.9
 
-      # Note: This uses an internal pip API and may not always work
-      # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
       - name: Cache pip
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements/base.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
+          key: pip-${{ hashFiles('requirements/*.txt') }}
+          restore-keys: pip-
 
       - name: Install package & dependencies
         run: |
           pip install "pip==22.2.1"  # todo: drop after resolving extras
-          pip install -e . -r requirements/devel.txt -r docs/requirements.txt \
-            --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
+          pip install -e . -r requirements/devel.txt -r docs/requirements.txt -f $TORCH_URL
           pip list
-        shell: bash
 
       - name: Test Documentation
+        working-directory: docs/
         env:
           SPHINX_MOCK_REQUIREMENTS: 0
         run: |
-          # First run the same pipeline as Read-The-Docs
-          cd docs
           make doctest
           make coverage
 
+
   make-docs:
     runs-on: ubuntu-20.04
-
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
           python-version: 3.9
 
-      # Note: This uses an internal pip API and may not always work
-      # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
       - name: Cache pip
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
+          key: pip-${{ hashFiles('requirements/*.txt') }}
+          restore-keys: pip-
 
-      - name: Install package
-        run: pip install -e . --find-links https://download.pytorch.org/whl/cpu/torch_stable.html
-        shell: bash
-
-      - name: Install dependencies
+      - name: Install package & dependencies
         run: |
           # install Texlive, see https://linuxconfig.org/how-to-install-latex-on-ubuntu-20-04-focal-fossa-linux
           sudo apt-get update && sudo apt-get install -y texlive-latex-extra dvipng texlive-pictures
           pip install "pip==22.2.1"  # todo: drop after resolving extras
-          pip install --requirement docs/requirements.txt
+          pip install -e . -r docs/requirements.txt -f $TORCH_URL
           pip list
-        shell: bash
 
       - name: Make Documentation
+        working-directory: docs/
         run: |
-          # First run the same pipeline as Read-The-Docs
-          cd docs
           make clean
           make html --debug --jobs 2 SPHINXOPTS="-W"
 

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -62,9 +62,10 @@ repos:
         additional_dependencies:
           - flake8-pytest-style
           - flake8-bandit
+          - pep8-naming
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: v0.0.270
+    rev: v0.0.272
     hooks:
       - id: ruff
         args: ["--fix"]
diff --git a/docs/source/models/self_supervised.rst b/docs/source/models/self_supervised.rst
@@ -497,7 +497,7 @@ To Train::
         num_samples=dm.num_unlabeled_samples,
         dataset='stl10',
         batch_size=batch_size,
-        nmb_crops=(2,4)
+        num_crops=(2,4)
     )
 
     # fit
@@ -601,7 +601,7 @@ To reproduce::
         --gaussian_blur
         --queue_length 0
         --jitter_strength 1.
-        --nmb_prototypes 512
+        --num_prototypes 512
 
     # finetune
     python swav_finetuner.py

diff --git a/pyproject.toml b/pyproject.toml
@@ -42,7 +42,6 @@ exclude_lines = [
 
 [tool.coverage.run]
 parallel = true
-# concurrency = "thread"
 relative_files = true
 
 
@@ -65,8 +64,8 @@ select = [
     "E", "W",  # see: https://pypi.org/project/pycodestyle
     "F",  # see: https://pypi.org/project/pyflakes
     "I",  # see: isort
+    "N",  # see: https://pypi.org/project/pep8-naming
 #    "D",  # see: https://pypi.org/project/pydocstyle
-#    "N",  # see: https://pypi.org/project/pep8-naming
 ]
  extend-select = [
      "C4",  # see: https://pypi.org/project/flake8-comprehensions

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -1,6 +1,6 @@
-numpy <1.25.0
+numpy <1.26.0
 pytorch-lightning >1.7.0, <2.0.0  # strict
-torchmetrics <0.11.0  # strict
+torchmetrics <0.12.0
 lightning-utilities >0.3.1  # this is needed for PL 1.7
 torchvision >=0.10.0  # todo: move to topic related extras
 tensorboard >=2.9.1, <2.14.0  # for `TensorBoardLogger`
diff --git a/requirements/loggers.txt b/requirements/loggers.txt
@@ -2,4 +2,4 @@
 # trains >=0.14.1
 matplotlib >3.0.0, <3.8.0
 wandb >0.13.0, <0.16.0
-scipy >1.5.0, <1.11.0
+scipy >1.5.0, <1.12.0
diff --git a/requirements/models.txt b/requirements/models.txt
@@ -1,7 +1,7 @@
 torchvision >=0.10.0
 scikit-learn >=1.0.2
 Pillow >9.0.0
-gym[atari] >=0.17.2, <0.20.0  # strict
+gym[atari] >=0.17.2, <0.22.0  # strict
 atari-py >0.2, !=0.2.6, <0.3  # strict
 box2d-py >2.3, <2.4  # strict
 opencv-python-headless >=4.5.5.62
diff --git a/requirements/test.txt b/requirements/test.txt
@@ -1,9 +1,10 @@
 coverage[toml] >7.0.0, <8.0.0
-pytest ==7.3.2
+pytest ==7.4.0
 pytest-cov ==4.1.0
+pytest-timeout ==2.1.0
 pytest-rerunfailures ==11.1.2
 
 scikit-learn >=1.0.2, <=1.2.2
-sparseml >1.0.0, <1.3.0
+sparseml >1.0.0, <1.6.0
 ale-py >=0.7, <=0.8.1
-jsonargparse[signatures] >4.0.0  # for LightningCLI
+jsonargparse[signatures] >4.0.0, <=4.22.0  # for LightningCLI