From 0e0ee9268faba516bec5fd56be9a4f3ce968ae5c Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Fri, 19 Jul 2024 19:16:37 +1200 Subject: [PATCH 01/16] Add a new `check` extra including `pyright` and `pre-commit`. --- Makefile | 2 +- requirements/dev.txt | 34 ++++++++++++++++++++++++++++------ setup.cfg | 3 +++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 3396b1b7..0250ce60 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ docs-clean: requirements/dev.txt: setup.cfg @# allows you to do this... @# make requirements | tee > requirements/some_file.txt - @pip-compile setup.cfg --rebuild --extra doc --extra test --output-file=- > $@ + @pip-compile setup.cfg --rebuild --extra doc --extra test --extra check --output-file=- > $@ binder/requirements.txt: requirements/dev.txt cp $< $@ diff --git a/requirements/dev.txt b/requirements/dev.txt index 5c1249c4..4f64aac0 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.11 # by the following command: # -# pip-compile --extra=doc --extra=test --output-file=- --strip-extras setup.cfg +# pip-compile --extra=doc --extra=test --extra=check --output-file=- --strip-extras setup.cfg # --index-url https://pypi.python.org/simple/ --trusted-host pypi.org @@ -47,7 +47,7 @@ azure-datalake-store==0.0.53 # via adlfs azure-identity==1.17.1 # via adlfs -azure-storage-blob==12.20.0 +azure-storage-blob==12.21.0 # via adlfs backcall==0.2.0 # via ipython @@ -67,6 +67,8 @@ cffi==1.16.0 # via # azure-datalake-store # cryptography +cfgv==3.4.0 + # via pre-commit charset-normalizer==3.3.2 # via requests click==8.1.7 @@ -93,12 +95,16 @@ decorator==5.1.1 # via # gcsfs # ipython +distlib==0.3.8 + # via virtualenv executing==2.0.1 # via stack-data fastjsonschema==2.20.0 # via nbformat fastparquet==2024.5.0 # via pins (setup.cfg) +filelock==3.15.4 + # via virtualenv frozenlist==1.4.1 # via # aiohttp @@ -141,6 +147,8 @@ griffe==0.48.0 # via quartodoc humanize==4.10.0 # via pins (setup.cfg) +identify==2.6.0 + # via pre-commit idna==3.7 # via # requests @@ -201,7 +209,7 @@ matplotlib-inline==0.1.7 # ipython mdurl==0.1.2 # via markdown-it-py -msal==1.29.0 +msal==1.30.0 # via # azure-datalake-store # azure-identity @@ -220,6 +228,10 @@ nbformat==5.10.4 # pins (setup.cfg) nest-asyncio==1.6.0 # via ipykernel +nodeenv==1.9.1 + # via + # pre-commit + # pyright numpy==2.0.0 # via # fastparquet @@ -247,13 +259,17 @@ pickleshare==0.7.5 pip-tools==7.4.1 # via pins (setup.cfg) platformdirs==4.2.2 - # via jupyter-core + # via + # jupyter-core + # virtualenv pluggy==1.5.0 # via pytest -plum-dispatch==2.5.1.post1 +plum-dispatch==2.5.2 # via quartodoc portalocker==2.10.1 # via msal-extensions +pre-commit==3.7.1 + # via pins (setup.cfg) prompt-toolkit==3.0.47 # via ipython proto-plus==1.24.0 @@ -271,7 +287,7 @@ pure-eval==0.2.2 # via stack-data py==1.11.0 # via pytest -pyarrow==16.1.0 +pyarrow==17.0.0 # via pins (setup.cfg) pyasn1==0.6.0 # via @@ -297,6 +313,8 @@ pyproject-hooks==1.1.0 # via # build # pip-tools +pyright==1.1.372 + # via pins (setup.cfg) pytest==7.1.3 # via # pins (setup.cfg) @@ -320,6 +338,7 @@ pytz==2024.1 pyyaml==6.0.1 # via # pins (setup.cfg) + # pre-commit # quartodoc pyzmq==26.0.3 # via @@ -389,6 +408,7 @@ typing-extensions==4.12.2 # azure-core # azure-identity # azure-storage-blob + # plum-dispatch # pydantic # pydantic-core # quartodoc @@ -398,6 +418,8 @@ urllib3==2.2.2 # via # botocore # requests +virtualenv==20.26.3 + # via pre-commit watchdog==4.0.1 # via quartodoc wcwidth==0.2.13 diff --git a/setup.cfg b/setup.cfg index ff0c547d..40de83c5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -65,6 +65,9 @@ test = fastparquet pyarrow +check = + pre-commit + pyright>=1.1.372 [bdist_wheel] universal = 1 From 4ec28413cdea4ba3d6f09f96d4d541a92b63ef91 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Fri, 19 Jul 2024 21:26:32 +1200 Subject: [PATCH 02/16] Configure pyright to run in CI. --- .github/workflows/code-checks.yml | 9 +++++++++ README.md | 2 +- pyproject.toml | 28 +++++++++++++++++++++++++--- setup.cfg | 2 +- 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 433489a6..07649720 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -16,3 +16,12 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v4 - uses: pre-commit/action@v2.0.3 + + pyright: + name: "Run Pyright" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: jakebailey/pyright-action@v1 + with: + version: 1.1.273 # Manually sync with setup.cfg diff --git a/README.md b/README.md index af007b66..31955ddb 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # pins -![PyPI - Version](https://img.shields.io/pypi/v/pins.svg) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pins) +![PyPI - Version](https://img.shields.io/pypi/v/pins.svg) ![PyPI - Python Version](https://img.shields.io/pypi/pyversions/pins) [![Checked with pyright](https://microsoft.github.io/pyright/img/pyright_badge.svg)](https://microsoft.github.io/pyright/) diff --git a/pyproject.toml b/pyproject.toml index d0b9249e..d7890f4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,8 +6,30 @@ build-backend = "setuptools.build_meta" testpaths = ["pins"] addopts = "--doctest-modules" doctest_optionflags = "NORMALIZE_WHITESPACE" -markers = [ - "skip_on_github: skip this test if running on github", -] +markers = ["skip_on_github: skip this test if running on github"] [tool.setuptools_scm] + +[tool.pyright] +include = ["pins"] +exclude = ["**/__pycache__"] +ignore = ["src/tests"] +pythonVersion = "3.12" # Use the maximum version supported by python-pins +pythonPlatform = "Linux" + +reportArgumentType = false +reportAssignmentType = false +reportAttributeAccessIssue = false +reportCallIssue = false +reportGeneralTypeIssues = false +reportIncompatibleMethodOverride = false +reportIncompatibleVariableOverride = false +reportIndexIssue = false +reportMissingImports = false +reportMissingTypeStubs = false +reportOptionalIterable = false +reportOptionalMemberAccess = false +reportOptionalSubscript = false +reportPossiblyUnboundVariable = false +reportReturnType = false +reportUnusedExpression = false diff --git a/setup.cfg b/setup.cfg index 40de83c5..9b0e020c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -67,7 +67,7 @@ test = check = pre-commit - pyright>=1.1.372 + pyright==1.1.372 # Pinned; manually sync with .github/workflows/code-checks.yml [bdist_wheel] universal = 1 From a3253cc985560437873241d8ca349acfbcd48a41 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Fri, 19 Jul 2024 21:31:37 +1200 Subject: [PATCH 03/16] Use pyright action v2 --- .github/workflows/code-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index 07649720..f5a92be7 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -22,6 +22,6 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: jakebailey/pyright-action@v1 + - uses: jakebailey/pyright-action@v2 with: version: 1.1.273 # Manually sync with setup.cfg From 0a6061b591a8e079ab214e9ab448fb6cdfbdbb66 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Fri, 19 Jul 2024 21:59:15 +1200 Subject: [PATCH 04/16] Install dependencies before running pyright. --- .github/workflows/code-checks.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index f5a92be7..d4a28f64 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -22,6 +22,14 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: 3.12 # Use the maximum version supported by python-pins + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install -e .[check] - uses: jakebailey/pyright-action@v2 with: version: 1.1.273 # Manually sync with setup.cfg From 4762898638eabf059840731053e3d080b5b44481 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Fri, 19 Jul 2024 22:07:54 +1200 Subject: [PATCH 05/16] Fix sync error. --- .github/workflows/code-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml index d4a28f64..59d01343 100644 --- a/.github/workflows/code-checks.yml +++ b/.github/workflows/code-checks.yml @@ -32,4 +32,4 @@ jobs: python -m pip install -e .[check] - uses: jakebailey/pyright-action@v2 with: - version: 1.1.273 # Manually sync with setup.cfg + version: 1.1.372 # Manually sync with setup.cfg From 2105df0edfe12261694e323607f616931c63fe9d Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Sat, 20 Jul 2024 14:22:01 +1200 Subject: [PATCH 06/16] Revert unnecessary bumps in dev.txt --- requirements/dev.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 4f64aac0..3635bd6f 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -47,7 +47,7 @@ azure-datalake-store==0.0.53 # via adlfs azure-identity==1.17.1 # via adlfs -azure-storage-blob==12.21.0 +azure-storage-blob==12.20.0 # via adlfs backcall==0.2.0 # via ipython @@ -169,7 +169,7 @@ ipython==8.12.0 # via # ipykernel # pins (setup.cfg) -isodate==0.6.1 + isodate==0.6.1 # via azure-storage-blob jedi==0.19.1 # via ipython @@ -209,7 +209,7 @@ matplotlib-inline==0.1.7 # ipython mdurl==0.1.2 # via markdown-it-py -msal==1.30.0 +msal==1.29.0 # via # azure-datalake-store # azure-identity @@ -264,7 +264,7 @@ platformdirs==4.2.2 # virtualenv pluggy==1.5.0 # via pytest -plum-dispatch==2.5.2 +plum-dispatch==2.5.1.post1 # via quartodoc portalocker==2.10.1 # via msal-extensions @@ -287,7 +287,7 @@ pure-eval==0.2.2 # via stack-data py==1.11.0 # via pytest -pyarrow==17.0.0 +pyarrow==16.1.0 # via pins (setup.cfg) pyasn1==0.6.0 # via @@ -408,7 +408,6 @@ typing-extensions==4.12.2 # azure-core # azure-identity # azure-storage-blob - # plum-dispatch # pydantic # pydantic-core # quartodoc From e6189d415d46834a1e16cce92ae98f8fdc1e6df4 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Sat, 20 Jul 2024 14:51:14 +1200 Subject: [PATCH 07/16] Ignore the tests folder correctly. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d7890f4d..ddd27848 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ markers = ["skip_on_github: skip this test if running on github"] [tool.pyright] include = ["pins"] exclude = ["**/__pycache__"] -ignore = ["src/tests"] +ignore = ["pins/tests"] pythonVersion = "3.12" # Use the maximum version supported by python-pins pythonPlatform = "Linux" From 986a98b536c20f0bc43be20a6ecd482f98581da4 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Sat, 20 Jul 2024 17:12:19 +1200 Subject: [PATCH 08/16] Add appdirs stubs --- requirements/dev.txt | 2 ++ setup.cfg | 1 + 2 files changed, 3 insertions(+) diff --git a/requirements/dev.txt b/requirements/dev.txt index 3635bd6f..30c614fe 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -403,6 +403,8 @@ traitlets==5.14.3 # matplotlib-inline # nbclient # nbformat +types-appdirs==1.4.3.5 + # via pins (setup.cfg) typing-extensions==4.12.2 # via # azure-core diff --git a/setup.cfg b/setup.cfg index 9b0e020c..7e61e325 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,6 +68,7 @@ test = check = pre-commit pyright==1.1.372 # Pinned; manually sync with .github/workflows/code-checks.yml + types-appdirs [bdist_wheel] universal = 1 From ebd603e8c707aa731b37b0d8685ab02cc851bbe9 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Sat, 20 Jul 2024 17:12:46 +1200 Subject: [PATCH 09/16] Re-enable reportUnusedExpression --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ddd27848..5799c83a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,4 +32,3 @@ reportOptionalMemberAccess = false reportOptionalSubscript = false reportPossiblyUnboundVariable = false reportReturnType = false -reportUnusedExpression = false From de4646a785d6314249323f900840e57bed7dbd99 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Sat, 20 Jul 2024 17:13:16 +1200 Subject: [PATCH 10/16] Add triaging comments for rule disables --- pyproject.toml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 5799c83a..a016a7fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,18 +17,58 @@ ignore = ["pins/tests"] pythonVersion = "3.12" # Use the maximum version supported by python-pins pythonPlatform = "Linux" +# ####################### +# Rules disabled and reasons why +# ####################### + +# None defaults without None in their type hint +# Variety of other issues reportArgumentType = false + +# Only one change needed - to avoid trying to treat Traversable as Path reportAssignmentType = false + +# Variety of issues reportAttributeAccessIssue = false + +# Passing extra args to the IFileSystem Protocol +# Missing overloads for as_df arg in pin_search and pin_versions reportCallIssue = false + +# Return type issues in CachePruner.versions and pins.constructors.board reportGeneralTypeIssues = false + +# Issues with ExtendMethodDoc +# RSConnect implementation of the IFileSystem Protocol reportIncompatibleMethodOverride = false + +# Only one change needed in RsConnectFs; protocol: ClassVar[str | tuple[str, ...]] reportIncompatibleVariableOverride = false + +# Using __delitem__ on a Mapping +# RSConnect API Reponse object issues reportIndexIssue = false + +# https://github.com/rstudio/pins-python/issues/262#issuecomment-2240875273 reportMissingImports = false + +# Missing stubs for fsspec: https://github.com/fsspec/filesystem_spec/issues/625 +# Missing stubs for joblib: https://github.com/joblib/joblib/issues/1516 reportMissingTypeStubs = false + +# RSConnect API _raw_query return type issue reportOptionalIterable = false + +# RSConnect API _raw_query return type issue +# prepare_pin_version unhandled case reportOptionalMemberAccess = false + +# RSConnect API _raw_query return type issue +# RSConnect implementation of the IFileSystem Protocol reportOptionalSubscript = false + +# Mostly False positives arising from fragmented if statements and integer comparisons reportPossiblyUnboundVariable = false + +# Variety of issues reportReturnType = false From e2a0641b04afed5365ac0a1b9da3d443e7b46a32 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Sun, 21 Jul 2024 07:23:25 +1200 Subject: [PATCH 11/16] Revert unnecessary project.toml formatting change --- pyproject.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a016a7fe..9d1fc4fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,9 @@ build-backend = "setuptools.build_meta" testpaths = ["pins"] addopts = "--doctest-modules" doctest_optionflags = "NORMALIZE_WHITESPACE" -markers = ["skip_on_github: skip this test if running on github"] +markers = [ + "skip_on_github: skip this test if running on github", +] [tool.setuptools_scm] From 8e34560c96e083bb91122d25390eabe83fefee95 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Sun, 21 Jul 2024 07:26:04 +1200 Subject: [PATCH 12/16] Revert unnecessary indentation in dev.txt --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 30c614fe..2b2a43a0 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -169,7 +169,7 @@ ipython==8.12.0 # via # ipykernel # pins (setup.cfg) - isodate==0.6.1 +isodate==0.6.1 # via azure-storage-blob jedi==0.19.1 # via ipython From bbeb3dc0a8444374efdeab0df6c99ed2614df610 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Wed, 24 Jul 2024 11:01:09 +1200 Subject: [PATCH 13/16] Update pyproject.toml Co-authored-by: Isabel Zimmerman <54685329+isabelizimm@users.noreply.github.com> --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 48b90007..317118ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,6 +73,7 @@ reportPossiblyUnboundVariable = false # Variety of issues reportReturnType = false +reportUnusedExpression = false [tool.ruff] line-length = 90 extend-exclude = ["docs"] From 839f5583662dccb99c0250116b42fc3439b7d120 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Wed, 24 Jul 2024 11:06:57 +1200 Subject: [PATCH 14/16] Move pyright comments to a tracking issue. --- pyproject.toml | 50 +------------------------------------------------- 1 file changed, 1 insertion(+), 49 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 317118ed..ec56b22c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,67 +17,19 @@ ignore = ["pins/tests"] pythonVersion = "3.12" # Use the maximum version supported by python-pins pythonPlatform = "Linux" -# ####################### -# Rules disabled and reasons why -# ####################### - -# None defaults without None in their type hint -# Variety of other issues +# Tracking compliance with these rules at https://github.com/rstudio/pins-python/issues/272 reportArgumentType = false - -# Only one change needed - to avoid trying to treat Traversable as Path reportAssignmentType = false - -# Variety of issues reportAttributeAccessIssue = false - -# Passing extra args to the IFileSystem Protocol -# Missing overloads for as_df arg in pin_search and pin_versions reportCallIssue = false - -# Return type issues in CachePruner.versions and pins.constructors.board reportGeneralTypeIssues = false - -# Issues with ExtendMethodDoc -# RSConnect implementation of the IFileSystem Protocol reportIncompatibleMethodOverride = false - -# Only one change needed in RsConnectFs; protocol: ClassVar[str | tuple[str, ...]] reportIncompatibleVariableOverride = false - -# Using __delitem__ on a Mapping -# RSConnect API Reponse object issues reportIndexIssue = false - -# https://github.com/rstudio/pins-python/issues/262#issuecomment-2240875273 reportMissingImports = false - -# Missing stubs for fsspec: https://github.com/fsspec/filesystem_spec/issues/625 -# Missing stubs for joblib: https://github.com/joblib/joblib/issues/1516 reportMissingTypeStubs = false - -# RSConnect API _raw_query return type issue reportOptionalIterable = false - -# RSConnect API _raw_query return type issue -# prepare_pin_version unhandled case reportOptionalMemberAccess = false - -# RSConnect API _raw_query return type issue -# RSConnect implementation of the IFileSystem Protocol reportOptionalSubscript = false - -# Mostly False positives arising from fragmented if statements and integer comparisons reportPossiblyUnboundVariable = false - -# Variety of issues reportReturnType = false - -reportUnusedExpression = false -[tool.ruff] -line-length = 90 -extend-exclude = ["docs"] - -[tool.ruff.lint] -select = ["E", "F", "W"] -ignore = ["E501"] From 39b227b12844bbeb8eaf189cb5dd46a4521574d0 Mon Sep 17 00:00:00 2001 From: Nathan McDougall Date: Wed, 24 Jul 2024 11:10:39 +1200 Subject: [PATCH 15/16] Revert removal of config sections in pyproject.toml. --- pyproject.toml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index ec56b22c..b3935bbc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,3 +33,12 @@ reportOptionalMemberAccess = false reportOptionalSubscript = false reportPossiblyUnboundVariable = false reportReturnType = false +reportUnusedExpression = false + +[tool.ruff] +line-length = 90 +extend-exclude = ["docs"] + +[tool.ruff.lint] +select = ["E", "F", "W"] +ignore = ["E501"] From 180c2bfa4ef04feaf16f04ad96340fcc557d8e82 Mon Sep 17 00:00:00 2001 From: isabelizimm Date: Thu, 25 Jul 2024 01:16:30 -0500 Subject: [PATCH 16/16] add check deps into toml --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 108c87cc..3497c51a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,11 @@ dependencies = [ [project.optional-dependencies] aws = ["s3fs"] azure = ["adlfs"] +check = [ + "pre-commit", + "pyright==1.1.372", # Pinned; manually sync with .github/workflows/code-checks.yml + "types-appdirs", +] doc = [ "ipykernel", "ipython<=8.12",