diff --git a/.editorconfig b/.editorconfig index 6056a3e8..5a54b3ef 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,7 +8,7 @@ end_of_line = lf indent_size = 4 indent_style = space insert_final_newline = true -max_line_length = 80 +max_line_length = 88 trim_trailing_whitespace = true [.editorconfig] @@ -29,15 +29,10 @@ indent_size = 2 max_line_length = off [{*.markdown,*.md,*.rst}] -max_line_length = off ij_visual_guides = none [{*.toml,Cargo.lock,Cargo.toml.orig,Gopkg.lock,Pipfile,poetry.lock}] max_line_length = off -[{*.ini, *.cfg}] -max_line_length = off - [{*.yaml,*.yml}] indent_size = 2 -max_line_length = off diff --git a/.github/renovate.json5 b/.github/renovate.json5 index 9c4bd555..9dcf2141 100644 --- a/.github/renovate.json5 +++ b/.github/renovate.json5 @@ -30,7 +30,7 @@ // Automerge patches, pin changes and digest changes. // Also groups these changes together. groupName: "bugfixes", - excludePackagePrefixes: ["lint", "types"], + excludeDepPatterns: ["lint/.*", "types/.*"], matchUpdateTypes: ["patch", "pin", "digest"], prPriority: 3, // Patches should go first! automerge: true @@ -38,7 +38,7 @@ { // Update all internal packages in one higher-priority PR groupName: "internal packages", - matchPackagePrefixes: ["craft-", "snap-"], + matchDepPatterns: ["craft-.*", "snap-.*"], matchLanguages: ["python"], prPriority: 2 }, @@ -59,10 +59,10 @@ // Minor changes can be grouped and automerged for dev dependencies, but are also deprioritised. groupName: "development dependencies (non-major)", groupSlug: "dev-dependencies", - matchPackagePrefixes: [ - "dev", - "lint", - "types" + matchDepPatterns: [ + "dev/.*", + "lint/.*", + "types/.*" ], matchUpdateTypes: ["minor", "patch", "pin", "digest"], prPriority: -1, @@ -74,7 +74,7 @@ groupSlug: "doc-dependencies", matchPackageNames: ["Sphinx", "furo"], matchPackagePatterns: ["[Ss]phinx.*$"], - matchPackagePrefixes: ["docs"], + matchDepPatterns: ["docs/.*"], }, { // Other major dependencies get deprioritised below minor dev dependencies. diff --git a/.github/workflows/check-renovate.yaml b/.github/workflows/check-renovate.yaml new file mode 100644 index 00000000..8da9f2b9 --- /dev/null +++ b/.github/workflows/check-renovate.yaml @@ -0,0 +1,36 @@ +name: Renovate check +on: + pull_request: + paths: + - ".github/workflows/check-renovate.yaml" + - ".github/renovate.json5" + + # Allows triggering the workflow manually from the Actions tab + workflow_dispatch: + inputs: + enable_ssh_access: + type: boolean + description: 'Enable ssh access' + required: false + default: false + +jobs: + renovate: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install renovate + run: npm install --global renovate + - name: Enable ssh access + uses: mxschmitt/action-tmate@v3 + if: ${{ inputs.enable_ssh_access }} + with: + limit-access-to-actor: true + - name: Check renovate config + run: renovate-config-validator .github/renovate.json5 + - name: Renovate dry-run + run: renovate --dry-run --autodiscover + env: + RENOVATE_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RENOVATE_USE_BASE_BRANCH_CONFIG: ${{ github.ref }} diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index a210f59a..db42f8c0 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -30,9 +30,9 @@ jobs: sudo apt-get install -y libapt-pkg-dev pip install tox - name: Lint documentation - run: tox run -e lint-docs + run: tox run --colored yes -e lint-docs - name: Build documentation - run: tox run -e build-docs + run: tox run --colored yes -e build-docs - name: Upload documentation uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a402e28b..27733d67 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -1,4 +1,4 @@ -name: Tests, linting, etc. +name: test on: push: branches: @@ -6,16 +6,21 @@ on: - "feature/*" - "hotfix/*" - "release/*" + - "renovate/*" pull_request: jobs: - linters: + lint: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 + - name: conventional commits + uses: webiny/action-conventional-commits@v1.3.0 + with: + allowed-commit-types: "build,chore,ci,docs,feat,fix,perf,refactor,style,test" - name: Setup Python uses: actions/setup-python@v5 with: @@ -26,7 +31,7 @@ jobs: echo "::group::Begin snap install" echo "Installing snaps in the background while running apt and pip..." sudo snap install --no-wait --classic pyright - sudo snap install --no-wait ruff shellcheck + sudo snap install --no-wait codespell shellcheck ruff echo "::endgroup::" echo "::group::apt-get" sudo apt update @@ -36,7 +41,7 @@ jobs: python -m pip install tox echo "::endgroup::" echo "::group::Create virtual environments for linting processes." - tox run -m lint --notest + tox run --colored yes -m lint --notest echo "::endgroup::" echo "::group::Wait for snap to complete" snap watch --last=install @@ -47,18 +52,19 @@ jobs: strategy: fail-fast: false matrix: - platform: [ubuntu-22.04] + platform: [ubuntu-22.04, ubuntu-24.04] runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Python versions on ${{ matrix.platform }} + - name: Set up Python uses: actions/setup-python@v5 with: python-version: | 3.10 3.12 + 3.13-dev cache: 'pip' - name: Setup LXD uses: canonical/setup-lxd@v0.1.1 @@ -75,9 +81,9 @@ jobs: echo "::endgroup::" mkdir -p results - name: Setup Tox environments - run: tox run -m unit-tests --notest - - name: Unit tests - run: .tox/.tox/bin/tox run --skip-pkg-install --no-list-dependencies --result-json results/tox-${{ matrix.platform }}.json -m unit-tests + run: tox run --colored yes -m tests --notest + - name: Test with tox + run: tox run --skip-pkg-install --no-list-dependencies --result-json results/tox-${{ matrix.platform }}.json --colored yes -m unit-tests env: PYTEST_ADDOPTS: "--no-header -vv -rN" - name: Upload code coverage @@ -95,19 +101,20 @@ jobs: strategy: fail-fast: false matrix: - platform: [ubuntu-22.04] + platform: [ubuntu-22.04, ubuntu-24.04] python: [py310, py312] runs-on: ${{ matrix.platform }} steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Set up Python versions on ${{ matrix.platform }} + - name: Set up Python uses: actions/setup-python@v5 with: python-version: | 3.10 3.12 + 3.13-dev cache: 'pip' - name: Setup LXD uses: canonical/setup-lxd@v0.1.1 diff --git a/.gitignore b/.gitignore index b19a46a2..4cb50447 100644 --- a/.gitignore +++ b/.gitignore @@ -139,3 +139,6 @@ dmypy.json # Ignore version module generated by setuptools_scm /*/_version.py + +# Visual Studio Code +.vscode/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a9b3a40..f84e07ac 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: "v4.4.0" hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -12,19 +12,16 @@ repos: - id: check-toml - id: fix-byte-order-marker - id: mixed-line-ending - - repo: https://github.com/charliermarsh/ruff-pre-commit - # renovate: datasource=pypi;depName=ruff - rev: "v0.0.267" + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.1.15" hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - repo: https://github.com/psf/black - # renovate: datasource=pypi;depName=black - rev: "23.3.0" + rev: "24.1.1" hooks: - id: black - repo: https://github.com/adrienverge/yamllint.git - # renovate: datasource=pypi;depName=yamllint - rev: "v1.31.0" + rev: "v1.33.0" hooks: - id: yamllint diff --git a/pyproject.toml b/pyproject.toml index 6a7deb5a..98514b9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,13 +47,14 @@ remote = [ "launchpadlib>=1.10.16", ] dev = [ + "build", "coverage[toml]==7.4.4", "hypothesis>=6.0", "pyfakefs~=5.3", - "pytest==8.1.1", + "pytest~=8.1", "pytest-check==2.3.1", - "pytest-cov==5.0.0", - "pytest-mock==3.14.0", + "pytest-cov~=5.0", + "pytest-mock~=3.14", "pytest-rerunfailures==14.0", "pytest-time>=0.3.1", # Pin requests because of https://github.com/msabramo/requests-unixsocket/issues/73 @@ -67,14 +68,14 @@ lint = [ "yamllint==1.35.1" ] types = [ - "mypy[reports]==1.9.0", + "mypy[reports]~=1.10.0", "pyright==1.1.359", "types-requests", "types-urllib3", ] docs = [ "canonical-sphinx~=0.1", - "sphinx-autobuild==2024.4.16", + "sphinx-autobuild~=2024.4", "sphinx-lint==0.9.1", ] apt = [ @@ -83,7 +84,7 @@ apt = [ [build-system] requires = [ - "setuptools==70.1.0", + "setuptools>=70.1", "setuptools_scm[toml]>=7.1" ] build-backend = "setuptools.build_meta" @@ -112,7 +113,7 @@ include = ["*craft*"] namespaces = false [tool.black] -target-version = ["py38"] +target-version = ["py310"] [tool.codespell] ignore-words-list = "buildd,crate,keyserver,comandos,ro,dedent,dedented" @@ -149,13 +150,17 @@ exclude_also = [ [tool.pyright] include = ["craft_application/**", "tests/**"] -# pyright might not like the annotations generated by setuptools_scm -exclude = ["craft_application/_version.py"] strict = ["craft_application"] pythonVersion = "3.10" pythonPlatform = "Linux" venvPath = ".tox" venv = "typing" +exclude = [ + "**/.*", + "**/__pycache__", + # pyright might not like the annotations generated by setuptools_scm + "**/_version.py", +] [tool.mypy] python_version = "3.10" @@ -170,7 +175,7 @@ exclude = [ warn_unused_configs = true warn_redundant_casts = true strict_equality = true -strict_concatenate = true +extra_checks = true warn_return_any = true disallow_subclassing_any = true disallow_untyped_decorators = true @@ -193,6 +198,13 @@ extend-exclude = [ "docs", "__pycache__", ] + +[tool.ruff.format] +docstring-code-format = true +line-ending = "lf" +quote-style = "double" + +[tool.ruff.lint] # Follow ST063 - Maintaining and updating linting specifications for updating these. lint.select = [ # Base linting rule selections. # See the internal document for discussion: @@ -269,34 +281,51 @@ lint.ignore = [ "SIM117", # Use a single `with` statement with multiple contexts instead of nested `with` statements # (reason: this creates long lines that get wrapped and reduces readability) + # Ignored due to conflicts with ruff's formatter: + # https://docs.astral.sh/ruff/formatter/#conflicting-lint-rules + "COM812", # Missing trailing comma - mostly the same, but marginal differences. + "ISC001", # Single-line implicit string concatenation. + # Ignored due to common usage in current code "TRY003", # Avoid specifying long messages outside the exception class ] +[tool.ruff.lint.flake8-annotations] +allow-star-arg-any = true + +[tool.ruff.lint.pydocstyle] +ignore-decorators = [ # Functions with these decorators don't have to have docstrings. + "typing.overload", # Default configuration + # The next four are all variations on override, so child classes don't have to repeat parent classes' docstrings. + "overrides.override", + "overrides.overrides", + "typing.override", + "typing_extensions.override", +] + +[tool.ruff.lint.pylint] +max-args = 8 + +[tool.ruff.lint.pep8-naming] +# Allow Pydantic's `@validator` decorator to trigger class method treatment. +classmethod-decorators = ["pydantic.validator", "pydantic.root_validator"] + [tool.ruff.lint.per-file-ignores] "tests/**.py" = [ # Some things we want for the moin project are unnecessary in tests. "D", # Ignore docstring rules in tests - "ANN", # Ignore type annotations in tests + "ANN", # Ignore type annotations in tests + "ARG", # Allow unused arguments in tests (e.g. for fake functions/methods/classes) "S101", # Allow assertions in tests "S103", # Allow `os.chmod` setting a permissive mask `0o555` on file or directory "S108", # Allow Probable insecure usage of temporary file or directory - "PLR0913", # Allow many arguments for test functions - "PT004", # Allow fixtures that don't return anything to not start with underscores +"PLR0913", # Allow many arguments for test functions (useful if we need many fixtures) + "PLR2004", # Allow magic values in tests + "SLF", # Allow accessing private members from tests. +] +"__init__.py" = [ + "I001", # isort leaves init files alone by default, this makes ruff ignore them too. + "F401", # Allows unused imports in __init__ files. ] -# isort leaves init files alone by default, this makes ruff ignore them too. -"__init__.py" = ["I001"] # TODO: Revert this # https://github.com/canonical/craft-application/issues/306 "tests/integration/launchpad/test_anonymous_access.py" = ["ERA001"] - -[tool.ruff.lint.pep8-naming] -classmethod-decorators = [ - "pydantic.root_validator", - "pydantic.validator", -] - -[tool.ruff.lint.pylint] -max-args = 8 - -[tool.ruff.lint.pydocstyle] -ignore-decorators = ["typing.overload", "overrides.override", "typing.override"] diff --git a/tests/integration/starbase/__init__.py b/tests/integration/starbase/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/test_setuptools.py b/tests/integration/test_setuptools.py new file mode 100644 index 00000000..a27c9843 --- /dev/null +++ b/tests/integration/test_setuptools.py @@ -0,0 +1,59 @@ +# Copyright 2023 Canonical Ltd. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License version 3, as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranties of MERCHANTABILITY, +# SATISFACTORY QUALITY, or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +"""Integration tests related to building the package.""" + +import re +import subprocess +import sys +from pathlib import Path +from zipfile import ZipFile + + +def test_packages(project_main_module, tmp_path, request): + """Check wheel generation from our pyproject.toml""" + root_dir = Path(request.config.rootdir) + out_dir = tmp_path + subprocess.check_call( + [sys.executable, "-m", "build", "--outdir", out_dir, root_dir], + ) + wheels = list(tmp_path.glob("*.whl")) + assert len(wheels) == 1 + wheel = wheels[0] + + main_module = project_main_module.__name__ + + project_files = [] + + dist_files = [] + dist_info_re = re.compile(f"{main_module}-.*.dist-info") + + invalid = [] + + with ZipFile(wheel) as wheel_zip: + names = [Path(p) for p in wheel_zip.namelist()] + assert len(names) > 1 + for name in names: + top = name.parts[0] + if top == main_module: + project_files.append(name) + elif dist_info_re.match(top): + dist_files.append(top) + else: + invalid = [] + + # Only the top-level "project_name" dir should be present, plus the + # project_name-xyz-dist-info/ entries. + assert project_files, f"No '{main_module}' modules were packaged!" + assert dist_files, "The dist-info directory was not created!" + assert not invalid, f"Invalid files were packaged: {invalid}" diff --git a/tests/unit/starbase/__init__.py b/tests/unit/starbase/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tox.ini b/tox.ini index 0a3fd290..46faf16e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,12 @@ [tox] env_list = # Environments to run when called with no parameters. - lint-{black,ruff,pyright,shellcheck,codespell,docs} - test-{py310,py311,py312} + format-{ruff,codespell} + pre-commit + lint-{ruff,mypy,pyright,shellcheck,codespell,docs,yaml} + unit-py3.{10,12} + integration-py3.10 +# Integration tests probably take a while, so we're only running them on Python +# 3.10, which is included in core22. minversion = 4.6 # Tox will use these requirements to bootstrap a venv if necessary. # tox-igore-env-name-mismatch allows us to have one virtualenv for all linting. @@ -13,7 +18,7 @@ requires = # renovate: datasource=pypi tox-ignore-env-name-mismatch>=0.2.0.post2 # renovate: datasource=pypi - tox-gh==1.3.1 + tox-gh==1.3.2 # Allow tox to access the user's $TMPDIR environment variable if set. # This workaround is required to avoid circular dependencies for TMPDIR, # since tox will otherwise attempt to use the environment's TMPDIR variable. @@ -27,6 +32,10 @@ env_tmp_dir = {user_tmp_dir:{env:XDG_RUNTIME_DIR:{work_dir}}}/tox_tmp/{env_name} set_env = TMPDIR={env_tmp_dir} COVERAGE_FILE={env_tmp_dir}/.coverage_{env_name} +pass_env = + CI + CRAFT_* + PYTEST_ADDOPTS [test] # Base configuration for unit and integration tests package = editable @@ -34,33 +43,37 @@ deps = python-apt@https://launchpad.net/ubuntu/+archive/primary/+sourcefiles/python-apt/2.4.0ubuntu2/python-apt_2.4.0ubuntu2.tar.xz .[dev] allowlist_externals = mkdir -commands_pre = mkdir -p results +commands_pre = mkdir -p {tox_root}/results -[testenv:test-{py310,py311,py312}] # Configuration for all tests using pytest +[testenv:{unit,integration}-py3.{8,9,10,11,12}] # Configuration for all tests using pytest base = testenv, test -description = Run unit tests with pytest +description = + unit: Run unit tests with pytest + integration: Run integration tests with pytest labels = - py310, py311, py312: tests, unit-tests -commands = pytest {tty:--color=yes} --cov --cov-report=xml:results/coverage-{env_name}.xml --junit-xml=results/test-results-{env_name}.xml tests/unit {posargs} - -[testenv:integration-{py310,py311,py312}] -base = testenv, test -description = Run integration tests with pytest -labels = - py310, py311, py312: tests, integration-tests -commands = pytest {tty:--color=yes} --junit-xml=results/test-results-{env_name}.xml tests/integration {posargs} + py3.{10,12}: tests + unit-py3.{10,12}: unit-tests + integration-py3.{10,12}: integration-tests +change_dir = + unit: tests/unit + integration: tests/integration +commands = pytest {tty:--color=yes} --cov={tox_root}/craft_application --cov-config={tox_root}/pyproject.toml --cov-report=xml:{tox_root}/results/coverage-{env_name}.xml --junit-xml={tox_root}/results/test-results-{env_name}.xml {posargs} [lint] # Standard linting configuration package = editable extras = lint env_dir = {work_dir}/linting runner = ignore_env_name_mismatch +allowlist_externals = + codespell: codespell + shellcheck: bash, xargs + ruff: ruff [shellcheck] find = git ls-files filter = file --mime-type -Nnf- | grep shellscript | cut -f1 -d: -[testenv:lint-{black,ruff,shellcheck,codespell,yaml}] +[testenv:lint-{ruff,shellcheck,codespell,yaml}] description = Lint the source code base = testenv, lint labels = lint @@ -70,8 +83,8 @@ allowlist_externals = commands_pre = shellcheck: bash -c '{[shellcheck]find} | {[shellcheck]filter} > {env_tmp_dir}/shellcheck_files' commands = - black: black --check --diff {tty:--color} {posargs} . - ruff: ruff check --respect-gitignore {posargs} . + ruff: ruff format --check --diff --respect-gitignore {posargs:.} + ruff: ruff check --respect-gitignore {posargs:.} shellcheck: xargs -ra {env_tmp_dir}/shellcheck_files shellcheck codespell: codespell --toml {tox_root}/pyproject.toml {posargs} yaml: yamllint {posargs} . @@ -85,28 +98,39 @@ labels = lint, type allowlist_externals = mypy: mkdir commands_pre = - mypy: mkdir -p .mypy_cache + mypy: mkdir -p {tox_root}/.mypy_cache commands = pyright: pyright {posargs} mypy: mypy --install-types --non-interactive {posargs:.} -[testenv:format-{black,ruff,codespell}] +[testenv:format-{ruff,codespell}] description = Automatically format source code base = testenv, lint labels = format allowlist_externals = ruff: ruff commands = - black: black {tty:--color} {posargs} . - ruff: ruff --fix --respect-gitignore {posargs} . + ruff: ruff format --respect-gitignore {posargs:.} + ruff: ruff check --fix --respect-gitignore {posargs} . codespell: codespell --toml {tox_root}/pyproject.toml --write-changes {posargs} +[testenv:pre-commit] +base = +deps = pre-commit +package = skip +no_package = true +env_dir = {work_dir}/pre-commit +runner = ignore_env_name_mismatch +description = Run pre-commit on staged files or arbitrary pre-commit commands (tox run -e pre-commit -- [args]) +commands = pre-commit {posargs:run} + [docs] # Sphinx documentation configuration extras = docs package = editable no_package = true env_dir = {work_dir}/docs runner = ignore_env_name_mismatch +source_dir = {tox_root}/{project_name} [testenv:build-docs] description = Build sphinx documentation @@ -124,5 +148,6 @@ commands = sphinx-autobuild {posargs:-b html --open-browser --port 8080} -W --wa [testenv:lint-docs] description = Lint the documentation with sphinx-lint base = docs -commands = sphinx-lint --ignore docs/_build/html --max-line-length 80 -e all {posargs} docs/ +commands = + sphinx-lint --ignore docs/_build/html --max-line-length 80 -e all {posargs} docs/ labels = lint