From 8ef9025f1344a254bdfa9aec1ecf1d889ed84d8e Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 17 Nov 2024 17:34:42 +0000 Subject: [PATCH] Update CI files --- .ci/assets/release_requirements.txt | 2 +- .ci/scripts/calc_constraints.py | 47 ++++++++++++++------ .ci/scripts/check_release.py | 7 +++ .ci/scripts/check_requirements.py | 49 +++++++++++++++------ .github/template_gitref | 2 +- .github/workflows/build.yml | 4 +- .github/workflows/ci.yml | 3 -- .github/workflows/create-branch.yml | 6 +-- .github/workflows/lint.yml | 36 ++++++++++----- .github/workflows/release.yml | 2 +- .github/workflows/scripts/before_install.sh | 5 --- .github/workflows/scripts/release.sh | 7 +-- lint_requirements.txt | 3 +- 13 files changed, 113 insertions(+), 60 deletions(-) diff --git a/.ci/assets/release_requirements.txt b/.ci/assets/release_requirements.txt index c064e94779..6635a87231 100644 --- a/.ci/assets/release_requirements.txt +++ b/.ci/assets/release_requirements.txt @@ -1,3 +1,3 @@ -bump2version +bump-my-version gitpython towncrier diff --git a/.ci/scripts/calc_constraints.py b/.ci/scripts/calc_constraints.py index 7ea35f039c..08050ab811 100755 --- a/.ci/scripts/calc_constraints.py +++ b/.ci/scripts/calc_constraints.py @@ -13,6 +13,11 @@ from packaging.version import Version import yaml +try: + import tomllib +except ImportError: + import tomli as tomllib + CORE_TEMPLATE_URL = "https://raw.githubusercontent.com/pulp/pulpcore/main/template_config.yml" @@ -59,11 +64,11 @@ def to_upper_bound(req): operator = "~=" version = Version(spec.version) if version.micro != 0: - max_version = f"{version.major}.{version.minor}.{version.micro-1}" + max_version = f"{version.major}.{version.minor}.{version.micro - 1}" elif version.minor != 0: - max_version = f"{version.major}.{version.minor-1}" + max_version = f"{version.major}.{version.minor - 1}" elif version.major != 0: - max_version = f"{version.major-1}.0" + max_version = f"{version.major - 1}.0" else: return f"# NO BETTER CONSTRAINT: {req}" return f"{requirement.name}{operator}{max_version}" @@ -100,18 +105,32 @@ def main(): parser.add_argument("filename", nargs="*") args = parser.parse_args() - with fileinput.input(files=args.filename) as req_file: - for line in req_file: - if line.strip().startswith("#"): - # Shortcut comment only lines - print(line.strip()) - else: - req, comment = split_comment(line) - if args.upper: - new_req = to_upper_bound(req) + modifier = to_upper_bound if args.upper else to_lower_bound + + req_files = [filename for filename in args.filename if not filename.endswith("pyproject.toml")] + pyp_files = [filename for filename in args.filename if filename.endswith("pyproject.toml")] + if req_files: + with fileinput.input(files=req_files) as req_file: + for line in req_file: + if line.strip().startswith("#"): + # Shortcut comment only lines + print(line.strip()) else: - new_req = to_lower_bound(req) - print(new_req + comment) + req, comment = split_comment(line) + new_req = modifier(req) + print(new_req + comment) + for filename in pyp_files: + with open(filename, "rb") as fp: + pyproject = tomllib.load(fp) + for req in pyproject["project"]["dependencies"]: + new_req = modifier(req) + print(new_req) + optional_dependencies = pyproject["project"].get("optional-dependencies") + if optional_dependencies: + for opt in optional_dependencies.values(): + for req in opt: + new_req = modifier(req) + print(new_req) if __name__ == "__main__": diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 9fab30a467..e569633329 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -86,6 +86,13 @@ def main(): ) if req_txt_diff: reasons.append("requirements.txt") + pyproject_diff = repo.git.diff( + f"{last_tag}", f"origin/{branch}", "--name-only", "--", "pyproject.toml" + ) + if pyproject_diff: + reasons.append( + "pyproject.toml changed (PLEASE check if dependencies are affected." + ) if reasons: curr_version = Version(last_tag) diff --git a/.ci/scripts/check_requirements.py b/.ci/scripts/check_requirements.py index c336aa6080..87b7fd1359 100755 --- a/.ci/scripts/check_requirements.py +++ b/.ci/scripts/check_requirements.py @@ -5,11 +5,13 @@ # # For more info visit https://github.com/pulp/plugin_template +import tomllib import warnings -from pkg_resources import Requirement +from packaging.requirements import Requirement CHECK_MATRIX = [ + ("pyproject.toml", True, True, True), ("requirements.txt", True, True, True), ("dev_requirements.txt", False, True, False), ("ci_requirements.txt", False, True, True), @@ -20,17 +22,33 @@ ("clitest_requirements.txt", False, True, True), ] -errors = [] -for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: - try: +def iterate_file(filename): + if filename == "pyproject.toml": + with open(filename, "rb") as fd: + pyproject_toml = tomllib.load(fd) + if "project" in pyproject_toml: + for nr, line in enumerate(pyproject_toml["project"]["dependencies"]): + yield nr, line + else: with open(filename, "r") as fd: for nr, line in enumerate(fd.readlines()): line = line.strip() if not line or line.startswith("#"): continue + if "#" in line: + line = line.split("#", maxsplit=1)[0] + yield nr, line.strip() + + +def main(): + errors = [] + + for filename, check_upperbound, check_prereleases, check_r in CHECK_MATRIX: + try: + for nr, line in iterate_file(filename): try: - req = Requirement.parse(line) + req = Requirement(line) except ValueError: if line.startswith("git+"): # The single exception... @@ -49,18 +67,21 @@ and req.name != "pulpcore-client" ): errors.append(f"{filename}:{nr}: Prerelease versions found in {line}.") - ops = [op for op, ver in req.specs] - spec = str(req.specs) + ops = [spec.operator for spec in req.specifier] if "~=" in ops: warnings.warn(f"{filename}:{nr}: Please avoid using ~= on {req.name}!") elif "<" not in ops and "<=" not in ops and "==" not in ops: if check_upperbound: errors.append(f"{filename}:{nr}: Upper bound missing in {line}.") - except FileNotFoundError: - # skip this test for plugins that don't use this requirements.txt - pass + except FileNotFoundError: + # skip this test for plugins that don't use this requirements.txt + pass + + if errors: + print("Dependency issues found:") + print("\n".join(errors)) + exit(1) + -if errors: - print("Dependency issues found:") - print("\n".join(errors)) - exit(1) +if __name__ == "__main__": + main() diff --git a/.github/template_gitref b/.github/template_gitref index 73f0df798b..d37a43c861 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-389-g444ab6c +2021.08.26-393-g0e700c1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5256baaac3..bdf7d903d2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,11 +34,11 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging twine wheel mkdocs jq + pip install build packaging twine wheel mkdocs jq echo ::endgroup:: - name: "Build package" run: | - python3 setup.py sdist bdist_wheel --python-tag py3 + python3 -m build twine check dist/* - name: "Install built packages" run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ee81cdc81..b6b2535a46 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,9 +42,6 @@ jobs: GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" run: | .github/workflows/scripts/check_commit.sh - - name: "Verify requirements files" - run: | - python .ci/scripts/check_requirements.py docs: uses: "./.github/workflows/docs.yml" diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index 703768ba2a..712c754e36 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -39,7 +39,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version packaging -r plugin_template/requirements.txt + pip install bump-my-version packaging -r plugin_template/requirements.txt echo ::endgroup:: - name: "Setting secrets" @@ -54,7 +54,7 @@ jobs: run: | # Just to be sure... git checkout main - NEW_BRANCH="$(bump2version --dry-run --list release | sed -Ene 's/^new_version=([[:digit:]]+\.[[:digit:]]+)\..*$/\1/p')" + NEW_BRANCH="$(bump-my-version show new_version --increment release | sed -Ene 's/^([[:digit:]]+\.[[:digit:]]+)\.[[:digit:]]+$/\1/p')" if [ -z "$NEW_BRANCH" ] then echo Could not determine the new branch name. @@ -70,7 +70,7 @@ jobs: - name: Bump version on main branch working-directory: pulpcore run: | - bump2version --no-commit minor + bump-my-version bump --no-commit minor - name: Remove entries from CHANGES directory working-directory: pulpcore diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c5490bbe7a..3c44618914 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,7 +16,7 @@ defaults: jobs: lint: - runs-on: ubuntu-latest + runs-on: "ubuntu-latest" steps: - uses: "actions/checkout@v4" @@ -34,26 +34,38 @@ jobs: pip install -r lint_requirements.txt echo ::endgroup:: - - name: Lint workflow files + - name: "Lint workflow files" run: | yamllint -s -d '{extends: relaxed, rules: {line-length: disable}}' .github/workflows + - name: "Verify bump version config" + run: | + bump-my-version bump --dry-run release + bump-my-version show-bump + # run black separately from flake8 to get a diff - - name: Run black + - name: "Run black" run: | black --version black --check --diff . # Lint code. - - name: Run flake8 - run: flake8 + - name: "Run flake8" + run: | + flake8 + + - name: "Run extra lint checks" + run: | + [ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh - - name: Run extra lint checks - run: "[ ! -x .ci/scripts/extra_linting.sh ] || .ci/scripts/extra_linting.sh" + - name: "Check for any files unintentionally left out of MANIFEST.in" + run: | + check-manifest - # check for any files unintentionally left out of MANIFEST.in - - name: Check manifest - run: check-manifest + - name: "Verify requirements files" + run: | + python .ci/scripts/check_requirements.py - - name: Check for gettext problems - run: sh .ci/scripts/check_gettext.sh + - name: "Check for common gettext problems" + run: | + sh .ci/scripts/check_gettext.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 593ec2f4a0..6d6849b20a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,7 +35,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install bump2version towncrier + pip install bump-my-version towncrier echo ::endgroup:: - name: "Configure Git with pulpbot name and email" diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index 2ee3b91208..9baa4bd160 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -45,11 +45,6 @@ if [ -f $PRE_BEFORE_INSTALL ]; then source $PRE_BEFORE_INSTALL fi -if [[ -n $(echo -e $COMMIT_MSG | grep -P "Required PR:.*") ]]; then - echo "The Required PR mechanism has been removed. Consider adding a scm requirement to requirements.txt." - exit 1 -fi - if [ "$GITHUB_EVENT_NAME" = "pull_request" ] || [ "${BRANCH_BUILD}" = "1" -a "${BRANCH}" != "main" ] then echo $COMMIT_MSG | sed -n -e 's/.*CI Base Image:\s*\([-_/[:alnum:]]*:[-_[:alnum:]]*\).*/ci_base: "\1"/p' >> .ci/ansible/vars/main.yaml diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh index 9525f229d9..a08353cdb5 100755 --- a/.github/workflows/scripts/release.sh +++ b/.github/workflows/scripts/release.sh @@ -10,7 +10,8 @@ then exit 1 fi -NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')" +# The tail is a necessary workaround to remove the warning from the output. +NEW_VERSION="$(bump-my-version show new_version --increment release | tail -n -1)" echo "Release ${NEW_VERSION}" if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] @@ -20,7 +21,7 @@ then fi towncrier build --yes --version "${NEW_VERSION}" -bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty -bump2version patch --commit +bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump-my-version bump patch --commit git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/lint_requirements.txt b/lint_requirements.txt index a8fb1161ea..859b6ac9ad 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -5,9 +5,10 @@ # # For more info visit https://github.com/pulp/plugin_template -# python packages handy for developers, but not required by pulp black==24.3.0 +bump-my-version check-manifest flake8 flake8-black +packaging yamllint