From 3efdab64448039daefc2c35ca5d014926df654e2 Mon Sep 17 00:00:00 2001 From: pulpbot Date: Sun, 28 Jan 2024 02:34:18 +0000 Subject: [PATCH] Update CI files [noissue] --- .ci/scripts/check_release.py | 13 +- .ci/scripts/collect_changes.py | 8 +- .github/template_gitref | 2 +- .github/workflows/create-branch.yml | 2 +- .github/workflows/publish.yml | 322 ++++++++++++++++ .github/workflows/release.yml | 361 +----------------- .../scripts/install_python_client.sh | 15 +- .../workflows/scripts/install_ruby_client.sh | 16 +- .github/workflows/scripts/release.sh | 26 ++ .github/workflows/scripts/script.sh | 105 +++-- .../scripts/update_backport_labels.py | 8 +- docs/Makefile | 8 +- docs/template_gitref | 2 +- lint_requirements.txt | 2 +- template_config.yml | 11 +- 15 files changed, 482 insertions(+), 419 deletions(-) create mode 100644 .github/workflows/publish.yml create mode 100755 .github/workflows/scripts/release.sh diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index db9a76a..a5dd896 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -29,7 +29,7 @@ def main(): "--branches", default="supported", help="A comma separated list of branches to check for releases. Can also use keyword: " - "'supported'. Defaults to 'supported', see `ci_update_branches` in " + "'supported'. Defaults to 'supported', see `supported_release_branches` in " "`plugin_template.yml`.", ) opts = parser.parse_args() @@ -46,12 +46,15 @@ def main(): if branches == "supported": with open(f"{d}/template_config.yml", mode="r") as f: tc = yaml.safe_load(f) - branches = tc["ci_update_branches"] - branches.append(DEFAULT_BRANCH) + branches = set(tc["supported_release_branches"]) + latest_release_branch = tc["latest_release_branch"] + if latest_release_branch is not None: + branches.add(latest_release_branch) + branches.add(DEFAULT_BRANCH) else: - branches = branches.split(",") + branches = set(branches.split(",")) - if diff := set(branches) - set(available_branches): + if diff := branches - set(available_branches): print(f"Supplied branches contains non-existent branches! {diff}") exit(1) diff --git a/.ci/scripts/collect_changes.py b/.ci/scripts/collect_changes.py index 202fccd..c2e4843 100755 --- a/.ci/scripts/collect_changes.py +++ b/.ci/scripts/collect_changes.py @@ -19,9 +19,11 @@ CHANGELOG_FILE = tc_settings.get("filename", "NEWS.rst") START_STRING = tc_settings.get( "start_string", - "\n" - if CHANGELOG_FILE.endswith(".md") - else ".. towncrier release notes start\n", + ( + "\n" + if CHANGELOG_FILE.endswith(".md") + else ".. towncrier release notes start\n" + ), ) TITLE_FORMAT = tc_settings.get("title_format", "{name} {version} ({project_date})") diff --git a/.github/template_gitref b/.github/template_gitref index bff7b99..06c4f52 100644 --- a/.github/template_gitref +++ b/.github/template_gitref @@ -1 +1 @@ -2021.08.26-301-g83f0607 +2021.08.26-310-g59cd732 diff --git a/.github/workflows/create-branch.yml b/.github/workflows/create-branch.yml index a421cf4..a0120f8 100644 --- a/.github/workflows/create-branch.yml +++ b/.github/workflows/create-branch.yml @@ -81,7 +81,7 @@ jobs: - name: Update CI branches in template_config working-directory: plugin_template run: | - python3 ./plugin-template pulp_gem --github --add-version-ci-update-branches "${NEW_BRANCH}" + python3 ./plugin-template pulp_gem --github --latest-release-branch "${NEW_BRANCH}" git add -A - name: Make a PR with version bump and without CHANGES/* diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..50f0b6d --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,322 @@ +# WARNING: DO NOT EDIT! +# +# This file was generated by plugin_template, and is managed by it. Please use +# './plugin-template --github pulp_gem' to update this file. +# +# For more info visit https://github.com/pulp/plugin_template + +--- +name: "Gem Publish Release" +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+" + +defaults: + run: + working-directory: "pulp_gem" + +jobs: + build: + uses: "./.github/workflows/build.yml" + + build-bindings-docs: + needs: + - "build" + runs-on: "ubuntu-latest" + # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests + env: + TEST: "docs" + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_gem" + + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.8" + + - uses: "actions/download-artifact@v3" + with: + name: "plugin_package" + path: "pulp_gem/dist/" + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "2.6" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs + echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_gem/.ci/assets/httpie/" >> $GITHUB_ENV + echo ::endgroup:: + + # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to + # setup the Pulp instance. + - name: "Before Install" + run: | + .github/workflows/scripts/before_install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Install" + run: | + .github/workflows/scripts/install.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + + - name: "Install Python client" + run: | + .github/workflows/scripts/install_python_client.sh + shell: "bash" + - name: "Install Ruby client" + run: | + .github/workflows/scripts/install_ruby_client.sh + shell: "bash" + - name: "Upload python client packages" + uses: "actions/upload-artifact@v3" + with: + name: "python-client.tar" + path: "pulp_gem/gem-python-client.tar" + if-no-files-found: "error" + + - name: "Upload python client docs" + uses: "actions/upload-artifact@v3" + with: + name: "python-client-docs.tar" + path: "pulp_gem/gem-python-client-docs.tar" + if-no-files-found: "error" + - name: "Upload ruby client packages" + uses: "actions/upload-artifact@v3" + with: + name: "ruby-client.tar" + path: "pulp_gem/gem-ruby-client.tar" + if-no-files-found: "error" + - name: Build docs + run: | + export DJANGO_SETTINGS_MODULE=pulpcore.app.settings + export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py + make -C docs/ PULP_URL="https://pulp" diagrams html + tar -cvf docs/docs.tar docs/_build + + - name: "Upload built docs" + uses: actions/upload-artifact@v3 + with: + name: "docs.tar" + path: "pulp_gem/docs/docs.tar" + + - name: "Logs" + if: always() + run: | + echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" + http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + docker images || true + docker ps -a || true + docker logs pulp || true + docker exec pulp ls -latr /etc/yum.repos.d/ || true + docker exec pulp cat /etc/yum.repos.d/* || true + docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" + publish-package: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_gem" + + - uses: "actions/download-artifact@v3" + with: + name: "plugin_package" + path: "pulp_gem/dist/" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.8" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install twine + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Deploy plugin to pypi" + run: | + .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.ref_name }} + publish-python-bindings: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_gem" + + - name: "Download Python client" + uses: "actions/download-artifact@v3" + with: + name: "python-client.tar" + path: "pulp_gem/" + + - name: "Untar python client packages" + run: | + tar -xvf gem-python-client.tar + + - uses: "actions/setup-python@v4" + with: + python-version: "3.8" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install twine + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Publish client to pypi" + run: | + bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.ref_name }} + publish-ruby-bindings: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_gem" + + - name: "Download Ruby client" + uses: "actions/download-artifact@v3" + with: + name: "ruby-client.tar" + path: "pulp_gem/" + + - name: "Untar Ruby client packages" + run: | + tar -xvf gem-ruby-client.tar + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "2.6" + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Publish client to rubygems" + run: | + bash .github/workflows/scripts/publish_client_gem.sh ${{ github.ref_name }} + publish-docs: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + + env: + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + + steps: + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + path: "pulp_gem" + + - uses: "actions/setup-python@v4" + with: + python-version: "3.8" + + - name: "Install python dependencies" + run: | + echo ::group::PYDEPS + pip install 'packaging~=21.3' requests + echo ::endgroup:: + + - name: "Setting secrets" + run: | + python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" + env: + SECRETS_CONTEXT: "${{ toJson(secrets) }}" + + - name: "Download built docs" + uses: "actions/download-artifact@v3" + with: + name: "docs.tar" + path: "pulp_gem/" + + - name: "Download Python client docs" + uses: "actions/download-artifact@v3" + with: + name: "python-client-docs.tar" + path: "pulp_gem/" + + - name: "Publish docs to pulpproject.org" + run: | + tar -xvf docs.tar + .github/workflows/scripts/publish_docs.sh tag ${{ github.ref_name }} + + create-gh-release: + runs-on: "ubuntu-latest" + needs: + - "build-bindings-docs" + - "publish-package" + - "publish-python-bindings" + - "publish-ruby-bindings" + - "publish-docs" + + steps: + - name: "Create release on GitHub" + uses: "actions/github-script@v7" + env: + TAG_NAME: "${{ github.ref_name }}" + with: + script: | + const { TAG_NAME } = process.env; + + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: TAG_NAME, + make_latest: "legacy", + }); diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index edb7faf..0524633 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,9 +9,6 @@ name: Gem Release Pipeline on: workflow_dispatch: - inputs: - release: - description: "Release tag (e.g. 3.2.1)" defaults: run: @@ -29,6 +26,7 @@ jobs: with: fetch-depth: 0 path: "pulp_gem" + token: ${{ secrets.RELEASE_TOKEN }} - uses: "actions/setup-python@v4" with: @@ -37,7 +35,7 @@ jobs: - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install 'packaging~=21.3' bump2version gitpython towncrier wheel requests + pip install bump2version towncrier echo ::endgroup:: - name: "Configure Git with pulpbot name and email" @@ -51,363 +49,12 @@ jobs: env: SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - name: "Create the release commit, tag it, create a post-release commit, and build plugin package" + - name: "Tag the release" run: | - python .github/workflows/scripts/release.py ${{ github.event.inputs.release }} - - - name: "Upload Package whl" - uses: "actions/upload-artifact@v3" - with: - name: "plugin_package" - path: "pulp_gem/dist/" - if-no-files-found: "error" - - - name: "Tar files" - run: | - tar -cvf pulp_gem.tar . - - - name: "Upload Artifact" - uses: "actions/upload-artifact@v3" - with: - name: "pulp_gem.tar" - path: "pulp_gem/pulp_gem.tar" - - - build-bindings-docs: - needs: - - "build-artifacts" - runs-on: "ubuntu-latest" - # Install scripts expect TEST to be set, 'docs' is most appropriate even though we don't run tests - env: - TEST: "docs" - - steps: - - uses: "actions/download-artifact@v3" - with: - name: "pulp_gem.tar" - path: "pulp_gem/" - - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - repository: "pulp/pulp-openapi-generator" - path: "pulp-openapi-generator" - - - uses: "actions/setup-python@v4" - with: - python-version: "3.8" - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: "Untar repository" - run: | - tar -xf pulp_gem.tar - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install towncrier twine wheel httpie docker netaddr boto3 ansible mkdocs - echo "HTTPIE_CONFIG_DIR=$GITHUB_WORKSPACE/pulp_gem/.ci/assets/httpie/" >> $GITHUB_ENV - echo ::endgroup:: - - # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to - # setup the Pulp instance. - - name: "Before Install" - run: | - .github/workflows/scripts/before_install.sh + .github/workflows/scripts/release.sh shell: "bash" env: PY_COLORS: "1" ANSIBLE_FORCE_COLOR: "1" GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install" - run: | - .github/workflows/scripts/install.sh - shell: "bash" - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" - - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - - name: "Install Ruby client" - run: | - .github/workflows/scripts/install_ruby_client.sh - shell: "bash" - - name: "Upload python client packages" - uses: "actions/upload-artifact@v3" - with: - name: "python-client.tar" - path: "pulp_gem/gem-python-client.tar" - if-no-files-found: "error" - - - name: "Upload python client docs" - uses: "actions/upload-artifact@v3" - with: - name: "python-client-docs.tar" - path: "pulp_gem/gem-python-client-docs.tar" - if-no-files-found: "error" - - name: "Upload ruby client packages" - uses: "actions/upload-artifact@v3" - with: - name: "ruby-client.tar" - path: "pulp_gem/gem-ruby-client.tar" - if-no-files-found: "error" - - name: Build docs - run: | - export DJANGO_SETTINGS_MODULE=pulpcore.app.settings - export PULP_SETTINGS=$PWD/.ci/ansible/settings/settings.py - make -C docs/ PULP_URL="https://pulp" diagrams html - tar -cvf docs/docs.tar docs/_build - - - name: "Upload built docs" - uses: actions/upload-artifact@v3 - with: - name: "docs.tar" - path: "pulp_gem/docs/docs.tar" - - - name: "Logs" - if: always() - run: | - echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true - docker images || true - docker ps -a || true - docker logs pulp || true - docker exec pulp ls -latr /etc/yum.repos.d/ || true - docker exec pulp cat /etc/yum.repos.d/* || true - docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" - - - publish-tag: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/download-artifact@v3" - with: - name: "pulp_gem.tar" - path: "pulp_gem/" - - - name: "Untar repository" - run: | - tar -xf pulp_gem.tar - - - name: "Configure Git with pulpbot name and email" - run: | - git config --global user.name 'pulpbot' - git config --global user.email 'pulp-infra@redhat.com' - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Push branch and tag to GitHub" - run: | - bash .github/workflows/scripts/push_branch_and_tag_to_github.sh ${{ github.event.inputs.release }} - publish-package: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - "publish-tag" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_gem" - - - uses: "actions/download-artifact@v3" - with: - name: "plugin_package" - path: "pulp_gem/dist/" - - - uses: "actions/setup-python@v4" - with: - python-version: "3.8" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install twine - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Deploy plugin to pypi" - run: | - .github/workflows/scripts/publish_plugin_pypi.sh ${{ github.event.inputs.release }} - publish-python-bindings: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - "publish-tag" - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_gem" - - - name: "Download Python client" - uses: "actions/download-artifact@v3" - with: - name: "python-client.tar" - path: "pulp_gem/" - - - name: "Untar python client packages" - run: | - tar -xvf gem-python-client.tar - - - uses: "actions/setup-python@v4" - with: - python-version: "3.8" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install twine - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Publish client to pypi" - run: | - bash .github/workflows/scripts/publish_client_pypi.sh ${{ github.event.inputs.release }} - publish-ruby-bindings: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - "publish-tag" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_gem" - - - name: "Download Ruby client" - uses: "actions/download-artifact@v3" - with: - name: "ruby-client.tar" - path: "pulp_gem/" - - - name: "Untar Ruby client packages" - run: | - tar -xvf gem-ruby-client.tar - - - uses: ruby/setup-ruby@v1 - with: - ruby-version: "2.6" - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Publish client to rubygems" - run: | - bash .github/workflows/scripts/publish_client_gem.sh ${{ github.event.inputs.release }} - publish-docs: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - "publish-tag" - - env: - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - steps: - - uses: "actions/checkout@v4" - with: - fetch-depth: 1 - path: "pulp_gem" - - - uses: "actions/setup-python@v4" - with: - python-version: "3.8" - - - name: "Install python dependencies" - run: | - echo ::group::PYDEPS - pip install 'packaging~=21.3' requests - echo ::endgroup:: - - - name: "Setting secrets" - run: | - python3 .github/workflows/scripts/secrets.py "$SECRETS_CONTEXT" - env: - SECRETS_CONTEXT: "${{ toJson(secrets) }}" - - - name: "Download built docs" - uses: "actions/download-artifact@v3" - with: - name: "docs.tar" - path: "pulp_gem/" - - - name: "Download Python client docs" - uses: "actions/download-artifact@v3" - with: - name: "python-client-docs.tar" - path: "pulp_gem/" - - - name: "Publish docs to pulpproject.org" - run: | - tar -xvf docs.tar - .github/workflows/scripts/publish_docs.sh tag ${{ github.event.inputs.release }} - - create-gh-release: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - "publish-tag" - - "publish-package" - - "publish-python-bindings" - - "publish-ruby-bindings" - - "publish-docs" - - steps: - - name: "Create release on GitHub" - uses: "actions/github-script@v7" - env: - TAG_NAME: "${{ inputs.release }}" - with: - script: | - const { TAG_NAME } = process.env; - - await github.rest.repos.createRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - tag_name: TAG_NAME, - make_latest: "legacy", - }); diff --git a/.github/workflows/scripts/install_python_client.sh b/.github/workflows/scripts/install_python_client.sh index 89cdad7..6df1ed4 100755 --- a/.github/workflows/scripts/install_python_client.sh +++ b/.github/workflows/scripts/install_python_client.sh @@ -14,7 +14,10 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. source .github/workflows/scripts/utils.sh -export PULP_URL="${PULP_URL:-https://pulp}" +PULP_URL="${PULP_URL:-https://pulp}" +export PULP_URL +PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" +export PULP_API_ROOT REPORTED_STATUS="$(pulp status)" REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "gem" -r '.versions[] | select(.component == $plugin) | .version')" @@ -22,7 +25,15 @@ VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import V pushd ../pulp-openapi-generator rm -rf pulp_gem-client -./generate.sh pulp_gem python "$VERSION" + +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" +then + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=gem" + USE_LOCAL_API_JSON=1 ./generate.sh pulp_gem python "$VERSION" +else + ./generate.sh pulp_gem python "$VERSION" +fi + pushd pulp_gem-client python setup.py sdist bdist_wheel --python-tag py3 diff --git a/.github/workflows/scripts/install_ruby_client.sh b/.github/workflows/scripts/install_ruby_client.sh index fc0f35e..ecc8136 100755 --- a/.github/workflows/scripts/install_ruby_client.sh +++ b/.github/workflows/scripts/install_ruby_client.sh @@ -14,8 +14,10 @@ cd "$(dirname "$(realpath -e "$0")")"/../../.. source .github/workflows/scripts/utils.sh -export PULP_URL="${PULP_URL:-https://pulp}" - +PULP_URL="${PULP_URL:-https://pulp}" +export PULP_URL +PULP_API_ROOT="${PULP_API_ROOT:-/pulp/}" +export PULP_API_ROOT REPORTED_STATUS="$(pulp status)" REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "gem" -r '.versions[] | select(.component == $plugin) | .version')" @@ -23,7 +25,15 @@ VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import V pushd ../pulp-openapi-generator rm -rf pulp_gem-client -./generate.sh pulp_gem ruby "$VERSION" + +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" +then + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=gem" + USE_LOCAL_API_JSON=1 ./generate.sh pulp_gem ruby "$VERSION" +else + ./generate.sh pulp_gem ruby "$VERSION" +fi + pushd pulp_gem-client gem build pulp_gem_client gem install --both "./pulp_gem_client-$VERSION.gem" diff --git a/.github/workflows/scripts/release.sh b/.github/workflows/scripts/release.sh new file mode 100755 index 0000000..9525f22 --- /dev/null +++ b/.github/workflows/scripts/release.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eu -o pipefail + +BRANCH=$(git branch --show-current) + +if ! [[ "${BRANCH}" =~ ^[0-9]+\.[0-9]+$ ]] +then + echo ERROR: This is not a release branch! + exit 1 +fi + +NEW_VERSION="$(bump2version --dry-run --list release | sed -ne 's/^new_version=//p')" +echo "Release ${NEW_VERSION}" + +if ! [[ "${NEW_VERSION}" == "${BRANCH}"* ]] +then + echo ERROR: Version does not match release branch + exit 1 +fi + +towncrier build --yes --version "${NEW_VERSION}" +bump2version release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty +bump2version patch --commit + +git push origin "${BRANCH}" "${NEW_VERSION}" diff --git a/.github/workflows/scripts/script.sh b/.github/workflows/scripts/script.sh index 1807885..841797d 100755 --- a/.github/workflows/scripts/script.sh +++ b/.github/workflows/scripts/script.sh @@ -51,32 +51,68 @@ password password # Some commands like ansible-galaxy specifically require 600 cmd_prefix bash -c "chmod 600 ~pulp/.netrc" -# Infer the client name from the package name by replacing "-" with "_". -# Use the component to infer the package name on older versions of pulpcore. - -if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] +# Generate and install binding +pushd ../pulp-openapi-generator +if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - pushd ../pulp-openapi-generator - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - popd + # Use app_label to generate api.json and package to produce the proper package name. + + if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ] + then + # Workaround: Domains are not supported by the published bindings. + # Generate new bindings for all packages. + for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS") + do + echo $item + COMPONENT="$(jq -r '.component' <<<"$item")" + VERSION="$(jq -r '.version' <<<"$item")" + MODULE="$(jq -r '.module' <<<"$item")" + PACKAGE="${MODULE%%.*}" + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" + USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" + cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" + sudo rm -rf "./${PACKAGE}-client" + done + else + # Sadly: Different pulpcore-versions aren't either... + for item in $(jq -r '.versions[]| select(.component!="gem")| tojson' <<<"$REPORTED_STATUS") + do + echo $item + COMPONENT="$(jq -r '.component' <<<"$item")" + VERSION="$(jq -r '.version' <<<"$item")" + MODULE="$(jq -r '.module' <<<"$item")" + PACKAGE="${MODULE%%.*}" + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" + USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" + cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" + sudo rm -rf "./${PACKAGE}-client" + done + fi else - # Sadly: Different pulpcore-versions aren't either... - pushd ../pulp-openapi-generator - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.component!="gem")|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - popd + # Infer the client name from the package name by replacing "-" with "_". + # Use the component to infer the package name on older versions of pulpcore. + + if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] + then + # Workaround: Domains are not supported by the published bindings. + # Generate new bindings for all packages. + for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') + do + ./generate.sh "${item}" python + cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" + sudo rm -rf "./${item}-client" + done + else + # Sadly: Different pulpcore-versions aren't either... + for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.component!="gem")|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') + do + ./generate.sh "${item}" python + cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" + sudo rm -rf "./${item}-client" + done + fi fi +popd # At this point, this is a safeguard only, so let's not make too much fuzz about the old status format. echo "$REPORTED_STATUS" | jq -r '.versions[]|select(.package)|(.package|sub("_"; "-")) + "-client==" + .version' > bindings_requirements.txt @@ -93,14 +129,14 @@ echo "Checking for uncommitted migrations..." cmd_user_prefix bash -c "django-admin makemigrations gem --check --dry-run" # Run unit tests. -cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes -p no:pulpcore --pyargs pulp_gem.tests.unit" +cmd_user_prefix bash -c "PULP_DATABASES__default__USER=postgres pytest -v -r sx --color=yes --suppress-no-test-exit-code -p no:pulpcore --pyargs pulp_gem.tests.unit" # Run functional tests if [[ "$TEST" == "performance" ]]; then if [[ -z ${PERFORMANCE_TEST+x} ]]; then - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_gem.tests.performance" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs pulp_gem.tests.performance" else - cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --pyargs --capture=no --durations=0 pulp_gem.tests.performance.test_${PERFORMANCE_TEST}" + cmd_user_prefix bash -c "pytest -vv -r sx --color=yes --suppress-no-test-exit-code --capture=no --durations=0 --pyargs pulp_gem.tests.performance.test_${PERFORMANCE_TEST}" fi exit fi @@ -108,15 +144,14 @@ fi if [ -f "$FUNC_TEST_SCRIPT" ]; then source "$FUNC_TEST_SCRIPT" else - if [[ "$GITHUB_WORKFLOW" == "Gem Nightly CI/CD" ]] - then - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_gem.tests.functional -m parallel -n 8 --nightly" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_gem.tests.functional -m 'not parallel' --nightly" - - else - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_gem.tests.functional -m parallel -n 8" - cmd_user_prefix bash -c "pytest -v -r sx --color=yes --pyargs pulp_gem.tests.functional -m 'not parallel'" - fi + if [[ "$GITHUB_WORKFLOW" =~ "Nightly" ]] + then + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_gem.tests.functional -m parallel -n 8 --nightly" + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_gem.tests.functional -m 'not parallel' --nightly" + else + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_gem.tests.functional -m parallel -n 8" + cmd_user_prefix bash -c "pytest -v -r sx --color=yes --suppress-no-test-exit-code --pyargs pulp_gem.tests.functional -m 'not parallel'" + fi fi pushd ../pulp-cli-gem pip install -r test_requirements.txt diff --git a/.github/workflows/scripts/update_backport_labels.py b/.github/workflows/scripts/update_backport_labels.py index c831203..e456656 100755 --- a/.github/workflows/scripts/update_backport_labels.py +++ b/.github/workflows/scripts/update_backport_labels.py @@ -32,10 +32,14 @@ def random_color(): assert response.status_code == 200 old_labels = set([x["name"] for x in response.json() if x["name"].startswith("backport-")]) -# get ci_update_branches from template_config.yml +# get list of branches from template_config.yml with open("./template_config.yml", "r") as f: plugin_template = yaml.safe_load(f) -new_labels = set(["backport-" + x for x in plugin_template["ci_update_branches"]]) +branches = set(plugin_template["supported_release_branches"]) +latest_release_branch = plugin_template["latest_release_branch"] +if latest_release_branch is not None: + branches.add(latest_release_branch) +new_labels = {"backport-" + x for x in branches} # delete old labels that are not in new labels for label in old_labels.difference(new_labels): diff --git a/docs/Makefile b/docs/Makefile index e3399c2..248471d 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -17,7 +17,6 @@ DIAGRAM_BUILD_DIR = _diagrams PULP_URL ?= http://localhost:24817 PULP_API_ROOT ?= /pulp/ - # Internal variables. PULP_V3_API_JSON_URL := ${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json PAPEROPT_a4 = -D latex_paper_size=a4 @@ -74,7 +73,12 @@ endif $(BUILDDIR)/html/api.json: mkdir -p $(BUILDDIR)/html - curl --fail -o $(BUILDDIR)/html/api.json "$(PULP_V3_API_JSON_URL)?plugin=pulp_gem&include_html=1" + if pulp debug has-plugin --name core --specifier ">=3.44.0.dev"; \ + then \ + curl --fail -o $(BUILDDIR)/html/api.json "$(PULP_V3_API_JSON_URL)?component=gem&include_html=1"; \ + else \ + curl --fail -o $(BUILDDIR)/html/api.json "$(PULP_V3_API_JSON_URL)?plugin=pulp_gem&include_html=1"; \ + fi html: $(BUILDDIR)/html/api.json $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/docs/template_gitref b/docs/template_gitref index bff7b99..06c4f52 100644 --- a/docs/template_gitref +++ b/docs/template_gitref @@ -1 +1 @@ -2021.08.26-301-g83f0607 +2021.08.26-310-g59cd732 diff --git a/lint_requirements.txt b/lint_requirements.txt index 94dbbc8..82422c2 100644 --- a/lint_requirements.txt +++ b/lint_requirements.txt @@ -6,7 +6,7 @@ # For more info visit https://github.com/pulp/plugin_template # python packages handy for developers, but not required by pulp -black +black==23.12.1 check-manifest flake8 flake8-black diff --git a/template_config.yml b/template_config.yml index 990bc25..968e88d 100644 --- a/template_config.yml +++ b/template_config.yml @@ -1,9 +1,8 @@ # This config represents the latest values used when running the plugin-template. Any settings that # were not present before running plugin-template have been added with their default values. -# generated with plugin_template@2021.08.26-301-g83f0607 +# generated with plugin_template@2021.08.26-310-g59cd732 -additional_repos: [] api_root: /pulp/ black: true check_commit_message: false @@ -12,11 +11,7 @@ check_manifest: true check_stray_pulpcore_imports: true ci_env: {} ci_trigger: '{pull_request: {branches: [''*'']}}' -ci_update_branches: -- '0.2' -- '0.3' ci_update_docs: true -ci_update_release_behavior: replace-previous-version cli_package: pulp-cli-gem cli_repo: https://github.com/pulp/pulp-cli-gem.git core_import_allowed: [] @@ -33,6 +28,7 @@ flake8_ignore: [] github_org: pulp issue_tracker: github kanban: true +latest_release_branch: null lint_requirements: true noissue_marker: '[noissue]' parallel_test_workers: 8 @@ -70,6 +66,9 @@ stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 stalebot_limit_to_pulls: true +supported_release_branches: +- '0.2' +- '0.3' sync_ci: true test_azure: true test_cli: true