From 05dbeb52df16b6d451159b13fc652fc9294157db Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Mon, 27 May 2024 12:46:41 +0200 Subject: [PATCH] Stop publishing bindings [noissue] --- plugin-template | 3 + .../github/.github/workflows/nightly.yml.j2 | 6 - .../github/.github/workflows/publish.yml.j2 | 136 +----------------- .../scripts/publish_client_gem.sh.j2 | 33 ----- .../scripts/publish_client_pypi.sh.j2 | 28 ---- .../workflows/scripts/publish_docs.sh.j2 | 14 -- .../.github/workflows/scripts/script.sh.j2 | 76 +++------- .../stage-changelog-for-default-branch.py.j2 | 65 --------- .../github/.github/workflows/test.yml.j2 | 44 ------ 9 files changed, 28 insertions(+), 377 deletions(-) delete mode 100755 templates/github/.github/workflows/scripts/publish_client_gem.sh.j2 delete mode 100755 templates/github/.github/workflows/scripts/publish_client_pypi.sh.j2 delete mode 100644 templates/github/.github/workflows/scripts/stage-changelog-for-default-branch.py.j2 diff --git a/plugin-template b/plugin-template index 5368f143..9adaccb8 100755 --- a/plugin-template +++ b/plugin-template @@ -108,6 +108,9 @@ DEPRECATED_FILES = { ".github/workflows/fips.yml", ".github/workflows/scripts/create_release_from_tag.sh", ".github/workflows/scripts/release.py", + ".github/workflows/scripts/publish_client_gem.sh", + ".github/workflows/scripts/publish_client_pypi.sh", + ".github/workflows/scripts/stage-changelog-for-default-branch.py", ".github/workflows/scripts/stage-changelog-for-master.py", ".github/workflows/scripts/update_ci.sh", ".travis", diff --git a/templates/github/.github/workflows/nightly.yml.j2 b/templates/github/.github/workflows/nightly.yml.j2 index 0ceb9970..50b6fad8 100644 --- a/templates/github/.github/workflows/nightly.yml.j2 +++ b/templates/github/.github/workflows/nightly.yml.j2 @@ -84,12 +84,6 @@ jobs: name: "docs.tar" path: "{{ plugin_name }}" - - name: Download Python client docs - uses: actions/download-artifact@v4 - with: - name: "python-client-docs.tar" - path: "{{ plugin_name }}" - {{ set_secrets() | indent(6) }} - name: Publish docs to pulpproject.org diff --git a/templates/github/.github/workflows/publish.yml.j2 b/templates/github/.github/workflows/publish.yml.j2 index e2635d3b..2b6303ed 100644 --- a/templates/github/.github/workflows/publish.yml.j2 +++ b/templates/github/.github/workflows/publish.yml.j2 @@ -24,8 +24,9 @@ defaults: jobs: build: uses: "./.github/workflows/build.yml" + {%- if publish_docs_to_pulpprojectdotorg %} - build-bindings-docs: + build-docs: needs: - "build" runs-on: "ubuntu-latest" @@ -50,54 +51,11 @@ jobs: {{ install_python_deps(["towncrier", "twine", "wheel", "httpie", "docker", "netaddr", "boto3", "ansible", "mkdocs"]) | indent(6) }} - # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to - # setup the Pulp instance. + # Building the docs requires accessing the OpenAPI specs endpoint, so we need to setup the Pulp instance. {{ run_script(name="Before Install", file="before_install.sh") | indent(6) }} {{ run_script(name="Install", file="install.sh") | indent(6) }} - {{ run_script(name="Install Python client", file="install_python_client.sh", withenv=False) | indent(6) }} - - {%- if deploy_client_to_rubygems %} - {{ run_script(name="Install Ruby client", file="install_ruby_client.sh", withenv=False) | indent(6) }} - {%- endif %} - - - name: "Upload python client packages" - uses: "actions/upload-artifact@v4" - with: - name: "python-client.tar" - path: | - {%- for plugin in plugins %} - {{ plugin_name }}/{{ plugin.app_label }}-python-client.tar - {%- endfor %} - if-no-files-found: "error" - overwrite: true - - - name: "Upload python client docs" - uses: "actions/upload-artifact@v4" - with: - name: "python-client-docs.tar" - path: | - {%- for plugin in plugins %} - {{ plugin_name }}/{{ plugin.app_label }}-python-client-docs.tar - {%- endfor %} - if-no-files-found: "error" - overwrite: true - - {%- if deploy_client_to_rubygems %} - - name: "Upload ruby client packages" - uses: "actions/upload-artifact@v4" - with: - name: "ruby-client.tar" - path: | - {%- for plugin in plugins %} - {{ plugin_name }}/{{ plugin.app_label }}-ruby-client.tar - {%- endfor %} - if-no-files-found: "error" - overwrite: true - {%- endif %} - - {%- if publish_docs_to_pulpprojectdotorg %} - name: Build docs run: | export DJANGO_SETTINGS_MODULE=pulpcore.app.settings @@ -112,15 +70,15 @@ jobs: path: "{{ plugin_name }}/docs/docs.tar" if-no-files-found: "error" overwrite: true - {%- endif %} {{ display_logs() | indent(6) }} + {%- endif %} {%- if deploy_to_pypi %} publish-package: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build" env: GITHUB_TOKEN: "{{ '${{ secrets.GITHUB_TOKEN }}' }}" @@ -144,79 +102,11 @@ jobs: .github/workflows/scripts/publish_plugin_pypi.sh {{ "${{ github.ref_name }}" }} {%- endif %} - {%- if deploy_client_to_pypi %} - publish-python-bindings: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "{{ '${{ secrets.GITHUB_TOKEN }}' }}" - - steps: - {{ checkout(depth=1, path=plugin_name) | indent(6) }} - - - name: "Download Python client" - uses: "actions/download-artifact@v4" - with: - name: "python-client.tar" - path: "{{ plugin_name }}/" - - - name: "Untar python client packages" - run: | - {%- for plugin in plugins %} - tar -xvf {{ plugin.app_label }}-python-client.tar - {%- endfor %} - - {{ setup_python() | indent(6) }} - - {{ install_python_deps(["twine"]) | indent(6) }} - - {{ set_secrets() | indent(6) }} - - - name: "Publish client to pypi" - run: | - bash .github/workflows/scripts/publish_client_pypi.sh {{ "${{ github.ref_name }}" }} - {%- endif %} - - {%- if deploy_client_to_rubygems %} - publish-ruby-bindings: - runs-on: "ubuntu-latest" - needs: - - "build-bindings-docs" - - env: - GITHUB_TOKEN: "{{ "${{ secrets.GITHUB_TOKEN }}" }}" - - steps: - {{ checkout(depth=1, path=plugin_name) | indent(6) }} - - - name: "Download Ruby client" - uses: "actions/download-artifact@v4" - with: - name: "ruby-client.tar" - path: "{{ plugin_name }}/" - - - name: "Untar Ruby client packages" - run: | - {%- for plugin in plugins %} - tar -xvf {{ plugin.app_label }}-ruby-client.tar - {%- endfor %} - - {{ setup_ruby() | indent(6) }} - - {{ set_secrets() | indent(6) }} - - - name: "Publish client to rubygems" - run: | - bash .github/workflows/scripts/publish_client_gem.sh {{ "${{ github.ref_name }}" }} - {%- endif %} - {%- if publish_docs_to_pulpprojectdotorg %} publish-docs: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build-docs" env: GITHUB_TOKEN: "{{ "${{ secrets.GITHUB_TOKEN }}" }}" @@ -236,12 +126,6 @@ jobs: name: "docs.tar" path: "{{ plugin_name }}/" - - name: "Download Python client docs" - uses: "actions/download-artifact@v4" - with: - name: "python-client-docs.tar" - path: "{{ plugin_name }}/" - - name: "Publish docs to pulpproject.org" run: | tar -xvf docs.tar @@ -251,16 +135,10 @@ jobs: create-gh-release: runs-on: "ubuntu-latest" needs: - - "build-bindings-docs" + - "build-docs" {%- if deploy_to_pypi %} - "publish-package" {%- endif %} - {%- if deploy_client_to_pypi %} - - "publish-python-bindings" - {%- endif %} - {%- if deploy_client_to_rubygems %} - - "publish-ruby-bindings" - {%- endif %} {%- if publish_docs_to_pulpprojectdotorg %} - "publish-docs" {%- endif %} diff --git a/templates/github/.github/workflows/scripts/publish_client_gem.sh.j2 b/templates/github/.github/workflows/scripts/publish_client_gem.sh.j2 deleted file mode 100755 index 09baf18b..00000000 --- a/templates/github/.github/workflows/scripts/publish_client_gem.sh.j2 +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -{% include 'header.j2' %} - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")"/../../.. - -VERSION="$1" - -if [[ -z "$VERSION" ]]; then - echo "No version specified." - exit 1 -fi - -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://rubygems.org/gems/{{ plugin_name | snake }}_client/versions/$VERSION")" - -if [ "$RESPONSE" == "200" ]; -then - echo "{{ plugin_name }} client $VERSION has already been released. Skipping." - exit -fi - -mkdir -p ~/.gem -touch ~/.gem/credentials -echo "--- -:rubygems_api_key: $RUBYGEMS_API_KEY" > ~/.gem/credentials -sudo chmod 600 ~/.gem/credentials - -{%- for plugin in plugins %} -gem push "{{ plugin.name | snake }}_client-${VERSION}.gem" -{%- endfor %} diff --git a/templates/github/.github/workflows/scripts/publish_client_pypi.sh.j2 b/templates/github/.github/workflows/scripts/publish_client_pypi.sh.j2 deleted file mode 100755 index 01834aca..00000000 --- a/templates/github/.github/workflows/scripts/publish_client_pypi.sh.j2 +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -{% include 'header.j2' %} - -set -euv - -# make sure this script runs at the repo root -cd "$(dirname "$(realpath -e "$0")")/../../.." - -VERSION="$1" - -if [[ -z "$VERSION" ]]; then - echo "No version specified." - exit 1 -fi -{%- for plugin in plugins %} - -RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/{{ plugin.name | dash }}-client/$VERSION/")" - -if [ "$RESPONSE" == "200" ]; -then - echo "{{ plugin.name }} client $VERSION has already been released. Skipping." -else - twine upload -u __token__ -p "$PYPI_API_TOKEN" \ - "dist/{{ plugin.name | snake }}_client-$VERSION-py3-none-any.whl" \ - "dist/{{ plugin.name | snake }}-client-$VERSION.tar.gz" -fi -{%- endfor %} diff --git a/templates/github/.github/workflows/scripts/publish_docs.sh.j2 b/templates/github/.github/workflows/scripts/publish_docs.sh.j2 index f7d707ef..ab0ba780 100755 --- a/templates/github/.github/workflows/scripts/publish_docs.sh.j2 +++ b/templates/github/.github/workflows/scripts/publish_docs.sh.j2 @@ -30,17 +30,3 @@ if [[ "$GITHUB_WORKFLOW" == "{{ plugin_app_label | camel }} changelog update" ]] # Do not build bindings docs on changelog update exit fi - -{%- for plugin in plugins %} - -mkdir -p ../{{ plugin.app_label }}-bindings -tar -xvf {{ plugin.app_label }}-python-client-docs.tar --directory ../{{ plugin.app_label }}-bindings -pushd ../{{ plugin.app_label }}-bindings - -# publish to docs.pulpproject.org/{{ plugin.name | snake }}_client -rsync -avzh site/ doc_builder_{{ plugin.name | snake }}@docs.pulpproject.org:/var/www/docs.pulpproject.org/{{ plugin.name | snake }}_client/ - -# publish to docs.pulpproject.org/{{ plugin.name | snake }}_client/en/{release} -rsync -avzh site/ doc_builder_{{ plugin.name | snake }}@docs.pulpproject.org:/var/www/docs.pulpproject.org/{{ plugin.name | snake }}_client/en/"$2" -popd -{%- endfor %} diff --git a/templates/github/.github/workflows/scripts/script.sh.j2 b/templates/github/.github/workflows/scripts/script.sh.j2 index d802e656..62ec8440 100755 --- a/templates/github/.github/workflows/scripts/script.sh.j2 +++ b/templates/github/.github/workflows/scripts/script.sh.j2 @@ -59,68 +59,28 @@ if pulp debug has-plugin --name "core" --specifier ">=3.44.0.dev" then # Use app_label to generate api.json and package to produce the proper package name. - if [ "$(jq -r '.domain_enabled' <<<"$REPORTED_STATUS")" = "true" ] - then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS") - do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" - PACKAGE="${MODULE%%.*}" - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" - USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" - done - else - # Sadly: Different pulpcore-versions aren't either... - for item in $(jq -r '.versions[] - {%- for plugin in plugins -%} - | select(.component!="{{ plugin.app_label }}") - {%- endfor -%} - | tojson' <<<"$REPORTED_STATUS") - do - echo $item - COMPONENT="$(jq -r '.component' <<<"$item")" - VERSION="$(jq -r '.version' <<<"$item")" - MODULE="$(jq -r '.module' <<<"$item")" - PACKAGE="${MODULE%%.*}" - curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" - USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" - cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" - sudo rm -rf "./${PACKAGE}-client" - done - fi + for item in $(jq -r '.versions[] | tojson' <<<"$REPORTED_STATUS") + do + echo $item + COMPONENT="$(jq -r '.component' <<<"$item")" + VERSION="$(jq -r '.version' <<<"$item")" + MODULE="$(jq -r '.module' <<<"$item")" + PACKAGE="${MODULE%%.*}" + curl --fail-with-body -k -o api.json "${PULP_URL}${PULP_API_ROOT}api/v3/docs/api.json?bindings&component=$COMPONENT" + USE_LOCAL_API_JSON=1 ./generate.sh "${PACKAGE}" python "${VERSION}" + cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" + sudo rm -rf "./${PACKAGE}-client" + done else # Infer the client name from the package name by replacing "-" with "_". # Use the component to infer the package name on older versions of pulpcore. - if [ "$(echo "$REPORTED_STATUS" | jq -r '.domain_enabled')" = "true" ] - then - # Workaround: Domains are not supported by the published bindings. - # Generate new bindings for all packages. - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - else - # Sadly: Different pulpcore-versions aren't either... - for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[] - {%- for plugin in plugins -%} - |select(.component!="{{ plugin.app_label }}") - {%- endfor -%} - |(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') - do - ./generate.sh "${item}" python - cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" - sudo rm -rf "./${item}-client" - done - fi + for item in $(echo "$REPORTED_STATUS" | jq -r '.versions[]|(.package // ("pulp_" + .component)|sub("pulp_core"; "pulpcore"))|sub("-"; "_")') + do + ./generate.sh "${item}" python + cmd_prefix pip3 install "/root/pulp-openapi-generator/${item}-client" + sudo rm -rf "./${item}-client" + done fi popd diff --git a/templates/github/.github/workflows/scripts/stage-changelog-for-default-branch.py.j2 b/templates/github/.github/workflows/scripts/stage-changelog-for-default-branch.py.j2 deleted file mode 100644 index ea6d7e85..00000000 --- a/templates/github/.github/workflows/scripts/stage-changelog-for-default-branch.py.j2 +++ /dev/null @@ -1,65 +0,0 @@ -{% include 'header.j2' %} - -import argparse -import os -import textwrap - -from git import Repo -from git.exc import GitCommandError - - -helper = textwrap.dedent( - """\ - Stage the changelog for a release on {{ plugin_default_branch }} branch. - - Example: - $ python .github/workflows/scripts/stage-changelog-for-default-branch.py 3.4.0 - - """ -) - -parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter, description=helper) - -parser.add_argument( - "release_version", - type=str, - help="The version string for the release.", -) - -args = parser.parse_args() - -release_version_arg = args.release_version - -release_path = os.path.dirname(os.path.abspath(__file__)) -plugin_path = release_path.split("/.github")[0] - -if not release_version_arg.endswith(".0"): - os._exit(os.system("python .ci/scripts/changelog.py")) - -print(f"\n\nRepo path: {plugin_path}") -repo = Repo(plugin_path) - -changelog_commit = None -# Look for a commit with the requested release version -for commit in repo.iter_commits(): - if f"{release_version_arg} changelog" == commit.message.split("\n")[0]: - changelog_commit = commit - break - if f"Add changelog for {release_version_arg}" == commit.message.split("\n")[0]: - changelog_commit = commit - break - -if not changelog_commit: - raise RuntimeError("Changelog commit for {release_version_arg} was not found.") - -git = repo.git -git.stash() -git.checkout("origin/{{ plugin_default_branch }}") -try: - git.cherry_pick(changelog_commit.hexsha) -except GitCommandError: - git.add("CHANGES/") - # Don't try opening an editor for the commit message - with git.custom_environment(GIT_EDITOR="true"): - git.cherry_pick("--continue") -git.reset("origin/{{ plugin_default_branch }}") diff --git a/templates/github/.github/workflows/test.yml.j2 b/templates/github/.github/workflows/test.yml.j2 index c6d906fa..c0c5aab7 100644 --- a/templates/github/.github/workflows/test.yml.j2 +++ b/templates/github/.github/workflows/test.yml.j2 @@ -59,20 +59,10 @@ jobs: {{ run_script(name="Before Install", file="before_install.sh") | indent(6) }} - {%- if deploy_client_to_rubygems %} - {{ setup_ruby(condition="${{ env.TEST == 'pulp' }}") | indent(6) }} - {%- endif %} - {{ run_script(name="Install", file="install.sh") | indent(6) }} {{ run_script(name="Before Script", file="before_script.sh", extra_env={"REDIS_DISABLED": "${{ contains('%s', matrix.env.TEST) }}" % (disabled_redis_runners | default(["TILT"]) | join(' '))}) | indent(6) }} - {{ run_script(name="Install Python client", file="install_python_client.sh", withenv=False) | indent(6) }} - - {%- if deploy_client_to_rubygems %} - {{ run_script(name="Install Ruby client", file="install_ruby_client.sh", withenv=False, condition="${{ env.TEST == 'pulp' }}") | indent(6) }} - {%- endif %} - {{ run_script(name="Script", file="script.sh") | indent(6) }} {%- if test_deprecations %} @@ -90,40 +80,6 @@ jobs: overwrite: true {%- endif %} - {%- for plugin in plugins %} - - name: Upload python client packages - if: {{ "${{ env.TEST == 'pulp' }}" }} - uses: actions/upload-artifact@v4 - with: - name: "python-client.tar" - path: "{{ plugin_name }}/{{ plugin.app_label }}-python-client.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - - - name: Upload python client docs - if: {{ "${{ env.TEST == 'pulp' }}" }} - uses: actions/upload-artifact@v4 - with: - name: "python-client-docs.tar" - path: "{{ plugin_name }}/{{ plugin.app_label }}-python-client-docs.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - - {%- if deploy_client_to_rubygems %} - - name: Upload Ruby client - if: {{ "${{ env.TEST == 'pulp' }}" }} - uses: actions/upload-artifact@v4 - with: - name: "ruby-client.tar" - path: "{{ plugin_name }}/{{ plugin.app_label }}-ruby-client.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - {%- endif %} - {%- endfor %} - {%- if docs_test %} - name: Upload built docs if: {{ "${{ env.TEST == 'docs' }}" }}