WIP: Account for multi-plugin repositories

This adds a `plugin` list to the template config. For "usual" plugins it's populated automatically. [noissue]
pulp · Nov 21, 2023 · 2a9dec9 · 2a9dec9
1 parent 664ee4a
commit 2a9dec9
Show file tree

Hide file tree

Showing 6 changed files with 83 additions and 90 deletions.
diff --git a/plugin-template b/plugin-template
@@ -51,6 +51,7 @@ DEFAULT_SETTINGS = {
     "plugin_app_label": None,
     "plugin_default_branch": "main",
     "plugin_name": None,
+    "plugins": None,
     "post_job_template": None,
     "pre_job_template": None,
     "publish_docs_to_pulpprojectdotorg": False,
@@ -249,6 +250,10 @@ def main():
                     for key in set(config.keys()) - set(DEFAULT_SETTINGS.keys()):
                         config.pop(key)
                         write_new_config = True
+                    if config["plugins"] is None:
+                        config["plugins"] = [
+                            {"name": config["plugin_name"], "app_label": config["plugin_app_label"]}
+                        ]
                     print(
                         "\nLoaded plugin template config from "
                         "{path}/template_config.yml.\n".format(path=plugin_root_dir)
@@ -470,6 +475,7 @@ def generate_config(plugin_name, plugin_app_label):
     config = DEFAULT_SETTINGS.copy()
     config["plugin_name"] = plugin_name
     config["plugin_app_label"] = plugin_app_label
+    config["plugins"] = [{"name": plugin_name, "app_label": plugin_app_label}]
     return config
 
 

diff --git a/templates/github/.github/workflows/release.yml.j2 b/templates/github/.github/workflows/release.yml.j2
@@ -80,14 +80,6 @@ jobs:
         run: |
           tar -xf {{ plugin_name }}.tar
 
-      # update to the branch's latest ci files rather than the ones from the release tag. this is
-      # helpful when there was a problem with the ci files during the release which needs to be
-      # fixed after the release tag has been created
-      - name: Update ci files
-        run: |
-          git checkout "origin/${GITHUB_REF##*/}" -- .ci
-          git checkout "origin/${GITHUB_REF##*/}" -- .github
-
       {{ install_httpie() | indent(6) }}
 
       # Building the bindings and docs requires accessing the OpenAPI specs endpoint, so we need to
@@ -108,25 +100,30 @@ jobs:
       {{ run_script(name="Install Ruby client", file="install_ruby_client.sh", withenv=False) | indent(6) }}
       {%- endif %}
 
+      {%- for plugin in plugins %}
       - name: "Upload python client packages"
         uses: actions/upload-artifact@v3
         with:
           name: "python-client.tar"
-          path: "{{ plugin_name }}/python-client.tar"
+          path: "{{ plugin_name }}/{{ plugin.app_label }}-python-client.tar"
+          if-no-files-found: "error"
 
       - name: "Upload python client docs"
         uses: actions/upload-artifact@v3
         with:
           name: "python-client-docs.tar"
-          path: "{{ plugin_name }}/python-client-docs.tar"
+          path: "{{ plugin_name }}/{{ plugin.app_label }}-python-client-docs.tar"
+          if-no-files-found: "error"
 
       {%- if deploy_client_to_rubygems %}
       - name: "Upload ruby client packages"
         uses: actions/upload-artifact@v3
         with:
           name: "ruby-client.tar"
-          path: "{{ plugin_name }}/ruby-client.tar"
+          path: "{{ plugin_name }}/{{ plugin.app_label }}-ruby-client.tar"
+          if-no-files-found: "error"
       {%- endif %}
+      {%- endfor %}
 
       {%- if publish_docs_to_pulpprojectdotorg and docs_test %}
       - name: Build docs
@@ -170,14 +167,6 @@ jobs:
         run: |
           tar -xf {{  plugin_name }}.tar
 
-      # update to the branch's latest ci files rather than the ones from the release tag. this is
-      # helpful when there was a problem with the ci files during the release which needs to be
-      # fixed after the release tag has been created
-      - name: Update ci files
-        run: |
-          git checkout "origin/${GITHUB_REF##*/}" -- .ci
-          git checkout "origin/${GITHUB_REF##*/}" -- .github
-
       {{ set_secrets() | indent(6) }}
 
       {{ install_python_deps("gitpython requests packaging~=21.3 tweepy") | indent(6) }}
@@ -216,11 +205,13 @@ jobs:
           name: "python-client.tar"
           path: "{{ plugin_name }}/"
 
-      - name: Untar python client packages
-        run: tar -xvf python-client.tar
+      - name: "Untar python client packages"
+        run: |
+          tar -xvf python-client.tar
 
-      - name: Publish client to pypi
-        run: bash .github/workflows/scripts/publish_client_pypi.sh
+      - name: "Publish client to pypi"
+        run: |
+          bash .github/workflows/scripts/publish_client_pypi.sh
       {%- endif %}
 
       {%- if deploy_client_to_rubygems %}
@@ -230,11 +221,13 @@ jobs:
           name: "ruby-client.tar"
           path: "{{ plugin_name }}/"
 
-      - name: Untar Ruby client packages
-        run: tar -xvf ruby-client.tar
+      - name: "Untar Ruby client packages"
+        run: |
+          tar -xvf ruby-client.tar
 
-      - name: Publish client to rubygems
-        run: bash .github/workflows/scripts/publish_client_gem.sh
+      - name: "Publish client to rubygems"
+        run: |
+          bash .github/workflows/scripts/publish_client_gem.sh
       {%- endif %}
 
       - name: Create release on GitHub

diff --git a/templates/github/.github/workflows/scripts/install_python_client.sh.j2 b/templates/github/.github/workflows/scripts/install_python_client.sh.j2
@@ -4,43 +4,38 @@
 
 set -mveuo pipefail
 
-export PULP_URL="${PULP_URL:-{{ pulp_scheme }}://pulp}"
-
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../../..
 
 source .github/workflows/scripts/utils.sh
 
+export PULP_URL="${PULP_URL:-{{ pulp_scheme }}://pulp}"
+
 pip install twine wheel
 
 REPORTED_STATUS="$(pulp status)"
 REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "{{ plugin_app_label }}" -r '.versions[] | select(.component == $plugin) | .version')"
 VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')"
 
-RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/{{ plugin_name | dash }}-client/$VERSION/")"
-
-if [ "$RESPONSE" == "200" ];
-then
-  # TODO This is probably not needed.
-  echo "{{ plugin_name }} client $VERSION has already been released. Installing from PyPI."
-  cms_prefix pip3 install "{{ plugin_name | dash }}-client==$VERSION"
-  mkdir -p dist
-  tar cvf python-client.tar ./dist
-  exit
-fi
-
-cd ../pulp-openapi-generator
-rm -rf {{ plugin_name | snake }}-client
-./generate.sh {{ plugin_name | snake }} python "$VERSION"
-cd {{ plugin_name | snake }}-client
+pushd ../pulp-openapi-generator
+{%- for plugin in plugins %}
+rm -rf {{ plugin.name | snake }}-client
+./generate.sh {{ plugin.name | snake }} python "$VERSION"
+pushd {{ plugin.name | snake }}-client
 python setup.py sdist bdist_wheel --python-tag py3
-cmd_prefix pip3 install "/root/pulp-openapi-generator/{{ plugin_name | snake }}-client/dist/{{ plugin_name | snake }}_client-${VERSION}-py3-none-any.whl"
-tar cvf ../../{{ plugin_name }}/python-client.tar ./dist
+
+twine check "dist/{{ plugin.name | snake }}_client-$VERSION-py3-none-any.whl" || exit 1
+twine check "dist/{{ plugin.name | snake }}-client-$VERSION.tar.gz" || exit 1
+
+cmd_prefix pip3 install "/root/pulp-openapi-generator/{{ plugin.name | snake }}-client/dist/{{ plugin.name | snake }}_client-${VERSION}-py3-none-any.whl"
+tar cvf ../../{{ plugin_name }}/{{ plugin.app_label }}-python-client.tar ./dist
 
 find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \;
 find ./docs/* -exec sed -i 's/README//g' {} \;
 cp README.md docs/index.md
 sed -i 's/docs\///g' docs/index.md
 find ./docs/* -exec sed -i 's/\.md//g' {} \;
-tar cvf ../../{{ plugin_name }}/python-client-docs.tar ./docs
-exit $?
+tar cvf ../../{{ plugin_name }}/{{ plugin.app_label }}-python-client-docs.tar ./docs
+popd
+{%- endfor %}
+popd
diff --git a/templates/github/.github/workflows/scripts/install_ruby_client.sh.j2 b/templates/github/.github/workflows/scripts/install_ruby_client.sh.j2
@@ -2,37 +2,28 @@
 
 {% include 'header.j2' %}
 
-set -euv
+set -mveuo pipefail
 
 # make sure this script runs at the repo root
 cd "$(dirname "$(realpath -e "$0")")"/../../..
 
+source .github/workflows/scripts/utils.sh
+
 export PULP_URL="${PULP_URL:-{{ pulp_scheme }}://pulp}"
 
-export REPORTED_VERSION=$(http $PULP_URL/pulp/api/v3/status/ | jq --arg plugin {{ plugin_app_label }} --arg legacy_plugin {{ plugin_name | snake }} -r '.versions[] | select(.component == $plugin or .component == $legacy_plugin) | .version')
-export DESCRIPTION="$(git describe --all --exact-match `git rev-parse HEAD`)"
-if [[ $DESCRIPTION == 'tags/'$REPORTED_VERSION ]]; then
-  export VERSION=${REPORTED_VERSION}
-else
-  export EPOCH="$(date +%s)"
-  export VERSION=${REPORTED_VERSION}${EPOCH}
-fi
-
-export response=$(curl --write-out %{http_code} --silent --output /dev/null https://rubygems.org/gems/{{ plugin_name | snake }}_client/versions/$VERSION)
-
-if [ "$response" == "200" ];
-then
-  echo "{{ plugin_name }} client $VERSION has already been released. Installing from RubyGems.org."
-  gem install {{ plugin_name | snake }}_client -v $VERSION
-  touch {{ plugin_name | snake }}_client-$VERSION.gem
-  tar cvf ruby-client.tar ./{{ plugin_name | snake }}_client-$VERSION.gem
-  exit
-fi
-
-cd ../pulp-openapi-generator
-rm -rf {{ plugin_name | snake }}-client
-./generate.sh {{ plugin_name | snake }} ruby $VERSION
-cd {{ plugin_name | snake }}-client
-gem build {{ plugin_name | snake }}_client
-gem install --both ./{{ plugin_name | snake }}_client-$VERSION.gem
-tar cvf ../../{{ plugin_name }}/ruby-client.tar ./{{ plugin_name | snake }}_client-$VERSION.gem
+
+REPORTED_STATUS="$(pulp status)"
+REPORTED_VERSION="$(echo "$REPORTED_STATUS" | jq --arg plugin "{{ plugin_app_label }}" -r '.versions[] | select(.component == $plugin) | .version')"
+VERSION="$(echo "$REPORTED_VERSION" | python -c 'from packaging.version import Version; print(Version(input()))')"
+
+pushd ../pulp-openapi-generator
+{%- for plugin in plugins %}
+rm -rf {{ plugin.name | snake }}-client
+./generate.sh {{ plugin.name | snake }} ruby "$VERSION"
+pushd {{ plugin.name | snake }}-client
+gem build {{ plugin.name | snake }}_client
+gem install --both "./{{ plugin.name | snake }}_client-$VERSION.gem"
+tar cvf ../../{{ plugin_name }}/{{ plugin.app_label }}-ruby-client.tar "./{{ plugin.name | snake }}_client-$VERSION.gem"
+popd
+{%- endfor %}
+popd
diff --git a/templates/github/.github/workflows/scripts/publish_client_pypi.sh.j2 b/templates/github/.github/workflows/scripts/publish_client_pypi.sh.j2
@@ -5,28 +5,28 @@
 set -euv
 
 # make sure this script runs at the repo root
-cd "$(dirname "$(realpath -e "$0")")"/../../..
+cd "$(dirname "$(realpath -e "$0")")/../../.."
 
 pip install twine
 
-export VERSION=$(ls dist | sed -rn 's/{{ plugin_name | snake }}-client-(.*)\.tar.gz/\1/p')
+VERSION="$(ls dist | sed -rn 's/{{ plugin_name | snake }}-client-(.*)\.tar.gz/\1/p')"
 
 if [[ -z "$VERSION" ]]; then
   echo "No client package found."
   exit
 fi
 
-export response=$(curl --write-out %{http_code} --silent --output /dev/null https://pypi.org/project/{{ plugin_name | dash }}-client/$VERSION/)
+RESPONSE="$(curl --write-out '%{http_code}' --silent --output /dev/null "https://pypi.org/project/{{ plugin_name | dash }}-client/$VERSION/")"
 
-if [ "$response" == "200" ];
+if [ "$RESPONSE" == "200" ];
 then
   echo "{{ plugin_name }} client $VERSION has already been released. Skipping."
   exit
 fi
 
-twine check dist/{{ plugin_name | snake }}_client-$VERSION-py3-none-any.whl || exit 1
-twine check dist/{{ plugin_name | snake }}-client-$VERSION.tar.gz || exit 1
-twine upload dist/{{ plugin_name | snake }}_client-$VERSION-py3-none-any.whl -u pulp -p $PYPI_PASSWORD
-twine upload dist/{{ plugin_name | snake }}-client-$VERSION.tar.gz -u pulp -p $PYPI_PASSWORD
-
-exit $?
+twine upload -u pulp -p "$PYPI_PASSWORD" \
+{%- for plugin in plugins %}
+"dist/{{ plugin.name | snake }}_client-$VERSION-py3-none-any.whl" \
+"dist/{{ plugin.name | snake }}-client-$VERSION.tar.gz" \
+{%- endfor %}
+;
diff --git a/templates/github/.github/workflows/test.yml.j2 b/templates/github/.github/workflows/test.yml.j2
@@ -90,28 +90,36 @@ jobs:
         id: deprecations
         run: echo deprecations-{{ "${{ matrix.env.TEST }}" }}=$(docker logs pulp 2>&1 | grep -i pulpcore.deprecation | base64 -w 0) >> $GITHUB_OUTPUT
 
+      {%- for plugin in plugins %}
       - name: Upload python client packages
         if: {{ "${{ env.TEST == 'pulp' }}" }}
         uses: actions/upload-artifact@v3
         with:
           name: "python-client.tar"
-          path: "{{ plugin_name }}/python-client.tar"
+          path: "{{ plugin_name }}/{{ plugin.app_label }}-python-client.tar"
+          if-no-files-found: "error"
+          retention-days: 5
 
       - name: Upload python client docs
         if: {{ "${{ env.TEST == 'pulp' }}" }}
         uses: actions/upload-artifact@v3
         with:
           name: "python-client-docs.tar"
-          path: "{{ plugin_name }}/python-client-docs.tar"
+          path: "{{ plugin_name }}/{{ plugin.app_label }}-python-client-docs.tar"
+          if-no-files-found: "error"
+          retention-days: 5
 
       {%- if deploy_client_to_rubygems %}
       - name: Upload Ruby client
         if: {{ "${{ env.TEST == 'pulp' }}" }}
         uses: actions/upload-artifact@v3
         with:
           name: "ruby-client.tar"
-          path: "{{ plugin_name }}/ruby-client.tar"
+          path: "{{ plugin_name }}/{{ plugin.app_label }}-ruby-client.tar"
+          if-no-files-found: "error"
+          retention-days: 5
       {%- endif %}
+      {%- endfor %}
 
       {%- if docs_test %}
       - name: Upload built docs