From 49fbe1c8f7c60ce534b20f76fcdae02155e860c0 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Fri, 21 Jun 2024 16:43:34 +0200 Subject: [PATCH] WIP --- .github/workflows/build.yml | 36 ++++++++++++- .github/workflows/test.yml | 35 ++++--------- .../github/.github/workflows/build.yml.j2 | 43 ++++++++++++++- .../scripts/build_python_client.sh.j2 | 52 +++++++++++++++++++ .../.github/workflows/scripts/script.sh.j2 | 4 ++ .../github/.github/workflows/test.yml.j2 | 33 ++++-------- 6 files changed, 155 insertions(+), 48 deletions(-) create mode 100755 templates/github/.github/workflows/scripts/build_python_client.sh.j2 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 306d913b..391e212a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,10 +27,16 @@ jobs: # Below this line we include the steps of the ci workflow of the generated plugin + - uses: "actions/checkout@v4" + with: + fetch-depth: 1 + repository: "pulp/pulp-openapi-generator" + path: "pulp-openapi-generator" + - uses: "actions/setup-python@v5" - name: "Install python dependencies" run: | echo ::group::PYDEPS - pip install packaging twine wheel + pip install packaging twine wheel mkdocs echo ::endgroup:: - name: "Build package" run: | @@ -61,3 +67,31 @@ jobs: if-no-files-found: "error" retention-days: 5 overwrite: true + - name: "Build bindings packages" + run: | + .github/workflows/scripts/build_python_client.sh + shell: "bash" + env: + PY_COLORS: "1" + ANSIBLE_FORCE_COLOR: "1" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" + - name: "Upload python client packages" + uses: "actions/upload-artifact@v4" + with: + name: "python-client.tar" + path: | + pulp_catdog/catdog-python-client.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Upload python client docs" + uses: "actions/upload-artifact@v4" + with: + name: "python-client-docs.tar" + path: | + pulp_catdog/catdog-python-client-docs.tar + if-no-files-found: "error" + retention-days: 5 + overwrite: true +... diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d9b21f46..3c12961d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,6 +57,16 @@ jobs: name: "plugin_package" path: "pulp_catdog/dist/" + - uses: "actions/download-artifact@v4" + with: + name: "python-client.tar" + path: "pulp-openapi-generator" + + - name: "Unpack client packages" + working-directory: "pulp-openapi-generator" + run: | + tar xvf python-client.tar + - name: "Install python dependencies" run: | echo ::group::PYDEPS @@ -102,11 +112,6 @@ jobs: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" GITHUB_CONTEXT: "${{ github.event.pull_request.commits_url }}" REDIS_DISABLED: "${{ contains('s3', matrix.env.TEST) }}" - - - name: "Install Python client" - run: | - .github/workflows/scripts/install_python_client.sh - shell: "bash" - name: "Install Ruby client" if: "${{ env.TEST == 'pulp' }}" run: | @@ -135,25 +140,6 @@ jobs: if-no-files-found: "error" retention-days: 5 overwrite: true - - name: Upload python client packages - if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v4 - with: - name: "python-client.tar" - path: "pulp_catdog/catdog-python-client.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - - - name: Upload python client docs - if: ${{ env.TEST == 'pulp' }} - uses: actions/upload-artifact@v4 - with: - name: "python-client-docs.tar" - path: "pulp_catdog/catdog-python-client-docs.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - name: Upload Ruby client if: ${{ env.TEST == 'pulp' }} uses: actions/upload-artifact@v4 @@ -184,3 +170,4 @@ jobs: docker exec pulp ls -latr /etc/yum.repos.d/ || true docker exec pulp cat /etc/yum.repos.d/* || true docker exec pulp bash -c "pip3 list && pip3 install pipdeptree && pipdeptree" +... diff --git a/templates/github/.github/workflows/build.yml.j2 b/templates/github/.github/workflows/build.yml.j2 index 1787c8e2..5b40e42b 100644 --- a/templates/github/.github/workflows/build.yml.j2 +++ b/templates/github/.github/workflows/build.yml.j2 @@ -3,6 +3,7 @@ checkout, setup_python, install_python_deps, + run_script, with context %} --- name: "Build" @@ -19,8 +20,9 @@ jobs: steps: {{ checkout(path=plugin_name) | indent(6) }} + {{ checkout(repository="pulp/pulp-openapi-generator", path="pulp-openapi-generator") | indent(6) }} {{ setup_python() | indent(6) }} - {{ install_python_deps(["packaging", "twine", "wheel"]) | indent(6) }} + {{ install_python_deps(["packaging", "twine", "wheel", "mkdocs"]) | indent(6) }} - name: "Build package" run: | python3 setup.py sdist bdist_wheel --python-tag py3 @@ -54,3 +56,42 @@ jobs: if-no-files-found: "error" retention-days: 5 overwrite: true + {{ run_script(name="Build bindings packages", file="build_python_client.sh") | indent(6) }} + - name: "Upload python client packages" + uses: "actions/upload-artifact@v4" + with: + name: "python-client.tar" + path: | + {%- for plugin in plugins %} + {{ plugin_name }}/{{ plugin.app_label }}-python-client.tar + {%- endfor %} + if-no-files-found: "error" + retention-days: 5 + overwrite: true + - name: "Upload python client docs" + uses: "actions/upload-artifact@v4" + with: + name: "python-client-docs.tar" + path: | + {%- for plugin in plugins %} + {{ plugin_name }}/{{ plugin.app_label }}-python-client-docs.tar + {%- endfor %} + if-no-files-found: "error" + retention-days: 5 + overwrite: true + {#- TODO + {%- if deploy_client_to_rubygems %} + - name: "Upload Ruby client" + uses: "actions/upload-artifact@v4" + with: + name: "ruby-client.tar" + path: | + {%- for plugin in plugins %} + {{ plugin_name }}/{{ plugin.app_label }}-ruby-client.tar + {%- endfor %} + if-no-files-found: "error" + retention-days: 5 + overwrite: true + {%- endif %} + #} +... diff --git a/templates/github/.github/workflows/scripts/build_python_client.sh.j2 b/templates/github/.github/workflows/scripts/build_python_client.sh.j2 new file mode 100755 index 00000000..54d42b56 --- /dev/null +++ b/templates/github/.github/workflows/scripts/build_python_client.sh.j2 @@ -0,0 +1,52 @@ +#!/bin/bash + +# This script expects all -api.json files to exist in the plugins root directory. +# It produces a -python-client.tar and -python-client-docs.tar file in the plugins root directory. + +{% include 'header.j2' %} + +set -mveuo pipefail + +# make sure this script runs at the repo root +cd "$(dirname "$(realpath -e "$0")")"/../../.. + +pushd ../pulp-openapi-generator +{%- for plugin in plugins %} +rm -rf "{{ plugin.name | snake }}-client" + +cp "../{{ plugin_name }}/{{ plugin.app_label }}-api.json" . +./gen-client.sh "{{ plugin.app_label }}-api.json" "{{ plugin.app_label }}" python "{{ plugin.name_snake }}" + +pushd {{ plugin.name | snake }}-client +python setup.py sdist bdist_wheel --python-tag py3 + +twine check "dist/{{ plugin.name | snake }}_client-"*"-py3-none-any.whl" +twine check "dist/{{ plugin.name | snake }}-client-"*".tar.gz" + +tar cvf ../../{{ plugin_name }}/{{ plugin.app_label }}-python-client.tar ./dist + +find ./docs/* -exec sed -i 's/Back to README/Back to HOME/g' {} \; +find ./docs/* -exec sed -i 's/README//g' {} \; +cp README.md docs/index.md +sed -i 's/docs\///g' docs/index.md +find ./docs/* -exec sed -i 's/\.md//g' {} \; + +cat >> mkdocs.yml << DOCSYAML +--- +site_name: {{ plugin.name | camel }} Client +site_description: {{ plugin.app_label | camel }} bindings +site_author: Pulp Team +site_url: https://docs.pulpproject.org/{{ plugin.name | snake }}_client/ +repo_name: pulp/{{ plugin.name | snake }} +repo_url: https://github.com/pulp/{{ plugin.name | snake }} +theme: readthedocs +DOCSYAML + +# Building the bindings docs +mkdocs build + +# Pack the built site. +tar cvf ../../{{ plugin_name }}/{{ plugin.app_label }}-python-client-docs.tar ./site +popd +{%- endfor %} +popd diff --git a/templates/github/.github/workflows/scripts/script.sh.j2 b/templates/github/.github/workflows/scripts/script.sh.j2 index 29d5c93a..de8b87c2 100755 --- a/templates/github/.github/workflows/scripts/script.sh.j2 +++ b/templates/github/.github/workflows/scripts/script.sh.j2 @@ -91,6 +91,10 @@ else cmd_prefix pip3 install "/root/pulp-openapi-generator/${PACKAGE}-client" sudo rm -rf "./${PACKAGE}-client" done + # install the prebuilt clients + {%- for plugin in plugins %} + cmd_prefix pip3 install "/root/pulp-openapi-generator/dist/{{ plugin.name | snake }}_client-"*"-py3-none-any.whl" + {%- endfor %} fi popd diff --git a/templates/github/.github/workflows/test.yml.j2 b/templates/github/.github/workflows/test.yml.j2 index c6d906fa..44b0b3ab 100644 --- a/templates/github/.github/workflows/test.yml.j2 +++ b/templates/github/.github/workflows/test.yml.j2 @@ -53,6 +53,16 @@ jobs: name: "plugin_package" path: "{{ plugin_name }}/dist/" + - uses: "actions/download-artifact@v4" + with: + name: "python-client.tar" + path: "pulp-openapi-generator" + + - name: "Unpack client packages" + working-directory: "pulp-openapi-generator" + run: | + tar xvf python-client.tar + {{ install_python_deps(["towncrier", "twine", "wheel", "httpie", "docker", "netaddr", "boto3", "ansible", "mkdocs"]) | indent(6) }} {{ setup_env() | indent(6) }} @@ -67,8 +77,6 @@ jobs: {{ run_script(name="Before Script", file="before_script.sh", extra_env={"REDIS_DISABLED": "${{ contains('%s', matrix.env.TEST) }}" % (disabled_redis_runners | default(["TILT"]) | join(' '))}) | indent(6) }} - {{ run_script(name="Install Python client", file="install_python_client.sh", withenv=False) | indent(6) }} - {%- if deploy_client_to_rubygems %} {{ run_script(name="Install Ruby client", file="install_ruby_client.sh", withenv=False, condition="${{ env.TEST == 'pulp' }}") | indent(6) }} {%- endif %} @@ -91,26 +99,6 @@ jobs: {%- endif %} {%- for plugin in plugins %} - - name: Upload python client packages - if: {{ "${{ env.TEST == 'pulp' }}" }} - uses: actions/upload-artifact@v4 - with: - name: "python-client.tar" - path: "{{ plugin_name }}/{{ plugin.app_label }}-python-client.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - - - name: Upload python client docs - if: {{ "${{ env.TEST == 'pulp' }}" }} - uses: actions/upload-artifact@v4 - with: - name: "python-client-docs.tar" - path: "{{ plugin_name }}/{{ plugin.app_label }}-python-client-docs.tar" - if-no-files-found: "error" - retention-days: 5 - overwrite: true - {%- if deploy_client_to_rubygems %} - name: Upload Ruby client if: {{ "${{ env.TEST == 'pulp' }}" }} @@ -137,3 +125,4 @@ jobs: {%- endif %} {{ display_logs() | indent(6) }} +...