Skip to content

Commit

Permalink
Add gen-client.sh
Browse files Browse the repository at this point in the history
This script focusses on generating the client from an api spec file. It
does not require access to a running pulp instance.

[noissue]
  • Loading branch information
mdellweg committed Jun 18, 2024
1 parent 6bc3825 commit ff896e2
Show file tree
Hide file tree
Showing 6 changed files with 155 additions and 167 deletions.
2 changes: 1 addition & 1 deletion .ci/ansible/start_container.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
follow_redirects: none
register: result
until: result.status == 200
retries: 6
retries: 20
delay: 5
rescue:
- name: "Output pulp container log"
Expand Down
44 changes: 9 additions & 35 deletions .github/workflows/scripts/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,52 +9,26 @@

set -euv

mkdir .ci/ansible/vars || true
echo "---" > .ci/ansible/vars/main.yaml
# make sure this script runs at the repo root
cd "$(dirname "$(realpath -e "$0")")"/../../..

TAG="${TAG:-latest}"

mkdir -p .ci/ansible/vars
cd .ci/ansible/

TAG=ci_build

if [ -n "${GITHUB_REF##*/}" ]; then
# Install the plugin only and use published PyPI packages for the rest
# Quoting ${TAG} ensures Ansible casts the tag as a string.
cat >> vars/main.yaml << VARSYAML
image:
name: pulp
tag: "${TAG}"
plugins:
- name: pulpcore
source: pulpcore
- name: pulp_file
source: "pulp_file"
services:
- name: pulp
image: "pulp:${TAG}"
volumes:
- ./settings:/etc/pulp
VARSYAML
else
cat >> vars/main.yaml << VARSYAML
image:
name: pulp
tag: "${TAG}"
plugins:
- name: pulp_file
source: "pulp_file"
- name: pulpcore
source: ./pulpcore
cat >> vars/main.yaml << VARSYAML
---
services:
- name: pulp
image: "pulp:${TAG}"
image: "ghcr.io/pulp/pulp:${TAG}"
volumes:
- ./settings:/etc/pulp
VARSYAML
fi

cat >> vars/main.yaml << VARSYAML
pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"]}
VARSYAML

ansible-playbook build_container.yaml
# ansible-playbook build_container.yaml
ansible-playbook start_container.yaml
36 changes: 16 additions & 20 deletions .github/workflows/scripts/script.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,39 +10,35 @@

set -mveuo pipefail

# make sure this script runs at the repo root
cd "$(dirname "$(realpath -e "$0")")"/../../..

# Needed for both starting the service and building the docs.
# Gets set in .github/settings.yml, but doesn't seem to inherited by
# this script.
export DJANGO_SETTINGS_MODULE=pulpcore.app.settings
export PULP_SETTINGS=$GITHUB_WORKSPACE/.ci/ansible/settings/settings.py

export DJANGO_SETTINGS_MODULE="pulpcore.app.settings"
export PULP_SETTINGS="$PWD/.ci/ansible/settings/settings.py"
export PULP_URL="http://pulp"

cd $GITHUB_WORKSPACE

./generate.sh pulpcore python
pip install ./pulpcore-client
./generate.sh pulp_file python
pip install ./pulp_file-client

python $GITHUB_WORKSPACE/.github/workflows/scripts/test_bindings.py
cd ../pulp-openapi-generator
python .github/workflows/scripts/test_bindings.py

rm -rf ./pulpcore-client
./generate.sh pulpcore ruby
pushd pulpcore-client
gem build pulpcore_client
gem install --both ./pulpcore_client-*.gem
popd

./generate.sh pulpcore ruby 0
cd pulpcore-client
gem build pulpcore_client
gem install --both ./pulpcore_client-0.gem
cd ..
rm -rf ./pulp_file-client

./generate.sh pulp_file ruby 0

cd pulp_file-client
./generate.sh pulp_file ruby
pushd pulp_file-client
gem build pulp_file_client
gem install --both ./pulp_file_client-0.gem
cd ..
ruby $GITHUB_WORKSPACE/.github/workflows/scripts/test_bindings.rb
gem install --both ./pulp_file_client-*.gem
popd

exit
ruby .github/workflows/scripts/test_bindings.rb
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,6 @@ target/

# written in generate.sh
.openapi-generator-ignore

# generated client packages
/*-client/
120 changes: 120 additions & 0 deletions gen-client.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
#!/bin/bash

set -eu

if [ $# -eq 0 ]
then
echo "No arguments provided"
echo "Usage: $0 <api_spec> <component> [<language> [<package>]]"
exit 1
fi

API_SPEC="$1"
COMPONENT="$2"
LANGUAGE="${3:-python}"
PACKAGE="${4:-pulp_${COMPONENT//-/_}}"

DOMAIN_ENABLED="$(jq -r '.info."x-pulp-domain-enabled" // false' < "${API_SPEC}")"
VERSION="$(jq -r --arg component "$COMPONENT" '.info."x-pulp-app-versions"[$component]' < "${API_SPEC}")"

echo "Version: ${VERSION}"
[ "${VERSION}" = "none" ]

OPENAPI_PYTHON_IMAGE="${OPENAPI_PYTHON_IMAGE:-docker.io/openapitools/openapi-generator-cli:v4.3.1}"
OPENAPI_RUBY_IMAGE="${OPENAPI_RUBY_IMAGE:-docker.io/openapitools/openapi-generator-cli:v4.3.1}"
OPENAPI_TYPESCRIPT_IMAGE="${OPENAPI_TYPESCRIPT_IMAGE:-docker.io/openapitools/openapi-generator-cli:v5.2.1}"

if command -v podman > /dev/null
then
CONTAINER_EXEC=podman
if [[ -n "${PULP_MCS_LABEL:-}" ]]
then
USER_COMMAND=("--userns=keep-id" "--security-opt" "label=level:$PULP_MCS_LABEL")
else
USER_COMMAND=("--userns=keep-id")
fi
ULIMIT_COMMAND=()
else
CONTAINER_EXEC=docker
if [[ -n "${PULP_MCS_LABEL:-}" ]]
then
USER_COMMAND=("-u" "$(id -u)" "--security-opt" "label=level:$PULP_MCS_LABEL")
else
USER_COMMAND=("-u" "$(id -u)")
fi
ULIMIT_COMMAND=("--ulimit" "nofile=122880:122880")
fi

# Mount volumes from parent container with `--volumes-from` option if `PARENT_CONTAINER_ID` is set.
if [ -z "${PARENT_CONTAINER_ID:-}" ]
then
if command -v getenforce > /dev/null && [ "$(getenforce)" == "Enforcing" ]
then
VOLUME_NAME="/local:Z"
else
VOLUME_NAME="/local"
fi
VOLUME_OPTION=("--volume" "${PWD}:${VOLUME_NAME}")
VOLUME_DIR="/local"
else
VOLUME_OPTION=("--volumes-from" "${PARENT_CONTAINER_ID}:rw")
VOLUME_DIR="${PWD}"
fi

if [ "$LANGUAGE" = "python" ]
then
$CONTAINER_EXEC run \
"${ULIMIT_COMMAND[@]}" \
"${USER_COMMAND[@]}" \
--rm \
"${VOLUME_OPTION[@]}" \
"$OPENAPI_PYTHON_IMAGE" generate \
-i "${VOLUME_DIR}/${API_SPEC}" \
-g python \
-o "${VOLUME_DIR}/${PACKAGE}-client" \
"--additional-properties=packageName=pulpcore.client.${PACKAGE},projectName=${PACKAGE}-client,packageVersion=${VERSION},domainEnabled=${DOMAIN_ENABLED}" \
-t "${VOLUME_DIR}/templates/python" \
--skip-validate-spec \
--strict-spec=false
cp python/__init__.py "${PACKAGE}-client/pulpcore/"
cp python/__init__.py "${PACKAGE}-client/pulpcore/client/"
fi

if [ "$LANGUAGE" = "ruby" ]
then
# https://github.com/OpenAPITools/openapi-generator/wiki/FAQ#how-to-skip-certain-files-during-code-generation
mkdir -p "$1-client"
echo git_push.sh > "${PACKAGE}-client/.openapi-generator-ignore"

python3 remove-cookie-auth.py
$CONTAINER_EXEC run \
"${ULIMIT_COMMAND[@]}" \
"${USER_COMMAND[@]}" \
--rm \
"${VOLUME_OPTION[@]}" \
"$OPENAPI_RUBY_IMAGE" generate \
-i "${VOLUME_DIR}/${API_SPEC}" \
-g ruby \
-o "${VOLUME_DIR}/${PACKAGE}-client" \
"--additional-properties=gemName=${PACKAGE}_client,gemLicense="GPLv2+",gemVersion=${VERSION},gemHomepage=https://github.com/pulp/${PACKAGE}" \
--library=faraday \
-t "${VOLUME_DIR}/templates/ruby" \
--skip-validate-spec \
--strict-spec=false
fi

if [ "$LANGUAGE" = "typescript" ]
then
$CONTAINER_EXEC run \
"${ULIMIT_COMMAND[@]}" \
"${USER_COMMAND[@]}" \
--rm \
"${VOLUME_OPTION[@]}" \
"$OPENAPI_TYPESCRIPT_IMAGE" generate \
-i "${VOLUME_DIR}/${API_SPEC}" \
-g typescript-axios \
-o "${VOLUME_DIR}/${PACKAGE}-client" \
-t "${VOLUME_DIR}/templates/typescript-axios" \
--skip-validate-spec \
--strict-spec=false
fi
117 changes: 6 additions & 111 deletions generate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,32 +8,12 @@ then
exit 1
fi

if command -v podman > /dev/null
# match the component name by removing the "pulp/pulp_" prefix
if [ "$1" = "pulpcore" ]
then
CONTAINER_EXEC=podman
if [[ -n "${PULP_MCS_LABEL:-}" ]]
then
USER_COMMAND=("--userns=keep-id" "--security-opt" "label=level:$PULP_MCS_LABEL")
else
USER_COMMAND=("--userns=keep-id")
fi
ULIMIT_COMMAND=()
COMPONENT="core"
else
CONTAINER_EXEC=docker
if [[ -n "${PULP_MCS_LABEL:-}" ]]
then
USER_COMMAND=("-u" "$(id -u)" "--security-opt" "label=level:$PULP_MCS_LABEL")
else
USER_COMMAND=("-u" "$(id -u)")
fi
ULIMIT_COMMAND=("--ulimit" "nofile=122880:122880")
fi

if command -v getenforce > /dev/null && [ "$(getenforce)" == "Enforcing" ]
then
VOLUME_NAME="/local:Z"
else
VOLUME_NAME="/local"
COMPONENT=${1#"pulp_"}
fi

# Skip downloading the api.json if `USE_LOCAL_API_JSON` is set.
Expand All @@ -45,7 +25,7 @@ then

# Download the schema
RETRY_COUNT=0
until curl --fail-with-body -k -o api.json "${PULP_URL}docs/api.json?bindings&plugin=$1"
until curl --fail-with-body -k -o api.json "${PULP_URL}docs/api.json?bindings&component=${COMPONENT}"
do
if [ $RETRY_COUNT -eq 10 ]
then
Expand All @@ -57,94 +37,9 @@ then
# Get the version of the pulpcore or plugin as reported by status API
fi

DOMAIN_ENABLED="$(jq -r '.info | ."x-pulp-domain-enabled" // false' < api.json)"

if [ $# -gt 2 ]
then
export VERSION=$3
else
# match the component name by removing the "pulp/pulp_" prefix
if [ "$1" = "pulpcore" ]
then
COMPONENT="core"
else
COMPONENT=${1#"pulp_"}
fi

VERSION="$(curl -k "${PULP_URL}status/" | jq --arg component "$COMPONENT" -r '.versions[] | select(.component == $component) | .version')"
fi

# Mount volumes from parent container with `--volumes-from` option if the
# `PARENT_CONTAINER_ID` is set.
if [ -z "${PARENT_CONTAINER_ID:-}" ]
then
VOLUME_OPTION=("--volume" "${PWD}:${VOLUME_NAME}")
VOLUME_DIR="/local"
else
VOLUME_OPTION=("--volumes-from" "${PARENT_CONTAINER_ID}:rw")
VOLUME_DIR="${PWD}"
fi

OPENAPI_PYTHON_IMAGE="${OPENAPI_PYTHON_IMAGE:-docker.io/openapitools/openapi-generator-cli:v4.3.1}"
OPENAPI_RUBY_IMAGE="${OPENAPI_RUBY_IMAGE:-docker.io/openapitools/openapi-generator-cli:v4.3.1}"
OPENAPI_TYPESCRIPT_IMAGE="${OPENAPI_TYPESCRIPT_IMAGE:-docker.io/openapitools/openapi-generator-cli:v5.2.1}"

echo ::group::BINDINGS
if [ "$2" = "python" ]
then
$CONTAINER_EXEC run \
"${ULIMIT_COMMAND[@]}" \
"${USER_COMMAND[@]}" \
--rm \
"${VOLUME_OPTION[@]}" \
"$OPENAPI_PYTHON_IMAGE" generate \
-i "${VOLUME_DIR}/api.json" \
-g python \
-o "${VOLUME_DIR}/$1-client" \
"--additional-properties=packageName=pulpcore.client.$1,projectName=$1-client,packageVersion=${VERSION},domainEnabled=${DOMAIN_ENABLED}" \
-t "${VOLUME_DIR}/templates/python" \
--skip-validate-spec \
--strict-spec=false
cp python/__init__.py "$1-client/pulpcore/"
cp python/__init__.py "$1-client/pulpcore/client/"
fi
if [ "$2" = 'ruby' ]
then
# https://github.com/OpenAPITools/openapi-generator/wiki/FAQ#how-to-skip-certain-files-during-code-generation
mkdir -p "$1-client"
echo git_push.sh > "$1-client/.openapi-generator-ignore"

python3 remove-cookie-auth.py
$CONTAINER_EXEC run \
"${ULIMIT_COMMAND[@]}" \
"${USER_COMMAND[@]}" \
--rm \
"${VOLUME_OPTION[@]}" \
"$OPENAPI_RUBY_IMAGE" generate \
-i "${VOLUME_DIR}/api.json" \
-g ruby \
-o "${VOLUME_DIR}/$1-client" \
"--additional-properties=gemName=$1_client,gemLicense="GPLv2+",gemVersion=${VERSION},gemHomepage=https://github.com/pulp/$1" \
--library=faraday \
-t "${VOLUME_DIR}/templates/ruby" \
--skip-validate-spec \
--strict-spec=false
fi
if [ "$2" = 'typescript' ]
then
$CONTAINER_EXEC run \
"${ULIMIT_COMMAND[@]}" \
"${USER_COMMAND[@]}" \
--rm \
"${VOLUME_OPTION[@]}" \
"$OPENAPI_TYPESCRIPT_IMAGE" generate \
-i "${VOLUME_DIR}/api.json" \
-g typescript-axios \
-o "${VOLUME_DIR}/$1-client" \
-t "${VOLUME_DIR}/templates/typescript-axios" \
--skip-validate-spec \
--strict-spec=false
fi
./gen-client.sh api.json "${COMPONENT}" "${2:-python}" "${1}"

echo ::endgroup::
if [[ -z "${USE_LOCAL_API_JSON:-}" ]]
Expand Down

0 comments on commit ff896e2

Please sign in to comment.