From e1e1dfcc086dffdcb40d5cf52295cf13b7f7aa66 Mon Sep 17 00:00:00 2001 From: sax Date: Tue, 11 Jun 2024 12:12:12 +0200 Subject: [PATCH] updates add ops actions --- .github/.gitignore | 1 + .github/actions/distro_hash/action.yml | 21 ++ .github/actions/docker_build/action.yml | 54 ++++++ .github/actions/docker_info/action.yml | 34 ++++ .github/actions/env/action.yml | 51 +++++ .github/actions/image_updated/action.yml | 233 +++++++++++++++++++++++ .github/actions/last_commit/action.yml | 15 ++ .github/actions/version/action.xyml | 99 ++++++++++ .github/actions/version/action.yml | 82 ++++++++ .gitignore | 2 + 10 files changed, 592 insertions(+) create mode 100644 .github/.gitignore create mode 100644 .github/actions/distro_hash/action.yml create mode 100644 .github/actions/docker_build/action.yml create mode 100644 .github/actions/docker_info/action.yml create mode 100644 .github/actions/env/action.yml create mode 100644 .github/actions/image_updated/action.yml create mode 100644 .github/actions/last_commit/action.yml create mode 100644 .github/actions/version/action.xyml create mode 100644 .github/actions/version/action.yml diff --git a/.github/.gitignore b/.github/.gitignore new file mode 100644 index 00000000..51c73eb0 --- /dev/null +++ b/.github/.gitignore @@ -0,0 +1 @@ +_workflows/* \ No newline at end of file diff --git a/.github/actions/distro_hash/action.yml b/.github/actions/distro_hash/action.yml new file mode 100644 index 00000000..53fc160d --- /dev/null +++ b/.github/actions/distro_hash/action.yml @@ -0,0 +1,21 @@ +# ref: https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action +name: 'Calculate Release Hash' +description: 'Calculate deps and os hash' +inputs: + files: + description: 'Files to use to calculate the hash' + required: true + default: "pdm.lock docker/bin/* docker/conf/* docker/Dockerfile" +outputs: + hash: # id of output + description: 'The time we greeted you' + value: ${{ steps.calc.outputs.hash }} + +runs: + using: 'composite' + steps: + - id: calc + shell: bash --noprofile --norc -eo pipefail -ux {0} + run: | + LOCK_SHA=$(sha1sum ${{ inputs.files }} | sha1sum | awk '{print $1}' | cut -c 1-8) + echo "hash=$LOCK_SHA" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/docker_build/action.yml b/.github/actions/docker_build/action.yml new file mode 100644 index 00000000..69fea0a1 --- /dev/null +++ b/.github/actions/docker_build/action.yml @@ -0,0 +1,54 @@ +# ref: https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action +name: 'Calculate version hash' +description: 'Calculate deps and os hash' +inputs: + username: + description: '' + required: true + password: + description: '' + required: true + image: + description: '' + required: true + target: + description: '' + required: true + + +runs: + using: 'composite' + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + tags: ${{vars.DOCKER_IMAGE}}:dev-latest + file: ./docker/Dockerfile + platforms: linux/amd64 + cache-from: type=registry,ref=${{vars.DOCKER_IMAGE}}:dev-latest + cache-to: type=inline + outputs: type=registry + target: python_dev_deps + build-args: | + BUILD_DATE="${{ steps.build_date.outputs.date }}" + CHECKSUM="${{ steps.release_hash.outputs.hash }}" + VERSION="${{ steps.version.outputs.version }}" + + - name: Calculate Release Hash + id: calc + shell: bash --noprofile --norc -eo pipefail {0} + run: | + docker pull ${{inputs.image}} + echo "----------" + docker inspect --format='{{json .Config.Labels}}' ${{inputs.image}}}}:dev-latest + docker run -t ${{inputs.image}}}}:dev-latest release-info.sh + echo "----------" diff --git a/.github/actions/docker_info/action.yml b/.github/actions/docker_info/action.yml new file mode 100644 index 00000000..a450f404 --- /dev/null +++ b/.github/actions/docker_info/action.yml @@ -0,0 +1,34 @@ +# ref: https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action +name: 'Retrieve Docker Image Information' +description: 'Calculate deps and os hash' +inputs: + image: + description: 'Files to use to calculate the hash' + required: true + username: + description: '' + required: true + password: + description: '' + required: true + +runs: + using: 'composite' + steps: +# - name: Configure Git +# shell: bash +# run: git config --global --add safe.directory $(realpath .) +# - name: ch +# uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: docker/login-action@v3 + with: + username: ${{ inputs.username }} + password: ${{ inputs.password }} + - name: Calculate Release Hash + shell: bash + run: | + docker pull ${{inputs.image}} + echo "----------" + docker inspect --format='{{json .Config.Labels}}' ${{inputs.image}} + docker run -t ${{inputs.image}} release-info.sh + echo "----------" diff --git a/.github/actions/env/action.yml b/.github/actions/env/action.yml new file mode 100644 index 00000000..d45fdb19 --- /dev/null +++ b/.github/actions/env/action.yml @@ -0,0 +1,51 @@ +name: 'version' +description: '' + +outputs: + stage: + description: '' + value: ${{ steps.version.outputs.stage }} + version: + description: '' + value: ${{ steps.version.outputs.version }} + commit: + description: '' + value: ${{ steps.version.outputs.commit }} + release: + description: '' + value: ${{ steps.version.outputs.release }} + date: + description: '' + value: ${{ steps.build_date.outputs.date }} + today: + description: '' + value: ${{ steps.build_date.outputs.today }} + timestamp: + description: '' + value: ${{ steps.build_date.outputs.timestamp }} + branch: + description: '' + value: ${{ steps.extract_branch.outputs.branch }} + + +runs: + using: 'composite' + steps: + - shell: bash --noprofile --norc -eo pipefail {0} + run: git config --global --add safe.directory $(realpath .) + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - id: version + uses: ./.github/actions/version + - id: extract_branch + name: Extract branch name + shell: bash + run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT + - id: build_date + shell: bash --noprofile --norc -eo pipefail -ux {0} + run: | + d1=$(date +"%a,%e %b %Y %H:%M %Z") + d2=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + d3=$(date -u +"%Y%m%d%H%M%S") + echo "today=$d1" >> $GITHUB_OUTPUT + echo "date=$d2" >> $GITHUB_OUTPUT + echo "timestamp=$d3" >> $GITHUB_OUTPUT diff --git a/.github/actions/image_updated/action.yml b/.github/actions/image_updated/action.yml new file mode 100644 index 00000000..a6908112 --- /dev/null +++ b/.github/actions/image_updated/action.yml @@ -0,0 +1,233 @@ +# ref: https://docs.github.com/en/actions/creating-actions/creating-a-docker-container-action +name: 'Check if image with valid checksum exists in dockerhub' +description: 'WARNING: Only works for single platform images' +inputs: + image: + description: 'Docker Image ' + required: true + checksum: + description: 'checksum' + required: true + label: + description: 'checksum label' + default: 'checksum' + required: false + username: + description: 'DockerHub username ' + required: false + password: + description: 'DockerHub password ' + required: false + architecture: + description: 'DockerHub architecture to build ' + required: false + default: amd64 + +outputs: + updated: + description: 'Returns true if the image exists and the checksum is valid' + value: ${{ steps.check.outputs.updated }} + exists: + description: 'Returns true if the image exists' + value: ${{ steps.check.outputs.exists }} + build_number: + description: 'Returns build number for the current branch' + value: ${{ steps.check.outputs.build_number }} + build_date: + description: 'Returns tha image build date' + value: ${{ steps.check.outputs.build_date }} + +runs: + using: 'composite' + steps: + - name: Setup + id: setup + shell: bash + run: | + ref=${{ inputs.image }} + architecture=${{ inputs.architecture }} + repo="${ref%:*}" + tag="${ref##*:}" + + res=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull") + token=$(echo $res | jq -r '.token') + if [[ -z "$token" ]];then + echo "::error title=⛔ error hint::Unable to get valid token" + exit 1 + fi + echo "token=$token" >> $GITHUB_OUTPUT + echo "repo=$repo" >> $GITHUB_OUTPUT + echo "tag=$tag" >> $GITHUB_OUTPUT + echo "architecture=$architecture" >> $GITHUB_OUTPUT + - name: Check Checksum + id: check + shell: bash + run: | + url="https://registry-1.docker.io/v2/${{steps.setup.outputs.repo}}/manifests/${{steps.setup.outputs.tag}}" + manifest=$(curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ + -H 'Authorization: Bearer ${{steps.setup.outputs.token}}' \ + -s $url) + + if [[ $manifest == *MANIFEST_UNKNOWN* ]];then + echo "exists=false" >> "$GITHUB_OUTPUT" + echo "updated=false" >> "$GITHUB_OUTPUT" + echo "build_number=1" >> "$GITHUB_OUTPUT" + echo "build_date=-" >> "$GITHUB_OUTPUT" + exit 0 + fi + if [[ $manifest == *errors\":* ]];then + code=$(echo $manifest | jq .errors[0].code) + message=$(echo $manifest | jq .errors[0].message) + echo "::error title=$code error hint::$message https://registry-1.docker.io/v2/${repo}/manifests/${tag}" + exit 1 + fi + echo "exists=true" >> $GITHUB_OUTPUT + + if [[ -z "$manifest" ]];then + echo "::error title=⛔ error hint::Unable to get manifest from $url" + exit 1 + fi + check1=$(echo $manifest | jq 'try(.manifests[])') + check2=$(echo $manifest | jq 'try(.config.digest)') + + if [[ -n "$check1" ]]; then + digest=$(echo $manifest | jq -r ".manifests| map(select(.platform.architecture | contains (\"${{steps.setup.outputs.architecture}}\"))) | .[].digest" 2>&1) + elif [[ -n "$check2" ]]; then + digest=$(echo $manifest | jq -r '.config.digest') + else + echo "::error title=⛔ error hint::Unable to detect digest" + exit 1 + fi + if [[ $digest == null ]]; then + echo "::error title=⛔ error hint::Digest is null" + exit 1 + fi + if [[ -z "$digest" ]];then + echo "::error title=⛔ error hint::Digest is empty" + exit 1 + fi + url=https://registry-1.docker.io/v2/${{steps.setup.outputs.repo}}/blobs/$digest + blob=$(curl \ + --silent \ + --location \ + -H "Accept: application/vnd.docker.disribution.manifest.v2+json" \ + -H 'Authorization: Bearer ${{steps.setup.outputs.token}}' \ + $url ) + if [[ -z "$blob" ]]; then + echo "Unable to get blob from: https://registry-1.docker.io/v2/$repo/blobs/$digest" + echo "updated=false" >> "$GITHUB_OUTPUT" + echo "build_number=1" >> "$GITHUB_OUTPUT" + echo "build_date=-" >> "$GITHUB_OUTPUT" + echo "::error file={name},line={line},endLine={endLine},title={title}::{message}" + exit 1 + fi + build_number=$(echo $blob | jq '.config.Labels.BuildNumber') + checksum=$(echo $blob | jq -r '.config.Labels.checksum') + build_date=$(echo $blob | jq -r '.config.Labels.date') + + if [[ $build_number =~ '^[0-9]+$' ]] ; then + build_number=$(( build_number + 1 )) + else + build_number=1 + fi + if [[ $checksum == ${{inputs.checksum}} ]]; then + echo "updated=true" >> $GITHUB_OUTPUT + else + echo "updated=false" >> $GITHUB_OUTPUT + fi + echo "build_number=${build_number}" >> $GITHUB_OUTPUT + echo "build_date=${build_date}" >> $GITHUB_OUTPUT + +# +# - name: Check Image Updated +# id: check +# continue-on-error: true +# shell: bash --noprofile --norc -eo pipefail -ux {0} +# run: | +# trap 'echo "exitcode=$?" >> "$GITHUB_OUTPUT"' EXIT +# +# ref=${{ inputs.image }} +# architecture=${{ inputs.architecture }} +# repo="${ref%:*}" +# tag="${ref##*:}" +# +# echo $repo:$tag +# +# res=$(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${repo}:pull") +# token=$(echo $res | jq -r '.token') +# +# if [[ -z "$token" ]];then +# echo "Unable to get Auth Token" +# echo $res +# echo "::error file={name},line={line},endLine={endLine},title={title}::{message}" +# exit 1 +# fi +# +# manifest=$(curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json" \ +# -H "Authorization: Bearer $token" \ +# -s "https://registry-1.docker.io/v2/${repo}/manifests/${tag}") +# +# if [[ -z "$manifest" ]];then +# echo "Unable to get manifest " +# echo "updated=false" >> "$GITHUB_OUTPUT" +# echo "build_number=1" >> "$GITHUB_OUTPUT" +# fi +# check=$(echo $manifest | jq 'try(.manifests[])') +# +# if [[ -n "$check" ]]; then +# echo "Multi platform manifest detected " +# digest=$(echo $manifest | jq -r ".manifests| map(select(.platform.architecture | contains (\"${architecture}\"))) | .[].digest" 2>&1) +# else +# echo "Single platform manifest found " +# digest=$(echo $manifest | jq -r '.config.digest' 2>&1) +# fi +# if [[ $digest == null ]]; then +# echo "Unable to get digest " +# echo $manifest | jq +# echo "updated=false" >> "$GITHUB_OUTPUT" +# echo "build_number=1" >> "$GITHUB_OUTPUT" +# fi +# if [[ -z "$digest" ]];then +# echo "Unable to get digest " +# echo $manifest | jq +# echo "updated=false" >> "$GITHUB_OUTPUT" +# echo "build_number=1" >> "$GITHUB_OUTPUT" +# echo "::error file={name},line={line},endLine={endLine},title={title}::{message}" +# exit 1 +# fi +# +# blob=$(curl \ +# --silent \ +# --location \ +# --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \ +# --header "Authorization: Bearer $token" \ +# "https://registry-1.docker.io/v2/$repo/blobs/$digest") +# +# if [[ -z "$blob" ]]; then +# echo "Unable to get blob from: https://registry-1.docker.io/v2/$repo/blobs/$digest" +# echo "updated=false" >> "$GITHUB_OUTPUT" +# echo "build_number=1" >> "$GITHUB_OUTPUT" +# echo "::error file={name},line={line},endLine={endLine},title={title}::{message}" +# exit 1 +# fi +# checksum=$(echo $blob | jq '.config.Labels.checksum') +# build_number=$(echo $blob | jq '.config.Labels.BuildNumber') +# +# if [[ $build_number == null ]]; then +# build_number=$((build_number+1)) +# else +# build_number = 1 +# fi +# +# ret='???' +# if [[ $checksum == null ]]; then +# echo "Invalid Blob from: https://registry-1.docker.io/v2/$repo/blobs/$digest" +# ret=false +# elif [[ $checksum == ${{inputs.checksum}} ]]; then +# ret=true +# else +# echo "Checksums do not match $checksum <> ${{inputs.checksum}}" +# ret=false +# fi +# echo "updated=${ret}" >> "$GITHUB_OUTPUT" +# echo "build_number=${build_number}" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/last_commit/action.yml b/.github/actions/last_commit/action.yml new file mode 100644 index 00000000..63a7d91c --- /dev/null +++ b/.github/actions/last_commit/action.yml @@ -0,0 +1,15 @@ +name: 'Get Last commit' +description: '' +runs: + using: "composite" + steps: + - name: Setup Environment (PR) + if: ${{ github.event_name == 'pull_request' }} + shell: bash + run: | + echo "last_commit_sha=${{ github.event.pull_request.head.sha }}" >> ${GITHUB_ENV} + - name: Setup Environment (Push) + if: ${{ github.event_name == 'push' }} + shell: bash + run: | + echo "last_commit_sha=${GITHUB_SHA}" >> ${GITHUB_ENV} \ No newline at end of file diff --git a/.github/actions/version/action.xyml b/.github/actions/version/action.xyml new file mode 100644 index 00000000..217adc62 --- /dev/null +++ b/.github/actions/version/action.xyml @@ -0,0 +1,99 @@ +name: 'version' +description: '' + +outputs: + stage: + description: 'The time we greeted you' + value: ${{ steps.parser.outputs.stage }} + version: + description: 'The time we greeted you' + value: ${{ steps.parser.outputs.version }} + commit: + description: 'The time we greeted you' + value: ${{ steps.parser.outputs.commit }} + sha: + description: 'The time we greeted you' + value: ${{ steps.parser.outputs.sha }} + + +runs: + using: 'composite' + steps: + - name: Configure Git + shell: bash --noprofile --norc -eo pipefail -ux {0} + run: | + git config --global --add safe.directory $(realpath .) + - name: ch + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Parse branch + shell: bash --noprofile --norc -eo pipefail -ux {0} + id: parser + run: | + # BASE=$(git describe --all --exact-match 2>/dev/null | sed 's=.*/==') + # BASE="1.2.3-rc1" + # VERSION="${BASE#[vV]}" + # VERSION_MAJOR="${VERSION%%.*}" + # VERSION_MINOR_PATCH="${VERSION#*.}" + # VERSION_MINOR="${VERSION_MINOR_PATCH%%.*}" + # VERSION_PATCH_PRE_RELEASE="${VERSION_MINOR_PATCH#*.}" + # VERSION_PATCH="${VERSION_PATCH_PRE_RELEASE%%[-+]*}" + # VERSION_PRE_RELEASE="" + # + # case "$VERSION" in + # *rc) + # VERSION_PRE_RELEASE="${VERSION#rc}" + # VERSION_PRE_RELEASE="${VERSION_PRE_RELEASE%%+*}" + # ;; + # *-*) + # VERSION_PRE_RELEASE="${VERSION#*-}" + # VERSION_PRE_RELEASE="${VERSION_PRE_RELEASE%%+*}" + # ;; + # esac + if [[ "${{ github.event_name }}" == pull_request ]]; then + sha=$(git rev-parse --short ${{ github.event.pull_request.head.sha }}) + elif [[ "${{ github.event_name }}" == push ]]; then + sha="${GITHUB_SHA}" + else + sha="not-available" + fi + + commit="${{ github.sha }}" + + if [[ ${{github.ref_name}} == develop ]]; then + stage=dev + version="${GITHUB_SHA::7}" + release=snapshot + elif [[ ${{github.ref_name}} == master ]]; then + stage=final + version=11111 + elif [[ ${{github.ref_name}} == releases/* ]]; then + stage=rc + version=${GITHUB_SHA::7} + elif [[ ${{github.ref_name}} == tags/* ]]; then + stage=final + version=${GITHUB_SHA::7} + else + stage=a + version=${GITHUB_SHA::7} + fi + + cat <> $GITHUB_OUTPUT + echo "stage=$stage" >> $GITHUB_OUTPUT + echo "release=$release" >> $GITHUB_OUTPUT + echo "sha=$sha" >> $GITHUB_OUTPUT + echo "commit=$commit" >> $GITHUB_OUTPUT +# +# - name: Describe +# id: describe +# shell: bash --noprofile --norc -eo pipefail -ux {0} +# run: | +# echo ${{ steps.branch.version }} +# ver=$(`git symbolic-ref HEAD 2> /dev/null | cut -b 12-`-`git log --pretty=format:"%h" -1`) +# echo "version=$ver" >> $GITHUB_OUTPUT diff --git a/.github/actions/version/action.yml b/.github/actions/version/action.yml new file mode 100644 index 00000000..5b8628a9 --- /dev/null +++ b/.github/actions/version/action.yml @@ -0,0 +1,82 @@ +name: 'version' +description: '' + +outputs: + stage: + description: 'Docker stage to use' + value: ${{ steps.parser.outputs.stage }} + version: + description: 'Docker T' + value: ${{ steps.parser.outputs.version }} + commit: + description: 'The time we greeted you' + value: ${{ steps.parser.outputs.commit }} + sha: + description: 'The time we greeted you' + value: ${{ steps.parser.outputs.sha }} + + +runs: + using: 'composite' + steps: + - name: Configure Git + shell: bash --noprofile --norc -eo pipefail {0} + run: git config --global --add safe.directory $(realpath .) + - name: ch + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Parse branch + shell: bash --noprofile --norc -eo pipefail -ux {0} + id: parser + run: | + release="not-available" + sha="not-available" + + if [[ ${{ github.event_name }} == pull_request ]]; then + sha=$(git rev-parse --short ${{ github.event.pull_request.head.sha }}) + elif [[ "${{ github.event_name }}" == push ]]; then + sha="${GITHUB_SHA}" + else + sha="not-available" + fi + + commit="${{ github.sha }}" + + if [[ ${{github.ref_name}} == develop ]]; then + stage=dev + version="${GITHUB_SHA::7}" + release=snapshot + elif [[ ${{github.ref_name}} == master ]]; then + stage=final + version=11111 + elif [[ ${{github.ref_name}} == releases/* ]]; then + stage=rc + version=${GITHUB_SHA::7} + elif [[ ${{github.ref_name}} == tags/* ]]; then + stage=final + version=${GITHUB_SHA::7} + else + stage=a + version=${GITHUB_SHA::7} + fi + + cat <> $GITHUB_OUTPUT + echo "version=$version" >> $GITHUB_OUTPUT + echo "stage=$stage" >> $GITHUB_OUTPUT + echo "release=$release" >> $GITHUB_OUTPUT + echo "sha=$sha" >> $GITHUB_OUTPUT + echo "commit=$commit" >> $GITHUB_OUTPUT +# +# - name: Describe +# id: describe +# shell: bash --noprofile --norc -eo pipefail -ux {0} +# run: | +# echo ${{ steps.branch.version }} +# ver=$(`git symbolic-ref HEAD 2> /dev/null | cut -b 12-`-`git log --pretty=format:"%h" -1`) +# echo "version=$ver" >> $GITHUB_OUTPUT diff --git a/.gitignore b/.gitignore index 2eaca4fe..50d6b3af 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,11 @@ *.egg-info __pycache__/ *.py[cod] + !tests/.coveragerc !.dockerignore !.flake8 +!.github !.gitignore !.gitlab/ !.tx/config