2024-06-14 22:43 - updates #145
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: | |
| push: | |
| branches: | |
| - develop | |
| - master | |
| - staging | |
| - release/* | |
| - feature/* | |
| - bugfix/* | |
| - hotfix/* | |
| pull_request: | |
| branches: [develop, master] | |
| types: [synchronize, opened, reopened, ready_for_review] | |
| concurrency: | |
| group: "${{ github.workflow }}-${{ github.ref }}" | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| setup: | |
| # if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name | |
| name: check files | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 3 | |
| defaults: | |
| run: | |
| shell: bash | |
| outputs: | |
| checksum: ${{ steps.release_hash.outputs.hash }} | |
| branch: ${{ steps.env.outputs.branch }} | |
| date: ${{ steps.env.outputs.date }} | |
| test_image: ${{ steps.env.outputs.test_image }} | |
| dist_image: ${{ steps.env.outputs.dist_image }} | |
| version: ${{ steps.env.outputs.version }} | |
| last_commit: ${{ steps.env.outputs.last_commit }} | |
| image_build_date: ${{ steps.env2.outputs.image_build_date }} | |
| image_build_number: ${{ steps.env2.outputs.image_build_number }} | |
| image_checksum: ${{ steps.env2.outputs.image_checksum }} | |
| updated: ${{ steps.env2.outputs.updated }} | |
| # docker: ${{ steps.changes.outputs.docker_base }} | |
| # python_files: ${{ steps.changes.outputs.python }} | |
| # tags: ${{ steps.meta.outputs.tags }} | |
| # tag: ${{ steps.meta.outputs.version }} | |
| # image_name: ${{ steps.image_name.outputs.name }} | |
| steps: | |
| # - run: git config --global --add safe.directory $(realpath .) | |
| - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 | |
| - id: changes | |
| name: Check for file changes | |
| uses: dorny/paths-filter@0bc4621a3135347011ad047f9ecf449bf72ce2bd # v3.0.0 | |
| with: | |
| base: ${{ github.ref }} | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| # - name: Docker meta | |
| # id: meta | |
| # uses: docker/metadata-action@v5 | |
| - id: release_hash | |
| uses: ./.github/actions/distro_hash | |
| - id: last_commit | |
| uses: ./.github/actions/last_commit | |
| - id: env | |
| run: | | |
| branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | |
| build_date=$(date +"%Y-%m-%d %H:%M") | |
| ver="${{steps.meta.outputs.version}}" | |
| ver="${branch/\//-}" | |
| echo "version=$ver" >> $GITHUB_OUTPUT | |
| echo "date=$build_date" >> $GITHUB_OUTPUT | |
| echo "branch=$branch" >> $GITHUB_OUTPUT | |
| echo "test_image=${{vars.DOCKER_IMAGE}}:test-$ver" >> $GITHUB_OUTPUT | |
| echo "dist_image=${{vars.DOCKER_IMAGE}}:$ver" >> $GITHUB_OUTPUT | |
| echo "last_commit=${{steps.last_commit.outputs.LAST_COMMIT_SHA}}" >> $GITHUB_OUTPUT | |
| - name: Image meta | |
| id: meta | |
| uses: ./.github/actions/image_meta | |
| with: | |
| image: ${{steps.env.outputs.test_image}} | |
| username: ${{ inputs.username }} | |
| password: ${{ inputs.password }} | |
| - id: env2 | |
| run: | | |
| echo "image_build_date=${{steps.meta.outputs.build_date}}" >> $GITHUB_OUTPUT | |
| echo "image_build_number=${{steps.meta.outputs.build_number}}" >> $GITHUB_OUTPUT | |
| echo "image_checksum=${{steps.meta.outputs.checksum}}" >> $GITHUB_OUTPUT | |
| if [[ "${{steps.meta.outputs.checksum}}" == "${{needs.release_hash.outputs.hash}}" ]];then | |
| echo "updated=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "updated=false" >> $GITHUB_OUTPUT | |
| fi | |
| # echo "${{ toJSON(steps.meta.outputs) }}" | |
| # echo "build_number=${{steps.meta.outputs.build_number}}" >> $GITHUB_OUTPUT | |
| # echo "exists=${{steps.meta.outputs.exists}}" >> $GITHUB_OUTPUT | |
| # echo "checksum=${{steps.meta.outputs.checksum}}" >> $GITHUB_OUTPUT | |
| # | |
| # | |
| # | |
| # - name: Docker meta | |
| # id: meta | |
| # uses: docker/metadata-action@v5 | |
| # - id: env | |
| # run: | | |
| # br=${{ steps.extract_branch.outputs.branch }} | |
| # build_date=$(date +"%Y-%m-%d %H:%M") | |
| # | |
| # echo "version=${br##*/}" >> $GITHUB_OUTPUT | |
| # echo "date=$build_date" >> $GITHUB_OUTPUT | |
| # echo "image=${tag%:*}" >> $GITHUB_OUTPUT | |
| # echo "name=${{vars.DOCKER_IMAGE}}:test-${{needs.setup.outputs.tag}}" >> $GITHUB_OUTPUT | |
| # | |
| # - name: Calculate Release Hash | |
| # id: release_hash | |
| # uses: ./.github/actions/distro_hash | |
| # - id: image_updated | |
| # name: Check if image exists and updated | |
| # uses: ./.github/actions/image_updated | |
| # with: | |
| # image: ${{inputs.image}} | |
| # checksum: ${{ steps.release_hash.outputs.hash }} | |
| # username: ${{ inputs.username }} | |
| # password: ${{ inputs.password }} | |
| # - shell: bash | |
| # run: | | |
| # echo "" | |
| # echo "run_id : ${{ github.run_id }}" | |
| # echo "date : ${{steps.setup.outputs.date}}" | |
| # echo "image : ${{steps.setup.outputs.image}}" | |
| # echo "tag : ${{inputs.image}}" | |
| # echo "target : ${{inputs.target}}" | |
| # echo "target-checksum : ${{inputs.checksum}}" | |
| # echo "docker-checksum : ${{steps.image_updated.outputs.checksum }}" | |
| # echo "image_exists : ${{steps.image_updated.outputs.exists}}" | |
| # echo "image_updated : ${{steps.image_updated.outputs.updated}}" | |
| # echo "build_number : ${{steps.image_updated.outputs.build_number}}" | |
| # echo "build_date : ${{steps.image_updated.outputs.build_date}}" | |
| # echo "force : ${{inputs.force}}" | |
| # echo "build : ${{steps.image_updated.outputs.updated != 'true' || inputs.force == 'true'}}" | |
| # echo "commit_sha : ${{env.LAST_COMMIT_SHA}}" | |
| # echo "=====================================" | |
| # echo "BUILD_DATE=${{ steps.setup.outputs.date }} | |
| # echo "CHECKSUM=${{ steps.release_hash.outputs.hash }} | |
| # echo "VERSION=${{ steps.version.outputs.version }} | |
| # echo "SOURCE_COMMIT=${{ env.LAST_COMMIT_SHA }} | |
| # echo "GITHUB_SERVER_URL=${{ github.server_url }} | |
| # echo "GITHUB_REPOSITORY=${{ github.repository }} | |
| # echo "=====================================" | |
| check: | |
| name: Build Testing Docker Image | |
| needs: [setup] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - run: | | |
| echo "" | |
| echo "${{ toJSON(needs.setup.outputs) }}" | |
| echo "${{ toJSON(needs.setup.env) }}" | |
| build: | |
| # if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name | |
| name: Build Testing Docker Image | |
| needs: [setup] | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| outputs: | |
| image: ${{ steps.image_name.outputs.name }} | |
| steps: | |
| - run: | | |
| echo "" | |
| echo "${{ toJSON(needs.setup.outputs) }}" | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Define target image name | |
| id: image_name | |
| run: | | |
| echo "name=${{vars.DOCKER_IMAGE}}:test-${{needs.setup.outputs.tag}}" >> $GITHUB_OUTPUT | |
| - id: doit | |
| uses: ./.github/actions/docker_build | |
| with: | |
| image: ${{steps.image_name.outputs.name}} | |
| checksum: ${{needs.setup.outputs.hash}} | |
| target: "python_dev_deps" | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| test: | |
| # if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name | |
| name: Run Tests | |
| needs: [build] | |
| runs-on: ubuntu-latest | |
| services: | |
| redis: | |
| image: redis | |
| db: | |
| image: postgres:14 | |
| env: | |
| POSTGRES_DATABASE: dedupe | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_USERNAME: postgres | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| env: | |
| DOCKER_DEFAULT_PLATFORM: linux/amd64 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - run: | | |
| echo "" | |
| echo "${{ toJSON(needs.build.outputs) }}" | |
| - name: Run tests | |
| run: | | |
| docker run --rm \ | |
| -e DATABASE_URL=postgres://postgres:postgres@db:5432/dedupe \ | |
| -e SECRET_KEY=secret_key \ | |
| -e CACHE_URL=redis://redis:6379/0 \ | |
| -e CELERY_BROKER_URL=redis://redis:6379/0 \ | |
| --network host \ | |
| -v $PWD:/code/app \ | |
| -w /code/app \ | |
| -t "${{needs.build.outputs.image}}" \ | |
| pytest tests -v | |
| - name: Upload coverage to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| env_vars: OS,PYTHON | |
| fail_ci_if_error: true | |
| files: coverage.xml | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| verbose: false | |
| name: codecov-${{env.GITHUB_REF_NAME}} | |
| release: | |
| # if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name | |
| name: Release deployment image | |
| needs: [setup, test] | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| outputs: | |
| image: ${{steps.image_name.outputs.name}} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Define target image name | |
| id: image_name | |
| run: | | |
| echo "name=${{vars.DOCKER_IMAGE}}:${{needs.setup.outputs.tag}}" >> $GITHUB_OUTPUT | |
| - id: doit | |
| uses: ./.github/actions/docker_build | |
| with: | |
| image: ${{steps.image_name.outputs.name}} | |
| checksum: ${{needs.setup.outputs.hash}} | |
| target: "dist" | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - run: | | |
| echo "::notice:: ${{steps.image_name.outputs.name}} pushed to dockerhub" | |
| trivy: | |
| name: Check Image with Trivy | |
| runs-on: ubuntu-latest | |
| needs: [release] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: DockerHub login | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Run Trivy vulnerability scanner | |
| uses: aquasecurity/trivy-action@master | |
| with: | |
| image-ref: ${{ needs.release.outputs.image }} | |
| format: 'sarif' | |
| output: 'trivy-results.sarif' | |
| # format: 'table' | |
| # exit-code: '0' | |
| # ignore-unfixed: true | |
| # vuln-type: 'os,library' | |
| # severity: 'CRITICAL,HIGH' | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: 'trivy-results.sarif' |