diff --git a/.ansible-lint b/.ansible-lint index 0e80b05..4ffc0ef 100644 --- a/.ansible-lint +++ b/.ansible-lint @@ -1,10 +1,9 @@ --- -# See https://ansible-lint.readthedocs.io/en/latest/configuring.html -# for a list of the configuration elements that can exist in this -# file. +# See https://ansible-lint.readthedocs.io/configuring/ for a list of +# the configuration elements that can exist in this file. enable_list: # Useful checks that one must opt-into. See here for more details: - # https://ansible-lint.readthedocs.io/en/latest/rules.html + # https://ansible-lint.readthedocs.io/rules/ - fcqn-builtins - no-log-password - no-same-owner diff --git a/.bandit.yml b/.bandit.yml index 2b618f6..663c521 100644 --- a/.bandit.yml +++ b/.bandit.yml @@ -3,7 +3,7 @@ # https://bandit.readthedocs.io/en/latest/config.html # Tests are first included by `tests`, and then excluded by `skips`. -# If `tests` is empty, all tests are are considered included. +# If `tests` is empty, all tests are considered included. tests: # - B101 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 371258c..3af99ba 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,3 +8,17 @@ # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. /.github/ @dav3r @felddy @jsf9k @mcdonnnj + +# These folks own all linting configuration files. +/.ansible-lint @dav3r @felddy @jsf9k @mcdonnnj +/.bandit.yml @dav3r @felddy @jsf9k @mcdonnnj +/.flake8 @dav3r @felddy @jsf9k @mcdonnnj +/.isort.cfg @dav3r @felddy @jsf9k @mcdonnnj +/.mdl_config.yaml @dav3r @felddy @jsf9k @mcdonnnj +/.pre-commit-config.yaml @dav3r @felddy @jsf9k @mcdonnnj +/.prettierignore @dav3r @felddy @jsf9k @mcdonnnj +/.yamllint @dav3r @felddy @jsf9k @mcdonnnj +/requirements.txt @dav3r @felddy @jsf9k @mcdonnnj +/requirements-dev.txt @dav3r @felddy @jsf9k @mcdonnnj +/requirements-test.txt @dav3r @felddy @jsf9k @mcdonnnj +/setup-env @dav3r @felddy @jsf9k @mcdonnnj diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b38be54..1cd0885 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -5,36 +5,55 @@ # these updates when the pull request(s) in the appropriate skeleton are merged # and Lineage processes these changes. -version: 2 updates: - - package-ecosystem: "docker" - directory: "/" + - directory: / + package-ecosystem: docker schedule: - interval: "weekly" + interval: weekly - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" + - directory: / ignore: + # Managed by cisagov/skeleton-generic - dependency-name: actions/cache - dependency-name: actions/checkout + - dependency-name: actions/setup-go - dependency-name: actions/setup-python +<<<<<<< HEAD +<<<<<<< HEAD # Managed by cisagov/cyhy-commander-docker +======= +======= + - dependency-name: cisagov/setup-env-github-action +>>>>>>> 0d48ebd47a28a887868ea3093e675e95f3843561 + - dependency-name: crazy-max/ghaction-dump-context + - dependency-name: crazy-max/ghaction-github-labeler + - dependency-name: crazy-max/ghaction-github-status + - dependency-name: GitHubSecurityLab/actions-permissions + - dependency-name: hashicorp/setup-packer + - dependency-name: hashicorp/setup-terraform + - dependency-name: mxschmitt/action-tmate + - dependency-name: step-security/harden-runner + # Managed by cisagov/skeleton-docker +>>>>>>> 1ea8a3fa98e790d66f8d5e10375f73c5be4d5fd5 # - dependency-name: actions/download-artifact - # - dependency-name: actions/github-script # - dependency-name: actions/upload-artifact # - dependency-name: docker/build-push-action # - dependency-name: docker/login-action + # - dependency-name: docker/metadata-action # - dependency-name: docker/setup-buildx-action # - dependency-name: docker/setup-qemu-action + # - dependency-name: github/codeql-action + package-ecosystem: github-actions + schedule: + interval: weekly - - package-ecosystem: "pip" - directory: "/" + - directory: / + package-ecosystem: pip schedule: - interval: "weekly" + interval: weekly - - package-ecosystem: "terraform" - directory: "/" + - directory: / + package-ecosystem: terraform schedule: - interval: "weekly" + interval: weekly +version: 2 diff --git a/.github/labels.yml b/.github/labels.yml new file mode 100644 index 0000000..014cc00 --- /dev/null +++ b/.github/labels.yml @@ -0,0 +1,73 @@ +--- +# Rather than breaking up descriptions into multiline strings we disable that +# specific rule in yamllint for this file. +# yamllint disable rule:line-length +- color: "eb6420" + description: This issue or pull request is awaiting the outcome of another issue or pull request + name: blocked +- color: "000000" + description: This issue or pull request involves changes to existing functionality + name: breaking change +- color: "d73a4a" + description: This issue or pull request addresses broken functionality + name: bug +- color: "07648d" + description: This issue will be advertised on code.gov's Open Tasks page (https://code.gov/open-tasks) + name: code.gov +- color: "0366d6" + description: Pull requests that update a dependency file + name: dependencies +- color: "2497ed" + description: Pull requests that update Docker code + name: docker +- color: "5319e7" + description: This issue or pull request improves or adds to documentation + name: documentation +- color: "cfd3d7" + description: This issue or pull request already exists or is covered in another issue or pull request + name: duplicate +- color: "b005bc" + description: A high-level objective issue encompassing multiple issues instead of a specific unit of work + name: epic +- color: "000000" + description: Pull requests that update GitHub Actions code + name: github-actions +- color: "0e8a16" + description: This issue or pull request is well-defined and good for newcomers + name: good first issue +- color: "ff7518" + description: Pull request that should count toward Hacktoberfest participation + name: hacktoberfest-accepted +- color: "a2eeef" + description: This issue or pull request will add or improve functionality, maintainability, or ease of use + name: improvement +- color: "fef2c0" + description: This issue or pull request is not applicable, incorrect, or obsolete + name: invalid +- color: "ce099a" + description: This pull request is ready to merge during the next Lineage Kraken release + name: kraken 🐙 +- color: "a4fc5d" + description: This issue or pull request requires further information + name: need info +- color: "fcdb45" + description: This pull request is awaiting an action or decision to move forward + name: on hold +- color: "ef476c" + description: This issue is a request for information or needs discussion + name: question +- color: "d73a4a" + description: This issue or pull request addresses a security issue + name: security +- color: "00008b" + description: This issue or pull request adds or otherwise modifies test code + name: test +- color: "1d76db" + description: This issue or pull request pulls in upstream updates + name: upstream update +- color: "d4c5f9" + description: This issue or pull request increments the version number + name: version bump +- color: "ffffff" + description: This issue will not be incorporated + name: wontfix diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cc56c8e..8425738 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,74 +2,128 @@ name: build on: + merge_group: + types: + - checks_requested + pull_request: push: branches: - "**" tags: - "v*.*.*" - pull_request: - schedule: - - cron: "0 10 * * *" # everyday at 10am repository_dispatch: # Respond to rebuild requests. See: https://github.com/cisagov/action-apb/ - types: [apb] + types: + - apb + schedule: + - cron: "0 10 * * *" # everyday at 10am workflow_dispatch: inputs: - remote-shell: - description: "Debug with remote shell" - required: true - default: "false" image-tag: + default: "dispatch" description: "Tag to apply to pushed images" required: true - default: "dispatch" + remote-shell: + default: "false" + description: "Debug with remote shell" + required: true + +# Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace, +# nounset, errexit, and pipefail. The `-x` will print all commands as they are +# run. Please see the GitHub Actions documentation for more information: +# https://docs.github.com/en/actions/using-jobs/setting-default-values-for-jobs +defaults: + run: + shell: bash -Eueo pipefail -x {0} env: BUILDX_CACHE_DIR: ~/.cache/buildx - CURL_CACHE_DIR: ~/.cache/curl IMAGE_NAME: cisagov/example PIP_CACHE_DIR: ~/.cache/pip - PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,\ - linux/arm64,linux/ppc64le,linux/s390x" PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit RUN_TMATE: ${{ secrets.RUN_TMATE }} + TERRAFORM_DOCS_REPO_BRANCH_NAME: improvement/support_atx_closed_markdown_headers + TERRAFORM_DOCS_REPO_DEPTH: 1 + TERRAFORM_DOCS_REPO_URL: https://github.com/mcdonnnj/terraform-docs.git jobs: + diagnostics: + name: Run diagnostics + # This job does not need any permissions + permissions: {} + runs-on: ubuntu-latest + steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - id: github-status + name: Check GitHub status + uses: crazy-max/ghaction-github-status@v4 + - id: dump-context + name: Dump context + uses: crazy-max/ghaction-dump-context@v2 lint: # Checks out the source and runs pre-commit hooks. Detects coding errors # and style deviations. - name: "Lint sources" + name: Lint sources + needs: + - diagnostics + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - id: setup-env uses: cisagov/setup-env-github-action@develop - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: - python-version: "3.10" + python-version: ${{ steps.setup-env.outputs.python-version }} # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - - uses: actions/setup-go@v2 + - id: setup-go + uses: actions/setup-go@v5 with: - go-version: "1.16" - - name: Store installed Go version - id: go-version - run: | - echo "::set-output name=version::"\ - "$(go version | sed 's/^go version go\([0-9.]\+\) .*/\1/')" - - name: Lookup Go cache directory - id: go-cache + # There is no expectation for actual Go code so we disable caching as + # it relies on the existence of a go.sum file. + cache: false + go-version: ${{ steps.setup-env.outputs.go-version }} + - id: go-cache + name: Lookup Go cache directory run: | - echo "::set-output name=dir::$(go env GOCACHE)" - - uses: actions/cache@v3 + echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT + - uses: actions/cache@v4 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ - go${{ steps.go-version.outputs.version }}-\ + go${{ steps.setup-go.outputs.go-version }}-\ packer${{ steps.setup-env.outputs.packer-version }}-\ tf${{ steps.setup-env.outputs.terraform-version }}-" with: + key: "${{ env.BASE_CACHE_KEY }}\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" # Note that the .terraform directory IS NOT included in the # cache because if we were caching, then we would need to use # the `-upgrade=true` option. This option blindly pulls down the @@ -79,45 +133,55 @@ jobs: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - ${{ env.CURL_CACHE_DIR }} ${{ steps.go-cache.outputs.dir }} - key: "${{ env.BASE_CACHE_KEY }}\ - ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - - name: Setup curl cache - run: mkdir -p ${{ env.CURL_CACHE_DIR }} - - name: Install Packer - env: - PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} - run: | - PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" - curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --location \ - "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" - sudo unzip -d /opt/packer \ - ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" - sudo mv /usr/local/bin/packer /usr/local/bin/packer-default - sudo ln -s /opt/packer/packer /usr/local/bin/packer - - uses: hashicorp/setup-terraform@v1 + - uses: hashicorp/setup-packer@v3 + with: + version: ${{ steps.setup-env.outputs.packer-version }} + - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - - name: Install shfmt + - name: Install go-critic + env: + PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic + PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install goimports env: - PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt - PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} + PACKAGE_URL: golang.org/x/tools/cmd/goimports + PACKAGE_VERSION: ${{ steps.setup-env.outputs.goimports-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install Terraform-docs + - name: Install gosec env: - PACKAGE_URL: github.com/terraform-docs/terraform-docs - PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }} + PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec + PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install staticcheck + env: + PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck + PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + # TODO: https://github.com/cisagov/skeleton-generic/issues/165 + # We are temporarily using @mcdonnnj's forked branch of terraform-docs + # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 + # is approved. This temporary fix will allow for ATX header support when + # terraform-docs is run during linting. + - name: Clone ATX headers branch from terraform-docs fork + run: | + git clone \ + --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ + --depth $TERRAFORM_DOCS_REPO_DEPTH \ + --single-branch \ + $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs + - name: Build and install terraform-docs binary + run: | + go build \ + -C /tmp/terraform-docs \ + -o $(go env GOPATH)/bin/terraform-docs - name: Install dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip setuptools wheel pip install --upgrade --requirement requirements-test.txt - name: Set up pre-commit hook environments run: pre-commit install-hooks @@ -127,170 +191,105 @@ jobs: uses: mxschmitt/action-tmate@v3 if: env.RUN_TMATE prepare: - # Calculates and publishes outputs that are used by other jobs. - # - # Outputs: - # created: - # The current date-time in RFC3339 format. - # repometa: - # The json metadata describing this repository. - # source_version: - # The source version as reported by the `bump_version.sh show` command. - # tags: - # A comma separated list of Docker tags to be applied to the images on - # Docker Hub. The tags will vary depending on: - # - The event that triggered the build. - # - The branch the build is based upon. - # - The git tag the build is based upon. - # - # When a build is based on a git tag of the form `v*.*.*` the image will - # be tagged on Docker Hub with multiple levels of version specificity. - # For example, a git tag of `v1.2.3+a` will generate Docker tags of - # `:1.2.3_a`, `:1.2.3`, `:1.2`, `:1`, and `:latest`. - # - # Builds targeting the default branch will be tagged with `:edge`. - # - # Builds from other branches will be tagged with the branch name. Solidi - # (`/` characters - commonly known as slashes) in branch names are - # replaced with hyphen-minuses (`-` characters) in the Docker tag. For - # more information about the solidus see these links: - # * https://www.compart.com/en/unicode/U+002F - # * https://en.wikipedia.org/wiki/Slash_(punctuation)#Encoding - # - # Builds triggered by a push event are tagged with a short hash in the - # form: sha-12345678 - # - # Builds triggered by a pull request are tagged with the pull request - # number in the form pr-123. - # - # Builds triggered using the GitHub GUI (workflow_dispatch) are tagged - # with the value specified by the user. - # - # Scheduled builds are tagged with `:nightly`. - name: "Prepare build variables" - runs-on: ubuntu-latest + # Generate Docker image metadata using the docker/metadata-action GitHub Action. + name: Prepare build variables + needs: + - diagnostics outputs: - created: ${{ steps.prep.outputs.created }} - repometa: ${{ steps.repo.outputs.result }} - source_version: ${{ steps.prep.outputs.source_version }} - tags: ${{ steps.prep.outputs.tags }} + labels: ${{ steps.generate-metadata.outputs.labels }} + tags: ${{ steps.generate-metadata.outputs.tags }} + permissions: + # actions/checkout needs this to fetch code + contents: read + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Gather repository metadata - id: repo - uses: actions/github-script@v5 + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 with: - script: | - const repo = await github.rest.repos.get(context.repo) - return repo.data - - name: Calculate output values - id: prep - run: | - VERSION=noop - SEMVER="^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?$" - if [ "${{ github.event_name }}" = "schedule" ]; then - VERSION=nightly - elif [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - VERSION=${{ github.event.inputs.image-tag }} - elif [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/} - elif [[ $GITHUB_REF == refs/heads/* ]]; then - VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -r 's#/+#-#g') - if [ "${{ github.event.repository.default_branch }}" = "$VERSION" ]; - then - VERSION=edge - fi - elif [[ $GITHUB_REF == refs/pull/* ]]; then - VERSION=pr-${{ github.event.number }} - fi - if [[ $VERSION =~ $SEMVER ]]; then - VERSION_NO_V=${VERSION#v} - MAJOR="${BASH_REMATCH[1]}" - MINOR="${BASH_REMATCH[2]}" - PATCH="${BASH_REMATCH[3]}" - TAGS="${IMAGE_NAME}:${VERSION_NO_V//+/_},${IMAGE_NAME}:${MAJOR}.${MINOR}.${PATCH},${IMAGE_NAME}:${MAJOR}.${MINOR},${IMAGE_NAME}:${MAJOR},${IMAGE_NAME}:latest" - else - TAGS="${IMAGE_NAME}:${VERSION}" - fi - if [ "${{ github.event_name }}" = "push" ]; then - TAGS="${TAGS},${IMAGE_NAME}:sha-${GITHUB_SHA::8}" - fi - for i in ${TAGS//,/ } - do - TAGS="${TAGS},ghcr.io/${i}" - done - echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') - echo ::set-output name=source_version::$(./bump_version.sh show) - echo ::set-output name=tags::${TAGS} - echo tags=${TAGS} + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - uses: actions/checkout@v4 + - id: generate-metadata + name: Generate Docker image metadata + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.IMAGE_NAME }} + ghcr.io/${{ env.IMAGE_NAME }} + tags: | + type=edge + type=raw,event=workflow_dispatch,value=${{ github.event.inputs.image-tag }} + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + type=schedule + type=semver,pattern={{major}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{version}} + type=sha - name: Setup tmate debug session uses: mxschmitt/action-tmate@v3 if: github.event.inputs.remote-shell == 'true' || env.RUN_TMATE build: # Builds a single test image for the native platform. This image is saved # as an artifact and loaded by the test job. - name: "Build test image" + name: Build test image + needs: + - diagnostics + - prepare + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest - needs: [prepare] steps: - - name: Checkout - uses: actions/checkout@v3 + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Cache Docker layers - uses: actions/cache@v3 + uses: actions/cache@v4 env: BASE_CACHE_KEY: buildx-${{ runner.os }}- with: - path: ${{ env.BUILDX_CACHE_DIR }} key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} + path: ${{ env.BUILDX_CACHE_DIR }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Create dist directory run: mkdir -p dist - name: Build image id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 with: - build-args: | - VERSION=${{ needs.prepare.outputs.source_version }} cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} - context: . - file: ./Dockerfile + labels: ${{ needs.prepare.outputs.labels }} outputs: type=docker,dest=dist/image.tar + # Uncomment the following option if you are building an image for use + # on Google Cloud Run or AWS Lambda. The current default image output + # is unable to run on either. Please see the following issue for more + # information: https://github.com/docker/buildx/issues/1533 + # provenance: false tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md - labels: "\ - org.opencontainers.image.created=${{ - needs.prepare.outputs.created }} - - org.opencontainers.image.description=${{ - fromJson(needs.prepare.outputs.repometa).description }} - - org.opencontainers.image.licenses=${{ - fromJson(needs.prepare.outputs.repometa).license.spdx_id }} - - org.opencontainers.image.revision=${{ github.sha }} - - org.opencontainers.image.source=${{ - fromJson(needs.prepare.outputs.repometa).clone_url }} - - org.opencontainers.image.title=${{ - fromJson(needs.prepare.outputs.repometa).name }} - - org.opencontainers.image.url=${{ - fromJson(needs.prepare.outputs.repometa).html_url }} - - org.opencontainers.image.version=${{ - needs.prepare.outputs.source_version }}" - name: Compress image run: gzip dist/image.tar - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist @@ -299,33 +298,49 @@ jobs: if: env.RUN_TMATE test: # Executes tests on the single-platform image created in the "build" job. - name: "Test image" + name: Test image + needs: + - diagnostics + - build + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest - needs: [build] steps: - - uses: actions/checkout@v3 + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - id: setup-env + uses: cisagov/setup-env-github-action@develop + - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: ${{ steps.setup-env.outputs.python-version }} - name: Cache testing environments - uses: actions/cache@v3 + uses: actions/cache@v4 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-" with: - path: ${{ env.PIP_CACHE_DIR }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}" + path: ${{ env.PIP_CACHE_DIR }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Install dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip setuptools wheel pip install --upgrade --requirement requirements-test.txt - name: Download docker image artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: dist path: dist @@ -339,88 +354,92 @@ jobs: uses: mxschmitt/action-tmate@v3 if: env.RUN_TMATE build-push-all: - # Builds the final set of images for each of the platforms listed in - # PLATFORMS environment variable. These images are tagged with the Docker - # tags calculated in the "prepare" job and pushed to Docker Hub and the - # GitHub Container Registry. The contents of README.md are pushed as the - # image's description to Docker Hub. This job is skipped when the - # triggering event is a pull request. - name: "Build and push all platforms" - runs-on: ubuntu-latest - needs: [lint, prepare, test] + # Builds the final set of images for each of the platforms specified in the + # "platforms" input for the docker/build-push-action Action. These images + # are tagged with the Docker tags calculated in the "prepare" job and + # pushed to Docker Hub and the GitHub Container Registry. The contents of + # README.md are pushed as the image's description to Docker Hub. This job + # is skipped when the triggering event is a pull request. if: github.event_name != 'pull_request' + name: Build and push all platforms + needs: + - diagnostics + - lint + - prepare + - test + permissions: + # actions/checkout needs this to fetch code + contents: read + # When Dependabot creates a PR it requires this permission in + # order to push Docker images to ghcr.io. + packages: write + runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.DOCKER_USERNAME }} - name: Login to GitHub Container Registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: + password: ${{ secrets.GITHUB_TOKEN }} registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Checkout - uses: actions/checkout@v3 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 - name: Cache Docker layers - uses: actions/cache@v3 + uses: actions/cache@v4 env: BASE_CACHE_KEY: buildx-${{ runner.os }}- with: - path: ${{ env.BUILDX_CACHE_DIR }} key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} + path: ${{ env.BUILDX_CACHE_DIR }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - - name: Create cross-platform support Dockerfile-x - run: ./buildx-dockerfile.sh - name: Build and push platform images to registries id: docker_build - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v6 with: - build-args: | - VERSION=${{ needs.prepare.outputs.source_version }} cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} - context: . - file: ./Dockerfile-x - platforms: ${{ env.PLATFORMS }} + labels: ${{ needs.prepare.outputs.labels }} + platforms: | + linux/amd64 + linux/arm/v6 + linux/arm/v7 + linux/arm64 + linux/ppc64le + linux/s390x + # Uncomment the following option if you are building an image for use + # on Google Cloud Run or AWS Lambda. The current default image output + # is unable to run on either. Please see the following issue for more + # information: https://github.com/docker/buildx/issues/1533 + # provenance: false push: true tags: ${{ needs.prepare.outputs.tags }} # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md - labels: "\ - org.opencontainers.image.created=${{ - needs.prepare.outputs.created }} - - org.opencontainers.image.description=${{ - fromJson(needs.prepare.outputs.repometa).description }} - - org.opencontainers.image.licenses=${{ - fromJson(needs.prepare.outputs.repometa).license.spdx_id }} - - org.opencontainers.image.revision=${{ github.sha }} - - org.opencontainers.image.source=${{ - fromJson(needs.prepare.outputs.repometa).clone_url }} - - org.opencontainers.image.title=${{ - fromJson(needs.prepare.outputs.repometa).name }} - - org.opencontainers.image.url=${{ - fromJson(needs.prepare.outputs.repometa).html_url }} - - org.opencontainers.image.version=${{ - needs.prepare.outputs.source_version }}" - - name: Publish README.md to Docker Hub - env: - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - run: ./push_readme.sh + - if: ${{ github.ref_name == 'develop' && github.event_name == 'push' }} + name: Update the Docker Hub description + uses: peter-evans/dockerhub-description@v4 + with: + password: ${{ secrets.DOCKER_PASSWORD }} + readme-filepath: README.md + repository: ${{ env.IMAGE_NAME }} + short-description: ${{ github.event.repository.description }} + username: ${{ secrets.DOCKER_USERNAME }} - name: Setup tmate debug session uses: mxschmitt/action-tmate@v3 if: env.RUN_TMATE diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 33d1999..642adda 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -1,69 +1,109 @@ --- - # For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. -name: "CodeQL" +name: CodeQL on: + merge_group: + types: + - checks_requested + pull_request: + # The branches here must be a subset of the ones in the push key + branches: + - develop push: # Dependabot triggered push events have read-only access, but uploading code # scanning requires write access. - branches-ignore: [dependabot/**] - pull_request: - # The branches below must be a subset of the branches above - branches: [develop] + branches-ignore: + - dependabot/** schedule: - cron: '0 21 * * 6' jobs: + diagnostics: + name: Run diagnostics + # This job does not need any permissions + permissions: {} + runs-on: ubuntu-latest + steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - id: github-status + name: Check GitHub status + uses: crazy-max/ghaction-github-status@v4 + - id: dump-context + name: Dump context + uses: crazy-max/ghaction-dump-context@v2 analyze: name: Analyze + needs: + - diagnostics runs-on: ubuntu-latest - + permissions: + # actions/checkout needs this to fetch code + contents: read + # required for all workflows + security-events: write strategy: fail-fast: false matrix: # Override automatic language detection by changing the below list - # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', - # 'python'] - language: ['python'] + # Supported options are go, javascript, csharp, python, cpp, and java + language: + - python # Learn more... # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a - # config file. By default, queries listed here will override any - # specified in a config file. Prefix the list here with "+" to use - # these queries and those in the config file. queries: - # ./path/to/local/query, your-org/your-repo/queries@main - # Autobuild attempts to build any compiled languages (C/C++, C#, or + # Autobuild attempts to build any compiled languages (C/C++, C#, or # Java). If this step fails, then you should remove it and run the build - # manually (see below) + # manually (see below). - name: Autobuild - uses: github/codeql-action/autobuild@v1 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following - # three lines and modify them (or add more) to build your code if your - # project uses a compiled language + # three lines and modify them (or add more) to build your code if your + # project uses a compiled language # - run: | - # make bootstrap - # make release + # make bootstrap + # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml new file mode 100644 index 0000000..0005147 --- /dev/null +++ b/.github/workflows/sync-labels.yml @@ -0,0 +1,66 @@ +--- +name: sync-labels + +on: + push: + paths: + - .github/labels.yml + - .github/workflows/sync-labels.yml + workflow_dispatch: + +permissions: + contents: read + +jobs: + diagnostics: + name: Run diagnostics + # This job does not need any permissions + permissions: {} + runs-on: ubuntu-latest + steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - id: github-status + name: Check GitHub status + uses: crazy-max/ghaction-github-status@v4 + - id: dump-context + name: Dump context + uses: crazy-max/ghaction-dump-context@v2 + labeler: + needs: + - diagnostics + permissions: + # actions/checkout needs this to fetch code + contents: read + # crazy-max/ghaction-github-labeler needs this to manage repository labels + issues: write + runs-on: ubuntu-latest + steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - uses: actions/checkout@v4 + - name: Sync repository labels + if: success() + uses: crazy-max/ghaction-github-labeler@v5 + with: + # This is a hideous ternary equivalent so we only do a dry run unless + # this workflow is triggered by the develop branch. + dry-run: ${{ github.ref_name == 'develop' && 'false' || 'true' }} diff --git a/.lgtm.yml b/.lgtm.yml deleted file mode 100644 index 8950263..0000000 --- a/.lgtm.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -extraction: - python: - python_setup: - version: 3 - requirements_files: - - requirements-test.txt - setup_py: false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6cc81b..4d0c4a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,23 +4,30 @@ default_language_version: python: python3 repos: + # Check the pre-commit configuration + - repo: meta + hooks: + - id: check-useless-excludes + - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.1.0 + rev: v5.0.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs - id: check-json - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-symlinks - id: check-toml + - id: check-vcs-permalinks - id: check-xml - id: debug-statements + - id: destroyed-symlinks - id: detect-aws-credentials args: - --allow-missing-credentials - id: detect-private-key - exclude: src/secrets/privkey.pem - id: end-of-file-fixer - exclude: files/(issue|motd) - id: mixed-line-ending args: - --fix=lf @@ -32,17 +39,17 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.31.1 + rev: v0.42.0 hooks: - id: markdownlint args: - --config=.mdl_config.yaml - - repo: https://github.com/pre-commit/mirrors-prettier - rev: v2.6.1 + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.3.3 hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.26.3 + rev: v1.35.1 hooks: - id: yamllint args: @@ -50,99 +57,182 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.14.2 + rev: 0.29.4 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v2.17.0 + rev: v4.0.1 hooks: - id: validate_manifest + # Go hooks + - repo: https://github.com/TekWizely/pre-commit-golang + rev: v1.0.0-rc.1 + hooks: + # Go Build + - id: go-build-repo-mod + # Style Checkers + - id: go-critic + # goimports + - id: go-imports-repo + args: + # Write changes to files + - -w + # Go Mod Tidy + - id: go-mod-tidy-repo + # GoSec + - id: go-sec-repo-mod + # StaticCheck + - id: go-staticcheck-repo-mod + # Go Test + - id: go-test-repo-mod + # Go Vet + - id: go-vet-repo-mod + # Nix hooks + - repo: https://github.com/nix-community/nixpkgs-fmt + rev: v1.3.0 + hooks: + - id: nixpkgs-fmt + # Shell script hooks - - repo: https://github.com/cisagov/pre-commit-shfmt - rev: v0.0.2 + - repo: https://github.com/scop/pre-commit-shfmt + rev: v3.10.0-1 hooks: - id: shfmt args: + # List files that will be formatted + - --list + # Write result to file instead of stdout + - --write # Indent by two spaces - - -i - - '2' + - --indent + - "2" # Binary operators may start a line - - -bn + - --binary-next-line # Switch cases are indented - - -ci + - --case-indent # Redirect operators are followed by a space - - -sr - - repo: https://github.com/detailyang/pre-commit-shell - rev: 1.0.5 + - --space-redirects + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.10.0.1 hooks: - - id: shell-lint + - id: shellcheck # Python hooks + # Run bandit on the "tests" tree with a configuration - repo: https://github.com/PyCQA/bandit - rev: 1.7.4 + rev: 1.7.10 hooks: - id: bandit name: bandit (tests tree) files: tests args: - --config=.bandit.yml - # Run bandit everything but tests directory + # Run bandit on everything except the "tests" tree - repo: https://github.com/PyCQA/bandit - rev: 1.7.0 + rev: 1.7.10 hooks: - id: bandit name: bandit (everything else) exclude: tests - - repo: https://github.com/psf/black - rev: 22.3.0 + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.10.0 hooks: - id: black - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.9.2 + - repo: https://github.com/PyCQA/flake8 + rev: 7.1.1 hooks: - id: flake8 additional_dependencies: - - flake8-docstrings + - flake8-docstrings==1.7.0 - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.942 + rev: v1.13.0 hooks: - id: mypy + - repo: https://github.com/pypa/pip-audit + rev: v2.7.3 + hooks: + - id: pip-audit + args: + # Add any pip requirements files to scan + - --requirement + - requirements-dev.txt + - --requirement + - requirements-test.txt + - --requirement + - requirements.txt - repo: https://github.com/asottile/pyupgrade - rev: v2.31.1 + rev: v3.19.0 hooks: - id: pyupgrade # Ansible hooks - - repo: https://github.com/ansible-community/ansible-lint - rev: v5.4.0 + - repo: https://github.com/ansible/ansible-lint + rev: v24.10.0 hooks: - id: ansible-lint - # files: molecule/default/playbook.yml + additional_dependencies: + # On its own ansible-lint does not pull in ansible, only + # ansible-core. Therefore, if an Ansible module lives in + # ansible instead of ansible-core, the linter will complain + # that the module is unknown. In these cases it is + # necessary to add the ansible package itself as an + # additional dependency, with the same pinning as is done in + # requirements-test.txt of cisagov/skeleton-ansible-role. + # + # Version 10 is required because the pip-audit pre-commit + # hook identifies a vulnerability in ansible-core 2.16.13, + # but all versions of ansible 9 have a dependency on + # ~=2.16.X. + # + # It is also a good idea to go ahead and upgrade to version + # 10 since version 9 is going EOL at the end of November: + # https://endoflife.date/ansible + # - ansible>=10,<11 + # ansible-core 2.16.3 through 2.16.6 suffer from the bug + # discussed in ansible/ansible#82702, which breaks any + # symlinked files in vars, tasks, etc. for any Ansible role + # installed via ansible-galaxy. Hence we never want to + # install those versions. + # + # Note that the pip-audit pre-commit hook identifies a + # vulnerability in ansible-core 2.16.13. The pin of + # ansible-core to >=2.17 effectively also pins ansible to + # >=10. + # + # It is also a good idea to go ahead and upgrade to + # ansible-core 2.17 since security support for ansible-core + # 2.16 ends this month: + # https://docs.ansible.com/ansible/devel/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix + # + # Note that any changes made to this dependency must also be + # made in requirements.txt in cisagov/skeleton-packer and + # requirements-test.txt in cisagov/skeleton-ansible-role. + - ansible-core>=2.17 # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.64.0 + rev: v1.96.1 hooks: - id: terraform_fmt - id: terraform_validate # Docker hooks - repo: https://github.com/IamTheFij/docker-pre-commit - rev: v2.1.0 + rev: v3.0.1 hooks: - id: docker-compose-check # Packer hooks - repo: https://github.com/cisagov/pre-commit-packer - rev: v0.0.2 + rev: v0.3.0 hooks: - - id: packer_validate - id: packer_fmt + - id: packer_validate diff --git a/.prettierignore b/.prettierignore index 738d402..324b87f 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,4 +1,5 @@ # Already being linted by pretty-format-json +*.lock *.json # Already being linted by mdl *.md diff --git a/.yamllint b/.yamllint index 76a1cce..2a119a6 100644 --- a/.yamllint +++ b/.yamllint @@ -8,6 +8,16 @@ rules: # this behavior. comments-indentation: disable + # yamllint does not allow inline mappings that exceed the line length by + # default. There are many scenarios where the inline mapping may be a key, + # hash, or other long value that would exceed the line length but cannot + # reasonably be broken across lines. + line-length: + # This rule implies the allow-non-breakable-words rule + allow-non-breakable-inline-mappings: true + # Allows a 10% overage from the default limit of 80 + max: 88 + # yamllint doesn't like when we use yes and no for true and false, # but that's pretty standard in Ansible. truthy: disable diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c935864..7600b3e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,9 +46,13 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -If you already have `pyenv` and `pyenv-virtualenv` configured you can -take advantage of the `setup-env` tool in this repo to automate the -entire environment configuration process. +We recommend using the `setup-env` script located in this repository, +as it automates the entire environment configuration process. The +dependencies required to run this script are +[GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), +[`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). +If these tools are already configured on your system, you can simply run the +following command: ```console ./setup-env @@ -57,13 +61,18 @@ entire environment configuration process. Otherwise, follow the steps below to manually configure your environment. -#### Installing and using `pyenv` and `pyenv-virtualenv` #### +#### Installing and using GNU `getopt`, `pyenv`, and `pyenv-virtualenv` #### -On the Mac, we recommend installing [brew](https://brew.sh/). Then -installation is as simple as `brew install pyenv pyenv-virtualenv` and +On macOS, we recommend installing [brew](https://brew.sh/). Then +installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and adding this to your profile: ```bash +# GNU getopt must be explicitly added to the path since it is +# keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean) +export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" + +# Setup pyenv export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init --path)" @@ -71,13 +80,15 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you +For Linux, Windows Subsystem for Linux (WSL), or macOS (if you don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). +GNU `getopt` is included in most Linux distributions as part of the +[`util-linux`](https://github.com/util-linux/util-linux) package. On WSL you should treat your platform as whatever Linux distribution you've chosen to install. diff --git a/Dockerfile b/Dockerfile index 248d6cf..38242be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,111 @@ -ARG VERSION=unspecified +# Official Docker images are in the form library/ while non-official +# images are in the form /. +FROM docker.io/library/python:3.13.2-alpine3.21 AS compile-stage -FROM python:3.10.1-alpine +### +# Unprivileged user variables +### +ARG CISA_USER="cisa" +ENV CISA_HOME="/home/${CISA_USER}" +ENV VIRTUAL_ENV="${CISA_HOME}/.venv" -ARG VERSION +# Versions of the Python packages installed directly +ENV PYTHON_PIP_VERSION=25.0.1 +ENV PYTHON_PIPENV_VERSION=2024.4.1 +ENV PYTHON_SETUPTOOLS_VERSION=75.8.0 +ENV PYTHON_WHEEL_VERSION=0.45.1 +### +# Install the specified versions of pip, setuptools, and wheel into the system +# Python environment; install the specified version of pipenv into the system Python +# environment; set up a Python virtual environment (venv); and install the specified +# versions of pip, setuptools, and wheel into the venv. +# +# Note that we use the --no-cache-dir flag to avoid writing to a local +# cache. This results in a smaller final image, at the cost of +# slightly longer install times. +### +RUN python3 -m pip install --no-cache-dir --upgrade \ + pip==${PYTHON_PIP_VERSION} \ + setuptools==${PYTHON_SETUPTOOLS_VERSION} \ + wheel==${PYTHON_WHEEL_VERSION} \ + && python3 -m pip install --no-cache-dir --upgrade \ + pipenv==${PYTHON_PIPENV_VERSION} \ + # Manually create the virtual environment + && python3 -m venv ${VIRTUAL_ENV} \ + # Ensure the core Python packages are installed in the virtual environment + && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ + pip==${PYTHON_PIP_VERSION} \ + setuptools==${PYTHON_SETUPTOOLS_VERSION} \ + wheel==${PYTHON_WHEEL_VERSION} + +### +# Check the Pipfile configuration and then install the Python dependencies into +# the virtual environment. +# +# Note that pipenv will install into a virtual environment if the VIRTUAL_ENV +# environment variable is set. +### +WORKDIR /tmp +COPY src/Pipfile src/Pipfile.lock ./ +RUN pipenv check --verbose \ + && pipenv install --clear --deploy --extra-pip-args "--no-cache-dir" --verbose + +# Official Docker images are in the form library/ while non-official +# images are in the form /. +FROM docker.io/library/python:3.13.2-alpine3.21 AS build-stage + +### # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md +# # Note: Additional labels are added by the build workflow. -LABEL org.opencontainers.image.authors="mark.feldhousen@cisa.dhs.gov" +### +# github@cisa.dhs.gov is a very generic email distribution, and it is +# unlikely that anyone on that distribution is familiar with the +# particulars of your repository. It is therefore *strongly* +# suggested that you use an email address here that is specific to the +# person or group that maintains this repository; for example: +# LABEL org.opencontainers.image.authors="vm-fusion-dev-group@trio.dhs.gov" +LABEL org.opencontainers.image.authors="github@cisa.dhs.gov" LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" +### +# Unprivileged user setup variables +### ARG CISA_UID=421 -ENV CISA_HOME="/home/cisa" -ENV ECHO_MESSAGE="Hello World from Dockerfile" +ARG CISA_GID=${CISA_UID} +ARG CISA_USER="cisa" +ENV CISA_GROUP=${CISA_USER} +ENV CISA_HOME="/home/${CISA_USER}" +ENV VIRTUAL_ENV="${CISA_HOME}/.venv" -RUN addgroup --system --gid ${CISA_UID} cisa \ - && adduser --system --uid ${CISA_UID} --ingroup cisa cisa +### +# Create unprivileged user +### +RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ + && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} -RUN apk --update --no-cache add \ -ca-certificates \ -openssl \ -py-pip +### +# Copy in the Python virtual environment created in compile-stage, symlink the +# Python binary in the venv to the system-wide Python, and add the venv to the PATH. +# +# Note that we symlink the Python binary in the venv to the system-wide Python so that +# any calls to `python3` will use our virtual environment. We are using short flags +# because the ln binary in Alpine Linux does not support long flags. The -f instructs +# ln to remove the existing file and the -s instructs ln to create a symbolic link. +### +COPY --from=compile-stage --chown=${CISA_USER}:${CISA_GROUP} ${VIRTUAL_ENV} ${VIRTUAL_ENV} +RUN ln -fs "$(command -v python3)" "${VIRTUAL_ENV}"/bin/python3 +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" +### +# Prepare to run +### +ENV ECHO_MESSAGE="Hello World from Dockerfile" WORKDIR ${CISA_HOME} - -RUN wget -O sourcecode.tgz https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz && \ - tar xzf sourcecode.tgz --strip-components=1 && \ - pip install --requirement requirements.txt && \ - ln -snf /run/secrets/quote.txt src/example/data/secret.txt && \ - rm sourcecode.tgz - -USER cisa - +USER ${CISA_USER}:${CISA_GROUP} EXPOSE 8080/TCP VOLUME ["/var/log"] ENTRYPOINT ["example"] -CMD ["--log-level", "DEBUG"] +CMD ["--log-level", "DEBUG", "8", "2"] diff --git a/README.md b/README.md index 1cbaf56..9e429b4 100644 --- a/README.md +++ b/README.md @@ -24,30 +24,30 @@ appropriate for Docker containers and the major languages that we use. To run the `cisagov/example` image via Docker: ```console -docker run cisagov/example:0.0.1 +docker run cisagov/example:0.2.0 ``` ### Running with Docker Compose ### -1. Create a `docker-compose.yml` file similar to the one below to use [Docker Compose](https://docs.docker.com/compose/). +1. Create a `compose.yml` file similar to the one below to use [Docker Compose](https://docs.docker.com/compose/). ```yaml --- - version: "3.7" + name: skeleton-docker services: example: - image: cisagov/example:0.0.1 - volumes: - - type: bind - source: - target: /var/log environment: - ECHO_MESSAGE="Hello from docker compose" + image: cisagov/example:0.2.0 ports: - - target: 8080 - published: 8080 - protocol: tcp + - protocol: tcp + published: "8080" + target: 8080 + volumes: + - source: + target: /var/log + type: bind ``` 1. Start the container and detach: @@ -70,11 +70,11 @@ environment variables. See the Better lock it in your pocket. ``` -1. Then add the secret to your `docker-compose.yml` file: +1. Then add the secret to your `compose.yml` file: ```yaml --- - version: "3.7" + name: skeleton-docker secrets: quote_txt: @@ -82,20 +82,20 @@ environment variables. See the services: example: - image: cisagov/example:0.0.1 - volumes: - - type: bind - source: - target: /var/log environment: - ECHO_MESSAGE="Hello from docker compose" + image: cisagov/example:0.2.0 ports: - - target: 8080 - published: 8080 - protocol: tcp + - protocol: tcp + published: "8080" + target: 8080 secrets: - source: quote_txt target: quote.txt + volumes: + - source: + target: /var/log + type: bind ``` ## Updating your container ## @@ -125,23 +125,52 @@ environment variables. See the 1. Pull the new image: ```console - docker pull cisagov/example:0.0.1 + docker pull cisagov/example:0.2.0 ``` 1. Recreate and run the container by following the [previous instructions](#running-with-docker). +## Updating Python dependencies ## + +This image uses [Pipenv] to manage Python dependencies using a [Pipfile](https://github.com/pypa/pipfile). +Both updating dependencies and changing the [Pipenv] configuration in `src/Pipfile` +will result in a modified `src/Pipfile.lock` file that should be committed to the +repository. + +> [!WARNING] +> The `src/Pipfile.lock` as generated will fail `pre-commit` checks due to JSON formatting. + +### Updating dependencies ### + +If you want to update existing dependencies you would run the following command +in the `src/` subdirectory: + +```console +pipenv lock +``` + +### Modifying dependencies ### + +If you want to add or remove dependencies you would update the `src/Pipfile` file +and then update dependencies as you would above. + +> [!NOTE] +> You should only specify packages that are direct requirements of +> your Docker configuration. Allow [Pipenv] to manage the dependencies +> of the specified packages. + ## Image tags ## The images of this container are tagged with [semantic versions](https://semver.org) of the underlying example project that they containerize. It is recommended that most users use a version tag (e.g. -`:0.0.1`). +`:0.2.0`). | Image:tag | Description | |-----------|-------------| -|`cisagov/example:1.2.3`| An exact release version. | -|`cisagov/example:1.2`| The most recent release matching the major and minor version numbers. | -|`cisagov/example:1`| The most recent release matching the major version number. | +|`cisagov/example:0.2.0`| An exact release version. | +|`cisagov/example:0.2`| The most recent release matching the major and minor version numbers. | +|`cisagov/example:0`| The most recent release matching the major version number. | |`cisagov/example:edge` | The most recent image built from a merge into the `develop` branch of this repository. | |`cisagov/example:nightly` | A nightly build of the `develop` branch of this repository. | |`cisagov/example:latest`| The most recent release image pushed to a container registry. Pulling an image using the `:latest` tag [should be avoided.](https://vsupalov.com/docker-latest-tag/) | @@ -163,7 +192,7 @@ The following ports are exposed by this container: |------|----------------| | 8080 | Example only; nothing is actually listening on the port | -The sample [Docker composition](docker-compose.yml) publishes the +The sample [Docker composition](compose.yml) publishes the exposed port at 8080. ## Environment variables ## @@ -196,8 +225,7 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --build-arg VERSION=0.0.1 \ - --tag cisagov/example:0.0.1 \ + --tag cisagov/example:0.2.0 \ https://github.com/cisagov/example.git#develop ``` @@ -227,9 +255,8 @@ Docker: docker buildx build \ --file Dockerfile-x \ --platform linux/amd64 \ - --build-arg VERSION=0.0.1 \ --output type=docker \ - --tag cisagov/example:0.0.1 . + --tag cisagov/example:0.2.0 . ``` ## New repositories from a skeleton ## @@ -256,3 +283,5 @@ dedication](https://creativecommons.org/publicdomain/zero/1.0/). All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. + +[Pipenv]: https://pypi.org/project/pipenv/ diff --git a/buildx-dockerfile.sh b/buildx-dockerfile.sh deleted file mode 100755 index 46710e9..0000000 --- a/buildx-dockerfile.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -# Create a Dockerfile suitable for a multi-platform build using buildx -# See: https://docs.docker.com/buildx/working-with-buildx/ - -set -o nounset -set -o errexit -set -o pipefail - -DOCKERFILE=Dockerfile -DOCKERFILEX=Dockerfile-x - -# We don't want this expression to expand. -# shellcheck disable=SC2016 -sed 's/^FROM /FROM --platform=$TARGETPLATFORM /g' < $DOCKERFILE > $DOCKERFILEX diff --git a/bump-version b/bump-version new file mode 100755 index 0000000..324fea9 --- /dev/null +++ b/bump-version @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +# bump-version [--push] [--label LABEL] (major | minor | patch | prerelease | build | finalize | show) +# bump-version --list-files + +set -o nounset +set -o errexit +set -o pipefail + +# Stores the canonical version for the project. +VERSION_FILE=src/version.txt +# Files that should be updated with the new version. +VERSION_FILES=("$VERSION_FILE" README.md) + +USAGE=$( + cat << END_OF_LINE +Update the version of the project. + +Usage: + ${0##*/} [--push] [--label LABEL] (major | minor | patch | prerelease | build | finalize | show) + ${0##*/} --list-files + ${0##*/} (-h | --help) + +Options: + -h | --help Show this message. + --push Perform a \`git push\` after updating the version. + --label LABEL Specify the label to use when updating the build or prerelease version. + --list-files List the files that will be updated when the version is bumped. +END_OF_LINE +) + +old_version=$(< "$VERSION_FILE") +# Comment out periods so they are interpreted as periods and don't +# just match any character +old_version_regex=${old_version//\./\\\.} +new_version="$old_version" + +bump_part="" +label="" +commit_prefix="Bump" +with_push=false +commands_with_label=("build" "prerelease") +commands_with_prerelease=("major" "minor" "patch") +with_prerelease=false + +####################################### +# Display an error message, the help information, and exit with a non-zero status. +# Arguments: +# Error message. +####################################### +function invalid_option() { + echo "$1" + echo "$USAGE" + exit 1 +} + +####################################### +# Bump the version using the provided command. +# Arguments: +# The version to bump. +# The command to bump the version. +# Returns: +# The new version. +####################################### +function bump_version() { + local temp_version + temp_version=$(python -c "import semver; print(semver.parse_version_info('$1').${2})") + echo "$temp_version" +} + +if [ $# -eq 0 ]; then + echo "$USAGE" + exit 1 +else + while [ $# -gt 0 ]; do + case $1 in + --push) + if [ "$with_push" = true ]; then + invalid_option "Push has already been set." + fi + + with_push=true + shift + ;; + --label) + if [ -n "$label" ]; then + invalid_option "Label has already been set." + fi + + label="$2" + shift 2 + ;; + build | finalize | major | minor | patch) + if [ -n "$bump_part" ]; then + invalid_option "Only one version part should be bumped at a time." + fi + + bump_part="$1" + shift + ;; + prerelease) + with_prerelease=true + shift + ;; + show) + echo "$old_version" + exit 0 + ;; + -h | --help) + echo "$USAGE" + exit 0 + ;; + --list-files) + printf '%s\n' "${VERSION_FILES[@]}" + exit 0 + ;; + *) + invalid_option "Invalid option: $1" + ;; + esac + done +fi + +if [ -n "$label" ] && [ "$with_prerelease" = false ] && [[ ! " ${commands_with_label[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then + invalid_option "Setting the label is only allowed for the following commands: ${commands_with_label[*]}" +fi + +if [ "$with_prerelease" = true ] && [ -n "$bump_part" ] && [[ ! " ${commands_with_prerelease[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then + invalid_option "Changing the prerelease is only allowed in conjunction with the following commands: ${commands_with_prerelease[*]}" +fi + +label_option="" +if [ -n "$label" ]; then + label_option="token='$label'" +fi + +if [ -n "$bump_part" ]; then + if [ "$bump_part" = "finalize" ]; then + commit_prefix="Finalize" + bump_command="finalize_version()" + elif [ "$bump_part" = "build" ]; then + bump_command="bump_${bump_part}($label_option)" + else + bump_command="bump_${bump_part}()" + fi + new_version=$(bump_version "$old_version" "$bump_command") + echo Changing version from "$old_version" to "$new_version" +fi + +if [ "$with_prerelease" = true ]; then + bump_command="bump_prerelease($label_option)" + temp_version=$(bump_version "$new_version" "$bump_command") + echo Changing version from "$new_version" to "$temp_version" + new_version="$temp_version" +fi + +tmp_file=/tmp/version.$$ +for version_file in "${VERSION_FILES[@]}"; do + if [ ! -f "$version_file" ]; then + echo Missing expected file: "$version_file" + exit 1 + fi + sed "s/$old_version_regex/$new_version/" "$version_file" > $tmp_file + mv $tmp_file "$version_file" +done + +git add "${VERSION_FILES[@]}" +git commit --message "$commit_prefix version from $old_version to $new_version" + +if [ "$with_push" = true ]; then + git push +fi diff --git a/bump_version.sh b/bump_version.sh deleted file mode 100755 index a6c8ed9..0000000 --- a/bump_version.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -# bump_version.sh (show|major|minor|patch|prerelease|build) - -set -o nounset -set -o errexit -set -o pipefail - -VERSION_FILE=src/version.txt -README_FILE=README.md - -HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" - -old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) - -if [ $# -ne 1 ]; then - echo "$HELP_INFORMATION" -else - case $1 in - major | minor | patch | prerelease | build) - new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - sed "s/$old_version/$new_version/" $README_FILE > $tmp_file - mv $tmp_file $README_FILE - git add $VERSION_FILE $README_FILE - git commit -m"Bump version from $old_version to $new_version" - git push - ;; - finalize) - new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - tmp_file=/tmp/version.$$ - sed "s/$old_version/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - sed "s/$old_version/$new_version/" $README_FILE > $tmp_file - mv $tmp_file $README_FILE - git add $VERSION_FILE $README_FILE - git commit -m"Bump version from $old_version to $new_version" - git push - ;; - show) - echo "$old_version" - ;; - *) - echo "$HELP_INFORMATION" - ;; - esac -fi diff --git a/docker-compose.yml b/compose.yml similarity index 89% rename from docker-compose.yml rename to compose.yml index e746ae9..75b708c 100644 --- a/docker-compose.yml +++ b/compose.yml @@ -1,8 +1,8 @@ --- -version: "3.7" - # This Docker composition file is used to build and test the container +name: skeleton-docker + secrets: quote_txt: file: ./src/secrets/quote.txt @@ -15,23 +15,23 @@ services: # e.g., --build-arg VERSION=0.0.1 context: . dockerfile: Dockerfile - image: cisagov/example - init: true - restart: "no" environment: - ECHO_MESSAGE=Hello World from docker compose! + image: cisagov/example + init: true ports: - - target: 8080 - published: 8080 + - mode: host protocol: tcp - mode: host + published: "8421" + target: 8421 + restart: "no" secrets: - source: quote_txt target: quote.txt example-version: # Run the container to collect version information + command: --version image: cisagov/example init: true restart: "no" - command: --version diff --git a/push_readme.sh b/push_readme.sh deleted file mode 100755 index 29b12aa..0000000 --- a/push_readme.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -# Push the README.md file to the Docker Hub repository - -# Requires the following environment variables to be set: -# DOCKER_PASSWORD, DOCKER_USERNAME, IMAGE_NAME - -set -o nounset -set -o errexit -set -o pipefail - -echo "Logging in and requesting JWT..." -token=$(curl --silent --request POST \ - --header "Content-Type: application/json" \ - --data \ - '{"username": "'"$DOCKER_USERNAME"'", "password": "'"$DOCKER_PASSWORD"'"}' \ - https://hub.docker.com/v2/users/login/ | jq --raw-output .token) - -echo "Pushing README file..." -code=$(jq --null-input --arg msg "$(< README.md)" \ - '{"registry":"registry-1.docker.io","full_description": $msg }' \ - | curl --silent --output /dev/null --location --write-out "%{http_code}" \ - https://hub.docker.com/v2/repositories/"${IMAGE_NAME}"/ \ - --data @- --request PATCH \ - --header "Content-Type: application/json" \ - --header "Authorization: JWT ${token}") - -if [[ "${code}" = "200" ]]; then - printf "Successfully pushed README to Docker Hub" -else - printf "Unable to push README to Docker Hub, response code: %s\n" "${code}" - exit 1 -fi diff --git a/requirements-dev.txt b/requirements-dev.txt index cb51627..8d26d98 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,3 @@ --requirement requirements-test.txt ipython -semver +pipenv diff --git a/requirements-test.txt b/requirements-test.txt index 5f3337c..16c2f58 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,4 +1,6 @@ --requirement requirements.txt pre-commit pytest -pytest-dockerc +python-on-whales +# The bump-version script requires at least version 3 of semver. +semver>=3 diff --git a/setup-env b/setup-env index f526cdb..059ccad 100755 --- a/setup-env +++ b/setup-env @@ -9,68 +9,130 @@ USAGE=$( Configure a development environment for this repository. It does the following: + - Allows the user to specify the Python version to use for the virtual environment. + - Allows the user to specify a name for the virtual environment. - Verifies pyenv and pyenv-virtualenv are installed. - - Creates a Python virtual environment. + - Creates the Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - Installs the requirements needed for development. - Installs git pre-commit hooks. - - Configures git upstream remote "lineage" repositories. + - Configures git remotes for upstream "lineage" repositories. Usage: - setup-env [options] [virt_env_name] + setup-env [--venv-name venv_name] [--python-version python_version] setup-env (-h | --help) Options: - -f --force Delete virtual enviroment if it already exists. - -h --help Show this message. - -i --install-hooks Install hook environments for all environments in the - pre-commit config file. + -f | --force Delete virtual enviroment if it already exists. + -h | --help Show this message. + -i | --install-hooks Install hook environments for all environments in the + pre-commit config file. + -l | --list-versions List available Python versions and select one interactively. + -v | --venv-name Specify the name of the virtual environment. + -p | --python-version Specify the Python version for the virtual environment. END_OF_LINE ) +# Display pyenv's installed Python versions +python_versions() { + pyenv versions --bare --skip-aliases --skip-envs +} + +check_python_version() { + local version=$1 + + # This is a valid regex for semantically correct Python version strings. + # For more information see here: + # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + # Break down the regex into readable parts major.minor.patch + local major="0|[1-9]\d*" + local minor="0|[1-9]\d*" + local patch="0|[1-9]\d*" + + # Splitting the prerelease part for readability + # Start of the prerelease + local prerelease="(?:-" + # Numeric or alphanumeric identifiers + local prerelease+="(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)" + # Additional dot-separated identifiers + local prerelease+="(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*" + # End of the prerelease, making it optional + local prerelease+=")?" + # Optional build metadata + local build="(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?" + + # Final regex composed of parts + local regex="^($major)\.($minor)\.($patch)$prerelease$build$" + + # This checks if the Python version does not match the regex pattern specified in $regex, + # using Perl for regex matching. If the pattern is not found, then prompt the user with + # the invalid version message. + if ! echo "$version" | perl -ne "exit(!/$regex/)"; then + echo "Invalid version of Python: Python follows semantic versioning," \ + "so any version string that is not a valid semantic version is an" \ + "invalid version of Python." + exit 1 + # Else if the Python version isn't installed then notify the user. + # grep -E is used for searching through text lines that match the specific verison. + elif ! python_versions | grep -E "^${version}$" > /dev/null; then + echo "Error: Python version $version is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $version" + fi +} + # Flag to force deletion and creation of virtual environment FORCE=0 -# Positional parameters -PARAMS="" +# Initialize the other flags +INSTALL_HOOKS=0 +LIST_VERSIONS=0 +PYTHON_VERSION="" +VENV_NAME="" -# Parse command line arguments -while (("$#")); do - case "$1" in - -f | --force) - FORCE=1 - shift - ;; - -h | --help) - echo "${USAGE}" - exit 0 - ;; - -i | --install-hooks) - INSTALL_HOOKS=1 - shift - ;; - -*) # unsupported flags - echo "Error: Unsupported flag $1" >&2 - exit 1 - ;; - *) # preserve positional arguments - PARAMS="$PARAMS $1" - shift - ;; - esac -done +# Define long options +LONGOPTS="force,help,install-hooks,list-versions,python-version:,venv-name:" + +# Define short options for getopt +SHORTOPTS="fhilp:v:" + +# Check for GNU getopt by matching a specific pattern ("getopt from util-linux") +# in its version output. This approach presumes the output format remains stable. +# Be aware that format changes could invalidate this check. +if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then + cat << 'END_OF_LINE' -# set positional arguments in their proper place -eval set -- "$PARAMS" + Please note, this script requires GNU getopt due to its enhanced + functionality and compatibility with certain script features that + are not supported by the POSIX getopt found in some systems, particularly + those with a non-GNU version of getopt. This distinction is crucial + as a system might have a non-GNU version of getopt installed by default, + which could lead to unexpected behavior. + + On macOS, we recommend installing brew (https://brew.sh/). Then installation + is as simple as `brew install gnu-getopt` and adding this to your + profile: + + export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" + + GNU getopt must be explicitly added to the PATH since it + is keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean). + +END_OF_LINE + exit 1 +fi # Check to see if pyenv is installed -if [ -z "$(command -v pyenv)" ] || [ -z "$(command -v pyenv-virtualenv)" ]; then +if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then echo "pyenv and pyenv-virtualenv are required." if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' - On the Mac, we recommend installing brew, https://brew.sh/. Then installation + On macOS, we recommend installing brew, https://brew.sh/. Then installation is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your profile: @@ -81,7 +143,7 @@ END_OF_LINE fi cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + For Linux, Windows Subsystem for Linux (WSL), or macOS (if you don't want to use "brew") you can use https://github.com/pyenv/pyenv-installer to install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the pyenv wiki page, @@ -100,16 +162,72 @@ END_OF_LINE exit 1 fi -set +o nounset +# Use GNU getopt to parse options +if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then + echo "Error parsing options" + exit 1 +fi +eval set -- "$PARSED" + +while true; do + case "$1" in + -f | --force) + FORCE=1 + shift + ;; + -h | --help) + echo "$USAGE" + exit 0 + ;; + -i | --install-hooks) + INSTALL_HOOKS=1 + shift + ;; + -l | --list-versions) + LIST_VERSIONS=1 + shift + ;; + -p | --python-version) + PYTHON_VERSION="$2" + shift 2 + # Check the Python version being passed in. + check_python_version "$PYTHON_VERSION" + ;; + -v | --venv-name) + VENV_NAME="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + # Unreachable due to GNU getopt handling all options + echo "Programming error" + exit 64 + ;; + esac +done + # Determine the virtual environment name -if [ "$1" ]; then +if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name - env_name=$1 + env_name="$VENV_NAME" else # Set the environment name to the last part of the working directory. env_name=${PWD##*/} fi -set -o nounset + +# List Python versions and select one interactively. +if [ $LIST_VERSIONS -ne 0 ]; then + echo Available Python versions: + python_versions + # Read the user's desired Python version. + # -r: treat backslashes as literal, -p: display prompt before input. + read -r -p "Enter the desired Python version: " PYTHON_VERSION + # Check the Python version being passed in. + check_python_version "$PYTHON_VERSION" +fi # Remove any lingering local configuration. if [ $FORCE -ne 0 ]; then @@ -118,7 +236,7 @@ if [ $FORCE -ne 0 ]; then elif [[ -f .python-version ]]; then cat << 'END_OF_LINE' An existing .python-version file was found. Either remove this file yourself - or re-run with --force option to have it deleted along with the associated + or re-run with the --force option to have it deleted along with the associated virtual environment. rm .python-version @@ -128,10 +246,18 @@ END_OF_LINE fi # Create a new virtual environment for this project -if ! pyenv virtualenv "${env_name}"; then +# +# If $PYTHON_VERSION is undefined then the current pyenv Python version will be used. +# +# We can't quote ${PYTHON_VERSION:=} below since if the variable is +# undefined then we want nothing to appear; this is the reason for the +# "shellcheck disable" line below. +# +# shellcheck disable=SC2086 +if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + environment yourself or re-run with the --force option to have it deleted. pyenv virtualenv-delete ${env_name} @@ -186,5 +312,5 @@ else: END_OF_LINE )" -# Qapla +# Qapla' echo "Success!" diff --git a/src/Pipfile b/src/Pipfile new file mode 100644 index 0000000..3d6e430 --- /dev/null +++ b/src/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +# List any Python dependencies for the image here +[packages] +# This should match the version of the image +example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0.2.0.tar.gz"} + +# This version should match the version of Python in the image +[requires] +python_full_version = "3.13.2" diff --git a/src/Pipfile.lock b/src/Pipfile.lock new file mode 100644 index 0000000..5211e41 --- /dev/null +++ b/src/Pipfile.lock @@ -0,0 +1,45 @@ +{ + "_meta": { + "hash": { + "sha256": "14f654f32f8b048255b0931440c8189b82643981e9d8d581eaceeef9d87603bb" + }, + "pipfile-spec": 6, + "requires": { + "python_full_version": "3.13.2" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, + "example": { + "file": "https://github.com/cisagov/skeleton-python-library/archive/v0.2.0.tar.gz" + }, + "schema": { + "hashes": [ + "sha256:5d976a5b50f36e74e2157b47097b60002bd4d42e65425fcc9c9befadb4255dde", + "sha256:7da553abd2958a19dc2547c388cde53398b39196175a9be59ea1caf5ab0a1807" + ], + "version": "==0.7.7" + }, + "setuptools": { + "hashes": [ + "sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6", + "sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3" + ], + "markers": "python_version >= '3.9'", + "version": "==75.8.0" + } + }, + "develop": {} +} diff --git a/src/version.txt b/src/version.txt index f102a9c..0ea3a94 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.0.1" +0.2.0 diff --git a/tag.sh b/tag.sh index e1f7447..0a0e607 100755 --- a/tag.sh +++ b/tag.sh @@ -4,6 +4,6 @@ set -o nounset set -o errexit set -o pipefail -version=$(./bump_version.sh show) +version=$(./bump-version show) git tag "v$version" && git push --tags diff --git a/tests/conftest.py b/tests/conftest.py index 90938e6..6092a48 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,18 +2,30 @@ https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins """ + # Third-Party Libraries import pytest +from python_on_whales import docker MAIN_SERVICE_NAME = "example" VERSION_SERVICE_NAME = f"{MAIN_SERVICE_NAME}-version" +VERSION_FILE = "src/version.txt" + + +@pytest.fixture(scope="session") +def dockerc(): + """Start up the Docker composition.""" + docker.compose.up(detach=True) + yield docker + docker.compose.down() + @pytest.fixture(scope="session") def main_container(dockerc): """Return the main container from the Docker composition.""" # find the container by name even if it is stopped already - return dockerc.containers(service_names=[MAIN_SERVICE_NAME], stopped=True)[0] + return dockerc.compose.ps(services=[MAIN_SERVICE_NAME], all=True)[0] @pytest.fixture(scope="session") @@ -23,7 +35,15 @@ def version_container(dockerc): The version container should just output the version of its underlying contents. """ # find the container by name even if it is stopped already - return dockerc.containers(service_names=[VERSION_SERVICE_NAME], stopped=True)[0] + return dockerc.compose.ps(services=[VERSION_SERVICE_NAME], all=True)[0] + + +@pytest.fixture(scope="session") +def project_version(): + """Return the version of the project.""" + with open(VERSION_FILE) as f: + project_version = f.read().strip() + return project_version def pytest_addoption(parser): diff --git a/tests/container_test.py b/tests/container_test.py index b92ff86..0937759 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,4 +1,3 @@ -#!/usr/bin/env pytest -vs """Tests for example container.""" # Standard Python Libraries @@ -7,22 +6,22 @@ # Third-Party Libraries import pytest +from semver import parse_version_info ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker compose!" READY_MESSAGE = "This is a debug message" -SECRET_QUOTE = ( - "There are no secrets better kept than the secrets everybody guesses." # nosec -) +DIVISION_MESSAGE = "8 / 2 == 4.000000" +SECRET_QUOTE = "Three may keep a secret, if two of them are dead." # nosec RELEASE_TAG = os.getenv("RELEASE_TAG") VERSION_FILE = "src/version.txt" def test_container_count(dockerc): """Verify the test composition and container.""" - # stopped parameter allows non-running containers in results + # all parameter allows non-running containers in results assert ( - len(dockerc.containers(stopped=True)) == 2 + len(dockerc.compose.ps(all=True)) == 2 ), "Wrong number of containers were started." @@ -30,7 +29,7 @@ def test_wait_for_ready(main_container): """Wait for container to be ready.""" TIMEOUT = 10 for i in range(TIMEOUT): - if READY_MESSAGE in main_container.logs().decode("utf-8"): + if READY_MESSAGE in main_container.logs(): break time.sleep(1) else: @@ -40,54 +39,51 @@ def test_wait_for_ready(main_container): ) -def test_wait_for_exits(main_container, version_container): +def test_wait_for_exits(dockerc, main_container, version_container): """Wait for containers to exit.""" - assert main_container.wait() == 0, "Container service (main) did not exit cleanly" assert ( - version_container.wait() == 0 + dockerc.wait(main_container.id) == 0 + ), "Container service (main) did not exit cleanly" + assert ( + dockerc.wait(version_container.id) == 0 ), "Container service (version) did not exit cleanly" -def test_output(main_container): +def test_output(dockerc, main_container): """Verify the container had the correct output.""" - main_container.wait() # make sure container exited if running test isolated - log_output = main_container.logs().decode("utf-8") + # make sure container exited if running test isolated + dockerc.wait(main_container.id) + log_output = main_container.logs() + assert DIVISION_MESSAGE in log_output, "Division message not found in log output." assert SECRET_QUOTE in log_output, "Secret not found in log output." @pytest.mark.skipif( RELEASE_TAG in [None, ""], reason="this is not a release (RELEASE_TAG not set)" ) -def test_release_version(): +def test_release_version(project_version): """Verify that release tag version agrees with the module version.""" - pkg_vars = {} - with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] assert ( RELEASE_TAG == f"v{project_version}" ), "RELEASE_TAG does not match the project version" -def test_log_version(version_container): +def test_log_version(dockerc, project_version, version_container): """Verify the container outputs the correct version to the logs.""" - version_container.wait() # make sure container exited if running test isolated - log_output = version_container.logs().decode("utf-8").strip() - pkg_vars = {} - with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] - assert ( - log_output == project_version + # make sure container exited if running test isolated + dockerc.wait(version_container.id) + log_version = parse_version_info(version_container.logs().strip()) + assert log_version == parse_version_info( + project_version ), f"Container version output to log does not match project version file {VERSION_FILE}" -def test_container_version_label_matches(version_container): +@pytest.mark.skipif( + RELEASE_TAG in [None, ""], reason="this is not a release (RELEASE_TAG not set)" +) +def test_container_version_label_matches(project_version, version_container): """Verify the container version label is the correct version.""" - pkg_vars = {} - with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] assert ( - version_container.labels["org.opencontainers.image.version"] == project_version + version_container.config.labels["org.opencontainers.image.version"] + == project_version ), "Dockerfile version label does not match project version"