From cd3c9681ddd792cb77c2f82154dc88ba1af6bf51 Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Mon, 30 Nov 2020 19:10:20 +0100 Subject: [PATCH 01/10] Move CI/CD from Travis to Github Actions. --- .github/workflows/test.yml | 79 +++++++++++++++++++ .travis.yml | 49 ------------ .../containers-logs.sh | 4 +- .../update-docker-compose.sh | 0 test/run.sh | 6 +- test/setup/setup-boulder.sh | 2 +- test/setup/setup-local.sh | 43 +++++----- test/setup/setup-nginx-proxy.sh | 4 +- test/tests/acme_accounts/run.sh | 2 +- test/tests/certs_san/run.sh | 2 +- test/tests/certs_single/run.sh | 2 +- test/tests/certs_single_domain/run.sh | 2 +- test/tests/certs_standalone/run.sh | 8 +- test/tests/container_restart/run.sh | 8 +- test/tests/default_cert/run.sh | 2 +- test/tests/docker_api/run.sh | 4 +- test/tests/force_renew/run.sh | 2 +- test/tests/location_config/run.sh | 4 +- test/tests/permissions_custom/run.sh | 2 +- test/tests/permissions_default/run.sh | 2 +- test/tests/private_keys/run.sh | 2 +- test/tests/symlinks/run.sh | 2 +- 22 files changed, 131 insertions(+), 100 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml rename test/{travis => github_actions}/containers-logs.sh (80%) rename test/{travis => github_actions}/update-docker-compose.sh (100%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..c2cb43c2 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,79 @@ +name: Test + +on: [push, pull_request] + +env: + IMAGE: jrcs/letsencrypt-nginx-proxy-companion + NGINX_CONTAINER_NAME: nginx-proxy + DOCKER_GEN_CONTAINER_NAME: nginx-proxy-gen + TEST_DOMAINS: le1.wtf,le2.wtf,le3.wtf + DOCKER_COMPOSE_VERSION: 1.24.0 + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Build Image + run: docker build -t "$IMAGE" . + - name: Inspect Image + run: docker inspect "$IMAGE" + - name: Get acme.sh Version + run: docker run --rm "$IMAGE" acme.sh --version + - name: List Docker Images + run: docker images + - name: Export Image Artifact + run: docker save $IMAGE > image.tar + - name: Upload Image Artifact + uses: actions/upload-artifact@v2 + with: + name: image.tar + path: image.tar + + test: + needs: build + strategy: + fail-fast: true + matrix: + setup: [2containers, 3containers] + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + # PREPARE RUNNER ENV + - name: Add Test Domains in /etc/hosts + run: | + test_domains=$(echo $TEST_DOMAINS | tr "," "\n") + for domain in $test_domains + do + echo "127.0.0.1 $domain" | sudo tee -a /etc/hosts + done + - run: cat /etc/hosts + - name: Add Test Dependencies + uses: actions/checkout@v2 + with: + repository: docker-library/official-images + path: official-images + - name: Setup Boulder + run: test/setup/setup-boulder.sh + - name: Setup NGINX Proxy + env: + SETUP: ${{ matrix.setup }} + run: test/setup/setup-nginx-proxy.sh + # ADD BUILDED IMAGE + - name: Download Builded Image + uses: actions/download-artifact@v2 + with: + name: image.tar + - name: Import Builded Image + run: docker load < image.tar + # TEST + - name: Test Image 1 + run: official-images/test/run.sh "$IMAGE" + - name: Test Image 2 + run: test/run.sh "$IMAGE" + - if: ${{ failure() }} + run: test/github_actions/containers-logs.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 956a6daf..00000000 --- a/.travis.yml +++ /dev/null @@ -1,49 +0,0 @@ -os: linux -dist: focal - -language: shell - -branches: - only: - - master - -addons: - hosts: - - le1.wtf - - le2.wtf - - le3.wtf - -env: - global: - - IMAGE=jrcs/letsencrypt-nginx-proxy-companion - - NGINX_CONTAINER_NAME=nginx-proxy - - DOCKER_GEN_CONTAINER_NAME=nginx-proxy-gen - - TEST_DOMAINS=le1.wtf,le2.wtf,le3.wtf - - DOCKER_COMPOSE_VERSION=1.24.0 - -jobs: - include: - - env: SETUP=2containers - - env: SETUP=3containers - -before_install: - - test/travis/update-docker-compose.sh - -install: - - docker build -t "$IMAGE" . - - docker inspect "$IMAGE" - - docker run --rm "$IMAGE" acme.sh --version - - docker images - -before_script: - - git clone https://github.com/docker-library/official-images.git official-images - - test/setup/setup-boulder.sh - - test/setup/setup-nginx-proxy.sh - - docker pull nginx:alpine - -script: - - official-images/test/run.sh "$IMAGE" - - test/run.sh "$IMAGE" - -after_failure: - - test/travis/containers-logs.sh diff --git a/test/travis/containers-logs.sh b/test/github_actions/containers-logs.sh similarity index 80% rename from test/travis/containers-logs.sh rename to test/github_actions/containers-logs.sh index ff94851d..103bd632 100755 --- a/test/travis/containers-logs.sh +++ b/test/github_actions/containers-logs.sh @@ -8,8 +8,8 @@ fold_end() { echo -e "\ntravis_fold:end:$1\r" } -if [[ -f "$TRAVIS_BUILD_DIR/test/travis/failed_tests.txt" ]]; then - mapfile -t containers < "$TRAVIS_BUILD_DIR/test/travis/failed_tests.txt" +if [[ -f "$GITHUB_WORKSPACE/test/github_actions/failed_tests.txt" ]]; then + mapfile -t containers < "$GITHUB_WORKSPACE/test/github_actions/failed_tests.txt" fi containers+=("$NGINX_CONTAINER_NAME") diff --git a/test/travis/update-docker-compose.sh b/test/github_actions/update-docker-compose.sh similarity index 100% rename from test/travis/update-docker-compose.sh rename to test/github_actions/update-docker-compose.sh diff --git a/test/run.sh b/test/run.sh index eb03c233..7c9abb8c 100755 --- a/test/run.sh +++ b/test/run.sh @@ -203,7 +203,7 @@ dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" self="$(basename "$0")" failed_tests=() -if [[ -z $TRAVIS ]] && [[ -f "$dir/local_test_env.sh" ]]; then +if [[ -z $GITHUB_ACTIONS ]] && [[ -f "$dir/local_test_env.sh" ]]; then # shellcheck source=/dev/null source "$dir/local_test_env.sh" fi @@ -448,9 +448,9 @@ done if [ "$didFail" ]; then ## Next five lines were added by jrcs/docker-letsencrypt-nginx-proxy-companion - if [[ $TRAVIS == 'true' ]]; then + if [[ $GITHUB_ACTIONS == 'true' ]]; then for test in "${failed_tests[@]}"; do - echo "$test" >> "$dir/travis/failed_tests.txt" + echo "$test" >> "$dir/github_actions/failed_tests.txt" done fi ## End of additional code diff --git a/test/setup/setup-boulder.sh b/test/setup/setup-boulder.sh index 2bc4fedd..ec948ebb 100755 --- a/test/setup/setup-boulder.sh +++ b/test/setup/setup-boulder.sh @@ -5,7 +5,7 @@ set -e acme_endpoint='http://boulder:4001/directory' setup_boulder() { - export GOPATH=${TRAVIS_BUILD_DIR}/go + export GOPATH=${GITHUB_WORKSPACE}/go [[ ! -d $GOPATH/src/github.com/letsencrypt/boulder ]] \ && git clone https://github.com/letsencrypt/boulder \ "$GOPATH/src/github.com/letsencrypt/boulder" diff --git a/test/setup/setup-local.sh b/test/setup/setup-local.sh index f05a72d0..0d6b09f7 100755 --- a/test/setup/setup-local.sh +++ b/test/setup/setup-local.sh @@ -6,22 +6,23 @@ function get_environment { dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" LOCAL_BUILD_DIR="$(cd "$dir/../.." && pwd)" - export TRAVIS_BUILD_DIR="$LOCAL_BUILD_DIR" + export GITHUB_WORKSPACE="$LOCAL_BUILD_DIR" # shellcheck source=/dev/null - [[ -f "${TRAVIS_BUILD_DIR}/test/local_test_env.sh" ]] && \ - source "${TRAVIS_BUILD_DIR}/test/local_test_env.sh" + [[ -f "${GITHUB_WORKSPACE}/test/local_test_env.sh" ]] && \ + source "${GITHUB_WORKSPACE}/test/local_test_env.sh" - # Get the environment variables from the .travis.yml file with sed - declare -a travis_yml - travis_yml[0]="$(sed -n 's/.*- NGINX_CONTAINER_NAME=//p' "$LOCAL_BUILD_DIR/.travis.yml")" - travis_yml[1]="$(sed -n 's/.*- DOCKER_GEN_CONTAINER_NAME=//p' "$LOCAL_BUILD_DIR/.travis.yml")" - travis_yml[2]="$(sed -n 's/.*- TEST_DOMAINS=//p' "$LOCAL_BUILD_DIR/.travis.yml")" + # Get the environment variables from the .github/workflows/test.yml file with sed + declare -a ci_test_yml + ci_test_yml[0]="$(sed -n 's/.* NGINX_CONTAINER_NAME: //p' "$LOCAL_BUILD_DIR/.github/workflows/test.yml")" + ci_test_yml[1]="$(sed -n 's/.* DOCKER_GEN_CONTAINER_NAME: //p' "$LOCAL_BUILD_DIR/.github/workflows/test.yml")" + ci_test_yml[2]="$(sed -n 's/.* TEST_DOMAINS: //p' "$LOCAL_BUILD_DIR/.github/workflows/test.yml")" - # If environment variable where sourced or manually set use them, else use those from .travis.yml - export NGINX_CONTAINER_NAME="${NGINX_CONTAINER_NAME:-${travis_yml[0]}}" - export DOCKER_GEN_CONTAINER_NAME="${DOCKER_GEN_CONTAINER_NAME:-${travis_yml[1]}}" - export TEST_DOMAINS="${TEST_DOMAINS:-${travis_yml[2]}}" + # If environment variable where sourced or manually set use them, else use those from + # .github/workflows/test.yml + export NGINX_CONTAINER_NAME="${NGINX_CONTAINER_NAME:-${ci_test_yml[0]}}" + export DOCKER_GEN_CONTAINER_NAME="${DOCKER_GEN_CONTAINER_NAME:-${ci_test_yml[1]}}" + export TEST_DOMAINS="${TEST_DOMAINS:-${ci_test_yml[2]}}" # Build the array containing domains to add to /etc/hosts IFS=',' read -r -a domains <<< "$TEST_DOMAINS" @@ -57,8 +58,8 @@ case $1 in get_environment # Prepare the env file that run.sh will source - cat > "${TRAVIS_BUILD_DIR}/test/local_test_env.sh" < "${GITHUB_WORKSPACE}/test/local_test_env.sh" < "${TRAVIS_BUILD_DIR}/nginx.tmpl" + curl https://raw.githubusercontent.com/jwilder/nginx-proxy/master/nginx.tmpl > "${GITHUB_WORKSPACE}/nginx.tmpl" docker run -d -p 80:80 -p 443:443 \ --name "$NGINX_CONTAINER_NAME" \ @@ -33,7 +33,7 @@ case $SETUP in docker run -d \ --name "$DOCKER_GEN_CONTAINER_NAME" \ --volumes-from "$NGINX_CONTAINER_NAME" \ - -v "${TRAVIS_BUILD_DIR}/nginx.tmpl:/etc/docker-gen/templates/nginx.tmpl:ro" \ + -v "${GITHUB_WORKSPACE}/nginx.tmpl:/etc/docker-gen/templates/nginx.tmpl:ro" \ -v /var/run/docker.sock:/tmp/docker.sock:ro \ --label com.github.jrcs.letsencrypt_nginx_proxy_companion.test_suite \ --network boulder_bluenet \ diff --git a/test/tests/acme_accounts/run.sh b/test/tests/acme_accounts/run.sh index a8d28d46..982ac73b 100755 --- a/test/tests/acme_accounts/run.sh +++ b/test/tests/acme_accounts/run.sh @@ -2,7 +2,7 @@ ## Test for ACME accounts handling. -if [[ -z $TRAVIS ]]; then +if [[ -z $GITHUB_ACTIONS ]]; then le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else le_container_name="$(basename "${0%/*}")" diff --git a/test/tests/certs_san/run.sh b/test/tests/certs_san/run.sh index ff8f5c0e..4e05f390 100755 --- a/test/tests/certs_san/run.sh +++ b/test/tests/certs_san/run.sh @@ -2,7 +2,7 @@ ## Test for SAN (Subject Alternative Names) certificates. -if [[ -z $TRAVIS ]]; then +if [[ -z $GITHUB_ACTIONS ]]; then le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else le_container_name="$(basename "${0%/*}")" diff --git a/test/tests/certs_single/run.sh b/test/tests/certs_single/run.sh index 26b895bf..f5220cef 100755 --- a/test/tests/certs_single/run.sh +++ b/test/tests/certs_single/run.sh @@ -2,7 +2,7 @@ ## Test for single domain certificates. -if [[ -z $TRAVIS ]]; then +if [[ -z $GITHUB_ACTIONS ]]; then le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else le_container_name="$(basename "${0%/*}")" diff --git a/test/tests/certs_single_domain/run.sh b/test/tests/certs_single_domain/run.sh index 172fd49d..fa544391 100755 --- a/test/tests/certs_single_domain/run.sh +++ b/test/tests/certs_single_domain/run.sh @@ -2,7 +2,7 @@ ## Test for spliting SAN certificates into single domain certificates by NGINX container env variables -if [[ -z $TRAVIS ]]; then +if [[ -z $GITHUB_ACTIONS ]]; then le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else le_container_name="$(basename "${0%/*}")" diff --git a/test/tests/certs_standalone/run.sh b/test/tests/certs_standalone/run.sh index fa845d8a..2a512429 100755 --- a/test/tests/certs_standalone/run.sh +++ b/test/tests/certs_standalone/run.sh @@ -2,7 +2,7 @@ ## Test for standalone certificates. -if [[ -z $TRAVIS ]]; then +if [[ -z $GITHUB_ACTIONS ]]; then le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else le_container_name="$(basename "${0%/*}")" @@ -24,7 +24,7 @@ function cleanup { trap cleanup EXIT # Create letsencrypt_user_data with a single domain cert -cat > "${TRAVIS_BUILD_DIR}/test/tests/certs_standalone/letsencrypt_user_data" < "${GITHUB_WORKSPACE}/test/tests/certs_standalone/letsencrypt_user_data" < "${TRAVIS_BUILD_DIR}/test/tests/certs_standalone/letsencrypt_user_data" < "${GITHUB_WORKSPACE}/test/tests/certs_standalone/letsencrypt_user_data" < "${TRAVIS_BUILD_DIR}/test/tests/container_restart/docker_event_out.txt" & + --format 'Container {{.Actor.Attributes.name}} restarted' > "${GITHUB_WORKSPACE}/test/tests/container_restart/docker_event_out.txt" & docker_events_pid=$! # Cleanup function with EXIT trap @@ -23,7 +23,7 @@ function cleanup { # Kill the Docker events listener kill $docker_events_pid && wait $docker_events_pid 2>/dev/null # Remove temporary files - rm -f "${TRAVIS_BUILD_DIR}/test/tests/container_restart/docker_event_out.txt" + rm -f "${GITHUB_WORKSPACE}/test/tests/container_restart/docker_event_out.txt" # Remove any remaining Nginx container(s) silently. for domain in "${domains[@]}"; do docker rm --force "$domain" &> /dev/null @@ -53,7 +53,7 @@ for domain in "${domains[@]}"; do # Check if container restarted timeout="$(date +%s)" timeout="$((timeout + 60))" - until grep "$domain" "${TRAVIS_BUILD_DIR}"/test/tests/container_restart/docker_event_out.txt; do + until grep "$domain" "${GITHUB_WORKSPACE}"/test/tests/container_restart/docker_event_out.txt; do if [[ "$(date +%s)" -gt "$timeout" ]]; then echo "Container $domain didn't restart in under one minute." break diff --git a/test/tests/default_cert/run.sh b/test/tests/default_cert/run.sh index bf3c6455..b3356955 100755 --- a/test/tests/default_cert/run.sh +++ b/test/tests/default_cert/run.sh @@ -2,7 +2,7 @@ ## Test for default certificate creation. -if [[ -z $TRAVIS ]]; then +if [[ -z $GITHUB_ACTIONS ]]; then le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else le_container_name="$(basename "${0%/*}")" diff --git a/test/tests/docker_api/run.sh b/test/tests/docker_api/run.sh index ed1add37..84acf098 100755 --- a/test/tests/docker_api/run.sh +++ b/test/tests/docker_api/run.sh @@ -78,7 +78,7 @@ case $SETUP in "$1" \ bash -c "$commands" 2>&1 - cat > "${TRAVIS_BUILD_DIR}/test/tests/docker_api/expected-std-out.txt" < "${GITHUB_WORKSPACE}/test/tests/docker_api/expected-std-out.txt" <&1 - cat > "${TRAVIS_BUILD_DIR}/test/tests/docker_api/expected-std-out.txt" < "${GITHUB_WORKSPACE}/test/tests/docker_api/expected-std-out.txt" < "$location_file" # Create le1.wtf configuration file, *.le3.wtf and test.* from inside the nginx container @@ -18,7 +18,7 @@ docker exec "$NGINX_CONTAINER_NAME" sh -c "echo '### This is a test comment' > / # Zero the default configuration file. docker exec "$NGINX_CONTAINER_NAME" sh -c "echo '' > /etc/nginx/vhost.d/default" -if [[ -z $TRAVIS ]]; then +if [[ -z $GITHUB_ACTIONS ]]; then le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else le_container_name="$(basename "${0%/*}")" diff --git a/test/tests/permissions_custom/run.sh b/test/tests/permissions_custom/run.sh index 6a7ca43a..8a242187 100755 --- a/test/tests/permissions_custom/run.sh +++ b/test/tests/permissions_custom/run.sh @@ -7,7 +7,7 @@ files_gid=1001 files_perms=640 folders_perms=750 -if [[ -z $TRAVIS ]]; then +if [[ -z $GITHUB_ACTIONS ]]; then le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else le_container_name="$(basename "${0%/*}")" diff --git a/test/tests/permissions_default/run.sh b/test/tests/permissions_default/run.sh index 11e34f34..0e28e48a 100755 --- a/test/tests/permissions_default/run.sh +++ b/test/tests/permissions_default/run.sh @@ -2,7 +2,7 @@ ## Test for sensitive files and folders permissions -if [[ -z $TRAVIS ]]; then +if [[ -z $GITHUB_ACTIONS ]]; then le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else le_container_name="$(basename "${0%/*}")" diff --git a/test/tests/private_keys/run.sh b/test/tests/private_keys/run.sh index d5886c3b..b3dee434 100755 --- a/test/tests/private_keys/run.sh +++ b/test/tests/private_keys/run.sh @@ -2,7 +2,7 @@ ## Test for private keys types -if [[ -z $TRAVIS ]]; then +if [[ -z $GITHUB_ACTIONS ]]; then le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else le_container_name="$(basename "${0%/*}")" diff --git a/test/tests/symlinks/run.sh b/test/tests/symlinks/run.sh index 9f660206..b74e57b0 100755 --- a/test/tests/symlinks/run.sh +++ b/test/tests/symlinks/run.sh @@ -2,7 +2,7 @@ ## Test for symlink creation / removal. -if [[ -z $TRAVIS ]]; then +if [[ -z $GITHUB_ACTIONS ]]; then le_container_name="$(basename "${0%/*}")_$(date "+%Y-%m-%d_%H.%M.%S")" else le_container_name="$(basename "${0%/*}")" From cfff67c726b76460b3d723d74206ba57280eaea1 Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Mon, 30 Nov 2020 19:10:46 +0100 Subject: [PATCH 02/10] CI/CD: Remove Docker Compose Update. --- .github/workflows/test.yml | 3 +-- test/github_actions/update-docker-compose.sh | 7 ------- 2 files changed, 1 insertion(+), 9 deletions(-) delete mode 100755 test/github_actions/update-docker-compose.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c2cb43c2..4f8d939d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,11 +3,10 @@ name: Test on: [push, pull_request] env: + DOCKER_GEN_CONTAINER_NAME: nginx-proxy-gen IMAGE: jrcs/letsencrypt-nginx-proxy-companion NGINX_CONTAINER_NAME: nginx-proxy - DOCKER_GEN_CONTAINER_NAME: nginx-proxy-gen TEST_DOMAINS: le1.wtf,le2.wtf,le3.wtf - DOCKER_COMPOSE_VERSION: 1.24.0 jobs: build: diff --git a/test/github_actions/update-docker-compose.sh b/test/github_actions/update-docker-compose.sh deleted file mode 100755 index 86fb757e..00000000 --- a/test/github_actions/update-docker-compose.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -sudo rm /usr/local/bin/docker-compose -curl -L "https://github.com/docker/compose/releases/download/$DOCKER_COMPOSE_VERSION/docker-compose-$(uname -s)-$(uname -m)" > docker-compose.temp -chmod +x docker-compose.temp -sudo mv docker-compose.temp /usr/local/bin/docker-compose -docker-compose --version From 17323b40a6bef7638d1d60f94d287002b005a4ad Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Mon, 30 Nov 2020 19:11:11 +0100 Subject: [PATCH 03/10] CI/CD: Split TestsJobs into Docker SPecifications Testing and Integration Testing. --- .github/workflows/test.yml | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4f8d939d..a9450ba4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,7 +31,28 @@ jobs: name: image.tar path: image.tar - test: + docker-specs-testing: + needs: build + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + with: + repository: docker-library/official-images + path: official-images + - name: Download Builded Image + uses: actions/download-artifact@v2 + with: + name: image.tar + - name: Import Builded Image + run: docker load < image.tar + - name: Docker Specifications Testing + run: official-images/test/run.sh "$IMAGE" + - if: ${{ failure() }} + run: test/github_actions/containers-logs.sh + + integration-testing: needs: build strategy: fail-fast: true @@ -50,12 +71,6 @@ jobs: do echo "127.0.0.1 $domain" | sudo tee -a /etc/hosts done - - run: cat /etc/hosts - - name: Add Test Dependencies - uses: actions/checkout@v2 - with: - repository: docker-library/official-images - path: official-images - name: Setup Boulder run: test/setup/setup-boulder.sh - name: Setup NGINX Proxy @@ -70,9 +85,7 @@ jobs: - name: Import Builded Image run: docker load < image.tar # TEST - - name: Test Image 1 - run: official-images/test/run.sh "$IMAGE" - - name: Test Image 2 + - name: Integration Testing run: test/run.sh "$IMAGE" - if: ${{ failure() }} run: test/github_actions/containers-logs.sh From a6aaac7d72e4b0ce5c2983e2589d3072110e329a Mon Sep 17 00:00:00 2001 From: Kevin Marilleau Date: Wed, 9 Dec 2020 14:01:21 +0100 Subject: [PATCH 04/10] CI/CD: split tests --- .github/workflows/test.yml | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a9450ba4..e437ebba 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,9 +55,26 @@ jobs: integration-testing: needs: build strategy: - fail-fast: true + fail-fast: false matrix: setup: [2containers, 3containers] + test-name: + [ + docker_api, + location_config, + default_cert, + certs_single, + certs_san, + certs_single_domain, + certs_standalone, + force_renew, + acme_accounts, + private_keys, + container_restart, + permissions_default, + permissions_custom, + symlinks, + ] runs-on: ubuntu-latest steps: @@ -86,6 +103,6 @@ jobs: run: docker load < image.tar # TEST - name: Integration Testing - run: test/run.sh "$IMAGE" + run: test/run.sh -t ${{ matrix.test-name }} "$IMAGE" - if: ${{ failure() }} run: test/github_actions/containers-logs.sh From 4692095e40f90dbb2a526dfa55a85c9b02385729 Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Mon, 21 Dec 2020 22:47:43 +0100 Subject: [PATCH 05/10] CI/CD: Use patched nginx-proxy & docker-gen images --- .github/workflows/test.yml | 63 ++++++++++++++++++++++----- Dockerfile | 5 ++- test/setup/docker-gen/Dockerfile | 37 ++++++++++++++++ test/setup/nginx-proxy/Dockerfile | 71 +++++++++++++++++++++++++++++++ 4 files changed, 164 insertions(+), 12 deletions(-) create mode 100644 test/setup/docker-gen/Dockerfile create mode 100644 test/setup/nginx-proxy/Dockerfile diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e437ebba..ade9e70b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ env: TEST_DOMAINS: le1.wtf,le2.wtf,le3.wtf jobs: - build: + companion-build: runs-on: ubuntu-latest steps: @@ -24,15 +24,42 @@ jobs: - name: List Docker Images run: docker images - name: Export Image Artifact - run: docker save $IMAGE > image.tar + run: docker save $IMAGE > companion.tar - name: Upload Image Artifact uses: actions/upload-artifact@v2 with: - name: image.tar - path: image.tar + name: companion.tar + path: companion.tar + + nginx-proxy-build: + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v2 + - name: Build patched nginx-proxy Image + run: docker build -t "jwilder/nginx-proxy:latest" ./test/setup/nginx-proxy + - name: Build patched docker-gen Image + run: docker build -t "jwilder/docker-gen:latest" ./test/setup/docker-gen + - name: List Docker Images + run: docker images + - name: Export Images Artifacts + run: | + docker save jwilder/nginx-proxy:latest > nginx-proxy.tar + docker save jwilder/docker-gen:latest > docker-gen.tar + - name: Upload nginx-proxy Image Artifact + uses: actions/upload-artifact@v2 + with: + name: nginx-proxy.tar + path: nginx-proxy.tar + - name: Upload docker-gen Image Artifact + uses: actions/upload-artifact@v2 + with: + name: docker-gen.tar + path: docker-gen.tar docker-specs-testing: - needs: build + needs: companion-build runs-on: ubuntu-latest steps: @@ -44,16 +71,18 @@ jobs: - name: Download Builded Image uses: actions/download-artifact@v2 with: - name: image.tar + name: companion.tar - name: Import Builded Image - run: docker load < image.tar + run: docker load < companion.tar - name: Docker Specifications Testing run: official-images/test/run.sh "$IMAGE" - if: ${{ failure() }} run: test/github_actions/containers-logs.sh integration-testing: - needs: build + needs: + - companion-build + - nginx-proxy-build strategy: fail-fast: false matrix: @@ -90,6 +119,18 @@ jobs: done - name: Setup Boulder run: test/setup/setup-boulder.sh + - name: Download nginx-proxy Patched Image + uses: actions/download-artifact@v2 + with: + name: nginx-proxy.tar + - name: Download docker-gen Patched Image + uses: actions/download-artifact@v2 + with: + name: docker-gen.tar + - name: Import nginx-proxy patched Images + run: | + docker load < nginx-proxy.tar + docker load < docker-gen.tar - name: Setup NGINX Proxy env: SETUP: ${{ matrix.setup }} @@ -98,11 +139,13 @@ jobs: - name: Download Builded Image uses: actions/download-artifact@v2 with: - name: image.tar + name: companion.tar - name: Import Builded Image - run: docker load < image.tar + run: docker load < companion.tar # TEST - name: Integration Testing + env: + SETUP: ${{ matrix.setup }} run: test/run.sh -t ${{ matrix.test-name }} "$IMAGE" - if: ${{ failure() }} run: test/github_actions/containers-logs.sh diff --git a/Dockerfile b/Dockerfile index 74b06975..0f6c35aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.15-alpine AS go-builder -ENV DOCKER_GEN_VERSION=0.7.4 +ENV DOCKER_GEN_VERSION=fix-current-container-id # Build docker-gen RUN apk add --no-cache --virtual .build-deps \ @@ -9,7 +9,8 @@ RUN apk add --no-cache --virtual .build-deps \ git \ make \ musl-dev \ - && go get github.com/jwilder/docker-gen \ + && go get github.com/buchdag/docker-gen \ + && mv /go/src/github.com/buchdag /go/src/github.com/jwilder \ && cd /go/src/github.com/jwilder/docker-gen \ && git -c advice.detachedHead=false checkout $DOCKER_GEN_VERSION \ && make get-deps \ diff --git a/test/setup/docker-gen/Dockerfile b/test/setup/docker-gen/Dockerfile new file mode 100644 index 00000000..63b22078 --- /dev/null +++ b/test/setup/docker-gen/Dockerfile @@ -0,0 +1,37 @@ +FROM golang:1.15-alpine AS build-docker-gen + +ARG DOCKER_GEN_VERSION=fix-current-container-id + +LABEL stage=intermediate + +# Install build dependencies for docker-gen +RUN apk add --update \ + curl \ + gcc \ + git \ + make \ + musl-dev + +# Build docker-gen +RUN go get github.com/buchdag/docker-gen \ + && mv /go/src/github.com/buchdag /go/src/github.com/jwilder \ + && cd /go/src/github.com/jwilder/docker-gen \ + && git -c advice.detachedHead=false checkout $DOCKER_GEN_VERSION \ + && make get-deps \ + && make all + +FROM alpine:3.8 + +LABEL maintainer="Nicolas Duchon " + +# DOCKER_GEN_VERSION environment variable is required by letsencrypt-nginx-proxy-companion +ENV DOCKER_GEN_VERSION=0.7.4 \ + DOCKER_HOST=unix:///tmp/docker.sock + +# Copy docker-gen binary from build stage +COPY --from=build-docker-gen /go/src/github.com/jwilder/docker-gen/docker-gen /usr/local/bin/ + +# Get latest nginx.tmpl +ADD https://raw.githubusercontent.com/jwilder/nginx-proxy/master/nginx.tmpl /etc/docker-gen/templates/ + +ENTRYPOINT ["/usr/local/bin/docker-gen"] \ No newline at end of file diff --git a/test/setup/nginx-proxy/Dockerfile b/test/setup/nginx-proxy/Dockerfile new file mode 100644 index 00000000..a5dab80f --- /dev/null +++ b/test/setup/nginx-proxy/Dockerfile @@ -0,0 +1,71 @@ +FROM golang:1.15-alpine AS go-builder + +ARG DOCKER_GEN_VERSION=fix-current-container-id +ARG FOREGO_VERSION=20180216151118 + +LABEL stage=intermediate + +# Install build dependencies for docker-gen and forego +RUN apk add --update \ + curl \ + gcc \ + git \ + make \ + musl-dev + +# Build forego +RUN go get github.com/ddollar/forego \ + && cd /go/src/github.com/ddollar/forego \ + && git -c advice.detachedHead=false checkout $FOREGO_VERSION \ + && make all + +# Build docker-gen +RUN go get github.com/buchdag/docker-gen \ + && mv /go/src/github.com/buchdag /go/src/github.com/jwilder \ + && cd /go/src/github.com/jwilder/docker-gen \ + && git -c advice.detachedHead=false checkout $DOCKER_GEN_VERSION \ + && make get-deps \ + && make all + +FROM nginx:1.19-alpine + +LABEL maintainer="Nicolas Duchon " + +# DOCKER_GEN_VERSION environment variable is required by letsencrypt-nginx-proxy-companion +ENV DOCKER_GEN_VERSION=0.7.4 \ + DOCKER_HOST=unix:///tmp/docker.sock + +# Install/update run dependencies +RUN apk add --update \ + bash \ + ca-certificates \ + curl \ + openssl \ + && rm -rf /var/cache/apk/* + +# Configure Nginx and apply fix for very long server names +RUN echo "daemon off;" >> /etc/nginx/nginx.conf \ + && sed -i 's/worker_processes 1/worker_processes auto/' /etc/nginx/nginx.conf + +# Copy forego and docker-gen binaries from build stage +COPY --from=go-builder /go/src/github.com/ddollar/forego/forego /usr/local/bin/ +COPY --from=go-builder /go/src/github.com/jwilder/docker-gen/docker-gen /usr/local/bin/ + +# Install nginx-proxy +RUN mkdir /src /app \ + && curl -sSL https://github.com/jwilder/nginx-proxy/archive/master.tar.gz \ + | tar -C /src -xz \ + && cp /src/nginx-proxy-master/Procfile /app/ \ + && cp /src/nginx-proxy-master/dhparam.pem.default /app/ \ + && cp /src/nginx-proxy-master/docker-entrypoint.sh /app/ \ + && cp /src/nginx-proxy-master/generate-dhparam.sh /app/ \ + && cp /src/nginx-proxy-master/nginx.tmpl /app/ \ + && cp /src/nginx-proxy-master/network_internal.conf /etc/nginx/ \ + && rm -rf /src + +WORKDIR /app + +VOLUME ["/etc/nginx/certs", "/etc/nginx/dhparam"] + +ENTRYPOINT ["/app/docker-entrypoint.sh"] +CMD ["forego", "start", "-r"] \ No newline at end of file From cf09ccbb80359fb2114b5d9f597b8013be83c323 Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Mon, 21 Dec 2020 23:14:23 +0100 Subject: [PATCH 06/10] CI/CD: Remove Travis stuff from containers-logs.sh --- test/github_actions/containers-logs.sh | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/test/github_actions/containers-logs.sh b/test/github_actions/containers-logs.sh index 103bd632..8a11a283 100755 --- a/test/github_actions/containers-logs.sh +++ b/test/github_actions/containers-logs.sh @@ -1,11 +1,7 @@ #!/bin/bash -fold_start() { - echo -e "travis_fold:start:$1\033[33;1m$2\033[0m" -} - -fold_end() { - echo -e "\ntravis_fold:end:$1\r" +bold_echo() { + echo -e "\033[33;1m$1\033[0m" } if [[ -f "$GITHUB_WORKSPACE/test/github_actions/failed_tests.txt" ]]; then @@ -17,12 +13,12 @@ containers+=("$NGINX_CONTAINER_NAME") containers+=("boulder") for container in "${containers[@]}"; do - fold_start "$container" "Docker container output for $container" + bold_echo "Docker container output for $container" docker logs "$container" - fold_end "$container" + docker inspect "$container" if [[ "$container" == "acme_accounts" ]]; then - fold_start "${container}_default" "Docker container output for ${container}_default" + bold_echo "Docker container output for ${container}_default" docker logs "${container}_default" - fold_end "${container}_default" + docker inspect "${container}_default" fi done From 992928816d4cc8a7d7aa6edb39ba64947d2780bf Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Mon, 21 Dec 2020 23:56:18 +0100 Subject: [PATCH 07/10] CI/CD: Fix certs_san test --- test/tests/certs_san/run.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/tests/certs_san/run.sh b/test/tests/certs_san/run.sh index 4e05f390..c6f21f3b 100755 --- a/test/tests/certs_san/run.sh +++ b/test/tests/certs_san/run.sh @@ -90,8 +90,7 @@ for hosts in "${letsencrypt_hosts[@]}"; do done docker stop "$container" &> /dev/null - docker exec "$le_container_name" rm -rf /etc/nginx/certs/le?.wtf* - docker exec "$le_container_name" rm -rf /etc/acme.sh/default/le?.wtf* + docker exec "$le_container_name" /app/cleanup_test_artifacts i=$(( i + 1 )) done From 000c279bb6c906b86d4d5a8855d8fe46394f7900 Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Tue, 22 Dec 2020 00:07:15 +0100 Subject: [PATCH 08/10] CI/CD: Use in-container artifact cleaning script --- test/tests/certs_single_domain/run.sh | 3 +-- test/tests/default_cert/run.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/tests/certs_single_domain/run.sh b/test/tests/certs_single_domain/run.sh index fa544391..50deb688 100755 --- a/test/tests/certs_single_domain/run.sh +++ b/test/tests/certs_single_domain/run.sh @@ -107,8 +107,7 @@ for hosts in "${letsencrypt_hosts[@]}"; do done docker stop "$container" &> /dev/null - docker exec "$le_container_name" rm -rf /etc/nginx/certs/le?.wtf* - docker exec "$le_container_name" rm -rf /etc/acme.sh/default/le?.wtf* + docker exec "$le_container_name" /app/cleanup_test_artifacts --default-cert i=$(( i + 1 )) done diff --git a/test/tests/default_cert/run.sh b/test/tests/default_cert/run.sh index b3356955..3a1c1454 100755 --- a/test/tests/default_cert/run.sh +++ b/test/tests/default_cert/run.sh @@ -57,7 +57,7 @@ done # the certificate or private key file are deleted for file in 'default.key' 'default.crt'; do old_default_cert_fingerprint="$(default_cert_fingerprint)" - docker exec "$le_container_name" rm -f /etc/nginx/certs/$file + docker exec "$le_container_name" /app/cleanup_test_artifacts --default-cert docker restart "$le_container_name" > /dev/null timeout="$(date +%s)" timeout="$((timeout + 60))" From 3d35839f3183b2c85086c3e9597e11a30f81ee18 Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Wed, 23 Dec 2020 10:39:04 +0100 Subject: [PATCH 09/10] CI/CD: Don't run tests for doc-only changes --- .github/workflows/test.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ade9e70b..01fbb58b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,6 +1,15 @@ -name: Test +name: Tests + +on: + push: + paths-ignore: + - 'docs/**' + - '*.md' + pull_request: + paths-ignore: + - 'docs/**' + - '*.md' -on: [push, pull_request] env: DOCKER_GEN_CONTAINER_NAME: nginx-proxy-gen From 3ac0f260e9eebfd5937c70f507c8fa85e311b93f Mon Sep 17 00:00:00 2001 From: Nicolas Duchon Date: Wed, 23 Dec 2020 19:15:11 +0100 Subject: [PATCH 10/10] CI/CD: Patch the companion's Dockerfile for tests --- .github/workflows/test.yml | 4 ++++ Dockerfile | 5 ++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01fbb58b..d46e5c1d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,6 +24,10 @@ jobs: steps: - name: Checkout Code uses: actions/checkout@v2 + - name: Patch the Dockerfile's docker-gen + run: | + sed -i 's#DOCKER_GEN_VERSION=.*$#DOCKER_GEN_VERSION=fix-current-container-id#g' Dockerfile + sed -i 's#go get github.com/jwilder/docker-gen#go get github.com/buchdag/docker-gen \&\& mv /go/src/github.com/buchdag /go/src/github.com/jwilder#g' Dockerfile - name: Build Image run: docker build -t "$IMAGE" . - name: Inspect Image diff --git a/Dockerfile b/Dockerfile index 0f6c35aa..74b06975 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM golang:1.15-alpine AS go-builder -ENV DOCKER_GEN_VERSION=fix-current-container-id +ENV DOCKER_GEN_VERSION=0.7.4 # Build docker-gen RUN apk add --no-cache --virtual .build-deps \ @@ -9,8 +9,7 @@ RUN apk add --no-cache --virtual .build-deps \ git \ make \ musl-dev \ - && go get github.com/buchdag/docker-gen \ - && mv /go/src/github.com/buchdag /go/src/github.com/jwilder \ + && go get github.com/jwilder/docker-gen \ && cd /go/src/github.com/jwilder/docker-gen \ && git -c advice.detachedHead=false checkout $DOCKER_GEN_VERSION \ && make get-deps \