From 5e9b9f156ee2e6a2a2bcc157ea9c97d9ee42fd4d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 12 Aug 2022 18:23:47 -0400 Subject: [PATCH 1/7] ci: make the docker script a bit more resilient Signed-off-by: William Woodruff --- ci/gha-docker-build | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/gha-docker-build b/ci/gha-docker-build index b92a8a6..fe97f0d 100755 --- a/ci/gha-docker-build +++ b/ci/gha-docker-build @@ -2,6 +2,8 @@ # gha-docker-build: build a MATE Docker image with some caching help +set -e + # This script only works in CI. if [[ -z "${GITHUB_ACTIONS}" ]]; then >&2 echo "Fatal: Only works in GitHub Actions." From 4967e8eab7ea44cc392d1f4d8346c11c3a38ea4f Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 12 Aug 2022 18:25:20 -0400 Subject: [PATCH 2/7] ci: remove more old files Signed-off-by: William Woodruff --- ci/README.md | 5 ----- ci/ssh.sh | 16 ---------------- 2 files changed, 21 deletions(-) delete mode 100644 ci/README.md delete mode 100644 ci/ssh.sh diff --git a/ci/README.md b/ci/README.md deleted file mode 100644 index 4ba9731..0000000 --- a/ci/README.md +++ /dev/null @@ -1,5 +0,0 @@ -All scripts in this folder are expected to be POSIX sh compliant. -The reasoning is that they may be executed on very minimal systems such as alpine containers or NixOS that only have `/bin/sh`. - -NB: This directory really should be named `scripts` and not `ci`, but the code churn required will have to wait for another day. -see: https://gitlab-ext.galois.com/mate/MATE/-/issues/1117 diff --git a/ci/ssh.sh b/ci/ssh.sh deleted file mode 100644 index 68c61a7..0000000 --- a/ci/ssh.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh - -# This script sets up an SSH agent with access to a key that the CI runner can -# use to clone submodules from private repos and push documentation to the -# webserver. - -set -eu - -eval "$(ssh-agent -s)" -echo "${STOCKFISH_PKEY}" | tr -d '\r' | ssh-add - -mkdir -p ~/.ssh -chmod 700 ~/.ssh -if [ -f /.dockerenv ]; then - printf '%s\n\t%s\n\n' 'Host *' 'StrictHostKeyChecking no' > ~/.ssh/config -fi -chmod 600 ~/.ssh/config From 49e7603d544b32f1a894299dc1c04d390a3e0843 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 12 Aug 2022 22:22:47 -0400 Subject: [PATCH 3/7] workflows/build: upload the bdist ...again Signed-off-by: William Woodruff --- .github/workflows/build.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9d8af63..731c0f6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,6 +48,16 @@ jobs: "ghcr.io/galoisinc/mate-dev:${ref}" \ ./shake.sh -j bdist + - name: Compress the MATE bdist + run: | + tar czf mate-bdist.tar.gz ./.out/bdist + + - name: Upload the MATE bdist + uses: actions/upload-artifact@v3 + with: + name: mate-bdist + path: mate-bdist.tar.gz + - name: Build and push the mate-dist image run: | # HACK: These files are owned by the container root, so we can't From 00c2ec20b31b68b4988b63ec2e22e9446ff3331a Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 15 Aug 2022 12:08:48 -0400 Subject: [PATCH 4/7] build, ci: try enabling the legacy tests Signed-off-by: William Woodruff --- .github/workflows/build.yml | 31 ++++++++++++++++++++----------- ci/gha-docker-build | 8 ++++++++ 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 731c0f6..cda8ad9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,19 +33,9 @@ jobs: - name: Build the MATE bdist run: | - # Extremely annoying: the `pull_request` event creates a merge commit, - # which means that `GITHUB_SHA` is completely useless to key off of. - # We have to do some additional sleuthing for these events to get the right - # commit. - if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then - ref="${ACTUAL_GITHUB_SHA_ON_PULL_REQUEST}" - else - ref="${GITHUB_SHA}" - fi - docker run \ --rm -v $(pwd):/mate \ - "ghcr.io/galoisinc/mate-dev:${ref}" \ + "ghcr.io/galoisinc/mate-dev:${GITHUB_RUN_ID}" \ ./shake.sh -j bdist - name: Compress the MATE bdist @@ -65,3 +55,22 @@ jobs: sudo chmod a+r -R .out/ ./ci/gha-docker-build "dist" "mate-dist" + + test-legacy: + needs: [build] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Pull mate-dev + run: | + docker pull "ghcr.io/galoisinc/mate-dev:${GITHUB_RUN_ID}" + + - name: Run legacy tests + run: | + # TODO: Figure out the right way to plumb the integration test setting. + docker run \ + -e MATE_INTEGRATION_TESTS=0 \ + --rm -v $(pwd):/mate -it \ + "ghcr.io/galoisinc/mate-dev:${GITHUB_RUN_ID}" \ + ./shake.sh --skip=build pytests -- -- -n logical -x diff --git a/ci/gha-docker-build b/ci/gha-docker-build index fe97f0d..a1cb444 100755 --- a/ci/gha-docker-build +++ b/ci/gha-docker-build @@ -52,6 +52,14 @@ docker build \ docker push "${full_image_name}:${ref}" +# Also tag it with the current workflow run ID, to make reuse in subsequent +# CI steps easier (no need to track the commit ref). +docker tag \ + "${full_image_name}:${ref}" \ + "${full_image_name}:${GITHUB_RUN_ID}" + +docker push "${full_image_name}:${GITHUB_RUN_ID}" + # Additionally, if we're on main or we're doing a manually triggered build, # tag the image as ":main". if [[ "${GITHUB_REF_NAME}" == "main" || "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then From a097ef661df77190cade050c41c894c10941344c Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 15 Aug 2022 12:35:11 -0400 Subject: [PATCH 5/7] workflows/build: fix docker login Signed-off-by: William Woodruff --- .github/workflows/build.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cda8ad9..8037f92 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -62,6 +62,13 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Login to Packages Container registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Pull mate-dev run: | docker pull "ghcr.io/galoisinc/mate-dev:${GITHUB_RUN_ID}" From 87c2eb14197a7c0cc4dd06254bac2520a15c412d Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 15 Aug 2022 13:03:35 -0400 Subject: [PATCH 6/7] workflows/build: tweak Signed-off-by: William Woodruff --- .github/workflows/build.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8037f92..81bc4fb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,11 +73,20 @@ jobs: run: | docker pull "ghcr.io/galoisinc/mate-dev:${GITHUB_RUN_ID}" + # - name: Fetch the MATE bdist + # uses: actions/download-artifact@v3 + # with: + # name: mate-bdist + + # - name: Extract the MATE bdist + # run: | + # tar xzf mate-dist.tar.gz + - name: Run legacy tests run: | # TODO: Figure out the right way to plumb the integration test setting. docker run \ -e MATE_INTEGRATION_TESTS=0 \ - --rm -v $(pwd):/mate -it \ + --rm -v $(pwd):/mate \ "ghcr.io/galoisinc/mate-dev:${GITHUB_RUN_ID}" \ ./shake.sh --skip=build pytests -- -- -n logical -x From 8f8ac76162958c4f2413f344a8248d87def19cf8 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 15 Aug 2022 13:29:12 -0400 Subject: [PATCH 7/7] workflows/build: fix clone Signed-off-by: William Woodruff --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 81bc4fb..c550d78 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,6 +61,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + with: + submodules: recursive - name: Login to Packages Container registry uses: docker/login-action@v2