From 677bc49f67b1ff0d8b3cdb82e10047715a85c1c2 Mon Sep 17 00:00:00 2001 From: Simon Zhu Date: Mon, 21 Oct 2024 17:24:46 -0400 Subject: [PATCH 1/4] init --- .evergreen/functions.yml | 21 +++++++++++ .../Dockerfile | 11 ++++++ .../build-e2e-docker-image.sh | 35 +++++++++++++++++++ 3 files changed, 67 insertions(+) create mode 100644 .evergreen/package-and-publish-e2e-docker-image/Dockerfile create mode 100644 .evergreen/package-and-publish-e2e-docker-image/build-e2e-docker-image.sh diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index acf5b475190..217f26d0908 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -132,6 +132,27 @@ post: params: file: src/.logs/*.xml functions: + build-e2e-docker-image: + command: shell.exec + type: setup + params: + shell: bash + working_dir: mms + include_expansions_in_env: [BAZELISK_HOME, AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN] + env: + XDG_RUNTIME_DIR: ${workdir}/docker/run + DOCKER_CONFIG: ${workdir}/mms/.docker/mms + script: | + set -exu + export DOCKER_HOST="unix://$XDG_RUNTIME_DIR/docker.sock" + PATH="${workdir}/bin:$PATH" bazel build ${bazel_args} ${bazel_target} + + package-and-publish-e2e-docker-image: + - *skip-if-patch-build + - *build-e2e-docker-image + - *check-if-e2e-docker-image-is-already-published + - *publish-e2e-docker-image + clone: - command: git.get_project type: system diff --git a/.evergreen/package-and-publish-e2e-docker-image/Dockerfile b/.evergreen/package-and-publish-e2e-docker-image/Dockerfile new file mode 100644 index 00000000000..dfcb8f79d03 --- /dev/null +++ b/.evergreen/package-and-publish-e2e-docker-image/Dockerfile @@ -0,0 +1,11 @@ +# "bullseye" is the debian distribution that ubuntu:20.04 is based on +ARG NODE_JS_VERSION=20 +FROM node:${NODE_JS_VERSION}-bullseye + +COPY . /compass-monorepo-root +WORKDIR /compass-monorepo-root + +RUN npm i -g npm@10.2.4 +RUN npm run bootstrap-ci + +CMD ["npm run test-web"] \ No newline at end of file diff --git a/.evergreen/package-and-publish-e2e-docker-image/build-e2e-docker-image.sh b/.evergreen/package-and-publish-e2e-docker-image/build-e2e-docker-image.sh new file mode 100644 index 00000000000..ec264dd5f87 --- /dev/null +++ b/.evergreen/package-and-publish-e2e-docker-image/build-e2e-docker-image.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +post_evergreen_status() { + status="$1" + type="$2" + desc="$3" + should_continue="$4" + curl --silent --show-error --header "Content-Type: application/json" --data \ + '{"status": "'"$status"'", "type": "'"$type"'", "desc": "'"$desc"'", "should_continue": '"$should_continue"'}' \ + "http://localhost:2285/task_status" || true +} + +# NOTE: do not -x here there are "env" vars in the commands +set -e + +MONOREPO_ROOT_DIR="$(cd $(dirname "$0")/../..; pwd)" +cd $MONOREPO_ROOT_DIR + +echo "building e2e tests image from ${PWD}" +docker build -t compass-e2e-tests:$github_commit --build-arg "NODE_JS_VERSION=$NODE_JS_VERSION" -f "./.evergreen/package-and-publish-e2e-docker-image/Dockerfile" . +echo "e2e tests image built" + +# skip if the image already exists +if aws ecr describe-images --region $ECR_REGION --repository-name=compass/e2e-tests --image-ids=imageTag=$github_commit; then + echo "Image with tag '${github_commit}' found in the 'compass/e2e-tests' repository. Setting task status to 'success'..." + post_evergreen_status success test "image tag '${github_commit}' already exists in the 'compass/e2e-tests' repository" false +else + echo "Image with tag '${github_commit}' not found in the 'compass/e2e-tests' repository. Continuing with remainder of task..." +fi + +echo "pushing e2e tests image to ECR" +aws ecr get-login-password --region $ECR_REGION | docker login --username AWS --password-stdin $ECR_REGISTRY +docker tag compass-e2e-tests:$github_commit $ECR_REGISTRY/compass/e2e-tests:$github_commit +docker push $ECR_REGISTRY/compass/e2e-tests:$github_commit +echo "image pushed" \ No newline at end of file From 3f4d89b4c67011cc07f9a52ca9f28f39ec6c3207 Mon Sep 17 00:00:00 2001 From: Simon Zhu Date: Mon, 21 Oct 2024 18:27:15 -0400 Subject: [PATCH 2/4] create tasks --- .evergreen/buildvariants-and-tasks.in.yml | 12 +++++++ .evergreen/functions.yml | 23 +++++------- .../Dockerfile | 11 ------ .../build-e2e-docker-image.sh | 35 ------------------- 4 files changed, 20 insertions(+), 61 deletions(-) delete mode 100644 .evergreen/package-and-publish-e2e-docker-image/Dockerfile delete mode 100644 .evergreen/package-and-publish-e2e-docker-image/build-e2e-docker-image.sh diff --git a/.evergreen/buildvariants-and-tasks.in.yml b/.evergreen/buildvariants-and-tasks.in.yml index a0ae901cefd..c3a19e32862 100644 --- a/.evergreen/buildvariants-and-tasks.in.yml +++ b/.evergreen/buildvariants-and-tasks.in.yml @@ -259,6 +259,12 @@ buildvariants: - name: e2e-coverage-<%= group.number %> <% } %> + - name: publish-e2e-tests + display_name: Publish E2E Tests Docker Image + run_on: ubuntu2004-large + tasks: + - name: publish-e2e-tests + - name: csfle-tests display_name: CSFLE Tests run_on: ubuntu2004-large @@ -528,3 +534,9 @@ for (const compassDistribution of COMPASS_DISTRIBUTIONS) { - func: create_static_analysis_report vars: first_party_dependency_filenames: <%= firstPartyDepsFilenames.map(f => '.sbom/' + f).join(',') %> + + - name: publish-e2e-tests + tags: ['run-on-pr'] + commands: + - func: clone + - func: package-and-publish-e2e-docker-image diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index 217f26d0908..c9968cfa4df 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -132,26 +132,19 @@ post: params: file: src/.logs/*.xml functions: - build-e2e-docker-image: + package-and-publish-e2e-docker-image: command: shell.exec - type: setup params: shell: bash - working_dir: mms - include_expansions_in_env: [BAZELISK_HOME, AWS_SECRET_ACCESS_KEY, AWS_ACCESS_KEY_ID, AWS_SESSION_TOKEN] env: - XDG_RUNTIME_DIR: ${workdir}/docker/run - DOCKER_CONFIG: ${workdir}/mms/.docker/mms + <<: *compass-env + AWS_ACCESS_KEY_ID: ${aws_key} + AWS_SECRET_ACCESS_KEY: ${aws_secret} + ECR_REGISTRY: ${ecr_registry} + include_expansions_in_env: [github_commit, ECR_REGION] script: | - set -exu - export DOCKER_HOST="unix://$XDG_RUNTIME_DIR/docker.sock" - PATH="${workdir}/bin:$PATH" bazel build ${bazel_args} ${bazel_target} - - package-and-publish-e2e-docker-image: - - *skip-if-patch-build - - *build-e2e-docker-image - - *check-if-e2e-docker-image-is-already-published - - *publish-e2e-docker-image + set -e + bash .evergreen/e2e-tests/package-and-publish-docker-image.sh clone: - command: git.get_project diff --git a/.evergreen/package-and-publish-e2e-docker-image/Dockerfile b/.evergreen/package-and-publish-e2e-docker-image/Dockerfile deleted file mode 100644 index dfcb8f79d03..00000000000 --- a/.evergreen/package-and-publish-e2e-docker-image/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# "bullseye" is the debian distribution that ubuntu:20.04 is based on -ARG NODE_JS_VERSION=20 -FROM node:${NODE_JS_VERSION}-bullseye - -COPY . /compass-monorepo-root -WORKDIR /compass-monorepo-root - -RUN npm i -g npm@10.2.4 -RUN npm run bootstrap-ci - -CMD ["npm run test-web"] \ No newline at end of file diff --git a/.evergreen/package-and-publish-e2e-docker-image/build-e2e-docker-image.sh b/.evergreen/package-and-publish-e2e-docker-image/build-e2e-docker-image.sh deleted file mode 100644 index ec264dd5f87..00000000000 --- a/.evergreen/package-and-publish-e2e-docker-image/build-e2e-docker-image.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -post_evergreen_status() { - status="$1" - type="$2" - desc="$3" - should_continue="$4" - curl --silent --show-error --header "Content-Type: application/json" --data \ - '{"status": "'"$status"'", "type": "'"$type"'", "desc": "'"$desc"'", "should_continue": '"$should_continue"'}' \ - "http://localhost:2285/task_status" || true -} - -# NOTE: do not -x here there are "env" vars in the commands -set -e - -MONOREPO_ROOT_DIR="$(cd $(dirname "$0")/../..; pwd)" -cd $MONOREPO_ROOT_DIR - -echo "building e2e tests image from ${PWD}" -docker build -t compass-e2e-tests:$github_commit --build-arg "NODE_JS_VERSION=$NODE_JS_VERSION" -f "./.evergreen/package-and-publish-e2e-docker-image/Dockerfile" . -echo "e2e tests image built" - -# skip if the image already exists -if aws ecr describe-images --region $ECR_REGION --repository-name=compass/e2e-tests --image-ids=imageTag=$github_commit; then - echo "Image with tag '${github_commit}' found in the 'compass/e2e-tests' repository. Setting task status to 'success'..." - post_evergreen_status success test "image tag '${github_commit}' already exists in the 'compass/e2e-tests' repository" false -else - echo "Image with tag '${github_commit}' not found in the 'compass/e2e-tests' repository. Continuing with remainder of task..." -fi - -echo "pushing e2e tests image to ECR" -aws ecr get-login-password --region $ECR_REGION | docker login --username AWS --password-stdin $ECR_REGISTRY -docker tag compass-e2e-tests:$github_commit $ECR_REGISTRY/compass/e2e-tests:$github_commit -docker push $ECR_REGISTRY/compass/e2e-tests:$github_commit -echo "image pushed" \ No newline at end of file From 228772bcd88694cacd49edaff772f1f0e49437e7 Mon Sep 17 00:00:00 2001 From: Simon Zhu Date: Mon, 21 Oct 2024 18:27:43 -0400 Subject: [PATCH 3/4] add docerfile --- .evergreen/e2e-tests/Dockerfile | 11 ++++++ .../package-and-publish-docker-image.sh | 35 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 .evergreen/e2e-tests/Dockerfile create mode 100644 .evergreen/e2e-tests/package-and-publish-docker-image.sh diff --git a/.evergreen/e2e-tests/Dockerfile b/.evergreen/e2e-tests/Dockerfile new file mode 100644 index 00000000000..dfcb8f79d03 --- /dev/null +++ b/.evergreen/e2e-tests/Dockerfile @@ -0,0 +1,11 @@ +# "bullseye" is the debian distribution that ubuntu:20.04 is based on +ARG NODE_JS_VERSION=20 +FROM node:${NODE_JS_VERSION}-bullseye + +COPY . /compass-monorepo-root +WORKDIR /compass-monorepo-root + +RUN npm i -g npm@10.2.4 +RUN npm run bootstrap-ci + +CMD ["npm run test-web"] \ No newline at end of file diff --git a/.evergreen/e2e-tests/package-and-publish-docker-image.sh b/.evergreen/e2e-tests/package-and-publish-docker-image.sh new file mode 100644 index 00000000000..ec264dd5f87 --- /dev/null +++ b/.evergreen/e2e-tests/package-and-publish-docker-image.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +post_evergreen_status() { + status="$1" + type="$2" + desc="$3" + should_continue="$4" + curl --silent --show-error --header "Content-Type: application/json" --data \ + '{"status": "'"$status"'", "type": "'"$type"'", "desc": "'"$desc"'", "should_continue": '"$should_continue"'}' \ + "http://localhost:2285/task_status" || true +} + +# NOTE: do not -x here there are "env" vars in the commands +set -e + +MONOREPO_ROOT_DIR="$(cd $(dirname "$0")/../..; pwd)" +cd $MONOREPO_ROOT_DIR + +echo "building e2e tests image from ${PWD}" +docker build -t compass-e2e-tests:$github_commit --build-arg "NODE_JS_VERSION=$NODE_JS_VERSION" -f "./.evergreen/package-and-publish-e2e-docker-image/Dockerfile" . +echo "e2e tests image built" + +# skip if the image already exists +if aws ecr describe-images --region $ECR_REGION --repository-name=compass/e2e-tests --image-ids=imageTag=$github_commit; then + echo "Image with tag '${github_commit}' found in the 'compass/e2e-tests' repository. Setting task status to 'success'..." + post_evergreen_status success test "image tag '${github_commit}' already exists in the 'compass/e2e-tests' repository" false +else + echo "Image with tag '${github_commit}' not found in the 'compass/e2e-tests' repository. Continuing with remainder of task..." +fi + +echo "pushing e2e tests image to ECR" +aws ecr get-login-password --region $ECR_REGION | docker login --username AWS --password-stdin $ECR_REGISTRY +docker tag compass-e2e-tests:$github_commit $ECR_REGISTRY/compass/e2e-tests:$github_commit +docker push $ECR_REGISTRY/compass/e2e-tests:$github_commit +echo "image pushed" \ No newline at end of file From 5362ef37a23192a73f59e9dc86366d34271f73b5 Mon Sep 17 00:00:00 2001 From: Simon Zhu Date: Mon, 21 Oct 2024 18:34:08 -0400 Subject: [PATCH 4/4] render evg config --- .evergreen/buildvariants-and-tasks.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.evergreen/buildvariants-and-tasks.yml b/.evergreen/buildvariants-and-tasks.yml index 12e0c34f457..e99273e2c12 100644 --- a/.evergreen/buildvariants-and-tasks.yml +++ b/.evergreen/buildvariants-and-tasks.yml @@ -242,6 +242,11 @@ buildvariants: - name: e2e-coverage-1 - name: e2e-coverage-2 - name: e2e-coverage-3 + - name: publish-e2e-tests + display_name: Publish E2E Tests Docker Image + run_on: ubuntu2004-large + tasks: + - name: publish-e2e-tests - name: csfle-tests display_name: CSFLE Tests run_on: ubuntu2004-large @@ -1774,3 +1779,9 @@ tasks: vars: first_party_dependency_filenames: >- .sbom/first-party-deps-compass-package-ubuntu.json,.sbom/first-party-deps-compass-package-windows.json,.sbom/first-party-deps-compass-package-rhel.json,.sbom/first-party-deps-compass-package-macos-x64.json,.sbom/first-party-deps-compass-package-macos-arm.json,.sbom/first-party-deps-compass-isolated-package-ubuntu.json,.sbom/first-party-deps-compass-isolated-package-windows.json,.sbom/first-party-deps-compass-isolated-package-rhel.json,.sbom/first-party-deps-compass-isolated-package-macos-x64.json,.sbom/first-party-deps-compass-isolated-package-macos-arm.json,.sbom/first-party-deps-compass-readonly-package-ubuntu.json,.sbom/first-party-deps-compass-readonly-package-windows.json,.sbom/first-party-deps-compass-readonly-package-rhel.json,.sbom/first-party-deps-compass-readonly-package-macos-x64.json,.sbom/first-party-deps-compass-readonly-package-macos-arm.json + - name: publish-e2e-tests + tags: + - run-on-pr + commands: + - func: clone + - func: package-and-publish-e2e-docker-image