From 04097fcfec77519ba60f764c50a724593b3361c9 Mon Sep 17 00:00:00 2001 From: Justin H Yang Date: Mon, 12 Aug 2024 11:39:38 -0700 Subject: [PATCH 1/3] copypasta from terra-github-workflows --- .github/workflows/rawls-swat-tests.yaml | 397 ++++++++++++++++++++++++ 1 file changed, 397 insertions(+) create mode 100644 .github/workflows/rawls-swat-tests.yaml diff --git a/.github/workflows/rawls-swat-tests.yaml b/.github/workflows/rawls-swat-tests.yaml new file mode 100644 index 0000000000..f129bf4142 --- /dev/null +++ b/.github/workflows/rawls-swat-tests.yaml @@ -0,0 +1,397 @@ +name: rawls-swat-tests +run-name: ${{ inputs.run-name }} +on: + workflow_dispatch: + inputs: + additional-args: + description: JSON of additional options + type: string + default: "{'logging':'true','java-version':'17','billing-project':''}" + required: false + run-name: + description: 'Specify the run name to fetch workflow run ID based on run-name' + required: false + type: string + bee-name: + description: 'hostname of the BEE to run tests against' + required: true + type: string + ENV: + description: 'test environment' + required: true + type: choice + options: + - dev + - alpha + - staging + - qa + ref: + description: 'the branch, tag or SHA to checkout' + required: false + type: string + test-group-name: + description: 'Rawls test group name' + required: false + type: string + default: '' + test-context: + description: "Name of the texting context" + required: true + type: choice + default: "pr-test" + options: + - pr-test + - dev-merge + - manual + test-command: + description: 'sbt test command' + required: true + type: string + e2e-env: + description: 'The name of a .env file that contains envvars for E2E tests.' + required: false + default: '' + user-subjects: + description: 'An e2e test users map in JSON format. This map includes pre-seeded users in BEE, and it also contains the definition of the service account used for impersonation within the user-subjects.' + required: false + default: '{"service_account":"firecloud-qa@broad-dsde-qa.iam.gserviceaccount.com","owners":["hermione.owner@quality.firecloud.org"],"students":["harry.potter@quality.firecloud.org"]}' + type: string + +env: + SERVICE_REPO: rawls + test_run_id: ${{ github.run_id }} + +jobs: + # We use a plural form here to indicate we may modify this job to support multiple owners in the future. + obtain-owner-tokens: + strategy: + matrix: + owner_subject: ${{ fromJson(inputs['user-subjects']).owners }} + svc_account: [ "${{ fromJson(inputs['user-subjects']).service_account }}" ] + + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + + steps: + - uses: actions/checkout@v4 + + - name: 'Obtain OAuth2 2.0 Access Token' + id: 'obtain-owner-token' + uses: google-github-actions/auth@v2 + with: + token_format: 'access_token' + workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider' + service_account: ${{ matrix.svc_account }} + access_token_scopes: 'profile, email, openid' + access_token_subject: ${{ matrix.owner_subject }} + export_environment_variables: false + + - name: 'Set up owner metadata: email, type, and auth token as a JSON string' + id: 'setup-owner-metadata' + run: | + echo "::add-mask::${{ steps.obtain-owner-token.outputs.access_token }}" + echo "{\"email\":\"${{ matrix.owner_subject }}\",\"type\":\"owner\",\"bearer\":\"${{ steps.obtain-owner-token.outputs.access_token }}\"}" > "${{ matrix.owner_subject }}.json" + + - name: 'Cache owner metadata to share with other jobs' + id: 'cache-owner-metadata' + uses: actions/upload-artifact@v3 + with: + name: users-metadata-cache-${{ inputs.run-name }} + path: | + ${{ matrix.owner_subject }}.json + retention-days: 1 + + # This job obtains access tokens for the rest of the user subjects that do not belong to owner type + obtain-tokens: + strategy: + matrix: + user_subject: ${{ fromJson(inputs['user-subjects']).students }} + svc_account: [ "${{ fromJson(inputs['user-subjects']).service_account }}" ] + + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + + steps: + - uses: actions/checkout@v4 + + - name: 'Obtain OAuth2 2.0 Access Token' + id: 'obtain-token' + uses: google-github-actions/auth@v2 + with: + token_format: 'access_token' + workload_identity_provider: 'projects/1038484894585/locations/global/workloadIdentityPools/github-wi-pool/providers/github-wi-provider' + service_account: ${{ matrix.svc_account }} + access_token_scopes: 'profile, email, openid' + access_token_subject: ${{ matrix.user_subject }} + export_environment_variables: false + + - name: 'Set up user metadata: email, type, and auth token as a JSON string' + id: 'setup-user-metadata' + run: | + echo "::add-mask::${{ steps.obtain-token.outputs.access_token }}" + echo "{\"email\":\"${{ matrix.user_subject }}\",\"type\":\"student\",\"bearer\":\"${{ steps.obtain-token.outputs.access_token }}\"}" > "${{ matrix.user_subject }}.json" + + - name: 'Cache user metadata to share with other jobs' + id: 'cache-user-metadata' + uses: actions/upload-artifact@v3 + with: + name: users-metadata-cache-${{ inputs.run-name }} + path: | + ${{ matrix.user_subject }}.json + retention-days: 1 + + # Serialize all users metadata into B64-encoded JSON string + serialize-users-metadata: + runs-on: ubuntu-latest + needs: [obtain-owner-tokens, obtain-tokens] + + steps: + - name: 'Pull users metadata from users-metadata-cache' + uses: actions/download-artifact@v3 + with: + name: users-metadata-cache-${{ inputs.run-name }} + + - name: 'Concatenate users metadata as a JSON string' + id: 'concatenate-users-metadata' + run: | + users_metadata_json="" + owners=$(echo '${{ inputs['user-subjects'] }}' | jq -r '.owners[]') + if [ -n "$owners" ]; then + for owner in $owners; do + owner_metadata=$(cat "${owner}.json") + if [ -n "$users_metadata_json" ]; then + users_metadata_json="${users_metadata_json}," + fi + users_metadata_json="${users_metadata_json}${owner_metadata}" + done + fi + subjects=$(echo '${{ inputs['user-subjects'] }}' | jq -r '.students[]') + if [ -n "$subjects" ]; then + for subject in $subjects; do + subject_metadata=$(cat "${subject}.json") + if [ -n "$users_metadata_json" ]; then + users_metadata_json="${users_metadata_json}," + fi + users_metadata_json="${users_metadata_json}${subject_metadata}" + done + fi + users_metadata_json="[${users_metadata_json}]" + users_metadata_json_b64=$(printf '%s' $users_metadata_json | base64 -w0) + echo "$users_metadata_json_b64" > users_metadata_json.b64 + + - name: 'Cache Users Metadata JSON (B64-encoded). This step is required in order to use add-mask to prevent leakage of secrets in subsequent jobs.' + id: 'cache-users-metadata-b64' + uses: actions/upload-artifact@v3 + with: + name: users-metadata-cache-${{ inputs.run-name }} + path: | + users_metadata_json.b64 + retention-days: 1 + + # This job will run Rawls swat tests against BEE_NAME. + rawls-swat-test-job: + runs-on: ubuntu-latest + needs: [serialize-users-metadata] + permissions: + contents: 'read' + id-token: 'write' + checks: 'write' + + steps: + - uses: 'actions/checkout@v4' + + - name: 'Pull users metadata from users-metadata-cache' + uses: actions/download-artifact@v3 + with: + name: users-metadata-cache-${{ inputs.run-name }} + + - name: Import Vault Secrets + uses: hashicorp/vault-action@v3 + with: + url: https://clotho.broadinstitute.org:8200 + method: approle + roleId: ${{ secrets.VAULT_ROLE_ID }} + secretId: ${{ secrets.VAULT_SECRET_ID }} + tlsSkipVerify: true + exportToken: true + secrets: | + secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json auth_provider_x509_cert_url | FIRECLOUD_ACCOUNT_AUTH_CERT_URL; + secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json auth_uri | FIRECLOUD_ACCOUNT_AUTH_URI; + secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json client_email | FIRECLOUD_ACCOUNT_CLIENT_EMAIL; + secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json client_id | FIRECLOUD_ACCOUNT_CLIENT_ID; + secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json client_x509_cert_url | FIRECLOUD_ACCOUNT_CLIENT_CERT_URL; + secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json private_key | FIRECLOUD_ACCOUNT_PVT_KEY; + secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json private_key_id | FIRECLOUD_ACCOUNT_PVT_KEY_ID; + secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json project_id | FIRECLOUD_ACCOUNT_PROJECT_ID; + secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json token_uri | FIRECLOUD_ACCOUNT_TOKEN_URI; + secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json type | FIRECLOUD_ACCOUNT_TYPE; + secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json auth_provider_x509_cert_url | RAWLS_ACCOUNT_AUTH_CERT_URL; + secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json auth_uri | RAWLS_ACCOUNT_AUTH_URI; + secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json client_email | RAWLS_ACCOUNT_CLIENT_EMAIL; + secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json client_id | RAWLS_ACCOUNT_CLIENT_ID; + secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json client_x509_cert_url | RAWLS_ACCOUNT_CLIENT_CERT_URL; + secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json private_key | RAWLS_ACCOUNT_PVT_KEY; + secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json private_key_id | RAWLS_ACCOUNT_PVT_KEY_ID; + secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json project_id | RAWLS_ACCOUNT_PROJECT_ID; + secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json token_uri | RAWLS_ACCOUNT_TOKEN_URI; + secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json type | RAWLS_ACCOUNT_TYPE; + secret/dsde/firecloud/${{ inputs.ENV }}/common/secrets firecloud_id | FIRECLOUD_ID; + secret/dsde/firecloud/${{ inputs.ENV }}/common/users automation_users_passwd | AUTOMATION_USERS_PASSWD; + secret/dsde/firecloud/${{ inputs.ENV }}/common/users billing_acct | BILLING_ACCT; + secret/dsde/firecloud/${{ inputs.ENV }}/common/users qa_users_passwd | QA_USERS_PASSWD; + secret/dsde/firecloud/${{ inputs.ENV }}/common/users service_acct_email | SERVICE_ACCT_EMAIL; + secret/dsde/firecloud/${{ inputs.ENV }}/common/users users | USERS; + secret/dsde/firecloud/${{ inputs.ENV }}/common/users users_passwd | USERS_PASSWD; + secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json auth_provider_x509_cert_url | SAM_FIRESTORE_ACCOUNT_AUTH_CERT_URL; + secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json auth_uri | SAM_FIRESTORE_ACCOUNT_AUTH_URI; + secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json client_email | SAM_FIRESTORE_ACCOUNT_CLIENT_EMAIL; + secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json client_id | SAM_FIRESTORE_ACCOUNT_CLIENT_ID; + secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json client_x509_cert_url | SAM_FIRESTORE_ACCOUNT_CLIENT_CERT_URL; + secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json private_key | SAM_FIRESTORE_ACCOUNT_PVT_KEY; + secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json private_key_id | SAM_FIRESTORE_ACCOUNT_PVT_KEY_ID; + secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json project_id | SAM_FIRESTORE_ACCOUNT_PROJECT_ID; + secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json token_uri | SAM_FIRESTORE_ACCOUNT_TOKEN_URI; + secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json type | SAM_FIRESTORE_ACCOUNT_TYPE; + + - id: prepare-secrets + name: Prepare secrets + run: | + echo firecloud-json='{"auth_provider_x509_cert_url":"${{ env.FIRECLOUD_ACCOUNT_AUTH_CERT_URL }}","auth_uri":"${{ env.FIRECLOUD_ACCOUNT_AUTH_URI }}","client_email":"${{ env.FIRECLOUD_ACCOUNT_CLIENT_EMAIL }}","client_id":"${{ env.FIRECLOUD_ACCOUNT_CLIENT_ID }}","client_x509_cert_url":"${{ env.FIRECLOUD_ACCOUNT_CLIENT_CERT_URL }}","private_key":${{ toJSON(env.FIRECLOUD_ACCOUNT_PVT_KEY) }},"private_key_id":"${{ env.FIRECLOUD_ACCOUNT_PVT_KEY_ID }}","project_id":"${{ env.FIRECLOUD_ACCOUNT_PROJECT_ID }}","token_uri":"${{ env.FIRECLOUD_ACCOUNT_TOKEN_URI }}","type":"${{ env.FIRECLOUD_ACCOUNT_TYPE }}"}' >> $GITHUB_OUTPUT + echo rawls-json='{"auth_provider_x509_cert_url":"${{ env.RAWLS_ACCOUNT_AUTH_CERT_URL }}","auth_uri":"${{ env.RAWLS_ACCOUNT_AUTH_URI }}","client_email":"${{ env.RAWLS_ACCOUNT_CLIENT_EMAIL }}","client_id":"${{ env.RAWLS_ACCOUNT_CLIENT_ID }}","client_x509_cert_url":"${{ env.RAWLS_ACCOUNT_CLIENT_CERT_URL }}","private_key":${{ toJSON(env.RAWLS_ACCOUNT_PVT_KEY) }},"private_key_id":"${{ env.RAWLS_ACCOUNT_PVT_KEY_ID }}","project_id":"${{ env.RAWLS_ACCOUNT_PROJECT_ID }}","token_uri":"${{ env.RAWLS_ACCOUNT_TOKEN_URI }}","type":"${{ env.RAWLS_ACCOUNT_TYPE }}"}' >> $GITHUB_OUTPUT + echo firecloud-secrets='{"firecloud_id":"${{ env.FIRECLOUD_ID }}"}' >> $GITHUB_OUTPUT + echo firecloud-users='{"automation_users_passwd":"${{ env.AUTOMATION_USERS_PASSWD }}","billing_acct":"${{ env.BILLING_ACCT }}","qa_users_passwd":"${{ env.QA_USERS_PASSWD }}","service_acct_email":"${{ env.SERVICE_ACCT_EMAIL }}","users":${{ toJSON(env.USERS) }},"users_passwd":"${{ env.USERS_PASSWD }}"}' >> $GITHUB_OUTPUT + echo sam-firestore-json='{"auth_provider_x509_cert_url":"${{ env.SAM_FIRESTORE_ACCOUNT_AUTH_CERT_URL }}","auth_uri":"${{ env.SAM_FIRESTORE_ACCOUNT_AUTH_URI }}","client_email":"${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_EMAIL }}","client_id":"${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_ID }}","client_x509_cert_url":"${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_CERT_URL }}","private_key":${{ toJSON(env.SAM_FIRESTORE_ACCOUNT_PVT_KEY) }},"private_key_id":"${{ env.SAM_FIRESTORE_ACCOUNT_PVT_KEY_ID }}","project_id":"${{ env.SAM_FIRESTORE_ACCOUNT_PROJECT_ID }}","token_uri":"${{ env.SAM_FIRESTORE_ACCOUNT_TOKEN_URI }}","type":"${{ env.SAM_FIRESTORE_ACCOUNT_TYPE }}"}' >> $GITHUB_OUTPUT + echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_AUTH_CERT_URL }}" + echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_AUTH_URI }}" + echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_AUTH_CERT_URL }}" + echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_AUTH_URI }}" + echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_CLIENT_EMAIL }}" + echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_CLIENT_ID }}" + echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_CLIENT_CERT_URL }}" + echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_PVT_KEY }}" + echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_PVT_KEY_ID }}" + echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_PROJECT_ID }}" + echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_TOKEN_URI }}" + echo "::add-mask::${{ env.RAWLS_ACCOUNT_AUTH_CERT_URL }}" + echo "::add-mask::${{ env.RAWLS_ACCOUNT_AUTH_URI }}" + echo "::add-mask::${{ env.RAWLS_ACCOUNT_CLIENT_EMAIL }}" + echo "::add-mask::${{ env.RAWLS_ACCOUNT_CLIENT_ID }}" + echo "::add-mask::${{ env.RAWLS_ACCOUNT_CLIENT_CERT_URL }}" + echo "::add-mask::${{ env.RAWLS_ACCOUNT_PVT_KEY }}" + echo "::add-mask::${{ env.RAWLS_ACCOUNT_PVT_KEY_ID }}" + echo "::add-mask::${{ env.RAWLS_ACCOUNT_PROJECT_ID }}" + echo "::add-mask::${{ env.RAWLS_ACCOUNT_TOKEN_URI }}" + echo "::add-mask::${{ env.RAWLS_ACCOUNT_TYPE }}" + echo "::add-mask::${{ env.FIRECLOUD_ID }}" + echo "::add-mask::${{ env.AUTOMATION_USERS_PASSWD }}" + echo "::add-mask::${{ env.BILLING_ACCT }}" + echo "::add-mask::${{ env.QA_USERS_PASSWD }}" + echo "::add-mask::${{ env.SERVICE_ACCT_EMAIL }}" + echo "::add-mask::${{ env.USERS }}" + echo "::add-mask::${{ env.USERS_PASSWD }}" + echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_AUTH_CERT_URL }}" + echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_AUTH_URI }}" + echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_EMAIL }}" + echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_ID }}" + echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_CERT_URL }}" + echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_PVT_KEY }}" + echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_PVT_KEY_ID }}" + echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_PROJECT_ID }}" + echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_TOKEN_URI }}" + echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_TYPE }}" + + - id: render-secrets + name: Render secrets + run: | + FIRECLOUD_JSON_KEY_B64=$(printf '%s' ${{ toJSON(steps.prepare-secrets.outputs.firecloud-json) }} | base64 -w0) + RAWLS_JSON_KEY_B64=$(printf '%s' ${{ toJSON(steps.prepare-secrets.outputs.rawls-json) }} | base64 -w0) + FIRECLOUD_SECRETS_B64=$(printf '%s' ${{ toJSON(steps.prepare-secrets.outputs.firecloud-secrets) }} | base64 -w0) + FIRECLOUD_USERS_B64=$(printf '%s' ${{ toJSON(steps.prepare-secrets.outputs.firecloud-users) }} | base64 -w0) + SAM_FIRESTORE_JSON_KEY_B64=$(printf '%s' ${{ toJSON(steps.prepare-secrets.outputs.sam-firestore-json) }} | base64 -w0) + + echo "::add-mask::$FIRECLOUD_JSON_KEY_B64" + echo firecloud-json-key-b64=$FIRECLOUD_JSON_KEY_B64 >> $GITHUB_OUTPUT + echo "::add-mask::$RAWLS_JSON_KEY_B64" + echo rawls-json-key-b64=$RAWLS_JSON_KEY_B64 >> $GITHUB_OUTPUT + echo "::add-mask::$FIRECLOUD_SECRETS_B64" + echo firecloud-secrets-b64=$FIRECLOUD_SECRETS_B64 >> $GITHUB_OUTPUT + echo "::add-mask::$FIRECLOUD_USERS_B64" + echo firecloud-users-b64=$FIRECLOUD_USERS_B64 >> $GITHUB_OUTPUT + echo "::add-mask::$SAM_FIRESTORE_JSON_KEY_B64" + echo sam-firestore-json-key-b64=$SAM_FIRESTORE_JSON_KEY_B64 >> $GITHUB_OUTPUT + + - id: render-secrets-v2 + name: Render secrets v2 (eventually replacing render-secrets and getting rid of reliance on sensitive SA keys) + run: | + users_metadata_json_b64="" + if [ -e "users_metadata_json.b64" ]; then + users_metadata_json_b64=$(cat users_metadata_json.b64) + if [ -n "$users_metadata_json_b64" ]; then + echo "::add-mask::$users_metadata_json_b64" + fi + echo $users_metadata_json_b64 + fi + echo "users-metadata-json-b64=$users_metadata_json_b64" >> $GITHUB_OUTPUT + + - id: run-rawls-swat-tests + name: 'Run Rawls Swat tests' + uses: ./.github/actions/run-swat-tests + with: + service_repo: ${{ env.SERVICE_REPO }} + ref: ${{ inputs.ref }} + bee_name: ${{ inputs.bee-name }} + broadbot_token: ${{ secrets.BROADBOT_TOKEN }} + firecloud-json-key-b64: ${{ steps.render-secrets.outputs.firecloud-json-key-b64 }} + firecloud-secrets-b64: ${{ steps.render-secrets.outputs.firecloud-secrets-b64 }} + firecloud-users-b64: ${{ steps.render-secrets.outputs.firecloud-users-b64 }} + rawls-json-key-b64: ${{ steps.render-secrets.outputs.rawls-json-key-b64 }} + sam-firestore-json-key-b64: ${{ steps.render-secrets.outputs.sam-firestore-json-key-b64 }} + test-group-name: ${{ inputs.test-group-name }} + sbt-test-command: "${{ inputs.test-command }}" + sbt-test-project: automation + test_run_id: ${{ env.test_run_id }} + test_env: ${{ inputs.ENV }} + java_version: ${{ fromJson(inputs.additional-args).java-version }} + e2e_env: ${{ inputs.e2e-env }} + billing_project: ${{ fromJson(inputs.additional-args).billing-project }} + users-metadata-b64: ${{ steps.render-secrets-v2.outputs.users-metadata-json-b64 }} + + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: always() + with: + check_name: report-${{ inputs.test-group-name }} + report_paths: 'rawls/automation/target/test-reports/TEST-*.xml' + fail_on_failure: false + require_tests: false + require_passed_tests: false + annotate_only: true + detailed_summary: true + + - name: Store Test Result Artifact + uses: actions/upload-artifact@v3 + if: always() + with: + name: test-reports + path: 'rawls/automation/target/test-reports/' + + upload-test-results: + needs: [rawls-swat-test-job] + if: ${{ always() && fromJson(inputs.additional-args).logging }} # always run (even in previous steps fail, unless told not to log) + permissions: + contents: 'read' + id-token: 'write' + uses: broadinstitute/dsp-reusable-workflows/.github/workflows/upload_test_results_to_biquery.yaml@main + # Always attempt to run if pull_request, as we want to report the appVersion even if the tests fail. + # never run on cron or other runs as we don't want extranaeous build reporting. + with: + service-name: 'rawls' + test-uuid: ${{ github.run_id }} # run id of this gha + environment: '${{ inputs.test-context }}' + artifact: 'test-reports' + big-query-table: 'broad-dsde-qa.automated_testing.test_results' + subuuid: ${{ github.run_id }} # run id of this gha From cd20371e06ac3f1393a3d674a1935373adc33826 Mon Sep 17 00:00:00 2001 From: Justin H Yang Date: Mon, 12 Aug 2024 11:53:05 -0700 Subject: [PATCH 2/3] changed ref for action to dsp-reusable-workflows --- .github/workflows/rawls-swat-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rawls-swat-tests.yaml b/.github/workflows/rawls-swat-tests.yaml index f129bf4142..8e457c25b7 100644 --- a/.github/workflows/rawls-swat-tests.yaml +++ b/.github/workflows/rawls-swat-tests.yaml @@ -339,7 +339,7 @@ jobs: - id: run-rawls-swat-tests name: 'Run Rawls Swat tests' - uses: ./.github/actions/run-swat-tests + uses: broadinstitute/dsp-reusable-workflows/.github/actions/run-swat-tests@DDO-3838-swat-test-move with: service_repo: ${{ env.SERVICE_REPO }} ref: ${{ inputs.ref }} From 7aa1590cc1ed5ef50ef8d25e7ef275f2af6f9138 Mon Sep 17 00:00:00 2001 From: Justin H Yang Date: Mon, 12 Aug 2024 11:59:45 -0700 Subject: [PATCH 3/3] assume remove vault calls for secerts --- .github/workflows/rawls-swat-tests.yaml | 73 +------------------------ 1 file changed, 3 insertions(+), 70 deletions(-) diff --git a/.github/workflows/rawls-swat-tests.yaml b/.github/workflows/rawls-swat-tests.yaml index 8e457c25b7..debb8c9673 100644 --- a/.github/workflows/rawls-swat-tests.yaml +++ b/.github/workflows/rawls-swat-tests.yaml @@ -219,26 +219,6 @@ jobs: tlsSkipVerify: true exportToken: true secrets: | - secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json auth_provider_x509_cert_url | FIRECLOUD_ACCOUNT_AUTH_CERT_URL; - secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json auth_uri | FIRECLOUD_ACCOUNT_AUTH_URI; - secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json client_email | FIRECLOUD_ACCOUNT_CLIENT_EMAIL; - secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json client_id | FIRECLOUD_ACCOUNT_CLIENT_ID; - secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json client_x509_cert_url | FIRECLOUD_ACCOUNT_CLIENT_CERT_URL; - secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json private_key | FIRECLOUD_ACCOUNT_PVT_KEY; - secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json private_key_id | FIRECLOUD_ACCOUNT_PVT_KEY_ID; - secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json project_id | FIRECLOUD_ACCOUNT_PROJECT_ID; - secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json token_uri | FIRECLOUD_ACCOUNT_TOKEN_URI; - secret/dsde/firecloud/${{ inputs.ENV }}/common/firecloud-account.json type | FIRECLOUD_ACCOUNT_TYPE; - secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json auth_provider_x509_cert_url | RAWLS_ACCOUNT_AUTH_CERT_URL; - secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json auth_uri | RAWLS_ACCOUNT_AUTH_URI; - secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json client_email | RAWLS_ACCOUNT_CLIENT_EMAIL; - secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json client_id | RAWLS_ACCOUNT_CLIENT_ID; - secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json client_x509_cert_url | RAWLS_ACCOUNT_CLIENT_CERT_URL; - secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json private_key | RAWLS_ACCOUNT_PVT_KEY; - secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json private_key_id | RAWLS_ACCOUNT_PVT_KEY_ID; - secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json project_id | RAWLS_ACCOUNT_PROJECT_ID; - secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json token_uri | RAWLS_ACCOUNT_TOKEN_URI; - secret/dsde/firecloud/${{ inputs.ENV }}/rawls/rawls-account.json type | RAWLS_ACCOUNT_TYPE; secret/dsde/firecloud/${{ inputs.ENV }}/common/secrets firecloud_id | FIRECLOUD_ID; secret/dsde/firecloud/${{ inputs.ENV }}/common/users automation_users_passwd | AUTOMATION_USERS_PASSWD; secret/dsde/firecloud/${{ inputs.ENV }}/common/users billing_acct | BILLING_ACCT; @@ -246,16 +226,7 @@ jobs: secret/dsde/firecloud/${{ inputs.ENV }}/common/users service_acct_email | SERVICE_ACCT_EMAIL; secret/dsde/firecloud/${{ inputs.ENV }}/common/users users | USERS; secret/dsde/firecloud/${{ inputs.ENV }}/common/users users_passwd | USERS_PASSWD; - secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json auth_provider_x509_cert_url | SAM_FIRESTORE_ACCOUNT_AUTH_CERT_URL; - secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json auth_uri | SAM_FIRESTORE_ACCOUNT_AUTH_URI; - secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json client_email | SAM_FIRESTORE_ACCOUNT_CLIENT_EMAIL; - secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json client_id | SAM_FIRESTORE_ACCOUNT_CLIENT_ID; - secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json client_x509_cert_url | SAM_FIRESTORE_ACCOUNT_CLIENT_CERT_URL; - secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json private_key | SAM_FIRESTORE_ACCOUNT_PVT_KEY; - secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json private_key_id | SAM_FIRESTORE_ACCOUNT_PVT_KEY_ID; - secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json project_id | SAM_FIRESTORE_ACCOUNT_PROJECT_ID; - secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json token_uri | SAM_FIRESTORE_ACCOUNT_TOKEN_URI; - secret/dsde/firecloud/${{ inputs.ENV }}/sam/sam-firestore-account.json type | SAM_FIRESTORE_ACCOUNT_TYPE; + - id: prepare-secrets name: Prepare secrets @@ -265,27 +236,6 @@ jobs: echo firecloud-secrets='{"firecloud_id":"${{ env.FIRECLOUD_ID }}"}' >> $GITHUB_OUTPUT echo firecloud-users='{"automation_users_passwd":"${{ env.AUTOMATION_USERS_PASSWD }}","billing_acct":"${{ env.BILLING_ACCT }}","qa_users_passwd":"${{ env.QA_USERS_PASSWD }}","service_acct_email":"${{ env.SERVICE_ACCT_EMAIL }}","users":${{ toJSON(env.USERS) }},"users_passwd":"${{ env.USERS_PASSWD }}"}' >> $GITHUB_OUTPUT echo sam-firestore-json='{"auth_provider_x509_cert_url":"${{ env.SAM_FIRESTORE_ACCOUNT_AUTH_CERT_URL }}","auth_uri":"${{ env.SAM_FIRESTORE_ACCOUNT_AUTH_URI }}","client_email":"${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_EMAIL }}","client_id":"${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_ID }}","client_x509_cert_url":"${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_CERT_URL }}","private_key":${{ toJSON(env.SAM_FIRESTORE_ACCOUNT_PVT_KEY) }},"private_key_id":"${{ env.SAM_FIRESTORE_ACCOUNT_PVT_KEY_ID }}","project_id":"${{ env.SAM_FIRESTORE_ACCOUNT_PROJECT_ID }}","token_uri":"${{ env.SAM_FIRESTORE_ACCOUNT_TOKEN_URI }}","type":"${{ env.SAM_FIRESTORE_ACCOUNT_TYPE }}"}' >> $GITHUB_OUTPUT - echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_AUTH_CERT_URL }}" - echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_AUTH_URI }}" - echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_AUTH_CERT_URL }}" - echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_AUTH_URI }}" - echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_CLIENT_EMAIL }}" - echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_CLIENT_ID }}" - echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_CLIENT_CERT_URL }}" - echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_PVT_KEY }}" - echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_PVT_KEY_ID }}" - echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_PROJECT_ID }}" - echo "::add-mask::${{ env.FIRECLOUD_ACCOUNT_TOKEN_URI }}" - echo "::add-mask::${{ env.RAWLS_ACCOUNT_AUTH_CERT_URL }}" - echo "::add-mask::${{ env.RAWLS_ACCOUNT_AUTH_URI }}" - echo "::add-mask::${{ env.RAWLS_ACCOUNT_CLIENT_EMAIL }}" - echo "::add-mask::${{ env.RAWLS_ACCOUNT_CLIENT_ID }}" - echo "::add-mask::${{ env.RAWLS_ACCOUNT_CLIENT_CERT_URL }}" - echo "::add-mask::${{ env.RAWLS_ACCOUNT_PVT_KEY }}" - echo "::add-mask::${{ env.RAWLS_ACCOUNT_PVT_KEY_ID }}" - echo "::add-mask::${{ env.RAWLS_ACCOUNT_PROJECT_ID }}" - echo "::add-mask::${{ env.RAWLS_ACCOUNT_TOKEN_URI }}" - echo "::add-mask::${{ env.RAWLS_ACCOUNT_TYPE }}" echo "::add-mask::${{ env.FIRECLOUD_ID }}" echo "::add-mask::${{ env.AUTOMATION_USERS_PASSWD }}" echo "::add-mask::${{ env.BILLING_ACCT }}" @@ -293,36 +243,19 @@ jobs: echo "::add-mask::${{ env.SERVICE_ACCT_EMAIL }}" echo "::add-mask::${{ env.USERS }}" echo "::add-mask::${{ env.USERS_PASSWD }}" - echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_AUTH_CERT_URL }}" - echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_AUTH_URI }}" - echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_EMAIL }}" - echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_ID }}" - echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_CLIENT_CERT_URL }}" - echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_PVT_KEY }}" - echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_PVT_KEY_ID }}" - echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_PROJECT_ID }}" - echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_TOKEN_URI }}" - echo "::add-mask::${{ env.SAM_FIRESTORE_ACCOUNT_TYPE }}" - id: render-secrets name: Render secrets run: | - FIRECLOUD_JSON_KEY_B64=$(printf '%s' ${{ toJSON(steps.prepare-secrets.outputs.firecloud-json) }} | base64 -w0) - RAWLS_JSON_KEY_B64=$(printf '%s' ${{ toJSON(steps.prepare-secrets.outputs.rawls-json) }} | base64 -w0) + FIRECLOUD_JSON_KEY_B64=$(printf '%s' ${{ toJSON( secrets.FIRECLOUD_JSON_KEY ) }} | base64 -w0) + RAWLS_JSON_KEY_B64=$(printf '%s' ${{ toJSON( secrets.RAWLS_JSON_KEY ) }} | base64 -w0) FIRECLOUD_SECRETS_B64=$(printf '%s' ${{ toJSON(steps.prepare-secrets.outputs.firecloud-secrets) }} | base64 -w0) FIRECLOUD_USERS_B64=$(printf '%s' ${{ toJSON(steps.prepare-secrets.outputs.firecloud-users) }} | base64 -w0) - SAM_FIRESTORE_JSON_KEY_B64=$(printf '%s' ${{ toJSON(steps.prepare-secrets.outputs.sam-firestore-json) }} | base64 -w0) - echo "::add-mask::$FIRECLOUD_JSON_KEY_B64" - echo firecloud-json-key-b64=$FIRECLOUD_JSON_KEY_B64 >> $GITHUB_OUTPUT - echo "::add-mask::$RAWLS_JSON_KEY_B64" - echo rawls-json-key-b64=$RAWLS_JSON_KEY_B64 >> $GITHUB_OUTPUT echo "::add-mask::$FIRECLOUD_SECRETS_B64" echo firecloud-secrets-b64=$FIRECLOUD_SECRETS_B64 >> $GITHUB_OUTPUT echo "::add-mask::$FIRECLOUD_USERS_B64" echo firecloud-users-b64=$FIRECLOUD_USERS_B64 >> $GITHUB_OUTPUT - echo "::add-mask::$SAM_FIRESTORE_JSON_KEY_B64" - echo sam-firestore-json-key-b64=$SAM_FIRESTORE_JSON_KEY_B64 >> $GITHUB_OUTPUT - id: render-secrets-v2 name: Render secrets v2 (eventually replacing render-secrets and getting rid of reliance on sensitive SA keys)