Skip to content

Commit 871db76

Browse files
rozzarozza
committed
Update the evergreen build to use the AWS secrets manager
JAVA-5886
1 parent f0ae963 commit 871db76

31 files changed

+118
-147
lines changed

‎.evergreen/.evg.yml

Lines changed: 35 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,11 @@ functions:
124124
perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|${DRIVERS_TOOLS}|g" $filename
125125
done
126126
127+
"assume-aws-test-secrets-role":
128+
- command: ec2.assume_role
129+
params:
130+
role_arn: ${aws_test_secrets_role}
131+
127132
"create-archive-tar-file":
128133
- command: shell.exec
129134
params:
@@ -154,9 +159,6 @@ functions:
154159
bash ${DRIVERS_TOOLS}/.evergreen/stop-orchestration.sh || true
155160
156161
"start-mongohoused":
157-
- command: ec2.assume_role
158-
params:
159-
role_arn: ${aws_test_secrets_role}
160162
- command: shell.exec
161163
params:
162164
include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ]
@@ -168,9 +170,6 @@ functions:
168170
DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-image.sh
169171
170172
"stop-mongohoused":
171-
- command: ec2.assume_role
172-
params:
173-
role_arn: ${aws_test_secrets_role}
174173
- command: shell.exec
175174
params:
176175
include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ]
@@ -182,6 +181,7 @@ functions:
182181
- command: shell.exec
183182
params:
184183
shell: "bash"
184+
include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ]
185185
script: |
186186
${PREPARE_SHELL}
187187
bash ${DRIVERS_TOOLS}/.evergreen/serverless/setup-secrets.sh ${VAULT_NAME}
@@ -192,6 +192,8 @@ functions:
192192
"stop-serverless":
193193
- command: shell.exec
194194
params:
195+
shell: "bash"
196+
include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ]
195197
script: |
196198
${PREPARE_SHELL}
197199
bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh || true
@@ -211,34 +213,6 @@ functions:
211213
cd ${DRIVERS_TOOLS}/.evergreen
212214
DRIVERS_TOOLS=${DRIVERS_TOOLS} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop || true
213215
214-
"start-kms-mock-server":
215-
- command: shell.exec
216-
params:
217-
background: true
218-
shell: "bash"
219-
script: |
220-
${PREPARE_SHELL}
221-
cd ${DRIVERS_TOOLS}/.evergreen/csfle
222-
. ./activate-kmstlsvenv.sh
223-
python -u kms_http_server.py -v --ca_file ../x509gen/ca.pem --cert_file ../x509gen/${CERT_FILE} --port 8000
224-
225-
"start-kms-kmip-server":
226-
- command: shell.exec
227-
params:
228-
shell: "bash"
229-
script: |
230-
${PREPARE_SHELL}
231-
cd ${DRIVERS_TOOLS}/.evergreen/csfle
232-
. ./activate-kmstlsvenv.sh
233-
- command: shell.exec
234-
params:
235-
shell: "bash"
236-
background: true
237-
script: |
238-
cd ${DRIVERS_TOOLS}/.evergreen/csfle
239-
. ./activate-kmstlsvenv.sh
240-
python -u kms_kmip_server.py
241-
242216
"stop-aws":
243217
- command: shell.exec
244218
params:
@@ -260,9 +234,6 @@ functions:
260234
rm -rf $DRIVERS_TOOLS || true
261235
262236
"add-aws-auth-variables-to-file":
263-
- command: ec2.assume_role
264-
params:
265-
role_arn: ${aws_test_secrets_role}
266237
- command: shell.exec
267238
type: "test"
268239
params:
@@ -463,8 +434,6 @@ functions:
463434
echo "Response Body: $response_body"
464435
echo "HTTP Status: $http_status"
465436
466-
467-
#
468437
# Test functions
469438
#
470439

@@ -474,25 +443,11 @@ functions:
474443
params:
475444
working_dir: "src"
476445
env:
477-
AWS_ACCESS_KEY_ID: ${aws_access_key_id}
478-
AWS_SECRET_ACCESS_KEY: ${aws_secret_access_key}
479-
AWS_ACCESS_KEY_ID_AWS_KMS_NAMED: ${aws_access_key_id_2}
480-
AWS_SECRET_ACCESS_KEY_AWS_KMS_NAMED: ${aws_secret_access_key_2}
481-
AWS_DEFAULT_REGION: us-east-1
482-
AZURE_TENANT_ID: ${azure_tenant_id}
483-
AZURE_CLIENT_ID: ${azure_client_id}
484-
AZURE_CLIENT_SECRET: ${azure_client_secret}
485-
GCP_EMAIL: ${gcp_email}
486-
GCP_PRIVATE_KEY: ${gcp_private_key}
487446
AZUREKMS_KEY_VAULT_ENDPOINT: ${testazurekms_keyvaultendpoint}
488447
AZUREKMS_KEY_NAME: ${testazurekms_keyname}
489448
script: |
490449
${PREPARE_SHELL}
491450
492-
. ${DRIVERS_TOOLS}/.evergreen/csfle/set-temp-creds.sh
493-
export AWS_TEMP_ACCESS_KEY_ID=$CSFLE_AWS_TEMP_ACCESS_KEY_ID
494-
export AWS_TEMP_SECRET_ACCESS_KEY=$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY
495-
export AWS_TEMP_SESSION_TOKEN=$CSFLE_AWS_TEMP_SESSION_TOKEN
496451
export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
497452
498453
AUTH="${AUTH}" SSL="${SSL}" MONGODB_URI="${MONGODB_URI}" TOPOLOGY="${TOPOLOGY}" \
@@ -816,9 +771,6 @@ functions:
816771
type: "test"
817772
params:
818773
working_dir: "src"
819-
env:
820-
AWS_ACCESS_KEY_ID: ${aws_access_key_id}
821-
AWS_SECRET_ACCESS_KEY: ${aws_secret_access_key}
822774
script: |
823775
${PREPARE_SHELL}
824776
set +o xtrace
@@ -830,26 +782,10 @@ functions:
830782
params:
831783
working_dir: "src"
832784
env:
833-
AWS_ACCESS_KEY_ID: ${aws_access_key_id}
834-
AWS_SECRET_ACCESS_KEY: ${aws_secret_access_key}
835-
AWS_ACCESS_KEY_ID_AWS_KMS_NAMED: ${aws_access_key_id_2}
836-
AWS_SECRET_ACCESS_KEY_AWS_KMS_NAMED: ${aws_secret_access_key_2}
837-
AWS_DEFAULT_REGION: us-east-1
838-
AZURE_TENANT_ID: ${azure_tenant_id}
839-
AZURE_CLIENT_ID: ${azure_client_id}
840-
AZURE_CLIENT_SECRET: ${azure_client_secret}
841-
GCP_EMAIL: ${gcp_email}
842-
GCP_PRIVATE_KEY: ${gcp_private_key}
843785
AZUREKMS_KEY_VAULT_ENDPOINT: ${testazurekms_keyvaultendpoint}
844786
AZUREKMS_KEY_NAME: ${testazurekms_keyname}
845787
script: |
846788
${PREPARE_SHELL}
847-
. ${DRIVERS_TOOLS}/.evergreen/csfle/set-temp-creds.sh
848-
849-
export AWS_TEMP_ACCESS_KEY_ID=$CSFLE_AWS_TEMP_ACCESS_KEY_ID
850-
export AWS_TEMP_SECRET_ACCESS_KEY=$CSFLE_AWS_TEMP_SECRET_ACCESS_KEY
851-
export AWS_TEMP_SESSION_TOKEN=$CSFLE_AWS_TEMP_SESSION_TOKEN
852-
853789
MONGODB_URI="${MONGODB_URI}" JAVA_VERSION="${JAVA_VERSION}" .evergreen/run-csfle-tests-with-mongocryptd.sh
854790
855791
"run-perf-tests":
@@ -899,13 +835,15 @@ pre:
899835
- func: "fix-absolute-paths"
900836

901837
post:
838+
- func: "stop-mongo-orchestration"
902839
- func: "upload-mo-artifacts"
903840
- func: "upload-test-results"
841+
- func: "assume-aws-test-secrets-role"
904842
- func: "stop-load-balancer"
905843
- func: "stop-serverless"
906844
- func: "stop-aws"
907-
- func: "stop-mongo-orchestration"
908845
- func: "stop-mongohoused"
846+
- func: "stop-csfle-servers"
909847
- func: "cleanup"
910848

911849
tasks:
@@ -936,23 +874,23 @@ tasks:
936874

937875
- name: "test-legacy-task"
938876
commands:
939-
- func: "start-kms-kmip-server"
877+
- func: "start-csfle-servers"
940878
- func: "start-mongo-orchestration"
941879
- func: "run-tests"
942880
vars:
943881
TESTS: 'driver-legacy:test'
944882

945883
- name: "test-sync-task"
946884
commands:
947-
- func: "start-kms-kmip-server"
885+
- func: "start-csfle-servers"
948886
- func: "start-mongo-orchestration"
949887
- func: "run-tests"
950888
vars:
951889
TESTS: 'driver-sync:test'
952890

953891
- name: "test-reactive-task"
954892
commands:
955-
- func: "start-kms-kmip-server"
893+
- func: "start-csfle-servers"
956894
- func: "start-mongo-orchestration"
957895
- func: "run-tests"
958896
vars:
@@ -1036,10 +974,7 @@ tasks:
1036974
# Might exceed 1 hour of execution.
1037975
exec_timeout_secs: 7200
1038976
commands:
1039-
- command: ec2.assume_role
1040-
params:
1041-
role_arn: ${aws_test_secrets_role}
1042-
duration_seconds: 1800
977+
- func: "assume-aws-test-secrets-role"
1043978
- func: "run-oidc-auth-test-k8s-test"
1044979
vars:
1045980
VARIANT: eks
@@ -1052,12 +987,13 @@ tasks:
1052987

1053988
- name: "serverless-test-task"
1054989
commands:
990+
- func: "assume-aws-test-secrets-role"
1055991
- func: "start-serverless"
1056992
- func: "run-serverless-tests"
1057993

1058994
- name: "accept-api-version-2-test-task"
1059995
commands:
1060-
- func: "start-kms-kmip-server"
996+
- func: "start-csfle-servers"
1061997
- func: "start-mongo-orchestration"
1062998
vars:
1063999
ORCHESTRATION_FILE: "versioned-api-testing.json"
@@ -1074,6 +1010,7 @@ tasks:
10741010
AUTH: "auth"
10751011
ORCHESTRATION_FILE: "auth-aws.json"
10761012
TOPOLOGY: "server"
1013+
- func: "assume-aws-test-secrets-role"
10771014
- func: "add-aws-auth-variables-to-file"
10781015
- func: "run-aws-auth-test-with-regular-aws-credentials"
10791016

@@ -1084,6 +1021,7 @@ tasks:
10841021
AUTH: "auth"
10851022
ORCHESTRATION_FILE: "auth-aws.json"
10861023
TOPOLOGY: "server"
1024+
- func: "assume-aws-test-secrets-role"
10871025
- func: "add-aws-auth-variables-to-file"
10881026
- func: "run-aws-auth-test-with-assume-role-credentials"
10891027

@@ -1094,6 +1032,7 @@ tasks:
10941032
AUTH: "auth"
10951033
ORCHESTRATION_FILE: "auth-aws.json"
10961034
TOPOLOGY: "server"
1035+
- func: "assume-aws-test-secrets-role"
10971036
- func: "add-aws-auth-variables-to-file"
10981037
- func: "run-aws-auth-test-with-aws-credentials-as-environment-variables"
10991038

@@ -1104,6 +1043,7 @@ tasks:
11041043
AUTH: "auth"
11051044
ORCHESTRATION_FILE: "auth-aws.json"
11061045
TOPOLOGY: "server"
1046+
- func: "assume-aws-test-secrets-role"
11071047
- func: "add-aws-auth-variables-to-file"
11081048
- func: "run-aws-auth-test-with-aws-credentials-and-session-token-as-environment-variables"
11091049

@@ -1114,6 +1054,7 @@ tasks:
11141054
AUTH: "auth"
11151055
ORCHESTRATION_FILE: "auth-aws.json"
11161056
TOPOLOGY: "server"
1057+
- func: "assume-aws-test-secrets-role"
11171058
- func: "add-aws-auth-variables-to-file"
11181059
- func: "run-aws-auth-test-with-aws-EC2 credentials"
11191060

@@ -1124,6 +1065,7 @@ tasks:
11241065
AUTH: "auth"
11251066
ORCHESTRATION_FILE: "auth-aws.json"
11261067
TOPOLOGY: "server"
1068+
- func: "assume-aws-test-secrets-role"
11271069
- func: "add-aws-auth-variables-to-file"
11281070
- func: "run-aws-auth-test-with-web-identity-credentials"
11291071

@@ -1526,6 +1468,7 @@ tasks:
15261468

15271469
- name: "atlas-data-lake-task"
15281470
commands:
1471+
- func: "assume-aws-test-secrets-role"
15291472
- func: "start-mongohoused"
15301473
- command: shell.exec
15311474
type: "test"
@@ -1664,9 +1607,7 @@ tasks:
16641607
TOPOLOGY: "server"
16651608
AUTH: "noauth"
16661609
SSL: "nossl"
1667-
- func: "start-kms-mock-server"
1668-
vars:
1669-
CERT_FILE: "expired.pem"
1610+
- func: "start-csfle-servers"
16701611
- func: "run-kms-tls-test"
16711612
vars:
16721613
KMS_TLS_ERROR_TYPE: "expired"
@@ -1682,7 +1623,7 @@ tasks:
16821623
TOPOLOGY: "server"
16831624
AUTH: "noauth"
16841625
SSL: "nossl"
1685-
- func: "start-kms-mock-server"
1626+
- func: "start-csfle-servers"
16861627
vars:
16871628
CERT_FILE: "wrong-host.pem"
16881629
- func: "run-kms-tls-test"
@@ -1695,6 +1636,7 @@ tasks:
16951636
- name: "test-csfle-aws-from-environment-task"
16961637
tags: [ "csfle-aws-from-environment" ]
16971638
commands:
1639+
- func: "start-csfle-servers"
16981640
- func: "start-mongo-orchestration"
16991641
vars:
17001642
TOPOLOGY: "server"
@@ -1704,7 +1646,7 @@ tasks:
17041646

17051647
- name: "csfle-tests-with-mongocryptd-task"
17061648
commands:
1707-
- func: "start-kms-kmip-server"
1649+
- func: "start-csfle-servers"
17081650
- func: "start-mongo-orchestration"
17091651
- func: "run-csfle-tests-with-mongocryptd"
17101652

@@ -2054,7 +1996,7 @@ task_groups:
20541996
export GCPKMS_PROJECT=${GCPKMS_PROJECT}
20551997
export GCPKMS_ZONE=${GCPKMS_ZONE}
20561998
export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME}
2057-
$DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh
1999+
$DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh || true
20582000
tasks:
20592001
- "test-gcp-kms-task"
20602002

@@ -2066,9 +2008,11 @@ task_groups:
20662008
- func: "prepare-resources"
20672009
- func: "fix-absolute-paths"
20682010
- func: "create-archive-tar-file"
2011+
- func: "assume-aws-test-secrets-role"
20692012
- command: shell.exec
20702013
params:
20712014
shell: "bash"
2015+
include_expansions_in_env: [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN" ]
20722016
script: |
20732017
${PREPARE_SHELL}
20742018
echo '${testazurekms_publickey}' > /tmp/testazurekms_publickey
@@ -2113,9 +2057,7 @@ task_groups:
21132057
- func: "fetch-source"
21142058
- func: "prepare-resources"
21152059
- func: "fix-absolute-paths"
2116-
- command: ec2.assume_role
2117-
params:
2118-
role_arn: ${aws_test_secrets_role}
2060+
- func: "assume-aws-test-secrets-role"
21192061
- command: subprocess.exec
21202062
params:
21212063
binary: bash
@@ -2141,9 +2083,7 @@ task_groups:
21412083
- func: "prepare-resources"
21422084
- func: "fix-absolute-paths"
21432085
- func: "create-archive-tar-file"
2144-
- command: ec2.assume_role
2145-
params:
2146-
role_arn: ${aws_test_secrets_role}
2086+
- func: "assume-aws-test-secrets-role"
21472087
- command: subprocess.exec
21482088
params:
21492089
binary: bash
@@ -2171,9 +2111,7 @@ task_groups:
21712111
- func: "prepare-resources"
21722112
- func: "fix-absolute-paths"
21732113
- func: "create-archive-tar-file"
2174-
- command: ec2.assume_role
2175-
params:
2176-
role_arn: ${aws_test_secrets_role}
2114+
- func: "assume-aws-test-secrets-role"
21772115
- command: subprocess.exec
21782116
params:
21792117
binary: bash
@@ -2202,9 +2140,7 @@ task_groups:
22022140
- func: "prepare-resources"
22032141
- func: "fix-absolute-paths"
22042142
- func: "create-archive-tar-file"
2205-
- command: ec2.assume_role
2206-
params:
2207-
role_arn: ${aws_test_secrets_role}
2143+
- func: "assume-aws-test-secrets-role"
22082144
- command: subprocess.exec
22092145
params:
22102146
binary: bash

0 commit comments

Comments
 (0)