Skip to content

Commit 916b3d1

Browse files
samuel40791765samuel40791765
authored
ec2-test-framework enhancements and graviton 4 testing (#1715)
1. Some of our ec2 instances from the ec2-test-framework were failing to be properly stopped. This adds logic to the job pruner to stop hanging instances. 2. Also parallelizing the test runs. GV2 is significantly slower which causes the run to be much longer. This also makes it much easier for us to extend testing against new test scripts. 3. Also added graviton 4 testing to the ec2-test-framework. By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license and the ISC license.
1 parent 15d5926 commit 916b3d1

14 files changed

+565
-258
lines changed

‎tests/ci/cdk/cdk/aws_lc_analytics_stack.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,4 +55,4 @@ def __init__(self,
5555
build_spec=BuildSpecLoader.load(spec_file_path))
5656
analytics.enable_batch_builds()
5757

58-
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics)
58+
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics, ec2_permissions=False)

‎tests/ci/cdk/cdk/aws_lc_android_ci_stack.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,4 +65,4 @@ def __init__(self,
6565
build_spec=BuildSpecLoader.load(spec_file_path))
6666
project.enable_batch_builds()
6767

68-
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
68+
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False)

‎tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ def __init__(self,
108108
})
109109
project.enable_batch_builds()
110110

111-
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
111+
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=True)
112112

113113
# Define logs for SSM.
114114
log_group_name = "{}-cw-logs".format(id)

‎tests/ci/cdk/cdk/aws_lc_github_ci_stack.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,4 +84,4 @@ def __init__(self,
8484
cfn_project.add_property_override("ResourceAccessRole", resource_access_role.role_arn)
8585
project.enable_batch_builds()
8686

87-
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
87+
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False)

‎tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,4 +128,4 @@ def __init__(self,
128128
"Type": "EFS"
129129
}])
130130

131-
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=fuzz_codebuild)
131+
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=fuzz_codebuild, ec2_permissions=False)

‎tests/ci/cdk/cdk/bm_framework_stack.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ def __init__(self,
7171
build_spec=BuildSpecLoader.load(spec_file_path))
7272
project.enable_batch_builds()
7373

74-
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
74+
PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False)
7575

7676
# use boto3 to determine if a cloudwatch logs group with the name we want exists, and if it doesn't, create it
7777
logs_client = boto3.client('logs', region_name=AWS_REGION)

‎tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ version: 0.2
77
batch:
88
build-list:
99
# Actual tests are ran on an Graviton2 ec2 instance via SSM Commands.
10-
- identifier: graviton2_tests
10+
- identifier: graviton2_tests_asan
1111
buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
1212
env:
1313
type: LINUX_CONTAINER
@@ -18,3 +18,44 @@ batch:
1818
EC2_AMI: "ami-0c29a2c5cf69b5a9c"
1919
EC2_INSTANCE_TYPE: "c6g.2xlarge"
2020
ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
21+
TARGET_TEST_SCRIPT: "./tests/ci/run_posix_sanitizers.sh"
22+
23+
- identifier: graviton2_tests_fips
24+
buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
25+
env:
26+
type: LINUX_CONTAINER
27+
privileged-mode: false
28+
compute-type: BUILD_GENERAL1_SMALL
29+
image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest
30+
variables:
31+
EC2_AMI: "ami-0c29a2c5cf69b5a9c"
32+
EC2_INSTANCE_TYPE: "c6g.2xlarge"
33+
ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
34+
TARGET_TEST_SCRIPT: "./tests/ci/run_fips_tests.sh"
35+
36+
# Actual tests are ran on an Graviton4 ec2 instance via SSM Commands.
37+
- identifier: graviton4_tests_asan
38+
buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
39+
env:
40+
type: LINUX_CONTAINER
41+
privileged-mode: false
42+
compute-type: BUILD_GENERAL1_SMALL
43+
image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest
44+
variables:
45+
EC2_AMI: "ami-0c29a2c5cf69b5a9c"
46+
EC2_INSTANCE_TYPE: "r8g.2xlarge"
47+
ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
48+
TARGET_TEST_SCRIPT: "./tests/ci/run_posix_sanitizers.sh"
49+
50+
- identifier: graviton4_tests_fips
51+
buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
52+
env:
53+
type: LINUX_CONTAINER
54+
privileged-mode: false
55+
compute-type: BUILD_GENERAL1_SMALL
56+
image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest
57+
variables:
58+
EC2_AMI: "ami-0c29a2c5cf69b5a9c"
59+
EC2_INSTANCE_TYPE: "r8g.2xlarge"
60+
ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
61+
TARGET_TEST_SCRIPT: "./tests/ci/run_fips_tests.sh"

‎tests/ci/cdk/cdk/components.py

Lines changed: 25 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@
44
aws_events as events, aws_events_targets as events_targets, aws_iam as iam, Duration
55

66
from constructs import Construct
7-
from util.metadata import GITHUB_REPO_OWNER, GITHUB_TOKEN_SECRET_NAME
7+
from util.metadata import AWS_REGION, AWS_ACCOUNT, GITHUB_REPO_OWNER, GITHUB_TOKEN_SECRET_NAME
88

99

1010
class PruneStaleGitHubBuilds(Construct):
11-
def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject) -> None:
11+
def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject, ec2_permissions: bool) -> None:
1212
super().__init__(scope, id)
1313

1414
github_token_secret = sm.Secret.from_secret_name_v2(scope=self,
@@ -36,12 +36,33 @@ def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject) ->
3636
actions=[
3737
"codebuild:BatchGetBuildBatches",
3838
"codebuild:ListBuildBatchesForProject",
39-
"codebuild:StopBuildBatch",
39+
"codebuild:StopBuildBatch"
4040
],
4141
resources=[project.project_arn]))
4242

43+
if ec2_permissions:
44+
lambda_function.add_to_role_policy(
45+
iam.PolicyStatement(effect=iam.Effect.ALLOW,
46+
actions=[
47+
"ec2:TerminateInstances",
48+
],
49+
resources=["arn:aws:ec2:{}:{}:instance/*".format(AWS_REGION, AWS_ACCOUNT)],
50+
conditions={
51+
"StringEquals": {
52+
"ec2:ResourceTag/ec2-framework-host": "ec2-framework-host"
53+
}
54+
}))
55+
# ec2:Describe* API actions do not support resource-level permissions.
56+
lambda_function.add_to_role_policy(
57+
iam.PolicyStatement(effect=iam.Effect.ALLOW,
58+
actions=[
59+
"ec2:DescribeInstances",
60+
],
61+
resources=["*"]))
62+
63+
4364
events.Rule(scope=self, id="PurgeEventRule",
44-
description="Purge stale GitHub codebuild jobs (once per minute)",
65+
description="Purge stale GitHub codebuild jobs and ec2 instances (once per minute)",
4566
enabled=True,
4667
schedule=events.Schedule.rate(Duration.minutes(1)),
4768
targets=[events_targets.LambdaFunction(handler=lambda_function)])

‎tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,8 @@ mainSteps:
5959
# Check if container was spun up succesfully. Then run test scripts and check the output.
6060
- >
6161
if [ -n "$exec_docker" ]; then
62-
chmod +x ./tests/ci/run_posix_sanitizers.sh
63-
$exec_docker ./tests/ci/run_posix_sanitizers.sh
64-
if [ $? != 0 ]; then
65-
exit 1
66-
fi
67-
chmod +x ./tests/ci/run_fips_tests.sh
68-
$exec_docker ./tests/ci/run_fips_tests.sh
62+
chmod +x {TARGET_TEST_SCRIPT}
63+
$exec_docker {TARGET_TEST_SCRIPT}
6964
if [ $? != 0 ]; then
7065
exit 1
7166
fi

‎tests/ci/codebuild/common/run_ec2_target.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,4 @@ env:
1010
phases:
1111
build:
1212
commands:
13-
- ./tests/ci/run_ec2_test_framework.sh "${EC2_AMI}" "${EC2_INSTANCE_TYPE}" "${ECR_DOCKER_TAG}"
13+
- ./tests/ci/run_ec2_test_framework.sh "${EC2_AMI}" "${EC2_INSTANCE_TYPE}" "${ECR_DOCKER_TAG}" "${TARGET_TEST_SCRIPT}"

0 commit comments

Comments
 (0)