ec2-test-framework enhancements and graviton 4 testing (#1715)

1. Some of our ec2 instances from the ec2-test-framework were failing to
be properly stopped. This adds logic to the job pruner to stop hanging
instances.
2. Also parallelizing the test runs. GV2 is significantly slower which
causes the run to be much longer. This also makes it much easier for us
to extend testing against new test scripts.
3. Also added graviton 4 testing to the ec2-test-framework.

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license and the ISC license.

tests/ci/cdk/cdk/aws_lc_analytics_stack.py

@@ -55,4 +55,4 @@ def __init__(self,
             build_spec=BuildSpecLoader.load(spec_file_path))
         analytics.enable_batch_builds()
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=analytics, ec2_permissions=False)

tests/ci/cdk/cdk/aws_lc_android_ci_stack.py

@@ -65,4 +65,4 @@ def __init__(self,
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False)

tests/ci/cdk/cdk/aws_lc_ec2_test_framework_ci_stack.py

@@ -108,7 +108,7 @@ def __init__(self,
             })
         project.enable_batch_builds()
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=True)
 
         # Define logs for SSM.
         log_group_name = "{}-cw-logs".format(id)

tests/ci/cdk/cdk/aws_lc_github_ci_stack.py

@@ -84,4 +84,4 @@ def __init__(self,
         cfn_project.add_property_override("ResourceAccessRole", resource_access_role.role_arn)
         project.enable_batch_builds()
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False)

tests/ci/cdk/cdk/aws_lc_github_fuzz_ci_stack.py

@@ -128,4 +128,4 @@ def __init__(self,
           "Type": "EFS"
         }])
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=fuzz_codebuild)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=fuzz_codebuild, ec2_permissions=False)

tests/ci/cdk/cdk/bm_framework_stack.py

@@ -71,7 +71,7 @@ def __init__(self,
             build_spec=BuildSpecLoader.load(spec_file_path))
         project.enable_batch_builds()
 
-        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project)
+        PruneStaleGitHubBuilds(scope=self, id="PruneStaleGitHubBuilds", project=project, ec2_permissions=False)
 
         # use boto3 to determine if a cloudwatch logs group with the name we want exists, and if it doesn't, create it
         logs_client = boto3.client('logs', region_name=AWS_REGION)

tests/ci/cdk/cdk/codebuild/ec2_test_framework_omnibus.yaml

@@ -7,7 +7,7 @@ version: 0.2
 batch:
   build-list:
     # Actual tests are ran on an Graviton2 ec2 instance via SSM Commands.
-    - identifier: graviton2_tests
+    - identifier: graviton2_tests_asan
       buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
       env:
         type: LINUX_CONTAINER
@@ -18,3 +18,44 @@ batch:
           EC2_AMI: "ami-0c29a2c5cf69b5a9c"
           EC2_INSTANCE_TYPE: "c6g.2xlarge"
           ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
+          TARGET_TEST_SCRIPT: "./tests/ci/run_posix_sanitizers.sh"
+
+    - identifier: graviton2_tests_fips
+      buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
+      env:
+        type: LINUX_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_SMALL
+        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest
+        variables:
+          EC2_AMI: "ami-0c29a2c5cf69b5a9c"
+          EC2_INSTANCE_TYPE: "c6g.2xlarge"
+          ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
+          TARGET_TEST_SCRIPT: "./tests/ci/run_fips_tests.sh"
+
+    # Actual tests are ran on an Graviton4 ec2 instance via SSM Commands.
+    - identifier: graviton4_tests_asan
+      buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
+      env:
+        type: LINUX_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_SMALL
+        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest
+        variables:
+          EC2_AMI: "ami-0c29a2c5cf69b5a9c"
+          EC2_INSTANCE_TYPE: "r8g.2xlarge"
+          ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
+          TARGET_TEST_SCRIPT: "./tests/ci/run_posix_sanitizers.sh"
+
+    - identifier: graviton4_tests_fips
+      buildspec: ./tests/ci/codebuild/common/run_ec2_target.yml
+      env:
+        type: LINUX_CONTAINER
+        privileged-mode: false
+        compute-type: BUILD_GENERAL1_SMALL
+        image: 620771051181.dkr.ecr.us-west-2.amazonaws.com/aws-lc-docker-images-linux-x86:ubuntu-20.04_clang-7x-bm-framework_latest
+        variables:
+          EC2_AMI: "ami-0c29a2c5cf69b5a9c"
+          EC2_INSTANCE_TYPE: "r8g.2xlarge"
+          ECR_DOCKER_TAG: "amazonlinux-2023_clang-15x_sanitizer"
+          TARGET_TEST_SCRIPT: "./tests/ci/run_fips_tests.sh"

tests/ci/cdk/cdk/components.py

@@ -4,11 +4,11 @@
     aws_events as events, aws_events_targets as events_targets, aws_iam as iam, Duration
 
 from constructs import Construct
-from util.metadata import GITHUB_REPO_OWNER, GITHUB_TOKEN_SECRET_NAME
+from util.metadata import AWS_REGION, AWS_ACCOUNT, GITHUB_REPO_OWNER, GITHUB_TOKEN_SECRET_NAME
 
 
 class PruneStaleGitHubBuilds(Construct):
-    def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject) -> None:
+    def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject, ec2_permissions: bool) -> None:
         super().__init__(scope, id)
 
         github_token_secret = sm.Secret.from_secret_name_v2(scope=self,
@@ -36,12 +36,33 @@ def __init__(self, scope: Construct, id: str, *, project: codebuild.IProject) ->
                                 actions=[
                                     "codebuild:BatchGetBuildBatches",
                                     "codebuild:ListBuildBatchesForProject",
-                                    "codebuild:StopBuildBatch",
+                                    "codebuild:StopBuildBatch"
                                 ],
                                 resources=[project.project_arn]))
 
+        if ec2_permissions:
+            lambda_function.add_to_role_policy(
+                iam.PolicyStatement(effect=iam.Effect.ALLOW,
+                                actions=[
+                                    "ec2:TerminateInstances",
+                                ],
+                                resources=["arn:aws:ec2:{}:{}:instance/*".format(AWS_REGION, AWS_ACCOUNT)],
+                                conditions={
+                                    "StringEquals": {
+                                        "ec2:ResourceTag/ec2-framework-host": "ec2-framework-host"
+                                    }
+                                }))
+            # ec2:Describe* API actions do not support resource-level permissions.
+            lambda_function.add_to_role_policy(
+                iam.PolicyStatement(effect=iam.Effect.ALLOW,
+                                    actions=[
+                                        "ec2:DescribeInstances",
+                                    ],
+                                    resources=["*"]))
+
+
         events.Rule(scope=self, id="PurgeEventRule",
-                    description="Purge stale GitHub codebuild jobs (once per minute)",
+                    description="Purge stale GitHub codebuild jobs and ec2 instances (once per minute)",
                     enabled=True,
                     schedule=events.Schedule.rate(Duration.minutes(1)),
                     targets=[events_targets.LambdaFunction(handler=lambda_function)])

tests/ci/cdk/cdk/ssm/general_test_run_ssm_document.yaml

@@ -59,13 +59,8 @@ mainSteps:
         # Check if container was spun up succesfully. Then run test scripts and check the output.
         - >
           if [ -n "$exec_docker" ]; then
-            chmod +x ./tests/ci/run_posix_sanitizers.sh
-            $exec_docker ./tests/ci/run_posix_sanitizers.sh
-            if [ $? != 0 ]; then
-              exit 1
-            fi
-            chmod +x ./tests/ci/run_fips_tests.sh
-            $exec_docker ./tests/ci/run_fips_tests.sh
+            chmod +x {TARGET_TEST_SCRIPT}
+            $exec_docker {TARGET_TEST_SCRIPT}
             if [ $? != 0 ]; then
               exit 1
             fi

tests/ci/codebuild/common/run_ec2_target.yml

@@ -10,4 +10,4 @@ env:
 phases:
   build:
     commands:
-      - ./tests/ci/run_ec2_test_framework.sh "${EC2_AMI}" "${EC2_INSTANCE_TYPE}" "${ECR_DOCKER_TAG}"
+      - ./tests/ci/run_ec2_test_framework.sh "${EC2_AMI}" "${EC2_INSTANCE_TYPE}" "${ECR_DOCKER_TAG}" "${TARGET_TEST_SCRIPT}"