fix(tests): make sure multiple e2e tests run concurrently (aws-powertools#1861)

Author: rubenfonseca

.github/workflows/run-e2e-tests.yml

@@ -26,6 +26,7 @@ jobs:
       id-token: write # needed to request JWT with GitHub's OIDC Token endpoint. docs: https://bit.ly/3MNgQO9
       contents: read
     strategy:
+      fail-fast: false # needed so if a version fails, the others will still be able to complete and cleanup
       matrix:
         version: ["3.7", "3.8", "3.9"]
     if: ${{ github.actor != 'dependabot[bot]' }}

parallel_run_e2e.py

@@ -8,7 +8,7 @@ def main():
     features = Path("tests/e2e").rglob("infrastructure.py")
     workers = len(list(features)) - 1
 
-    command = f"poetry run pytest -n {workers} --dist loadfile -o log_cli=true tests/e2e"
+    command = f"poetry run pytest -n {workers} -o log_cli=true tests/e2e"
     result = subprocess.run(command.split(), shell=False)
     sys.exit(result.returncode)
 

tests/e2e/event_handler/conftest.py

@@ -3,7 +3,7 @@
 from tests.e2e.event_handler.infrastructure import EventHandlerStack
 
 
-@pytest.fixture(autouse=True, scope="module")
+@pytest.fixture(autouse=True, scope="package")
 def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 

tests/e2e/event_handler/test_header_serializer.py

@@ -36,6 +36,7 @@ def lambda_function_url_endpoint(infrastructure: dict) -> str:
     return infrastructure.get("LambdaFunctionUrl", "")
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_alb_headers_serializer(alb_basic_listener_endpoint):
     # GIVEN
     url = f"{alb_basic_listener_endpoint}/todos"
@@ -74,6 +75,7 @@ def test_alb_headers_serializer(alb_basic_listener_endpoint):
     assert response.cookies.get(last_cookie.name) == last_cookie.value
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_alb_multi_value_headers_serializer(alb_multi_value_header_listener_endpoint):
     # GIVEN
     url = f"{alb_multi_value_header_listener_endpoint}/todos"
@@ -112,6 +114,7 @@ def test_alb_multi_value_headers_serializer(alb_multi_value_header_listener_endp
         assert response.cookies.get(cookie.name) == cookie.value
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_api_gateway_rest_headers_serializer(apigw_rest_endpoint):
     # GIVEN
     url = f"{apigw_rest_endpoint}todos"
@@ -147,6 +150,7 @@ def test_api_gateway_rest_headers_serializer(apigw_rest_endpoint):
         assert response.cookies.get(cookie.name) == cookie.value
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_api_gateway_http_headers_serializer(apigw_http_endpoint):
     # GIVEN
     url = f"{apigw_http_endpoint}todos"
@@ -182,6 +186,7 @@ def test_api_gateway_http_headers_serializer(apigw_http_endpoint):
         assert response.cookies.get(cookie.name) == cookie.value
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_lambda_function_url_headers_serializer(lambda_function_url_endpoint):
     # GIVEN
     url = f"{lambda_function_url_endpoint}todos"  # the function url endpoint already has the trailing /

tests/e2e/event_handler/test_paths_ending_with_slash.py

@@ -33,6 +33,7 @@ def lambda_function_url_endpoint(infrastructure: dict) -> str:
     return infrastructure.get("LambdaFunctionUrl", "")
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_api_gateway_rest_trailing_slash(apigw_rest_endpoint):
     # GIVEN API URL ends in a trailing slash
     url = f"{apigw_rest_endpoint}todos/"
@@ -51,6 +52,7 @@ def test_api_gateway_rest_trailing_slash(apigw_rest_endpoint):
     assert response.status_code == 200
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_api_gateway_http_trailing_slash(apigw_http_endpoint):
     # GIVEN the URL for the API ends in a trailing slash API gateway should return a 404
     url = f"{apigw_http_endpoint}todos/"
@@ -67,6 +69,7 @@ def test_api_gateway_http_trailing_slash(apigw_http_endpoint):
         )
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_lambda_function_url_trailing_slash(lambda_function_url_endpoint):
     # GIVEN the URL for the API ends in a trailing slash it should behave as if there was not one
     url = f"{lambda_function_url_endpoint}todos/"  # the function url endpoint already has the trailing /
@@ -83,6 +86,7 @@ def test_lambda_function_url_trailing_slash(lambda_function_url_endpoint):
         )
 
 
+@pytest.mark.xdist_group(name="event_handler")
 def test_alb_url_trailing_slash(alb_multi_value_header_listener_endpoint):
     # GIVEN url has a trailing slash - it should behave as if there was not one
     url = f"{alb_multi_value_header_listener_endpoint}/todos/"

tests/e2e/idempotency/conftest.py

@@ -3,8 +3,8 @@
 from tests.e2e.idempotency.infrastructure import IdempotencyDynamoDBStack
 
 
-@pytest.fixture(autouse=True, scope="module")
-def infrastructure(tmp_path_factory, worker_id):
+@pytest.fixture(autouse=True, scope="package")
+def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 
     Yields

tests/e2e/idempotency/test_idempotency_dynamodb.py

@@ -27,6 +27,7 @@ def idempotency_table_name(infrastructure: dict) -> str:
     return infrastructure.get("DynamoDBTable", "")
 
 
+@pytest.mark.xdist_group(name="idempotency")
 def test_ttl_caching_expiration_idempotency(ttl_cache_expiration_handler_fn_arn: str):
     # GIVEN
     payload = json.dumps({"message": "Lambda Powertools - TTL 5s"})
@@ -56,6 +57,7 @@ def test_ttl_caching_expiration_idempotency(ttl_cache_expiration_handler_fn_arn:
     assert third_execution_response != second_execution_response
 
 
+@pytest.mark.xdist_group(name="idempotency")
 def test_ttl_caching_timeout_idempotency(ttl_cache_timeout_handler_fn_arn: str):
     # GIVEN
     payload_timeout_execution = json.dumps({"sleep": 5, "message": "Lambda Powertools - TTL 1s"})
@@ -79,6 +81,7 @@ def test_ttl_caching_timeout_idempotency(ttl_cache_timeout_handler_fn_arn: str):
     assert payload_working_execution == execution_working_response
 
 
+@pytest.mark.xdist_group(name="idempotency")
 def test_parallel_execution_idempotency(parallel_execution_handler_fn_arn: str):
     # GIVEN
     arguments = json.dumps({"message": "Lambda Powertools - Parallel execution"})

tests/e2e/logger/conftest.py

@@ -3,8 +3,8 @@
 from tests.e2e.logger.infrastructure import LoggerStack
 
 
-@pytest.fixture(autouse=True, scope="module")
-def infrastructure(tmp_path_factory, worker_id):
+@pytest.fixture(autouse=True, scope="package")
+def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 
     Yields

tests/e2e/logger/test_logger.py

@@ -17,6 +17,7 @@ def basic_handler_fn_arn(infrastructure: dict) -> str:
     return infrastructure.get("BasicHandlerArn", "")
 
 
+@pytest.mark.xdist_group(name="logger")
 def test_basic_lambda_logs_visible(basic_handler_fn, basic_handler_fn_arn):
     # GIVEN
     message = "logs should be visible with default settings"

tests/e2e/metrics/conftest.py

@@ -3,8 +3,8 @@
 from tests.e2e.metrics.infrastructure import MetricsStack
 
 
-@pytest.fixture(autouse=True, scope="module")
-def infrastructure(tmp_path_factory, worker_id):
+@pytest.fixture(autouse=True, scope="package")
+def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 
     Yields

tests/e2e/metrics/test_metrics.py

@@ -28,6 +28,7 @@ def cold_start_fn_arn(infrastructure: dict) -> str:
 METRIC_NAMESPACE = "powertools-e2e-metric"
 
 
+@pytest.mark.xdist_group(name="metrics")
 def test_basic_lambda_metric_is_visible(basic_handler_fn: str, basic_handler_fn_arn: str):
     # GIVEN
     metric_name = data_builder.build_metric_name()
@@ -47,6 +48,7 @@ def test_basic_lambda_metric_is_visible(basic_handler_fn: str, basic_handler_fn_
     assert metric_values == [3.0]
 
 
+@pytest.mark.xdist_group(name="metrics")
 def test_cold_start_metric(cold_start_fn_arn: str, cold_start_fn: str):
     # GIVEN
     metric_name = "ColdStart"

tests/e2e/parameters/conftest.py

@@ -3,8 +3,8 @@
 from tests.e2e.parameters.infrastructure import ParametersStack
 
 
-@pytest.fixture(autouse=True, scope="module")
-def infrastructure(tmp_path_factory, worker_id):
+@pytest.fixture(autouse=True, scope="package")
+def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 
     Yields

tests/e2e/parameters/infrastructure.py

@@ -27,7 +27,7 @@ def create_resources(self):
             iam.PolicyStatement(
                 effect=iam.Effect.ALLOW,
                 actions=[
-                    "ssm:GetParameter",
+                    "ssm:GetParameters",
                 ],
                 resources=[f"arn:aws:ssm:{self.region}:{self.account_id}:parameter/powertools/e2e/parameters/*"],
             )

tests/e2e/parameters/test_appconfig.py

@@ -35,6 +35,7 @@ def parameter_appconfig_freeform_profile(infrastructure: dict) -> str:
     return infrastructure.get("AppConfigProfile", "")
 
 
+@pytest.mark.xdist_group(name="parameters")
 def test_get_parameter_appconfig_freeform(
     parameter_appconfig_freeform_handler_fn_arn: str,
     parameter_appconfig_freeform_value: str,

tests/e2e/parameters/test_ssm.py

@@ -17,7 +17,7 @@ def parameters_list(infrastructure: dict) -> List[str]:
     return json.loads(param_list)
 
 
-#
+@pytest.mark.xdist_group(name="parameters")
 def test_get_parameters_by_name(
     ssm_get_parameters_by_name_fn_arn: str,
     parameters_list: str,

tests/e2e/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+addopts = -ra -vv --dist loadgroup

tests/e2e/streaming/conftest.py

@@ -3,8 +3,8 @@
 from tests.e2e.streaming.infrastructure import StreamingStack
 
 
-@pytest.fixture(autouse=True, scope="module")
-def infrastructure(tmp_path_factory, worker_id):
+@pytest.fixture(autouse=True, scope="package")
+def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 
     Yields

tests/e2e/streaming/test_s3_object.py

@@ -21,6 +21,7 @@ def s3_object_handler_fn_arn(infrastructure: dict) -> str:
     return infrastructure.get("S3ObjectHandler", "")
 
 
+@pytest.mark.xdist_group(name="streaming")
 def get_object_version(bucket, key) -> str:
     s3 = boto3.client("s3")
     versions = s3.list_object_versions(Bucket=bucket)
@@ -43,13 +44,15 @@ def get_lambda_result_payload(s3_object_handler_fn_arn: str, payload: dict) -> d
     return json.loads(handler_result["Payload"].read())
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_size(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "plain.txt"}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("size") == 12
     assert result.get("body") == "hello world"
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_versioned_object_size(s3_object_handler_fn_arn, versioned_bucket_name):
     key = "plain.txt"
     payload = {
@@ -62,18 +65,21 @@ def test_s3_versioned_object_size(s3_object_handler_fn_arn, versioned_bucket_nam
     assert result.get("body") == "hello world"
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_non_existent(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "NOTEXISTENT.txt"}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("error") == "Not found"
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_csv_constructor(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "csv.txt", "is_csv": True}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("body") == {"name": "hello", "value": "world"}
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_versioned_object_csv_constructor(s3_object_handler_fn_arn, versioned_bucket_name):
     key = "csv.txt"
     payload = {
@@ -86,24 +92,28 @@ def test_s3_versioned_object_csv_constructor(s3_object_handler_fn_arn, versioned
     assert result.get("body") == {"name": "hello", "value": "world"}
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_csv_transform(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "csv.txt", "transform_csv": True}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("body") == {"name": "hello", "value": "world"}
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_csv_transform_in_place(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "csv.txt", "transform_csv": True, "in_place": True}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("body") == {"name": "hello", "value": "world"}
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_csv_gzip_constructor(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "csv.txt.gz", "is_csv": True, "is_gzip": True}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("body") == {"name": "hello", "value": "world"}
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_versioned_object_csv_gzip_constructor(s3_object_handler_fn_arn, versioned_bucket_name):
     key = "csv.txt.gz"
     payload = {
@@ -117,12 +127,14 @@ def test_s3_versioned_object_csv_gzip_constructor(s3_object_handler_fn_arn, vers
     assert result.get("body") == {"name": "hello", "value": "world"}
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_gzip_constructor(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "plain.txt.gz", "is_gzip": True}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("body") == "hello world"
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_versioned_object_gzip_constructor(s3_object_handler_fn_arn, versioned_bucket_name):
     key = "plain.txt.gz"
     payload = {
@@ -135,39 +147,45 @@ def test_s3_versioned_object_gzip_constructor(s3_object_handler_fn_arn, versione
     assert result.get("body") == "hello world"
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_gzip_transform(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "plain.txt.gz", "transform_gzip": True}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("body") == "hello world"
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_gzip_transform_in_place(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "plain.txt.gz", "transform_gzip": True, "in_place": True}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("body") == "hello world"
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_zip_transform(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "fileset.zip", "transform_zip": True}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("manifest") == ["1.txt", "2.txt"]
     assert result.get("body") == "This is file 2"
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_zip_transform_in_place(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "fileset.zip", "transform_zip": True, "in_place": True}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("manifest") == ["1.txt", "2.txt"]
     assert result.get("body") == "This is file 2"
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_zip_lzma_transform(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "fileset.zip.lzma", "transform_zip_lzma": True}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)
     assert result.get("manifest") == ["1.txt", "2.txt"]
     assert result.get("body") == "This is file 2"
 
 
+@pytest.mark.xdist_group(name="streaming")
 def test_s3_object_zip_lzma_transform_in_place(s3_object_handler_fn_arn, regular_bucket_name):
     payload = {"bucket": regular_bucket_name, "key": "fileset.zip.lzma", "transform_zip_lzma": True, "in_place": True}
     result = get_lambda_result_payload(s3_object_handler_fn_arn, payload)

tests/e2e/tracer/conftest.py

@@ -3,7 +3,7 @@
 from tests.e2e.tracer.infrastructure import TracerStack
 
 
-@pytest.fixture(autouse=True, scope="module")
+@pytest.fixture(autouse=True, scope="package")
 def infrastructure():
     """Setup and teardown logic for E2E test infrastructure
 

tests/e2e/tracer/test_tracer.py

@@ -36,6 +36,7 @@ def async_fn(infrastructure: dict) -> str:
     return infrastructure.get("AsyncCapture", "")
 
 
+@pytest.mark.xdist_group(name="tracer")
 def test_lambda_handler_trace_is_visible(basic_handler_fn_arn: str, basic_handler_fn: str):
     # GIVEN
     service = data_builder.build_service_name()
@@ -64,6 +65,7 @@ def test_lambda_handler_trace_is_visible(basic_handler_fn_arn: str, basic_handle
     assert len(trace.get_subsegment(name=method_subsegment)) == 2
 
 
+@pytest.mark.xdist_group(name="tracer")
 def test_lambda_handler_trace_multiple_functions_same_name(same_function_name_arn: str, same_function_name_fn: str):
     # GIVEN
     service = data_builder.build_service_name()
@@ -90,6 +92,7 @@ def test_lambda_handler_trace_multiple_functions_same_name(same_function_name_ar
     assert len(trace.get_subsegment(name=method_subsegment_comments)) == 1
 
 
+@pytest.mark.xdist_group(name="tracer")
 def test_async_trace_is_visible(async_fn_arn: str, async_fn: str):
     # GIVEN
     service = data_builder.build_service_name()