From 864bac94a36680378c6108e4cbae9e508acb4160 Mon Sep 17 00:00:00 2001 From: Igor D'Astolfo Date: Fri, 7 Mar 2025 14:52:32 +0100 Subject: [PATCH 1/8] Added parameter to disable tests during bootstrap pipeline --- src/template.yml | 81 ++++++++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 31 deletions(-) diff --git a/src/template.yml b/src/template.yml index f1118633c..d744a615c 100644 --- a/src/template.yml +++ b/src/template.yml @@ -111,6 +111,16 @@ Parameters: - ERROR - CRITICAL + PerformTests: + Description: >- + Should ADF perform tox tests during bootstrap? + Valid options are: true, false + Type: String + Default: "false" + AllowedValues: + - "true" + - "false" + AllowBootstrappingOfManagementAccount: Description: >- Would ADF need to bootstrap the Management Account of your AWS @@ -381,7 +391,7 @@ Resources: - !GetAtt JumpRoleApplication.Outputs.ManagerFunctionArn AccountFileProcessingFunction: - Type: 'AWS::Serverless::Function' + Type: "AWS::Serverless::Function" Properties: Handler: process_account_files.lambda_handler Description: >- @@ -399,7 +409,7 @@ Resources: Variables: MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId - ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] + ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel ACCOUNT_MANAGEMENT_STATEMACHINE_ARN: !Ref AccountManagementStateMachine ADF_PRIVILEGED_CROSS_ACCOUNT_ROLE_NAME: !Ref CrossAccountAccessRoleName @@ -450,7 +460,7 @@ Resources: - !Ref AccountProcessingLambdaBasePolicy AccountAliasConfigFunction: - Type: 'AWS::Serverless::Function' + Type: "AWS::Serverless::Function" Properties: Handler: configure_account_alias.lambda_handler Description: ADF - Account Management - Account Alias Configuration @@ -463,7 +473,7 @@ Resources: AWS_PARTITION: !Ref AWS::Partition MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId - ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] + ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel ADF_PRIVILEGED_CROSS_ACCOUNT_ROLE_NAME: !Ref CrossAccountAccessRoleName FunctionName: adf-account-management-config-alias @@ -499,7 +509,7 @@ Resources: - !Sub "arn:${AWS::Partition}:organizations::${AWS::AccountId}:account/${Organization.OrganizationId}/*" AccountTagConfigFunction: - Type: 'AWS::Serverless::Function' + Type: "AWS::Serverless::Function" Properties: Handler: configure_account_tags.lambda_handler Description: ADF - Account Management - Account Tag Configuration @@ -511,7 +521,7 @@ Resources: Variables: MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId - ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] + ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel FunctionName: adf-account-management-config-tags Role: !GetAtt AccountTagConfigFunctionRole.Arn @@ -550,7 +560,7 @@ Resources: - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/target_regions" AccountRegionConfigFunction: - Type: 'AWS::Serverless::Function' + Type: "AWS::Serverless::Function" Properties: Handler: configure_account_regions.lambda_handler Description: ADF - Account Management - Account Region Configuration @@ -562,7 +572,7 @@ Resources: Variables: MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId - ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] + ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel FunctionName: adf-account-management-config-region Role: !GetAtt AccountRegionConfigFunctionRole.Arn @@ -570,7 +580,7 @@ Resources: BuildMethod: python3.12 AccountOUConfigFunction: - Type: 'AWS::Serverless::Function' + Type: "AWS::Serverless::Function" Properties: Handler: configure_account_ou.lambda_handler Description: ADF - Account Management - Account OU Configuration @@ -582,7 +592,7 @@ Resources: Variables: MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId - ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] + ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel FunctionName: adf-account-management-config-ou Role: !GetAtt AccountOUConfigFunctionRole.Arn @@ -620,7 +630,7 @@ Resources: Resource: "*" GetAccountRegionsFunction: - Type: 'AWS::Serverless::Function' + Type: "AWS::Serverless::Function" Properties: Handler: get_account_regions.lambda_handler Description: ADF - Account Management - Get Default Regions @@ -633,7 +643,7 @@ Resources: AWS_PARTITION: !Ref AWS::Partition MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId - ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] + ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel ADF_PRIVILEGED_CROSS_ACCOUNT_ROLE_NAME: !Ref CrossAccountAccessRoleName FunctionName: adf-account-management-get-regions @@ -660,7 +670,7 @@ Resources: - !Ref AccountProcessingLambdaBasePolicy DeleteDefaultVPCFunction: - Type: 'AWS::Serverless::Function' + Type: "AWS::Serverless::Function" Properties: Handler: delete_default_vpc.lambda_handler Description: ADF - Account Management - Delete the Default VPCs @@ -673,7 +683,7 @@ Resources: AWS_PARTITION: !Ref AWS::Partition MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId - ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] + ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel ADF_PRIVILEGED_CROSS_ACCOUNT_ROLE_NAME: !Ref CrossAccountAccessRoleName FunctionName: adf-account-management-delete-default-vpc @@ -700,7 +710,7 @@ Resources: - !Ref AccountProcessingLambdaBasePolicy CreateAccountFunction: - Type: 'AWS::Serverless::Function' + Type: "AWS::Serverless::Function" Properties: Handler: create_account.lambda_handler Description: ADF - Account Management - Create Account @@ -712,7 +722,7 @@ Resources: Variables: MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId - ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] + ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel ADF_PRIVILEGED_CROSS_ACCOUNT_ROLE_NAME: !Ref CrossAccountAccessRoleName FunctionName: adf-account-management-create-account @@ -747,7 +757,7 @@ Resources: Resource: "*" RegisterAccountForSupportFunction: - Type: 'AWS::Serverless::Function' + Type: "AWS::Serverless::Function" Properties: Handler: register_account_for_support.lambda_handler Description: ADF - Account Management - Register support level @@ -759,7 +769,7 @@ Resources: Variables: MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId - ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] + ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel FunctionName: adf-account-management-register-support-level Role: !GetAtt RegisterAccountForSupportFunctionRole.Arn @@ -1101,7 +1111,6 @@ Resources: } } ### Account processing end - ADFSharedPythonLambdaLayerVersion: Type: "AWS::Serverless::LayerVersion" Properties: @@ -1152,7 +1161,7 @@ Resources: Location: account_bootstrapping_jump_role.yml Parameters: OrganizationId: !GetAtt Organization.OrganizationId - ADFVersion: !FindInMap ['Metadata', 'ADF', 'Version'] + ADFVersion: !FindInMap ["Metadata", "ADF", "Version"] LambdaLayer: !Ref ADFSharedPythonLambdaLayerVersion CrossAccountAccessRoleName: !Ref CrossAccountAccessRoleName DeploymentAccountId: !GetAtt DeploymentAccount.AccountId @@ -1311,8 +1320,7 @@ Resources: Resource: !GetAtt BootstrapTemplatesBucket.Arn - Effect: "Allow" Action: "s3:GetObject" - Resource: - !Sub "${BootstrapTemplatesBucket.Arn}/*" + Resource: !Sub "${BootstrapTemplatesBucket.Arn}/*" CrossAccountDeployBootstrapFunction: Type: "AWS::Serverless::Function" @@ -1491,9 +1499,9 @@ Resources: Statement: - Effect: Allow Action: - - 'support:CreateCase' - - 'support:DescribeSeverityLevels' - Resource: '*' + - "support:CreateCase" + - "support:DescribeSeverityLevels" + Resource: "*" CodeBuildPolicy: Type: "AWS::IAM::ManagedPolicy" @@ -1651,6 +1659,8 @@ Resources: Value: !Ref DeploymentAccountMainRegion - Name: ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN Value: !Ref AccountBootstrappingStateMachine + - Name: ADF_PERFORM_TESTS + Value: !Ref PerformTests Type: LINUX_CONTAINER Name: "adf-bootstrap-pipeline-build" ServiceRole: !GetAtt BootstrapCodeBuildRole.Arn @@ -1665,15 +1675,24 @@ Resources: commands: - >- pip install - -r requirements-dev.txt -r adf-build/requirements.txt - -r adf-build/requirements-dev.txt -r adf-build/shared/requirements.txt - -r adf-build/shared/requirements-dev.txt -r adf-build/shared/helpers/requirements.txt - -r adf-build/shared/helpers/requirements-dev.txt + --root-user-action ignore --quiet - - tox + - | + if [ "$ADF_PERFORM_TESTS" = "true" ] ; then + pip install \ + -r requirements-dev.txt \ + -r adf-build/requirements-dev.txt \ + -r adf-build/shared/requirements-dev.txt \ + -r adf-build/shared/helpers/requirements-dev.txt \ + --quiet \ + --root-user-action ignore + tox + else + echo "Skipping tests" + fi - >- docker run --privileged --rm public.ecr.aws/eks-distro-build-tooling/binfmt-misc:qemu-v7.0.0 @@ -2282,7 +2301,7 @@ Resources: Variables: MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId DEPLOYMENT_REGION: !Ref DeploymentAccountMainRegion - ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] + ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel Policies: - Version: "2012-10-17" From c989ff4d0e90510cd9df2be9730cbcaa47e52210 Mon Sep 17 00:00:00 2001 From: Igor D'Astolfo Date: Fri, 7 Mar 2025 17:43:21 +0100 Subject: [PATCH 2/8] Added default run-tests entry in adfconfig --- .../initial_commit/adfconfig.yml.j2 | 5 +++++ src/template.yml | 17 ++++------------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/src/lambda_codebase/initial_commit/adfconfig.yml.j2 b/src/lambda_codebase/initial_commit/adfconfig.yml.j2 index 8eef343ad..46eb201b3 100644 --- a/src/lambda_codebase/initial_commit/adfconfig.yml.j2 +++ b/src/lambda_codebase/initial_commit/adfconfig.yml.j2 @@ -37,6 +37,11 @@ config: allow-empty-target: disabled # ^ Needs to be set to "enabled" to activate. Defaults to "disabled" when # not set. + + # Tells whether or not run tests during the bootstrap pipeline, + # Needs to be set to "enabled" to activate. Defaults to "disabled" when not set + run-tests: disabled + # Optional config for multi-organization deployments of ADF: # org: # # Optional: Use this variable to define the AWS Organization in case of diff --git a/src/template.yml b/src/template.yml index d744a615c..66ed26804 100644 --- a/src/template.yml +++ b/src/template.yml @@ -111,16 +111,6 @@ Parameters: - ERROR - CRITICAL - PerformTests: - Description: >- - Should ADF perform tox tests during bootstrap? - Valid options are: true, false - Type: String - Default: "false" - AllowedValues: - - "true" - - "false" - AllowBootstrappingOfManagementAccount: Description: >- Would ADF need to bootstrap the Management Account of your AWS @@ -1659,8 +1649,6 @@ Resources: Value: !Ref DeploymentAccountMainRegion - Name: ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN Value: !Ref AccountBootstrappingStateMachine - - Name: ADF_PERFORM_TESTS - Value: !Ref PerformTests Type: LINUX_CONTAINER Name: "adf-bootstrap-pipeline-build" ServiceRole: !GetAtt BootstrapCodeBuildRole.Arn @@ -1673,6 +1661,8 @@ Resources: python: 3.12 pre_build: commands: + - pip install yq --root-user-action ignore --quiet + - ADF_PERFORM_TESTS=`cat adfconfig.yml | yq -r '.config."run-tests" // "disabled"'` - >- pip install -r adf-build/requirements.txt @@ -1681,7 +1671,8 @@ Resources: --root-user-action ignore --quiet - | - if [ "$ADF_PERFORM_TESTS" = "true" ] ; then + if [ "$ADF_PERFORM_TESTS" = "enabled" ] ; then + echo "Performing tests" pip install \ -r requirements-dev.txt \ -r adf-build/requirements-dev.txt \ From fc710f2efe873390db28fd7b746fc41255af8541 Mon Sep 17 00:00:00 2001 From: Igor D'Astolfo <130134134+igordust@users.noreply.github.com> Date: Mon, 10 Mar 2025 10:34:34 +0100 Subject: [PATCH 3/8] Update src/lambda_codebase/initial_commit/adfconfig.yml.j2 Co-authored-by: Simon Kok --- src/lambda_codebase/initial_commit/adfconfig.yml.j2 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lambda_codebase/initial_commit/adfconfig.yml.j2 b/src/lambda_codebase/initial_commit/adfconfig.yml.j2 index 46eb201b3..aa45ca7f9 100644 --- a/src/lambda_codebase/initial_commit/adfconfig.yml.j2 +++ b/src/lambda_codebase/initial_commit/adfconfig.yml.j2 @@ -38,10 +38,11 @@ config: # ^ Needs to be set to "enabled" to activate. Defaults to "disabled" when # not set. - # Tells whether or not run tests during the bootstrap pipeline, - # Needs to be set to "enabled" to activate. Defaults to "disabled" when not set - run-tests: disabled - + bootstrap-pipeline: + # Configure whether or not to run tests in the bootstrap pipeline. + # Needs to be set to "enabled" to activate. Defaults to "enabled" when not set. + run-tests: enabled + # Optional config for multi-organization deployments of ADF: # org: # # Optional: Use this variable to define the AWS Organization in case of From 5af192bd0554594c6624f4ed4c217c4d367b91f9 Mon Sep 17 00:00:00 2001 From: Igor D'Astolfo Date: Mon, 10 Mar 2025 10:36:04 +0100 Subject: [PATCH 4/8] Set 'enabled' as default value when run-tests is not set --- src/template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/template.yml b/src/template.yml index 66ed26804..697a5cee6 100644 --- a/src/template.yml +++ b/src/template.yml @@ -1662,7 +1662,7 @@ Resources: pre_build: commands: - pip install yq --root-user-action ignore --quiet - - ADF_PERFORM_TESTS=`cat adfconfig.yml | yq -r '.config."run-tests" // "disabled"'` + - ADF_PERFORM_TESTS=`cat adfconfig.yml | yq -r '.config."run-tests" // "enabled"'` - >- pip install -r adf-build/requirements.txt From 480647e663a5fe2abad42412dc8eff95589ef5ba Mon Sep 17 00:00:00 2001 From: Igor D'Astolfo Date: Mon, 10 Mar 2025 10:46:15 +0100 Subject: [PATCH 5/8] Added run-tests description in documentation --- docs/admin-guide.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 731eda722..67118d01a 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -20,8 +20,7 @@ - [Bootstrapping Recommendations](#bootstrapping-recommendations) - [Pipelines](#pipelines) - [Pipeline Parameters](#pipeline-parameters) - - [Using AWS CodeConnections for Bitbucket, GitHub, or - GitLab](#using-aws-codeconnections-for-bitbucket-github-or-gitlab) + - [Using AWS CodeConnections for Bitbucket, GitHub, or GitLab](#using-aws-codeconnections-for-bitbucket-github-or-gitlab) - [AWS CodeStar Connection](#aws-codestar-connection) - [Chaining Pipelines](#chaining-pipelines) - [Service Control Policies](#service-control-policies) @@ -253,6 +252,16 @@ Config has five components in `main-notification-endpoint`, `scp`, `scm`, The CodeCommit provider guide provides more details: [providers-guide.md.yml: CodeCommit](./providers-guide.md#codecommit). + - `run-tests` controls whether the bootstrap pipeline must run all ADF unit + tests or not. If not set, the default value is `enabled`. + Allowed values are `enabled` or `disabled` + Unit tests are important for the ADF development lifecycle to maintain code + stability and avoid regressions, but for installations where users aren't + updating the ADF core, they can be safely skipped to gain some improvement + in the bootstrap pipeline execution time. Disabling the tests means that + the development packages won't be installed and all tests will be skipped, + leading to a noticeable improvement on the overall execution time. + ## Accounts ### Management account From 838cd6beb3d07f190037f4632822d869556ac450 Mon Sep 17 00:00:00 2001 From: Igor D'Astolfo <130134134+igordust@users.noreply.github.com> Date: Mon, 10 Mar 2025 10:47:06 +0100 Subject: [PATCH 6/8] Update src/template.yml Co-authored-by: Simon Kok --- src/template.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/template.yml b/src/template.yml index 697a5cee6..1f12a5a70 100644 --- a/src/template.yml +++ b/src/template.yml @@ -1101,6 +1101,7 @@ Resources: } } ### Account processing end + ADFSharedPythonLambdaLayerVersion: Type: "AWS::Serverless::LayerVersion" Properties: From f717f3445fff7f64c368225cb4baae559a9f03ac Mon Sep 17 00:00:00 2001 From: Igor D'Astolfo Date: Mon, 10 Mar 2025 10:53:38 +0100 Subject: [PATCH 7/8] Moved run-tests under bootstrap-pipeline configuration entry --- docs/admin-guide.md | 21 ++++++++++++--------- src/template.yml | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 67118d01a..ff3176ba8 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -252,15 +252,18 @@ Config has five components in `main-notification-endpoint`, `scp`, `scm`, The CodeCommit provider guide provides more details: [providers-guide.md.yml: CodeCommit](./providers-guide.md#codecommit). - - `run-tests` controls whether the bootstrap pipeline must run all ADF unit - tests or not. If not set, the default value is `enabled`. - Allowed values are `enabled` or `disabled` - Unit tests are important for the ADF development lifecycle to maintain code - stability and avoid regressions, but for installations where users aren't - updating the ADF core, they can be safely skipped to gain some improvement - in the bootstrap pipeline execution time. Disabling the tests means that - the development packages won't be installed and all tests will be skipped, - leading to a noticeable improvement on the overall execution time. + - `bootstrap-pipeline` allows to configure certain aspects of the bootstrap pipeline + - `run-tests` controls whether the bootstrap pipeline must run all ADF unit + tests or not. If not set, the default value is `enabled`. + + Allowed values are `enabled` or `disabled` + + Unit tests are important for the ADF development lifecycle to maintain code + stability and avoid regressions, but for installations where users aren't + updating the ADF core, they can be safely skipped to gain some improvement + in the bootstrap pipeline execution time. Disabling the tests means that + the development packages won't be installed and all tests will be skipped, + leading to a noticeable improvement on the overall execution time. ## Accounts diff --git a/src/template.yml b/src/template.yml index 1f12a5a70..46fa7f187 100644 --- a/src/template.yml +++ b/src/template.yml @@ -1663,7 +1663,7 @@ Resources: pre_build: commands: - pip install yq --root-user-action ignore --quiet - - ADF_PERFORM_TESTS=`cat adfconfig.yml | yq -r '.config."run-tests" // "enabled"'` + - ADF_PERFORM_TESTS=`cat adfconfig.yml | yq -r '.config."bootstrap-pipeline"."run-tests" // "enabled"'` - >- pip install -r adf-build/requirements.txt From 727584ee8062c54777527f8be9c49214cdf6b529 Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Tue, 11 Mar 2025 09:30:47 +0100 Subject: [PATCH 8/8] Remove trailing whitespaces in docs/admin-guide.md --- docs/admin-guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index ff3176ba8..a7e1b6abd 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -257,12 +257,12 @@ Config has five components in `main-notification-endpoint`, `scp`, `scm`, tests or not. If not set, the default value is `enabled`. Allowed values are `enabled` or `disabled` - + Unit tests are important for the ADF development lifecycle to maintain code stability and avoid regressions, but for installations where users aren't updating the ADF core, they can be safely skipped to gain some improvement in the bootstrap pipeline execution time. Disabling the tests means that - the development packages won't be installed and all tests will be skipped, + the development packages won't be installed and all tests will be skipped, leading to a noticeable improvement on the overall execution time. ## Accounts