diff --git a/.codebuild/cleanup_workflow.yml b/.codebuild/cleanup_workflow.yml index 4d7fdfe7b..e5ca317dc 100644 --- a/.codebuild/cleanup_workflow.yml +++ b/.codebuild/cleanup_workflow.yml @@ -9,4 +9,4 @@ phases: build: commands: - yarn production-build - - cd packages/amplify-e2e-tests && yarn clean-cb-e2e-resources \ No newline at end of file + - cd packages/amplify-codegen-e2e-tests && yarn clean-cb-e2e-resources \ No newline at end of file diff --git a/packages/amplify-codegen-e2e-core/src/init/initProjectHelper.ts b/packages/amplify-codegen-e2e-core/src/init/initProjectHelper.ts index 879b3d66c..b8c888e41 100644 --- a/packages/amplify-codegen-e2e-core/src/init/initProjectHelper.ts +++ b/packages/amplify-codegen-e2e-core/src/init/initProjectHelper.ts @@ -1,4 +1,4 @@ -import { nspawn as spawn, getCLIPath, singleSelect, addCircleCITags } from '..'; +import { nspawn as spawn, getCLIPath, singleSelect, addCITags } from '..'; import { KEY_DOWN_ARROW, AmplifyFrontend } from '../utils'; import { amplifyRegions } from '../configure'; import { v4 as uuid } from 'uuid'; @@ -32,7 +32,7 @@ export function initJSProjectWithProfile(cwd: string, settings: Object = {}): Pr }; } - addCircleCITags(cwd); + addCITags(cwd); const cliArgs = ['init']; const providerConfigSpecified = !!s.providerConfig && typeof s.providerConfig === 'object'; @@ -89,7 +89,7 @@ export function initJSProjectWithProfile(cwd: string, settings: Object = {}): Pr export function initAndroidProjectWithProfile(cwd: string, settings: Object): Promise { const s = { ...defaultSettings, ...settings }; - addCircleCITags(cwd); + addCITags(cwd); return new Promise((resolve, reject) => { spawn(getCLIPath(), ['init'], { @@ -120,7 +120,7 @@ export function initAndroidProjectWithProfile(cwd: string, settings: Object): Pr .wait('Try "amplify add api" to create a backend API and then "amplify push" to deploy everything') .run((err: Error) => { if (!err) { - addCircleCITags(cwd); + addCITags(cwd); resolve(); } else { @@ -133,7 +133,7 @@ export function initAndroidProjectWithProfile(cwd: string, settings: Object): Pr export function initIosProjectWithProfile(cwd: string, settings: Object): Promise { const s = { ...defaultSettings, ...settings }; - addCircleCITags(cwd); + addCITags(cwd); return new Promise((resolve, reject) => { spawn(getCLIPath(), ['init'], { @@ -163,7 +163,7 @@ export function initIosProjectWithProfile(cwd: string, settings: Object): Promis .wait('Try "amplify add api" to create a backend API and then "amplify push" to deploy everything') .run((err: Error) => { if (!err) { - addCircleCITags(cwd); + addCITags(cwd); resolve(); } else { @@ -176,7 +176,7 @@ export function initIosProjectWithProfile(cwd: string, settings: Object): Promis export function initFlutterProjectWithProfile(cwd: string, settings: Object): Promise { const s = { ...defaultSettings, ...settings }; - addCircleCITags(cwd); + addCITags(cwd); return new Promise((resolve, reject) => { let chain = spawn(getCLIPath(), ['init'], { cwd, stripColors: true }) @@ -220,7 +220,7 @@ export function initProjectWithAccessKey( ): Promise { const s = { ...defaultSettings, ...settings }; - addCircleCITags(cwd); + addCITags(cwd); return new Promise((resolve, reject) => { let chain = spawn(getCLIPath(), ['init'], { @@ -275,7 +275,7 @@ export function initProjectWithAccessKey( } export function initNewEnvWithAccessKey(cwd: string, s: { envName: string; accessKeyId: string; secretAccessKey: string }): Promise { - addCircleCITags(cwd); + addCITags(cwd); return new Promise((resolve, reject) => { let chain = spawn(getCLIPath(), ['init'], { @@ -314,7 +314,7 @@ export function initNewEnvWithAccessKey(cwd: string, s: { envName: string; acces } export function initNewEnvWithProfile(cwd: string, s: { envName: string }): Promise { - addCircleCITags(cwd); + addCITags(cwd); return new Promise((resolve, reject) => { spawn(getCLIPath(), ['init'], { @@ -354,7 +354,7 @@ export function amplifyInitSandbox(cwd: string, settings: {}): Promise { }; } - addCircleCITags(cwd); + addCITags(cwd); return new Promise((resolve, reject) => { spawn(getCLIPath(), ['init'], { cwd, stripColors: true, env }) diff --git a/packages/amplify-codegen-e2e-core/src/utils/add-circleci-tags.ts b/packages/amplify-codegen-e2e-core/src/utils/add-ci-tags.ts similarity index 55% rename from packages/amplify-codegen-e2e-core/src/utils/add-circleci-tags.ts rename to packages/amplify-codegen-e2e-core/src/utils/add-ci-tags.ts index 6ec1623ba..3a557e593 100644 --- a/packages/amplify-codegen-e2e-core/src/utils/add-circleci-tags.ts +++ b/packages/amplify-codegen-e2e-core/src/utils/add-ci-tags.ts @@ -9,6 +9,49 @@ declare global { } } } +export const addCITags = (projectPath: string): void => { + if (process.env && process.env['CODEBUILD']) { + addCodeBuildCITags(projectPath); + } + else if(process.env && process.env['CIRCLECI']) { + addCircleCITags(projectPath); + } +} + +/** + * Add CI tags for code build + * Refer https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html + * @param projectPath + */ +export const addCodeBuildCITags = (projectPath: string): void => { + const tags = stateManager.getProjectTags(projectPath); + + const addTagIfNotExist = (key: string, value: string): void => { + if (!tags.find(t => t.Key === key)) { + tags.push({ + Key: key, + Value: value, + }); + } + }; + addTagIfNotExist('codebuild', sanitizeTagValue(process.env['CODEBUILD'] || 'N/A')); + addTagIfNotExist('codebuild:batch_build_identifier', sanitizeTagValue(process.env['CODEBUILD_BATCH_BUILD_IDENTIFIER'] || 'N/A')); + addTagIfNotExist('codebuild:build_id', sanitizeTagValue(process.env['CODEBUILD_BUILD_ID'] || 'N/A')); + // exposed by custom CLI test environment + if (global.getTestName) { + addTagIfNotExist('jest:test_name', sanitizeTagValue(global.getTestName().substr(0, 255) || 'N/A')); + } + if (global.getHookName) { + addTagIfNotExist('jest:hook_name', sanitizeTagValue(global.getHookName().substr(0, 255) || 'N/A')); + } + if (global.getDescibeBlocks) { + global.getDescibeBlocks().forEach((blockName, i) => { + addTagIfNotExist(`jest:describe_${i + 1}`, sanitizeTagValue(blockName.substr(0, 255) || 'N/A')); + }); + } + + stateManager.setProjectFileTags(projectPath, tags); +}; export const addCircleCITags = (projectPath: string): void => { if (process.env && process.env['CIRCLECI']) { diff --git a/packages/amplify-codegen-e2e-core/src/utils/index.ts b/packages/amplify-codegen-e2e-core/src/utils/index.ts index 7ee8cb13c..f2583e257 100644 --- a/packages/amplify-codegen-e2e-core/src/utils/index.ts +++ b/packages/amplify-codegen-e2e-core/src/utils/index.ts @@ -4,7 +4,7 @@ import * as rimraf from 'rimraf'; import { config } from 'dotenv'; import execa from 'execa'; -export * from './add-circleci-tags'; +export * from './add-ci-tags'; export * from './api'; export * from './appsync'; export * from './envVars'; diff --git a/packages/amplify-codegen-e2e-core/src/utils/pinpoint.ts b/packages/amplify-codegen-e2e-core/src/utils/pinpoint.ts index be62de1cf..bf9cff28f 100644 --- a/packages/amplify-codegen-e2e-core/src/utils/pinpoint.ts +++ b/packages/amplify-codegen-e2e-core/src/utils/pinpoint.ts @@ -1,5 +1,5 @@ import { Pinpoint } from 'aws-sdk'; -import { getCLIPath, nspawn as spawn, singleSelect, amplifyRegions, addCircleCITags, KEY_DOWN_ARROW } from '..'; +import { getCLIPath, nspawn as spawn, singleSelect, amplifyRegions, addCITags, KEY_DOWN_ARROW } from '..'; import _ from 'lodash'; const settings = { @@ -70,7 +70,7 @@ export async function pinpointAppExist(pinpointProjectId: string): Promise { - addCircleCITags(cwd); + addCITags(cwd); return new Promise((resolve, reject) => { let chain = spawn(getCLIPath(), ['init'], { diff --git a/packages/amplify-codegen-e2e-core/src/utils/sdk-calls.ts b/packages/amplify-codegen-e2e-core/src/utils/sdk-calls.ts index f29210974..4987ec612 100644 --- a/packages/amplify-codegen-e2e-core/src/utils/sdk-calls.ts +++ b/packages/amplify-codegen-e2e-core/src/utils/sdk-calls.ts @@ -42,8 +42,8 @@ export const bucketNotExists = async (bucket: string) => { } }; -export const deleteS3Bucket = async (bucket: string) => { - const s3 = new S3(); +export const deleteS3Bucket = async (bucket: string, providedS3Client: S3 | undefined = undefined) => { + const s3 = providedS3Client ? providedS3Client : new S3(); let continuationToken: Required> = undefined; const objectKeyAndVersion = []; let truncated = false; diff --git a/packages/amplify-codegen-e2e-tests/package.json b/packages/amplify-codegen-e2e-tests/package.json index 16cbb9ab9..f030252d2 100644 --- a/packages/amplify-codegen-e2e-tests/package.json +++ b/packages/amplify-codegen-e2e-tests/package.json @@ -19,7 +19,8 @@ "scripts": { "e2e": "npm run setup-profile && jest --verbose", "setup-profile": "ts-node ./src/configure_tests.ts", - "clean-e2e-resources": "ts-node ./src/cleanup-e2e-resources.ts" + "clean-e2e-resources": "ts-node ./src/cleanup-e2e-resources.ts", + "clean-cb-e2e-resources": "ts-node ./src/cleanup-cb-e2e-resources.ts" }, "dependencies": { "@aws-amplify/amplify-codegen-e2e-core": "1.4.9", diff --git a/packages/amplify-codegen-e2e-tests/src/cleanup-cb-e2e-resources.ts b/packages/amplify-codegen-e2e-tests/src/cleanup-cb-e2e-resources.ts new file mode 100644 index 000000000..9424f5b81 --- /dev/null +++ b/packages/amplify-codegen-e2e-tests/src/cleanup-cb-e2e-resources.ts @@ -0,0 +1,735 @@ +/* eslint-disable spellcheck/spell-checker, camelcase, jsdoc/require-jsdoc, @typescript-eslint/no-explicit-any */ +import { CodeBuild } from 'aws-sdk'; +import { config } from 'dotenv'; +import yargs from 'yargs'; +import * as aws from 'aws-sdk'; +import _ from 'lodash'; +import fs from 'fs-extra'; +import path from 'path'; +import { deleteS3Bucket, sleep } from '@aws-amplify/amplify-codegen-e2e-core'; + +// Ensure to update scripts/split-e2e-tests.ts is also updated this gets updated +const AWS_REGIONS_TO_RUN_TESTS = [ + 'us-east-1', + 'us-east-2', + 'us-west-2', + 'eu-west-2', + 'eu-central-1', + 'ap-northeast-1', + 'ap-southeast-1', + 'ap-southeast-2', +]; + +const reportPath = path.normalize(path.join(__dirname, '..', 'amplify-e2e-reports', 'stale-resources.json')); + +const MULTI_JOB_APP = ''; +const ORPHAN = ''; +const UNKNOWN = ''; + +type StackInfo = { + stackName: string; + stackStatus: string; + resourcesFailedToDelete?: string[]; + tags: Record; + region: string; + jobId: string; + cbInfo?: CodeBuild.Build; +}; + +type AmplifyAppInfo = { + appId: string; + name: string; + region: string; + backends: Record; +}; + +type S3BucketInfo = { + name: string; + jobId?: string; + cbInfo?: CodeBuild.Build; +}; + +type IamRoleInfo = { + name: string; + cbInfo?: CodeBuild.Build; +}; + +type ReportEntry = { + jobId?: string; + buildBatchArn?: string; + buildComplete?: boolean; + cbJobDetails?: CodeBuild.Build; + buildStatus?: string; + amplifyApps: Record; + stacks: Record; + buckets: Record; + roles: Record; +}; + +type JobFilterPredicate = (job: ReportEntry) => boolean; + +type CBJobInfo = { + buildBatchArn: string; + projectName: string; + buildComplete: boolean; + cbJobDetails: CodeBuild.Build; + buildStatus: string; +}; + +type AWSAccountInfo = { + accountId: string; + accessKeyId: string; + secretAccessKey: string; + sessionToken: string; +}; + +const BUCKET_TEST_REGEX = /test/; +const IAM_TEST_REGEX = /!RotateE2eAwsToken-e2eTestContextRole|-integtest$|^amplify-|^eu-|^us-|^ap-/; +const STALE_DURATION_MS = 2 * 60 * 60 * 1000; // 2 hours in milliseconds + +const isCI = (): boolean => process.env.CI && process.env.CODEBUILD ? true : false; +/* + * Exit on expired token as all future requests will fail. + */ +const handleExpiredTokenException = (): void => { + console.log('Token expired. Exiting...'); + process.exit(); +}; + +/** + * We define a resource as viable for deletion if it matches TEST_REGEX in the name, and if it is > STALE_DURATION_MS old. + */ +const testBucketStalenessFilter = (resource: aws.S3.Bucket): boolean => { + const isTestResource = resource.Name.match(BUCKET_TEST_REGEX); + const isStaleResource = (Date.now() - resource.CreationDate.getMilliseconds()) > STALE_DURATION_MS; + return isTestResource && isStaleResource; +}; + +const testRoleStalenessFilter = (resource: aws.IAM.Role): boolean => { + const isTestResource = resource.RoleName.match(IAM_TEST_REGEX); + const isStaleResource = (Date.now() - resource.CreateDate.getMilliseconds()) > STALE_DURATION_MS; + return isTestResource && isStaleResource; +}; + +/** + * Get all S3 buckets in the account, and filter down to the ones we consider stale. + */ +const getOrphanS3TestBuckets = async (account: AWSAccountInfo): Promise => { + const s3Client = new aws.S3(getAWSConfig(account)); + const listBucketResponse = await s3Client.listBuckets().promise(); + const staleBuckets = listBucketResponse.Buckets.filter(testBucketStalenessFilter); + return staleBuckets.map(it => ({ name: it.Name })); +}; + +/** + * Get all iam roles in the account, and filter down to the ones we consider stale. + */ +const getOrphanTestIamRoles = async (account: AWSAccountInfo): Promise => { + const iamClient = new aws.IAM(getAWSConfig(account)); + const listRoleResponse = await iamClient.listRoles({ MaxItems: 1000 }).promise(); + const staleRoles = listRoleResponse.Roles.filter(testRoleStalenessFilter); + return staleRoles.map(it => ({ name: it.RoleName })); +}; + +/** + * Get the relevant AWS config object for a given account and region. + */ + const getAWSConfig = ({ accessKeyId, secretAccessKey, sessionToken }: AWSAccountInfo, region?: string): unknown => ({ + credentials: { + accessKeyId, + secretAccessKey, + sessionToken, + }, + ...(region ? { region } : {}), + maxRetries: 10, +}); + +/** + * Returns a list of Amplify Apps in the region. The apps includes information about the CircleCI build that created the app + * This is determined by looking at tags of the backend environments that are associated with the Apps + * @param account aws account to query for amplify Apps + * @param region aws region to query for amplify Apps + * @returns Promise a list of Amplify Apps in the region with build info + */ +const getAmplifyApps = async (account: AWSAccountInfo, region: string): Promise => { + const amplifyClient = new aws.Amplify(getAWSConfig(account, region)); + const amplifyApps = await amplifyClient.listApps({ maxResults: 50 }).promise(); // keeping it to 50 as max supported is 50 + const result: AmplifyAppInfo[] = []; + for (const app of amplifyApps.apps) { + const backends: Record = {}; + try { + const backendEnvironments = await amplifyClient.listBackendEnvironments({ appId: app.appId, maxResults: 50 }).promise(); + for (const backendEnv of backendEnvironments.backendEnvironments) { + const buildInfo = await getStackDetails(backendEnv.stackName, account, region); + if (buildInfo) { + backends[backendEnv.environmentName] = buildInfo; + } + } + } catch (e) { + console.log(e); + } + result.push({ + appId: app.appId, + name: app.name, + region, + backends, + }); + } + return result; +}; + +/** + * Return the CodeBuild job id looking at `codebuild:build_id` in the tags + * @param tags Tags associated with the resource + * @returns build number or undefined + */ +const getJobId = (tags: aws.CloudFormation.Tags = []): string | undefined => { + const jobId = tags.find(tag => tag.Key === 'codebuild:build_id')?.Value; + return jobId; +}; + +/** + * Gets detail about a stack including the details about CircleCI job that created the stack. If a stack + * has status of `DELETE_FAILED` then it also includes the list of physical id of resources that caused + * deletion failures + * + * @param stackName name of the stack + * @param account account + * @param region region + * @returns stack details + */ +const getStackDetails = async (stackName: string, account: AWSAccountInfo, region: string): Promise => { + const cfnClient = new aws.CloudFormation(getAWSConfig(account, region)); + const stack = await cfnClient.describeStacks({ StackName: stackName }).promise(); + const tags = stack.Stacks.length && stack.Stacks[0].Tags; + const stackStatus = stack.Stacks[0].StackStatus; + let resourcesFailedToDelete: string[] = []; + if (stackStatus === 'DELETE_FAILED') { + // TODO: We need to investigate if we should go ahead and remove the resources to prevent account getting cluttered + const resources = await cfnClient.listStackResources({ StackName: stackName }).promise(); + resourcesFailedToDelete = resources.StackResourceSummaries.filter(r => r.ResourceStatus === 'DELETE_FAILED').map( + r => r.LogicalResourceId, + ); + } + const jobId = getJobId(tags); + return { + stackName, + stackStatus, + resourcesFailedToDelete, + region, + tags: tags.reduce((acc, tag) => ({ ...acc, [tag.Key]: tag.Value }), {}), + jobId + }; +}; + +const getStacks = async (account: AWSAccountInfo, region: string): Promise => { + const cfnClient = new aws.CloudFormation(getAWSConfig(account, region)); + const stacks = await cfnClient + .listStacks({ + StackStatusFilter: [ + 'CREATE_COMPLETE', + 'ROLLBACK_FAILED', + 'DELETE_FAILED', + 'UPDATE_COMPLETE', + 'UPDATE_ROLLBACK_FAILED', + 'UPDATE_ROLLBACK_COMPLETE', + 'IMPORT_COMPLETE', + 'IMPORT_ROLLBACK_FAILED', + 'IMPORT_ROLLBACK_COMPLETE', + ], + }) + .promise(); + + // We are interested in only the root stacks that are deployed by amplify-cli + const rootStacks = stacks.StackSummaries.filter(stack => !stack.RootId); + const results: StackInfo[] = []; + for (const stack of rootStacks) { + try { + const details = await getStackDetails(stack.StackName, account, region); + if (details) { + results.push(details); + } + } catch { + // don't want to barf and fail e2e tests + } + } + return results; +}; + +const getCodeBuildClient = (): CodeBuild => { + return new CodeBuild({ + apiVersion: '2016-10-06', + region: 'us-east-1', + }); +}; + +const getJobCodeBuildDetails = async (jobIds: string[]): Promise => { + if (jobIds.length === 0) { + return []; + } + const client = getCodeBuildClient(); + try { + const { builds } = await client.batchGetBuilds({ ids: jobIds }).promise(); + return builds; + } catch(e) { + console.log(e); + } +}; + +const getS3Buckets = async (account: AWSAccountInfo): Promise => { + const s3Client = new aws.S3(getAWSConfig(account)); + const buckets = await s3Client.listBuckets().promise(); + const result: S3BucketInfo[] = []; + for (const bucket of buckets.Buckets) { + try { + const bucketDetails = await s3Client.getBucketTagging({ Bucket: bucket.Name }).promise(); + const jobId = getJobId(bucketDetails.TagSet); + if (jobId) { + result.push({ + name: bucket.Name, + jobId + }); + } + } catch (e) { + if (e.code !== 'NoSuchTagSet' && e.code !== 'NoSuchBucket') { + throw e; + } + result.push({ + name: bucket.Name, + }); + } + } + return result; +}; + +/** + * extract and moves CircleCI job details + */ +const extractCCIJobInfo = ( + record: S3BucketInfo | StackInfo | AmplifyAppInfo, + buildInfos: Record + ): CBJobInfo => { + const buildId = _.get(record, ['0', 'jobId']); + return { + buildBatchArn: _.get(buildInfos, [ buildId, '0', 'buildBatchArn' ]), + projectName: _.get(buildInfos, [ buildId, '0', 'projectName' ]), + buildComplete: _.get(buildInfos, [ buildId, '0', 'buildComplete' ]), + cbJobDetails: _.get(buildInfos, [ buildId, '0' ]), + buildStatus: _.get(buildInfos, [ buildId, '0', 'buildStatus' ]) + }; +} + + +/** + * Merges stale resources and returns a list grouped by the CodeBuild jobId. Amplify Apps that don't have + * any backend environment are grouped as Orphan apps and apps that have Backend created by different CodeBuild jobs are + * grouped as MULTI_JOB_APP. Any resource that do not have a CodeBuild job is grouped under UNKNOWN + */ +const mergeResourcesByCCIJob = async ( + amplifyApp: AmplifyAppInfo[], + cfnStacks: StackInfo[], + s3Buckets: S3BucketInfo[], + orphanS3Buckets: S3BucketInfo[], + orphanIamRoles: IamRoleInfo[], +): Promise> => { + const result: Record = {}; + + const stacksByJobId = _.groupBy(cfnStacks, (stack: StackInfo) => _.get(stack, ['jobId'], UNKNOWN)); + + const bucketByJobId = _.groupBy(s3Buckets, (bucketInfo: S3BucketInfo) => _.get(bucketInfo, ['jobId'], UNKNOWN)); + + const amplifyAppByJobId = _.groupBy(amplifyApp, (appInfo: AmplifyAppInfo) => { + if (Object.keys(appInfo.backends).length === 0) { + return ORPHAN; + } + + const buildIds = _.groupBy(appInfo.backends, backendInfo => _.get(backendInfo, ['jobId'], UNKNOWN)); + if (Object.keys(buildIds).length === 1) { + return Object.keys(buildIds)[0]; + } + + return MULTI_JOB_APP; + }); + const codeBuildJobIds: string[] = _.uniq([...Object.keys(stacksByJobId), ...Object.keys(bucketByJobId), ...Object.keys(amplifyAppByJobId)]) + .filter((jobId: string) => jobId !== UNKNOWN && jobId !== ORPHAN && jobId !== MULTI_JOB_APP) + const buildInfos = await getJobCodeBuildDetails(codeBuildJobIds); + const buildInfosByJobId = _.groupBy(buildInfos, (build: CodeBuild.Build) => _.get(build, ['id'])); + _.mergeWith( + result, + _.pickBy(amplifyAppByJobId, (__, key) => key !== MULTI_JOB_APP), + (val, src, key) => ({ + ...val, + ...extractCCIJobInfo(src, buildInfosByJobId), + jobId: key, + amplifyApps: src, + }), + ); + + _.mergeWith( + result, + stacksByJobId, + (__: unknown, key: string) => key !== ORPHAN, + (val, src, key) => ({ + ...val, + ...extractCCIJobInfo(src, buildInfosByJobId), + jobId: key, + stacks: src, + }), + ); + + _.mergeWith(result, bucketByJobId, (val, src, key) => ({ + ...val, + ...extractCCIJobInfo(src, buildInfosByJobId), + jobId: key, + buckets: src, + })); + + const orphanBuckets = { + [ORPHAN]: orphanS3Buckets, + }; + + _.mergeWith(result, orphanBuckets, (val, src, key) => ({ + ...val, + jobId: key, + buckets: src, + })); + + const orphanIamRolesGroup = { + [ORPHAN]: orphanIamRoles, + }; + + _.mergeWith(result, orphanIamRolesGroup, (val, src, key) => ({ + ...val, + jobId: key, + roles: src, + })); + + return result; +}; + +const deleteAmplifyApps = async (account: AWSAccountInfo, accountIndex: number, apps: AmplifyAppInfo[]): Promise => { + await Promise.all(apps.map(app => deleteAmplifyApp(account, accountIndex, app))); +}; + +const deleteAmplifyApp = async (account: AWSAccountInfo, accountIndex: number, app: AmplifyAppInfo): Promise => { + const { name, appId, region } = app; + console.log(`${generateAccountInfo(account, accountIndex)} Deleting App ${name}(${appId})`); + const amplifyClient = new aws.Amplify(getAWSConfig(account, region)); + try { + await amplifyClient.deleteApp({ appId }).promise(); + } catch (e) { + console.log(`${generateAccountInfo(account, accountIndex)} Deleting Amplify App ${appId} failed with the following error`, e); + if (e.code === 'ExpiredTokenException') { + handleExpiredTokenException(); + } + } +}; + +const deleteIamRoles = async (account: AWSAccountInfo, accountIndex: number, roles: IamRoleInfo[]): Promise => { + // Sending consecutive delete role requests is throwing Rate limit exceeded exception. + // We introduce a brief delay between batches + const batchSize = 20; + for (var i = 0; i < roles.length; i += batchSize) { + const rolesToDelete = roles.slice(i, i + batchSize); + await Promise.all(rolesToDelete.map(role => deleteIamRole(account, accountIndex, role))); + await sleep(5000); + } +}; + +const deleteIamRole = async (account: AWSAccountInfo, accountIndex: number, role: IamRoleInfo): Promise => { + const { name: roleName } = role; + try { + console.log(`${generateAccountInfo(account, accountIndex)} Deleting Iam Role ${roleName}`); + const iamClient = new aws.IAM(getAWSConfig(account)); + await deleteAttachedRolePolicies(account, accountIndex, roleName); + await deleteRolePolicies(account, accountIndex, roleName); + await iamClient.deleteRole({ RoleName: roleName }).promise(); + } catch (e) { + console.log(`${generateAccountInfo(account, accountIndex)} Deleting iam role ${roleName} failed with error ${e.message}`); + if (e.code === 'ExpiredTokenException') { + handleExpiredTokenException(); + } + } +}; + +const deleteAttachedRolePolicies = async ( + account: AWSAccountInfo, + accountIndex: number, + roleName: string, +): Promise => { + const iamClient = new aws.IAM(getAWSConfig(account)); + const rolePolicies = await iamClient.listAttachedRolePolicies({ RoleName: roleName }).promise(); + await Promise.all(rolePolicies.AttachedPolicies.map(policy => detachIamAttachedRolePolicy(account, accountIndex, roleName, policy))); +}; + +const detachIamAttachedRolePolicy = async ( + account: AWSAccountInfo, + accountIndex: number, + roleName: string, + policy: aws.IAM.AttachedPolicy, +): Promise => { + try { + console.log(`${generateAccountInfo(account, accountIndex)} Detach Iam Attached Role Policy ${policy.PolicyName}`); + const iamClient = new aws.IAM(getAWSConfig(account)); + await iamClient.detachRolePolicy({ RoleName: roleName, PolicyArn: policy.PolicyArn }).promise(); + } catch (e) { + console.log(`${generateAccountInfo(account, accountIndex)} Detach iam role policy ${policy.PolicyName} failed with error ${e.message}`); + if (e.code === 'ExpiredTokenException') { + handleExpiredTokenException(); + } + } +}; + +const deleteRolePolicies = async ( + account: AWSAccountInfo, + accountIndex: number, + roleName: string, +): Promise => { + const iamClient = new aws.IAM(getAWSConfig(account)); + const rolePolicies = await iamClient.listRolePolicies({ RoleName: roleName }).promise(); + await Promise.all(rolePolicies.PolicyNames.map(policy => deleteIamRolePolicy(account, accountIndex, roleName, policy))); +}; + +const deleteIamRolePolicy = async ( + account: AWSAccountInfo, + accountIndex: number, + roleName: string, + policyName: string, +): Promise => { + try { + console.log(`${generateAccountInfo(account, accountIndex)} Deleting Iam Role Policy ${policyName}`); + const iamClient = new aws.IAM(getAWSConfig(account)); + await iamClient.deleteRolePolicy({ RoleName: roleName, PolicyName: policyName }).promise(); + } catch (e) { + console.log(`${generateAccountInfo(account, accountIndex)} Deleting iam role policy ${policyName} failed with error ${e.message}`); + if (e.code === 'ExpiredTokenException') { + handleExpiredTokenException(); + } + } +}; + +const deleteBuckets = async (account: AWSAccountInfo, accountIndex: number, buckets: S3BucketInfo[]): Promise => { + await Promise.all(buckets.map(bucket => deleteBucket(account, accountIndex, bucket))); +}; + +const deleteBucket = async (account: AWSAccountInfo, accountIndex: number, bucket: S3BucketInfo): Promise => { + const { name } = bucket; + try { + console.log(`${generateAccountInfo(account, accountIndex)} Deleting S3 Bucket ${name}`); + const s3 = new aws.S3(getAWSConfig(account)); + await deleteS3Bucket(name, s3); + } catch (e) { + console.log(`${generateAccountInfo(account, accountIndex)} Deleting bucket ${name} failed with error ${e.message}`); + if (e.code === 'ExpiredTokenException') { + handleExpiredTokenException(); + } + } +}; + +const deleteCfnStacks = async (account: AWSAccountInfo, accountIndex: number, stacks: StackInfo[]): Promise => { + await Promise.all(stacks.map(stack => deleteCfnStack(account, accountIndex, stack))); +}; + +const deleteCfnStack = async (account: AWSAccountInfo, accountIndex: number, stack: StackInfo): Promise => { + const { stackName, region, resourcesFailedToDelete } = stack; + const resourceToRetain = resourcesFailedToDelete.length ? resourcesFailedToDelete : undefined; + console.log(`${generateAccountInfo(account, accountIndex)} Deleting CloudFormation stack ${stackName}`); + try { + const cfnClient = new aws.CloudFormation(getAWSConfig(account, region)); + await cfnClient.deleteStack({ StackName: stackName, RetainResources: resourceToRetain }).promise(); + await cfnClient.waitFor('stackDeleteComplete', { StackName: stackName }).promise(); + } catch (e) { + console.log(`Deleting CloudFormation stack ${stackName} failed with error ${e.message}`); + if (e.code === 'ExpiredTokenException') { + handleExpiredTokenException(); + } + } +}; + +const generateReport = (jobs: _.Dictionary): void => { + fs.ensureFileSync(reportPath); + fs.writeFileSync(reportPath, JSON.stringify(jobs, null, 4)); +}; + +/** + * While we basically fan-out deletes elsewhere in this script, leaving the app->cfn->bucket delete process + * serial within a given account, it's not immediately clear if this is necessary, but seems possibly valuable. + */ +const deleteResources = async ( + account: AWSAccountInfo, + accountIndex: number, + staleResources: Record, +): Promise => { + for (const jobId of Object.keys(staleResources)) { + const resources = staleResources[jobId]; + if (resources.amplifyApps) { + await deleteAmplifyApps(account, accountIndex, Object.values(resources.amplifyApps)); + } + + if (resources.stacks) { + await deleteCfnStacks(account, accountIndex, Object.values(resources.stacks)); + } + + if (resources.buckets) { + await deleteBuckets(account, accountIndex, Object.values(resources.buckets)); + } + + if (resources.roles) { + await deleteIamRoles(account, accountIndex, Object.values(resources.roles)); + } + } +}; + +/** + * Grab the right CircleCI filter based on args passed in. + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const getFilterPredicate = (args: any): JobFilterPredicate => { + const filterByJobId = (jobId: string) => (job: ReportEntry) => job.jobId === jobId; + const filterByBuildBatchArn = (buildBatchArn: string) => (job: ReportEntry) => job.buildBatchArn === buildBatchArn; + const filterAllStaleResources = () => (job: ReportEntry) => job.buildComplete || job.jobId === ORPHAN; + + if (args._.length === 0) { + return filterAllStaleResources(); + } + if (args._[0] === 'buildBatchArn') { + return filterByBuildBatchArn(args.buildBatchArn as string); + } + if (args._[0] === 'job') { + return filterByJobId(args.jobId as string); + } + throw Error('Invalid args config'); +}; + +/** + * Retrieve the accounts to process for potential cleanup. By default we will attempt + * to get all accounts within the root account organization. + */ +const getAccountsToCleanup = async (): Promise => { + // This script runs using the codebuild project role to begin with + const stsClient = new aws.STS({ + apiVersion: '2011-06-15' + }); + const assumeRoleResForE2EParent = await stsClient + .assumeRole({ + RoleArn: process.env.TEST_ACCOUNT_ROLE, + RoleSessionName: `testSession${Math.floor(Math.random() * 100000)}`, + // One hour + DurationSeconds: 1 * 60 * 60, + }) + .promise(); + const e2eParentAccountCred = { + accessKeyId: assumeRoleResForE2EParent.Credentials.AccessKeyId, + secretAccessKey: assumeRoleResForE2EParent.Credentials.SecretAccessKey, + sessionToken: assumeRoleResForE2EParent.Credentials.SessionToken + } + const stsClientForE2E = new aws.STS({ + apiVersion: '2011-06-15', + credentials: e2eParentAccountCred + }); + const parentAccountIdentity = await stsClientForE2E.getCallerIdentity().promise() + const orgApi = new aws.Organizations({ + apiVersion: '2016-11-28', + // the region where the organization exists + region: 'us-east-1', + credentials: e2eParentAccountCred + }); + try { + const orgAccounts = await orgApi.listAccounts().promise(); + const accountCredentialPromises = orgAccounts.Accounts.map(async account => { + if (account.Id === parentAccountIdentity.Account) { + return { + accountId: account.Id, + ...e2eParentAccountCred + }; + } + const randomNumber = Math.floor(Math.random() * 100000); + const assumeRoleRes = await stsClientForE2E + .assumeRole({ + RoleArn: `arn:aws:iam::${account.Id}:role/OrganizationAccountAccessRole`, + RoleSessionName: `testSession${randomNumber}`, + // One hour + DurationSeconds: 1 * 60 * 60, + }) + .promise(); + return { + accountId: account.Id, + accessKeyId: assumeRoleRes.Credentials.AccessKeyId, + secretAccessKey: assumeRoleRes.Credentials.SecretAccessKey, + sessionToken: assumeRoleRes.Credentials.SessionToken, + }; + }); + return await Promise.all(accountCredentialPromises); + } catch (e) { + console.error(e); + console.log('Error assuming child account role. This could be because the script is already running from within a child account. Running on current AWS account only.'); + return [ + { + accountId: parentAccountIdentity.Account, + ...e2eParentAccountCred + }, + ]; + } +}; + +const cleanupAccount = async (account: AWSAccountInfo, accountIndex: number, filterPredicate: JobFilterPredicate): Promise => { + const appPromises = AWS_REGIONS_TO_RUN_TESTS.map(region => getAmplifyApps(account, region)); + const stackPromises = AWS_REGIONS_TO_RUN_TESTS.map(region => getStacks(account, region)); + const bucketPromise = getS3Buckets(account); + const orphanBucketPromise = getOrphanS3TestBuckets(account); + const orphanIamRolesPromise = getOrphanTestIamRoles(account); + + const apps = (await Promise.all(appPromises)).flat(); + const stacks = (await Promise.all(stackPromises)).flat(); + const buckets = await bucketPromise; + const orphanBuckets = await orphanBucketPromise; + const orphanIamRoles = await orphanIamRolesPromise; + + const allResources = await mergeResourcesByCCIJob(apps, stacks, buckets, orphanBuckets, orphanIamRoles); + const staleResources = _.pickBy(allResources, filterPredicate); + + generateReport(staleResources); + await deleteResources(account, accountIndex, staleResources); + console.log(`${generateAccountInfo(account, accountIndex)} Cleanup done!`); +}; + +const generateAccountInfo = (account: AWSAccountInfo, accountIndex: number): string => { + return (`[ACCOUNT ${accountIndex}][${account.accountId}]`); +}; + +/** + * Execute the cleanup script. + * Cleanup will happen in parallel across all accounts within a given organization, + * based on the requested filter parameters (i.e. for a given workflow, job, or all stale resources). + * Logs are emitted for given account ids anywhere we've fanned out, but we use an indexing scheme instead + * of account ids since the logs these are written to will be effectively public. + */ +const cleanup = async (): Promise => { + const args = yargs + .command('*', 'clean up all the stale resources') + .command('buildBatchArn ', 'clean all the resources created by batch build', _yargs => { + _yargs.positional('buildBatchArn', { + describe: 'ARN of batch build', + type: 'string', + demandOption: '', + }); + }) + .command('job ', 'clean all the resource created by a job', _yargs => { + _yargs.positional('jobId', { + describe: 'job id of the job', + type: 'string', + }); + }) + .help().argv; + config(); + + const filterPredicate = getFilterPredicate(args); + const accounts = await getAccountsToCleanup(); + accounts.map((account, i) => { + console.log(`${generateAccountInfo(account, i)} is under cleanup`); + }); + await Promise.all(accounts.map((account, i) => cleanupAccount(account, i, filterPredicate))); + console.log('Done cleaning all accounts!'); +}; + +cleanup(); diff --git a/packages/amplify-codegen-e2e-tests/src/init-special-cases/index.ts b/packages/amplify-codegen-e2e-tests/src/init-special-cases/index.ts index d26ab4d28..0490c7ffa 100644 --- a/packages/amplify-codegen-e2e-tests/src/init-special-cases/index.ts +++ b/packages/amplify-codegen-e2e-tests/src/init-special-cases/index.ts @@ -1,5 +1,5 @@ import path from 'path'; -import { nspawn as spawn, getCLIPath, singleSelect, amplifyRegions, addCircleCITags, KEY_DOWN_ARROW } from '@aws-amplify/amplify-codegen-e2e-core'; +import { nspawn as spawn, getCLIPath, singleSelect, amplifyRegions, addCITags, KEY_DOWN_ARROW } from '@aws-amplify/amplify-codegen-e2e-core'; import fs from 'fs-extra'; import os from 'os'; @@ -42,7 +42,7 @@ export async function initWithoutCredentialFileAndNoNewUserSetup(projRoot) { } async function initWorkflow(cwd: string, settings: { accessKeyId: string; secretAccessKey: string; region: string }): Promise { - addCircleCITags(cwd); + addCITags(cwd); return new Promise((resolve, reject) => { let chain = spawn(getCLIPath(), ['init'], {