From f98f810939dd5e23ca4c2e3262aac95f939061d9 Mon Sep 17 00:00:00 2001 From: phani-srikar Date: Mon, 10 Jul 2023 21:51:06 -0700 Subject: [PATCH 1/6] feat: add e2e split tests script --- .circleci/config.base.yml | 2 +- .circleci/config.yml | 2 +- .codebuild/cleanup_workflow.yml | 2 +- .codebuild/e2e_workflow.yml | 92 +- ...ifact-storage-path-allow-list-codebuild.ts | 8 +- package.json | 6 +- scripts/cci/api.ts | 117 +++ .../get-test-timing-from-cci-job-metrics.ts | 37 + scripts/cci/job.data.json | 787 ++++++++++++++++++ scripts/cci/test-timings.data.json | 5 + scripts/cci/utils.ts | 70 ++ scripts/split-e2e-tests-codebuild.ts | 207 +++++ shared-scripts.sh | 2 +- 13 files changed, 1326 insertions(+), 11 deletions(-) create mode 100644 scripts/cci/api.ts create mode 100644 scripts/cci/get-test-timing-from-cci-job-metrics.ts create mode 100644 scripts/cci/job.data.json create mode 100644 scripts/cci/test-timings.data.json create mode 100644 scripts/cci/utils.ts create mode 100644 scripts/split-e2e-tests-codebuild.ts diff --git a/.circleci/config.base.yml b/.circleci/config.base.yml index f1da5a4f..4762bd86 100644 --- a/.circleci/config.base.yml +++ b/.circleci/config.base.yml @@ -303,7 +303,7 @@ jobs: yarn clean-e2e-resources no_output_timeout: 20m - store_artifacts: - path: ~/repo/packages/amplify-e2e-tests/amplify-e2e-reports + path: ~/repo/packages/amplify-codegen-e2e-tests/amplify-e2e-reports working_directory: ~/repo verify-api-extract: diff --git a/.circleci/config.yml b/.circleci/config.yml index f6e9a51d..e3a5af6b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -306,7 +306,7 @@ jobs: yarn clean-e2e-resources no_output_timeout: 20m - store_artifacts: - path: ~/repo/packages/amplify-e2e-tests/amplify-e2e-reports + path: ~/repo/packages/amplify-codegen-e2e-tests/amplify-e2e-reports verify-api-extract: docker: *ref_4 working_directory: ~/repo diff --git a/.codebuild/cleanup_workflow.yml b/.codebuild/cleanup_workflow.yml index 4d7fdfe7..e5ca317d 100644 --- a/.codebuild/cleanup_workflow.yml +++ b/.codebuild/cleanup_workflow.yml @@ -9,4 +9,4 @@ phases: build: commands: - yarn production-build - - cd packages/amplify-e2e-tests && yarn clean-cb-e2e-resources \ No newline at end of file + - cd packages/amplify-codegen-e2e-tests && yarn clean-cb-e2e-resources \ No newline at end of file diff --git a/.codebuild/e2e_workflow.yml b/.codebuild/e2e_workflow.yml index 74d4e57b..6ecafc38 100644 --- a/.codebuild/e2e_workflow.yml +++ b/.codebuild/e2e_workflow.yml @@ -1,8 +1,8 @@ +# auto generated file. DO NOT EDIT manually version: 0.2 env: shell: bash compute-type: BUILD_GENERAL1_MEDIUM - batch: fast-fail: false build-graph: @@ -28,3 +28,93 @@ batch: compute-type: BUILD_GENERAL1_MEDIUM depend-on: - build_linux + - identifier: add_codegen_android_add_codegen_ios_add_codegen_js_build_app_android + buildspec: .codebuild/run_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + TEST_SUITE: >- + src/__tests__/add-codegen-android.test.ts|src/__tests__/add-codegen-ios.test.ts|src/__tests__/add-codegen-js.test.ts|src/__tests__/build-app-android.test.ts + CLI_REGION: us-east-1 + depend-on: + - publish_to_local_registry + - identifier: >- + build_app_ts_configure_codegen_android_configure_codegen_ios_configure_codegen_js + buildspec: .codebuild/run_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + TEST_SUITE: >- + src/__tests__/build-app-ts.test.ts|src/__tests__/configure-codegen-android.test.ts|src/__tests__/configure-codegen-ios.test.ts|src/__tests__/configure-codegen-js.test.ts + CLI_REGION: us-east-2 + depend-on: + - publish_to_local_registry + - identifier: >- + datastore_modelgen_android_datastore_modelgen_flutter_datastore_modelgen_ios_datastore_modelgen_js + buildspec: .codebuild/run_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + TEST_SUITE: >- + src/__tests__/datastore-modelgen-android.test.ts|src/__tests__/datastore-modelgen-flutter.test.ts|src/__tests__/datastore-modelgen-ios.test.ts|src/__tests__/datastore-modelgen-js.test.ts + CLI_REGION: eu-west-2 + depend-on: + - publish_to_local_registry + - identifier: env_codegen_feature_flags_graphql_codegen_android_graphql_codegen_ios + buildspec: .codebuild/run_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + TEST_SUITE: >- + src/__tests__/env-codegen.test.ts|src/__tests__/feature-flags.test.ts|src/__tests__/graphql-codegen-android.test.ts|src/__tests__/graphql-codegen-ios.test.ts + CLI_REGION: eu-central-1 + depend-on: + - publish_to_local_registry + - identifier: >- + graphql_codegen_js_graphql_documents_generator_model_introspection_codegen_pull_codegen + buildspec: .codebuild/run_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + TEST_SUITE: >- + src/__tests__/graphql-codegen-js.test.ts|src/__tests__/graphql-documents-generator.test.ts|src/__tests__/model-introspection-codegen.test.ts|src/__tests__/pull-codegen.test.ts + CLI_REGION: ap-northeast-1 + depend-on: + - publish_to_local_registry + - identifier: >- + push_codegen_android_push_codegen_ios_push_codegen_js_remove_codegen_android + buildspec: .codebuild/run_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + TEST_SUITE: >- + src/__tests__/push-codegen-android.test.ts|src/__tests__/push-codegen-ios.test.ts|src/__tests__/push-codegen-js.test.ts|src/__tests__/remove-codegen-android.test.ts + CLI_REGION: ap-southeast-1 + depend-on: + - publish_to_local_registry + - identifier: remove_codegen_ios_remove_codegen_js + buildspec: .codebuild/run_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + TEST_SUITE: >- + src/__tests__/remove-codegen-ios.test.ts|src/__tests__/remove-codegen-js.test.ts + CLI_REGION: ap-southeast-2 + depend-on: + - publish_to_local_registry + - identifier: build_app_swift + buildspec: .codebuild/run_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_SMALL + variables: + TEST_SUITE: src/__tests__/build-app-swift.test.ts + CLI_REGION: us-west-2 + USE_PARENT_ACCOUNT: 1 + depend-on: + - publish_to_local_registry + - identifier: cleanup_e2e_resources + buildspec: .codebuild/cleanup_e2e_resources.yml + env: + compute-type: BUILD_GENERAL1_SMALL + depend-on: + - add_codegen_android_add_codegen_ios_add_codegen_js_build_app_android diff --git a/.codebuild/scripts/artifact-storage-path-allow-list-codebuild.ts b/.codebuild/scripts/artifact-storage-path-allow-list-codebuild.ts index f89fa334..60824994 100644 --- a/.codebuild/scripts/artifact-storage-path-allow-list-codebuild.ts +++ b/.codebuild/scripts/artifact-storage-path-allow-list-codebuild.ts @@ -9,10 +9,10 @@ * artifacts: files: - - $CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports/* + - $CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports/* * * From the above job, 'path' includes the following: - * $CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports + * $CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports * * Those paths must be included in this list. * @@ -27,6 +27,6 @@ * will automatically normalize these paths for Windows if it detects it. */ export const ARTIFACT_STORAGE_PATH_ALLOW_LIST_CODEBUILD = [ - '$CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/', - '$CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports' + '$CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/', + '$CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports' ]; \ No newline at end of file diff --git a/package.json b/package.json index 04939650..52c6e184 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,9 @@ "view-test-artifact": "./scripts/view-test-artifacts.sh", "cleanup-stale-resources": "source ./scripts/cloud-utils.sh && cleanupStaleResources", "cloud-e2e-cb": "source scripts/cloud-utils.sh && cloudE2E", - "cloud-e2e-cb-beta": "source scripts/cloud-utils.sh && cloudE2EBeta" + "cloud-e2e-cb-beta": "source scripts/cloud-utils.sh && cloudE2EBeta", + "split-codebuild-e2e-tests": "yarn ts-node ./scripts/split-e2e-tests-codebuild.ts && git add .codebuild/e2e_workflow.yml", + "update-test-timing-data": "ts-node ./scripts/cci/get-test-timing-from-cci-job-metrics.ts" }, "bugs": { "url": "https://github.com/aws-amplify/amplify-codegen/issues" @@ -55,7 +57,7 @@ "hooks": { "commit-msg": "commitlint -E HUSKY_GIT_PARAMS", "pre-push": "npm run lint && npm run test-changed", - "pre-commit": "yarn split-e2e-tests && pretty-quick --staged" + "pre-commit": "yarn split-codebuild-e2e-tests" } }, "author": "Amazon Web Services", diff --git a/scripts/cci/api.ts b/scripts/cci/api.ts new file mode 100644 index 00000000..68c6ec56 --- /dev/null +++ b/scripts/cci/api.ts @@ -0,0 +1,117 @@ +import axios from 'axios'; + +type ReportingWindow = 'last-7-days' | 'last-90-days' | 'last-24-hours' | 'last-30-days' | 'last-60-days'; +export type CircleCIClientDefaults = { + defaultBranch: string; + defaultWorkflow: string; + vcs: string; + projectSlug: string; + projectName: string; +}; +export class CircleCIAPIClient { + private headers; + private options: CircleCIClientDefaults; + private slug: string; + constructor(token: string, options: CircleCIClientDefaults) { + this.headers = { + 'Circle-Token': token, + }; + this.options = options; + this.slug = `${options.vcs}/${options.projectSlug}/${options.projectName}`; + } + + /** + * Returns a sequence of jobs for a workflow. + * + * https://circleci.com/docs/api/v2/index.html#operation/listWorkflowJobs + * @returns + */ + getWorkflowJobs = async (workflowId: string = this.options.defaultWorkflow) => { + const result = await axios.get(`https://circleci.com/api/v2/workflow/${workflowId}/job`, { + headers: this.headers, + }); + return result.data; + }; + /** + * Returns a job's details. + * + * https://circleci.com/docs/api/v2/index.html#operation/getJobDetails + * @param jobId + * @returns + */ + getJobDetails = async (jobId: string) => { + const result = await axios.get(`https://circleci.com/api/v2/project/${this.slug}/job/${jobId}`, { + headers: this.headers, + }); + return result.data; + }; + /** + * Returns a single job's artifacts. + * + * https://circleci.com/docs/api/v2/index.html#operation/getJobArtifacts + * @param jobId + * @returns + */ + getJobArtifacts = async (jobId: string) => { + const result = await axios.get(`https://circleci.com/api/v2/project/${this.slug}/${jobId}/artifacts`, { + headers: this.headers, + }); + return result.data; + }; + /** + * Get test metadata for a single job + * + * https://circleci.com/docs/api/v2/index.html#operation/getTests + * @param jobId + * @returns + */ + getJobTests = async (jobId: string) => { + const result = await axios.get(`https://circleci.com/api/v2/project/${this.slug}/${jobId}/tests`, { + headers: this.headers, + }); + return result.data; + }; + /** + * Get summary metrics for a project workflow's jobs. + * + * https://circleci.com/docs/api/v2/index.html#operation/getProjectWorkflowJobMetrics + * + * @param workflowName + * @param branch + * @param reportingWindow + * @returns + */ + getAllJobMetrics = async ( + workflowName: string = this.options.defaultWorkflow, + branch: string = this.options.defaultBranch, + reportingWindow: ReportingWindow = 'last-30-days', + ) => { + const result = await axios.get(`https://circleci.com/api/v2/insights/${this.slug}/workflows/${workflowName}/jobs`, { + headers: this.headers, + params: { + branch: branch, + 'reporting-window': reportingWindow, + }, + }); + return result.data; + }; + + /** + * Get test metrics for a project's workflows. + * + * https://circleci.com/docs/api/v2/index.html#operation/getProjectWorkflowTestMetrics + * @param workflowName + * @param branch + * @param reportingWindow + * @returns + */ + getAllTestMetrics = async (workflowName: string = this.options.defaultWorkflow, branch: string = this.options.defaultBranch) => { + const result = await axios.get(`https://circleci.com/api/v2/insights/${this.slug}/workflows/${workflowName}/test-metrics`, { + headers: this.headers, + params: { + branch: branch, + }, + }); + return result.data; + }; +} \ No newline at end of file diff --git a/scripts/cci/get-test-timing-from-cci-job-metrics.ts b/scripts/cci/get-test-timing-from-cci-job-metrics.ts new file mode 100644 index 00000000..5ee83941 --- /dev/null +++ b/scripts/cci/get-test-timing-from-cci-job-metrics.ts @@ -0,0 +1,37 @@ +import { join } from 'path'; +import { REPO_ROOT, getCCIClient, getTestFiles, getTestNameFromPath, getTimingsFromJobsData, saveJobMetrics, saveTestTimings } from './utils'; + +async function main(): Promise { + const client = getCCIClient(); + console.log('Fetching job metrics...'); + const data = await client.getAllJobMetrics(); + saveJobMetrics(data); + + const testSuites = getTestFiles(join(REPO_ROOT, 'packages', 'amplify-codegen-e2e-tests')); + + const jobTimings = getTimingsFromJobsData(); + const testRuntimes = testSuites.map(t => { + const oldName = getTestNameFromPath(t); + if (jobTimings.has(oldName)) { + return { + test: t, + medianRuntime: jobTimings.get(oldName) as number, + }; + } else { + console.log('Could not find timing for:', t); + return { + test: t, + medianRuntime: 10, // default for unknown + }; + } + }); + testRuntimes.sort((a, b) => { + return a.medianRuntime - b.medianRuntime; + }); + saveTestTimings({ + lastUpdated: new Date().toISOString(), + totalTestFiles: testRuntimes.length, + timingData: testRuntimes, + }); +} +main(); \ No newline at end of file diff --git a/scripts/cci/job.data.json b/scripts/cci/job.data.json new file mode 100644 index 00000000..0b386416 --- /dev/null +++ b/scripts/cci/job.data.json @@ -0,0 +1,787 @@ +{ + "next_page_token": null, + "items": [ + { + "name": "add-codegen-android-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 390, + "mean": 418, + "median": 426, + "p95": 432, + "max": 434, + "standard_deviation": 17, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 831, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:33.479Z", + "window_end": "2023-07-10T22:11:18.542Z" + }, + { + "name": "add-codegen-ios-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 350, + "mean": 380, + "median": 383, + "p95": 411, + "max": 417, + "standard_deviation": 25, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 757, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:32.473Z", + "window_end": "2023-07-10T22:10:38.898Z" + }, + { + "name": "add-codegen-js-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 522, + "mean": 532, + "median": 530, + "p95": 547, + "max": 551, + "standard_deviation": 10, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 1060, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:32.445Z", + "window_end": "2023-07-10T22:13:06.147Z" + }, + { + "name": "build", + "metrics": { + "total_runs": 8, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 0, + "mean": 111, + "median": 127, + "p95": 193, + "max": 195, + "standard_deviation": 75, + "total_duration": 0 + }, + "success_rate": 0.75, + "total_credits_used": 291, + "throughput": 0.2666666667 + }, + "window_start": "2023-06-19T17:28:54.389Z", + "window_end": "2023-07-10T22:03:07.678Z" + }, + { + "name": "build-app-android-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 618, + "mean": 741, + "median": 717, + "p95": 893, + "max": 917, + "standard_deviation": 111, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 1481, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:34.079Z", + "window_end": "2023-07-10T22:19:26.837Z" + }, + { + "name": "build-app-swift-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 1177, + "mean": 1221, + "median": 1210, + "p95": 1285, + "max": 1304, + "standard_deviation": 44, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 12130, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:33.769Z", + "window_end": "2023-07-10T22:24:41.323Z" + }, + { + "name": "build-app-ts-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 917, + "mean": 986, + "median": 975, + "p95": 1057, + "max": 1066, + "standard_deviation": 54, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 1967, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:32.161Z", + "window_end": "2023-07-10T22:21:20.385Z" + }, + { + "name": "configure-codegen-android-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 337, + "mean": 378, + "median": 380, + "p95": 409, + "max": 410, + "standard_deviation": 29, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 752, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:31.581Z", + "window_end": "2023-07-10T22:10:41.952Z" + }, + { + "name": "configure-codegen-ios-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 415, + "mean": 434, + "median": 433, + "p95": 453, + "max": 458, + "standard_deviation": 14, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 861, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:33.151Z", + "window_end": "2023-07-10T22:11:30.111Z" + }, + { + "name": "configure-codegen-js-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 352, + "mean": 403, + "median": 396, + "p95": 462, + "max": 471, + "standard_deviation": 45, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 804, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:34.375Z", + "window_end": "2023-07-10T22:11:27.834Z" + }, + { + "name": "datastore-modelgen-android-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 424, + "mean": 445, + "median": 449, + "p95": 462, + "max": 466, + "standard_deviation": 14, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 888, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:32.073Z", + "window_end": "2023-07-10T22:11:40.442Z" + }, + { + "name": "datastore-modelgen-flutter-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 542, + "mean": 564, + "median": 551, + "p95": 613, + "max": 632, + "standard_deviation": 33, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 1124, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:31.390Z", + "window_end": "2023-07-10T22:14:42.234Z" + }, + { + "name": "datastore-modelgen-ios-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 502, + "mean": 532, + "median": 534, + "p95": 551, + "max": 553, + "standard_deviation": 18, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 1059, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:33.381Z", + "window_end": "2023-07-10T22:13:15.115Z" + }, + { + "name": "datastore-modelgen-js-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 435, + "mean": 454, + "median": 454, + "p95": 475, + "max": 481, + "standard_deviation": 15, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 902, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:34.377Z", + "window_end": "2023-07-10T22:12:13.663Z" + }, + { + "name": "deploy", + "metrics": { + "total_runs": 3, + "failed_runs": 0, + "successful_runs": 3, + "median_credits_used": 0, + "duration_metrics": { + "min": 42, + "mean": 44, + "median": 45, + "p95": 45, + "max": 46, + "standard_deviation": 2, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 42, + "throughput": 0.1 + }, + "window_start": "2023-06-19T17:53:02.522Z", + "window_end": "2023-06-29T17:10:35.513Z" + }, + { + "name": "done_with_node_e2e_tests", + "metrics": { + "total_runs": 3, + "failed_runs": 0, + "successful_runs": 3, + "median_credits_used": 0, + "duration_metrics": { + "min": 25, + "mean": 26, + "median": 26, + "p95": 28, + "max": 29, + "standard_deviation": 2, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 24, + "throughput": 0.1 + }, + "window_start": "2023-06-19T17:52:37.213Z", + "window_end": "2023-06-29T17:09:49.727Z" + }, + { + "name": "env-codegen-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 1, + "successful_runs": 5, + "median_credits_used": 0, + "duration_metrics": { + "min": 479, + "mean": 584, + "median": 597, + "p95": 624, + "max": 629, + "standard_deviation": 53, + "total_duration": 0 + }, + "success_rate": 0.8333333333, + "total_credits_used": 1165, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:32.807Z", + "window_end": "2023-07-10T22:12:10.121Z" + }, + { + "name": "feature-flags-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 403, + "mean": 432, + "median": 435, + "p95": 445, + "max": 446, + "standard_deviation": 15, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 861, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:33.104Z", + "window_end": "2023-07-10T22:11:24.854Z" + }, + { + "name": "graphql-codegen-android-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 362, + "mean": 388, + "median": 397, + "p95": 404, + "max": 405, + "standard_deviation": 20, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 772, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:33.116Z", + "window_end": "2023-07-10T22:10:44.059Z" + }, + { + "name": "graphql-codegen-ios-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 414, + "mean": 428, + "median": 430, + "p95": 439, + "max": 442, + "standard_deviation": 10, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 848, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:31.872Z", + "window_end": "2023-07-10T22:11:23.361Z" + }, + { + "name": "graphql-codegen-js-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 362, + "mean": 384, + "median": 382, + "p95": 412, + "max": 419, + "standard_deviation": 19, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 761, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:31.069Z", + "window_end": "2023-07-10T22:11:11.346Z" + }, + { + "name": "graphql-documents-generator-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 2, + "successful_runs": 4, + "median_credits_used": 0, + "duration_metrics": { + "min": 626, + "mean": 745, + "median": 787, + "p95": 805, + "max": 807, + "standard_deviation": 79, + "total_duration": 0 + }, + "success_rate": 0.6666666667, + "total_credits_used": 1486, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:33.428Z", + "window_end": "2023-07-10T22:14:38.281Z" + }, + { + "name": "model-introspection-codegen-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 3, + "successful_runs": 3, + "median_credits_used": 0, + "duration_metrics": { + "min": 540, + "mean": 585, + "median": 593, + "p95": 608, + "max": 612, + "standard_deviation": 25, + "total_duration": 0 + }, + "success_rate": 0.5, + "total_credits_used": 1165, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:32.848Z", + "window_end": "2023-07-10T22:13:48.972Z" + }, + { + "name": "publish_to_local_registry", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 61, + "mean": 79, + "median": 78, + "p95": 95, + "max": 99, + "standard_deviation": 12, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 153, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:30:54.032Z", + "window_end": "2023-07-10T22:04:09.237Z" + }, + { + "name": "pull-codegen-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 1, + "successful_runs": 5, + "median_credits_used": 0, + "duration_metrics": { + "min": 631, + "mean": 713, + "median": 718, + "p95": 754, + "max": 762, + "standard_deviation": 43, + "total_duration": 0 + }, + "success_rate": 0.8333333333, + "total_credits_used": 1422, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:32.477Z", + "window_end": "2023-07-10T22:14:41.457Z" + }, + { + "name": "push-codegen-android-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 1, + "successful_runs": 5, + "median_credits_used": 0, + "duration_metrics": { + "min": 524, + "mean": 735, + "median": 779, + "p95": 788, + "max": 790, + "standard_deviation": 104, + "total_duration": 0 + }, + "success_rate": 0.8333333333, + "total_credits_used": 1468, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:32.825Z", + "window_end": "2023-07-10T22:12:54.137Z" + }, + { + "name": "push-codegen-ios-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 1, + "successful_runs": 5, + "median_credits_used": 0, + "duration_metrics": { + "min": 471, + "mean": 649, + "median": 682, + "p95": 705, + "max": 710, + "standard_deviation": 88, + "total_duration": 0 + }, + "success_rate": 0.8333333333, + "total_credits_used": 1292, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:34.036Z", + "window_end": "2023-07-10T22:12:00.368Z" + }, + { + "name": "push-codegen-js-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 1, + "successful_runs": 5, + "median_credits_used": 0, + "duration_metrics": { + "min": 541, + "mean": 754, + "median": 785, + "p95": 826, + "max": 829, + "standard_deviation": 106, + "total_duration": 0 + }, + "success_rate": 0.8333333333, + "total_credits_used": 1502, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:31.734Z", + "window_end": "2023-07-10T22:13:10.326Z" + }, + { + "name": "remove-codegen-android-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 357, + "mean": 383, + "median": 383, + "p95": 408, + "max": 409, + "standard_deviation": 21, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 761, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:32.235Z", + "window_end": "2023-07-10T22:10:59.009Z" + }, + { + "name": "remove-codegen-ios-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 362, + "mean": 402, + "median": 394, + "p95": 442, + "max": 447, + "standard_deviation": 31, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 799, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:33.734Z", + "window_end": "2023-07-10T22:11:38.351Z" + }, + { + "name": "remove-codegen-js-e2e-test", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 490, + "mean": 517, + "median": 521, + "p95": 529, + "max": 530, + "standard_deviation": 15, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 1029, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:32:31.843Z", + "window_end": "2023-07-10T22:13:01.480Z" + }, + { + "name": "test-l", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 137, + "mean": 144, + "median": 139, + "p95": 160, + "max": 164, + "standard_deviation": 10, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 285, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:30:54.665Z", + "window_end": "2023-07-10T22:05:25.354Z" + }, + { + "name": "test-w", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 487, + "mean": 622, + "median": 616, + "p95": 778, + "max": 807, + "standard_deviation": 123, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 2451, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:30:54.893Z", + "window_end": "2023-07-10T22:14:40.901Z" + }, + { + "name": "verify-api-extract", + "metrics": { + "total_runs": 6, + "failed_runs": 0, + "successful_runs": 6, + "median_credits_used": 0, + "duration_metrics": { + "min": 46, + "mean": 67, + "median": 71, + "p95": 74, + "max": 75, + "standard_deviation": 10, + "total_duration": 0 + }, + "success_rate": 1, + "total_credits_used": 130, + "throughput": 0.2 + }, + "window_start": "2023-06-19T17:30:54.371Z", + "window_end": "2023-07-10T22:04:22.075Z" + } + ] +} \ No newline at end of file diff --git a/scripts/cci/test-timings.data.json b/scripts/cci/test-timings.data.json new file mode 100644 index 00000000..830c96e2 --- /dev/null +++ b/scripts/cci/test-timings.data.json @@ -0,0 +1,5 @@ +{ + "lastUpdated": "2023-07-11T04:44:38.813Z", + "totalTestFiles": 0, + "timingData": [] +} \ No newline at end of file diff --git a/scripts/cci/utils.ts b/scripts/cci/utils.ts new file mode 100644 index 00000000..098854f1 --- /dev/null +++ b/scripts/cci/utils.ts @@ -0,0 +1,70 @@ +import { CircleCIAPIClient, CircleCIClientDefaults } from './api'; +import * as fs from 'fs-extra'; +import * as glob from 'glob'; +import { join } from 'path'; + +export const REPO_ROOT = join(__dirname, '..', '..'); +const JOB_METRICS_PATH = join(REPO_ROOT, 'scripts', 'cci', 'job.data.json'); +const TEST_TIMINGS_PATH = join(REPO_ROOT, 'scripts', 'cci', 'test-timings.data.json'); + +export const ClientDefaults: CircleCIClientDefaults = { + defaultBranch: 'main', + defaultWorkflow: 'build_test_deploy', + vcs: 'github', + projectSlug: 'aws-amplify', + projectName: 'amplify-codegen', +}; + +export const getCCIClient = () => { + if (!process.env.CIRCLECI_TOKEN) { + throw new Error('CIRCLECI_TOKEN is not set. Export it to your terminal, then try again.'); + } + return new CircleCIAPIClient(process.env.CIRCLECI_TOKEN, ClientDefaults); +} + +export function saveJobMetrics(data: any): any { + console.log(`saving job metrics to ${JOB_METRICS_PATH}`); + fs.writeFileSync(JOB_METRICS_PATH, JSON.stringify(data, null, 2)); +} + +export function getTestFiles(dir: string, pattern = 'src/**/*.test.ts'): string[] { + return glob.sync(pattern, { cwd: dir }); +} + +export function getTimingsFromJobsData() { + const jobData = JSON.parse(fs.readFileSync(JOB_METRICS_PATH, 'utf-8')); + const jobTimings: Map = new Map(); + for (let job of jobData.items) { + const testName = getTestNameFromJobName(job.name); + const duration = Math.floor(job.metrics.duration_metrics.median / 60); + if (jobTimings.has(testName)) { + jobTimings.set(testName, Math.max(jobTimings.get(testName)!, duration)); + } else { + jobTimings.set(testName, duration); + } + } + return jobTimings; +} + +function getTestNameFromJobName(jobName: string) { + // first, remove any - from the name + const endIndex = jobName.lastIndexOf('-l'); + const name = jobName.substring(0, endIndex); + return name; +} + +export const getTestNameFromPath = (testSuitePath: string): string => { + const startIndex = testSuitePath.lastIndexOf('/') + 1; + const endIndex = testSuitePath.lastIndexOf('.test'); + return testSuitePath + .substring(startIndex, endIndex) + .split('.e2e') + .join('') + .split('.') + .join('-'); +}; + +export function saveTestTimings(data: any): any { + console.log(`saving timing data to ${TEST_TIMINGS_PATH}`); + fs.writeFileSync(TEST_TIMINGS_PATH, JSON.stringify(data, null, 2)); +} \ No newline at end of file diff --git a/scripts/split-e2e-tests-codebuild.ts b/scripts/split-e2e-tests-codebuild.ts new file mode 100644 index 00000000..16431826 --- /dev/null +++ b/scripts/split-e2e-tests-codebuild.ts @@ -0,0 +1,207 @@ +import * as glob from 'glob'; +import * as fs from 'fs-extra'; +import { join } from 'path'; +import * as yaml from 'js-yaml'; + +// Ensure to update packages/amplify-codegen-e2e-tests/src/cleanup-e2e-resources.ts is also updated this gets updated +const AWS_REGIONS_TO_RUN_TESTS = [ + 'us-east-1', + 'us-east-2', + 'us-west-2', + 'eu-west-2', + 'eu-central-1', + 'ap-northeast-1', + 'ap-southeast-1', + 'ap-southeast-2', +]; + +// some tests require additional time, the parent account can handle longer tests (up to 90 minutes) +const USE_PARENT_ACCOUNT = [ + 'src/__tests__/build-app-swift.test.ts', +]; +const REPO_ROOT = join(__dirname, '..'); +const TEST_TIMINGS_PATH = join(REPO_ROOT, 'scripts', 'cci', 'test-timings.data.json'); +const CODEBUILD_CONFIG_BASE_PATH = join(REPO_ROOT, '.codebuild', 'e2e_workflow_base.yml'); +const CODEBUILD_GENERATE_CONFIG_PATH = join(REPO_ROOT, '.codebuild', 'e2e_workflow.yml'); +const RUN_SOLO = [ + 'src/__tests__/build-app-swift.test.ts', +]; +const runJobOnAndroid = new Set(['build-app-android-e2e-test']); +const runJobOnMacOS = new Set(['build-app-swift-e2e-test']); + +export function loadConfigBase() { + return yaml.load(fs.readFileSync(CODEBUILD_CONFIG_BASE_PATH, 'utf8')); +} +export function saveConfig(config: any): void { + const output = ['# auto generated file. DO NOT EDIT manually', yaml.dump(config, { noRefs: true })]; + fs.writeFileSync(CODEBUILD_GENERATE_CONFIG_PATH, output.join('\n')); +} +export function loadTestTimings(): { timingData: { test: string; medianRuntime: number }[] } { + return JSON.parse(fs.readFileSync(TEST_TIMINGS_PATH, 'utf-8')); +} +function getTestFiles(dir: string, pattern = 'src/**/*.test.ts'): string[] { + return glob.sync(pattern, { cwd: dir }); +} +type COMPUTE_TYPE = 'BUILD_GENERAL1_MEDIUM' | 'BUILD_GENERAL1_LARGE'; +type BatchBuildJob = { + identifier: string; + env: { + 'compute-type': COMPUTE_TYPE; + variables: [string: string]; + }; +}; +type ConfigBase = { + batch: { + 'build-graph': BatchBuildJob[]; + 'fast-fail': boolean; + }; + env: { + 'compute-type': COMPUTE_TYPE; + shell: 'bash'; + variables: [string: string]; + }; +}; +const MAX_WORKERS = 4; +type OS_TYPE = 'w' | 'l'; +type CandidateJob = { + region: string; + os: OS_TYPE; + tests: string[]; + useParentAccount: boolean; + runSolo: boolean; +}; +const createJob = (os: OS_TYPE, jobIdx: number, runSolo: boolean = false): CandidateJob => { + const region = AWS_REGIONS_TO_RUN_TESTS[jobIdx % AWS_REGIONS_TO_RUN_TESTS.length]; + return { + region, + os, + tests: [], + useParentAccount: false, + runSolo, + }; +}; +const getTestNameFromPath = (testSuitePath: string): string => { + const startIndex = testSuitePath.lastIndexOf('/') + 1; + const endIndex = testSuitePath.lastIndexOf('.test'); + return testSuitePath + .substring(startIndex, endIndex) + .split('.e2e') + .join('') + .split('.') + .join('-'); +}; +const splitTests = ( + baseJobLinux: any, + testDirectory: string, + pickTests?: ((testSuites: string[]) => string[]), +) => { + const output: any[] = []; + let testSuites = getTestFiles(testDirectory); + if (pickTests && typeof pickTests === 'function') { + testSuites = pickTests(testSuites); + } + if (testSuites.length === 0) { + return output; + } + const testFileRunTimes = loadTestTimings().timingData; + + testSuites.sort((a, b) => { + const runtimeA = testFileRunTimes.find((t:any) => t.test === a)?.medianRuntime ?? 30; + const runtimeB = testFileRunTimes.find((t:any) => t.test === b)?.medianRuntime ?? 30; + return runtimeA - runtimeB; + }); + const generateJobsForOS = (os: OS_TYPE) => { + const soloJobs: CandidateJob[] = []; + let jobIdx = 0; + const osJobs = [createJob(os, jobIdx)]; + jobIdx++; + for (let test of testSuites) { + const currentJob = osJobs[osJobs.length - 1]; + + const USE_PARENT = USE_PARENT_ACCOUNT.some((usesParent) => test.startsWith(usesParent)); + + if (RUN_SOLO.find((solo) => test === solo)) { + const newSoloJob = createJob(os, jobIdx, true); + jobIdx++; + newSoloJob.tests.push(test); + + if (USE_PARENT) { + newSoloJob.useParentAccount = true; + } + soloJobs.push(newSoloJob); + continue; + } + + // add the test + currentJob.tests.push(test); + + if (USE_PARENT) { + currentJob.useParentAccount = true; + } + + // create a new job once the current job is full; + if (currentJob.tests.length >= MAX_WORKERS) { + osJobs.push(createJob(os, jobIdx)); + jobIdx++; + } + } + return [...osJobs, ...soloJobs]; + }; + const linuxJobs = generateJobsForOS('l'); + const getIdentifier = (os: string, names: string) => { + const jobName = `${names.replace(/-/g, '_')}`.substring(0, 127); + return jobName; + }; + const result: any[] = []; + linuxJobs.forEach((j) => { + if (j.tests.length !== 0) { + const names = j.tests.map((tn) => getTestNameFromPath(tn)).join('_'); + const tmp = { + ...JSON.parse(JSON.stringify(baseJobLinux)), // deep clone base job + identifier: getIdentifier(j.os, names), + }; + tmp.env.variables = {}; + tmp.env.variables.TEST_SUITE = j.tests.join('|'); + tmp.env.variables.CLI_REGION = j.region; + if (j.useParentAccount) { + tmp.env.variables.USE_PARENT_ACCOUNT = 1; + } + if (j.runSolo) { + tmp.env['compute-type'] = 'BUILD_GENERAL1_SMALL'; + } + result.push(tmp); + } + }); + return result; +}; +function main(): void { + const configBase: any = loadConfigBase(); + const baseBuildGraph = configBase.batch['build-graph']; + const splitE2ETests = splitTests( + { + identifier: 'run_e2e_tests', + buildspec: '.codebuild/run_e2e_tests.yml', + env: { + 'compute-type': 'BUILD_GENERAL1_MEDIUM', + }, + 'depend-on': ['publish_to_local_registry'], + }, + join(REPO_ROOT, 'packages', 'amplify-codegen-e2e-tests'), + ); + + let allBuilds = [...splitE2ETests]; + const cleanupResources = { + identifier: 'cleanup_e2e_resources', + buildspec: '.codebuild/cleanup_e2e_resources.yml', + env: { + 'compute-type': 'BUILD_GENERAL1_SMALL' + }, + 'depend-on': [allBuilds[0].identifier] + } + console.log(`Total number of splitted jobs: ${allBuilds.length}`) + let currentBatch = [...baseBuildGraph, ...allBuilds, cleanupResources]; + configBase.batch['build-graph'] = currentBatch; + saveConfig(configBase); +} + +main(); diff --git a/shared-scripts.sh b/shared-scripts.sh index cff95f36..ea0dbdc8 100644 --- a/shared-scripts.sh +++ b/shared-scripts.sh @@ -311,7 +311,7 @@ function runE2eTest { if [ -z "$FIRST_RUN" ] || [ "$FIRST_RUN" == "true" ]; then echo "using Amplify CLI version: "$(amplify --version) - cd $(pwd)/packages/amplify-e2e-tests + cd $(pwd)/packages/amplify-codegen-e2e-tests fi if [ -f $FAILED_TEST_REGEX_FILE ]; then From f0ee81b0d3747fb6498bf73a6d996a6e3c47b506 Mon Sep 17 00:00:00 2001 From: phani-srikar Date: Mon, 10 Jul 2023 22:44:33 -0700 Subject: [PATCH 2/6] fix: handle android and swift build tests separately --- .codebuild/e2e_workflow.yml | 67 +++++++----- .codebuild/e2e_workflow_base.yml | 19 ++++ .codebuild/scripts/local_publish_helpers.sh | 4 +- scripts/cci/test-timings.data.json | 115 +++++++++++++++++++- scripts/cci/utils.ts | 14 +-- scripts/split-e2e-tests-codebuild.ts | 11 +- 6 files changed, 181 insertions(+), 49 deletions(-) diff --git a/.codebuild/e2e_workflow.yml b/.codebuild/e2e_workflow.yml index 6ecafc38..0dd2c121 100644 --- a/.codebuild/e2e_workflow.yml +++ b/.codebuild/e2e_workflow.yml @@ -28,88 +28,98 @@ batch: compute-type: BUILD_GENERAL1_MEDIUM depend-on: - build_linux - - identifier: add_codegen_android_add_codegen_ios_add_codegen_js_build_app_android + - identifier: build_app_android buildspec: .codebuild/run_e2e_tests.yml env: compute-type: BUILD_GENERAL1_MEDIUM variables: - TEST_SUITE: >- - src/__tests__/add-codegen-android.test.ts|src/__tests__/add-codegen-ios.test.ts|src/__tests__/add-codegen-js.test.ts|src/__tests__/build-app-android.test.ts - CLI_REGION: us-east-1 + TEST_SUITE: src/__tests__/build-app-android.test.ts + CLI_REGION: us-east-2 + SKIP_SET_NPM_PREFIX: true + depend-on: + - publish_to_local_registry + - identifier: build_app_swift + buildspec: .codebuild/run_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + TEST_SUITE: src/__tests__/build-app-swift.test.ts + CLI_REGION: us-east-2 depend-on: - publish_to_local_registry - identifier: >- - build_app_ts_configure_codegen_android_configure_codegen_ios_configure_codegen_js + add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android buildspec: .codebuild/run_e2e_tests.yml env: compute-type: BUILD_GENERAL1_MEDIUM variables: TEST_SUITE: >- - src/__tests__/build-app-ts.test.ts|src/__tests__/configure-codegen-android.test.ts|src/__tests__/configure-codegen-ios.test.ts|src/__tests__/configure-codegen-js.test.ts - CLI_REGION: us-east-2 + src/__tests__/add-codegen-ios.test.ts|src/__tests__/configure-codegen-android.test.ts|src/__tests__/configure-codegen-js.test.ts|src/__tests__/graphql-codegen-android.test.ts + CLI_REGION: us-east-1 depend-on: - publish_to_local_registry - identifier: >- - datastore_modelgen_android_datastore_modelgen_flutter_datastore_modelgen_ios_datastore_modelgen_js + graphql_codegen_js_remove_codegen_android_remove_codegen_ios_add_codegen_android buildspec: .codebuild/run_e2e_tests.yml env: compute-type: BUILD_GENERAL1_MEDIUM variables: TEST_SUITE: >- - src/__tests__/datastore-modelgen-android.test.ts|src/__tests__/datastore-modelgen-flutter.test.ts|src/__tests__/datastore-modelgen-ios.test.ts|src/__tests__/datastore-modelgen-js.test.ts - CLI_REGION: eu-west-2 + src/__tests__/graphql-codegen-js.test.ts|src/__tests__/remove-codegen-android.test.ts|src/__tests__/remove-codegen-ios.test.ts|src/__tests__/add-codegen-android.test.ts + CLI_REGION: us-east-2 depend-on: - publish_to_local_registry - - identifier: env_codegen_feature_flags_graphql_codegen_android_graphql_codegen_ios + - identifier: >- + configure_codegen_ios_datastore_modelgen_android_datastore_modelgen_js_feature_flags buildspec: .codebuild/run_e2e_tests.yml env: compute-type: BUILD_GENERAL1_MEDIUM variables: TEST_SUITE: >- - src/__tests__/env-codegen.test.ts|src/__tests__/feature-flags.test.ts|src/__tests__/graphql-codegen-android.test.ts|src/__tests__/graphql-codegen-ios.test.ts - CLI_REGION: eu-central-1 + src/__tests__/configure-codegen-ios.test.ts|src/__tests__/datastore-modelgen-android.test.ts|src/__tests__/datastore-modelgen-js.test.ts|src/__tests__/feature-flags.test.ts + CLI_REGION: us-west-2 depend-on: - publish_to_local_registry - identifier: >- - graphql_codegen_js_graphql_documents_generator_model_introspection_codegen_pull_codegen + graphql_codegen_ios_add_codegen_js_datastore_modelgen_ios_remove_codegen_js buildspec: .codebuild/run_e2e_tests.yml env: compute-type: BUILD_GENERAL1_MEDIUM variables: TEST_SUITE: >- - src/__tests__/graphql-codegen-js.test.ts|src/__tests__/graphql-documents-generator.test.ts|src/__tests__/model-introspection-codegen.test.ts|src/__tests__/pull-codegen.test.ts - CLI_REGION: ap-northeast-1 + src/__tests__/graphql-codegen-ios.test.ts|src/__tests__/add-codegen-js.test.ts|src/__tests__/datastore-modelgen-ios.test.ts|src/__tests__/remove-codegen-js.test.ts + CLI_REGION: eu-west-2 depend-on: - publish_to_local_registry - identifier: >- - push_codegen_android_push_codegen_ios_push_codegen_js_remove_codegen_android + datastore_modelgen_flutter_env_codegen_model_introspection_codegen_pull_codegen buildspec: .codebuild/run_e2e_tests.yml env: compute-type: BUILD_GENERAL1_MEDIUM variables: TEST_SUITE: >- - src/__tests__/push-codegen-android.test.ts|src/__tests__/push-codegen-ios.test.ts|src/__tests__/push-codegen-js.test.ts|src/__tests__/remove-codegen-android.test.ts - CLI_REGION: ap-southeast-1 + src/__tests__/datastore-modelgen-flutter.test.ts|src/__tests__/env-codegen.test.ts|src/__tests__/model-introspection-codegen.test.ts|src/__tests__/pull-codegen.test.ts + CLI_REGION: eu-central-1 depend-on: - publish_to_local_registry - - identifier: remove_codegen_ios_remove_codegen_js + - identifier: >- + push_codegen_ios_push_codegen_android_graphql_documents_generator_push_codegen_js buildspec: .codebuild/run_e2e_tests.yml env: compute-type: BUILD_GENERAL1_MEDIUM variables: TEST_SUITE: >- - src/__tests__/remove-codegen-ios.test.ts|src/__tests__/remove-codegen-js.test.ts - CLI_REGION: ap-southeast-2 + src/__tests__/push-codegen-ios.test.ts|src/__tests__/push-codegen-android.test.ts|src/__tests__/graphql-documents-generator.test.ts|src/__tests__/push-codegen-js.test.ts + CLI_REGION: ap-northeast-1 depend-on: - publish_to_local_registry - - identifier: build_app_swift + - identifier: build_app_ts buildspec: .codebuild/run_e2e_tests.yml env: - compute-type: BUILD_GENERAL1_SMALL + compute-type: BUILD_GENERAL1_MEDIUM variables: - TEST_SUITE: src/__tests__/build-app-swift.test.ts - CLI_REGION: us-west-2 - USE_PARENT_ACCOUNT: 1 + TEST_SUITE: src/__tests__/build-app-ts.test.ts + CLI_REGION: ap-southeast-1 depend-on: - publish_to_local_registry - identifier: cleanup_e2e_resources @@ -117,4 +127,5 @@ batch: env: compute-type: BUILD_GENERAL1_SMALL depend-on: - - add_codegen_android_add_codegen_ios_add_codegen_js_build_app_android + - >- + add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android diff --git a/.codebuild/e2e_workflow_base.yml b/.codebuild/e2e_workflow_base.yml index 74d4e57b..4fea9bdb 100644 --- a/.codebuild/e2e_workflow_base.yml +++ b/.codebuild/e2e_workflow_base.yml @@ -28,3 +28,22 @@ batch: compute-type: BUILD_GENERAL1_MEDIUM depend-on: - build_linux + - identifier: build_app_android + buildspec: .codebuild/run_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + TEST_SUITE: src/__tests__/build-app-android.test.ts + CLI_REGION: us-east-2 + SKIP_SET_NPM_PREFIX: true + depend-on: + - publish_to_local_registry + - identifier: build_app_swift + buildspec: .codebuild/run_e2e_tests.yml + env: + compute-type: BUILD_GENERAL1_MEDIUM + variables: + TEST_SUITE: src/__tests__/build-app-swift.test.ts + CLI_REGION: us-east-2 + depend-on: + - publish_to_local_registry diff --git a/.codebuild/scripts/local_publish_helpers.sh b/.codebuild/scripts/local_publish_helpers.sh index c96de99e..a775ca6d 100644 --- a/.codebuild/scripts/local_publish_helpers.sh +++ b/.codebuild/scripts/local_publish_helpers.sh @@ -44,7 +44,9 @@ function unsetSudoNpmRegistryUrl { function changeNpmGlobalPath { mkdir -p ~/.npm-global - npm config set prefix '~/.npm-global' + if [ -z $SKIP_SET_NPM_PREFIX ]; then + npm config set prefix '~/.npm-global' + fi export PATH=~/.npm-global/bin:$PATH } diff --git a/scripts/cci/test-timings.data.json b/scripts/cci/test-timings.data.json index 830c96e2..327302e7 100644 --- a/scripts/cci/test-timings.data.json +++ b/scripts/cci/test-timings.data.json @@ -1,5 +1,114 @@ { - "lastUpdated": "2023-07-11T04:44:38.813Z", - "totalTestFiles": 0, - "timingData": [] + "lastUpdated": "2023-07-11T05:07:36.310Z", + "totalTestFiles": 27, + "timingData": [ + { + "test": "src/__tests__/add-codegen-ios.test.ts", + "medianRuntime": 6 + }, + { + "test": "src/__tests__/configure-codegen-android.test.ts", + "medianRuntime": 6 + }, + { + "test": "src/__tests__/configure-codegen-js.test.ts", + "medianRuntime": 6 + }, + { + "test": "src/__tests__/graphql-codegen-android.test.ts", + "medianRuntime": 6 + }, + { + "test": "src/__tests__/graphql-codegen-js.test.ts", + "medianRuntime": 6 + }, + { + "test": "src/__tests__/remove-codegen-android.test.ts", + "medianRuntime": 6 + }, + { + "test": "src/__tests__/remove-codegen-ios.test.ts", + "medianRuntime": 6 + }, + { + "test": "src/__tests__/add-codegen-android.test.ts", + "medianRuntime": 7 + }, + { + "test": "src/__tests__/configure-codegen-ios.test.ts", + "medianRuntime": 7 + }, + { + "test": "src/__tests__/datastore-modelgen-android.test.ts", + "medianRuntime": 7 + }, + { + "test": "src/__tests__/datastore-modelgen-js.test.ts", + "medianRuntime": 7 + }, + { + "test": "src/__tests__/feature-flags.test.ts", + "medianRuntime": 7 + }, + { + "test": "src/__tests__/graphql-codegen-ios.test.ts", + "medianRuntime": 7 + }, + { + "test": "src/__tests__/add-codegen-js.test.ts", + "medianRuntime": 8 + }, + { + "test": "src/__tests__/datastore-modelgen-ios.test.ts", + "medianRuntime": 8 + }, + { + "test": "src/__tests__/remove-codegen-js.test.ts", + "medianRuntime": 8 + }, + { + "test": "src/__tests__/datastore-modelgen-flutter.test.ts", + "medianRuntime": 9 + }, + { + "test": "src/__tests__/env-codegen.test.ts", + "medianRuntime": 9 + }, + { + "test": "src/__tests__/model-introspection-codegen.test.ts", + "medianRuntime": 9 + }, + { + "test": "src/__tests__/build-app-android.test.ts", + "medianRuntime": 11 + }, + { + "test": "src/__tests__/pull-codegen.test.ts", + "medianRuntime": 11 + }, + { + "test": "src/__tests__/push-codegen-ios.test.ts", + "medianRuntime": 11 + }, + { + "test": "src/__tests__/push-codegen-android.test.ts", + "medianRuntime": 12 + }, + { + "test": "src/__tests__/graphql-documents-generator.test.ts", + "medianRuntime": 13 + }, + { + "test": "src/__tests__/push-codegen-js.test.ts", + "medianRuntime": 13 + }, + { + "test": "src/__tests__/build-app-ts.test.ts", + "medianRuntime": 16 + }, + { + "test": "src/__tests__/build-app-swift.test.ts", + "medianRuntime": 20 + } + ] } \ No newline at end of file diff --git a/scripts/cci/utils.ts b/scripts/cci/utils.ts index 098854f1..469f1a6b 100644 --- a/scripts/cci/utils.ts +++ b/scripts/cci/utils.ts @@ -35,7 +35,7 @@ export function getTimingsFromJobsData() { const jobData = JSON.parse(fs.readFileSync(JOB_METRICS_PATH, 'utf-8')); const jobTimings: Map = new Map(); for (let job of jobData.items) { - const testName = getTestNameFromJobName(job.name); + const testName = job.name; const duration = Math.floor(job.metrics.duration_metrics.median / 60); if (jobTimings.has(testName)) { jobTimings.set(testName, Math.max(jobTimings.get(testName)!, duration)); @@ -46,22 +46,14 @@ export function getTimingsFromJobsData() { return jobTimings; } -function getTestNameFromJobName(jobName: string) { - // first, remove any - from the name - const endIndex = jobName.lastIndexOf('-l'); - const name = jobName.substring(0, endIndex); - return name; -} - export const getTestNameFromPath = (testSuitePath: string): string => { const startIndex = testSuitePath.lastIndexOf('/') + 1; const endIndex = testSuitePath.lastIndexOf('.test'); return testSuitePath .substring(startIndex, endIndex) - .split('.e2e') - .join('') .split('.') - .join('-'); + .join('-') + + '-e2e-test'; }; export function saveTestTimings(data: any): any { diff --git a/scripts/split-e2e-tests-codebuild.ts b/scripts/split-e2e-tests-codebuild.ts index 16431826..0a557db2 100644 --- a/scripts/split-e2e-tests-codebuild.ts +++ b/scripts/split-e2e-tests-codebuild.ts @@ -16,18 +16,16 @@ const AWS_REGIONS_TO_RUN_TESTS = [ ]; // some tests require additional time, the parent account can handle longer tests (up to 90 minutes) -const USE_PARENT_ACCOUNT = [ - 'src/__tests__/build-app-swift.test.ts', -]; +const USE_PARENT_ACCOUNT = []; const REPO_ROOT = join(__dirname, '..'); const TEST_TIMINGS_PATH = join(REPO_ROOT, 'scripts', 'cci', 'test-timings.data.json'); const CODEBUILD_CONFIG_BASE_PATH = join(REPO_ROOT, '.codebuild', 'e2e_workflow_base.yml'); const CODEBUILD_GENERATE_CONFIG_PATH = join(REPO_ROOT, '.codebuild', 'e2e_workflow.yml'); -const RUN_SOLO = [ +const RUN_SOLO = []; +const EXCLUDE_TESTS = [ 'src/__tests__/build-app-swift.test.ts', + 'src/__tests__/build-app-android.test.ts', ]; -const runJobOnAndroid = new Set(['build-app-android-e2e-test']); -const runJobOnMacOS = new Set(['build-app-swift-e2e-test']); export function loadConfigBase() { return yaml.load(fs.readFileSync(CODEBUILD_CONFIG_BASE_PATH, 'utf8')); @@ -187,6 +185,7 @@ function main(): void { 'depend-on': ['publish_to_local_registry'], }, join(REPO_ROOT, 'packages', 'amplify-codegen-e2e-tests'), + (testSuites) => testSuites.filter((ts) => !EXCLUDE_TESTS.includes(ts)), ); let allBuilds = [...splitE2ETests]; From 9a01a285706e6e0399453aec5764d9b9db01974c Mon Sep 17 00:00:00 2001 From: phani-srikar Date: Mon, 10 Jul 2023 22:45:21 -0700 Subject: [PATCH 3/6] chore: lint fix --- scripts/split-e2e-tests-codebuild.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/split-e2e-tests-codebuild.ts b/scripts/split-e2e-tests-codebuild.ts index 0a557db2..624df27f 100644 --- a/scripts/split-e2e-tests-codebuild.ts +++ b/scripts/split-e2e-tests-codebuild.ts @@ -187,7 +187,7 @@ function main(): void { join(REPO_ROOT, 'packages', 'amplify-codegen-e2e-tests'), (testSuites) => testSuites.filter((ts) => !EXCLUDE_TESTS.includes(ts)), ); - + let allBuilds = [...splitE2ETests]; const cleanupResources = { identifier: 'cleanup_e2e_resources', From 412b93369d2931ecc22bf86fdacc804aaea886c3 Mon Sep 17 00:00:00 2001 From: phani-srikar Date: Tue, 11 Jul 2023 15:36:32 -0700 Subject: [PATCH 4/6] chore: remove swift and android tests --- .codebuild/e2e_workflow.yml | 19 ------------------- .codebuild/e2e_workflow_base.yml | 19 ------------------- 2 files changed, 38 deletions(-) diff --git a/.codebuild/e2e_workflow.yml b/.codebuild/e2e_workflow.yml index 0dd2c121..3d06ed2e 100644 --- a/.codebuild/e2e_workflow.yml +++ b/.codebuild/e2e_workflow.yml @@ -28,25 +28,6 @@ batch: compute-type: BUILD_GENERAL1_MEDIUM depend-on: - build_linux - - identifier: build_app_android - buildspec: .codebuild/run_e2e_tests.yml - env: - compute-type: BUILD_GENERAL1_MEDIUM - variables: - TEST_SUITE: src/__tests__/build-app-android.test.ts - CLI_REGION: us-east-2 - SKIP_SET_NPM_PREFIX: true - depend-on: - - publish_to_local_registry - - identifier: build_app_swift - buildspec: .codebuild/run_e2e_tests.yml - env: - compute-type: BUILD_GENERAL1_MEDIUM - variables: - TEST_SUITE: src/__tests__/build-app-swift.test.ts - CLI_REGION: us-east-2 - depend-on: - - publish_to_local_registry - identifier: >- add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android buildspec: .codebuild/run_e2e_tests.yml diff --git a/.codebuild/e2e_workflow_base.yml b/.codebuild/e2e_workflow_base.yml index 4fea9bdb..74d4e57b 100644 --- a/.codebuild/e2e_workflow_base.yml +++ b/.codebuild/e2e_workflow_base.yml @@ -28,22 +28,3 @@ batch: compute-type: BUILD_GENERAL1_MEDIUM depend-on: - build_linux - - identifier: build_app_android - buildspec: .codebuild/run_e2e_tests.yml - env: - compute-type: BUILD_GENERAL1_MEDIUM - variables: - TEST_SUITE: src/__tests__/build-app-android.test.ts - CLI_REGION: us-east-2 - SKIP_SET_NPM_PREFIX: true - depend-on: - - publish_to_local_registry - - identifier: build_app_swift - buildspec: .codebuild/run_e2e_tests.yml - env: - compute-type: BUILD_GENERAL1_MEDIUM - variables: - TEST_SUITE: src/__tests__/build-app-swift.test.ts - CLI_REGION: us-east-2 - depend-on: - - publish_to_local_registry From 525f72cc84bb16d23ec32877af1890a6b940cb4c Mon Sep 17 00:00:00 2001 From: phani-srikar Date: Tue, 11 Jul 2023 16:12:27 -0700 Subject: [PATCH 5/6] chore: git ignore jobs data file --- .gitignore | 3 +- scripts/cci/job.data.json | 787 -------------------------------------- 2 files changed, 2 insertions(+), 788 deletions(-) delete mode 100644 scripts/cci/job.data.json diff --git a/.gitignore b/.gitignore index 5008007a..5b8f7018 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ test.out.log *.tsbuildinfo package-lock.json .idea -scripts/.env \ No newline at end of file +scripts/.env +scripts/cci/job.data.json \ No newline at end of file diff --git a/scripts/cci/job.data.json b/scripts/cci/job.data.json deleted file mode 100644 index 0b386416..00000000 --- a/scripts/cci/job.data.json +++ /dev/null @@ -1,787 +0,0 @@ -{ - "next_page_token": null, - "items": [ - { - "name": "add-codegen-android-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 390, - "mean": 418, - "median": 426, - "p95": 432, - "max": 434, - "standard_deviation": 17, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 831, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:33.479Z", - "window_end": "2023-07-10T22:11:18.542Z" - }, - { - "name": "add-codegen-ios-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 350, - "mean": 380, - "median": 383, - "p95": 411, - "max": 417, - "standard_deviation": 25, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 757, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:32.473Z", - "window_end": "2023-07-10T22:10:38.898Z" - }, - { - "name": "add-codegen-js-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 522, - "mean": 532, - "median": 530, - "p95": 547, - "max": 551, - "standard_deviation": 10, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 1060, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:32.445Z", - "window_end": "2023-07-10T22:13:06.147Z" - }, - { - "name": "build", - "metrics": { - "total_runs": 8, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 0, - "mean": 111, - "median": 127, - "p95": 193, - "max": 195, - "standard_deviation": 75, - "total_duration": 0 - }, - "success_rate": 0.75, - "total_credits_used": 291, - "throughput": 0.2666666667 - }, - "window_start": "2023-06-19T17:28:54.389Z", - "window_end": "2023-07-10T22:03:07.678Z" - }, - { - "name": "build-app-android-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 618, - "mean": 741, - "median": 717, - "p95": 893, - "max": 917, - "standard_deviation": 111, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 1481, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:34.079Z", - "window_end": "2023-07-10T22:19:26.837Z" - }, - { - "name": "build-app-swift-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 1177, - "mean": 1221, - "median": 1210, - "p95": 1285, - "max": 1304, - "standard_deviation": 44, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 12130, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:33.769Z", - "window_end": "2023-07-10T22:24:41.323Z" - }, - { - "name": "build-app-ts-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 917, - "mean": 986, - "median": 975, - "p95": 1057, - "max": 1066, - "standard_deviation": 54, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 1967, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:32.161Z", - "window_end": "2023-07-10T22:21:20.385Z" - }, - { - "name": "configure-codegen-android-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 337, - "mean": 378, - "median": 380, - "p95": 409, - "max": 410, - "standard_deviation": 29, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 752, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:31.581Z", - "window_end": "2023-07-10T22:10:41.952Z" - }, - { - "name": "configure-codegen-ios-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 415, - "mean": 434, - "median": 433, - "p95": 453, - "max": 458, - "standard_deviation": 14, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 861, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:33.151Z", - "window_end": "2023-07-10T22:11:30.111Z" - }, - { - "name": "configure-codegen-js-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 352, - "mean": 403, - "median": 396, - "p95": 462, - "max": 471, - "standard_deviation": 45, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 804, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:34.375Z", - "window_end": "2023-07-10T22:11:27.834Z" - }, - { - "name": "datastore-modelgen-android-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 424, - "mean": 445, - "median": 449, - "p95": 462, - "max": 466, - "standard_deviation": 14, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 888, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:32.073Z", - "window_end": "2023-07-10T22:11:40.442Z" - }, - { - "name": "datastore-modelgen-flutter-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 542, - "mean": 564, - "median": 551, - "p95": 613, - "max": 632, - "standard_deviation": 33, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 1124, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:31.390Z", - "window_end": "2023-07-10T22:14:42.234Z" - }, - { - "name": "datastore-modelgen-ios-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 502, - "mean": 532, - "median": 534, - "p95": 551, - "max": 553, - "standard_deviation": 18, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 1059, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:33.381Z", - "window_end": "2023-07-10T22:13:15.115Z" - }, - { - "name": "datastore-modelgen-js-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 435, - "mean": 454, - "median": 454, - "p95": 475, - "max": 481, - "standard_deviation": 15, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 902, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:34.377Z", - "window_end": "2023-07-10T22:12:13.663Z" - }, - { - "name": "deploy", - "metrics": { - "total_runs": 3, - "failed_runs": 0, - "successful_runs": 3, - "median_credits_used": 0, - "duration_metrics": { - "min": 42, - "mean": 44, - "median": 45, - "p95": 45, - "max": 46, - "standard_deviation": 2, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 42, - "throughput": 0.1 - }, - "window_start": "2023-06-19T17:53:02.522Z", - "window_end": "2023-06-29T17:10:35.513Z" - }, - { - "name": "done_with_node_e2e_tests", - "metrics": { - "total_runs": 3, - "failed_runs": 0, - "successful_runs": 3, - "median_credits_used": 0, - "duration_metrics": { - "min": 25, - "mean": 26, - "median": 26, - "p95": 28, - "max": 29, - "standard_deviation": 2, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 24, - "throughput": 0.1 - }, - "window_start": "2023-06-19T17:52:37.213Z", - "window_end": "2023-06-29T17:09:49.727Z" - }, - { - "name": "env-codegen-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 1, - "successful_runs": 5, - "median_credits_used": 0, - "duration_metrics": { - "min": 479, - "mean": 584, - "median": 597, - "p95": 624, - "max": 629, - "standard_deviation": 53, - "total_duration": 0 - }, - "success_rate": 0.8333333333, - "total_credits_used": 1165, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:32.807Z", - "window_end": "2023-07-10T22:12:10.121Z" - }, - { - "name": "feature-flags-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 403, - "mean": 432, - "median": 435, - "p95": 445, - "max": 446, - "standard_deviation": 15, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 861, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:33.104Z", - "window_end": "2023-07-10T22:11:24.854Z" - }, - { - "name": "graphql-codegen-android-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 362, - "mean": 388, - "median": 397, - "p95": 404, - "max": 405, - "standard_deviation": 20, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 772, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:33.116Z", - "window_end": "2023-07-10T22:10:44.059Z" - }, - { - "name": "graphql-codegen-ios-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 414, - "mean": 428, - "median": 430, - "p95": 439, - "max": 442, - "standard_deviation": 10, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 848, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:31.872Z", - "window_end": "2023-07-10T22:11:23.361Z" - }, - { - "name": "graphql-codegen-js-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 362, - "mean": 384, - "median": 382, - "p95": 412, - "max": 419, - "standard_deviation": 19, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 761, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:31.069Z", - "window_end": "2023-07-10T22:11:11.346Z" - }, - { - "name": "graphql-documents-generator-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 2, - "successful_runs": 4, - "median_credits_used": 0, - "duration_metrics": { - "min": 626, - "mean": 745, - "median": 787, - "p95": 805, - "max": 807, - "standard_deviation": 79, - "total_duration": 0 - }, - "success_rate": 0.6666666667, - "total_credits_used": 1486, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:33.428Z", - "window_end": "2023-07-10T22:14:38.281Z" - }, - { - "name": "model-introspection-codegen-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 3, - "successful_runs": 3, - "median_credits_used": 0, - "duration_metrics": { - "min": 540, - "mean": 585, - "median": 593, - "p95": 608, - "max": 612, - "standard_deviation": 25, - "total_duration": 0 - }, - "success_rate": 0.5, - "total_credits_used": 1165, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:32.848Z", - "window_end": "2023-07-10T22:13:48.972Z" - }, - { - "name": "publish_to_local_registry", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 61, - "mean": 79, - "median": 78, - "p95": 95, - "max": 99, - "standard_deviation": 12, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 153, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:30:54.032Z", - "window_end": "2023-07-10T22:04:09.237Z" - }, - { - "name": "pull-codegen-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 1, - "successful_runs": 5, - "median_credits_used": 0, - "duration_metrics": { - "min": 631, - "mean": 713, - "median": 718, - "p95": 754, - "max": 762, - "standard_deviation": 43, - "total_duration": 0 - }, - "success_rate": 0.8333333333, - "total_credits_used": 1422, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:32.477Z", - "window_end": "2023-07-10T22:14:41.457Z" - }, - { - "name": "push-codegen-android-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 1, - "successful_runs": 5, - "median_credits_used": 0, - "duration_metrics": { - "min": 524, - "mean": 735, - "median": 779, - "p95": 788, - "max": 790, - "standard_deviation": 104, - "total_duration": 0 - }, - "success_rate": 0.8333333333, - "total_credits_used": 1468, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:32.825Z", - "window_end": "2023-07-10T22:12:54.137Z" - }, - { - "name": "push-codegen-ios-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 1, - "successful_runs": 5, - "median_credits_used": 0, - "duration_metrics": { - "min": 471, - "mean": 649, - "median": 682, - "p95": 705, - "max": 710, - "standard_deviation": 88, - "total_duration": 0 - }, - "success_rate": 0.8333333333, - "total_credits_used": 1292, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:34.036Z", - "window_end": "2023-07-10T22:12:00.368Z" - }, - { - "name": "push-codegen-js-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 1, - "successful_runs": 5, - "median_credits_used": 0, - "duration_metrics": { - "min": 541, - "mean": 754, - "median": 785, - "p95": 826, - "max": 829, - "standard_deviation": 106, - "total_duration": 0 - }, - "success_rate": 0.8333333333, - "total_credits_used": 1502, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:31.734Z", - "window_end": "2023-07-10T22:13:10.326Z" - }, - { - "name": "remove-codegen-android-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 357, - "mean": 383, - "median": 383, - "p95": 408, - "max": 409, - "standard_deviation": 21, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 761, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:32.235Z", - "window_end": "2023-07-10T22:10:59.009Z" - }, - { - "name": "remove-codegen-ios-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 362, - "mean": 402, - "median": 394, - "p95": 442, - "max": 447, - "standard_deviation": 31, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 799, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:33.734Z", - "window_end": "2023-07-10T22:11:38.351Z" - }, - { - "name": "remove-codegen-js-e2e-test", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 490, - "mean": 517, - "median": 521, - "p95": 529, - "max": 530, - "standard_deviation": 15, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 1029, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:32:31.843Z", - "window_end": "2023-07-10T22:13:01.480Z" - }, - { - "name": "test-l", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 137, - "mean": 144, - "median": 139, - "p95": 160, - "max": 164, - "standard_deviation": 10, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 285, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:30:54.665Z", - "window_end": "2023-07-10T22:05:25.354Z" - }, - { - "name": "test-w", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 487, - "mean": 622, - "median": 616, - "p95": 778, - "max": 807, - "standard_deviation": 123, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 2451, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:30:54.893Z", - "window_end": "2023-07-10T22:14:40.901Z" - }, - { - "name": "verify-api-extract", - "metrics": { - "total_runs": 6, - "failed_runs": 0, - "successful_runs": 6, - "median_credits_used": 0, - "duration_metrics": { - "min": 46, - "mean": 67, - "median": 71, - "p95": 74, - "max": 75, - "standard_deviation": 10, - "total_duration": 0 - }, - "success_rate": 1, - "total_credits_used": 130, - "throughput": 0.2 - }, - "window_start": "2023-06-19T17:30:54.371Z", - "window_end": "2023-07-10T22:04:22.075Z" - } - ] -} \ No newline at end of file From f79745da948d75f125bd231ff976db23e9b7a691 Mon Sep 17 00:00:00 2001 From: phani-srikar Date: Tue, 11 Jul 2023 17:45:02 -0700 Subject: [PATCH 6/6] fix: remove changelog step --- shared-scripts.sh | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/shared-scripts.sh b/shared-scripts.sh index ea0dbdc8..c253edd5 100644 --- a/shared-scripts.sh +++ b/shared-scripts.sh @@ -142,17 +142,6 @@ function _publishToLocalRegistry { unsetNpmRegistryUrl # copy [verdaccio-cache] to s3 storeCache $CODEBUILD_SRC_DIR/../verdaccio-cache verdaccio-cache - - _generateChangeLog -} - -function _generateChangeLog { - echo "Generate Change Log" - git reset --hard HEAD - yarn update-versions - yarn ts-node scripts/unified-changelog.ts - # copy [changelog] to s3 - storeCacheFile $CODEBUILD_SRC_DIR/UNIFIED_CHANGELOG.md UNIFIED_CHANGELOG.md } function _installCLIFromLocalRegistry {