Split E2E tests into batches

.circleci/config.base.yml

@@ -303,7 +303,7 @@ jobs:
             yarn clean-e2e-resources
           no_output_timeout: 20m
       - store_artifacts:
-          path: ~/repo/packages/amplify-e2e-tests/amplify-e2e-reports
+          path: ~/repo/packages/amplify-codegen-e2e-tests/amplify-e2e-reports
     working_directory: ~/repo
 
   verify-api-extract:

.circleci/config.yml

@@ -306,7 +306,7 @@ jobs:
             yarn clean-e2e-resources
           no_output_timeout: 20m
       - store_artifacts:
-          path: ~/repo/packages/amplify-e2e-tests/amplify-e2e-reports
+          path: ~/repo/packages/amplify-codegen-e2e-tests/amplify-e2e-reports
   verify-api-extract:
     docker: *ref_4
     working_directory: ~/repo

.codebuild/cleanup_workflow.yml

@@ -9,4 +9,4 @@ phases:
   build:
     commands:
       - yarn production-build
-      - cd packages/amplify-e2e-tests && yarn clean-cb-e2e-resources
+      - cd packages/amplify-codegen-e2e-tests && yarn clean-cb-e2e-resources

.codebuild/e2e_workflow.yml

@@ -1,8 +1,8 @@
+# auto generated file. DO NOT EDIT manually
 version: 0.2
 env:
   shell: bash
   compute-type: BUILD_GENERAL1_MEDIUM
-
 batch:
   fast-fail: false
   build-graph:
@@ -28,3 +28,85 @@ batch:
         compute-type: BUILD_GENERAL1_MEDIUM
       depend-on:
         - build_linux
+    - identifier: >-
+        add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/add-codegen-ios.test.ts|src/__tests__/configure-codegen-android.test.ts|src/__tests__/configure-codegen-js.test.ts|src/__tests__/graphql-codegen-android.test.ts
+          CLI_REGION: us-east-1
+      depend-on:
+        - publish_to_local_registry
+    - identifier: >-
+        graphql_codegen_js_remove_codegen_android_remove_codegen_ios_add_codegen_android
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/graphql-codegen-js.test.ts|src/__tests__/remove-codegen-android.test.ts|src/__tests__/remove-codegen-ios.test.ts|src/__tests__/add-codegen-android.test.ts
+          CLI_REGION: us-east-2
+      depend-on:
+        - publish_to_local_registry
+    - identifier: >-
+        configure_codegen_ios_datastore_modelgen_android_datastore_modelgen_js_feature_flags
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/configure-codegen-ios.test.ts|src/__tests__/datastore-modelgen-android.test.ts|src/__tests__/datastore-modelgen-js.test.ts|src/__tests__/feature-flags.test.ts
+          CLI_REGION: us-west-2
+      depend-on:
+        - publish_to_local_registry
+    - identifier: >-
+        graphql_codegen_ios_add_codegen_js_datastore_modelgen_ios_remove_codegen_js
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/graphql-codegen-ios.test.ts|src/__tests__/add-codegen-js.test.ts|src/__tests__/datastore-modelgen-ios.test.ts|src/__tests__/remove-codegen-js.test.ts
+          CLI_REGION: eu-west-2
+      depend-on:
+        - publish_to_local_registry
+    - identifier: >-
+        datastore_modelgen_flutter_env_codegen_model_introspection_codegen_pull_codegen
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/datastore-modelgen-flutter.test.ts|src/__tests__/env-codegen.test.ts|src/__tests__/model-introspection-codegen.test.ts|src/__tests__/pull-codegen.test.ts
+          CLI_REGION: eu-central-1
+      depend-on:
+        - publish_to_local_registry
+    - identifier: >-
+        push_codegen_ios_push_codegen_android_graphql_documents_generator_push_codegen_js
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: >-
+            src/__tests__/push-codegen-ios.test.ts|src/__tests__/push-codegen-android.test.ts|src/__tests__/graphql-documents-generator.test.ts|src/__tests__/push-codegen-js.test.ts
+          CLI_REGION: ap-northeast-1
+      depend-on:
+        - publish_to_local_registry
+    - identifier: build_app_ts
+      buildspec: .codebuild/run_e2e_tests.yml
+      env:
+        compute-type: BUILD_GENERAL1_MEDIUM
+        variables:
+          TEST_SUITE: src/__tests__/build-app-ts.test.ts
+          CLI_REGION: ap-southeast-1
+      depend-on:
+        - publish_to_local_registry
+    - identifier: cleanup_e2e_resources
+      buildspec: .codebuild/cleanup_e2e_resources.yml
+      env:
+        compute-type: BUILD_GENERAL1_SMALL
+      depend-on:
+        - >-
+          add_codegen_ios_configure_codegen_android_configure_codegen_js_graphql_codegen_android

.codebuild/scripts/artifact-storage-path-allow-list-codebuild.ts

@@ -9,10 +9,10 @@
  *
     artifacts:
       files:
-        - $CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports/*
+        - $CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports/*
  *
  * From the above job, 'path' includes the following:
- *  $CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports
+ *  $CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports
  *
  * Those paths must be included in this list.
  *
@@ -27,6 +27,6 @@
  * will automatically normalize these paths for Windows if it detects it.
  */
 export const ARTIFACT_STORAGE_PATH_ALLOW_LIST_CODEBUILD = [
-  '$CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/',
-  '$CODEBUILD_SRC_DIR/packages/amplify-e2e-tests/amplify-e2e-reports'
+  '$CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/',
+  '$CODEBUILD_SRC_DIR/packages/amplify-codegen-e2e-tests/amplify-e2e-reports'
 ];

.codebuild/scripts/local_publish_helpers.sh

@@ -44,7 +44,9 @@ function unsetSudoNpmRegistryUrl {
 
 function changeNpmGlobalPath {
     mkdir -p ~/.npm-global
-    npm config set prefix '~/.npm-global'
+    if [ -z $SKIP_SET_NPM_PREFIX ]; then
+        npm config set prefix '~/.npm-global'
+    fi
     export PATH=~/.npm-global/bin:$PATH
 }
 

.gitignore

@@ -26,4 +26,5 @@ test.out.log
 *.tsbuildinfo
 package-lock.json
 .idea
-scripts/.env
+scripts/.env
+scripts/cci/job.data.json

package.json

@@ -39,7 +39,9 @@
     "view-test-artifact": "./scripts/view-test-artifacts.sh",
     "cleanup-stale-resources": "source ./scripts/cloud-utils.sh && cleanupStaleResources",
     "cloud-e2e-cb": "source scripts/cloud-utils.sh && cloudE2E",
-    "cloud-e2e-cb-beta": "source scripts/cloud-utils.sh && cloudE2EBeta"
+    "cloud-e2e-cb-beta": "source scripts/cloud-utils.sh && cloudE2EBeta",
+    "split-codebuild-e2e-tests": "yarn ts-node ./scripts/split-e2e-tests-codebuild.ts && git add .codebuild/e2e_workflow.yml",
+    "update-test-timing-data": "ts-node ./scripts/cci/get-test-timing-from-cci-job-metrics.ts"
   },
   "bugs": {
     "url": "https://github.com/aws-amplify/amplify-codegen/issues"
@@ -55,7 +57,7 @@
     "hooks": {
       "commit-msg": "commitlint -E HUSKY_GIT_PARAMS",
       "pre-push": "npm run lint && npm run test-changed",
-      "pre-commit": "yarn split-e2e-tests && pretty-quick --staged"
+      "pre-commit": "yarn split-codebuild-e2e-tests"
     }
   },
   "author": "Amazon Web Services",

scripts/cci/api.ts

@@ -0,0 +1,117 @@
+import axios from 'axios';
+
+type ReportingWindow = 'last-7-days' | 'last-90-days' | 'last-24-hours' | 'last-30-days' | 'last-60-days';
+export type CircleCIClientDefaults = {
+  defaultBranch: string;
+  defaultWorkflow: string;
+  vcs: string;
+  projectSlug: string;
+  projectName: string;
+};
+export class CircleCIAPIClient {
+  private headers;
+  private options: CircleCIClientDefaults;
+  private slug: string;
+  constructor(token: string, options: CircleCIClientDefaults) {
+    this.headers = {
+      'Circle-Token': token,
+    };
+    this.options = options;
+    this.slug = `${options.vcs}/${options.projectSlug}/${options.projectName}`;
+  }
+
+  /**
+   * Returns a sequence of jobs for a workflow.
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/listWorkflowJobs
+   * @returns
+   */
+  getWorkflowJobs = async (workflowId: string = this.options.defaultWorkflow) => {
+    const result = await axios.get(`https://circleci.com/api/v2/workflow/${workflowId}/job`, {
+      headers: this.headers,
+    });
+    return result.data;
+  };
+  /**
+   * Returns a job's details.
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/getJobDetails
+   * @param jobId
+   * @returns
+   */
+  getJobDetails = async (jobId: string) => {
+    const result = await axios.get(`https://circleci.com/api/v2/project/${this.slug}/job/${jobId}`, {
+      headers: this.headers,
+    });
+    return result.data;
+  };
+  /**
+   * Returns a single job's artifacts.
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/getJobArtifacts
+   * @param jobId
+   * @returns
+   */
+  getJobArtifacts = async (jobId: string) => {
+    const result = await axios.get(`https://circleci.com/api/v2/project/${this.slug}/${jobId}/artifacts`, {
+      headers: this.headers,
+    });
+    return result.data;
+  };
+  /**
+   * Get test metadata for a single job
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/getTests
+   * @param jobId
+   * @returns
+   */
+  getJobTests = async (jobId: string) => {
+    const result = await axios.get(`https://circleci.com/api/v2/project/${this.slug}/${jobId}/tests`, {
+      headers: this.headers,
+    });
+    return result.data;
+  };
+  /**
+   * Get summary metrics for a project workflow's jobs.
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/getProjectWorkflowJobMetrics
+   *
+   * @param workflowName
+   * @param branch
+   * @param reportingWindow
+   * @returns
+   */
+  getAllJobMetrics = async (
+    workflowName: string = this.options.defaultWorkflow,
+    branch: string = this.options.defaultBranch,
+    reportingWindow: ReportingWindow = 'last-30-days',
+  ) => {
+    const result = await axios.get(`https://circleci.com/api/v2/insights/${this.slug}/workflows/${workflowName}/jobs`, {
+      headers: this.headers,
+      params: {
+        branch: branch,
+        'reporting-window': reportingWindow,
+      },
+    });
+    return result.data;
+  };
+
+  /**
+   * Get test metrics for a project's workflows.
+   *
+   * https://circleci.com/docs/api/v2/index.html#operation/getProjectWorkflowTestMetrics
+   * @param workflowName
+   * @param branch
+   * @param reportingWindow
+   * @returns
+   */
+  getAllTestMetrics = async (workflowName: string = this.options.defaultWorkflow, branch: string = this.options.defaultBranch) => {
+    const result = await axios.get(`https://circleci.com/api/v2/insights/${this.slug}/workflows/${workflowName}/test-metrics`, {
+      headers: this.headers,
+      params: {
+        branch: branch,
+      },
+    });
+    return result.data;
+  };
+}

scripts/cci/get-test-timing-from-cci-job-metrics.ts

@@ -0,0 +1,37 @@
+import { join } from 'path';
+import { REPO_ROOT, getCCIClient, getTestFiles, getTestNameFromPath, getTimingsFromJobsData, saveJobMetrics, saveTestTimings } from './utils';
+
+async function main(): Promise<void> {
+  const client = getCCIClient();
+  console.log('Fetching job metrics...');
+  const data = await client.getAllJobMetrics();
+  saveJobMetrics(data);
+
+  const testSuites = getTestFiles(join(REPO_ROOT, 'packages', 'amplify-codegen-e2e-tests'));
+
+  const jobTimings = getTimingsFromJobsData();
+  const testRuntimes = testSuites.map(t => {
+    const oldName = getTestNameFromPath(t);
+    if (jobTimings.has(oldName)) {
+      return {
+        test: t,
+        medianRuntime: jobTimings.get(oldName) as number,
+      };
+    } else {
+      console.log('Could not find timing for:', t);
+      return {
+        test: t,
+        medianRuntime: 10, // default for unknown
+      };
+    }
+  });
+  testRuntimes.sort((a, b) => {
+    return a.medianRuntime - b.medianRuntime;
+  });
+  saveTestTimings({
+    lastUpdated: new Date().toISOString(),
+    totalTestFiles: testRuntimes.length,
+    timingData: testRuntimes,
+  });
+}
+main();