Add retry mechanism to create amplify e2e test (aws-amplify#2220)

* Add retry mechanism to create amplify e2e test * Add retry mechanism to create amplify e2e test * this.
awsluja · Nov 12, 2024 · 9fd0642 · 9fd0642
1 parent 38d6986
commit 9fd0642
Show file tree

Hide file tree

Showing 4 changed files with 55 additions and 37 deletions.
diff --git a/.changeset/serious-spoons-fetch.md b/.changeset/serious-spoons-fetch.md
@@ -0,0 +1,2 @@
+---
+---
diff --git a/packages/integration-tests/src/package_manager_sanity_checks.test.ts b/packages/integration-tests/src/package_manager_sanity_checks.test.ts
@@ -26,7 +26,7 @@ import {
   runWithPackageManager,
 } from './process-controller/process_controller.js';
 import { amplifyAtTag } from './constants.js';
-import { runWithRetry } from './retry.js';
+import { RetryPredicates, runWithRetry } from './retry.js';
 
 void describe('getting started happy path', async () => {
   let branchBackendIdentifier: BackendIdentifier;
@@ -83,34 +83,21 @@ void describe('getting started happy path', async () => {
       return;
     }
 
-    await runWithRetry(
-      async () => {
-        if (packageManager === 'yarn-classic') {
-          await execa('yarn', ['add', 'create-amplify'], { cwd: tempDir });
-          await execaCommand(
-            './node_modules/.bin/create-amplify --yes --debug',
-            {
-              cwd: tempDir,
-              env: { npm_config_user_agent: 'yarn/1.22.21' },
-            }
-          );
-        } else {
-          await runPackageManager(
-            packageManager,
-            ['create', amplifyAtTag, '--yes'],
-            tempDir
-          ).run();
-        }
-      },
-      (error) => {
-        // Retry on network-related errors or command failures
-        return (
-          error.message.includes('exit code 1') &&
-          (error.message.includes('aws-amplify') ||
-            error.message.includes('Command failed with exit code 1: yarn add'))
-        );
+    await runWithRetry(async () => {
+      if (packageManager === 'yarn-classic') {
+        await execa('yarn', ['add', 'create-amplify'], { cwd: tempDir });
+        await execaCommand('./node_modules/.bin/create-amplify --yes --debug', {
+          cwd: tempDir,
+          env: { npm_config_user_agent: 'yarn/1.22.21' },
+        });
+      } else {
+        await runPackageManager(
+          packageManager,
+          ['create', amplifyAtTag, '--yes'],
+          tempDir
+        ).run();
       }
-    );
+    }, RetryPredicates.createAmplifyRetryPredicate);
 
     const pathPrefix = path.join(tempDir, 'amplify');
 

diff --git a/packages/integration-tests/src/retry.ts b/packages/integration-tests/src/retry.ts
@@ -1,3 +1,5 @@
+export type RetryPredicate = (error: Error) => boolean;
+
 /**
  * Executes an asynchronous operation with retry logic.
  * This function attempts to execute the provided callable function multiple times
@@ -6,7 +8,7 @@
  */
 export const runWithRetry = async <T>(
   callable: () => Promise<T>,
-  retryPredicate: (error: Error) => boolean,
+  retryPredicate: RetryPredicate,
   maxAttempts = 3
 ): Promise<T> => {
   const collectedErrors: Error[] = [];
@@ -21,6 +23,10 @@ export const runWithRetry = async <T>(
         if (!retryPredicate(error)) {
           throw error;
         }
+      } else {
+        // re-throw non-Error.
+        // This should never happen, but we should be aware if it does.
+        throw error;
       }
     }
   }
@@ -30,3 +36,23 @@ export const runWithRetry = async <T>(
     `All ${maxAttempts} attempts failed`
   );
 };
+
+/**
+ * Known retry predicates that repeat in multiple places.
+ */
+export class RetryPredicates {
+  static createAmplifyRetryPredicate: RetryPredicate = (
+    error: Error
+  ): boolean => {
+    const message = error.message.toLowerCase();
+    // Note: we can't assert on whole stdout or stderr because
+    // they're not always captured in the error due to settings we need for
+    // ProcessController to work.
+    const didProcessExitWithError = message.includes('exit code 1');
+    const isKnownProcess =
+      (message.includes('yarn add') && message.includes('aws-amplify')) ||
+      message.includes('npm create amplify') ||
+      message.includes('pnpm create amplify');
+    return didProcessExitWithError && isKnownProcess;
+  };
+}
diff --git a/packages/integration-tests/src/test-e2e/create_amplify.test.ts b/packages/integration-tests/src/test-e2e/create_amplify.test.ts
@@ -9,6 +9,7 @@ import { testConcurrencyLevel } from './test_concurrency.js';
 import { findBaselineCdkVersion } from '../cdk_version_finder.js';
 import { amplifyAtTag } from '../constants.js';
 import { NpmProxyController } from '../npm_proxy_controller.js';
+import { RetryPredicates, runWithRetry } from '../retry.js';
 
 void describe(
   'create-amplify script',
@@ -78,14 +79,16 @@ void describe(
             );
           }
 
-          await execa(
-            'npm',
-            ['create', amplifyAtTag, '--yes', '--', '--debug'],
-            {
-              cwd: tempDir,
-              stdio: 'inherit',
-            }
-          );
+          await runWithRetry(async () => {
+            await execa(
+              'npm',
+              ['create', amplifyAtTag, '--yes', '--', '--debug'],
+              {
+                cwd: tempDir,
+                stdio: 'inherit',
+              }
+            );
+          }, RetryPredicates.createAmplifyRetryPredicate);
 
           // Override CDK installation with baseline version
           await execa(