feat(asyn/await): parallelize commands

examples/eslintrc-yml/shepherd.yml

@@ -1,12 +1,10 @@
-id: 2018.08.15-eslintrc-yml
-title: Rename all .eslintrc files to .eslintrc.yml
+id: 2024.10.06-test-migration
+title: |
+  feat: test migration
 adapter:
   type: github
-  search_query: repo:NerdWalletOSS/shepherd-demo path:/ filename:.eslintrc
+  search_query: repo:NerdWalletOSS/shepherd path:/
 hooks:
-  should_migrate:
-    - ls .eslintrc
-  apply:
-    - mv .eslintrc .eslintrc.yml
-  pr_message:
-    - echo "Hey! This PR renames `.eslintrc` to `.eslintrc.yml`"
+  apply: 'touch $SHEPHERD_REPO_DIR/testfile.js && echo "some content" > $SHEPHERD_REPO_DIR/testfile.js'
+  pr_message: |
+    echo "e2e test suite"

src/commands/checkout.ts

@@ -1,85 +1,145 @@
 import fs from 'fs-extra';
-
 import IRepoAdapter, { IRepo } from '../adapters/base.js';
 import { IMigrationContext } from '../migration-context.js';
 import executeSteps from '../util/execute-steps.js';
-import forEachRepo from '../util/for-each-repo.js';
 import { updateRepoList } from '../util/persisted-data.js';
+import forEachRepo from '../util/for-each-repo.js';
+import chalk from 'chalk';
 
 const removeRepoDirectories = async (adapter: IRepoAdapter, repo: IRepo) => {
   await fs.remove(adapter.getRepoDir(repo));
   await fs.remove(adapter.getDataDir(repo));
 };
 
-export default async (context: IMigrationContext) => {
+const loadRepos = async (
+  context: IMigrationContext,
+  onRetry: (numSeconds: number) => void
+): Promise<IRepo[]> => {
   const {
     migration: { selectedRepos },
     adapter,
     logger,
   } = context;
-  function onRetry(numSeconds: number) {
-    logger.info(`Hit rate limit; waiting ${numSeconds} seconds and retrying.`);
-  }
-
-  let repos;
-
   if (selectedRepos) {
     logger.info(`Using ${selectedRepos.length} selected repos`);
-    repos = selectedRepos;
+    return selectedRepos;
   } else {
     const spinner = logger.spinner('Loading candidate repos');
-    repos = (await adapter.getCandidateRepos(onRetry)) || [];
+    const repos = (await adapter.getCandidateRepos(onRetry)) || [];
     spinner.succeed(`Loaded ${repos.length} repos`);
+    return repos;
+  }
+};
+
+/**
+ * Handles the checkout process for a given repository.
+ *
+ * @param context - The migration context containing the adapter and logger.
+ * @param repo - The repository to be checked out.
+ * @param checkedOutRepos - An array to store successfully checked out repositories.
+ * @param discardedRepos - An array to store repositories that were discarded during the process.
+ * @param repoLogs - An array to store log messages related to the checkout process.
+ *
+ * @returns A promise that resolves when the checkout process is complete.
+ *
+ * @throws Will log an error and skip the repository if the checkout process fails.
+ *
+ * The function performs the following steps:
+ * 1. Attempts to check out the repository using the adapter.
+ * 2. Creates necessary directories for the repository.
+ * 3. Runs the 'should_migrate' steps and discards the repository if they fail.
+ * 4. Runs the 'post_checkout' steps and discards the repository if they fail.
+ * 5. Adds the repository to the checkedOutRepos array if all steps succeed.
+ */
+const handleRepoCheckout = async (
+  context: IMigrationContext,
+  repo: IRepo,
+  checkedOutRepos: IRepo[],
+  discardedRepos: IRepo[],
+  repoLogs: string[]
+) => {
+  const { adapter, logger } = context;
+  try {
+    await adapter.checkoutRepo(repo);
+    repoLogs.push('Checked out repo');
+  } catch (e: any) {
+    logger.error(e);
+    repoLogs.push('Failed to check out repo; skipping');
+    return;
   }
-  context.migration.repos = repos;
-  const checkedOutRepos: IRepo[] = [];
-  const discardedRepos: IRepo[] = [];
 
-  const options = { warnMissingDirectory: false };
-
-  await forEachRepo(context, options, async (repo) => {
-    const spinner = logger.spinner('Checking out repo');
-    try {
-      await adapter.checkoutRepo(repo);
-      spinner.succeed('Checked out repo');
-    } catch (e: any) {
-      logger.error(e);
-      spinner.fail('Failed to check out repo; skipping');
-      return;
-    }
-    // We need to create the data directory before running should_migrate
-    await fs.mkdirs(adapter.getDataDir(repo));
-
-    logger.info('> Running should_migrate steps');
-    const stepsResults = await executeSteps(context, repo, 'should_migrate');
-    if (!stepsResults.succeeded) {
-      discardedRepos.push(repo);
-      await removeRepoDirectories(adapter, repo);
-      logger.failIcon('Error running should_migrate steps; skipping');
-    } else {
-      logger.succeedIcon('Completed all should_migrate steps successfully');
-
-      logger.info('> Running post_checkout steps');
-      const postCheckoutStepsResults = await executeSteps(context, repo, 'post_checkout');
-      if (!postCheckoutStepsResults.succeeded) {
-        discardedRepos.push(repo);
-        await removeRepoDirectories(adapter, repo);
-        logger.failIcon('Error running post_checkout steps; skipping');
-      } else {
-        logger.succeedIcon('Completed all post_checkout steps successfully');
-        checkedOutRepos.push(repo);
-      }
-    }
+  await fs.mkdirs(adapter.getDataDir(repo));
+
+  repoLogs.push('> Running should_migrate steps');
+  const shouldMigrateResults = await executeSteps(context, repo, 'should_migrate');
+  if (!shouldMigrateResults.succeeded) {
+    discardedRepos.push(repo);
+    await removeRepoDirectories(adapter, repo);
+    repoLogs.push('Error running should_migrate steps; skipping');
+    return;
+  }
+
+  repoLogs.push('Completed all should_migrate steps successfully');
+  repoLogs.push('> Running post_checkout steps');
+  const postCheckoutResults = await executeSteps(context, repo, 'post_checkout');
+  if (!postCheckoutResults.succeeded) {
+    discardedRepos.push(repo);
+    await removeRepoDirectories(adapter, repo);
+    repoLogs.push('Error running post_checkout steps; skipping');
+  } else {
+    repoLogs.push('Completed all post_checkout steps successfully');
+    checkedOutRepos.push(repo);
+  }
+};
+
+const logRepoInfo = (
+  repo: IRepo,
+  count: number,
+  total: number,
+  adapter: IRepoAdapter,
+  repoLogs: string[]
+): void => {
+  const indexString = chalk.dim(`${count}/${total}`);
+  repoLogs.push(chalk.bold(`\n[${adapter.stringifyRepo(repo)}] ${indexString}`));
+};
+
+const checkoutRepos = async (
+  context: IMigrationContext,
+  repos: IRepo[],
+  checkedOutRepos: IRepo[],
+  discardedRepos: IRepo[]
+) => {
+  const { adapter, logger } = context;
+  let count = 1;
+
+  logger.info('Checking out repos');
+  await forEachRepo(context, { warnMissingDirectory: false }, async (repo) => {
+    const repoLogs: string[] = [];
+    logRepoInfo(repo, count++, repos.length, adapter, repoLogs);
+    await handleRepoCheckout(context, repo, checkedOutRepos, discardedRepos, repoLogs);
+    repoLogs.forEach((log) => logger.info(log));
   });
 
   logger.info('');
   logger.info(`Checked out ${checkedOutRepos.length} out of ${repos.length} repos`);
+};
 
-  const mappedCheckedOutRepos: IRepo[] = [];
-  for (const repo of checkedOutRepos) {
-    mappedCheckedOutRepos.push(await adapter.mapRepoAfterCheckout(repo));
-  }
+export default async (context: IMigrationContext) => {
+  const { adapter, logger } = context;
+  const onRetry = (numSeconds: number) =>
+    logger.info(`Hit rate limit; waiting ${numSeconds} seconds and retrying.`);
+
+  const repos = await loadRepos(context, onRetry);
+  context.migration.repos = repos;
+
+  const checkedOutRepos: IRepo[] = [];
+  const discardedRepos: IRepo[] = [];
+
+  await checkoutRepos(context, repos, checkedOutRepos, discardedRepos);
+
+  const mappedCheckedOutRepos = await Promise.all(
+    checkedOutRepos.map((repo) => adapter.mapRepoAfterCheckout(repo))
+  );
 
-  // We'll persist this list of repos for use in future steps
   await updateRepoList(context, mappedCheckedOutRepos, discardedRepos);
 };

src/util/for-each-repo.ts

@@ -3,12 +3,52 @@ import fs from 'fs-extra';
 import { IRepo } from '../adapters/base.js';
 import { IMigrationContext } from '../migration-context.js';
 
-type RepoHandler = (repo: IRepo) => Promise<void>;
+type RepoHandler = (repo: IRepo, header: string) => Promise<void>;
 
 interface IOptions {
   warnMissingDirectory?: boolean;
 }
 
+const getRepos = (migrationRepos: IRepo[], selectedRepos: IRepo[], adapter: any): IRepo[] => {
+  if (selectedRepos && selectedRepos.length) {
+    return selectedRepos.map((r) => {
+      const existingRepo = migrationRepos.find((repo) => adapter.reposEqual(r, repo));
+      return existingRepo || r;
+    });
+  }
+  return migrationRepos || [];
+};
+
+const logRepoInfo = (repo: IRepo, index: number, total: number, adapter: any) => {
+  const indexString = chalk.dim(`${index}/${total}`);
+  return chalk.bold(`\n[${adapter.stringifyRepo(repo)}] ${indexString}`);
+};
+
+const checkRepoDirectory = async (repoDir: string, warnMissingDirectory: boolean) => {
+  if (warnMissingDirectory && !(await fs.pathExists(repoDir))) {
+    return `Directory ${repoDir} does not exist`;
+  }
+  return '';
+};
+
+/**
+ * Processes each repository in the migration context using the provided handler.
+ *
+ * @param context - The migration context containing repositories, logger, and adapter.
+ * @param param1 - Either a repository handler function or options for processing.
+ * @param param2 - Optional repository handler function if `param1` is options.
+ *
+ * @returns A promise that resolves when all repositories have been processed.
+ *
+ * The function extracts repositories from the migration context and applies the provided handler
+ * to each repository. It logs warnings for missing directories and errors encountered during processing.
+ *
+ * @example
+ * ```typescript
+ * await forEachRepo(context, handler);
+ * await forEachRepo(context, options, handler);
+ * ```
+ */
 export default async (
   context: IMigrationContext,
   param1: RepoHandler | IOptions,
@@ -20,47 +60,34 @@ export default async (
     adapter,
   } = context;
 
-  let handler: RepoHandler;
-  let options: IOptions;
-  if (typeof param1 === 'function') {
-    // No options were provided
-    options = {};
-    handler = param1;
-  } else {
-    // We got options!
-    options = param1;
-    handler = param2 as RepoHandler;
-  }
+  const { handler, options } =
+    typeof param1 === 'function'
+      ? { handler: param1, options: {} as IOptions }
+      : { handler: param2 as RepoHandler, options: param1 };
 
-  // We want to show these warnings by default and allow opt-out of them
   const { warnMissingDirectory = true } = options;
+  const repos = getRepos(migrationRepos || [], selectedRepos || [], adapter);
 
-  // If `selectedRepos` is specified, we should use that instead of the full repo list
-  let repos;
-  if (selectedRepos && selectedRepos.length) {
-    // If this repo was already checked out, it may have additional metadata
-    // associated with it that came from the adapter's mapRepoAfterCheckout
-    // Let's rely on the migrations from the list on disk if at all possible
-    repos = selectedRepos.map((r) => {
-      const existingRepo = (migrationRepos || []).find((repo) => adapter.reposEqual(r, repo));
-      return existingRepo || r;
-    });
-  } else {
-    repos = migrationRepos || [];
-  }
-
-  let index = 0;
-  for (const repo of repos) {
-    index += 1;
-    const indexString = chalk.dim(`${index}/${repos.length}`);
-    logger.info(chalk.bold(`\n[${adapter.stringifyRepo(repo)}] ${indexString}`));
-
-    // Quick sanity check in case we're working from user-selected repos
+  const _handler = async function (repo: IRepo, index: number, repos: IRepo[]) {
     const repoDir = adapter.getRepoDir(repo);
-    if (warnMissingDirectory && !(await fs.pathExists(repoDir))) {
-      logger.error(`Directory ${repoDir} does not exist`);
+    const dirCheckMessage = await checkRepoDirectory(repoDir, warnMissingDirectory);
+    if (dirCheckMessage) {
+      logger.warn(dirCheckMessage);
     }
 
-    await handler(repo);
-  }
+    try {
+      const header = logRepoInfo(repo, index + 1, repos.length, adapter);
+      await handler(repo, header);
+    } catch (error: unknown) {
+      if (error instanceof Error) {
+        logger.error(`Error processing repo ${adapter.stringifyRepo(repo)}: ${error.message}`);
+      } else {
+        logger.error(`Error processing repo ${adapter.stringifyRepo(repo)}: ${String(error)}`);
+      }
+    }
+  };
+
+  const repoListWithHandlersApplied = repos.map(_handler);
+
+  await Promise.all(repoListWithHandlersApplied);
 };