From 95d6815b206f62be41d305bc1b081b1917608ff9 Mon Sep 17 00:00:00 2001 From: Aori Nevo Date: Sun, 6 Oct 2024 19:48:20 -0400 Subject: [PATCH] feat(asyn/await): parallelize commands --- examples/eslintrc-yml/shepherd.yml | 16 ++- src/commands/checkout.ts | 168 +++++++++++++++++++---------- src/util/for-each-repo.ts | 103 +++++++++++------- 3 files changed, 186 insertions(+), 101 deletions(-) diff --git a/examples/eslintrc-yml/shepherd.yml b/examples/eslintrc-yml/shepherd.yml index 41e61350..8825ddb5 100644 --- a/examples/eslintrc-yml/shepherd.yml +++ b/examples/eslintrc-yml/shepherd.yml @@ -1,12 +1,10 @@ -id: 2018.08.15-eslintrc-yml -title: Rename all .eslintrc files to .eslintrc.yml +id: 2024.10.06-test-migration +title: | + feat: test migration adapter: type: github - search_query: repo:NerdWalletOSS/shepherd-demo path:/ filename:.eslintrc + search_query: repo:NerdWalletOSS/shepherd path:/ hooks: - should_migrate: - - ls .eslintrc - apply: - - mv .eslintrc .eslintrc.yml - pr_message: - - echo "Hey! This PR renames `.eslintrc` to `.eslintrc.yml`" + apply: 'touch $SHEPHERD_REPO_DIR/testfile.js && echo "some content" > $SHEPHERD_REPO_DIR/testfile.js' + pr_message: | + echo "e2e test suite" diff --git a/src/commands/checkout.ts b/src/commands/checkout.ts index 8d63d039..b8fe531f 100644 --- a/src/commands/checkout.ts +++ b/src/commands/checkout.ts @@ -1,85 +1,145 @@ import fs from 'fs-extra'; - import IRepoAdapter, { IRepo } from '../adapters/base.js'; import { IMigrationContext } from '../migration-context.js'; import executeSteps from '../util/execute-steps.js'; -import forEachRepo from '../util/for-each-repo.js'; import { updateRepoList } from '../util/persisted-data.js'; +import forEachRepo from '../util/for-each-repo.js'; +import chalk from 'chalk'; const removeRepoDirectories = async (adapter: IRepoAdapter, repo: IRepo) => { await fs.remove(adapter.getRepoDir(repo)); await fs.remove(adapter.getDataDir(repo)); }; -export default async (context: IMigrationContext) => { +const loadRepos = async ( + context: IMigrationContext, + onRetry: (numSeconds: number) => void +): Promise => { const { migration: { selectedRepos }, adapter, logger, } = context; - function onRetry(numSeconds: number) { - logger.info(`Hit rate limit; waiting ${numSeconds} seconds and retrying.`); - } - - let repos; - if (selectedRepos) { logger.info(`Using ${selectedRepos.length} selected repos`); - repos = selectedRepos; + return selectedRepos; } else { const spinner = logger.spinner('Loading candidate repos'); - repos = (await adapter.getCandidateRepos(onRetry)) || []; + const repos = (await adapter.getCandidateRepos(onRetry)) || []; spinner.succeed(`Loaded ${repos.length} repos`); + return repos; + } +}; + +/** + * Handles the checkout process for a given repository. + * + * @param context - The migration context containing the adapter and logger. + * @param repo - The repository to be checked out. + * @param checkedOutRepos - An array to store successfully checked out repositories. + * @param discardedRepos - An array to store repositories that were discarded during the process. + * @param repoLogs - An array to store log messages related to the checkout process. + * + * @returns A promise that resolves when the checkout process is complete. + * + * @throws Will log an error and skip the repository if the checkout process fails. + * + * The function performs the following steps: + * 1. Attempts to check out the repository using the adapter. + * 2. Creates necessary directories for the repository. + * 3. Runs the 'should_migrate' steps and discards the repository if they fail. + * 4. Runs the 'post_checkout' steps and discards the repository if they fail. + * 5. Adds the repository to the checkedOutRepos array if all steps succeed. + */ +const handleRepoCheckout = async ( + context: IMigrationContext, + repo: IRepo, + checkedOutRepos: IRepo[], + discardedRepos: IRepo[], + repoLogs: string[] +) => { + const { adapter, logger } = context; + try { + await adapter.checkoutRepo(repo); + repoLogs.push('Checked out repo'); + } catch (e: any) { + logger.error(e); + repoLogs.push('Failed to check out repo; skipping'); + return; } - context.migration.repos = repos; - const checkedOutRepos: IRepo[] = []; - const discardedRepos: IRepo[] = []; - const options = { warnMissingDirectory: false }; - - await forEachRepo(context, options, async (repo) => { - const spinner = logger.spinner('Checking out repo'); - try { - await adapter.checkoutRepo(repo); - spinner.succeed('Checked out repo'); - } catch (e: any) { - logger.error(e); - spinner.fail('Failed to check out repo; skipping'); - return; - } - // We need to create the data directory before running should_migrate - await fs.mkdirs(adapter.getDataDir(repo)); - - logger.info('> Running should_migrate steps'); - const stepsResults = await executeSteps(context, repo, 'should_migrate'); - if (!stepsResults.succeeded) { - discardedRepos.push(repo); - await removeRepoDirectories(adapter, repo); - logger.failIcon('Error running should_migrate steps; skipping'); - } else { - logger.succeedIcon('Completed all should_migrate steps successfully'); - - logger.info('> Running post_checkout steps'); - const postCheckoutStepsResults = await executeSteps(context, repo, 'post_checkout'); - if (!postCheckoutStepsResults.succeeded) { - discardedRepos.push(repo); - await removeRepoDirectories(adapter, repo); - logger.failIcon('Error running post_checkout steps; skipping'); - } else { - logger.succeedIcon('Completed all post_checkout steps successfully'); - checkedOutRepos.push(repo); - } - } + await fs.mkdirs(adapter.getDataDir(repo)); + + repoLogs.push('> Running should_migrate steps'); + const shouldMigrateResults = await executeSteps(context, repo, 'should_migrate'); + if (!shouldMigrateResults.succeeded) { + discardedRepos.push(repo); + await removeRepoDirectories(adapter, repo); + repoLogs.push('Error running should_migrate steps; skipping'); + return; + } + + repoLogs.push('Completed all should_migrate steps successfully'); + repoLogs.push('> Running post_checkout steps'); + const postCheckoutResults = await executeSteps(context, repo, 'post_checkout'); + if (!postCheckoutResults.succeeded) { + discardedRepos.push(repo); + await removeRepoDirectories(adapter, repo); + repoLogs.push('Error running post_checkout steps; skipping'); + } else { + repoLogs.push('Completed all post_checkout steps successfully'); + checkedOutRepos.push(repo); + } +}; + +const logRepoInfo = ( + repo: IRepo, + count: number, + total: number, + adapter: IRepoAdapter, + repoLogs: string[] +): void => { + const indexString = chalk.dim(`${count}/${total}`); + repoLogs.push(chalk.bold(`\n[${adapter.stringifyRepo(repo)}] ${indexString}`)); +}; + +const checkoutRepos = async ( + context: IMigrationContext, + repos: IRepo[], + checkedOutRepos: IRepo[], + discardedRepos: IRepo[] +) => { + const { adapter, logger } = context; + let count = 1; + + logger.info('Checking out repos'); + await forEachRepo(context, { warnMissingDirectory: false }, async (repo) => { + const repoLogs: string[] = []; + logRepoInfo(repo, count++, repos.length, adapter, repoLogs); + await handleRepoCheckout(context, repo, checkedOutRepos, discardedRepos, repoLogs); + repoLogs.forEach((log) => logger.info(log)); }); logger.info(''); logger.info(`Checked out ${checkedOutRepos.length} out of ${repos.length} repos`); +}; - const mappedCheckedOutRepos: IRepo[] = []; - for (const repo of checkedOutRepos) { - mappedCheckedOutRepos.push(await adapter.mapRepoAfterCheckout(repo)); - } +export default async (context: IMigrationContext) => { + const { adapter, logger } = context; + const onRetry = (numSeconds: number) => + logger.info(`Hit rate limit; waiting ${numSeconds} seconds and retrying.`); + + const repos = await loadRepos(context, onRetry); + context.migration.repos = repos; + + const checkedOutRepos: IRepo[] = []; + const discardedRepos: IRepo[] = []; + + await checkoutRepos(context, repos, checkedOutRepos, discardedRepos); + + const mappedCheckedOutRepos = await Promise.all( + checkedOutRepos.map((repo) => adapter.mapRepoAfterCheckout(repo)) + ); - // We'll persist this list of repos for use in future steps await updateRepoList(context, mappedCheckedOutRepos, discardedRepos); }; diff --git a/src/util/for-each-repo.ts b/src/util/for-each-repo.ts index b6d1676e..f0390a61 100644 --- a/src/util/for-each-repo.ts +++ b/src/util/for-each-repo.ts @@ -3,12 +3,52 @@ import fs from 'fs-extra'; import { IRepo } from '../adapters/base.js'; import { IMigrationContext } from '../migration-context.js'; -type RepoHandler = (repo: IRepo) => Promise; +type RepoHandler = (repo: IRepo, header: string) => Promise; interface IOptions { warnMissingDirectory?: boolean; } +const getRepos = (migrationRepos: IRepo[], selectedRepos: IRepo[], adapter: any): IRepo[] => { + if (selectedRepos && selectedRepos.length) { + return selectedRepos.map((r) => { + const existingRepo = migrationRepos.find((repo) => adapter.reposEqual(r, repo)); + return existingRepo || r; + }); + } + return migrationRepos || []; +}; + +const logRepoInfo = (repo: IRepo, index: number, total: number, adapter: any) => { + const indexString = chalk.dim(`${index}/${total}`); + return chalk.bold(`\n[${adapter.stringifyRepo(repo)}] ${indexString}`); +}; + +const checkRepoDirectory = async (repoDir: string, warnMissingDirectory: boolean) => { + if (warnMissingDirectory && !(await fs.pathExists(repoDir))) { + return `Directory ${repoDir} does not exist`; + } + return ''; +}; + +/** + * Processes each repository in the migration context using the provided handler. + * + * @param context - The migration context containing repositories, logger, and adapter. + * @param param1 - Either a repository handler function or options for processing. + * @param param2 - Optional repository handler function if `param1` is options. + * + * @returns A promise that resolves when all repositories have been processed. + * + * The function extracts repositories from the migration context and applies the provided handler + * to each repository. It logs warnings for missing directories and errors encountered during processing. + * + * @example + * ```typescript + * await forEachRepo(context, handler); + * await forEachRepo(context, options, handler); + * ``` + */ export default async ( context: IMigrationContext, param1: RepoHandler | IOptions, @@ -20,47 +60,34 @@ export default async ( adapter, } = context; - let handler: RepoHandler; - let options: IOptions; - if (typeof param1 === 'function') { - // No options were provided - options = {}; - handler = param1; - } else { - // We got options! - options = param1; - handler = param2 as RepoHandler; - } + const { handler, options } = + typeof param1 === 'function' + ? { handler: param1, options: {} as IOptions } + : { handler: param2 as RepoHandler, options: param1 }; - // We want to show these warnings by default and allow opt-out of them const { warnMissingDirectory = true } = options; + const repos = getRepos(migrationRepos || [], selectedRepos || [], adapter); - // If `selectedRepos` is specified, we should use that instead of the full repo list - let repos; - if (selectedRepos && selectedRepos.length) { - // If this repo was already checked out, it may have additional metadata - // associated with it that came from the adapter's mapRepoAfterCheckout - // Let's rely on the migrations from the list on disk if at all possible - repos = selectedRepos.map((r) => { - const existingRepo = (migrationRepos || []).find((repo) => adapter.reposEqual(r, repo)); - return existingRepo || r; - }); - } else { - repos = migrationRepos || []; - } - - let index = 0; - for (const repo of repos) { - index += 1; - const indexString = chalk.dim(`${index}/${repos.length}`); - logger.info(chalk.bold(`\n[${adapter.stringifyRepo(repo)}] ${indexString}`)); - - // Quick sanity check in case we're working from user-selected repos + const _handler = async function (repo: IRepo, index: number, repos: IRepo[]) { const repoDir = adapter.getRepoDir(repo); - if (warnMissingDirectory && !(await fs.pathExists(repoDir))) { - logger.error(`Directory ${repoDir} does not exist`); + const dirCheckMessage = await checkRepoDirectory(repoDir, warnMissingDirectory); + if (dirCheckMessage) { + logger.warn(dirCheckMessage); } - await handler(repo); - } + try { + const header = logRepoInfo(repo, index + 1, repos.length, adapter); + await handler(repo, header); + } catch (error: unknown) { + if (error instanceof Error) { + logger.error(`Error processing repo ${adapter.stringifyRepo(repo)}: ${error.message}`); + } else { + logger.error(`Error processing repo ${adapter.stringifyRepo(repo)}: ${String(error)}`); + } + } + }; + + const repoListWithHandlersApplied = repos.map(_handler); + + await Promise.all(repoListWithHandlersApplied); };