From 9ddeba518579215ff361eb5b296689b696d3d7f4 Mon Sep 17 00:00:00 2001 From: Sasha Mysak Date: Tue, 27 Aug 2024 02:46:11 +0200 Subject: [PATCH] changes to logic --- apps/backend/src/publishHandler.ts | 8 +++- apps/cli/src/commands/init.ts | 36 ++++++++++++---- apps/cli/src/commands/publish.ts | 15 ++++--- apps/cli/src/fetch-codemod.ts | 36 ++++------------ apps/cli/src/flags.ts | 6 +++ apps/cli/src/main.ts | 2 + apps/cli/src/safe-arguments.ts | 6 ++- apps/cli/src/worker.ts | 5 ++- apps/docs/deploying-codemods/cli.mdx | 11 +++++ .../guides/sharing/publishing-codemods.mdx | 2 + .../app/(website)/studio/main/RunOptions.tsx | 25 ++++++----- .../printer/src/schemata/worker-thread.ts | 1 - packages/runner/src/runner.ts | 7 +++- packages/runner/src/schemata/flow-settings.ts | 1 + packages/runner/src/source-code.ts | 42 ++++++++----------- packages/utilities/src/package-boilerplate.ts | 2 +- 16 files changed, 118 insertions(+), 87 deletions(-) diff --git a/apps/backend/src/publishHandler.ts b/apps/backend/src/publishHandler.ts index 5b9655157..e8daa692a 100644 --- a/apps/backend/src/publishHandler.ts +++ b/apps/backend/src/publishHandler.ts @@ -22,7 +22,7 @@ import { import type { UserDataPopulatedRequest } from "@codemod-com/auth"; import { prisma } from "@codemod-com/database"; // Direct import because tree-shaking helps this to not throw. -import { getCodemodExecutable } from "@codemod-com/runner/dist/source-code.js"; +import { BUILT_SOURCE_PATH } from "@codemod-com/runner/dist/source-code.js"; import { buildCodemodSlug, codemodNameRegex, @@ -34,6 +34,7 @@ import { unzip, } from "@codemod-com/utilities"; +import { readFile } from "node:fs/promises"; import { buildRevalidateHelper } from "./revalidate.js"; import { environment } from "./util.js"; @@ -147,7 +148,10 @@ export const publishHandler: RouteHandler<{ throwOnNotFound: false, }); - const built = await getCodemodExecutable(unpackPath).catch(() => null); + const built = await readFile( + join(unpackPath, BUILT_SOURCE_PATH), + "utf8", + ).catch(() => null); if (path === null || built === null) { return reply.code(400).send({ diff --git a/apps/cli/src/commands/init.ts b/apps/cli/src/commands/init.ts index 5adecd989..7e4f3b078 100644 --- a/apps/cli/src/commands/init.ts +++ b/apps/cli/src/commands/init.ts @@ -12,6 +12,11 @@ import inquirer from "inquirer"; import terminalLink from "terminal-link"; import { type Printer, chalk } from "@codemod-com/printer"; +import { + BUILT_SOURCE_PATH, + bundleJS, + getCodemodExecutable, +} from "@codemod-com/runner"; import { ALL_ENGINES, type KnownEngines, @@ -31,12 +36,16 @@ import { oraCheckmark } from "#utils/constants.js"; import { detectCodemodEngine } from "#utils/detectCodemodEngine.js"; import { isFile } from "#utils/general.js"; +// @TODO: decompose export const handleInitCliCommand = async (options: { printer: Printer; target: string; source?: string; engine?: string; + esm?: boolean; + build?: boolean; noLogs?: boolean; + noFixtures?: boolean; useDefaultName?: boolean; }) => { const { @@ -44,8 +53,11 @@ export const handleInitCliCommand = async (options: { target, source, engine, - useDefaultName = false, + esm, + build = true, noLogs = false, + noFixtures = false, + useDefaultName = false, } = options; if (source) { @@ -135,6 +147,9 @@ export const handleInitCliCommand = async (options: { downloadInput.codemodBody = await readFile(source, "utf-8"); } + if (noFixtures) { + downloadInput.cases = []; + } const files = getCodemodProjectFiles(downloadInput); const codemodBaseDir = join(target ?? process.cwd(), downloadInput.name); @@ -154,11 +169,7 @@ export const handleInitCliCommand = async (options: { }); } catch (err) { for (const createdPath of created) { - try { - await unlink(join(codemodBaseDir, createdPath)); - } catch (err) { - // - } + await unlink(join(codemodBaseDir, createdPath)).catch(() => null); } throw new Error( @@ -172,10 +183,19 @@ export const handleInitCliCommand = async (options: { } } - if (noLogs) { - return codemodBaseDir; + if (build) { + const outputPath = join(codemodBaseDir, BUILT_SOURCE_PATH); + const bundledCode = + source && isSourceAFile + ? await bundleJS({ entry: source, output: outputPath, esm }) + : await getCodemodExecutable(codemodBaseDir, esm); + + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, bundledCode); } + if (noLogs) return codemodBaseDir; + printer.printConsoleMessage( "info", chalk.cyan("Codemod package created at", `${chalk.bold(codemodBaseDir)}.`), diff --git a/apps/cli/src/commands/publish.ts b/apps/cli/src/commands/publish.ts index b64fb19ec..b684d823c 100644 --- a/apps/cli/src/commands/publish.ts +++ b/apps/cli/src/commands/publish.ts @@ -6,6 +6,7 @@ import inquirer from "inquirer"; import * as semver from "semver"; import * as v from "valibot"; +import { tmpdir } from "node:os"; import { type Printer, chalk } from "@codemod-com/printer"; import { BUILT_SOURCE_PATH, getCodemodExecutable } from "@codemod-com/runner"; import type { TelemetrySender } from "@codemod-com/telemetry"; @@ -23,15 +24,15 @@ import { extractPrintableApiError, getCodemod, publish } from "#api.js"; import { getCurrentUserOrLogin } from "#auth-utils.js"; import { handleInitCliCommand } from "#commands/init.js"; import type { TelemetryEvent } from "#telemetry.js"; -import { codemodDirectoryPath } from "#utils/constants.js"; import { isFile } from "#utils/general.js"; export const handlePublishCliCommand = async (options: { printer: Printer; source: string; telemetry: TelemetrySender; + esm?: boolean; }) => { - let { source, printer, telemetry } = options; + let { source, printer, telemetry, esm } = options; const { token, allowedNamespaces, organizations } = await getCurrentUserOrLogin({ @@ -39,7 +40,6 @@ export const handlePublishCliCommand = async (options: { printer, }); - const tempDirectory = join(codemodDirectoryPath, "temp"); const formData = new FormData(); const excludedPaths = [ "node_modules/**", @@ -53,8 +53,11 @@ export const handlePublishCliCommand = async (options: { source = await handleInitCliCommand({ printer, source, - target: tempDirectory, + target: tmpdir(), noLogs: true, + noFixtures: true, + build: true, + esm, }); const { choice } = await inquirer.prompt<{ choice: string }>({ @@ -260,7 +263,7 @@ export const handlePublishCliCommand = async (options: { ); if (codemodRc.engine !== "recipe") { - const builtExecutable = await getCodemodExecutable(source).catch( + const builtExecutable = await getCodemodExecutable(source, esm).catch( () => null, ); @@ -315,7 +318,7 @@ export const handlePublishCliCommand = async (options: { message: errorMessage, }); } finally { - if (source.includes(tempDirectory)) { + if (source.includes(tmpdir())) { await fs.promises.rm(source, { recursive: true, force: true }); } } diff --git a/apps/cli/src/fetch-codemod.ts b/apps/cli/src/fetch-codemod.ts index 716c77782..dcdfd7c52 100644 --- a/apps/cli/src/fetch-codemod.ts +++ b/apps/cli/src/fetch-codemod.ts @@ -1,5 +1,6 @@ -import { createHash, randomBytes } from "node:crypto"; -import { lstat, mkdir, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { lstat, mkdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { join, parse as pathParse, resolve } from "node:path"; import type { AxiosError } from "axios"; import inquirer from "inquirer"; @@ -7,14 +8,12 @@ import semver from "semver"; import { flatten } from "valibot"; import { type Printer, chalk } from "@codemod-com/printer"; -import { bundleJS } from "@codemod-com/runner"; import { type Codemod, type CodemodValidationInput, buildCodemodSlug, doubleQuotify, getCodemodRc, - isJavaScriptName, isRecipeCodemod, parseCodemod, parseEngineOptions, @@ -123,34 +122,15 @@ export const fetchCodemod = async (options: FetchOptions): Promise => { } // Standalone codemod - // For standalone codemods, before creating a compatible package, we attempt to - // build the binary because it might have other dependencies in the folder - const tempFolderPath = join(codemodDirectoryPath, "temp"); - await mkdir(tempFolderPath, { recursive: true }); - - let codemodPath = nameOrPath; - if (isJavaScriptName(nameOrPath)) { - codemodPath = join( - tempFolderPath, - `${randomBytes(8).toString("hex")}.js`, - ); - - try { - const executable = await bundleJS({ entry: nameOrPath }); - await writeFile(codemodPath, executable); - } catch (err) { - throw new Error( - `Error bundling codemod: ${(err as Error).message ?? "Unknown error"}`, - ); - } - } - const codemodPackagePath = await handleInitCliCommand({ printer, - source: codemodPath, - target: tempFolderPath, + source: nameOrPath, + target: tmpdir(), useDefaultName: true, noLogs: true, + noFixtures: true, + build: true, + esm: argv.esm, }); const { config } = await getCodemodRc({ diff --git a/apps/cli/src/flags.ts b/apps/cli/src/flags.ts index 861e30df4..3727cfe13 100644 --- a/apps/cli/src/flags.ts +++ b/apps/cli/src/flags.ts @@ -41,6 +41,12 @@ export const buildGlobalOptions = (y: Argv) => alias: "v", description: "Show version number", }) + .option("esm", { + type: "boolean", + description: + "Use to specify that you intend to use ESM-specific features in your codemods.", + default: false, + }) .option("json", { alias: "j", type: "boolean", diff --git a/apps/cli/src/main.ts b/apps/cli/src/main.ts index 13474a820..891edfd7b 100644 --- a/apps/cli/src/main.ts +++ b/apps/cli/src/main.ts @@ -263,6 +263,7 @@ export const main = async () => { printer, source: args.source ?? process.cwd(), telemetry: telemetryService, + esm: args.esm, }); }); }, @@ -326,6 +327,7 @@ export const main = async () => { printer, target: args.target ?? process.cwd(), engine: args.engine, + esm: args.esm, }), ); }, diff --git a/apps/cli/src/safe-arguments.ts b/apps/cli/src/safe-arguments.ts index 52568adc6..fea61da1c 100644 --- a/apps/cli/src/safe-arguments.ts +++ b/apps/cli/src/safe-arguments.ts @@ -12,7 +12,11 @@ export const buildSafeArgumentRecord = async ( if (codemod.type === "standalone") { // no checks performed for local codemods // b/c no source of truth for the arguments - return argvRecord as ArgumentRecord; + const { ...safeRecord } = argvRecord; + delete safeRecord._; + delete safeRecord.include; + delete safeRecord.exclude; + return safeRecord as ArgumentRecord; } const validateStringArg = (options: { diff --git a/apps/cli/src/worker.ts b/apps/cli/src/worker.ts index 84841acce..d13e62560 100644 --- a/apps/cli/src/worker.ts +++ b/apps/cli/src/worker.ts @@ -32,7 +32,9 @@ const messageHandler = async (m: unknown) => { try { message = decodeMainThreadMessage(m); } catch (err) { - throw new Error(`Failed to decode message: ${String(err)}`); + throw new Error( + `Failed to decode message: ${String(err)} - ${JSON.stringify(m)}`, + ); } if (message.kind === "initialization") { @@ -118,7 +120,6 @@ const messageHandler = async (m: unknown) => { kind: "error", message: error instanceof Error ? error.message : String(error), path: error instanceof PathAwareError ? error.path : undefined, - stack: error instanceof Error ? error.stack : undefined, } satisfies WorkerThreadMessage); } }; diff --git a/apps/docs/deploying-codemods/cli.mdx b/apps/docs/deploying-codemods/cli.mdx index 6b1aa2f20..31b3121fb 100644 --- a/apps/docs/deploying-codemods/cli.mdx +++ b/apps/docs/deploying-codemods/cli.mdx @@ -261,6 +261,17 @@ codemod [codemod-name] --format + + If you want to use ESM-specific features like top-level `await` or `import.meta` in your codemods, you can use this flag. + + You can also rename your codemod entry-point file to use `.mjs` or `.mts` extension to omit using this compatibility flag. + +```bash +codemod [codemod-name] --json +``` + + + Can be used to disable caching downloaded codemod files. diff --git a/apps/docs/guides/sharing/publishing-codemods.mdx b/apps/docs/guides/sharing/publishing-codemods.mdx index 44d89db7c..c314580ac 100644 --- a/apps/docs/guides/sharing/publishing-codemods.mdx +++ b/apps/docs/guides/sharing/publishing-codemods.mdx @@ -54,6 +54,8 @@ Creating and publishing a [Codemod-compatible package](/guides/building-codemods To generate a [Codemod-compatible package](/guides/building-codemods/package-requirements), Codemod CLI will ask you a few questions about your codemod to generate the [`codemodrc.json` configuration file](/guides/building-codemods/package-requirements#codemodrc-json-reference) automatically for you. + If you want to use ESM-specific features like top-level `await` or `import.meta`, you can rename it to use `.mjs` or `.mts` extension or specify `--esm` flag to treat the source file as ESM package explicitly. + ```bash codemod publish [path] --source ``` diff --git a/apps/frontend/app/(website)/studio/main/RunOptions.tsx b/apps/frontend/app/(website)/studio/main/RunOptions.tsx index cd43c5fa1..f2fcf1704 100644 --- a/apps/frontend/app/(website)/studio/main/RunOptions.tsx +++ b/apps/frontend/app/(website)/studio/main/RunOptions.tsx @@ -41,6 +41,19 @@ export const RunOptions = () => { const modStore = useModStore(); const { engine, getAllSnippets } = useSnippetsStore(); + const allSnippets = getAllSnippets(); + const cases = allSnippets.before.reduce( + (acc, before, i) => { + const after = allSnippets.after[i]; + if (!after) { + return acc; + } + + return acc.concat({ before, after }); + }, + [] as { before: string; after: string }[], + ); + const { session } = useSession(); const { getToken } = useAuth(); @@ -77,17 +90,7 @@ export const RunOptions = () => { // TODO: temporary fix, most likely we need to upgrade monaco editor or babel or whatever is responsible // for taking the code from the web-editor and converting it to string codemodBody: modStore.content.replace(/\n *as\n *const/g, " as const"), - cases: allSnippets.before.reduce( - (acc, before, i) => { - const after = allSnippets.after[i]; - if (!after) { - return acc; - } - - return acc.concat({ before, after }); - }, - [] as { before: string; after: string }[], - ), + cases: cases.length ? cases : undefined, engine, username: session.user.username ?? session.user.fullName, }); diff --git a/packages/printer/src/schemata/worker-thread.ts b/packages/printer/src/schemata/worker-thread.ts index f4c8e5b93..89e4b3917 100644 --- a/packages/printer/src/schemata/worker-thread.ts +++ b/packages/printer/src/schemata/worker-thread.ts @@ -12,7 +12,6 @@ const workerThreadMessageSchema = v.union([ kind: v.literal("error"), message: v.string(), path: v.optional(v.string()), - stack: v.optional(v.string()), }), v.object({ kind: v.literal("console"), diff --git a/packages/runner/src/runner.ts b/packages/runner/src/runner.ts index f3e8bb6c8..b755daf8d 100644 --- a/packages/runner/src/runner.ts +++ b/packages/runner/src/runner.ts @@ -122,7 +122,7 @@ export class Runner { }) { const { codemod, flowSettings, onError, onSuccess, printer } = options; const cloudRunner = await this._options.runnerService?.startCodemodRun({ - source: await getCodemodExecutable(codemod.path), + source: await getCodemodExecutable(codemod.path, flowSettings.esm), engine: codemod.config.engine as "workflow", args: codemod.safeArgumentRecord, }); @@ -484,7 +484,10 @@ export class Runner { return await onSuccess?.({ codemod, output: "", commands: [] }); } - const codemodSource = await getCodemodExecutable(codemod.path); + const codemodSource = await getCodemodExecutable( + codemod.path, + flowSettings.esm, + ); const transformer = await getTransformer(codemodSource); if (codemod.config.engine === "workflow") { diff --git a/packages/runner/src/schemata/flow-settings.ts b/packages/runner/src/schemata/flow-settings.ts index f31005c95..2bb16c2bd 100644 --- a/packages/runner/src/schemata/flow-settings.ts +++ b/packages/runner/src/schemata/flow-settings.ts @@ -46,6 +46,7 @@ export const flowSettingsSchema = v.object({ json: v.optional(v.boolean(), DEFAULT_USE_JSON), threads: v.optional(v.pipe(v.number(), v.minValue(0)), DEFAULT_THREAD_COUNT), cloud: v.optional(v.boolean(), false), + esm: v.optional(v.boolean(), false), }); export type FlowSettings = v.InferOutput; diff --git a/packages/runner/src/source-code.ts b/packages/runner/src/source-code.ts index 09ec61d9e..8408480cd 100644 --- a/packages/runner/src/source-code.ts +++ b/packages/runner/src/source-code.ts @@ -1,5 +1,5 @@ import { createHash } from "node:crypto"; -import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { dirname, join, resolve } from "node:path"; import esbuild from "esbuild"; @@ -11,7 +11,7 @@ export type TransformFunction = ( ) => unknown | Promise; type NonDefaultExports = { - __esModule?: true; + __esModule?: boolean; default?: TransformFunction; handleSourceFile?: TransformFunction; transform?: TransformFunction; @@ -20,6 +20,9 @@ type NonDefaultExports = { filemod?: TransformFunction; }; +export const isESMExtension = (path: string) => + path.endsWith(".mjs") || path.endsWith(".mts"); + export const temporaryLoadedModules = new Map< string, TransformFunction | null @@ -52,12 +55,11 @@ export const getTransformer = async (source: string, name?: string) => { } catch (err) { // ESM try { - const tempDir = tmpdir(); - const tempFilePath = join(tempDir, `temp-module-${hashDigest}.mjs`); - - const alreadyLoaded = temporaryLoadedModules.get(tempFilePath); + const alreadyLoaded = temporaryLoadedModules.get(hashDigest); if (alreadyLoaded) return alreadyLoaded; + const tempFilePath = join(tmpdir(), `temp-module-${hashDigest}.mjs`); + await writeFile(tempFilePath, source); const module = (await import( `file://${tempFilePath}` @@ -76,10 +78,9 @@ export const getTransformer = async (source: string, name?: string) => { null : null; - temporaryLoadedModules.set(tempFilePath, transformer); + temporaryLoadedModules.set(hashDigest, transformer); return transformer; } catch (err) { - console.log(err); return null; } } @@ -95,24 +96,25 @@ export const bundleJS = async (options: { const { entry, output = join(dirname(entry), BUILT_SOURCE_PATH), - esm = true, + esm: argvEsm, } = options; + const isESM = isESMExtension(entry) || argvEsm; const EXTERNAL_DEPENDENCIES = ["jscodeshift", "ts-morph", "@ast-grep/napi"]; const buildOptions: Parameters[0] = { entryPoints: [entry], bundle: true, - external: esm ? undefined : EXTERNAL_DEPENDENCIES, + external: isESM ? undefined : EXTERNAL_DEPENDENCIES, platform: "node", minify: true, minifyWhitespace: true, - format: esm ? "esm" : "cjs", + format: isESM ? "esm" : "cjs", legalComments: "inline", outfile: output, write: false, // to the in-memory file system logLevel: "error", - mainFields: esm ? ["module", "main"] : undefined, - banner: esm + mainFields: isESM ? ["module", "main"] : undefined, + banner: isESM ? { js: ` import { fileURLToPath } from 'url'; @@ -141,7 +143,7 @@ const require = createRequire(import.meta.url); return sourceCode; }; -export const getCodemodExecutable = async (source: string, write?: boolean) => { +export const getCodemodExecutable = async (source: string, esm?: boolean) => { const outputFilePath = join(resolve(source), BUILT_SOURCE_PATH); try { return await readFile(outputFilePath, { encoding: "utf8" }); @@ -158,15 +160,5 @@ export const getCodemodExecutable = async (source: string, write?: boolean) => { return readFile(entryPoint, { encoding: "utf8" }); } - const bundledCode = await bundleJS({ - entry: entryPoint, - output: outputFilePath, - }); - - if (write) { - await mkdir(dirname(outputFilePath), { recursive: true }); - await writeFile(outputFilePath, bundledCode); - } - - return bundledCode; + return bundleJS({ entry: entryPoint, output: outputFilePath, esm }); }; diff --git a/packages/utilities/src/package-boilerplate.ts b/packages/utilities/src/package-boilerplate.ts index fe5dfa9c6..83f67b8be 100644 --- a/packages/utilities/src/package-boilerplate.ts +++ b/packages/utilities/src/package-boilerplate.ts @@ -668,7 +668,7 @@ export function getCodemodProjectFiles(input: ProjectDownloadInput) { throw new Error(`Unknown engine: ${input.engine}`); } - if (!input.cases || input.cases.length === 0) { + if (input.cases === undefined) { input.cases = [ { before: `const toReplace = "hello";`,