diff --git a/.gitignore b/.gitignore index d0901c12..0e64e1e1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .nyc_output/ +artifacts configs coverage node_modules diff --git a/README.md b/README.md index 6cd747f5..b81a926b 100644 --- a/README.md +++ b/README.md @@ -482,7 +482,7 @@ git cherry-pick --continue git push # create pull request -e pr --backport 1234 +e pr open --backport 1234 ``` ## Common Usage diff --git a/package.json b/package.json index 94291a99..bb6ee415 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "command-exists": "^1.2.8", "commander": "^9.0.0", "debug": "^4.3.1", + "extract-zip": "^2.0.1", "inquirer": "^8.2.4", "node-gyp": "^10.0.1", "open": "^6.4.0", diff --git a/src/download.js b/src/download.js index 41ac0bb5..5fffc4d0 100644 --- a/src/download.js +++ b/src/download.js @@ -1,29 +1,9 @@ const fs = require('fs'); const stream = require('stream'); const { pipeline } = require('stream/promises'); -const ProgressBar = require('progress'); const { fatal } = require('./utils/logging'); - -const MB_BYTES = 1024 * 1024; - -const progressStream = function (total, tokens) { - var pt = new stream.PassThrough(); - - pt.on('pipe', function (stream) { - const bar = new ProgressBar(tokens, { total: Math.round(total) }); - - pt.on('data', function (chunk) { - const elapsed = new Date() - bar.start; - const rate = bar.curr / (elapsed / 1000); - bar.tick(chunk.length, { - mbRate: (rate / MB_BYTES).toFixed(2), - }); - }); - }); - - return pt; -}; +const { progressStream } = require('./utils/download'); const write = fs.createWriteStream(process.argv[3]); diff --git a/src/e b/src/e index 92ec439b..65f02ec5 100755 --- a/src/e +++ b/src/e @@ -169,7 +169,7 @@ program .command('backport [pr]', 'Assists with manual backport processes') .command('show ', 'Show info about the current build config') .command('test [specRunnerArgs...]', `Run Electron's spec runner`) - .command('pr [options]', 'Open a GitHub URL where you can PR your changes') + .command('pr [subcommand]', 'Work with PRs to electron/electron') .command('patches ', 'Refresh the patches in $root/src/electron/patches/$target') .command('open ', 'Open a GitHub URL for the given commit hash / pull # / issue #') .command('auto-update', 'Check for build-tools updates or enable/disable automatic updates') diff --git a/src/e-pr.js b/src/e-pr.js index 0ceb756b..1435e210 100644 --- a/src/e-pr.js +++ b/src/e-pr.js @@ -1,15 +1,26 @@ #!/usr/bin/env node const childProcess = require('child_process'); +const fs = require('fs'); +const os = require('os'); const path = require('path'); +const { Readable } = require('stream'); +const { pipeline } = require('stream/promises'); + +const extractZip = require('extract-zip'); const querystring = require('querystring'); const semver = require('semver'); - const open = require('open'); const program = require('commander'); +const { Octokit } = require('@octokit/rest'); +const inquirer = require('inquirer'); +const { progressStream } = require('./utils/download'); +const { getGitHubAuthToken } = require('./utils/github-auth'); const { current } = require('./evm-config'); -const { color, fatal } = require('./utils/logging'); +const { color, fatal, logError } = require('./utils/logging'); + +const d = require('debug')('build-tools:pr'); // Adapted from https://github.com/electron/clerk function findNoteInPRBody(body) { @@ -134,27 +145,23 @@ function pullRequestSource(source) { } program + .command('open', null, { isDefault: true }) .description('Open a GitHub URL where you can PR your changes') - .option( - '-s, --source ', - 'Where the changes are coming from', - guessPRSource(current()), - ) - .option( - '-t, --target ', - 'Where the changes are going to', - guessPRTarget(current()), - ) + .option('-s, --source [source_branch]', 'Where the changes are coming from') + .option('-t, --target [target_branch]', 'Where the changes are going to') .option('-b, --backport ', 'Pull request being backported') .action(async (options) => { - if (!options.source) { + const source = options.source || guessPRSource(current()); + const target = options.target || guessPRTarget(current()); + + if (!source) { fatal(`'source' is required to create a PR`); - } else if (!options.target) { + } else if (!target) { fatal(`'target' is required to create a PR`); } const repoBaseUrl = 'https://github.com/electron/electron'; - const comparePath = `${options.target}...${pullRequestSource(options.source)}`; + const comparePath = `${target}...${pullRequestSource(source)}`; const queryParams = { expand: 1 }; if (!options.backport) { @@ -188,5 +195,233 @@ program } return open(`${repoBaseUrl}/compare/${comparePath}?${querystring.stringify(queryParams)}`); - }) - .parse(process.argv); + }); + +program + .command('download-dist ') + .description('Download a pull request dist') + .option( + '--platform [platform]', + 'Platform to download dist for. Defaults to current platform.', + process.platform, + ) + .option( + '--arch [arch]', + 'Architecture to download dist for. Defaults to current arch.', + process.arch, + ) + .option( + '-o, --output ', + 'Specify the output directory for downloaded artifacts. ' + + 'Defaults to ~/.electron_build_tools/artifacts/pr_{number}_{commithash}_{platform}_{arch}', + ) + .option( + '-s, --skip-confirmation', + 'Skip the confirmation prompt before downloading the dist.', + !!process.env.CI, + ) + .action(async (pullRequestNumber, options) => { + if (!pullRequestNumber) { + fatal(`Pull request number is required to download a PR`); + } + + d('checking auth...'); + const auth = await getGitHubAuthToken(['repo']); + const octokit = new Octokit({ auth }); + + d('fetching pr info...'); + let pullRequest; + try { + const { data } = await octokit.pulls.get({ + owner: 'electron', + repo: 'electron', + pull_number: pullRequestNumber, + }); + pullRequest = data; + } catch (error) { + console.error(`Failed to get pull request: ${error}`); + return; + } + + if (!options.skipConfirmation) { + const isElectronRepo = pullRequest.head.repo.full_name !== 'electron/electron'; + const { proceed } = await inquirer.prompt([ + { + type: 'confirm', + default: false, + name: 'proceed', + message: `You are about to download artifacts from: + +“${pullRequest.title} (#${pullRequest.number})” by ${pullRequest.user.login} +${pullRequest.head.repo.html_url}${isElectronRepo ? ' (fork)' : ''} +${pullRequest.state !== 'open' ? '\n❗❗❗ The pull request is closed, only proceed if you trust the source ❗❗❗\n' : ''} +Proceed?`, + }, + ]); + + if (!proceed) return; + } + + d('fetching workflow runs...'); + let workflowRuns; + try { + const { data } = await octokit.actions.listWorkflowRunsForRepo({ + owner: 'electron', + repo: 'electron', + branch: pullRequest.head.ref, + name: 'Build', + event: 'pull_request', + status: 'completed', + per_page: 10, + sort: 'created', + direction: 'desc', + }); + workflowRuns = data.workflow_runs; + } catch (error) { + console.error(`Failed to list workflow runs: ${error}`); + return; + } + + const latestBuildWorkflowRun = workflowRuns.find((run) => run.name === 'Build'); + if (!latestBuildWorkflowRun) { + fatal(`No 'Build' workflow runs found for pull request #${pullRequestNumber}`); + } + const shortCommitHash = latestBuildWorkflowRun.head_sha.substring(0, 7); + + d('fetching artifacts...'); + let artifacts; + try { + const { data } = await octokit.actions.listWorkflowRunArtifacts({ + owner: 'electron', + repo: 'electron', + run_id: latestBuildWorkflowRun.id, + }); + artifacts = data.artifacts; + } catch (error) { + console.error(`Failed to list artifacts: ${error}`); + return; + } + + const artifactPlatform = options.platform === 'win32' ? 'win' : options.platform; + const artifactName = `generated_artifacts_${artifactPlatform}_${options.arch}`; + const artifact = artifacts.find((artifact) => artifact.name === artifactName); + if (!artifact) { + console.error(`Failed to find artifact: ${artifactName}`); + return; + } + + let outputDir; + + if (options.output) { + outputDir = path.resolve(options.output); + + if (!(await fs.promises.stat(outputDir).catch(() => false))) { + fatal(`The output directory '${options.output}' does not exist`); + } + } else { + const artifactsDir = path.resolve(__dirname, '..', 'artifacts'); + const defaultDir = path.resolve( + artifactsDir, + `pr_${pullRequest.number}_${shortCommitHash}_${options.platform}_${options.arch}`, + ); + + // Clean up the directory if it exists + try { + await fs.promises.rm(defaultDir, { recursive: true, force: true }); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + + // Create the directory + await fs.promises.mkdir(defaultDir, { recursive: true }); + + outputDir = defaultDir; + } + + console.log( + `Downloading artifact '${artifactName}' from pull request #${pullRequestNumber}...`, + ); + + // Download the artifact to a temporary directory + const tempDir = path.join(os.tmpdir(), 'electron-tmp'); + await fs.promises.rm(tempDir, { recursive: true, force: true }); + await fs.promises.mkdir(tempDir); + + const { url } = await octokit.actions.downloadArtifact.endpoint({ + owner: 'electron', + repo: 'electron', + artifact_id: artifact.id, + archive_format: 'zip', + }); + + const response = await fetch(url, { + headers: { + Authorization: `Bearer ${auth}`, + }, + }); + + if (!response.ok) { + fatal(`Could not find artifact: ${url} got ${response.status}`); + } + + const total = parseInt(response.headers.get('content-length'), 10); + const artifactDownloadStream = Readable.fromWeb(response.body); + + try { + const artifactZipPath = path.join(tempDir, `${artifactName}.zip`); + const artifactFileStream = fs.createWriteStream(artifactZipPath); + await pipeline( + artifactDownloadStream, + // Show download progress + ...(process.env.CI ? [] : [progressStream(total, '[:bar] :mbRateMB/s :percent :etas')]), + artifactFileStream, + ); + + // Extract artifact zip + d('unzipping artifact to %s', tempDir); + await extractZip(artifactZipPath, { dir: tempDir }); + + // Check if dist.zip exists within the extracted artifact + const distZipPath = path.join(tempDir, 'dist.zip'); + if (!(await fs.promises.stat(distZipPath).catch(() => false))) { + throw new Error(`dist.zip not found in build artifact.`); + } + + // Extract dist.zip + // NOTE: 'extract-zip' is used as it correctly extracts symlinks. + d('unzipping dist.zip to %s', outputDir); + await extractZip(distZipPath, { dir: outputDir }); + + const platformExecutables = { + win32: 'electron.exe', + darwin: 'Electron.app/', + linux: 'electron', + }; + + const executableName = platformExecutables[options.platform]; + if (!executableName) { + throw new Error(`Unable to find executable for platform '${options.platform}'`); + } + + const executablePath = path.join(outputDir, executableName); + if (!(await fs.promises.stat(executablePath).catch(() => false))) { + throw new Error(`${executableName} not found within dist.zip.`); + } + + console.log(`${color.success} Downloaded to ${outputDir}`); + } catch (error) { + logError(error); + process.exitCode = 1; // wait for cleanup + } finally { + // Cleanup temporary files + try { + await fs.promises.rm(tempDir, { recursive: true }); + } catch { + // ignore + } + } + }); + +program.parse(process.argv); diff --git a/src/utils/download.js b/src/utils/download.js new file mode 100644 index 00000000..c8475b1b --- /dev/null +++ b/src/utils/download.js @@ -0,0 +1,26 @@ +const stream = require('stream'); +const ProgressBar = require('progress'); + +const MB_BYTES = 1024 * 1024; + +const progressStream = function (total, tokens) { + var pt = new stream.PassThrough(); + + pt.on('pipe', function (_stream) { + const bar = new ProgressBar(tokens, { total: Math.round(total) }); + + pt.on('data', function (chunk) { + const elapsed = new Date() - bar.start; + const rate = bar.curr / (elapsed / 1000); + bar.tick(chunk.length, { + mbRate: (rate / MB_BYTES).toFixed(2), + }); + }); + }); + + return pt; +}; + +module.exports = { + progressStream, +}; diff --git a/src/utils/logging.js b/src/utils/logging.js index d6d9caa5..1c8f3da6 100644 --- a/src/utils/logging.js +++ b/src/utils/logging.js @@ -20,16 +20,21 @@ const color = { warn: chalk.bgYellowBright.black('WARN'), }; -function fatal(e, code = 1) { +function logError(e) { if (typeof e === 'string') { console.error(`${color.err} ${e}`); } else { console.error(`${color.err} ${e.stack ? e.stack : e.message}`); } +} + +function fatal(e, code = 1) { + logError(e); process.exit(code); } module.exports = { color, fatal, + logError, }; diff --git a/yarn.lock b/yarn.lock index 9cb0f17c..92e85f48 100644 --- a/yarn.lock +++ b/yarn.lock @@ -775,11 +775,25 @@ resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== +"@types/node@*": + version "22.10.2" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.10.2.tgz#a485426e6d1fdafc7b0d4c7b24e2c78182ddabb9" + integrity sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ== + dependencies: + undici-types "~6.20.0" + "@types/unist@*", "@types/unist@^2.0.0": version "2.0.6" resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.6.tgz#250a7b16c3b91f672a24552ec64678eeb1d3a08d" integrity sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ== +"@types/yauzl@^2.9.1": + version "2.10.3" + resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.10.3.tgz#e9b2808b4f109504a03cda958259876f61017999" + integrity sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q== + dependencies: + "@types/node" "*" + "@vitest/expect@2.1.3": version "2.1.3" resolved "https://registry.yarnpkg.com/@vitest/expect/-/expect-2.1.3.tgz#4b9a6fff22be4c4cd5d57e687cfda611b514b0ad" @@ -1151,6 +1165,11 @@ btoa-lite@^1.0.0: resolved "https://registry.yarnpkg.com/btoa-lite/-/btoa-lite-1.0.0.tgz#337766da15801210fdd956c22e9c6891ab9d0337" integrity sha1-M3dm2hWAEhD92VbCLpxokaudAzc= +buffer-crc32@~0.2.3: + version "0.2.13" + resolved "https://registry.yarnpkg.com/buffer-crc32/-/buffer-crc32-0.2.13.tgz#0d333e3f00eac50aa1454abd30ef8c2a5d9a7242" + integrity sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ== + buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" @@ -2005,6 +2024,17 @@ external-editor@^3.0.3: iconv-lite "^0.4.24" tmp "^0.0.33" +extract-zip@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.1.tgz#663dca56fe46df890d5f131ef4a06d22bb8ba13a" + integrity sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg== + dependencies: + debug "^4.1.1" + get-stream "^5.1.0" + yauzl "^2.10.0" + optionalDependencies: + "@types/yauzl" "^2.9.1" + fast-deep-equal@^3.1.1, fast-deep-equal@^3.1.3: version "3.1.3" resolved "https://registry.yarnpkg.com/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz#3a7d56b559d6cbc3eb512325244e619a65c6c525" @@ -2027,6 +2057,13 @@ fastq@^1.6.0: dependencies: reusify "^1.0.4" +fd-slicer@~1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.1.0.tgz#25c7c89cb1f9077f8891bbe61d8f390eae256f1e" + integrity sha512-cE1qsB/VwyQozZ+q1dGxR8LBYNZeofhEdUNGSMbQD3Gw2lAzX9Zb3uIU6Ebc/Fmyjo9AWWfnn0AUCHqtevs/8g== + dependencies: + pend "~1.2.0" + figures@^3.0.0: version "3.2.0" resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" @@ -2257,6 +2294,13 @@ get-stdin@~9.0.0: resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-9.0.0.tgz#3983ff82e03d56f1b2ea0d3e60325f39d703a575" integrity sha512-dVKBjfWisLAicarI2Sf+JuBE/DghV4UzNAVe9yhEJuzeREd3JhOTE9cUaJTeSa77fsbQUK3pcOpJfM59+VKZaA== +get-stream@^5.1.0: + version "5.2.0" + resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-5.2.0.tgz#4966a1795ee5ace65e706c4b7beb71257d6e22d3" + integrity sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA== + dependencies: + pump "^3.0.0" + get-stream@^8.0.1: version "8.0.1" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-8.0.1.tgz#def9dfd71742cd7754a7761ed43749a27d02eca2" @@ -4082,6 +4126,11 @@ pathval@^2.0.0: resolved "https://registry.yarnpkg.com/pathval/-/pathval-2.0.0.tgz#7e2550b422601d4f6b8e26f1301bc8f15a741a25" integrity sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA== +pend@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/pend/-/pend-1.2.0.tgz#7a57eb550a6783f9115331fcf4663d5c8e007a50" + integrity sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg== + picocolors@^1.0.0, picocolors@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1" @@ -5036,6 +5085,11 @@ unbox-primitive@^1.0.2: has-symbols "^1.0.3" which-boxed-primitive "^1.0.2" +undici-types@~6.20.0: + version "6.20.0" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.20.0.tgz#8171bf22c1f588d1554d55bf204bc624af388433" + integrity sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg== + unique-filename@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230" @@ -5439,6 +5493,14 @@ yargs@^15.0.2: y18n "^4.0.0" yargs-parser "^18.1.2" +yauzl@^2.10.0: + version "2.10.0" + resolved "https://registry.yarnpkg.com/yauzl/-/yauzl-2.10.0.tgz#c7eb17c93e112cb1086fa6d8e51fb0667b79a5f9" + integrity sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g== + dependencies: + buffer-crc32 "~0.2.3" + fd-slicer "~1.1.0" + yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b"