From 6de579fb9d8e1a6c901178165aaa3a8351f8fb42 Mon Sep 17 00:00:00 2001 From: Meir Blachman Date: Thu, 15 Apr 2021 21:52:53 +0300 Subject: [PATCH] add registry option (#20) * add registry option * cleanup --- bin/download-tgz | 5 +++++ lib/commands.js | 28 ++++++++++++---------------- lib/config.js | 13 +++++++++++++ lib/constants.js | 5 ----- lib/crawler.js | 7 ++++--- lib/downloader.js | 27 +++++++++++++++++---------- lib/npm-search.js | 6 +++--- lib/url-resolver.js | 14 ++++++++++++++ 8 files changed, 68 insertions(+), 37 deletions(-) create mode 100644 lib/config.js delete mode 100644 lib/constants.js create mode 100644 lib/url-resolver.js diff --git a/bin/download-tgz b/bin/download-tgz index d5aeec5..93aab1d 100644 --- a/bin/download-tgz +++ b/bin/download-tgz @@ -21,12 +21,14 @@ program .command('package-lock ') .description('download tarballs based on a package-lock.json') .option('--directory [directory]') + .option('--registry [registry]') .action((uri, command) => commands.packageLockCommand(uri, command)); program .command('package [version]') .description('download tarballs based on a package and a version') .option('--directory [directory]') + .option('--registry [registry]') .option('--devDependencies') .option('--peerDependencies') .action((name, version, command) => commands.packageCommand(name, version, command)); @@ -35,6 +37,7 @@ program .command('package-json ') .description('download tarballs based on a package.json') .option('--directory [directory]') + .option('--registry [registry]') .option('--devDependencies') .option('--peerDependencies') .action((uri, command) => commands.packageJsonCommand(uri, command)); @@ -43,6 +46,7 @@ program .command('search ') .description('download tarballs based on a npm registry search results') .option('--directory [directory]') + .option('--registry [registry]') .option('--devDependencies') .option('--peerDependencies') .action((keyword, command) => commands.searchCommand(keyword, command)); @@ -51,6 +55,7 @@ program .command('generate [version]') .description('generates the download links for a giving package and a version') .option('--outputFile [outputFile]') + .option('--registry [registry]') .option('--devDependencies') .option('--peerDependencies') .action((name, version, command) => commands.generateCommand(name, version, command)); diff --git a/lib/commands.js b/lib/commands.js index bd430fc..e3d4790 100644 --- a/lib/commands.js +++ b/lib/commands.js @@ -7,66 +7,62 @@ const generator = require('./generator'); /** * @param {string} uri - * @param {{ directory: string }} options + * @param {{ directory: string, registry?: string }} options */ async function packageLockCommand(uri, options = {}) { const packageLock = await retrieveFile(uri); - downloader.downloadFromPackageLock(packageLock, options.directory); + downloader.downloadFromPackageLock(packageLock, options); } /** * @param {string} name * @param {string} version - * @param {{ directory: string, devDependencies: boolean, peerDependencies: boolean }} options + * @param {{ directory: string, devDependencies: boolean, peerDependencies: boolean, registry?: string }} options */ async function packageCommand(name, version, options = {}) { const tarballsSet = await crawler.getDependencies({ name, version, - devDependencies: options.devDependencies, - peerDependencies: options.peerDependencies, + ...options }); - downloader.downloadFromIterable(tarballsSet, options.directory); + downloader.downloadFromIterable(tarballsSet, options); } /** * @param {string} uri - * @param {{ directory: string, devDependencies: boolean, peerDependencies: boolean }} options + * @param {{ directory: string, devDependencies: boolean, peerDependencies: boolean, registry?: string }} options */ async function packageJsonCommand(uri, options = {}) { const packageJson = await retrieveFile(uri); const tarballsSet = await crawler.getPackageJsonDependencies({ packageJson, - devDependencies: options.devDependencies, - peerDependencies: options.peerDependencies, + ...options }); - downloader.downloadFromIterable(tarballsSet, options.directory); + downloader.downloadFromIterable(tarballsSet, options); } /** * @param {string} keyword -* @param {{ directory: string, devDependencies: boolean, peerDependencies: boolean }} options +* @param {{ directory: string, devDependencies: boolean, peerDependencies: boolean, registry?: string }} options */ async function searchCommand(keyword, options = {}) { const packageJson = await generatePackageJson({ keyword }); const tarballsSet = await crawler.getPackageJsonDependencies({ packageJson, - devDependencies: options.devDependencies, - peerDependencies: options.peerDependencies, + ...options }); downloader.downloadFromIterable(tarballsSet, options.directory); } /** * @param {string} keyword -* @param {{ outputFile: string, devDependencies: boolean, peerDependencies: boolean }} options +* @param {{ outputFile: string, devDependencies: boolean, peerDependencies: boolean, registry?: string }} options */ async function generateCommand(name, version, options) { const tarballsSet = await crawler.getDependencies({ name, version, - devDependencies: options.devDependencies, - peerDependencies: options.peerDependencies + ...options }); generator.saveToFile(Array.from(tarballsSet), options.outputFile); } diff --git a/lib/config.js b/lib/config.js new file mode 100644 index 0000000..2d8e3c3 --- /dev/null +++ b/lib/config.js @@ -0,0 +1,13 @@ +const defaultNpmRegistry = 'http://registry.npmjs.org'; + +/** + * + * @param {{ registry?: string }} options + */ +function getNpmRegistry(options = {}) { + return options.registry || defaultNpmRegistry; +} + +module.exports = { + getNpmRegistry +}; diff --git a/lib/constants.js b/lib/constants.js deleted file mode 100644 index 5edd504..0000000 --- a/lib/constants.js +++ /dev/null @@ -1,5 +0,0 @@ -const npmRegistry = 'http://registry.npmjs.org'; - -module.exports = { - npmRegistry, -}; diff --git a/lib/crawler.js b/lib/crawler.js index d4fab36..2fc052a 100644 --- a/lib/crawler.js +++ b/lib/crawler.js @@ -5,7 +5,7 @@ const logger = require('./logger'); require('colors'); -const { npmRegistry } = require('./constants'); +const { getNpmRegistry } = require('./config'); let cacheHits = 1; let registryHits = 1; @@ -20,6 +20,7 @@ const tarballs = new Set(); * @property {boolean} devDependencies * @property {boolean} peerDependencies * @property {string} outputPrefix + * @property {string?} registry * * @param { DependenciesOptions } options * @returns { Promise> } @@ -71,8 +72,8 @@ async function getPackageJsonDependencies(options) { return tarballs; } -async function _retrievePackageVersion({ name, version, outputPrefix = '' }) { - const uri = `${npmRegistry}/${name.replace('/', '%2F')}`; +async function _retrievePackageVersion({ name, version, outputPrefix = '', registry = '' }) { + const uri = `${getNpmRegistry({ registry })}/${name.replace('/', '%2F')}`; if (packagesCache.has(name)) { logger(['cache'.yellow, cacheHits], `retrieving ${outputPrefix}${name.cyan} ${(version || '').cyan}`); diff --git a/lib/downloader.js b/lib/downloader.js index 2ffe95e..0c027aa 100644 --- a/lib/downloader.js +++ b/lib/downloader.js @@ -5,33 +5,40 @@ const fs = require('fs'); const tar = require('tar'); const logger = require('./logger'); const downloadFileAsync = require('./download-file'); +const urlResolver = require('./url-resolver'); require('colors'); -function downloadFromPackageLock(packageLock, directory) { +/** + * @param {{directory: string, registry?: string}} options + */ +function downloadFromPackageLock(packageLock, options) { const tarballs = []; - _enumerateDependencies(tarballs, packageLock.dependencies); + _enumerateDependencies(tarballs, packageLock.dependencies, options); - return _downloadTarballs(tarballs, directory); + return _downloadTarballs(tarballs, options.directory); } /** * @param { Iterable | ArrayLike } tarballsIterable - * @param { string } directory + * @param {{directory: string, registry?: string}} options */ -function downloadFromIterable(tarballsIterable, directory) { +function downloadFromIterable(tarballsIterable, options) { const tarballs = Array.from(tarballsIterable) - .map(url => ({ url, directory: _convertUrlToDirectory(url) })); - return _downloadTarballs(tarballs, directory); + .map(url => ({ url: urlResolver.resolve(url, options), directory: _convertUrlToDirectory(url) })); + return _downloadTarballs(tarballs, options.directory); } -function _enumerateDependencies(tarballs, dependencies) { +/** + * @param {{directory: string, registry?: string}} options + */ +function _enumerateDependencies(tarballs, dependencies, options) { for (const [dependencyName, dependency] of Object.entries(dependencies)) { if (dependency.resolved) { - tarballs.push({ url: dependency.resolved, directory: dependencyName }); + tarballs.push({ url: urlResolver.resolve(dependency.resolved, options), directory: dependencyName }); } if (dependency.dependencies) { - _enumerateDependencies(tarballs, dependency.dependencies); + _enumerateDependencies(tarballs, dependency.dependencies, options); } } } diff --git a/lib/npm-search.js b/lib/npm-search.js index 7f6405c..0d96516 100644 --- a/lib/npm-search.js +++ b/lib/npm-search.js @@ -1,12 +1,12 @@ const request = require('request-promise'); -const { npmRegistry } = require('./constants'); +const { getNpmRegistry } = require('./config'); /** * - * @param {{ keyword: string }} options + * @param {{ keyword: string, registry?: string }} options */ async function generatePackageJson(options) { - const uri = `${npmRegistry}/-/v1/search?text=keywords:${options.keyword}&size=1000`; + const uri = `${getNpmRegistry(options)}/-/v1/search?text=keywords:${options.keyword}&size=1000`; const response = await request({ uri, json: true }); const packageJson = response.objects.reduce((prev, curr) => { diff --git a/lib/url-resolver.js b/lib/url-resolver.js new file mode 100644 index 0000000..498e000 --- /dev/null +++ b/lib/url-resolver.js @@ -0,0 +1,14 @@ +/** + * @param {string} url + * @param {{ registry?: string }} options + */ +function resolve(url, options = {}) { + if (!options.registry) return url; + + const urlObject = new URL(url); + return urlObject.href.replace(urlObject.origin, options.registry); +} + +module.exports = { + resolve +}; \ No newline at end of file