Skip to content

Commit 1b8a3be

Browse files
HyperBrainHyperBrain
authored
Merge pull request #330 from serverless-heaven/329-packager-interface
Abstract packager into universal interface.
2 parents d38a385 + fee4947 commit 1b8a3be

File tree

8 files changed

+663
-409
lines changed

8 files changed

+663
-409
lines changed

‎.npmignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ node_modules
22
coverage
33
examples
44
tests
5+
*.test.js

‎lib/packExternalModules.js

Lines changed: 111 additions & 157 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@
33
const BbPromise = require('bluebird');
44
const _ = require('lodash');
55
const path = require('path');
6-
const childProcess = require('child_process');
76
const fse = require('fs-extra');
87
const isBuiltinModule = require('is-builtin-module');
98

9+
const Packagers = require('./packagers');
10+
1011
function rebaseFileReferences(pathToPackageRoot, moduleVersion) {
1112
if (/^file:[^/]{2}/.test(moduleVersion)) {
1213
const filePath = _.replace(moduleVersion, /^file:/, '');
@@ -16,18 +17,6 @@ function rebaseFileReferences(pathToPackageRoot, moduleVersion) {
1617
return moduleVersion;
1718
}
1819

19-
function rebasePackageLock(pathToPackageRoot, module) {
20-
if (module.version) {
21-
module.version = rebaseFileReferences(pathToPackageRoot, module.version);
22-
}
23-
24-
if (module.dependencies) {
25-
_.forIn(module.dependencies, moduleDependency => {
26-
rebasePackageLock(pathToPackageRoot, moduleDependency);
27-
});
28-
}
29-
}
30-
3120
/**
3221
* Add the given modules to a package json's dependencies.
3322
*/
@@ -200,156 +189,121 @@ module.exports = {
200189
const packagePath = includes.packagePath || './package.json';
201190
const packageJsonPath = path.join(process.cwd(), packagePath);
202191

203-
this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);
204-
// Get first level dependency graph
205-
const command = 'npm ls -prod -json -depth=1'; // Only prod dependencies
206-
207-
const ignoredNpmErrors = [
208-
{ npmError: 'extraneous', log: false },
209-
{ npmError: 'missing', log: false },
210-
{ npmError: 'peer dep missing', log: true },
211-
];
212-
213-
return BbPromise.fromCallback(cb => {
214-
childProcess.exec(command, {
215-
cwd: path.dirname(packageJsonPath),
216-
maxBuffer: this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024,
217-
encoding: 'utf8'
218-
}, (err, stdout, stderr) => {
219-
if (err) {
220-
// Only exit with an error if we have critical npm errors for 2nd level inside
221-
const errors = _.split(stderr, '\n');
222-
const failed = _.reduce(errors, (failed, error) => {
223-
if (failed) {
224-
return true;
225-
}
226-
return !_.isEmpty(error) && !_.some(ignoredNpmErrors, ignoredError => _.startsWith(error, `npm ERR! ${ignoredError.npmError}`));
227-
}, false);
228-
229-
if (failed) {
230-
return cb(err);
231-
}
232-
}
233-
return cb(null, stdout);
234-
});
235-
})
236-
.then(depJson => BbPromise.try(() => JSON.parse(depJson)))
237-
.then(dependencyGraph => {
238-
const problems = _.get(dependencyGraph, 'problems', []);
239-
if (this.options.verbose && !_.isEmpty(problems)) {
240-
this.serverless.cli.log(`Ignoring ${_.size(problems)} NPM errors:`);
241-
_.forEach(problems, problem => {
242-
this.serverless.cli.log(`=> ${problem}`);
243-
});
244-
}
245-
246-
// (1) Generate dependency composition
247-
const compositeModules = _.uniq(_.flatMap(stats.stats, compileStats => {
248-
const externalModules = _.concat(
249-
getExternalModules.call(this, compileStats),
250-
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
251-
);
252-
return getProdModules.call(this, externalModules, packagePath, dependencyGraph);
253-
}));
254-
removeExcludedModules.call(this, compositeModules, packageForceExcludes, true);
255-
256-
if (_.isEmpty(compositeModules)) {
257-
// The compiled code does not reference any external modules at all
258-
this.serverless.cli.log('No external modules needed');
259-
return BbPromise.resolve();
260-
}
261-
262-
// (1.a) Install all needed modules
263-
const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');
264-
const compositePackageJson = path.join(compositeModulePath, 'package.json');
265-
266-
// (1.a.1) Create a package.json
267-
const compositePackage = {
268-
name: this.serverless.service.service,
269-
version: '1.0.0',
270-
description: `Packaged externals for ${this.serverless.service.service}`,
271-
private: true
272-
};
273-
const relPath = path.relative(compositeModulePath, path.dirname(packageJsonPath));
274-
addModulesToPackageJson(compositeModules, compositePackage, relPath);
275-
this.serverless.utils.writeFileSync(compositePackageJson, JSON.stringify(compositePackage, null, 2));
276-
277-
// (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades
278-
const packageLockPath = path.join(path.dirname(packageJsonPath), 'package-lock.json');
279-
return BbPromise.fromCallback(cb => fse.pathExists(packageLockPath, cb))
280-
.then(exists => {
281-
if (exists) {
282-
this.serverless.cli.log('Package lock found - Using locked versions');
283-
try {
284-
const packageLockJson = this.serverless.utils.readFileSync(packageLockPath);
285-
/**
286-
* We should not be modifying 'package-lock.json'
287-
* because this file should be treat as internal to npm.
288-
*
289-
* Rebase package-lock is a temporary workaround and must be
290-
* removed as soon as https://github.com/npm/npm/issues/19183 gets fixed.
291-
*/
292-
rebasePackageLock(relPath, packageLockJson);
293-
294-
this.serverless.utils.writeFileSync(path.join(compositeModulePath, 'package-lock.json'), JSON.stringify(packageLockJson, null, 2));
295-
} catch(err) {
296-
this.serverless.cli.log(`Warning: Could not read lock file: ${err.message}`);
297-
}
192+
// Determine and create packager
193+
return BbPromise.try(() => Packagers.get.call(this, 'npm'))
194+
.then(packager => {
195+
// Get first level dependency graph
196+
this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);
197+
const maxExecBufferSize = this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024;
198+
199+
return packager.getProdDependencies(path.dirname(packageJsonPath), 1, maxExecBufferSize)
200+
.then(dependencyGraph => {
201+
const problems = _.get(dependencyGraph, 'problems', []);
202+
if (this.options.verbose && !_.isEmpty(problems)) {
203+
this.serverless.cli.log(`Ignoring ${_.size(problems)} NPM errors:`);
204+
_.forEach(problems, problem => {
205+
this.serverless.cli.log(`=> ${problem}`);
206+
});
298207
}
299-
return BbPromise.resolve();
300-
})
301-
.then(() => {
302-
const start = _.now();
303-
this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));
304-
return BbPromise.fromCallback(cb => {
305-
childProcess.exec('npm install', {
306-
cwd: compositeModulePath,
307-
maxBuffer: this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024,
308-
encoding: 'utf8'
309-
}, cb);
310-
})
311-
.then(() => this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`))
312-
.return(stats.stats);
313-
})
314-
.mapSeries(compileStats => {
315-
const modulePath = compileStats.compilation.compiler.outputPath;
316-
317-
// Create package.json
318-
const modulePackageJson = path.join(modulePath, 'package.json');
319-
const modulePackage = {
320-
dependencies: {}
321-
};
322-
const prodModules = getProdModules.call(this,
323-
_.concat(
208+
209+
// (1) Generate dependency composition
210+
const compositeModules = _.uniq(_.flatMap(stats.stats, compileStats => {
211+
const externalModules = _.concat(
324212
getExternalModules.call(this, compileStats),
325213
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
326-
), packagePath, dependencyGraph);
327-
removeExcludedModules.call(this, prodModules, packageForceExcludes);
328-
const relPath = path.relative(modulePath, path.dirname(packageJsonPath));
329-
addModulesToPackageJson(prodModules, modulePackage, relPath);
330-
this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));
331-
332-
// GOOGLE: Copy modules only if not google-cloud-functions
333-
// GCF Auto installs the package json
334-
if (_.get(this.serverless, 'service.provider.name') === 'google') {
214+
);
215+
return getProdModules.call(this, externalModules, packagePath, dependencyGraph);
216+
}));
217+
removeExcludedModules.call(this, compositeModules, packageForceExcludes, true);
218+
219+
if (_.isEmpty(compositeModules)) {
220+
// The compiled code does not reference any external modules at all
221+
this.serverless.cli.log('No external modules needed');
335222
return BbPromise.resolve();
336223
}
337-
338-
const startCopy = _.now();
339-
return BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, 'node_modules'), path.join(modulePath, 'node_modules'), callback))
340-
.tap(() => this.options.verbose && this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`))
224+
225+
// (1.a) Install all needed modules
226+
const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');
227+
const compositePackageJson = path.join(compositeModulePath, 'package.json');
228+
229+
// (1.a.1) Create a package.json
230+
const compositePackage = {
231+
name: this.serverless.service.service,
232+
version: '1.0.0',
233+
description: `Packaged externals for ${this.serverless.service.service}`,
234+
private: true
235+
};
236+
const relPath = path.relative(compositeModulePath, path.dirname(packageJsonPath));
237+
addModulesToPackageJson(compositeModules, compositePackage, relPath);
238+
this.serverless.utils.writeFileSync(compositePackageJson, JSON.stringify(compositePackage, null, 2));
239+
240+
// (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades
241+
const packageLockPath = path.join(path.dirname(packageJsonPath), packager.lockfileName);
242+
return BbPromise.fromCallback(cb => fse.pathExists(packageLockPath, cb))
243+
.then(exists => {
244+
if (exists) {
245+
this.serverless.cli.log('Package lock found - Using locked versions');
246+
try {
247+
const packageLockJson = this.serverless.utils.readFileSync(packageLockPath);
248+
/**
249+
* We should not be modifying 'package-lock.json'
250+
* because this file should be treat as internal to npm.
251+
*
252+
* Rebase package-lock is a temporary workaround and must be
253+
* removed as soon as https://github.com/npm/npm/issues/19183 gets fixed.
254+
*/
255+
packager.rebaseLockfile(relPath, packageLockJson);
256+
257+
this.serverless.utils.writeFileSync(path.join(compositeModulePath, packager.lockfileName), JSON.stringify(packageLockJson, null, 2));
258+
} catch(err) {
259+
this.serverless.cli.log(`Warning: Could not read lock file: ${err.message}`);
260+
}
261+
}
262+
return BbPromise.resolve();
263+
})
341264
.then(() => {
342-
// Prune extraneous packages - removes not needed ones
343-
const startPrune = _.now();
344-
return BbPromise.fromCallback(callback => {
345-
childProcess.exec('npm prune', {
346-
cwd: modulePath
347-
}, callback);
348-
})
349-
.tap(() => this.options.verbose && this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`));
350-
});
351-
})
352-
.return();
265+
const start = _.now();
266+
this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));
267+
return packager.install(compositeModulePath, maxExecBufferSize)
268+
.then(() => this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`))
269+
.return(stats.stats);
270+
})
271+
.mapSeries(compileStats => {
272+
const modulePath = compileStats.compilation.compiler.outputPath;
273+
274+
// Create package.json
275+
const modulePackageJson = path.join(modulePath, 'package.json');
276+
const modulePackage = {
277+
dependencies: {}
278+
};
279+
const prodModules = getProdModules.call(this,
280+
_.concat(
281+
getExternalModules.call(this, compileStats),
282+
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
283+
), packagePath, dependencyGraph);
284+
removeExcludedModules.call(this, prodModules, packageForceExcludes);
285+
const relPath = path.relative(modulePath, path.dirname(packageJsonPath));
286+
addModulesToPackageJson(prodModules, modulePackage, relPath);
287+
this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));
288+
289+
// GOOGLE: Copy modules only if not google-cloud-functions
290+
// GCF Auto installs the package json
291+
if (_.get(this.serverless, 'service.provider.name') === 'google') {
292+
return BbPromise.resolve();
293+
}
294+
295+
const startCopy = _.now();
296+
return BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, 'node_modules'), path.join(modulePath, 'node_modules'), callback))
297+
.tap(() => this.options.verbose && this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`))
298+
.then(() => {
299+
// Prune extraneous packages - removes not needed ones
300+
const startPrune = _.now();
301+
return packager.prune(modulePath, maxExecBufferSize)
302+
.tap(() => this.options.verbose && this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`));
303+
});
304+
})
305+
.return();
306+
});
353307
});
354308
}
355309
};

‎lib/packagers/index.js

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
'use strict';
2+
/**
3+
* Factory for supported packagers.
4+
*
5+
* All packagers must implement the following interface:
6+
*
7+
* interface Packager {
8+
*
9+
* static get lockfileName(): string;
10+
* static getProdDependencies(cwd: string, depth: number = 1, maxExecBufferSize = undefined): BbPromise<Object>;
11+
* static rebaseLockfile(pathToPackageRoot: string, lockfile: Object): void;
12+
* static install(cwd: string, maxExecBufferSize = undefined): BbPromise<void>;
13+
* static prune(cwd: string): BbPromise<void>;
14+
*
15+
* }
16+
*/
17+
18+
const _ = require('lodash');
19+
const npm = require('./npm');
20+
21+
const registeredPackagers = {
22+
npm: npm
23+
};
24+
25+
/**
26+
* Factory method.
27+
* @this ServerlessWebpack - Active plugin instance
28+
* @param {string} packagerId - Well known packager id.
29+
* @returns {BbPromise<Packager>} - Promised packager to allow packagers be created asynchronously.
30+
*/
31+
module.exports.get = function(packagerId) {
32+
if (!_.has(registeredPackagers, packagerId)) {
33+
const message = `Could not find packager '${packagerId}'`;
34+
this.serverless.cli.log(`ERROR: ${message}`);
35+
throw new this.serverless.classes.Error(message);
36+
}
37+
return registeredPackagers[packagerId];
38+
};

0 commit comments

Comments
 (0)