Skip to content

Commit

Permalink
Abstract packager into own interface
Browse files Browse the repository at this point in the history 
Extended pack external unit tests

Added unit tests for npm packager

Adapted unit tests

Exclude test.js files from nyc statistics

Added unit test for packager factory

Extract optional lock file adaption

Fixed ESLint

ESLint fixes

Moved npm to a separate packager class.
  • Loading branch information
Frank Schmid committed Mar 1, 2018
1 parent d38a385 commit fee4947
Show file tree
Hide file tree
Showing 8 changed files with 663 additions and 409 deletions.
1 change: 1 addition & 0 deletions .npmignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,4 @@ node_modules
coverage
examples
tests
*.test.js
268 changes: 111 additions & 157 deletions lib/packExternalModules.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@
const BbPromise = require('bluebird');
const _ = require('lodash');
const path = require('path');
const childProcess = require('child_process');
const fse = require('fs-extra');
const isBuiltinModule = require('is-builtin-module');

const Packagers = require('./packagers');

function rebaseFileReferences(pathToPackageRoot, moduleVersion) {
if (/^file:[^/]{2}/.test(moduleVersion)) {
const filePath = _.replace(moduleVersion, /^file:/, '');
Expand All @@ -16,18 +17,6 @@ function rebaseFileReferences(pathToPackageRoot, moduleVersion) {
return moduleVersion;
}

function rebasePackageLock(pathToPackageRoot, module) {
if (module.version) {
module.version = rebaseFileReferences(pathToPackageRoot, module.version);
}

if (module.dependencies) {
_.forIn(module.dependencies, moduleDependency => {
rebasePackageLock(pathToPackageRoot, moduleDependency);
});
}
}

/**
* Add the given modules to a package json's dependencies.
*/
Expand Down Expand Up @@ -200,156 +189,121 @@ module.exports = {
const packagePath = includes.packagePath || './package.json';
const packageJsonPath = path.join(process.cwd(), packagePath);

this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);
// Get first level dependency graph
const command = 'npm ls -prod -json -depth=1'; // Only prod dependencies

const ignoredNpmErrors = [
{ npmError: 'extraneous', log: false },
{ npmError: 'missing', log: false },
{ npmError: 'peer dep missing', log: true },
];

return BbPromise.fromCallback(cb => {
childProcess.exec(command, {
cwd: path.dirname(packageJsonPath),
maxBuffer: this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024,
encoding: 'utf8'
}, (err, stdout, stderr) => {
if (err) {
// Only exit with an error if we have critical npm errors for 2nd level inside
const errors = _.split(stderr, '\n');
const failed = _.reduce(errors, (failed, error) => {
if (failed) {
return true;
}
return !_.isEmpty(error) && !_.some(ignoredNpmErrors, ignoredError => _.startsWith(error, `npm ERR! ${ignoredError.npmError}`));
}, false);

if (failed) {
return cb(err);
}
}
return cb(null, stdout);
});
})
.then(depJson => BbPromise.try(() => JSON.parse(depJson)))
.then(dependencyGraph => {
const problems = _.get(dependencyGraph, 'problems', []);
if (this.options.verbose && !_.isEmpty(problems)) {
this.serverless.cli.log(`Ignoring ${_.size(problems)} NPM errors:`);
_.forEach(problems, problem => {
this.serverless.cli.log(`=> ${problem}`);
});
}

// (1) Generate dependency composition
const compositeModules = _.uniq(_.flatMap(stats.stats, compileStats => {
const externalModules = _.concat(
getExternalModules.call(this, compileStats),
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
);
return getProdModules.call(this, externalModules, packagePath, dependencyGraph);
}));
removeExcludedModules.call(this, compositeModules, packageForceExcludes, true);

if (_.isEmpty(compositeModules)) {
// The compiled code does not reference any external modules at all
this.serverless.cli.log('No external modules needed');
return BbPromise.resolve();
}

// (1.a) Install all needed modules
const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');
const compositePackageJson = path.join(compositeModulePath, 'package.json');

// (1.a.1) Create a package.json
const compositePackage = {
name: this.serverless.service.service,
version: '1.0.0',
description: `Packaged externals for ${this.serverless.service.service}`,
private: true
};
const relPath = path.relative(compositeModulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(compositeModules, compositePackage, relPath);
this.serverless.utils.writeFileSync(compositePackageJson, JSON.stringify(compositePackage, null, 2));

// (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades
const packageLockPath = path.join(path.dirname(packageJsonPath), 'package-lock.json');
return BbPromise.fromCallback(cb => fse.pathExists(packageLockPath, cb))
.then(exists => {
if (exists) {
this.serverless.cli.log('Package lock found - Using locked versions');
try {
const packageLockJson = this.serverless.utils.readFileSync(packageLockPath);
/**
* We should not be modifying 'package-lock.json'
* because this file should be treat as internal to npm.
*
* Rebase package-lock is a temporary workaround and must be
* removed as soon as https://github.com/npm/npm/issues/19183 gets fixed.
*/
rebasePackageLock(relPath, packageLockJson);

this.serverless.utils.writeFileSync(path.join(compositeModulePath, 'package-lock.json'), JSON.stringify(packageLockJson, null, 2));
} catch(err) {
this.serverless.cli.log(`Warning: Could not read lock file: ${err.message}`);
}
// Determine and create packager
return BbPromise.try(() => Packagers.get.call(this, 'npm'))
.then(packager => {
// Get first level dependency graph
this.options.verbose && this.serverless.cli.log(`Fetch dependency graph from ${packageJsonPath}`);
const maxExecBufferSize = this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024;

return packager.getProdDependencies(path.dirname(packageJsonPath), 1, maxExecBufferSize)
.then(dependencyGraph => {
const problems = _.get(dependencyGraph, 'problems', []);
if (this.options.verbose && !_.isEmpty(problems)) {
this.serverless.cli.log(`Ignoring ${_.size(problems)} NPM errors:`);
_.forEach(problems, problem => {
this.serverless.cli.log(`=> ${problem}`);
});
}
return BbPromise.resolve();
})
.then(() => {
const start = _.now();
this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));
return BbPromise.fromCallback(cb => {
childProcess.exec('npm install', {
cwd: compositeModulePath,
maxBuffer: this.serverless.service.custom.packExternalModulesMaxBuffer || 200 * 1024,
encoding: 'utf8'
}, cb);
})
.then(() => this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`))
.return(stats.stats);
})
.mapSeries(compileStats => {
const modulePath = compileStats.compilation.compiler.outputPath;

// Create package.json
const modulePackageJson = path.join(modulePath, 'package.json');
const modulePackage = {
dependencies: {}
};
const prodModules = getProdModules.call(this,
_.concat(

// (1) Generate dependency composition
const compositeModules = _.uniq(_.flatMap(stats.stats, compileStats => {
const externalModules = _.concat(
getExternalModules.call(this, compileStats),
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
), packagePath, dependencyGraph);
removeExcludedModules.call(this, prodModules, packageForceExcludes);
const relPath = path.relative(modulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(prodModules, modulePackage, relPath);
this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));

// GOOGLE: Copy modules only if not google-cloud-functions
// GCF Auto installs the package json
if (_.get(this.serverless, 'service.provider.name') === 'google') {
);
return getProdModules.call(this, externalModules, packagePath, dependencyGraph);
}));
removeExcludedModules.call(this, compositeModules, packageForceExcludes, true);

if (_.isEmpty(compositeModules)) {
// The compiled code does not reference any external modules at all
this.serverless.cli.log('No external modules needed');
return BbPromise.resolve();
}

const startCopy = _.now();
return BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, 'node_modules'), path.join(modulePath, 'node_modules'), callback))
.tap(() => this.options.verbose && this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`))

// (1.a) Install all needed modules
const compositeModulePath = path.join(this.webpackOutputPath, 'dependencies');
const compositePackageJson = path.join(compositeModulePath, 'package.json');

// (1.a.1) Create a package.json
const compositePackage = {
name: this.serverless.service.service,
version: '1.0.0',
description: `Packaged externals for ${this.serverless.service.service}`,
private: true
};
const relPath = path.relative(compositeModulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(compositeModules, compositePackage, relPath);
this.serverless.utils.writeFileSync(compositePackageJson, JSON.stringify(compositePackage, null, 2));

// (1.a.2) Copy package-lock.json if it exists, to prevent unwanted upgrades
const packageLockPath = path.join(path.dirname(packageJsonPath), packager.lockfileName);
return BbPromise.fromCallback(cb => fse.pathExists(packageLockPath, cb))
.then(exists => {
if (exists) {
this.serverless.cli.log('Package lock found - Using locked versions');
try {
const packageLockJson = this.serverless.utils.readFileSync(packageLockPath);
/**
* We should not be modifying 'package-lock.json'
* because this file should be treat as internal to npm.
*
* Rebase package-lock is a temporary workaround and must be
* removed as soon as https://github.com/npm/npm/issues/19183 gets fixed.
*/
packager.rebaseLockfile(relPath, packageLockJson);

this.serverless.utils.writeFileSync(path.join(compositeModulePath, packager.lockfileName), JSON.stringify(packageLockJson, null, 2));
} catch(err) {
this.serverless.cli.log(`Warning: Could not read lock file: ${err.message}`);
}
}
return BbPromise.resolve();
})
.then(() => {
// Prune extraneous packages - removes not needed ones
const startPrune = _.now();
return BbPromise.fromCallback(callback => {
childProcess.exec('npm prune', {
cwd: modulePath
}, callback);
})
.tap(() => this.options.verbose && this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`));
});
})
.return();
const start = _.now();
this.serverless.cli.log('Packing external modules: ' + compositeModules.join(', '));
return packager.install(compositeModulePath, maxExecBufferSize)
.then(() => this.options.verbose && this.serverless.cli.log(`Package took [${_.now() - start} ms]`))
.return(stats.stats);
})
.mapSeries(compileStats => {
const modulePath = compileStats.compilation.compiler.outputPath;

// Create package.json
const modulePackageJson = path.join(modulePath, 'package.json');
const modulePackage = {
dependencies: {}
};
const prodModules = getProdModules.call(this,
_.concat(
getExternalModules.call(this, compileStats),
_.map(packageForceIncludes, whitelistedPackage => ({ external: whitelistedPackage }))
), packagePath, dependencyGraph);
removeExcludedModules.call(this, prodModules, packageForceExcludes);
const relPath = path.relative(modulePath, path.dirname(packageJsonPath));
addModulesToPackageJson(prodModules, modulePackage, relPath);
this.serverless.utils.writeFileSync(modulePackageJson, JSON.stringify(modulePackage, null, 2));

// GOOGLE: Copy modules only if not google-cloud-functions
// GCF Auto installs the package json
if (_.get(this.serverless, 'service.provider.name') === 'google') {
return BbPromise.resolve();
}

const startCopy = _.now();
return BbPromise.fromCallback(callback => fse.copy(path.join(compositeModulePath, 'node_modules'), path.join(modulePath, 'node_modules'), callback))
.tap(() => this.options.verbose && this.serverless.cli.log(`Copy modules: ${modulePath} [${_.now() - startCopy} ms]`))
.then(() => {
// Prune extraneous packages - removes not needed ones
const startPrune = _.now();
return packager.prune(modulePath, maxExecBufferSize)
.tap(() => this.options.verbose && this.serverless.cli.log(`Prune: ${modulePath} [${_.now() - startPrune} ms]`));
});
})
.return();
});
});
}
};
38 changes: 38 additions & 0 deletions lib/packagers/index.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
'use strict';
/**
* Factory for supported packagers.
*
* All packagers must implement the following interface:
*
* interface Packager {
*
* static get lockfileName(): string;
* static getProdDependencies(cwd: string, depth: number = 1, maxExecBufferSize = undefined): BbPromise<Object>;
* static rebaseLockfile(pathToPackageRoot: string, lockfile: Object): void;
* static install(cwd: string, maxExecBufferSize = undefined): BbPromise<void>;
* static prune(cwd: string): BbPromise<void>;
*
* }
*/

const _ = require('lodash');
const npm = require('./npm');

const registeredPackagers = {
npm: npm
};

/**
* Factory method.
* @this ServerlessWebpack - Active plugin instance
* @param {string} packagerId - Well known packager id.
* @returns {BbPromise<Packager>} - Promised packager to allow packagers be created asynchronously.
*/
module.exports.get = function(packagerId) {
if (!_.has(registeredPackagers, packagerId)) {
const message = `Could not find packager '${packagerId}'`;
this.serverless.cli.log(`ERROR: ${message}`);
throw new this.serverless.classes.Error(message);
}
return registeredPackagers[packagerId];
};
Loading

0 comments on commit fee4947

Please sign in to comment.