New: Use woker-farm to handle webpack multi compile

This improves memory footprint of compilations using package
individually. By only extracting the properties we need from webpack
stats/result we can clean up a lot more memory every compile. This also
uses seperate processes to compile each function which can clean up all
the memory related to compilations and build multiple functions at same
time.

lib/compile.js

@@ -2,41 +2,58 @@
 
 const _ = require('lodash');
 const BbPromise = require('bluebird');
-const webpack = require('webpack');
 const tty = require('tty');
 
+const { compiler } = require('./compiler');
+const { multiCompiler } = require('./multiCompiler/compiler');
+
 module.exports = {
   compile() {
     this.serverless.cli.log('Bundling with Webpack...');
+    const consoleStats = this.webpackConfig.stats ||
+      _.get(this, 'webpackConfig[0].stats') || {
+      colors: tty.isatty(process.stdout.fd),
+      hash: false,
+      version: false,
+      chunks: false,
+      children: false
+    };
+
+    const configOptions = {
+      servicePath: this.serverless.config.servicePath,
+      out: this.options.out
+    };
+
+    const compileOptions = {
+      webpackConfigFilePath: this.webpackConfigFilePath,
+      webpackConfig: this.webpackConfig,
+
+      entryFunctions: this.entryFunctions,
+
+      configOptions,
+      consoleStats
+    };
 
-    const compiler = webpack(this.webpackConfig);
+    if (this.multiCompile) {
+      this.options.verbose && this.serverless.cli.log('Using multi-thread function compiler');
+    }
 
-    return BbPromise.fromCallback(cb => compiler.run(cb)).then(stats => {
-      if (!this.multiCompile) {
-        stats = { stats: [stats] };
-      }
+    const webpackCompiler = this.multiCompile ? multiCompiler(compileOptions) : compiler(compileOptions);
 
+    return webpackCompiler.then(stats => {
       const compileOutputPaths = [];
-      const consoleStats = this.webpackConfig.stats ||
-        _.get(this, 'webpackConfig[0].stats') || {
-        colors: tty.isatty(process.stdout.fd),
-        hash: false,
-        version: false,
-        chunks: false,
-        children: false
-      };
 
       _.forEach(stats.stats, compileStats => {
-        const statsOutput = compileStats.toString(consoleStats);
+        const statsOutput = compileStats.cliOutput;
         if (statsOutput) {
           this.serverless.cli.consoleLog(statsOutput);
         }
 
-        if (compileStats.compilation.errors.length) {
+        if (compileStats.errors.length) {
           throw new Error('Webpack compilation error, see above');
         }
 
-        compileOutputPaths.push(compileStats.compilation.compiler.outputPath);
+        compileOutputPaths.push(compileStats.outputPath);
       });
 
       this.compileOutputPaths = compileOutputPaths;

lib/compiler.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const BbPromise = require('bluebird');
+const webpack = require('webpack');
+
+const { processWebpackStats } = require('./processWebpackStats');
+const { setOptionsOnConfig } = require('./processConfig');
+
+module.exports = {
+  compiler(options) {
+    const webpackConfig = options.webpackConfig;
+    const configOptions = options.configOptions;
+    const consoleStats = options.consoleStats;
+
+    const config = setOptionsOnConfig(webpackConfig, configOptions);
+
+    const compiler = webpack(config);
+
+    return BbPromise.fromCallback(cb => compiler.run(cb)).then(stats => {
+      const result = processWebpackStats(stats, consoleStats);
+
+      return {
+        stats: [result]
+      };
+    });
+  }
+};

lib/multiCompiler/compiler.js

@@ -0,0 +1,45 @@
+'use strict';
+
+const _ = require('lodash');
+const BbPromise = require('bluebird');
+const workerFarm = require('worker-farm');
+
+module.exports = {
+  multiCompiler(options) {
+    const workerOptions = {
+      maxCallsPerWorker: 1
+    };
+    const workerThreadSourcePath = require.resolve('./workerThread');
+    const methods = ['runWebpack'];
+
+    const workers = workerFarm(workerOptions, workerThreadSourcePath, methods);
+
+    const configOptions = options.configOptions;
+
+    const threads = _.map(options.entryFunctions, entryFunc => {
+      const webpackConfigFilePath = options.webpackConfigFilePath;
+      const workerOptions = {
+        webpackConfigFilePath,
+        configOptions: {
+          ...configOptions,
+          entryFunc
+        },
+        consoleStats: options.consoleStats
+      };
+
+      if (!webpackConfigFilePath) {
+        workerOptions.webpackConfig = options.webpackConfig;
+      }
+
+      return BbPromise.fromCallback(cb => {
+        workers.runWebpack(workerOptions, cb);
+      });
+    });
+
+    return Promise.all(threads).then(stats => {
+      return {
+        stats
+      };
+    });
+  }
+};

lib/multiCompiler/workerThread.js

@@ -0,0 +1,43 @@
+'use strict';
+
+const webpack = require('webpack');
+const { setOptionsOnConfig } = require('../processConfig');
+const { processWebpackStats } = require('../processWebpackStats');
+
+module.exports = {
+  runWebpack(options, callback) {
+    const webpackConfigFilePath = options.webpackConfigFilePath;
+    const configOptions = options.configOptions;
+    const consoleStats = options.consoleStats;
+
+    let webpackConfig = null;
+    try {
+      if (webpackConfigFilePath) {
+        webpackConfig = require(webpackConfigFilePath);
+      } else if (options.webpackConfig) {
+        webpackConfig = options.webpackConfig;
+      } else {
+        throw new Error('Missing config');
+      }
+    } catch (error) {
+      callback(new Error('Failed to load config'));
+      return;
+    }
+
+    const entryConfig = setOptionsOnConfig(webpackConfig, configOptions);
+
+    const compiler = webpack(entryConfig);
+
+    compiler.run((error, stats) => {
+      if (error) {
+        callback(new Error('Failed to compile'));
+
+        return;
+      }
+
+      const result = processWebpackStats(stats, consoleStats);
+
+      callback(null, result);
+    });
+  }
+};

lib/packExternalModules.js

@@ -4,7 +4,6 @@ const BbPromise = require('bluebird');
 const _ = require('lodash');
 const path = require('path');
 const fse = require('fs-extra');
-const isBuiltinModule = require('is-builtin-module');
 
 const Packagers = require('./packagers');
 
@@ -143,58 +142,6 @@ function getProdModules(externalModules, packagePath, dependencyGraph, forceExcl
   return prodModules;
 }
 
-function getExternalModuleName(module) {
-  const path = /^external "(.*)"$/.exec(module.identifier())[1];
-  const pathComponents = path.split('/');
-  const main = pathComponents[0];
-
-  // this is a package within a namespace
-  if (main.charAt(0) == '@') {
-    return `${main}/${pathComponents[1]}`;
-  }
-
-  return main;
-}
-
-function isExternalModule(module) {
-  return _.startsWith(module.identifier(), 'external ') && !isBuiltinModule(getExternalModuleName(module));
-}
-
-/**
- * Find the original module that required the transient dependency. Returns
- * undefined if the module is a first level dependency.
- * @param {Object} issuer - Module issuer
- */
-function findExternalOrigin(issuer) {
-  if (!_.isNil(issuer) && _.startsWith(issuer.rawRequest, './')) {
-    return findExternalOrigin(issuer.issuer);
-  }
-  return issuer;
-}
-
-function getExternalModules(stats) {
-  if (!stats.compilation.chunks) {
-    return [];
-  }
-  const externals = new Set();
-  for (const chunk of stats.compilation.chunks) {
-    if (!chunk.modulesIterable) {
-      continue;
-    }
-
-    // Explore each module within the chunk (built inputs):
-    for (const module of chunk.modulesIterable) {
-      if (isExternalModule(module)) {
-        externals.add({
-          origin: _.get(findExternalOrigin(module.issuer), 'rawRequest'),
-          external: getExternalModuleName(module)
-        });
-      }
-    }
-  }
-  return Array.from(externals);
-}
-
 module.exports = {
   /**
    * We need a performant algorithm to install the packages for each single
@@ -259,7 +206,7 @@ module.exports = {
         const compositeModules = _.uniq(
           _.flatMap(stats.stats, compileStats => {
             const externalModules = _.concat(
-              getExternalModules.call(this, compileStats),
+              compileStats.externalModules,
               _.map(packageForceIncludes, whitelistedPackage => ({
                 external: whitelistedPackage
               }))
@@ -328,7 +275,7 @@ module.exports = {
               .return(stats.stats);
           })
           .mapSeries(compileStats => {
-            const modulePath = compileStats.compilation.compiler.outputPath;
+            const modulePath = compileStats.outputPath;
 
             // Create package.json
             const modulePackageJson = path.join(modulePath, 'package.json');
@@ -346,7 +293,7 @@ module.exports = {
             const prodModules = getProdModules.call(
               this,
               _.concat(
-                getExternalModules.call(this, compileStats),
+                compileStats.externalModules,
                 _.map(packageForceIncludes, whitelistedPackage => ({
                   external: whitelistedPackage
                 }))

lib/packageModules.js

@@ -76,7 +76,7 @@ module.exports = {
     return BbPromise.mapSeries(stats.stats, (compileStats, index) => {
       const entryFunction = _.get(this.entryFunctions, index, {});
       const filename = `${entryFunction.funcName || this.serverless.service.getServiceObject().name}.zip`;
-      const modulePath = compileStats.compilation.compiler.outputPath;
+      const modulePath = compileStats.outputPath;
 
       const startZip = _.now();
       return zip

lib/processConfig.js

@@ -0,0 +1,47 @@
+'use strict';
+
+const _ = require('lodash');
+const path = require('path');
+
+module.exports = {
+  setOptionsOnConfig(webpackConfig, options) {
+    // Default context
+    if (!webpackConfig.context) {
+      webpackConfig.context = options.servicePath;
+    }
+
+    // Default target
+    if (!webpackConfig.target) {
+      webpackConfig.target = 'node';
+    }
+
+    // Default output
+    if (!webpackConfig.output || _.isEmpty(webpackConfig.output)) {
+      const outputPath = path.join(options.servicePath, '.webpack');
+      webpackConfig.output = {
+        libraryTarget: 'commonjs',
+        path: outputPath,
+        filename: '[name].js'
+      };
+    }
+
+    // Custom output path
+    if (options.out) {
+      webpackConfig.output.path = path.join(options.servicePath, options.out);
+    }
+
+    // In case of individual packaging we have to create a separate config for each function
+    if (options.entryFunc) {
+      const entryFunc = options.entryFunc;
+      webpackConfig.entry = {
+        [entryFunc.entry.key]: entryFunc.entry.value
+      };
+      const compileName = entryFunc.funcName || _.camelCase(entryFunc.entry.key);
+      webpackConfig.output.path = path.join(webpackConfig.output.path, compileName);
+    } else {
+      webpackConfig.output.path = path.join(webpackConfig.output.path, 'service');
+    }
+
+    return webpackConfig;
+  }
+};