added local plugin to only update modified files

Author: blakeandrewwood

.dockerignore

@@ -3,4 +3,5 @@
 **/build
 **/build-pdf
 **/build-swagger
-**/build-ngindox
+**/build-ngindox
+.revision

.gitignore

@@ -9,3 +9,4 @@ npm-debug.log
 .DS_Store
 ./**/*.tmp
 .tmp
+.revision

index.js

@@ -10,10 +10,12 @@ const serve            = require('metalsmith-serve');
 const webpack          = require('metalsmith-webpack2');
 const anchor           = require('markdown-it-anchor');
 const attrs            = require('markdown-it-attrs');
-const incremental      = require('metalsmith-incremental').default;
+//const incremental      = require('metalsmith-incremental').default;
 const timer            = require('metalsmith-timer');
 
 // Local Plugins
+const reduce                  = require('./plugins/metalsmith-revision').reduce;
+const restore                 = require('./plugins/metalsmith-revision').restore;
 const hierarchy               = require('./plugins/metalsmith-hierarchy');
 const hierarchyRss            = require('./plugins/metalsmith-hierarchy-rss');
 const headings                = require('./plugins/metalsmith-headings');
@@ -115,13 +117,8 @@ CB.use(hierarchyRss({
 CB.use(timer('CB: Hierarchy RSS'))
 
 // Filter unmodified files
-CB.use(incremental({
-  plugin: 'filter',
-  rename: {
-    from: /.md$/,
-    to: '.html',
-  }
-}))
+CB.use(reduce())
+CB.use(timer('CB: Reduce'))
 
 //
 // Slow Plugins
@@ -184,26 +181,27 @@ CB.use(timer('CB: Layouts'))
 //
 
 // Restore unmodified files
-CB.use(incremental({
-  plugin: 'cache'
-}))
+CB.use(restore())
+CB.use(timer('CB: Reduce'))
 
 // Enable watching
 if(process.env.NODE_ENV === 'development') {
+  /*
   CB.use(incremental({
     plugin: 'watch',
     paths: {
-      './layouts/*': '*',
+      'layouts/*': '*',
     }
-    /*
-    paths: {
-      './pages/*': '*',
-      './layouts/*': '*',
-      './scss/*': '*',
-      './js/*': '*',
-    },
-    */
   }))
+  */
+  CB.use(
+    watch({
+      paths: {
+        'pages/**/*': '**/*.md',
+        'layouts/**/*': '**/*.pug',
+      },
+    })
+  )
 }
 
 // Search Indexing
@@ -227,7 +225,9 @@ if(process.env.NODE_ENV == "pdf") {
 
 // In case you have restored all files with cache plugin
 // call filter plugin as last middleware
+/*
 CB.use(incremental({ plugin: 'filter' }))
+*/
 
 // Serve
 if(process.env.NODE_ENV == "development") {
@@ -248,7 +248,8 @@ if(process.env.NODE_ENV === 'development') {
   AB.use(
     watch({
       paths: {
-        "js/**/*": "**/*.js",
+        'js/**/*': '**/*.js',
+        'scss/**/*': '**/*.scss',
       },
     })
   )

package-lock.json

@@ -46,7 +46,6 @@
     "metalsmith-assets": "^0.1.0",
     "metalsmith-branch": "0.0.5",
     "metalsmith-headings": "^0.2.0",
-    "metalsmith-incremental": "^0.1.1",
     "metalsmith-layouts": "^1.8.1",
     "metalsmith-markdownit": "^0.4.0",
     "metalsmith-permalinks": "^0.5.0",

package.json

@@ -0,0 +1,109 @@
+const resolve = require('path').resolve;
+const relative = require('path').relative;
+const fs = require('fs');
+const crypto = require('crypto');
+//const recurse = require('recursive-readdir');
+
+function checksum (str, algorithm, encoding) {
+  return crypto
+    .createHash(algorithm || 'md5')
+    .update(str, 'utf8')
+    .digest(encoding || 'hex')
+}
+
+const default_options = { layout: false, layoutDir: './layouts' }
+
+function plugin(opts) {
+  const options = Object.assign({}, default_options, opts)
+  return (files, metalsmith, done) => {
+
+    const configPath = resolve(metalsmith._directory, '.revision')
+    const hash_table = { layouts: {}, src: {} }
+    const layoutDirectory = resolve(metalsmith._directory, options.layoutDir)
+    const revision = fs.existsSync(configPath)
+      ? JSON.parse(fs.readFileSync(configPath, 'utf-8'))
+      : null;
+
+    if(revision == null) {
+      fs.writeFileSync(configPath, JSON.stringify(hash_table), 'utf-8')
+      done();
+      return;
+    }
+
+    if(!metalsmith.revision) {
+      metalsmith.revision = {};
+    }
+
+    metalsmith.revision.cachedFiles = Object.assign({}, files);
+
+    Object.keys(files).forEach(file => {
+      const hash = checksum(files[file].contents)
+      hash_table.src[file] = hash
+      if(revision.src[file] === hash) {
+        delete files[file];
+      }
+    });
+
+    console.log('Updating %s files', Object.keys(files).length);
+
+    fs.writeFileSync(configPath, JSON.stringify(hash_table), 'utf-8')
+    done()
+
+    /*
+    if(options.layout){
+      recurse(
+        layoutDirectory
+      , (err, layouts) => {
+          if(err) throw(err)
+          layouts.map(l => {
+            const path = relative(layoutDirectory, l)
+            const content = fs.readFileSync(l, 'utf-8')
+            const hash = checksum(content)
+            hash_table.layouts[path] = hash
+        })
+
+        Object.keys(files).forEach(file => {
+          const hash = checksum(files[file].contents)
+          hash_table.src[file] = hash
+          if(
+            (revision !== null && revision.src[file] === hash) &&
+            ( (!options.layout) ||
+              (options.layout && revision.layouts[files[file].layout] === hash_table.layouts[files[file].layout]) ) )
+            delete files[file]
+        })
+
+        fs.writeFileSync(configPath, JSON.stringify(hash_table), 'utf-8')
+        done()
+      })
+    }
+    else{
+      Object.keys(files).forEach(file => {
+        const hash = checksum(files[file].contents)
+        hash_table.src[file] = hash
+        if(
+          (revision !== null && revision.src[file] === hash) &&
+          ( (!options.layout) ||
+            (options.layout && revision.layouts[files[file].layout] === hash_table.layouts[files[file].layout]) ) )
+          delete files[file]
+      })
+
+      fs.writeFileSync(configPath, JSON.stringify(hash_table), 'utf-8')
+      done()
+    }
+    */
+  }
+}
+
+function restore(opts) {
+  const options = Object.assign({}, default_options, opts)
+  return (files, metalsmith, done) => {
+    files = metalsmith.revision.cachedFiles;
+    console.log('Restored %s files.', Object.keys(files).length);
+    done();
+  }
+}
+
+module.exports = {
+  reduce: plugin,
+  restore: restore
+};