diff --git a/lib/index.js b/lib/index.js index 21ea232..3076f63 100644 --- a/lib/index.js +++ b/lib/index.js @@ -8,7 +8,8 @@ var AWS = require('aws-sdk'), mime = require('mime'), pascalCase = require('pascal-case'), File = require('vinyl'), - gutil = require('gulp-util'); + gutil = require('gulp-util'), + _ = require('lodash'); var PLUGIN_NAME = 'gulp-awspublish'; @@ -174,6 +175,7 @@ module.exports.reporter = function(param) { function Publisher(config) { this.config = config; this.client = new AWS.S3(config); + this._cacheOptions = { headers: false }; // load cache try { @@ -181,6 +183,19 @@ function Publisher(config) { } catch (err) { this._cache = {}; } + + // Migrate old style caches that used only ETags: + // { "path1": "etag1", "path2": "etag2" } + for (var key in this._cache) { + var value = this._cache[key]; + + if (typeof value === 'string') { + this._cache = { + etag: value, + headers: {} + }; + } + } } /** @@ -199,14 +214,120 @@ Publisher.prototype.getCacheFilename = function() { return '.awspublish-' + bucket; }; +Publisher.prototype._isFileCached = function(file, etag) { + + var fileCache = this._cache[file.s3.path]; + + if (!fileCache) return false; + + if (fileCache.etag !== etag) return false; + + // Header comparison is turned off. + if (!this._cacheOptions.headers) return true; + + var checkedHeaders = {}; + + for (var name in file.s3.headers) { + + checkedHeaders[name] = true; + + if (name in fileCache.headers) { + if (file.s3.headers[name] !== fileCache.headers[name]) { + // Modified header. + return false; + } + } + else { + // New header. + return false; + } + } + + for (var name in fileCache.headers) { + if (name in checkedHeaders) continue; + + // Deleted header. + return false; + } + + return true; +}; + +Publisher.prototype._shouldSkipFile = function(file, response, etag) { + + if (response.ETag !== etag) return false; + + // Normalize S3 headers. + var responseHeaders = {}; + if (response.CacheControl) { responseHeaders['Cache-Control'] = response.CacheControl; } + if (response.ContentType) { responseHeaders['Content-Type'] = response.ContentType; } + if (response.ContentDisposition) { responseHeaders['Content-Disposition'] = response.ContentDisposition; } + if (response.ContentLanguage) { responseHeaders['Content-Language'] = response.ContentLanguage; } + if (response.Expires) { responseHeaders['Expires'] = response.Expires; } + if (response.ContentEncoding) { responseHeaders['Content-Encoding'] = response.ContentEncoding; } + + if (response.Metadata) { + for (var name in response.Metadata) { + responseHeaders['x-amz-meta-' + name] = response.Metadata[name]; + } + } + + var checkedHeaders = {}; + + for (var name in file.s3.headers) { + + checkedHeaders[name] = true; + + // Have to ignore the ACL header since it's a canned ACL name that gets + // transformed into an ACL and isn't returned in the response. The + // S3 client's 'getObjectAcl()' could be used, but the response ACL + // would have to be mapped back to the canned ACL name. + if (name === 'x-amz-acl') { + continue; + } + + if (name in responseHeaders) { + if (file.s3.headers[name] !== responseHeaders[name]) { + // Modified header. + return false; + } + } + else { + // New header. + return false; + } + } + + for (var name in responseHeaders.headers) { + + if (name in checkedHeaders) continue; + + // Deleted header. + return false; + } + + return true; +}; + /** * create a through stream that save file in cache + * @options {Object} options option hash * + * available options are: + * - headers {Boolean} include file headers when checking if the file is cached + * @return {Stream} * @api public */ -Publisher.prototype.cache = function() { +Publisher.prototype.cache = function(options) { + + if (!options) { + options = this._cacheOptions; + } + + this._cacheOptions = options; + var _this = this, counter = 0; @@ -226,7 +347,10 @@ Publisher.prototype.cache = function() { // update others } else if (file.s3.etag) { - _this._cache[file.s3.path] = file.s3.etag; + _this._cache[file.s3.path] = { + etag: file.s3.etag, + headers: _.clone(file.s3.headers) + }; } // save cache every 10 files @@ -281,68 +405,67 @@ Publisher.prototype.publish = function (headers, options) { } // check if file.contents is a `Buffer` - if (file.isBuffer()) { + if (!file.isBuffer()) return; - initFile(file); + initFile(file); - // calculate etag - etag = '"' + md5Hash(file.contents) + '"'; + // calculate etag + etag = '"' + md5Hash(file.contents) + '"'; - // delete - stop here - if (file.s3.state === 'delete') return cb(null, file); + // delete - stop here + if (file.s3.state === 'delete') return cb(null, file); - // check if file is identical as the one in cache - if (!options.force && _this._cache[file.s3.path] === etag) { - file.s3.state = 'cache'; - return cb(null, file); - } + // add content-type header + if (!file.s3.headers['Content-Type']) file.s3.headers['Content-Type'] = getContentType(file); - // add content-type header - if (!file.s3.headers['Content-Type']) file.s3.headers['Content-Type'] = getContentType(file); + // add content-length header + if (!file.s3.headers['Content-Length']) file.s3.headers['Content-Length'] = file.contents.length; - // add content-length header - if (!file.s3.headers['Content-Length']) file.s3.headers['Content-Length'] = file.contents.length; + // add extra headers + for (header in headers) file.s3.headers[header] = headers[header]; - // add extra headers - for (header in headers) file.s3.headers[header] = headers[header]; + // check if file is identical as the one in cache + if (!options.force && _this._isFileCached(file, etag)) { + file.s3.state = 'cache'; + return cb(null, file); + } - if (options.simulate) return cb(null, file); + if (options.simulate) return cb(null, file); - // get s3 headers - _this.client.headObject({ Key: file.s3.path }, function(err, res) { - //ignore 403 and 404 errors since we're checking if a file exists on s3 - if (err && [403, 404].indexOf(err.statusCode) < 0) return cb(err); + // get s3 headers + _this.client.headObject({ Key: file.s3.path }, function(err, res) { + //ignore 403 and 404 errors since we're checking if a file exists on s3 + if (err && [403, 404].indexOf(err.statusCode) < 0) return cb(err); - res = res || {}; + res = res || {}; - // skip: no updates allowed - var noUpdate = options.createOnly && res.ETag; + // skip: no updates allowed + var noUpdate = options.createOnly && res.ETag; - // skip: file are identical - var noChange = !options.force && res.ETag === etag; + // skip: file is identical + var noChange = !options.force && _this._shouldSkipFile(file, res, etag); - if (noUpdate || noChange) { - file.s3.state = 'skip'; - file.s3.etag = etag; - file.s3.date = new Date(res.LastModified); - cb(err, file); + if (noUpdate || noChange) { + file.s3.state = 'skip'; + file.s3.etag = etag; + file.s3.date = new Date(res.LastModified); + cb(err, file); - // update: file are different - } else { - file.s3.state = res.ETag - ? 'update' - : 'create'; + // update: file are different + } else { + file.s3.state = res.ETag + ? 'update' + : 'create'; - _this.client.putObject(toAwsParams(file), function(err) { - if (err) return cb(err); + _this.client.putObject(toAwsParams(file), function(err) { + if (err) return cb(err); - file.s3.date = new Date(); - file.s3.etag = etag; - cb(err, file); - }); - } - }); - } + file.s3.date = new Date(); + file.s3.etag = etag; + cb(err, file); + }); + } + }); }); }; diff --git a/package.json b/package.json index eb64822..c1fd401 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "aws-sdk": "^2.1.16", "clone": "0.x", "gulp-util": "2.x", + "lodash": "^3.10.1", "mime": "1.x", "pad-component": "0.x", "pascal-case": "^1.1.0",