Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix #73 - Change cache to take header changes into account #78

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Fix #73 - Change cache to take header changes into account #78

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
cf7737d
Early exit instead of long condition.
TheCloudlessSky Aug 5, 2015
3039801
Move cache checking to methods.
TheCloudlessSky Aug 5, 2015
afe02f1
Add lodash and save headers to cache.
TheCloudlessSky Aug 5, 2015
d6cac7c
Compare headers for caching/skipping files.
TheCloudlessSky Aug 6, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
221 changes: 172 additions & 49 deletions lib/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ var AWS = require('aws-sdk'),
mime = require('mime'),
pascalCase = require('pascal-case'),
File = require('vinyl'),
gutil = require('gulp-util');
gutil = require('gulp-util'),
_ = require('lodash');

var PLUGIN_NAME = 'gulp-awspublish';

Expand Down Expand Up @@ -174,13 +175,27 @@ module.exports.reporter = function(param) {
function Publisher(config) {
this.config = config;
this.client = new AWS.S3(config);
this._cacheOptions = { headers: false };

// load cache
try {
this._cache = JSON.parse(fs.readFileSync(this.getCacheFilename(), 'utf8'));
} catch (err) {
this._cache = {};
}

// Migrate old style caches that used only ETags:
// { "path1": "etag1", "path2": "etag2" }
for (var key in this._cache) {
var value = this._cache[key];

if (typeof value === 'string') {
this._cache = {
etag: value,
headers: {}
};
}
}
}

/**
Expand All @@ -199,14 +214,120 @@ Publisher.prototype.getCacheFilename = function() {
return '.awspublish-' + bucket;
};

Publisher.prototype._isFileCached = function(file, etag) {

var fileCache = this._cache[file.s3.path];

if (!fileCache) return false;

if (fileCache.etag !== etag) return false;

// Header comparison is turned off.
if (!this._cacheOptions.headers) return true;

var checkedHeaders = {};

for (var name in file.s3.headers) {

checkedHeaders[name] = true;

if (name in fileCache.headers) {
if (file.s3.headers[name] !== fileCache.headers[name]) {
// Modified header.
return false;
}
}
else {
// New header.
return false;
}
}

for (var name in fileCache.headers) {
if (name in checkedHeaders) continue;

// Deleted header.
return false;
}

return true;
};

Publisher.prototype._shouldSkipFile = function(file, response, etag) {

if (response.ETag !== etag) return false;

// Normalize S3 headers.
var responseHeaders = {};
if (response.CacheControl) { responseHeaders['Cache-Control'] = response.CacheControl; }
if (response.ContentType) { responseHeaders['Content-Type'] = response.ContentType; }
if (response.ContentDisposition) { responseHeaders['Content-Disposition'] = response.ContentDisposition; }
if (response.ContentLanguage) { responseHeaders['Content-Language'] = response.ContentLanguage; }
if (response.Expires) { responseHeaders['Expires'] = response.Expires; }
if (response.ContentEncoding) { responseHeaders['Content-Encoding'] = response.ContentEncoding; }

if (response.Metadata) {
for (var name in response.Metadata) {
responseHeaders['x-amz-meta-' + name] = response.Metadata[name];
}
}

var checkedHeaders = {};

for (var name in file.s3.headers) {

checkedHeaders[name] = true;

// Have to ignore the ACL header since it's a canned ACL name that gets
// transformed into an ACL and isn't returned in the response. The
// S3 client's 'getObjectAcl()' could be used, but the response ACL
// would have to be mapped back to the canned ACL name.
if (name === 'x-amz-acl') {
continue;
}

if (name in responseHeaders) {
if (file.s3.headers[name] !== responseHeaders[name]) {
// Modified header.
return false;
}
}
else {
// New header.
return false;
}
}

for (var name in responseHeaders.headers) {

if (name in checkedHeaders) continue;

// Deleted header.
return false;
}

return true;
};

/**
* create a through stream that save file in cache
* @options {Object} options option hash
*
* available options are:
* - headers {Boolean} include file headers when checking if the file is cached

* @return {Stream}
* @api public
*/

Publisher.prototype.cache = function() {
Publisher.prototype.cache = function(options) {

if (!options) {
options = this._cacheOptions;
}

this._cacheOptions = options;

var _this = this,
counter = 0;

Expand All @@ -226,7 +347,10 @@ Publisher.prototype.cache = function() {

// update others
} else if (file.s3.etag) {
_this._cache[file.s3.path] = file.s3.etag;
_this._cache[file.s3.path] = {
etag: file.s3.etag,
headers: _.clone(file.s3.headers)
};
}

// save cache every 10 files
Expand Down Expand Up @@ -281,68 +405,67 @@ Publisher.prototype.publish = function (headers, options) {
}

// check if file.contents is a `Buffer`
if (file.isBuffer()) {
if (!file.isBuffer()) return;

initFile(file);
initFile(file);

// calculate etag
etag = '"' + md5Hash(file.contents) + '"';
// calculate etag
etag = '"' + md5Hash(file.contents) + '"';

// delete - stop here
if (file.s3.state === 'delete') return cb(null, file);
// delete - stop here
if (file.s3.state === 'delete') return cb(null, file);

// check if file is identical as the one in cache
if (!options.force && _this._cache[file.s3.path] === etag) {
file.s3.state = 'cache';
return cb(null, file);
}
// add content-type header
if (!file.s3.headers['Content-Type']) file.s3.headers['Content-Type'] = getContentType(file);

// add content-type header
if (!file.s3.headers['Content-Type']) file.s3.headers['Content-Type'] = getContentType(file);
// add content-length header
if (!file.s3.headers['Content-Length']) file.s3.headers['Content-Length'] = file.contents.length;

// add content-length header
if (!file.s3.headers['Content-Length']) file.s3.headers['Content-Length'] = file.contents.length;
// add extra headers
for (header in headers) file.s3.headers[header] = headers[header];

// add extra headers
for (header in headers) file.s3.headers[header] = headers[header];
// check if file is identical as the one in cache
if (!options.force && _this._isFileCached(file, etag)) {
file.s3.state = 'cache';
return cb(null, file);
}

if (options.simulate) return cb(null, file);
if (options.simulate) return cb(null, file);

// get s3 headers
_this.client.headObject({ Key: file.s3.path }, function(err, res) {
//ignore 403 and 404 errors since we're checking if a file exists on s3
if (err && [403, 404].indexOf(err.statusCode) < 0) return cb(err);
// get s3 headers
_this.client.headObject({ Key: file.s3.path }, function(err, res) {
//ignore 403 and 404 errors since we're checking if a file exists on s3
if (err && [403, 404].indexOf(err.statusCode) < 0) return cb(err);

res = res || {};
res = res || {};

// skip: no updates allowed
var noUpdate = options.createOnly && res.ETag;
// skip: no updates allowed
var noUpdate = options.createOnly && res.ETag;

// skip: file are identical
var noChange = !options.force && res.ETag === etag;
// skip: file is identical
var noChange = !options.force && _this._shouldSkipFile(file, res, etag);

if (noUpdate || noChange) {
file.s3.state = 'skip';
file.s3.etag = etag;
file.s3.date = new Date(res.LastModified);
cb(err, file);
if (noUpdate || noChange) {
file.s3.state = 'skip';
file.s3.etag = etag;
file.s3.date = new Date(res.LastModified);
cb(err, file);

// update: file are different
} else {
file.s3.state = res.ETag
? 'update'
: 'create';
// update: file are different
} else {
file.s3.state = res.ETag
? 'update'
: 'create';

_this.client.putObject(toAwsParams(file), function(err) {
if (err) return cb(err);
_this.client.putObject(toAwsParams(file), function(err) {
if (err) return cb(err);

file.s3.date = new Date();
file.s3.etag = etag;
cb(err, file);
});
}
});
}
file.s3.date = new Date();
file.s3.etag = etag;
cb(err, file);
});
}
});
});
};

Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
"aws-sdk": "^2.1.16",
"clone": "0.x",
"gulp-util": "2.x",
"lodash": "^3.10.1",
"mime": "1.x",
"pad-component": "0.x",
"pascal-case": "^1.1.0",
Expand Down