From df9afe2d03a7ef3d34b5f7f1456a3dc015afd90e Mon Sep 17 00:00:00 2001 From: Jason Abbott Date: Tue, 11 Apr 2017 18:31:12 -0600 Subject: [PATCH] update view caching --- .vscode/extensions.json | 1 - .vscode/settings.json | 7 +-- CHANGELOG.md | 4 ++ lib/cache.js | 75 ++++++++++++++++++++++++++- lib/constants.js | 8 ++- lib/controller.js | 19 +++++-- lib/factory.js | 38 ++++++-------- lib/routes.js | 13 +++-- lib/startup/configure-file-source.js | 18 ------- lib/startup/configure-photo-source.js | 24 --------- package.json | 28 +++++----- test/controller.test.js | 18 +++---- test/flickr.test.js | 19 +++---- test/json-ld.test.js | 4 +- typings/trailimage.d.ts | 9 +--- views/partials/head-files.hbs | 2 +- 16 files changed, 158 insertions(+), 129 deletions(-) delete mode 100644 lib/startup/configure-file-source.js delete mode 100644 lib/startup/configure-photo-source.js diff --git a/.vscode/extensions.json b/.vscode/extensions.json index f42d9ca5..606bbe5d 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -8,7 +8,6 @@ "cspotcode.vscode-mocha-latte", "codezombiech.gitignore", "pranaygp.vscode-css-peek", - "joelday.docthis", "ziyasal.vscode-open-in-github", "donjayamanne.githistory", "anseki.vscode-color", diff --git a/.vscode/settings.json b/.vscode/settings.json index 79cf4fb2..aed0457f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,9 +9,6 @@ "files.associations": { "procfile": "yaml" }, - "files.exclude": { - ".idea/": true - }, "mocha.env": { "FLICKR_API_KEY": "${env.FLICKR_API_KEY}", @@ -24,7 +21,5 @@ "mocha.files.glob": "test/**/*.test.js", "mocha.options": { "ui": "bdd" - }, - - "npm-intellisense.scanDevDependencies": true + } } \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 0407202c..053367b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.2.3 +- Update dependencies +- Use memory instead of Redis to cache views + ## 2.2.2 - Fix missing JSON-LD @type for blog entries diff --git a/lib/cache.js b/lib/cache.js index cab8c3ae..b3080213 100644 --- a/lib/cache.js +++ b/lib/cache.js @@ -8,6 +8,10 @@ const compress = require('zlib'); const prefix = 'api:'; const viewKey = 'view'; const mapKey = 'map'; +/** + * @type {object.} + */ +const memory = {}; /** * Whether key with prefix exists @@ -104,9 +108,78 @@ module.exports = { ), /** - * Cache rendered views + * Cache rendered views in memory */ view: { + /** + * @param {string} key Page slug + * @returns {Promise.} + */ + getItem: key => Promise.resolve(memory[key]), + + /** + * @returns {Promise.} + */ + keys: ()=> Promise.resolve(Object.keys(memory)), + + /** + * Add or replace value at key + * @param {string} key Page slug + * @param {Buffer|string} text HTML or JSON + * @returns {Promise.} + */ + add: (key, text) => createItem(key, text).then(item => { + if (config.cache.views) { memory[key] = item; } + return Promise.resolve(item); + }), + + create: createItem, + + /** + * Whether cache view exists + * @param {string} key + * @returns {Promise.} + */ + exists: key => Promise.resolve(is.defined(memory, key)), + + /** + * Add value only if it doesn't already exist (mainly for testing) + * @param {string} key Page slug + * @param {Buffer|string} buffer Zipped view content + * @returns {Promise} + */ + addIfMissing(key, buffer) { + return (config.cache.views) + ? this.exists(key).then(exists => exists ? Promise.resolve() : this.add(key, buffer)) + : Promise.resolve(); + }, + + /** + * Remove cached page views + * @param {string|string[]} keys + * @returns {Promise} + */ + remove: keys => { + if (is.array(keys)) { + keys.forEach(k => delete memory[k]); + } else { + delete memory[keys]; + } + return Promise.resolve(); + }, + + /** + * In-memory cache doesn't need to serialize the page buffer + * @param {ViewCacheItem} item + * @returns {object} + */ + serialize: item => item + }, + + /** + * Cache rendered views in Redis + */ + redisView: { /** * @param {string} key Page slug * @returns {Promise.} diff --git a/lib/constants.js b/lib/constants.js index 3d977f21..132e1c91 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -27,13 +27,17 @@ module.exports = { UNAVAILABLE: 503 }, alphabet: ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z'], - // provider names used internally by Winston + /** + * Provider names used internally by Winston + */ logTo: { REDIS: 'redis', CONSOLE: 'console', FILE: 'file' }, - // route placeholders that become req.params values + /** + * Route placeholders that become req.params values + */ route: { CATEGORY: 'category', MONTH: 'month', diff --git a/lib/controller.js b/lib/controller.js index fcb77b37..6491d461 100644 --- a/lib/controller.js +++ b/lib/controller.js @@ -22,6 +22,8 @@ const menuKeys = [ template.page.SITEMAP ]; +//= Map ======================================================================= + // can be replaced with injection let google = require('./google'); @@ -82,6 +84,8 @@ function mapGPX(req, res) { } } +//= Post ====================================================================== + /** * @param {BlogResponse} res * @param {string} key Post key @@ -135,8 +139,7 @@ function postWithID(req, res) { */ function latestPost(req, res) { postView(res, library.posts[0].key); } -//endregion -//region Photos +//= Photo ===================================================================== /** * Small HTML table of EXIF values for given photo @@ -227,6 +230,8 @@ function normalizeTag(slug) { return (is.defined(config.photoTagChanges, slug)) ? config.photoTagChanges[slug] : slug; } +//= Category ================================================================== + /** * @param res * @param {String} path @@ -333,6 +338,8 @@ function renderCategory(render, template, category, linkData, options, childCoun })); } +//= Menu ====================================================================== + // https://npmjs.org/package/uglify-js const uglify = require('uglify-js'); @@ -359,6 +366,8 @@ function mobileMenu(req, res) { }); } +//= Auth ====================================================================== + /** * @see https://github.com/google/google-api-nodejs-client/#generating-an-authentication-url */ @@ -433,6 +442,8 @@ function googleAuth(req, res) { } } +//= Admin ===================================================================== + /** * @param res * @param {string[]} viewKeys @@ -682,6 +693,8 @@ function deleteJsonCache(req, res) { // return p; // } +//= RSS ======================================================================= + const MAX_RSS_RETRIES = 10; let rssRetries = 0; @@ -725,7 +738,7 @@ function rssFeed(req, res) { }); } res.set('Content-Type', C.mimeType.XML); - res.send(feed.render('rss-2.0')); + res.send(feed.rss2()); } module.exports = { diff --git a/lib/factory.js b/lib/factory.js index cd5c8fc7..328bb8a7 100644 --- a/lib/factory.js +++ b/lib/factory.js @@ -13,11 +13,11 @@ const library = require('./library'); let flickr = require('./flickr'); let google = require('./google'); -//region Library +//= Library =================================================================== /** - * @param {Boolean} [emptyIfLoaded] - * @return {Promise.} Resolve with list of changed post keys + * @param {boolean} [emptyIfLoaded] + * @returns {Promise.} Resolve with list of changed post keys */ function buildLibrary(emptyIfLoaded = true) { // store existing post keys to compute changes @@ -76,7 +76,7 @@ function buildLibrary(emptyIfLoaded = true) { /** * @this {Library} - * @param {Photo|String} photo + * @param {Photo|string} photo * @returns {Promise} */ function getPostWithPhoto(photo) { @@ -140,14 +140,13 @@ function correlatePosts() { } } -//endregion -//region Categories +//= Category ================================================================== /** * Add Flickr collection to library singleton as category * @param {Flickr.Collection} collection - * @param {Boolean} root Whether a root level collection - * @returns {Category|Object} + * @param {boolean} root Whether a root level collection + * @returns {Category|object} */ function buildCategory(collection, root = false) { let exclude = config.flickr.excludeSets; @@ -207,14 +206,14 @@ function buildCategory(collection, root = false) { } /** - * @param {String} key + * @param {string} key * @this {Category} category * @returns {Category} */ function getSubcategory(key) { return this.subcategories.find(c => c.title === key || c.key === key); } /** - * @param {String} key + * @param {string} key * @this {Category} category * @returns {Boolean} */ @@ -262,8 +261,7 @@ function ensureCategoryLoaded() { return Promise.all(this.posts.map(p => p.getInfo().then(p => p.getPhotos()))); } -//endregion -//region Posts +//= Posts ===================================================================== /** * Create post from Flickr photo set @@ -456,8 +454,7 @@ function postName() { return p.title + ((p.isPartial) ? config.library.subtitleSeparator + ' ' + p.subTitle : ''); } -//endregion -//region Photos +//= Photos ==================================================================== /** * Load photos for post and calculate summaries @@ -675,8 +672,7 @@ function serializePhotoCoordinates() { this.photoCoordinates = (is.empty(map)) ? null : encodeURIComponent('size:tiny' + map); } -// endregion -// region EXIF +//= EXIF ====================================================================== /** * @param {String} photoID @@ -765,8 +761,7 @@ function sanitizeExif(exif) { return exif; } -// endregion -// region Tracks and Waypoints +//= Tracks and Waypoints ====================================================== const cache = require('./cache'); @@ -835,13 +830,12 @@ function addPhotoFeature(post, geo, resolve) { }); } -//endregion -//region Video +//= Video ===================================================================== /** * Get video ID and dimensions * @param {Flickr.SetInfo} setInfo - * @returns {Object} + * @returns {object} */ function buildVideoInfo(setInfo) { const d = setInfo.description._content; @@ -861,8 +855,6 @@ function buildVideoInfo(setInfo) { } } -//endregion - module.exports = { buildLibrary, map: { diff --git a/lib/routes.js b/lib/routes.js index 1f6ec635..12c214ed 100644 --- a/lib/routes.js +++ b/lib/routes.js @@ -3,11 +3,12 @@ const config = require('./config'); const C = require('./constants'); const c = require('./controller'); const library = require('./library'); -// route placeholders +/** + * Route placeholders + * @type {object} + */ const ph = C.route; -//region Sub-routes - /** * Need to capture top-level route parameters * @see http://expressjs.com/en/4x/api.html#express.router @@ -25,7 +26,7 @@ function adminRoutes() { } /** - * @param {String} photoID Pattern + * @param {string} photoID Pattern * @returns {core.Router} */ function postRoutes(photoID) { @@ -41,7 +42,7 @@ function postRoutes(photoID) { /** * Series should load the PDF, GPX and GeoJSON for the main post - * @param {String} photoID Pattern + * @param {string} photoID Pattern * @returns {core.Router} */ function seriesRoutes(photoID) { @@ -67,8 +68,6 @@ function categoryRoutes() { return r; } -//endregion - /** * @param app Express instance * @see http://expressjs.com/en/4x/api.html diff --git a/lib/startup/configure-file-source.js b/lib/startup/configure-file-source.js deleted file mode 100644 index 305004f1..00000000 --- a/lib/startup/configure-file-source.js +++ /dev/null @@ -1,18 +0,0 @@ -const Blog = require('../'); -const GoogleProvider = require('@trailimage/google-provider'); - -module.exports = function() { - let c = GoogleProvider.Config(); - - c.apiKey = config.env('GOOGLE_DRIVE_KEY'); - c.tracksFolder = '0B0lgcM9JCuSbMWluNjE4LVJtZWM'; - c.httpProxy = config.proxy; - - c.auth.clientID = config.env('GOOGLE_CLIENT_ID'); - c.auth.clientSecret = config.env('GOOGLE_SECRET'); - c.auth.url.callback = `http://www.${config.domain}/auth/google`; - c.auth.accessToken = process.env['GOOGLE_ACCESS_TOKEN']; - c.auth.refreshToken = process.env['GOOGLE_REFRESH_TOKEN']; - - Blog.active.file = new GoogleProvider.File(c); -}; \ No newline at end of file diff --git a/lib/startup/configure-photo-source.js b/lib/startup/configure-photo-source.js deleted file mode 100644 index d9281d65..00000000 --- a/lib/startup/configure-photo-source.js +++ /dev/null @@ -1,24 +0,0 @@ -'use strict'; - -const Blog = require('../'); -const FlickrProvider = require('@trailimage/flickr-provider'); - -module.exports = function() { - const config = Blog.config; - /** @type FlickrProvider.Config */ - let c = FlickrProvider.Config(); - - c.userID = '60950751@N04'; - c.appID = '72157631007435048'; - c.featureSets.push({ id: '72157632729508554', title: 'Ruminations' }); - c.excludeSets.push('72157631638576162'); - c.httpProxy = config.proxy; - - c.auth.clientID = config.env('FLICKR_API_KEY'); - c.auth.clientSecret = config.env('FLICKR_SECRET'); - c.auth.url.callback = `http://www.${config.domain}/auth/flickr`; - c.auth.accessToken = process.env['FLICKR_ACCESS_TOKEN']; - c.auth.tokenSecret = process.env['FLICKR_TOKEN_SECRET']; - - Blog.active.photo = new FlickrProvider.Photo(c); -}; \ No newline at end of file diff --git a/package.json b/package.json index 3b6c45d6..dd5fab61 100644 --- a/package.json +++ b/package.json @@ -1,14 +1,14 @@ { "name": "trail-image", "description": "Trail Image blog", - "version": "2.2.0", + "version": "2.2.3", "private": false, "author": { "name": "Jason Abbott" }, "license": "MIT", "engines": { - "node": ">=7.5.0" + "node": ">=7.8.0" }, "scripts": { "test-pretty": "node_modules/.bin/mocha --recursive --reporter mocha-pretty-spec-reporter", @@ -18,19 +18,19 @@ "mapfont": "webfont-dl \"http://fonts.googleapis.com/css?family=Droid+Sans:400,700|Orbitron:700'\" --css-rel=./fonts -o dist/fonts/mapfont.css" }, "dependencies": { - "body-parser": "^1.16.0", + "body-parser": "^1.17.0", "compression": "^1.6.0", "connect-wwwhisper": "^0.1.16", - "express": "^4.14.0", + "express": "^4.15.0", "express-hbs": "^1.0.0", - "feed": "^0.3.0", + "feed": "^1.0.0", "google-auth-library": "^0.10.0", - "googleapis": "^16.1.0", + "googleapis": "^19.0.0", "node-fetch": "^1.6.3", "oauth": "^0.9.0", "pdfkit": "^0.8.0", - "redis": "^2.6.2", - "uglify-js": "^2.7.4", + "redis": "^2.7.0", + "uglify-js": "^2.8.0", "winston": "^2.3.0", "winston-redis": "^1.0.0", "xmldom": "^0.1.19" @@ -39,20 +39,20 @@ "babel-eslint": "^7.1.1", "bootstrap": "git+https://github.com/twbs/bootstrap.git#master", "chai": "^3.5.0", - "eslint": "^3.15.0", - "eslint-loader": "^1.6.1", + "eslint": "^3.19.0", + "eslint-loader": "^1.7.0", "eslint-plugin-react": "^6.10.0", "gulp": "^3.9.1", "gulp-concat": "^2.6.1", "gulp-cssnano": "^2.1.2", "gulp-less": "^3.3.0", - "gulp-mocha": "^3.0.1", - "gulp-uglify": "^2.0.0", + "gulp-mocha": "^4.3.0", + "gulp-uglify": "^2.1.0", "merge2": "^1.0.2", "mocha": "^3.2.0", "mocha-pretty-spec-reporter": "0.1.0-beta.2", - "stylelint": "^7.6.0", - "stylelint-config-standard": "^15.0.0", + "stylelint": "^7.10.0", + "stylelint-config-standard": "^16.0.0", "webfont-dl": "^0.1.2" }, "repository": { diff --git a/test/controller.test.js b/test/controller.test.js index acd4ec0d..f469d4fc 100644 --- a/test/controller.test.js +++ b/test/controller.test.js @@ -13,8 +13,8 @@ const ph = C.route; /** * Expect standard Handlexitebars template response - * @param {String} name Template name - * @returns {Object} + * @param {string} name Template name + * @returns {object} */ function expectTemplate(name) { expect(res.httpStatus).equals(C.httpStatus.OK); @@ -24,7 +24,7 @@ function expectTemplate(name) { } /** - * @param {String} path Redirection target + * @param {string} path Redirection target */ function expectRedirect(path) { expect(res.redirected).to.exist; @@ -34,7 +34,7 @@ function expectRedirect(path) { /** * Expectations for JSON responses - * @returns {String|Object} response content + * @returns {string|object} response content */ function expectJSON() { expect(res.httpStatus).equals(C.httpStatus.OK); @@ -47,8 +47,8 @@ function expectJSON() { /** * Run exists() method for each key and confirm it does or does not exist - * @param {String[]} keys - * @param {Boolean} [exists] + * @param {string[]} keys + * @param {boolean} [exists] * @returns {Promise} */ function expectInCache(keys, exists = true) { @@ -58,8 +58,6 @@ function expectInCache(keys, exists = true) { .then(results => { results.forEach(r => expect(r).equals(exists)); }); } -//endregion - describe('Controller', ()=> { before(done => { const googleMock = require('./mocks/google.mock'); @@ -314,17 +312,17 @@ describe('Controller', ()=> { + '' + nl + tab + '' + nl + tab + tab + '' + title + '' + nl - + tab + tab + '' + description + '' + nl + tab + tab + '' + url + '' + nl + + tab + tab + '' + description + '' + nl + tab + tab + '' + updated.toUTCString() + '' + nl + tab + tab + 'http://blogs.law.harvard.edu/tech/rss' + nl + + tab + tab + 'Feed for Node.js' + nl + tab + tab + '' + nl + tab + tab + tab + '' + title + '' + nl + tab + tab + tab + '' + image + '' + nl + tab + tab + tab + '' + url + '' + nl + tab + tab + '' + nl + tab + tab + '' + copyright + '' + nl - + tab + tab + 'Feed for Node.js' + nl + tab + '' + nl + ''; diff --git a/test/flickr.test.js b/test/flickr.test.js index 6f5c9c7f..3018892e 100644 --- a/test/flickr.test.js +++ b/test/flickr.test.js @@ -4,6 +4,7 @@ const flickr = require('../lib/flickr'); const config = require('../lib/config'); const featureSetID = config.flickr.featureSets[0].id; const featurePhotoID = '8459503474'; +const longTimeout = 5000; describe('Flickr', ()=> { // disable caching so calls hit API @@ -11,15 +12,15 @@ describe('Flickr', ()=> { it('retrieves all collections', ()=> flickr.getCollections().then(json => { expect(json).to.be.instanceOf(Array); - })); + })).timeout(longTimeout * 2); it('catches non-existent set request', ()=> flickr.getSetInfo('45').catch(error => { expect(error).equals('Flickr photosets.getInfo failed for photoset_id 45'); - })); + })).timeout(longTimeout); it('retrieves set information', ()=> flickr.getSetInfo(featureSetID).then(json => { expect(json.id).equals(featureSetID); - })); + })).timeout(longTimeout); it('retrieves set photos', ()=> flickr.getSetPhotos(featureSetID).then(json => { expect(json).has.property('id', featureSetID); @@ -28,30 +29,30 @@ describe('Flickr', ()=> { // should retrieve all size URLs needed to display post expect(json.photo[0]).to.include.keys(s); }); - })); + })).timeout(longTimeout); it('retrieves photo EXIF', ()=> flickr.getExif(featurePhotoID).then(json => { expect(json).to.be.instanceOf(Array); - })); + })).timeout(longTimeout); it('retrieves photo sizes', ()=> flickr.getPhotoSizes(featurePhotoID).then(json => { expect(json).to.be.instanceOf(Array); expect(json[0]).to.include.keys('url'); - })); + })).timeout(longTimeout); it('retrieves all photo tags', ()=> flickr.getAllPhotoTags().then(json => { expect(json).to.be.instanceOf(Array); - })); + })).timeout(longTimeout); it('retrieves photo context', ()=> flickr.getPhotoContext(featurePhotoID).then(json => { expect(json).to.be.instanceOf(Array); expect(json[0]).has.property('id', featureSetID); - })); + })).timeout(longTimeout); it('searches for photos', ()=> flickr.photoSearch('horse').then(json => { expect(json).to.be.instanceOf(Array); expect(json[0]).has.property('owner', config.flickr.userID); - })); + })).timeout(longTimeout); // restore caching for other tests after(() => { config.cache.json = true; }); diff --git a/test/json-ld.test.js b/test/json-ld.test.js index f7d6a614..dada81af 100644 --- a/test/json-ld.test.js +++ b/test/json-ld.test.js @@ -44,9 +44,9 @@ describe('JSON-LD', ()=> { + '"@type":"Person"},"name":"Spring Fish & Chips","headline":"Spring Fish & Chips",' + '"description":"Photography’s highest form is sometimes likened to poetry, capturing experiences that defy denotation. Both are diminished to the extent they require explanation, some say, so hopefully coercing them to explain each other is a right born of two wrongs.",' + '"image":{"url":"https://farm9.staticflickr.com/8109/8459503474_7fcb90b3e9_b.jpg","width":1024,"height":688,"@type":"ImageObject"},' - + '"publisher":{"name":"Trail Image","logo":{"url":"http://www.trailimage.com/img/logo-large.png","width":200,"height":200,"@type":"ImageObject"},' + + '"publisher":{"name":"Trail Image","logo":{"url":"http://www.trailimage.com/img/logo-title.png","width":308,"height":60,"@type":"ImageObject"},' + '"@type":"Organization"},"mainEntityOfPage":{"@id":"http://www.trailimage.com/spring-fish--chips","@type":"WebPage"},' - + '"datePublished":"2011-01-01T02:14:07.000Z","dateModified":"2016-06-04T22:07:20.000Z","articleSection":"2016,Boise River,Family,Bicycle","@type":"","@context":"http://schema.org"}'; + + '"datePublished":"2011-01-01T02:14:07.000Z","dateModified":"2016-06-04T22:07:20.000Z","articleSection":"2016,Boise River,Family,Bicycle","@type":"BlogPosting","@context":"http://schema.org"}'; const source = ld.serialize(ld.fromPost(post)); expect(source).equals(target); diff --git a/typings/trailimage.d.ts b/typings/trailimage.d.ts index 73489121..816cff09 100644 --- a/typings/trailimage.d.ts +++ b/typings/trailimage.d.ts @@ -10,8 +10,6 @@ interface FlickrOptions { args: Object } -//region IO - interface ViewCacheItem { buffer: Buffer, eTag: string @@ -58,9 +56,6 @@ interface MockResponse extends BlogRequest { setHeader(key: string, value: string): MockResponse } -//endregion -//region Models - interface Category { title: string, key: string, @@ -155,6 +150,4 @@ interface Size { width: number, height: number, isEmpty: boolean -} - -//endregion \ No newline at end of file +} \ No newline at end of file diff --git a/views/partials/head-files.hbs b/views/partials/head-files.hbs index f3736b45..47b8c8b4 100644 --- a/views/partials/head-files.hbs +++ b/views/partials/head-files.hbs @@ -5,7 +5,7 @@ {{! update admin.hbs to match }} - + {{! other scripts are loaded dynamically by responsive.js }} \ No newline at end of file