From fc7bd720582c2aa8344e1dee29b1c973d4b356c6 Mon Sep 17 00:00:00 2001 From: "dr.dimitru" Date: Fri, 10 Jun 2022 00:07:54 +0300 Subject: [PATCH] =?UTF-8?q?=F0=9F=93=A6=20v1.6.6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - ๐Ÿงน Codebase cleanup - ๐Ÿ‘ทโ€โ™‚๏ธ Avoid exception on `.abort()` --- .versions | 6 +++--- lib/index.js | 58 ++++++++++++++++++++++++++-------------------------- package.js | 2 +- package.json | 4 ++-- 4 files changed, 35 insertions(+), 35 deletions(-) diff --git a/.versions b/.versions index 14a16b6..636d383 100644 --- a/.versions +++ b/.versions @@ -1,6 +1,6 @@ allow-deny@1.1.1 babel-compiler@7.9.0 -babel-runtime@1.5.0 +babel-runtime@1.5.1 base64@1.0.12 binary-heap@1.0.11 boilerplate-generator@1.7.1 @@ -22,7 +22,7 @@ geojson-utils@1.0.10 http@1.4.2 id-map@1.1.1 inter-process-messaging@0.1.1 -local-test:ostrio:spiderable-middleware@1.6.5 +local-test:ostrio:spiderable-middleware@1.6.6 logging@1.3.1 meteor@1.10.0 minimongo@1.8.0 @@ -35,7 +35,7 @@ mongo-dev-server@1.1.0 mongo-id@1.0.8 npm-mongo@4.3.1 ordered-dict@1.1.0 -ostrio:spiderable-middleware@1.6.5 +ostrio:spiderable-middleware@1.6.6 promise@0.12.0 random@1.2.0 react-fast-refresh@0.2.3 diff --git a/lib/index.js b/lib/index.js index ba45929..88b2d46 100644 --- a/lib/index.js +++ b/lib/index.js @@ -34,15 +34,15 @@ var _warn = function warn (...args) { console.warn.call(console, '[WARN] [Spiderable-Middleware]', ...args); }; -request.defaultOptions.debug = false; -request.defaultOptions.headers = { 'User-Agent': 'spiderable-middleware/1.6.4', Accept: '*/*' }; -request.defaultOptions.noStorage = true; -request.defaultOptions.rawBody = true; -request.defaultOptions.retry = true; -request.defaultOptions.retries = 3; -request.defaultOptions.retryDelay = 128; -request.defaultOptions.timeout = 102400; -request.defaultOptions.wait = true; +request.defaultOptions.debug = false; +request.defaultOptions.headers = { 'User-Agent': 'spiderable-middleware/1.6.4', Accept: '*/*' }; +request.defaultOptions.noStorage = true; +request.defaultOptions.rawBody = true; +request.defaultOptions.retry = true; +request.defaultOptions.retries = 3; +request.defaultOptions.retryDelay = 128; +request.defaultOptions.timeout = 102400; +request.defaultOptions.wait = true; request.defaultOptions.badStatuses = [502, 503, 504, 599]; request.defaultOptions.isBadStatus = function (statusCode, badStatuses) { return badStatuses.includes(statusCode); @@ -55,15 +55,15 @@ module.exports = (function () { opts = _opts; } - this.auth = opts.auth; - this.debug = opts.debug || process.env.DEBUG === 'true' || process.env.DEBUG === true || false; - var ignore = opts.ignore || false; - this.only = opts.only || false; - this.onlyRE = opts.onlyRE || false; - this.botsUA = opts.botsUA || Spiderable.prototype.botsUA; - this.rootURL = opts.rootURL || process.env.ROOT_URL; - this.timeout = opts.timeout || 180000; - this.staticExt = opts.staticExt || re.staticExt; + this.auth = opts.auth; + this.debug = opts.debug || process.env.DEBUG === 'true' || process.env.DEBUG === true || false; + var ignore = opts.ignore || false; + this.only = opts.only || false; + this.onlyRE = opts.onlyRE || false; + this.botsUA = opts.botsUA || Spiderable.prototype.botsUA; + this.rootURL = opts.rootURL || process.env.ROOT_URL; + this.timeout = opts.timeout || 180000; + this.staticExt = opts.staticExt || re.staticExt; this.serviceURL = opts.serviceURL || process.env.SPIDERABLE_SERVICE_URL || process.env.PRERENDER_SERVICE_URL || 'https://render.ostr.io'; this.ignoredHeaders = opts.ignoredHeaders || Spiderable.prototype.ignoredHeaders; this.requestOptions = opts.requestOptions || {}; @@ -128,7 +128,7 @@ module.exports = (function () { throw new Error('{serviceURL} is malformed! Must start with protocol http or https'); } - this.rootURL = this.rootURL.replace(re.trailingSlash, strs.empty).replace(re.beginningSlash, strs.empty); + this.rootURL = this.rootURL.replace(re.trailingSlash, strs.empty).replace(re.beginningSlash, strs.empty); this.serviceURL = this.serviceURL.replace(re.trailingSlash, strs.empty).replace(re.beginningSlash, strs.empty); if (ignore) { @@ -151,14 +151,14 @@ module.exports = (function () { var urlObj = url.parse(req.url, true); if ((urlObj.query && urlObj.query._escaped_fragment_ !== void 0) || this.botsRE.test(req.headers[strs.ua] || strs.empty)) { var hasIgnored = false; - var hasOnly = false; + var hasOnly = false; if (this.staticExt.test(urlObj.pathname)) { return next(); } if (this.onlyRE) { - hasOnly = this.onlyRE.test(urlObj.pathname); + hasOnly = this.onlyRE.test(urlObj.pathname); hasIgnored = !hasOnly; } @@ -169,13 +169,13 @@ module.exports = (function () { if (Object.prototype.toString.call(this.only[i]) === strs.objs.string) { if (this.only[i] === urlObj.pathname) { hasIgnored = false; - hasOnly = true; + hasOnly = true; break; } } else if (Object.prototype.toString.call(this.only[i]) === strs.objs.regexp) { if (this.only[i].test(urlObj.pathname)) { hasIgnored = false; - hasOnly = true; + hasOnly = true; break; } } else { @@ -200,14 +200,14 @@ module.exports = (function () { } reqUrl += '/' + urlObj.pathname; - reqUrl = reqUrl.replace(/([^:]\/)\/+/g, '$1'); - reqUrl = (this.serviceURL + '/?url=' + encodeURIComponent(reqUrl)); + reqUrl = reqUrl.replace(/([^:]\/)\/+/g, '$1'); + reqUrl = (this.serviceURL + '/?url=' + encodeURIComponent(reqUrl)); if (req.headers[strs.ua]) { reqUrl += '&bot=' + encodeURIComponent(req.headers[strs.ua]); } - var opts = Object.assign({}, this.requestOptions, { + var opts = Object.assign({}, this.requestOptions, { uri: reqUrl, auth: this.auth || false, debug: this.debug @@ -215,11 +215,11 @@ module.exports = (function () { try { var usedHeaders = []; - var _headersRE = this.headersRE; - var serviceReq = request(opts, function (error, resp) { + var _headersRE = this.headersRE; + var serviceReq = request(opts, function (error, resp) { if (error) { // DO NOT THROW AN ERROR ABOUT ABORTED REQUESTS - if (!req.aborted) { + if (!req.aborted && error.statusCode !== 499) { _warn('Error while connecting to external service:', error); next(); } diff --git a/package.js b/package.js index 522e422..e8cf55e 100644 --- a/package.js +++ b/package.js @@ -1,6 +1,6 @@ Package.describe({ name: 'ostrio:spiderable-middleware', - version: '1.6.5', + version: '1.6.6', summary: 'Allow JavaScript websites to be perfectly crawled and indexed by search engines (SEO)', git: 'https://github.com/veliovgroup/spiderable-middleware', documentation: 'README.md' diff --git a/package.json b/package.json index 0dec60e..6cbe0cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "spiderable-middleware", - "version": "1.6.5", + "version": "1.6.6", "description": "Perfect SEO for JavaScript websites. Pre-rendering โ€” it's just like SSR with simple integration and no coding", "main": "./lib/index.js", "scripts": { @@ -47,6 +47,6 @@ "express": "^4.18.1", "mocha": "^10.0.0", "request": "^2.88.2", - "underscore": "^1.13.3" + "underscore": "^1.13.4" } }