From e0f228f3b28808d57ca8f4c156efc073031b1e90 Mon Sep 17 00:00:00 2001 From: Touzbi Date: Mon, 10 Nov 2014 21:24:38 +0200 Subject: [PATCH] test 'should not skip one single url when skipDuplicates is active' now passing --- lib/crawler.js | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/lib/crawler.js b/lib/crawler.js index 0619f68..6d01718 100644 --- a/lib/crawler.js +++ b/lib/crawler.js @@ -262,22 +262,19 @@ Crawler.prototype._makeCrawlerRequest = function _makeCrawlerRequest (options) { Crawler.prototype._executeCrawlerRequest = function _executeCrawlerRequest (options) { var self = this; + var cacheData = self.cache[options.uri]; - if (useCache(options)) { + //If a query has already been made to self URL, don't callback again + if (useCache(options) && cacheData) { - var cacheData = self.cache[options.uri]; - - //If a query has already been made to self URL, don't callback again - if (cacheData) { - - // Make sure we actually have cached data, and not just a note - // that the page was already crawled - if (_.isArray(cacheData)) { - self._onContent(null, options, cacheData[0], true); - } else { - self.emit('pool:release', options); - } + // Make sure we actually have cached data, and not just a note + // that the page was already crawled + if (_.isArray(cacheData)) { + self._onContent(null, options, cacheData[0], true); + } else { + self.emit('pool:release', options); } + } else { self._buildHttpRequest(options); }