Skip to content

Commit

Permalink
test 'should not skip one single url when skipDuplicates is active' n…
Browse files Browse the repository at this point in the history
…ow passing
  • Loading branch information
tzbee committed Nov 10, 2014
1 parent dbb3b62 commit e0f228f
Showing 1 changed file with 10 additions and 13 deletions.
23 changes: 10 additions & 13 deletions lib/crawler.js
Original file line number Diff line number Diff line change
Expand Up @@ -262,22 +262,19 @@ Crawler.prototype._makeCrawlerRequest = function _makeCrawlerRequest (options) {

Crawler.prototype._executeCrawlerRequest = function _executeCrawlerRequest (options) {
var self = this;
var cacheData = self.cache[options.uri];

if (useCache(options)) {
//If a query has already been made to self URL, don't callback again
if (useCache(options) && cacheData) {

var cacheData = self.cache[options.uri];

//If a query has already been made to self URL, don't callback again
if (cacheData) {

// Make sure we actually have cached data, and not just a note
// that the page was already crawled
if (_.isArray(cacheData)) {
self._onContent(null, options, cacheData[0], true);
} else {
self.emit('pool:release', options);
}
// Make sure we actually have cached data, and not just a note
// that the page was already crawled
if (_.isArray(cacheData)) {
self._onContent(null, options, cacheData[0], true);
} else {
self.emit('pool:release', options);
}

} else {
self._buildHttpRequest(options);
}
Expand Down

0 comments on commit e0f228f

Please sign in to comment.