feat(lru): add lru instead of previous response

candreoliveira · Jul 6, 2020 · 7fc0f87 · 7fc0f87
1 parent 11a54c6
commit 7fc0f87
Show file tree

Hide file tree

Showing 3 changed files with 32 additions and 23 deletions.
diff --git a/lib/crawler.js b/lib/crawler.js
@@ -4,6 +4,8 @@ const pick = require("lodash/pick");
 const isEmpty = require("lodash/isEmpty");
 const uniq = require("lodash/uniq");
 const devices = require("puppeteer/DeviceDescriptors");
+const LRU = require("lru-cache");
+
 const {
   resolveUrl,
   debugConsole,
@@ -29,6 +31,7 @@ class Crawler {
     this._options = options;
     this._depth = depth;
     this._previousUrl = previousUrl;
+    this._LRU = new LRU({ max: 100000, maxAge: 1000 * 60 * 60 });
   }
 
   /**
@@ -65,21 +68,11 @@ class Crawler {
     }
 
     if (xrequest) {
-      if (
-        // eslint-disable-next-line operator-linebreak
-        this._previousResponse &&
-        // eslint-disable-next-line quotes
-        typeof this._previousResponse === "object"
-      ) {
-        this._previousResponse[response.url] = output;
-      } else {
-        this._previousResponse = {};
-        this._previousResponse[response.url] = output;
-      }
+      this._LRU.set(response.url, output, 1000 * 90);
     }
 
     return {
-      ...((this._previousResponse || {})[response.url] || {}),
+      ...(this._LRU.get(response.url) || {}),
       ...output,
     };
   }

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -28,6 +28,7 @@
     "debug": "3.1.0",
     "jquery": "3.3.1",
     "lodash": "4.17.5",
+    "lru-cache": "^5.1.1",
     "puppeteer": "1.19.0",
     "request": "2.87.0",
     "request-promise": "4.2.2",