From 7fc0f874b8c06fc2441c2afde8c0f91d6a0d2134 Mon Sep 17 00:00:00 2001
From: Carlos Oliveira <candreoliveira@gmail.com>
Date: Mon, 6 Jul 2020 14:46:29 -0300
Subject: [PATCH] feat(lru): add lru instead of previous response

---
 lib/crawler.js    | 17 +++++------------
 package-lock.json | 37 ++++++++++++++++++++++++++-----------
 package.json      |  1 +
 3 files changed, 32 insertions(+), 23 deletions(-)

diff --git a/lib/crawler.js b/lib/crawler.js
index 362a645e..063b1b9c 100644
--- a/lib/crawler.js
+++ b/lib/crawler.js
@@ -4,6 +4,8 @@ const pick = require("lodash/pick");
 const isEmpty = require("lodash/isEmpty");
 const uniq = require("lodash/uniq");
 const devices = require("puppeteer/DeviceDescriptors");
+const LRU = require("lru-cache");
+
 const {
   resolveUrl,
   debugConsole,
@@ -29,6 +31,7 @@ class Crawler {
     this._options = options;
     this._depth = depth;
     this._previousUrl = previousUrl;
+    this._LRU = new LRU({ max: 100000, maxAge: 1000 * 60 * 60 });
   }
 
   /**
@@ -65,21 +68,11 @@ class Crawler {
     }
 
     if (xrequest) {
-      if (
-        // eslint-disable-next-line operator-linebreak
-        this._previousResponse &&
-        // eslint-disable-next-line quotes
-        typeof this._previousResponse === "object"
-      ) {
-        this._previousResponse[response.url] = output;
-      } else {
-        this._previousResponse = {};
-        this._previousResponse[response.url] = output;
-      }
+      this._LRU.set(response.url, output, 1000 * 90);
     }
 
     return {
-      ...((this._previousResponse || {})[response.url] || {}),
+      ...(this._LRU.get(response.url) || {}),
       ...output,
     };
   }
diff --git a/package-lock.json b/package-lock.json
index 9de7bdf9..099d7a69 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -2310,7 +2310,25 @@
             "lru-cache": "^4.0.1",
             "shebang-command": "^1.2.0",
             "which": "^1.2.9"
+          },
+          "dependencies": {
+            "lru-cache": {
+              "version": "4.1.5",
+              "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
+              "integrity": "sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==",
+              "dev": true,
+              "requires": {
+                "pseudomap": "^1.0.2",
+                "yallist": "^2.1.2"
+              }
+            }
           }
+        },
+        "yallist": {
+          "version": "2.1.2",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
+          "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=",
+          "dev": true
         }
       }
     },
@@ -3154,7 +3172,7 @@
     },
     "get-stream": {
       "version": "3.0.0",
-      "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
       "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=",
       "dev": true
     },
@@ -4690,13 +4708,11 @@
       }
     },
     "lru-cache": {
-      "version": "4.1.3",
-      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.3.tgz",
-      "integrity": "sha512-fFEhvcgzuIoJVUF8fYr5KR0YqxD238zgObTps31YdADwPPAp82a4M8TrckkWyx7ekNlf9aBcVn81cFwwXngrJA==",
-      "dev": true,
+      "version": "5.1.1",
+      "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
+      "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==",
       "requires": {
-        "pseudomap": "^1.0.2",
-        "yallist": "^2.1.2"
+        "yallist": "^3.0.2"
       }
     },
     "makeerror": {
@@ -7325,10 +7341,9 @@
       "dev": true
     },
     "yallist": {
-      "version": "2.1.2",
-      "resolved": "https://registry.npmjs.org/yallist/-/yallist-2.1.2.tgz",
-      "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=",
-      "dev": true
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
+      "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
     },
     "yargs": {
       "version": "11.1.0",
diff --git a/package.json b/package.json
index a0733832..1b4a4397 100644
--- a/package.json
+++ b/package.json
@@ -28,6 +28,7 @@
     "debug": "3.1.0",
     "jquery": "3.3.1",
     "lodash": "4.17.5",
+    "lru-cache": "^5.1.1",
     "puppeteer": "1.19.0",
     "request": "2.87.0",
     "request-promise": "4.2.2",