From d5f530ca3de4b2f8a9ce66741891e79dd2d786af Mon Sep 17 00:00:00 2001
From: Ubuntu <howardzchung@gmail.com>
Date: Sat, 22 Jun 2024 05:51:02 +0000
Subject: [PATCH] implement backup scanner, don't try to fetch api on parsed
 matches

---
 config.ts            |  1 +
 ecosystem.config.js  |  7 +++++++
 global.d.ts          |  3 ++-
 store/buildStatus.ts |  1 +
 store/queries.ts     | 38 ++++++++++++++++++++------------------
 svc/scanner.ts       | 11 ++++++++---
 6 files changed, 39 insertions(+), 22 deletions(-)

diff --git a/config.ts b/config.ts
index f7f0fb27f..c6240e6c0 100755
--- a/config.ts
+++ b/config.ts
@@ -75,6 +75,7 @@ const defaults = {
   API_KEY_GEN_THRESHOLD: '0', // Account ID requirement (delta from max) for generating API keys
   SERVICE_REGISTRY_HOST: '', // Host for external services to register themselves at
   USE_SERVICE_REGISTRY: '', // Use the service registry for determining gc and parser urls
+  SCANNER_OFFSET: '0', // Delay in match seq num value to run secondary scanner (to pick up missing matches)
 };
 if (process.env.NODE_ENV === 'development') {
   // force PORT to null in development so we can run multiple web services without conflict
diff --git a/ecosystem.config.js b/ecosystem.config.js
index 9d611dd61..1ea37ef66 100644
--- a/ecosystem.config.js
+++ b/ecosystem.config.js
@@ -51,6 +51,13 @@ let arr = [
     name: 'scanner',
     group: 'backend',
   },
+  // {
+  //   name: 'scanner2',
+  //   group: 'backend',
+  //   env: {
+  //     SCANNER_OFFSET: '50000',
+  //   }
+  // },
   {
     name: 'backupscanner',
     group: 'disabled',
diff --git a/global.d.ts b/global.d.ts
index f4bb3a552..ef3019fbd 100644
--- a/global.d.ts
+++ b/global.d.ts
@@ -454,7 +454,8 @@ type MetricName =
   | 'incomplete_archive'
   | 'gen_api_key_invalid'
   | 'parser_job'
-  | 'oldparse';
+  | 'oldparse'
+  | 'secondary_scanner';
 
 // Object to map player_slot to basic info
 type PGroup = {
diff --git a/store/buildStatus.ts b/store/buildStatus.ts
index 8a4feafb4..79ae990c0 100644
--- a/store/buildStatus.ts
+++ b/store/buildStatus.ts
@@ -139,6 +139,7 @@ export async function buildStatus() {
     error_last_day: async () => countDay('500_error'),
     web_crash_last_day: async () => countDay('web_crash'),
     skip_seq_num_last_day: async () => countDay('skip_seq_num'),
+    secondary_scanner_last_day: async () => countDay('secondary_scanner'),
     api_paths: async () => {
       const results = await redis.zrangebyscore(
         'api_paths',
diff --git a/store/queries.ts b/store/queries.ts
index a400b0196..e310a6813 100644
--- a/store/queries.ts
+++ b/store/queries.ts
@@ -731,24 +731,6 @@ export async function getMatchDataFromBlobWithMetadata(
     has_gcdata: Boolean(gcdata),
     has_parsed: Boolean(parsed),
   };
-
-  if (!api && backfill) {
-    redisCount(redis, 'steam_api_backfill');
-    api = await tryFetchApiData(matchId, true);
-    if (api) {
-      odData.backfill_api = true;
-    }
-  }
-  if (!api) {
-    return [null, null];
-  }
-  if (!gcdata && backfill) {
-    redisCount(redis, 'steam_gc_backfill');
-    // gcdata = await tryFetchGcData(matchId, getPGroup(api));
-    if (gcdata) {
-      odData.backfill_gc = true;
-    }
-  }
   if (backfill) {
     archived = await tryReadArchivedMatch(matchId);
     if (archived) {
@@ -756,6 +738,26 @@ export async function getMatchDataFromBlobWithMetadata(
     }
   }
 
+  if (!archived) {
+    if (!api && backfill) {
+      redisCount(redis, 'steam_api_backfill');
+      api = await tryFetchApiData(matchId, true);
+      if (api) {
+        odData.backfill_api = true;
+      }
+    }
+    if (!api) {
+      return [null, null];
+    }
+    if (!gcdata && backfill) {
+      redisCount(redis, 'steam_gc_backfill');
+      // gcdata = await tryFetchGcData(matchId, getPGroup(api));
+      if (gcdata) {
+        odData.backfill_gc = true;
+      }
+    }
+  }
+
   // Merge the results together
   const final: Match | ParsedMatch = {
     ...archived,
diff --git a/svc/scanner.ts b/svc/scanner.ts
index 168ec6f4d..486eea8dc 100755
--- a/svc/scanner.ts
+++ b/svc/scanner.ts
@@ -14,7 +14,7 @@ const SCANNER_WAIT = 5000;
 const SCANNER_WAIT_CATCHUP = SCANNER_WAIT / parallelism;
 
 async function scanApi(seqNum: number) {
-  let nextSeqNum = seqNum;
+  let nextSeqNum = seqNum - Number(config.SCANNER_OFFSET);
   while (true) {
     const container = generateJob('api_sequence', {
       start_at_match_seq_num: nextSeqNum,
@@ -64,15 +64,20 @@ async function processMatch(match: ApiMatch) {
     return;
   }
   // check if match was previously processed
-  const result = await redis.get(`scanner_insert:${match.match_id}`);
+  const result = await redis.zscore('scanner_insert', match.match_id);
   // console.log(match.match_id, result);
   // don't insert this match if we already processed it recently
   if (!result) {
+    if (Number(config.SCANNER_OFFSET)) {
+      // secondary scanner picked up a missing match
+      redisCount(redis, 'secondary_scanner');
+    }
     await insertMatch(match, {
       type: 'api',
       origin: 'scanner',
     });
-    await redis.setex(`scanner_insert:${match.match_id}`, 3600 * 4, 1);
+    await redis.zadd('scanner_insert', match.match_id, match.match_id);
+    await redis.zremrangebyrank('scanner_insert', '0', '-100000');
   }
 }