add logic to prefer archive for parsed matches

dev/archiveTest.mjs

@@ -1,17 +1,21 @@
-import { archivePut, archiveGet } from '../store/archive.js';
-import { getMatchData, getPlayerMatchData } from '../store/queries.js';
+import { archivePut } from '../store/archive.js';
+import { getArchivedMatch, getMatchData, getPlayerMatchData } from '../store/queries.js';
 
 // Read some match data
-const match = await getMatchData(7465883253);
-const players = await getPlayerMatchData(7465883253);
-const blob = Buffer.from(JSON.stringify({ ...match, players }));
+const match = {...await getMatchData(7465883253), players: await getPlayerMatchData(7465883253)};
+const blob = Buffer.from(JSON.stringify(match));
 
 // Archive it
-await archivePut(match.match_id.toString(), blob);
+const putRes = await archivePut(match.match_id.toString(), blob);
+console.log(putRes);
 
 // Read it back
-const readBack = await archiveGet(match.match_id.toString());
+const readBack = await getArchivedMatch(match.match_id.toString());
 
-console.log(blob.length, readBack.length);
+console.log(JSON.stringify(match).length, JSON.stringify(readBack).length);
+
+// Verify we get back null for invalid match id
+const nullMatch = await getArchivedMatch(123);
+console.log(nullMatch);
 
 // Confirm API returns the same data whether we used the archive or not

store/archive.js

@@ -30,7 +30,7 @@ async function stream2buffer(stream) {
 
 async function archiveGet(key) {
   if (!client) {
-    return;
+    return null;
   }
   const command = new GetObjectCommand({
     Bucket: config.MATCH_ARCHIVE_S3_BUCKET,
@@ -39,7 +39,7 @@ async function archiveGet(key) {
   try {
     const data = await client.send(command);
     if (!data.Body) {
-      return;
+      return null;
     }
     const buffer = await stream2buffer(data.Body);
     const result = gunzipSync(buffer);
@@ -50,27 +50,33 @@ async function archiveGet(key) {
     );
     return result;
   } catch (e) {
-    return;
+    console.error('[ARCHIVE] get error:', e.Code);
+    return null;
   }
 }
 
 async function archivePut(key, blob) {
   if (!client) {
-    return;
+    return null;
+  }
+  try {
+    const data = gzipSync(blob);
+    const command = new PutObjectCommand({
+      Bucket: config.MATCH_ARCHIVE_S3_BUCKET,
+      Key: key,
+      Body: data,
+    });
+    const result = await client.send(command);
+    console.log(
+      '[ARCHIVE] original %s bytes, archived %s bytes',
+      blob.length,
+      data.length
+    );
+    return result;
+  } catch(e) {
+    console.error('[ARCHIVE] put error:', e.Code);
+    return null;
   }
-  const data = gzipSync(blob);
-  const command = new PutObjectCommand({
-    Bucket: config.MATCH_ARCHIVE_S3_BUCKET,
-    Key: key,
-    Body: data,
-  });
-  const result = await client.send(command);
-  console.log(
-    '[ARCHIVE] original %s bytes, archived %s bytes',
-    blob.length,
-    data.length
-  );
-  return result;
 }
 
 module.exports = {

store/buildMatch.js

@@ -10,11 +10,11 @@ const utility = require('../util/utility');
 const cassandra = require('./cassandra');
 const redis = require('./redis');
 const db = require('./db');
-const { archiveGet } = require('./archive');
 const {
   getPlayerMatchData,
   getMatchData,
   insertMatchPromise,
+  getArchivedMatch,
 } = require('./queries');
 
 const { computeMatchData } = compute;
@@ -119,14 +119,21 @@ async function getMatch(matchId) {
   if (!matchId || Number.isNaN(Number(matchId)) || Number(matchId) <= 0) {
     return Promise.resolve();
   }
-  let match = await getMatchData(matchId);
-  if (!match) {
-    // check the parsed match archive to see if we have it
-    const blob = await archiveGet(matchId.toString());
-    if (blob) {
-      match = JSON.parse(blob);
-      utility.redisCount(redis, 'match_archive_read');
-    }
+  // Check if the match is parsed
+  // if so we prefer the archive since Cassandra may contain an unparsed version
+  const isParsed = Boolean(
+    (
+      await db.raw(
+        'select match_id from parsed_matches where match_id = ?',
+        [match.match_id]
+      )
+    ).rows[0]
+  );
+  let match = null;
+  if (isParsed) {
+    match = await getArchivedMatch(matchId) || await getMatchData();
+  } else {
+    match = await getMatchData(matchId) || await getArchivedMatch(matchId);
   }
   if (!match) {
     // if we still don't have it, try backfilling it from Steam API and then check again

store/queries.js

@@ -16,6 +16,7 @@ const { es, INDEX } = require('./elasticsearch');
 const cassandra = require('./cassandra');
 const cacheFunctions = require('./cacheFunctions');
 const benchmarksUtil = require('../util/benchmarksUtil');
+const { archiveGet } = require('./archive');
 
 const {
   redisCount,
@@ -1690,6 +1691,19 @@ async function getPlayerMatchData(matchId) {
   return deserializedResult;
 }
 
+async function getArchivedMatch(matchId) {
+  try {
+    const result = JSON.parse(await archiveGet(matchId.toString()));
+    if (result) {
+      utility.redisCount(redis, 'match_archive_read');
+      return result;
+    }
+  } catch(e) {
+    console.error(e);
+  }
+  return null;
+}
+
 module.exports = {
   upsert,
   insertPlayer,
@@ -1720,4 +1734,5 @@ module.exports = {
   getMatchRankTier,
   getMatchData,
   getPlayerMatchData,
+  getArchivedMatch,
 };