lay out upsertmatchblobs intention

config.js

@@ -55,7 +55,7 @@ const defaults = {
   SCENARIOS_SAMPLE_PERCENT: 100, // percent of parsed matches to sample for scenarios
   BENCHMARKS_SAMPLE_PERCENT: 100, // percent of parsed matches to sample for benchmarks
   ENABLE_MATCH_CACHE: '', // set to enable caching matches in Redis
-  ENABLE_PLAYER_CACHE: 1, // enable/disable player aggregation caching
+  ENABLE_PLAYER_CACHE: '', // enable/disable player aggregation caching
   ENABLE_RANDOM_MMR_UPDATE: '', // set to request MMR updates after ranked matches
   MAXIMUM_AGE_SCENARIOS_ROWS: 4, // maximum allowed age of scenarios rows in weeks
   MATCH_CACHE_SECONDS: 60, // number of seconds to cache matches

processors/processUploadProps.js

@@ -15,7 +15,7 @@ function processUploadProps(entries) {
         container.match_id = dota.matchId_;
         container.game_mode = dota.gameMode_;
         container.radiant_win = dota.gameWinner_ === 2;
-        // TODO following needs some extraction/transformation
+        // NOTE: following needs some extraction/transformation
         // container.picks_bans = dota.picksBans_;
         // require('fs').writeFileSync('./outputEpilogue.json', JSON.stringify(JSON.parse(e.key)));
         break;

sql/create_tables.sql

@@ -454,7 +454,7 @@ CREATE TABLE IF NOT EXISTS subscriber (
 CREATE TABLE IF NOT EXISTS match_blobs (
   PRIMARY KEY (match_id),
   match_id bigint,
-  basic json,
+  api json,
   gcdata json,
   replay json
 );

store/buildMatch.js

@@ -139,7 +139,7 @@ async function getMatch(matchId) {
       playersMatchData = await getPlayerMatchData(matchId);
     }
   } catch (e) {
-    // TODO we can probably remove this try/catch after bad data is fixed
+    // TODO (howard) we can probably remove this try/catch after bad data is fixed
     console.error(e);
     if (
       e.message.startsWith('Unexpected') ||

store/queries.js

@@ -1340,7 +1340,16 @@ function insertMatch(match, options, cb) {
     });
   }
 
-  function updatePlayerCaches(cb) {
+  function upsertMatchBlobs(cb) {
+    // TODO (howard) this function is meant to eventually replace the cassandra match/player_match tables
+    // It's a temporary store (postgres table) holding data for each possible stage of ingestion, api/gcdata/replay/meta etc.
+    // We store a match blob in the row for each stage
+    // in buildMatch we can assemble the data from all these pieces
+    // After some retention period we stick the data in match archive and delete it
+    cb();
+  }
+
+  function updateCassandraPlayerCaches(cb) {
     // console.log('[INSERTMATCH] upserting into Cassandra player_caches');
     const copy = createMatchCopy(match, players);
     return insertPlayerCache(copy, cb);
@@ -1372,11 +1381,11 @@ function insertMatch(match, options, cb) {
     return cb();
   }
 
-  function clearMatchCache(cb) {
+  function clearRedisMatch(cb) {
     redis.del(`match:${match.match_id}`, cb);
   }
 
-  function clearPlayerCaches(cb) {
+  function clearRedisPlayer(cb) {
     async.each(
       (match.players || []).filter((player) => Boolean(player.account_id)),
       (player, cb) => {
@@ -1557,10 +1566,11 @@ function insertMatch(match, options, cb) {
       upsertMatchPostgres: (cb) => upsertMatchPostgres(cb),
       getAverageRank,
       upsertMatchCassandra,
+      upsertMatchBlobs,
       upsertParsedMatch,
-      updatePlayerCaches,
-      clearMatchCache,
-      clearPlayerCaches,
+      updateCassandraPlayerCaches,
+      clearRedisMatch,
+      clearRedisPlayer,
       telemetry,
       decideCounts,
       decideScenarios,

svc/cassandraDelete.js

@@ -73,7 +73,7 @@ async function start() {
       config.MATCH_ARCHIVE_S3_ENDPOINT &&
         (await Promise.all(parsedIds.map((id) => doArchive(id))));
 
-      // TODO remove insert once backfill complete
+      // TODO (howard) remove insert once backfill complete
       await Promise.all(
         parsedIds.map((id) =>
           db.raw(
@@ -97,7 +97,7 @@ async function doArchive(matchId) {
   );
   const result = await archivePut(matchId.toString(), blob);
   if (result) {
-    // TODO Delete from Cassandra after archival
+    // TODO (howard) Delete from Cassandra after archival
     // await cassandra.execute(
     //   "DELETE from player_matches where match_id = ?",
     //   [matchId],

test/test.js

@@ -371,7 +371,7 @@ describe('teamRanking', () => {
       });
   });
 });
-// TODO test against an unparsed match to catch exceptions caused by code expecting parsed data
+// TODO also test on unparsed match to catch exceptions caused by code expecting parsed data
 describe('api', () => {
   it('should get API spec', function testAPISpec(cb) {
     this.timeout(5000);

util/getGcData.js

@@ -42,7 +42,7 @@ async function getGcDataFromRetriever(match, cb) {
         'retrieverCounts',
         moment().startOf('hour').add(1, 'hour').format('X')
       );
-      // TODO add discovered account_ids to database and fetch account data/rank medal
+      // TODO (howard) add discovered account_ids to database and fetch account data/rank medal
       try {
         const players = body.match.players.map((p, i) => ({
           player_slot: p.player_slot,