From 962ae5a9538cb9e7c7b5243584ca2c17de79c94b Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 30 Nov 2023 18:17:03 +0000 Subject: [PATCH] remove match logs parsing --- processors/createParsedDataBlob.js | 12 +---- processors/processLogParse.js | 52 ---------------------- sql/create_tables.sql | 64 --------------------------- store/queries.js | 70 +++++------------------------- svc/parser.js | 5 +-- util/utility.js | 5 +-- 6 files changed, 15 insertions(+), 193 deletions(-) delete mode 100644 processors/processLogParse.js diff --git a/processors/createParsedDataBlob.js b/processors/createParsedDataBlob.js index 61db7dcd9..5dce0fb2e 100644 --- a/processors/createParsedDataBlob.js +++ b/processors/createParsedDataBlob.js @@ -2,7 +2,6 @@ const { Console } = require("console"); const readline = require("readline"); const processAllPlayers = require("./processAllPlayers"); const processTeamfights = require("./processTeamfights"); -const processLogParse = require("./processLogParse"); // const processUploadProps = require('../processors/processUploadProps'); const processParsedData = require("./processParsedData"); const processMetadata = require("./processMetadata"); @@ -82,7 +81,7 @@ function getParseSchema() { }; } -function createParsedDataBlob(entries, matchId, doLogParse) { +function createParsedDataBlob(entries, matchId) { const logConsole = new Console(process.stderr); logConsole.time("metadata"); @@ -113,19 +112,12 @@ function createParsedDataBlob(entries, matchId, doLogParse) { parsedData.radiant_gold_adv = ap.radiant_gold_adv; parsedData.radiant_xp_adv = ap.radiant_xp_adv; - logConsole.time("doLogParse"); - if (doLogParse) { - parsedData.logs = processLogParse(entries, meta); - } - logConsole.timeEnd("doLogParse"); - return parsedData; } const entries = []; let complete = false; const matchId = process.argv[2]; -const doLogParse = Boolean(process.argv[3]); const parseStream = readline.createInterface({ input: process.stdin, }); @@ -138,7 +130,7 @@ parseStream.on("line", (e) => { }); parseStream.on("close", () => { if (complete) { - const parsedData = createParsedDataBlob(entries, matchId, doLogParse); + const parsedData = createParsedDataBlob(entries, matchId); process.stdout.write(JSON.stringify(parsedData), null, (err) => { process.exit(Number(err)); }); diff --git a/processors/processLogParse.js b/processors/processLogParse.js deleted file mode 100644 index 7a5016b51..000000000 --- a/processors/processLogParse.js +++ /dev/null @@ -1,52 +0,0 @@ -const insignificantDeaths = ["npc_dota_creep", "npc_dota_neutral"]; - -function translate(s) { - return s === "dota_unknown" ? null : s; -} - -/** - * A processor to reduce the event stream to only logs we want to persist - * */ -function processReduce(entries, meta) { - const result = entries - .filter((e) => { - if ( - e.type === "DOTA_COMBATLOG_PURCHASE" || - (e.type === "DOTA_COMBATLOG_DEATH" && - insignificantDeaths.every( - (prefix) => e.targetname.indexOf(prefix) !== 0 - )) - ) { - return Boolean(e.time); - } - return false; - }) - .map((e) => { - const e2 = { - ...e, - match_id: meta.match_id, - attackername_slot: - meta.slot_to_playerslot[meta.hero_to_slot[e.attackername]], - targetname_slot: - meta.slot_to_playerslot[meta.hero_to_slot[e.targetname]], - sourcename_slot: - meta.slot_to_playerslot[meta.hero_to_slot[e.sourcename]], - targetsourcename_slot: - meta.slot_to_playerslot[meta.hero_to_slot[e.targetname]], - player1_slot: meta.slot_to_playerslot[e.player1], - player_slot: e.player_slot || meta.slot_to_playerslot[e.slot], - inflictor: translate(e.inflictor), - }; - return e2; - }); - /* - var count = {}; - result.forEach(function(r) - { - count[r.type] = (count[r.type] || 0) + 1; - }); - console.log(count); - */ - return result; -} -module.exports = processReduce; diff --git a/sql/create_tables.sql b/sql/create_tables.sql index ec93ab86f..1eea2d956 100644 --- a/sql/create_tables.sql +++ b/sql/create_tables.sql @@ -243,69 +243,6 @@ CREATE TABLE IF NOT EXISTS notable_players ( locked_until integer ); -CREATE TABLE IF NOT EXISTS match_logs ( - match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE, - time int, - type varchar(100), - team smallint, - unit varchar(100), - key varchar(1000), - value int, - slot smallint, - player_slot smallint, - player1 int, - player2 int, - attackerhero boolean, - targethero boolean, - attackerillusion boolean, - targetillusion boolean, - inflictor varchar(100), - gold_reason smallint, - xp_reason smallint, - attackername varchar(100), - targetname varchar(100), - sourcename varchar(100), - targetsourcename varchar(100), - valuename varchar(100), - gold int, - lh int, - xp int, - x smallint, - y smallint, - z smallint, - entityleft boolean, - ehandle int, - stuns real, - hero_id smallint, - life_state smallint, - level smallint, - kills smallint, - deaths smallint, - assists smallint, - denies smallint, - attackername_slot smallint, - targetname_slot smallint, - sourcename_slot smallint, - targetsourcename_slot smallint, - player1_slot smallint, - obs_placed int, - sen_placed int, - creeps_stacked int, - camps_stacked int, - rune_pickups int -); -CREATE INDEX IF NOT EXISTS match_logs_match_id_idx ON match_logs(match_id); -CREATE INDEX IF NOT EXISTS match_logs_match_id_player_slot_idx ON match_logs(match_id, player_slot) WHERE player_slot IS NOT NULL; -CREATE INDEX IF NOT EXISTS match_logs_match_id_player1_slot_idx ON match_logs(match_id, player1_slot) WHERE player1_slot IS NOT NULL; -CREATE INDEX IF NOT EXISTS match_logs_match_id_attackername_slot_idx ON match_logs(match_id, attackername_slot) WHERE attackername_slot IS NOT NULL; -CREATE INDEX IF NOT EXISTS match_logs_match_id_targetname_slot_idx ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL; -CREATE INDEX IF NOT EXISTS match_logs_match_id_sourcename_slot_idx ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL; -CREATE INDEX IF NOT EXISTS match_logs_match_id_targetsourcename_slot_idx ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL; -CREATE INDEX IF NOT EXISTS match_logs_match_id_valuename_idx ON match_logs(match_id, valuename) WHERE valuename IS NOT NULL; -CREATE INDEX IF NOT EXISTS match_logs_match_id_type_idx ON match_logs(match_id, type); -CREATE INDEX IF NOT EXISTS match_logs_valuename_idx ON match_logs(valuename) WHERE valuename IS NOT NULL; -CREATE INDEX IF NOT EXISTS match_logs_type_idx ON match_logs(type); - CREATE TABLE IF NOT EXISTS picks_bans( match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE, is_pick boolean, @@ -526,7 +463,6 @@ BEGIN GRANT SELECT ON team_match TO readonly; GRANT SELECT ON match_patch TO readonly; GRANT SELECT ON picks_bans TO readonly; - GRANT SELECT ON match_logs TO readonly; GRANT SELECT ON notable_players TO readonly; GRANT SELECT ON public_matches TO readonly; GRANT SELECT ON public_player_matches TO readonly; diff --git a/store/queries.js b/store/queries.js index 2e4dd84f6..6ce425995 100644 --- a/store/queries.js +++ b/store/queries.js @@ -1064,30 +1064,7 @@ function insertMatch(match, options, cb) { cb(); } - function decideLogParse(cb) { - if (match.leagueid) { - db.select("leagueid") - .from("leagues") - .where("tier", "premium") - .orWhere("tier", "professional") - .asCallback((err, leagueids) => { - if (err) { - return cb(err); - } - options.doLogParse = - options.doLogParse || - utility.isProMatch( - match, - leagueids.map((l) => l.leagueid) - ); - return cb(err); - }); - } else { - cb(); - } - } - - function updateMatchGcData(cb) { + function updateMatchSeriesType(cb) { if (options.type === "gcdata") { db.raw( "UPDATE matches SET series_id = ?, series_type = ? WHERE match_id = ?", @@ -1115,9 +1092,12 @@ function insertMatch(match, options, cb) { return cb(); } - function upsertMatch(cb) { - if (!options.doLogParse) { - // Skip this if not a pro match (doLogParse true) and not inserting gcdata (series_id/type) + async function upsertMatchPostgres(cb) { + // Check if leagueid is premium/professional + const result = match.leagueid && await db.raw(`select leagueid from leagues where leagueid = ? and (tier = 'premium' OR tier = 'professional')`, [match.leagueid]); + const pass = result?.rows?.length > 0 && utility.isProMatch(match); + if (!pass) { + // Skip this if not a pro match return cb(); } // console.log('[INSERTMATCH] upserting into Postgres'); @@ -1243,32 +1223,6 @@ function insertMatch(match, options, cb) { return updateTeamRankings(match, options).then(cb).catch(cb); } - function upsertMatchLogs(cb) { - if (!match.logs) { - return cb(); - } - return trx - .raw("DELETE FROM match_logs WHERE match_id = ?", [match.match_id]) - .asCallback((err) => { - if (err) { - return cb(err); - } - return async.eachLimit( - match.logs, - 10, - (e, cb) => { - cleanRowPostgres(db, "match_logs", e, (err, cleanedRow) => { - if (err) { - return cb(err); - } - return trx("match_logs").insert(cleanedRow).asCallback(cb); - }); - }, - cb - ); - }); - } - function exit(err) { if (err) { console.error(err); @@ -1287,7 +1241,6 @@ function insertMatch(match, options, cb) { upsertMatchPatch, upsertTeamMatch, upsertTeamRankings, - upsertMatchLogs, }, exit @@ -1555,8 +1508,7 @@ function insertMatch(match, options, cb) { if (err) { return cb(err); } - const { doLogParse } = options; - const doParse = hasTrackedPlayer || options.forceParse || doLogParse; + const doParse = hasTrackedPlayer || options.forceParse; if (doParse) { let priority = options.priority; if (match.leagueid) { @@ -1576,7 +1528,6 @@ function insertMatch(match, options, cb) { duration: match.duration, replay_blob_key: match.replay_blob_key, pgroup: match.pgroup, - doLogParse, ability_upgrades: abilityUpgrades, allowBackup: options.allowBackup, origin: options.origin, @@ -1596,9 +1547,8 @@ function insertMatch(match, options, cb) { async.series( { preprocess, - decideLogParse, - updateMatchGcData, - upsertMatch, + updateMatchSeriesType, + upsertMatchPostgres: (cb) => upsertMatchPostgres(cb), getAverageRank, upsertMatchCassandra, upsertParsedMatch, diff --git a/svc/parser.js b/svc/parser.js index 03ac0e644..63d54cf82 100755 --- a/svc/parser.js +++ b/svc/parser.js @@ -35,9 +35,7 @@ function runParse(match, job, cb) { url && url.slice(-3) === "bz2" ? "bunzip2" : "cat" } | curl -X POST -T - ${ config.PARSER_HOST - } | node processors/createParsedDataBlob.js ${match.match_id} ${Boolean( - match.doLogParse - )}`, + } | node processors/createParsedDataBlob.js ${match.match_id}`, { shell: true, maxBuffer: 10 * 1024 * 1024 }, (err, stdout) => { if (err) { @@ -49,7 +47,6 @@ function runParse(match, job, cb) { { type: "parsed", skipParse: true, - doLogParse: match.doLogParse, }, cb ); diff --git a/util/utility.js b/util/utility.js index 49c54e383..1307ebea8 100644 --- a/util/utility.js +++ b/util/utility.js @@ -398,7 +398,7 @@ function isSignificant(match) { /** * Determines if a match is a pro match * */ -function isProMatch(match, leagueids) { +function isProMatch(match) { return Boolean( isSignificant(match) && match.leagueid && @@ -409,8 +409,7 @@ function isProMatch(match, leagueids) { match.players && match.players.every((player) => player.level > 1) && match.players.every((player) => player.xp_per_min > 0) && - match.players.every((player) => player.hero_id > 0) && - leagueids.includes(match.leagueid) + match.players.every((player) => player.hero_id > 0) ); }