Skip to content

Commit

Permalink
remove match logs parsing
Browse files Browse the repository at this point in the history
  • Loading branch information
@howardchung
howardchung committed Nov 30, 2023
1 parent 8d6b181 commit 962ae5a
Show file tree
Hide file tree
Showing 6 changed files with 15 additions and 193 deletions.
12 changes: 2 additions & 10 deletions processors/createParsedDataBlob.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ const { Console } = require("console");
const readline = require("readline");
const processAllPlayers = require("./processAllPlayers");
const processTeamfights = require("./processTeamfights");
const processLogParse = require("./processLogParse");
// const processUploadProps = require('../processors/processUploadProps');
const processParsedData = require("./processParsedData");
const processMetadata = require("./processMetadata");
Expand Down Expand Up @@ -82,7 +81,7 @@ function getParseSchema() {
};
}

function createParsedDataBlob(entries, matchId, doLogParse) {
function createParsedDataBlob(entries, matchId) {
const logConsole = new Console(process.stderr);

logConsole.time("metadata");
Expand Down Expand Up @@ -113,19 +112,12 @@ function createParsedDataBlob(entries, matchId, doLogParse) {
parsedData.radiant_gold_adv = ap.radiant_gold_adv;
parsedData.radiant_xp_adv = ap.radiant_xp_adv;

logConsole.time("doLogParse");
if (doLogParse) {
parsedData.logs = processLogParse(entries, meta);
}
logConsole.timeEnd("doLogParse");

return parsedData;
}

const entries = [];
let complete = false;
const matchId = process.argv[2];
const doLogParse = Boolean(process.argv[3]);
const parseStream = readline.createInterface({
input: process.stdin,
});
Expand All @@ -138,7 +130,7 @@ parseStream.on("line", (e) => {
});
parseStream.on("close", () => {
if (complete) {
const parsedData = createParsedDataBlob(entries, matchId, doLogParse);
const parsedData = createParsedDataBlob(entries, matchId);
process.stdout.write(JSON.stringify(parsedData), null, (err) => {
process.exit(Number(err));
});
Expand Down
52 changes: 0 additions & 52 deletions processors/processLogParse.js
View file

This file was deleted.

64 changes: 0 additions & 64 deletions sql/create_tables.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,69 +243,6 @@ CREATE TABLE IF NOT EXISTS notable_players (
locked_until integer
);

CREATE TABLE IF NOT EXISTS match_logs (
match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE,
time int,
type varchar(100),
team smallint,
unit varchar(100),
key varchar(1000),
value int,
slot smallint,
player_slot smallint,
player1 int,
player2 int,
attackerhero boolean,
targethero boolean,
attackerillusion boolean,
targetillusion boolean,
inflictor varchar(100),
gold_reason smallint,
xp_reason smallint,
attackername varchar(100),
targetname varchar(100),
sourcename varchar(100),
targetsourcename varchar(100),
valuename varchar(100),
gold int,
lh int,
xp int,
x smallint,
y smallint,
z smallint,
entityleft boolean,
ehandle int,
stuns real,
hero_id smallint,
life_state smallint,
level smallint,
kills smallint,
deaths smallint,
assists smallint,
denies smallint,
attackername_slot smallint,
targetname_slot smallint,
sourcename_slot smallint,
targetsourcename_slot smallint,
player1_slot smallint,
obs_placed int,
sen_placed int,
creeps_stacked int,
camps_stacked int,
rune_pickups int
);
CREATE INDEX IF NOT EXISTS match_logs_match_id_idx ON match_logs(match_id);
CREATE INDEX IF NOT EXISTS match_logs_match_id_player_slot_idx ON match_logs(match_id, player_slot) WHERE player_slot IS NOT NULL;
CREATE INDEX IF NOT EXISTS match_logs_match_id_player1_slot_idx ON match_logs(match_id, player1_slot) WHERE player1_slot IS NOT NULL;
CREATE INDEX IF NOT EXISTS match_logs_match_id_attackername_slot_idx ON match_logs(match_id, attackername_slot) WHERE attackername_slot IS NOT NULL;
CREATE INDEX IF NOT EXISTS match_logs_match_id_targetname_slot_idx ON match_logs(match_id, targetname_slot) WHERE targetname_slot IS NOT NULL;
CREATE INDEX IF NOT EXISTS match_logs_match_id_sourcename_slot_idx ON match_logs(match_id, sourcename_slot) WHERE sourcename_slot IS NOT NULL;
CREATE INDEX IF NOT EXISTS match_logs_match_id_targetsourcename_slot_idx ON match_logs(match_id, targetsourcename_slot) WHERE targetsourcename_slot IS NOT NULL;
CREATE INDEX IF NOT EXISTS match_logs_match_id_valuename_idx ON match_logs(match_id, valuename) WHERE valuename IS NOT NULL;
CREATE INDEX IF NOT EXISTS match_logs_match_id_type_idx ON match_logs(match_id, type);
CREATE INDEX IF NOT EXISTS match_logs_valuename_idx ON match_logs(valuename) WHERE valuename IS NOT NULL;
CREATE INDEX IF NOT EXISTS match_logs_type_idx ON match_logs(type);

CREATE TABLE IF NOT EXISTS picks_bans(
match_id bigint REFERENCES matches(match_id) ON DELETE CASCADE,
is_pick boolean,
Expand Down Expand Up @@ -526,7 +463,6 @@ BEGIN
GRANT SELECT ON team_match TO readonly;
GRANT SELECT ON match_patch TO readonly;
GRANT SELECT ON picks_bans TO readonly;
GRANT SELECT ON match_logs TO readonly;
GRANT SELECT ON notable_players TO readonly;
GRANT SELECT ON public_matches TO readonly;
GRANT SELECT ON public_player_matches TO readonly;
Expand Down
70 changes: 10 additions & 60 deletions store/queries.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1064,30 +1064,7 @@ function insertMatch(match, options, cb) {
cb();
}

function decideLogParse(cb) {
if (match.leagueid) {
db.select("leagueid")
.from("leagues")
.where("tier", "premium")
.orWhere("tier", "professional")
.asCallback((err, leagueids) => {
if (err) {
return cb(err);
}
options.doLogParse =
options.doLogParse ||
utility.isProMatch(
match,
leagueids.map((l) => l.leagueid)
);
return cb(err);
});
} else {
cb();
}
}

function updateMatchGcData(cb) {
function updateMatchSeriesType(cb) {
if (options.type === "gcdata") {
db.raw(
"UPDATE matches SET series_id = ?, series_type = ? WHERE match_id = ?",
Expand Down Expand Up @@ -1115,9 +1092,12 @@ function insertMatch(match, options, cb) {
return cb();
}

function upsertMatch(cb) {
if (!options.doLogParse) {
// Skip this if not a pro match (doLogParse true) and not inserting gcdata (series_id/type)
async function upsertMatchPostgres(cb) {
// Check if leagueid is premium/professional
const result = match.leagueid && await db.raw(`select leagueid from leagues where leagueid = ? and (tier = 'premium' OR tier = 'professional')`, [match.leagueid]);
const pass = result?.rows?.length > 0 && utility.isProMatch(match);
if (!pass) {
// Skip this if not a pro match
return cb();
}
// console.log('[INSERTMATCH] upserting into Postgres');
Expand Down Expand Up @@ -1243,32 +1223,6 @@ function insertMatch(match, options, cb) {
return updateTeamRankings(match, options).then(cb).catch(cb);
}

function upsertMatchLogs(cb) {
if (!match.logs) {
return cb();
}
return trx
.raw("DELETE FROM match_logs WHERE match_id = ?", [match.match_id])
.asCallback((err) => {
if (err) {
return cb(err);
}
return async.eachLimit(
match.logs,
10,
(e, cb) => {
cleanRowPostgres(db, "match_logs", e, (err, cleanedRow) => {
if (err) {
return cb(err);
}
return trx("match_logs").insert(cleanedRow).asCallback(cb);
});
},
cb
);
});
}

function exit(err) {
if (err) {
console.error(err);
Expand All @@ -1287,7 +1241,6 @@ function insertMatch(match, options, cb) {
upsertMatchPatch,
upsertTeamMatch,
upsertTeamRankings,
upsertMatchLogs,
},

exit
Expand Down Expand Up @@ -1555,8 +1508,7 @@ function insertMatch(match, options, cb) {
if (err) {
return cb(err);
}
const { doLogParse } = options;
const doParse = hasTrackedPlayer || options.forceParse || doLogParse;
const doParse = hasTrackedPlayer || options.forceParse;
if (doParse) {
let priority = options.priority;
if (match.leagueid) {
Expand All @@ -1576,7 +1528,6 @@ function insertMatch(match, options, cb) {
duration: match.duration,
replay_blob_key: match.replay_blob_key,
pgroup: match.pgroup,
doLogParse,
ability_upgrades: abilityUpgrades,
allowBackup: options.allowBackup,
origin: options.origin,
Expand All @@ -1596,9 +1547,8 @@ function insertMatch(match, options, cb) {
async.series(
{
preprocess,
decideLogParse,
updateMatchGcData,
upsertMatch,
updateMatchSeriesType,
upsertMatchPostgres: (cb) => upsertMatchPostgres(cb),
getAverageRank,
upsertMatchCassandra,
upsertParsedMatch,
Expand Down
5 changes: 1 addition & 4 deletions svc/parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,7 @@ function runParse(match, job, cb) {
url && url.slice(-3) === "bz2" ? "bunzip2" : "cat"
} | curl -X POST -T - ${
config.PARSER_HOST
} | node processors/createParsedDataBlob.js ${match.match_id} ${Boolean(
match.doLogParse
)}`,
} | node processors/createParsedDataBlob.js ${match.match_id}`,
{ shell: true, maxBuffer: 10 * 1024 * 1024 },
(err, stdout) => {
if (err) {
Expand All @@ -49,7 +47,6 @@ function runParse(match, job, cb) {
{
type: "parsed",
skipParse: true,
doLogParse: match.doLogParse,
},
cb
);
Expand Down
5 changes: 2 additions & 3 deletions util/utility.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -398,7 +398,7 @@ function isSignificant(match) {
/**
* Determines if a match is a pro match
* */
function isProMatch(match, leagueids) {
function isProMatch(match) {
return Boolean(
isSignificant(match) &&
match.leagueid &&
Expand All @@ -409,8 +409,7 @@ function isProMatch(match, leagueids) {
match.players &&
match.players.every((player) => player.level > 1) &&
match.players.every((player) => player.xp_per_min > 0) &&
match.players.every((player) => player.hero_id > 0) &&
leagueids.includes(match.leagueid)
match.players.every((player) => player.hero_id > 0)
);
}

Expand Down

0 comments on commit 962ae5a

Please sign in to comment.