From 363a9a06f0cebee21990060384b02c86b8d71037 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Thu, 30 Nov 2023 04:13:41 +0000 Subject: [PATCH] delete old skill related code --- dev/export.js | 112 ---------------------------------------- dev/skillToCassandra.js | 35 ------------- svc/skill.js | 89 ------------------------------- util/utility.js | 8 --- 4 files changed, 244 deletions(-) delete mode 100644 dev/export.js delete mode 100644 dev/skillToCassandra.js delete mode 100644 svc/skill.js diff --git a/dev/export.js b/dev/export.js deleted file mode 100644 index c7951103f..000000000 --- a/dev/export.js +++ /dev/null @@ -1,112 +0,0 @@ -/* -let db = require('../db'), - fs = require('fs'), - zlib = require('zlib'), - moment = require('moment'), - path = require('path'), - JSONStream = require('JSONStream'); - -console.log(process.argv.length); -if (process.argv.length != 4) { - console.log('Not enough arguments'); - process.exit(); -} - -const filePath = process.argv[2]; -const startingMatchId = process.argv[3]; - -try { - var stat = fs.statSync(filePath); - if (!stat.isDirectory) { - console.log('Not a valid directory'); - process.exit(1); - } -} catch (e) { - console.log(e); - process.exit(1); -} - -const fileName = path.join(filePath, `dump-${moment().format('YYYY-MM-DD')}.json.gz`); - -try { - var stat = fs.statSync(fileName); - - if (stat.isFile()) { - console.log('Export file already exists'); - process.exit(1); - } -} catch (e) { - if (e.code !== 'ENOENT') { - process.exit(1); - } -} - -let count = 0, - max = 500000, - jsstream = JSONStream.stringify(), - gzip = zlib.createGzip(), - write = fs.createWriteStream(fileName); - -jsstream.pipe(gzip).pipe(write); - -console.log(`Exporting parsed matches since match_id ${startingMatchId}`); - -const stream = db.select('*') - .from('matches') - .leftJoin('match_skill', 'matches.match_id', 'match_skill.match_id') - .where('version', '>', 0) - .where('matches.match_id', '>', startingMatchId) - .orderBy('matches.match_id', 'desc') - .stream(); - -stream.on('data', (match) => { - stream.pause(); - db.select() - .from('player_matches') - .where({ - 'player_matches.match_id': Number(match.match_id), - }) - .orderBy('player_slot', 'asc') - .asCallback((err, players) => { - if (err) { - console.log(err); - stream.resume(); - return; - } - - count++; - delete match.pgroup; - delete match.url; - players.forEach((p) => { - delete p.match_id; - }); - - match.players = players; - - jsstream.write(match); - stream.resume(); - - if (count % 10000 === 0) { - console.log('Exported %s, matchID %s', count, match.match_id); - } - - if (count > max) { - stream.end(); - } - }); -}); - -stream.on('end', () => { - jsstream.end(); -}); - -jsstream.on('end', () => { - gzip.end(); -}); - -gzip.on('end', () => { - write.end(); - console.log('Done. Exported %s', count); - process.exit(); -}); -*/ diff --git a/dev/skillToCassandra.js b/dev/skillToCassandra.js deleted file mode 100644 index a09c2cec4..000000000 --- a/dev/skillToCassandra.js +++ /dev/null @@ -1,35 +0,0 @@ -const JSONStream = require("JSONStream"); -const db = require("../store/db"); -const queries = require("../store/queries"); - -const { insertMatchSkillCassandra } = queries; -const args = process.argv.slice(2); -const startId = Number(args[0]) || 0; -// var end_id = Number(args[1]) || Number.MAX_VALUE; - -function done(err) { - if (err) { - console.error(err); - } - console.log("done!"); - process.exit(Number(err)); -} - -const stream = db - .select() - .from("match_skill") - .where("match_id", ">=", startId) - .orderBy("match_id", "asc") - .stream(); -stream.on("end", done); -stream.pipe(JSONStream.parse()); -stream.on("data", (m) => { - stream.pause(); - insertMatchSkillCassandra(m, (err) => { - if (err) { - throw err; - } - console.log(m.match_id); - return stream.resume(); - }); -}); diff --git a/svc/skill.js b/svc/skill.js deleted file mode 100644 index 11ea01f92..000000000 --- a/svc/skill.js +++ /dev/null @@ -1,89 +0,0 @@ -/** - * Worker checking the GetMatchHistory endpoint to get skill data for matches - * */ -const constants = require("dotaconstants"); -const async = require("async"); -const config = require("../config"); -const utility = require("../util/utility"); -const queries = require("../store/queries"); - -const apiKeys = config.STEAM_API_KEY.split(","); -const parallelism = Math.min(3, apiKeys.length); -const skills = [1, 2, 3]; -const heroes = Object.keys(constants.heroes); -const permute = []; - -function getPageData(start, options, cb) { - const container = utility.generateJob("api_skill", { - skill: options.skill, - hero_id: options.hero_id, - start_at_match_id: start, - }); - utility.getData( - { - url: container.url, - }, - (err, data) => { - if (err) { - return cb(err); - } - if (!data || !data.result || !data.result.matches) { - return getPageData(start, options, cb); - } - // data is in data.result.matches - const { matches } = data.result; - return async.eachSeries( - matches, - (m, cb) => { - cb(); - // insertMatchSkillCassandra({ - // match_id: m.match_id, - // skill: options.skill, - // players: m.players, - // }, cb); - }, - (err) => { - if (err) { - return cb(err); - } - // repeat until results_remaining===0 - if (data.result.results_remaining === 0) { - return cb(err); - } - const nextStart = matches[matches.length - 1].match_id - 1; - return getPageData(nextStart, options, cb); - } - ); - } - ); -} - -function scanSkill() { - async.eachLimit( - permute, - parallelism, - (object, cb) => { - // use api_skill - const start = null; - getPageData(start, object, cb); - }, - (err) => { - if (err) { - throw err; - } - return scanSkill(); - } - ); -} - -for (let i = 0; i < heroes.length; i += 1) { - for (let j = 0; j < skills.length; j += 1) { - permute.push({ - skill: skills[j], - hero_id: heroes[i], - }); - } -} -// permute = [{skill:1,hero_id:1}]; -console.log(permute.length); -scanSkill(); diff --git a/util/utility.js b/util/utility.js index 6629f9cfc..49c54e383 100644 --- a/util/utility.js +++ b/util/utility.js @@ -120,14 +120,6 @@ function generateJob(type, payload) { payload, }; }, - api_skill() { - return { - url: `${apiUrl}/IDOTA2Match_570/GetMatchHistory/v0001/?key=${apiKey}&start_at_match_id=${payload.start_at_match_id}&skill=${payload.skill}&hero_id=${payload.hero_id}&min_players=10`, - title: [type, payload.skill].join(), - type: "api", - payload, - }; - }, api_live() { return { url: `${apiUrl}/IDOTA2Match_570/GetLiveLeagueGames/v0001/?key=${apiKey}`,