From fbd4b88bc9370cc2b95f18a9df06bf4e955dfb46 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sun, 29 Sep 2024 00:33:04 +0200 Subject: [PATCH 01/19] package-lock.json update --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 455223f..ac2015a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "screeps-grafana-go_carbon", - "version": "1.0.5", + "version": "1.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "screeps-grafana-go_carbon", - "version": "1.0.5", + "version": "1.1.0", "dependencies": { "axios": "^0.27.2", "dotenv": "^16.0.2", From de341b29e8e9e17474bd25f9725ceff7a7648102 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Fri, 27 Sep 2024 22:42:54 +0200 Subject: [PATCH 02/19] Fix push status port I missed the port being hardcoded in the stat-pusher. This makes it properly go through the environment. The setup harness will auto-use 10004 if --pushStatusPort is set, otherwise set it to the specified value. --- README.md | 3 ++- bin/server.js | 10 ++++++++++ docker-compose.example.yml | 2 +- src/pushStats/index.js | 8 ++++---- src/setup/setup.js | 4 ++-- 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 7f7ed8e..36eb69f 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,8 @@ Update all .example files and/or folders to match your needs. This step is not r * `--grafanaDomain`: Overwrite grafana.ini domain * `--grafanaPort`: port for Grafana to run on * `--relayPort`: port for relay-ng to run on (default: 2003) -* `--pushStatusPort`: port for the stats-getter push API (default: disabled) +* `--pushStatusPort`: port for the stats-getter push API (default: false) + true will set it to 10004, otherwise specify a port number it'll listen to #### Exporting diff --git a/bin/server.js b/bin/server.js index 2293d88..881a82b 100644 --- a/bin/server.js +++ b/bin/server.js @@ -34,6 +34,16 @@ const logger = createLogger({ async function main() { argv.grafanaPort = argv.grafanaPort ?? await getPort({ portRange: [3000, 4000] }); argv.serverPort = argv.serverPort ?? 21025; + if (argv.pushStatusPort === true) { + argv.pushStatusPort = 10004; + } else { + const port = Number(argv.pushStatusPort); + if (!Number.isNaN(port)) { + argv.pushStatusPort = port; + } else { + delete argv.pushStatusPort; + } + } const cli = { cmd: argv._.shift(), diff --git a/docker-compose.example.yml b/docker-compose.example.yml index 54bce82..310074b 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -49,7 +49,7 @@ services: environment: - PREFIX= - SERVER_PORT=21025 - - INCLUDE_PUSH_STATUS_API=false + - PUSH_STATUS_PORT= networks: - stats logging: *default-logging \ No newline at end of file diff --git a/src/pushStats/index.js b/src/pushStats/index.js index 06d20b8..f328a99 100644 --- a/src/pushStats/index.js +++ b/src/pushStats/index.js @@ -12,7 +12,7 @@ import express from 'express'; import ApiFunc from './apiFunctions.js'; const app = express(); -const port = 10004; +const pushStatusPort = Number(process.env.PUSH_STATUS_PORT); let lastUpload = new Date().getTime(); const users = JSON.parse(fs.readFileSync('users.json')); @@ -233,9 +233,9 @@ cron.schedule('*/30 * * * * *', async () => { }); }); -if (process.env.INCLUDE_PUSH_STATUS_API === 'true') { - app.listen(port, () => { - console.log(`App listening at http://localhost:${port}`); +if (pushStatusPort) { + app.listen(pushStatusPort, () => { + console.log(`App listening at http://localhost:${pushStatusPort}`); }); app.get('/', (req, res) => { const diffCompleteMinutes = Math.ceil( diff --git a/src/setup/setup.js b/src/setup/setup.js index ef8d682..540c25a 100644 --- a/src/setup/setup.js +++ b/src/setup/setup.js @@ -57,8 +57,8 @@ async function UpdateDockerComposeFile() { } if (argv.pushStatusPort) { contents = contents.replace( - 'INCLUDE_PUSH_STATUS_API=false', - `INCLUDE_PUSH_STATUS_API=true${regexEscape} ports:${regexEscape} - ${argv.pushStatusPort}:${argv.pushStatusPort}`, + 'PUSH_STATUS_PORT=', + `PUSH_STATUS_PORT=${argv.pushStatusPort}${regexEscape} ports:${regexEscape} - ${argv.pushStatusPort}:${argv.pushStatusPort}`, ); } if (argv.prefix) { From a6f8bd66643aedb64d7d7196fa4dfbc10c2ec7df Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Fri, 27 Sep 2024 23:11:39 +0200 Subject: [PATCH 03/19] Make the users.json file permanent Instead of having the setup potentially copy over the example user file, just ask the user to set it up himself. This stops mistakes from happening with `--force`. Also mount the file into the container instead of copying it, so that an update just needs a container restart and not a rebuild. Hence, remove the rebuilding from the start script and add it as a developer-intended command to the npm scripts (since the only reason to rebuild would be if the pushStats scripts is edited). --- README.md | 23 +++++++++++++++++++++-- docker-compose.example.yml | 1 + package.json | 1 + src/pushStats/Dockerfile | 3 +-- src/setup/setup.js | 20 ++++++-------------- 5 files changed, 30 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 36eb69f..9002bb9 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,27 @@ ## Setup -1. Update all .example files and/or folders to match your needs. This step is not required if you are using the default setup. -2. Add your own Grafana variables in `grafanaConfig/.env.grafana`. This file will be updated after a volume reset. +1. Edit `example.env` and `docker-compose.example.yml` to match your needs. This step is not required if you are using the default setup. +2. Copy `users.example.json` to `users.json` and edit it according to [User Setup](#User-Setup). +3. The configuration files for both Grafana and Graphite are in `config/grafana` and `config/graphite` respectively. +4. If you have a dashboard you want to auto-add, you can drop their JSON files into `config/grafana/provisioning/dashboards` +and they'll be auto-added to the instance. + +## Usage + +* `npm run start`: start the containers +* `npm run logs`: check the container's logs +* `npm run stop`: stop the containers +* `npm run reset`: remove the containers +* `npm run reset:hard`: remove the containers and their volumes +* `npm run rebuild`: rebuild the pushStats container and restart it; needed if you make changes to its code. + +See the scripts section in the package.json file. + +Go to [localhost:3000](http://localhost:3000) (if you used port 3000) and login with `admin` and `password` (or your custom set login info). + +Its possible to use https for your grafana instance, check out this [tutorial](https://www.turbogeek.co.uk/grafana-how-to-configure-ssl-https-in-grafana/) for example on how to do this, enough info online about it. I dont support this (yet) + ### User Setup diff --git a/docker-compose.example.yml b/docker-compose.example.yml index 310074b..a2a461e 100644 --- a/docker-compose.example.yml +++ b/docker-compose.example.yml @@ -44,6 +44,7 @@ services: dockerfile: ./src/pushStats/Dockerfile volumes: - ./logs/statsGetter:/app/logs + - ./users.json:/app/users.json depends_on: - graphite environment: diff --git a/package.json b/package.json index 3af7bab..ef4d089 100644 --- a/package.json +++ b/package.json @@ -5,6 +5,7 @@ "setup": "node bin/server.js setup", "start": "node bin/server.js start --grafanaPort=3000", "start:test": "node bin/server.js start --grafanaPort=3000 --force", + "rebuild": "docker compose build --no-cache", "lint": "eslint src/**/*.js && eslint dashboards/**/*.js", "lint:fix": "eslint src/**/*.js --fix && eslint dashboards/**/*.js --fix", "update-stats-getter": "docker-compose up --detach --build" diff --git a/src/pushStats/Dockerfile b/src/pushStats/Dockerfile index 0d43f8d..308f935 100644 --- a/src/pushStats/Dockerfile +++ b/src/pushStats/Dockerfile @@ -6,7 +6,6 @@ ENV NODE_ENV=production COPY ./src/pushStats . -COPY ./users.json ./users.json -RUN npm install +RUN npm clean-install CMD ["node", "index.js"] \ No newline at end of file diff --git a/src/setup/setup.js b/src/setup/setup.js index 540c25a..ee19c9e 100644 --- a/src/setup/setup.js +++ b/src/setup/setup.js @@ -69,18 +69,6 @@ async function UpdateDockerComposeFile() { logger.info('Docker-compose file created'); } -function UpdateUsersFile() { - const usersFile = join(__dirname, '../../users.json'); - if (fs.existsSync(usersFile) && !argv.force) { - return logger.warn('Users file already exists, use --force to overwrite it'); - } - - const exampleUsersFilePath = join(__dirname, '../../users.example.json'); - const exampleUsersText = fs.readFileSync(exampleUsersFilePath, 'utf8'); - fs.writeFileSync(usersFile, exampleUsersText); - logger.info('Users file created'); -} - function UpdateGrafanaConfigFolder() { const configDirPath = join(__dirname, '../../grafanaConfig'); if (fs.existsSync(configDirPath) && !argv.force) { @@ -148,7 +136,12 @@ async function Setup(cli) { argv = cli.args; logger = cli.logger; - UpdateUsersFile(); + const usersFile = join(__dirname, '../../users.json'); + if (!fs.existsSync(usersFile)) { + logger.error('missing users.json file'); + process.exit(-1); + } + UpdateEnvFile(); await UpdateDockerComposeFile(); UpdateGrafanaConfigFolder(); @@ -161,7 +154,6 @@ module.exports.commands = async function Commands(grafanaApiUrl) { const commands = [ { command: `docker compose down ${argv.removeVolumes ? '--volumes' : ''} --remove-orphans`, name: 'docker-compose down' }, - { command: 'docker compose build --no-cache', name: 'docker-compose build' }, { command: 'docker compose up -d', name: 'docker-compose up' }, ]; From fe12a67fac9a15e44e795b1fed120319d71880ab Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Fri, 27 Sep 2024 23:26:44 +0200 Subject: [PATCH 04/19] This doesn't exist anymore --- src/setup/setup.js | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/setup/setup.js b/src/setup/setup.js index ee19c9e..23948a1 100644 --- a/src/setup/setup.js +++ b/src/setup/setup.js @@ -45,15 +45,8 @@ async function UpdateDockerComposeFile() { contents = contents.replace('3000:3000', `${argv.grafanaPort}:${argv.grafanaPort}`); contents = contents.replace('http://localhost:3000/login', `http://localhost:${argv.grafanaPort}/login`); - if (argv.relayPort) { - contents = contents.replace('2003:2003', `${argv.relayPort}:2003`); - } else { - contents = contents.replace(createRegexWithEscape('ports:\r\n - 2003:2003'), ''); - } if (argv.serverPort) { - contents = contents - .replace('http://localhost:21025/web', `http://localhost:${argv.serverPort}/web`) - .replace('SERVER_PORT: 21025', `SERVER_PORT: ${argv.serverPort}`); + contents = contents.replace('SERVER_PORT: 21025', `SERVER_PORT: ${argv.serverPort}`); } if (argv.pushStatusPort) { contents = contents.replace( From c7b4160103421f7ec5d78be03074d1c7db789b0a Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 28 Sep 2024 00:38:27 +0200 Subject: [PATCH 05/19] Remove commented line --- src/pushStats/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pushStats/index.js b/src/pushStats/index.js index f328a99..301b010 100644 --- a/src/pushStats/index.js +++ b/src/pushStats/index.js @@ -203,7 +203,6 @@ class ManageStats { lastUpload = new Date().getTime(); resolve(true); }); - // resolve(true); }); } From 67ee97c42ca7cc12dc585062c07a28b530eeb795 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 28 Sep 2024 01:05:16 +0200 Subject: [PATCH 06/19] This is unused --- package.json | 1 - src/setup/setup.js | 10 ---------- 2 files changed, 11 deletions(-) diff --git a/package.json b/package.json index ef4d089..90c0f92 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,5 @@ "docker-compose.example.yml", "example.env", ".dockerignore", - "go-carbon-storage" ] } diff --git a/src/setup/setup.js b/src/setup/setup.js index 23948a1..d180c30 100644 --- a/src/setup/setup.js +++ b/src/setup/setup.js @@ -106,16 +106,6 @@ function UpdateGrafanaConfigFolder() { } function resetFolders() { - const carbonStoragePath = join(__dirname, '../../go-carbon-storage'); - let carbonStorageExists = fs.existsSync(carbonStoragePath); - if (carbonStorageExists && argv.removeWhisper) { - fs.rmdirSync(carbonStoragePath, { recursive: true }); - carbonStorageExists = false; - } - if (!carbonStorageExists) { - fs.mkdirSync(carbonStoragePath, { recursive: true }); - } - const logsPath = join(__dirname, '../../logs'); let logsExist = fs.existsSync(logsPath); if (logsExist && argv.deleteLogs) { From 1c3240ddc151c72aca8a3d737474903f88ce457c Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 28 Sep 2024 00:38:10 +0200 Subject: [PATCH 07/19] Staticify Compose file This moves everything the setup step did to the `.env` & Compose files into the files themselves, which makes the Compose file fully static, with the `.env` file in charge of carrying the setup. --- .env.example | 12 +++++ .gitignore | 2 - ...-compose.example.yml => docker-compose.yml | 12 ++--- example.env | 4 -- package.json | 6 +-- src/setup/setup.js | 50 ------------------- 6 files changed, 21 insertions(+), 65 deletions(-) create mode 100644 .env.example rename docker-compose.example.yml => docker-compose.yml (80%) delete mode 100644 example.env diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..8e36176 --- /dev/null +++ b/.env.example @@ -0,0 +1,12 @@ +# The port where the Screeps server is listen at +SERVER_PORT=21025 + +# External port where Grafana listens to +GRAFANA_PORT=3000 + +# The port where the Push Status API will listen to +# Optional, if unset, the port won't be opened +PUSH_STATUS_PORT= + +# A prefix to use when pushing keys to Graphite +PREFIX= diff --git a/.gitignore b/.gitignore index 65df0e2..251f6b7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ # Files -docker-compose.yml .env -users.json # Folders node_modules diff --git a/docker-compose.example.yml b/docker-compose.yml similarity index 80% rename from docker-compose.example.yml rename to docker-compose.yml index a2a461e..5314700 100644 --- a/docker-compose.example.yml +++ b/docker-compose.yml @@ -29,9 +29,9 @@ services: - ./grafanaConfig/grafana:/etc/grafana - ./logs/grafana:/var/log/grafana ports: - - 3000:3000 + - ${GRAFANA_PORT}:${GRAFANA_PORT} healthcheck: - test: "curl -fsSL -o /dev/null http://localhost:3000/login" + test: "curl -fsSL -o /dev/null http://localhost:${GRAFANA_PORT}/login" interval: 10s timeout: 5s retries: 3 @@ -48,9 +48,9 @@ services: depends_on: - graphite environment: - - PREFIX= - - SERVER_PORT=21025 - - PUSH_STATUS_PORT= + - PREFIX=${PREFIX} + - SERVER_PORT=${SERVER_PORT} + - PUSH_STATUS_PORT=${PUSH_STATUS_PORT} networks: - stats - logging: *default-logging \ No newline at end of file + logging: *default-logging diff --git a/example.env b/example.env deleted file mode 100644 index 6dccac6..0000000 --- a/example.env +++ /dev/null @@ -1,4 +0,0 @@ -SERVER_PORT=21025 -GRAFANA_PORT=3000 -COMPOSE_PROJECT_NAME=screeps-grafana -COMPOSE_FILE=./docker-compose.yml diff --git a/package.json b/package.json index 90c0f92..672242a 100644 --- a/package.json +++ b/package.json @@ -31,8 +31,8 @@ "dashboards", "grafanaConfig.example", "users.example.json", - "docker-compose.example.yml", - "example.env", - ".dockerignore", + "docker-compose.yml", + "env.example", + ".dockerignore" ] } diff --git a/src/setup/setup.js b/src/setup/setup.js index d180c30..50602f1 100644 --- a/src/setup/setup.js +++ b/src/setup/setup.js @@ -14,54 +14,6 @@ function createRegexWithEscape(string) { return new RegExp(string.replace('\r\n', regexEscape)); } -function UpdateEnvFile() { - const envFile = join(__dirname, '../../.env'); - if (fs.existsSync(envFile) && !argv.force) { - return logger.warn('Env file already exists, use --force to overwrite it'); - } - - const exampleEnvFilePath = join(__dirname, '../../example.env'); - let contents = fs.readFileSync(exampleEnvFilePath, 'utf8'); - contents = contents - .replace('GRAFANA_PORT=3000', `GRAFANA_PORT=${argv.grafanaPort}`) - .replace('COMPOSE_PROJECT_NAME=screeps-grafana', `COMPOSE_PROJECT_NAME=screeps-grafana-${argv.grafanaPort}`) - .replace('COMPOSE_FILE=./docker-compose.yml', `COMPOSE_FILE=${join(__dirname, '../../docker-compose.yml')}`); - if (argv.serverPort) { - contents = contents.replace('SERVER_PORT=21025', `SERVER_PORT=${argv.serverPort}`); - } - - fs.writeFileSync(envFile, contents); - logger.info('Env file created'); -} - -async function UpdateDockerComposeFile() { - const dockerComposeFile = join(__dirname, '../../docker-compose.yml'); - if (fs.existsSync(dockerComposeFile) && !argv.force) { - return logger.warn('Docker-compose file already exists, use --force to overwrite it'); - } - - const exampleDockerComposeFile = join(__dirname, '../../docker-compose.example.yml'); - let contents = fs.readFileSync(exampleDockerComposeFile, 'utf8'); - contents = contents.replace('3000:3000', `${argv.grafanaPort}:${argv.grafanaPort}`); - contents = contents.replace('http://localhost:3000/login', `http://localhost:${argv.grafanaPort}/login`); - - if (argv.serverPort) { - contents = contents.replace('SERVER_PORT: 21025', `SERVER_PORT: ${argv.serverPort}`); - } - if (argv.pushStatusPort) { - contents = contents.replace( - 'PUSH_STATUS_PORT=', - `PUSH_STATUS_PORT=${argv.pushStatusPort}${regexEscape} ports:${regexEscape} - ${argv.pushStatusPort}:${argv.pushStatusPort}`, - ); - } - if (argv.prefix) { - contents = contents.replace('PREFIX=', `PREFIX=${argv.prefix}`); - } - - fs.writeFileSync(dockerComposeFile, contents); - logger.info('Docker-compose file created'); -} - function UpdateGrafanaConfigFolder() { const configDirPath = join(__dirname, '../../grafanaConfig'); if (fs.existsSync(configDirPath) && !argv.force) { @@ -125,8 +77,6 @@ async function Setup(cli) { process.exit(-1); } - UpdateEnvFile(); - await UpdateDockerComposeFile(); UpdateGrafanaConfigFolder(); } From 54ae8b83a5a5b5ef3fbd8a4a0c16f972a02af064 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 28 Sep 2024 00:40:42 +0200 Subject: [PATCH 08/19] Staticify Grafana setup This just mounts the config files directly into the containers and passes the value we wanna tweak through the environment instead. --- .env.example | 13 +++++ .../grafana/grafana.ini | 0 .../provisioning/dashboards/telemetry.yaml | 0 .../provisioning/datasources/carbonapi.yaml | 0 .../graphite/aggregation-rules.conf | 0 .../graphite/blacklist.conf | 0 .../graphite/brubeck.json | 0 .../graphite/carbon.amqp.conf | 0 .../graphite/carbon.conf | 0 .../graphite/dashboard.conf | 0 .../graphite/go-carbon.conf | 0 .../graphite/graphTemplates.conf | 0 .../graphite/relay-rules.conf | 0 .../graphite/rewrite-rules.conf | 0 .../graphite/storage-aggregation.conf | 0 .../graphite/storage-schemas.conf | 0 .../graphite/whitelist.conf | 0 docker-compose.yml | 15 ++++-- package.json | 16 +++--- src/setup/setup.js | 53 ------------------- 20 files changed, 33 insertions(+), 64 deletions(-) rename {grafanaConfig.example => config}/grafana/grafana.ini (100%) rename {grafanaConfig.example => config}/grafana/provisioning/dashboards/telemetry.yaml (100%) rename {grafanaConfig.example => config}/grafana/provisioning/datasources/carbonapi.yaml (100%) rename {grafanaConfig.example => config}/graphite/aggregation-rules.conf (100%) rename {grafanaConfig.example => config}/graphite/blacklist.conf (100%) rename {grafanaConfig.example => config}/graphite/brubeck.json (100%) rename {grafanaConfig.example => config}/graphite/carbon.amqp.conf (100%) rename {grafanaConfig.example => config}/graphite/carbon.conf (100%) rename {grafanaConfig.example => config}/graphite/dashboard.conf (100%) rename {grafanaConfig.example => config}/graphite/go-carbon.conf (100%) rename {grafanaConfig.example => config}/graphite/graphTemplates.conf (100%) rename {grafanaConfig.example => config}/graphite/relay-rules.conf (100%) rename {grafanaConfig.example => config}/graphite/rewrite-rules.conf (100%) rename {grafanaConfig.example => config}/graphite/storage-aggregation.conf (100%) rename {grafanaConfig.example => config}/graphite/storage-schemas.conf (100%) rename {grafanaConfig.example => config}/graphite/whitelist.conf (100%) diff --git a/.env.example b/.env.example index 8e36176..ae7277f 100644 --- a/.env.example +++ b/.env.example @@ -10,3 +10,16 @@ PUSH_STATUS_PORT= # A prefix to use when pushing keys to Graphite PREFIX= + +# Admin user for Grafana +ADMIN_USER=admin +ADMIN_PASSWORD=password + +# The external hostname for the server Grafana runs on +HOSTNAME=localhost + +# Email address Grafana uses for emails +EMAIL_ADDRESS=admin@grafana.localhost + +# Whether anonymous access to Grafana is allowed +ANONYMOUS_AUTH_ENABLED=false diff --git a/grafanaConfig.example/grafana/grafana.ini b/config/grafana/grafana.ini similarity index 100% rename from grafanaConfig.example/grafana/grafana.ini rename to config/grafana/grafana.ini diff --git a/grafanaConfig.example/grafana/provisioning/dashboards/telemetry.yaml b/config/grafana/provisioning/dashboards/telemetry.yaml similarity index 100% rename from grafanaConfig.example/grafana/provisioning/dashboards/telemetry.yaml rename to config/grafana/provisioning/dashboards/telemetry.yaml diff --git a/grafanaConfig.example/grafana/provisioning/datasources/carbonapi.yaml b/config/grafana/provisioning/datasources/carbonapi.yaml similarity index 100% rename from grafanaConfig.example/grafana/provisioning/datasources/carbonapi.yaml rename to config/grafana/provisioning/datasources/carbonapi.yaml diff --git a/grafanaConfig.example/graphite/aggregation-rules.conf b/config/graphite/aggregation-rules.conf similarity index 100% rename from grafanaConfig.example/graphite/aggregation-rules.conf rename to config/graphite/aggregation-rules.conf diff --git a/grafanaConfig.example/graphite/blacklist.conf b/config/graphite/blacklist.conf similarity index 100% rename from grafanaConfig.example/graphite/blacklist.conf rename to config/graphite/blacklist.conf diff --git a/grafanaConfig.example/graphite/brubeck.json b/config/graphite/brubeck.json similarity index 100% rename from grafanaConfig.example/graphite/brubeck.json rename to config/graphite/brubeck.json diff --git a/grafanaConfig.example/graphite/carbon.amqp.conf b/config/graphite/carbon.amqp.conf similarity index 100% rename from grafanaConfig.example/graphite/carbon.amqp.conf rename to config/graphite/carbon.amqp.conf diff --git a/grafanaConfig.example/graphite/carbon.conf b/config/graphite/carbon.conf similarity index 100% rename from grafanaConfig.example/graphite/carbon.conf rename to config/graphite/carbon.conf diff --git a/grafanaConfig.example/graphite/dashboard.conf b/config/graphite/dashboard.conf similarity index 100% rename from grafanaConfig.example/graphite/dashboard.conf rename to config/graphite/dashboard.conf diff --git a/grafanaConfig.example/graphite/go-carbon.conf b/config/graphite/go-carbon.conf similarity index 100% rename from grafanaConfig.example/graphite/go-carbon.conf rename to config/graphite/go-carbon.conf diff --git a/grafanaConfig.example/graphite/graphTemplates.conf b/config/graphite/graphTemplates.conf similarity index 100% rename from grafanaConfig.example/graphite/graphTemplates.conf rename to config/graphite/graphTemplates.conf diff --git a/grafanaConfig.example/graphite/relay-rules.conf b/config/graphite/relay-rules.conf similarity index 100% rename from grafanaConfig.example/graphite/relay-rules.conf rename to config/graphite/relay-rules.conf diff --git a/grafanaConfig.example/graphite/rewrite-rules.conf b/config/graphite/rewrite-rules.conf similarity index 100% rename from grafanaConfig.example/graphite/rewrite-rules.conf rename to config/graphite/rewrite-rules.conf diff --git a/grafanaConfig.example/graphite/storage-aggregation.conf b/config/graphite/storage-aggregation.conf similarity index 100% rename from grafanaConfig.example/graphite/storage-aggregation.conf rename to config/graphite/storage-aggregation.conf diff --git a/grafanaConfig.example/graphite/storage-schemas.conf b/config/graphite/storage-schemas.conf similarity index 100% rename from grafanaConfig.example/graphite/storage-schemas.conf rename to config/graphite/storage-schemas.conf diff --git a/grafanaConfig.example/graphite/whitelist.conf b/config/graphite/whitelist.conf similarity index 100% rename from grafanaConfig.example/graphite/whitelist.conf rename to config/graphite/whitelist.conf diff --git a/docker-compose.yml b/docker-compose.yml index 5314700..cf594be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,8 +16,8 @@ services: graphite: image: graphiteapp/graphite-statsd volumes: + - ./config/graphite:/opt/graphite/conf - ./logs/graphite:/var/log - - ./grafanaConfig/graphite:/opt/graphite/conf - graphite_data:/opt/graphite/storage networks: - stats @@ -25,11 +25,18 @@ services: grafana: image: grafana/grafana-oss:9.3.6-ubuntu volumes: - - grafana_data:/var/lib/grafana - - ./grafanaConfig/grafana:/etc/grafana + - ./config/grafana/grafana.ini:/etc/grafana/grafana.ini + - ./config/grafana/provisioning:/etc/grafana/provisioning - ./logs/grafana:/var/log/grafana + - grafana_data:/var/lib/grafana ports: - - ${GRAFANA_PORT}:${GRAFANA_PORT} + - ${GRAFANA_PORT}:3000 + environment: + - GF_SECURITY_ADMIN_USER=${ADMIN_USER} + - GF_SECURITY_ADMIN_PASSWORD=${ADMIN_PASSWORD} + - GF_DOMAIN=${HOSTNAME} + - GF_SMTP_FROM_ADDRESS=${EMAIL_ADDRESS} + - GF_AUTH_ANONYMOUS_ENABLED=${ANONYMOUS_AUTH_ENABLED} healthcheck: test: "curl -fsSL -o /dev/null http://localhost:${GRAFANA_PORT}/login" interval: 10s diff --git a/package.json b/package.json index 672242a..3f718a6 100644 --- a/package.json +++ b/package.json @@ -2,13 +2,15 @@ "name": "screeps-grafana-go_carbon", "version": "1.1.0", "scripts": { - "setup": "node bin/server.js setup", - "start": "node bin/server.js start --grafanaPort=3000", - "start:test": "node bin/server.js start --grafanaPort=3000 --force", - "rebuild": "docker compose build --no-cache", + "start": "docker compose up -d", + "start:logs": "docker compose up -d && docker compose logs -ft", + "logs": "docker compose logs -ft", + "stop": "docker compose stop", + "reset": "docker compose down", + "reset:hard": "docker compose down -v", + "rebuild": "docker compose build --no-cache && docker compose down stats-getter && docker compose up -d stats-getter", "lint": "eslint src/**/*.js && eslint dashboards/**/*.js", - "lint:fix": "eslint src/**/*.js --fix && eslint dashboards/**/*.js --fix", - "update-stats-getter": "docker-compose up --detach --build" + "lint:fix": "eslint src/**/*.js --fix && eslint dashboards/**/*.js --fix" }, "dependencies": { "axios": "^0.27.2", @@ -29,7 +31,7 @@ "files": [ "src", "dashboards", - "grafanaConfig.example", + "config", "users.example.json", "docker-compose.yml", "env.example", diff --git a/src/setup/setup.js b/src/setup/setup.js index 50602f1..598edf2 100644 --- a/src/setup/setup.js +++ b/src/setup/setup.js @@ -1,5 +1,4 @@ const fs = require('fs'); -const fse = require('fs-extra'); const { join } = require('path'); const { execSync } = require('child_process'); @@ -7,56 +6,6 @@ let argv; /** @type {import('winston').Logger} */ let logger; -const isWindows = process.platform === 'win32'; -const regexEscape = isWindows ? '\r\n' : '\n'; - -function createRegexWithEscape(string) { - return new RegExp(string.replace('\r\n', regexEscape)); -} - -function UpdateGrafanaConfigFolder() { - const configDirPath = join(__dirname, '../../grafanaConfig'); - if (fs.existsSync(configDirPath) && !argv.force) { - return logger.warn('Grafana config folder already exists, use --force to overwrite it'); - } - - fse.copySync(join(__dirname, '../../grafanaConfig.example'), configDirPath); - const grafanaIniFile = join(configDirPath, './grafana/grafana.ini'); - let grafanaIniText = fs.readFileSync(grafanaIniFile, 'utf8'); - - if (argv.username) grafanaIniText = grafanaIniText.replace(/admin_user = (.*)/, `admin_user = ${argv.username}`); - if (argv.password) grafanaIniText = grafanaIniText.replace(/admin_password = (.*)/, `admin_password = ${argv.password}`); - if (argv.grafanaDomain) { - grafanaIniText = grafanaIniText.replace('domain = localhost', `domain = ${argv.grafanaDomain}`); - grafanaIniText = grafanaIniText.replace('from_address = admin@localhost', `from_address = admin@${argv.grafanaDomain}`); - } - if (argv.grafanaPort) { - grafanaIniText = grafanaIniText.replace('http_port = 3000', `http_port = ${argv.grafanaPort}`); - } - grafanaIniText = grafanaIniText.replace( - createRegexWithEscape('enable anonymous access\r\nenabled = (.*)'), - `enable anonymous access${regexEscape}enabled = ${argv.enableAnonymousAccess ? 'true' : 'false'}`, - ); - fs.writeFileSync(grafanaIniFile, grafanaIniText); - - // This can just be set manually in the config folder. - /* - const storageSchemasFile = join(grafanaConfigFolder, './go-carbon/storage-schemas.conf'); - let storageSchemasText = fs.readFileSync(storageSchemasFile, 'utf8'); - const { defaultRetention } = argv; - - if (defaultRetention) { - storageSchemasText = storageSchemasText.replace( - createRegexWithEscape('pattern = .*\r\nretentions = (.*)'), - `pattern = .*${regexEscape}retentions = ${defaultRetention}`, - ); - } - fs.writeFileSync(storageSchemasFile, storageSchemasText); - */ - - logger.info('Grafana config folder created'); -} - function resetFolders() { const logsPath = join(__dirname, '../../logs'); let logsExist = fs.existsSync(logsPath); @@ -76,8 +25,6 @@ async function Setup(cli) { logger.error('missing users.json file'); process.exit(-1); } - - UpdateGrafanaConfigFolder(); } module.exports = Setup; From e7a1350c455443f273304e54d284440ca99690f7 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 28 Sep 2024 01:05:04 +0200 Subject: [PATCH 09/19] Fix the Graphite datasource UID so the serviceInfo Dashboard can expect it --- .../provisioning/dashboards}/serviceInfo.json | 0 .../provisioning/datasources/carbonapi.yaml | 2 + dashboards/helper.js | 46 ------------------- package.json | 1 - 4 files changed, 2 insertions(+), 47 deletions(-) rename {dashboards => config/grafana/provisioning/dashboards}/serviceInfo.json (100%) delete mode 100644 dashboards/helper.js diff --git a/dashboards/serviceInfo.json b/config/grafana/provisioning/dashboards/serviceInfo.json similarity index 100% rename from dashboards/serviceInfo.json rename to config/grafana/provisioning/dashboards/serviceInfo.json diff --git a/config/grafana/provisioning/datasources/carbonapi.yaml b/config/grafana/provisioning/datasources/carbonapi.yaml index 8931eed..800710e 100644 --- a/config/grafana/provisioning/datasources/carbonapi.yaml +++ b/config/grafana/provisioning/datasources/carbonapi.yaml @@ -19,6 +19,8 @@ datasources: orgId: 1 # url url: http://graphite:8080/ + # uid, used to link that source with the provisioned dashboards + uid: d34EAUnVz # database password, if used password: # database user, if used diff --git a/dashboards/helper.js b/dashboards/helper.js deleted file mode 100644 index 0bd04e6..0000000 --- a/dashboards/helper.js +++ /dev/null @@ -1,46 +0,0 @@ -const fs = require('fs'); -const { join } = require('path'); - -function transformDashboard(dashboard) { - delete dashboard.id; - delete dashboard.uid; - for (let i = 0; i < dashboard.templating.list.length; i += 1) { - const { datasource } = dashboard.templating.list[i]; - if (datasource) { - delete datasource.type; - delete datasource.uid; - } - } - - for (let i = 0; i < dashboard.panels.length; i += 1) { - const panel = dashboard.panels[i]; - if (panel.panels) { - for (let y = 0; y < panel.panels.length; y += 1) { - const subPanel = panel.panels[y]; - delete subPanel.datasource.uid; - delete subPanel.datasource.type; - } - } else if (panel.type !== 'row') { - delete panel.datasource.uid; - delete panel.datasource.type; - } - } - return { dashboard, overwrite: true }; -} - -module.exports = function GetDashboards() { - const setupDashboards = {}; - - const dashboardPath = __dirname; - const dashboardFileNames = fs.readdirSync(dashboardPath).filter((d) => d.endsWith('.json')); - dashboardFileNames.forEach((name) => { - try { - const filePath = join(dashboardPath, `${name}`); - const rawDashboardText = fs.readFileSync(filePath); - setupDashboards[name] = transformDashboard(JSON.parse(rawDashboardText)); - } catch (error) { - console.log(error); - } - }); - return setupDashboards; -}; diff --git a/package.json b/package.json index 3f718a6..d9a95c1 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,6 @@ }, "files": [ "src", - "dashboards", "config", "users.example.json", "docker-compose.yml", From bf249c16b9a5b39002b6a18d4721fd05b1a3246c Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 28 Sep 2024 01:10:30 +0200 Subject: [PATCH 10/19] Remove the old server stuff now that we're fully static --- bin/server.js | 74 ---------------------------------------------- package-lock.json | 3 -- package.json | 3 -- src/deletePath.js | 41 ------------------------- src/setup/setup.js | 53 --------------------------------- src/setup/start.js | 67 ----------------------------------------- 6 files changed, 241 deletions(-) delete mode 100644 bin/server.js delete mode 100644 src/deletePath.js delete mode 100644 src/setup/setup.js delete mode 100644 src/setup/start.js diff --git a/bin/server.js b/bin/server.js deleted file mode 100644 index 881a82b..0000000 --- a/bin/server.js +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env node - -const { execSync } = require('child_process'); -const { join } = require('path'); -require('dotenv').config({ path: join(__dirname, '../.env') }); - -const nodeVersion = process.versions.node; -const nodeVersionMajor = Number(nodeVersion.split('.')[0]); -const { getPort } = nodeVersionMajor >= 14 ? require('get-port-please') : { getPort: async () => 3000 }; - -const minimist = require('minimist'); -const { createLogger, format, transports } = require('winston'); - -const setup = require('../src/setup/setup'); -const start = require('../src/setup/start'); - -const argv = minimist(process.argv.slice(2)); - -const { combine, timestamp, prettyPrint } = format; -const logger = createLogger({ - transports: [ - new transports.Console({ - format: combine(format.colorize(), format.simple()), - }), - new transports.File({ - filename: 'logs/setup.log', - format: combine( - timestamp(), - prettyPrint(), - ), - })], -}); - -async function main() { - argv.grafanaPort = argv.grafanaPort ?? await getPort({ portRange: [3000, 4000] }); - argv.serverPort = argv.serverPort ?? 21025; - if (argv.pushStatusPort === true) { - argv.pushStatusPort = 10004; - } else { - const port = Number(argv.pushStatusPort); - if (!Number.isNaN(port)) { - argv.pushStatusPort = port; - } else { - delete argv.pushStatusPort; - } - } - - const cli = { - cmd: argv._.shift(), - args: argv, - logger, - }; - - switch (cli.cmd) { - case 'setup': - setup(cli); - break; - - case 'start': - start(cli); - break; - - case 'stop': - logger.info(`Stopping server from ${process.env.COMPOSE_FILE}`); - execSync('docker-compose stop'); - break; - - default: - logger.error(`expected command, got "${cli.cmd}"`); - break; - } -} - -main(); diff --git a/package-lock.json b/package-lock.json index ac2015a..78c5ca0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,9 +15,6 @@ "minimist": "^1.2.7", "winston": "^3.8.1" }, - "bin": { - "screeps-grafana-go_carbon": "bin/setup.js" - }, "devDependencies": { "eslint": "^8.23.1", "eslint-config-airbnb-base": "^15.0.0", diff --git a/package.json b/package.json index d9a95c1..c05b510 100644 --- a/package.json +++ b/package.json @@ -25,9 +25,6 @@ "eslint-config-airbnb-base": "^15.0.0", "eslint-plugin-import": "^2.26.0" }, - "bin": { - "screeps-grafana-go_carbon": "./bin/server.js" - }, "files": [ "src", "config", diff --git a/src/deletePath.js b/src/deletePath.js deleted file mode 100644 index c3f79a9..0000000 --- a/src/deletePath.js +++ /dev/null @@ -1,41 +0,0 @@ -const fs = require('fs'); -const { join } = require('path'); - -const minimist = require('minimist'); - -const argv = minimist(process.argv.slice(2)); -console.dir(argv); - -const whisperPath = join(__dirname, '../whisper/'); -let { statsPath } = argv; -if (!statsPath) { - console.error('Please provide a path to the stats'); - process.exit(1); -} - -if (!fs.existsSync(whisperPath)) { - console.log('No whisper folder found, manually delete the stats are not working while whisper export is disabled.'); - process.exit(1); -} else if (statsPath.startsWith('.')) { - console.log('Please provide a path without a leading dot'); - process.exit(1); -} else if (statsPath.endsWith('.')) { - console.log('Please provide a path without a trailing dot'); - process.exit(1); -} -statsPath = statsPath.split('.').join('/'); - -function deletePath(path) { - if (fs.existsSync(path)) { - fs.rm(path, { recursive: true }, (err) => { - if (err) { - console.error(err); - process.exit(1); - } - console.log(`Deleted ${path}`); - }); - return; - } - console.log(`Path not found: ${path}`); -} -deletePath(join(whisperPath, statsPath)); diff --git a/src/setup/setup.js b/src/setup/setup.js deleted file mode 100644 index 598edf2..0000000 --- a/src/setup/setup.js +++ /dev/null @@ -1,53 +0,0 @@ -const fs = require('fs'); -const { join } = require('path'); -const { execSync } = require('child_process'); - -let argv; -/** @type {import('winston').Logger} */ -let logger; - -function resetFolders() { - const logsPath = join(__dirname, '../../logs'); - let logsExist = fs.existsSync(logsPath); - if (logsExist && argv.deleteLogs) { - fs.rmdirSync(logsPath, { recursive: true }); - logsExist = false; - } - if (!logsExist) fs.mkdirSync(logsPath, { recursive: true }); -} - -async function Setup(cli) { - argv = cli.args; - logger = cli.logger; - - const usersFile = join(__dirname, '../../users.json'); - if (!fs.existsSync(usersFile)) { - logger.error('missing users.json file'); - process.exit(-1); - } -} - -module.exports = Setup; - -module.exports.commands = async function Commands(grafanaApiUrl) { - logger.info(`Grafana API URL: ${grafanaApiUrl}, serverPort: ${argv.serverPort}`); - - const commands = [ - { command: `docker compose down ${argv.removeVolumes ? '--volumes' : ''} --remove-orphans`, name: 'docker-compose down' }, - { command: 'docker compose up -d', name: 'docker-compose up' }, - ]; - - logger.info('Executing start commands:'); - for (let i = 0; i < commands.length; i += 1) { - const commandInfo = commands[i]; - try { - logger.info(`Running command ${commandInfo.name}`); - execSync(commandInfo.command, { stdio: argv.debug ? 'inherit' : 'ignore' }); - if (commandInfo.name.startsWith('docker-compose down')) resetFolders(); - } catch (error) { - logger.error(`Command ${commandInfo.name} errored`, error); - logger.error('Stopping setup'); - process.exit(1); - } - } -}; diff --git a/src/setup/start.js b/src/setup/start.js deleted file mode 100644 index 72bcb4d..0000000 --- a/src/setup/start.js +++ /dev/null @@ -1,67 +0,0 @@ -const dotenv = require('dotenv'); -const axios = require('axios'); -const { join } = require('path'); -const fs = require('fs'); - -let grafanaApiUrl; - -const setup = require('./setup.js'); -const getDashboards = require('../../dashboards/helper.js'); - -/** @type {import('winston').Logger} */ -let logger; - -function sleep(milliseconds) { - // eslint-disable-next-line no-promise-executor-return - return new Promise((resolve) => setTimeout(resolve, milliseconds)); -} - -const dashboards = getDashboards(); -let adminLogin; - -function handleSuccess(type) { - logger.info(`${type} dashboard setup done`); -} - -function handleError(type, err) { - logger.error(`${type} dashboard error: `, err); -} - -async function SetupServiceInfoDashboard() { - const type = 'Service-Info'; - try { - const dashboard = dashboards.serviceInfo; - await axios({ - url: `${grafanaApiUrl}/dashboards/db`, - method: 'post', - auth: adminLogin, - data: dashboard, - }); - handleSuccess(type); - } catch (err) { - handleError(type, err); - } -} - -async function Start(cli) { - logger = cli.logger; - await setup(cli); - dotenv.config({ path: join(__dirname, '../../.env') }); - - const grafanaIni = fs.readFileSync(join(__dirname, '../../grafanaConfig/grafana/grafana.ini'), 'utf8'); - const username = grafanaIni.match(/admin_user = (.*)/)[1]; - const password = grafanaIni.match(/admin_password = (.*)/)[1]; - adminLogin = { username, password }; - - dotenv.config({ path: join(__dirname, '../../grafanaConfig/.env.grafana') }); - - grafanaApiUrl = `http://localhost:${cli.args.grafanaPort}/api`; - await setup.commands(grafanaApiUrl); - logger.info('Pre setup done! Waiting for Grafana to start...'); - await sleep(30 * 1000); - - await SetupServiceInfoDashboard(); - logger.info('Setup done!'); -} - -module.exports = Start; From 9c79662c64e84a62c6637a36d4b05e5b49a0e887 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 28 Sep 2024 01:17:55 +0200 Subject: [PATCH 11/19] Overlay the directory hierarchy properly This saves Grafana from logging errors because the directories don't exist. --- config/grafana/provisioning/access-control/.gitignore | 0 config/grafana/provisioning/alerting/.gitignore | 0 config/grafana/provisioning/notifiers/.gitignore | 0 config/grafana/provisioning/plugins/.gitignore | 0 4 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 config/grafana/provisioning/access-control/.gitignore create mode 100644 config/grafana/provisioning/alerting/.gitignore create mode 100644 config/grafana/provisioning/notifiers/.gitignore create mode 100644 config/grafana/provisioning/plugins/.gitignore diff --git a/config/grafana/provisioning/access-control/.gitignore b/config/grafana/provisioning/access-control/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/config/grafana/provisioning/alerting/.gitignore b/config/grafana/provisioning/alerting/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/config/grafana/provisioning/notifiers/.gitignore b/config/grafana/provisioning/notifiers/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/config/grafana/provisioning/plugins/.gitignore b/config/grafana/provisioning/plugins/.gitignore new file mode 100644 index 0000000..e69de29 From 8229573f67bae0b7961924a1c00944949597fe2e Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 28 Sep 2024 01:26:38 +0200 Subject: [PATCH 12/19] Properly set up dashboard provisioning --- .../dashboards/{telemetry.yaml => dashboard.yml} | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) rename config/grafana/provisioning/dashboards/{telemetry.yaml => dashboard.yml} (76%) diff --git a/config/grafana/provisioning/dashboards/telemetry.yaml b/config/grafana/provisioning/dashboards/dashboard.yml similarity index 76% rename from config/grafana/provisioning/dashboards/telemetry.yaml rename to config/grafana/provisioning/dashboards/dashboard.yml index 07d4e8b..fa0f40c 100644 --- a/config/grafana/provisioning/dashboards/telemetry.yaml +++ b/config/grafana/provisioning/dashboards/dashboard.yml @@ -2,23 +2,23 @@ apiVersion: 1 providers: # an unique provider name. Required - - name: 'telemetry' + - name: 'serviceInfo' # Org id. Default to 1 orgId: 1 # name of the dashboard folder. - folder: 'Telemetry' + folder: '' # folder UID. will be automatically generated if not specified folderUid: '' # provider type. Default to 'file' type: file # disable dashboard deletion - disableDeletion: true + disableDeletion: false # how often Grafana will scan for changed dashboards - updateIntervalSeconds: 60 + updateIntervalSeconds: 10 # allow updating provisioned dashboards from the UI - allowUiUpdates: false + allowUiUpdates: true options: # path to dashboard files on disk. Required when using the 'file' type - path: /etc/grafana/dashboards + path: /etc/grafana/provisioning/dashboards # use folder names from filesystem to create folders in Grafana - foldersFromFilesStructure: false \ No newline at end of file + foldersFromFilesStructure: true \ No newline at end of file From 1d08bb0e0c8f1b8324a9b1c21dfd7839ac260f92 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 28 Sep 2024 01:54:14 +0200 Subject: [PATCH 13/19] Randomly make the service dashboard better --- .../provisioning/dashboards/serviceInfo.json | 814 ++++++++++++------ 1 file changed, 532 insertions(+), 282 deletions(-) diff --git a/config/grafana/provisioning/dashboards/serviceInfo.json b/config/grafana/provisioning/dashboards/serviceInfo.json index 9823c82..e959866 100644 --- a/config/grafana/provisioning/dashboards/serviceInfo.json +++ b/config/grafana/provisioning/dashboards/serviceInfo.json @@ -1,317 +1,567 @@ { - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - }, - "type": "dashboard" - } - ] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, - "id": 1, - "links": [], - "liveNow": false, - "panels": [ + "annotations": { + "list": [ { + "builtIn": 1, "datasource": { - "type": "graphite", - "uid": "d34EAUnVz" + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "thresholdsStyle": { - "mode": "off" + { + "color": "red", + "value": 80 } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "refId": "A", + "target": "carbon.*.*.cpuUsage" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 0 + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "id": 4, - "options": { - "legend": { - "calcs": [ - "mean" - ], - "displayMode": "table", - "placement": "bottom" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "LlpKjYR4z" + "refId": "A", + "target": "aliasByNode(carbon.agents.*.memUsage, 2)" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" }, - "refId": "A", - "target": "groupByNode(aliasByNode(openmetric.carbon.*.*.metricsReceived, 3), 0, 'sumSeries')" + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } - ], - "title": "Metrics recieved", - "type": "timeseries" + }, + "overrides": [] }, - { - "datasource": { - "type": "graphite", - "uid": "d34EAUnVz" + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "refId": "A", + "target": "groupByNode(aliasByNode(carbon.*.*.metricsReceived, 3), 0, 'sumSeries')" + } + ], + "title": "Metrics recieved", + "type": "timeseries" + }, + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 0 + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, - "id": 5, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "refId": "A", + "target": "alias(stats.statsd.processing_time, 'Total')" + } + ], + "title": "Processing time", + "type": "timeseries" + }, + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "tooltip": { - "mode": "single", - "sort": "none" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } }, - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "LlpKjYR4z" - }, - "refId": "A", - "target": "groupByNode(alias(openmetric.carbon.*.cache.metrics, 'Total'), 0, 'sum')" - } - ], - "title": "Metrics processed", - "type": "timeseries" + "overrides": [] }, - { - "datasource": { - "type": "graphite", - "uid": "d34EAUnVz" + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 7, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.6", + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "refId": "A", + "target": "stats.gauges.statsd.timestamp_lag" + } + ], + "title": "Timestamp lag", + "transformations": [], + "type": "gauge" + }, + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": true, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" }, - "decimals": 2, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "overrides": [] + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "gridPos": { - "h": 7, - "w": 24, - "x": 0, - "y": 9 + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true }, - "id": 2, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "bottom" + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "graphite", + "uid": "d34EAUnVz" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "graphite", - "uid": "LlpKjYR4z" - }, - "refId": "A", - "target": "groupByNode(alias(openmetric.carbon.*.cache.size, 'Total'), 0, 'sum')" - } - ], - "title": "Cache size", - "type": "timeseries" - } - ], - "refresh": "15m", - "schemaVersion": 36, - "style": "dark", - "tags": [], - "templating": { - "list": [] - }, - "time": { - "from": "now-24h", - "to": "now" - }, - "timepicker": {}, - "timezone": "", - "title": "Service info", - "uid": "weVPAU7Vk", - "version": 19, - "weekStart": "" - } \ No newline at end of file + "refId": "A", + "target": "groupByNode(alias(carbon.agents.*.cache.size, 'Total'), 0, 'sum')" + } + ], + "title": "Cache size", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Service info", + "uid": "weVPAU7Vk", + "version": 1, + "weekStart": "" +} \ No newline at end of file From e03b15beb4cfdf37a9ad0822f582059f950cfbdf Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 28 Sep 2024 03:10:20 +0200 Subject: [PATCH 14/19] Remove the weird separator from the logs --- src/pushStats/index.js | 63 +++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/src/pushStats/index.js b/src/pushStats/index.js index 301b010..6e06224 100644 --- a/src/pushStats/index.js +++ b/src/pushStats/index.js @@ -56,11 +56,8 @@ const cronLogger = createLogger({ class ManageStats { groupedStats; - message; - constructor() { this.groupedStats = {}; - this.message = '----------------------------------------------------------------\r\n'; } async handleUsers(type) { @@ -92,28 +89,21 @@ class ManageStats { const { groupedStats } = this; - if (type === 'mmo') { + if (type === 'mmo' || type === 'season') { if (Object.keys(groupedStats).length > 0) { - if (!await ManageStats.reportStats({ stats: groupedStats })) return console.log('Error while pushing stats'); - - console.log(`[${type}] Pushed stats to graphite`); - - return console.log(this.message); + const push = await ManageStats.reportStats({ stats: groupedStats }); + if (push) { + console.log('Error while pushing stats'); + } else { + console.log(`[${type}] Pushed stats to graphite`); + } + return; } - if (beginningOfMinute) return console.log('No stats to push'); - return undefined; - } - if (type === 'season') { - if (Object.keys(groupedStats).length > 0) { - if (!await ManageStats.reportStats({ stats: groupedStats })) return console.log('Error while pushing stats'); - - console.log(`[${type}] Pushed stats to graphite`); - - return console.log(this.message); + if (beginningOfMinute) { + console.log('No stats to push'); } - if (beginningOfMinute) return console.log('No stats to push'); - return undefined; + return; } const privateUser = users.find((user) => user.type === 'private' && user.host); @@ -133,20 +123,30 @@ class ManageStats { } } - if (!await ManageStats.reportStats({ stats: groupedStats, serverStats, adminUtilsServerStats })) return console.log('Error while pushing stats'); - let statsPushed = ''; + const push = await ManageStats.reportStats({ + stats: groupedStats, + serverStats, + adminUtilsServerStats, + }); + if (!push) { + console.log('Error while pushing stats'); + return; + } + const typesPushed = []; if (Object.keys(groupedStats).length > 0) { - statsPushed = `Pushed ${type} stats`; + typesPushed.push(type); } if (serverStats) { - statsPushed += statsPushed.length > 0 ? ', server stats' : 'Pushed server stats'; + typesPushed.push('server stats'); } if (adminUtilsServerStats) { - statsPushed += statsPushed.length > 0 ? ', adminUtilsServerStats' : 'Pushed server stats'; + typesPushed.push('adminUtilsServerStats'); + } + if (typesPushed.length) { + logger.info(`> Pushed ${typesPushed.join(', ')} to graphite`); + } else { + logger.info('> Pushed no stats to graphite'); } - this.message += statsPushed.length > 0 ? `> ${statsPushed} to graphite` : '> Pushed no stats to graphite'; - logger.info(this.message); - return console.log(this.message); } static async getLoginInfo(userinfo) { @@ -224,9 +224,8 @@ const groupedUsers = users.reduce((group, user) => { }, {}); cron.schedule('*/30 * * * * *', async () => { - const message = `Cron event hit: ${new Date()}`; - console.log(`\r\n${message}\n`); - cronLogger.info(message); + console.log(`Cron event hit: ${new Date()}`); + cronLogger.info(`Cron event hit: ${new Date()}`); Object.keys(groupedUsers).forEach((type) => { new ManageStats(groupedUsers[type]).handleUsers(type); }); From a7d4e3c89dbe4524f0a248877ca998184537e6b0 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 28 Sep 2024 16:54:29 +0200 Subject: [PATCH 15/19] Typescript stuff a little --- src/pushStats/apiFunctions.js | 99 +++++++++++-- src/pushStats/index.js | 66 +++++++-- src/pushStats/package-lock.json | 250 ++++++++++++++++++++++++++++++++ src/pushStats/package.json | 8 +- src/pushStats/types.d.ts | 13 ++ tsconfig.json | 109 ++++++++++++++ 6 files changed, 521 insertions(+), 24 deletions(-) create mode 100644 src/pushStats/types.d.ts create mode 100644 tsconfig.json diff --git a/src/pushStats/apiFunctions.js b/src/pushStats/apiFunctions.js index c7356d8..52d662f 100644 --- a/src/pushStats/apiFunctions.js +++ b/src/pushStats/apiFunctions.js @@ -14,7 +14,8 @@ import { createLogger, format, transports } from 'winston'; // eslint-disable-next-line import/no-unresolved import 'winston-daily-rotate-file'; -const users = JSON.parse(fs.readFileSync('users.json')); +/** @type {UserInfo[]} */ +const users = JSON.parse(fs.readFileSync('users.json').toString('utf8')); const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); @@ -40,6 +41,11 @@ const logger = createLogger({ transports: [transport], }); +/** + * + * @param {string} data + * @returns + */ async function gz(data) { if (!data) return {}; const buf = Buffer.from(data.slice(3), 'base64'); @@ -47,6 +53,11 @@ async function gz(data) { return JSON.parse(ret.toString()); } +/** + * + * @param {any} obj + * @returns + */ function removeNonNumbers(obj) { if (!obj) return obj; @@ -64,11 +75,12 @@ function removeNonNumbers(obj) { return obj; } +/** @type {string | undefined} */ let privateHost; let serverPort = 21025; function getPrivateHost() { - serverPort = process.env.SERVER_PORT || 21025; + serverPort = parseInt(process.env.SERVER_PORT, 10) || 21025; const hosts = [ 'localhost', 'host.docker.internal', @@ -109,6 +121,12 @@ if (!privateHost && needsPrivateHost) { TryToGetPrivateHost(); } +/** + * + * @param {string} host + * @param {UserType} type + * @returns + */ async function getHost(host, type) { if (type === 'mmo') return 'screeps.com'; if (type === 'season') return 'screeps.com/season'; @@ -116,7 +134,17 @@ async function getHost(host, type) { return privateHost; } +/** + * + * @param {Omit & { token?: string }} info + * @param {string} path + * @param {'GET'|'POST'} method + * @param {{}} body + * @returns {http.RequestOptions & {body: {}, isHTTPS: boolean}} + */ async function getRequestOptions(info, path, method = 'GET', body = {}) { + /** @type {Record} */ const headers = { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(JSON.stringify(body)), @@ -134,6 +162,12 @@ async function getRequestOptions(info, path, method = 'GET', body = {}) { isHTTPS: info.type === 'mmo', }; } + +/** + * + * @param {https.RequestOptions & { body?: {}, isHTTPS?: boolean }} options + * @returns + */ async function req(options) { const reqBody = JSON.stringify(options.body); const { isHTTPS } = options; @@ -171,12 +205,13 @@ async function req(options) { .then((result) => { if (result === 'Timeout') { logger.log('info', 'Timeout hit!', new Date(), JSON.stringify(options), reqBody); - return; + return undefined; + } + if (typeof result === 'string' && result.startsWith('Rate limit exceeded')) { + logger.log('error', { data: result, options }); + } else { + logger.log('info', { data: `${JSON.stringify(result).length / 1000} MB`, options }); } - // is result string - if (typeof result === 'string' && result.startsWith('Rate limit exceeded')) logger.log('error', { data: result, options }); - else logger.log('info', { data: `${JSON.stringify(result).length / 1000} MB`, options }); - // eslint-disable-next-line consistent-return return result; }) .catch((result) => { @@ -186,8 +221,13 @@ async function req(options) { } export default class { + /** + * + * @param {UserInfo} info + * @returns + */ static async getPrivateServerToken(info) { - const options = await getRequestOptions({ type: 'private', username: info.username, host: info.host }, '/api/auth/signin', 'POST', { + const options = await getRequestOptions(info, '/api/auth/signin', 'POST', { email: info.username, password: info.password, }); @@ -196,13 +236,18 @@ export default class { return res.token; } + /** + * + * @param {UserInfo} info + * @param {string} shard + * @param {string} statsPath + * @returns + */ static async getMemory(info, shard, statsPath = 'stats') { const options = await getRequestOptions(info, `/api/user/memory?path=${statsPath}&shard=${shard}`, 'GET'); const res = await req(options); - if (res) { - console.log(`Got memory from ${info.username} in ${shard} `); - } else { + if (!res) { return undefined; } @@ -210,6 +255,12 @@ export default class { return data; } + /** + * + * @param {UserInfo} info + * @param {string} shard + * @returns + */ static async getSegmentMemory(info, shard) { const options = await getRequestOptions(info, `/api/user/memory-segment?segment=${info.segment}&shard=${shard}`, 'GET'); const res = await req(options); @@ -222,21 +273,36 @@ export default class { } } + /** + * + * @param {UserInfo} info + * @returns + */ static async getUserinfo(info) { const options = await getRequestOptions(info, '/api/auth/me', 'GET'); const res = await req(options); return res; } + /** + * + * @param {UserInfo} info + * @returns + */ static async getLeaderboard(info) { const options = await getRequestOptions(info, `/api/leaderboard/find?username=${info.username}&mode=world`, 'GET'); const res = await req(options); return res; } + /** + * + * @param {string | undefined} host + * @returns + */ static async getServerStats(host) { const serverHost = host || privateHost; - const options = await getRequestOptions({ host: serverHost }, '/api/stats/server', 'GET'); + const options = await getRequestOptions(/** @type {UserInfo} */ ({ host: serverHost }), '/api/stats/server', 'GET'); const res = await req(options); if (!res || !res.users) { logger.error(res); @@ -245,9 +311,14 @@ export default class { return removeNonNumbers(res); } + /** + * + * @param {string | undefined} host + * @returns + */ static async getAdminUtilsServerStats(host) { const serverHost = host || privateHost; - const options = await getRequestOptions({ host: serverHost }, '/stats', 'GET'); + const options = await getRequestOptions(/** @type {UserInfo} */ ({ host: serverHost }), '/stats', 'GET'); const res = await req(options); if (!res || !res.gametime) { logger.error(res); @@ -255,7 +326,9 @@ export default class { } delete res.ticks.ticks; + /** @type {Record} */ const mUsers = {}; + // @ts-expect-error res.users.forEach((user) => { mUsers[user.username] = user; }); diff --git a/src/pushStats/index.js b/src/pushStats/index.js index 6e06224..852e03d 100644 --- a/src/pushStats/index.js +++ b/src/pushStats/index.js @@ -15,7 +15,8 @@ const app = express(); const pushStatusPort = Number(process.env.PUSH_STATUS_PORT); let lastUpload = new Date().getTime(); -const users = JSON.parse(fs.readFileSync('users.json')); +/** @type {UserInfo[]} */ +const users = JSON.parse(fs.readFileSync('users.json').toString('utf8')); dotenv.config(); const pushTransport = new transports.DailyRotateFile({ @@ -54,16 +55,23 @@ const cronLogger = createLogger({ }); class ManageStats { + /** @type {Record} */ groupedStats; constructor() { this.groupedStats = {}; } + /** + * + * @param {"mmo"|"season"|"private"} type + * @returns + */ async handleUsers(type) { console.log(`[${type}] Handling Users`); const beginningOfMinute = new Date().getSeconds() < 15; + /** @type {(string | unknown)[]} */ const getStatsFunctions = []; users.forEach((user) => { try { @@ -74,12 +82,11 @@ class ManageStats { if (user.type === 'mmo' && shouldContinue) return; if (user.type === 'season' && shouldContinue) return; - for (let y = 0; y < user.shards.length; y += 1) { - const shard = user.shards[y]; - getStatsFunctions.push(this.getStats(user, shard, this.message)); + for (const shard of user.shards) { + getStatsFunctions.push(this.getStats(user, shard)); } } catch (error) { - logger.error(error.message); + logger.error(error); } }); @@ -112,6 +119,7 @@ class ManageStats { const adminUtilsServerStats = await ApiFunc.getAdminUtilsServerStats(host); if (adminUtilsServerStats) { try { + /** @type {Record} */ const groupedAdminStatsUsers = {}; for (const [username, user] of Object.entries(adminUtilsServerStats)) { groupedAdminStatsUsers[username] = user; @@ -123,15 +131,18 @@ class ManageStats { } } - const push = await ManageStats.reportStats({ + const stats = { stats: groupedStats, serverStats, adminUtilsServerStats, - }); + }; + + const push = await ManageStats.reportStats(stats); if (!push) { console.log('Error while pushing stats'); return; } + /** @type {string[]} */ const typesPushed = []; if (Object.keys(groupedStats).length > 0) { typesPushed.push(type); @@ -149,6 +160,11 @@ class ManageStats { } } + /** + * + * @param {UserInfo} userinfo + * @returns + */ static async getLoginInfo(userinfo) { if (userinfo.type === 'private') { userinfo.token = await ApiFunc.getPrivateServerToken(userinfo); @@ -156,6 +172,11 @@ class ManageStats { return userinfo.token; } + /** + * + * @param {UserInfo} userinfo + * @returns {Promise<{ rank: number, score: number }>} + */ static async addLeaderboardData(userinfo) { try { const leaderboard = await ApiFunc.getLeaderboard(userinfo); @@ -169,6 +190,12 @@ class ManageStats { } } + /** + * + * @param {UserInfo} userinfo + * @param {string} shard + * @returns {Promise} + */ async getStats(userinfo, shard) { try { await ManageStats.getLoginInfo(userinfo); @@ -183,6 +210,13 @@ class ManageStats { } } + /** + * + * @param {UserInfo} userinfo + * @param {string} shard + * @param {*} stats + * @returns + */ async processStats(userinfo, shard, stats) { if (Object.keys(stats).length === 0) return; const me = await ApiFunc.getUserinfo(userinfo); @@ -191,6 +225,11 @@ class ManageStats { this.pushStats(userinfo, stats, shard); } + /** + * + * @param {*} stats + * @returns + */ static async reportStats(stats) { return new Promise((resolve) => { console.log(`Writing stats ${JSON.stringify(stats)} to graphite`); @@ -206,6 +245,13 @@ class ManageStats { }); } + /** + * + * @param {UserInfo} userinfo + * @param {*} stats + * @param {string} shard + * @returns + */ pushStats(userinfo, stats, shard) { if (Object.keys(stats).length === 0) return; const username = userinfo.replaceName !== undefined ? userinfo.replaceName : userinfo.username; @@ -221,13 +267,13 @@ const groupedUsers = users.reduce((group, user) => { group[type] = group[type] ?? []; group[type].push(user); return group; -}, {}); +}, /** @type {Record} */ ({})); cron.schedule('*/30 * * * * *', async () => { console.log(`Cron event hit: ${new Date()}`); cronLogger.info(`Cron event hit: ${new Date()}`); Object.keys(groupedUsers).forEach((type) => { - new ManageStats(groupedUsers[type]).handleUsers(type); + new ManageStats().handleUsers(/** @type {UserType} */(type)); }); }); @@ -237,7 +283,7 @@ if (pushStatusPort) { }); app.get('/', (req, res) => { const diffCompleteMinutes = Math.ceil( - Math.abs(parseInt(new Date().getTime(), 10) - parseInt(lastUpload, 10)) / (1000 * 60), + Math.abs(new Date().getTime() - lastUpload) / (1000 * 60), ); res.json({ result: diffCompleteMinutes < 300, lastUpload, diffCompleteMinutes }); }); diff --git a/src/pushStats/package-lock.json b/src/pushStats/package-lock.json index 9a00fe2..55dc2d1 100644 --- a/src/pushStats/package-lock.json +++ b/src/pushStats/package-lock.json @@ -12,6 +12,12 @@ "node-cron": "^3.0.1", "winston": "^3.8.1", "winston-daily-rotate-file": "^4.7.1" + }, + "devDependencies": { + "@types/express": "^5.0.0", + "@types/graphite": "^0.1.2", + "@types/node": "^22.7.4", + "@types/node-cron": "^3.0.11" } }, "node_modules/@colors/colors": { @@ -32,6 +38,128 @@ "kuler": "^2.0.0" } }, + "node_modules/@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "node_modules/@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/express": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.0.tgz", + "integrity": "sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^5.0.0", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "node_modules/@types/express-serve-static-core": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.0.0.tgz", + "integrity": "sha512-AbXMTZGt40T+KON9/Fdxx0B2WK5hsgxcfXJLr5bFpZ7b4JCex2WyQPTEKdXqfHiY5nKKBScZ7yCoO6Pvgxfvnw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "node_modules/@types/graphite": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@types/graphite/-/graphite-0.1.2.tgz", + "integrity": "sha512-k+esqcUwtDaAZbTXf6J06Z82ZNbndJohxh9/PqvtWwYtqJZxSzpD7HBR7oN5ADXGE67MbqDCs1+1/TpJxIz2zQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.7.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.4.tgz", + "integrity": "sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.19.2" + } + }, + "node_modules/@types/node-cron": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.11.tgz", + "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/qs": { + "version": "6.9.16", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.16.tgz", + "integrity": "sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "node_modules/@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, "node_modules/@types/triple-beam": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", @@ -882,6 +1010,13 @@ "node": ">= 0.6" } }, + "node_modules/undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true, + "license": "MIT" + }, "node_modules/unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", @@ -987,6 +1122,115 @@ "kuler": "^2.0.0" } }, + "@types/body-parser": { + "version": "1.19.5", + "resolved": "https://registry.npmjs.org/@types/body-parser/-/body-parser-1.19.5.tgz", + "integrity": "sha512-fB3Zu92ucau0iQ0JMCFQE7b/dv8Ot07NI3KaZIkIUNXq82k4eBAqUaneXfleGY9JWskeS9y+u0nXMyspcuQrCg==", + "dev": true, + "requires": { + "@types/connect": "*", + "@types/node": "*" + } + }, + "@types/connect": { + "version": "3.4.38", + "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", + "integrity": "sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, + "@types/express": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.0.tgz", + "integrity": "sha512-DvZriSMehGHL1ZNLzi6MidnsDhUZM/x2pRdDIKdwbUNqqwHxMlRdkxtn6/EPKyqKpHqTl/4nRZsRNLpZxZRpPQ==", + "dev": true, + "requires": { + "@types/body-parser": "*", + "@types/express-serve-static-core": "^5.0.0", + "@types/qs": "*", + "@types/serve-static": "*" + } + }, + "@types/express-serve-static-core": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.0.0.tgz", + "integrity": "sha512-AbXMTZGt40T+KON9/Fdxx0B2WK5hsgxcfXJLr5bFpZ7b4JCex2WyQPTEKdXqfHiY5nKKBScZ7yCoO6Pvgxfvnw==", + "dev": true, + "requires": { + "@types/node": "*", + "@types/qs": "*", + "@types/range-parser": "*", + "@types/send": "*" + } + }, + "@types/graphite": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@types/graphite/-/graphite-0.1.2.tgz", + "integrity": "sha512-k+esqcUwtDaAZbTXf6J06Z82ZNbndJohxh9/PqvtWwYtqJZxSzpD7HBR7oN5ADXGE67MbqDCs1+1/TpJxIz2zQ==", + "dev": true + }, + "@types/http-errors": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@types/http-errors/-/http-errors-2.0.4.tgz", + "integrity": "sha512-D0CFMMtydbJAegzOyHjtiKPLlvnm3iTZyZRSZoLq2mRhDdmLfIWOCYPfQJ4cu2erKghU++QvjcUjp/5h7hESpA==", + "dev": true + }, + "@types/mime": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/mime/-/mime-1.3.5.tgz", + "integrity": "sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==", + "dev": true + }, + "@types/node": { + "version": "22.7.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.7.4.tgz", + "integrity": "sha512-y+NPi1rFzDs1NdQHHToqeiX2TIS79SWEAw9GYhkkx8bD0ChpfqC+n2j5OXOCpzfojBEBt6DnEnnG9MY0zk1XLg==", + "dev": true, + "requires": { + "undici-types": "~6.19.2" + } + }, + "@types/node-cron": { + "version": "3.0.11", + "resolved": "https://registry.npmjs.org/@types/node-cron/-/node-cron-3.0.11.tgz", + "integrity": "sha512-0ikrnug3/IyneSHqCBeslAhlK2aBfYek1fGo4bP4QnZPmiqSGRK+Oy7ZMisLWkesffJvQ1cqAcBnJC+8+nxIAg==", + "dev": true + }, + "@types/qs": { + "version": "6.9.16", + "resolved": "https://registry.npmjs.org/@types/qs/-/qs-6.9.16.tgz", + "integrity": "sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A==", + "dev": true + }, + "@types/range-parser": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.7.tgz", + "integrity": "sha512-hKormJbkJqzQGhziax5PItDUTMAM9uE2XXQmM37dyd4hVM+5aVl7oVxMVUiVQn2oCQFN/LKCZdvSM0pFRqbSmQ==", + "dev": true + }, + "@types/send": { + "version": "0.17.4", + "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.4.tgz", + "integrity": "sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==", + "dev": true, + "requires": { + "@types/mime": "^1", + "@types/node": "*" + } + }, + "@types/serve-static": { + "version": "1.15.7", + "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.7.tgz", + "integrity": "sha512-W8Ym+h8nhuRwaKPaDw34QUkwsGi6Rc4yYqvKFo5rm2FUEhCFbzVWrxXUxuKK8TASjWsysJY0nsmNCGhCOIsrOw==", + "dev": true, + "requires": { + "@types/http-errors": "*", + "@types/node": "*", + "@types/send": "*" + } + }, "@types/triple-beam": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.2.tgz", @@ -1634,6 +1878,12 @@ "mime-types": "~2.1.24" } }, + "undici-types": { + "version": "6.19.8", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", + "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==", + "dev": true + }, "unpipe": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", diff --git a/src/pushStats/package.json b/src/pushStats/package.json index 5bff340..4ad6a2f 100644 --- a/src/pushStats/package.json +++ b/src/pushStats/package.json @@ -8,5 +8,11 @@ "winston": "^3.8.1", "winston-daily-rotate-file": "^4.7.1" }, - "type": "module" + "type": "module", + "devDependencies": { + "@types/express": "^5.0.0", + "@types/graphite": "^0.1.2", + "@types/node": "^22.7.4", + "@types/node-cron": "^3.0.11" + } } diff --git a/src/pushStats/types.d.ts b/src/pushStats/types.d.ts new file mode 100644 index 0000000..bcfc0cf --- /dev/null +++ b/src/pushStats/types.d.ts @@ -0,0 +1,13 @@ +type UserType = "mmo"|"season"|"private"; + +interface UserInfo { + type: UserType; + username: string; + host: string; + replaceName: string; + password: string; + token: string; + prefix: string; + segment: number; + shards: string[]; +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..9f1f160 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,109 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "ES2021", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "Node16", /* Specify what module code is generated. */ + // "rootDir": "./", /* Specify the root folder within your source files. */ + // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + "rootDirs": [".", "src/pushStats"], /* Allow multiple folders to be treated as one when resolving modules. */ + "typeRoots": ["./node_modules/@types", "./src/pushStats/node_modules/@types"], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ + // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ + // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ + // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + // "outDir": "./", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +} From c560fcdc1c46edeb2abd698e36e5000f388706a6 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sun, 29 Sep 2024 00:21:41 +0200 Subject: [PATCH 16/19] Remove dotenv There's no .env file in that container, and everything is passed in by the compose file already. --- src/pushStats/apiFunctions.js | 7 ------- src/pushStats/index.js | 3 --- src/pushStats/package-lock.json | 17 ----------------- src/pushStats/package.json | 1 - 4 files changed, 28 deletions(-) diff --git a/src/pushStats/apiFunctions.js b/src/pushStats/apiFunctions.js index 52d662f..6f975bc 100644 --- a/src/pushStats/apiFunctions.js +++ b/src/pushStats/apiFunctions.js @@ -5,9 +5,6 @@ import util from 'util'; import zlib from 'zlib'; import fs from 'fs'; // import users from './users.json' assert {type: 'json'}; -import { fileURLToPath } from 'url'; -import * as dotenv from 'dotenv'; -import { join, dirname } from 'path'; import { createLogger, format, transports } from 'winston'; @@ -16,10 +13,6 @@ import 'winston-daily-rotate-file'; /** @type {UserInfo[]} */ const users = JSON.parse(fs.readFileSync('users.json').toString('utf8')); - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); -dotenv.config({ path: join(__dirname, './.env') }); const needsPrivateHost = users.some((u) => u.type !== 'mmo' && !u.host); const gunzipAsync = util.promisify(zlib.gunzip); diff --git a/src/pushStats/index.js b/src/pushStats/index.js index 852e03d..131b89d 100644 --- a/src/pushStats/index.js +++ b/src/pushStats/index.js @@ -5,8 +5,6 @@ import graphite from 'graphite'; import { createLogger, format, transports } from 'winston'; // eslint-disable-next-line import/no-unresolved import 'winston-daily-rotate-file'; -import fs from 'fs'; -import * as dotenv from 'dotenv'; // eslint-disable-next-line import/no-unresolved import express from 'express'; import ApiFunc from './apiFunctions.js'; @@ -17,7 +15,6 @@ let lastUpload = new Date().getTime(); /** @type {UserInfo[]} */ const users = JSON.parse(fs.readFileSync('users.json').toString('utf8')); -dotenv.config(); const pushTransport = new transports.DailyRotateFile({ filename: 'logs/push-%DATE%.log', diff --git a/src/pushStats/package-lock.json b/src/pushStats/package-lock.json index 55dc2d1..b654e04 100644 --- a/src/pushStats/package-lock.json +++ b/src/pushStats/package-lock.json @@ -6,7 +6,6 @@ "": { "dependencies": { "axios": "^0.27.2", - "dotenv": "^16.0.2", "express": "^4.18.2", "graphite": "^0.1.4", "node-cron": "^3.0.1", @@ -360,17 +359,6 @@ "npm": "1.2.8000 || >= 1.4.16" } }, - "node_modules/dotenv": { - "version": "16.3.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", - "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/motdotla/dotenv?sponsor=1" - } - }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -1396,11 +1384,6 @@ "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" }, - "dotenv": { - "version": "16.3.1", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.3.1.tgz", - "integrity": "sha512-IPzF4w4/Rd94bA9imS68tZBaYyBWSCE47V1RGuMrB94iyTOIEwRmVL2x/4An+6mETpLrKJ5hQkB8W4kFAadeIQ==" - }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", diff --git a/src/pushStats/package.json b/src/pushStats/package.json index 4ad6a2f..c87dfec 100644 --- a/src/pushStats/package.json +++ b/src/pushStats/package.json @@ -1,7 +1,6 @@ { "dependencies": { "axios": "^0.27.2", - "dotenv": "^16.0.2", "express": "^4.18.2", "graphite": "^0.1.4", "node-cron": "^3.0.1", From 3818fae916bb466bf4f7e68a19992dc6ef979ca2 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sun, 29 Sep 2024 00:42:24 +0200 Subject: [PATCH 17/19] Another big batch of changes User loading has been moved to its own file and validated for correctness; now, an user definition will be complete (there's a valid username, replaceName, host/port based on type, token/password), so that the rest of the code can just use those values. Now each time the cron triggers, the users file is parsed again, and users marked as type: "private" with nothing else will use a local Screeps server automatically if found; use an explicit host/port to make them static. --- .eslintrc.js | 1 + src/pushStats/apiFunctions.js | 94 +++-------------- src/pushStats/index.js | 184 +++++++++++++++------------------- src/pushStats/types.d.ts | 1 + src/pushStats/users.js | 105 +++++++++++++++++++ 5 files changed, 203 insertions(+), 182 deletions(-) create mode 100644 src/pushStats/users.js diff --git a/.eslintrc.js b/.eslintrc.js index defab5b..5b95e04 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -17,5 +17,6 @@ module.exports = { "no-underscore-dangle":"off", "no-param-reassign": ["error", { "props": false }], "no-restricted-syntax": "off", + "no-continue": "off", } } diff --git a/src/pushStats/apiFunctions.js b/src/pushStats/apiFunctions.js index 6f975bc..334190a 100644 --- a/src/pushStats/apiFunctions.js +++ b/src/pushStats/apiFunctions.js @@ -1,20 +1,13 @@ import http from 'http'; import https from 'https'; -import net from 'net'; import util from 'util'; import zlib from 'zlib'; -import fs from 'fs'; -// import users from './users.json' assert {type: 'json'}; import { createLogger, format, transports } from 'winston'; // eslint-disable-next-line import/no-unresolved import 'winston-daily-rotate-file'; -/** @type {UserInfo[]} */ -const users = JSON.parse(fs.readFileSync('users.json').toString('utf8')); -const needsPrivateHost = users.some((u) => u.type !== 'mmo' && !u.host); - const gunzipAsync = util.promisify(zlib.gunzip); const { combine, timestamp, prettyPrint } = format; @@ -68,65 +61,6 @@ function removeNonNumbers(obj) { return obj; } -/** @type {string | undefined} */ -let privateHost; -let serverPort = 21025; - -function getPrivateHost() { - serverPort = parseInt(process.env.SERVER_PORT, 10) || 21025; - const hosts = [ - 'localhost', - 'host.docker.internal', - '172.17.0.1', - ]; - for (let h = 0; h < hosts.length; h += 1) { - const host = hosts[h]; - const sock = new net.Socket(); - sock.setTimeout(2500); - // eslint-disable-next-line no-loop-func - sock.on('connect', () => { - sock.destroy(); - privateHost = host; - }) - .on('error', () => { - sock.destroy(); - }) - .on('timeout', () => { - sock.destroy(); - }) - .connect(serverPort, host); - } -} - -async function TryToGetPrivateHost() { - if (!privateHost && needsPrivateHost) { - getPrivateHost(); - if (!privateHost) console.log('No private host found to make connection with yet! Trying again in 60 seconds.'); - else console.log(`Private host found! Continuing with ${privateHost}.`); - - // eslint-disable-next-line - await new Promise((resolve) => setTimeout(resolve, 60 * 1000)); - TryToGetPrivateHost(); - } -} - -if (!privateHost && needsPrivateHost) { - TryToGetPrivateHost(); -} - -/** - * - * @param {string} host - * @param {UserType} type - * @returns - */ -async function getHost(host, type) { - if (type === 'mmo') return 'screeps.com'; - if (type === 'season') return 'screeps.com/season'; - if (host) return host; - return privateHost; -} - /** * * @param {Omit} */ const headers = { 'Content-Type': 'application/json', @@ -146,8 +80,8 @@ async function getRequestOptions(info, path, method = 'GET', body = {}) { if (info.username) headers['X-Username'] = info.username; if (info.token) headers['X-Token'] = info.token; return { - host: await getHost(info.host, info.type), - port: info.type === 'mmo' ? 443 : serverPort, + host: info.host, + port: info.port, path, method, headers, @@ -220,7 +154,7 @@ export default class { * @returns */ static async getPrivateServerToken(info) { - const options = await getRequestOptions(info, '/api/auth/signin', 'POST', { + const options = getRequestOptions(info, '/api/auth/signin', 'POST', { email: info.username, password: info.password, }); @@ -237,7 +171,7 @@ export default class { * @returns */ static async getMemory(info, shard, statsPath = 'stats') { - const options = await getRequestOptions(info, `/api/user/memory?path=${statsPath}&shard=${shard}`, 'GET'); + const options = getRequestOptions(info, `/api/user/memory?path=${statsPath}&shard=${shard}`, 'GET'); const res = await req(options); if (!res) { @@ -255,7 +189,7 @@ export default class { * @returns */ static async getSegmentMemory(info, shard) { - const options = await getRequestOptions(info, `/api/user/memory-segment?segment=${info.segment}&shard=${shard}`, 'GET'); + const options = getRequestOptions(info, `/api/user/memory-segment?segment=${info.segment}&shard=${shard}`, 'GET'); const res = await req(options); if (!res || res.data == null) return {}; try { @@ -272,7 +206,7 @@ export default class { * @returns */ static async getUserinfo(info) { - const options = await getRequestOptions(info, '/api/auth/me', 'GET'); + const options = getRequestOptions(info, '/api/auth/me', 'GET'); const res = await req(options); return res; } @@ -283,7 +217,7 @@ export default class { * @returns */ static async getLeaderboard(info) { - const options = await getRequestOptions(info, `/api/leaderboard/find?username=${info.username}&mode=world`, 'GET'); + const options = getRequestOptions(info, `/api/leaderboard/find?username=${info.username}&mode=world`, 'GET'); const res = await req(options); return res; } @@ -291,11 +225,11 @@ export default class { /** * * @param {string | undefined} host + * @param {number} port * @returns */ - static async getServerStats(host) { - const serverHost = host || privateHost; - const options = await getRequestOptions(/** @type {UserInfo} */ ({ host: serverHost }), '/api/stats/server', 'GET'); + static async getServerStats(host, port) { + const options = getRequestOptions(/** @type {UserInfo} */ ({ host, port }), '/api/stats/server', 'GET'); const res = await req(options); if (!res || !res.users) { logger.error(res); @@ -307,11 +241,11 @@ export default class { /** * * @param {string | undefined} host + * @param {number} port * @returns */ - static async getAdminUtilsServerStats(host) { - const serverHost = host || privateHost; - const options = await getRequestOptions(/** @type {UserInfo} */ ({ host: serverHost }), '/stats', 'GET'); + static async getAdminUtilsServerStats(host, port) { + const options = getRequestOptions(/** @type {UserInfo} */ ({ host, port }), '/stats', 'GET'); const res = await req(options); if (!res || !res.gametime) { logger.error(res); diff --git a/src/pushStats/index.js b/src/pushStats/index.js index 131b89d..6f6c4d9 100644 --- a/src/pushStats/index.js +++ b/src/pushStats/index.js @@ -8,14 +8,12 @@ import 'winston-daily-rotate-file'; // eslint-disable-next-line import/no-unresolved import express from 'express'; import ApiFunc from './apiFunctions.js'; +import loadUsers from './users.js'; const app = express(); const pushStatusPort = Number(process.env.PUSH_STATUS_PORT); let lastUpload = new Date().getTime(); -/** @type {UserInfo[]} */ -const users = JSON.parse(fs.readFileSync('users.json').toString('utf8')); - const pushTransport = new transports.DailyRotateFile({ filename: 'logs/push-%DATE%.log', auditFile: 'logs/push-audit.json', @@ -61,23 +59,24 @@ class ManageStats { /** * - * @param {"mmo"|"season"|"private"} type + * @param {string} host + * @param {UserInfo[]} hostUsers * @returns */ - async handleUsers(type) { - console.log(`[${type}] Handling Users`); + async handleUsers(host, hostUsers) { + console.log(`[${host}] Handling Users`); const beginningOfMinute = new Date().getSeconds() < 15; - /** @type {(string | unknown)[]} */ + /** @type {(Promise)[]} */ const getStatsFunctions = []; - users.forEach((user) => { + for (const user of hostUsers) { try { - if (user.type !== type) return; + if (user.host !== host) continue; const rightMinuteForShard = new Date().getMinutes() % user.shards.length === 0; const shouldContinue = !beginningOfMinute || !rightMinuteForShard; - if (user.type === 'mmo' && shouldContinue) return; - if (user.type === 'season' && shouldContinue) return; + if (user.type === 'mmo' && shouldContinue) continue; + if (user.type === 'season' && shouldContinue) continue; for (const shard of user.shards) { getStatsFunctions.push(this.getStats(user, shard)); @@ -85,88 +84,59 @@ class ManageStats { } catch (error) { logger.error(error); } - }); + } - console.log(`[${type}] Getting ${getStatsFunctions.length} statistics`); + console.log(`[${host}] Getting ${getStatsFunctions.length} statistics`); await Promise.all(getStatsFunctions); - const { groupedStats } = this; - - if (type === 'mmo' || type === 'season') { - if (Object.keys(groupedStats).length > 0) { - const push = await ManageStats.reportStats({ stats: groupedStats }); - if (push) { - console.log('Error while pushing stats'); - } else { - console.log(`[${type}] Pushed stats to graphite`); - } - return; - } + /** @type {Record} */ + const stats = { + stats: this.groupedStats, + }; - if (beginningOfMinute) { - console.log('No stats to push'); - } - return; - } + if (!host.startsWith('screeps.com')) { + const serverStats = await ApiFunc.getServerStats(host, hostUsers[0].port); + const adminUtilsServerStats = await ApiFunc.getAdminUtilsServerStats(host, hostUsers[0].port); + if (adminUtilsServerStats) { + try { + /** @type {Record} */ + const groupedAdminStatsUsers = {}; + for (const [username, user] of Object.entries(adminUtilsServerStats)) { + groupedAdminStatsUsers[username] = user; + } - const privateUser = users.find((user) => user.type === 'private' && user.host); - const host = privateUser ? privateUser.host : undefined; - const serverStats = await ApiFunc.getServerStats(host); - const adminUtilsServerStats = await ApiFunc.getAdminUtilsServerStats(host); - if (adminUtilsServerStats) { - try { - /** @type {Record} */ - const groupedAdminStatsUsers = {}; - for (const [username, user] of Object.entries(adminUtilsServerStats)) { - groupedAdminStatsUsers[username] = user; + adminUtilsServerStats.users = groupedAdminStatsUsers; + } catch (error) { + console.log(error); } - - adminUtilsServerStats.users = groupedAdminStatsUsers; - } catch (error) { - console.log(error); } + console.log(`[${host}] Server stats: ${serverStats ? 'yes' : 'no'}, adminUtils: ${adminUtilsServerStats ? 'yes' : 'no'}`); + stats.serverStats = serverStats; + stats.adminUtilsServerStats = adminUtilsServerStats; } - const stats = { - stats: groupedStats, - serverStats, - adminUtilsServerStats, - }; - const push = await ManageStats.reportStats(stats); if (!push) { - console.log('Error while pushing stats'); + console.log(`[${host}] Error while pushing stats`); return; } /** @type {string[]} */ const typesPushed = []; - if (Object.keys(groupedStats).length > 0) { - typesPushed.push(type); + if (Object.keys(stats.stats).length > 0) { + typesPushed.push(host); } - if (serverStats) { + if (stats.serverStats) { typesPushed.push('server stats'); } - if (adminUtilsServerStats) { - typesPushed.push('adminUtilsServerStats'); + if (stats.adminUtilsServerStats) { + typesPushed.push('admin-utils stats'); } if (typesPushed.length) { - logger.info(`> Pushed ${typesPushed.join(', ')} to graphite`); + logger.info(`> [${host}] Pushed ${typesPushed.join(', ')}`); } else { - logger.info('> Pushed no stats to graphite'); - } - } - - /** - * - * @param {UserInfo} userinfo - * @returns - */ - static async getLoginInfo(userinfo) { - if (userinfo.type === 'private') { - userinfo.token = await ApiFunc.getPrivateServerToken(userinfo); + logger.info(`> [${host}] Pushed no stats`); } - return userinfo.token; } /** @@ -190,32 +160,31 @@ class ManageStats { /** * * @param {UserInfo} userinfo - * @param {string} shard - * @returns {Promise} + * @returns */ - async getStats(userinfo, shard) { - try { - await ManageStats.getLoginInfo(userinfo); - const stats = userinfo.segment === undefined - ? await ApiFunc.getMemory(userinfo, shard) - : await ApiFunc.getSegmentMemory(userinfo, shard); - - await this.processStats(userinfo, shard, stats); - return 'success'; - } catch (error) { - return error; + static async getLoginInfo(userinfo) { + if (userinfo.type === 'private') { + userinfo.token = await ApiFunc.getPrivateServerToken(userinfo); } + return userinfo.token; } /** * * @param {UserInfo} userinfo * @param {string} shard - * @param {*} stats - * @returns + * @returns {Promise} */ - async processStats(userinfo, shard, stats) { + async getStats(userinfo, shard) { + await ManageStats.getLoginInfo(userinfo); + const stats = userinfo.segment === undefined + ? await ApiFunc.getMemory(userinfo, shard) + : await ApiFunc.getSegmentMemory(userinfo, shard); + if (Object.keys(stats).length === 0) return; + + console.log(`Got memory from ${userinfo.username} in ${shard}`); + const me = await ApiFunc.getUserinfo(userinfo); if (me) stats.power = me.power || 0; stats.leaderboard = await ManageStats.addLeaderboardData(userinfo); @@ -229,7 +198,10 @@ class ManageStats { */ static async reportStats(stats) { return new Promise((resolve) => { - console.log(`Writing stats ${JSON.stringify(stats)} to graphite`); + if (Object.keys(stats).length === 0) { + resolve(false); + } + console.debug(`Writing stats ${JSON.stringify(stats)}`); client.write({ [`${process.env.PREFIX ? `${process.env.PREFIX}.` : ''}screeps`]: stats }, (err) => { if (err) { console.log(err); @@ -250,28 +222,36 @@ class ManageStats { * @returns */ pushStats(userinfo, stats, shard) { - if (Object.keys(stats).length === 0) return; - const username = userinfo.replaceName !== undefined ? userinfo.replaceName : userinfo.username; - this.groupedStats[(userinfo.prefix ? `${userinfo.prefix}.` : '') + username] = { [shard]: stats }; + const statSize = Object.keys(stats).length; + if (statSize === 0) return; + const username = userinfo.replaceName ? userinfo.replaceName : userinfo.username; + const userStatsKey = (userinfo.prefix ? `${userinfo.prefix}.` : '') + username; - console.log(`Pushing stats for ${(userinfo.prefix ? `${userinfo.prefix}.` : '') + username} in ${shard}`); + console.log(`[${userinfo.host}] Pushing ${statSize} stats for ${userStatsKey} in ${shard}`); + if (!this.groupedStats[userStatsKey]) { + this.groupedStats[userStatsKey] = { [shard]: stats }; + } else { + this.groupedStats[userStatsKey][shard] = stats; + } } } -const groupedUsers = users.reduce((group, user) => { - const { type } = user; - // eslint-disable-next-line no-param-reassign - group[type] = group[type] ?? []; - group[type].push(user); - return group; -}, /** @type {Record} */ ({})); - cron.schedule('*/30 * * * * *', async () => { console.log(`Cron event hit: ${new Date()}`); cronLogger.info(`Cron event hit: ${new Date()}`); - Object.keys(groupedUsers).forEach((type) => { - new ManageStats().handleUsers(/** @type {UserType} */(type)); - }); + /** @type {UserInfo[]} */ + const users = await loadUsers(); + + const usersByHost = users.reduce((group, user) => { + const { host } = user; + group[host] = group[host] ?? []; + group[host].push(user); + return group; + }, /** @type {Record} */ ({})); + + for (const [host, usersForHost] of Object.entries(usersByHost)) { + new ManageStats().handleUsers(host, usersForHost); + } }); if (pushStatusPort) { diff --git a/src/pushStats/types.d.ts b/src/pushStats/types.d.ts index bcfc0cf..2fd4cca 100644 --- a/src/pushStats/types.d.ts +++ b/src/pushStats/types.d.ts @@ -4,6 +4,7 @@ interface UserInfo { type: UserType; username: string; host: string; + port: number; replaceName: string; password: string; token: string; diff --git a/src/pushStats/users.js b/src/pushStats/users.js new file mode 100644 index 0000000..ef3b781 --- /dev/null +++ b/src/pushStats/users.js @@ -0,0 +1,105 @@ +import fs from 'fs'; +import net from 'net'; + +const SERVER_PORT = parseInt(/** @type {string} */ (process.env.SERVER_PORT), 10) ?? 21025; + +/** + * Check whether there's a server nearby + * @returns {Promise<[string, number]>} + */ +async function checkLocalhostServer() { + const hosts = [ + 'localhost', + 'host.docker.internal', + '172.17.0.1', + ]; + /** @type {Promise<[string, number]>[]} */ + const promises = []; + for (const host of hosts) { + const p = new Promise((resolve, reject) => { + const sock = new net.Socket(); + sock.setTimeout(2500); + console.log(`[${host}:${SERVER_PORT}] attempting connection`); + sock + .on('connect', () => { + sock.destroy(); + resolve([host, SERVER_PORT]); + }) + .on('error', () => { + sock.destroy(); + reject(); + }) + .on('timeout', () => { + sock.destroy(); + reject(); + }) + .connect(SERVER_PORT, host); + }); + promises.push(p); + } + return Promise.any(promises); +} + +/** + * + * @param {UserType} type + * @returns {[string, number]} + */ +function getHostInfoFromType(type) { + switch (type) { + case 'mmo': + return ['screeps.com', 443]; + case 'season': + return ['screeps.com/season', 443]; + default: + throw new Error(`no idea what type ${type} is`); + } +} + +export default async function loadUsers() { + /** @type {UserInfo[]} */ + const users = JSON.parse(fs.readFileSync('users.json').toString('utf8')); + /** @type {UserInfo[]} */ + const validUsers = []; + const localServer = await checkLocalhostServer(); + for (const user of users) { + if (typeof user.username !== 'string' || user.username.length <= 0) { + console.log('Missing username!'); + continue; + } + if (user.username.includes('.') && !user.replaceName) { + // Just yank the dot from the name + user.replaceName = user.username.replace(/\./g, ''); + } + if (user.type && !['mmo', 'season', 'private'].includes(user.type)) { + console.log(`Invalid type for user ${user.username}, ignoring.`); + continue; + } + if (!user.host) { + try { + if (user.type === 'private') { + [user.host, user.port] = localServer; + } else { + [user.host, user.port] = getHostInfoFromType(user.type); + } + } catch { + console.log(`Cannot get host for user ${user.username}, ignoring.`); + continue; + } + } + if (!user.host || !user.port) { + console.log(`Missing host or port for user ${user.username}, ignoring.`); + continue; + } + if (!user.password && !user.token) { + console.log(`Missing password or token for user ${user.username}, ignoring.`); + continue; + } + if (!Array.isArray(user.shards) || !user.shards.every((s) => typeof s === 'string')) { + console.log(`Missing or invalid shard for user ${user.username}, ignoring.`); + continue; + } + validUsers.push(user); + } + return validUsers; +} From 7bb16e34ad7be7a75047e610bafb2c60fcf13e7b Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sun, 29 Sep 2024 01:00:22 +0200 Subject: [PATCH 18/19] Update README --- README.md | 39 ++------------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index 9002bb9..5b863ce 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ ## Setup -1. Edit `example.env` and `docker-compose.example.yml` to match your needs. This step is not required if you are using the default setup. +1. Copy `.env.example` to `.env` and edit to match your needs. 2. Copy `users.example.json` to `users.json` and edit it according to [User Setup](#User-Setup). 3. The configuration files for both Grafana and Graphite are in `config/grafana` and `config/graphite` respectively. 4. If you have a dashboard you want to auto-add, you can drop their JSON files into `config/grafana/provisioning/dashboards` @@ -78,6 +78,7 @@ If the private server is not hosted on localhost, add the host to the user: "shards": ["screeps"], "password": "password", "host": "192.168.1.10", + "port": 21025, } ``` @@ -93,39 +94,3 @@ If the segment of the stats is not memory, add it to the user: "segment": 0, } ``` - -Update all .example files and/or folders to match your needs. This step is not required if you are using the default setup. - -### Run Commands - -#### Config - -* `--force`: force the non .example config files to be overwritten. -* `--debug`: listen to setup Docker logs -* `--username`: overwrite the username for the Grafana admin user -* `--password`: overwrite the password for the Grafana admin user -* `--enableAnonymousAccess`: enable anonymous access to Grafana - -#### Network - -* `--grafanaDomain`: Overwrite grafana.ini domain -* `--grafanaPort`: port for Grafana to run on -* `--relayPort`: port for relay-ng to run on (default: 2003) -* `--pushStatusPort`: port for the stats-getter push API (default: false) - true will set it to 10004, otherwise specify a port number it'll listen to - -#### Exporting - -* `--deleteLogs`: deletes the logs folder -* `--removeWhisper`: Deletes the carbon whisper folder -* `--removeVolumes`: Remove all volumes, including the grafana database. - -## Usage - -* `npm run setup`: to execute setup only -* `npm run start`: to configure and start it -* For other run commands like eslint, check out package.json scripts object. - -Go to [localhost:3000](http://localhost:3000) (if you used port 3000) and login with `admin` and `password` (or your custom set login info). - -Its possible to use https for your grafana instance, check out this [tutorial](https://www.turbogeek.co.uk/grafana-how-to-configure-ssl-https-in-grafana/) for example on how to do this, enough info online about it. I dont support this (yet) From 4646d868dff35e9a779c4ac76deae92bb1579170 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sun, 29 Sep 2024 01:41:47 +0200 Subject: [PATCH 19/19] Fix linting --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index c05b510..b3af475 100644 --- a/package.json +++ b/package.json @@ -9,8 +9,8 @@ "reset": "docker compose down", "reset:hard": "docker compose down -v", "rebuild": "docker compose build --no-cache && docker compose down stats-getter && docker compose up -d stats-getter", - "lint": "eslint src/**/*.js && eslint dashboards/**/*.js", - "lint:fix": "eslint src/**/*.js --fix && eslint dashboards/**/*.js --fix" + "lint": "eslint src/**/*.js", + "lint:fix": "eslint src/**/*.js --fix" }, "dependencies": { "axios": "^0.27.2",