Merge pull request #1130 from AletheiaFact/service-runner-removal

Remove service-runner dependency and refactor code accordingly.

.env.example

@@ -35,7 +35,7 @@ SMTP_REFRESH_TOKEN=
 
 # FEATURE FLAG #######
 GITLAB_FEATURE_FLAG_URL=
-ENV=
+ENV=local
 GITLAB_FEATURE_FLAG_INSTANCE_ID=
 # AWS
 AWS_SDK_BUCKET=aletheia

config.development.yaml

@@ -1,36 +1,5 @@
-# Number of worker processes to spawn.
-# Set to 0 to run everything in a single process without clustering.
-# Use 'ncpu' to run as many workers as there are CPU units
-num_workers: 2
-
-# Number of workers to start in parallel after the first worker.
-# The first worker is always started independently. After it has completed
-# its start-up, this number controls the number of workers to start in
-# parallel until `num_workers` have been started. Note that setting this
-# number to a too high a value might lead to high resource consumption
-# (especially of CPU) during the start-up process.
-startup_concurrency: 1
-
-# Log error messages and gracefully restart a worker if v8 reports that it
-# uses more heap (note: not RSS) than this many mb.
-worker_heap_limit_mb: 750
-
-# The maximum interval in ms that can pass between two beat messages
-# sent by each worker to the master before it is killed
-worker_heartbeat_timeout: 15000
-
-# Logger info
-logging:
-  level: trace
-  streams:
-  - type: stdout # log to stdout
-    named_levels: true # emit log level name instead of index. e.g. INFO vs 30
-
-# Using config.yaml with wikimedia service-runner pattern
 services:
   - name: service-aletheia
-    # path relative to the __dirname variable
-    module: dist/server/main.js
     conf:
       port: 3000
       cors: '*'

config.example.yaml

@@ -1,23 +1,5 @@
-# Number of worker processes to spawn.
-# Use 'ncpu' to run as many workers as there are CPU units
-num_workers: 1
-
-# Log error messages and gracefully restart a worker if v8 reports that it
-# uses more heap (note: not RSS) than this many mb.
-# worker_heap_limit_mb: 250
-
-# Logger info
-logging:
-  level: trace
-  streams:
-  - type: stdout # log to stdout
-    named_levels: true # emit log level name instead of index. e.g. INFO vs 30
-
-# Using config.yaml with wikimedia service-runner pattern
 services:
   - name: service-aletheia
-    # path relative to the __dirname variable
-    module: dist/server/main.js
     conf:
       port: 3000
       cors: '*'

config.production.yaml

@@ -1,36 +1,5 @@
-# Number of worker processes to spawn.
-# Set to 0 to run everything in a single process without clustering.
-# Use 'ncpu' to run as many workers as there are CPU units
-num_workers: 2
-
-# Number of workers to start in parallel after the first worker.
-# The first worker is always started independently. After it has completed
-# its start-up, this number controls the number of workers to start in
-# parallel until `num_workers` have been started. Note that setting this
-# number to a too high a value might lead to high resource consumption
-# (especially of CPU) during the start-up process.
-startup_concurrency: 1
-
-# Log error messages and gracefully restart a worker if v8 reports that it
-# uses more heap (note: not RSS) than this many mb.
-worker_heap_limit_mb: 750
-
-# The maximum interval in ms that can pass between two beat messages
-# sent by each worker to the master before it is killed
-worker_heartbeat_timeout: 15000
-
-# Logger info
-logging:
-  level: trace
-  streams:
-  - type: stdout # log to stdout
-    named_levels: true # emit log level name instead of index. e.g. INFO vs 30
-
-# Using config.yaml with wikimedia service-runner pattern
 services:
   - name: service-aletheia
-    # path relative to the __dirname variable
-    module: dist/server/main.js
     conf:
       port: 3000
       cors: '*'

config.seed.example.yaml

@@ -1,20 +1,5 @@
-# Number of worker processes to spawn.
-# Set to 0 to run everything in a single process without clustering.
-# Use 'ncpu' to run as many workers as there are CPU units
-num_workers: 1
-
-# Logger info
-logging:
-  level: trace
-  streams:
-    - type: stdout # log to stdout
-      named_levels: true # emit log level name instead of index. e.g. INFO vs 30
-
-# Using config.yaml with wikimedia service-runner pattern
 services:
   - name: service-aletheia
-    # path relative to the __dirname variable
-    module: dist/server/seed.js
     conf:
       web_url: https://test.aletheiafact.org
       smtp_host: {env(SMTP_CLIENT_HOST)}

config.seed.test.ci.yaml

@@ -1,20 +1,5 @@
-# Number of worker processes to spawn.
-# Set to 0 to run everything in a single process without clustering.
-# Use 'ncpu' to run as many workers as there are CPU units
-num_workers: 0
-
-# Logger info
-logging:
-  level: trace
-  streams:
-    - type: stdout # log to stdout
-      named_levels: true # emit log level name instead of index. e.g. INFO vs 30
-
-# Using config.yaml with wikimedia service-runner pattern
 services:
   - name: ci-seed-command
-    # path relative to the __dirname variable
-    module: dist/server/seed.js
     conf:
       disable_smtp: true
       db:

config.test.ci.yaml

@@ -1,28 +1,5 @@
-# Number of worker processes to spawn.
-# Use 'ncpu' to run as many workers as there are CPU units
-num_workers: 0
-
-# Log error messages and gracefully restart a worker if v8 reports that it
-# uses more heap (note: not RSS) than this many mb.
-# worker_heap_limit_mb: 250
-
-# Logger info
-logging:
-  level: trace
-  streams:
-  - type: stdout # log to stdout
-    named_levels: true # emit log level name instead of index. e.g. INFO vs 30
-
-# Using config.yaml with wikimedia service-runner pattern
 services:
-  - name: ci-websocket-aletheia
-    module: dist/server/websocket.js
-    conf:
-      port: 5001
-
   - name: ci-aletheia-server
-    # path relative to the __dirname variable
-    module: dist/server/main.js
     conf:
       port: 3000
       cors: '*'

config.websocket.development.yaml

@@ -1,22 +1,4 @@
-# Number of worker processes to spawn.
-# Use 'ncpu' to run as many workers as there are CPU units
-num_workers: 0
-
-# Log error messages and gracefully restart a worker if v8 reports that it
-# uses more heap (note: not RSS) than this many mb.
-# worker_heap_limit_mb: 250
-
-# Logger info
-logging:
-  level: trace
-  streams:
-  - type: stdout # log to stdout
-    named_levels: true # emit log level name instead of index. e.g. INFO vs 30
-
-# Using config.yaml with wikimedia service-runner pattern
 services:
   - name: websocket-aletheia-development
-    # path relative to the __dirname variable
-    module: dist/server/websocket.js
     conf:
       port: 5051

config.websocket.production.yaml

@@ -1,22 +1,4 @@
-# Number of worker processes to spawn.
-# Use 'ncpu' to run as many workers as there are CPU units
-num_workers: 0
-
-# Log error messages and gracefully restart a worker if v8 reports that it
-# uses more heap (note: not RSS) than this many mb.
-# worker_heap_limit_mb: 250
-
-# Logger info
-logging:
-  level: trace
-  streams:
-  - type: stdout # log to stdout
-    named_levels: true # emit log level name instead of index. e.g. INFO vs 30
-
-# Using config.yaml with wikimedia service-runner pattern
 services:
   - name: websocket-aletheia-development
-    # path relative to the __dirname variable
-    module: dist/server/websocket.js
     conf:
       port: 5051

package.json

@@ -6,10 +6,10 @@
   "main": "./dist/server/main.js",
   "scripts": {
     "prepare": "husky install",
-    "start": "service-runner",
+    "start": "node dist/server/main.js",
     "copy-html-files": "copyfiles -u 1 server/**/*.html dist/server/",
-    "seed": "yarn build-ts && yarn build-nest && yarn copy-html-files && node dist/server/command.js -c config.seed.yaml",
-    "seed:ci": "yarn build-ts && yarn build-nest && yarn copy-html-files && node dist/server/command.js -c config.seed.test.ci.yaml",
+    "seed": "yarn build-ts && yarn build-nest && yarn copy-html-files && node dist/server/seed.js -c config.seed.yaml",
+    "seed:ci": "yarn build-ts && yarn build-nest && yarn copy-html-files && node dist/server/seed.js -c config.seed.test.ci.yaml",
     "dev": "yarn copy-static-assets && concurrently -k -p \"[{name}]\" -n \"TypeScript,Node\" -c \"cyan.bold,green.bold\" \"yarn watch-ts\" \"yarn watch-node\"",
     "test": "env-cmd --silent jest --detectOpenHandles --forceExit",
     "build": "yarn build-ts && yarn build-nest && yarn copy-static-assets && yarn copy-html-files && yarn build-next",
@@ -23,8 +23,8 @@
     "lint:fix": "tsc --noEmit && eslint \"**/*.{js,ts,tsx}\" --quiet --fix",
     "copy-static-assets": "ts-node scripts/copyStaticAssets.ts",
     "watch-debug": "yarn copy-static-assets && concurrently -k -p \"[{name}]\" -n \"TypeScript,Node\" -c \"cyan.bold,green.bold\" \"yarn watch-ts\" \"yarn serve-debug\"",
-    "watch-node": "nodemon dist/server/server.js  --signal SIGTERM",
-    "serve-debug": "nodemon --inspect dist/server/server.js  --signal SIGTERM",
+    "watch-node": "nodemon dist/server/main.js  --signal SIGTERM",
+    "serve-debug": "nodemon --inspect dist/server/main.js  --signal SIGTERM",
     "clean": "rimraf **/{node_modules,dist}",
     "compodoc": "npx compodoc -p tsconfig.json -s -d ./docs/compodoc",
     "proxy": "ory proxy --no-jwt --port 4000 http://localhost:3000/",
@@ -149,7 +149,6 @@
     "rehype-raw": "^6.1.1",
     "remirror": "^2.0.11",
     "rxjs": "^7.3.0",
-    "service-runner": "^2.8.4",
     "set-cookie-parser": "^2.4.8",
     "sitemap": "5",
     "slugify": "^1.6.1",
@@ -246,5 +245,8 @@
       "eslint --fix"
     ]
   },
+  "resolutions": {
+    "@types/mime": "3.0.4"
+  },
   "packageManager": "[email protected]"
 }

server/app.module.ts

@@ -63,19 +63,17 @@ export class AppModule implements NestModule {
     }
 
     static register(options): DynamicModule {
-        // TODO: interface app with service-runner metrics interface
-
         const imports = [
             MongooseModule.forRoot(
-                options.config.db.connection_uri,
-                options.config.db.options
+                options.db.connection_uri,
+                options.db.options
             ),
             ConfigModule.forRoot({
-                load: [() => options.config || {}],
+                load: [() => options || {}],
             }),
             ThrottlerModule.forRoot({
-                ttl: options.config.throttle.ttl,
-                limit: options.config.throttle.limit,
+                ttl: options.throttle.ttl,
+                limit: options.throttle.limit,
             }),
             UsersModule,
             WikidataModule,
@@ -110,16 +108,16 @@ export class AppModule implements NestModule {
             CommentModule,
             NameSpaceModule,
         ];
-        if (options.config.feature_flag) {
+        if (options.feature_flag) {
             imports.push(
                 UnleashModule.forRoot({
-                    url: options.config.feature_flag.url,
-                    appName: options.config.feature_flag.appName,
-                    instanceId: options.config.feature_flag.instanceId,
+                    url: options.feature_flag.url,
+                    appName: options.feature_flag.appName,
+                    instanceId: options.feature_flag.instanceId,
                 })
             );
         }
-        if (options.config.novu) {
+        if (options.novu) {
             imports.push(NotificationModule);
         }
         return {

server/configLoader.ts

@@ -0,0 +1,38 @@
+const fs = require("fs");
+const yaml = require("js-yaml");
+
+// Function extracted from service-runner codebase
+function _replaceEnvVars(config) {
+    const envRegex = /{\s*env\(([^,\s)]+),?\s*([^)]+)?\)\s*}/g;
+    if (Buffer.isBuffer(config)) {
+        config = config.toString();
+    }
+    return config.replace(envRegex, (match, envName, defValue) => {
+        if (process.env[envName] !== undefined) {
+            return process.env[envName];
+        }
+        if (defValue !== undefined) {
+            return defValue;
+        }
+        return "";
+    });
+}
+function loadConfig() {
+    const defaultConfigFilePath = "config.yaml";
+    const configFileIndex = process.argv.findIndex((arg) => arg === "-c");
+    const configFilePath =
+        configFileIndex !== -1 ? process.argv[configFileIndex + 1] : null;
+
+    let configContent = fs.readFileSync(
+        configFilePath || defaultConfigFilePath,
+        "utf8"
+    );
+    configContent = _replaceEnvVars(configContent);
+
+    const doc = yaml.load(configContent);
+    const { conf: options } = doc.services[0];
+
+    return options;
+}
+
+export default loadConfig;