From 91161443d5dc0f0023f16c66a0f38c03a9eae7cf Mon Sep 17 00:00:00 2001 From: Svetoslav Petkov Date: Fri, 25 Oct 2024 14:57:38 +0300 Subject: [PATCH] Whiteboard event tracking prototype (#40) * accommodate new excalidraw version * version bump * npm audit fix * wip * wip * wip * proper logging modes * rabbit suggestions * removed unused import * suggestions * changed a variable name * version bump * audit fix --- config.yml | 20 +++ package-lock.json | 169 +++++++++++++++--- package.json | 3 +- src/config/config.type.ts | 20 +++ src/config/index.ts | 1 + src/config/whiteboard.event.logging.mode.ts | 8 + .../elastic.response.error.ts | 25 +++ .../elasticsearch.client.factory.ts | 56 ++++++ .../elasticsearch.client.injection.token.ts | 1 + .../elasticsearch.client.provider.ts | 13 ++ .../handle.elastic.error.ts | 47 +++++ src/elasticsearch-client/index.ts | 6 + src/elasticsearch-client/is.elastic.error.ts | 6 + .../is.elastic.response.error.ts | 8 + src/excalidraw-backend/server.ts | 14 +- .../utils/reconcile.files.ts | 1 + src/excalidraw/types/excalidraw.element.ts | 3 + .../elasticsearch/elasticsearch.module.ts | 10 ++ .../elasticsearch/elasticsearch.service.ts | 163 +++++++++++++++++ src/services/util/util.module.ts | 6 +- src/services/util/util.service.ts | 50 +++++- .../detect-changes/detect.changes.spec.ts | 91 ++++++++++ src/util/detect-changes/detect.changes.ts | 131 ++++++++++++++ 23 files changed, 818 insertions(+), 34 deletions(-) create mode 100644 src/config/whiteboard.event.logging.mode.ts create mode 100644 src/elasticsearch-client/elastic.response.error.ts create mode 100644 src/elasticsearch-client/elasticsearch.client.factory.ts create mode 100644 src/elasticsearch-client/elasticsearch.client.injection.token.ts create mode 100644 src/elasticsearch-client/elasticsearch.client.provider.ts create mode 100644 src/elasticsearch-client/handle.elastic.error.ts create mode 100644 src/elasticsearch-client/index.ts create mode 100644 src/elasticsearch-client/is.elastic.error.ts create mode 100644 src/elasticsearch-client/is.elastic.response.error.ts create mode 100644 src/services/elasticsearch/elasticsearch.module.ts create mode 100644 src/services/elasticsearch/elasticsearch.service.ts create mode 100644 src/util/detect-changes/detect.changes.spec.ts create mode 100644 src/util/detect-changes/detect.changes.ts diff --git a/config.yml b/config.yml index ca81d22..50eb880 100644 --- a/config.yml +++ b/config.yml @@ -12,6 +12,15 @@ rabbitmq: # heartbeat heartbeat: ${RABBITMQ_HEARTBEAT}:30 +elasticsearch: + host: ${ELASTICSEARCH_URL} + api_key: ${ELASTICSEARCH_API_KEY} + retries: ${ELASTICSEARCH_RETRIES}:3 + timeout: ${ELASTICSEARCH_TIMEOUT}:30000 + tls: + ca_cert_path: ${ELASTIC_TLS_CA_CERT_PATH}:none + rejectUnauthorized: ${ELASTIC_TLS_REJECT_UNAUTHORIZED}:false + monitoring: logging: # A flag setting whether Winston Console transport will be enabled. @@ -25,6 +34,17 @@ monitoring: # The logging format will be in json - useful for parsing # if disabled - will be in a human-readable form json: ${LOGGING_FORMAT_JSON}:false + events: + # The index responsible for tracking the whiteboard events happening in a particular session + whiteboard_event_index: ${ELASTIC_INDEX_WHITEBOARD_EVENTS}:whiteboard-change-events + # The amount of logged information + # None = disabled; + # Lite = user + event type; + # Full = user + event type + delta of what was changed + mode: ${WHITEBOARD_EVENTS_LOGGING_MODE}:none + # MILLISECONDS of how long are the events buffered in memory before being sent + interval: ${WHITEBOARD_EVENTS_LOGGING_INTERVAL}:3000 + settings: # application level settings diff --git a/package-lock.json b/package-lock.json index 3c32c7c..19b9328 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,14 +1,15 @@ { "name": "whiteboard-collaboration-service", - "version": "0.5.0", + "version": "0.5.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "whiteboard-collaboration-service", - "version": "0.5.0", + "version": "0.5.1", "license": "EUPL-1.2", "dependencies": { + "@elastic/elasticsearch": "8.12.2", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.2.2", "@nestjs/core": "^10.0.0", @@ -927,6 +928,43 @@ "kuler": "^2.0.0" } }, + "node_modules/@elastic/elasticsearch": { + "version": "8.12.2", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.12.2.tgz", + "integrity": "sha512-04NvH3LIgcv1Uwguorfw2WwzC9Lhfsqs9f0L6uq6MrCw0lqe/HOQ6E8vJ6EkHAA15iEfbhtxOtenbZVVcE+mAQ==", + "license": "Apache-2.0", + "dependencies": { + "@elastic/transport": "^8.4.1", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@elastic/transport": { + "version": "8.8.1", + "resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.8.1.tgz", + "integrity": "sha512-4RQIiChwNIx3B0O+2JdmTq/Qobj6+1g2RQnSv1gt4V2SVfAYjGwOKu0ZMKEHQOXYNG6+j/Chero2G9k3/wXLEw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "1.x", + "debug": "^4.3.4", + "hpagent": "^1.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0", + "tslib": "^2.4.0", + "undici": "^6.12.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@elastic/transport/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -1983,14 +2021,14 @@ "license": "0BSD" }, "node_modules/@nestjs/platform-express": { - "version": "10.4.4", - "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.4.4.tgz", - "integrity": "sha512-y52q1MxhbHaT3vAgWd08RgiYon0lJgtTa8U6g6gV0KI0IygwZhDQFJVxnrRDUdxQGIP5CKHmfQu3sk9gTNFoEA==", + "version": "10.4.6", + "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.4.6.tgz", + "integrity": "sha512-HcyCpAKccAasrLSGRTGWv5BKRs0rwTIFOSsk6laNyqfqvgvYcJQAedarnm4jmaemtmSJ0PFI9PmtEZADd2ahCg==", "license": "MIT", "dependencies": { "body-parser": "1.20.3", "cors": "2.8.5", - "express": "4.21.0", + "express": "4.21.1", "multer": "1.4.4-lts.1", "tslib": "2.7.0" }, @@ -2118,6 +2156,15 @@ "npm": ">=5.0.0" } }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -3967,9 +4014,9 @@ "dev": true }, "node_modules/cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==", "license": "MIT", "engines": { "node": ">= 0.6" @@ -4744,9 +4791,9 @@ } }, "node_modules/express": { - "version": "4.21.0", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.0.tgz", - "integrity": "sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng==", + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", + "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", "license": "MIT", "dependencies": { "accepts": "~1.3.8", @@ -4754,7 +4801,7 @@ "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.6.0", + "cookie": "0.7.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -5483,6 +5530,15 @@ "node": ">=8" } }, + "node_modules/hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==", + "license": "MIT", + "engines": { + "node": ">=14" + } + }, "node_modules/html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -7985,6 +8041,12 @@ "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "license": "BSD-3-Clause" + }, "node_modules/semver": { "version": "7.6.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", @@ -8991,6 +9053,15 @@ "node": ">= 4.0.0" } }, + "node_modules/undici": { + "version": "6.20.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.20.1.tgz", + "integrity": "sha512-AjQF1QsmqfJys+LXfGTNum+qw4S88CojRInG/6t31W/1fk6G59s92bnAvGz5Cmur+kQv2SURXEvvudLmbrE8QA==", + "license": "MIT", + "engines": { + "node": ">=18.17" + } + }, "node_modules/undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", @@ -10148,6 +10219,36 @@ "kuler": "^2.0.0" } }, + "@elastic/elasticsearch": { + "version": "8.12.2", + "resolved": "https://registry.npmjs.org/@elastic/elasticsearch/-/elasticsearch-8.12.2.tgz", + "integrity": "sha512-04NvH3LIgcv1Uwguorfw2WwzC9Lhfsqs9f0L6uq6MrCw0lqe/HOQ6E8vJ6EkHAA15iEfbhtxOtenbZVVcE+mAQ==", + "requires": { + "@elastic/transport": "^8.4.1", + "tslib": "^2.4.0" + } + }, + "@elastic/transport": { + "version": "8.8.1", + "resolved": "https://registry.npmjs.org/@elastic/transport/-/transport-8.8.1.tgz", + "integrity": "sha512-4RQIiChwNIx3B0O+2JdmTq/Qobj6+1g2RQnSv1gt4V2SVfAYjGwOKu0ZMKEHQOXYNG6+j/Chero2G9k3/wXLEw==", + "requires": { + "@opentelemetry/api": "1.x", + "debug": "^4.3.4", + "hpagent": "^1.0.0", + "ms": "^2.1.3", + "secure-json-parse": "^2.4.0", + "tslib": "^2.4.0", + "undici": "^6.12.0" + }, + "dependencies": { + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" + } + } + }, "@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -10876,13 +10977,13 @@ } }, "@nestjs/platform-express": { - "version": "10.4.4", - "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.4.4.tgz", - "integrity": "sha512-y52q1MxhbHaT3vAgWd08RgiYon0lJgtTa8U6g6gV0KI0IygwZhDQFJVxnrRDUdxQGIP5CKHmfQu3sk9gTNFoEA==", + "version": "10.4.6", + "resolved": "https://registry.npmjs.org/@nestjs/platform-express/-/platform-express-10.4.6.tgz", + "integrity": "sha512-HcyCpAKccAasrLSGRTGWv5BKRs0rwTIFOSsk6laNyqfqvgvYcJQAedarnm4jmaemtmSJ0PFI9PmtEZADd2ahCg==", "requires": { "body-parser": "1.20.3", "cors": "2.8.5", - "express": "4.21.0", + "express": "4.21.1", "multer": "1.4.4-lts.1", "tslib": "2.7.0" }, @@ -10968,6 +11069,11 @@ "node-fetch": "^2.6.1" } }, + "@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==" + }, "@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -12402,9 +12508,9 @@ "dev": true }, "cookie": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", - "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==" + "version": "0.7.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.1.tgz", + "integrity": "sha512-6DnInpx7SJ2AK3+CTUE/ZM0vWTUboZCegxhC2xiIydHR9jNuTAASBrfEpHhiGOZw/nX51bHt6YQl8jsGo4y/0w==" }, "cookie-signature": { "version": "1.0.6", @@ -12947,16 +13053,16 @@ } }, "express": { - "version": "4.21.0", - "resolved": "https://registry.npmjs.org/express/-/express-4.21.0.tgz", - "integrity": "sha512-VqcNGcj/Id5ZT1LZ/cfihi3ttTn+NJmkli2eZADigjq29qTlWi/hAQ43t/VLPq8+UX06FCEx3ByOYet6ZFblng==", + "version": "4.21.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.21.1.tgz", + "integrity": "sha512-YSFlK1Ee0/GC8QaO91tHcDxJiE/X4FbpAyQWkxAvG6AXCuR65YzK8ua6D9hvi/TzUfZMpc+BwuM1IPw8fmQBiQ==", "requires": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "1.20.3", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.6.0", + "cookie": "0.7.1", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -13508,6 +13614,11 @@ "integrity": "sha512-QFLV0taWQOZtvIRIAdBChesmogZrtuXvVWsFHZTk2SU+anspqZ2vMnoLg7IE1+Uk16N19APic1BuF8bC8c2m5g==", "dev": true }, + "hpagent": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/hpagent/-/hpagent-1.2.0.tgz", + "integrity": "sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA==" + }, "html-escaper": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", @@ -15349,6 +15460,11 @@ } } }, + "secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==" + }, "semver": { "version": "7.6.2", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", @@ -16068,6 +16184,11 @@ "resolved": "https://registry.npmjs.org/uid2/-/uid2-1.0.0.tgz", "integrity": "sha512-+I6aJUv63YAcY9n4mQreLUt0d4lvwkkopDNmpomkAUz0fAkEMV9pRWxN0EjhW1YfRhcuyHg2v3mwddCDW1+LFQ==" }, + "undici": { + "version": "6.20.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.20.1.tgz", + "integrity": "sha512-AjQF1QsmqfJys+LXfGTNum+qw4S88CojRInG/6t31W/1fk6G59s92bnAvGz5Cmur+kQv2SURXEvvudLmbrE8QA==" + }, "undici-types": { "version": "6.19.8", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz", diff --git a/package.json b/package.json index 71ced66..c1aee2b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "whiteboard-collaboration-service", - "version": "0.5.0", + "version": "0.5.1", "description": "Alkemio Whiteboard Collaboration Service for Excalidraw backend", "author": "Alkemio Foundation", "private": false, @@ -16,6 +16,7 @@ "test": "jest" }, "dependencies": { + "@elastic/elasticsearch": "8.12.2", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.2.2", "@nestjs/core": "^10.0.0", diff --git a/src/config/config.type.ts b/src/config/config.type.ts index ae0b1c5..a2f3628 100644 --- a/src/config/config.type.ts +++ b/src/config/config.type.ts @@ -1,3 +1,5 @@ +import { WhiteboardEventLoggingModeType } from './whiteboard.event.logging.mode'; + export interface ConfigType { rabbitmq: { connection: { @@ -8,11 +10,29 @@ export interface ConfigType { heartbeat: number; }; }; + elasticsearch: { + host: string; + api_key: string; + retries: number; + timeout: number; + tls: { + ca_cert_path: string | 'none'; + rejectUnauthorized: boolean; + }; + indices: { + whiteboard_events: string; + }; + }; monitoring: { logging: { enabled: boolean; level: string; json: boolean; + events: { + whiteboard_event_index: string; + interval: number; + mode: WhiteboardEventLoggingModeType; + }; }; }; settings: { diff --git a/src/config/index.ts b/src/config/index.ts index 94cd823..2f02503 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -1,3 +1,4 @@ export * from './config.type'; export * from './configuration'; export * from './winston.config'; +export * from './whiteboard.event.logging.mode'; diff --git a/src/config/whiteboard.event.logging.mode.ts b/src/config/whiteboard.event.logging.mode.ts new file mode 100644 index 0000000..84e04de --- /dev/null +++ b/src/config/whiteboard.event.logging.mode.ts @@ -0,0 +1,8 @@ +export const WhiteboardEventLoggingMode = { + none: 'none', + lite: 'lite', + full: 'full', +} as const; + +export type WhiteboardEventLoggingModeType = + (typeof WhiteboardEventLoggingMode)[keyof typeof WhiteboardEventLoggingMode]; diff --git a/src/elasticsearch-client/elastic.response.error.ts b/src/elasticsearch-client/elastic.response.error.ts new file mode 100644 index 0000000..35e075e --- /dev/null +++ b/src/elasticsearch-client/elastic.response.error.ts @@ -0,0 +1,25 @@ +export type ElasticResponseError = { + meta: { + body: { + error: { + reason: string; + type: string; + root_cause: [{ reason: string; type: string }]; + }; + }; + headers: Record; + meta: { + aborted: boolean; + attempts: number; + connection: Record; + context: unknown; + name: string; + request: Record; + }; + statusCode: Record; + warnings: unknown; + }; + name: string; + message: string; + stack: string; +}; diff --git a/src/elasticsearch-client/elasticsearch.client.factory.ts b/src/elasticsearch-client/elasticsearch.client.factory.ts new file mode 100644 index 0000000..3f1d6d9 --- /dev/null +++ b/src/elasticsearch-client/elasticsearch.client.factory.ts @@ -0,0 +1,56 @@ +import fs from 'fs'; +import { LoggerService } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { Client } from '@elastic/elasticsearch'; +import { ConfigType } from '../config'; + +export const elasticSearchClientFactory = async ( + logger: LoggerService, + configService: ConfigService, +): Promise => { + const { host, retries, timeout, api_key, tls } = configService.get( + 'elasticsearch', + { + infer: true, + }, + ); + + const rejectUnauthorized = tls.rejectUnauthorized ?? false; + let tlsOptions; + + // Ensure the path to the certificate inside the container is correct + if (tls.ca_cert_path === 'none') { + tlsOptions = { rejectUnauthorized }; + } else { + // This should match the mountPath in your Kubernetes deployment YAML + const certPath = tls.ca_cert_path; + if (!fs.existsSync(certPath)) { + logger.error(`Certificate not found at path: ${certPath}`); + return undefined; + } + const cert = fs.readFileSync(certPath); + tlsOptions = { + rejectUnauthorized: true, + ca: cert, + }; + } + + if (!host) { + logger.warn('Elasticsearch host URL not provided!'); + return undefined; + } + + if (!api_key) { + logger.error('Elasticsearch API key not provided!'); + return undefined; + } + + return new Client({ + node: host, + maxRetries: retries, + requestTimeout: timeout, + resurrectStrategy: 'ping', + auth: { apiKey: api_key }, + tls: tlsOptions, + }); +}; diff --git a/src/elasticsearch-client/elasticsearch.client.injection.token.ts b/src/elasticsearch-client/elasticsearch.client.injection.token.ts new file mode 100644 index 0000000..ba0fa1b --- /dev/null +++ b/src/elasticsearch-client/elasticsearch.client.injection.token.ts @@ -0,0 +1 @@ +export const ELASTICSEARCH_CLIENT_PROVIDER = 'elasticsearch-client-provider'; diff --git a/src/elasticsearch-client/elasticsearch.client.provider.ts b/src/elasticsearch-client/elasticsearch.client.provider.ts new file mode 100644 index 0000000..7812a63 --- /dev/null +++ b/src/elasticsearch-client/elasticsearch.client.provider.ts @@ -0,0 +1,13 @@ +import { Client } from '@elastic/elasticsearch'; +import { FactoryProvider } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { WINSTON_MODULE_NEST_PROVIDER } from 'nest-winston'; +import { ELASTICSEARCH_CLIENT_PROVIDER } from './elasticsearch.client.injection.token'; +import { elasticSearchClientFactory } from './elasticsearch.client.factory'; + +export const ElasticsearchClientProvider: FactoryProvider = + { + provide: ELASTICSEARCH_CLIENT_PROVIDER, + inject: [WINSTON_MODULE_NEST_PROVIDER, ConfigService], + useFactory: elasticSearchClientFactory, + }; diff --git a/src/elasticsearch-client/handle.elastic.error.ts b/src/elasticsearch-client/handle.elastic.error.ts new file mode 100644 index 0000000..2a29b4b --- /dev/null +++ b/src/elasticsearch-client/handle.elastic.error.ts @@ -0,0 +1,47 @@ +import { randomUUID } from 'crypto'; +import { isElasticResponseError } from './is.elastic.response.error'; +import { isElasticError } from './is.elastic.error'; + +export type HandledElasticError = { + message: string; + uuid: string; + name?: string; + status?: number; +}; + +const UNKNOWN_STATUS = -1; + +export const handleElasticError = (error: unknown): HandledElasticError => { + const errorId = randomUUID(); + + if (isElasticResponseError(error)) { + // not really how to handle multiple status codes + const status = Number( + Object.keys(error.meta.statusCode)?.[0] ?? UNKNOWN_STATUS + ); + return { + message: error.message, + uuid: errorId, + name: error.name, + status, + }; + } else if (isElasticError(error)) { + return { + message: error.error.type, + uuid: errorId, + name: 'ErrorResponseBase', + status: error.status, + }; + } else if (error instanceof Error) { + return { + message: error.message, + uuid: errorId, + name: error.name, + }; + } else { + return { + message: String(error), + uuid: errorId, + }; + } +}; diff --git a/src/elasticsearch-client/index.ts b/src/elasticsearch-client/index.ts new file mode 100644 index 0000000..68c17f3 --- /dev/null +++ b/src/elasticsearch-client/index.ts @@ -0,0 +1,6 @@ +export * from './elasticsearch.client.provider'; + +export * from './elasticsearch.client.injection.token'; + +export * from './is.elastic.error'; +export * from './is.elastic.response.error'; diff --git a/src/elasticsearch-client/is.elastic.error.ts b/src/elasticsearch-client/is.elastic.error.ts new file mode 100644 index 0000000..d70c1cd --- /dev/null +++ b/src/elasticsearch-client/is.elastic.error.ts @@ -0,0 +1,6 @@ +import { ErrorResponseBase } from '@elastic/elasticsearch/lib/api/types'; + +export const isElasticError = (error: unknown): error is ErrorResponseBase => { + const err = error as ErrorResponseBase; + return !!err?.status && !!err?.error && !!err?.error.type; +}; diff --git a/src/elasticsearch-client/is.elastic.response.error.ts b/src/elasticsearch-client/is.elastic.response.error.ts new file mode 100644 index 0000000..51a0254 --- /dev/null +++ b/src/elasticsearch-client/is.elastic.response.error.ts @@ -0,0 +1,8 @@ +import { ElasticResponseError } from './elastic.response.error'; + +export const isElasticResponseError = ( + error: unknown, +): error is ElasticResponseError => { + const e = error as ElasticResponseError; + return !!e.meta && !!e.stack && !!e.message; +}; diff --git a/src/excalidraw-backend/server.ts b/src/excalidraw-backend/server.ts index ec41bdc..35b09de 100644 --- a/src/excalidraw-backend/server.ts +++ b/src/excalidraw-backend/server.ts @@ -254,6 +254,14 @@ export class Server { return; } + this.utilService.reportChanges( + roomID, + socket.data.userInfo.email, + (this.snapshots.get(roomID)?.content.elements ?? + []) as ExcalidrawElement[], + eventData.payload.elements as ExcalidrawElement[], + ); + this.createAndStoreLatestSnapshot( roomID, eventData.payload.elements, @@ -347,13 +355,13 @@ export class Server { remoteElements: readonly ExcalidrawElement[], remoteFileStore: DeepReadonly, ) { - const snapshot = this.snapshots.get(roomId); - if (!snapshot) { + const oldSnapshot = this.snapshots.get(roomId); + if (!oldSnapshot) { return; } const reconciledSnapshot = InMemorySnapshot.reconcile( - snapshot, + oldSnapshot, remoteElements, remoteFileStore, ); diff --git a/src/excalidraw-backend/utils/reconcile.files.ts b/src/excalidraw-backend/utils/reconcile.files.ts index 84a1ef8..6086c54 100644 --- a/src/excalidraw-backend/utils/reconcile.files.ts +++ b/src/excalidraw-backend/utils/reconcile.files.ts @@ -22,6 +22,7 @@ export const reconcileFiles = ( }; continue; } + // todo: investigate if it's still the case /** uncomment this when Excalidraw starts sending the element and the file at the same time * otherwise it's causing a bug where the file is sent but the element that should fit the image is not * then another event is sent with the host element but not the file diff --git a/src/excalidraw/types/excalidraw.element.ts b/src/excalidraw/types/excalidraw.element.ts index e09a32c..9a68114 100644 --- a/src/excalidraw/types/excalidraw.element.ts +++ b/src/excalidraw/types/excalidraw.element.ts @@ -9,6 +9,9 @@ type ExcalidrawBaseElement = { Always kept in sync with the array order by `syncMovedIndices` and `syncInvalidIndices`. Could be null, i.e. for new elements which were not yet assigned to the scene. */ index: FractionalIndex | null; + updated: number; + isDeleted: boolean; + boundElements: any[] | null; }; export type ExcalidrawImageElement = ExcalidrawBaseElement & { diff --git a/src/services/elasticsearch/elasticsearch.module.ts b/src/services/elasticsearch/elasticsearch.module.ts new file mode 100644 index 0000000..1c3033f --- /dev/null +++ b/src/services/elasticsearch/elasticsearch.module.ts @@ -0,0 +1,10 @@ +import { Module } from '@nestjs/common'; +import { ElasticsearchClientProvider } from '../../elasticsearch-client'; +import { ElasticsearchService } from './elasticsearch.service'; + +@Module({ + imports: [], + providers: [ElasticsearchClientProvider, ElasticsearchService], + exports: [ElasticsearchService], +}) +export class ElasticsearchModule {} diff --git a/src/services/elasticsearch/elasticsearch.service.ts b/src/services/elasticsearch/elasticsearch.service.ts new file mode 100644 index 0000000..5289745 --- /dev/null +++ b/src/services/elasticsearch/elasticsearch.service.ts @@ -0,0 +1,163 @@ +import { Inject, Injectable, LoggerService } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { WINSTON_MODULE_NEST_PROVIDER } from 'nest-winston'; +import { throttle } from 'lodash'; +import { Client as ElasticClient } from '@elastic/elasticsearch'; +import { ErrorCause } from '@elastic/elasticsearch/lib/api/types'; +import { + DetectedChanges, + DetectedChangesType, +} from '../../util/detect-changes/detect.changes'; +import { ExcalidrawElement } from '../../excalidraw/types'; +import { ELASTICSEARCH_CLIENT_PROVIDER } from '../../elasticsearch-client'; +import { + ConfigType, + WhiteboardEventLoggingMode, + WhiteboardEventLoggingModeType, +} from '../../config'; + +type ErroredDocument = { + status: number | undefined; + error: ErrorCause | undefined; + operation: unknown; + document: unknown; +}; + +type BaseChangeEventDocument = { + '@timestamp': Date; + createdBy: string; + whiteboardId: string; + types: DetectedChangesType[]; +}; + +type LiteChangeEventDocument = BaseChangeEventDocument; +type FullChangeEventDocument = BaseChangeEventDocument & + DetectedChanges; + +type WhiteboardChangeEventDocument = + | LiteChangeEventDocument + | FullChangeEventDocument; + +@Injectable() +export class ElasticsearchService { + private readonly sendDataThrottled; + private readonly dataToSend: WhiteboardChangeEventDocument[] = []; + private readonly eventIndex: string; + private readonly eventLoggingMode: WhiteboardEventLoggingModeType; + + constructor( + @Inject(WINSTON_MODULE_NEST_PROVIDER) private logger: LoggerService, + @Inject(ELASTICSEARCH_CLIENT_PROVIDER) + private readonly elasticClient: ElasticClient | undefined, + private readonly configService: ConfigService, + ) { + const eventsConfig = this.configService.get('monitoring.logging.events', { + infer: true, + }); + this.eventIndex = eventsConfig.whiteboard_event_index; + this.eventLoggingMode = eventsConfig.mode; + + this.sendDataThrottled = throttle( + this.sendBufferedEventData.bind(this), + eventsConfig.interval, + ); + } + + public sendWhiteboardChangeEvent( + roomId: string, + createdBy: string, + changes: DetectedChanges, + ): void { + if (this.eventLoggingMode === WhiteboardEventLoggingMode.none) { + return; + } + + const baseDocument: BaseChangeEventDocument = { + '@timestamp': new Date(), + whiteboardId: roomId, + createdBy, + types: this.eventType(changes), + }; + + if (this.eventLoggingMode === WhiteboardEventLoggingMode.lite) { + this.dataToSend.push(baseDocument); + } + + if (this.eventLoggingMode === WhiteboardEventLoggingMode.full) { + this.dataToSend.push({ + ...baseDocument, + ...changes, + }); + } + + this.sendDataThrottled(); + } + + private async sendBufferedEventData(): Promise { + await this.ingestBulk(this.dataToSend, this.eventIndex); + this.dataToSend.length = 0; + } + + private async ingestBulk(data: unknown[], index: string): Promise { + if (!this.elasticClient) { + return; + } + + if (!data.length) { + return; + } + + const operations = data.flatMap((doc) => [ + { create: { _index: index } }, + doc, + ]); + + const bulkResponse = await this.elasticClient.bulk({ operations }); + + if (bulkResponse.errors) { + const erroredDocuments: ErroredDocument[] = []; + // The items array has the same order of the dataset we just indexed. + // The presence of the `error` key indicates that the operation + // that we did for the document has failed. + bulkResponse.items.forEach((action, i) => { + const operation = Object.keys(action)[0] as keyof typeof action; + if (action[operation]?.error) { + erroredDocuments.push({ + // If the status is 429 it means that you can retry the document, + // otherwise it's very likely a mapping error, and you should + // fix the document before to try it again. + status: action[operation]?.status, + error: action[operation]?.error, + operation: operations[i * 2], + document: operations[i * 2 + 1], + }); + } + }); + this.logger.error( + `[${index}] - ${erroredDocuments.length} documents errored. ${ + data.length - erroredDocuments.length + } documents indexed.`, + ); + } else { + this.logger.verbose?.(`[${index}] - ${data.length} documents indexed`); + } + } + + private eventType(changes: DetectedChanges): DetectedChangesType[] { + const types: DetectedChangesType[] = []; + + if (changes.inserted) { + types.push(DetectedChangesType.inserted); + } + + if (changes.updated) { + types.push(DetectedChangesType.updated); + } + + if (changes.deleted) { + types.push(DetectedChangesType.deleted); + } + + return types.length > 0 ? types : [DetectedChangesType.unknown]; + } +} diff --git a/src/services/util/util.module.ts b/src/services/util/util.module.ts index 296f1ae..66a6c6d 100644 --- a/src/services/util/util.module.ts +++ b/src/services/util/util.module.ts @@ -1,10 +1,12 @@ import { Module } from '@nestjs/common'; import { WhiteboardIntegrationModule } from '../whiteboard-integration/whiteboard.integration.module'; +import { ElasticsearchClientProvider } from '../../elasticsearch-client'; +import { ElasticsearchModule } from '../elasticsearch/elasticsearch.module'; import { UtilService } from './util.service'; @Module({ - imports: [WhiteboardIntegrationModule], - providers: [UtilService], + imports: [WhiteboardIntegrationModule, ElasticsearchModule], + providers: [UtilService, ElasticsearchClientProvider], exports: [UtilService], }) export class UtilModule {} diff --git a/src/services/util/util.service.ts b/src/services/util/util.service.ts index eca5234..b0b8acb 100644 --- a/src/services/util/util.service.ts +++ b/src/services/util/util.service.ts @@ -1,4 +1,6 @@ import { Inject, Injectable, LoggerService } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import { WINSTON_MODULE_NEST_PROVIDER } from 'nest-winston'; import { WhiteboardIntegrationService } from '../whiteboard-integration/whiteboard.integration.service'; import { UserInfo } from '../whiteboard-integration/user.info'; import { @@ -9,18 +11,35 @@ import { SaveInputData, WhoInputData, } from '../whiteboard-integration/inputs'; -import { ExcalidrawContent } from '../../excalidraw/types'; -import { isFetchErrorData } from '../whiteboard-integration/outputs'; +import { ExcalidrawContent, ExcalidrawElement } from '../../excalidraw/types'; import { excalidrawInitContent } from '../../util'; -import { WINSTON_MODULE_NEST_PROVIDER } from 'nest-winston'; import { DeepReadonly } from '../../excalidraw-backend/utils'; +import { isFetchErrorData } from '../whiteboard-integration/outputs'; +import { detectChanges } from '../../util/detect-changes/detect.changes'; +import { ElasticsearchService } from '../elasticsearch/elasticsearch.service'; +import { + ConfigType, + WhiteboardEventLoggingMode, + WhiteboardEventLoggingModeType, +} from '../../config'; @Injectable() export class UtilService { + private readonly eventLoggingMode: WhiteboardEventLoggingModeType; + constructor( @Inject(WINSTON_MODULE_NEST_PROVIDER) private logger: LoggerService, private readonly integrationService: WhiteboardIntegrationService, - ) {} + private readonly elasticService: ElasticsearchService, + private readonly configService: ConfigService, + ) { + this.eventLoggingMode = this.configService.get( + 'monitoring.logging.events.mode', + { + infer: true, + }, + ); + } public async getUserInfo(opts: { cookie?: string; @@ -83,4 +102,27 @@ export class UtilService { return excalidrawInitContent; } } + + public reportChanges( + roomId: string, + createdBy: string, + oldEl: ExcalidrawElement[], + newEl: ExcalidrawElement[], + ) { + if (this.eventLoggingMode === WhiteboardEventLoggingMode.none) { + return; + } + // we need the changes to calculate the types correctly + const changes = detectChanges(oldEl, newEl, [ + 'version', + 'versionNonce', + 'updated', + 'boundElements', + ]); + if (!changes) { + return; + } + + this.elasticService.sendWhiteboardChangeEvent(roomId, createdBy, changes); + } } diff --git a/src/util/detect-changes/detect.changes.spec.ts b/src/util/detect-changes/detect.changes.spec.ts new file mode 100644 index 0000000..720142d --- /dev/null +++ b/src/util/detect-changes/detect.changes.spec.ts @@ -0,0 +1,91 @@ +import { detectChanges } from './detect.changes'; + +describe('detectChanges', () => { + it('should return null if both arrays are empty', () => { + const result = detectChanges([], []); + expect(result).toBeNull(); + }); + + it('should detect inserted elements', () => { + const result = detectChanges([], [element1, element2]); + expect(result).toEqual({ inserted: [element1, element2] }); + }); + + it('should detect deleted elements', () => { + const result = detectChanges([element1, element2], []); + expect(result).toEqual({ deleted: [{ id: '1' }, { id: '2' }] }); + }); + + it('should detect updated elements', () => { + const updatedElement1 = { ...element1, someField: 'newValue1' }; + const result = detectChanges([element1], [updatedElement1]); + expect(result).toEqual({ + updated: [ + { + id: '1', + someField: { + old: 'value1', + new: 'newValue1', + }, + }, + ], + }); + }); + + it('should detect mixed changes', () => { + const updatedElement1 = { ...element1, someField: 'newValue1' }; + const result = detectChanges( + [element1, element2], + [updatedElement1, element3], + ); + expect(result).toEqual({ + inserted: [element3], + updated: [ + { + id: '1', + someField: { + old: 'value1', + new: 'newValue1', + }, + }, + ], + }); + }); + + it('should ignore specified fields', () => { + const updatedElement1 = { ...element1, someField: 'newValue1' }; + const result = detectChanges([element1], [updatedElement1], [ + 'someField', + ] as any); + expect(result).toBeNull(); + }); + + it('should handle elements marked as deleted', () => { + const deletedElement1 = { ...element1, isDeleted: true }; + const result = detectChanges([element1, element4], [deletedElement1]); + expect(result).toEqual({ + deleted: [{ id: '1' }], + }); + }); +}); + +const element1: any = { + id: '1', + isDeleted: false, + someField: 'value1', +}; +const element2: any = { + id: '2', + isDeleted: false, + someField: 'value2', +}; +const element3: any = { + id: '3', + isDeleted: false, + someField: 'value3', +}; +const element4: any = { + id: '4', + isDeleted: true, + someField: 'value4', +}; diff --git a/src/util/detect-changes/detect.changes.ts b/src/util/detect-changes/detect.changes.ts new file mode 100644 index 0000000..9d8654f --- /dev/null +++ b/src/util/detect-changes/detect.changes.ts @@ -0,0 +1,131 @@ +import { isEqual } from 'lodash'; +import { ExcalidrawElement } from '../../excalidraw/types'; + +type Identifiable = { + id: string; +}; + +type Changes = { + id: string; +} & Partial<{ + [P in keyof T as P extends 'id' ? never : P]: { + old?: T[P]; + new?: T[P]; + }; +}>; + +export type DetectedChanges = { + inserted?: Array; + updated?: Array>; + deleted?: Array<{ + id: string; + }>; +}; + +export const enum DetectedChangesType { + inserted = 'inserted', + updated = 'updated', + deleted = 'deleted', + unknown = 'unknown', +} + +/** + * Detects deep changes between two arrays of ExcalidrawElements, and returns an object with the changes. + * Returns null if no changes are detected. + * @param oldArrInput + * @param newArrInput + * @param fieldsToIgnore + */ +export const detectChanges = ( + oldArrInput: Array, + newArrInput: Array, + fieldsToIgnore?: Array, +): DetectedChanges | null => { + const oldLen = oldArrInput.length; + const newLen = newArrInput.length; + // if both are empty, no changes + if (oldLen === 0 && newLen === 0) { + return null; + } + // if oldArr is empty, all new are inserted + if (oldLen === 0) { + return { inserted: newArrInput }; + } + // if new is empty, all oldArr are deleted + if (newLen === 0) { + return { deleted: oldArrInput.map((item) => ({ id: item.id })) }; + } + + // filter out the fields we want to ignore + const newArr = newArrInput.map((item) => + removeFields(item, fieldsToIgnore ?? []), + ); + + const oldArr = oldArrInput.map((item) => + removeFields(item, fieldsToIgnore ?? []), + ); + + const changes: DetectedChanges = {}; + + const oldMap = new Map(oldArr.map((item) => [item.id, item])); + for (const newItem of newArr) { + const oldItem = oldMap.get(newItem.id); + // if the new item is not in the old array, it is inserted + if (!oldItem) { + changes.inserted = changes.inserted ?? []; + changes.inserted.push(newItem); + continue; + } + // a match is found - the element is updated or deleted + // if both are deleted - ignore + if (oldItem.isDeleted && newItem.isDeleted) { + continue; + } + // if the new item is deleted, it is considered deleted + if (newItem.isDeleted) { + changes.deleted = changes.deleted ?? []; + changes.deleted.push({ id: newItem.id }); + continue; + } + // a match is found and the item is not deleted + // search what has been updated + const updatedFields: Changes = { id: newItem.id }; + let isUpdated = false; + // compare every field of the new item against every field of the old item + for (const key in newItem) { + const typedKey = key as keyof ExcalidrawElement; + + if (isEqual(newItem[typedKey], oldItem[typedKey])) { + continue; + } + updatedFields[typedKey] = { + // TS2322: Type string | number | boolean | any[] | FractionalIndex | null + // is not assignable to type undefined + // Type null is not assignable to type undefined + // @ts-expect-error updatedFields[typedKey] can be of undefined type and assigning another type to it brings a type error + old: oldItem[typedKey], + // @ts-expect-error same + new: newItem[typedKey], + }; + isUpdated = true; + } + + if (isUpdated) { + changes.updated = changes.updated ?? []; + changes.updated.push(updatedFields); + } + } + // return null if no changes detected + return Object.keys(changes).length > 0 ? changes : null; +}; + +function removeFields( + obj: T, + fields: K[], +): T { + const result = { ...obj }; + fields.forEach((field) => { + delete result[field]; + }); + return result; +}