From 1aa792e9bc60b6b9193a7ba6eb19bd52a187458b Mon Sep 17 00:00:00 2001 From: Florian Maunier Date: Fri, 6 Sep 2024 17:09:21 +0200 Subject: [PATCH] test(transform): split transform and adding a basic test --- .github/workflows/pull_request.workflow.yml | 8 +- package.json | 4 +- src/index.ts | 45 +----- src/transform.ts | 45 ++++++ src/types/overrides.d.ts | 2 +- tests/fixtures/testLogs.ts | 156 ++++++++++++++++++++ tests/unit/transform.test.ts | 47 ++++++ 7 files changed, 261 insertions(+), 46 deletions(-) create mode 100644 src/transform.ts create mode 100644 tests/fixtures/testLogs.ts create mode 100644 tests/unit/transform.test.ts diff --git a/.github/workflows/pull_request.workflow.yml b/.github/workflows/pull_request.workflow.yml index 234fe29..2cf6b8e 100644 --- a/.github/workflows/pull_request.workflow.yml +++ b/.github/workflows/pull_request.workflow.yml @@ -47,8 +47,8 @@ jobs: - name: Run types tests run: npm run test:types - functional-tests: - name: Functional Tests + unit-tests: + name: Unit tests runs-on: ubuntu-latest needs: [lint, check-types] strategy: @@ -67,5 +67,5 @@ jobs: - name: Install dependencies run: npm ci - - name: Run functional Tests - run: npm run test:functional + - name: Run unit tests + run: npm run test:unit diff --git a/package.json b/package.json index 86373cc..f4d47be 100644 --- a/package.json +++ b/package.json @@ -9,8 +9,8 @@ "lint": "eslint .", "lint:fix": "eslint . --fix", "prepare": "husky", - "test": "npm run test:functional && npm run test:types", - "test:functional": "jest --runInBand", + "test": "npm run test:unit && npm run test:types", + "test:unit": "jest --runInBand", "test:types": "tsc -p tsconfig.dist.json --noEmit" }, "keywords": [], diff --git a/src/index.ts b/src/index.ts index ba4ea83..df90ad2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,51 +1,18 @@ import { ecsFormat } from '@elastic/ecs-pino-format'; -import { pipeline, Transform } from 'node:stream'; +import { pipeline } from 'node:stream'; import build from 'pino-abstract-transport'; -import { deserializeError } from 'serialize-error'; +import { getTransform } from './transform'; import { PinoTransportEcsOptions } from './types'; -export default async function pinoTransportEcs(options: PinoTransportEcsOptions) { +export default async function pinoTransportEcs(options?: PinoTransportEcsOptions) { const pinoConfigEcs = ecsFormat(options); return build( (source) => { - const myTransportStream = new Transform({ - autoDestroy: true, - objectMode: true, - transform(line, enc, cb) { - if (options.additionalBindings) { - line = { ...line, ...options.additionalBindings }; - } + const transformStream = getTransform(source, pinoConfigEcs, options); - if (pinoConfigEcs.messageKey && source.messageKey !== pinoConfigEcs.messageKey) { - line = { ...line, [pinoConfigEcs.messageKey]: line[source.messageKey] }; - delete line[source.messageKey]; - } - - line['@timestamp'] = new Date(line.time).toISOString(); - delete line.time; - - line = { - ...line, - ...pinoConfigEcs.formatters!.level!(source.levels.labels[line.level], line.level), - }; - - delete line.level; - - if (line.err) { - line.err = deserializeError(line.err); - } - - line = pinoConfigEcs.formatters!.bindings!(line); - line = pinoConfigEcs.formatters!.log!(line); - - this.push(`${JSON.stringify(line)}\n`); - cb(); - }, - }); - - pipeline(source, myTransportStream, () => {}); - return myTransportStream; + pipeline(source, transformStream, () => {}); + return transformStream; }, { enablePipelining: true, diff --git a/src/transform.ts b/src/transform.ts new file mode 100644 index 0000000..d40f6c0 --- /dev/null +++ b/src/transform.ts @@ -0,0 +1,45 @@ +import { Transform } from 'node:stream'; +import { LoggerOptions } from 'pino'; +import build, { type PinoConfig } from 'pino-abstract-transport'; +import { deserializeError } from 'serialize-error'; +import { PinoTransportEcsOptions } from './types'; + +export const getTransform = ( + source: Transform & build.OnUnknown & PinoConfig, + pinoConfigEcs: LoggerOptions, + options?: PinoTransportEcsOptions, +) => + new Transform({ + autoDestroy: true, + objectMode: true, + transform(line, enc, cb) { + if (options?.additionalBindings) { + line = { ...line, ...options.additionalBindings }; + } + + if (pinoConfigEcs.messageKey && source.messageKey !== pinoConfigEcs.messageKey) { + line = { ...line, [pinoConfigEcs.messageKey]: line[source.messageKey] }; + delete line[source.messageKey]; + } + + line['@timestamp'] = new Date(line.time).toISOString(); + delete line.time; + + line = { + ...line, + ...pinoConfigEcs.formatters!.level!(source.levels.labels[line.level], line.level), + }; + + delete line.level; + + if (line.err) { + line.err = deserializeError(line.err); + } + + line = pinoConfigEcs.formatters!.bindings!(line); + line = pinoConfigEcs.formatters!.log!(line); + + this.push(`${JSON.stringify(line)}\n`); + cb(); + }, + }); diff --git a/src/types/overrides.d.ts b/src/types/overrides.d.ts index 2ec3b83..64ca629 100644 --- a/src/types/overrides.d.ts +++ b/src/types/overrides.d.ts @@ -50,7 +50,7 @@ declare module 'pino-abstract-transport' { expectPinoConfig?: boolean; }; - type PinoConfig = { + export type PinoConfig = { errorKey: string; messageKey: string; levels: { diff --git a/tests/fixtures/testLogs.ts b/tests/fixtures/testLogs.ts new file mode 100644 index 0000000..e2db072 --- /dev/null +++ b/tests/fixtures/testLogs.ts @@ -0,0 +1,156 @@ +export const testLogs: Record[] = [ + { + level: 30, + time: 1725632753065, + pid: 492854, + hostname: 'izanami', + msg: 'hello world', + }, + { + level: 50, + time: 1725632753065, + pid: 492854, + hostname: 'izanami', + msg: 'this is at error level', + }, + { + level: 30, + time: 1725632753065, + pid: 492854, + hostname: 'izanami', + msg: 'the answer is 42', + }, + { + level: 30, + time: 1725632753065, + pid: 492854, + hostname: 'izanami', + obj: 42, + msg: 'hello world', + }, + { + level: 30, + time: 1725632753065, + pid: 492854, + hostname: 'izanami', + obj: 42, + b: 2, + msg: 'hello world', + }, + { + level: 30, + time: 1725632753065, + pid: 492854, + hostname: 'izanami', + nested: { + obj: 42, + }, + msg: 'nested', + }, + { + level: 40, + time: 1725632753065, + pid: 492854, + hostname: 'izanami', + msg: 'WARNING!', + }, + { + level: 50, + time: 1725632753065, + pid: 492854, + hostname: 'izanami', + err: { + type: 'Error', + message: 'an error', + stack: + 'Error: an error\n at Object. (/home/kuzzle/pino-transport-ecs/tests/application/app.ts:34:14)\n at Module._compile (node:internal/modules/cjs/loader:1358:14)\n at Module.m._compile (/home/kuzzle/pino-transport-ecs/node_modules/ts-node/src/index.ts:1618:23)\n at Module._extensions..js (node:internal/modules/cjs/loader:1416:10)\n at Object.require.extensions. [as .ts] (/home/kuzzle/pino-transport-ecs/node_modules/ts-node/src/index.ts:1621:12)\n at Module.load (node:internal/modules/cjs/loader:1208:32)\n at Function.Module._load (node:internal/modules/cjs/loader:1024:12)\n at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:174:12)\n at node:internal/main/run_main_module:28:49', + }, + msg: 'an error', + }, + { + level: 30, + time: 1725632753069, + pid: 492854, + hostname: 'izanami', + a: 'property', + msg: 'hello child!', + }, + { + level: 30, + time: 1725632753069, + pid: 492854, + hostname: 'izanami', + a: 'property', + another: 'property', + msg: 'hello baby..', + }, + { + level: 20, + time: 1725632753070, + pid: 492854, + hostname: 'izanami', + msg: 'this is a debug statement', + }, + { + level: 20, + time: 1725632753070, + pid: 492854, + hostname: 'izanami', + another: 'property', + msg: 'this is a debug statement via child', + }, + { + level: 10, + time: 1725632753070, + pid: 492854, + hostname: 'izanami', + msg: 'this is a trace statement', + }, + { + level: 20, + time: 1725632753070, + pid: 492854, + hostname: 'izanami', + msg: 'this is a "debug" statement with "', + }, + { + level: 30, + time: 1725632753070, + pid: 492854, + hostname: 'izanami', + err: { + type: 'Error', + message: 'kaboom', + stack: + 'Error: kaboom\n at Object. (/home/kuzzle/pino-transport-ecs/tests/application/app.ts:53:13)\n at Module._compile (node:internal/modules/cjs/loader:1358:14)\n at Module.m._compile (/home/kuzzle/pino-transport-ecs/node_modules/ts-node/src/index.ts:1618:23)\n at Module._extensions..js (node:internal/modules/cjs/loader:1416:10)\n at Object.require.extensions. [as .ts] (/home/kuzzle/pino-transport-ecs/node_modules/ts-node/src/index.ts:1621:12)\n at Module.load (node:internal/modules/cjs/loader:1208:32)\n at Function.Module._load (node:internal/modules/cjs/loader:1024:12)\n at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:174:12)\n at node:internal/main/run_main_module:28:49', + }, + msg: 'kaboom', + }, + { + level: 30, + time: 1725632753070, + pid: 492854, + hostname: 'izanami', + msg: null, + }, + { + level: 30, + time: 1725632753070, + pid: 492854, + hostname: 'izanami', + err: { + type: 'Error', + message: 'kaboom', + stack: + 'Error: kaboom\n at Object. (/home/kuzzle/pino-transport-ecs/tests/application/app.ts:56:13)\n at Module._compile (node:internal/modules/cjs/loader:1358:14)\n at Module.m._compile (/home/kuzzle/pino-transport-ecs/node_modules/ts-node/src/index.ts:1618:23)\n at Module._extensions..js (node:internal/modules/cjs/loader:1416:10)\n at Object.require.extensions. [as .ts] (/home/kuzzle/pino-transport-ecs/node_modules/ts-node/src/index.ts:1621:12)\n at Module.load (node:internal/modules/cjs/loader:1208:32)\n at Function.Module._load (node:internal/modules/cjs/loader:1024:12)\n at Function.executeUserEntryPoint [as runMain] (node:internal/modules/run_main:174:12)\n at node:internal/main/run_main_module:28:49', + }, + msg: 'with', + }, + { + level: 30, + time: 1725632753074, + pid: 492854, + hostname: 'izanami', + msg: 'after setImmediate', + }, +]; diff --git a/tests/unit/transform.test.ts b/tests/unit/transform.test.ts new file mode 100644 index 0000000..6d1fa50 --- /dev/null +++ b/tests/unit/transform.test.ts @@ -0,0 +1,47 @@ +import ecsFormat from '@elastic/ecs-pino-format'; +import { Readable, Transform } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import build, { PinoConfig } from 'pino-abstract-transport'; +import { getTransform } from '../../src/transform'; +import { testLogs } from '../fixtures/testLogs'; + +describe('Transform', () => { + it('should convert all the examples messages without error', async () => { + const testLogsStream: Transform & build.OnUnknown & PinoConfig = Readable.from( + testLogs, + ) as Transform & build.OnUnknown & PinoConfig; + + testLogsStream.levels = { + labels: { + 10: 'trace', + 20: 'debug', + 30: 'info', + 40: 'warn', + 50: 'error', + 60: 'fatal', + }, + values: { + trace: 10, + debug: 20, + info: 30, + warn: 40, + error: 50, + fatal: 60, + }, + }; + testLogsStream.messageKey = 'msg'; + testLogsStream.errorKey = 'err'; + + const pinoConfigEcs = ecsFormat(); + const transformStream = getTransform(testLogsStream, pinoConfigEcs, {}); + + const pipelinePromise = pipeline(testLogsStream, transformStream); + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (const line of transformStream) { + // void data + } + + await expect(pipelinePromise).resolves.toBeUndefined(); + }); +});