log-server: fix log number being exported (#186)

* log-server: fix log number being exported fixes #185 * changeset
QuentinRoy · Nov 22, 2023 · cd62201 · cd62201
1 parent 0a34617
commit cd62201
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 18 deletions.
diff --git a/.changeset/lucky-emus-yell.md b/.changeset/lucky-emus-yell.md
@@ -0,0 +1,5 @@
+---
+'@lightmill/log-server': patch
+---
+
+Fix extra column "number" being exported
diff --git a/packages/log-server/__tests__/app.test.ts b/packages/log-server/__tests__/app.test.ts
@@ -648,7 +648,6 @@ describe('logs', () => {
         [
           {
             "experimentId": "getLogs:experimentId-1",
-            "number": 1,
             "runId": "getLogs:runId-1",
             "type": "getLogs:type-1",
             "values": {
@@ -658,7 +657,6 @@ describe('logs', () => {
           },
           {
             "experimentId": "getLogs:experimentId-2",
-            "number": 2,
             "runId": "getLogs:runId-2",
             "type": "getLogs:type-2",
             "values": {
@@ -683,7 +681,6 @@ describe('logs', () => {
         [
           {
             "experimentId": "getLogs:experimentId-1",
-            "number": 1,
             "runId": "getLogs:runId-1",
             "type": "getLogs:type-1",
             "values": {
@@ -693,7 +690,6 @@ describe('logs', () => {
           },
           {
             "experimentId": "getLogs:experimentId-2",
-            "number": 2,
             "runId": "getLogs:runId-2",
             "type": "getLogs:type-2",
             "values": {
@@ -715,9 +711,9 @@ describe('logs', () => {
         .expect(200);
       expect(store.getLogs).toHaveBeenCalledWith({ experimentId: 'exp' });
       expect(result.text).toMatchInlineSnapshot(`
-        "type,run_id,mock_col1,mock_col2,mock_col3,number
-        getLogs:type-1,getLogs:runId-1,log1-mock-value1,log1-mock-value2,,1
-        getLogs:type-2,getLogs:runId-2,log2-mock-value1,log2-mock-value2,log2-mock-value3,2
+        "type,run_id,mock_col1,mock_col2,mock_col3
+        getLogs:type-1,getLogs:runId-1,log1-mock-value1,log1-mock-value2,
+        getLogs:type-2,getLogs:runId-2,log2-mock-value1,log2-mock-value2,log2-mock-value3
         "
       `);
     });
@@ -731,7 +727,6 @@ describe('logs', () => {
         [
           {
             "experimentId": "getLogs:experimentId-1",
-            "number": 1,
             "runId": "getLogs:runId-1",
             "type": "getLogs:type-1",
             "values": {
@@ -741,7 +736,6 @@ describe('logs', () => {
           },
           {
             "experimentId": "getLogs:experimentId-2",
-            "number": 2,
             "runId": "getLogs:runId-2",
             "type": "getLogs:type-2",
             "values": {

diff --git a/packages/log-server/package.json b/packages/log-server/package.json
@@ -36,6 +36,7 @@
     "@types/supertest": "^2.0.16",
     "@types/yargs": "^17.0.32",
     "cross-env": "^7.0.3",
+    "string-dedent": "^3.0.1",
     "supertest": "^6.3.3",
     "tsx": "^4.2.0",
     "type-fest": "^4.8.2",

diff --git a/packages/log-server/src/export.ts b/packages/log-server/src/export.ts
@@ -4,14 +4,19 @@ import { mapKeys, pickBy, pipe } from 'remeda';
 import { Log, Store } from './store.js';
 import { toSnakeCase } from './utils.js';
 
-const logColumns: Array<keyof Log> = ['type', 'experimentId', 'runId'];
+const csvLogColumns: Array<keyof Log> = ['type', 'experimentId', 'runId'];
+const jsonLogColumns: Array<keyof Log> = [
+  'type',
+  'experimentId',
+  'runId',
+  'values',
+];
 const renamedLogColumns: Partial<Record<keyof Log, string>> = {};
-const ignoredLogValues: Array<keyof Log> = ['values'];
 
 export function csvExportStream(
-  store: Store,
+  store: Pick<Store, 'getLogs' | 'getLogValueNames'>,
   filter: Parameters<Store['getLogs']>[0] &
-    Parameters<Store['getLogValueNames']>[0],
+    Parameters<Store['getLogValueNames']>[0] = {},
 ): Readable {
   return pipeline(
     async function* () {
@@ -21,9 +26,9 @@ export function csvExportStream(
         (filter?.type == null || columnName !== 'type') &&
         (filter?.experimentId == null || columnName !== 'experimentId') &&
         (filter?.runId == null || columnName !== 'runId') &&
-        !ignoredLogValues.includes(columnName);
+        csvLogColumns.includes(columnName);
       let columns = [
-        ...logColumns
+        ...csvLogColumns
           .filter(logColumnFilter)
           .map((c) => renamedLogColumns[c] ?? c),
         ...valueColumns,
@@ -65,8 +70,8 @@ export function csvExportStream(
 }
 
 export function jsonExportStream(
-  store: Store,
-  filter: Parameters<Store['getLogs']>[0],
+  store: Pick<Store, 'getLogs'>,
+  filter: Parameters<Store['getLogs']>[0] = {},
 ) {
   return Readable.from(stringifyLogs(store.getLogs(filter)));
 }
@@ -78,7 +83,11 @@ async function* stringifyLogs(logs: AsyncIterable<Log>) {
     yield started ? ',' : '';
     started = true;
     yield JSON.stringify(
-      mapKeys(log, (key) => renamedLogColumns[key] ?? key),
+      pipe(
+        log,
+        pickBy((v, k) => jsonLogColumns.includes(k)),
+        mapKeys((key) => renamedLogColumns[key] ?? key),
+      ),
       stringifyDateSerializer,
     );
   }

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml