diff --git a/backend/src/export/CSVFormatter.test.ts b/backend/src/export/CSVFormatter.test.ts new file mode 100644 index 0000000..0f27997 --- /dev/null +++ b/backend/src/export/CSVFormatter.test.ts @@ -0,0 +1,54 @@ +import { Writable } from 'node:stream'; +import { CSVFormatter } from './CSVFormatter'; + +describe('CSVFormatter', () => { + it('streams the given CSV table into a buffer', async () => { + const formatter = new CSVFormatter(); + const stream = TestStream(); + await formatter.stream(stream.writable, [ + ['a', 'b'], + ['c', 'd'], + ]); + expect(stream.getValue()).toEqual('a,b\nc,d\n'); + }); + + it('quotes values with special characters', async () => { + const formatter = new CSVFormatter(); + const stream = TestStream(); + await formatter.stream(stream.writable, [['a\nb', 'c"']]); + expect(stream.getValue()).toEqual('"a\nb","c"""\n'); + }); + + it('consumes iterators in the input', async () => { + const formatter = new CSVFormatter(); + const stream = TestStream(); + let advance = () => {}; + const delay = new Promise((resolve) => { + advance = resolve; + }); + const rowGenerator = (async function* () { + yield ['a', 'b']; + await delay; + yield ['c', 'd']; + })(); + const promise = formatter.stream(stream.writable, rowGenerator); + await new Promise(process.nextTick); + expect(stream.getValue()).toEqual('a,b\n'); + advance(); + await promise; + expect(stream.getValue()).toEqual('a,b\nc,d\n'); + }); +}); + +const TestStream = () => { + const chunks: string[] = []; + return { + writable: new Writable({ + write(chunk, _, callback) { + chunks.push(chunk); + callback(); + }, + }), + getValue: () => chunks.join(''), + }; +}; diff --git a/backend/src/export/CSVFormatter.ts b/backend/src/export/CSVFormatter.ts new file mode 100644 index 0000000..f158dc2 --- /dev/null +++ b/backend/src/export/CSVFormatter.ts @@ -0,0 +1,62 @@ +import type { Writable } from 'stream'; + +type MaybeAsyncIterable = Iterable | AsyncIterable; + +export class CSVFormatter { + private readonly escapedQuote: string; + constructor( + private readonly delimiter = ',', + private readonly newline = '\n', + private readonly quote = '"', + ) { + this.escapedQuote = quote + quote; + } + + private encodeCSVCell(target: Writable, content: string) { + if (SIMPLE_CSV_CELL.test(content)) { + target.write(content); + } else { + target.write(this.quote); + target.write(content.replaceAll(this.quote, this.escapedQuote)); + target.write(this.quote); + } + } + + async stream( + target: Writable, + rows: MaybeAsyncIterable>, + ) { + for await (const row of rows) { + let col = 0; + if (Symbol.iterator in row) { + for (const cell of row) { + if (target.writableNeedDrain) { + await awaitDrain(target); + } + if (col) { + target.write(this.delimiter); + } + this.encodeCSVCell(target, cell); + ++col; + } + } else { + for await (const cell of row) { + if (target.writableNeedDrain) { + await awaitDrain(target); + } + if (col) { + target.write(this.delimiter); + } + this.encodeCSVCell(target, cell); + ++col; + } + } + target.write(this.newline); + } + } +} + +const SIMPLE_CSV_CELL = /^[^"':;\\\r\n\t ]*$/i; + +const awaitDrain = (target: Writable) => + new Promise((resolve) => target.once('drain', resolve)); diff --git a/backend/src/export/JSONFormatter.test.ts b/backend/src/export/JSONFormatter.test.ts new file mode 100644 index 0000000..5325830 --- /dev/null +++ b/backend/src/export/JSONFormatter.test.ts @@ -0,0 +1,68 @@ +import { Writable } from 'node:stream'; +import { JSONFormatter } from './JSONFormatter'; + +describe('JSONFormatter', () => { + it('streams the given JSON object into a buffer', async () => { + const formatter = JSONFormatter.Builder().build(); + const stream = TestStream(); + await formatter.stream(stream.writable, { foo: 1.2, bar: ['value'] }); + expect(stream.getValue()).toEqual('{"foo":1.2,"bar":["value"]}'); + }); + + it('includes whitespace if configured', async () => { + const formatter = JSONFormatter.Builder().withIndent(2).build(); + const stream = TestStream(); + await formatter.stream(stream.writable, { foo: 1.2, bar: ['value'] }); + expect(stream.getValue()).toEqual( + '{\n "foo": 1.2,\n "bar": [\n "value"\n ]\n}', + ); + }); + + it('ignores undefined properties', async () => { + const formatter = JSONFormatter.Builder().build(); + const stream = TestStream(); + await formatter.stream(stream.writable, { + foo: false, + bar: null, + baz: undefined, + list: [null, undefined], + }); + expect(stream.getValue()).toEqual( + '{"foo":false,"bar":null,"list":[null,null]}', + ); + }); + + it('consumes iterators in the input', async () => { + const formatter = JSONFormatter.Builder().build(); + const stream = TestStream(); + let advance = () => {}; + const delay = new Promise((resolve) => { + advance = resolve; + }); + const itemGenerator = (async function* () { + yield 1; + yield 2; + await delay; + yield 3; + })(); + const promise = formatter.stream(stream.writable, { list: itemGenerator }); + await new Promise(process.nextTick); + expect(stream.getValue()).toEqual('{"list":[1,2'); + advance(); + await promise; + expect(stream.getValue()).toEqual('{"list":[1,2,3]}'); + }); +}); + +const TestStream = () => { + const chunks: string[] = []; + return { + writable: new Writable({ + write(chunk, _, callback) { + chunks.push(chunk); + callback(); + }, + }), + getValue: () => chunks.join(''), + }; +}; diff --git a/backend/src/export/JSONFormatter.ts b/backend/src/export/JSONFormatter.ts new file mode 100644 index 0000000..38b059a --- /dev/null +++ b/backend/src/export/JSONFormatter.ts @@ -0,0 +1,186 @@ +import type { Writable } from 'stream'; + +type JSONWriter = ( + value: T, + context: JSONFormatterContext, +) => Promise | void; + +interface JSONType { + detector: (v: unknown) => v is T; + writer: JSONWriter; +} + +interface JSONFormatterContext { + write(v: string): void; + stream(object: unknown, nesting?: number): Promise; + joiner( + begin: string, + end: string, + nesting?: number, + ): { next: () => void; end: () => void }; + newline(nesting?: number): void; + whitespace: boolean; +} + +class Builder { + private readonly types: JSONType[] = []; + private indent: number | string = ''; + private nestingLimit: number = 20; + + constructor( + private readonly target: ( + ...args: [JSONType[], number | string, number] + ) => JSONFormatter, + ) {} + + withType(type: JSONType) { + this.types.unshift(type as JSONType); + return this; + } + + withIndent(indent: number | string) { + this.indent = indent; + return this; + } + + withNestingLimit(nestingLimit: number) { + this.nestingLimit = nestingLimit; + return this; + } + + build() { + return this.target([...this.types], this.indent, this.nestingLimit); + } +} + +export class JSONFormatter { + private readonly newline: + | ((target: Writable, nesting: number) => void) + | null; + private readonly whitespace: boolean; + + private constructor( + private readonly types: JSONType[], + indent: number | string, + nestingLimit: number, + ) { + if (indent) { + const indentStep = + typeof indent === 'number' ? ' '.repeat(indent) : indent; + const newlines: string[] = []; + for (let i = 0; i <= nestingLimit; ++i) { + newlines.push('\n' + indentStep.repeat(i)); + } + this.newline = (target: Writable, nesting: number) => + target.write(newlines[Math.min(nesting, nestingLimit)]); + this.whitespace = true; + } else { + this.newline = null; + this.whitespace = false; + } + } + + static Builder() { + return new Builder((...args) => new JSONFormatter(...args)) + .withType(JSONFormatter.OBJECT) + .withType(JSONFormatter.ASYNC_ITERABLE) + .withType(JSONFormatter.ITERABLE); + } + + private joiner( + target: Writable, + nesting: number, + begin: string, + end: string, + ) { + let count = 0; + target.write(begin); + return { + next: () => { + if (count) { + target.write(','); + } + this.newline?.(target, nesting + 1); + ++count; + }, + end: () => { + if (count) { + this.newline?.(target, nesting); + } + target.write(end); + }, + }; + } + + async stream(target: Writable, object: unknown, nesting: number = 0) { + if (target.writableNeedDrain) { + await awaitDrain(target); + } + for (const { detector, writer } of this.types) { + if (detector(object)) { + await writer(object, { + write: (v) => target.write(v), + stream: (subObject: unknown, subNesting: number = 0) => + this.stream(target, subObject, nesting + subNesting), + joiner: (begin: string, end: string, subNesting: number = 0) => + this.joiner(target, nesting + subNesting, begin, end), + newline: (subNesting: number = 0) => + this.newline?.(target, nesting + subNesting), + whitespace: this.whitespace, + }); + return; + } + } + target.write(JSON.stringify(object ?? null)); + } + + static readonly ITERABLE: JSONType> = { + detector: (object): object is Iterable => + Boolean( + object && typeof object === 'object' && Symbol.iterator in object, + ), + async writer(object, { joiner, stream }) { + const j = joiner('[', ']'); + for (const v of object) { + j.next(); + await stream(v, 1); + } + j.end(); + }, + }; + + static readonly ASYNC_ITERABLE: JSONType> = { + detector: (object): object is AsyncIterable => + Boolean( + object && typeof object === 'object' && Symbol.asyncIterator in object, + ), + async writer(object, { joiner, stream }) { + const j = joiner('[', ']'); + for await (const v of object) { + j.next(); + await stream(v, 1); + } + j.end(); + }, + }; + + static readonly OBJECT: JSONType = { + detector: (object): object is object => + Boolean(object && typeof object === 'object'), + async writer(object, { write, joiner, stream, whitespace }) { + const j = joiner('{', '}'); + for (const [k, v] of Object.entries(object)) { + if (v !== undefined) { + j.next(); + write(JSON.stringify(k)); + write(whitespace ? ': ' : ':'); + await stream(v, 1); + } + } + j.end(); + }, + }; +} + +const awaitDrain = (target: Writable) => + new Promise((resolve) => target.once('drain', resolve)); diff --git a/backend/src/export/RetroJsonExport.ts b/backend/src/export/RetroJsonExport.ts index b1d6c09..f20970c 100644 --- a/backend/src/export/RetroJsonExport.ts +++ b/backend/src/export/RetroJsonExport.ts @@ -7,6 +7,8 @@ import { type RetroArchive, } from '../shared/api-entities'; +type MaybeAsyncIterable = Iterable | AsyncIterable; + export interface RetroItemAttachmentJsonExport { type: string; url: string; @@ -37,10 +39,10 @@ export interface RetroJsonExport { url: string; name: string; current: RetroDataJsonExport; - archives?: RetroArchiveJsonExport[]; + archives?: AsyncIterable; } -export function exportTimestamp(timestamp: number): string { +function exportTimestamp(timestamp: number): string { const date = new Date(timestamp); return date.toISOString(); } @@ -49,7 +51,7 @@ export function importTimestamp(isoDate: string): number { return Date.parse(isoDate); } -export function exportRetroItemAttachment( +function exportRetroItemAttachment( attachment: RetroItemAttachment, ): RetroItemAttachmentJsonExport { return { @@ -58,7 +60,7 @@ export function exportRetroItemAttachment( }; } -export function importRetroItemAttachment( +function importRetroItemAttachment( attachment: RetroItemAttachmentJsonExport, ): RetroItemAttachment { return { @@ -67,7 +69,7 @@ export function importRetroItemAttachment( }; } -export function exportRetroItem(item: RetroItem): RetroItemJsonExport { +function exportRetroItem(item: RetroItem): RetroItemJsonExport { const result: RetroItemJsonExport = { created: exportTimestamp(item.created), category: item.category, @@ -84,7 +86,7 @@ export function exportRetroItem(item: RetroItem): RetroItemJsonExport { return result; } -export function importRetroItem(item: RetroItemJsonExport): RetroItem { +function importRetroItem(item: RetroItemJsonExport): RetroItem { return { id: randomUUID(), created: importTimestamp(item.created), @@ -99,7 +101,7 @@ export function importRetroItem(item: RetroItemJsonExport): RetroItem { }; } -export function exportRetroData(archive: RetroData): RetroDataJsonExport { +function exportRetroData(archive: RetroData): RetroDataJsonExport { return { format: archive.format, options: archive.options, @@ -107,7 +109,7 @@ export function exportRetroData(archive: RetroData): RetroDataJsonExport { }; } -export function importRetroData(archive: RetroDataJsonExport): RetroData { +export function importRetroDataJson(archive: RetroDataJsonExport): RetroData { return { format: archive.format, options: archive.options, @@ -115,26 +117,33 @@ export function importRetroData(archive: RetroDataJsonExport): RetroData { }; } -export function exportRetroArchive( - archive: RetroArchive, -): RetroArchiveJsonExport { +function exportRetroArchive(archive: RetroArchive): RetroArchiveJsonExport { return { created: exportTimestamp(archive.created), snapshot: exportRetroData(archive), }; } -export function exportRetro( +export function exportRetroJson( retro: Retro, - archives?: Readonly, + archives?: MaybeAsyncIterable, ): RetroJsonExport { const result: RetroJsonExport = { url: retro.slug, name: retro.name, current: exportRetroData(retro), }; - if (archives && archives.length > 0) { - result.archives = archives.map(exportRetroArchive); + if (archives) { + result.archives = map(archives, exportRetroArchive); } return result; } + +async function* map( + input: MaybeAsyncIterable, + fn: (i: I) => O, +): AsyncGenerator { + for await (const item of input) { + yield fn(item); + } +} diff --git a/backend/src/export/RetroTableExport.ts b/backend/src/export/RetroTableExport.ts new file mode 100644 index 0000000..3ca49ba --- /dev/null +++ b/backend/src/export/RetroTableExport.ts @@ -0,0 +1,67 @@ +import { + type Retro, + type RetroArchive, + type RetroItem, +} from '../shared/api-entities'; + +type MaybeAsyncIterable = Iterable | AsyncIterable; + +function exportItems( + items: RetroItem[], + archive: RetroArchive | null, + index: number, +): string[][] { + let archiveID = 'current'; + if (archive) { + archiveID = `#${index + 1} (${dateString(new Date(archive.created))})`; + } + return [...items] + .sort(RETRO_ITEM_ORDER) + .map((item) => [ + archiveID, + CATEGORIES.get(item.category)?.label ?? item.category, + item.message, + String(item.votes), + item.category === 'action' + ? item.doneTime > 0 + ? 'Complete' + : '' + : item.doneTime > 0 + ? 'Discussed' + : '', + ]); +} + +function dateString(date: Date) { + return date.toISOString().split('T')[0]; +} + +export async function* exportRetroTable( + retro: Retro, + archives?: MaybeAsyncIterable, +): AsyncGenerator { + yield ['Archive', 'Category', 'Message', 'Votes', 'State']; + yield* exportItems(retro.items, null, 0); + if (archives) { + let i = 0; + for await (const archive of archives) { + yield* exportItems(archive.items, archive, i); + ++i; + } + } +} + +const CATEGORIES = new Map([ + ['happy', { label: 'Happy' }], + ['meh', { label: 'Question' }], + ['sad', { label: 'Sad' }], + ['action', { label: 'Action' }], +]); + +const CATEGORY_ORDER = [...CATEGORIES.keys()]; + +const RETRO_ITEM_ORDER = (a: RetroItem, b: RetroItem) => + CATEGORY_ORDER.indexOf(a.category) - CATEGORY_ORDER.indexOf(b.category) || + a.doneTime - b.doneTime || + a.votes - b.votes || + a.created - b.created; diff --git a/backend/src/helpers/exportedJsonParsers.ts b/backend/src/helpers/exportedJsonParsers.ts index 2d44278..d5149dd 100644 --- a/backend/src/helpers/exportedJsonParsers.ts +++ b/backend/src/helpers/exportedJsonParsers.ts @@ -38,7 +38,23 @@ export const extractExportedRetroData = json.object({ items: json.array(extractExportedRetroItem), }); -export const extractExportedRetro = json.object({ +type Sync = { + [k in keyof T]: T[k] extends string | undefined + ? T[k] + : T[k] extends AsyncIterable + ? Sync[] + : T[k] extends Iterable + ? Sync[] + : T[k] extends AsyncIterable | undefined + ? Sync[] | undefined + : T[k] extends Iterable | undefined + ? Sync[] | undefined + : T[k] extends object + ? Sync + : T[k]; +}; + +export const extractExportedRetro = json.object>({ url: json.string, name: json.string, current: extractExportedRetroData, diff --git a/backend/src/helpers/routeHelpers.ts b/backend/src/helpers/routeHelpers.ts new file mode 100644 index 0000000..28a48cc --- /dev/null +++ b/backend/src/helpers/routeHelpers.ts @@ -0,0 +1,11 @@ +import type { RequestHandler } from 'express'; + +export function safe(handler: H): H { + return (async (req, res, next) => { + try { + await handler(req, res, next); + } catch (e) { + next(e); + } + }) as H; +} diff --git a/backend/src/routers/ApiRetrosRouter.ts b/backend/src/routers/ApiRetrosRouter.ts index f168c20..e2fbc17 100644 --- a/backend/src/routers/ApiRetrosRouter.ts +++ b/backend/src/routers/ApiRetrosRouter.ts @@ -6,19 +6,27 @@ import { type RetroAuthService } from '../services/RetroAuthService'; import { type RetroService } from '../services/RetroService'; import { type RetroArchiveService } from '../services/RetroArchiveService'; import { - exportRetro, - importRetroData, + exportRetroJson, + importRetroDataJson, importTimestamp, } from '../export/RetroJsonExport'; import { extractExportedRetro } from '../helpers/exportedJsonParsers'; import { json } from '../helpers/json'; +import { safe } from '../helpers/routeHelpers'; import { logError } from '../log'; +import { JSONFormatter } from '../export/JSONFormatter'; +import { CSVFormatter } from '../export/CSVFormatter'; +import { exportRetroTable } from '../export/RetroTableExport'; const MIN_PASSWORD_LENGTH = 8; const MAX_PASSWORD_LENGTH = 512; const JSON_BODY = WebSocketExpress.json({ limit: 512 * 1024 }); +const formattedJSON = JSONFormatter.Builder().withIndent(2).build(); + +const formattedCSV = new CSVFormatter(); + export class ApiRetrosRouter extends Router { public readonly softClose: (timeout: number) => Promise; @@ -41,13 +49,17 @@ export class ApiRetrosRouter extends Router { ); this.softClose = (timeout) => wsHandlerFactory.softClose(timeout); - this.get('/', userAuthMiddleware, async (_, res) => { - const userId = WebSocketExpress.getAuthData(res).sub!; + this.get( + '/', + userAuthMiddleware, + safe(async (_, res) => { + const userId = WebSocketExpress.getAuthData(res).sub!; - res.json({ - retros: await retroService.getRetroListForUser(userId), - }); - }); + res.json({ + retros: await retroService.getRetroListForUser(userId), + }); + }), + ); this.post('/', userAuthMiddleware, JSON_BODY, async (req, res) => { try { @@ -78,7 +90,7 @@ export class ApiRetrosRouter extends Router { if (importJson) { await retroService.retroBroadcaster.update(id, [ 'merge', - importRetroData(importJson.current), + importRetroDataJson(importJson.current), ]); const archives = importJson.archives || []; @@ -87,7 +99,7 @@ export class ApiRetrosRouter extends Router { archives.map((exportedArchive) => retroArchiveService.createArchive( id, - importRetroData(exportedArchive.snapshot), + importRetroDataJson(exportedArchive.snapshot), importTimestamp(exportedArchive.created), ), ), @@ -130,11 +142,11 @@ export class ApiRetrosRouter extends Router { ); this.get( - '/:retroId/export/json', + '/:retroId/export/:format', WebSocketExpress.requireAuthScope('read'), WebSocketExpress.requireAuthScope('readArchives'), - async (req, res) => { - const { retroId } = req.params; + safe(async (req, res) => { + const { retroId, format } = req.params; const retro = await retroService.getRetro(retroId); if (!retro) { @@ -143,16 +155,36 @@ export class ApiRetrosRouter extends Router { } const archives = await retroArchiveService.getRetroArchiveList(retroId); - const fileName = `${retro.slug}-export.json`; - const data = exportRetro(retro, archives); - res.header( - 'content-disposition', - `attachment; filename="${encodeURIComponent(fileName)}"`, - ); - res.header('content-type', 'application/json; charset=utf-8'); - res.send(JSON.stringify(data, null, 2)).end(); - }, + switch (format) { + case 'json': { + res.header( + 'content-disposition', + `attachment; filename="${encodeURIComponent(`${retro.slug}-export.json`)}"`, + ); + res.header('content-type', 'application/json; charset=utf-8'); + await formattedJSON.stream(res, exportRetroJson(retro, archives)); + res.end(); + break; + } + case 'csv': { + res.header( + 'content-disposition', + `attachment; filename="${encodeURIComponent(`${retro.slug}-export.csv`)}"`, + ); + res.header( + 'content-type', + 'text/csv; charset=utf-8; header=present', + ); + await formattedCSV.stream(res, exportRetroTable(retro, archives)); + res.end(); + break; + } + default: + res.status(404).end(); + break; + } + }), ); this.use( diff --git a/frontend/src/components/archive-list/ArchiveListPage.tsx b/frontend/src/components/archive-list/ArchiveListPage.tsx index 7fc05a7..34fb626 100644 --- a/frontend/src/components/archive-list/ArchiveListPage.tsx +++ b/frontend/src/components/archive-list/ArchiveListPage.tsx @@ -37,6 +37,14 @@ export const ArchiveListPage = memo(({ retroToken, retro }: PropsT) => { > Export as JSON + {' / '} + + Export items as CSV + );