diff --git a/.changeset/rotten-maps-clap.md b/.changeset/rotten-maps-clap.md new file mode 100644 index 00000000..1174c9b6 --- /dev/null +++ b/.changeset/rotten-maps-clap.md @@ -0,0 +1,5 @@ +--- +'@lightmill/log-server': minor +--- + +Significantly increase default select query limit of SQLiteStore diff --git a/.changeset/soft-walls-type.md b/.changeset/soft-walls-type.md new file mode 100644 index 00000000..6326510c --- /dev/null +++ b/.changeset/soft-walls-type.md @@ -0,0 +1,5 @@ +--- +'@lightmill/log-server': minor +--- + +Display count of exported logs during export when output is a file. diff --git a/packages/log-server/src/cli.ts b/packages/log-server/src/cli.ts index 997d484f..b618d841 100644 --- a/packages/log-server/src/cli.ts +++ b/packages/log-server/src/cli.ts @@ -12,6 +12,7 @@ import loglevel from 'loglevel'; import yargs from 'yargs'; import { SQLiteStore, LogServer } from './index.js'; import { csvExportStream, jsonExportStream } from './export.js'; +import { Transform } from 'node:stream'; // Constants and setup // ------------------- @@ -121,9 +122,39 @@ async function exportLogs({ : jsonExportStream(store, filter); if (output === undefined) { stream.pipe(process.stdout).on('error', handleError); - } else { - stream.pipe(createWriteStream(output)).on('error', handleError); + return; } + let startDate = new Date(); + let logCount = 0; + process.stdout.write(`${logCount.toLocaleString('en')} logs exported...`); + stream + .pipe( + new Transform({ + writableObjectMode: true, + transform(chunk, encoding, callback) { + process.stdout.cursorTo(0); + logCount += 1; + process.stdout.write( + `${logCount.toLocaleString('en')} logs exported...`, + ); + callback(null, chunk); + }, + }), + ) + .pipe(createWriteStream(output)) + .on('error', handleError) + .on('finish', () => { + process.stdout.clearLine(0); + process.stdout.cursorTo(0); + let durationInSeconds = (Date.now() - startDate.getTime()) / 1000; + process.stdout.write( + `${logCount.toLocaleString( + 'en', + )} logs exported in ${durationInSeconds.toLocaleString( + 'en', + )} seconds.\n`, + ); + }); } type MigrateDatabaseParameter = { diff --git a/packages/log-server/src/store.ts b/packages/log-server/src/store.ts index bd99c30a..f80ee6cb 100644 --- a/packages/log-server/src/store.ts +++ b/packages/log-server/src/store.ts @@ -18,7 +18,7 @@ import loglevel, { LogLevelDesc } from 'loglevel'; import { groupBy, last } from 'remeda'; import { arrayify } from './utils.js'; -const DEFAULT_SELECT_QUERY_LIMIT = 10000; +const DEFAULT_SELECT_QUERY_LIMIT = 1000000; const __dirname = url.fileURLToPath(new URL('.', import.meta.url)); const migrationFolder = path.join(__dirname, 'db-migrations');