node: overhaul of database and storage [INT-355]

packages/node/src/BacktraceClient.ts

@@ -7,49 +7,105 @@ import {
     SessionFiles,
     VariableDebugIdMapProvider,
 } from '@backtrace/sdk-core';
-import path from 'path';
+import nodeFs from 'fs';
 import { BacktraceConfiguration, BacktraceSetupConfiguration } from './BacktraceConfiguration.js';
 import { BacktraceNodeRequestHandler } from './BacktraceNodeRequestHandler.js';
 import { AGENT } from './agentDefinition.js';
+import {
+    BacktraceFileAttachmentFactory,
+    NodeFsBacktraceFileAttachmentFactory,
+} from './attachment/BacktraceFileAttachment.js';
 import { FileAttachmentsManager } from './attachment/FileAttachmentsManager.js';
 import { transformAttachment } from './attachment/transformAttachments.js';
 import { FileBreadcrumbsStorage } from './breadcrumbs/FileBreadcrumbsStorage.js';
 import { BacktraceClientBuilder } from './builder/BacktraceClientBuilder.js';
 import { BacktraceNodeClientSetup } from './builder/BacktraceClientSetup.js';
 import { NodeOptionReader } from './common/NodeOptionReader.js';
+import { toArray } from './common/asyncGenerator.js';
 import { NodeDiagnosticReportConverter } from './converter/NodeDiagnosticReportConverter.js';
-import { FsNodeFileSystem } from './storage/FsNodeFileSystem.js';
-import { NodeFileSystem } from './storage/interfaces/NodeFileSystem.js';
+import {
+    AttachmentBacktraceDatabaseRecordSender,
+    AttachmentBacktraceDatabaseRecordSerializer,
+} from './database/AttachmentBacktraceDatabaseRecord.js';
+import {
+    ReportBacktraceDatabaseRecordWithAttachmentsFactory,
+    ReportBacktraceDatabaseRecordWithAttachmentsSender,
+    ReportBacktraceDatabaseRecordWithAttachmentsSerializer,
+} from './database/ReportBacktraceDatabaseRecordWithAttachments.js';
+import { assertDatabasePath } from './database/utils.js';
+import { BacktraceStorageModule } from './storage/BacktraceStorage.js';
+import { BacktraceStorageModuleFactory } from './storage/BacktraceStorageModuleFactory.js';
+import { NodeFsBacktraceStorageModuleFactory } from './storage/NodeFsBacktraceStorage.js';
+import { NodeFs } from './storage/nodeFs.js';
 
 export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration> {
     private _listeners: Record<string, NodeJS.UnhandledRejectionListener | NodeJS.UncaughtExceptionListener> = {};
 
-    protected get nodeFileSystem() {
-        return this.fileSystem as NodeFileSystem | undefined;
+    protected readonly storageFactory: BacktraceStorageModuleFactory;
+    protected readonly fileAttachmentFactory: BacktraceFileAttachmentFactory;
+    protected readonly fs: NodeFs;
+
+    protected get databaseNodeFsStorage() {
+        return this.databaseStorage as BacktraceStorageModule | undefined;
     }
 
     constructor(clientSetup: BacktraceNodeClientSetup) {
-        const fileSystem = clientSetup.fileSystem ?? new FsNodeFileSystem();
+        const storageFactory = clientSetup.storageFactory ?? new NodeFsBacktraceStorageModuleFactory();
+        const fs = clientSetup.fs ?? nodeFs;
+        const fileAttachmentFactory = new NodeFsBacktraceFileAttachmentFactory(fs);
+        const storage =
+            clientSetup.database?.storage ??
+            (clientSetup.options.database?.enable
+                ? storageFactory.create({
+                      path: assertDatabasePath(clientSetup.options.database.path),
+                      createDirectory: clientSetup.options.database.createDatabaseDirectory,
+                      fs,
+                  })
+                : undefined);
+
         super({
             sdkOptions: AGENT,
             requestHandler: new BacktraceNodeRequestHandler(clientSetup.options),
             debugIdMapProvider: new VariableDebugIdMapProvider(global as DebugIdContainer),
+            database:
+                clientSetup.options.database?.enable && storage
+                    ? {
+                          storage,
+                          reportRecordFactory: ReportBacktraceDatabaseRecordWithAttachmentsFactory.default(),
+                          ...clientSetup.database,
+                          recordSenders: (submission) => ({
+                              report: new ReportBacktraceDatabaseRecordWithAttachmentsSender(submission),
+                              attachment: new AttachmentBacktraceDatabaseRecordSender(submission),
+                              ...clientSetup.database?.recordSenders?.(submission),
+                          }),
+                          recordSerializers: {
+                              report: new ReportBacktraceDatabaseRecordWithAttachmentsSerializer(fileAttachmentFactory),
+                              attachment: new AttachmentBacktraceDatabaseRecordSerializer(fileAttachmentFactory),
+                              ...clientSetup.database?.recordSerializers,
+                          },
+                      }
+                    : undefined,
             ...clientSetup,
-            fileSystem,
             options: {
                 ...clientSetup.options,
-                attachments: clientSetup.options.attachments?.map(transformAttachment),
+                attachments: clientSetup.options.attachments?.map(transformAttachment(fileAttachmentFactory)),
             },
         });
 
+        this.storageFactory = storageFactory;
+        this.fileAttachmentFactory = fileAttachmentFactory;
+        this.fs = fs;
+
         const breadcrumbsManager = this.modules.get(BreadcrumbsManager);
-        if (breadcrumbsManager && this.sessionFiles) {
-            breadcrumbsManager.setStorage(FileBreadcrumbsStorage.factory(this.sessionFiles, fileSystem));
+        if (breadcrumbsManager && this.sessionFiles && storage) {
+            breadcrumbsManager.setStorage(
+                FileBreadcrumbsStorage.factory(this.sessionFiles, storage, this.fileAttachmentFactory),
+            );
         }
 
-        if (this.sessionFiles && clientSetup.options.database?.captureNativeCrashes) {
-            this.addModule(FileAttributeManager, FileAttributeManager.create(fileSystem));
-            this.addModule(FileAttachmentsManager, FileAttachmentsManager.create(fileSystem));
+        if (this.sessionFiles && storage && clientSetup.options.database?.captureNativeCrashes) {
+            this.addModule(FileAttributeManager, FileAttributeManager.create(storage));
+            this.addModule(FileAttachmentsManager, FileAttachmentsManager.create(storage, fileAttachmentFactory));
         }
     }
 
@@ -58,6 +114,7 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
 
         try {
             super.initialize();
+
             this.captureUnhandledErrors(
                 this.options.captureUnhandledErrors,
                 this.options.captureUnhandledPromiseRejections,
@@ -242,18 +299,19 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
     }
 
     private async loadNodeCrashes() {
-        if (!this.database || !this.nodeFileSystem || !this.options.database?.captureNativeCrashes) {
+        if (!this.database || !this.options.database?.captureNativeCrashes) {
             return;
         }
 
         const reportName = process.report?.filename;
-        const databasePath = process.report?.directory
-            ? process.report.directory
-            : (this.options.database?.path ?? process.cwd());
+        const storage = this.storageFactory.create({
+            path: process.report?.directory ? process.report.directory : (this.options.database?.path ?? process.cwd()),
+            fs: this.fs,
+        });
 
         let databaseFiles: string[];
         try {
-            databaseFiles = await this.nodeFileSystem.readDir(databasePath);
+            databaseFiles = await toArray(storage.keys());
         } catch {
             return;
         }
@@ -271,11 +329,14 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
 
         const reports: [path: string, report: BacktraceReport, sessionFiles?: SessionFiles][] = [];
         for (const recordName of recordNames) {
-            const recordPath = path.join(databasePath, recordName);
             try {
-                const recordJson = await this.nodeFileSystem.readFile(recordPath);
+                const recordJson = await storage.get(recordName);
+                if (!recordJson) {
+                    continue;
+                }
+
                 const report = converter.convert(JSON.parse(recordJson));
-                reports.push([recordPath, report]);
+                reports.push([recordName, report]);
             } catch {
                 // Do nothing, skip the report
             }
@@ -292,17 +353,21 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
             currentSession = currentSession?.getPreviousSession();
         }
 
-        for (const [recordPath, report, session] of reports) {
+        for (const [recordName, report, session] of reports) {
             try {
                 if (session) {
                     report.attachments.push(
-                        ...FileBreadcrumbsStorage.getSessionAttachments(session, this.nodeFileSystem),
+                        ...FileBreadcrumbsStorage.getSessionAttachments(session, this.fileAttachmentFactory),
                     );
 
-                    const fileAttributes = FileAttributeManager.createFromSession(session, this.nodeFileSystem);
+                    const fileAttributes = FileAttributeManager.createFromSession(session, storage);
                     Object.assign(report.attributes, await fileAttributes.get());
 
-                    const fileAttachments = FileAttachmentsManager.createFromSession(session, this.nodeFileSystem);
+                    const fileAttachments = FileAttachmentsManager.createFromSession(
+                        session,
+                        storage,
+                        this.fileAttachmentFactory,
+                    );
                     report.attachments.push(...(await fileAttachments.get()));
 
                     report.attributes['application.session'] = session.sessionId;
@@ -318,7 +383,7 @@ export class BacktraceClient extends BacktraceCoreClient<BacktraceConfiguration>
                 // Do nothing, skip the report
             } finally {
                 try {
-                    await this.nodeFileSystem.unlink(recordPath);
+                    await storage.remove(recordName);
                 } catch {
                     // Do nothing
                 }

packages/node/src/BacktraceConfiguration.ts

@@ -1,10 +1,37 @@
-import { BacktraceAttachment, BacktraceConfiguration as CoreConfiguration } from '@backtrace/sdk-core';
+import {
+    BacktraceAttachment,
+    BacktraceConfiguration as CoreConfiguration,
+    DisabledBacktraceDatabaseConfiguration as CoreDisabledBacktraceDatabaseConfiguration,
+    EnabledBacktraceDatabaseConfiguration as CoreEnabledBacktraceDatabaseConfiguration,
+} from '@backtrace/sdk-core';
 import { Readable } from 'stream';
 
-export interface BacktraceSetupConfiguration extends Omit<CoreConfiguration, 'attachments'> {
+export interface EnabledBacktraceDatabaseConfiguration extends CoreEnabledBacktraceDatabaseConfiguration {
+    /**
+     * Path where the SDK can store data.
+     */
+    path: string;
+    /**
+     * Determine if the directory should be auto created by the SDK.
+     * @default true
+     */
+    createDatabaseDirectory?: boolean;
+}
+
+export interface DisabledBacktraceDatabaseConfiguration
+    extends CoreDisabledBacktraceDatabaseConfiguration,
+        Omit<Partial<EnabledBacktraceDatabaseConfiguration>, 'enable'> {}
+
+export type BacktraceDatabaseConfiguration =
+    | EnabledBacktraceDatabaseConfiguration
+    | DisabledBacktraceDatabaseConfiguration;
+
+export interface BacktraceSetupConfiguration extends Omit<CoreConfiguration, 'attachments' | 'database'> {
     attachments?: Array<BacktraceAttachment<Buffer | Readable | string | Uint8Array> | string>;
+    database?: BacktraceDatabaseConfiguration;
 }
 
-export interface BacktraceConfiguration extends Omit<CoreConfiguration, 'attachments'> {
+export interface BacktraceConfiguration extends Omit<CoreConfiguration, 'attachments' | 'database'> {
     attachments?: BacktraceAttachment<Buffer | Readable | string | Uint8Array>[];
+    database?: BacktraceDatabaseConfiguration;
 }

packages/node/src/attachment/BacktraceFileAttachment.ts

@@ -1,24 +1,37 @@
-import { BacktraceFileAttachment as CoreBacktraceFileAttachment } from '@backtrace/sdk-core';
-import fs from 'fs';
+import { BacktraceAttachment } from '@backtrace/sdk-core';
+import nodeFs from 'fs';
 import path from 'path';
 import { Readable } from 'stream';
-import { NodeFileSystem } from '../storage/interfaces/NodeFileSystem.js';
+import { NodeFs } from '../storage/nodeFs.js';
 
-export class BacktraceFileAttachment implements CoreBacktraceFileAttachment<Readable> {
+export class BacktraceFileAttachment implements BacktraceAttachment<Readable> {
     public readonly name: string;
 
     constructor(
         public readonly filePath: string,
         name?: string,
-        private readonly _fileSystem?: NodeFileSystem,
+        private readonly _fs: Pick<NodeFs, 'existsSync' | 'createReadStream'> = nodeFs,
     ) {
         this.name = name ?? path.basename(this.filePath);
     }
 
     public get(): Readable | undefined {
-        if (!(this._fileSystem ?? fs).existsSync(this.filePath)) {
+        if (!this._fs.existsSync(this.filePath)) {
             return undefined;
         }
-        return (this._fileSystem ?? fs).createReadStream(this.filePath);
+
+        return this._fs.createReadStream(this.filePath);
+    }
+}
+
+export interface BacktraceFileAttachmentFactory {
+    create(filePath: string, name?: string): BacktraceFileAttachment;
+}
+
+export class NodeFsBacktraceFileAttachmentFactory implements BacktraceFileAttachmentFactory {
+    constructor(private readonly _fs: Pick<NodeFs, 'existsSync' | 'createReadStream'> = nodeFs) {}
+
+    public create(filePath: string, name?: string): BacktraceFileAttachment {
+        return new BacktraceFileAttachment(filePath, name, this._fs);
     }
 }

packages/node/src/attachment/FileAttachmentsManager.ts

@@ -2,10 +2,10 @@ import {
     AttachmentManager,
     BacktraceModule,
     BacktraceModuleBindData,
-    FileSystem,
+    BacktraceStorage,
     SessionFiles,
 } from '@backtrace/sdk-core';
-import { BacktraceFileAttachment } from './BacktraceFileAttachment.js';
+import { BacktraceFileAttachment, BacktraceFileAttachmentFactory } from './BacktraceFileAttachment.js';
 
 const ATTACHMENT_FILE_NAME = 'bt-attachments';
 
@@ -15,17 +15,22 @@ export class FileAttachmentsManager implements BacktraceModule {
     private _attachmentsManager?: AttachmentManager;
 
     constructor(
-        private readonly _fileSystem: FileSystem,
+        private readonly _storage: BacktraceStorage,
+        private readonly _fileAttachmentFactory: BacktraceFileAttachmentFactory,
         private _fileName?: string,
     ) {}
 
-    public static create(fileSystem: FileSystem) {
-        return new FileAttachmentsManager(fileSystem);
+    public static create(storage: BacktraceStorage, fileAttachmentFactory: BacktraceFileAttachmentFactory) {
+        return new FileAttachmentsManager(storage, fileAttachmentFactory);
     }
 
-    public static createFromSession(sessionFiles: SessionFiles, fileSystem: FileSystem) {
+    public static createFromSession(
+        sessionFiles: SessionFiles,
+        fileSystem: BacktraceStorage,
+        fileAttachmentFactory: BacktraceFileAttachmentFactory,
+    ) {
         const fileName = sessionFiles.getFileName(ATTACHMENT_FILE_NAME);
-        return new FileAttachmentsManager(fileSystem, fileName);
+        return new FileAttachmentsManager(fileSystem, fileAttachmentFactory, fileName);
     }
 
     public initialize(): void {
@@ -56,9 +61,12 @@ export class FileAttachmentsManager implements BacktraceModule {
         }
 
         try {
-            const content = await this._fileSystem.readFile(this._fileName);
+            const content = await this._storage.get(this._fileName);
+            if (!content) {
+                return [];
+            }
             const attachments = JSON.parse(content) as SavedAttachment[];
-            return attachments.map(([path, name]) => new BacktraceFileAttachment(path, name));
+            return attachments.map(([path, name]) => this._fileAttachmentFactory.create(path, name));
         } catch {
             return [];
         }
@@ -74,6 +82,6 @@ export class FileAttachmentsManager implements BacktraceModule {
             .filter((f): f is BacktraceFileAttachment => f instanceof BacktraceFileAttachment)
             .map<SavedAttachment>((f) => [f.filePath, f.name]);
 
-        await this._fileSystem.writeFile(this._fileName, JSON.stringify(fileAttachments));
+        await this._storage.set(this._fileName, JSON.stringify(fileAttachments));
     }
 }

packages/node/src/attachment/isFileAttachment.ts

@@ -0,0 +1,9 @@
+import { BacktraceAttachment } from '@backtrace/sdk-core';
+import { BacktraceFileAttachment } from './BacktraceFileAttachment.js';
+
+export function isFileAttachment(attachment: BacktraceAttachment): attachment is BacktraceFileAttachment {
+    return (
+        attachment instanceof BacktraceFileAttachment ||
+        ('filePath' in attachment && typeof attachment.filePath === 'string')
+    );
+}

packages/node/src/attachment/transformAttachments.ts

@@ -1,15 +1,17 @@
 import { BacktraceAttachment } from '@backtrace/sdk-core';
 import { Readable } from 'stream';
 import { BacktraceSetupConfiguration } from '../BacktraceConfiguration.js';
-import { BacktraceFileAttachment } from './BacktraceFileAttachment.js';
+import { BacktraceFileAttachmentFactory } from './BacktraceFileAttachment.js';
 
 /**
  * Transform a client attachment into the attachment model.
  */
-export function transformAttachment(
-    attachment: NonNullable<BacktraceSetupConfiguration['attachments']>[number] | BacktraceAttachment,
-): BacktraceAttachment<Buffer | Readable | string | Uint8Array> {
-    return typeof attachment === 'string'
-        ? new BacktraceFileAttachment(attachment)
-        : (attachment as BacktraceAttachment<Buffer | Readable | string | Uint8Array>);
-}
+export const transformAttachment =
+    (fileAttachmentFactory: BacktraceFileAttachmentFactory) =>
+    (
+        attachment: NonNullable<BacktraceSetupConfiguration['attachments']>[number] | BacktraceAttachment,
+    ): BacktraceAttachment<Buffer | Readable | string | Uint8Array> => {
+        return typeof attachment === 'string'
+            ? fileAttachmentFactory.create(attachment)
+            : (attachment as BacktraceAttachment<Buffer | Readable | string | Uint8Array>);
+    };