Add USFM serializer

packages/core/src/diagnostic/diagnostic-provider.ts

@@ -1,17 +1,8 @@
 import { Observable } from 'rxjs';
 
-import { Document } from '../document/document';
-import { DocumentManager } from '../document/document-manager';
 import { Diagnostic } from './diagnostic';
 import { DiagnosticFix } from './diagnostic-fix';
 
-export type DiagnosticProviderFactory<T extends Document = Document> = (
-  DocumentManager: DocumentManager<T>,
-) => DiagnosticProvider;
-export type DiagnosticProviderConstructor<T extends Document = Document> = new (
-  documentManager: DocumentManager<T>,
-) => DiagnosticProvider;
-
 export interface DiagnosticsChanged {
   uri: string;
   version?: number;

packages/core/src/diagnostic/index.ts

@@ -1,9 +1,4 @@
 export type { Diagnostic } from './diagnostic';
 export { DiagnosticSeverity } from './diagnostic';
 export type { DiagnosticFix } from './diagnostic-fix';
-export type {
-  DiagnosticProvider,
-  DiagnosticProviderConstructor,
-  DiagnosticProviderFactory,
-  DiagnosticsChanged,
-} from './diagnostic-provider';
+export type { DiagnosticProvider, DiagnosticsChanged } from './diagnostic-provider';

packages/core/src/document/document-manager.test.ts

@@ -113,6 +113,6 @@ class TestEnvironment {
       return { uri: document.uri, format: 'plaintext', version, content: changes[0].text };
     });
 
-    this.docManager = new DocumentManager(this.docReader, this.docFactory);
+    this.docManager = new DocumentManager(this.docFactory, this.docReader);
   }
 }

packages/core/src/document/document-manager.ts

@@ -34,8 +34,8 @@ export class DocumentManager<T extends Document> {
   private readonly changedSubject = new Subject<DocumentChanged<T>>();
 
   constructor(
-    private readonly reader: DocumentReader | undefined,
     private readonly factory: DocumentFactory<T>,
+    private readonly reader?: DocumentReader,
   ) {}
 
   get created$(): Observable<DocumentCreated<T>> {

packages/core/src/document/index.ts

@@ -17,6 +17,7 @@ export { ScriptureOptBreak } from './scripture-optbreak';
 export { ScriptureParagraph } from './scripture-paragraph';
 export { ScriptureRef } from './scripture-ref';
 export { ScriptureRow } from './scripture-row';
+export type { ScriptureSerializer } from './scripture-serializer';
 export { ScriptureSidebar } from './scripture-sidebar';
 export { ScriptureTable } from './scripture-table';
 export { ScriptureText } from './scripture-text';

packages/core/src/document/scripture-book.ts

@@ -2,6 +2,8 @@ import { ScriptureContainer } from './scripture-container';
 import { ScriptureNodeType } from './scripture-node';
 
 export class ScriptureBook extends ScriptureContainer {
+  public readonly style = 'id';
+
   constructor(public readonly code: string) {
     super();
   }

packages/core/src/document/scripture-chapter.ts

@@ -7,11 +7,11 @@ export class ScriptureChapter extends ScriptureMilestone {
     public readonly number: string,
     public readonly altNumber?: string,
     public readonly pubNumber?: string,
-    public readonly sid?: string,
-    public readonly eid?: string,
+    sid?: string,
+    eid?: string,
     range?: Range,
   ) {
-    super('c', sid, eid, undefined, range);
+    super('c', true, sid, eid, undefined, range);
   }
 
   get type(): ScriptureNodeType {

packages/core/src/document/scripture-milestone.ts

@@ -5,6 +5,7 @@ import { ScriptureNodeType } from './scripture-node';
 export class ScriptureMilestone extends ScriptureLeaf {
   constructor(
     public readonly style: string,
+    public readonly isStart: boolean,
     public readonly sid?: string,
     public readonly eid?: string,
     public readonly attributes: Record<string, string> = {},

packages/core/src/document/scripture-serializer.ts

@@ -0,0 +1,5 @@
+import { ScriptureNode } from './scripture-node';
+
+export interface ScriptureSerializer {
+  serialize(nodes: ScriptureNode[] | ScriptureNode): string;
+}

packages/core/src/document/scripture-sidebar.ts

@@ -2,8 +2,9 @@ import { ScriptureContainer } from './scripture-container';
 import { ScriptureNodeType } from './scripture-node';
 
 export class ScriptureSidebar extends ScriptureContainer {
+  public readonly style = 'esb';
+
   constructor(
-    public readonly style: string,
     public readonly category?: string,
     children?: ScriptureContainer[],
   ) {

packages/core/src/document/scripture-verse.ts

@@ -7,11 +7,11 @@ export class ScriptureVerse extends ScriptureMilestone {
     public readonly number: string,
     public readonly altNumber?: string,
     public readonly pubNumber?: string,
-    public readonly sid?: string,
-    public readonly eid?: string,
+    sid?: string,
+    eid?: string,
     range?: Range,
   ) {
-    super('v', sid, eid, undefined, range);
+    super('v', true, sid, eid, undefined, range);
   }
 
   get type(): ScriptureNodeType {

packages/core/src/formatting/index.ts

@@ -1,5 +1 @@
-export type {
-  OnTypeFormattingProvider,
-  OnTypeFormattingProviderConstructor,
-  OnTypeFormattingProviderFactory,
-} from './on-type-formatting-provider';
+export type { OnTypeFormattingProvider } from './on-type-formatting-provider';

packages/core/src/formatting/on-type-formatting-provider.ts

@@ -1,14 +1,5 @@
 import { Position } from '../common/position';
 import { TextEdit } from '../common/text-edit';
-import { Document } from '../document/document';
-import { DocumentManager } from '../document/document-manager';
-
-export type OnTypeFormattingProviderFactory<T extends Document = Document> = (
-  DocumentManager: DocumentManager<T>,
-) => OnTypeFormattingProvider;
-export type OnTypeFormattingProviderConstructor<T extends Document = Document> = new (
-  documentManager: DocumentManager<T>,
-) => OnTypeFormattingProvider;
 
 export interface OnTypeFormattingProvider {
   readonly id: string;

packages/core/src/workspace/workspace.ts

@@ -4,50 +4,23 @@ import { Position } from '../common/position';
 import { TextEdit } from '../common/text-edit';
 import { Diagnostic } from '../diagnostic/diagnostic';
 import { DiagnosticFix } from '../diagnostic/diagnostic-fix';
-import {
-  DiagnosticProvider,
-  DiagnosticProviderConstructor,
-  DiagnosticProviderFactory,
-  DiagnosticsChanged,
-} from '../diagnostic/diagnostic-provider';
-import { Document } from '../document/document';
-import { DocumentFactory } from '../document/document-factory';
-import { DocumentManager } from '../document/document-manager';
-import { DocumentReader } from '../document/document-reader';
-import {
-  OnTypeFormattingProvider,
-  OnTypeFormattingProviderConstructor,
-  OnTypeFormattingProviderFactory,
-} from '../formatting/on-type-formatting-provider';
+import { DiagnosticProvider, DiagnosticsChanged } from '../diagnostic/diagnostic-provider';
+import { OnTypeFormattingProvider } from '../formatting/on-type-formatting-provider';
 
-export interface WorkspaceConfig<T extends Document = Document> {
-  documentReader?: DocumentReader;
-  documentFactory: DocumentFactory<T>;
-  diagnosticProviders?: (DiagnosticProviderFactory<T> | DiagnosticProviderConstructor<T>)[];
-  onTypeFormattingProviders?: (OnTypeFormattingProviderFactory<T> | OnTypeFormattingProviderConstructor<T>)[];
+export interface WorkspaceConfig {
+  diagnosticProviders?: DiagnosticProvider[];
+  onTypeFormattingProviders?: OnTypeFormattingProvider[];
 }
 
-export class Workspace<T extends Document = Document> {
+export class Workspace {
   private readonly diagnosticProviders: Map<string, DiagnosticProvider>;
   private readonly onTypeFormattingProviders: Map<string, OnTypeFormattingProvider>;
   private readonly lastDiagnosticChangedEvents = new Map<string, DiagnosticsChanged[]>();
 
-  public readonly documentManager: DocumentManager<T>;
   public readonly diagnosticsChanged$: Observable<DiagnosticsChanged>;
 
-  constructor(config: WorkspaceConfig<T>) {
-    this.documentManager = new DocumentManager(config.documentReader, config.documentFactory);
-    this.diagnosticProviders = new Map(
-      config.diagnosticProviders?.map((factory) => {
-        let provider: DiagnosticProvider;
-        try {
-          provider = new (factory as DiagnosticProviderConstructor<T>)(this.documentManager);
-        } catch {
-          provider = (factory as DiagnosticProviderFactory<T>)(this.documentManager);
-        }
-        return [provider.id, provider];
-      }),
-    );
+  constructor(config: WorkspaceConfig) {
+    this.diagnosticProviders = new Map(config.diagnosticProviders?.map((provider) => [provider.id, provider]));
     this.diagnosticsChanged$ = merge(
       ...Array.from(this.diagnosticProviders.values()).map((provider, i) =>
         provider.diagnosticsChanged$.pipe(
@@ -58,15 +31,7 @@ export class Workspace<T extends Document = Document> {
       ),
     ).pipe(map((e) => this.getCombinedDiagnosticChangedEvent(e.uri, e.version)));
     this.onTypeFormattingProviders = new Map(
-      config.onTypeFormattingProviders?.map((factory) => {
-        let provider: OnTypeFormattingProvider;
-        try {
-          provider = new (factory as OnTypeFormattingProviderConstructor<T>)(this.documentManager);
-        } catch {
-          provider = (factory as OnTypeFormattingProviderFactory<T>)(this.documentManager);
-        }
-        return [provider.id, provider];
-      }),
+      config.onTypeFormattingProviders?.map((provider) => [provider.id, provider]),
     );
   }
 

packages/eslint-config/library.mjs

@@ -34,6 +34,7 @@ export default tseslint.config(
       '@typescript-eslint/no-unsafe-call': 'off',
       '@typescript-eslint/no-unsafe-assignment': 'off',
       '@typescript-eslint/no-unsafe-return': 'off',
+      '@typescript-eslint/no-unsafe-argument': 'off',
     },
   },
   {

packages/usfm/src/index.ts

@@ -1,2 +1,3 @@
 export { UsfmDocument } from './usfm-document';
 export { UsfmDocumentFactory } from './usfm-document-factory';
+export { UsfmScriptureSerializer } from './usfm-scripture-serializer';

packages/usfm/src/usfm-document.ts

@@ -233,8 +233,8 @@ class UsfmDocumentBuilder extends UsfmParserHandlerBase {
     this.endContainer(state, false);
   }
 
-  startSidebar(state: UsfmParserState, marker: string, category: string | undefined): void {
-    this.startContainer(state, new ScriptureSidebar(marker, category));
+  startSidebar(state: UsfmParserState, _marker: string, category: string | undefined): void {
+    this.startContainer(state, new ScriptureSidebar(category));
   }
 
   endSidebar(state: UsfmParserState, _marker: string, closed: boolean): void {
@@ -279,12 +279,13 @@ class UsfmDocumentBuilder extends UsfmParserHandlerBase {
   milestone(
     state: UsfmParserState,
     marker: string,
-    _startMilestone: boolean,
+    startMilestone: boolean,
     attributes: readonly UsfmAttribute[] | undefined,
   ): void {
     this.appendChild(
       new ScriptureMilestone(
         marker,
+        startMilestone,
         undefined,
         undefined,
         UsfmDocumentBuilder.convertAttributes(attributes),

packages/usfm/src/usfm-scripture-serializer.ts

@@ -0,0 +1,124 @@
+import {
+  ScriptureBook,
+  ScriptureCell,
+  ScriptureChapter,
+  ScriptureCharacterStyle,
+  ScriptureDocument,
+  ScriptureMilestone,
+  ScriptureNode,
+  ScriptureNote,
+  ScriptureOptBreak,
+  ScriptureParagraph,
+  ScriptureRef,
+  ScriptureRow,
+  ScriptureSerializer,
+  ScriptureSidebar,
+  ScriptureTable,
+  ScriptureText,
+  ScriptureVerse,
+} from '@sillsdev/lynx';
+import { UsfmStylesheet, UsfmToken, UsfmTokenizer, UsfmTokenType } from '@sillsdev/machine/corpora';
+
+export class UsfmScriptureSerializer implements ScriptureSerializer {
+  private readonly tokenizer: UsfmTokenizer;
+
+  constructor(private readonly stylesheet: UsfmStylesheet) {
+    this.tokenizer = new UsfmTokenizer(stylesheet);
+  }
+
+  serialize(nodes: ScriptureNode[] | ScriptureNode): string {
+    const tokens = this.toTokens(nodes);
+    return this.tokenizer.detokenize(tokens, false, true);
+  }
+
+  private *toTokens(node: ScriptureNode | readonly ScriptureNode[]): Iterable<UsfmToken> {
+    if (Array.isArray(node)) {
+      for (const child of node) {
+        yield* this.toTokens(child as ScriptureNode);
+      }
+    } else if (node instanceof ScriptureDocument) {
+      yield* this.toTokens(node.children);
+    } else if (node instanceof ScriptureText) {
+      yield new UsfmToken(UsfmTokenType.Text, undefined, node.text);
+    } else if (node instanceof ScriptureBook) {
+      yield new UsfmToken(UsfmTokenType.Book, node.style, undefined, undefined, node.code);
+      yield* this.toTokens(node.children);
+    } else if (node instanceof ScriptureChapter) {
+      yield new UsfmToken(UsfmTokenType.Chapter, node.style, undefined, undefined, node.number);
+      if (node.altNumber != null) {
+        yield new UsfmToken(UsfmTokenType.Character, 'ca', undefined, 'ca*');
+        yield new UsfmToken(UsfmTokenType.Text, undefined, node.altNumber);
+        yield new UsfmToken(UsfmTokenType.End, 'ca*');
+      }
+      if (node.pubNumber != null) {
+        yield new UsfmToken(UsfmTokenType.Paragraph, 'cp');
+        yield new UsfmToken(UsfmTokenType.Text, undefined, node.pubNumber);
+      }
+    } else if (node instanceof ScriptureParagraph) {
+      yield new UsfmToken(UsfmTokenType.Paragraph, node.style, undefined, node.style + '*');
+      yield* this.toTokens(node.children);
+    } else if (node instanceof ScriptureVerse) {
+      yield new UsfmToken(UsfmTokenType.Verse, node.style, undefined, undefined, node.number);
+      if (node.altNumber != null) {
+        yield new UsfmToken(UsfmTokenType.Character, 'va', undefined, 'va*');
+        yield new UsfmToken(UsfmTokenType.Text, undefined, node.altNumber);
+        yield new UsfmToken(UsfmTokenType.End, 'va*');
+      }
+      if (node.pubNumber != null) {
+        yield new UsfmToken(UsfmTokenType.Character, 'vp', undefined, 'vp*');
+        yield new UsfmToken(UsfmTokenType.Text, undefined, node.pubNumber);
+        yield new UsfmToken(UsfmTokenType.End, 'vp*');
+      }
+    } else if (node instanceof ScriptureCharacterStyle) {
+      yield new UsfmToken(UsfmTokenType.Character, node.style, undefined, node.style + '*');
+      yield* this.toTokens(node.children);
+      yield new UsfmToken(UsfmTokenType.End, node.style + '*');
+    } else if (node instanceof ScriptureMilestone) {
+      let type: UsfmTokenType;
+      let endMarker: string | undefined;
+      if (node.isStart) {
+        type = UsfmTokenType.Milestone;
+        const tag = this.stylesheet.getTag(node.style);
+        endMarker = tag.endMarker;
+      } else {
+        type = UsfmTokenType.MilestoneEnd;
+        endMarker = undefined;
+      }
+      yield new UsfmToken(type, node.style, undefined, endMarker);
+    } else if (node instanceof ScriptureNote) {
+      yield new UsfmToken(UsfmTokenType.Note, node.style, undefined, node.style + '*', node.caller);
+      if (node.category != null) {
+        yield new UsfmToken(UsfmTokenType.Character, 'cat', undefined, 'cat*');
+        yield new UsfmToken(UsfmTokenType.Text, undefined, node.category);
+        yield new UsfmToken(UsfmTokenType.End, 'cat*');
+      }
+      yield* this.toTokens(node.children);
+    } else if (node instanceof ScriptureRef) {
+      yield new UsfmToken(UsfmTokenType.Character, 'ref', undefined, 'ref*');
+      yield new UsfmToken(UsfmTokenType.Text, undefined, `${node.display}|${node.target}`);
+      yield new UsfmToken(UsfmTokenType.End, 'ref*');
+    } else if (node instanceof ScriptureOptBreak) {
+      yield new UsfmToken(UsfmTokenType.Text, undefined, '\\');
+    } else if (node instanceof ScriptureSidebar) {
+      yield new UsfmToken(UsfmTokenType.Paragraph, node.style);
+      if (node.category != null) {
+        yield new UsfmToken(UsfmTokenType.Character, 'esbc', undefined, 'esbc*');
+        yield new UsfmToken(UsfmTokenType.Text, undefined, node.category);
+        yield new UsfmToken(UsfmTokenType.End, 'esbc*');
+      }
+      yield* this.toTokens(node.children);
+    } else if (node instanceof ScriptureTable) {
+      yield* this.toTokens(node.children);
+    } else if (node instanceof ScriptureRow) {
+      yield new UsfmToken(UsfmTokenType.Paragraph, 'tr');
+      yield* this.toTokens(node.children);
+    } else if (node instanceof ScriptureCell) {
+      let marker = node.style;
+      if (node.colSpan > 0) {
+        marker += '-' + node.colSpan.toString();
+      }
+      yield new UsfmToken(UsfmTokenType.Character, marker);
+      yield* this.toTokens(node.children);
+    }
+  }
+}