From f6f17edc9c0dfe202e80c757c702e48de0ebbe17 Mon Sep 17 00:00:00 2001 From: Damien Daspit Date: Mon, 4 Nov 2024 12:57:27 -0500 Subject: [PATCH] Add USFM serializer --- package-lock.json | 8 +- .../src/diagnostic/diagnostic-provider.ts | 9 -- packages/core/src/diagnostic/index.ts | 7 +- .../src/document/document-manager.test.ts | 2 +- .../core/src/document/document-manager.ts | 2 +- packages/core/src/document/index.ts | 1 + packages/core/src/document/scripture-book.ts | 2 + .../core/src/document/scripture-chapter.ts | 6 +- .../core/src/document/scripture-milestone.ts | 1 + .../core/src/document/scripture-serializer.ts | 5 + .../core/src/document/scripture-sidebar.ts | 3 +- packages/core/src/document/scripture-verse.ts | 6 +- packages/core/src/formatting/index.ts | 6 +- .../formatting/on-type-formatting-provider.ts | 9 -- packages/core/src/workspace/workspace.ts | 53 ++---- packages/eslint-config/library.mjs | 1 + packages/usfm/package.json | 4 +- packages/usfm/src/index.ts | 1 + packages/usfm/src/usfm-document.ts | 7 +- .../src/usfm-scripture-serializer.test.ts | 68 ++++++++ .../usfm/src/usfm-scripture-serializer.ts | 153 ++++++++++++++++++ packages/vscode/src/server.ts | 28 ++-- .../src/verse-order-diagnostic-provider.ts | 23 ++- 23 files changed, 292 insertions(+), 113 deletions(-) create mode 100644 packages/core/src/document/scripture-serializer.ts create mode 100644 packages/usfm/src/usfm-scripture-serializer.test.ts create mode 100644 packages/usfm/src/usfm-scripture-serializer.ts diff --git a/package-lock.json b/package-lock.json index 8c4e485..136f622 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1062,9 +1062,9 @@ "link": true }, "node_modules/@sillsdev/machine": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sillsdev/machine/-/machine-3.0.1.tgz", - "integrity": "sha512-1qoxIBT5mOA6eUo36HxVKuoNnAzqqiV0xilWiA2PE53h+hTSxBVFb+pYVV39OhQ0HplTIV0KohzaRdAIUrUYCA==", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@sillsdev/machine/-/machine-3.0.2.tgz", + "integrity": "sha512-dp7FghTIPyFSFnWFmYCwXTMRgOlZetsd0ne9bTE6P3++fjpGiRcOrDSm9Zr1MP6sJqiL5WoCZ4Zr/CT/+q3n/A==", "license": "MIT", "dependencies": { "@sillsdev/scripture": "^2.0.1", @@ -6700,7 +6700,7 @@ "license": "MIT", "dependencies": { "@sillsdev/lynx": "*", - "@sillsdev/machine": "^3.0.1" + "@sillsdev/machine": "^3.0.2" }, "devDependencies": { "@repo/eslint-config": "*", diff --git a/packages/core/src/diagnostic/diagnostic-provider.ts b/packages/core/src/diagnostic/diagnostic-provider.ts index 01998e8..d051290 100644 --- a/packages/core/src/diagnostic/diagnostic-provider.ts +++ b/packages/core/src/diagnostic/diagnostic-provider.ts @@ -1,17 +1,8 @@ import { Observable } from 'rxjs'; -import { Document } from '../document/document'; -import { DocumentManager } from '../document/document-manager'; import { Diagnostic } from './diagnostic'; import { DiagnosticFix } from './diagnostic-fix'; -export type DiagnosticProviderFactory = ( - DocumentManager: DocumentManager, -) => DiagnosticProvider; -export type DiagnosticProviderConstructor = new ( - documentManager: DocumentManager, -) => DiagnosticProvider; - export interface DiagnosticsChanged { uri: string; version?: number; diff --git a/packages/core/src/diagnostic/index.ts b/packages/core/src/diagnostic/index.ts index 653c2a7..87719c4 100644 --- a/packages/core/src/diagnostic/index.ts +++ b/packages/core/src/diagnostic/index.ts @@ -1,9 +1,4 @@ export type { Diagnostic } from './diagnostic'; export { DiagnosticSeverity } from './diagnostic'; export type { DiagnosticFix } from './diagnostic-fix'; -export type { - DiagnosticProvider, - DiagnosticProviderConstructor, - DiagnosticProviderFactory, - DiagnosticsChanged, -} from './diagnostic-provider'; +export type { DiagnosticProvider, DiagnosticsChanged } from './diagnostic-provider'; diff --git a/packages/core/src/document/document-manager.test.ts b/packages/core/src/document/document-manager.test.ts index 6050aba..9db362a 100644 --- a/packages/core/src/document/document-manager.test.ts +++ b/packages/core/src/document/document-manager.test.ts @@ -113,6 +113,6 @@ class TestEnvironment { return { uri: document.uri, format: 'plaintext', version, content: changes[0].text }; }); - this.docManager = new DocumentManager(this.docReader, this.docFactory); + this.docManager = new DocumentManager(this.docFactory, this.docReader); } } diff --git a/packages/core/src/document/document-manager.ts b/packages/core/src/document/document-manager.ts index b745e7d..d017c6e 100644 --- a/packages/core/src/document/document-manager.ts +++ b/packages/core/src/document/document-manager.ts @@ -34,8 +34,8 @@ export class DocumentManager { private readonly changedSubject = new Subject>(); constructor( - private readonly reader: DocumentReader | undefined, private readonly factory: DocumentFactory, + private readonly reader?: DocumentReader, ) {} get created$(): Observable> { diff --git a/packages/core/src/document/index.ts b/packages/core/src/document/index.ts index 4cab0fe..9fdf1e7 100644 --- a/packages/core/src/document/index.ts +++ b/packages/core/src/document/index.ts @@ -17,6 +17,7 @@ export { ScriptureOptBreak } from './scripture-optbreak'; export { ScriptureParagraph } from './scripture-paragraph'; export { ScriptureRef } from './scripture-ref'; export { ScriptureRow } from './scripture-row'; +export type { ScriptureSerializer } from './scripture-serializer'; export { ScriptureSidebar } from './scripture-sidebar'; export { ScriptureTable } from './scripture-table'; export { ScriptureText } from './scripture-text'; diff --git a/packages/core/src/document/scripture-book.ts b/packages/core/src/document/scripture-book.ts index e43c8e0..5642b67 100644 --- a/packages/core/src/document/scripture-book.ts +++ b/packages/core/src/document/scripture-book.ts @@ -2,6 +2,8 @@ import { ScriptureContainer } from './scripture-container'; import { ScriptureNodeType } from './scripture-node'; export class ScriptureBook extends ScriptureContainer { + public readonly style = 'id'; + constructor(public readonly code: string) { super(); } diff --git a/packages/core/src/document/scripture-chapter.ts b/packages/core/src/document/scripture-chapter.ts index 1d498a1..bd5697b 100644 --- a/packages/core/src/document/scripture-chapter.ts +++ b/packages/core/src/document/scripture-chapter.ts @@ -7,11 +7,11 @@ export class ScriptureChapter extends ScriptureMilestone { public readonly number: string, public readonly altNumber?: string, public readonly pubNumber?: string, - public readonly sid?: string, - public readonly eid?: string, + sid?: string, + eid?: string, range?: Range, ) { - super('c', sid, eid, undefined, range); + super('c', true, sid, eid, undefined, range); } get type(): ScriptureNodeType { diff --git a/packages/core/src/document/scripture-milestone.ts b/packages/core/src/document/scripture-milestone.ts index 9699482..58fefd6 100644 --- a/packages/core/src/document/scripture-milestone.ts +++ b/packages/core/src/document/scripture-milestone.ts @@ -5,6 +5,7 @@ import { ScriptureNodeType } from './scripture-node'; export class ScriptureMilestone extends ScriptureLeaf { constructor( public readonly style: string, + public readonly isStart: boolean, public readonly sid?: string, public readonly eid?: string, public readonly attributes: Record = {}, diff --git a/packages/core/src/document/scripture-serializer.ts b/packages/core/src/document/scripture-serializer.ts new file mode 100644 index 0000000..b274661 --- /dev/null +++ b/packages/core/src/document/scripture-serializer.ts @@ -0,0 +1,5 @@ +import { ScriptureNode } from './scripture-node'; + +export interface ScriptureSerializer { + serialize(nodes: ScriptureNode[] | ScriptureNode): string; +} diff --git a/packages/core/src/document/scripture-sidebar.ts b/packages/core/src/document/scripture-sidebar.ts index 316ba19..ea9dc4c 100644 --- a/packages/core/src/document/scripture-sidebar.ts +++ b/packages/core/src/document/scripture-sidebar.ts @@ -2,8 +2,9 @@ import { ScriptureContainer } from './scripture-container'; import { ScriptureNodeType } from './scripture-node'; export class ScriptureSidebar extends ScriptureContainer { + public readonly style = 'esb'; + constructor( - public readonly style: string, public readonly category?: string, children?: ScriptureContainer[], ) { diff --git a/packages/core/src/document/scripture-verse.ts b/packages/core/src/document/scripture-verse.ts index bb0aeb2..9ee49ce 100644 --- a/packages/core/src/document/scripture-verse.ts +++ b/packages/core/src/document/scripture-verse.ts @@ -7,11 +7,11 @@ export class ScriptureVerse extends ScriptureMilestone { public readonly number: string, public readonly altNumber?: string, public readonly pubNumber?: string, - public readonly sid?: string, - public readonly eid?: string, + sid?: string, + eid?: string, range?: Range, ) { - super('v', sid, eid, undefined, range); + super('v', true, sid, eid, undefined, range); } get type(): ScriptureNodeType { diff --git a/packages/core/src/formatting/index.ts b/packages/core/src/formatting/index.ts index d1c6a8f..fb54780 100644 --- a/packages/core/src/formatting/index.ts +++ b/packages/core/src/formatting/index.ts @@ -1,5 +1 @@ -export type { - OnTypeFormattingProvider, - OnTypeFormattingProviderConstructor, - OnTypeFormattingProviderFactory, -} from './on-type-formatting-provider'; +export type { OnTypeFormattingProvider } from './on-type-formatting-provider'; diff --git a/packages/core/src/formatting/on-type-formatting-provider.ts b/packages/core/src/formatting/on-type-formatting-provider.ts index 34d7497..471433a 100644 --- a/packages/core/src/formatting/on-type-formatting-provider.ts +++ b/packages/core/src/formatting/on-type-formatting-provider.ts @@ -1,14 +1,5 @@ import { Position } from '../common/position'; import { TextEdit } from '../common/text-edit'; -import { Document } from '../document/document'; -import { DocumentManager } from '../document/document-manager'; - -export type OnTypeFormattingProviderFactory = ( - DocumentManager: DocumentManager, -) => OnTypeFormattingProvider; -export type OnTypeFormattingProviderConstructor = new ( - documentManager: DocumentManager, -) => OnTypeFormattingProvider; export interface OnTypeFormattingProvider { readonly id: string; diff --git a/packages/core/src/workspace/workspace.ts b/packages/core/src/workspace/workspace.ts index a4f2645..df3e3c9 100644 --- a/packages/core/src/workspace/workspace.ts +++ b/packages/core/src/workspace/workspace.ts @@ -4,50 +4,23 @@ import { Position } from '../common/position'; import { TextEdit } from '../common/text-edit'; import { Diagnostic } from '../diagnostic/diagnostic'; import { DiagnosticFix } from '../diagnostic/diagnostic-fix'; -import { - DiagnosticProvider, - DiagnosticProviderConstructor, - DiagnosticProviderFactory, - DiagnosticsChanged, -} from '../diagnostic/diagnostic-provider'; -import { Document } from '../document/document'; -import { DocumentFactory } from '../document/document-factory'; -import { DocumentManager } from '../document/document-manager'; -import { DocumentReader } from '../document/document-reader'; -import { - OnTypeFormattingProvider, - OnTypeFormattingProviderConstructor, - OnTypeFormattingProviderFactory, -} from '../formatting/on-type-formatting-provider'; +import { DiagnosticProvider, DiagnosticsChanged } from '../diagnostic/diagnostic-provider'; +import { OnTypeFormattingProvider } from '../formatting/on-type-formatting-provider'; -export interface WorkspaceConfig { - documentReader?: DocumentReader; - documentFactory: DocumentFactory; - diagnosticProviders?: (DiagnosticProviderFactory | DiagnosticProviderConstructor)[]; - onTypeFormattingProviders?: (OnTypeFormattingProviderFactory | OnTypeFormattingProviderConstructor)[]; +export interface WorkspaceConfig { + diagnosticProviders?: DiagnosticProvider[]; + onTypeFormattingProviders?: OnTypeFormattingProvider[]; } -export class Workspace { +export class Workspace { private readonly diagnosticProviders: Map; private readonly onTypeFormattingProviders: Map; private readonly lastDiagnosticChangedEvents = new Map(); - public readonly documentManager: DocumentManager; public readonly diagnosticsChanged$: Observable; - constructor(config: WorkspaceConfig) { - this.documentManager = new DocumentManager(config.documentReader, config.documentFactory); - this.diagnosticProviders = new Map( - config.diagnosticProviders?.map((factory) => { - let provider: DiagnosticProvider; - try { - provider = new (factory as DiagnosticProviderConstructor)(this.documentManager); - } catch { - provider = (factory as DiagnosticProviderFactory)(this.documentManager); - } - return [provider.id, provider]; - }), - ); + constructor(config: WorkspaceConfig) { + this.diagnosticProviders = new Map(config.diagnosticProviders?.map((provider) => [provider.id, provider])); this.diagnosticsChanged$ = merge( ...Array.from(this.diagnosticProviders.values()).map((provider, i) => provider.diagnosticsChanged$.pipe( @@ -58,15 +31,7 @@ export class Workspace { ), ).pipe(map((e) => this.getCombinedDiagnosticChangedEvent(e.uri, e.version))); this.onTypeFormattingProviders = new Map( - config.onTypeFormattingProviders?.map((factory) => { - let provider: OnTypeFormattingProvider; - try { - provider = new (factory as OnTypeFormattingProviderConstructor)(this.documentManager); - } catch { - provider = (factory as OnTypeFormattingProviderFactory)(this.documentManager); - } - return [provider.id, provider]; - }), + config.onTypeFormattingProviders?.map((provider) => [provider.id, provider]), ); } diff --git a/packages/eslint-config/library.mjs b/packages/eslint-config/library.mjs index 81bb2d6..7cc92fa 100644 --- a/packages/eslint-config/library.mjs +++ b/packages/eslint-config/library.mjs @@ -34,6 +34,7 @@ export default tseslint.config( '@typescript-eslint/no-unsafe-call': 'off', '@typescript-eslint/no-unsafe-assignment': 'off', '@typescript-eslint/no-unsafe-return': 'off', + '@typescript-eslint/no-unsafe-argument': 'off', }, }, { diff --git a/packages/usfm/package.json b/packages/usfm/package.json index 9cd7e60..7de2ea1 100644 --- a/packages/usfm/package.json +++ b/packages/usfm/package.json @@ -30,8 +30,8 @@ "author": "SIL Global", "license": "MIT", "dependencies": { - "@sillsdev/machine": "^3.0.1", - "@sillsdev/lynx": "*" + "@sillsdev/lynx": "*", + "@sillsdev/machine": "^3.0.2" }, "devDependencies": { "@repo/eslint-config": "*", diff --git a/packages/usfm/src/index.ts b/packages/usfm/src/index.ts index cb4be41..eb04c16 100644 --- a/packages/usfm/src/index.ts +++ b/packages/usfm/src/index.ts @@ -1,2 +1,3 @@ export { UsfmDocument } from './usfm-document'; export { UsfmDocumentFactory } from './usfm-document-factory'; +export { UsfmScriptureSerializer } from './usfm-scripture-serializer'; diff --git a/packages/usfm/src/usfm-document.ts b/packages/usfm/src/usfm-document.ts index ba1363b..2e1bf06 100644 --- a/packages/usfm/src/usfm-document.ts +++ b/packages/usfm/src/usfm-document.ts @@ -233,8 +233,8 @@ class UsfmDocumentBuilder extends UsfmParserHandlerBase { this.endContainer(state, false); } - startSidebar(state: UsfmParserState, marker: string, category: string | undefined): void { - this.startContainer(state, new ScriptureSidebar(marker, category)); + startSidebar(state: UsfmParserState, _marker: string, category: string | undefined): void { + this.startContainer(state, new ScriptureSidebar(category)); } endSidebar(state: UsfmParserState, _marker: string, closed: boolean): void { @@ -279,12 +279,13 @@ class UsfmDocumentBuilder extends UsfmParserHandlerBase { milestone( state: UsfmParserState, marker: string, - _startMilestone: boolean, + startMilestone: boolean, attributes: readonly UsfmAttribute[] | undefined, ): void { this.appendChild( new ScriptureMilestone( marker, + startMilestone, undefined, undefined, UsfmDocumentBuilder.convertAttributes(attributes), diff --git a/packages/usfm/src/usfm-scripture-serializer.test.ts b/packages/usfm/src/usfm-scripture-serializer.test.ts new file mode 100644 index 0000000..92bf20e --- /dev/null +++ b/packages/usfm/src/usfm-scripture-serializer.test.ts @@ -0,0 +1,68 @@ +import { UsfmStylesheet } from '@sillsdev/machine/corpora'; +import { describe, expect, it } from 'vitest'; + +import { UsfmDocument } from './usfm-document'; +import { UsfmScriptureSerializer } from './usfm-scripture-serializer'; + +describe('UsfmScriptureSerializer', () => { + it('single paragraph', () => { + const usfm = '\\p This is a paragraph.'; + const result = serialize(usfm); + expect(result).toEqual(usfm); + }); + + it('multiple paragraphs', () => { + const usfm = `\\p This is a paragraph. +\\p This is another paragraph.`; + const result = serialize(usfm); + expect(result).toEqual(usfm); + }); + + it('nested character style', () => { + const usfm = '\\add an addition containing the word \\+nd Lord\\+nd*\\add*'; + const result = serialize(usfm); + expect(result).toEqual(usfm); + }); + + it('attributes', () => { + const usfm = '\\fig At once they left their nets.|src="avnt016.jpg" size="span" ref="1.18"\\fig*'; + const result = serialize(usfm); + expect(result).toEqual(usfm); + }); + + it('default attribute', () => { + const usfm = '\\w gracious|grace\\w*'; + const result = serialize(usfm); + expect(result).toEqual(usfm); + }); + + it('chapter and verse', () => { + const usfm = `\\c 1 +\\p +\\v 1 This is a verse.`; + const result = serialize(usfm); + expect(result).toEqual(usfm); + }); + + it('table', () => { + const usfm = `\\tr \\th1 Tribe \\th2 Leader \\thr3 Number +\\tr \\tc1 Reuben \\tc2 Elizur son of Shedeur \\tcr3 46,500 +\\tr \\tc1 Simeon \\tc2 Shelumiel son of Zurishaddai \\tcr3 59,300 +\\tr \\tc1 Gad \\tc2 Eliasaph son of Deuel \\tcr3 45,650 +\\tr \\tcr1-2 Total: \\tcr3 151,450`; + const result = serialize(usfm); + expect(result).toEqual(usfm); + }); +}); + +function serialize(usfm: string): string { + const stylesheet = new UsfmStylesheet('usfm.sty'); + const document = new UsfmDocument('uri', 1, usfm, stylesheet); + const serializer = new UsfmScriptureSerializer(stylesheet); + + return normalize(serializer.serialize(document)); +} + +function normalize(text: string): string { + return text.replace(/\r?\n/g, '\n').trim(); +} diff --git a/packages/usfm/src/usfm-scripture-serializer.ts b/packages/usfm/src/usfm-scripture-serializer.ts new file mode 100644 index 0000000..07d064e --- /dev/null +++ b/packages/usfm/src/usfm-scripture-serializer.ts @@ -0,0 +1,153 @@ +import { + ScriptureBook, + ScriptureCell, + ScriptureChapter, + ScriptureCharacterStyle, + ScriptureDocument, + ScriptureMilestone, + ScriptureNode, + ScriptureNote, + ScriptureOptBreak, + ScriptureParagraph, + ScriptureRef, + ScriptureRow, + ScriptureSerializer, + ScriptureSidebar, + ScriptureTable, + ScriptureText, + ScriptureVerse, +} from '@sillsdev/lynx'; +import { UsfmAttribute, UsfmStylesheet, UsfmToken, UsfmTokenizer, UsfmTokenType } from '@sillsdev/machine/corpora'; + +export class UsfmScriptureSerializer implements ScriptureSerializer { + private readonly tokenizer: UsfmTokenizer; + + constructor(private readonly stylesheet: UsfmStylesheet) { + this.tokenizer = new UsfmTokenizer(stylesheet); + } + + serialize(nodes: ScriptureNode[] | ScriptureNode): string { + const tokens = this.toTokens(nodes, false, false); + return this.tokenizer.detokenize(tokens, false, nodes instanceof ScriptureDocument); + } + + private *toTokens( + node: ScriptureNode | readonly ScriptureNode[], + nested: boolean, + endOfParagraph: boolean, + ): Iterable { + if (Array.isArray(node)) { + for (let i = 0; i < node.length; i++) { + yield* this.toTokens(node[i] as ScriptureNode, nested, endOfParagraph && i === node.length - 1); + } + } else if (node instanceof ScriptureDocument) { + yield* this.toTokens(node.children, nested, false); + } else if (node instanceof ScriptureText) { + let text = node.text; + if (endOfParagraph) { + text += ' '; + } + yield new UsfmToken(UsfmTokenType.Text, undefined, text); + } else if (node instanceof ScriptureBook) { + yield new UsfmToken(UsfmTokenType.Book, node.style, undefined, undefined, node.code); + yield* this.toTokens(node.children, nested, false); + } else if (node instanceof ScriptureChapter) { + yield new UsfmToken(UsfmTokenType.Chapter, node.style, undefined, undefined, node.number); + if (node.altNumber != null) { + yield new UsfmToken(UsfmTokenType.Character, 'ca', undefined, 'ca*'); + yield new UsfmToken(UsfmTokenType.Text, undefined, node.altNumber); + yield new UsfmToken(UsfmTokenType.End, 'ca*'); + } + if (node.pubNumber != null) { + yield new UsfmToken(UsfmTokenType.Paragraph, 'cp'); + yield new UsfmToken(UsfmTokenType.Text, undefined, node.pubNumber); + } + } else if (node instanceof ScriptureParagraph) { + yield new UsfmToken(UsfmTokenType.Paragraph, node.style, undefined, node.style + '*'); + yield* this.toTokens(node.children, nested, true); + } else if (node instanceof ScriptureVerse) { + yield new UsfmToken(UsfmTokenType.Verse, node.style, undefined, undefined, node.number); + if (node.altNumber != null) { + yield new UsfmToken(UsfmTokenType.Character, 'va', undefined, 'va*'); + yield new UsfmToken(UsfmTokenType.Text, undefined, node.altNumber); + yield new UsfmToken(UsfmTokenType.End, 'va*'); + } + if (node.pubNumber != null) { + yield new UsfmToken(UsfmTokenType.Character, 'vp', undefined, 'vp*'); + yield new UsfmToken(UsfmTokenType.Text, undefined, node.pubNumber); + yield new UsfmToken(UsfmTokenType.End, 'vp*'); + } + } else if (node instanceof ScriptureCharacterStyle) { + let marker = node.style; + if (nested) { + marker = '+' + marker; + } + const token = new UsfmToken(UsfmTokenType.Character, marker, undefined, marker + '*'); + const attributes: UsfmAttribute[] = []; + for (const key in node.attributes) { + const value = node.attributes[key]; + attributes.push(new UsfmAttribute(key, value)); + } + if (attributes.length > 0) { + const attrToken = new UsfmToken(UsfmTokenType.Attribute, marker); + const tag = this.stylesheet.getTag(node.style); + token.setAttributes(attributes, tag.defaultAttributeName); + yield attrToken; + } + yield token; + yield* this.toTokens(node.children, true, endOfParagraph); + if (attributes.length > 0) { + const attrToken = new UsfmToken(UsfmTokenType.Attribute, marker); + attrToken.copyAttributes(token); + yield attrToken; + } + yield new UsfmToken(UsfmTokenType.End, marker + '*'); + } else if (node instanceof ScriptureMilestone) { + let type: UsfmTokenType; + let endMarker: string | undefined; + if (node.isStart) { + type = UsfmTokenType.Milestone; + const tag = this.stylesheet.getTag(node.style); + endMarker = tag.endMarker; + } else { + type = UsfmTokenType.MilestoneEnd; + endMarker = undefined; + } + yield new UsfmToken(type, node.style, undefined, endMarker); + } else if (node instanceof ScriptureNote) { + yield new UsfmToken(UsfmTokenType.Note, node.style, undefined, node.style + '*', node.caller); + if (node.category != null) { + yield new UsfmToken(UsfmTokenType.Character, 'cat', undefined, 'cat*'); + yield new UsfmToken(UsfmTokenType.Text, undefined, node.category); + yield new UsfmToken(UsfmTokenType.End, 'cat*'); + } + yield* this.toTokens(node.children, nested, endOfParagraph); + } else if (node instanceof ScriptureRef) { + yield new UsfmToken(UsfmTokenType.Character, 'ref', undefined, 'ref*'); + yield new UsfmToken(UsfmTokenType.Text, undefined, `${node.display}|${node.target}`); + yield new UsfmToken(UsfmTokenType.End, 'ref*'); + } else if (node instanceof ScriptureOptBreak) { + yield new UsfmToken(UsfmTokenType.Text, undefined, '\\'); + } else if (node instanceof ScriptureSidebar) { + yield new UsfmToken(UsfmTokenType.Paragraph, node.style); + if (node.category != null) { + yield new UsfmToken(UsfmTokenType.Character, 'esbc', undefined, 'esbc*'); + yield new UsfmToken(UsfmTokenType.Text, undefined, node.category); + yield new UsfmToken(UsfmTokenType.End, 'esbc*'); + } + yield* this.toTokens(node.children, nested, true); + } else if (node instanceof ScriptureTable) { + yield* this.toTokens(node.children, nested, false); + } else if (node instanceof ScriptureRow) { + yield new UsfmToken(UsfmTokenType.Paragraph, 'tr'); + yield* this.toTokens(node.children, nested, true); + } else if (node instanceof ScriptureCell) { + let marker = node.style; + if (node.colSpan > 0) { + marker += '-' + node.colSpan.toString(); + } + yield new UsfmToken(UsfmTokenType.Character, marker); + yield* this.toTokens(node.children, nested, endOfParagraph); + } + } +} diff --git a/packages/vscode/src/server.ts b/packages/vscode/src/server.ts index fd71439..6d91b60 100644 --- a/packages/vscode/src/server.ts +++ b/packages/vscode/src/server.ts @@ -1,8 +1,9 @@ -import { Diagnostic, Workspace } from '@sillsdev/lynx'; -import { UsfmDocumentFactory } from '@sillsdev/lynx-usfm'; +import { Diagnostic, DocumentManager, ScriptureDocument, Workspace } from '@sillsdev/lynx'; +import { UsfmDocumentFactory, UsfmScriptureSerializer } from '@sillsdev/lynx-usfm'; import { UsfmStylesheet } from '@sillsdev/machine/corpora'; import { CodeAction, + CodeActionKind, createConnection, type DocumentDiagnosticReport, DocumentDiagnosticReportKind, @@ -19,11 +20,13 @@ import { VerseOrderDiagnosticProvider } from './verse-order-diagnostic-provider' // Also include all preview / proposed LSP features. const connection = createConnection(ProposedFeatures.all); -// Create a simple text document manager. +const stylesheet = new UsfmStylesheet('usfm.sty'); +const documentFactory = new UsfmDocumentFactory(stylesheet); +const scriptureSerializer = new UsfmScriptureSerializer(stylesheet); +const documentManager = new DocumentManager(documentFactory); const workspace = new Workspace({ - documentFactory: new UsfmDocumentFactory(new UsfmStylesheet('usfm.sty')), - diagnosticProviders: [VerseOrderDiagnosticProvider], - onTypeFormattingProviders: [SmartQuoteFormattingProvider], + diagnosticProviders: [new VerseOrderDiagnosticProvider(documentManager, scriptureSerializer)], + onTypeFormattingProviders: [new SmartQuoteFormattingProvider(documentManager)], }); let hasWorkspaceFolderCapability = false; @@ -73,8 +76,9 @@ connection.onCodeAction(async (params) => { actions.push( ...(await workspace.getDiagnosticFixes(params.textDocument.uri, diagnostic as Diagnostic)).map((fix) => ({ title: fix.title, - kind: 'quickfix', + kind: CodeActionKind.QuickFix, diagnostics: [diagnostic], + isPreferred: fix.isPreferred, edit: { changes: { [params.textDocument.uri]: fix.edits, @@ -87,7 +91,7 @@ connection.onCodeAction(async (params) => { }); connection.onDidOpenTextDocument((params) => { - void workspace.documentManager.fireOpened( + void documentManager.fireOpened( params.textDocument.uri, params.textDocument.languageId, params.textDocument.version, @@ -96,15 +100,11 @@ connection.onDidOpenTextDocument((params) => { }); connection.onDidCloseTextDocument((params) => { - void workspace.documentManager.fireClosed(params.textDocument.uri); + void documentManager.fireClosed(params.textDocument.uri); }); connection.onDidChangeTextDocument((params) => { - void workspace.documentManager.fireChanged( - params.textDocument.uri, - params.contentChanges, - params.textDocument.version, - ); + void documentManager.fireChanged(params.textDocument.uri, params.contentChanges, params.textDocument.version); }); connection.onDocumentOnTypeFormatting(async (params) => { diff --git a/packages/vscode/src/verse-order-diagnostic-provider.ts b/packages/vscode/src/verse-order-diagnostic-provider.ts index 119ec41..6cda052 100644 --- a/packages/vscode/src/verse-order-diagnostic-provider.ts +++ b/packages/vscode/src/verse-order-diagnostic-provider.ts @@ -8,6 +8,7 @@ import { ScriptureChapter, ScriptureDocument, ScriptureNodeType, + ScriptureSerializer, ScriptureVerse, } from '@sillsdev/lynx'; import { map, merge, Observable, switchMap } from 'rxjs'; @@ -16,7 +17,10 @@ export class VerseOrderDiagnosticProvider implements DiagnosticProvider { public readonly id = 'verse-order'; public readonly diagnosticsChanged$: Observable; - constructor(private readonly documentManager: DocumentManager) { + constructor( + private readonly documentManager: DocumentManager, + private readonly serializer: ScriptureSerializer, + ) { this.diagnosticsChanged$ = merge( documentManager.opened$.pipe( map((e) => ({ @@ -60,7 +64,7 @@ export class VerseOrderDiagnosticProvider implements DiagnosticProvider { edits: [ { range: { start: diagnostic.range.start, end: diagnostic.range.start }, - newText: `\\v ${verseNumber.toString()} `, + newText: this.serializer.serialize(new ScriptureVerse(verseNumber.toString())), }, ], }); @@ -72,9 +76,11 @@ export class VerseOrderDiagnosticProvider implements DiagnosticProvider { const diagnostics: Diagnostic[] = []; const verseNodes: [number, ScriptureVerse][] = []; + let chapterNumber = '0'; for (const node of doc.findNodes([ScriptureNodeType.Chapter, ScriptureNodeType.Verse])) { if (node instanceof ScriptureChapter) { - diagnostics.push(...this.findMissingVerse(verseNodes)); + diagnostics.push(...this.findMissingVerse(chapterNumber, verseNodes)); + chapterNumber = node.number; verseNodes.length = 0; } else if (node instanceof ScriptureVerse) { const verseNumber = parseInt(node.number); @@ -85,7 +91,7 @@ export class VerseOrderDiagnosticProvider implements DiagnosticProvider { range: prevVerseNode.range, severity: DiagnosticSeverity.Error, code: 1, - message: 'Verses are out of order.', + message: `Verse ${prevVerseNumber.toString()} occurs out of order in chapter ${chapterNumber}.`, source: this.id, }); } @@ -94,22 +100,23 @@ export class VerseOrderDiagnosticProvider implements DiagnosticProvider { } } - diagnostics.push(...this.findMissingVerse(verseNodes)); + diagnostics.push(...this.findMissingVerse(chapterNumber, verseNodes)); return diagnostics; } - private findMissingVerse(verseNodes: [number, ScriptureVerse][]): Diagnostic[] { + private findMissingVerse(chapterNumber: string, verseNodes: [number, ScriptureVerse][]): Diagnostic[] { verseNodes.sort((a, b) => a[0] - b[0]); const diagnostics: Diagnostic[] = []; for (const [i, [number, node]] of verseNodes.entries()) { if (number !== i + 1) { + const missingVerse = number - 1; diagnostics.push({ range: node.range, severity: DiagnosticSeverity.Warning, code: 2, - message: 'Verse is missing.', + message: `Verse ${missingVerse.toString()} is missing from chapter ${chapterNumber}. Insert the missing verse to fix.`, source: this.id, - data: number - 1, + data: missingVerse, }); } }