From 16918b70a5a130ab87babc525f4a491ed7c213ee Mon Sep 17 00:00:00 2001 From: Ajani Bilby <11359344+AjaniBilby@users.noreply.github.com> Date: Tue, 18 Jul 2023 16:42:04 +1000 Subject: [PATCH] ~Fixed memory overgrowth --- dist/bnf.d.ts | 41 ++++++++++++++------------- dist/shared.js | 10 +++++-- docs/source/changelog.md | 11 +++++++ docs/source/static/dist/bnf-parser.js | 36 ++++++++++++++++------- package.json | 2 +- source/cli.ts | 7 ++++- source/types.ts | 16 ++++++----- source/wasm/run.ts | 12 ++++++-- 8 files changed, 91 insertions(+), 44 deletions(-) diff --git a/dist/bnf.d.ts b/dist/bnf.d.ts index d16df24..258a113 100644 --- a/dist/bnf.d.ts +++ b/dist/bnf.d.ts @@ -1,4 +1,5 @@ import type _Shared from './shared.js'; +export type _Literal = { type: "literal", value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }; export type Term_Program = { type: 'program', start: number, @@ -23,7 +24,7 @@ export type Term_W = { count: number, ref: _Shared.ReferenceRange, value: [ - (Term_Comment | { type: 'literal', value: '\x20', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x09', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x0a', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x0d\x0a', start: number, end: number, count: number, ref: _Shared.ReferenceRange }) + (Term_Comment | _Literal & {value: "\x20"} | _Literal & {value: "\x09"} | _Literal & {value: "\x0a"} | _Literal & {value: "\x0d\x0a"}) ] } export declare function Parse_W (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -40,9 +41,9 @@ export type Term_Comment = { count: number, ref: _Shared.ReferenceRange, value: [ - { type: 'literal', value: '\x23', start: number, end: number, count: number, ref: _Shared.ReferenceRange }, - { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }, - { type: 'literal', value: '\x0a', start: number, end: number, count: number, ref: _Shared.ReferenceRange } + _Literal & {value: "\x23"}, + _Literal, + _Literal & {value: "\x0a"} ] } export declare function Parse_Comment (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -60,7 +61,7 @@ export type Term_Name = { ref: _Shared.ReferenceRange, value: [ Term_Letter, - { type: '(...)*', value: Array<(Term_Letter | Term_Digit | { type: 'literal', value: '\x5f', start: number, end: number, count: number, ref: _Shared.ReferenceRange })>, start: number, end: number, count: number, ref: _Shared.ReferenceRange } + { type: '(...)*', value: Array<(Term_Letter | Term_Digit | _Literal & {value: "\x5f"})>, start: number, end: number, count: number, ref: _Shared.ReferenceRange } ] } export declare function Parse_Name (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -77,7 +78,7 @@ export type Term_Letter = { count: number, ref: _Shared.ReferenceRange, value: [ - ({ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }) + (_Literal | _Literal) ] } export declare function Parse_Letter (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -94,7 +95,7 @@ export type Term_Digit = { count: number, ref: _Shared.ReferenceRange, value: [ - { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } + _Literal ] } export declare function Parse_Digit (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -111,7 +112,7 @@ export type Term_Hex = { count: number, ref: _Shared.ReferenceRange, value: [ - ({ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }) + (_Literal | _Literal | _Literal) ] } export declare function Parse_Hex (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -145,7 +146,7 @@ export type Term_Frag = { count: number, ref: _Shared.ReferenceRange, value: [ - (Term_Escape | Term_Byte | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }) + (Term_Escape | Term_Byte | _Literal) ] } export declare function Parse_Frag (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -162,7 +163,7 @@ export type Term_Escape = { count: number, ref: _Shared.ReferenceRange, value: [ - { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } + _Literal ] } export declare function Parse_Escape (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -179,7 +180,7 @@ export type Term_Byte = { count: number, ref: _Shared.ReferenceRange, value: [ - { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } + _Literal ] } export declare function Parse_Byte (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -196,7 +197,7 @@ export type Term_Def = { count: number, ref: _Shared.ReferenceRange, value: [ - { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }, + _Literal, Term_Expr ] } @@ -222,7 +223,7 @@ export type Term_Expr = { count: number, ref: _Shared.ReferenceRange, value: [ - { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }, + _Literal, Term_Expr_arg ] }>, start: number, end: number, count: number, ref: _Shared.ReferenceRange } @@ -243,8 +244,8 @@ export type Term_Expr_arg = { ref: _Shared.ReferenceRange, value: [ Term_Expr_prefix, - (Term_Constant | Term_Expr_brackets | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }), - { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } + (Term_Constant | Term_Expr_brackets | _Literal), + _Literal ] } export declare function Parse_Expr_arg (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -261,9 +262,9 @@ export type Term_Expr_prefix = { count: number, ref: _Shared.ReferenceRange, value: [ - { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }, - { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }, - { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } + _Literal, + _Literal, + _Literal ] } export declare function Parse_Expr_prefix (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -280,7 +281,7 @@ export type Term_Expr_infix = { count: number, ref: _Shared.ReferenceRange, value: [ - ({ type: 'literal', value: '\x2d\x3e', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x7c', start: number, end: number, count: number, ref: _Shared.ReferenceRange }) + (_Literal & {value: "\x2d\x3e"} | _Literal & {value: "\x7c"}) ] } export declare function Parse_Expr_infix (i: string, refMapping?: boolean): _Shared.ParseError | { @@ -297,7 +298,7 @@ export type Term_Expr_suffix = { count: number, ref: _Shared.ReferenceRange, value: [ - ({ type: 'literal', value: '\x2a', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x3f', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x2b', start: number, end: number, count: number, ref: _Shared.ReferenceRange }) + (_Literal & {value: "\x2a"} | _Literal & {value: "\x3f"} | _Literal & {value: "\x2b"}) ] } export declare function Parse_Expr_suffix (i: string, refMapping?: boolean): _Shared.ParseError | { diff --git a/dist/shared.js b/dist/shared.js index 13f2af7..d4084f3 100644 --- a/dist/shared.js +++ b/dist/shared.js @@ -1,10 +1,16 @@ const OFFSET = {"TYPE":0,"TYPE_LEN":4,"START":8,"END":12,"COUNT":16,"DATA":20}; export function InitParse(ctx, data) { - const memory = ctx.exports.memory; - memory.grow(1); // grow memory if needed + const bytesPerPage = 64 * 1024; // Convert the string to UTF-8 bytes const utf8Encoder = new TextEncoder(); const stringBytes = utf8Encoder.encode(data); + const memory = ctx.exports.memory; + // ONLY grow memory if needed + const chunks = memory.buffer.byteLength / bytesPerPage; + const desireChunks = stringBytes.byteLength * 6 / bytesPerPage; + if (desireChunks > chunks) { + memory.grow(desireChunks - chunks); + } // Copy the string bytes to WebAssembly memory const wasmMemory = new Uint8Array(memory.buffer); wasmMemory.set(stringBytes, ctx.exports.input.value); diff --git a/docs/source/changelog.md b/docs/source/changelog.md index 01f046b..2170b80 100644 --- a/docs/source/changelog.md +++ b/docs/source/changelog.md @@ -1,5 +1,16 @@ # Changelog +## Version 4.0.4 + +### Fixes: + - [x] Memory over growth: Available memory kept growing after every single parse + - [x] Better error handling for `bnf-compile` when given a bad directory + +## Version 4.0.3 + +### Additions: + - [x] `_Literal` helper type in compiled artifacts + ## Version 4.0.2 ### Fixes: diff --git a/docs/source/static/dist/bnf-parser.js b/docs/source/static/dist/bnf-parser.js index 0aefb86..92d3327 100644 --- a/docs/source/static/dist/bnf-parser.js +++ b/docs/source/static/dist/bnf-parser.js @@ -1500,7 +1500,7 @@ function CompileOmit() { return ''; } function CompileGather() { - return TemplateNode(`'literal'`, 'string'); + return "_Literal"; } function CompileTerm(expr) { const once = CompileTermOnce(expr); @@ -1515,10 +1515,10 @@ function CompileTermOnce(expr) { return 'Term_' + expr.value[0].toUpperCase() + expr.value.slice(1); } function CompileNot() { - return TemplateNode(`'literal'`, 'string'); + return "_Literal"; } function CompileRange() { - return TemplateNode(`'literal'`, 'string'); + return "_Literal"; } function CompileLiteral(expr) { const once = CompileLiteralOnce(expr); @@ -1531,7 +1531,7 @@ function CompileLiteral(expr) { } function CompileLiteralOnce(expr) { let safe = expr.value.replace(/[^a-zA-Z0-9]/g, (char) => '\\x' + char.charCodeAt(0).toString(16).padStart(2, '0')); - return TemplateNode(`'literal'`, `'${safe}'`); + return `_Literal & {value: "${safe}"}`; } function CompileRepeat(innerType, repetitions) { switch (repetitions) { @@ -1571,9 +1571,11 @@ function CompileRule(rule) { `export declare function Parse_${capName} (i: string, refMapping?: boolean): _Shared.ParseError | {\n\troot: _Shared.SyntaxNode & ${typeName},\n\treachBytes: number,\n\treach: null | _Shared.Reference,\n\tisPartial: boolean\n}\n`; } function CompileTypes(lang) { - return `import type _Shared from './shared.js';\n` + [...lang.terms.keys()] - .map(x => CompileRule(lang.terms.get(x))) // hush Typescript it's okay - .join('\n'); + return `import type _Shared from './shared.js';\n` + + `export type _Literal = { type: "literal", value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange };\n` + + [...lang.terms.keys()] + .map(x => CompileRule(lang.terms.get(x))) // hush Typescript it's okay + .join('\n'); } function FlattenConstant(syntax) { @@ -1745,11 +1747,17 @@ function Create(wasm) { return bundle; } function InitParse$1(ctx, data) { - const memory = ctx.exports.memory; - memory.grow(1); // grow memory if needed + const bytesPerPage = 64 * 1024; // Convert the string to UTF-8 bytes const utf8Encoder = new TextEncoder(); const stringBytes = utf8Encoder.encode(data); + const memory = ctx.exports.memory; + // ONLY grow memory if needed + const chunks = memory.buffer.byteLength / bytesPerPage; + const desireChunks = stringBytes.byteLength * 6 / bytesPerPage; + if (desireChunks > chunks) { + memory.grow(desireChunks - chunks); + } // Copy the string bytes to WebAssembly memory const wasmMemory = new Uint8Array(memory.buffer); wasmMemory.set(stringBytes, ctx.exports.input.value); @@ -1902,11 +1910,17 @@ var run = /*#__PURE__*/Object.freeze({ const OFFSET = {"TYPE":0,"TYPE_LEN":4,"START":8,"END":12,"COUNT":16,"DATA":20}; function InitParse(ctx, data) { - const memory = ctx.exports.memory; - memory.grow(1); // grow memory if needed + const bytesPerPage = 64 * 1024; // Convert the string to UTF-8 bytes const utf8Encoder = new TextEncoder(); const stringBytes = utf8Encoder.encode(data); + const memory = ctx.exports.memory; + // ONLY grow memory if needed + const chunks = memory.buffer.byteLength / bytesPerPage; + const desireChunks = stringBytes.byteLength * 6 / bytesPerPage; + if (desireChunks > chunks) { + memory.grow(desireChunks - chunks); + } // Copy the string bytes to WebAssembly memory const wasmMemory = new Uint8Array(memory.buffer); wasmMemory.set(stringBytes, ctx.exports.input.value); diff --git a/package.json b/package.json index 7e49d08..e526a87 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "bnf-parser", - "version": "4.0.2", + "version": "4.0.4", "description": "Deterministic BNF compiler/parser", "homepage": "https://bnf-parser.ajanibilby.com", "main": "./bin/index.js", diff --git a/source/cli.ts b/source/cli.ts index f45c934..6f3fdca 100644 --- a/source/cli.ts +++ b/source/cli.ts @@ -51,10 +51,15 @@ export { ready };\n`; const root = process.argv[2] || "./"; +if (!existsSync(root)) { + console.error(`Unknown path ${root}`); + process.exit(1); +} + const isFile = statSync(root).isFile(); const root_dir = isFile ? dirname(root) : root.slice(0, -1); -if (!existsSync(root)) { +if (!existsSync(root_dir)) { console.error(`Unknown path ${root}`); process.exit(1); } diff --git a/source/types.ts b/source/types.ts index e359055..51dcd2e 100644 --- a/source/types.ts +++ b/source/types.ts @@ -65,7 +65,7 @@ function CompileOmit(): string { } function CompileGather(): string { - return TemplateNode(`'literal'`, 'string'); + return "_Literal"; } @@ -85,11 +85,11 @@ function CompileTermOnce(expr: Term): string { function CompileNot(): string { - return TemplateNode(`'literal'`, 'string'); + return "_Literal"; } function CompileRange(): string { - return TemplateNode(`'literal'`, 'string'); + return "_Literal"; } @@ -109,7 +109,7 @@ function CompileLiteralOnce(expr: Literal): string { (char) => '\\x' + char.charCodeAt(0).toString(16).padStart(2, '0') ); - return TemplateNode(`'literal'`, `'${safe}'`); + return `_Literal & {value: "${safe}"}`; } @@ -160,7 +160,9 @@ function CompileRule(rule: Rule) { export function CompileTypes(lang: Parser) { - return `import type _Shared from './shared.js';\n`+ [...lang.terms.keys()] - .map(x => CompileRule(lang.terms.get(x) as any)) // hush Typescript it's okay - .join('\n'); + return `import type _Shared from './shared.js';\n`+ + `export type _Literal = { type: "literal", value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange };\n` + + [...lang.terms.keys()] + .map(x => CompileRule(lang.terms.get(x) as any)) // hush Typescript it's okay + .join('\n'); } \ No newline at end of file diff --git a/source/wasm/run.ts b/source/wasm/run.ts index 7d54f83..0a852a7 100644 --- a/source/wasm/run.ts +++ b/source/wasm/run.ts @@ -26,13 +26,21 @@ export function Create(wasm: BufferSource){ } function InitParse(ctx: WasmParser, data: string) { - const memory = ctx.exports.memory; - memory.grow(1); // grow memory if needed + const bytesPerPage = 64 * 1024 // Convert the string to UTF-8 bytes const utf8Encoder = new TextEncoder(); const stringBytes = utf8Encoder.encode(data); + const memory = ctx.exports.memory; + + // ONLY grow memory if needed + const chunks = memory.buffer.byteLength / bytesPerPage; + const desireChunks = stringBytes.byteLength * 10 / bytesPerPage; + if (desireChunks > chunks) { + memory.grow(desireChunks - chunks); + } + // Copy the string bytes to WebAssembly memory const wasmMemory = new Uint8Array(memory.buffer); wasmMemory.set(stringBytes, ctx.exports.input.value);