Skip to content

Commit

Permalink
~Fixed memory overgrowth
Browse files Browse the repository at this point in the history
  • Loading branch information
@AjaniBilby
AjaniBilby committed Jul 18, 2023
1 parent 737de85 commit 16918b7
Show file tree
Hide file tree
Showing 8 changed files with 91 additions and 44 deletions.
41 changes: 21 additions & 20 deletions dist/bnf.d.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type _Shared from './shared.js';
export type _Literal = { type: "literal", value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange };
export type Term_Program = {
type: 'program',
start: number,
Expand All @@ -23,7 +24,7 @@ export type Term_W = {
count: number,
ref: _Shared.ReferenceRange,
value: [
(Term_Comment | { type: 'literal', value: '\x20', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x09', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x0a', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x0d\x0a', start: number, end: number, count: number, ref: _Shared.ReferenceRange })
(Term_Comment | _Literal & {value: "\x20"} | _Literal & {value: "\x09"} | _Literal & {value: "\x0a"} | _Literal & {value: "\x0d\x0a"})
]
}
export declare function Parse_W (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand All @@ -40,9 +41,9 @@ export type Term_Comment = {
count: number,
ref: _Shared.ReferenceRange,
value: [
{ type: 'literal', value: '\x23', start: number, end: number, count: number, ref: _Shared.ReferenceRange },
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange },
{ type: 'literal', value: '\x0a', start: number, end: number, count: number, ref: _Shared.ReferenceRange }
_Literal & {value: "\x23"},
_Literal,
_Literal & {value: "\x0a"}
]
}
export declare function Parse_Comment (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand All @@ -60,7 +61,7 @@ export type Term_Name = {
ref: _Shared.ReferenceRange,
value: [
Term_Letter,
{ type: '(...)*', value: Array<(Term_Letter | Term_Digit | { type: 'literal', value: '\x5f', start: number, end: number, count: number, ref: _Shared.ReferenceRange })>, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
{ type: '(...)*', value: Array<(Term_Letter | Term_Digit | _Literal & {value: "\x5f"})>, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
]
}
export declare function Parse_Name (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand All @@ -77,7 +78,7 @@ export type Term_Letter = {
count: number,
ref: _Shared.ReferenceRange,
value: [
({ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange })
(_Literal | _Literal)
]
}
export declare function Parse_Letter (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand All @@ -94,7 +95,7 @@ export type Term_Digit = {
count: number,
ref: _Shared.ReferenceRange,
value: [
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
_Literal
]
}
export declare function Parse_Digit (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand All @@ -111,7 +112,7 @@ export type Term_Hex = {
count: number,
ref: _Shared.ReferenceRange,
value: [
({ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange })
(_Literal | _Literal | _Literal)
]
}
export declare function Parse_Hex (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand Down Expand Up @@ -145,7 +146,7 @@ export type Term_Frag = {
count: number,
ref: _Shared.ReferenceRange,
value: [
(Term_Escape | Term_Byte | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange })
(Term_Escape | Term_Byte | _Literal)
]
}
export declare function Parse_Frag (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand All @@ -162,7 +163,7 @@ export type Term_Escape = {
count: number,
ref: _Shared.ReferenceRange,
value: [
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
_Literal
]
}
export declare function Parse_Escape (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand All @@ -179,7 +180,7 @@ export type Term_Byte = {
count: number,
ref: _Shared.ReferenceRange,
value: [
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
_Literal
]
}
export declare function Parse_Byte (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand All @@ -196,7 +197,7 @@ export type Term_Def = {
count: number,
ref: _Shared.ReferenceRange,
value: [
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange },
_Literal,
Term_Expr
]
}
Expand All @@ -222,7 +223,7 @@ export type Term_Expr = {
count: number,
ref: _Shared.ReferenceRange,
value: [
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange },
_Literal,
Term_Expr_arg
]
}>, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
Expand All @@ -243,8 +244,8 @@ export type Term_Expr_arg = {
ref: _Shared.ReferenceRange,
value: [
Term_Expr_prefix,
(Term_Constant | Term_Expr_brackets | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }),
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
(Term_Constant | Term_Expr_brackets | _Literal),
_Literal
]
}
export declare function Parse_Expr_arg (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand All @@ -261,9 +262,9 @@ export type Term_Expr_prefix = {
count: number,
ref: _Shared.ReferenceRange,
value: [
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange },
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange },
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
_Literal,
_Literal,
_Literal
]
}
export declare function Parse_Expr_prefix (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand All @@ -280,7 +281,7 @@ export type Term_Expr_infix = {
count: number,
ref: _Shared.ReferenceRange,
value: [
({ type: 'literal', value: '\x2d\x3e', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x7c', start: number, end: number, count: number, ref: _Shared.ReferenceRange })
(_Literal & {value: "\x2d\x3e"} | _Literal & {value: "\x7c"})
]
}
export declare function Parse_Expr_infix (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand All @@ -297,7 +298,7 @@ export type Term_Expr_suffix = {
count: number,
ref: _Shared.ReferenceRange,
value: [
({ type: 'literal', value: '\x2a', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x3f', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x2b', start: number, end: number, count: number, ref: _Shared.ReferenceRange })
(_Literal & {value: "\x2a"} | _Literal & {value: "\x3f"} | _Literal & {value: "\x2b"})
]
}
export declare function Parse_Expr_suffix (i: string, refMapping?: boolean): _Shared.ParseError | {
Expand Down
10 changes: 8 additions & 2 deletions dist/shared.js
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,16 @@
const OFFSET = {"TYPE":0,"TYPE_LEN":4,"START":8,"END":12,"COUNT":16,"DATA":20};
export function InitParse(ctx, data) {
const memory = ctx.exports.memory;
memory.grow(1); // grow memory if needed
const bytesPerPage = 64 * 1024;
// Convert the string to UTF-8 bytes
const utf8Encoder = new TextEncoder();
const stringBytes = utf8Encoder.encode(data);
const memory = ctx.exports.memory;
// ONLY grow memory if needed
const chunks = memory.buffer.byteLength / bytesPerPage;
const desireChunks = stringBytes.byteLength * 6 / bytesPerPage;
if (desireChunks > chunks) {
memory.grow(desireChunks - chunks);
}
// Copy the string bytes to WebAssembly memory
const wasmMemory = new Uint8Array(memory.buffer);
wasmMemory.set(stringBytes, ctx.exports.input.value);
Expand Down
11 changes: 11 additions & 0 deletions docs/source/changelog.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,16 @@
# Changelog

## Version 4.0.4

### Fixes:
- [x] Memory over growth: Available memory kept growing after every single parse
- [x] Better error handling for `bnf-compile` when given a bad directory

## Version 4.0.3

### Additions:
- [x] `_Literal` helper type in compiled artifacts

## Version 4.0.2

### Fixes:
Expand Down
36 changes: 25 additions & 11 deletions docs/source/static/dist/bnf-parser.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1500,7 +1500,7 @@ function CompileOmit() {
return '';
}
function CompileGather() {
return TemplateNode(`'literal'`, 'string');
return "_Literal";
}
function CompileTerm(expr) {
const once = CompileTermOnce(expr);
Expand All @@ -1515,10 +1515,10 @@ function CompileTermOnce(expr) {
return 'Term_' + expr.value[0].toUpperCase() + expr.value.slice(1);
}
function CompileNot() {
return TemplateNode(`'literal'`, 'string');
return "_Literal";
}
function CompileRange() {
return TemplateNode(`'literal'`, 'string');
return "_Literal";
}
function CompileLiteral(expr) {
const once = CompileLiteralOnce(expr);
Expand All @@ -1531,7 +1531,7 @@ function CompileLiteral(expr) {
}
function CompileLiteralOnce(expr) {
let safe = expr.value.replace(/[^a-zA-Z0-9]/g, (char) => '\\x' + char.charCodeAt(0).toString(16).padStart(2, '0'));
return TemplateNode(`'literal'`, `'${safe}'`);
return `_Literal & {value: "${safe}"}`;
}
function CompileRepeat(innerType, repetitions) {
switch (repetitions) {
Expand Down Expand Up @@ -1571,9 +1571,11 @@ function CompileRule(rule) {
`export declare function Parse_${capName} (i: string, refMapping?: boolean): _Shared.ParseError | {\n\troot: _Shared.SyntaxNode & ${typeName},\n\treachBytes: number,\n\treach: null | _Shared.Reference,\n\tisPartial: boolean\n}\n`;
}
function CompileTypes(lang) {
return `import type _Shared from './shared.js';\n` + [...lang.terms.keys()]
.map(x => CompileRule(lang.terms.get(x))) // hush Typescript it's okay
.join('\n');
return `import type _Shared from './shared.js';\n` +
`export type _Literal = { type: "literal", value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange };\n` +
[...lang.terms.keys()]
.map(x => CompileRule(lang.terms.get(x))) // hush Typescript it's okay
.join('\n');
}

function FlattenConstant(syntax) {
Expand Down Expand Up @@ -1745,11 +1747,17 @@ function Create(wasm) {
return bundle;
}
function InitParse$1(ctx, data) {
const memory = ctx.exports.memory;
memory.grow(1); // grow memory if needed
const bytesPerPage = 64 * 1024;
// Convert the string to UTF-8 bytes
const utf8Encoder = new TextEncoder();
const stringBytes = utf8Encoder.encode(data);
const memory = ctx.exports.memory;
// ONLY grow memory if needed
const chunks = memory.buffer.byteLength / bytesPerPage;
const desireChunks = stringBytes.byteLength * 6 / bytesPerPage;
if (desireChunks > chunks) {
memory.grow(desireChunks - chunks);
}
// Copy the string bytes to WebAssembly memory
const wasmMemory = new Uint8Array(memory.buffer);
wasmMemory.set(stringBytes, ctx.exports.input.value);
Expand Down Expand Up @@ -1902,11 +1910,17 @@ var run = /*#__PURE__*/Object.freeze({

const OFFSET = {"TYPE":0,"TYPE_LEN":4,"START":8,"END":12,"COUNT":16,"DATA":20};
function InitParse(ctx, data) {
const memory = ctx.exports.memory;
memory.grow(1); // grow memory if needed
const bytesPerPage = 64 * 1024;
// Convert the string to UTF-8 bytes
const utf8Encoder = new TextEncoder();
const stringBytes = utf8Encoder.encode(data);
const memory = ctx.exports.memory;
// ONLY grow memory if needed
const chunks = memory.buffer.byteLength / bytesPerPage;
const desireChunks = stringBytes.byteLength * 6 / bytesPerPage;
if (desireChunks > chunks) {
memory.grow(desireChunks - chunks);
}
// Copy the string bytes to WebAssembly memory
const wasmMemory = new Uint8Array(memory.buffer);
wasmMemory.set(stringBytes, ctx.exports.input.value);
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "bnf-parser",
"version": "4.0.2",
"version": "4.0.4",
"description": "Deterministic BNF compiler/parser",
"homepage": "https://bnf-parser.ajanibilby.com",
"main": "./bin/index.js",
Expand Down
7 changes: 6 additions & 1 deletion source/cli.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,15 @@ export { ready };\n`;


const root = process.argv[2] || "./";
if (!existsSync(root)) {
console.error(`Unknown path ${root}`);
process.exit(1);
}

const isFile = statSync(root).isFile();
const root_dir = isFile ? dirname(root) : root.slice(0, -1);

if (!existsSync(root)) {
if (!existsSync(root_dir)) {
console.error(`Unknown path ${root}`);
process.exit(1);
}
Expand Down
16 changes: 9 additions & 7 deletions source/types.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ function CompileOmit(): string {
}

function CompileGather(): string {
return TemplateNode(`'literal'`, 'string');
return "_Literal";
}


Expand All @@ -85,11 +85,11 @@ function CompileTermOnce(expr: Term): string {


function CompileNot(): string {
return TemplateNode(`'literal'`, 'string');
return "_Literal";
}

function CompileRange(): string {
return TemplateNode(`'literal'`, 'string');
return "_Literal";
}


Expand All @@ -109,7 +109,7 @@ function CompileLiteralOnce(expr: Literal): string {
(char) => '\\x' + char.charCodeAt(0).toString(16).padStart(2, '0')
);

return TemplateNode(`'literal'`, `'${safe}'`);
return `_Literal & {value: "${safe}"}`;
}


Expand Down Expand Up @@ -160,7 +160,9 @@ function CompileRule(rule: Rule) {


export function CompileTypes(lang: Parser) {
return `import type _Shared from './shared.js';\n`+ [...lang.terms.keys()]
.map(x => CompileRule(lang.terms.get(x) as any)) // hush Typescript it's okay
.join('\n');
return `import type _Shared from './shared.js';\n`+
`export type _Literal = { type: "literal", value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange };\n` +
[...lang.terms.keys()]
.map(x => CompileRule(lang.terms.get(x) as any)) // hush Typescript it's okay
.join('\n');
}
12 changes: 10 additions & 2 deletions source/wasm/run.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,21 @@ export function Create(wasm: BufferSource){
}

function InitParse(ctx: WasmParser, data: string) {
const memory = ctx.exports.memory;
memory.grow(1); // grow memory if needed
const bytesPerPage = 64 * 1024

// Convert the string to UTF-8 bytes
const utf8Encoder = new TextEncoder();
const stringBytes = utf8Encoder.encode(data);

const memory = ctx.exports.memory;

// ONLY grow memory if needed
const chunks = memory.buffer.byteLength / bytesPerPage;
const desireChunks = stringBytes.byteLength * 10 / bytesPerPage;
if (desireChunks > chunks) {
memory.grow(desireChunks - chunks);
}

// Copy the string bytes to WebAssembly memory
const wasmMemory = new Uint8Array(memory.buffer);
wasmMemory.set(stringBytes, ctx.exports.input.value);
Expand Down

0 comments on commit 16918b7

Please sign in to comment.