From a4d22965ca73179f5cb17e346abb4f38bdb305e3 Mon Sep 17 00:00:00 2001 From: Agustin Mendez Date: Tue, 24 Dec 2019 18:59:27 -0300 Subject: [PATCH] fix: memory layout issue (#58) --- TODO.md | 31 +++++ examples/tests/struct-memory-layout.spec.md | 122 ++++++++++++++++++++ src/compiler/phases/canonicalPhase.ts | 12 +- src/compiler/phases/codeGenerationPhase.ts | 58 +++++++--- src/compiler/phases/semanticPhase.ts | 48 ++++---- src/compiler/types.ts | 20 +++- src/grammar.ts | 2 +- src/index-bin.ts | 61 +++++++++- test/Parser.spec.ts | 30 +++++ 9 files changed, 331 insertions(+), 53 deletions(-) create mode 100644 examples/tests/struct-memory-layout.spec.md diff --git a/TODO.md b/TODO.md index 426c336b..277930c1 100644 --- a/TODO.md +++ b/TODO.md @@ -13,6 +13,37 @@ - for in sugar syntax - robust type alias use cases and test suites +```lys +// this should yield an error +match x { + case 1 -> ??? + case 1 -> ??? + case 1 -> ??? + case 1 -> ??? +} +``` + +```lys +// doesnt work +match x { + else char -> ... +} +``` + +```lys +// assignment as firts code block stmt dowsnt work +match parse(rule, ruleName, parser, level + 1) { + case is Nil -> break + case ast is AST -> { + ret = match ret { + case is Nil -> ast + else -> AstCons(ret, ast) + } + continue + } +} +``` + every declaration with the same name of different impl must have the same visibility exhaustive test of implicit coercion diff --git a/examples/tests/struct-memory-layout.spec.md b/examples/tests/struct-memory-layout.spec.md new file mode 100644 index 00000000..96f2bb3d --- /dev/null +++ b/examples/tests/struct-memory-layout.spec.md @@ -0,0 +1,122 @@ +# Test + +Smoke tests of memory layout + +#### main.lys + +```lys +import support::test +import system::charset::utf8 + +enum TokenType { + EndOfFile + Identifier + Unknown + NewLine + Whitespace + StringLiteral + NumberLiteral + ParenthesesOpen + ParenthesesClose + MacroDecoration + VectorOpen + VectorClose + Operator + LineComment + MultiLineComment + CurlyBracesOpen + CurlyBracesClose + Comma +} + +struct Token(tokenType: TokenType, start: u32, end: u32) + + +enum ParserRule { + Terminal(tokenType: TokenType) + StrictTerminal(tokenType: TokenType, value: string) + NonTerminal(name: string) + Or(lhs: ParserRule, rhs: ParserRule) + OneOrMore(rule: ParserRule) + ZeroOrMore(rule: ParserRule) + Cons(head: ParserRule, tail: ParserRule) + Cut(head: ParserRule, tail: ParserRule) + Optional(rule: ParserRule) + Fail(message: string) + LookAhead(rule: ParserRule) + NegativeLookAhead(rule: ParserRule) + Discard(rule: ParserRule) + Push(name: string, rule: ParserRule) + PushIfManyChildren(name: string, rule: ParserRule) +} + +enum AstNode { + Rule0 + Leaf(token: Token, value: string) + Node(name: string, child: AstNode) + Numbers(a: u8, b: u16, c: u32, d: u64) + SyntaxError(token: Token, message: string) + UnexpectedToken(token: Token, value: string) + AstCons(head: AstNode, tail: AstNode) +} + + +#[export] +fun test(): void = { + START("test basics") + + + var t = 0x0 + + + t = Numbers.^property$0_offset + mustEqual(t, 0x0, "Numbers.property0 must start at 0") + t = Numbers.^property$1_offset + mustEqual(t, 0x1, "Numbers.property1 must start at 1") + t = Numbers.^property$2_offset + mustEqual(t, 0x3, "Numbers.property2 must start at 3") + t = Numbers.^property$3_offset + mustEqual(t, 0x7, "Numbers.property3 must start at 7") + t = Numbers.^allocationSize + mustEqual(t, 0xF, "Numbers.allocationSize must be 15") + + t = Node.^property$0_offset + mustEqual(t, 0x0, "Node.property0 must start at 0") + t = Node.^property$1_offset + mustEqual(t, 0x8, "Node.property1 must start at 8") + t = Node.^allocationSize + mustEqual(t, 0x10, "Node.allocationSize must be 16") + + t = Leaf.^property$0_offset + mustEqual(t, 0x0, "Leaf.property0 must start at 0") + t = Leaf.^property$1_offset + mustEqual(t, 0x8, "Leaf.property1 must start at 8") + t = Leaf.^allocationSize + mustEqual(t, 0x10, "Leaf.allocationSize must be 16") + + t = SyntaxError.^property$0_offset + mustEqual(t, 0x0, "SyntaxError.property0 must start at 0") + t = SyntaxError.^property$1_offset + mustEqual(t, 0x8, "SyntaxError.property1 must start at 8") + t = SyntaxError.^allocationSize + mustEqual(t, 0x10, "SyntaxError.allocationSize must be 16") + + t = AstCons.^property$0_offset + mustEqual(t, 0x0, "AstCons.property0 must start at 0") + t = AstCons.^property$1_offset + mustEqual(t, 0x8, "AstCons.property1 must start at 8") + t = AstCons.^allocationSize + mustEqual(t, 0x10, "AstCons.allocationSize must be 16") + + t = Token.^property$0_offset + mustEqual(t, 0x0, "Token.property0 must start at 0") + t = Token.^property$1_offset + mustEqual(t, 0x8, "Token.property1 must start at 8") + t = Token.^property$2_offset + mustEqual(t, 12 as u32, "Token.property2 must start at 12") + t = Token.^allocationSize + mustEqual(t, 0x10, "Token.allocationSize must be 16") + + END() +} +``` diff --git a/src/compiler/phases/canonicalPhase.ts b/src/compiler/phases/canonicalPhase.ts index b32d5459..5f122a67 100644 --- a/src/compiler/phases/canonicalPhase.ts +++ b/src/compiler/phases/canonicalPhase.ts @@ -542,8 +542,8 @@ const visitor = { return new Nodes.IfNode(astNode, condition, truePart); } }, - SyntaxError(_: Nodes.ASTNode) { - return null; + SyntaxError(node: Nodes.ASTNode) { + return new PositionCapableError(node.errors[0].message, node); }, StructDirective(astNode: Nodes.ASTNode) { const children = astNode.children.slice(); @@ -648,7 +648,13 @@ function visit(astNode: Nodes.ASTNode): T & any { throw new Error('astNode is null'); } if ((visitor as any)[astNode.type]) { - return (visitor as any)[astNode.type](astNode); + const x = (visitor as any)[astNode.type](astNode); + + if (!x) { + throw new PositionCapableError('Error visiting node ' + astNode.type, astNode); + } + + return x; } else { throw new PositionCapableError(`Visitor not implemented for ${astNode.type}`, astNode); } diff --git a/src/compiler/phases/codeGenerationPhase.ts b/src/compiler/phases/codeGenerationPhase.ts index c877b5ca..4c83f645 100644 --- a/src/compiler/phases/codeGenerationPhase.ts +++ b/src/compiler/phases/codeGenerationPhase.ts @@ -422,6 +422,7 @@ function emit(node: Nodes.Node, document: Nodes.DocumentNode): any { export class CodeGenerationPhaseResult { programAST: any; buffer?: Uint8Array; + sourceMap: string | null = null; constructor(public document: Nodes.DocumentNode, public parsingContext: ParsingContext) { failWithErrors(`Compilation`, parsingContext); @@ -464,30 +465,53 @@ export class CodeGenerationPhaseResult { const module = binaryen.readBinary(binary.buffer); - module.runPasses(['duplicate-function-elimination']); + if (!debug) { + module.runPasses(['duplicate-function-elimination']); + } module.runPasses(['remove-unused-module-elements']); if (module.validate() === 0) { this.parsingContext.messageCollector.error(new LysCompilerError('binaryen validation failed', this.document)); } - let last = module.emitBinary(); - - if (optimize) { - do { - module.optimize(); - let next = module.emitBinary(); - if (next.length >= last.length) { - // a if (next.length > last.length) { - // a this.parsingContext.system.write('Last converge was suboptimial.\n'); - // a } - break; - } - last = next; - } while (true); - } + if (debug) { + let last = module.emitBinary('sourceMap.map'); + + if (optimize) { + do { + module.optimize(); + let next = module.emitBinary('sourceMap.map'); + if (next.binary.length >= last.binary.length) { + // a if (next.length > last.length) { + // a this.parsingContext.system.write('Last converge was suboptimial.\n'); + // a } + break; + } + last = next; + } while (true); + } - this.buffer = last; + this.buffer = last.binary; + this.sourceMap = last.sourceMap; + } else { + let last = module.emitBinary(); + + if (optimize) { + do { + module.optimize(); + let next = module.emitBinary(); + if (next.length >= last.length) { + // a if (next.length > last.length) { + // a this.parsingContext.system.write('Last converge was suboptimial.\n'); + // a } + break; + } + last = next; + } while (true); + } + + this.buffer = last; + } module.dispose(); diff --git a/src/compiler/phases/semanticPhase.ts b/src/compiler/phases/semanticPhase.ts index 35233735..d77c27d3 100644 --- a/src/compiler/phases/semanticPhase.ts +++ b/src/compiler/phases/semanticPhase.ts @@ -127,7 +127,7 @@ function explicitDecorator( } function processFunctionDecorations(node: Nodes.FunDirectiveNode, parsingContext: ParsingContext) { - if (node.decorators && node.decorators.length) { + if (node && node.decorators && node.decorators.length) { node.decorators.forEach($ => { switch ($.decoratorName.name) { case 'extern': @@ -153,7 +153,7 @@ function processFunctionDecorations(node: Nodes.FunDirectiveNode, parsingContext } function rejectDecorator(node: Nodes.DirectiveNode, parsingContext: ParsingContext) { - if (node.decorators && node.decorators.length) { + if (node && node.decorators && node.decorators.length) { node.decorators.forEach($ => { parsingContext.messageCollector.error( `Unknown decorator "${$.decoratorName.name}" for ${node.nodeName}`, @@ -190,30 +190,32 @@ const overloadFunctions = function( document.directives[ix] = overloaded; } } else { - rejectDecorator(node, parsingContext); - - if (node instanceof Nodes.ImplDirective) { - overloadFunctions(node, parsingContext); - } else if (node instanceof Nodes.TraitDirectiveNode) { - node.directives.forEach($ => { - if ($ instanceof Nodes.FunDirectiveNode) { - if ($.functionNode.body) { - parsingContext.messageCollector.error( - `Unexpected function body. Traits only accept signatures.`, - $.functionNode.body.astNode - ); - } - if ($.decorators.length > 0) { - $.decorators.forEach($ => { + if (node) { + rejectDecorator(node, parsingContext); + + if (node instanceof Nodes.ImplDirective) { + overloadFunctions(node, parsingContext); + } else if (node instanceof Nodes.TraitDirectiveNode) { + node.directives.forEach($ => { + if ($ instanceof Nodes.FunDirectiveNode) { + if ($.functionNode.body) { parsingContext.messageCollector.error( - `Unexpected decorator. Traits only accept signatures.`, - $.astNode + `Unexpected function body. Traits only accept signatures.`, + $.functionNode.body.astNode ); - }); + } + if ($.decorators.length > 0) { + $.decorators.forEach($ => { + parsingContext.messageCollector.error( + `Unexpected decorator. Traits only accept signatures.`, + $.astNode + ); + }); + } } - } - }); - overloadFunctions(node, parsingContext); + }); + overloadFunctions(node, parsingContext); + } } } }); diff --git a/src/compiler/types.ts b/src/compiler/types.ts index 48799879..5aefe7c2 100644 --- a/src/compiler/types.ts +++ b/src/compiler/types.ts @@ -294,13 +294,16 @@ export class StackType extends Type { schema() { return { byteSize: StackType.of('u32', NativeTypes.i32, 4), - allocationSize: StackType.of('u32', NativeTypes.i32, 4) + allocationSize: StackType.of('u32', NativeTypes.i32, 4), + stackSize: StackType.of('u32', NativeTypes.i32, 4) }; } getSchemaValue(name: string) { if (name === 'byteSize') { return this.byteSize; + } else if (name === 'stackSize') { + return this.byteSize; } else if (name === 'allocationSize') { return this.byteSize; } @@ -354,6 +357,7 @@ export class RefType extends Type { schema() { return { byteSize: StackType.of('u32', NativeTypes.i32, 4), + stackSize: StackType.of('u32', NativeTypes.i32, 4), allocationSize: StackType.of('u32', NativeTypes.i32, 4) }; } @@ -361,6 +365,8 @@ export class RefType extends Type { getSchemaValue(name: string) { if (name === 'byteSize') { return 8; + } else if (name === 'stackSize') { + return 8; } else if (name === 'allocationSize') { return 8; } @@ -681,7 +687,8 @@ export class UnionType extends Type { schema() { return { - byteSize: u32 + byteSize: u32, + stackSize: u32 }; } @@ -705,6 +712,8 @@ export class UnionType extends Type { } } else if (name === 'allocationSize') { return 8; + } else if (name === 'stackSize') { + return 8; } throw new Error(`Cannot read schema property ${name} of ${this.inspect(10)}`); } @@ -797,6 +806,7 @@ export class TypeAlias extends Type { if (baseType instanceof StructType) { result['allocationSize'] = u32; + result['stackSize'] = u32; const properties = this.getOrderedProperties(); @@ -818,7 +828,9 @@ export class TypeAlias extends Type { } else { const baseType = getUnderlyingTypeFromAlias(this); if (baseType instanceof StructType) { - if (name === 'allocationSize') { + if (name === 'stackSize') { + return 8; + } else if (name === 'allocationSize') { const properties = this.getOrderedProperties(); let offset = 0; @@ -846,7 +858,7 @@ export class TypeAlias extends Type { break; } const fn = getNonVoidFunction(TypeHelpers.getNodeType(prop.name) as IntersectionType, ctx); - offset += fn!.returnType!.getSchemaValue('allocationSize', ctx); + offset += fn!.returnType!.getSchemaValue('stackSize', ctx); } return offset; diff --git a/src/grammar.ts b/src/grammar.ts index 7befc161..5ec37586 100644 --- a/src/grammar.ts +++ b/src/grammar.ts @@ -3,7 +3,7 @@ import { Parser, Grammars, IRule } from 'ebnf'; export const grammar = ` {ws=explicit} -Document ::= WS* Directives WS* EOF? {ws=implicit} +Document ::= WS* Directives WS* EOF? Directives ::= Directive WS* Directives? WS* {pin=1,recoverUntil=DIRECTIVE_RECOVERY,fragment=true} Directive ::= FunctionDirective diff --git a/src/index-bin.ts b/src/index-bin.ts index 21e0f691..2e3bb4a1 100644 --- a/src/index-bin.ts +++ b/src/index-bin.ts @@ -2,13 +2,14 @@ import * as arg from 'arg'; import { ParsingContext } from './compiler/ParsingContext'; import { dirname, basename, relative } from 'path'; import { generateTestInstance } from './utils/testEnvironment'; -import { getTestResults } from './utils/libs/test'; +import { getTestResults, TestDescription } from './utils/libs/test'; import { nodeSystem } from './support/NodeSystem'; import { writeFileSync } from 'fs'; import { ForegroundColors, formatColorAndReset } from './utils/colors'; import { LysError } from './utils/errorPrinter'; import { compile } from './index'; import { loadFromMD } from './utils/loadFromMD'; +import { printNode } from './utils/nodePrinter'; function mkdirRecursive(dir: string) { // we explicitly don't use `path.sep` to have it platform independent; @@ -42,6 +43,7 @@ export async function main(cwd: string, argv: string[]) { '--lib': [String], '--test': Boolean, '--debug': Boolean, + '--desugar': Boolean, '--run': '--test' }, { @@ -53,6 +55,7 @@ export async function main(cwd: string, argv: string[]) { args['--lib'] = args['--lib'] || []; const DEBUG = !!args['--debug']; + const DESUGAR = !!args['--desugar']; const file = args._[0]; @@ -123,12 +126,16 @@ export async function main(cwd: string, argv: string[]) { const optimize = !args['--no-optimize']; - await codeGen.validate(optimize, DEBUG); + await codeGen.validate(optimize, DEBUG || args['--wast']); if (args['--wast']) { nodeSystem.writeFile(outFileFullWithoutExtension + '.wast', codeGen.emitText()); } + if (codeGen.sourceMap) { + nodeSystem.writeFile(nodeSystem.resolvePath(outPath, 'sourceMap.map'), codeGen.sourceMap); + } + if (!codeGen.buffer) { throw new LysError('Could not generate WASM binary'); } @@ -176,6 +183,17 @@ exports.default = async function() { nodeSystem.writeFile(outFileFullWithoutExtension + '.js', src.join('\n')); + if (DESUGAR) { + parsingContext.modulesInContext.forEach(module => { + if (module.fileName.startsWith(nodeSystem.cwd)) { + const relativePath = nodeSystem.relative(nodeSystem.cwd, module.fileName); + const targetFile = nodeSystem.resolvePath(outPath + '/desugar/', relativePath); + mkdirRecursive(dirname(targetFile)); + nodeSystem.writeFile(targetFile, printNode(module)); + } + }); + } + if (args['--test']) { const testInstance = await generateTestInstance(codeGen.buffer, libs); @@ -215,13 +233,46 @@ exports.default = async function() { } } + let totalTime = 0; + let totalTests = 0; + let totalPass = 0; + + const summarizeItem = (result: TestDescription) => { + totalTime += result.endTime - result.startTime; + totalTests++; + if (result.passed) { + totalPass++; + } + result.tests && result.tests.forEach(summarizeItem); + }; + + const summarize = (results: TestDescription[]) => { + results.forEach(summarizeItem); + }; + if (testResults && testResults.length) { + summarize(testResults); + nodeSystem.writeFile('_lys_test_results.json', JSON.stringify(testResults, null, 2)); - if (testResults.some($ => !$.passed)) { - console.error(formatColorAndReset('\n\n Some tests failed.\n\n', ForegroundColors.Red)); + if (totalPass !== totalTests) { + console.error( + formatColorAndReset( + `\n\n Some tests failed. Passed: ${totalPass} Failed: ${totalTests - totalPass} (${( + totalTime / 1000 + ).toFixed(2)}s)\n\n`, + ForegroundColors.Red + ) + ); process.exit(1); } else { - console.error(formatColorAndReset('\n\n All tests passed.\n\n', ForegroundColors.Green)); + console.error( + formatColorAndReset( + `\n\n All tests passed. Passed: ${totalPass} Failed: ${totalTests - totalPass} (${( + totalTime / 1000 + ).toFixed(2)}s)\n\n`, + ForegroundColors.Green + ) + ); } } } diff --git a/test/Parser.spec.ts b/test/Parser.spec.ts index 4a46c1d1..be99c23b 100644 --- a/test/Parser.spec.ts +++ b/test/Parser.spec.ts @@ -688,6 +688,36 @@ describe('Parser', () => { test` fun x(): string = "\\"'\`\\\\" ` + + // test` + // fun x(): string = "// a comment inside a string" + + // fun singleLineComment(): void = { + // START("Single line comment") + // { + // val p = Parser("// asd") + + // validateToken(p, LineComment, "// asd") + // validateToken(p, EndOfFile, "") + + // END() + // } + + // START("Single line comment 2") + // { + // val p = Parser("asd // asd\n asd") + + // validateToken(p, Identifier, "asd") + // validateToken(p, LineComment, "// asd") + // validateToken(p, NewLine, "\n") + // validateToken(p, Whitespace, " ") + // validateToken(p, Identifier, "asd") + // validateToken(p, EndOfFile, "") + + // END() + // } + // } + // ` }); describe('bin op', () => {