From b5c1a7da5fba9d22eef105346cb29b28dd46a181 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 24 Sep 2024 11:51:37 +0200 Subject: [PATCH 01/21] Started working on decent test support --- .../recovery/PicoMassRecoveryTests.rsc | 28 +++++ .../recovery/RascalMassRecoveryTests.rsc | 15 +++ .../concrete/recovery/RecoveryTestSupport.rsc | 100 ++++++++++++++++++ .../parser/uptr/UPTRNodeFactory.java | 5 +- 4 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc new file mode 100644 index 0000000000..339d69df26 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc @@ -0,0 +1,28 @@ +module lang::rascal::tests::concrete::recovery::PicoMassRecoveryTests + +import lang::pico::\syntax::Main; +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; + +str picoFac = "begin declare input : natural, + output : natural, + repnr : natural, + rep : natural; + input := 14; + output := 1; + % check this out % + while input - 1 do + rep := output; + repnr := input; + while repnr - 1 do + output := output + rep; + repnr := repnr - 1 + od; + input := input - 1 + od +end"; + +bool testFacDeletions() { + TestStats stats = testSingleCharDeletions(#Program, picoFac); + printStats(stats); + return true; +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc new file mode 100644 index 0000000000..c5d220efb3 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc @@ -0,0 +1,15 @@ +module lang::rascal::tests::concrete::recovery::RascalMassRecoveryTests + +import lang::rascal::\syntax::Rascal; +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import IO; +import ParseTree; + +start[Module] (value input, loc origin) rascalParser = parser(#start[Module], allowAmbiguity=true, allowRecovery=true); +str source = readFile(|std:///lang/rascal/vis/ImportGraph.rsc|); + +bool testRascalDeletions() { + TestStats stats = testSingleCharDeletions(rascalParser, source); + printStats(stats); + return true; +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc new file mode 100644 index 0000000000..57377b3f31 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -0,0 +1,100 @@ +module lang::rascal::tests::concrete::recovery::RecoveryTestSupport + +import ParseTree; +import String; +import IO; +import util::Benchmark; + +public data TestStats = testStats(int totalAttempts=0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, list[tuple[loc,int]] slowParses = []); + +private TestStats testRecovery(&T (value input, loc origin) parser, str input, loc source) { + TestStats stats = testStats(totalAttempts=1); + + int begin = realTime(); + try { + Tree t = parser(input, source); + if (hasErrors(t)) { + stats.successfulRecoveries = 1; + print("+"); + + //errors = findAllErrors(t); + //count = size(errors); + //best = findBestError(t); + //println("successful recovery, errors found, best: , loc: "); + //for (err <- errors) { + // println(""); + //} + } else { + stats.successfulParses = 1; + print("."); + } + } catch ParseError(_): { + stats.parseErrors = 1; + print("?"); + } + int duration = realTime() - begin; + + slowParse = []; + if (duration > 200) { + stats.slowParses = []; + print("!"); + } + + return stats; +} + +TestStats testSingleCharDeletions(type[&T] grammar, str input) = testSingleCharDeletions(parser(grammar, allowAmbiguity=true, allowRecovery=true), input); + +TestStats testSingleCharDeletions(&T (value input, loc origin) parser, str input) { + TestStats totalStats = testStats(); + int len = size(input); + int i = 0; + + while (i < len) { + str modifiedInput = substring(input, 0, i) + substring(input, i+1); + TestStats singleRunStats = testRecovery(parser, modifiedInput, |uknown:///?deleted=<"">|); + totalStats = mergeStats(totalStats, singleRunStats); + i = i+1; + } + + return totalStats; +} + +TestStats mergeStats(TestStats stats1, TestStats stats2) { + TestStats result = stats1; + result.totalAttempts += stats2.totalAttempts; + result.successfulParses += stats2.successfulParses; + result.successfulRecoveries += stats2.successfulRecoveries; + result.failedRecoveries += stats2.failedRecoveries; + result.parseErrors += stats2.parseErrors; + result.slowParses += stats2.slowParses; + return result; +} + +private int percentage(int number, int total) { + return (100*number)/total; +} + +void printStats(TestStats stats) { + println(); + println("Total parses: "); + println("Succesful parses: ( % of total)"); + int totalFailed = stats.totalAttempts - stats.successfulParses; + println("Succesful recoveries: ( % of failed)"); + println("Failed recoveries: ( % of failed)"); + println("Parse errors: ( % of failed)"); + + if (stats.slowParses == []) { + println("No slow parses."); + } else { + println(" slow parses:"); + for ( <- stats.slowParses) { + println(": ms."); + } + } +} + +void testRecovery(type[&T] grammar, loc input) { + TestStats stats = testSingleCharDeletions(grammar, readFile(input)); + printStats(stats); +} \ No newline at end of file diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index 3b5e8a84c6..34ff966fcc 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -145,8 +145,9 @@ public Object getProductionFromNode(ITree node){ @Override public ITree createSkippedNode(int[] characters) { - IList chars = Arrays.stream(characters).mapToObj(VF::character).collect(VF.listWriter()); - return VF.appl(SKIPPED, chars); + //IList chars = Arrays.stream(characters).mapToObj(VF::character).collect(VF.listWriter()); + //return VF.appl(SKIPPED, VF.list(charValues)); + return createLiteralNode(characters, SKIPPED); } public ITree createErrorNode(ArrayList children, Object production) { From b193ce432c1409375dd23a0104be24885897063f Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 24 Sep 2024 11:57:20 +0200 Subject: [PATCH 02/21] Merge branch 'error-recovery' into recovery/testing --- pom.xml | 31 ++++++++++ .../lang/rascal/grammar/definition/Layout.rsc | 9 ++- .../lang/rascal/matching/Fingerprint.rsc | 6 +- .../lang/rascal/syntax/tests/ImplodeTests.rsc | 4 +- .../basic/CompilerIssues/LIstMatchInOR.rsc | 16 ++--- .../basic/CompilerIssues/VariableInOr.rsc | 10 +++ .../lang/rascal/tests/basic/Locations.rsc | 13 +++- .../rascal/tests/library/util/MaybeTests.rsc | 62 +++++++++++++++++++ .../library/lang/rascal/vis/ImportGraph.rsc | 6 +- src/org/rascalmpl/library/util/Reflective.rsc | 10 ++- src/org/rascalmpl/library/util/ShellExec.java | 20 +++--- .../uptr/recovery/ToTokenRecoverer.java | 7 ++- .../parser/util/ParseStateVisualizer.java | 2 - .../repl/TerminalProgressBarMonitor.java | 11 +++- 14 files changed, 174 insertions(+), 33 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/basic/CompilerIssues/VariableInOr.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/library/util/MaybeTests.rsc diff --git a/pom.xml b/pom.xml index 5e11002459..62348c7e09 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,16 @@ 0.28.1 + + + BSD-2-Clause + https://opensource.org/license/BSD-2-Clause + repo + + Note, some older files are still licensed under Eclipse v1 + + + src @@ -55,6 +65,9 @@ rascal-DEPENDENCIES.txt + + ${project.build.directory}/generated-resources + test @@ -270,6 +283,19 @@ -Drascal.compile.skip -Drascal.tutor.skip -DskipTests + + org.codehaus.mojo + license-maven-plugin + 2.4.0 + + + download-licenses + + download-licenses + + + + org.apache.maven.plugins maven-shade-plugin @@ -294,6 +320,11 @@ org/rascalmpl/uri/resolvers.config io/usethesource/vallang/type/types.config + + + + false + diff --git a/src/org/rascalmpl/library/lang/rascal/grammar/definition/Layout.rsc b/src/org/rascalmpl/library/lang/rascal/grammar/definition/Layout.rsc index c9e2c3b6cf..264db1e4ac 100644 --- a/src/org/rascalmpl/library/lang/rascal/grammar/definition/Layout.rsc +++ b/src/org/rascalmpl/library/lang/rascal/grammar/definition/Layout.rsc @@ -86,8 +86,13 @@ list[Symbol] intermix(list[Symbol] syms, Symbol l, set[Symbol] others) { return syms; } -private bool sepInOthers(Symbol sep, set[Symbol] others) // TODO: factored out due to compiler issue - = sep in others || (seq([a,_,b]) := sep && (a in others || b in others)); +private bool sepInOthers(Symbol sep, set[Symbol] others) + // TODO: factored out due to compiler issue + // TODO: rewritten to satisfy stricter variable rules for or operator + { if(sep in others) return true; + if(seq([a,_,b]) := sep) return (a in others || b in others); + return false; + } private Symbol regulars(Symbol s, Symbol l, set[Symbol] others) { return visit(s) { diff --git a/src/org/rascalmpl/library/lang/rascal/matching/Fingerprint.rsc b/src/org/rascalmpl/library/lang/rascal/matching/Fingerprint.rsc index fa06484a54..2bea1824da 100644 --- a/src/org/rascalmpl/library/lang/rascal/matching/Fingerprint.rsc +++ b/src/org/rascalmpl/library/lang/rascal/matching/Fingerprint.rsc @@ -43,6 +43,9 @@ extend ParseTree; import Node; import List; +@synopsis{Remove outer label from symbol, if any} +private Symbol delabel(Symbol s) = label(_, Symbol t) := s ? t : s; + @synopsis{Computes a unique fingerprint for each kind of tree based on the identity of the top-level tree node.} @description{ Concrete fingerprint implements the pattern matching contract: @@ -56,8 +59,7 @@ To complete the function for the other kinds of trees, even though less importan implement a sensible encoding that follows the contract and tries to differentiate as much as possible between different values. } int concreteFingerprint(appl(Production p, list[Tree] _)) = concreteFingerprint(p); -int concreteFingerprint(amb({appl(prod(Symbol s, _, _), list[Tree] _), _})) = internalHashCode("amb") + 43 * internalHashCode(t) - when label(_, Symbol t) := s || Symbol t := s; +int concreteFingerprint(amb({appl(prod(Symbol s, _, _), list[Tree] _), _})) = internalHashCode("amb") + 43 * internalHashCode(delabel(s)); int concreteFingerprint(amb({})) = internalHashCode("amb"); int concreteFingerprint(char(int ch)) = internalHashCode("char") + internalHashCode(ch); int concreteFingerprint(cycle(Symbol s, int _)) = internalHashCode("cycle") + 13 * internalHashCode(s); diff --git a/src/org/rascalmpl/library/lang/rascal/syntax/tests/ImplodeTests.rsc b/src/org/rascalmpl/library/lang/rascal/syntax/tests/ImplodeTests.rsc index 476897ea5e..61f3813df4 100644 --- a/src/org/rascalmpl/library/lang/rascal/syntax/tests/ImplodeTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/syntax/tests/ImplodeTests.rsc @@ -12,11 +12,11 @@ import lang::rascal::\syntax::tests::ImplodeTestGrammar; import ParseTree; import Exception; -public data Num(loc src=|unknown:///|, map[int,list[str]] comments = ()) = \int(str n); +public data Num(loc src=|unknown:///|, map[int,list[str]] comments = ()); public data Exp(loc src=|unknown:///|, map[int,list[str]] comments = ()) = id(str name) | eq(Exp e1, Exp e2) | number(Num n); public Exp number(Num::\int("0")) = Exp::number(Num::\int("01")); -public data Number(loc src=|unknown:///|, map[int,list[str]] comments = ()) = \int(str n); +public data Number(loc src=|unknown:///|, map[int,list[str]] comments = ()); public data Expr(loc src=|unknown:///|, map[int,list[str]] comments = ()) = id(str name) | eq(Expr e1, Expr e2) | number(Number n); public Expr number(Number::\int("0")) = Expr::number(Number::\int("02")); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/basic/CompilerIssues/LIstMatchInOR.rsc b/src/org/rascalmpl/library/lang/rascal/tests/basic/CompilerIssues/LIstMatchInOR.rsc index 26dd0e03e1..c2c9027e00 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/basic/CompilerIssues/LIstMatchInOR.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/basic/CompilerIssues/LIstMatchInOR.rsc @@ -8,14 +8,14 @@ data Symbol ; - @ignoreCompiler{Generates incorrect code} -test bool listMatchInOR() { - bool unquotable(Symbol x) - = par([*Symbol _]) := x - || strt() := x - ; - return unquotable(\strt()); -} +//@ignoreCompiler{Generates incorrect code} +//test bool listMatchInOR() { +// bool unquotable(Symbol x) +// = par([*Symbol _]) := x +// || strt() := x +// ; +// return unquotable(\strt()); +//} data Symbol diff --git a/src/org/rascalmpl/library/lang/rascal/tests/basic/CompilerIssues/VariableInOr.rsc b/src/org/rascalmpl/library/lang/rascal/tests/basic/CompilerIssues/VariableInOr.rsc new file mode 100644 index 0000000000..22ed4a3da7 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/basic/CompilerIssues/VariableInOr.rsc @@ -0,0 +1,10 @@ +module lang::rascal::tests::basic::CompilerIssues::VariableInOr + +data D = d1(int n) | label(D d); + +@ignoreCompiler{Generates erroneous Java code} +int f(D d) = e.n + when label(D e) := d || e := d; + +@ignoreCompiler{Generates erroneous Java code} +test bool testF() = f(label(d1(10))) == 10; \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc b/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc index ef8cb6b5c3..98415cd27c 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc @@ -610,4 +610,15 @@ test bool mvnSchemeTest() { return true; -} \ No newline at end of file +}@synopsis{Nested clones in different scopes are acceptable} +int fun_with_clone(int n){ + if(n > 0){ + int h(int n) = 2*n; + return h(n); + } else { + int h(int n) = 2*n; + return h(n); + } +} + +test bool noCodeClone() = fun_with_clone(3) == 6; \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/util/MaybeTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/util/MaybeTests.rsc new file mode 100644 index 0000000000..0551b1434f --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/util/MaybeTests.rsc @@ -0,0 +1,62 @@ +module lang::rascal::tests::library::util::MaybeTests + +import util::Maybe; + +test bool nothingStrEqual(){ + Maybe[str] x = nothing(); + Maybe[str] y = nothing(); + return x == nothing() && y == nothing() && x == y; +} + +test bool just2nothing(){ + x = just(3); + x = nothing(); + return x == nothing(); +} + +test bool inc1(){ + Maybe[int] x = just(3); + x.val += 1; + return x.val == 4; +} + +test bool inc2(){ + x = just(3); + x.val += 1; + return x.val == 4; +} + +test bool inc3(){ + x = just((1 : "a")); + x.val[1] ? "aaa" += "xxx"; + return x.val == (1:"axxx"); +} + +test bool inc4(){ + x = just((1 : "a")); + x.val[2] ? "aaa" += "xxx"; + return x.val == (1:"a",2:"aaaxxx"); +} + +data X = contain( + Maybe[bool] p, + Maybe[int] kw1 = just(2), + Maybe[str] kw2 = nothing() + ); + +test bool contain1(){ + c = contain(nothing()); + return c.p == nothing() && c.kw1.val == 2 && c.kw2 == nothing(); +} + +test bool contain2(){ + c = contain(nothing(), kw1 = nothing(), kw2 = just("hi")); + return c.p == nothing() && c.kw1 == nothing() && c.kw2 == just("hi"); +} + +test bool contain3(){ + c = contain(nothing()); + c.kw1 = nothing(); + c.kw2 = just("hi"); + return c.p == nothing() && c.kw1 == nothing() && c.kw2 == just("hi"); +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc index 0efc06a82a..b05d70f66e 100644 --- a/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc +++ b/src/org/rascalmpl/library/lang/rascal/vis/ImportGraph.rsc @@ -46,8 +46,8 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { // let's start with a simple graph and elaborate on details in later versions g = { | <- m.imports, hideExternals ==> to notin m.external} + { | <- m.extends, hideExternals ==> to notin m.external} - + { <"_", "_", to> | to <- top(m.imports + m.extends) } // pull up the top modules - + { | from <- bottom(m.imports + m.extends), hideExternals ==> from notin m.external} // pull the bottom modules down. + // + { <"_", "_", to> | to <- top(m.imports + m.extends) } // pull up the top modules + // + { | from <- bottom(m.imports + m.extends), hideExternals ==> from notin m.external} // pull the bottom modules down. ; loc modLinker(str name) { @@ -59,7 +59,7 @@ void importGraph(PathConfig pcfg, bool hideExternals=true) { default loc modLinker(value _) = |nothing:///|; - showInteractiveContent(graph(g, \layout=defaultDagreLayout(), nodeLinker=modLinker), title="Rascal Import/Extend Graph"); + showInteractiveContent(graph(g, \layout=defaultDagreLayout(), nodeLinker=modLinker), title="Rascal Import/Extend Graph", edgeStyle=defaultEdgeStyle()[\curve-style=taxi()]); } @synopsis{Container for everything we need to know about the modules in a project to visualize it.} diff --git a/src/org/rascalmpl/library/util/Reflective.rsc b/src/org/rascalmpl/library/util/Reflective.rsc index 07bfeb7c62..2a9a3f7d5a 100644 --- a/src/org/rascalmpl/library/util/Reflective.rsc +++ b/src/org/rascalmpl/library/util/Reflective.rsc @@ -103,8 +103,14 @@ PathConfig applyManifests(PathConfig cfg) { return cfg; } -str makeFileName(str qualifiedModuleName, str extension = "rsc") = - replaceAll(qualifiedModuleName, "::", "/") + (isEmpty(extension) ? "" : ("." + extension)); +str makeFileName(str qualifiedModuleName, str extension = "rsc") { + str qnameSlashes = replaceAll(qualifiedModuleName, "::", "/"); + int n = findLast(qnameSlashes, "/"); + str prefix = extension == "rsc" ? "" : "$"; + str package = extension == "rsc" ? "" : "rascal/"; + qnameSlashes = n < 0 ? "" + qnameSlashes : qnameSlashes[0..n] + "/" + qnameSlashes[n+1..]; + return "">"; +} loc getSearchPathLoc(str filePath, PathConfig pcfg){ for(loc dir <- pcfg.srcs + pcfg.libs){ diff --git a/src/org/rascalmpl/library/util/ShellExec.java b/src/org/rascalmpl/library/util/ShellExec.java index 92dd0e5d25..fcba6088bf 100644 --- a/src/org/rascalmpl/library/util/ShellExec.java +++ b/src/org/rascalmpl/library/util/ShellExec.java @@ -19,6 +19,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.rascalmpl.exceptions.RuntimeExceptionFactory; import org.rascalmpl.uri.URIResolverRegistry; @@ -34,7 +35,8 @@ import io.usethesource.vallang.IValueFactory; public class ShellExec { - private static Map runningProcesses = new HashMap<>(); + private static Map runningProcesses = new ConcurrentHashMap<>(); + private static Map processExitCodes = new ConcurrentHashMap<>(); private static Map processInputStreams = new HashMap<>(); private static Map processErrorStreams = new HashMap<>(); private static Map processOutputStreams = new HashMap<>(); @@ -107,16 +109,18 @@ public IInteger exitCode(IInteger pid) { Process p = runningProcesses.get(pid); if (p == null) { + IInteger storedExitCode = processExitCodes.get(pid); + if (storedExitCode == null) { throw RuntimeExceptionFactory.illegalArgument(pid, "unknown process"); } + return storedExitCode; + } - while (true) { try { return vf.integer(p.waitFor()); } catch (InterruptedException e) { - continue; - } + throw RuntimeExceptionFactory.javaException(e, null, null); } } @@ -229,11 +233,12 @@ public synchronized void killProcess(IInteger processId, IBool force) { } } - new Thread("zombie process clean up") { + Thread waitForCleared = new Thread("zombie process clean up") { public void run() { while (true) { try { runningProcess.waitFor(); + processExitCodes.put(processId, vf.integer(runningProcess.exitValue())); runningProcesses.remove(processId); return; } @@ -244,8 +249,9 @@ public void run() { } } }; - }.start(); - + }; + waitForCleared.setDaemon(true); + waitForCleared.start(); return; } diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index f495e4902d..16759bd825 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -20,17 +20,14 @@ import org.rascalmpl.parser.gtd.ExpectsProvider; import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; -import org.rascalmpl.parser.gtd.result.EpsilonNode; import org.rascalmpl.parser.gtd.result.SkippedNode; import org.rascalmpl.parser.gtd.stack.AbstractExpandableStackNode; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.CaseInsensitiveLiteralStackNode; import org.rascalmpl.parser.gtd.stack.EmptyStackNode; import org.rascalmpl.parser.gtd.stack.EpsilonStackNode; -import org.rascalmpl.parser.gtd.stack.ListStackNode; import org.rascalmpl.parser.gtd.stack.LiteralStackNode; import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; -import org.rascalmpl.parser.gtd.stack.OptionalStackNode; import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; import org.rascalmpl.parser.gtd.stack.SkippingStackNode; import org.rascalmpl.parser.gtd.stack.StackNodeVisitorAdapter; @@ -49,6 +46,8 @@ import io.usethesource.vallang.IConstructor; public class ToTokenRecoverer implements IRecoverer { + private static final boolean VISUALIZE_RECOVERY_NODES = false; + private URI uri; private IdDispenser stackNodeIdDispenser; private ExpectsProvider expectsProvider; @@ -84,8 +83,10 @@ private DoubleArrayList, AbstractNode> reviveNod recoveryNodes .sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); + if (VISUALIZE_RECOVERY_NODES) { ParseStateVisualizer visualizer = new ParseStateVisualizer("Recovery"); visualizer.visualizeRecoveryNodes(recoveryNodes); + } for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); diff --git a/src/org/rascalmpl/parser/util/ParseStateVisualizer.java b/src/org/rascalmpl/parser/util/ParseStateVisualizer.java index e62d84f6c0..bbe3d573b9 100644 --- a/src/org/rascalmpl/parser/util/ParseStateVisualizer.java +++ b/src/org/rascalmpl/parser/util/ParseStateVisualizer.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; -import java.io.StringReader; import java.net.URI; import java.nio.file.Files; import java.nio.file.StandardCopyOption; @@ -44,7 +43,6 @@ import org.rascalmpl.parser.gtd.result.SkippedNode; import org.rascalmpl.parser.gtd.result.SortContainerNode; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; -import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.util.ArrayList; import org.rascalmpl.parser.gtd.util.DoubleArrayList; diff --git a/src/org/rascalmpl/repl/TerminalProgressBarMonitor.java b/src/org/rascalmpl/repl/TerminalProgressBarMonitor.java index e155bc26c4..f2c349fb2c 100644 --- a/src/org/rascalmpl/repl/TerminalProgressBarMonitor.java +++ b/src/org/rascalmpl/repl/TerminalProgressBarMonitor.java @@ -419,7 +419,16 @@ static int getCursorPosition(PrintWriter writer, InputStream in) throws IOExcept byte[] col = new byte[32]; int len = in.read(col); - String echo = new String(col, 0, len, Configuration.getEncoding()); + String echo; + + try { + echo = new String(col, 0, len, Configuration.getEncoding()); + } + catch (StringIndexOutOfBoundsException e) { + // this happens if there is some other input on stdin (for example a pipe) + // TODO: the input is now read and can't be processed again. + echo = ""; + } if (!echo.startsWith("\u001B[") || !echo.contains(";")) { return -1; From eef3ddcc4f3c5d39aa17f7a3210b5adb2d6ac3ae Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 24 Sep 2024 15:39:16 +0200 Subject: [PATCH 03/21] Added Java and C recovery tests --- .../library/lang/c90/examples/hello-world.c | 19 ++++++++++++ .../library/lang/pico/examples/fac.pico | 18 +++++++++++ .../recovery/LanguageRecoveryTests.rsc | 10 ++++++ .../concrete/recovery/RecoveryTestSupport.rsc | 31 +++++++++++++++++-- 4 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/c90/examples/hello-world.c create mode 100644 src/org/rascalmpl/library/lang/pico/examples/fac.pico create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc diff --git a/src/org/rascalmpl/library/lang/c90/examples/hello-world.c b/src/org/rascalmpl/library/lang/c90/examples/hello-world.c new file mode 100644 index 0000000000..3e04f5ac33 --- /dev/null +++ b/src/org/rascalmpl/library/lang/c90/examples/hello-world.c @@ -0,0 +1,19 @@ + +int print(const char *text); + +void printHello(char *name) { + print("Hello "); + print(name); + print("!"); +} + +int main(int argc, char *argv[]) { + char *name; + if (argc > 1) { + name = argv[1]; + } else { + name = "World"; + } + + printHello(name); +} diff --git a/src/org/rascalmpl/library/lang/pico/examples/fac.pico b/src/org/rascalmpl/library/lang/pico/examples/fac.pico new file mode 100644 index 0000000000..95a05f3e2e --- /dev/null +++ b/src/org/rascalmpl/library/lang/pico/examples/fac.pico @@ -0,0 +1,18 @@ +begin declare input : natural, + output : natural, + repnr : natural, + rep : natural, + s1 : string, + s2 : string; + input := 14; + output := 1; + while input - 1 do + rep := output; + repnr := input; + while repnr - 1 do + output := output + rep; + repnr := repnr - 1 + od; + input := input - 1 + od +end \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc new file mode 100644 index 0000000000..042844ab59 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc @@ -0,0 +1,10 @@ +module lang::rascal::tests::concrete::recovery::LanguageRecoveryTests + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; + +void runLanguageTests() { + testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); + testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); + testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); + testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 57377b3f31..225d0b3e95 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -1,9 +1,16 @@ + +@bootstrapParser module lang::rascal::tests::concrete::recovery::RecoveryTestSupport +import lang::rascal::\syntax::Rascal; import ParseTree; import String; import IO; import util::Benchmark; +import Grammar; + +import lang::rascal::grammar::definition::Modules; + public data TestStats = testStats(int totalAttempts=0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, list[tuple[loc,int]] slowParses = []); @@ -94,7 +101,27 @@ void printStats(TestStats stats) { } } -void testRecovery(type[&T] grammar, loc input) { - TestStats stats = testSingleCharDeletions(grammar, readFile(input)); +private str syntaxLocToModuleName(loc syntaxFile) { + str path = replaceLast(substring(syntaxFile.path, 1), ".rsc", ""); + return replaceAll(path, "/", "::"); +} + +loc zippedFile(str zip, str path) { + loc res = getResource("m3/snakes-and-ladders-project-source.zip"); + loc zipFile = res[scheme="jar+"][path=res.path + "!/"]; + return zipFile + path; +} + +void testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { + Module \module = parse(#start[Module], syntaxFile).top; + str modName = syntaxLocToModuleName(syntaxFile); + gram = modules2grammar(modName, {\module}); + + if (sym:\start(\sort(topSort)) <- gram.starts) { + println("Error recovery of () on :"); + TestStats stats = testSingleCharDeletions(type(sym, gram.rules), readFile(testInput)); printStats(stats); + } else { + println("Cannot find top sort in "); + } } \ No newline at end of file From ec4794ddbfc6d6cecd21c4df1fa43351228ab76c Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 25 Sep 2024 07:56:52 +0200 Subject: [PATCH 04/21] Removed obsolete tests, replaced by tests in LanguageRecoveryTests.rsc --- .../recovery/PicoMassRecoveryTests.rsc | 28 ------------------- .../recovery/RascalMassRecoveryTests.rsc | 15 ---------- 2 files changed, 43 deletions(-) delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc deleted file mode 100644 index 339d69df26..0000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoMassRecoveryTests.rsc +++ /dev/null @@ -1,28 +0,0 @@ -module lang::rascal::tests::concrete::recovery::PicoMassRecoveryTests - -import lang::pico::\syntax::Main; -import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; - -str picoFac = "begin declare input : natural, - output : natural, - repnr : natural, - rep : natural; - input := 14; - output := 1; - % check this out % - while input - 1 do - rep := output; - repnr := input; - while repnr - 1 do - output := output + rep; - repnr := repnr - 1 - od; - input := input - 1 - od -end"; - -bool testFacDeletions() { - TestStats stats = testSingleCharDeletions(#Program, picoFac); - printStats(stats); - return true; -} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc deleted file mode 100644 index c5d220efb3..0000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalMassRecoveryTests.rsc +++ /dev/null @@ -1,15 +0,0 @@ -module lang::rascal::tests::concrete::recovery::RascalMassRecoveryTests - -import lang::rascal::\syntax::Rascal; -import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; -import IO; -import ParseTree; - -start[Module] (value input, loc origin) rascalParser = parser(#start[Module], allowAmbiguity=true, allowRecovery=true); -str source = readFile(|std:///lang/rascal/vis/ImportGraph.rsc|); - -bool testRascalDeletions() { - TestStats stats = testSingleCharDeletions(rascalParser, source); - printStats(stats); - return true; -} \ No newline at end of file From bab859a090d8cb0dd8361fb1fa5cef8443cf07ec Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 25 Sep 2024 09:27:18 +0200 Subject: [PATCH 05/21] Added "delete until end of line" test --- .../recovery/LanguageRecoveryTests.rsc | 13 ++++-- .../concrete/recovery/RecoveryTestSupport.rsc | 41 +++++++++++++++++-- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc index 042844ab59..dc07c4fb46 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc @@ -2,9 +2,14 @@ module lang::rascal::tests::concrete::recovery::LanguageRecoveryTests import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +void runCTest() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); +void runPicoTest() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); +void runJavaTest() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); +void runRascalTest() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); + void runLanguageTests() { - testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); - testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); - testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); - testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); + runPicoTest(); + runCTest(); + runJavaTest(); + runRascalTest(); } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 225d0b3e95..9116f32c7f 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -67,6 +67,24 @@ TestStats testSingleCharDeletions(&T (value input, loc origin) parser, str input return totalStats; } +TestStats testDeleteUntilEol(&T (value input, loc origin) parser, str input) { + TestStats totalStats = testStats(); + int lineStart = 0; + list[int] lineEndings = findAll(input, "\n"); + + for (int lineEnd <- lineEndings) { + lineLength = lineEnd - lineStart; + for (int pos <- [lineStart..lineEnd]) { + modifiedInput = substring(input, 0, pos) + substring(input, lineEnd); + TestStats singleRunStats = testRecovery(parser, modifiedInput, |uknown:///?deletedUntilEol=<",">|); + totalStats = mergeStats(totalStats, singleRunStats); + } + lineStart = lineEnd+1; + } + + return totalStats; +} + TestStats mergeStats(TestStats stats1, TestStats stats2) { TestStats result = stats1; result.totalAttempts += stats2.totalAttempts; @@ -107,7 +125,7 @@ private str syntaxLocToModuleName(loc syntaxFile) { } loc zippedFile(str zip, str path) { - loc res = getResource("m3/snakes-and-ladders-project-source.zip"); + loc res = getResource(zip); loc zipFile = res[scheme="jar+"][path=res.path + "!/"]; return zipFile + path; } @@ -115,12 +133,27 @@ loc zippedFile(str zip, str path) { void testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { Module \module = parse(#start[Module], syntaxFile).top; str modName = syntaxLocToModuleName(syntaxFile); - gram = modules2grammar(modName, {\module}); + Grammar gram = modules2grammar(modName, {\module}); if (sym:\start(\sort(topSort)) <- gram.starts) { + println("--------------------------------------------------------------------------------"); println("Error recovery of () on :"); - TestStats stats = testSingleCharDeletions(type(sym, gram.rules), readFile(testInput)); - printStats(stats); + type[value] begin = type(sym, gram.rules); + testParser = parser(begin, allowAmbiguity=true, allowRecovery=true); + str input = readFile(testInput); + + println("Single char deletions:"); + TestStats singleCharDeletionStats = testSingleCharDeletions(testParser, input); + printStats(singleCharDeletionStats); + TestStats totalStats = singleCharDeletionStats; + + println("Deletes until end-of-line:"); + TestStats deleteUntilEolStats = testDeleteUntilEol(testParser, input); + printStats(deleteUntilEolStats); + totalStats = mergeStats(totalStats, deleteUntilEolStats); + + println("Total stats:"); + printStats(totalStats); } else { println("Cannot find top sort in "); } From 5444c2834759b413b2d7d00092c41535019e3188 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 26 Sep 2024 11:44:54 +0200 Subject: [PATCH 06/21] Improved performance of `findBestError`, improved statistics reporting --- src/org/rascalmpl/library/ParseTree.rsc | 48 +---- .../concrete/recovery/BasicRecoveryTests.rsc | 6 +- .../recovery/LanguageRecoveryTests.rsc | 13 +- .../concrete/recovery/RecoveryTestSupport.rsc | 175 +++++++++++------- .../parser/uptr/UPTRNodeFactory.java | 2 - 5 files changed, 134 insertions(+), 110 deletions(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 8279e18dc1..c2a7950e5d 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -144,7 +144,7 @@ extend Message; extend List; import String; -import Node; +import Set; @synopsis{The Tree data type as produced by the parser.} @description{ @@ -807,20 +807,19 @@ str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) = stringCh This filter removes error trees until no ambiguities caused by error recovery are left. Note that regular ambiguous trees remain in the parse forest. } -Tree defaultErrorDisambiguationFilter(t: appl(Production prod, args)) { - Tree result = appl(prod, [defaultErrorDisambiguationFilter(arg) | arg <- args]); - return setKeywordParameters(result, getKeywordParameters(t)); +Tree defaultErrorDisambiguationFilter(Tree t) { + return visit(t) { + case a:amb(_) => ambDisambiguation(a) + }; } -Tree defaultErrorDisambiguationFilter(amb(set[Tree] alternatives)) { +private Tree ambDisambiguation(amb(set[Tree] alternatives)) { // Go depth-first - set[Tree] disambiguatedAlts = { defaultErrorDisambiguationFilter(alt) | Tree alt <- alternatives }; - - set[Tree] errorTrees = { alt | Tree alt <- disambiguatedAlts, /appl(error(_,_,_), _) := alt }; - set[Tree] nonErrorTrees = { alt | Tree alt <- disambiguatedAlts, /appl(error(_,_,_), _) !:= alt }; + rel[int score, Tree alt] scoredErrorTrees = { | Tree alt <- alternatives }; + set[Tree] nonErrorTrees = scoredErrorTrees[0]; if (nonErrorTrees == {}) { - return getBestErrorTree(errorTrees); + return (getFirstFrom(scoredErrorTrees) | it.score > c.score ? c : it | c <- scoredErrorTrees).alt; } if ({Tree single} := nonErrorTrees) { @@ -832,34 +831,7 @@ Tree defaultErrorDisambiguationFilter(amb(set[Tree] alternatives)) { return amb(nonErrorTrees); } -private Tree getBestErrorTree(set[Tree] trees) { - Tree best = char(0); - int bestErrorCount = -1; - int bestErrorLength = 0; - - for (tree <- trees) { - list[Tree] errors = findAllErrors(tree); - int errorCount = size(errors); - int errorLength = 0; - - for (err <- errors) { - errorLength += getSkipped(err).src.length; - } - - if (bestErrorCount == -1 || errorCount < bestErrorCount || (errorCount == bestErrorCount && errorLength < bestErrorLength)) { - best = tree; - bestErrorCount = errorCount; - bestErrorLength = errorLength; - } - } - - if (bestErrorCount != -1) { - return best; - } - - // trees must have been empty - fail; -} +private int scoreErrors(Tree t) = (0 | it + getSkipped(e).src.length | /e:appl(error(_,_,_),_) := t); // Handle char and cycle nodes default Tree defaultErrorDisambiguationFilter(Tree t) = t; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc index c9edce905b..ab6d77258c 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc @@ -24,7 +24,7 @@ syntax T = ABC End; syntax ABC = 'a' 'b' 'c'; syntax End = "$"; -private Tree parseS(str input, bool visualize=false) +Tree parseS(str input, bool visualize=false) = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool basicOk() { @@ -38,10 +38,10 @@ test bool abx() { test bool axc() { Tree t = parseS("a x c $"); - return getErrorText(findFirstError(t)) == "x c"; + return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x c"; } test bool ax() { Tree t = parseS("a x $"); - return getErrorText(findFirstError(t)) == "x "; + return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x "; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc index dc07c4fb46..aa72c00691 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc @@ -2,10 +2,15 @@ module lang::rascal::tests::concrete::recovery::LanguageRecoveryTests import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; -void runCTest() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); -void runPicoTest() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); -void runJavaTest() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); -void runRascalTest() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); +void runCTest() { testRecoveryC(); } +void runPicoTest() { testRecoveryPico(); } +void runJavaTest() { testRecoveryJava(); } +void runRascalTest() { testRecoveryRascal(); } + +TestStats testRecoveryC() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); +TestStats testRecoveryPico() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); +TestStats testRecoveryJava() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); +TestStats testRecoveryRascal() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); void runLanguageTests() { runPicoTest(); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 9116f32c7f..7beb136cfe 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -1,5 +1,3 @@ - -@bootstrapParser module lang::rascal::tests::concrete::recovery::RecoveryTestSupport import lang::rascal::\syntax::Rascal; @@ -8,67 +6,97 @@ import String; import IO; import util::Benchmark; import Grammar; +import analysis::statistics::Descriptive; import lang::rascal::grammar::definition::Modules; +public data TestMeasurement(loc source=|unknown:///|, int duration=0) = successfulParse() | recovered(int errorSize=0) | parseError(); +public data TestStats = testStats(int slowParseLimit, int recoverySuccessLimit, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, list[TestMeasurement] measurements=[]); -public data TestStats = testStats(int totalAttempts=0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, list[tuple[loc,int]] slowParses = []); +private TestMeasurement testRecovery(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, loc source) { + int startTime = 0; + int duration = 0; + TestMeasurement measurement = successfulParse(); + try { + startTime = realTime(); + Tree t = standardParser(input, source); + duration = realTime() - startTime; + measurement = successfulParse(source=source, duration=duration); + } catch ParseError(_): { + startTime = realTime(); + try { + Tree t = recoveryParser(input, source); + duration = realTime() - startTime; + Tree best = findBestError(t); + errorSize = size(getErrorText(best)); + measurement = recovered(source=source, duration=duration, errorSize=errorSize); + } catch ParseError(_): { + duration = realTime() - startTime; + measurement = parseError(source=source, duration=duration); + } + } -private TestStats testRecovery(&T (value input, loc origin) parser, str input, loc source) { - TestStats stats = testStats(totalAttempts=1); + return measurement; +} - int begin = realTime(); - try { - Tree t = parser(input, source); - if (hasErrors(t)) { - stats.successfulRecoveries = 1; +TestStats updateStats(TestStats stats, TestMeasurement measurement) { + switch (measurement) { + case successfulParse(): { + print("."); + stats.successfulParses += 1; + } + case recovered(errorSize=errorSize): + if (errorSize <= stats.recoverySuccessLimit) { print("+"); - - //errors = findAllErrors(t); - //count = size(errors); - //best = findBestError(t); - //println("successful recovery, errors found, best: , loc: "); - //for (err <- errors) { - // println(""); - //} + stats.successfulRecoveries += 1; } else { - stats.successfulParses = 1; - print("."); + print("-"); + stats.failedRecoveries += 1; } - } catch ParseError(_): { - stats.parseErrors = 1; + case parseError(): { print("?"); + stats.parseErrors += 1; + } } - int duration = realTime() - begin; - slowParse = []; - if (duration > 200) { - stats.slowParses = []; + if (measurement.duration > stats.slowParseLimit) { print("!"); + stats.slowParses += 1; } + stats.measurements = stats.measurements + measurement; + return stats; } -TestStats testSingleCharDeletions(type[&T] grammar, str input) = testSingleCharDeletions(parser(grammar, allowAmbiguity=true, allowRecovery=true), input); +TestStats mergeStats(TestStats stats1, TestStats stats2) { + return testStats( + stats1.slowParseLimit, stats1.recoverySuccessLimit, + successfulParses = stats1.successfulParses + stats2.successfulParses, + successfulRecoveries = stats1.successfulRecoveries + stats2.successfulRecoveries, + failedRecoveries = stats1.failedRecoveries + stats2.failedRecoveries, + parseErrors = stats1.parseErrors + stats2.parseErrors, + slowParses = stats1.slowParses + stats2.slowParses, + measurements = stats1.measurements + stats2.measurements); +} -TestStats testSingleCharDeletions(&T (value input, loc origin) parser, str input) { - TestStats totalStats = testStats(); +TestStats testSingleCharDeletions(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int slowParseLimit, int recoverySuccessLimit) { + TestStats stats = testStats(slowParseLimit, recoverySuccessLimit); int len = size(input); int i = 0; while (i < len) { str modifiedInput = substring(input, 0, i) + substring(input, i+1); - TestStats singleRunStats = testRecovery(parser, modifiedInput, |uknown:///?deleted=<"">|); - totalStats = mergeStats(totalStats, singleRunStats); + TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deleted=<"">|); + stats = updateStats(stats, measurement); i = i+1; } - return totalStats; + return stats; } -TestStats testDeleteUntilEol(&T (value input, loc origin) parser, str input) { - TestStats totalStats = testStats(); +TestStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int slowParseLimit, int recoverySuccessLimit) { + TestStats stats = testStats(slowParseLimit, recoverySuccessLimit); int lineStart = 0; list[int] lineEndings = findAll(input, "\n"); @@ -76,24 +104,13 @@ TestStats testDeleteUntilEol(&T (value input, loc origin) parser, str input) { lineLength = lineEnd - lineStart; for (int pos <- [lineStart..lineEnd]) { modifiedInput = substring(input, 0, pos) + substring(input, lineEnd); - TestStats singleRunStats = testRecovery(parser, modifiedInput, |uknown:///?deletedUntilEol=<",">|); - totalStats = mergeStats(totalStats, singleRunStats); + TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deletedUntilEol=<",">|); + stats = updateStats(stats, measurement); } lineStart = lineEnd+1; } - return totalStats; -} - -TestStats mergeStats(TestStats stats1, TestStats stats2) { - TestStats result = stats1; - result.totalAttempts += stats2.totalAttempts; - result.successfulParses += stats2.successfulParses; - result.successfulRecoveries += stats2.successfulRecoveries; - result.failedRecoveries += stats2.failedRecoveries; - result.parseErrors += stats2.parseErrors; - result.slowParses += stats2.slowParses; - return result; + return stats; } private int percentage(int number, int total) { @@ -102,21 +119,39 @@ private int percentage(int number, int total) { void printStats(TestStats stats) { println(); - println("Total parses: "); - println("Succesful parses: ( % of total)"); - int totalFailed = stats.totalAttempts - stats.successfulParses; + int measurementCount = size(stats.measurements); + println("Total parses: "); + println("Succesful parses: ( % of total)"); + int totalFailed = measurementCount - stats.successfulParses; println("Succesful recoveries: ( % of failed)"); println("Failed recoveries: ( % of failed)"); println("Parse errors: ( % of failed)"); - if (stats.slowParses == []) { + if (stats.slowParses == 0) { println("No slow parses."); } else { - println(" slow parses:"); - for ( <- stats.slowParses) { - println(": ms."); - } + slowest = (getFirstFrom(stats.measurements) | it.duration < e.duration ? it : e | e <- stats.measurements); + println(" slow parses, slowest parse: ( ms)"); } + + println(); + println("95th percentiles:"); + + list[int] successfulParseTimes = [ duration | successfulParse(duration=duration) <- stats.measurements ]; + list[int] successfulRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; + list[int] failedRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize > stats.recoverySuccessLimit ]; + list[int] parseErrorTimes = [ duration | parseError(duration=duration) <- stats.measurements ]; + + println("Succesful parse time: ms"); + println("Succesful recovery time: ms"); + println("Failed recovery time: ms"); + println("Parse error time: ms"); + + list[int] errorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements ]; + println("Recovery error size characters"); + + list[int] successfulErrorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; + println("Successful recovery size: characters"); } private str syntaxLocToModuleName(loc syntaxFile) { @@ -130,31 +165,45 @@ loc zippedFile(str zip, str path) { return zipFile + path; } -void testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { +TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { Module \module = parse(#start[Module], syntaxFile).top; str modName = syntaxLocToModuleName(syntaxFile); Grammar gram = modules2grammar(modName, {\module}); if (sym:\start(\sort(topSort)) <- gram.starts) { - println("--------------------------------------------------------------------------------"); + println("=========================================================================="); println("Error recovery of () on :"); type[value] begin = type(sym, gram.rules); - testParser = parser(begin, allowAmbiguity=true, allowRecovery=true); + standardParser = parser(begin, allowAmbiguity=true, allowRecovery=false); + recoveryParser = parser(begin, allowAmbiguity=true, allowRecovery=true); str input = readFile(testInput); + int startTime = realTime(); + standardParser(input, testInput); + int referenceDuration = realTime() - startTime; + int slowParseLimit = referenceDuration*100; + int recoverySuccessLimit = size(input)/4; + + println(); println("Single char deletions:"); - TestStats singleCharDeletionStats = testSingleCharDeletions(testParser, input); + TestStats singleCharDeletionStats = testSingleCharDeletions(standardParser, recoveryParser, input, slowParseLimit, recoverySuccessLimit); printStats(singleCharDeletionStats); TestStats totalStats = singleCharDeletionStats; + println(); println("Deletes until end-of-line:"); - TestStats deleteUntilEolStats = testDeleteUntilEol(testParser, input); + TestStats deleteUntilEolStats = testDeleteUntilEol(standardParser, recoveryParser, input, slowParseLimit, recoverySuccessLimit); printStats(deleteUntilEolStats); totalStats = mergeStats(totalStats, deleteUntilEolStats); - println("Total stats:"); + println(); + println("Overall stats"); + print("-------------"); printStats(totalStats); + println(); + + return totalStats; } else { - println("Cannot find top sort in "); + throw "Cannot find top sort in "; } } \ No newline at end of file diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index 34ff966fcc..006187c7f9 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -145,8 +145,6 @@ public Object getProductionFromNode(ITree node){ @Override public ITree createSkippedNode(int[] characters) { - //IList chars = Arrays.stream(characters).mapToObj(VF::character).collect(VF.listWriter()); - //return VF.appl(SKIPPED, VF.list(charValues)); return createLiteralNode(characters, SKIPPED); } From f8be72f1fbe8256a8bbf77806a2992c5f08f5441 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 26 Sep 2024 14:28:18 +0200 Subject: [PATCH 07/21] Added some more languages to test and the output of a baseline run --- .../library/lang/diff/unified/UnifiedDiff.rsc | 2 + .../lang/diff/unified/examples/example.diff | 39 ++ .../library/lang/dot/examples/parse-state.dot | 46 ++ .../lang/dot/examples/parser-state.dot | 65 +++ .../rascalmpl/library/lang/dot/syntax/Dot.rsc | 2 +- .../recovery/ErrorRecoveryBenchmark.rsc | 29 ++ .../recovery/LanguageRecoveryTests.rsc | 20 - .../rascal/tests/concrete/recovery/NOTES.md | 413 ++++++++++++++++++ .../concrete/recovery/RecoveryTestSupport.rsc | 24 +- 9 files changed, 611 insertions(+), 29 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/diff/unified/examples/example.diff create mode 100644 src/org/rascalmpl/library/lang/dot/examples/parse-state.dot create mode 100644 src/org/rascalmpl/library/lang/dot/examples/parser-state.dot create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md diff --git a/src/org/rascalmpl/library/lang/diff/unified/UnifiedDiff.rsc b/src/org/rascalmpl/library/lang/diff/unified/UnifiedDiff.rsc index d4e84750c7..4e2d114f00 100644 --- a/src/org/rascalmpl/library/lang/diff/unified/UnifiedDiff.rsc +++ b/src/org/rascalmpl/library/lang/diff/unified/UnifiedDiff.rsc @@ -3,6 +3,8 @@ @contributor{Tijs van der Storm - storm@cwi.nl (CWI)} module lang::diff::unified::UnifiedDiff +start syntax DiffFile = Diff; + syntax Diff = Header old Header new Chunk* chunks ; diff --git a/src/org/rascalmpl/library/lang/diff/unified/examples/example.diff b/src/org/rascalmpl/library/lang/diff/unified/examples/example.diff new file mode 100644 index 0000000000..ea75dfad7a --- /dev/null +++ b/src/org/rascalmpl/library/lang/diff/unified/examples/example.diff @@ -0,0 +1,39 @@ +--- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java ++++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +@@ -1,6 +1,7 @@ + package org.rascalmpl.parser.uptr; + + import java.net.URI; ++import java.util.Arrays; + import java.util.IdentityHashMap; + import java.util.Map; + +@@ -21,7 +22,9 @@ import org.rascalmpl.values.parsetrees.ProductionAdapter; + import org.rascalmpl.values.parsetrees.TreeAdapter; + + public class UPTRNodeFactory implements INodeConstructorFactory{ +- private final static RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); ++ private static final RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); ++ private static final IConstructor SKIPPED = VF.constructor(RascalValueFactory.Production_Skipped, VF.constructor(RascalValueFactory.Symbol_IterStar, VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Range, VF.integer(1), VF.integer(Character.MAX_CODE_POINT)))))); ++ + private boolean allowAmb; + + public UPTRNodeFactory(boolean allowAmbiguity){ +@@ -141,7 +144,14 @@ public class UPTRNodeFactory implements INodeConstructorFactory children, Object production) { ++ IConstructor prod = (IConstructor) production; ++ IConstructor errorProd = VF.constructor(RascalValueFactory.Production_Error, prod.get(0), prod, VF.integer(children.size()-1)); ++ return buildAppl(children, errorProd); ++ } ++ + } diff --git a/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot b/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot new file mode 100644 index 0000000000..bda91c2557 --- /dev/null +++ b/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot @@ -0,0 +1,46 @@ +digraph Parser { +"Parser"["label"="Parser\nInput: \"void f(){if(1){}}\"\nLocation: 0 ('v')\nStep 2: Reducing terminals"]; +"todo-1"["label"="<0> 0", "shape"="record"]; +"-2"["label"="Epsilon: \n.0@0 ,matchable,end\n?\nin: 'sort(\"Tag\") -> regular(\iter-star-seps(sort(\"Tag\"),[layouts(\"LAYOUTLIST\")]))'"]; +"124"["label"="SeparatedList: 124\n.0@0 ,expandable,end\n124\nin: 'default -> tags'"]; +"12858"["label"="NonTerminal: Tags\n.0@0 \nTags\nin: Tags Visibility Signature '=' Expression 'when' 12878 ';'"]; +"-1"["label"="NonTerminal: FunctionDeclaration\n.0@-1 \nFunctionDeclaration"]; +"12858" -> "-1"; +"124" -> "12858"; +"-2" -> "124"; +"todo-1":"0":sw -> "-2"["label"="Stack"]; +"46484886"["shape"="octagon", "label"="Epsilon"]; +"todo-1":"0":se -> "46484886"["label"="Node"]; +"todoLists":"1" -> "todo-1"; +"todoLists"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3 | <4> 4 | <5> 5 | <6> 6 | <7> 7 | <8> 8 | <9> 9 | <10> 10 | <11> 11 | <12> 12 | <13> 13 | <14> 14 | <15> 15", "shape"="record"]; +"Parser" -> "todoLists"["label"="todo lists"]; +"stacksToExpand"["label"="", "shape"="record"]; +"Parser" -> "stacksToExpand"["label"="stacks to expand"]; +"terminalsToReduce"["label"="<0> 0", "shape"="record", "color"="red"]; +"terminalsToReduce":"0":sw -> "-2"["label"="Stack"]; +"terminalsToReduce":"0":se -> "46484886"["label"="Node"]; +"Parser" -> "terminalsToReduce"["label"="terminals to reduce"]; +"nonTerminalsToReduce"["label"="", "shape"="record"]; +"Parser" -> "nonTerminalsToReduce"["label"="non-terminals to reduce"]; +"122"["label"="NonTerminal: Tag\n.0@0 ,end\nTag\nin: 'sort(\"Tag\") -> regular(\iter-star-seps(sort(\"Tag\"),[layouts(\"LAYOUTLIST\")]))'"]; +"122" -> "124"; +"unexpandableNodes":"0" -> "122"; +"unexpandableNodes"["label"="<0> 0", "shape"="record"]; +"12824"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"FunctionDeclaration\")' ':' 12828 0"]; +"unmatchableLeafNodes":"0" -> "12824"; +"128"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"Tags\")' ':' 132 0"]; +"unmatchableLeafNodes":"1" -> "128"; +"2043"["label"="Literal: \n.0@-1 ,matchable\n'@'\nin: '@' Name '=' Expression"]; +"unmatchableLeafNodes":"2" -> "2043"; +"2065"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 '\iter-star(sort(\"Tag\"))' ':' 2069 0"]; +"unmatchableLeafNodes":"3" -> "2065"; +"unmatchableLeafNodes"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3", "shape"="record"]; +"unmatchableMidProductionNodes"["shape"="record", "label"=""]; +"filteredNodes"["label"="", "shape"="record"]; +"error"["label"="Errors"]; +"Parser" -> "error"["label"="error tracking"]; +"error" -> "unexpandableNodes"["label"="unexpandable"]; +"error" -> "unmatchableLeafNodes"["label"="unmatchable leafs"]; +"error" -> "unmatchableMidProductionNodes"["label"="unmatchable mid-prod"]; +"error" -> "filteredNodes"["label"="filtered"]; +} diff --git a/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot b/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot new file mode 100644 index 0000000000..e0405c03a8 --- /dev/null +++ b/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot @@ -0,0 +1,65 @@ +digraph Parser { +"Parser"["label"="Parser\nInput: \"void f(){if(1){}}\"\nLocation: 0 ('v')\nStep 5: Reducing terminals"]; +"todo-1"["label"="<0> 0", "shape"="record"]; +"-2"["label"="Epsilon: \n.0@0 ,matchable,end\n?\nin: 'lex(\"LAYOUT\") -> regular(\iter-star(lex(\"LAYOUT\")))'"]; +"7226"["label"="List: 7226\n.0@0 ,expandable,end\n7226\nin: 'LAYOUTLIST -> \iter-star(lex(\"LAYOUT\"))'"]; +"12860"["label"="NonTerminal: LAYOUTLIST\n.1@0 \nlayouts_LAYOUTLIST\nin: Tags Visibility Signature '=' Expression 'when' 12878 ';'"]; +"-1"["label"="NonTerminal: FunctionDeclaration\n.0@-1 \nFunctionDeclaration"]; +"12860" -> "-1"; +"7226" -> "12860"; +"-2" -> "7226"; +"todo-1":"0" -> "-2"["label"="Stack"]; +"46484886"["shape"="octagon", "label"="Epsilon"]; +"todo-1":"0":se -> "46484886"["label"="Node"]; +"todoLists":"1" -> "todo-1"; +"todoLists"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3 | <4> 4 | <5> 5 | <6> 6 | <7> 7 | <8> 8 | <9> 9 | <10> 10 | <11> 11 | <12> 12 | <13> 13 | <14> 14 | <15> 15", "shape"="record"]; +"Parser" -> "todoLists"["label"="todo lists"]; +"stacksToExpand"["label"="", "shape"="record"]; +"Parser" -> "stacksToExpand"["label"="stacks to expand"]; +"terminalsToReduce"["label"="<0> 0", "shape"="record", "color"="red"]; +"terminalsToReduce":"0":sw -> "-2"["label"="Stack"]; +"terminalsToReduce":"0":se -> "46484886"["label"="Node"]; +"Parser" -> "terminalsToReduce"["label"="terminals to reduce"]; +"nonTerminalsToReduce"["label"="", "shape"="record"]; +"Parser" -> "nonTerminalsToReduce"["label"="non-terminals to reduce"]; +"122"["label"="NonTerminal: Tag\n.0@0 ,end\nTag\nin: 'sort(\"Tag\") -> regular(\iter-star-seps(sort(\"Tag\"),[layouts(\"LAYOUTLIST\")]))'"]; +"124"["label"="SeparatedList: 124\n.0@0 ,expandable,end\n124\nin: 'default -> tags'"]; +"12858"["label"="NonTerminal: Tags\n.0@0 \nTags\nin: Tags Visibility Signature '=' Expression 'when' 12878 ';'"]; +"12858" -> "-1"; +"124" -> "12858"; +"122" -> "124"; +"unexpandableNodes":"0" -> "122"; +"13120"["label"="NonTerminal: Comment\n.0@0 ,end\nComment\nin: 'LAYOUT -> Comment'"]; +"7221"["label"="NonTerminal: LAYOUT\n.0@0 ,end\nLAYOUT\nin: 'lex(\"LAYOUT\") -> regular(\iter-star(lex(\"LAYOUT\")))'"]; +"7221" -> "7226"; +"13120" -> "7221"; +"unexpandableNodes":"1" -> "13120"; +"unexpandableNodes"["label"="<0> 0 | <1> 1", "shape"="record"]; +"12824"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"FunctionDeclaration\")' ':' 12828 0"]; +"unmatchableLeafNodes":"0" -> "12824"; +"128"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"Tags\")' ':' 132 0"]; +"unmatchableLeafNodes":"1" -> "128"; +"2043"["label"="Literal: \n.0@-1 ,matchable\n'@'\nin: '@' Name '=' Expression"]; +"unmatchableLeafNodes":"2" -> "2043"; +"2065"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 '\iter-star(sort(\"Tag\"))' ':' 2069 0"]; +"unmatchableLeafNodes":"3" -> "2065"; +"13122"["label"="Char: \n.0@-1 ,matchable,end\n9-13,32,133,160,5760,6158,8192-8202,8232-8233,8239,8287,12288\nin: 'LAYOUT -> [range(9,13),range(32,32),range(133,133),range(160,160),range(5760,5760),range(6158,6158),range(8192,8202),range(8232,8233),range(8239,8239),range(8287,8287),range(12288,12288)]'"]; +"unmatchableLeafNodes":"4" -> "13122"; +"13125"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 '\iter-star(sort(\"LAYOUT\"))' ':' 13129 0"]; +"unmatchableLeafNodes":"5" -> "13125"; +"7373"["label"="Literal: \n.0@-1 ,matchable\n'/*'\nin: '/*' 7379 '*/'"]; +"unmatchableLeafNodes":"6" -> "7373"; +"7382"["label"="Literal: \n.0@-1 ,matchable\n'//'\nin: '//' 7386"]; +"unmatchableLeafNodes":"7" -> "7382"; +"7389"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"Comment\")' ':' 7393 0"]; +"unmatchableLeafNodes":"8" -> "7389"; +"unmatchableLeafNodes"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3 | <4> 4 | <5> 5 | <6> 6 | <7> 7 | <8> 8", "shape"="record"]; +"unmatchableMidProductionNodes"["shape"="record", "label"=""]; +"filteredNodes"["label"="", "shape"="record"]; +"error"["label"="Errors"]; +"Parser" -> "error"["label"="error tracking"]; +"error" -> "unexpandableNodes"["label"="unexpandable"]; +"error" -> "unmatchableLeafNodes"["label"="unmatchable leafs"]; +"error" -> "unmatchableMidProductionNodes"["label"="unmatchable mid-prod"]; +"error" -> "filteredNodes"["label"="filtered"]; +} diff --git a/src/org/rascalmpl/library/lang/dot/syntax/Dot.rsc b/src/org/rascalmpl/library/lang/dot/syntax/Dot.rsc index 7fcf9b98ca..d50f8f66a5 100644 --- a/src/org/rascalmpl/library/lang/dot/syntax/Dot.rsc +++ b/src/org/rascalmpl/library/lang/dot/syntax/Dot.rsc @@ -68,7 +68,7 @@ syntax NodeId | Id Port ; -syntax Port = ":" Id Id? +syntax Port = ":" Id (":" Id)? // | ":" Id // | ":" CompassPt ; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc new file mode 100644 index 0000000000..a0459cc523 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -0,0 +1,29 @@ +module lang::rascal::tests::concrete::recovery::ErrorRecoveryBenchmark + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; + +void runTestC() { testRecoveryC(); } +void runTestDiff() { testRecoveryDiff(); } +void runTestDot() { testRecoveryDot(); } +void runTestJava() { testRecoveryJava(); } +void runTestJson() { testRecoveryJson(); } +void runTestPico() { testRecoveryPico(); } +void runTestRascal() { testRecoveryRascal(); } + +TestStats testRecoveryC() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); +TestStats testRecoveryDiff() = testErrorRecovery(|std:///lang/diff/unified/UnifiedDiff.rsc|, "DiffFile", |std:///lang/diff/unified/examples/example.diff|); +TestStats testRecoveryDot() = testErrorRecovery(|std:///lang/dot/syntax/Dot.rsc|, "DOT", |std:///lang/dot/examples/parser-state.dot|); +TestStats testRecoveryJava() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); +TestStats testRecoveryJson() = testErrorRecovery(|std:///lang/json/syntax/JSON.rsc|, "JSONText", |std:///lang/json/examples/ex01.json|); +TestStats testRecoveryPico() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); +TestStats testRecoveryRascal() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); + +void runLanguageTests() { + testRecoveryC(); + testRecoveryDiff(); + testRecoveryDot(); + testRecoveryJava(); + testRecoveryJson(); + testRecoveryPico(); + testRecoveryRascal(); +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc deleted file mode 100644 index aa72c00691..0000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/LanguageRecoveryTests.rsc +++ /dev/null @@ -1,20 +0,0 @@ -module lang::rascal::tests::concrete::recovery::LanguageRecoveryTests - -import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; - -void runCTest() { testRecoveryC(); } -void runPicoTest() { testRecoveryPico(); } -void runJavaTest() { testRecoveryJava(); } -void runRascalTest() { testRecoveryRascal(); } - -TestStats testRecoveryC() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); -TestStats testRecoveryPico() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); -TestStats testRecoveryJava() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); -TestStats testRecoveryRascal() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); - -void runLanguageTests() { - runPicoTest(); - runCTest(); - runJavaTest(); - runRascalTest(); -} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md new file mode 100644 index 0000000000..5eb4723935 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md @@ -0,0 +1,413 @@ + +# Error recovery benchmark + +Next to some basic integration tests, this directory contains a simple error recovery benchmark (see `ErrorRecoveryBenchmark.rsc`). +This benchmark is meant to provide feedback on the quality and speed of error recovery and to be able to track progress in these areas. + +The benchmark tests error recovery for a number of languages by performing modifications on a valid input file and attempting to parse the result. +Currently only two types of modifications are implemented: + +- Deletion of single characters +- Deletion until the next end-of-line character + +We anticipate more types of modifications later, possibly based on the parse tree instead of the raw source file. + +## Baseline test + +Below the output of a baseline test is included. This provides a basic idea of what information the benchmark provides and also serves as a comparison for assessing the result of future improvements. + +Note that a benchmark test is documented by a string of characters including: + +- `.`: After modification the input still parses without errors +- '+': After modification error recovery is able to recover by skipping a reasonable amount of input +- '-': After modification error recovery is able to recover by skipping an excessive amount of input (>25% of the contents of the file) +- '?': After modification parsing results in an unrecoverable parse error + +The rest of the output should (hopefully) be self-descriptive: + +``` +rascal>import lang::rascal::tests::concrete::recovery::ErrorRecoveryBenchmark; +rascal>runLanguageTests(); +========================================================================== +Error recovery of |std:///lang/c90/syntax/C.rsc| (TranslationUnit) on |std:///lang/c90/examples/hello-world.c|: +☑ Generating parser; MoreParameters Specifier Declarator TypeQualifier TranslationUnit AnonymousIdentifier FunctionDefinition MultiLineCommentBodyToken NonCommaExpression Expression Asterisk Declaration StorageClass StringConstant AbstractDeclarator PrototypeDeclarator Globa 🕛 0:00:03.468 + +Single char deletions: +...........+................++....................+..........+.+.........++......+++.........+....++.........++.+++..-............?....................+++.+......................................................................?....++.-.....+.++.?.................++++....+.++++.?...........+.+.....?+....?................+....+?..?.. +Total parses: 333 +Succesful parses: 281 (84 % of total) +Succesful recoveries: 42 (80 % of failed) +Failed recoveries: 2 (3 % of failed) +Parse errors: 8 (15 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 1/1/2 ms +Succesful recovery time : 3/2/8 ms +Failed recovery time : 1/2/2 ms +Parse error time : 1/1/2 ms +Recovery error size : 17/5/83 characters +Successful recovery size : 13/4/50 characters + +Deletes until end-of-line: +...........++++++++++++++++++...++++++++++++++++++++++?----++.........++++++++++....+++++++++++....++++++++++.-..?++??++++???????????????????????++............................................................+++++?????.????+++++++++++??......++++++++++++++....??+++??......+++++++??????+.???.....????++++++??????.?. +Total parses: 314 +Succesful parses: 119 (37 % of total) +Succesful recoveries: 128 (65 % of failed) +Failed recoveries: 5 (2 % of failed) +Parse errors: 62 (31 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/2 ms +Succesful recovery time : 2/2/9 ms +Failed recovery time : 2/2/3 ms +Parse error time : 1/1/2 ms +Recovery error size : 38/36/82 characters +Successful recovery size : 36/35/82 characters + +Overall stats +------------- +Total parses: 647 +Succesful parses: 400 (61 % of total) +Succesful recoveries: 170 (68 % of failed) +Failed recoveries: 7 (2 % of failed) +Parse errors: 70 (28 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 1/1/2 ms +Succesful recovery time : 2/2/9 ms +Failed recovery time : 1/2/3 ms +Parse error time : 1/1/2 ms +Recovery error size : 33/27/83 characters +Successful recovery size : 30/26/82 characters + +========================================================================== +Error recovery of |std:///lang/diff/unified/UnifiedDiff.rsc| (DiffFile) on |std:///lang/diff/unified/examples/example.diff|: + +Single char deletions: +++++....................................................+++++....................................................++++++.++++.++++.+...................................+.+.....................+.........................+..................................+......................+.----...---...----...........................................................-....................................................-.-.........................................................................................-..........................................................................................................-..........................................................................................................-...............................................................................................................................................................................................................................................................................................................................-.-...........................-..-.................................................????....???......???..................................................................................?...?............................................................................................................................................................................................?.................................................?...?.?..............................................................................?.................................................?..................................................................................................................................?.........................................?...?.?.? +Total parses: 1776 +Succesful parses: 1702 (95 % of total) +Succesful recoveries: 30 (40 % of failed) +Failed recoveries: 21 (28 % of failed) +Parse errors: 23 (31 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/2 ms +Succesful recovery time : 1/1/2 ms +Failed recovery time : 5/6/9 ms +Parse error time : 0/0/1 ms +Recovery error size : 386/161/837 characters +Successful recovery size : 96/96/161 characters + +Deletes until end-of-line: +++++....................................................++++....................................................++++++++++++++++..................................++....................+........................+.................................+.....................+-----!-!-!-----!-!----..........................................................-...................................................--........................................................................................-.........................................................................................................-.........................................................................................................+..............................................................................................................................................................................................................................................................................................................................--..........................-.-................................................????????????????????.................................................................................?..??.............?.......................................................?..................................................?.....?......................................................?................................................?..??.............................................................................?................................................?.................................................................................................................................?........................................?..??. +Total parses: 1737 +Succesful parses: 1643 (94 % of total) +Succesful recoveries: 31 (32 % of failed) +Failed recoveries: 26 (27 % of failed) +Parse errors: 37 (39 % of failed) +5 slow parses, slowest parse: |unknown:///?deletedUntilEol=1768,1770| (0 ms) + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/2 ms +Succesful recovery time : 1/2/2 ms +Failed recovery time : 6/6/17 ms +Parse error time : 1/1/2 ms +Recovery error size : 398/160/777 characters +Successful recovery size : 102/147/161 characters + +Overall stats +------------- +Total parses: 3513 +Succesful parses: 3345 (95 % of total) +Succesful recoveries: 61 (36 % of failed) +Failed recoveries: 47 (27 % of failed) +Parse errors: 60 (35 % of failed) +5 slow parses, slowest parse: |unknown:///?deletedUntilEol=1768,1770| (0 ms) + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/2 ms +Succesful recovery time : 1/1/2 ms +Failed recovery time : 6/6/13 ms +Parse error time : 0/1/2 ms +Recovery error size : 392/161/837 characters +Successful recovery size : 99/109/161 characters + +========================================================================== +Error recovery of |std:///lang/dot/syntax/Dot.rsc| (DOT) on |std:///lang/dot/examples/parser-state.dot|: +☑ Generating parser; DotAttr NodeId Id EdgeStatement EdgeRhs Nod LAYOUT AttrStatement EdgeOp Comment AttrTag DOT Edg StatementList NodeStatement AttrList Statement StatementOptional Subgraph Graph AttrList0 Port 🕛 0:00:01.307 + +Single char deletions: +++??????.......?..+......+-+.....+++...............+..................+................................................++...+......--+.....+++.....+..+.....+++......++...+..+-+.....+++............................................+.......+.............................+.......+.....++...-....--+.....+++..........................................................................+.......+....++...+.....+-+.....+++.................................................................................................................++...+..+++.....+++.............................................................++...+.....+.++.+..+...+....+.++.+.....+...+..+.++.+....+...+......+.+.+.++.+..+++.....+++.....++...+........+++.....+++.......+..+.....+++.......++...+......+.+.+....++.+........+++.....+++....++...+.........+.+.+.++.+......+...+.........+++.....+++.........................................................................................................................................+..+.....+++......++...+......+.++.+.........+++.....+++..........++...+..............+++.....++++..+.....+++......++...+......+.++.+..............+++.....+++................++...+.................+++.....+++.....+..+.....+++......+..+.....+++...++...+.................+.+.+....++.+..+++.....+++.....++...+.................+.+.+....++.+........+-+.....+++....++...+......+.++.+.................+-+.....+++...................++...+....................+-+.....++++..+.....+++......++...+......+.-+.+....................+-+.....+++.......................++...-...--+.....+++............................................+....+...................................+....+............+...........+......++...-...--+.....+++....................................................................++...+.....+++.....+++.............................................................................................++...+.....+.++.+..-...-...-.++.-.....-...-...-.++.-...-...-.................-.-.-.-+.-...-...-.....--+.....+++.................................................................++...-....-++.....+++.................................................+.......+.............................+.......+.....++...+....+.++.+....+...+.....+.++.+....+...+.................+.+.+.++.+.....+...+.................+-+.....+++.............+..+.....+++......++...+.....+++.....+++.........................................+....................+...............++...+....................+.+.+.++.+.....+...+...+++.....+++.........................................+.....+.............++...+....................+.+.+.++.+...+...+....+++.....+++.............................................................++...+....................+.+.+.++.+....+...+....+++.....+++....................................................+....+...............++...+....................+.+.+.-+.+....+...+.....+++.....+++..............................................................................................................................................................................................................................................................................................++...+....................+.+.+.++.+.....+...+.....+++.....+++....................................................+.......+................++...+....................+.+.+.++.+.....+...+....+++.....+++.....................................................++...+....................+.+.+.++.+....+...+....+++.....+++................................................++...+....................+.+.+.++.+....+...+....+++.....+++.........................................+........+..............++...+....................+.+.+.+?.+....+...+....................+++.....+++.....................................................................+..+.....+++......++...?.............................?++.....+++......+..+.....+++++...?.............?++.....++++..+.....+++......++...?.....?++.....+++......++...?......?.+?.?.....?++.....+++..............++...?.....?.+?.?.................?++.....+++............++...?.....?.+?.?....................?++.....+++.................++...?.....?.+?.?.............................?++.....+++....................++...?.....?.+?.?.............?++.....++?........??...?.. +Total parses: 4239 +Succesful parses: 3555 (83 % of total) +Succesful recoveries: 599 (87 % of failed) +Failed recoveries: 42 (6 % of failed) +Parse errors: 43 (6 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 5/5/8 ms +Succesful recovery time : 9/9/17 ms +Failed recovery time : 10/10/17 ms +Parse error time : 3/3/6 ms +Recovery error size : 233/44/2220 characters +Successful recovery size : 65/38/241 characters + +Deletes until end-of-line: +????????????????..-------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++.++++++++++++++++++++++++++++++++++...---.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++..+..+++...+++++..+..++++++...+++..+..+++++...+++++++..++..+..+++.++++++++++++++++...+++++++++.+++++++++++++++++++++++++++++++++++++...+++++++..++.+...+..+++++++++.+++++++++++++++...++++++++++..++..+..+++++++...++++++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++..+..++++++++++.+++++++++++++++++++++...+++++++++++++++.+++++++++++++++++++++++++++++...+++++++..+..+++++++++++++++.+++++++++++++++++++++++++++...++++++++++++++++++.+++++++++++++++++++++++++++++++++++++++++++++++++...++++++++++++++++++.+++.+...+..+++.++++++++++++++++...++++++++++++++++++..++.....+..+++++++++.+++++++++++++++...+++++++..+..++++++++++++++++++.++++++++++++++++++++++++++++++...+++++++++++++++++++++.+++++++++++++++++++++++++++++...-------..+..---------------------.++++++++++++++++++++++++++++++++++...----.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...------..+..---...----..+..------...----..+..----...------------------..--..+..----...------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++..+..+++++...++++++..+..+++++...++++++++++++++++++..++..+..++++++...++++++++++++++++++.++++++++++++++++++++++++++++++++++++++++++...++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..?..+++++...?????????????????????.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...??????????????????????????????.+++++++++++++++++++++++++++++...??????????????.+++++++++++++++++++++++++++++...??????.+++++++++++++++++...???????..?..??????.+++++++++++++++++++++++++...??????..?..??????????????????.+++++++++++++++++++++++...??????..?..?????????????????????.++++++++++++++++++++++++++++...??????..?..??????????????????????????????.+++++++++++++++++++++++++++++++...??????..?????????????????.???????????????????..?. +Total parses: 4174 +Succesful parses: 402 (9 % of total) +Succesful recoveries: 3438 (91 % of failed) +Failed recoveries: 99 (2 % of failed) +Parse errors: 235 (6 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 6/6/13 ms +Succesful recovery time : 9/8/18 ms +Failed recovery time : 11/10/25 ms +Parse error time : 4/4/7 ms +Recovery error size : 150/63/391 characters +Successful recovery size : 81/61/297 characters + +Overall stats +------------- +Total parses: 8413 +Succesful parses: 3957 (47 % of total) +Succesful recoveries: 4037 (90 % of failed) +Failed recoveries: 141 (3 % of failed) +Parse errors: 278 (6 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 5/5/9 ms +Succesful recovery time : 9/8/18 ms +Failed recovery time : 11/10/19 ms +Parse error time : 3/4/7 ms +Recovery error size : 163/61/408 characters +Successful recovery size : 79/57/289 characters + +========================================================================== +Error recovery of |std:///lang/java/syntax/Java15.rsc| (CompilationUnit) on |jar+file:///C:/Users/pieter/Documents/projects/wk/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|: +☑ Generating parser; SwitchBlock CharLiteral BinaryExponent ClassOrInterfaceType VarDec Super VarMod Id Type ClassBody Interfaces Asterisk CommentPart DimExpr ImportDec ArrayAccess PackageDec MethodSpec DeciFloatExponentPart BlockStm EnumConstArgs AbstractMethodDec DeciNumer 🕛 0:00:08.222 + +Single char deletions: +++++++++......-..++++++.++++++..........+++++++++.......-...++++++...........-....+....+....+........+.-........+....+.........++..+...+..........++++++.....................++.+...++++++.....+..?.?. +Total parses: 198 +Succesful parses: 131 (66 % of total) +Succesful recoveries: 61 (91 % of failed) +Failed recoveries: 4 (5 % of failed) +Parse errors: 2 (2 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 1/1/2 ms +Succesful recovery time : 2/2/6 ms +Failed recovery time : 2/2/4 ms +Parse error time : 0/1/1 ms +Recovery error size : 18/14/69 characters +Successful recovery size : 14/13/31 characters + +Deletes until end-of-line: +.++++++------------------------------------------------++++++++++++++++++++-------------------------...+++++++++++++++++++++++..+.......++++++++++++++++++++++++++++++++...+++++++++++??? +Total parses: 185 +Succesful parses: 16 (8 % of total) +Succesful recoveries: 93 (55 % of failed) +Failed recoveries: 73 (43 % of failed) +Parse errors: 3 (1 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 1/1/2 ms +Succesful recovery time : 2/2/5 ms +Failed recovery time : 1/2/3 ms +Parse error time : 0/1/1 ms +Recovery error size : 48/42/112 characters +Successful recovery size : 18/11/45 characters + +Overall stats +------------- +Total parses: 383 +Succesful parses: 147 (38 % of total) +Succesful recoveries: 154 (65 % of failed) +Failed recoveries: 77 (32 % of failed) +Parse errors: 5 (2 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 1/1/2 ms +Succesful recovery time : 2/2/6 ms +Failed recovery time : 1/2/4 ms +Parse error time : 0/1/1 ms +Recovery error size : 39/21/112 characters +Successful recovery size : 16/12/42 characters + +========================================================================== +Error recovery of |std:///lang/json/syntax/JSON.rsc| (JSONText) on |std:///lang/json/examples/ex01.json|: +☑ Generating parser; Member Object Value NumericLiteral StringChar IntegerLiteral RealLiteral Array JSONText UnicodeEscape StringLiteral 🕛 0:00:00.619 + +Single char deletions: +?........?.....??.?............+.....++..+..?............+......++.+..?............+.....++..+....................??............?.........??.?................+...++....+......................................++................+......-+....+................-.....+-..+...-............?+............?...??.?...+....+....+......?........?..? +Total parses: 337 +Succesful parses: 285 (84 % of total) +Succesful recoveries: 27 (51 % of failed) +Failed recoveries: 4 (7 % of failed) +Parse errors: 21 (40 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/0/1 ms +Succesful recovery time : 0/1/1 ms +Failed recovery time : 0/0/1 ms +Parse error time : 0/1/1 ms +Recovery error size : 34/21/124 characters +Successful recovery size : 21/16/57 characters + +Deletes until end-of-line: +?.????????????????............+++++++??????............++++++++?????............????????????????????????????????.????????????????????????................++++++++++++++++++++++++++++++++++++++++++++++++++................--------+++++.-----------------------------.-----------+.??????????????????+????????????????????.???????. +Total parses: 324 +Succesful parses: 75 (23 % of total) +Succesful recoveries: 72 (28 % of failed) +Failed recoveries: 48 (19 % of failed) +Parse errors: 129 (51 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/0/1 ms +Succesful recovery time : 0/1/2 ms +Failed recovery time : 0/1/1 ms +Parse error time : 0/0/1 ms +Recovery error size : 86/57/294 characters +Successful recovery size : 42/57/57 characters + +Overall stats +------------- +Total parses: 661 +Succesful parses: 360 (54 % of total) +Succesful recoveries: 99 (32 % of failed) +Failed recoveries: 52 (17 % of failed) +Parse errors: 150 (49 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/0/1 ms +Succesful recovery time : 0/1/2 ms +Failed recovery time : 0/1/1 ms +Parse error time : 0/0/1 ms +Recovery error size : 76/57/293 characters +Successful recovery size : 37/39/57 characters + +========================================================================== +Error recovery of |std:///lang/pico/syntax/Main.rsc| (Program) on |std:///lang/pico/examples/fac.pico|: +☑ Generating parser; Type Id WhitespaceAndComment Statement Natural Program Declarations String Expression IdType 🕛 0:00:00.719 + +Single char deletions: +?????.-------.......+.+++++++-.........................+.++++++++............................+.++++++++....................+.++++++++................+..+.+++++++................+..+.------+..............++...+...............++.+-........-----.......+.-.--.................++.......+..................++......-............+++++.......+.+.++......................++........+....+.....................++.......+.+............+++..................--.......+.+........--..???.. +Total parses: 472 +Succesful parses: 355 (75 % of total) +Succesful recoveries: 81 (69 % of failed) +Failed recoveries: 28 (23 % of failed) +Parse errors: 8 (6 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/1 ms +Succesful recovery time : 2/3/5 ms +Failed recovery time : 1/2/4 ms +Parse error time : 0/0/1 ms +Recovery error size : 87/10/463 characters +Successful recovery size : 16/6/70 characters + +Deletes until end-of-line: +?????--------..++++++++++++++-..................++++++++++++++++......................+++++++++++++++................+++++++++++++................+++++++++++.-------------------------+........+++++++++++........-----------.+++++++-----------------.............+++++++++++++............+++++++-------.++++++++++++++++++++++++++++...............++++++++++++++++++++++.+++++++++++++++++++++++......++.------------+.------------------++......++.+++++++-.???. +Total parses: 454 +Succesful parses: 151 (33 % of total) +Succesful recoveries: 195 (64 % of failed) +Failed recoveries: 100 (33 % of failed) +Parse errors: 8 (2 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/1 ms +Succesful recovery time : 2/2/4 ms +Failed recovery time : 1/1/4 ms +Parse error time : 0/0/1 ms +Recovery error size : 101/41/268 characters +Successful recovery size : 28/26/70 characters + +Overall stats +------------- +Total parses: 926 +Succesful parses: 506 (54 % of total) +Succesful recoveries: 276 (65 % of failed) +Failed recoveries: 128 (30 % of failed) +Parse errors: 16 (3 % of failed) +No slow parses. + +Statistics (avg/median/95th percentile): +Succesful parse time : 0/1/1 ms +Succesful recovery time : 2/2/5 ms +Failed recovery time : 1/1/4 ms +Parse error time : 0/0/1 ms +Recovery error size : 97/33/272 characters +Successful recovery size : 25/15/70 characters + +========================================================================== +Error recovery of |std:///lang/rascal/syntax/Rascal.rsc| (Module) on |std:///lang/rascal/vis/ImportGraph.rsc|: + +Single char deletions: +?........+...............................................................................................?.?...........+......................................................................................................................................................................................................................................................................................................................................................................................................................?.?........+....................................................................................................................................................................................................................................................................................................?.?........+..................................................................................................?.?.!......!....!..!.!.!????????..!.!.+!+......++...++.............??????+....++..........+.??????+...++......+.??????+....++......++.......++..........++.......+.??????+....++......+++......++......+.??????+.........+.??????+....++..........+.??????+....++...........+.+++++++..+.+++++++........++......++.....-..+........+..............................................................................................?.....?...........?.....................+..................+.?................+!+.......+++............++.............................++.+..+........?......................................................................................?.....?...........?.....................+..................+.?.....++...............................................................................................+.....................+................+.........++++................+!.................................++.+..+........?.................................................................................................?.....?...........?.....................+..................-!.?.....+.+................+.........+?..........++...........................................................................+....+.+!.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++!++++++.........+.......+.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+.....++............................................................................................++...........................................................................................................................................+......+.........+.........+........+.+.........++.+....+.++.+......+....++.....................+.........++++..............++++++.+.......+...++.....+......+++++++....+.........+......++.+.+.......+...++............................+.....+...+.........................+++...........+.........+.......++..........................++..........+................+?++................+++++.+..+........+........................................................................................?.+++++.............+.............+........+...+............+.+.+++........+...+...............+.+++........+...+....+.......+.+.+++........+...+....+.......+.+.+++........+...+....+.......+.++.+?..?........?..........................................................................................?.............?...............?....+...+.....+.?..............+.+.....+...+.+...++..............++.................+.+...............+.+...+............++.................?.............+...............+.+.+.+.......+...+..........++...............+.+.+.+.......+...+..........++...............+.+.+.+.......+...+..........++.................+.+.+...........+..........+.....++..........................+..................+++..+..................+.+..+..................++.......................+.?..?........?.............................................................?.............?...............?........?.?.....+++.?..................?............+....++..........+....+.....+.....+.+.................+.++..........++++++.............+...................+.+.+....++...................+.+.+....+.+.....+...................+.+.+....+.+.....+.....................+.++....+.....++.........+?.....?.....++++++..........+.+.+.............................++?.? +Total parses: 4314 +Succesful parses: 3815 (88 % of total) +Succesful recoveries: 393 (78 % of failed) +Failed recoveries: 2 (0 % of failed) +Parse errors: 104 (20 % of failed) +14 slow parses, slowest parse: |unknown:///?deleted=1137| (1 ms) + +Statistics (avg/median/95th percentile): +Succesful parse time : 14/12/20 ms +Succesful recovery time : 29/21/69 ms +Failed recovery time : 183/184/351 ms +Parse error time : 10/5/40 ms +Recovery error size : 65/19/274 characters +Successful recovery size : 59/18/233 characters + +Deletes until end-of-line: +.+........?????????????????????????????????????????????????????????????????????????????????????????????????++++++++++++.................................................................................................................................................................................................................................................................................................................................................................................................................??+++++++++...............................................................................................................................................................................................................................................................................................??+++++++++................................................................................................?.?..............????????....+?......+?...+?...........?????++++++++++++++++++.?????+++++++++++++.?????++++++++++++++++++++++++++++++++++++++++++++.?????+++++++++++++++++++++++++++++++.?????+++++++++++.?????++++++++++++++++++.?????+++++++++++++++++++.+++++++++.+++++-------------------------.+........???????????????????????????????????????????????????????????????????????????????????????????????++++++???????????+++++++++++++++++++++++++++++++++++++++++??.....+!+!+!+!+!++!+!+!+!+!+++++++++++++++++++++++++++++++++++++++++++++++++++++++++.+........???????????????????????????????????????????????????????????????????????????????????????++++++???????????+++++++++++++++++++++++++++++++++++++++++??.....+..........................................................................................+++++..++++++++++++++++++++++++++++++++++++++++++++++++++.....+!+!+!+!+!++!+!+!+!+!++++++++++++++++++++++++++++++++++++.+........????????????????????????????????????????????????????????????????????????????????????????????????????????????????????-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!??-!-!-!-!-!?????????????-!----!??.........???????????????++++++++++?.........+...........................................................................+++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.........++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....+............................................................................................+...........................................................................................................................................+++++++?????----++-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.........+++.+++++++++++++?????+++????????+?+++++++.....+++++++++++++++++++++++++++++++..+++++++++++++.....+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!++++++????+????????????????????????????????+????????????????+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++?+????????????????????+++.+........?????????????????????????????????????????????????????????????????????????????????????????+++++++++++++++++++++++++++++++++........++++.........+..+..++++.....+++++++++++++++++++++++++++.....++++++++++++++++++++++++++.....++++++++++++++++++++++++++++++++++++++++++++++++++++++??.?........????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????+++++++++++??????.......+........?++++++++++++++++++++++++++++++++...........+......????????????????????++++++++++++++++++++++++++++++++++++++++++++++...........+..+..++?+++++++++++++++++++++++++.........++++++++++?+++++++++++++++++++++++++.........++++++++++?+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....++++++++++++++++++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....+++++++++++++++++?.?........???????????????????????????????????????????????????????????????????????????????????????????????????????????????.........++++++++++++++++++++++++++++.........+++++++++++++++++++..+++++++++++++++++++++++++++++++++++++++++++++++...............+..+..++?+++++.............++++++++++?++++++++++++.............++++++++++?++++++++++++++++++++++++++++++++++++++++++++++++???????????????+++++++++++++++++++++++++.??????????????++????????????+? +Total parses: 4206 +Succesful parses: 1542 (36 % of total) +Succesful recoveries: 1611 (60 % of failed) +Failed recoveries: 55 (2 % of failed) +Parse errors: 998 (37 % of failed) +64 slow parses, slowest parse: |unknown:///?deletedUntilEol=963,996| (1 ms) + +Statistics (avg/median/95th percentile): +Succesful parse time : 13/13/21 ms +Succesful recovery time : 28/18/66 ms +Failed recovery time : 135/18/319 ms +Parse error time : 7/6/14 ms +Recovery error size : 158/40/998 characters +Successful recovery size : 116/38/561 characters + +Overall stats +------------- +Total parses: 8520 +Succesful parses: 5357 (62 % of total) +Succesful recoveries: 2004 (63 % of failed) +Failed recoveries: 57 (1 % of failed) +Parse errors: 1102 (34 % of failed) +78 slow parses, slowest parse: |unknown:///?deletedUntilEol=963,996| (1 ms) + +Statistics (avg/median/95th percentile): +Succesful parse time : 14/13/20 ms +Succesful recovery time : 28/18/67 ms +Failed recovery time : 137/18/331 ms +Parse error time : 7/6/14 ms +Recovery error size : 140/36/981 characters +Successful recovery size : 105/32/407 characters +``` diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 7beb136cfe..4c7058ba19 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -7,6 +7,7 @@ import IO; import util::Benchmark; import Grammar; import analysis::statistics::Descriptive; +import util::Math; import lang::rascal::grammar::definition::Modules; @@ -135,23 +136,30 @@ void printStats(TestStats stats) { } println(); - println("95th percentiles:"); + println("Statistics (average/median/95th percentile):"); + + void printStats(str label, list[int] values, str unit) { + int mean = sum(values)/size(values); + print(left(label, 40)); + print(": "); + println("// "); + } list[int] successfulParseTimes = [ duration | successfulParse(duration=duration) <- stats.measurements ]; list[int] successfulRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; list[int] failedRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize > stats.recoverySuccessLimit ]; list[int] parseErrorTimes = [ duration | parseError(duration=duration) <- stats.measurements ]; - println("Succesful parse time: ms"); - println("Succesful recovery time: ms"); - println("Failed recovery time: ms"); - println("Parse error time: ms"); + printStats("Succesful parse time", successfulParseTimes, "ms"); + printStats("Succesful recovery time", successfulRecoveryTimes, "ms"); + printStats("Failed recovery time", failedRecoveryTimes, "ms"); + printStats("Parse error time", parseErrorTimes, "ms"); list[int] errorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements ]; - println("Recovery error size characters"); + printStats("Recovery error size", errorSizes, "characters"); list[int] successfulErrorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; - println("Successful recovery size: characters"); + printStats("Successful recovery size", successfulErrorSizes, "characters"); } private str syntaxLocToModuleName(loc syntaxFile) { @@ -181,7 +189,7 @@ TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { int startTime = realTime(); standardParser(input, testInput); int referenceDuration = realTime() - startTime; - int slowParseLimit = referenceDuration*100; + int slowParseLimit = referenceDuration*10; int recoverySuccessLimit = size(input)/4; println(); From 06547e6dca48ca4ed2e79de17d063f3a9273475d Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 26 Sep 2024 15:52:05 +0200 Subject: [PATCH 08/21] Reintroduced 'private' --- .../lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc index ab6d77258c..26655e8edb 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc @@ -24,7 +24,7 @@ syntax T = ABC End; syntax ABC = 'a' 'b' 'c'; syntax End = "$"; -Tree parseS(str input, bool visualize=false) +private Tree parseS(str input, bool visualize=false) = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool basicOk() { From f805b579d193ed70d9f5b5dd1a68999f90fb20d7 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Fri, 27 Sep 2024 12:26:04 +0200 Subject: [PATCH 09/21] Added removed compass direction --- src/org/rascalmpl/library/lang/dot/examples/parser-state.dot | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot b/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot index e0405c03a8..183d2b050a 100644 --- a/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot +++ b/src/org/rascalmpl/library/lang/dot/examples/parser-state.dot @@ -8,7 +8,7 @@ digraph Parser { "12860" -> "-1"; "7226" -> "12860"; "-2" -> "7226"; -"todo-1":"0" -> "-2"["label"="Stack"]; +"todo-1":"0":sw -> "-2"["label"="Stack"]; "46484886"["shape"="octagon", "label"="Epsilon"]; "todo-1":"0":se -> "46484886"["label"="Node"]; "todoLists":"1" -> "todo-1"; From 6cb060be62ef4bdae96866434cfdb424bd0f48d7 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Fri, 27 Sep 2024 12:27:01 +0200 Subject: [PATCH 10/21] Added Rascal batch testing support --- .../recovery/ErrorRecoveryBenchmark.rsc | 3 ++ .../concrete/recovery/RecoveryTestSupport.rsc | 50 +++++++++++++++++-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc index a0459cc523..f6a9b292fa 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -26,4 +26,7 @@ void runLanguageTests() { testRecoveryJson(); testRecoveryPico(); testRecoveryRascal(); +} +void runRascalBatchTest() { + batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 3, 4096); } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 4c7058ba19..298f1c14c1 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -173,7 +173,10 @@ loc zippedFile(str zip, str path) { return zipFile + path; } -TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { +TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) = +testErrorRecovery(syntaxFile, topSort, testInput, readFile(testInput)); + +TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str input, int slowParseLimit=0, int recoverySuccessLimit=0) { Module \module = parse(#start[Module], syntaxFile).top; str modName = syntaxLocToModuleName(syntaxFile); Grammar gram = modules2grammar(modName, {\module}); @@ -184,13 +187,17 @@ TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { type[value] begin = type(sym, gram.rules); standardParser = parser(begin, allowAmbiguity=true, allowRecovery=false); recoveryParser = parser(begin, allowAmbiguity=true, allowRecovery=true); - str input = readFile(testInput); + if (slowParseLimit == 0) { int startTime = realTime(); standardParser(input, testInput); int referenceDuration = realTime() - startTime; - int slowParseLimit = referenceDuration*10; - int recoverySuccessLimit = size(input)/4; + slowParseLimit = referenceDuration*10; + } + + if (recoverySuccessLimit == 0) { + recoverySuccessLimit = size(input)/4; + } println(); println("Single char deletions:"); @@ -214,4 +221,37 @@ TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) { } else { throw "Cannot find top sort in "; } -} \ No newline at end of file +}TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int maxFileSize) { + int count = 0; + + int slowParseLimit = 200; + int recoverySuccessLimit = 100; + + TestStats totalStats = testStats(slowParseLimit, recoverySuccessLimit); + + println("Batch testing in directory "); + for (entry <- listEntries(dir)) { + loc file = dir + entry; + if (isFile(file)) { + if (endsWith(file.path, ext)) { + str content = readFile(file); + if (size(content) <= maxFileSize) { + TestStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content, slowParseLimit = slowParseLimit, recoverySuccessLimit = recoverySuccessLimit); + mergeStats(totalStats, fileStats); + count += 1; + } + } + } else if (isDirectory(file)) { + TestStats dirStats = batchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles-count, maxFileSize); + totalStats = mergeStats(totalStats, dirStats); + count += size(dirStats.measurements); + } + + if (count > maxFiles) { + return totalStats; + } + } + + return totalStats; +} + From a92b7cf87ed242f2dc2bfb6289df423a5d4e9b92 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Fri, 27 Sep 2024 12:33:34 +0200 Subject: [PATCH 11/21] Merge branch 'error-recovery' into recovery/testing --- .../rascalmpl/library/lang/rascal/tests/basic/Locations.rsc | 4 +++- src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc b/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc index 98415cd27c..3843d3081d 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc @@ -610,7 +610,9 @@ test bool mvnSchemeTest() { return true; -}@synopsis{Nested clones in different scopes are acceptable} +} + +@synopsis{Nested clones in different scopes are acceptable} int fun_with_clone(int n){ if(n > 0){ int h(int n) = 2*n; diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index f361aa9393..93c64e2da0 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -590,7 +590,7 @@ public int updateOvertakenNode(AbstractStackNode

predecessor, AbstractNode re // Initialize the prefixes map. int edgesMapSize = edgesMap.size(); - int possibleMaxSize = edgesMapSize + edgesMapSize; + int possibleMaxSize = edgesMapSize + edgesMapToAdd.size(); if(prefixesMap == null){ prefixesMap = new ArrayList[possibleMaxSize]; }else{ From fb147b6566c9e0b4c8fd24779f75fcaf106fff72 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Sat, 28 Sep 2024 19:22:07 +0200 Subject: [PATCH 12/21] Fixed issue with infinite recursion during parsing --- .../recovery/ErrorRecoveryBenchmark.rsc | 10 +++++++++- .../concrete/recovery/RecoveryTestSupport.rsc | 18 ++++++++++++------ .../parser/uptr/recovery/ToTokenRecoverer.java | 14 +++++++++++--- 3 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc index f6a9b292fa..20132d3593 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -2,6 +2,9 @@ module lang::rascal::tests::concrete::recovery::ErrorRecoveryBenchmark import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import IO; +import util::Benchmark; + void runTestC() { testRecoveryC(); } void runTestDiff() { testRecoveryDiff(); } void runTestDot() { testRecoveryDot(); } @@ -28,5 +31,10 @@ void runLanguageTests() { testRecoveryRascal(); } void runRascalBatchTest() { - batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 3, 4096); + int startTime = realTime(); + TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 100, 4096); + int duration = realTime() - startTime; + println("================================================================"); + println("Rascal batch test done in seconds, total result:"); + printStats(stats); } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 298f1c14c1..d5bb63e069 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -12,7 +12,7 @@ import util::Math; import lang::rascal::grammar::definition::Modules; public data TestMeasurement(loc source=|unknown:///|, int duration=0) = successfulParse() | recovered(int errorSize=0) | parseError(); -public data TestStats = testStats(int slowParseLimit, int recoverySuccessLimit, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, list[TestMeasurement] measurements=[]); +public data TestStats = testStats(int slowParseLimit, int recoverySuccessLimit, int filesTested=0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, list[TestMeasurement] measurements=[]); private TestMeasurement testRecovery(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, loc source) { int startTime = 0; @@ -73,6 +73,7 @@ TestStats updateStats(TestStats stats, TestMeasurement measurement) { TestStats mergeStats(TestStats stats1, TestStats stats2) { return testStats( stats1.slowParseLimit, stats1.recoverySuccessLimit, + filesTested = stats1.filesTested + stats2.filesTested, successfulParses = stats1.successfulParses + stats2.successfulParses, successfulRecoveries = stats1.successfulRecoveries + stats2.successfulRecoveries, failedRecoveries = stats1.failedRecoveries + stats2.failedRecoveries, @@ -139,11 +140,15 @@ void printStats(TestStats stats) { println("Statistics (average/median/95th percentile):"); void printStats(str label, list[int] values, str unit) { - int mean = sum(values)/size(values); print(left(label, 40)); print(": "); + if (values == []) { + print("-/-/- "); + } else { + int mean = sum(values)/size(values); println("// "); } + } list[int] successfulParseTimes = [ duration | successfulParse(duration=duration) <- stats.measurements ]; list[int] successfulRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; @@ -210,10 +215,10 @@ TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu TestStats deleteUntilEolStats = testDeleteUntilEol(standardParser, recoveryParser, input, slowParseLimit, recoverySuccessLimit); printStats(deleteUntilEolStats); totalStats = mergeStats(totalStats, deleteUntilEolStats); + totalStats.filesTested = 1; println(); - println("Overall stats"); - print("-------------"); + println("Cumulative stats for "); printStats(totalStats); println(); @@ -234,17 +239,18 @@ TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu loc file = dir + entry; if (isFile(file)) { if (endsWith(file.path, ext)) { + println("Testing file ( left)"); str content = readFile(file); if (size(content) <= maxFileSize) { TestStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content, slowParseLimit = slowParseLimit, recoverySuccessLimit = recoverySuccessLimit); - mergeStats(totalStats, fileStats); + totalStats = mergeStats(totalStats, fileStats); count += 1; } } } else if (isDirectory(file)) { TestStats dirStats = batchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles-count, maxFileSize); totalStats = mergeStats(totalStats, dirStats); - count += size(dirStats.measurements); + count += dirStats.filesTested; } if (count > maxFiles) { diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 16759bd825..9bdb300ec6 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -52,6 +52,8 @@ public class ToTokenRecoverer implements IRecoverer { private IdDispenser stackNodeIdDispenser; private ExpectsProvider expectsProvider; + private Set processedNodes = new HashSet<>(); + public ToTokenRecoverer(URI uri, ExpectsProvider expectsProvider, IdDispenser stackNodeIdDispenser) { this.uri = uri; this.expectsProvider = expectsProvider; @@ -67,7 +69,6 @@ public DoubleArrayList, AbstractNode> reviveStac // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to // improve error recovery. - ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); @@ -346,6 +347,14 @@ private DoubleArrayList, AbstractNode> reviveFai new DoubleArrayList<>(); for (int i = failedNodes.size() - 1; i >= 0; --i) { + AbstractStackNode failedNode = failedNodes.get(i); + + // Protect against endless loop + long id = (long) failedNode.getId() << 32 | failedNode.getStartLocation(); + if (!processedNodes.add(id)) { + continue; + } + findRecoveryNodes(failedNodes.get(i), recoveryNodes); } @@ -368,7 +377,7 @@ private static void collectUnexpandableNodes(Stack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes) { for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { @@ -377,7 +386,6 @@ private static void collectUnmatchableMidProductionNodes(int location, AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference // updates of the static version - // Merge the information on the predecessors into the failed node. for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); From 3047dde81f492fb8ad5a3cfb45689ac779c33c55 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Sat, 28 Sep 2024 19:41:26 +0200 Subject: [PATCH 13/21] Merge branch 'error-recovery' into recovery/testing --- .../rascalmpl/library/lang/rascal/tests/basic/Locations.rsc | 1 - .../rascal/tests/concrete/recovery/BasicRecoveryTests.rsc | 6 +++--- .../rascal/tests/concrete/recovery/RascalRecoveryTests.rsc | 2 ++ 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc b/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc index 3843d3081d..67e7bad9d5 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/basic/Locations.rsc @@ -611,7 +611,6 @@ test bool mvnSchemeTest() { return true; } - @synopsis{Nested clones in different scopes are acceptable} int fun_with_clone(int n){ if(n > 0){ diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc index 26655e8edb..23b3af5d2f 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc @@ -33,15 +33,15 @@ test bool basicOk() { test bool abx() { Tree t = parseS("a b x $"); - return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x "; + return getErrorText(findBestError(t)) == "x "; } test bool axc() { Tree t = parseS("a x c $"); - return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x c"; + return getErrorText(findBestError(t)) == "x c"; } test bool ax() { Tree t = parseS("a x $"); - return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x "; + return getErrorText(findBestError(t)) == "x "; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc index c2a5b02701..9e0332422c 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc @@ -125,6 +125,7 @@ test bool rascalIfBodyEmpty() { } // Not working yet: +/* test bool rascalMissingOpeningParen() { Tree t = parseRascal("module A void f){} void g() { }"); @@ -159,3 +160,4 @@ test bool rascalIfMissingSemi() { println("error text: "); return getErrorText(findFirstError(t)) == ";"; } +*/ \ No newline at end of file From 020b4f3dc9faf11e7449da74d5a725d6f3037f41 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 30 Sep 2024 08:51:02 +0200 Subject: [PATCH 14/21] Improved benchmarking, both in speed and stat quality --- .../recovery/ErrorRecoveryBenchmark.rsc | 17 +- .../concrete/recovery/RecoveryTestSupport.rsc | 279 ++++++++++++------ 2 files changed, 199 insertions(+), 97 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc index 20132d3593..442420e60e 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -13,13 +13,13 @@ void runTestJson() { testRecoveryJson(); } void runTestPico() { testRecoveryPico(); } void runTestRascal() { testRecoveryRascal(); } -TestStats testRecoveryC() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); -TestStats testRecoveryDiff() = testErrorRecovery(|std:///lang/diff/unified/UnifiedDiff.rsc|, "DiffFile", |std:///lang/diff/unified/examples/example.diff|); -TestStats testRecoveryDot() = testErrorRecovery(|std:///lang/dot/syntax/Dot.rsc|, "DOT", |std:///lang/dot/examples/parser-state.dot|); -TestStats testRecoveryJava() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); -TestStats testRecoveryJson() = testErrorRecovery(|std:///lang/json/syntax/JSON.rsc|, "JSONText", |std:///lang/json/examples/ex01.json|); -TestStats testRecoveryPico() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); -TestStats testRecoveryRascal() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); +FileStats testRecoveryC() = testErrorRecovery(|std:///lang/c90/syntax/C.rsc|, "TranslationUnit", |std:///lang/c90/examples/hello-world.c|); +FileStats testRecoveryDiff() = testErrorRecovery(|std:///lang/diff/unified/UnifiedDiff.rsc|, "DiffFile", |std:///lang/diff/unified/examples/example.diff|); +FileStats testRecoveryDot() = testErrorRecovery(|std:///lang/dot/syntax/Dot.rsc|, "DOT", |std:///lang/dot/examples/parser-state.dot|); +FileStats testRecoveryJava() = testErrorRecovery(|std:///lang/java/syntax/Java15.rsc|, "CompilationUnit", zippedFile("m3/snakes-and-ladders-project-source.zip", "src/snakes/LastSquare.java")); +FileStats testRecoveryJson() = testErrorRecovery(|std:///lang/json/syntax/JSON.rsc|, "JSONText", |std:///lang/json/examples/ex01.json|); +FileStats testRecoveryPico() = testErrorRecovery(|std:///lang/pico/syntax/Main.rsc|, "Program", |std:///lang/pico/examples/fac.pico|); +FileStats testRecoveryRascal() = testErrorRecovery(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///lang/rascal/vis/ImportGraph.rsc|); void runLanguageTests() { testRecoveryC(); @@ -32,8 +32,9 @@ void runLanguageTests() { } void runRascalBatchTest() { int startTime = realTime(); - TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 100, 4096); + TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 1000, 10000); int duration = realTime() - startTime; + println(); println("================================================================"); println("Rascal batch test done in seconds, total result:"); printStats(stats); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index d5bb63e069..4ee8d9da24 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -8,11 +8,17 @@ import util::Benchmark; import Grammar; import analysis::statistics::Descriptive; import util::Math; +import Set; +import List; import lang::rascal::grammar::definition::Modules; +alias FrequencyTable = map[int val, int count]; + public data TestMeasurement(loc source=|unknown:///|, int duration=0) = successfulParse() | recovered(int errorSize=0) | parseError(); -public data TestStats = testStats(int slowParseLimit, int recoverySuccessLimit, int filesTested=0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, list[TestMeasurement] measurements=[]); +public data FileStats = fileStats(int totalParses = 0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, FrequencyTable parseTimeRatios=()); + +public data TestStats = testStats(int filesTested=0, int testCount=0, FrequencyTable successfulParsePct=(), FrequencyTable successfulRecoveryPct=(), FrequencyTable failedRecoveryPct=(), FrequencyTable parseErrorPct=(), FrequencyTable slowParsePct=(), FrequencyTable parseTimeRatios=()); private TestMeasurement testRecovery(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, loc source) { int startTime = 0; @@ -40,65 +46,122 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser return measurement; } -TestStats updateStats(TestStats stats, TestMeasurement measurement) { +FileStats updateStats(FileStats stats, TestMeasurement measurement, int referenceParseTime, int recoverySuccessLimit) { + stats.totalParses += 1; + + int ratio = measurement.duration/referenceParseTime; + int parseTimeRatio = ratio == 0 ? 0 : round(log2(measurement.duration/referenceParseTime)); + switch (measurement) { case successfulParse(): { print("."); stats.successfulParses += 1; } - case recovered(errorSize=errorSize): - if (errorSize <= stats.recoverySuccessLimit) { + case recovered(errorSize=errorSize): { + stats.parseTimeRatios = increment(stats.parseTimeRatios, parseTimeRatio); + if (errorSize <= recoverySuccessLimit) { print("+"); stats.successfulRecoveries += 1; } else { print("-"); stats.failedRecoveries += 1; } + } case parseError(): { + stats.parseTimeRatios = increment(stats.parseTimeRatios, parseTimeRatio); print("?"); stats.parseErrors += 1; } } - if (measurement.duration > stats.slowParseLimit) { + if (measurement.duration > referenceParseTime*10) { print("!"); stats.slowParses += 1; } - stats.measurements = stats.measurements + measurement; - return stats; } -TestStats mergeStats(TestStats stats1, TestStats stats2) { - return testStats( - stats1.slowParseLimit, stats1.recoverySuccessLimit, - filesTested = stats1.filesTested + stats2.filesTested, +FileStats mergeFileStats(FileStats stats1, FileStats stats2) { + return fileStats( + totalParses = stats1.totalParses + stats2.totalParses, successfulParses = stats1.successfulParses + stats2.successfulParses, successfulRecoveries = stats1.successfulRecoveries + stats2.successfulRecoveries, failedRecoveries = stats1.failedRecoveries + stats2.failedRecoveries, parseErrors = stats1.parseErrors + stats2.parseErrors, slowParses = stats1.slowParses + stats2.slowParses, - measurements = stats1.measurements + stats2.measurements); + parseTimeRatios = mergeFrequencyTables(stats1.parseTimeRatios, stats2.parseTimeRatios) + ); +} + +FrequencyTable increment(FrequencyTable frequencyTable, int val) { + if (val in frequencyTable) { + frequencyTable[val] += 1; + } else { + frequencyTable[val] = 1; + } + + return frequencyTable; +} + +TestStats consolidateStats(TestStats cumulativeStats, FileStats fileStats) { + int totalFailed = fileStats.totalParses - fileStats.successfulParses; + + cumulativeStats.successfulParsePct = increment(cumulativeStats.successfulParsePct, percentage(fileStats.successfulParses, fileStats.totalParses)); + cumulativeStats.successfulRecoveryPct = increment(cumulativeStats.successfulRecoveryPct, percentage(fileStats.successfulRecoveries, totalFailed)); + cumulativeStats.failedRecoveryPct = increment(cumulativeStats.failedRecoveryPct, percentage(fileStats.failedRecoveries, totalFailed)); + cumulativeStats.parseErrorPct = increment(cumulativeStats.parseErrorPct, percentage(fileStats.parseErrors, totalFailed)); + cumulativeStats.slowParsePct = increment(cumulativeStats.slowParsePct, percentage(fileStats.slowParses, totalFailed)); + cumulativeStats.parseTimeRatios = mergeFrequencyTables(cumulativeStats.parseTimeRatios, fileStats.parseTimeRatios); + + cumulativeStats.filesTested += 1; + cumulativeStats.testCount += fileStats.totalParses; + + return cumulativeStats; } -TestStats testSingleCharDeletions(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int slowParseLimit, int recoverySuccessLimit) { - TestStats stats = testStats(slowParseLimit, recoverySuccessLimit); +map[int,int] mergeFrequencyTables(map[int,int] hist1, map[int,int] hist2) { + for (int pct <- hist2) { + if (pct in hist1) { + hist1[pct] += hist2[pct]; + } else { + hist1[pct] = hist2[pct]; + } + } + + return hist1; +} + +TestStats mergeStats(TestStats stats, TestStats stats2) { + stats.filesTested += stats2.filesTested; + stats.testCount += stats2.testCount; + stats.successfulParsePct = mergeFrequencyTables(stats.successfulParsePct, stats2.successfulParsePct); + stats.successfulRecoveryPct = mergeFrequencyTables(stats.successfulRecoveryPct, stats2.successfulRecoveryPct); + stats.failedRecoveryPct = mergeFrequencyTables(stats.failedRecoveryPct, stats2.failedRecoveryPct); + stats.parseErrorPct = mergeFrequencyTables(stats.parseErrorPct, stats2.parseErrorPct); + stats.slowParsePct = mergeFrequencyTables(stats.slowParsePct, stats2.slowParsePct); + stats.parseTimeRatios = mergeFrequencyTables(stats.parseTimeRatios, stats2.parseTimeRatios); + + return stats; +} + +FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit) { + FileStats stats = fileStats(); int len = size(input); int i = 0; while (i < len) { str modifiedInput = substring(input, 0, i) + substring(input, i+1); TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deleted=<"">|); - stats = updateStats(stats, measurement); + stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); i = i+1; } return stats; } -TestStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int slowParseLimit, int recoverySuccessLimit) { - TestStats stats = testStats(slowParseLimit, recoverySuccessLimit); +FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit) { + FileStats stats = fileStats(); int lineStart = 0; list[int] lineEndings = findAll(input, "\n"); @@ -107,7 +170,7 @@ TestStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (va for (int pos <- [lineStart..lineEnd]) { modifiedInput = substring(input, 0, pos) + substring(input, lineEnd); TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deletedUntilEol=<",">|); - stats = updateStats(stats, measurement); + stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); } lineStart = lineEnd+1; } @@ -116,55 +179,101 @@ TestStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (va } private int percentage(int number, int total) { - return (100*number)/total; + return total == 0 ? 0 : (100*number)/total; } -void printStats(TestStats stats) { - println(); - int measurementCount = size(stats.measurements); - println("Total parses: "); - println("Succesful parses: ( % of total)"); - int totalFailed = measurementCount - stats.successfulParses; - println("Succesful recoveries: ( % of failed)"); - println("Failed recoveries: ( % of failed)"); - println("Parse errors: ( % of failed)"); - - if (stats.slowParses == 0) { - println("No slow parses."); - } else { - slowest = (getFirstFrom(stats.measurements) | it.duration < e.duration ? it : e | e <- stats.measurements); - println(" slow parses, slowest parse: ( ms)"); +int statLabelWidth = 40; +int statFieldWidth = 7; + + +void printFileStats(FileStats fileStats) { + void printStat(str label, int stat, int total, bool printPct=true) { + int pct = total == 0 ? 0 : stat*100/total; + print(left(label + ":", statLabelWidth)); + str pctStr = printPct ? " ( %)" : ""; + println(left("", statFieldWidth)); } println(); - println("Statistics (average/median/95th percentile):"); + printStat("Total parses", fileStats.totalParses, fileStats.totalParses); + printStat("Successful parses", fileStats.successfulParses, fileStats.totalParses); + int failedParses = fileStats.totalParses - fileStats.successfulParses; + printStat("Successful recoveries", fileStats.successfulRecoveries, failedParses); + printStat("Failed recoveries", fileStats.failedRecoveries, failedParses); + printStat("Parse errors", fileStats.parseErrors, failedParses); + printStat("Slow parses", fileStats.slowParses, failedParses); + printFrequencyTableHeader(); + printFrequencyTableStats("Parse time ratios", fileStats.parseTimeRatios, unit = "log2/%"); +} - void printStats(str label, list[int] values, str unit) { - print(left(label, 40)); - print(": "); - if (values == []) { - print("-/-/- "); - } else { - int mean = sum(values)/size(values); - println("// "); +void printFrequencyTableHeader() { + print(left("", statLabelWidth+1)); + print(right("mean", statFieldWidth)); + print(right("median", statFieldWidth)); + print(right("95 %", statFieldWidth)); + print(right("min", statFieldWidth)); + println(right("max", statFieldWidth)); } + +void printFrequencyTableStats(str label, FrequencyTable frequencyTable, str unit = "%") { + print(left(label + " ():", statLabelWidth)); + + int totalCount = (0 | it+frequencyTable[val] | val <- frequencyTable); + + int total = 0; + int median = 0; + int medianCount = 0; + int cumulativeCount = 0; + int ninetyFivePercentileLimit = round(totalCount * 0.95); + int ninetyFivePercentile = -1; + int minVal = 1000000000; + int maxVal = -1000000000; + + for (val <- sort(toList(frequencyTable.val))) { + minVal = min(minVal, val); + maxVal = max(maxVal, val); + + int count = frequencyTable[val]; + cumulativeCount += count; + + if (ninetyFivePercentile == -1 && cumulativeCount >= ninetyFivePercentileLimit) { + ninetyFivePercentile = val; } - list[int] successfulParseTimes = [ duration | successfulParse(duration=duration) <- stats.measurements ]; - list[int] successfulRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; - list[int] failedRecoveryTimes = [ duration | recovered(duration=duration, errorSize=errorSize) <- stats.measurements, errorSize > stats.recoverySuccessLimit ]; - list[int] parseErrorTimes = [ duration | parseError(duration=duration) <- stats.measurements ]; + total += val*count; - printStats("Succesful parse time", successfulParseTimes, "ms"); - printStats("Succesful recovery time", successfulRecoveryTimes, "ms"); - printStats("Failed recovery time", failedRecoveryTimes, "ms"); - printStats("Parse error time", parseErrorTimes, "ms"); + if (count > medianCount) { + medianCount = count; + median = val; + } + } - list[int] errorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements ]; - printStats("Recovery error size", errorSizes, "characters"); + if (totalCount == 0) { + print("-"); + } else { + int mean = total/totalCount; + print(right("", statFieldWidth)); + print(right("", statFieldWidth)); + print(right("", statFieldWidth)); + print(right("", statFieldWidth)); + println(right("", statFieldWidth)); + } +} + +void printStats(TestStats stats) { + if (stats.filesTested != 1) { + println("Files tested: "); + } + println("Total parses: "); + printFrequencyTableHeader(); + printFrequencyTableStats("Succesful parses", stats.successfulParsePct); + printFrequencyTableStats("Succesful recoveries", stats.successfulRecoveryPct); + printFrequencyTableStats("Failed recoveries", stats.failedRecoveryPct); + printFrequencyTableStats("Parse errors", stats.parseErrorPct); + printFrequencyTableStats("Slow parses", stats.slowParsePct); + printFrequencyTableStats("Parse time ratios", stats.parseTimeRatios, unit = "log2/%"); - list[int] successfulErrorSizes = [ errorSize | recovered(errorSize=errorSize) <- stats.measurements, errorSize <= stats.recoverySuccessLimit ]; - printStats("Successful recovery size", successfulErrorSizes, "characters"); + println(); } private str syntaxLocToModuleName(loc syntaxFile) { @@ -178,86 +287,78 @@ loc zippedFile(str zip, str path) { return zipFile + path; } -TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) = -testErrorRecovery(syntaxFile, topSort, testInput, readFile(testInput)); +FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) = testErrorRecovery(syntaxFile, topSort, testInput, readFile(testInput)); -TestStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str input, int slowParseLimit=0, int recoverySuccessLimit=0) { +FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str input) { Module \module = parse(#start[Module], syntaxFile).top; str modName = syntaxLocToModuleName(syntaxFile); Grammar gram = modules2grammar(modName, {\module}); if (sym:\start(\sort(topSort)) <- gram.starts) { - println("=========================================================================="); println("Error recovery of () on :"); type[value] begin = type(sym, gram.rules); standardParser = parser(begin, allowAmbiguity=true, allowRecovery=false); recoveryParser = parser(begin, allowAmbiguity=true, allowRecovery=true); - if (slowParseLimit == 0) { int startTime = realTime(); standardParser(input, testInput); - int referenceDuration = realTime() - startTime; - slowParseLimit = referenceDuration*10; - } + int referenceParseTime = realTime() - startTime; - if (recoverySuccessLimit == 0) { recoverySuccessLimit = size(input)/4; - } println(); println("Single char deletions:"); - TestStats singleCharDeletionStats = testSingleCharDeletions(standardParser, recoveryParser, input, slowParseLimit, recoverySuccessLimit); - printStats(singleCharDeletionStats); - TestStats totalStats = singleCharDeletionStats; + FileStats singleCharDeletionStats = testSingleCharDeletions(standardParser, recoveryParser, input, referenceParseTime, recoverySuccessLimit); + printFileStats(singleCharDeletionStats); println(); println("Deletes until end-of-line:"); - TestStats deleteUntilEolStats = testDeleteUntilEol(standardParser, recoveryParser, input, slowParseLimit, recoverySuccessLimit); - printStats(deleteUntilEolStats); - totalStats = mergeStats(totalStats, deleteUntilEolStats); - totalStats.filesTested = 1; + FileStats deleteUntilEolStats = testDeleteUntilEol(standardParser, recoveryParser, input, referenceParseTime, recoverySuccessLimit); + printFileStats(deleteUntilEolStats); + FileStats stats = mergeFileStats(singleCharDeletionStats, deleteUntilEolStats); println(); - println("Cumulative stats for "); - printStats(totalStats); - println(); - - return totalStats; + println("-----------------------------------------------------------"); + println("Total test stats for :"); + printFileStats(stats); + return stats; } else { throw "Cannot find top sort in "; } }TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int maxFileSize) { int count = 0; - int slowParseLimit = 200; - int recoverySuccessLimit = 100; - - TestStats totalStats = testStats(slowParseLimit, recoverySuccessLimit); + TestStats cumulativeStats = testStats(); - println("Batch testing in directory

"); + println("Batch testing in directory (maxFiles=, maxFileSize=)"); for (entry <- listEntries(dir)) { loc file = dir + entry; if (isFile(file)) { if (endsWith(file.path, ext)) { - println("Testing file ( left)"); str content = readFile(file); if (size(content) <= maxFileSize) { - TestStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content, slowParseLimit = slowParseLimit, recoverySuccessLimit = recoverySuccessLimit); - totalStats = mergeStats(totalStats, fileStats); + println("========================================================================"); + println("Testing file ( left)"); + FileStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content); + cumulativeStats = consolidateStats(cumulativeStats, fileStats); + println(); + println("------------------------------------------------------------------------"); + println("Cumulative stats after testing :"); + printStats(cumulativeStats); count += 1; } } } else if (isDirectory(file)) { TestStats dirStats = batchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles-count, maxFileSize); - totalStats = mergeStats(totalStats, dirStats); + cumulativeStats = mergeStats(cumulativeStats, dirStats); count += dirStats.filesTested; } - if (count > maxFiles) { - return totalStats; + if (count >= maxFiles) { + break; } } - return totalStats; + return cumulativeStats; } From 8fa0d78bf5c7400f6b51b5d31cba3f3dadc682ec Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 30 Sep 2024 13:09:58 +0200 Subject: [PATCH 15/21] Fixed cumulative stats display --- .../recovery/ErrorRecoveryBenchmark.rsc | 21 ++++++++++++++++--- .../concrete/recovery/RecoveryTestSupport.rsc | 17 ++++++--------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc index 442420e60e..85a69c60a7 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -4,6 +4,8 @@ import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; import IO; import util::Benchmark; +import String; +import List; void runTestC() { testRecoveryC(); } void runTestDiff() { testRecoveryDiff(); } @@ -30,12 +32,25 @@ void runLanguageTests() { testRecoveryPico(); testRecoveryRascal(); } -void runRascalBatchTest() { + +void runRascalBatchTest(int maxFiles=1000, int maxFileSize=4000) { int startTime = realTime(); - TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", 1000, 10000); + TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", maxFiles, maxFileSize); int duration = realTime() - startTime; println(); - println("================================================================"); + println("========================im========================================"); println("Rascal batch test done in seconds, total result:"); printStats(stats); +}int main(list[str] args) { + int maxFiles = 1000; + int maxFileSize = 4000; + if (size(args) == 2) { + maxFiles = toInt(args[0]); + maxFileSize = toInt(args[1]); + } else if (size(args) != 0) { + println("Usage: ErrorRecoveryBenchmark "); + } + + runRascalBatchTest(maxFiles=maxFiles, maxFileSize=maxFileSize); + return 0; } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 4ee8d9da24..cd50959a9b 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -203,7 +203,7 @@ void printFileStats(FileStats fileStats) { printStat("Parse errors", fileStats.parseErrors, failedParses); printStat("Slow parses", fileStats.slowParses, failedParses); printFrequencyTableHeader(); - printFrequencyTableStats("Parse time ratios", fileStats.parseTimeRatios, unit = "log2/%"); + printFrequencyTableStats("Parse time ratios", fileStats.parseTimeRatios, unit = "log2(ratio)"); } void printFrequencyTableHeader() { @@ -325,11 +325,9 @@ FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu } else { throw "Cannot find top sort in "; } -}TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int maxFileSize) { - int count = 0; - - TestStats cumulativeStats = testStats(); +} +TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int maxFileSize, TestStats cumulativeStats=testStats()) { println("Batch testing in directory (maxFiles=, maxFileSize=)"); for (entry <- listEntries(dir)) { loc file = dir + entry; @@ -338,23 +336,20 @@ FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu str content = readFile(file); if (size(content) <= maxFileSize) { println("========================================================================"); - println("Testing file ( left)"); + println("Testing file # ( of left)"); FileStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content); cumulativeStats = consolidateStats(cumulativeStats, fileStats); println(); println("------------------------------------------------------------------------"); println("Cumulative stats after testing :"); printStats(cumulativeStats); - count += 1; } } } else if (isDirectory(file)) { - TestStats dirStats = batchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles-count, maxFileSize); - cumulativeStats = mergeStats(cumulativeStats, dirStats); - count += dirStats.filesTested; + cumulativeStats = batchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles, maxFileSize, cumulativeStats=cumulativeStats); } - if (count >= maxFiles) { + if (cumulativeStats.filesTested >= maxFiles) { break; } } From 4c41029f1d0f775faf05a4819bd4f732427cf743 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 30 Sep 2024 13:14:48 +0200 Subject: [PATCH 16/21] Fixed division-by-zero when reference parse time happens to be 0 --- .../rascal/tests/concrete/recovery/RecoveryTestSupport.rsc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index cd50959a9b..d06badfd34 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -49,8 +49,8 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser FileStats updateStats(FileStats stats, TestMeasurement measurement, int referenceParseTime, int recoverySuccessLimit) { stats.totalParses += 1; - int ratio = measurement.duration/referenceParseTime; - int parseTimeRatio = ratio == 0 ? 0 : round(log2(measurement.duration/referenceParseTime)); + int ratio = referenceParseTime == 0 ? measurement.duration : measurement.duration/referenceParseTime; + int parseTimeRatio = ratio == 0 ? 0 : round(log2(ratio)); switch (measurement) { case successfulParse(): { From e85ecfded7ed9f152e2319e51254790d69dbd425 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 09:47:20 +0200 Subject: [PATCH 17/21] Added bug tests and some support for selective testing --- .gitattributes | 1 + .../concrete/recovery/RecoveryTestSupport.rsc | 17 ++++++-- .../recovery/bugs/InfiniteLoop2Bug.rsc | 14 +++++++ .../recovery/bugs/InfiniteLoopBug.rsc | 20 ++++++++++ .../recovery/bugs/InfiniteLoopInput.txt | 3 ++ .../concrete/recovery/bugs/OvertakeBug.rsc | 11 ++++++ .../recovery/bugs/OvertakeBugInput.txt | 39 +++++++++++++++++++ 7 files changed, 102 insertions(+), 3 deletions(-) create mode 100644 .gitattributes create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoop2Bug.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopBug.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopInput.txt create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBug.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBugInput.txt diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..2ce24899f2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.dot diff=-astextplain diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index d06badfd34..6f606992f6 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -33,6 +33,7 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser startTime = realTime(); try { Tree t = recoveryParser(input, source); + rprintln(t); duration = realTime() - startTime; Tree best = findBestError(t); errorSize = size(getErrorText(best)); @@ -160,14 +161,23 @@ FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, & return stats; } -FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit) { +FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit, int begin=0, int end=-1) { FileStats stats = fileStats(); - int lineStart = 0; + int lineStart = begin; list[int] lineEndings = findAll(input, "\n"); for (int lineEnd <- lineEndings) { lineLength = lineEnd - lineStart; for (int pos <- [lineStart..lineEnd]) { + + // Check boundaries (only used for quick bug testing) + if (end != -1 && end < pos) { + return stats; + } + if (pos < begin) { + continue; + } + modifiedInput = substring(input, 0, pos) + substring(input, lineEnd); TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deletedUntilEol=<",">|); stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); @@ -295,7 +305,6 @@ FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu Grammar gram = modules2grammar(modName, {\module}); if (sym:\start(\sort(topSort)) <- gram.starts) { - println("Error recovery of () on :"); type[value] begin = type(sym, gram.rules); standardParser = parser(begin, allowAmbiguity=true, allowRecovery=false); recoveryParser = parser(begin, allowAmbiguity=true, allowRecovery=true); @@ -306,6 +315,8 @@ FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu recoverySuccessLimit = size(input)/4; + println("Error recovery of () on , reference parse time: ms."); + println(); println("Single char deletions:"); FileStats singleCharDeletionStats = testSingleCharDeletions(standardParser, recoveryParser, input, referenceParseTime, recoverySuccessLimit); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoop2Bug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoop2Bug.rsc new file mode 100644 index 0000000000..4beae613ec --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoop2Bug.rsc @@ -0,0 +1,14 @@ +module lang::rascal::tests::concrete::recovery::bugs::InfiniteLoop2Bug + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; + +void testBug() { + standardParser = parser(#start[Module], allowRecovery=false, allowAmbiguity=true); + recoveryParser = parser(#start[Module], allowRecovery=true, allowAmbiguity=true); + input = readFile(|std:///lang/rascal/grammar/tests/TestGrammars.rsc|); + testDeleteUntilEol(standardParser, recoveryParser, input, 200, 100, begin=278, end=278); +} + diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopBug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopBug.rsc new file mode 100644 index 0000000000..8710b829d5 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopBug.rsc @@ -0,0 +1,20 @@ +module lang::rascal::tests::concrete::recovery::bugs::InfiniteLoopBug + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; + +void testInfiniteLoop1() { + str input = readFile(|std:///lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopInput.txt|); + p = parser(#start[Module], allowRecovery=true, allowAmbiguity=true); + println("starting parse"); + p(input, |unknown:///?visualize=false|); +} + +void testInfiniteLoop2() { + standardParser = parser(#start[Module], allowRecovery=false, allowAmbiguity=true); + recoveryParser = parser(#start[Module], allowRecovery=true, allowAmbiguity=true); + input = readFile(|std:///analysis/m3/FlowGraph.rsc|); + testDeleteUntilEol(standardParser, recoveryParser, input, 200, 100); +} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopInput.txt b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopInput.txt new file mode 100644 index 0000000000..26359b3e55 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoopInput.txt @@ -0,0 +1,3 @@ +module M +alias G = rel[loc, set[E +data P = t() | f(); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBug.rsc new file mode 100644 index 0000000000..2880cef3a3 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBug.rsc @@ -0,0 +1,11 @@ +module lang::rascal::tests::concrete::recovery::bugs::OvertakeBug + +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; + +bool testOvertakeBug() { + str input = readFile(|std:///lang/rascal/tests/concrete/recovery/bugs/OvertakeBugInput.txt|); + parser(#Module, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=false|); + return true; +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBugInput.txt b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBugInput.txt new file mode 100644 index 0000000000..a6821039bc --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakeBugInput.txt @@ -0,0 +1,39 @@ +module analysis::diff::edits::ExecuteTextEdits + +extend analysis::diff::edits::TextEdits; +import IO; +import String; +import List; + +void executeDocumentEdits(list[DocumentEdit] edits) { + for (e <- edits) { + executeDocumentEdit(e); + } +} + +void executeDocumentEdit(removed(loc f)) { + remove(f.top; +} + +void executeDocumentEdit(created(loc f)) { + writeFile(f, ""); +} + +void executeDocumentEdit(renamed(loc from, loc to)) { + move(from.top, to.top, overwrite=true); +} + +void executeDocumentEdit(changed(loc file, list[TextEdit] edits)) { + assert isSorted(edits, less=bool (TextEdit e1, TextEdit e2) { + return e1.range.offset < e2.range.offset; + }); + + str content = readFile(file); + + for (replace(loc range, str repl) <- reverse(edits)) { + assert range.top == file.top; + content = ""; + } + + writeFile(file.top, content); +} \ No newline at end of file From 2b8e20517f0431f188a6cbb808d9cb2b935003af Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 09:56:44 +0200 Subject: [PATCH 18/21] Removed unnecessary changes --- .../library/lang/dot/examples/parse-state.dot | 46 ------------------- .../uptr/recovery/ToTokenRecoverer.java | 2 +- 2 files changed, 1 insertion(+), 47 deletions(-) delete mode 100644 src/org/rascalmpl/library/lang/dot/examples/parse-state.dot diff --git a/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot b/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot deleted file mode 100644 index bda91c2557..0000000000 --- a/src/org/rascalmpl/library/lang/dot/examples/parse-state.dot +++ /dev/null @@ -1,46 +0,0 @@ -digraph Parser { -"Parser"["label"="Parser\nInput: \"void f(){if(1){}}\"\nLocation: 0 ('v')\nStep 2: Reducing terminals"]; -"todo-1"["label"="<0> 0", "shape"="record"]; -"-2"["label"="Epsilon: \n.0@0 ,matchable,end\n?\nin: 'sort(\"Tag\") -> regular(\iter-star-seps(sort(\"Tag\"),[layouts(\"LAYOUTLIST\")]))'"]; -"124"["label"="SeparatedList: 124\n.0@0 ,expandable,end\n124\nin: 'default -> tags'"]; -"12858"["label"="NonTerminal: Tags\n.0@0 \nTags\nin: Tags Visibility Signature '=' Expression 'when' 12878 ';'"]; -"-1"["label"="NonTerminal: FunctionDeclaration\n.0@-1 \nFunctionDeclaration"]; -"12858" -> "-1"; -"124" -> "12858"; -"-2" -> "124"; -"todo-1":"0":sw -> "-2"["label"="Stack"]; -"46484886"["shape"="octagon", "label"="Epsilon"]; -"todo-1":"0":se -> "46484886"["label"="Node"]; -"todoLists":"1" -> "todo-1"; -"todoLists"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3 | <4> 4 | <5> 5 | <6> 6 | <7> 7 | <8> 8 | <9> 9 | <10> 10 | <11> 11 | <12> 12 | <13> 13 | <14> 14 | <15> 15", "shape"="record"]; -"Parser" -> "todoLists"["label"="todo lists"]; -"stacksToExpand"["label"="", "shape"="record"]; -"Parser" -> "stacksToExpand"["label"="stacks to expand"]; -"terminalsToReduce"["label"="<0> 0", "shape"="record", "color"="red"]; -"terminalsToReduce":"0":sw -> "-2"["label"="Stack"]; -"terminalsToReduce":"0":se -> "46484886"["label"="Node"]; -"Parser" -> "terminalsToReduce"["label"="terminals to reduce"]; -"nonTerminalsToReduce"["label"="", "shape"="record"]; -"Parser" -> "nonTerminalsToReduce"["label"="non-terminals to reduce"]; -"122"["label"="NonTerminal: Tag\n.0@0 ,end\nTag\nin: 'sort(\"Tag\") -> regular(\iter-star-seps(sort(\"Tag\"),[layouts(\"LAYOUTLIST\")]))'"]; -"122" -> "124"; -"unexpandableNodes":"0" -> "122"; -"unexpandableNodes"["label"="<0> 0", "shape"="record"]; -"12824"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"FunctionDeclaration\")' ':' 12828 0"]; -"unmatchableLeafNodes":"0" -> "12824"; -"128"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 'sort(\"Tags\")' ':' 132 0"]; -"unmatchableLeafNodes":"1" -> "128"; -"2043"["label"="Literal: \n.0@-1 ,matchable\n'@'\nin: '@' Name '=' Expression"]; -"unmatchableLeafNodes":"2" -> "2043"; -"2065"["label"="Char: \n.0@-1 ,matchable\n0\nin: 0 '\iter-star(sort(\"Tag\"))' ':' 2069 0"]; -"unmatchableLeafNodes":"3" -> "2065"; -"unmatchableLeafNodes"["label"="<0> 0 | <1> 1 | <2> 2 | <3> 3", "shape"="record"]; -"unmatchableMidProductionNodes"["shape"="record", "label"=""]; -"filteredNodes"["label"="", "shape"="record"]; -"error"["label"="Errors"]; -"Parser" -> "error"["label"="error tracking"]; -"error" -> "unexpandableNodes"["label"="unexpandable"]; -"error" -> "unmatchableLeafNodes"["label"="unmatchable leafs"]; -"error" -> "unmatchableMidProductionNodes"["label"="unmatchable mid-prod"]; -"error" -> "filteredNodes"["label"="filtered"]; -} diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 9bdb300ec6..992cf8fd3a 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -377,7 +377,7 @@ private static void collectUnexpandableNodes(Stack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes) { for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { From 883d864f541c1717c82c4b3856480bf08df25a30 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 09:58:28 +0200 Subject: [PATCH 19/21] Removed unnecessary changes --- src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 992cf8fd3a..45efd546c3 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -69,6 +69,7 @@ public DoubleArrayList, AbstractNode> reviveStac // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to // improve error recovery. + ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); @@ -386,6 +387,7 @@ private static void collectUnmatchableMidProductionNodes(int location, AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference // updates of the static version + // Merge the information on the predecessors into the failed node. for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); From b223f07061d425c548a9451b21a7b16bf55cb71c Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 10:35:00 +0200 Subject: [PATCH 20/21] Printing a newline after each line of input that is processed --- .../rascal/tests/concrete/recovery/NOTES.md | 1132 +++++++++++------ .../concrete/recovery/RecoveryTestSupport.rsc | 5 +- 2 files changed, 782 insertions(+), 355 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md index 5eb4723935..a4c0087718 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md @@ -22,392 +22,816 @@ Note that a benchmark test is documented by a string of characters including: - '+': After modification error recovery is able to recover by skipping a reasonable amount of input - '-': After modification error recovery is able to recover by skipping an excessive amount of input (>25% of the contents of the file) - '?': After modification parsing results in an unrecoverable parse error +- A newline is printed after each line of the input file The rest of the output should (hopefully) be self-descriptive: ``` +$ /usr/bin/env C:\\Program\ Files\\Eclipse\ Adoptium\\jdk-11.0.14.101-hotspot\\bin\\java.exe -agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=localhost:58567 @C:\\Users\\pieter\\AppData\\Local\\Temp\\cp_89hnnok9nfsf1vtykxy3jiv5n.argfile org.rascalmpl.shell.RascalShell +Windows console mode could not be enabled, errno=1 +Version: Rascal version not specified in META-INF/MANIFEST.MF??? +INFO: detected |lib://rascal| at |file:///D:/rascal/rascal/target/classes/| rascal>import lang::rascal::tests::concrete::recovery::ErrorRecoveryBenchmark; +☑ Loading modules 🕗 0:00:03.047 +ok rascal>runLanguageTests(); -========================================================================== -Error recovery of |std:///lang/c90/syntax/C.rsc| (TranslationUnit) on |std:///lang/c90/examples/hello-world.c|: -☑ Generating parser; MoreParameters Specifier Declarator TypeQualifier TranslationUnit AnonymousIdentifier FunctionDefinition MultiLineCommentBodyToken NonCommaExpression Expression Asterisk Declaration StorageClass StringConstant AbstractDeclarator PrototypeDeclarator Globa 🕛 0:00:03.468 +☑ Loading modules 🕐 0:00:02.331 +☑ Generating parser; RealLiteral Tags ModuleActuals Renamings KeywordFormal MidPathChars OptionalExpression Expression Class PathPart Signature ShellCommand Catch DataTarget KeywordFormals Renaming Tag Type LocationLiteral Declaration FunctionBody Field Range OptionalComma PrePathChars StringConstant Variable Type 🕛 0:00:12.763 +☑ Generating parser; MoreParameters Specifier Declarator TypeQualifier TranslationUnit AnonymousIdentifier FunctionDefinition MultiLineCommentBodyToken NonCommaExpression Expression Asterisk Declaration StorageClass StringConstant AbstractDeclarator PrototypeDeclarator GlobalDeclaration Comment Parameter Parameter 🕛 0:00:03.927 +Error recovery of |std:///lang/c90/syntax/C.rsc| (TranslationUnit) on |std:///lang/c90/examples/hello-world.c|, reference parse time: 47 ms. Single char deletions: -...........+................++....................+..........+.+.........++......+++.........+....++.........++.+++..-............?....................+++.+......................................................................?....++.-.....+.++.?.................++++....+.++++.?...........+.+.....?+....?................+....+?..?.. -Total parses: 333 -Succesful parses: 281 (84 % of total) -Succesful recoveries: 42 (80 % of failed) -Failed recoveries: 2 (3 % of failed) -Parse errors: 8 (15 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 1/1/2 ms -Succesful recovery time : 3/2/8 ms -Failed recovery time : 1/2/2 ms -Parse error time : 1/1/2 ms -Recovery error size : 17/5/83 characters -Successful recovery size : 13/4/50 characters +..........+................+-. +. +................+..........+.+ +........++......+++ +........+....++ +........++.+++ +.- +. +.........?....................+++.+........................................................ +.............? +...++.-.....+.++.? +................++++ +...+.++++.? +..........+.+.....?+ +...? +. +.............+....+? +.? +. +Total parses: 314 (100%) +Successful parses: 262 (83%) +Successful recoveries: 41 (78%) +Failed recoveries: 3 (5%) +Parse errors: 8 (15%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -...........++++++++++++++++++...++++++++++++++++++++++?----++.........++++++++++....+++++++++++....++++++++++.-..?++??++++???????????????????????++............................................................+++++?????.????+++++++++++??......++++++++++++++....??+++??......+++++++??????+.???.....????++++++??????.?. -Total parses: 314 -Succesful parses: 119 (37 % of total) -Succesful recoveries: 128 (65 % of failed) -Failed recoveries: 5 (2 % of failed) -Parse errors: 62 (31 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/2 ms -Succesful recovery time : 2/2/9 ms -Failed recovery time : 2/2/3 ms -Parse error time : 1/1/2 ms -Recovery error size : 38/36/82 characters -Successful recovery size : 36/35/82 characters - -Overall stats -------------- -Total parses: 647 -Succesful parses: 400 (61 % of total) -Succesful recoveries: 170 (68 % of failed) -Failed recoveries: 7 (2 % of failed) -Parse errors: 70 (28 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 1/1/2 ms -Succesful recovery time : 2/2/9 ms -Failed recovery time : 1/2/3 ms -Parse error time : 1/1/2 ms -Recovery error size : 33/27/83 characters -Successful recovery size : 30/26/82 characters - -========================================================================== -Error recovery of |std:///lang/diff/unified/UnifiedDiff.rsc| (DiffFile) on |std:///lang/diff/unified/examples/example.diff|: + +..........+++++++++++++++++-. + +++++++++++++++++++++++?----++ +........++++++++++ +...+++++++++++ +...++++++++++ +- + +?++??++++???????????????????????++........................................................ +...+++++????? +????+++++++++++?? +.....++++++++++++++ +...??+++?? +.....+++++++??????+ +??? + +...?????+++++?????? +? + +Total parses: 295 (100%) +Successful parses: 100 (33%) +Successful recoveries: 126 (64%) +Failed recoveries: 6 (3%) +Parse errors: 63 (32%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 + +----------------------------------------------------------- +Total test stats for |std:///lang/c90/examples/hello-world.c|: + +Total parses: 609 (100%) +Successful parses: 362 (59%) +Successful recoveries: 167 (67%) +Failed recoveries: 9 (3%) +Parse errors: 71 (28%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 +☑ Generating parser; Range Content Month Date DiffFile Diff Hours Decimal TimeZone DateTime Sign ChunkStart Year Day Line Minutes Seconds Chunk Time Header Indicator FileName 🕛 0:00:01.032 +Error recovery of |std:///lang/diff/unified/UnifiedDiff.rsc| (DiffFile) on |std:///lang/diff/unified/examples/example.diff|, reference parse time: 20 ms. Single char deletions: -++++....................................................+++++....................................................++++++.++++.++++.+...................................+.+.....................+.........................+..................................+......................+.----...---...----...........................................................-....................................................-.-.........................................................................................-..........................................................................................................-..........................................................................................................-...............................................................................................................................................................................................................................................................................................................................-.-...........................-..-.................................................????....???......???..................................................................................?...?............................................................................................................................................................................................?.................................................?...?.?..............................................................................?.................................................?..................................................................................................................................?.........................................?...?.?.? -Total parses: 1776 -Succesful parses: 1702 (95 % of total) -Succesful recoveries: 30 (40 % of failed) -Failed recoveries: 21 (28 % of failed) -Parse errors: 23 (31 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/2 ms -Succesful recovery time : 1/1/2 ms -Failed recovery time : 5/6/9 ms -Parse error time : 0/0/1 ms -Recovery error size : 386/161/837 characters -Successful recovery size : 96/96/161 characters +++++.................................................... ++++++.................................................... +++++++.++++.++++ +.+.................................. +.+ +.+.................... +.+........................ +.+................................. +.+..................... +.+ +.----...---...----.......................................................... +.-................................................... +.- +.-........................................................................................ +.-......................................................................................................... +.-......................................................................................................... +.-.............................................................................................................................................................................................................................................................................................................................. +.- +.-.......................... +.-. +.-................................................ +.????....???......???................................................................................. +.?.. +.? +............... +......................................................... +.................................................... +....... +........................................................ +.?................................................ +.?.. +.? +.?............................................................................. +.?................................................ +.?................................................................................................................................. +.?........................................ +.?.. +.? +.?. +? +Total parses: 1776 (100%) +Successful parses: 1702 (95%) +Successful recoveries: 30 (40%) +Failed recoveries: 21 (28%) +Parse errors: 23 (31%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -++++....................................................++++....................................................++++++++++++++++..................................++....................+........................+.................................+.....................+-----!-!-!-----!-!----..........................................................-...................................................--........................................................................................-.........................................................................................................-.........................................................................................................+..............................................................................................................................................................................................................................................................................................................................--..........................-.-................................................????????????????????.................................................................................?..??.............?.......................................................?..................................................?.....?......................................................?................................................?..??.............................................................................?................................................?.................................................................................................................................?........................................?..??. -Total parses: 1737 -Succesful parses: 1643 (94 % of total) -Succesful recoveries: 31 (32 % of failed) -Failed recoveries: 26 (27 % of failed) -Parse errors: 37 (39 % of failed) -5 slow parses, slowest parse: |unknown:///?deletedUntilEol=1768,1770| (0 ms) - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/2 ms -Succesful recovery time : 1/2/2 ms -Failed recovery time : 6/6/17 ms -Parse error time : 1/1/2 ms -Recovery error size : 398/160/777 characters -Successful recovery size : 102/147/161 characters - -Overall stats -------------- -Total parses: 3513 -Succesful parses: 3345 (95 % of total) -Succesful recoveries: 61 (36 % of failed) -Failed recoveries: 47 (27 % of failed) -Parse errors: 60 (35 % of failed) -5 slow parses, slowest parse: |unknown:///?deletedUntilEol=1768,1770| (0 ms) - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/2 ms -Succesful recovery time : 1/1/2 ms -Failed recovery time : 6/6/13 ms -Parse error time : 0/1/2 ms -Recovery error size : 392/161/837 characters -Successful recovery size : 99/109/161 characters - -========================================================================== -Error recovery of |std:///lang/dot/syntax/Dot.rsc| (DOT) on |std:///lang/dot/examples/parser-state.dot|: -☑ Generating parser; DotAttr NodeId Id EdgeStatement EdgeRhs Nod LAYOUT AttrStatement EdgeOp Comment AttrTag DOT Edg StatementList NodeStatement AttrList Statement StatementOptional Subgraph Graph AttrList0 Port 🕛 0:00:01.307 +++++.................................................... +++++.................................................... ++++++++++++++++ ++.................................. ++ ++.................... ++........................ ++................................. ++..................... ++ +-----------------.......................................................... +-................................................... +- +-........................................................................................ +-......................................................................................................... +-......................................................................................................... ++.............................................................................................................................................................................................................................................................................................................................. +- +-.......................... +-. +-................................................ +????????????????????................................................................................. +?.. +? +?............. +?....................................................... +?.................................................. +?..... +?...................................................... +?................................................ +?.. +? +?............................................................................. +?................................................ +?................................................................................................................................. +?........................................ +?.. +? +?. + +Total parses: 1737 (100%) +Successful parses: 1643 (94%) +Successful recoveries: 31 (32%) +Failed recoveries: 26 (27%) +Parse errors: 37 (39%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 + +----------------------------------------------------------- +Total test stats for |std:///lang/diff/unified/examples/example.diff|: + +Total parses: 3513 (100%) +Successful parses: 3345 (95%) +Successful recoveries: 61 (36%) +Failed recoveries: 47 (27%) +Parse errors: 60 (35%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 +☑ Generating parser; DotAttr NodeId Id EdgeStatement EdgeRhs Nod LAYOUT AttrStatement EdgeOp Comment AttrTag DOT Edg StatementList NodeStatement AttrList Statement StatementOptional Subgraph Graph AttrList0 Port 🕛 0:00:01.529 +Error recovery of |std:///lang/dot/syntax/Dot.rsc| (DOT) on |std:///lang/dot/examples/parser-state.dot|, reference parse time: 25 ms. Single char deletions: -++??????.......?..+......+-+.....+++...............+..................+................................................++...+......--+.....+++.....+..+.....+++......++...+..+-+.....+++............................................+.......+.............................+.......+.....++...-....--+.....+++..........................................................................+.......+....++...+.....+-+.....+++.................................................................................................................++...+..+++.....+++.............................................................++...+.....+.++.+..+...+....+.++.+.....+...+..+.++.+....+...+......+.+.+.++.+..+++.....+++.....++...+........+++.....+++.......+..+.....+++.......++...+......+.+.+....++.+........+++.....+++....++...+.........+.+.+.++.+......+...+.........+++.....+++.........................................................................................................................................+..+.....+++......++...+......+.++.+.........+++.....+++..........++...+..............+++.....++++..+.....+++......++...+......+.++.+..............+++.....+++................++...+.................+++.....+++.....+..+.....+++......+..+.....+++...++...+.................+.+.+....++.+..+++.....+++.....++...+.................+.+.+....++.+........+-+.....+++....++...+......+.++.+.................+-+.....+++...................++...+....................+-+.....++++..+.....+++......++...+......+.-+.+....................+-+.....+++.......................++...-...--+.....+++............................................+....+...................................+....+............+...........+......++...-...--+.....+++....................................................................++...+.....+++.....+++.............................................................................................++...+.....+.++.+..-...-...-.++.-.....-...-...-.++.-...-...-.................-.-.-.-+.-...-...-.....--+.....+++.................................................................++...-....-++.....+++.................................................+.......+.............................+.......+.....++...+....+.++.+....+...+.....+.++.+....+...+.................+.+.+.++.+.....+...+.................+-+.....+++.............+..+.....+++......++...+.....+++.....+++.........................................+....................+...............++...+....................+.+.+.++.+.....+...+...+++.....+++.........................................+.....+.............++...+....................+.+.+.++.+...+...+....+++.....+++.............................................................++...+....................+.+.+.++.+....+...+....+++.....+++....................................................+....+...............++...+....................+.+.+.-+.+....+...+.....+++.....+++..............................................................................................................................................................................................................................................................................................++...+....................+.+.+.++.+.....+...+.....+++.....+++....................................................+.......+................++...+....................+.+.+.++.+.....+...+....+++.....+++.....................................................++...+....................+.+.+.++.+....+...+....+++.....+++................................................++...+....................+.+.+.++.+....+...+....+++.....+++.........................................+........+..............++...+....................+.+.+.+?.+....+...+....................+++.....+++.....................................................................+..+.....+++......++...?.............................?++.....+++......+..+.....+++++...?.............?++.....++++..+.....+++......++...?.....?++.....+++......++...?......?.+?.?.....?++.....+++..............++...?.....?.+?.?.................?++.....+++............++...?.....?.+?.?....................?++.....+++.................++...?.....?.+?.?.............................?++.....+++....................++...?.....?.+?.?.............?++.....++?........??...?.. -Total parses: 4239 -Succesful parses: 3555 (83 % of total) -Succesful recoveries: 599 (87 % of failed) -Failed recoveries: 42 (6 % of failed) -Parse errors: 43 (6 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 5/5/8 ms -Succesful recovery time : 9/9/17 ms -Failed recovery time : 10/10/17 ms -Parse error time : 3/3/6 ms -Recovery error size : 233/44/2220 characters -Successful recovery size : 65/38/241 characters +++??????.......? +.+......+-+.....+++...............+..................+................................................++. +.+......--+.....+++.....+..+.....+++......++. +.+..+-+.....+++............................................+.......+.............................+.......+.....++. +.-....--+.....+++..........................................................................+.......+....++. +.+.....+-+.....+++.................................................................................................................++. +.+..+++.....+++.............................................................++. +.+.....+.++.+..+. +.+....+.++.+.....+. +.+..+.++.+....+. +.+......+.+.+....++.+..+++.....+++.....++. +.+........+++.....+++.......+..+.....+++.......++. +.+......+.+.+....++.+........+++.....+++....++. +.+.........+.+.+.++.+......+. +.+.........+++.....+++.........................................................................................................................................+..+.....+++......++. +.+......+.++.+.........+++.....+++..........++. +.+..............+++.....++++..+.....+++......++. +.+......+.++.+..............+++.....+++................++. +.+.................+++.....+++.....+..+.....+++......+..+.....+++...++. +.+.................+.+.+....++.+..+++.....+++.....++. +.+.................+.+.+....++.+........+-+.....+++....++. +.+......+.++.+.................+-+.....+++...................++. +.+....................+-+.....++++..+.....+++......++. +.+......+.-+.+....................+-+.....+++.......................++. +.-...--+.....+++............................................+....+...................................+....+............+...........+......++. +.-...--+.....+++....................................................................++. +.+.....+++.....+++.............................................................................................++. +.+.....+.++.+..-. +.-...-.++.-.....-. +.-...-.++.-...-. +.-.................-.-.-.-+.-...-. +.-.....--+.....+++.................................................................++. +.-....-++.....+++.................................................+.......+.............................+.......+.....++. +.+....+.++.+....+. +.+.....+.++.+....+. +.+.................+.+.+.++.+.....+. +.+.................+-+.....+++.............+..+.....+++......++. +.+.....+++.....+++.........................................+....................+...............++. +.+....................+.+.+.++.+.....+. +.+...+++.....+++.........................................+.....+.............++. +.+....................+.+.+.++.+...+. +.+....+++.....+++.............................................................++. +.+....................+.+.+.++.+....+. +.+....+++.....+++....................................................+....+...............++. +.+....................+.+.+.-+.+....+. +.+.....+++.....+++..............................................................................................................................................................................................................................................................................................++. +.+....................+.+.+.++.+.....+. +.+.....+++.....+++....................................................+.......+................++. +.+....................+.+.+.++.+.....+. +.+....+++.....+++.....................................................++. +.+....................+.+.+.++.+....+. +.+....+++.....+++................................................++. +.+....................+.+.+.++.+....+. +.+....+++.....+++.........................................+........+..............++. +.+....................+.+.+.+?.+....+. +.+....................+++.....+++.....................................................................+..+.....+++......++. +.?.............................?++.....+++......+..+.....+++++. +.?.............?++.....++++..+.....+++......++. +.?.....?++.....+++......++. +.?......?.+?.?.....?++.....+++..............++. +.?.....?.+?.?.................?++.....+++............++. +.?.....?.+?.?....................?++.....+++.................++. +.?.....?.+?.?.............................?++.....+++....................++. +.?.....?.+?.?.............?++.....++?........??. +.? +. +Total parses: 4177 (100%) +Successful parses: 3493 (83%) +Successful recoveries: 599 (87%) +Failed recoveries: 42 (6%) +Parse errors: 43 (6%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -????????????????..-------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++.++++++++++++++++++++++++++++++++++...---.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++..+..+++...+++++..+..++++++...+++..+..+++++...+++++++..++..+..+++.++++++++++++++++...+++++++++.+++++++++++++++++++++++++++++++++++++...+++++++..++.+...+..+++++++++.+++++++++++++++...++++++++++..++..+..+++++++...++++++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++..+..++++++++++.+++++++++++++++++++++...+++++++++++++++.+++++++++++++++++++++++++++++...+++++++..+..+++++++++++++++.+++++++++++++++++++++++++++...++++++++++++++++++.+++++++++++++++++++++++++++++++++++++++++++++++++...++++++++++++++++++.+++.+...+..+++.++++++++++++++++...++++++++++++++++++..++.....+..+++++++++.+++++++++++++++...+++++++..+..++++++++++++++++++.++++++++++++++++++++++++++++++...+++++++++++++++++++++.+++++++++++++++++++++++++++++...-------..+..---------------------.++++++++++++++++++++++++++++++++++...----.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...------..+..---...----..+..------...----..+..----...------------------..--..+..----...------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++..+..+++++...++++++..+..+++++...++++++++++++++++++..++..+..++++++...++++++++++++++++++.++++++++++++++++++++++++++++++++++++++++++...++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..++++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..+..+++++...+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...+++++++++++++++++++++..++..?..+++++...?????????????????????.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++...??????????????????????????????.+++++++++++++++++++++++++++++...??????????????.+++++++++++++++++++++++++++++...??????.+++++++++++++++++...???????..?..??????.+++++++++++++++++++++++++...??????..?..??????????????????.+++++++++++++++++++++++...??????..?..?????????????????????.++++++++++++++++++++++++++++...??????..?..??????????????????????????????.+++++++++++++++++++++++++++++++...??????..?????????????????.???????????????????..?. -Total parses: 4174 -Succesful parses: 402 (9 % of total) -Succesful recoveries: 3438 (91 % of failed) -Failed recoveries: 99 (2 % of failed) -Parse errors: 235 (6 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 6/6/13 ms -Succesful recovery time : 9/8/18 ms -Failed recovery time : 11/10/25 ms -Parse error time : 4/4/7 ms -Recovery error size : 150/63/391 characters -Successful recovery size : 81/61/297 characters - -Overall stats -------------- -Total parses: 8413 -Succesful parses: 3957 (47 % of total) -Succesful recoveries: 4037 (90 % of failed) -Failed recoveries: 141 (3 % of failed) -Parse errors: 278 (6 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 5/5/9 ms -Succesful recovery time : 9/8/18 ms -Failed recovery time : 11/10/19 ms -Parse error time : 3/4/7 ms -Recovery error size : 163/61/408 characters -Successful recovery size : 79/57/289 characters - -========================================================================== -Error recovery of |std:///lang/java/syntax/Java15.rsc| (CompilationUnit) on |jar+file:///C:/Users/pieter/Documents/projects/wk/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|: -☑ Generating parser; SwitchBlock CharLiteral BinaryExponent ClassOrInterfaceType VarDec Super VarMod Id Type ClassBody Interfaces Asterisk CommentPart DimExpr ImportDec ArrayAccess PackageDec MethodSpec DeciFloatExponentPart BlockStm EnumConstArgs AbstractMethodDec DeciNumer 🕛 0:00:08.222 +???????????????? +.-------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++.++++++++++++++++++++++++++++++++++. +.---.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.++++++..+..+++. +.+++++..+..++++++. +.+++..+..+++++. +.+++++++..++.....+..+++.++++++++++++++++. +.+++++++++.+++++++++++++++++++++++++++++++++++++. +.+++++++..++.+...+..+++++++++.+++++++++++++++. +.++++++++++..++..+..+++++++. +.++++++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++..+..++++++++++.+++++++++++++++++++++. +.+++++++++++++++.+++++++++++++++++++++++++++++. +.+++++++..+..+++++++++++++++.+++++++++++++++++++++++++++. +.++++++++++++++++++.+++++++++++++++++++++++++++++++++++++++++++++++++. +.++++++++++++++++++.+++.+...+..+++.++++++++++++++++. +.++++++++++++++++++..++.....+..+++++++++.+++++++++++++++. +.+++++++..+..++++++++++++++++++.++++++++++++++++++++++++++++++. +.+++++++++++++++++++++.+++++++++++++++++++++++++++++. +.-------..+..---------------------.++++++++++++++++++++++++++++++++++. +.----.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.------..+..---. +.----..+..------. +.----..+..----. +.------------------..--..+..----. +.------.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++..+..+++++. +.++++++..+..+++++. +.++++++++++++++++++..++..+..++++++. +.++++++++++++++++++.++++++++++++++++++++++++++++++++++++++++++. +.++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..++++++. +.++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..+++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..+++++. +.++++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..++++++. +.++++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..++++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..+++++. +.+++++.+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..+..+++++. +.+++++.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.+++++++++++++++++++++..++..?..+++++. +.?????????????????????.++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++. +.??????????????????????????????.+++++++++++++++++++++++++++++. +.??????????????.+++++++++++++++++++++++++++++. +.??????.+++++++++++++++++. +.???????..?..??????.+++++++++++++++++++++++++. +.??????..?..??????????????????.+++++++++++++++++++++++. +.??????..?..?????????????????????.++++++++++++++++++++++++++++. +.??????..?..??????????????????????????????.+++++++++++++++++++++++++++++++. +.??????..?????????????????.???????????????????. +? + +Total parses: 4112 (100%) +Successful parses: 340 (8%) +Successful recoveries: 3438 (91%) +Failed recoveries: 99 (2%) +Parse errors: 235 (6%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 1 + +----------------------------------------------------------- +Total test stats for |std:///lang/dot/examples/parser-state.dot|: + +Total parses: 8289 (100%) +Successful parses: 3833 (46%) +Successful recoveries: 4037 (90%) +Failed recoveries: 141 (3%) +Parse errors: 278 (6%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 1 +☑ Generating parser; SwitchBlock CharLiteral BinaryExponent ClassOrInterfaceType VarDec Super VarMod Id Type ClassBody Interfaces Asterisk CommentPart DimExpr ImportDec ArrayAccess PackageDec MethodSpec DeciFloatExponentPart BlockStm EnumConstArgs AbstractMethodDec DeciNumeral AbstractMethodMod StringLiteral Field 🕛 0:00:07.668 +Error recovery of |std:///lang/java/syntax/Java15.rsc| (CompilationUnit) on |jar+file:///D:/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|, reference parse time: 91 ms. Single char deletions: -++++++++......-..++++++.++++++..........+++++++++.......-...++++++...........-....+....+....+........+.-........+....+.........++..+...+..........++++++.....................++.+...++++++.....+..?.?. -Total parses: 198 -Succesful parses: 131 (66 % of total) -Succesful recoveries: 61 (91 % of failed) -Failed recoveries: 4 (5 % of failed) -Parse errors: 2 (2 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 1/1/2 ms -Succesful recovery time : 2/2/6 ms -Failed recovery time : 2/2/4 ms -Parse error time : 0/1/1 ms -Recovery error size : 18/14/69 characters -Successful recovery size : 14/13/31 characters +++++++++......- +. +.++++++.++++++..........+++++++++.......- +. +..++++++...........-....+....+....+........+.- +........+....+.........++ +..+ +. +..+........ +..++++++.....................++.+ +...++++++.....+ +..? +.? +. +Total parses: 198 (100%) +Successful parses: 131 (66%) +Successful recoveries: 61 (91%) +Failed recoveries: 4 (5%) +Parse errors: 2 (2%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -.++++++------------------------------------------------++++++++++++++++++++-------------------------...+++++++++++++++++++++++..+.......++++++++++++++++++++++++++++++++...+++++++++++??? -Total parses: 185 -Succesful parses: 16 (8 % of total) -Succesful recoveries: 93 (55 % of failed) -Failed recoveries: 73 (43 % of failed) -Parse errors: 3 (1 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 1/1/2 ms -Succesful recovery time : 2/2/5 ms -Failed recovery time : 1/2/3 ms -Parse error time : 0/1/1 ms -Recovery error size : 48/42/112 characters -Successful recovery size : 18/11/45 characters - -Overall stats -------------- -Total parses: 383 -Succesful parses: 147 (38 % of total) -Succesful recoveries: 154 (65 % of failed) -Failed recoveries: 77 (32 % of failed) -Parse errors: 5 (2 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 1/1/2 ms -Succesful recovery time : 2/2/6 ms -Failed recovery time : 1/2/4 ms -Parse error time : 0/1/1 ms -Recovery error size : 39/21/112 characters -Successful recovery size : 16/12/42 characters - -========================================================================== -Error recovery of |std:///lang/json/syntax/JSON.rsc| (JSONText) on |std:///lang/json/examples/ex01.json|: -☑ Generating parser; Member Object Value NumericLiteral StringChar IntegerLiteral RealLiteral Array JSONText UnicodeEscape StringLiteral 🕛 0:00:00.619 +.++++++-------- + +---------------------------------------- + +++++++++++++++++++++------------------------- +...+++++++++++++++++++++ +++ + +..+....... +++++++++++++++++++++++++++++++++ +...+++++++++++ +?? +? + +Total parses: 185 (100%) +Successful parses: 16 (8%) +Successful recoveries: 93 (55%) +Failed recoveries: 73 (43%) +Parse errors: 3 (1%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 + +----------------------------------------------------------- +Total test stats for |jar+file:///D:/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|: + +Total parses: 383 (100%) +Successful parses: 147 (38%) +Successful recoveries: 154 (65%) +Failed recoveries: 77 (32%) +Parse errors: 5 (2%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 +☑ Generating parser; Member Object Value NumericLiteral StringChar IntegerLiteral RealLiteral Array JSONText UnicodeEscape StringLiteral 🕛 0:00:00.691 +Error recovery of |std:///lang/json/syntax/JSON.rsc| (JSONText) on |std:///lang/json/examples/ex01.json|, reference parse time: 12 ms. Single char deletions: -?........?.....??.?............+.....++..+..?............+......++.+..?............+.....++..+....................??............?.........??.?................+...++....+......................................++................+......-+....+................-.....+-..+...-............?+............?...??.?...+....+....+......?........?..? -Total parses: 337 -Succesful parses: 285 (84 % of total) -Succesful recoveries: 27 (51 % of failed) -Failed recoveries: 4 (7 % of failed) -Parse errors: 21 (40 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/0/1 ms -Succesful recovery time : 0/1/1 ms -Failed recovery time : 0/0/1 ms -Parse error time : 0/1/1 ms -Recovery error size : 34/21/124 characters -Successful recovery size : 21/16/57 characters +?. +.......?.....??.?. +...........+.....++..+..?. +...........+......++.+..?. +...........+.....++..+....................??. +...........?.........??.?. +...............+...++....+......................................++. +...............+......-+....+. +...............-.....+-..+...-. +...........?+. +...........?...??.?...+....+....+......?. +.......?. +.? +Total parses: 337 (100%) +Successful parses: 285 (84%) +Successful recoveries: 27 (51%) +Failed recoveries: 4 (7%) +Parse errors: 21 (40%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -?.????????????????............+++++++??????............++++++++?????............????????????????????????????????.????????????????????????................++++++++++++++++++++++++++++++++++++++++++++++++++................--------+++++.-----------------------------.-----------+.??????????????????+????????????????????.???????. -Total parses: 324 -Succesful parses: 75 (23 % of total) -Succesful recoveries: 72 (28 % of failed) -Failed recoveries: 48 (19 % of failed) -Parse errors: 129 (51 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/0/1 ms -Succesful recovery time : 0/1/2 ms -Failed recovery time : 0/1/1 ms -Parse error time : 0/0/1 ms -Recovery error size : 86/57/294 characters -Successful recovery size : 42/57/57 characters - -Overall stats -------------- -Total parses: 661 -Succesful parses: 360 (54 % of total) -Succesful recoveries: 99 (32 % of failed) -Failed recoveries: 52 (17 % of failed) -Parse errors: 150 (49 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/0/1 ms -Succesful recovery time : 0/1/2 ms -Failed recovery time : 0/1/1 ms -Parse error time : 0/0/1 ms -Recovery error size : 76/57/293 characters -Successful recovery size : 37/39/57 characters - -========================================================================== -Error recovery of |std:///lang/pico/syntax/Main.rsc| (Program) on |std:///lang/pico/examples/fac.pico|: -☑ Generating parser; Type Id WhitespaceAndComment Statement Natural Program Declarations String Expression IdType 🕛 0:00:00.719 +?. +????????????????. +...........+++++++??????. +...........++++++++?????. +...........????????????????????????????????. +????????????????????????. +...............++++++++++++++++++++++++++++++++++++++++++++++++++. +...............--------+++++. +-----------------------------. +-----------+. +??????????????????+????????????????????. +???????. + +Total parses: 324 (100%) +Successful parses: 75 (23%) +Successful recoveries: 72 (28%) +Failed recoveries: 48 (19%) +Parse errors: 129 (51%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 + +----------------------------------------------------------- +Total test stats for |std:///lang/json/examples/ex01.json|: + +Total parses: 661 (100%) +Successful parses: 360 (54%) +Successful recoveries: 99 (32%) +Failed recoveries: 52 (17%) +Parse errors: 150 (49%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 +☑ Generating parser; Type Id WhitespaceAndComment Statement Natural Program Declarations String Expression IdType 🕛 0:00:00.660 +Error recovery of |std:///lang/pico/syntax/Main.rsc| (Program) on |std:///lang/pico/examples/fac.pico|, reference parse time: 9 ms. Single char deletions: -?????.-------.......+.+++++++-.........................+.++++++++............................+.++++++++....................+.++++++++................+..+.+++++++................+..+.------+..............++...+...............++.+-........-----.......+.-.--.................++.......+..................++......-............+++++.......+.+.++......................++........+....+.....................++.......+.+............+++..................--.......+.+........--..???.. -Total parses: 472 -Succesful parses: 355 (75 % of total) -Succesful recoveries: 81 (69 % of failed) -Failed recoveries: 28 (23 % of failed) -Parse errors: 8 (6 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/1 ms -Succesful recovery time : 2/3/5 ms -Failed recovery time : 1/2/4 ms -Parse error time : 0/0/1 ms -Recovery error size : 87/10/463 characters -Successful recovery size : 16/6/70 characters +?????.-------.......+.+++++++-.. +......................+.++++++++...... +.....................+.++++++++ +...................+.++++++++ +...............+..+.+++++++ +...............+..+.------+ +.............++...+ +..............++.+- +.......-----.......+.-.--. +...............++.......+ +.................++......- +...........+++++.......+.+.++ +.....................++........+....+ +....................++.......+.+ +...........+++ +.................??.......+.+ +.......-- +.??? +. +Total parses: 454 (100%) +Successful parses: 337 (74%) +Successful recoveries: 81 (69%) +Failed recoveries: 26 (22%) +Parse errors: 10 (8%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 Deletes until end-of-line: -?????--------..++++++++++++++-..................++++++++++++++++......................+++++++++++++++................+++++++++++++................+++++++++++.-------------------------+........+++++++++++........-----------.+++++++-----------------.............+++++++++++++............+++++++-------.++++++++++++++++++++++++++++...............++++++++++++++++++++++.+++++++++++++++++++++++......++.------------+.------------------++......++.+++++++-.???. -Total parses: 454 -Succesful parses: 151 (33 % of total) -Succesful recoveries: 195 (64 % of failed) -Failed recoveries: 100 (33 % of failed) -Parse errors: 8 (2 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/1 ms -Succesful recovery time : 2/2/4 ms -Failed recovery time : 1/1/4 ms -Parse error time : 0/0/1 ms -Recovery error size : 101/41/268 characters -Successful recovery size : 28/26/70 characters - -Overall stats -------------- -Total parses: 926 -Succesful parses: 506 (54 % of total) -Succesful recoveries: 276 (65 % of failed) -Failed recoveries: 128 (30 % of failed) -Parse errors: 16 (3 % of failed) -No slow parses. - -Statistics (avg/median/95th percentile): -Succesful parse time : 0/1/1 ms -Succesful recovery time : 2/2/5 ms -Failed recovery time : 1/1/4 ms -Parse error time : 0/0/1 ms -Recovery error size : 97/33/272 characters -Successful recovery size : 25/15/70 characters - -========================================================================== -Error recovery of |std:///lang/rascal/syntax/Rascal.rsc| (Module) on |std:///lang/rascal/vis/ImportGraph.rsc|: +?????--------..++++++++++++++-.. +...............++++++++++++++++...... +...............+++++++++++++++ +...............+++++++++++++ +...............+++++++++++ +-------------------------+ +.......+++++++++++ +.......----------- ++++++++-----------------. +...........+++++++++++++ +...........+++++++------- +++++++++++++++++++++++++++++ +..............++++++++++++++++++++++ ++++++++++++++++++++++++......++ +------------+ +??????????????????++......++ ++++++++- +??? + +Total parses: 436 (100%) +Successful parses: 133 (30%) +Successful recoveries: 195 (64%) +Failed recoveries: 82 (27%) +Parse errors: 26 (8%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 + +----------------------------------------------------------- +Total test stats for |std:///lang/pico/examples/fac.pico|: + +Total parses: 890 (100%) +Successful parses: 470 (52%) +Successful recoveries: 276 (65%) +Failed recoveries: 108 (25%) +Parse errors: 36 (8%) +Slow parses: 0 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 0 0 0 +Error recovery of |std:///lang/rascal/syntax/Rascal.rsc| (Module) on |std:///lang/rascal/vis/ImportGraph.rsc|, reference parse time: 20 ms. Single char deletions: -?........+...............................................................................................?.?...........+......................................................................................................................................................................................................................................................................................................................................................................................................................?.?........+....................................................................................................................................................................................................................................................................................................?.?........+..................................................................................................?.?.!......!....!..!.!.!????????..!.!.+!+......++...++.............??????+....++..........+.??????+...++......+.??????+....++......++.......++..........++.......+.??????+....++......+++......++......+.??????+.........+.??????+....++..........+.??????+....++...........+.+++++++..+.+++++++........++......++.....-..+........+..............................................................................................?.....?...........?.....................+..................+.?................+!+.......+++............++.............................++.+..+........?......................................................................................?.....?...........?.....................+..................+.?.....++...............................................................................................+.....................+................+.........++++................+!.................................++.+..+........?.................................................................................................?.....?...........?.....................+..................-!.?.....+.+................+.........+?..........++...........................................................................+....+.+!.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++!++++++.........+.......+.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+.....++............................................................................................++...........................................................................................................................................+......+.........+.........+........+.+.........++.+....+.++.+......+....++.....................+.........++++..............++++++.+.......+...++.....+......+++++++....+.........+......++.+.+.......+...++............................+.....+...+.........................+++...........+.........+.......++..........................++..........+................+?++................+++++.+..+........+........................................................................................?.+++++.............+.............+........+...+............+.+.+++........+...+...............+.+++........+...+....+.......+.+.+++........+...+....+.......+.+.+++........+...+....+.......+.++.+?..?........?..........................................................................................?.............?...............?....+...+.....+.?..............+.+.....+...+.+...++..............++.................+.+...............+.+...+............++.................?.............+...............+.+.+.+.......+...+..........++...............+.+.+.+.......+...+..........++...............+.+.+.+.......+...+..........++.................+.+.+...........+..........+.....++..........................+..................+++..+..................+.+..+..................++.......................+.?..?........?.............................................................?.............?...............?........?.?.....+++.?..................?............+....++..........+....+.....+.....+.+.................+.++..........++++++.............+...................+.+.+....++...................+.+.+....+.+.....+...................+.+.+....+.+.....+.....................+.++....+.....++.........+?.....?.....++++++..........+.+.+.............................++?.? -Total parses: 4314 -Succesful parses: 3815 (88 % of total) -Succesful recoveries: 393 (78 % of failed) -Failed recoveries: 2 (0 % of failed) -Parse errors: 104 (20 % of failed) -14 slow parses, slowest parse: |unknown:///?deleted=1137| (1 ms) - -Statistics (avg/median/95th percentile): -Succesful parse time : 14/12/20 ms -Succesful recovery time : 29/21/69 ms -Failed recovery time : 183/184/351 ms -Parse error time : 10/5/40 ms -Recovery error size : 65/19/274 characters -Successful recovery size : 59/18/233 characters +?........+...............................................................................................? +.?...........+ +...................................................................................................... +.................................................................................................... +................................................................................................................ +........................................................................................... +.? +.?........+ +.................................................................................................... +............................................................................ +............................................................................................... +.................... +.? +.?........+ +................................................................................................. +.? +.?............... +????????....++......++...++........... +. +.??????+....++..........+ +.??????+...++......+ +.??????+....++......++.......++..........++.......+ +.??????+....++......+++......++......+ +.??????+.........+ +.??????+....++..........+ +.??????+....++...........+ +.+++++++..+ +.+++++++........++......++.....- +. +.+........+..............................................................................................? +.....?...........?.....................+..................+.? +................+!+.......+++............++.............................++ +.+ +. +.+........?......................................................................................? +.....?...........?.....................+..................+.? +.....++..................................................................................... +..........+.....................+................+.........++++ +................+!.................................++ +.+ +. +.+........?.................................................................................................? +.....?...........?.....................+..................-!.? +.....+.+................+.........+? +..... +.....++........................................................................... ++....+.+!.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+ +.......+.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+ +.....++....................................................................................... +.....++........................................................................................................................................... ++......+ +. +........+.........+........+.+ +.........++.+....+.++.+......+....?+ +.....................+ +.........++++. +.............++++++.+.......+...++ +.....+ +. +.....+++++++....+.........+......++.+.+.......+...++ +. +...........................+.....+...+.........................+++...........+.........+.......++..........................++..........+................+?++................+++++ +.+ +. +.+........+........................................................................................? +.+++++.............+.............+ +........+...+............+.+.+++ +........+...+...............+.+++ +........+...+....+.......+.+.+++ +........+...+....+.......+.+.+++ +........+...+....+.......+.++ +.+? +. +.?........?..........................................................................................? +.............?...............?....+...+.....+.? +..............+.+.....+...+.+...++..............++ +..... +............+.+...............+.+...+............++ +. +................?.............+ +...............+.+.+.+.......+...+..........++ +...............+.+.+.+.......+...+..........++ +...............+.+.+.+.......+...+..........++ +.................+.+.+...........+..........+ +.....++ +. +.........................+..................+++..+..................+.+..+..................++ +. +......................+ +.? +. +.?........?.............................................................? +.............?...............?........?.? +.....+++.? +..................?............+....++ +. +.........+....+.....+.....+.+.................+.++ +. +.........++++++.............+ +...................+.+.+....++ +...................+.+.+....+.+.....+ +...................+.+.+....+.+.....+ +.....................+.++....+.....++ +.........+? +.....? +.....++++++..........+.+.+. +............................++? +.? +Total parses: 4314 (100%) +Successful parses: 3815 (88%) +Successful recoveries: 392 (78%) +Failed recoveries: 2 (0%) +Parse errors: 105 (21%) +Slow parses: 4 (0%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 1 0 4 Deletes until end-of-line: -.+........?????????????????????????????????????????????????????????????????????????????????????????????????++++++++++++.................................................................................................................................................................................................................................................................................................................................................................................................................??+++++++++...............................................................................................................................................................................................................................................................................................??+++++++++................................................................................................?.?..............????????....+?......+?...+?...........?????++++++++++++++++++.?????+++++++++++++.?????++++++++++++++++++++++++++++++++++++++++++++.?????+++++++++++++++++++++++++++++++.?????+++++++++++.?????++++++++++++++++++.?????+++++++++++++++++++.+++++++++.+++++-------------------------.+........???????????????????????????????????????????????????????????????????????????????????????????????++++++???????????+++++++++++++++++++++++++++++++++++++++++??.....+!+!+!+!+!++!+!+!+!+!+++++++++++++++++++++++++++++++++++++++++++++++++++++++++.+........???????????????????????????????????????????????????????????????????????????????????????++++++???????????+++++++++++++++++++++++++++++++++++++++++??.....+..........................................................................................+++++..++++++++++++++++++++++++++++++++++++++++++++++++++.....+!+!+!+!+!++!+!+!+!+!++++++++++++++++++++++++++++++++++++.+........????????????????????????????????????????????????????????????????????????????????????????????????????????????????????-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!??-!-!-!-!-!?????????????-!----!??.........???????????????++++++++++?.........+...........................................................................+++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.........++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....+............................................................................................+...........................................................................................................................................+++++++?????----++-+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.........+++.+++++++++++++?????+++????????+?+++++++.....+++++++++++++++++++++++++++++++..+++++++++++++.....+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!++++++????+????????????????????????????????+????????????????+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++?+????????????????????+++.+........?????????????????????????????????????????????????????????????????????????????????????????+++++++++++++++++++++++++++++++++........++++.........+..+..++++.....+++++++++++++++++++++++++++.....++++++++++++++++++++++++++.....++++++++++++++++++++++++++++++++++++++++++++++++++++++??.?........????????????????????????????????????????????????????????????????????????????????????????????????????????????????????????+++++++++++??????.......+........?++++++++++++++++++++++++++++++++...........+......????????????????????++++++++++++++++++++++++++++++++++++++++++++++...........+..+..++?+++++++++++++++++++++++++.........++++++++++?+++++++++++++++++++++++++.........++++++++++?+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....++++++++++++++++++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++.....+++++++++++++++++?.?........???????????????????????????????????????????????????????????????????????????????????????????????????????????????.........++++++++++++++++++++++++++++.........+++++++++++++++++++..+++++++++++++++++++++++++++++++++++++++++++++++...............+..+..++?+++++.............++++++++++?++++++++++++.............++++++++++?++++++++++++++++++++++++++++++++++++++++++++++++???????????????+++++++++++++++++++++++++.??????????????++????????????+? -Total parses: 4206 -Succesful parses: 1542 (36 % of total) -Succesful recoveries: 1611 (60 % of failed) -Failed recoveries: 55 (2 % of failed) -Parse errors: 998 (37 % of failed) -64 slow parses, slowest parse: |unknown:///?deletedUntilEol=963,996| (1 ms) - -Statistics (avg/median/95th percentile): -Succesful parse time : 13/13/21 ms -Succesful recovery time : 28/18/66 ms -Failed recovery time : 135/18/319 ms -Parse error time : 7/6/14 ms -Recovery error size : 158/40/998 characters -Successful recovery size : 116/38/561 characters - -Overall stats -------------- -Total parses: 8520 -Succesful parses: 5357 (62 % of total) -Succesful recoveries: 2004 (63 % of failed) -Failed recoveries: 57 (1 % of failed) -Parse errors: 1102 (34 % of failed) -78 slow parses, slowest parse: |unknown:///?deletedUntilEol=963,996| (1 ms) - -Statistics (avg/median/95th percentile): -Succesful parse time : 14/13/20 ms -Succesful recovery time : 28/18/67 ms -Failed recovery time : 137/18/331 ms -Parse error time : 7/6/14 ms -Recovery error size : 140/36/981 characters -Successful recovery size : 105/32/407 characters +.+........???????????????????????????????????????????????????????????????????????????????????????????????? +?++++++++++++ +..................................................................................................... +................................................................................................... +............................................................................................................... +.......................................................................................... +? +?+++++++++ +................................................................................................... +........................................................................... +.............................................................................................. +................... +? +?+++++++++ +................................................................................................ +? +.?.............. +????????....+?......+?...+?.......... + +.?????++++++++++++++++++ +.?????+++++++++++++ +.?????++++++++++++++++++++++++++++++++++++++++++++ +.?????+++++++++++++++++++++++++++++++ +.?????+++++++++++ +.?????++++++++++++++++++ +.?????+++++++++++++++++++ +.+++++++++ +.+++++------------------------- + +.+........??????????????????????????????????????????????????????????????????????????????????????????????? +++++++???????????+++++++++++++++++++++++++++++++++++++++++?? +.....+!+!+!+!+!++!+!+!+!+!++++++++++++++++++++++++?+++++++++++++++++++++++++++++++ ++ + +.+........??????????????????????????????????????????????????????????????????????????????????????? +++++++???????????+++++++++++++++++++++++++++++++++++++++++?? +.....+..................................................................................... +.....+++++..++++++++++++++++++++++++++++++++++++++++++++++++++ +.....+!+!+!+!+!++!+!+!+!+!+++++++++++++++++++++++++++++++++++ ++ + +.+........?????????????????????????????????????????????????????????????????????????????????????????????????? +??????????????????-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!??-!-!-!-!-!?????????????-!----!?? +.........???????????????++++++++++? +.... +.....+........................................................................... ++++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.........++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ +.....+....................................................................................... +.....+........................................................................................................................................... ++++++++ + +?????----++-+++++++++++++++++ ++++++++++++++++++++++++++++++++++++ ++++++++++++++++++++++ +.........+++. ++++++++++++++?????+++????????+?++ ++++++ + +.....+++++++++++++++++++++++++++++++..+++++++++++++ + +.....+!+!+!+!+!+!++!+!+!+!+!+!+!+!+!+!+!+!+!+!+!++++++????+????????????????????????????????+????????????????+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++?+????????????????????++ ++ + +.+........????????????????????????????????????????????????????????????????????????????????????????? ++++++++++++++++++++++++++++++++++ +........++++.........+..+..++++ +.....+++++++++++++++++++++++++++ +.....++++++++++++++++++++++++++ +.....++++++++++++++++++++++++++ +++++++++++++++++++++++++++++ +?? + +.?........??????????????????????????????????????????????????????????????????????????????????????????? +?????????????????????????????+++++++++++?????? +.......+........?++++++++++++++++++++++++++++++++ +.... +.......+......????????????????????++++++++++++++++ + +++++++++++++++++++++++++++++++ +...........+..+..++?+++++++++++++++++++++++++ +.........++++++++++?+++++++++++++++++++++++++ +.........++++++++++?+++++++++++++++++++++++++ +++++++++++++++++++++++++++++++++++++++++++++ +++++++ + +.....++++++++++++++++++++..++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ + +.....+++++++++++++++++ +? + +.?........?????????????????????????????????????????????????????????????? +???????????????????????????????????????? +????????? +.........++++++++++++++++++++++++++++ + +.........+++++++++++++++++++..+++++++++++++++++++ + +++++++++++++++++++++++++++++ +...............+..+..++?+++++ +.............++++++++++?++++++++++++ +.............++++++++++?++++++++++++ +++++++++++++++++++++++++++++++++++++ +?????????? +????? ++++++++++++++++++++++++++. +??????????????++????????????+? + +Total parses: 4206 (100%) +Successful parses: 1542 (36%) +Successful recoveries: 1610 (60%) +Failed recoveries: 55 (2%) +Parse errors: 999 (37%) +Slow parses: 63 (2%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 1 0 4 + +----------------------------------------------------------- +Total test stats for |std:///lang/rascal/vis/ImportGraph.rsc|: + +Total parses: 8520 (100%) +Successful parses: 5357 (62%) +Successful recoveries: 2002 (63%) +Failed recoveries: 57 (1%) +Parse errors: 1104 (34%) +Slow parses: 67 (2%) + mean median 95% min max +Parse time ratios (log2(ratio)): 0 0 1 0 4 +ok ``` diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 6f606992f6..ff53cb0b7e 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -33,7 +33,6 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser startTime = realTime(); try { Tree t = recoveryParser(input, source); - rprintln(t); duration = realTime() - startTime; Tree best = findBestError(t); errorSize = size(getErrorText(best)); @@ -156,6 +155,9 @@ FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, & TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deleted=<"">|); stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); i = i+1; + if (i < len && substring(input, i, i+1) == "\n") { + println(); + } } return stats; @@ -183,6 +185,7 @@ FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (va stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); } lineStart = lineEnd+1; + println(); } return stats; From 5d256fd38728fab7a92b02509d93d2ed61c3c5d6 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 11:16:04 +0200 Subject: [PATCH 21/21] Fixed off-by-one error in newline printing --- .../rascal/tests/concrete/recovery/NOTES.md | 537 +++++++++--------- .../concrete/recovery/RecoveryTestSupport.rsc | 2 +- 2 files changed, 269 insertions(+), 270 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md index a4c0087718..3067de0465 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NOTES.md @@ -35,31 +35,30 @@ rascal>import lang::rascal::tests::concrete::recovery::ErrorRecoveryBenchmark; ☑ Loading modules 🕗 0:00:03.047 ok rascal>runLanguageTests(); -☑ Loading modules 🕐 0:00:02.331 -☑ Generating parser; RealLiteral Tags ModuleActuals Renamings KeywordFormal MidPathChars OptionalExpression Expression Class PathPart Signature ShellCommand Catch DataTarget KeywordFormals Renaming Tag Type LocationLiteral Declaration FunctionBody Field Range OptionalComma PrePathChars StringConstant Variable Type 🕛 0:00:12.763 -☑ Generating parser; MoreParameters Specifier Declarator TypeQualifier TranslationUnit AnonymousIdentifier FunctionDefinition MultiLineCommentBodyToken NonCommaExpression Expression Asterisk Declaration StorageClass StringConstant AbstractDeclarator PrototypeDeclarator GlobalDeclaration Comment Parameter Parameter 🕛 0:00:03.927 -Error recovery of |std:///lang/c90/syntax/C.rsc| (TranslationUnit) on |std:///lang/c90/examples/hello-world.c|, reference parse time: 47 ms. +☑ Generating parser; MoreParameters Specifier Declarator TypeQualifier TranslationUnit AnonymousIdentifier FunctionDefinition MultiLineCommentBodyToken NonCommaExpression Expression Asterisk Declaration StorageClass StringConstant AbstractDeclarator PrototypeDeclarator GlobalDeclaration Comment Parameter Parameter 🕛 0:00:04.556 +Error recovery of |std:///lang/c90/syntax/C.rsc| (TranslationUnit) on |std:///lang/c90/examples/hello-world.c|, reference parse time: 42 ms. Single char deletions: -..........+................+-. . -................+..........+.+ -........++......+++ -........+....++ -........++.+++ -.- +.........+................+-.. . -.........?....................+++.+........................................................ -.............? -...++.-.....+.++.? -................++++ -...+.++++.? -..........+.+.....?+ -...? +...............+..........+.+. +.......++......+++. +.......+....++. +.......++.+++. +-. . -.............+....+? -.? +........?....................+++.+......................................................... +............?. +..++.-.....+.++.?. +...............++++. +..+.++++.?. +.........+.+.....?+. +..?. . +............+....+?. +?. + Total parses: 314 (100%) Successful parses: 262 (83%) Successful recoveries: 41 (78%) @@ -110,50 +109,50 @@ Parse errors: 71 (28%) Slow parses: 0 (0%) mean median 95% min max Parse time ratios (log2(ratio)): 0 0 0 0 0 -☑ Generating parser; Range Content Month Date DiffFile Diff Hours Decimal TimeZone DateTime Sign ChunkStart Year Day Line Minutes Seconds Chunk Time Header Indicator FileName 🕛 0:00:01.032 -Error recovery of |std:///lang/diff/unified/UnifiedDiff.rsc| (DiffFile) on |std:///lang/diff/unified/examples/example.diff|, reference parse time: 20 ms. +☑ Generating parser; Range Content Month Date DiffFile Diff Hours Decimal TimeZone DateTime Sign ChunkStart Year Day Line Minutes Seconds Chunk Time Header Indicator FileName 🕛 0:00:00.677 +Error recovery of |std:///lang/diff/unified/UnifiedDiff.rsc| (DiffFile) on |std:///lang/diff/unified/examples/example.diff|, reference parse time: 10 ms. Single char deletions: -++++.................................................... -+++++.................................................... -++++++.++++.++++ -.+.................................. -.+ -.+.................... -.+........................ -.+................................. -.+..................... -.+ -.----...---...----.......................................................... -.-................................................... -.- -.-........................................................................................ -.-......................................................................................................... -.-......................................................................................................... -.-.............................................................................................................................................................................................................................................................................................................................. -.- -.-.......................... -.-. -.-................................................ -.????....???......???................................................................................. -.?.. -.? +++++....................................................+ +++++....................................................+ ++++++.++++.++++. ++................................... ++. ++..................... ++......................... ++.................................. ++...................... ++. +----...---...----........................................................... +-.................................................... +-. +-......................................................................................... +-.......................................................................................................... +-.......................................................................................................... +-............................................................................................................................................................................................................................................................................................................................... +-. +-........................... +-.. +-................................................. +????....???......???.................................................................................. +?... +?. ............... ......................................................... .................................................... ....... ........................................................ -.?................................................ -.?.. -.? -.?............................................................................. -.?................................................ -.?................................................................................................................................. -.?........................................ -.?.. -.? -.?. -? +?................................................. +?... +?. +?.............................................................................. +?................................................. +?.................................................................................................................................. +?......................................... +?... +?. +?.? + Total parses: 1776 (100%) Successful parses: 1702 (95%) Successful recoveries: 30 (40%) @@ -211,7 +210,7 @@ Failed recoveries: 26 (27%) Parse errors: 37 (39%) Slow parses: 0 (0%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 0 0 0 +Parse time ratios (log2(ratio)): 0 0 0 0 1 ----------------------------------------------------------- Total test stats for |std:///lang/diff/unified/examples/example.diff|: @@ -223,77 +222,77 @@ Failed recoveries: 47 (27%) Parse errors: 60 (35%) Slow parses: 0 (0%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 0 0 0 -☑ Generating parser; DotAttr NodeId Id EdgeStatement EdgeRhs Nod LAYOUT AttrStatement EdgeOp Comment AttrTag DOT Edg StatementList NodeStatement AttrList Statement StatementOptional Subgraph Graph AttrList0 Port 🕛 0:00:01.529 -Error recovery of |std:///lang/dot/syntax/Dot.rsc| (DOT) on |std:///lang/dot/examples/parser-state.dot|, reference parse time: 25 ms. +Parse time ratios (log2(ratio)): 0 0 0 0 1 +☑ Generating parser; DotAttr NodeId Id EdgeStatement EdgeRhs Nod LAYOUT AttrStatement EdgeOp Comment AttrTag DOT Edg StatementList NodeStatement AttrList Statement StatementOptional Subgraph Graph AttrList0 Port 🕛 0:00:01.346 +Error recovery of |std:///lang/dot/syntax/Dot.rsc| (DOT) on |std:///lang/dot/examples/parser-state.dot|, reference parse time: 38 ms. Single char deletions: -++??????.......? -.+......+-+.....+++...............+..................+................................................++. -.+......--+.....+++.....+..+.....+++......++. -.+..+-+.....+++............................................+.......+.............................+.......+.....++. -.-....--+.....+++..........................................................................+.......+....++. -.+.....+-+.....+++.................................................................................................................++. -.+..+++.....+++.............................................................++. -.+.....+.++.+..+. -.+....+.++.+.....+. -.+..+.++.+....+. -.+......+.+.+....++.+..+++.....+++.....++. -.+........+++.....+++.......+..+.....+++.......++. -.+......+.+.+....++.+........+++.....+++....++. -.+.........+.+.+.++.+......+. -.+.........+++.....+++.........................................................................................................................................+..+.....+++......++. -.+......+.++.+.........+++.....+++..........++. -.+..............+++.....++++..+.....+++......++. -.+......+.++.+..............+++.....+++................++. -.+.................+++.....+++.....+..+.....+++......+..+.....+++...++. -.+.................+.+.+....++.+..+++.....+++.....++. -.+.................+.+.+....++.+........+-+.....+++....++. -.+......+.++.+.................+-+.....+++...................++. -.+....................+-+.....++++..+.....+++......++. -.+......+.-+.+....................+-+.....+++.......................++. -.-...--+.....+++............................................+....+...................................+....+............+...........+......++. -.-...--+.....+++....................................................................++. -.+.....+++.....+++.............................................................................................++. -.+.....+.++.+..-. -.-...-.++.-.....-. -.-...-.++.-...-. -.-.................-.-.-.-+.-...-. -.-.....--+.....+++.................................................................++. -.-....-++.....+++.................................................+.......+.............................+.......+.....++. -.+....+.++.+....+. -.+.....+.++.+....+. -.+.................+.+.+.++.+.....+. -.+.................+-+.....+++.............+..+.....+++......++. -.+.....+++.....+++.........................................+....................+...............++. -.+....................+.+.+.++.+.....+. -.+...+++.....+++.........................................+.....+.............++. -.+....................+.+.+.++.+...+. -.+....+++.....+++.............................................................++. -.+....................+.+.+.++.+....+. -.+....+++.....+++....................................................+....+...............++. -.+....................+.+.+.-+.+....+. -.+.....+++.....+++..............................................................................................................................................................................................................................................................................................++. -.+....................+.+.+.++.+.....+. -.+.....+++.....+++....................................................+.......+................++. -.+....................+.+.+.++.+.....+. -.+....+++.....+++.....................................................++. -.+....................+.+.+.++.+....+. -.+....+++.....+++................................................++. -.+....................+.+.+.++.+....+. -.+....+++.....+++.........................................+........+..............++. -.+....................+.+.+.+?.+....+. -.+....................+++.....+++.....................................................................+..+.....+++......++. -.?.............................?++.....+++......+..+.....+++++. -.?.............?++.....++++..+.....+++......++. -.?.....?++.....+++......++. -.?......?.+?.?.....?++.....+++..............++. -.?.....?.+?.?.................?++.....+++............++. -.?.....?.+?.?....................?++.....+++.................++. -.?.....?.+?.?.............................?++.....+++....................++. -.?.....?.+?.?.............?++.....++?........??. -.? -. +++??????.......?. ++......+-+.....+++...............+..................+................................................++.. ++......--+.....+++.....+..+.....+++......++.. ++..+-+.....+++............................................+.......+.............................+.......+.....++.. +-....--+.....+++..........................................................................+.......+....++.. ++.....+-+.....+++.................................................................................................................++.. ++..+++.....+++.............................................................++.. ++.....+.++.+..+.. ++....+.++.+.....+.. ++..+.++.+....+.. ++......+.+.+....++.+..+++.....+++.....++.. ++........+++.....+++.......+..+.....+++.......++.. ++......+.+.+....++.+........+++.....+++....++.. ++.........+.+.+.++.+......+.. ++.........+++.....+++.........................................................................................................................................+..+.....+++......++.. ++......+.++.+.........+++.....+++..........++.. ++..............+++.....++++..+.....+++......++.. ++......+.++.+..............+++.....+++................++.. ++.................+++.....+++.....+..+.....+++......+..+.....+++...++.. ++.................+.+.+....++.+..+++.....+++.....++.. ++.................+.+.+....++.+........+-+.....+++....++.. ++......+.++.+.................+-+.....+++...................++.. ++....................+-+.....++++..+.....+++......++.. ++......+.-+.+....................+-+.....+++.......................++.. +-...--+.....+++............................................+....+...................................+....+............+...........+......++.. +-...--+.....+++....................................................................++.. ++.....+++.....+++.............................................................................................++.. ++.....+.++.+..-.. +-...-.++.-.....-.. +-...-.++.-...-.. +-.................-.-.-.-+.-...-.. +-.....--+.....+++.................................................................++.. +-....-++.....+++.................................................+.......+.............................+.......+.....++.. ++....+.++.+....+.. ++.....+.++.+....+.. ++.................+.+.+.++.+.....+.. ++.................+-+.....+++.............+..+.....+++......++.. ++.....+++.....+++.........................................+....................+...............++.. ++....................+.+.+.++.+.....+.. ++...+++.....+++.........................................+.....+.............++.. ++....................+.+.+.++.+...+.. ++....+++.....+++.............................................................++.. ++....................+.+.+.++.+....+.. ++....+++.....+++....................................................+....+...............++.. ++....................+.+.+.-+.+....+.. ++.....+++.....+++..............................................................................................................................................................................................................................................................................................++.. ++....................+.+.+.++.+.....+.. ++.....+++.....+++....................................................+.......+................++.. ++....................+.+.+.++.+.....+.. ++....+++.....+++.....................................................++.. ++....................+.+.+.++.+....+.. ++....+++.....+++................................................++.. ++....................+.+.+.++.+....+.. ++....+++.....+++.........................................+........+..............++.. ++....................+.+.+.+?.+....+.. ++....................+++.....+++.....................................................................+..+.....+++......++.. +?.............................?++.....+++......+..+.....+++++.. +?.............?++.....++++..+.....+++......++.. +?.....?++.....+++......++.. +?......?.+?.?.....?++.....+++..............++.. +?.....?.+?.?.................?++.....+++............++.. +?.....?.+?.?....................?++.....+++.................++.. +?.....?.+?.?.............................?++.....+++....................++.. +?.....?.+?.?.............?++.....++?........??.. +?. + Total parses: 4177 (100%) Successful parses: 3493 (83%) Successful recoveries: 599 (87%) @@ -377,7 +376,7 @@ Failed recoveries: 99 (2%) Parse errors: 235 (6%) Slow parses: 0 (0%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 0 0 1 +Parse time ratios (log2(ratio)): 0 0 0 0 0 ----------------------------------------------------------- Total test stats for |std:///lang/dot/examples/parser-state.dot|: @@ -389,25 +388,25 @@ Failed recoveries: 141 (3%) Parse errors: 278 (6%) Slow parses: 0 (0%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 0 0 1 -☑ Generating parser; SwitchBlock CharLiteral BinaryExponent ClassOrInterfaceType VarDec Super VarMod Id Type ClassBody Interfaces Asterisk CommentPart DimExpr ImportDec ArrayAccess PackageDec MethodSpec DeciFloatExponentPart BlockStm EnumConstArgs AbstractMethodDec DeciNumeral AbstractMethodMod StringLiteral Field 🕛 0:00:07.668 -Error recovery of |std:///lang/java/syntax/Java15.rsc| (CompilationUnit) on |jar+file:///D:/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|, reference parse time: 91 ms. +Parse time ratios (log2(ratio)): 0 0 0 0 0 +☑ Generating parser; SwitchBlock CharLiteral BinaryExponent ClassOrInterfaceType VarDec Super VarMod Id Type ClassBody Interfaces Asterisk CommentPart DimExpr ImportDec ArrayAccess PackageDec MethodSpec DeciFloatExponentPart BlockStm EnumConstArgs AbstractMethodDec DeciNumeral AbstractMethodMod StringLiteral Field 🕛 0:00:13.312 +Error recovery of |std:///lang/java/syntax/Java15.rsc| (CompilationUnit) on |jar+file:///D:/rascal/rascal/test/org/rascalmpl/test/data/m3/snakes-and-ladders-project-source.zip!/src/snakes/LastSquare.java|, reference parse time: 118 ms. Single char deletions: -++++++++......- +++++++++......-. . -.++++++.++++++..........+++++++++.......- +++++++.++++++..........+++++++++.......-. . -..++++++...........-....+....+....+........+.- -........+....+.........++ -..+ -. -..+........ -..++++++.....................++.+ -...++++++.....+ -..? -.? +.++++++...........-....+....+....+........+.-. +.......+....+.........++. +.+. . +.+......... +.++++++.....................++.+. +..++++++.....+. +.?. +?. + Total parses: 198 (100%) Successful parses: 131 (66%) Successful recoveries: 61 (91%) @@ -452,23 +451,23 @@ Parse errors: 5 (2%) Slow parses: 0 (0%) mean median 95% min max Parse time ratios (log2(ratio)): 0 0 0 0 0 -☑ Generating parser; Member Object Value NumericLiteral StringChar IntegerLiteral RealLiteral Array JSONText UnicodeEscape StringLiteral 🕛 0:00:00.691 -Error recovery of |std:///lang/json/syntax/JSON.rsc| (JSONText) on |std:///lang/json/examples/ex01.json|, reference parse time: 12 ms. +☑ Generating parser; Member Object Value NumericLiteral StringChar IntegerLiteral RealLiteral Array JSONText UnicodeEscape StringLiteral 🕛 0:00:01.228 +Error recovery of |std:///lang/json/syntax/JSON.rsc| (JSONText) on |std:///lang/json/examples/ex01.json|, reference parse time: 17 ms. Single char deletions: -?. -.......?.....??.?. -...........+.....++..+..?. -...........+......++.+..?. -...........+.....++..+....................??. -...........?.........??.?. -...............+...++....+......................................++. -...............+......-+....+. -...............-.....+-..+...-. -...........?+. -...........?...??.?...+....+....+......?. -.......?. -.? +?.. +......?.....??.?.. +..........+.....++..+..?.. +..........+......++.+..?.. +..........+.....++..+....................??.. +..........?.........??.?.. +..............+...++....+......................................++.. +..............+......-+....+.. +..............-.....+-..+...-.. +..........?+.. +..........?...??.?...+....+....+......?.. +......?.. +? Total parses: 337 (100%) Successful parses: 285 (84%) Successful recoveries: 27 (51%) @@ -512,29 +511,29 @@ Parse errors: 150 (49%) Slow parses: 0 (0%) mean median 95% min max Parse time ratios (log2(ratio)): 0 0 0 0 0 -☑ Generating parser; Type Id WhitespaceAndComment Statement Natural Program Declarations String Expression IdType 🕛 0:00:00.660 -Error recovery of |std:///lang/pico/syntax/Main.rsc| (Program) on |std:///lang/pico/examples/fac.pico|, reference parse time: 9 ms. +☑ Generating parser; Type Id WhitespaceAndComment Statement Natural Program Declarations String Expression IdType 🕛 0:00:01.139 +Error recovery of |std:///lang/pico/syntax/Main.rsc| (Program) on |std:///lang/pico/examples/fac.pico|, reference parse time: 16 ms. Single char deletions: -?????.-------.......+.+++++++-.. -......................+.++++++++...... -.....................+.++++++++ -...................+.++++++++ -...............+..+.+++++++ -...............+..+.------+ -.............++...+ -..............++.+- -.......-----.......+.-.--. -...............++.......+ -.................++......- -...........+++++.......+.+.++ -.....................++........+....+ -....................++.......+.+ -...........+++ -.................??.......+.+ -.......-- -.??? -. +?????.-------.......+.+++++++-... +.....................+.++++++++....... +....................+.++++++++. +..................+.++++++++. +..............+..+.+++++++. +..............+..+.------+. +............++...+. +.............++.+-. +......-----.......+.-.--.. +..............++.......+. +................++......-. +..........+++++.......+.+.++. +....................++........+....+. +...................++.......+.+. +..........+++. +................??.......+.+. +......--. +???. + Total parses: 454 (100%) Successful parses: 337 (74%) Successful recoveries: 81 (69%) @@ -584,125 +583,125 @@ Parse errors: 36 (8%) Slow parses: 0 (0%) mean median 95% min max Parse time ratios (log2(ratio)): 0 0 0 0 0 -Error recovery of |std:///lang/rascal/syntax/Rascal.rsc| (Module) on |std:///lang/rascal/vis/ImportGraph.rsc|, reference parse time: 20 ms. +Error recovery of |std:///lang/rascal/syntax/Rascal.rsc| (Module) on |std:///lang/rascal/vis/ImportGraph.rsc|, reference parse time: 17 ms. Single char deletions: -?........+...............................................................................................? -.?...........+ +?........+...............................................................................................?. +?...........+. ...................................................................................................... .................................................................................................... ................................................................................................................ ........................................................................................... -.? -.?........+ +?. +?........+. .................................................................................................... ............................................................................ ............................................................................................... .................... -.? -.?........+ +?. +?........+. ................................................................................................. -.? -.?............... -????????....++......++...++........... +?. +?...............? +???????....++......++...++............ . -.??????+....++..........+ -.??????+...++......+ -.??????+....++......++.......++..........++.......+ -.??????+....++......+++......++......+ -.??????+.........+ -.??????+....++..........+ -.??????+....++...........+ -.+++++++..+ -.+++++++........++......++.....- +??????+....++..........+. +??????+...++......+. +??????+....++......++.......++..........++.......+. +??????+....++......+++......++......+. +??????+.........+. +??????+....++..........+. +??????+....++...........+. ++++++++..+. ++++++++........++......++.....-. . -.+........+..............................................................................................? -.....?...........?.....................+..................+.? -................+!+.......+++............++.............................++ -.+ ++........+..............................................................................................?. +....?...........?.....................+..................+.?. +...............+!+.......+++............++.............................++!. ++. . -.+........?......................................................................................? -.....?...........?.....................+..................+.? -.....++..................................................................................... -..........+.....................+................+.........++++ -................+!.................................++ -.+ ++........?......................................................................................?. +....?...........?.....................+..................+.?. +....++...................................................................................... +.........+.....................+!................+.........++++. +...............+!.................................++. ++. . -.+........?.................................................................................................? -.....?...........?.....................+..................-!.? -.....+.+................+.........+? ++........?.................................................................................................?. +....?...........?.....................+..................-!.?. +....+!.+................+.........+?. ..... -.....++........................................................................... -+....+.+!.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+ -.......+.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+ -.....++....................................................................................... -.....++........................................................................................................................................... -+......+ +....++...........................................................................+ +....+!.+!.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+. +......+.+.+....+.+.++.?.+.+.+......?.+....+.......++...............++..?.++++++++.........+. +....++........................................................................................ +....++...........................................................................................................................................+ +......+. . -........+.........+........+.+ -.........++.+....+.++.+......+....?+ -.....................+ -.........++++. -.............++++++.+.......+...++ -.....+ +.......+.........+........+.+. +........++.+....+.++.+......+....?+. +....................+. +........++++.. +............++++++.+.......+...++. +....+. . -.....+++++++....+.........+......++.+.+.......+...++ +....+++++++....+.........+......++.+.+.......+...++. . -...........................+.....+...+.........................+++...........+.........+.......++..........................++..........+................+?++................+++++ -.+ +..........................+.....+...+.........................+++...........+.........+.......++..........................++..........+................+?++................+++++. ++. . -.+........+........................................................................................? -.+++++.............+.............+ -........+...+............+.+.+++ -........+...+...............+.+++ -........+...+....+.......+.+.+++ -........+...+....+.......+.+.+++ -........+...+....+.......+.++ -.+? ++........+........................................................................................?. ++++++.............+.............+. +.......+...+............+.+.+++. +.......+...+...............+.+++. +.......+...+....+.......+.+.+++. +.......+...+....+.......+.+.+++. +.......+...+....+.......+.++. ++?. . -.?........?..........................................................................................? -.............?...............?....+...+.....+.? -..............+.+.....+...+.+...++..............++ +?........?..........................................................................................?. +............?...............?....+...+.....+.?. +.............+!.+.....+...+.+...++..............++. ..... -............+.+...............+.+...+............++ +...........+.+...............+.+...+............++. . -................?.............+ -...............+.+.+.+.......+...+..........++ -...............+.+.+.+.......+...+..........++ -...............+.+.+.+.......+...+..........++ -.................+.+.+...........+..........+ -.....++ +...............?.............+. +..............+.+.+.+.......+...+..........++. +..............+.+.+.+.......+...+..........++. +..............+.+.+.+.......+...+..........++. +................+.+.+...........+..........+. +....++. . -.........................+..................+++..+..................+.+..+..................++ +........................+..................+++..+..................+.+..+..................++. . -......................+ -.? +.....................+. +?. . -.?........?.............................................................? -.............?...............?........?.? -.....+++.? -..................?............+....++ +?........?.............................................................?. +............?...............?........?.?. +....+++.?. +.................?............+....++. . -.........+....+.....+.....+.+.................+.++ +........+....+.....+.....+.+.................+.++. . -.........++++++.............+ -...................+.+.+....++ -...................+.+.+....+.+.....+ -...................+.+.+....+.+.....+ -.....................+.++....+.....++ -.........+? -.....? -.....++++++..........+.+.+. -............................++? -.? +........++++++.............+. +..................+.+.+....++. +..................+.+.+....+.+.....+. +..................+.+.+....+.+.....+. +....................+.++....+.....++. +........+?. +....?. +....++++++..........+.+.+.. +...........................++?. +? Total parses: 4314 (100%) Successful parses: 3815 (88%) Successful recoveries: 392 (78%) Failed recoveries: 2 (0%) Parse errors: 105 (21%) -Slow parses: 4 (0%) +Slow parses: 9 (1%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 1 0 4 +Parse time ratios (log2(ratio)): 0 0 2 0 5 Deletes until end-of-line: .+........???????????????????????????????????????????????????????????????????????????????????????????????? @@ -736,14 +735,14 @@ Deletes until end-of-line: .+........??????????????????????????????????????????????????????????????????????????????????????????????? ++++++???????????+++++++++++++++++++++++++++++++++++++++++?? -.....+!+!+!+!+!++!+!+!+!+!++++++++++++++++++++++++?+++++++++++++++++++++++++++++++ +.....+!+!+!+!+!++!+!+!+!+!++++++++++++++++++++++++?+++++++++++++++++++++++++++++++! + .+........??????????????????????????????????????????????????????????????????????????????????????? ++++++???????????+++++++++++++++++++++++++++++++++++++++++?? .....+..................................................................................... .....+++++..++++++++++++++++++++++++++++++++++++++++++++++++++ -.....+!+!+!+!+!++!+!+!+!+!+++++++++++++++++++++++++++++++++++ +.....+!+!+!+!+!++!+!+!+!+!+++++++++++++!++++++++++++++++++++++ + .+........?????????????????????????????????????????????????????????????????????????????????????????????????? @@ -766,7 +765,7 @@ Deletes until end-of-line: .....+++++++++++++++++++++++++++++++..+++++++++++++ -.....+!+!+!+!+!+!++!+!+!+!+!+!+!+!+!+!+!+!+!+!+!++++++????+????????????????????????????????+????????????????+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++?+????????????????????++ +.....+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!+!++++++????+????????????????????????????????+????????????????+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++?+????????????????????++ + .+........????????????????????????????????????????????????????????????????????????????????????????? @@ -818,9 +817,9 @@ Successful parses: 1542 (36%) Successful recoveries: 1610 (60%) Failed recoveries: 55 (2%) Parse errors: 999 (37%) -Slow parses: 63 (2%) +Slow parses: 66 (2%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 1 0 4 +Parse time ratios (log2(ratio)): 0 0 2 0 5 ----------------------------------------------------------- Total test stats for |std:///lang/rascal/vis/ImportGraph.rsc|: @@ -830,8 +829,8 @@ Successful parses: 5357 (62%) Successful recoveries: 2002 (63%) Failed recoveries: 57 (1%) Parse errors: 1104 (34%) -Slow parses: 67 (2%) +Slow parses: 75 (2%) mean median 95% min max -Parse time ratios (log2(ratio)): 0 0 1 0 4 +Parse time ratios (log2(ratio)): 0 0 2 0 5 ok ``` diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index ff53cb0b7e..b16ea8f0ec 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -154,10 +154,10 @@ FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, & str modifiedInput = substring(input, 0, i) + substring(input, i+1); TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deleted=<"">|); stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); - i = i+1; if (i < len && substring(input, i, i+1) == "\n") { println(); } + i = i+1; } return stats;