From a36039542e4c5db686005adf5f09bc678e817744 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 14:21:39 +0200 Subject: [PATCH 01/11] Reimplemented error tree disambiguation in Java --- src/org/rascalmpl/library/ParseTree.rsc | 32 +--- .../concrete/recovery/ErrorRecovery.java | 181 ++++++++++++++++++ .../recovery/ErrorRecoveryBenchmark.rsc | 4 +- ...Loop2Bug.rsc => SlowDisambiguationBug.rsc} | 2 +- 4 files changed, 187 insertions(+), 32 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecovery.java rename src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/{InfiniteLoop2Bug.rsc => SlowDisambiguationBug.rsc} (86%) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index cbacbb70c4..fa9a43b4e3 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -784,7 +784,7 @@ list[Tree] findAllErrors(Tree tree) = [err | /err:appl(error(_, _, _), _) := tr Tree findFirstError(/err:appl(error(_, _, _), _)) = err; @synopsis{Find the best error from a tree containing errors. This function will fail if `tree` does not contain an error.} -Tree findBestError(Tree tree) = findFirstError(defaultErrorDisambiguationFilter(tree)); +Tree findBestError(Tree tree) = findFirstError(disambiguateErrors(tree)); @synopsis{Get the symbol (sort) of the failing production} Symbol getErrorSymbol(appl(error(Symbol sym, _, _), _)) = sym; @@ -803,35 +803,9 @@ If you want the text of the whole error tree, you can just use string interpolat } str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) = stringChars([c | char(c) <- chars]); +@javaClass{org.rascalmpl.library.lang.rascal.tests.concrete.recovery.ErrorRecovery} @synopsis{Error recovery often produces ambiguous trees where errors can be recovered in multiple ways. This filter removes error trees until no ambiguities caused by error recovery are left. Note that regular ambiguous trees remain in the parse forest. } -Tree defaultErrorDisambiguationFilter(Tree t) { - return visit(t) { - case a:amb(_) => ambDisambiguation(a) - }; -} - -private Tree ambDisambiguation(amb(set[Tree] alternatives)) { - // Go depth-first - rel[int score, Tree alt] scoredErrorTrees = { | Tree alt <- alternatives }; - set[Tree] nonErrorTrees = scoredErrorTrees[0]; - - if (nonErrorTrees == {}) { - return (getFirstFrom(scoredErrorTrees) | it.score > c.score ? c : it | c <- scoredErrorTrees).alt; - } - - if ({Tree single} := nonErrorTrees) { - // One ambiguity left, no ambiguity concerns here - return single; - } - - // Multiple non-error trees left, return an ambiguity node with just the non-error trees - return amb(nonErrorTrees); -} - -private int scoreErrors(Tree t) = (0 | it + getSkipped(e).src.length | /e:appl(error(_,_,_),_) := t); - -// Handle char and cycle nodes -default Tree defaultErrorDisambiguationFilter(Tree t) = t; +java Tree disambiguateErrors(Tree t); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecovery.java b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecovery.java new file mode 100644 index 0000000000..39ed617c59 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecovery.java @@ -0,0 +1,181 @@ +package org.rascalmpl.library.lang.rascal.tests.concrete.recovery; + +import java.io.PrintWriter; + +import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.interpreter.utils.IResourceLocationProvider; +import org.rascalmpl.values.IRascalValueFactory; +import org.rascalmpl.values.RascalValueFactory; +import org.rascalmpl.values.parsetrees.ITree; +import org.rascalmpl.values.parsetrees.ProductionAdapter; +import org.rascalmpl.values.parsetrees.TreeAdapter; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IList; +import io.usethesource.vallang.IListWriter; +import io.usethesource.vallang.ISet; +import io.usethesource.vallang.ISetWriter; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.IValueFactory; +import io.usethesource.vallang.type.Type; +import io.usethesource.vallang.type.TypeStore; + +public class ErrorRecovery { + private final IValueFactory values; + private final IRascalValueFactory rascalValues; + private final PrintWriter out; + private final TypeStore store; + + public ErrorRecovery(IValueFactory values, IRascalValueFactory rascalValues, PrintWriter out, TypeStore store, IRascalMonitor monitor, IResourceLocationProvider resourceProvider) { + super(); + + this.values = values; + this.rascalValues = rascalValues; + this.store = store; + this.out = out; + } + + private static class ScoredTree { + public final IConstructor tree; + public final int score; + + public ScoredTree(IConstructor tree, int score) { + this.tree = tree; + this.score = score; + } + } + + /** + * Disambiguate error trees, faster implementation of the original Rascal variant: +Tree defaultErrorDisambiguationFilter(Tree t) { + return visit(t) { + case a:amb(_) => ambDisambiguation(a) + }; +} + +private Tree ambDisambiguation(amb(set[Tree] alternatives)) { + // Go depth-first + rel[int score, Tree alt] scoredErrorTrees = { | Tree alt <- alternatives }; + set[Tree] nonErrorTrees = scoredErrorTrees[0]; + + if (nonErrorTrees == {}) { + return (getFirstFrom(scoredErrorTrees) | it.score > c.score ? c : it | c <- scoredErrorTrees).alt; + } + + if ({Tree single} := nonErrorTrees) { + // One ambiguity left, no ambiguity concerns here + return single; + } + + // Multiple non-error trees left, return an ambiguity node with just the non-error trees + return amb(nonErrorTrees); +} + +private int scoreErrors(Tree t) = (0 | it + getSkipped(e).src.length | /e:appl(error(_,_,_),_) := t); + +// Handle char and cycle nodes +default Tree defaultErrorDisambiguationFilter(Tree t) = t; + **/ + + public IConstructor disambiguateErrors(IConstructor arg) { + long start = System.currentTimeMillis(); + IConstructor result = disambiguate(arg).tree; + long duration = System.currentTimeMillis() - start; + System.err.println("disambiguateErrors took " + duration + " ms."); + return result; + } + + private ScoredTree disambiguate(IConstructor tree) { + Type type = tree.getConstructorType(); + + if (type == RascalValueFactory.Tree_Appl) { + return disambiguateAppl((ITree) tree); + } else if (type == RascalValueFactory.Tree_Amb) { + return disambiguateAmb(tree); + } + + return new ScoredTree(tree, 0); + } + + private ScoredTree disambiguateAppl(ITree appl) { + if (ProductionAdapter.isSkipped(appl.getProduction())) { + return new ScoredTree(appl, ((IList) appl.get(1)).length()); + } + + IList args = TreeAdapter.getArgs(appl); + int totalScore = 0; + IValue[] disambiguatedArgs = new IValue[args.size()]; + + int index=0; + boolean anyChanges = false; + + // Disambiguate and score all children + for (IValue arg : args) { + ScoredTree disambiguatedArg = disambiguate((IConstructor) arg); + totalScore += disambiguatedArg.score; + disambiguatedArgs[index++] = disambiguatedArg.tree; + if (disambiguatedArg.tree != arg) { + anyChanges = true; + } + } + + // Only build a new tree if at least one of the arguments has changed + ITree resultTree; + if (anyChanges) { + IListWriter argWriter = rascalValues.listWriter(); + for (IValue arg : disambiguatedArgs) { + argWriter.append(arg); + } + resultTree = TreeAdapter.setArgs(appl, argWriter.done()); + } else { + resultTree = appl; + } + + + return new ScoredTree(resultTree, totalScore); + } + + private ScoredTree disambiguateAmb(IConstructor amb) { + ISet alts = (ISet) amb.get(0); + + ISetWriter nonErrorAlts = null; + ScoredTree bestErrorAlt = null; + for (IValue alt : alts) { + ScoredTree disambiguatedAlt = disambiguate((IConstructor) alt); + if (disambiguatedAlt.score == 0) { + // Non-error tree + if (nonErrorAlts == null) { + nonErrorAlts = values.setWriter(); + } + nonErrorAlts.insert(disambiguatedAlt.tree); + } else { + // Only keep the best of the error trees + if (bestErrorAlt == null || bestErrorAlt.score > disambiguatedAlt.score) { + bestErrorAlt = disambiguatedAlt; + } + } + } + + if (nonErrorAlts == null) { + return bestErrorAlt; + } + + ISet newAlts = nonErrorAlts.done(); + int altCount = newAlts.size(); + + IConstructor resultTree; + if (altCount == alts.size()) { + // All children are without errors, return the original tree + resultTree = amb; + } else if (altCount == 1) { + // One child without errors remains, dissolve the amb tree + resultTree = (IConstructor)newAlts.iterator().next(); + } else { + // Create a new amb tree with the remaining non-error trees + resultTree = rascalValues.amb(newAlts); + } + + return new ScoredTree(resultTree, 0); + } + +} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc index 6642b08d60..b3fa1063fa 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -38,7 +38,7 @@ void runRascalBatchTest(int maxFiles=1000, int maxFileSize=4000) { TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", maxFiles, maxFileSize); int duration = realTime() - startTime; println(); - println("========================im========================================"); + println("================================================================"); println("Rascal batch test done in seconds, total result:"); printStats(stats); } @@ -50,7 +50,7 @@ int main(list[str] args) { maxFiles = toInt(args[0]); maxFileSize = toInt(args[1]); } else if (size(args) != 0) { - println("Usage: ErrorRecoveryBenchmark "); + println("Usage: ErrorRecoveryBenchmark \ \"); } runRascalBatchTest(maxFiles=maxFiles, maxFileSize=maxFileSize); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoop2Bug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/SlowDisambiguationBug.rsc similarity index 86% rename from src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoop2Bug.rsc rename to src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/SlowDisambiguationBug.rsc index 4beae613ec..a7b6759a64 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/InfiniteLoop2Bug.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/SlowDisambiguationBug.rsc @@ -1,4 +1,4 @@ -module lang::rascal::tests::concrete::recovery::bugs::InfiniteLoop2Bug +module lang::rascal::tests::concrete::recovery::bugs::SlowDisambiguationBug import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; import lang::rascal::\syntax::Rascal; From d4543e79bfa5433937e33a1ab1fa0d07bbd0e0eb Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 14:26:34 +0200 Subject: [PATCH 02/11] Removed references to obsolete 'defaultErrorDisambiguationFilter' --- .../rascal/tests/concrete/recovery/NestedRecoveryTests.rsc | 2 +- .../rascal/tests/concrete/recovery/RascalRecoveryTests.rsc | 7 +++---- .../tests/concrete/recovery/ToyRascalRecoveryTests.rsc | 3 +-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NestedRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NestedRecoveryTests.rsc index 528c20d9eb..d82a2ac736 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NestedRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NestedRecoveryTests.rsc @@ -35,5 +35,5 @@ test bool nestedOk() { test bool nestedTypo() { Tree t = parseS("a b x c"); - return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x "; + return getErrorText(findBestError(t)) == "x "; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc index 9e0332422c..39c303217c 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc @@ -28,8 +28,7 @@ Tree parseRascal(type[&T] t, str input, bool visualize=false) { println("- "); } - Tree disambiguated = defaultErrorDisambiguationFilter(result); - println("Best error: "); + println("Best error: "); } return result; @@ -92,7 +91,7 @@ test bool rascalMissingCloseParen() { Tree t = parseRascal("module A void f({} void g(){}"); assert getErrorText(findFirstError(t)) == "void g("; - assert getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "("; + assert getErrorText(findBestError(t)) == "("; return true; } @@ -102,7 +101,7 @@ test bool rascalFunctionDeclarationMissingCloseParen() { assert getErrorText(findFirstError(t)) == "void g("; - Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); + Tree error = findBest(t); assert getErrorText(error) == "("; loc location = getSkipped(error).src; assert location.begin.column == 16 && location.length == 1; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ToyRascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ToyRascalRecoveryTests.rsc index 75c7f47899..4e6122ff3c 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ToyRascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ToyRascalRecoveryTests.rsc @@ -28,8 +28,7 @@ Tree parseToyRascal(str input, bool visualize=false) { println("- "); } - Tree disambiguated = defaultErrorDisambiguationFilter(result); - println("Best error: "); + println("Best error: "); } return result; From 2305a54650c3e4b58121c1592ca45c100fdd2354 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 2 Oct 2024 15:21:45 +0200 Subject: [PATCH 03/11] Added support for "automatic" disambiguation of error trees --- src/org/rascalmpl/library/ParseTree.rsc | 6 +-- .../concrete/recovery/BasicRecoveryTests.rsc | 12 +++++- ...very.java => ParseErrorDisambiguator.java} | 43 ++++++++----------- .../concrete/recovery/PicoRecoveryTests.rsc | 2 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 19 ++------ .../values/RascalFunctionValueFactory.java | 10 ++++- 6 files changed, 45 insertions(+), 47 deletions(-) rename src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/{ErrorRecovery.java => ParseErrorDisambiguator.java} (80%) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index fa9a43b4e3..426dc1ff28 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -803,9 +803,9 @@ If you want the text of the whole error tree, you can just use string interpolat } str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) = stringChars([c | char(c) <- chars]); -@javaClass{org.rascalmpl.library.lang.rascal.tests.concrete.recovery.ErrorRecovery} +@javaClass{org.rascalmpl.library.lang.rascal.tests.concrete.recovery.ParseErrorDisambiguator} @synopsis{Error recovery often produces ambiguous trees where errors can be recovered in multiple ways. This filter removes error trees until no ambiguities caused by error recovery are left. -Note that regular ambiguous trees remain in the parse forest. +Note that regular ambiguous trees remain in the parse forest unless `allowAmbiguity` is set to false in which case an error is thrown. } -java Tree disambiguateErrors(Tree t); +java Tree disambiguateErrors(Tree t, bool allowAmbiguity=true); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc index 23b3af5d2f..8e372f7732 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc @@ -42,6 +42,14 @@ test bool axc() { } test bool ax() { - Tree t = parseS("a x $"); - return getErrorText(findBestError(t)) == "x "; + str input = "a x $"; + + Tree t = parseS(input); + assert size(findAllErrors(t)) == 3; + assert getErrorText(findBestError(t)) == "x "; + + Tree autoDisambiguated = parser(#S, allowRecovery=true, allowAmbiguity=false)(input, |unknown:///|); + assert size(findAllErrors(autoDisambiguated)) == 1; + + return getErrorText(findFirstError(autoDisambiguated)) == getErrorText(findBestError(t)); } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecovery.java b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ParseErrorDisambiguator.java similarity index 80% rename from src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecovery.java rename to src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ParseErrorDisambiguator.java index 39ed617c59..d2e99f71a0 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecovery.java +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ParseErrorDisambiguator.java @@ -1,38 +1,29 @@ package org.rascalmpl.library.lang.rascal.tests.concrete.recovery; -import java.io.PrintWriter; - -import org.rascalmpl.debug.IRascalMonitor; -import org.rascalmpl.interpreter.utils.IResourceLocationProvider; +import org.rascalmpl.interpreter.asserts.Ambiguous; import org.rascalmpl.values.IRascalValueFactory; import org.rascalmpl.values.RascalValueFactory; import org.rascalmpl.values.parsetrees.ITree; import org.rascalmpl.values.parsetrees.ProductionAdapter; import org.rascalmpl.values.parsetrees.TreeAdapter; +import io.usethesource.vallang.IBool; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IList; import io.usethesource.vallang.IListWriter; import io.usethesource.vallang.ISet; import io.usethesource.vallang.ISetWriter; import io.usethesource.vallang.IValue; -import io.usethesource.vallang.IValueFactory; import io.usethesource.vallang.type.Type; -import io.usethesource.vallang.type.TypeStore; -public class ErrorRecovery { - private final IValueFactory values; +public class ParseErrorDisambiguator { private final IRascalValueFactory rascalValues; - private final PrintWriter out; - private final TypeStore store; + private boolean allowAmbiguity; - public ErrorRecovery(IValueFactory values, IRascalValueFactory rascalValues, PrintWriter out, TypeStore store, IRascalMonitor monitor, IResourceLocationProvider resourceProvider) { + public ParseErrorDisambiguator(IRascalValueFactory rascalValues) { super(); - this.values = values; this.rascalValues = rascalValues; - this.store = store; - this.out = out; } private static class ScoredTree { @@ -77,12 +68,9 @@ private Tree ambDisambiguation(amb(set[Tree] alternatives)) { default Tree defaultErrorDisambiguationFilter(Tree t) = t; **/ - public IConstructor disambiguateErrors(IConstructor arg) { - long start = System.currentTimeMillis(); - IConstructor result = disambiguate(arg).tree; - long duration = System.currentTimeMillis() - start; - System.err.println("disambiguateErrors took " + duration + " ms."); - return result; + public synchronized IConstructor disambiguateErrors(IConstructor arg, IBool allowAmbiguity) { + this.allowAmbiguity = allowAmbiguity != null && allowAmbiguity.getValue(); + return disambiguate(arg).tree; } private ScoredTree disambiguate(IConstructor tree) { @@ -91,7 +79,7 @@ private ScoredTree disambiguate(IConstructor tree) { if (type == RascalValueFactory.Tree_Appl) { return disambiguateAppl((ITree) tree); } else if (type == RascalValueFactory.Tree_Amb) { - return disambiguateAmb(tree); + return disambiguateAmb((ITree) tree); } return new ScoredTree(tree, 0); @@ -135,7 +123,7 @@ private ScoredTree disambiguateAppl(ITree appl) { return new ScoredTree(resultTree, totalScore); } - private ScoredTree disambiguateAmb(IConstructor amb) { + private ScoredTree disambiguateAmb(ITree amb) { ISet alts = (ISet) amb.get(0); ISetWriter nonErrorAlts = null; @@ -145,7 +133,7 @@ private ScoredTree disambiguateAmb(IConstructor amb) { if (disambiguatedAlt.score == 0) { // Non-error tree if (nonErrorAlts == null) { - nonErrorAlts = values.setWriter(); + nonErrorAlts = rascalValues.setWriter(); } nonErrorAlts.insert(disambiguatedAlt.tree); } else { @@ -163,16 +151,21 @@ private ScoredTree disambiguateAmb(IConstructor amb) { ISet newAlts = nonErrorAlts.done(); int altCount = newAlts.size(); - IConstructor resultTree; + ITree resultTree; if (altCount == alts.size()) { // All children are without errors, return the original tree resultTree = amb; } else if (altCount == 1) { // One child without errors remains, dissolve the amb tree - resultTree = (IConstructor)newAlts.iterator().next(); + resultTree = (ITree) newAlts.iterator().next(); } else { // Create a new amb tree with the remaining non-error trees resultTree = rascalValues.amb(newAlts); + + // We have an ambiguity between non-error trees + if (!allowAmbiguity) { + throw new Ambiguous(resultTree); + } } return new ScoredTree(resultTree, 0); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoRecoveryTests.rsc index 564b8840ee..729545b085 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoRecoveryTests.rsc @@ -113,4 +113,4 @@ test bool picoMissingSemiSmall() { end"); return checkError(t, "output := 0\n od"); -} \ No newline at end of file +} diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index ced74501b4..a296939595 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -10,7 +10,6 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URI; -import java.util.concurrent.atomic.AtomicBoolean; import org.rascalmpl.parser.gtd.debug.IDebugListener; import org.rascalmpl.parser.gtd.exception.ParseError; @@ -881,18 +880,8 @@ private void move(AbstractStackNode

node, AbstractNode result) { } if (node.isEndNode()) { - if (!result.isEmpty() || node.getId() == AbstractExpandableStackNode.DEFAULT_LIST_EPSILON_ID) { // Only go - // into the - // nullable - // fix path - // for - // nullables - // (special - // list - // epsilons - // can be - // ignored - // as well). + // Only go into the nullable fix path for nullables (special list epsilons can be ignored as well). + if (!result.isEmpty() || node.getId() == AbstractExpandableStackNode.DEFAULT_LIST_EPSILON_ID) { updateEdges(node, result); } else { @@ -1357,8 +1346,8 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] * Initiates parsing. */ @SuppressWarnings("unchecked") - protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] input, IRecoverer

recoverer, - IDebugListener

debugListener) { + protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] input, + IRecoverer

recoverer, IDebugListener

debugListener) { if (debugListener == null) { debugListener = new NopDebugListener<>(); } diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 1725085649..a9e06ed779 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -40,6 +40,7 @@ import org.rascalmpl.interpreter.result.ResultFactory; import org.rascalmpl.interpreter.staticErrors.UndeclaredNonTerminal; import org.rascalmpl.library.lang.rascal.syntax.RascalParser; +import org.rascalmpl.library.lang.rascal.tests.concrete.recovery.ParseErrorDisambiguator; import org.rascalmpl.parser.ParserGenerator; import org.rascalmpl.parser.gtd.IGTD; import org.rascalmpl.parser.gtd.debug.IDebugListener; @@ -583,7 +584,14 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in recoverer = new ToTokenRecoverer(uri, parserInstance, new StackNodeIdDispenser(parserInstance)); //debugListener = new DebugLogger(new PrintWriter(System.out, true)); } - return (ITree) parserInstance.parse(methodName, uri, input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); + ITree parseForest = (ITree) parserInstance.parse(methodName, uri, input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowRecovery || allowAmbiguity), recoverer, debugListener); + + if (!allowAmbiguity && allowRecovery && (filters == null || filters.isEmpty())) { + // Filter error-induced ambiguities + parseForest = (ITree) new ParseErrorDisambiguator((RascalValueFactory) ValueFactoryFactory.getValueFactory()).disambiguateErrors(parseForest, null); + } + + return parseForest; } } From 2eb653e6e35f2796eec17f739833310b08919cd2 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 3 Oct 2024 11:42:41 +0200 Subject: [PATCH 04/11] Error disambiguation can actually return a tree without errors. Also, optimized the case where no changes are detected in subtrees. --- src/org/rascalmpl/library/ParseTree.rsc | 22 ++- .../concrete/recovery/BasicRecoveryTests.rsc | 9 +- .../concrete/recovery/NestedRecoveryTests.rsc | 3 +- .../recovery/ParseErrorDisambiguator.java | 126 +++++++----------- .../concrete/recovery/PicoRecoveryTests.rsc | 3 +- .../concrete/recovery/RascalRecoveryTests.rsc | 11 +- .../concrete/recovery/RecoveryTestSupport.rsc | 76 ++++++----- .../recovery/ToyRascalRecoveryTests.rsc | 9 +- .../recovery/bugs/NoErrorsAfterDisambBug.rsc | 14 ++ .../values/RascalFunctionValueFactory.java | 6 +- 10 files changed, 154 insertions(+), 125 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/NoErrorsAfterDisambBug.rsc diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 426dc1ff28..948f3a132b 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -144,7 +144,7 @@ extend Message; extend List; import String; -import Set; +import util::Maybe; @synopsis{The Tree data type as produced by the parser.} @description{ @@ -355,6 +355,16 @@ The latter option terminates much faster, i.e. always in cubic time, and always while constructing ambiguous parse forests may grow to O(n^p+1), where p is the length of the longest production rule and n is the length of the input. +The `allowRecovery` can be set to `true` to enable error recovery. This is an experimental feature. +When error recovery is enabled, the parser will attempt to recover from parse errors and continue parsing. +If successful, a parse tree with error and skipped productions is returned (see the definition of `Production` above). +A number of functions is provided to analyze trees with errors, for example `hasErrors`, `getSkipped`, and `getErrorText`. +Note that the resulting parse forest can contain a lot of error nodes. `disambiguateErrors` can be used to prune the forest +and leave a tree with a single (or even zero) errors based on simple heuristics. +When `allowAmbiguity` is set to false, `allowRecovery` is set to true, and `filters` is empty, this disambiguation is done +automatically so you should end up with a tree with no error ambiguities. Regular ambiguities can still occur +and will result in an error. + The `filters` set contains functions which may be called optionally after the parse algorithm has finished and just before the Tree representation is built. The set of functions contain alternative functions, only on of them is successfully applied to each node in a tree. If such a function fails to apply, the other ones are tried. There is no fixed-point computation, so @@ -784,7 +794,15 @@ list[Tree] findAllErrors(Tree tree) = [err | /err:appl(error(_, _, _), _) := tr Tree findFirstError(/err:appl(error(_, _, _), _)) = err; @synopsis{Find the best error from a tree containing errors. This function will fail if `tree` does not contain an error.} -Tree findBestError(Tree tree) = findFirstError(disambiguateErrors(tree)); +Maybe[Tree] findBestError(Tree tree) { + Tree disambiguated = disambiguateErrors(tree); + if (/err:appl(error(_, _, _), _) := disambiguated) { + return just(err); + } + + // All errors have disappeared + return nothing(); +} @synopsis{Get the symbol (sort) of the failing production} Symbol getErrorSymbol(appl(error(Symbol sym, _, _), _)) = sym; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc index 8e372f7732..5464f3e977 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/BasicRecoveryTests.rsc @@ -15,6 +15,7 @@ module lang::rascal::tests::concrete::recovery::BasicRecoveryTests import ParseTree; +import util::Maybe; layout Layout = [\ ]* !>> [\ ]; @@ -33,12 +34,12 @@ test bool basicOk() { test bool abx() { Tree t = parseS("a b x $"); - return getErrorText(findBestError(t)) == "x "; + return getErrorText(findBestError(t).val) == "x "; } test bool axc() { Tree t = parseS("a x c $"); - return getErrorText(findBestError(t)) == "x c"; + return getErrorText(findBestError(t).val) == "x c"; } test bool ax() { @@ -46,10 +47,10 @@ test bool ax() { Tree t = parseS(input); assert size(findAllErrors(t)) == 3; - assert getErrorText(findBestError(t)) == "x "; + assert getErrorText(findBestError(t).val) == "x "; Tree autoDisambiguated = parser(#S, allowRecovery=true, allowAmbiguity=false)(input, |unknown:///|); assert size(findAllErrors(autoDisambiguated)) == 1; - return getErrorText(findFirstError(autoDisambiguated)) == getErrorText(findBestError(t)); + return getErrorText(findFirstError(autoDisambiguated)) == getErrorText(findBestError(t).val); } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NestedRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NestedRecoveryTests.rsc index d82a2ac736..4d4804711b 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NestedRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/NestedRecoveryTests.rsc @@ -15,6 +15,7 @@ module lang::rascal::tests::concrete::recovery::NestedRecoveryTests import ParseTree; +import util::Maybe; layout Layout = [\ ]* !>> [\ ]; @@ -35,5 +36,5 @@ test bool nestedOk() { test bool nestedTypo() { Tree t = parseS("a b x c"); - return getErrorText(findBestError(t)) == "x "; + return getErrorText(findBestError(t).val) == "x "; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ParseErrorDisambiguator.java b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ParseErrorDisambiguator.java index d2e99f71a0..a032d7b7d3 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ParseErrorDisambiguator.java +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ParseErrorDisambiguator.java @@ -17,13 +17,12 @@ import io.usethesource.vallang.type.Type; public class ParseErrorDisambiguator { - private final IRascalValueFactory rascalValues; + private final IRascalValueFactory rascalValues; private boolean allowAmbiguity; public ParseErrorDisambiguator(IRascalValueFactory rascalValues) { - super(); - - this.rascalValues = rascalValues; + super(); + this.rascalValues = rascalValues; } private static class ScoredTree { @@ -37,130 +36,107 @@ public ScoredTree(IConstructor tree, int score) { } /** - * Disambiguate error trees, faster implementation of the original Rascal variant: -Tree defaultErrorDisambiguationFilter(Tree t) { - return visit(t) { - case a:amb(_) => ambDisambiguation(a) - }; -} - -private Tree ambDisambiguation(amb(set[Tree] alternatives)) { - // Go depth-first - rel[int score, Tree alt] scoredErrorTrees = { | Tree alt <- alternatives }; - set[Tree] nonErrorTrees = scoredErrorTrees[0]; - - if (nonErrorTrees == {}) { - return (getFirstFrom(scoredErrorTrees) | it.score > c.score ? c : it | c <- scoredErrorTrees).alt; - } - - if ({Tree single} := nonErrorTrees) { - // One ambiguity left, no ambiguity concerns here - return single; - } - - // Multiple non-error trees left, return an ambiguity node with just the non-error trees - return amb(nonErrorTrees); -} - -private int scoreErrors(Tree t) = (0 | it + getSkipped(e).src.length | /e:appl(error(_,_,_),_) := t); - -// Handle char and cycle nodes -default Tree defaultErrorDisambiguationFilter(Tree t) = t; - **/ + * Disambiguate error trees. Error recovery often produces ambiguous trees where errors can be recovered in multiple ways. + * This filter removes error trees until no ambiguities caused by error recovery are left. + * Regular ambiguous trees remain in the parse forest unless `allowAmbiguity` is set to false in which case an error is thrown. + * It can happen that the original tree has errors but the result does not: + * when a tree has an ambiguity where one branch has an error and the other has not the error branch is pruned leaving an error-free tree + * possibly without ambiguities. + */ public synchronized IConstructor disambiguateErrors(IConstructor arg, IBool allowAmbiguity) { - this.allowAmbiguity = allowAmbiguity != null && allowAmbiguity.getValue(); - return disambiguate(arg).tree; + return disambiguate(arg, allowAmbiguity.getValue()).tree; } - private ScoredTree disambiguate(IConstructor tree) { + private ScoredTree disambiguate(IConstructor tree, boolean allowAmbiguity) { Type type = tree.getConstructorType(); - if (type == RascalValueFactory.Tree_Appl) { - return disambiguateAppl((ITree) tree); + if (type == RascalValueFactory.Tree_Appl) { + return disambiguateAppl((ITree) tree, allowAmbiguity); } else if (type == RascalValueFactory.Tree_Amb) { - return disambiguateAmb((ITree) tree); + return disambiguateAmb((ITree) tree, allowAmbiguity); } + // Other trees (cycle, char) do not have subtrees so they have a score of 0 return new ScoredTree(tree, 0); } - private ScoredTree disambiguateAppl(ITree appl) { + private ScoredTree disambiguateAppl(ITree appl, boolean allowAmbiguity) { if (ProductionAdapter.isSkipped(appl.getProduction())) { return new ScoredTree(appl, ((IList) appl.get(1)).length()); } IList args = TreeAdapter.getArgs(appl); int totalScore = 0; - IValue[] disambiguatedArgs = new IValue[args.size()]; - - int index=0; - boolean anyChanges = false; + IListWriter disambiguatedArgs = null; // Disambiguate and score all children - for (IValue arg : args) { - ScoredTree disambiguatedArg = disambiguate((IConstructor) arg); + for (int i=0; i disambiguatedAlt.score) { - bestErrorAlt = disambiguatedAlt; + if (errorAltWithBestScore == null || errorAltWithBestScore.score > disambiguatedAlt.score) { + errorAltWithBestScore = disambiguatedAlt; } } } - if (nonErrorAlts == null) { - return bestErrorAlt; + if (alternativesWithoutErrors == null) { + assert errorAltWithBestScore != null : "No trees with and no trees without errors?"; + return errorAltWithBestScore; } - ISet newAlts = nonErrorAlts.done(); - int altCount = newAlts.size(); + ISet remainingAlts = alternativesWithoutErrors.done(); ITree resultTree; - if (altCount == alts.size()) { + if (remainingAlts.size() == originalAlts.size()) { // All children are without errors, return the original tree resultTree = amb; - } else if (altCount == 1) { + } else if (remainingAlts.size() == 1) { // One child without errors remains, dissolve the amb tree - resultTree = (ITree) newAlts.iterator().next(); + resultTree = (ITree) remainingAlts.iterator().next(); } else { // Create a new amb tree with the remaining non-error trees - resultTree = rascalValues.amb(newAlts); + resultTree = rascalValues.amb(remainingAlts); // We have an ambiguity between non-error trees if (!allowAmbiguity) { diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoRecoveryTests.rsc index 729545b085..0f705e5c18 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/PicoRecoveryTests.rsc @@ -20,12 +20,13 @@ import ParseTree; import IO; import String; +import util::Maybe; Tree parsePico(str input, bool visualize=false) = parser(#Program, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); bool checkError(Tree t, str expectedError) { - str bestError = getErrorText(findBestError(t)); + str bestError = getErrorText(findBestError(t).val); println("best error: , expected: "); return size(bestError) == size(expectedError); } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc index 39c303217c..6183465762 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RascalRecoveryTests.rsc @@ -18,6 +18,7 @@ import lang::rascal::\syntax::Rascal; import ParseTree; import IO; +import util::Maybe; Tree parseRascal(type[&T] t, str input, bool visualize=false) { Tree result = parser(t, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); @@ -28,7 +29,7 @@ Tree parseRascal(type[&T] t, str input, bool visualize=false) { println("- "); } - println("Best error: "); + println("Best error: "); } return result; @@ -91,7 +92,7 @@ test bool rascalMissingCloseParen() { Tree t = parseRascal("module A void f({} void g(){}"); assert getErrorText(findFirstError(t)) == "void g("; - assert getErrorText(findBestError(t)) == "("; + assert getErrorText(findBestError(t).val) == "("; return true; } @@ -101,7 +102,7 @@ test bool rascalFunctionDeclarationMissingCloseParen() { assert getErrorText(findFirstError(t)) == "void g("; - Tree error = findBest(t); + Tree error = findBestError(t).val; assert getErrorText(error) == "("; loc location = getSkipped(error).src; assert location.begin.column == 16 && location.length == 1; @@ -111,14 +112,14 @@ test bool rascalFunctionDeclarationMissingCloseParen() { test bool rascalIfMissingExpr() { Tree t = parseFunctionDeclaration("void f(){if(){1;}}", visualize=false); - return getErrorText(findFirstError(t)) == ")"; + return getErrorText(findBestError(t).val) == ")"; } test bool rascalIfBodyEmpty() { Tree t = parseRascal("module A void f(){1;} void g(){if(1){}} void h(){1;}"); println("error: "); - assert getErrorText(findBestError(t)) == "} void h(){1"; + assert getErrorText(findBestError(t).val) == "} void h(){1"; return true; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index ac73a8f53f..a28811499f 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -8,6 +8,7 @@ import util::Benchmark; import Grammar; import analysis::statistics::Descriptive; import util::Math; +import util::Maybe; import Set; import List; @@ -15,10 +16,10 @@ import lang::rascal::grammar::definition::Modules; alias FrequencyTable = map[int val, int count]; -public data TestMeasurement(loc source=|unknown:///|, int duration=0) = successfulParse() | recovered(int errorSize=0) | parseError(); -public data FileStats = fileStats(int totalParses = 0, int successfulParses=0, int successfulRecoveries=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, FrequencyTable parseTimeRatios=()); +public data TestMeasurement(loc source=|unknown:///|, int duration=0) = successfulParse() | recovered(int errorSize=0) | parseError() | successfulDisambiguation(); +public data FileStats = fileStats(int totalParses = 0, int successfulParses=0, int successfulRecoveries=0, int successfulDisambiguations=0, int failedRecoveries=0, int parseErrors=0, int slowParses=0, FrequencyTable parseTimeRatios=()); -public data TestStats = testStats(int filesTested=0, int testCount=0, FrequencyTable successfulParsePct=(), FrequencyTable successfulRecoveryPct=(), FrequencyTable failedRecoveryPct=(), FrequencyTable parseErrorPct=(), FrequencyTable slowParsePct=(), FrequencyTable parseTimeRatios=()); +public data TestStats = testStats(int filesTested=0, int testCount=0, FrequencyTable successfulParses=(), FrequencyTable successfulRecoveries=(), FrequencyTable successfulDisambiguations=(), FrequencyTable failedRecoveries=(), FrequencyTable parseErrors=(), FrequencyTable slowParses=(), FrequencyTable parseTimeRatios=()); private TestMeasurement testRecovery(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, loc source) { int startTime = 0; @@ -34,9 +35,15 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser try { Tree t = recoveryParser(input, source); duration = realTime() - startTime; - Tree best = findBestError(t); - errorSize = size(getErrorText(best)); - measurement = recovered(source=source, duration=duration, errorSize=errorSize); + Maybe[Tree] best = findBestError(t); + totalDuration = realTime() - startTime; + println("parse time: , disamb time: "); + if (nothing() := best) { + measurement = successfulDisambiguation(source=source, duration=duration); + } else { + errorSize = size(getErrorText(best.val)); + measurement = recovered(source=source, duration=duration, errorSize=errorSize); + } } catch ParseError(_): { duration = realTime() - startTime; measurement = parseError(source=source, duration=duration); @@ -67,6 +74,10 @@ FileStats updateStats(FileStats stats, TestMeasurement measurement, int referenc stats.failedRecoveries += 1; } } + case successfulDisambiguation(): { + print("&"); + stats.successfulDisambiguations += 1; + } case parseError(): { stats.parseTimeRatios = increment(stats.parseTimeRatios, parseTimeRatio); print("?"); @@ -87,6 +98,7 @@ FileStats mergeFileStats(FileStats stats1, FileStats stats2) { totalParses = stats1.totalParses + stats2.totalParses, successfulParses = stats1.successfulParses + stats2.successfulParses, successfulRecoveries = stats1.successfulRecoveries + stats2.successfulRecoveries, + successfulDisambiguations = stats1.successfulDisambiguations + stats2.successfulDisambiguations, failedRecoveries = stats1.failedRecoveries + stats2.failedRecoveries, parseErrors = stats1.parseErrors + stats2.parseErrors, slowParses = stats1.slowParses + stats2.slowParses, @@ -107,11 +119,12 @@ FrequencyTable increment(FrequencyTable frequencyTable, int val) { TestStats consolidateStats(TestStats cumulativeStats, FileStats fileStats) { int totalFailed = fileStats.totalParses - fileStats.successfulParses; - cumulativeStats.successfulParsePct = increment(cumulativeStats.successfulParsePct, percentage(fileStats.successfulParses, fileStats.totalParses)); - cumulativeStats.successfulRecoveryPct = increment(cumulativeStats.successfulRecoveryPct, percentage(fileStats.successfulRecoveries, totalFailed)); - cumulativeStats.failedRecoveryPct = increment(cumulativeStats.failedRecoveryPct, percentage(fileStats.failedRecoveries, totalFailed)); - cumulativeStats.parseErrorPct = increment(cumulativeStats.parseErrorPct, percentage(fileStats.parseErrors, totalFailed)); - cumulativeStats.slowParsePct = increment(cumulativeStats.slowParsePct, percentage(fileStats.slowParses, totalFailed)); + cumulativeStats.successfulParses = increment(cumulativeStats.successfulParses, percentage(fileStats.successfulParses, fileStats.totalParses)); + cumulativeStats.successfulRecoveries = increment(cumulativeStats.successfulRecoveries, percentage(fileStats.successfulRecoveries, totalFailed)); + cumulativeStats.successfulDisambiguations = increment(cumulativeStats.successfulDisambiguations, percentage(fileStats.successfulDisambiguations, totalFailed)); + cumulativeStats.failedRecoveries = increment(cumulativeStats.failedRecoveries, percentage(fileStats.failedRecoveries, totalFailed)); + cumulativeStats.parseErrors = increment(cumulativeStats.parseErrors, percentage(fileStats.parseErrors, totalFailed)); + cumulativeStats.slowParses = increment(cumulativeStats.slowParses, percentage(fileStats.slowParses, totalFailed)); cumulativeStats.parseTimeRatios = mergeFrequencyTables(cumulativeStats.parseTimeRatios, fileStats.parseTimeRatios); cumulativeStats.filesTested += 1; @@ -135,22 +148,23 @@ map[int,int] mergeFrequencyTables(map[int,int] hist1, map[int,int] hist2) { TestStats mergeStats(TestStats stats, TestStats stats2) { stats.filesTested += stats2.filesTested; stats.testCount += stats2.testCount; - stats.successfulParsePct = mergeFrequencyTables(stats.successfulParsePct, stats2.successfulParsePct); - stats.successfulRecoveryPct = mergeFrequencyTables(stats.successfulRecoveryPct, stats2.successfulRecoveryPct); - stats.failedRecoveryPct = mergeFrequencyTables(stats.failedRecoveryPct, stats2.failedRecoveryPct); - stats.parseErrorPct = mergeFrequencyTables(stats.parseErrorPct, stats2.parseErrorPct); - stats.slowParsePct = mergeFrequencyTables(stats.slowParsePct, stats2.slowParsePct); + stats.successfulParses = mergeFrequencyTables(stats.successfulParses, stats2.successfulParses); + stats.successfulRecoveries = mergeFrequencyTables(stats.successfulRecoveries, stats2.successfulRecoveries); + stats.successfulDisambiguations = mergeFrequencyTables(stats.successfulDisambiguations, stats2.successfulDisambiguations); + stats.failedRecoveries = mergeFrequencyTables(stats.failedRecoveries, stats2.failedRecoveries); + stats.parseErrors = mergeFrequencyTables(stats.parseErrors, stats2.parseErrors); + stats.slowParses = mergeFrequencyTables(stats.slowParses, stats2.slowParses); stats.parseTimeRatios = mergeFrequencyTables(stats.parseTimeRatios, stats2.parseTimeRatios); return stats; } -FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit) { +FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit, int begin=0, int end=-1) { FileStats stats = fileStats(); int len = size(input); - int i = 0; + int i = begin; - while (i < len) { + while (i < len && (end == -1 || i<=end)) { str modifiedInput = substring(input, 0, i) + substring(input, i+1); TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deleted=<"">|); stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); @@ -171,7 +185,6 @@ FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (va for (int lineEnd <- lineEndings) { lineLength = lineEnd - lineStart; for (int pos <- [lineStart..lineEnd]) { - // Check boundaries (only used for quick bug testing) if (end != -1 && end < pos) { return stats; @@ -196,14 +209,14 @@ private int percentage(int number, int total) { } int statLabelWidth = 40; -int statFieldWidth = 7; +int statFieldWidth = 8; void printFileStats(FileStats fileStats) { - void printStat(str label, int stat, int total, bool printPct=true) { + void printStat(str label, int stat, int total, bool prints=true) { int pct = total == 0 ? 0 : stat*100/total; print(left(label + ":", statLabelWidth)); - str pctStr = printPct ? " (%)" : ""; + str pctStr = prints ? " (%)" : ""; println(left("", statFieldWidth)); } @@ -225,7 +238,8 @@ void printFrequencyTableHeader() { print(right("median", statFieldWidth)); print(right("95%", statFieldWidth)); print(right("min", statFieldWidth)); - println(right("max", statFieldWidth)); + print(right("max", statFieldWidth)); + println(right("total", statFieldWidth)); } void printFrequencyTableStats(str label, FrequencyTable frequencyTable, str unit = "%") { @@ -269,7 +283,8 @@ void printFrequencyTableStats(str label, FrequencyTable frequencyTable, str unit print(right("", statFieldWidth)); print(right("", statFieldWidth)); print(right("", statFieldWidth)); - println(right("", statFieldWidth)); + print(right("", statFieldWidth)); + println(right("", statFieldWidth)); } } @@ -279,11 +294,12 @@ void printStats(TestStats stats) { } println("Total parses: "); printFrequencyTableHeader(); - printFrequencyTableStats("Succesful parses", stats.successfulParsePct); - printFrequencyTableStats("Succesful recoveries", stats.successfulRecoveryPct); - printFrequencyTableStats("Failed recoveries", stats.failedRecoveryPct); - printFrequencyTableStats("Parse errors", stats.parseErrorPct); - printFrequencyTableStats("Slow parses", stats.slowParsePct); + printFrequencyTableStats("Succesful parses", stats.successfulParses); + printFrequencyTableStats("Succesful recoveries", stats.successfulRecoveries); + printFrequencyTableStats("Succesful disambiguations", stats.successfulRecoveries); + printFrequencyTableStats("Failed recoveries", stats.failedRecoveries); + printFrequencyTableStats("Parse errors", stats.parseErrors); + printFrequencyTableStats("Slow parses", stats.slowParses); printFrequencyTableStats("Parse time ratios", stats.parseTimeRatios, unit = "log2/%"); println(); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ToyRascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ToyRascalRecoveryTests.rsc index 4e6122ff3c..a248631e7a 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ToyRascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ToyRascalRecoveryTests.rsc @@ -18,6 +18,7 @@ import lang::rascal::tests::concrete::recovery::ToyRascal; import ParseTree; import IO; +import util::Maybe; Tree parseToyRascal(str input, bool visualize=false) { Tree result = parser(#start[FunctionDeclaration], allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); @@ -28,7 +29,7 @@ Tree parseToyRascal(str input, bool visualize=false) { println("- "); } - println("Best error: "); + println("Best error: "); } return result; @@ -41,15 +42,15 @@ test bool toyRascalOk() { test bool toyRascalMissingOpenParen() { Tree t = parseToyRascal("f){}", visualize=true); - return hasErrors(t) && getErrorText(findBestError(t)) == ")"; + return hasErrors(t) && getErrorText(findBestError(t).val) == ")"; } test bool toyRascalMissingCloseParen() { Tree t = parseToyRascal("f({}", visualize=true); - return hasErrors(t) && getErrorText(findBestError(t)) == "("; + return hasErrors(t) && getErrorText(findBestError(t).val) == "("; } test bool toyRascalMissingIfBody() { Tree t = parseToyRascal("f(){if(1){}}", visualize=true); - return hasErrors(t) && getErrorText(findBestError(t)) == "}"; + return hasErrors(t) && getErrorText(findBestError(t).val) == "}"; } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/NoErrorsAfterDisambBug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/NoErrorsAfterDisambBug.rsc new file mode 100644 index 0000000000..190e179941 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/NoErrorsAfterDisambBug.rsc @@ -0,0 +1,14 @@ +module lang::rascal::tests::concrete::recovery::bugs::NoErrorsAfterDisambBug + + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; + +void testBug() { + standardParser = parser(#start[Module], allowRecovery=false, allowAmbiguity=true); + recoveryParser = parser(#start[Module], allowRecovery=true, allowAmbiguity=true); + input = readFile(|std:///lang/rascal/tests/basic/ListRelations.rsc|); + testSingleCharDeletions(standardParser, recoveryParser, input, 200, 100, begin=1916, end=1916); +} diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index a9e06ed779..8b8f82ff34 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -582,13 +582,13 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in URI uri = location.getURI(); if (allowRecovery) { recoverer = new ToTokenRecoverer(uri, parserInstance, new StackNodeIdDispenser(parserInstance)); - //debugListener = new DebugLogger(new PrintWriter(System.out, true)); } ITree parseForest = (ITree) parserInstance.parse(methodName, uri, input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowRecovery || allowAmbiguity), recoverer, debugListener); - if (!allowAmbiguity && allowRecovery && (filters == null || filters.isEmpty())) { + if (!allowAmbiguity && allowRecovery && filters.isEmpty()) { // Filter error-induced ambiguities - parseForest = (ITree) new ParseErrorDisambiguator((RascalValueFactory) ValueFactoryFactory.getValueFactory()).disambiguateErrors(parseForest, null); + RascalValueFactory valueFactory = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); + parseForest = (ITree) new ParseErrorDisambiguator(valueFactory).disambiguateErrors(parseForest, valueFactory.bool(false)); } return parseForest; From bf3e5aeb65091bd669263289a81df78fae71e6fc Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 3 Oct 2024 13:10:11 +0200 Subject: [PATCH 05/11] Moved disambiguator to a better location (and fixed a bug) --- src/org/rascalmpl/library/ParseTree.rsc | 2 +- .../concrete/recovery/RecoveryTestSupport.rsc | 20 ++++++++++--------- .../recovery/ParseErrorDisambiguator.java | 3 +-- 3 files changed, 13 insertions(+), 12 deletions(-) rename src/org/rascalmpl/{library/lang/rascal/tests/concrete => parser/gtd}/recovery/ParseErrorDisambiguator.java (98%) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 948f3a132b..3fff07aa9d 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -821,7 +821,7 @@ If you want the text of the whole error tree, you can just use string interpolat } str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) = stringChars([c | char(c) <- chars]); -@javaClass{org.rascalmpl.library.lang.rascal.tests.concrete.recovery.ParseErrorDisambiguator} +@javaClass{org.rascalmpl.parser.gtd.recovery.ParseErrorDisambiguator} @synopsis{Error recovery often produces ambiguous trees where errors can be recovered in multiple ways. This filter removes error trees until no ambiguities caused by error recovery are left. Note that regular ambiguous trees remain in the parse forest unless `allowAmbiguity` is set to false in which case an error is thrown. diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index a28811499f..65f4a44e96 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -36,9 +36,9 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser Tree t = recoveryParser(input, source); duration = realTime() - startTime; Maybe[Tree] best = findBestError(t); - totalDuration = realTime() - startTime; - println("parse time: , disamb time: "); - if (nothing() := best) { + //totalDuration = realTime() - startTime; + //println("parse time: , disamb time: "); + if (best == nothing()) { measurement = successfulDisambiguation(source=source, duration=duration); } else { errorSize = size(getErrorText(best.val)); @@ -75,6 +75,7 @@ FileStats updateStats(FileStats stats, TestMeasurement measurement, int referenc } } case successfulDisambiguation(): { + stats.parseTimeRatios = increment(stats.parseTimeRatios, parseTimeRatio); print("&"); stats.successfulDisambiguations += 1; } @@ -209,7 +210,7 @@ private int percentage(int number, int total) { } int statLabelWidth = 40; -int statFieldWidth = 8; +int statFieldWidth = 10; void printFileStats(FileStats fileStats) { @@ -225,11 +226,12 @@ void printFileStats(FileStats fileStats) { printStat("Successful parses", fileStats.successfulParses, fileStats.totalParses); int failedParses = fileStats.totalParses - fileStats.successfulParses; printStat("Successful recoveries", fileStats.successfulRecoveries, failedParses); + printStat("Successful disambiguations", fileStats.successfulDisambiguations, failedParses); printStat("Failed recoveries", fileStats.failedRecoveries, failedParses); printStat("Parse errors", fileStats.parseErrors, failedParses); printStat("Slow parses", fileStats.slowParses, failedParses); printFrequencyTableHeader(); - printFrequencyTableStats("Parse time ratios", fileStats.parseTimeRatios, unit = "log2(ratio)"); + printFrequencyTableStats("Parse time ratios", fileStats.parseTimeRatios, unit = "log2(ratio)", printTotal=false); } void printFrequencyTableHeader() { @@ -242,7 +244,7 @@ void printFrequencyTableHeader() { println(right("total", statFieldWidth)); } -void printFrequencyTableStats(str label, FrequencyTable frequencyTable, str unit = "%") { +void printFrequencyTableStats(str label, FrequencyTable frequencyTable, str unit = "%", bool printTotal=true) { print(left(label + " ():", statLabelWidth)); int totalCount = (0 | it+frequencyTable[val] | val <- frequencyTable); @@ -284,7 +286,7 @@ void printFrequencyTableStats(str label, FrequencyTable frequencyTable, str unit print(right("", statFieldWidth)); print(right("", statFieldWidth)); print(right("", statFieldWidth)); - println(right("", statFieldWidth)); + println(right("", statFieldWidth)); } } @@ -296,11 +298,11 @@ void printStats(TestStats stats) { printFrequencyTableHeader(); printFrequencyTableStats("Succesful parses", stats.successfulParses); printFrequencyTableStats("Succesful recoveries", stats.successfulRecoveries); - printFrequencyTableStats("Succesful disambiguations", stats.successfulRecoveries); + printFrequencyTableStats("Succesful disambiguations", stats.successfulDisambiguations); printFrequencyTableStats("Failed recoveries", stats.failedRecoveries); printFrequencyTableStats("Parse errors", stats.parseErrors); printFrequencyTableStats("Slow parses", stats.slowParses); - printFrequencyTableStats("Parse time ratios", stats.parseTimeRatios, unit = "log2/%"); + printFrequencyTableStats("Parse time ratios", stats.parseTimeRatios, unit = "log2/%", printTotal=false); println(); } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ParseErrorDisambiguator.java b/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java similarity index 98% rename from src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ParseErrorDisambiguator.java rename to src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java index a032d7b7d3..c3e9d5d43b 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ParseErrorDisambiguator.java +++ b/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java @@ -18,7 +18,6 @@ public class ParseErrorDisambiguator { private final IRascalValueFactory rascalValues; - private boolean allowAmbiguity; public ParseErrorDisambiguator(IRascalValueFactory rascalValues) { super(); @@ -78,7 +77,7 @@ private ScoredTree disambiguateAppl(ITree appl, boolean allowAmbiguity) { if (disambiguatedArg.tree != arg) { if (disambiguatedArgs == null) { disambiguatedArgs = rascalValues.listWriter(); - for (int j=0; j<=i; j++) { + for (int j=0; j Date: Thu, 3 Oct 2024 14:27:38 +0200 Subject: [PATCH 06/11] Fixed imports after moving a class --- .../rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java | 2 +- src/org/rascalmpl/values/RascalFunctionValueFactory.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java b/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java index c3e9d5d43b..b477798b22 100644 --- a/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java +++ b/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java @@ -1,4 +1,4 @@ -package org.rascalmpl.library.lang.rascal.tests.concrete.recovery; +package org.rascalmpl.parser.gtd.recovery; import org.rascalmpl.interpreter.asserts.Ambiguous; import org.rascalmpl.values.IRascalValueFactory; diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 8b8f82ff34..3cb60fd790 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -40,7 +40,6 @@ import org.rascalmpl.interpreter.result.ResultFactory; import org.rascalmpl.interpreter.staticErrors.UndeclaredNonTerminal; import org.rascalmpl.library.lang.rascal.syntax.RascalParser; -import org.rascalmpl.library.lang.rascal.tests.concrete.recovery.ParseErrorDisambiguator; import org.rascalmpl.parser.ParserGenerator; import org.rascalmpl.parser.gtd.IGTD; import org.rascalmpl.parser.gtd.debug.IDebugListener; @@ -48,6 +47,7 @@ import org.rascalmpl.parser.gtd.exception.UndeclaredNonTerminalException; import org.rascalmpl.parser.gtd.io.InputConverter; import org.rascalmpl.parser.gtd.recovery.IRecoverer; +import org.rascalmpl.parser.gtd.recovery.ParseErrorDisambiguator; import org.rascalmpl.parser.gtd.result.action.IActionExecutor; import org.rascalmpl.parser.gtd.result.out.DefaultNodeFlattener; import org.rascalmpl.parser.gtd.util.StackNodeIdDispenser; From 243fab24a20b2687156ca4a596585bb825a188e9 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 3 Oct 2024 18:19:50 +0200 Subject: [PATCH 07/11] Fixed bug in disambiguator (insert is not the same as append!) --- .../tests/concrete/recovery/bugs/LostSkipBug.rsc | 14 ++++++++++++++ src/org/rascalmpl/parser/gtd/SGTDBF.java | 13 +++++++------ .../gtd/recovery/ParseErrorDisambiguator.java | 15 ++++++++------- 3 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/LostSkipBug.rsc diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/LostSkipBug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/LostSkipBug.rsc new file mode 100644 index 0000000000..173af413af --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/LostSkipBug.rsc @@ -0,0 +1,14 @@ +module lang::rascal::tests::concrete::recovery::bugs::LostSkipBug + + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; + +void testBug() { + standardParser = parser(#start[Module], allowRecovery=false, allowAmbiguity=true); + recoveryParser = parser(#start[Module], allowRecovery=true, allowAmbiguity=true); + input = readFile(|std:///analysis/diff/edits/ExecuteTextEdits.rsc|); + testSingleCharDeletions(standardParser, recoveryParser, input, 200, 100, begin=235, end=235); +} diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 130de09530..d690606b9f 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -1678,13 +1678,14 @@ private IConstructor fixErrorAppl(ITree tree, newChild = introduceErrorNodes(child, nodeConstructorFactory); } - if (newChild != child || errorTree) { - if (newChildren == null) { - newChildren = new ArrayList<>(childCount); - for (int j=0; j(childCount); + for (int j=0; j Date: Fri, 4 Oct 2024 09:23:28 +0200 Subject: [PATCH 08/11] Fixed array index out of bounds issue --- .../concrete/recovery/RecoveryTestSupport.rsc | 1 - .../recovery/bugs/OvertakenNullableBug.rsc | 14 ++++++++++++++ .../parser/gtd/stack/AbstractStackNode.java | 7 ++++++- 3 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakenNullableBug.rsc diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 65f4a44e96..1be6b0a657 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -193,7 +193,6 @@ FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (va if (pos < begin) { continue; } - modifiedInput = substring(input, 0, pos) + substring(input, lineEnd); TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deletedUntilEol=<",">|); stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakenNullableBug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakenNullableBug.rsc new file mode 100644 index 0000000000..6d82aafd0f --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OvertakenNullableBug.rsc @@ -0,0 +1,14 @@ +module lang::rascal::tests::concrete::recovery::bugs::OvertakenNullableBug + + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; + +void testBug() { + standardParser = parser(#start[Module], allowRecovery=false, allowAmbiguity=true); + recoveryParser = parser(#start[Module], allowRecovery=true, allowAmbiguity=true); + input = readFile(|std:///lang/rascal/tests/library/analysis/statistics/DescriptiveTests.rsc|); + testDeleteUntilEol(standardParser, recoveryParser, input, 200, 100, begin=561, end=561); +} diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index fee21a394a..67cd564b52 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -590,6 +590,9 @@ public int updateOvertakenNode(AbstractStackNode

predecessor, AbstractNode re // Initialize the prefixes map. int edgesMapSize = edgesMap.size(); + // Before error recovery: int possibleMaxSize = edgesMapSize + edgesMapSize; + // It is unclear why error recovery can cause more edges to be added than previously accounted for, + // although this might just have been a bug. int possibleMaxSize = edgesMapSize + edgesMapToAdd.size(); if(prefixesMap == null){ prefixesMap = new ArrayList[possibleMaxSize]; @@ -651,7 +654,9 @@ public int updateOvertakenNullableNode(AbstractStackNode

predecessor, Abstrac // Initialize the prefixes map. int edgesMapSize = edgesMap.size(); - int possibleMaxSize = edgesMapSize + potentialNewEdges; + // Before error recovery: int possibleMaxSize = edgesMapSize + potentialNewEdges; + // It is unclear why error recovery can cause more edges to be added than previously accounted for. + int possibleMaxSize = edgesMapSize + edgesMapToAdd.size(); if(prefixesMap == null){ prefixesMap = new ArrayList[possibleMaxSize]; From 4818fde9b17c89f380cd46fe82538d8bf219e653 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Fri, 4 Oct 2024 09:49:54 +0200 Subject: [PATCH 09/11] Removed 'synchronized' --- .../rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java b/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java index aea2aaa83f..60cae6efca 100644 --- a/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java +++ b/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java @@ -43,7 +43,7 @@ public ScoredTree(IConstructor tree, int score) { * possibly without ambiguities. */ - public synchronized IConstructor disambiguateErrors(IConstructor arg, IBool allowAmbiguity) { + public IConstructor disambiguateErrors(IConstructor arg, IBool allowAmbiguity) { return disambiguate(arg, allowAmbiguity.getValue()).tree; } From 27e4e693c79c8954d39bec369d6b63c9a75fcb77 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 7 Oct 2024 11:31:24 +0200 Subject: [PATCH 10/11] Implemented preservation of sharing Both when introducing error nodes and when disambiguating errors --- rascal-recovery-stats.csv | 178 ++++++++++++++++++ .../recovery/ErrorRecoveryBenchmark.rsc | 27 ++- .../concrete/recovery/RecoveryTestSupport.rsc | 65 +++++-- .../concrete/recovery/bugs/OutOfMemoryBug.rsc | 15 ++ src/org/rascalmpl/parser/gtd/SGTDBF.java | 13 +- .../gtd/recovery/ParseErrorDisambiguator.java | 32 +++- 6 files changed, 294 insertions(+), 36 deletions(-) create mode 100644 rascal-recovery-stats.csv create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OutOfMemoryBug.rsc diff --git a/rascal-recovery-stats.csv b/rascal-recovery-stats.csv new file mode 100644 index 0000000000..3c9605e269 --- /dev/null +++ b/rascal-recovery-stats.csv @@ -0,0 +1,178 @@ +source,size,result,duration,disambiguationDuration,errorSize +|std:///analysis/grammars/Dependency.rsc?deletedChar=0|,1070,recovery,0,2,341 +|std:///analysis/grammars/Dependency.rsc?deletedChar=1|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=2|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=3|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=4|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=5|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=6|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=7|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=8|,1070,recovery,0,2,272 +|std:///analysis/grammars/Dependency.rsc?deletedChar=9|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=10|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=11|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=12|,1070,success,7,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=13|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=14|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=15|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=16|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=17|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=18|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=19|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=20|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=21|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=22|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=23|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=24|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=25|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=26|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=27|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=28|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=29|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=30|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=31|,1070,success,7,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=32|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=33|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=34|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=35|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=36|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=37|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=38|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=39|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=40|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=41|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=42|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=43|,1070,success,10,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=44|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=45|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=46|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=47|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=48|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=49|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=50|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=51|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=52|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=53|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=54|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=55|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=56|,1070,success,7,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=57|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=58|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=59|,1070,success,7,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=60|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=61|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=62|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=63|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=64|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=65|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=66|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=67|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=68|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=69|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=70|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=71|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=72|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=73|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=74|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=75|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=76|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=77|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=78|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=79|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=80|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=81|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=82|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=83|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=84|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=85|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=86|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=87|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=88|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=89|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=90|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=91|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=92|,1070,success,3,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=93|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=94|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=95|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=96|,1070,success,11,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=97|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=98|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=99|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=100|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=101|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=102|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=103|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=104|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=105|,1070,success,7,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=106|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=107|,1070,success,7,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=108|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=109|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=110|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=111|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=112|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=113|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=114|,1070,success,3,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=115|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=116|,1070,success,3,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=117|,1070,success,3,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=118|,1070,success,3,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=119|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=120|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=121|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=122|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=123|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=124|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=125|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=126|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=127|,1070,success,3,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=128|,1070,success,3,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=129|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=130|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=131|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=132|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=133|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=134|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=135|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=136|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=137|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=138|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=139|,1070,success,7,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=140|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=141|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=142|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=143|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=144|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=145|,1070,success,7,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=146|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=147|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=148|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=149|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=150|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=151|,1070,success,10,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=152|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=153|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=154|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=155|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=156|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=157|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=158|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=159|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=160|,1070,success,3,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=161|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=162|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=163|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=164|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=165|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=166|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=167|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=168|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=169|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=170|,1070,success,6,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=171|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=172|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=173|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=174|,1070,success,5,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=175|,1070,success,4,-1,0 +|std:///analysis/grammars/Dependency.rsc?deletedChar=176|,1070,success,5,-1,0 diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc index b3fa1063fa..777afb171e 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/ErrorRecoveryBenchmark.rsc @@ -33,9 +33,9 @@ void runLanguageTests() { testRecoveryRascal(); } -void runRascalBatchTest(int maxFiles=1000, int maxFileSize=4000) { +void runRascalBatchTest(int maxFiles=1000, int minFileSize=0, int maxFileSize=4000, int fromFile=0) { int startTime = realTime(); - TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", maxFiles, maxFileSize); + TestStats stats = batchRecoveryTest(|std:///lang/rascal/syntax/Rascal.rsc|, "Module", |std:///|, ".rsc", maxFiles, minFileSize, maxFileSize, fromFile, |cwd:///rascal-recovery-stats.csv|); int duration = realTime() - startTime; println(); println("================================================================"); @@ -45,14 +45,25 @@ void runRascalBatchTest(int maxFiles=1000, int maxFileSize=4000) { int main(list[str] args) { int maxFiles = 1000; - int maxFileSize = 4000; - if (size(args) == 2) { + int maxFileSize = 1000000; + int minFileSize = 0; + int fromFile = 0; + if (size(args) > 0) { maxFiles = toInt(args[0]); - maxFileSize = toInt(args[1]); - } else if (size(args) != 0) { - println("Usage: ErrorRecoveryBenchmark \ \"); } + if (size(args) > 1) { + minFileSize = toInt(args[1]); + } + if (size(args) > 2) { + maxFileSize = toInt(args[2]); + } + if (size(args) > 3) { + fromFile = toInt(args[3]); + } else { + println("Usage: ErrorRecoveryBenchmark [\ [\ [\ [\]]]]"); + } + + runRascalBatchTest(maxFiles=maxFiles, minFileSize=minFileSize, maxFileSize=maxFileSize, fromFile=fromFile); - runRascalBatchTest(maxFiles=maxFiles, maxFileSize=maxFileSize); return 0; } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 1be6b0a657..3bd63489f6 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -21,23 +21,28 @@ public data FileStats = fileStats(int totalParses = 0, int successfulParses=0, i public data TestStats = testStats(int filesTested=0, int testCount=0, FrequencyTable successfulParses=(), FrequencyTable successfulRecoveries=(), FrequencyTable successfulDisambiguations=(), FrequencyTable failedRecoveries=(), FrequencyTable parseErrors=(), FrequencyTable slowParses=(), FrequencyTable parseTimeRatios=()); -private TestMeasurement testRecovery(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, loc source) { +private TestMeasurement testRecovery(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, loc source, loc statFile) { int startTime = 0; int duration = 0; + int disambDuration = -1; + int errorSize=0; + str result = "?"; TestMeasurement measurement = successfulParse(); try { startTime = realTime(); Tree t = standardParser(input, source); duration = realTime() - startTime; measurement = successfulParse(source=source, duration=duration); + result = "success"; } catch ParseError(_): { startTime = realTime(); try { Tree t = recoveryParser(input, source); - duration = realTime() - startTime; + int parseEndTime = realTime(); + duration = realTime() - parseEndTime; Maybe[Tree] best = findBestError(t); - //totalDuration = realTime() - startTime; - //println("parse time: , disamb time: "); + disambDuration = realTime() - parseEndTime; + result = "recovery"; if (best == nothing()) { measurement = successfulDisambiguation(source=source, duration=duration); } else { @@ -45,11 +50,16 @@ private TestMeasurement testRecovery(&T (value input, loc origin) standardParser measurement = recovered(source=source, duration=duration, errorSize=errorSize); } } catch ParseError(_): { + result = "error"; duration = realTime() - startTime; measurement = parseError(source=source, duration=duration); } } + if (statFile != |unknown:///|) { + appendToFile(statFile, ",,,,,\n"); + } + return measurement; } @@ -160,14 +170,15 @@ TestStats mergeStats(TestStats stats, TestStats stats2) { return stats; } -FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit, int begin=0, int end=-1) { +FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, loc source, str input, int referenceParseTime, int recoverySuccessLimit, int begin=0, int end=-1, loc statFile=|unknown:///|) { FileStats stats = fileStats(); int len = size(input); int i = begin; while (i < len && (end == -1 || i<=end)) { str modifiedInput = substring(input, 0, i) + substring(input, i+1); - TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deleted=<"">|); + source.query = "deletedChar="; + TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, source, statFile); stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); if (i < len && substring(input, i, i+1) == "\n") { println(); @@ -178,11 +189,12 @@ FileStats testSingleCharDeletions(&T (value input, loc origin) standardParser, & return stats; } -FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, str input, int referenceParseTime, int recoverySuccessLimit, int begin=0, int end=-1) { +FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (value input, loc origin) recoveryParser, loc source, str input, int referenceParseTime, int recoverySuccessLimit, int begin=0, int end=-1, loc statFile=|unknown:///|) { FileStats stats = fileStats(); int lineStart = begin; list[int] lineEndings = findAll(input, "\n"); + int line = 1; for (int lineEnd <- lineEndings) { lineLength = lineEnd - lineStart; for (int pos <- [lineStart..lineEnd]) { @@ -194,11 +206,13 @@ FileStats testDeleteUntilEol(&T (value input, loc origin) standardParser, &T (va continue; } modifiedInput = substring(input, 0, pos) + substring(input, lineEnd); - TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, |unknown:///?deletedUntilEol=<",">|); + source.query = "deletedUntilEol=,,"; + TestMeasurement measurement = testRecovery(standardParser, recoveryParser, modifiedInput, source, statFile); stats = updateStats(stats, measurement, referenceParseTime, recoverySuccessLimit); } lineStart = lineEnd+1; println(); + line = line+1; } return stats; @@ -319,7 +333,7 @@ loc zippedFile(str zip, str path) { FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput) = testErrorRecovery(syntaxFile, topSort, testInput, readFile(testInput)); -FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str input) { +FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str input, loc statFile=|unknown:///|) { Module \module = parse(#start[Module], syntaxFile).top; str modName = syntaxLocToModuleName(syntaxFile); Grammar gram = modules2grammar(modName, {\module}); @@ -339,12 +353,12 @@ FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu println(); println("Single char deletions:"); - FileStats singleCharDeletionStats = testSingleCharDeletions(standardParser, recoveryParser, input, referenceParseTime, recoverySuccessLimit); + FileStats singleCharDeletionStats = testSingleCharDeletions(standardParser, recoveryParser, testInput, input, referenceParseTime, recoverySuccessLimit, statFile=statFile); printFileStats(singleCharDeletionStats); println(); println("Deletes until end-of-line:"); - FileStats deleteUntilEolStats = testDeleteUntilEol(standardParser, recoveryParser, input, referenceParseTime, recoverySuccessLimit); + FileStats deleteUntilEolStats = testDeleteUntilEol(standardParser, recoveryParser, testInput, input, referenceParseTime, recoverySuccessLimit, statFile=statFile); printFileStats(deleteUntilEolStats); FileStats stats = mergeFileStats(singleCharDeletionStats, deleteUntilEolStats); @@ -358,17 +372,34 @@ FileStats testErrorRecovery(loc syntaxFile, str topSort, loc testInput, str inpu } } -TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int maxFileSize, TestStats cumulativeStats=testStats()) { - println("Batch testing in directory

(maxFiles=, maxFileSize=)"); +private int fileNr = 0; +private int fromFile = 0; + +TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int minFileSize, int maxFileSize, int from, loc statFile) { + fileNr = 0; + fromFile = from; + + return runBatchRecoveryTest(syntaxFile, topSort, dir, ext, maxFiles, minFileSize, maxFileSize, statFile, testStats()); +} + +TestStats runBatchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int maxFiles, int minFileSize, int maxFileSize, loc statFile, TestStats cumulativeStats) { + println("Batch testing in directory (maxFiles=, maxFileSize=, fromFile=)"); + writeFile(statFile, "source,size,result,duration,disambiguationDuration,errorSize\n"); for (entry <- listEntries(dir)) { loc file = dir + entry; if (isFile(file)) { if (endsWith(file.path, ext)) { str content = readFile(file); - if (size(content) <= maxFileSize) { + int contentSize = size(content); + if (contentSize >= minFileSize && contentSize < maxFileSize) { + fileNr += 1; + if (fileNr < fromFile) { + println("Skipping file #: , (\< )"); + continue; + } println("========================================================================"); - println("Testing file # ( of left)"); - FileStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content); + println("Testing file # ( of left)"); + FileStats fileStats = testErrorRecovery(syntaxFile, topSort, file, content, statFile=statFile); cumulativeStats = consolidateStats(cumulativeStats, fileStats); println(); println("------------------------------------------------------------------------"); @@ -377,7 +408,7 @@ TestStats batchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, int m } } } else if (isDirectory(file)) { - cumulativeStats = batchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles, maxFileSize, cumulativeStats=cumulativeStats); + cumulativeStats = runBatchRecoveryTest(syntaxFile, topSort, file, ext, maxFiles, minFileSize, maxFileSize, statFile, cumulativeStats); } if (cumulativeStats.filesTested >= maxFiles) { diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OutOfMemoryBug.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OutOfMemoryBug.rsc new file mode 100644 index 0000000000..cd0e54775d --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/bugs/OutOfMemoryBug.rsc @@ -0,0 +1,15 @@ +module lang::rascal::tests::concrete::recovery::bugs::OutOfMemoryBug + + +import lang::rascal::tests::concrete::recovery::RecoveryTestSupport; +import lang::rascal::\syntax::Rascal; +import ParseTree; +import IO; + +void testBug() { + standardParser = parser(#start[Module], allowRecovery=false, allowAmbiguity=true); + recoveryParser = parser(#start[Module], allowRecovery=true, allowAmbiguity=true); + loc source = |std:///lang/rascal/tests/functionality/PatternSet3.rsc|; + input = readFile(source); + testDeleteUntilEol(standardParser, recoveryParser, source, input, 200, 150, begin=581, end=581); +} diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index d690606b9f..0125814206 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -10,6 +10,9 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URI; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import org.rascalmpl.parser.gtd.debug.IDebugListener; import org.rascalmpl.parser.gtd.exception.ParseError; @@ -140,6 +143,7 @@ public abstract class SGTDBF implements IGTD { // Error recovery private IRecoverer

recoverer; + private Map processedTrees = new java.util.HashMap<>(); // Used to preserve sharing during error node introduction // Debugging private IDebugListener

debugListener; @@ -1630,7 +1634,11 @@ private T introduceErrorNodes(T tree, INodeConstructorFactory nodeConstruc private IConstructor introduceErrorNodes(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { - IConstructor result; + IConstructor result = processedTrees.get(tree); + if (result != null) { + return result; + } + Type type = tree.getConstructorType(); if (type == RascalValueFactory.Tree_Appl) { result = fixErrorAppl((ITree) tree, nodeConstructorFactory); @@ -1653,11 +1661,14 @@ else if (type == RascalValueFactory.Tree_Cycle) { result = result.asWithKeywordParameters().setParameter(RascalValueFactory.Location, loc); } + processedTrees.put(tree, result); + return result; } private IConstructor fixErrorAppl(ITree tree, INodeConstructorFactory nodeConstructorFactory) { + IValue prod = TreeAdapter.getProduction(tree); IList childList = TreeAdapter.getArgs(tree); diff --git a/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java b/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java index 60cae6efca..5b0fe6421d 100644 --- a/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java +++ b/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java @@ -1,5 +1,8 @@ package org.rascalmpl.parser.gtd.recovery; +import java.util.HashMap; +import java.util.Map; + import org.rascalmpl.interpreter.asserts.Ambiguous; import org.rascalmpl.values.IRascalValueFactory; import org.rascalmpl.values.RascalValueFactory; @@ -44,23 +47,32 @@ public ScoredTree(IConstructor tree, int score) { */ public IConstructor disambiguateErrors(IConstructor arg, IBool allowAmbiguity) { - return disambiguate(arg, allowAmbiguity.getValue()).tree; + return disambiguate(arg, allowAmbiguity.getValue(), new HashMap<>()).tree; } - private ScoredTree disambiguate(IConstructor tree, boolean allowAmbiguity) { + private ScoredTree disambiguate(IConstructor tree, boolean allowAmbiguity, Map processedTrees) { + ScoredTree result = processedTrees.get(tree); + if (result != null) { + return result; + } + Type type = tree.getConstructorType(); if (type == RascalValueFactory.Tree_Appl) { - return disambiguateAppl((ITree) tree, allowAmbiguity); + result = disambiguateAppl((ITree) tree, allowAmbiguity, processedTrees); } else if (type == RascalValueFactory.Tree_Amb) { - return disambiguateAmb((ITree) tree, allowAmbiguity); + result = disambiguateAmb((ITree) tree, allowAmbiguity, processedTrees); + } else { + // Other trees (cycle, char) do not have subtrees so they have a score of 0 + result = new ScoredTree(tree, 0); } - // Other trees (cycle, char) do not have subtrees so they have a score of 0 - return new ScoredTree(tree, 0); + processedTrees.put(tree, result); + + return result; } - private ScoredTree disambiguateAppl(ITree appl, boolean allowAmbiguity) { + private ScoredTree disambiguateAppl(ITree appl, boolean allowAmbiguity, Map processedTrees) { if (ProductionAdapter.isSkipped(appl.getProduction())) { return new ScoredTree(appl, ((IList) appl.get(1)).length()); } @@ -72,7 +84,7 @@ private ScoredTree disambiguateAppl(ITree appl, boolean allowAmbiguity) { // Disambiguate and score all children for (int i=0; i processedTrees) { ISet originalAlts = (ISet) amb.get(0); ISetWriter alternativesWithoutErrors = null; ScoredTree errorAltWithBestScore = null; for (IValue alt : originalAlts) { - ScoredTree disambiguatedAlt = disambiguate((IConstructor) alt, allowAmbiguity); + ScoredTree disambiguatedAlt = disambiguate((IConstructor) alt, allowAmbiguity, processedTrees); if (disambiguatedAlt.score == 0) { // Non-error tree if (alternativesWithoutErrors == null) { From d5f60904cb6767da84f1d32c31c042392e230d23 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 7 Oct 2024 13:28:35 +0200 Subject: [PATCH 11/11] Slight optimization in ParseErrorDisambiguator --- rascal-recovery-stats.csv | 178 ------------------ .../concrete/recovery/RecoveryTestSupport.rsc | 1 - .../gtd/recovery/ParseErrorDisambiguator.java | 82 ++++---- 3 files changed, 47 insertions(+), 214 deletions(-) delete mode 100644 rascal-recovery-stats.csv diff --git a/rascal-recovery-stats.csv b/rascal-recovery-stats.csv deleted file mode 100644 index 3c9605e269..0000000000 --- a/rascal-recovery-stats.csv +++ /dev/null @@ -1,178 +0,0 @@ -source,size,result,duration,disambiguationDuration,errorSize -|std:///analysis/grammars/Dependency.rsc?deletedChar=0|,1070,recovery,0,2,341 -|std:///analysis/grammars/Dependency.rsc?deletedChar=1|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=2|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=3|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=4|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=5|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=6|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=7|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=8|,1070,recovery,0,2,272 -|std:///analysis/grammars/Dependency.rsc?deletedChar=9|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=10|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=11|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=12|,1070,success,7,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=13|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=14|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=15|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=16|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=17|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=18|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=19|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=20|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=21|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=22|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=23|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=24|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=25|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=26|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=27|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=28|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=29|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=30|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=31|,1070,success,7,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=32|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=33|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=34|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=35|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=36|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=37|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=38|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=39|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=40|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=41|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=42|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=43|,1070,success,10,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=44|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=45|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=46|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=47|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=48|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=49|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=50|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=51|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=52|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=53|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=54|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=55|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=56|,1070,success,7,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=57|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=58|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=59|,1070,success,7,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=60|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=61|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=62|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=63|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=64|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=65|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=66|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=67|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=68|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=69|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=70|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=71|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=72|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=73|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=74|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=75|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=76|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=77|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=78|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=79|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=80|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=81|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=82|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=83|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=84|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=85|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=86|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=87|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=88|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=89|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=90|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=91|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=92|,1070,success,3,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=93|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=94|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=95|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=96|,1070,success,11,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=97|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=98|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=99|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=100|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=101|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=102|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=103|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=104|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=105|,1070,success,7,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=106|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=107|,1070,success,7,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=108|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=109|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=110|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=111|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=112|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=113|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=114|,1070,success,3,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=115|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=116|,1070,success,3,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=117|,1070,success,3,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=118|,1070,success,3,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=119|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=120|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=121|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=122|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=123|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=124|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=125|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=126|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=127|,1070,success,3,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=128|,1070,success,3,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=129|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=130|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=131|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=132|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=133|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=134|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=135|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=136|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=137|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=138|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=139|,1070,success,7,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=140|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=141|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=142|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=143|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=144|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=145|,1070,success,7,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=146|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=147|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=148|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=149|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=150|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=151|,1070,success,10,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=152|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=153|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=154|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=155|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=156|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=157|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=158|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=159|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=160|,1070,success,3,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=161|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=162|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=163|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=164|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=165|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=166|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=167|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=168|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=169|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=170|,1070,success,6,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=171|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=172|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=173|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=174|,1070,success,5,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=175|,1070,success,4,-1,0 -|std:///analysis/grammars/Dependency.rsc?deletedChar=176|,1070,success,5,-1,0 diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc index 3bd63489f6..2bf0689fdb 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/recovery/RecoveryTestSupport.rsc @@ -418,4 +418,3 @@ TestStats runBatchRecoveryTest(loc syntaxFile, str topSort, loc dir, str ext, in return cumulativeStats; } - diff --git a/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java b/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java index 5b0fe6421d..2ed8f01950 100644 --- a/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java +++ b/src/org/rascalmpl/parser/gtd/recovery/ParseErrorDisambiguator.java @@ -51,12 +51,8 @@ public IConstructor disambiguateErrors(IConstructor arg, IBool allowAmbiguity) { } private ScoredTree disambiguate(IConstructor tree, boolean allowAmbiguity, Map processedTrees) { - ScoredTree result = processedTrees.get(tree); - if (result != null) { - return result; - } - Type type = tree.getConstructorType(); + ScoredTree result; if (type == RascalValueFactory.Tree_Appl) { result = disambiguateAppl((ITree) tree, allowAmbiguity, processedTrees); @@ -67,51 +63,63 @@ private ScoredTree disambiguate(IConstructor tree, boolean allowAmbiguity, Map processedTrees) { - if (ProductionAdapter.isSkipped(appl.getProduction())) { - return new ScoredTree(appl, ((IList) appl.get(1)).length()); + ScoredTree result = processedTrees.get(appl); + if (result != null) { + return result; } - IList args = TreeAdapter.getArgs(appl); - int totalScore = 0; - IListWriter disambiguatedArgs = null; - - // Disambiguate and score all children - for (int i=0; i processedTrees) { + ScoredTree result = processedTrees.get(amb); + if (result != null) { + return result; + } + ISet originalAlts = (ISet) amb.get(0); ISetWriter alternativesWithoutErrors = null; @@ -134,9 +142,10 @@ private ScoredTree disambiguateAmb(ITree amb, boolean allowAmbiguity, Map