From 9deefa361bbc0b4c68c2ff405b057001c8d0b45c Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 30 Mar 2022 16:45:59 +0200 Subject: [PATCH 01/87] recovered Recoverer from git history --- .../parser/uptr/recovery/Recoverer.java | 205 ++++++++++++++++++ 1 file changed, 205 insertions(+) create mode 100644 src/org/rascalmpl/parser/uptr/recovery/Recoverer.java diff --git a/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java b/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java new file mode 100644 index 00000000000..0a69c83636b --- /dev/null +++ b/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java @@ -0,0 +1,205 @@ +/******************************************************************************* + * Copyright (c) 2009-2013 CWI + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + + * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI + * * Arnold Lankamp - Arnold.Lankamp@cwi.nl +*******************************************************************************/ +package org.rascalmpl.parser.uptr.recovery; + +import org.eclipse.imp.pdb.facts.IConstructor; +import org.rascalmpl.parser.gtd.recovery.IRecoverer; +import org.rascalmpl.parser.gtd.result.AbstractNode; +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; +import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; +import org.rascalmpl.parser.gtd.stack.SkippingStackNode; +import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; +import org.rascalmpl.parser.gtd.util.ArrayList; +import org.rascalmpl.parser.gtd.util.DoubleArrayList; +import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.IntegerObjectList; +import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; +import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.values.uptr.ProductionAdapter; + +public class Recoverer implements IRecoverer{ + // TODO: its a magic constant, and it may clash with other generated constants + // should generate implementation of static int getLastId() in generated parser to fix this. + private int recoveryId = 100000; + + private final int[][] continuationCharactersList; + + private final ObjectKeyedIntegerMap robust; + + public Recoverer(IConstructor[] robustProds, int[][] continuationCharactersList){ + super(); + + this.continuationCharactersList = continuationCharactersList; + + this.robust = new ObjectKeyedIntegerMap(); + + for (int i = robustProds.length - 1; i >= 0; --i) { + robust.put(robustProds[i], i); + } + } + + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ + DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); + + for(int i = recoveryNodes.size() - 1; i >= 0; --i) { + AbstractStackNode recoveryNode = recoveryNodes.getFirst(i); + ArrayList prods = recoveryNodes.getSecond(i); + + // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). + for(int j = prods.size() - 1; j >= 0; --j){ + IConstructor prod = prods.get(j); + + AbstractStackNode continuer = new RecoveryPointStackNode(recoveryId++, prod, recoveryNode); + + int startLocation = recoveryNode.getStartLocation(); + + int[] until = continuationCharactersList[robust.get(prod)]; + AbstractStackNode recoverLiteral = new SkippingStackNode(recoveryId++, until, input, startLocation, prod); + recoverLiteral = recoverLiteral.getCleanCopy(startLocation); + recoverLiteral.initEdges(); + EdgesSet edges = new EdgesSet(1); + edges.add(continuer); + + recoverLiteral.addEdges(edges, startLocation); + + continuer.setIncomingEdges(edges); + + recoveredNodes.add(recoverLiteral, recoverLiteral.getResult()); + } + } + + return recoveredNodes; + } + + private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { + DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); + + for(int i = failedNodes.size() - 1; i >= 0; --i){ + findRecoveryNodes(failedNodes.get(i), recoveryNodes); + } + + return reviveNodes(input, location, recoveryNodes); + } + + private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { + for(int i = unexpandableNodes.getSize() - 1; i >= 0; --i){ + failedNodes.add(unexpandableNodes.get(i)); + } + } + + private static void collectUnmatchableMidProductionNodes(int location, DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ + for(int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i){ + DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); + AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version + + // Merge the information on the predecessors into the failed node. + for(int j = failedNodePredecessors.size() - 1; j >= 0; --j){ + AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); + AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); + failedNode.updateNode(predecessor, predecessorResult); + } + + failedNodes.add(failedNode); + } + } + + private boolean isRobust(IConstructor prod) { + return robust.contains(prod); + } + + /** + * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. + */ + private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { + ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap>(); + Stack> todo = new Stack>(); + + todo.push(failer); + + while (!todo.isEmpty()) { + AbstractStackNode node = todo.pop(); + if(visited.contains(node)) continue; // Don't follow cycles + + visited.put(node, 0); + + ArrayList recoveryProductions = new ArrayList(); + collectProductions(node, recoveryProductions); + + if(recoveryProductions.size() > 0){ + recoveryNodes.add(node, recoveryProductions); + } + + IntegerObjectList> edges = node.getEdges(); + + for(int i = edges.size() - 1; i >= 0; --i){ + EdgesSet edgesList = edges.getValue(i); + + if (edgesList != null) { + for(int j = edgesList.size() - 1; j >= 0; --j){ + AbstractStackNode parent = edgesList.get(j); + + todo.push(parent); + } + } + } + } + } + + // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) + private void collectProductions(AbstractStackNode node, ArrayList productions) { + AbstractStackNode[] production = node.getProduction(); + if(production == null) return; // The root node does not have a production, so ignore it. + + int dot = node.getDot(); + + if(node.isEndNode()){ + IConstructor parentProduction = node.getParentProduction(); + if(isRobust(parentProduction)){ + productions.add(parentProduction); + + if(ProductionAdapter.isList(parentProduction)) return; // Don't follow productions in lists productions, since they are 'cyclic'. + } + } + + for(int i = dot + 1; i < production.length; ++i){ + AbstractStackNode currentNode = production[i]; + if(currentNode.isEndNode()){ + IConstructor parentProduction = currentNode.getParentProduction(); + if(isRobust(parentProduction)){ + productions.add(parentProduction); + } + } + + AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); + if(alternateProductions != null){ + for(int j = alternateProductions.length - 1; j >= 0; --j){ + collectProductions(alternateProductions[j][i], productions); + } + } + } + } + + public DoubleArrayList, AbstractNode> reviveStacks(int[] input, + int location, + Stack> unexpandableNodes, + Stack> unmatchableLeafNodes, + DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, + DoubleStack, AbstractNode> filteredNodes) { + ArrayList> failedNodes = new ArrayList>(); + collectUnexpandableNodes(unexpandableNodes, failedNodes); + collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); + //collectFilteredNodes(filteredNodes, failedNodes); + + return reviveFailedNodes(input, location, failedNodes); + } +} From 9ae895c908e87fa8f152d8fc50a24a03745a542a Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 09:58:30 +0200 Subject: [PATCH 02/87] fixed imports and whitespace --- .../parser/uptr/recovery/Recoverer.java | 39 ++++++++++--------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java b/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java index 0a69c83636b..76f1f6485c2 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2009-2013 CWI + * Copyright (c) 2009-2022 NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -12,7 +12,6 @@ *******************************************************************************/ package org.rascalmpl.parser.uptr.recovery; -import org.eclipse.imp.pdb.facts.IConstructor; import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; @@ -25,7 +24,9 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; -import org.rascalmpl.values.uptr.ProductionAdapter; +import org.rascalmpl.values.parsetrees.ProductionAdapter; + +import io.usethesource.vallang.IConstructor; public class Recoverer implements IRecoverer{ // TODO: its a magic constant, and it may clash with other generated constants @@ -51,12 +52,12 @@ public Recoverer(IConstructor[] robustProds, int[][] continuationCharactersList) private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); - for(int i = recoveryNodes.size() - 1; i >= 0; --i) { + for (int i = recoveryNodes.size() - 1; i >= 0; --i) { AbstractStackNode recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). - for(int j = prods.size() - 1; j >= 0; --j){ + for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); AbstractStackNode continuer = new RecoveryPointStackNode(recoveryId++, prod, recoveryNode); @@ -84,7 +85,7 @@ private DoubleArrayList, AbstractNode> reviveNod private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); - for(int i = failedNodes.size() - 1; i >= 0; --i){ + for (int i = failedNodes.size() - 1; i >= 0; --i) { findRecoveryNodes(failedNodes.get(i), recoveryNodes); } @@ -92,18 +93,18 @@ private DoubleArrayList, AbstractNode> reviveFai } private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { - for(int i = unexpandableNodes.getSize() - 1; i >= 0; --i){ + for (int i = unexpandableNodes.getSize() - 1; i >= 0; --i) { failedNodes.add(unexpandableNodes.get(i)); } } private static void collectUnmatchableMidProductionNodes(int location, DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ - for(int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i){ + for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version // Merge the information on the predecessors into the failed node. - for(int j = failedNodePredecessors.size() - 1; j >= 0; --j){ + for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); failedNode.updateNode(predecessor, predecessorResult); @@ -135,13 +136,13 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr ArrayList recoveryProductions = new ArrayList(); collectProductions(node, recoveryProductions); - if(recoveryProductions.size() > 0){ + if (recoveryProductions.size() > 0) { recoveryNodes.add(node, recoveryProductions); } IntegerObjectList> edges = node.getEdges(); - for(int i = edges.size() - 1; i >= 0; --i){ + for (int i = edges.size() - 1; i >= 0; --i) { EdgesSet edgesList = edges.getValue(i); if (edgesList != null) { @@ -162,27 +163,29 @@ private void collectProductions(AbstractStackNode node, ArrayList< int dot = node.getDot(); - if(node.isEndNode()){ + if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); if(isRobust(parentProduction)){ productions.add(parentProduction); - if(ProductionAdapter.isList(parentProduction)) return; // Don't follow productions in lists productions, since they are 'cyclic'. + if (ProductionAdapter.isList(parentProduction)) { + return; // Don't follow productions in lists productions, since they are 'cyclic'. + } } } - for(int i = dot + 1; i < production.length; ++i){ + for (int i = dot + 1; i < production.length; ++i) { AbstractStackNode currentNode = production[i]; - if(currentNode.isEndNode()){ + if (currentNode.isEndNode()) { IConstructor parentProduction = currentNode.getParentProduction(); - if(isRobust(parentProduction)){ + if (isRobust(parentProduction)) { productions.add(parentProduction); } } AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); - if(alternateProductions != null){ - for(int j = alternateProductions.length - 1; j >= 0; --j){ + if (alternateProductions != null) { + for (int j = alternateProductions.length - 1; j >= 0; --j) { collectProductions(alternateProductions[j][i], productions); } } From 4e43070c15d65ff35de6df74c7787254bab81cbe Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 10:15:31 +0200 Subject: [PATCH 03/87] added documentation --- .../parser/gtd/recovery/IRecoverer.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java b/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java index f7db2b352ab..40dc4d52257 100644 --- a/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java +++ b/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java @@ -17,7 +17,22 @@ import org.rascalmpl.parser.gtd.util.DoubleStack; import org.rascalmpl.parser.gtd.util.Stack; -public interface IRecoverer

{ +public interface IRecoverer

{ + /** + * reviveStacks is called when the parser is unable to make more progress and the end-of-input + * has not been reached. The parameters provide insight into the current state of the parser + * and some of its history. With this information new stack nodes may be generated for the parser + * to continue with. It is up to the reviveStacks method to make sure the parser still generates + * derivation trees that cover the entire input. + * + * @param input the 24-bit unicode input character array + * @param location the current character offset in the input that the parser got stuck on + * @param unexpandableNodes these are non-terminals that were predicted at this location but did not fly + * @param unmatchableLeafNodes these are the terminals that were predicted but did not fly + * @param unmatchableMidProductionNodes these are quasi-non-terminals due to prefix sharing that did not fly + * @param filteredNodes these are non-terminals nodes that did not fly due to a disambiguation filter + * @return a list of new predictions for the parser to continue with + */ DoubleArrayList, AbstractNode> reviveStacks(int[] input, int location, Stack> unexpandableNodes, From ff99ac9bc7eae6dfeb257914319628270f1e0a56 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 10:16:05 +0200 Subject: [PATCH 04/87] renamed recoverer and simplified to whitespace only and all context-free non-terminals --- ...er.java => ToNextWhitespaceRecoverer.java} | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) rename src/org/rascalmpl/parser/uptr/recovery/{Recoverer.java => ToNextWhitespaceRecoverer.java} (91%) diff --git a/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java similarity index 91% rename from src/org/rascalmpl/parser/uptr/recovery/Recoverer.java rename to src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 76f1f6485c2..7d2ceddbb79 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/Recoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -28,26 +28,12 @@ import io.usethesource.vallang.IConstructor; -public class Recoverer implements IRecoverer{ +public class ToNextWhitespaceRecoverer implements IRecoverer{ // TODO: its a magic constant, and it may clash with other generated constants // should generate implementation of static int getLastId() in generated parser to fix this. private int recoveryId = 100000; + private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; - private final int[][] continuationCharactersList; - - private final ObjectKeyedIntegerMap robust; - - public Recoverer(IConstructor[] robustProds, int[][] continuationCharactersList){ - super(); - - this.continuationCharactersList = continuationCharactersList; - - this.robust = new ObjectKeyedIntegerMap(); - - for (int i = robustProds.length - 1; i >= 0; --i) { - robust.put(robustProds[i], i); - } - } private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); @@ -64,8 +50,7 @@ private DoubleArrayList, AbstractNode> reviveNod int startLocation = recoveryNode.getStartLocation(); - int[] until = continuationCharactersList[robust.get(prod)]; - AbstractStackNode recoverLiteral = new SkippingStackNode(recoveryId++, until, input, startLocation, prod); + AbstractStackNode recoverLiteral = new SkippingStackNode(recoveryId++, WHITESPACE, input, startLocation, prod); recoverLiteral = recoverLiteral.getCleanCopy(startLocation); recoverLiteral.initEdges(); EdgesSet edges = new EdgesSet(1); @@ -114,10 +99,6 @@ private static void collectUnmatchableMidProductionNodes(int location, DoubleSta } } - private boolean isRobust(IConstructor prod) { - return robust.contains(prod); - } - /** * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. */ @@ -165,7 +146,7 @@ private void collectProductions(AbstractStackNode node, ArrayList< if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); - if(isRobust(parentProduction)){ + if(ProductionAdapter.isContextFree(parentProduction)){ productions.add(parentProduction); if (ProductionAdapter.isList(parentProduction)) { @@ -178,7 +159,7 @@ private void collectProductions(AbstractStackNode node, ArrayList< AbstractStackNode currentNode = production[i]; if (currentNode.isEndNode()) { IConstructor parentProduction = currentNode.getParentProduction(); - if (isRobust(parentProduction)) { + if (ProductionAdapter.isContextFree(parentProduction)) { productions.add(parentProduction); } } From d2a0b668e1c584f76b91ee35b296554b9eda946d Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:20:55 +0200 Subject: [PATCH 05/87] removed unused javadoc plugin that produced warnings --- pom.xml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pom.xml b/pom.xml index 3d285611905..ae8b8ea02a2 100644 --- a/pom.xml +++ b/pom.xml @@ -92,14 +92,6 @@ - - org.apache.maven.plugins - maven-javadoc-plugin - 3.2.0 - - -Xdoclint:none - - org.codehaus.mojo buildnumber-maven-plugin From d9439a450020942fe80617379277c75c5020be63 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:21:07 +0200 Subject: [PATCH 06/87] fixed compiler warnings --- .../library/lang/java/m3/internal/JarConverter.java | 7 ------- src/org/rascalmpl/semantics/dynamic/Tree.java | 11 ++++------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/src/org/rascalmpl/library/lang/java/m3/internal/JarConverter.java b/src/org/rascalmpl/library/lang/java/m3/internal/JarConverter.java index a202cf5087a..0061d3ee329 100644 --- a/src/org/rascalmpl/library/lang/java/m3/internal/JarConverter.java +++ b/src/org/rascalmpl/library/lang/java/m3/internal/JarConverter.java @@ -266,7 +266,6 @@ private void setClassRelations(ClassReader classReader, String compUnitRelative) ISourceLocation classLogical = resolver.resolveBinding(classNode, null); ISourceLocation classPhysical = M3LocationUtil.makeLocation(compUnitPhysical, classReader.header, classReader.b.length); IConstructor cons = resolver.resolveType(classNode, null); - @SuppressWarnings("unchecked") List annotations = composeAnnotations(classNode.visibleAnnotations, classNode.invisibleAnnotations); addToContainment(compUnitLogical, classLogical); @@ -291,7 +290,6 @@ private void setClassRelations(ClassReader classReader, String compUnitRelative) */ private void setInnerClassRelations(ClassNode classNode, ISourceLocation classLogical) { // cn.innerClasses and cn.outerClass are not providing consistent information. - @SuppressWarnings("unchecked") List innerClasses = classNode.innerClasses; if (innerClasses != null) { @@ -322,7 +320,6 @@ private void setInnerClassRelations(ClassNode classNode, ISourceLocation classLo * @param classNode - class node where fields are declared * @param classLogical - class location */ - @SuppressWarnings("unchecked") private void setFieldRelations(ClassNode classNode, ISourceLocation classLogical) { List fields = classNode.fields; @@ -356,7 +353,6 @@ private void setFieldRelations(ClassNode classNode, ISourceLocation classLogical * @param classNode - class node where fields are declared * @param classLogical - class location */ - @SuppressWarnings("unchecked") private void setMethodRelations(ClassNode classNode, ISourceLocation classLogical) { List methods = classNode.methods; @@ -400,7 +396,6 @@ private void setMethodOverridesRelation(String superClass, MethodNode methodNode if (classReader != null) { ClassNode classNode = new ClassNode(); classReader.accept(classNode, ClassReader.SKIP_DEBUG); - @SuppressWarnings("unchecked") List superMethods = classNode.methods; if (superMethods != null) { @@ -425,7 +420,6 @@ private void setMethodOverridesRelation(String superClass, MethodNode methodNode * @param methodLogical - logical location of the method */ private void setExceptionRelations(MethodNode methodNode, ISourceLocation methodLogical) { - @SuppressWarnings("unchecked") List exceptions = methodNode.exceptions; for (String exception : exceptions) { @@ -468,7 +462,6 @@ private void setInstructionRelations(MethodNode methodNode, ISourceLocation meth InsnList instructions = methodNode.instructions; if(instructions != null) { - @SuppressWarnings("unchecked") ListIterator iterator = instructions.iterator(); while (iterator.hasNext()) { diff --git a/src/org/rascalmpl/semantics/dynamic/Tree.java b/src/org/rascalmpl/semantics/dynamic/Tree.java index 51b36ffa629..f69f8c750c5 100644 --- a/src/org/rascalmpl/semantics/dynamic/Tree.java +++ b/src/org/rascalmpl/semantics/dynamic/Tree.java @@ -27,15 +27,17 @@ import org.rascalmpl.interpreter.matching.IMatchingResult; import org.rascalmpl.interpreter.matching.LiteralPattern; import org.rascalmpl.interpreter.matching.NodePattern; -import org.rascalmpl.interpreter.matching.QualifiedNamePattern; import org.rascalmpl.interpreter.matching.SetPattern; import org.rascalmpl.interpreter.matching.TypedVariablePattern; import org.rascalmpl.interpreter.result.Result; import org.rascalmpl.interpreter.staticErrors.UndeclaredVariable; import org.rascalmpl.interpreter.staticErrors.UninitializedVariable; -import org.rascalmpl.interpreter.utils.Names; import org.rascalmpl.types.NonTerminalType; import org.rascalmpl.types.RascalTypeFactory; +import org.rascalmpl.values.RascalValueFactory; +import org.rascalmpl.values.parsetrees.ProductionAdapter; +import org.rascalmpl.values.parsetrees.SymbolAdapter; +import org.rascalmpl.values.parsetrees.TreeAdapter; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IList; @@ -45,11 +47,6 @@ import io.usethesource.vallang.IValue; import io.usethesource.vallang.type.Type; -import org.rascalmpl.values.RascalValueFactory; -import org.rascalmpl.values.parsetrees.ProductionAdapter; -import org.rascalmpl.values.parsetrees.SymbolAdapter; -import org.rascalmpl.values.parsetrees.TreeAdapter; - /** * These classes special case Expression.CallOrTree for concrete syntax patterns */ From d87b4b85291cd6f0a219a3b2e11be5ba36d45afa Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:29:11 +0200 Subject: [PATCH 07/87] fixed warnings --- .../rascalmpl/interpreter/matching/VariableBecomesPattern.java | 1 - src/org/rascalmpl/interpreter/result/ResultFactory.java | 2 -- 2 files changed, 3 deletions(-) diff --git a/src/org/rascalmpl/interpreter/matching/VariableBecomesPattern.java b/src/org/rascalmpl/interpreter/matching/VariableBecomesPattern.java index 66e2530f716..0370efb1b42 100644 --- a/src/org/rascalmpl/interpreter/matching/VariableBecomesPattern.java +++ b/src/org/rascalmpl/interpreter/matching/VariableBecomesPattern.java @@ -17,7 +17,6 @@ import java.util.HashMap; import java.util.List; -import org.rascalmpl.ast.AbstractAST; import org.rascalmpl.ast.Expression; import org.rascalmpl.interpreter.IEvaluatorContext; import org.rascalmpl.interpreter.env.Environment; diff --git a/src/org/rascalmpl/interpreter/result/ResultFactory.java b/src/org/rascalmpl/interpreter/result/ResultFactory.java index 27917622418..4607b274ce0 100644 --- a/src/org/rascalmpl/interpreter/result/ResultFactory.java +++ b/src/org/rascalmpl/interpreter/result/ResultFactory.java @@ -13,10 +13,8 @@ import org.rascalmpl.exceptions.ImplementationError; import org.rascalmpl.interpreter.IEvaluatorContext; -import org.rascalmpl.interpreter.utils.TreeAsNode; import org.rascalmpl.types.RascalType; import org.rascalmpl.values.RascalValueFactory; -import org.rascalmpl.values.parsetrees.ITree; import io.usethesource.vallang.IBool; import io.usethesource.vallang.IConstructor; From 4897d88fbb5b285ea5a0a5b0f3fb575365387aec Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:29:26 +0200 Subject: [PATCH 08/87] fixed warnings --- src/org/rascalmpl/util/HeapDumper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/org/rascalmpl/util/HeapDumper.java b/src/org/rascalmpl/util/HeapDumper.java index d2376434353..4af965e9ded 100644 --- a/src/org/rascalmpl/util/HeapDumper.java +++ b/src/org/rascalmpl/util/HeapDumper.java @@ -25,7 +25,6 @@ * * Use it to dump the heap to a file for further analysis. */ -@SuppressWarnings("restriction") public class HeapDumper { private static final class InstanceKeeper { /*package*/ From 58034115b6d59f342b01a1c62e8321771b385d7e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:29:57 +0200 Subject: [PATCH 09/87] added boolean parameter \'robust\' to parsing API --- src/org/rascalmpl/library/ParseTree.rsc | 16 ++++++++-------- src/org/rascalmpl/library/Prelude.java | 8 ++++---- .../rascalmpl/values/IRascalValueFactory.java | 9 +++++---- .../values/RascalFunctionValueFactory.java | 18 ++++++++++-------- 4 files changed, 27 insertions(+), 24 deletions(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index cedafe73c61..2f2385d6ab1 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -449,14 +449,14 @@ catch ParseError(loc l): { } ---- } -public &T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, |unknown:///|); +public &T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool hasSideEffects=false, bool robust=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, robust=robust, filters=filters)(input, |unknown:///|); -public &T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, origin); +public &T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool hasSideEffects=false, bool robust=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, robust=robust, filters=filters)(input, origin); -public &T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, input); +public &T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool hasSideEffects=false, bool robust=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, robust=robust, filters=filters)(input, input); @doc{ .Synopsis @@ -487,7 +487,7 @@ The parse function behaves differently depending of the given keyword parameters * parse forest to be constructed in polynomial time. } @javaClass{org.rascalmpl.library.Prelude} -public java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, set[Tree(Tree)] filters={}); +public java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, bool robust=false, set[Tree(Tree)] filters={}); @doc{ .Synopsis @@ -499,7 +499,7 @@ This parser generator behaves the same as the `parser` function, but it produces nonterminal parameter. This can be used to select a specific non-terminal from the grammar to use as start-symbol for parsing. } @javaClass{org.rascalmpl.library.Prelude} -public java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, set[Tree(Tree)] filters={}); +public java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, bool robust=false, set[Tree(Tree)] filters={}); @doc{ .Synopsis parse the input but instead of returning the entire tree, return the trees for the first ambiguous substring. diff --git a/src/org/rascalmpl/library/Prelude.java b/src/org/rascalmpl/library/Prelude.java index e096c3b0e5c..258d54c4aef 100644 --- a/src/org/rascalmpl/library/Prelude.java +++ b/src/org/rascalmpl/library/Prelude.java @@ -2237,12 +2237,12 @@ public INode arbNode() { protected final TypeReifier tr; - public IFunction parser(IValue start, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { - return rascalValues.parser(start, allowAmbiguity, hasSideEffects, firstAmbiguity, filters); + public IFunction parser(IValue start, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, IBool robust, ISet filters) { + return rascalValues.parser(start, allowAmbiguity, hasSideEffects, firstAmbiguity, robust, filters); } - public IFunction parsers(IValue start, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { - return rascalValues.parsers(start, allowAmbiguity, hasSideEffects, firstAmbiguity, filters); + public IFunction parsers(IValue start, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, IBool robust, ISet filters) { + return rascalValues.parsers(start, allowAmbiguity, hasSideEffects, firstAmbiguity, robust, filters); } // REFLECT -- copy in {@link PreludeCompiled} diff --git a/src/org/rascalmpl/values/IRascalValueFactory.java b/src/org/rascalmpl/values/IRascalValueFactory.java index 405924792bf..f43d662b1ef 100644 --- a/src/org/rascalmpl/values/IRascalValueFactory.java +++ b/src/org/rascalmpl/values/IRascalValueFactory.java @@ -87,12 +87,15 @@ default IFunction function(Type functionType, BiFunction Date: Thu, 31 Mar 2022 12:37:15 +0200 Subject: [PATCH 10/87] wired boolean parameter for robustness from Rascal function down to parser implementation --- src/org/rascalmpl/interpreter/Evaluator.java | 18 +++++++++-------- src/org/rascalmpl/interpreter/IEvaluator.java | 8 ++++---- .../semantics/dynamic/Expression.java | 4 ++-- .../values/RascalFunctionValueFactory.java | 20 +++++++++---------- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/src/org/rascalmpl/interpreter/Evaluator.java b/src/org/rascalmpl/interpreter/Evaluator.java index 2069f732aba..4a2929545b4 100755 --- a/src/org/rascalmpl/interpreter/Evaluator.java +++ b/src/org/rascalmpl/interpreter/Evaluator.java @@ -103,6 +103,7 @@ import org.rascalmpl.parser.uptr.UPTRNodeFactory; import org.rascalmpl.parser.uptr.action.NoActionExecutor; import org.rascalmpl.parser.uptr.action.RascalFunctionActionExecutor; +import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.uri.URIUtil; import org.rascalmpl.values.parsetrees.ITree; @@ -714,7 +715,7 @@ public IValue call(QualifiedName qualifiedName, Map kwArgs, IValu } @Override - public ITree parseObject(IConstructor grammar, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects) { + public ITree parseObject(IConstructor grammar, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects, boolean robust) { IConstructor startSort = (IConstructor) grammar.get("symbol"); IGTD parser; String name; @@ -728,16 +729,17 @@ public ITree parseObject(IConstructor grammar, ISet filters, ISourceLocation loc ? new RascalFunctionActionExecutor(filters, !hasSideEffects) : new NoActionExecutor(); - return (ITree) parser.parse(name, location.getURI(), input, exec, new DefaultNodeFlattener(), new UPTRNodeFactory(allowAmbiguity), (IRecoverer) null); + IRecoverer recoverer = robust ? new ToNextWhitespaceRecoverer() : null; + return (ITree) parser.parse(name, location.getURI(), input, exec, new DefaultNodeFlattener(), new UPTRNodeFactory(allowAmbiguity), recoverer); } @Override - public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean hasSideEffects){ + public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean hasSideEffects, boolean robust){ IRascalMonitor old = setMonitor(monitor); try { char[] input = getResourceContent(location); - return parseObject(startSort, filters, location, input, allowAmbiguity, hasSideEffects); + return parseObject(startSort, filters, location, input, allowAmbiguity, hasSideEffects, robust); } catch(IOException ioex){ throw RuntimeExceptionFactory.io(vf.string(ioex.getMessage()), getCurrentAST(), getStackTrace()); @@ -748,10 +750,10 @@ public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, } @Override - public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, String input, boolean allowAmbiguity, boolean hasSideEffects) { + public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, String input, boolean allowAmbiguity, boolean hasSideEffects, boolean robust) { IRascalMonitor old = setMonitor(monitor); try { - return parseObject(startSort, filters, URIUtil.invalidLocation(), input.toCharArray(), allowAmbiguity, hasSideEffects); + return parseObject(startSort, filters, URIUtil.invalidLocation(), input.toCharArray(), allowAmbiguity, hasSideEffects, robust); } finally { setMonitor(old); @@ -759,10 +761,10 @@ public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, } @Override - public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, String input, ISourceLocation loc, boolean allowAmbiguity, boolean hasSideEffects) { + public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, ISet filters, String input, ISourceLocation loc, boolean allowAmbiguity, boolean hasSideEffects, boolean robust) { IRascalMonitor old = setMonitor(monitor); try{ - return parseObject(startSort, filters, loc, input.toCharArray(), allowAmbiguity, hasSideEffects); + return parseObject(startSort, filters, loc, input.toCharArray(), allowAmbiguity, hasSideEffects, robust); }finally{ setMonitor(old); } diff --git a/src/org/rascalmpl/interpreter/IEvaluator.java b/src/org/rascalmpl/interpreter/IEvaluator.java index 35f8065cfb1..2bb5fc69db6 100644 --- a/src/org/rascalmpl/interpreter/IEvaluator.java +++ b/src/org/rascalmpl/interpreter/IEvaluator.java @@ -144,16 +144,16 @@ public Result evalMore(IRascalMonitor monitor, String commands, */ public IValue call(String adt, String name, IValue... args); - public IConstructor parseObject(IConstructor startSort, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects); + public IConstructor parseObject(IConstructor startSort, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects, boolean robust); public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, - ISet filters, String input, ISourceLocation loc, boolean allowAmbiguity, boolean hasSideEffects); + ISet filters, String input, ISourceLocation loc, boolean allowAmbiguity, boolean hasSideEffects, boolean robust); public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, - ISet filters, String input, boolean allowAmbiguity, boolean hasSideEffects); + ISet filters, String input, boolean allowAmbiguity, boolean hasSideEffects, boolean robust); public IConstructor parseObject(IRascalMonitor monitor, IConstructor startSort, - ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean hasSideEffects); + ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean hasSideEffects, boolean robust); /** * Freeze the global state of this evaluator so that it can no longer be updated. diff --git a/src/org/rascalmpl/semantics/dynamic/Expression.java b/src/org/rascalmpl/semantics/dynamic/Expression.java index 95adc37aedc..78117d90e3b 100644 --- a/src/org/rascalmpl/semantics/dynamic/Expression.java +++ b/src/org/rascalmpl/semantics/dynamic/Expression.java @@ -1089,11 +1089,11 @@ public Result interpret(IEvaluator> __eval) { if (result.getStaticType().isString()) { tree = __eval.parseObject(value, VF.set(), this.getLocation(), - ((IString) result.getValue()).getValue().toCharArray(), true, false); + ((IString) result.getValue()).getValue().toCharArray(), true, false, false); } else if (result.getStaticType().isSourceLocation()) { tree = __eval.parseObject(__eval, value, VF.set(), - ((ISourceLocation) result.getValue()), true, false); + ((ISourceLocation) result.getValue()), true, false, false); } assert tree != null; // because we checked earlier diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 616f4f57877..76a150ddd7d 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -229,10 +229,10 @@ else if (parameters[0].getType().isSourceLocation()) { } if (parameters[0].getType().isString()) { - return parse(grammar, filters, (IString) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects, ctx); + return parse(grammar, filters, (IString) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects, robust, ctx); } else if (parameters[0].getType().isSourceLocation()) { - return parse(grammar, filters, (ISourceLocation) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects, ctx); + return parse(grammar, filters, (ISourceLocation) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects, robust, ctx); } } @@ -243,7 +243,7 @@ protected Throw fail(IValue... parameters) { return RuntimeExceptionFactory.callFailed(URIUtil.rootLocation("unknown"), Arrays.stream(parameters).collect(ctx.getValueFactory().listWriter())); } - protected IValue parse(IValue start, ISet filters, IString input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects, IEvaluatorContext ctx) { + protected IValue parse(IValue start, ISet filters, IString input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects, boolean robust, IEvaluatorContext ctx) { try { Type reified = start.getType(); if (origin == null) { @@ -251,7 +251,7 @@ protected IValue parse(IValue start, ISet filters, IString input, ISourceLocati } IConstructor grammar = checkPreconditions(start, reified); - return ctx.getEvaluator().parseObject(grammar, filters, origin, input.getValue().toCharArray(), allowAmbiguity, hasSideEffects); + return ctx.getEvaluator().parseObject(grammar, filters, origin, input.getValue().toCharArray(), allowAmbiguity, hasSideEffects, robust); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -271,7 +271,7 @@ protected IValue firstAmbiguity(IValue start, IString input, IEvaluatorContext c IConstructor grammar = checkPreconditions(start, reified); try { - return ctx.getEvaluator().parseObject(grammar, vf.set(), URIUtil.invalidLocation(), input.getValue().toCharArray(), false, false); + return ctx.getEvaluator().parseObject(grammar, vf.set(), URIUtil.invalidLocation(), input.getValue().toCharArray(), false, false, false); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -290,7 +290,7 @@ protected IValue firstAmbiguity(IValue start, ISourceLocation input, IEvaluatorC IConstructor grammar = checkPreconditions(start, reified); try { - return ctx.getEvaluator().parseObject(grammar, vf.set(), input, readAll(input), false, false); + return ctx.getEvaluator().parseObject(grammar, vf.set(), input, readAll(input), false, false, false); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -317,7 +317,7 @@ private IString printSymbol(IConstructor symbol) { return vf.string(SymbolAdapter.toString(symbol, false)); } - protected IValue parse(IValue start, ISet filters, ISourceLocation input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects, IEvaluatorContext ctx) { + protected IValue parse(IValue start, ISet filters, ISourceLocation input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects, boolean robust, IEvaluatorContext ctx) { Type reified = start.getType(); IConstructor grammar = checkPreconditions(start, reified); @@ -326,7 +326,7 @@ protected IValue parse(IValue start, ISet filters, ISourceLocation input, ISourc } try { - return ctx.getEvaluator().parseObject(grammar, vf.set(), input, readAll(input), allowAmbiguity, hasSideEffects); + return ctx.getEvaluator().parseObject(grammar, vf.set(), input, readAll(input), allowAmbiguity, hasSideEffects, robust); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -397,10 +397,10 @@ else if (parameters[1].getType().isSourceLocation()) { } if (parameters[1].getType().isString()) { - return parse(parameters[0], filters, (IString) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects, ctx); + return parse(parameters[0], filters, (IString) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects, robust, ctx); } else if (parameters[1].getType().isSourceLocation()) { - return parse(parameters[0], filters, (ISourceLocation) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects, ctx); + return parse(parameters[0], filters, (ISourceLocation) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects, robust, ctx); } } From 7318e8f4d11a565e289cfb21c8c56852469109dd Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 31 Mar 2022 12:49:17 +0200 Subject: [PATCH 11/87] added override --- .../parser/uptr/recovery/ToNextWhitespaceRecoverer.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 7d2ceddbb79..bdb6c8665e1 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -173,6 +173,7 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } + @Override public DoubleArrayList, AbstractNode> reviveStacks(int[] input, int location, Stack> unexpandableNodes, From f6e13d88bf8ebf965497298c6b3fb0d5e6e59dfc Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 11 Apr 2022 17:19:24 +0200 Subject: [PATCH 12/87] minor additions to make recovery work and removed dead code --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 2 +- .../parser/gtd/stack/SkippingStackNode.java | 4 +- .../recovery/ToNextWhitespaceRecoverer.java | 51 ++----------------- 3 files changed, 9 insertions(+), 48 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 07e35861917..6e0c8e31845 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -815,7 +815,7 @@ private boolean findStacksToReduce(){ } } - if (recoverer != null) { + if (recoverer != null && location < input.length) { DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (recoveredNodes.size() > 0) { for (int i = 0; i < recoveredNodes.size(); i++) { diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index eb243b3b6c0..63164ab3bc7 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -91,7 +91,9 @@ public int hashCode(){ } public boolean isEqual(AbstractStackNode

stackNode){ - if(!(stackNode instanceof SkippingStackNode)) return false; + if ( !(stackNode instanceof SkippingStackNode)) { + return false; + } SkippingStackNode

otherNode = (SkippingStackNode

) stackNode; diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index bdb6c8665e1..f195e71a0b3 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -24,7 +24,6 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; -import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; @@ -110,24 +109,20 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr while (!todo.isEmpty()) { AbstractStackNode node = todo.pop(); - if(visited.contains(node)) continue; // Don't follow cycles - visited.put(node, 0); - - ArrayList recoveryProductions = new ArrayList(); - collectProductions(node, recoveryProductions); - - if (recoveryProductions.size() > 0) { - recoveryNodes.add(node, recoveryProductions); + if (visited.contains(node)) { + continue; // Don't follow cycles } + visited.put(node, 0); + IntegerObjectList> edges = node.getEdges(); for (int i = edges.size() - 1; i >= 0; --i) { EdgesSet edgesList = edges.getValue(i); if (edgesList != null) { - for(int j = edgesList.size() - 1; j >= 0; --j){ + for (int j = edgesList.size() - 1; j >= 0; --j) { AbstractStackNode parent = edgesList.get(j); todo.push(parent); @@ -137,42 +132,6 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr } } - // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) - private void collectProductions(AbstractStackNode node, ArrayList productions) { - AbstractStackNode[] production = node.getProduction(); - if(production == null) return; // The root node does not have a production, so ignore it. - - int dot = node.getDot(); - - if (node.isEndNode()) { - IConstructor parentProduction = node.getParentProduction(); - if(ProductionAdapter.isContextFree(parentProduction)){ - productions.add(parentProduction); - - if (ProductionAdapter.isList(parentProduction)) { - return; // Don't follow productions in lists productions, since they are 'cyclic'. - } - } - } - - for (int i = dot + 1; i < production.length; ++i) { - AbstractStackNode currentNode = production[i]; - if (currentNode.isEndNode()) { - IConstructor parentProduction = currentNode.getParentProduction(); - if (ProductionAdapter.isContextFree(parentProduction)) { - productions.add(parentProduction); - } - } - - AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); - if (alternateProductions != null) { - for (int j = alternateProductions.length - 1; j >= 0; --j) { - collectProductions(alternateProductions[j][i], productions); - } - } - } - } - @Override public DoubleArrayList, AbstractNode> reviveStacks(int[] input, int location, From ba0fb43fc8207d12999924ac3c72aff398aa1cfc Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 13 Apr 2022 16:28:46 +0200 Subject: [PATCH 13/87] recovered a missing piece from the recovery code --- .../recovery/ToNextWhitespaceRecoverer.java | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index f195e71a0b3..bd94e061868 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -24,6 +24,7 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; @@ -116,6 +117,12 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr visited.put(node, 0); + ArrayList recoveryProductions = new ArrayList(); + collectProductions(node, recoveryProductions); + if (recoveryProductions.size() > 0) { + recoveryNodes.add(node, recoveryProductions); + } + IntegerObjectList> edges = node.getEdges(); for (int i = edges.size() - 1; i >= 0; --i) { @@ -132,6 +139,45 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr } } + // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) + private void collectProductions(AbstractStackNode node, ArrayList productions) { + AbstractStackNode[] production = node.getProduction(); + if (production == null) { + return; // The root node does not have a production, so ignore it. + } + + int dot = node.getDot(); + + if (node.isEndNode()) { + IConstructor parentProduction = node.getParentProduction(); + if (ProductionAdapter.isContextFree(parentProduction)){ + productions.add(parentProduction); + + if (ProductionAdapter.isList(parentProduction)) { + return; // Don't follow productions in lists productions, since they are 'cyclic'. + } + } + } + + for (int i = dot + 1; i < production.length; ++i) { + AbstractStackNode currentNode = production[i]; + if (currentNode.isEndNode()) { + IConstructor parentProduction = currentNode.getParentProduction(); + if (ProductionAdapter.isContextFree(parentProduction)) { + productions.add(parentProduction); + } + } + + AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); + if (alternateProductions != null) { + for (int j = alternateProductions.length - 1; j >= 0; --j) { + collectProductions(alternateProductions[j][i], productions); + } + } + } + } + + @Override public DoubleArrayList, AbstractNode> reviveStacks(int[] input, int location, From 650224af21b441a874f0e9241f85474470c8bcd9 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 14 Apr 2022 17:23:15 +0200 Subject: [PATCH 14/87] Debugging error recovery Sometimes recovery nodes start before the current location where the parser failed to continue. Since the parser works with a short queue of schedulede TODO's around the current cursor, we might end up outside of this queue when recovering. This breaks several unspecified invariants of the SGTBF implementations. For now I added a detection that a recovery node is to be planned before the currently retained history and filter that recovery node. The next step will be to make sure backtracking over the current location is made possible. --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 6e0c8e31845..83b7578afb7 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -823,7 +823,9 @@ private boolean findStacksToReduce(){ int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation()); - addTodo(recovered, levelsFromHere, recoveredNodes.getSecond(i)); +// if (levelsFromHere >= 0) { // TODO experien + addTodo(recovered, levelsFromHere, recoveredNodes.getSecond(i)); +// } } return findStacksToReduce(); } @@ -853,6 +855,13 @@ private void addTodo(AbstractStackNode

node, int length, AbstractNode result) queueDepth = length + 1; queueIndex = 0; } + else if (length < 0) { + if (length + queueIndex < 0) { + // the queue is not long enough back into the past locations + // to cover this recovery node, so we must skip it + return; + } + } int insertLocation = (queueIndex + length) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[insertLocation]; From 6d5d49b64a2a166dd19c0da9f7e7786e3e29634b Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 15 Apr 2022 10:54:51 +0200 Subject: [PATCH 15/87] added possibility for recovered nodes to start back in time (at earlier input locations) --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 81 +++++++++++++++++++----- 1 file changed, 64 insertions(+), 17 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 83b7578afb7..c2777fbea8f 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -786,7 +786,7 @@ private boolean findFirstStacksToReduce(){ if (recoveredNodes.size() > 0) { // TODO Do something with the revived node. Is this the right location to do this? for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); - addTodo(recovered, recovered.getLength(), recoveredNodes.getSecond(i)); + queueMatchableNode(recovered, recovered.getLength(), recoveredNodes.getSecond(i)); } return findStacksToReduce(); } @@ -821,11 +821,9 @@ private boolean findStacksToReduce(){ for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); - int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation()); +// int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation()); -// if (levelsFromHere >= 0) { // TODO experien - addTodo(recovered, levelsFromHere, recoveredNodes.getSecond(i)); -// } + queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), recoveredNodes.getSecond(i)); } return findStacksToReduce(); } @@ -844,8 +842,9 @@ public boolean parseErrorHasOccurred(){ * Inserts a stack bottom into the todo-list. */ @SuppressWarnings("unchecked") - private void addTodo(AbstractStackNode

node, int length, AbstractNode result){ - if(result == null) throw new RuntimeException(); + private void queueMatchableNode(AbstractStackNode

node, int length, AbstractNode result){ + assert result != null; + int queueDepth = todoLists.length; if(length >= queueDepth){ DoubleStack, AbstractNode>[] oldTodoLists = todoLists; @@ -855,13 +854,6 @@ private void addTodo(AbstractStackNode

node, int length, AbstractNode result) queueDepth = length + 1; queueIndex = 0; } - else if (length < 0) { - if (length + queueIndex < 0) { - // the queue is not long enough back into the past locations - // to cover this recovery node, so we must skip it - return; - } - } int insertLocation = (queueIndex + length) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[insertLocation]; @@ -872,6 +864,61 @@ else if (length < 0) { terminalsTodo.push(node, result); } + /** + * Inserts a recovery node into the todo-list, and possibly + * rewinds the parser to an earlier location in the input + */ + @SuppressWarnings("unchecked") + private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int length, AbstractNode result){ + assert result != null; + + int queueDepth = todoLists.length; + + if (startPosition < location) { + // Have to reset the parser to an earlier location to at least + // be able to process the new node. Cannot throw away the queue, + // because there are possibly already other recovery tokens in the queue. + // However, we may assume that the queue before the current index is + // done, based on the way we cycle the queue now. The queue is + // looking forward to the future and we never re-use past entries. + + int negativeOffset = location - startPosition; + + DoubleStack, AbstractNode>[] oldTodoLists = todoLists; + todoLists = new DoubleStack[negativeOffset + Math.max(queueDepth, length) + 1]; + System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset, queueDepth - queueIndex); + System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset, queueIndex); + + // reset the parser! + queueIndex = 0; + location = startPosition; + + DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; + if (terminalsTodo == null){ + terminalsTodo = new DoubleStack, AbstractNode>(); + todoLists[queueIndex] = terminalsTodo; + } + else { + assert false: "this should never happen"; + } + + terminalsTodo.push(node, result); + } + else if (startPosition == location) { + // this is the normal case where new matchable nodes are discovered + // for the current parsing location, so reuse the code for queuing + queueMatchableNode(node, length, result); + } + else { + // This would mean we have discovered a recovery node for a location + // we have not been yet. That would be odd because then there would + // not have been a parse error and we wouldn't need recovery... + throw new RuntimeException("discovered a future recovery? " + node); + } + } + + + /** * Handles the retrieved alternatives for the given stack. */ @@ -913,7 +960,7 @@ private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cached first = first.getCleanCopyWithResult(location, result); - addTodo(first, length, result); + queueMatchableNode(first, length, result); }else{ first = first.getCleanCopy(location); stacksToExpand.push(first); @@ -974,7 +1021,7 @@ private void expandStack(AbstractStackNode

stack){ } if(stack.isMatchable()){ // Eager matching optimization related. - addTodo(stack, stack.getLength(), stack.getResult()); + queueMatchableNode(stack, stack.getLength(), stack.getResult()); }else if(!stack.isExpandable()){ // A 'normal' non-terminal. EdgesSet

cachedEdges = cachedEdgesForExpect.get(stack.getName()); if(cachedEdges == null){ @@ -1051,7 +1098,7 @@ private void expandStack(AbstractStackNode

stack){ } child = child.getCleanCopyWithResult(location, result); - addTodo(child, length, result); + queueMatchableNode(child, length, result); }else{ child = child.getCleanCopy(location); stacksToExpand.push(child); From 78ee76be767a99c77256e38dce44618af01d4f5f Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 15 Apr 2022 11:02:26 +0200 Subject: [PATCH 16/87] fixed off-by-one: error nodes should be scheduled one character ahead because the next parser loop iteration always wants to advance one character --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index c2777fbea8f..1ac29bbd8db 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -886,17 +886,17 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int DoubleStack, AbstractNode>[] oldTodoLists = todoLists; todoLists = new DoubleStack[negativeOffset + Math.max(queueDepth, length) + 1]; - System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset, queueDepth - queueIndex); - System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset, queueIndex); + System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset + 1, queueDepth - queueIndex); + System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset + 1, queueIndex); // reset the parser! queueIndex = 0; location = startPosition; - DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; + DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; if (terminalsTodo == null){ terminalsTodo = new DoubleStack, AbstractNode>(); - todoLists[queueIndex] = terminalsTodo; + todoLists[queueIndex + 1] = terminalsTodo; } else { assert false: "this should never happen"; From d9f8fc971dfdc9b01324a46a0a5e8543b8efadeb Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 15 Apr 2022 11:12:30 +0200 Subject: [PATCH 17/87] fixed another off-by-one --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 1ac29bbd8db..08a60529fd4 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -891,7 +891,7 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int // reset the parser! queueIndex = 0; - location = startPosition; + location = startPosition + 1; DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; if (terminalsTodo == null){ From 685198dadd38c6b4114f42877782fc70413da47c Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 15 Apr 2022 11:28:08 +0200 Subject: [PATCH 18/87] added initial construction of skipped nodes --- src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java | 4 +++- src/org/rascalmpl/values/RascalValueFactory.java | 1 + src/org/rascalmpl/values/parsetrees/ProductionAdapter.java | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index 8a6199fa2a0..3bb39dbda0d 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -1,6 +1,7 @@ package org.rascalmpl.parser.uptr; import java.net.URI; +import java.util.Arrays; import java.util.IdentityHashMap; import java.util.Map; @@ -142,6 +143,7 @@ public Object getProductionFromNode(ITree node){ @Override public ITree createRecoveryNode(int[] characters) { - throw new UnsupportedOperationException(); + IList chars = Arrays.stream(characters).mapToObj(ch -> VF.character(ch)).collect(VF.listWriter()); + return VF.appl(VF.constructor(RascalValueFactory.Production_Skipped), chars); } } diff --git a/src/org/rascalmpl/values/RascalValueFactory.java b/src/org/rascalmpl/values/RascalValueFactory.java index c7bca997e1b..34f57cf0b29 100644 --- a/src/org/rascalmpl/values/RascalValueFactory.java +++ b/src/org/rascalmpl/values/RascalValueFactory.java @@ -152,6 +152,7 @@ public class RascalValueFactory extends AbstractValueFactoryAdapter implements I public static final Type Production_Priority = tf.constructor(uptr, Production, "priority", Symbol, "def", tf.listType(Production), "choices"); public static final Type Production_Composition = tf.constructor(uptr, Production, "composition", Production, "lhs", Production, "rhs"); public static final Type Production_Associativity = tf.constructor(uptr, Production, "associativity", Symbol, "def", Associativity, "assoc", tf.setType(Production), "alternatives"); + public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped"); /* Constructors for Attr */ public static final Type Attr_Assoc = tf.constructor(uptr, Attr, "assoc", Associativity, "assoc"); diff --git a/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java b/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java index 591d781a674..be88043ebb0 100644 --- a/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/ProductionAdapter.java @@ -138,6 +138,10 @@ public static boolean isDefault(IConstructor tree) { public static boolean isRegular(IConstructor tree) { return tree.getConstructorType() == RascalValueFactory.Production_Regular; } + + public static boolean isSkipped(IConstructor tree) { + return tree.getConstructorType() == RascalValueFactory.Production_Skipped; + } public static boolean isSeparatedList(IConstructor tree) { IConstructor rhs = getType(tree); From 1d5435bac11fba6eb32451213cbe1b73fb99cc97 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 15 Apr 2022 11:37:35 +0200 Subject: [PATCH 19/87] gave skipped productions a type such that all trees have a type --- src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java | 3 ++- src/org/rascalmpl/values/RascalValueFactory.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index 3bb39dbda0d..5988650baaa 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -23,6 +23,7 @@ public class UPTRNodeFactory implements INodeConstructorFactory{ private final static RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); + private final static IConstructor SKIPPED = VF.constructor(RascalValueFactory.Production_Skipped, VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Range, VF.integer(1), VF.integer(Character.MAX_CODE_POINT))))); private boolean allowAmb; public UPTRNodeFactory(boolean allowAmbiguity){ @@ -144,6 +145,6 @@ public Object getProductionFromNode(ITree node){ @Override public ITree createRecoveryNode(int[] characters) { IList chars = Arrays.stream(characters).mapToObj(ch -> VF.character(ch)).collect(VF.listWriter()); - return VF.appl(VF.constructor(RascalValueFactory.Production_Skipped), chars); + return VF.appl(SKIPPED, chars); } } diff --git a/src/org/rascalmpl/values/RascalValueFactory.java b/src/org/rascalmpl/values/RascalValueFactory.java index 34f57cf0b29..832eebc7800 100644 --- a/src/org/rascalmpl/values/RascalValueFactory.java +++ b/src/org/rascalmpl/values/RascalValueFactory.java @@ -152,7 +152,7 @@ public class RascalValueFactory extends AbstractValueFactoryAdapter implements I public static final Type Production_Priority = tf.constructor(uptr, Production, "priority", Symbol, "def", tf.listType(Production), "choices"); public static final Type Production_Composition = tf.constructor(uptr, Production, "composition", Production, "lhs", Production, "rhs"); public static final Type Production_Associativity = tf.constructor(uptr, Production, "associativity", Symbol, "def", Associativity, "assoc", tf.setType(Production), "alternatives"); - public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped"); + public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped", Symbol, "def"); /* Constructors for Attr */ public static final Type Attr_Assoc = tf.constructor(uptr, Attr, "assoc", Associativity, "assoc"); From 5265ef09cbacb8426a6cb96827e5bf1803617c47 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 20 Apr 2022 11:56:30 +0200 Subject: [PATCH 20/87] updated template --- .../ISSUE_TEMPLATE/stable-release-manual-testing-template.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/stable-release-manual-testing-template.md b/.github/ISSUE_TEMPLATE/stable-release-manual-testing-template.md index 645e4465a8e..2bd1728f945 100644 --- a/.github/ISSUE_TEMPLATE/stable-release-manual-testing-template.md +++ b/.github/ISSUE_TEMPLATE/stable-release-manual-testing-template.md @@ -34,7 +34,7 @@ First a "pre-release" of the supporting compiler/typechecker tools must be done, # Manual feature tests -- [ ] Eclipse download and install latest unstable release from update site https://update.rascal-mpl.org/unstable +- [ ] Eclipse download and install latest unstable release from update site https://releases.usethesource.io/maven/org/rascalmpl/rascal-update-site/ - [ ] Open a Rascal REPL using the toolbar button - [ ] Can create new Rascal project using the wizard - [ ] Can create new Rascal module using the wizard From a2162c9316a437fc6d53cc543779ae74cf33b39e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 20 Apr 2022 15:38:00 +0200 Subject: [PATCH 21/87] bumped rascal-maven-plugin to 0.8.0 to see if we can benefit from optimizations --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8c16b33c736..a03c136c879 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.rascalmpl rascal-maven-plugin - 0.7.4 + 0.8.0 true ${project.build.outputDirectory} From 91fb32bb77df44314ba693713f9739886e67f95a Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 21 Apr 2022 17:23:53 +0200 Subject: [PATCH 22/87] upgraded to rascal-maven-plugin 0.8.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a03c136c879..12667de5e3a 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.rascalmpl rascal-maven-plugin - 0.8.0 + 0.8.1 true ${project.build.outputDirectory} From 018826fc305dde4f9a079a1772d45b569ea9d205 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 21 Apr 2022 18:31:28 +0200 Subject: [PATCH 23/87] [maven-release-plugin] prepare release v0.23.1 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 12667de5e3a..9d2872e64b8 100644 --- a/pom.xml +++ b/pom.xml @@ -3,12 +3,12 @@ org.rascalmpl rascal - 0.23.1-SNAPSHOT + 0.23.1 jar scm:git:ssh://git@github.com/usethesource/rascal.git - HEAD + v0.23.1 From 7b6e8b82a6dc89d456385ff561c49087691cf084 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 21 Apr 2022 18:31:33 +0200 Subject: [PATCH 24/87] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 9d2872e64b8..03de13d9e2f 100644 --- a/pom.xml +++ b/pom.xml @@ -3,12 +3,12 @@ org.rascalmpl rascal - 0.23.1 + 0.23.2-SNAPSHOT jar scm:git:ssh://git@github.com/usethesource/rascal.git - v0.23.1 + HEAD From 4aefd529d5d21dc2d7c0ff21c5b3469612a241ed Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 22 Apr 2022 15:56:45 +0200 Subject: [PATCH 25/87] make sure that (a) BasicIDEServices are registered for the commandline version of Rascal and (b) the edit command is wired to the edit IDEService --- .../rascal/tutor/TutorCommandExecutor.java | 7 +++++-- src/org/rascalmpl/library/util/TermREPL.java | 5 ++++- src/org/rascalmpl/repl/BaseREPL.java | 6 +++--- src/org/rascalmpl/repl/ILanguageProtocol.java | 4 +++- .../rascalmpl/repl/RascalInterpreterREPL.java | 7 ++++--- .../semantics/dynamic/ShellCommand.java | 19 +++++++++++++++++++ src/org/rascalmpl/shell/REPLRunner.java | 13 +++++++++---- 7 files changed, 47 insertions(+), 14 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tutor/TutorCommandExecutor.java b/src/org/rascalmpl/library/lang/rascal/tutor/TutorCommandExecutor.java index 533245d5f2c..4261a9a8de3 100644 --- a/src/org/rascalmpl/library/lang/rascal/tutor/TutorCommandExecutor.java +++ b/src/org/rascalmpl/library/lang/rascal/tutor/TutorCommandExecutor.java @@ -16,6 +16,8 @@ import java.util.Map; import java.util.UUID; +import org.rascalmpl.ideservices.BasicIDEServices; +import org.rascalmpl.ideservices.IDEServices; import org.rascalmpl.interpreter.Evaluator; import org.rascalmpl.library.Prelude; import org.rascalmpl.library.util.PathConfig; @@ -41,14 +43,15 @@ public TutorCommandExecutor(PathConfig pcfg) throws IOException, URISyntaxExcept repl = new RascalInterpreterREPL(false, false, null) { @Override - protected Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr) { + protected Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices services) { Evaluator eval = ShellEvaluatorFactory.getDefaultEvaluator(input, stdout, stderr); eval.getConfiguration().setRascalJavaClassPathProperty(javaCompilerPathAsString(pcfg.getJavaCompilerPath())); + eval.setMonitor(services); return eval; } }; - repl.initialize(shellInputNotUsed, shellStandardOutput, shellErrorOutput); + repl.initialize(shellInputNotUsed, shellStandardOutput, shellErrorOutput, null); repl.setMeasureCommandTime(false); } diff --git a/src/org/rascalmpl/library/util/TermREPL.java b/src/org/rascalmpl/library/util/TermREPL.java index fa9f98907d8..75becc5460d 100644 --- a/src/org/rascalmpl/library/util/TermREPL.java +++ b/src/org/rascalmpl/library/util/TermREPL.java @@ -14,6 +14,7 @@ import java.util.function.Function; import org.rascalmpl.exceptions.RuntimeExceptionFactory; +import org.rascalmpl.ideservices.IDEServices; import org.rascalmpl.interpreter.Evaluator; import org.rascalmpl.interpreter.IEvaluatorContext; import org.rascalmpl.interpreter.result.AbstractFunction; @@ -102,6 +103,7 @@ public static class TheREPL implements ILanguageProtocol { private final AbstractFunction completor; private final IValueFactory vf; private final AbstractFunction stacktrace; + private IDEServices services; public TheREPL(IValueFactory vf, IString title, IString welcome, IString prompt, IString quit, ISourceLocation history, IFunction handler, IFunction completor, IValue stacktrace, InputStream input, OutputStream stderr, OutputStream stdout) { @@ -144,10 +146,11 @@ public void stackTraceRequested() { } @Override - public void initialize(InputStream input, OutputStream stdout, OutputStream stderr) { + public void initialize(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices services) { this.stdout = stdout; this.stderr = stderr; this.input = input; + this.services = services; } @Override diff --git a/src/org/rascalmpl/repl/BaseREPL.java b/src/org/rascalmpl/repl/BaseREPL.java index 459df65964f..74217675357 100755 --- a/src/org/rascalmpl/repl/BaseREPL.java +++ b/src/org/rascalmpl/repl/BaseREPL.java @@ -110,7 +110,7 @@ else if (prettyPrompt) { else { this.errorWriter = new FilterWriter(reader.getOutput()) { }; // create a basic wrapper to avoid locking on stdout and stderr } - initialize(stdin, terminal.wrapOutIfNeeded(stdout) /*JURGEN LET OP reader.getOutput()*/, terminal.wrapOutIfNeeded(stderr)); + initialize(stdin, terminal.wrapOutIfNeeded(stdout) /*JURGEN LET OP reader.getOutput()*/, terminal.wrapOutIfNeeded(stderr), ideServices); if (supportsCompletion()) { reader.addCompleter(new Completer(){ @Override @@ -155,8 +155,8 @@ public static char ctrl(char ch) { * @throws IOException * @throws URISyntaxException */ - protected void initialize(InputStream input, OutputStream stdout, OutputStream stderr) throws IOException, URISyntaxException { - language.initialize(input, stdout, stderr); + protected void initialize(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices services) throws IOException, URISyntaxException { + language.initialize(input, stdout, stderr, services); } /** diff --git a/src/org/rascalmpl/repl/ILanguageProtocol.java b/src/org/rascalmpl/repl/ILanguageProtocol.java index 627ef161199..9a32ca88762 100755 --- a/src/org/rascalmpl/repl/ILanguageProtocol.java +++ b/src/org/rascalmpl/repl/ILanguageProtocol.java @@ -27,6 +27,8 @@ import java.io.OutputStream; import java.util.Map; +import org.rascalmpl.ideservices.IDEServices; + public interface ILanguageProtocol { @@ -35,7 +37,7 @@ public interface ILanguageProtocol { * @param stdout the output stream to write normal output to. * @param stderr the error stream to write error messages on, depending on the environment and options passed, will print in red. */ - void initialize(InputStream input, OutputStream stdout, OutputStream stderr); + void initialize(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices services); /** * Will be called everytime a new prompt is printed. diff --git a/src/org/rascalmpl/repl/RascalInterpreterREPL.java b/src/org/rascalmpl/repl/RascalInterpreterREPL.java index 5b9b6a0a9bf..5333e75a9b6 100644 --- a/src/org/rascalmpl/repl/RascalInterpreterREPL.java +++ b/src/org/rascalmpl/repl/RascalInterpreterREPL.java @@ -21,6 +21,7 @@ import org.rascalmpl.exceptions.StackTrace; import org.rascalmpl.exceptions.Throw; +import org.rascalmpl.ideservices.IDEServices; import org.rascalmpl.interpreter.Configuration; import org.rascalmpl.interpreter.Evaluator; import org.rascalmpl.interpreter.control_exceptions.InterruptException; @@ -70,11 +71,11 @@ public boolean getMeasureCommandTime() { } @Override - public void initialize(InputStream input, OutputStream stdout, OutputStream stderr) { - eval = constructEvaluator(input, stdout, stderr); + public void initialize(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices ideServices) { + eval = constructEvaluator(input, stdout, stderr, ideServices); } - protected abstract Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr); + protected abstract Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices ideServices); @Override public PrintWriter getErrorWriter() { diff --git a/src/org/rascalmpl/semantics/dynamic/ShellCommand.java b/src/org/rascalmpl/semantics/dynamic/ShellCommand.java index 56f1e03a75d..d1e9ed311e1 100644 --- a/src/org/rascalmpl/semantics/dynamic/ShellCommand.java +++ b/src/org/rascalmpl/semantics/dynamic/ShellCommand.java @@ -15,6 +15,8 @@ import org.rascalmpl.ast.Expression; import org.rascalmpl.ast.QualifiedName; +import org.rascalmpl.debug.IRascalMonitor; +import org.rascalmpl.ideservices.IDEServices; import org.rascalmpl.interpreter.Configuration; import org.rascalmpl.interpreter.IEvaluator; import org.rascalmpl.interpreter.control_exceptions.QuitException; @@ -34,6 +36,23 @@ public Edit(ISourceLocation __param1, IConstructor tree, QualifiedName __param2) @Override public Result interpret(IEvaluator> __eval) { + IRascalMonitor monitor = __eval.getMonitor(); + + if (monitor instanceof IDEServices) { + IDEServices services = (IDEServices) monitor; + String name = Names.fullName(getName()); + + ISourceLocation uri = __eval.getRascalResolver().resolveModule(name); + if (uri == null) { + __eval.getOutPrinter().println("module " + name + " can not be found in the search path."); + } + + services.edit(uri); + } + else { + __eval.getOutPrinter().println("The current Rascal execution environment does not know how to start an editor."); + } + return org.rascalmpl.interpreter.result.ResultFactory.nothing(); } } diff --git a/src/org/rascalmpl/shell/REPLRunner.java b/src/org/rascalmpl/shell/REPLRunner.java index 708e7259978..0e82f29a4ce 100755 --- a/src/org/rascalmpl/shell/REPLRunner.java +++ b/src/org/rascalmpl/shell/REPLRunner.java @@ -5,9 +5,12 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.PrintWriter; import java.net.URISyntaxException; import java.util.Map; +import org.rascalmpl.ideservices.BasicIDEServices; +import org.rascalmpl.ideservices.IDEServices; import org.rascalmpl.interpreter.Evaluator; import org.rascalmpl.repl.BaseREPL; import org.rascalmpl.repl.ILanguageProtocol; @@ -33,11 +36,11 @@ private static File getHistoryFile() throws IOException { public REPLRunner(InputStream stdin, OutputStream stderr, OutputStream stdout, Terminal term) throws IOException, URISyntaxException { super(makeInterpreter(stdin, stderr, stdout, true, term.isAnsiSupported(), getHistoryFile(), term), null, - stdin, stderr, stdout, true, term.isAnsiSupported(), getHistoryFile(), term, null); + stdin, stderr, stdout, true, term.isAnsiSupported(), getHistoryFile(), term, new BasicIDEServices(new PrintWriter(stderr))); } public REPLRunner(ILanguageProtocol language) throws IOException, URISyntaxException { - super(language, null, null, null, null, true, true, new File(""), null, null); + super(language, null, null, null, null, true, true, new File(""), null, new BasicIDEServices(new PrintWriter(System.err))); } private static ILanguageProtocol makeInterpreter(InputStream stdin, OutputStream stderr, OutputStream stdout, @@ -46,8 +49,10 @@ private static ILanguageProtocol makeInterpreter(InputStream stdin, OutputStream RascalInterpreterREPL repl = new RascalInterpreterREPL(prettyPrompt, allowColors, getHistoryFile()) { @Override - protected Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr) { - return ShellEvaluatorFactory.getDefaultEvaluator(input, stdout, stderr); + protected Evaluator constructEvaluator(InputStream input, OutputStream stdout, OutputStream stderr, IDEServices services) { + Evaluator eval = ShellEvaluatorFactory.getDefaultEvaluator(input, stdout, stderr); + eval.setMonitor(services); + return eval; } @Override From dde9bd95da711fd53a620b2e0e5dcf1eed7cf98f Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 25 Apr 2022 09:46:50 +0200 Subject: [PATCH 26/87] basic IDE services can now also browse contents of files which are not in the scheme by copyinhthe contents to a tmp file --- .../ideservices/BasicIDEServices.java | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/org/rascalmpl/ideservices/BasicIDEServices.java b/src/org/rascalmpl/ideservices/BasicIDEServices.java index 8334de5bda2..8215b0b0a72 100644 --- a/src/org/rascalmpl/ideservices/BasicIDEServices.java +++ b/src/org/rascalmpl/ideservices/BasicIDEServices.java @@ -21,6 +21,8 @@ import java.nio.file.Paths; import org.rascalmpl.interpreter.ConsoleRascalMonitor; +import org.rascalmpl.uri.URIResolverRegistry; +import org.rascalmpl.uri.URIUtil; import io.usethesource.vallang.ISourceLocation; @@ -65,12 +67,21 @@ public void browse(URI uri){ } @Override - public void edit(ISourceLocation loc){ - if(loc.getScheme() != "file"){ - stderr.println("Can only edit files using the \"file\" scheme"); - return; + public void edit(ISourceLocation loc) { + try { + if (loc.getScheme() != "file") { + ISourceLocation tmp = URIUtil.correctLocation("tmp", "", "rascal-edits"); + tmp = URIUtil.getChildLocation(tmp, loc.getScheme()); + tmp = URIUtil.getChildLocation(tmp, loc.getPath()); + URIResolverRegistry.getInstance().copy(loc, tmp, false, true); + loc = URIResolverRegistry.getInstance().logicalToPhysical(tmp); } + edit(Paths.get(loc.getURI())); + } + catch (IOException e) { + stderr.println("Can not edit " + loc + " because: " + e); + } } /* (non-Javadoc) @@ -86,7 +97,7 @@ public void edit(Path path) { stderr.println(e.getMessage()); } } else { - stderr.println("Desktop not supported, cannout open editor"); + stderr.println("Desktop not supported, cannot open editor"); } } From 517df9cfb368ca58ec78f49f08acc14cd570f09b Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 10 Jun 2024 11:45:32 +0200 Subject: [PATCH 27/87] Started working on a simple recovery test --- src/org/rascalmpl/library/ParseTree.rsc | 1 + .../parser/gtd/util/IdDispenser.java | 5 + .../recovery/ToNextWhitespaceRecoverer.java | 14 +- .../rascalmpl/test/parser/IParserTest.java | 20 +++ .../rascalmpl/test/parser/RecoveryTests.java | 137 ++++++++++++++++++ 5 files changed, 171 insertions(+), 6 deletions(-) create mode 100644 src/org/rascalmpl/parser/gtd/util/IdDispenser.java create mode 100644 test/org/rascalmpl/test/parser/RecoveryTests.java diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 3289f88fd2c..2813f58b661 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -189,6 +189,7 @@ data Production | \reference(Symbol def, str cons) // <5> ; +data Production = skipped(Symbol def, Production prod, int validPrefix); @synopsis{Attributes in productions.} @description{ diff --git a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java new file mode 100644 index 00000000000..3952d7fa3c0 --- /dev/null +++ b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java @@ -0,0 +1,5 @@ +package org.rascalmpl.parser.gtd.util; + +public interface IdDispenser { + int dispenseId(); +} diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index bd94e061868..70730462e27 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -21,6 +21,7 @@ import org.rascalmpl.parser.gtd.util.ArrayList; import org.rascalmpl.parser.gtd.util.DoubleArrayList; import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.IdDispenser; import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; @@ -28,12 +29,10 @@ import io.usethesource.vallang.IConstructor; -public class ToNextWhitespaceRecoverer implements IRecoverer{ - // TODO: its a magic constant, and it may clash with other generated constants - // should generate implementation of static int getLastId() in generated parser to fix this. - private int recoveryId = 100000; +public class ToNextWhitespaceRecoverer implements IRecoverer { private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; + private IdDispenser stackNodeIdDispenser; private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); @@ -46,11 +45,11 @@ private DoubleArrayList, AbstractNode> reviveNod for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); - AbstractStackNode continuer = new RecoveryPointStackNode(recoveryId++, prod, recoveryNode); + AbstractStackNode continuer = new RecoveryPointStackNode(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); int startLocation = recoveryNode.getStartLocation(); - AbstractStackNode recoverLiteral = new SkippingStackNode(recoveryId++, WHITESPACE, input, startLocation, prod); + AbstractStackNode recoverLiteral = new SkippingStackNode(stackNodeIdDispenser.dispenseId(), WHITESPACE, input, startLocation, prod); recoverLiteral = recoverLiteral.getCleanCopy(startLocation); recoverLiteral.initEdges(); EdgesSet edges = new EdgesSet(1); @@ -177,6 +176,9 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } + public ToNextWhitespaceRecoverer(IdDispenser stackNodeIdDispenser) { + this.stackNodeIdDispenser = stackNodeIdDispenser; + } @Override public DoubleArrayList, AbstractNode> reviveStacks(int[] input, diff --git a/test/org/rascalmpl/test/parser/IParserTest.java b/test/org/rascalmpl/test/parser/IParserTest.java index 9cf98df7cb2..a9598a8239e 100644 --- a/test/org/rascalmpl/test/parser/IParserTest.java +++ b/test/org/rascalmpl/test/parser/IParserTest.java @@ -13,13 +13,33 @@ import java.io.IOException; +import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IValue; import io.usethesource.vallang.IValueFactory; + +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.values.ValueFactoryFactory; import org.rascalmpl.values.parsetrees.ITree; public interface IParserTest{ public final static IValueFactory VF = ValueFactoryFactory.getValueFactory(); + + @SafeVarargs + public static AbstractStackNode[] createExpectArray(IConstructor prod, AbstractStackNode... nodes) { + @SuppressWarnings({"unchecked", "cast"}) + AbstractStackNode[] expectArray = (AbstractStackNode[]) new AbstractStackNode[nodes.length]; + + int index = 0; + for (AbstractStackNode node : nodes) { + expectArray[index] = node; + node.setProduction(expectArray); + index++; + } + + expectArray[index-1].setAlternativeProduction(prod); + + return expectArray; + } ITree executeParser(); diff --git a/test/org/rascalmpl/test/parser/RecoveryTests.java b/test/org/rascalmpl/test/parser/RecoveryTests.java new file mode 100644 index 00000000000..869b74bca6e --- /dev/null +++ b/test/org/rascalmpl/test/parser/RecoveryTests.java @@ -0,0 +1,137 @@ +package org.rascalmpl.test.parser; + +import java.io.IOException; +import java.io.StringReader; + +import org.junit.Assert; +import org.junit.Test; +import org.rascalmpl.parser.gtd.SGTDBF; +import org.rascalmpl.parser.gtd.recovery.IRecoverer; +import org.rascalmpl.parser.gtd.result.out.DefaultNodeFlattener; +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; +import org.rascalmpl.parser.gtd.stack.LiteralStackNode; +import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; +import org.rascalmpl.parser.uptr.UPTRNodeFactory; +import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; +import org.rascalmpl.values.RascalValueFactory; +import org.rascalmpl.values.ValueFactoryFactory; +import org.rascalmpl.values.parsetrees.ITree; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.ISourceLocation; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.io.StandardTextReader; + +/** + * S -> A ws B ws C + * A -> [a] ws [a] + * B -> [b] ws [b] + * C -> [c] ws [c] + * + * ws -> [\ ] + */ +public class RecoveryTests extends SGTDBF implements IParserTest{ + private final static IConstructor SYMBOL_START_S = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("S")); + + private final static IConstructor SYMBOL_ws = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("ws")); + private final static IConstructor SYMBOL_char_space = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(32)))); + + private final static IConstructor SYMBOL_A = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("A")); + private final static IConstructor SYMBOL_a = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("a")); + private final static IConstructor SYMBOL_char_a = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(97)))); + private final static IConstructor PROD_A_a_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_A, VF.list(SYMBOL_a, SYMBOL_ws, SYMBOL_a), VF.set()); + + private final static IConstructor SYMBOL_B = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("B")); + private final static IConstructor SYMBOL_b = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("b")); + private final static IConstructor SYMBOL_char_b = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(98)))); + private final static IConstructor PROD_B_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_B, VF.list(SYMBOL_b, SYMBOL_ws, SYMBOL_b), VF.set()); + + private final static IConstructor SYMBOL_C = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("C")); + private final static IConstructor SYMBOL_c = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("c")); + private final static IConstructor SYMBOL_char_c = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(99)))); + private final static IConstructor PROD_C_c_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_C, VF.list(SYMBOL_c, SYMBOL_ws, SYMBOL_c), VF.set()); + + private final static IConstructor PROD_S_A_B_C = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_START_S, VF.list(SYMBOL_A, SYMBOL_ws, SYMBOL_B, SYMBOL_ws, SYMBOL_C), VF.set()); + + private final static IConstructor PROD_ws = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_ws, VF.list(SYMBOL_char_space), VF.set()); + private final static IConstructor PROD_a_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_a, VF.list(SYMBOL_char_a), VF.set()); + private final static IConstructor PROD_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_b, VF.list(SYMBOL_char_b), VF.set()); + private final static IConstructor PROD_c_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_c, VF.list(SYMBOL_char_c), VF.set()); + + public AbstractStackNode[] S(){ + return IParserTest.createExpectArray(PROD_S_A_B_C, + new NonTerminalStackNode(1, 0, "A"), + new LiteralStackNode(2, 1, PROD_ws, new int[]{' '}), + new NonTerminalStackNode(3, 2, "B"), + new LiteralStackNode(4, 3, PROD_ws, new int[]{' '}), + new NonTerminalStackNode(5, 4, "C") + ); + } + + public AbstractStackNode[] A(){ + return IParserTest.createExpectArray(PROD_A_a_a, + new LiteralStackNode(6, 0, PROD_a_a, new int[]{'a'}), + new LiteralStackNode(7, 1, PROD_ws, new int[]{' '}), + new LiteralStackNode(8, 2, PROD_a_a, new int[]{'a'}) + ); + } + + public AbstractStackNode[] B(){ + return IParserTest.createExpectArray(PROD_B_b_b, + new LiteralStackNode(9, 0, PROD_b_b, new int[]{'b'}), + new LiteralStackNode(10, 1, PROD_ws, new int[]{' '}), + new LiteralStackNode(11, 2, PROD_b_b, new int[]{'b'}) + ); + } + + public AbstractStackNode[] C(){ + return IParserTest.createExpectArray(PROD_C_c_c, + new LiteralStackNode(12, 0, PROD_c_c, new int[]{'c'}), + new LiteralStackNode(13, 1, PROD_ws, new int[]{' '}), + new LiteralStackNode(14, 2, PROD_c_c, new int[]{'c'}) + ); + } + + private int nextFreeStackNodeId = 100; + + @Override + protected int getFreeStackNodeId() { + return nextFreeStackNodeId++; + } + + private ITree parse(String s) { + IRecoverer recoverer = new ToNextWhitespaceRecoverer(() -> nextFreeStackNodeId++); + return parse("S" /* NONTERMINAL_START_S */, null, s.toCharArray(), + new DefaultNodeFlattener(), new UPTRNodeFactory(false), recoverer, null); + } + + private ITree toTree(String s) { + try { + return (ITree) new StandardTextReader().read(ValueFactoryFactory.getValueFactory(), RascalValueFactory.uptr, RascalValueFactory.Tree, new StringReader(s)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ITree executeParser() { + return parse("a a b b c c"); + } + + @Override + public IValue getExpectedResult() { + return toTree("appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"); + } + + @Test + public void testOk() { + Assert.assertEquals(getExpectedResult(), executeParser()); + } + + @Test + public void testMissingCharRecovery() { + String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; + Assert.assertEquals(toTree(expected), parse("a a b c c")); + } + +} From c779820cb9332ed3b56623056c796829cf96c098 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 17 Jun 2024 09:33:56 +0200 Subject: [PATCH 28/87] Added 'toString' methods and fixed issue in recoverer. - Added a lot of toString methods to help understading when debugging as VSCode uses toString to display the value of objects - Fixed iteration order in recoverer. I am not sure if the incoming nodes are always ordered by location, if not we might need to sort them in the future. --- .vscode/launch.json | 9 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 57 ++++++- .../parser/gtd/debug/IDebugListener.java | 13 ++ .../gtd/result/AbstractContainerNode.java | 36 +++++ .../rascalmpl/parser/gtd/result/CharNode.java | 6 + .../parser/gtd/result/EpsilonNode.java | 5 + .../gtd/result/ExpandableContainerNode.java | 5 + .../parser/gtd/result/LiteralNode.java | 11 ++ .../parser/gtd/result/RecoveredNode.java | 5 + .../parser/gtd/result/SkippedNode.java | 9 ++ .../parser/gtd/result/SortContainerNode.java | 5 + .../result/error/ErrorListContainerNode.java | 5 + .../result/error/ErrorSortContainerNode.java | 6 + .../parser/gtd/result/error/ExpectedNode.java | 8 + .../parser/gtd/result/struct/Link.java | 4 + .../gtd/stack/AbstractMatchableStackNode.java | 2 +- .../parser/gtd/stack/AbstractStackNode.java | 73 ++++++++- .../parser/gtd/stack/LiteralStackNode.java | 15 +- .../gtd/stack/NonTerminalStackNode.java | 9 +- .../parser/gtd/stack/SkippingStackNode.java | 22 ++- .../parser/gtd/stack/edge/EdgesSet.java | 33 +++- .../rascalmpl/parser/gtd/util/ArrayList.java | 13 ++ .../parser/gtd/util/DoubleArrayList.java | 16 ++ .../parser/gtd/util/DoubleStack.java | 15 ++ .../parser/gtd/util/IntegerObjectList.java | 16 ++ .../parser/uptr/debug/DebugLogger.java | 56 +++++++ .../recovery/ToNextWhitespaceRecoverer.java | 51 +++--- src/org/rascalmpl/parser/util/DebugUtil.java | 39 +++++ .../rascalmpl/unicode/UnicodeConverter.java | 12 ++ .../rascalmpl/test/parser/IParserTest.java | 2 +- .../rascalmpl/test/parser/RecoveryTests2.java | 148 ++++++++++++++++++ 31 files changed, 655 insertions(+), 51 deletions(-) create mode 100644 src/org/rascalmpl/parser/util/DebugUtil.java create mode 100644 src/org/rascalmpl/unicode/UnicodeConverter.java create mode 100644 test/org/rascalmpl/test/parser/RecoveryTests2.java diff --git a/.vscode/launch.json b/.vscode/launch.json index aa506d012ad..8d7a2f6a52c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,6 +4,13 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ + { + "type": "java", + "name": "Simple2", + "request": "launch", + "mainClass": "org.rascalmpl.test.parser.Simple2", + "projectName": "rascal" + }, { "type": "java", "name": "Launch DocRunner", @@ -38,7 +45,7 @@ "request": "launch", "mainClass": "org.rascalmpl.shell.RascalShell", "projectName": "rascal", - "cwd" : "${workspaceFolder}/../rascal-tutor", + "cwd": "${workspaceFolder}/../rascal-tutor", "vmArgs": "-Xss80m -Xmx2g -ea" }, { diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 771f3e6157c..458568f5643 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -54,24 +54,53 @@ public abstract class SGTDBF implements IGTD{ private URI inputURI; private int[] input; - + private int location; + protected int lookAheadChar; + + // A mapping between character location and line/column. private final PositionStore positionStore; + // Terminals that matched. Circular buffer indexed by length of the terminal. Each entry contains the node to reduce and the result node + // This is a circular buffer where queueIndex determines the start of the buffer. + // At each position, a stack is maintained of all terminals to reduce of a certain length. + // So at queueIndex+3, all terminals of length 3 that need reducing are stored. private DoubleStack, AbstractNode>[] todoLists; private int queueIndex; + // Stack of non-terminal nodes to expand + // - Nodes are removed in expand, which pops and expands all stack nodes on this stack + // - Nodes are added in: + // - parse: the root node is pushed + // - updateNextNode: next node of the production is pushed + // - updateAlternativeNextNode: next node of a prefix-shared production is pushed + // - handleExpects: non-matchable first element of each alternative is pushed + // - expandStack: when an expandable stack is expanded, all non-matchable children are pushed private final Stack> stacksToExpand; + + // The current stack of non-terminals to reduce. Each stack has a container node to accumulate results. + // - Nodes are removed in `reduce` where all productions are advanced one dot past the non-terminal node + // - Nodes are added in: + // - handleEdgeList: result container node is created and all edges are pushed with the same result container node + // - handleEdgeListWithRestrictions: result container node is created and all edges are pushed with the same result container node + // - expandStack: non-matchable, non-expandable nodes (and their existing result container node) are added if their name can be found in `cachedEdgesForExpect`. + // - expandStack: expandable nodes that are nullable? Might be a cycle thing private final DoubleStack, AbstractContainerNode

> stacksWithNonTerminalsToReduce; + + // The current stack of non-terminals to reduce: it contains the matchable node with the smallest length from todoLists. + // - Nodes are removed in `reduce` where all productions are advanced one dot past the matchable node + // - Variable is assigned in: + // - findFirstStacksToReduce: the first non-empty `todoList` is assigned to this variable + // - findStacksToReduce: again the first non-empty `todoList` is assigned to this variable + // - parse: variable is used in main reduce/expand loop to determine when it is time to look for more `stacksToReduce`. private DoubleStack, AbstractNode> stacksWithTerminalsToReduce; private final HashMap> cachedEdgesForExpect; private final IntegerKeyedDoubleValueHashMap, DoubleArrayList, AbstractNode>> sharedNextNodes; - private int location; - - protected int lookAheadChar; + // Reflection is used to get the expects for each non-terminal. + // This cache is used so the reflection call is only needed once. private final HashMap[]> expectCache; private final IntegerObjectList> sharedLastExpects; @@ -81,7 +110,7 @@ public abstract class SGTDBF implements IGTD{ // Error reporting private final Stack> unexpandableNodes; - private final Stack> unmatchableLeafNodes; + private final Stack> unmatchableLeafNodes; // Leaf nodes (for instance literals) that failed to match private final DoubleStack, AbstractNode>, AbstractStackNode

> unmatchableMidProductionNodes; private final DoubleStack, AbstractNode> filteredNodes; @@ -791,7 +820,13 @@ private boolean findFirstStacksToReduce(){ } if (recoverer != null) { + if (debugListener != null) { + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + } DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + if (debugListener != null) { + debugListener.revived(recoveredNodes); + } if (recoveredNodes.size() > 0) { // TODO Do something with the revived node. Is this the right location to do this? for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); @@ -825,10 +860,16 @@ private boolean findStacksToReduce(){ } if (recoverer != null && location < input.length) { + if (debugListener != null) { + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + } DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + debugListener.revived(recoveredNodes); if (recoveredNodes.size() > 0) { + + // for (int i = recoveredNodes.size()-1; i>= 0; i--) { for (int i = 0; i < recoveredNodes.size(); i++) { - AbstractStackNode

recovered = recoveredNodes.getFirst(i); + AbstractStackNode

recovered = recoveredNodes.getFirst(i); // int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation()); @@ -900,12 +941,12 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int // reset the parser! queueIndex = 0; - location = startPosition + 1; + location = startPosition + 1; DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; if (terminalsTodo == null){ terminalsTodo = new DoubleStack, AbstractNode>(); - todoLists[queueIndex + 1] = terminalsTodo; + todoLists[queueIndex + 1] = terminalsTodo; // Why the +1 and not length? To get the recovered node to be processed first? } else { assert false: "this should never happen"; diff --git a/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java b/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java index ae412bde870..fc6a8a2bec3 100644 --- a/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java +++ b/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java @@ -16,6 +16,9 @@ import org.rascalmpl.parser.gtd.result.struct.Link; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; +import org.rascalmpl.parser.gtd.util.DoubleArrayList; +import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.Stack; public interface IDebugListener

{ void shifting(int offset, int[] input, PositionStore positionStore); @@ -47,4 +50,14 @@ public interface IDebugListener

{ void filteredByEnterFilter(AbstractStackNode

node); void filteredByCompletionFilter(AbstractStackNode

node, AbstractNode result); + + void reviving(int[] input, + int location, + Stack> unexpandableNodes, + Stack> unmatchableLeafNodes, + DoubleStack, AbstractNode>, + AbstractStackNode

> unmatchableMidProductionNodes, + DoubleStack, AbstractNode> filteredNodes); + + void revived(DoubleArrayList, AbstractNode> recoveredNodes); } diff --git a/src/org/rascalmpl/parser/gtd/result/AbstractContainerNode.java b/src/org/rascalmpl/parser/gtd/result/AbstractContainerNode.java index 1a9d8373f59..f57f543ec94 100644 --- a/src/org/rascalmpl/parser/gtd/result/AbstractContainerNode.java +++ b/src/org/rascalmpl/parser/gtd/result/AbstractContainerNode.java @@ -15,6 +15,9 @@ import org.rascalmpl.parser.gtd.result.struct.Link; import org.rascalmpl.parser.gtd.util.ArrayList; +import org.rascalmpl.parser.util.DebugUtil; + +import io.usethesource.vallang.IConstructor; /** * All nodes in the resulting tree that can contain other nodes are a subtype @@ -147,4 +150,37 @@ public ArrayList

getAdditionalProductions(){ public ArrayList getAdditionalAlternatives(){ return alternatives; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("offset=" + offset); + builder.append(",endOffset=" + endOffset); + if (isNullable) { + builder.append(",nullable"); + } + if (isSeparator) { + builder.append(",separator"); + } + if (isLayout) { + builder.append(",layout"); + } + if (firstAlternative != null) { + builder.append(",alternatives=["); + builder.append(firstAlternative); + builder.append(":"); + builder.append(DebugUtil.prodToString((IConstructor) firstProduction)); + + if (alternatives != null) { + for (int i=0; i getPrefixes(){ public AbstractNode getNode(){ return node; } + + public String toString() { + return "Link[node=" + node + ", prefixes=" + (prefixes == null ? 0 : prefixes.size()) + "]"; + } } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java index 41951569714..3afee7e1894 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java @@ -65,5 +65,5 @@ public AbstractStackNode

getEmptyChild(){ public final boolean isMatchable(){ return true; } - + } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index 54e7fb5d70d..f0df8513e42 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -11,6 +11,8 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.stack; +import java.util.Arrays; + import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.result.struct.Link; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; @@ -20,6 +22,9 @@ import org.rascalmpl.parser.gtd.util.BitSet; import org.rascalmpl.parser.gtd.util.IntegerList; import org.rascalmpl.parser.gtd.util.IntegerObjectList; +import org.rascalmpl.parser.util.DebugUtil; + +import io.usethesource.vallang.IConstructor; @SuppressWarnings({"unchecked", "cast"}) public abstract class AbstractStackNode

{ @@ -28,8 +33,8 @@ public abstract class AbstractStackNode

{ protected AbstractStackNode

[] production; protected AbstractStackNode

[][] alternateProductions; - - protected IntegerObjectList> edgesMap; + + protected IntegerObjectList> edgesMap; // : key=startLocation, value=EdgesSet a that location protected ArrayList[] prefixesMap; protected EdgesSet

incomingEdges; @@ -732,7 +737,69 @@ public IntegerList getPropagatedReductions(){ return propagatedReductions; } - + + public String toShortString() { + return "id=" + id + ",dot=" + dot + ",start=" + startLocation; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append(id); + builder.append('.'); + builder.append(dot); + builder.append('@'); + builder.append(startLocation); + if (production != null) { + builder.append(",prod=["); + boolean first = true; + for (AbstractStackNode

prodElem : production) { + if (first) { + first = false; + } else { + builder.append(","); + } + builder.append(prodElem.getId()); + } + builder.append("]"); + } + if (isEndNode) { + builder.append(",endNode"); + } + if (isSeparator) { + builder.append(",separator"); + } + if (isLayout) { + builder.append(",layout"); + } + + if (alternateProductions != null && alternateProductions.length != 0) { + builder.append(",alternateProductions=" + Arrays.toString(alternateProductions)); + } + if (edgesMap != null && edgesMap.size() != 0) { + builder.append(",edges=" + edgesMap); + } + if (prefixesMap != null && prefixesMap.length != 0) { + builder.append(",prefixes=" + Arrays.toString(prefixesMap)); + } + if (incomingEdges != null && incomingEdges.size() != 0) { + builder.append(",incomingEdges=" + incomingEdges); + } + if (alternativeProduction != null) { + builder.append(",alternativeProduction=" + DebugUtil.prodToString((IConstructor)alternativeProduction)); + } + if (propagatedPrefixes != null) { + builder.append(",propagatedPrefixes=" + propagatedPrefixes); + } + if (propagatedReductions != null) { + builder.append(",propagatedReductions=" + propagatedReductions); + } + + // Do not print filters for now. + + return builder.toString(); + } + // Matchables. /** * Matches the symbol associated with this node to the input at the specified location. diff --git a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java index 0dafc11f539..cd0db6bf2f2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java @@ -15,6 +15,7 @@ import org.rascalmpl.parser.gtd.result.LiteralNode; import org.rascalmpl.parser.gtd.stack.filter.ICompletionFilter; import org.rascalmpl.parser.gtd.stack.filter.IEnterFilter; +import org.rascalmpl.unicode.UnicodeConverter; public final class LiteralStackNode

extends AbstractMatchableStackNode

{ private final int[] literal; @@ -78,15 +79,13 @@ public AbstractNode getResult(){ } public String toString(){ - StringBuilder sb = new StringBuilder(); - for (int i : literal) { - sb.appendCodePoint(i); - } - sb.append(getId()); - sb.append('('); - sb.append(startLocation); - sb.append(')'); + StringBuilder sb = new StringBuilder("lit['"); + sb.append(UnicodeConverter.unicodeArrayToString(literal)); + sb.append("',"); + sb.append(super.toString()); + sb.append(']'); + return sb.toString(); } diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index 7f00fe8397b..247562026f7 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -77,12 +77,11 @@ public AbstractNode getResult(){ } public String toString(){ - StringBuilder sb = new StringBuilder(); + StringBuilder sb = new StringBuilder("NonTerminal["); sb.append(expectIdentifier); - sb.append(getId()); - sb.append('('); - sb.append(startLocation); - sb.append(')'); + sb.append(","); + sb.append(super.toString()); + sb.append("]"); return sb.toString(); } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 63164ab3bc7..84590af3d54 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -76,7 +76,7 @@ public AbstractNode getResult(){ return result; } - public String toString(){ + /*Original: public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(getId()); sb.append('('); @@ -84,12 +84,28 @@ public String toString(){ sb.append(')'); return sb.toString(); - } + }*/ + @Override + public String toString() { + return "SkippingStackNode[result=" + result + "," + super.toString() + "]"; + } + + @Override public int hashCode(){ return getParentProduction().hashCode(); } - + + @SuppressWarnings("unchecked") + @Override + public boolean equals(Object rhs) { + if (rhs instanceof AbstractStackNode) { + return isEqual((AbstractStackNode

)rhs); + } + + return false; + } + public boolean isEqual(AbstractStackNode

stackNode){ if ( !(stackNode instanceof SkippingStackNode)) { return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java b/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java index b778f6677a3..f95daf557c7 100644 --- a/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java +++ b/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java @@ -28,11 +28,13 @@ public class EdgesSet

{ private AbstractStackNode

[] edges; private int size; - + private int lastVisitedLevel = -1; private IntegerMap lastVisitedFilteredLevel; private AbstractContainerNode

lastResults; + + // Indexed by `resultStoreId`. private IntegerObjectList> lastFilteredResults; public EdgesSet(){ @@ -140,4 +142,33 @@ public int size(){ public void clear(){ size = 0; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("EdgesSet[{"); + + for (int i=0; i 0) { + builder.append(","); + } + builder.append(edges[i].getName() + "=" + edges[i].getId() + "." + edges[i].getDot() + "@" + edges[i].getStartLocation()); + } + + builder.append("}"); + + if (lastVisitedLevel >= 0) { + builder.append(",lastVisitedLevel=" + lastVisitedLevel); + } + if (lastResults != null) { + builder.append(",lastResults=" + lastResults); + } + + // Skip 'filtered' fields for now + + builder.append("]"); + + return builder.toString(); + } + + } diff --git a/src/org/rascalmpl/parser/gtd/util/ArrayList.java b/src/org/rascalmpl/parser/gtd/util/ArrayList.java index d93c1740981..899fcdbb0d4 100644 --- a/src/org/rascalmpl/parser/gtd/util/ArrayList.java +++ b/src/org/rascalmpl/parser/gtd/util/ArrayList.java @@ -104,4 +104,17 @@ public int size(){ public Object[] getBackingArray(){ return data; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("["); + for (int i=0; i 0) { + builder.append(","); + } + builder.append(data[i]); + } + builder.append("]"); + return builder.toString(); + } } diff --git a/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java b/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java index e1e6d4ea9ea..69b9e5ef0ef 100644 --- a/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java +++ b/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java @@ -102,4 +102,20 @@ public void ditryClear(){ public int size(){ return size; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("list["); + for (int i=0; i"); + } + builder.append("]\n"); + return builder.toString(); + } + } diff --git a/src/org/rascalmpl/parser/gtd/util/DoubleStack.java b/src/org/rascalmpl/parser/gtd/util/DoubleStack.java index 9251e89a519..3c2c6f3a6d7 100644 --- a/src/org/rascalmpl/parser/gtd/util/DoubleStack.java +++ b/src/org/rascalmpl/parser/gtd/util/DoubleStack.java @@ -171,4 +171,19 @@ public void clear(){ public void dirtyClear(){ size = 0; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("stack["); + for (int i=0; i"); + } + builder.append("]\n"); + return builder.toString(); + } } diff --git a/src/org/rascalmpl/parser/gtd/util/IntegerObjectList.java b/src/org/rascalmpl/parser/gtd/util/IntegerObjectList.java index 92d1009b02a..30b63fd46db 100644 --- a/src/org/rascalmpl/parser/gtd/util/IntegerObjectList.java +++ b/src/org/rascalmpl/parser/gtd/util/IntegerObjectList.java @@ -125,4 +125,20 @@ public void clear(){ public void dirtyClear(){ size = 0; } + + public String toString() { + StringBuilder builder = new StringBuilder("["); + + for (int i=0; i 0) { + builder.append(","); + } + builder.append(keys[i]); + builder.append("="); + builder.append(values[i]); + } + + builder.append("]"); + return builder.toString(); + } } diff --git a/src/org/rascalmpl/parser/uptr/debug/DebugLogger.java b/src/org/rascalmpl/parser/uptr/debug/DebugLogger.java index 89ba535a3bc..fb05ba5d95d 100644 --- a/src/org/rascalmpl/parser/uptr/debug/DebugLogger.java +++ b/src/org/rascalmpl/parser/uptr/debug/DebugLogger.java @@ -9,6 +9,9 @@ import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.util.ArrayList; +import org.rascalmpl.parser.gtd.util.DoubleArrayList; +import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.Stack; import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; @@ -153,4 +156,57 @@ public void filteredByEnterFilter(AbstractStackNode node){ public void filteredByCompletionFilter(AbstractStackNode node, AbstractNode result){ out.println(String.format("Filtered by completion filter: %s", node)); } + + @Override + public void reviving(int[] input, int location, Stack> unexpandableNodes, + Stack> unmatchableLeafNodes, + DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, + DoubleStack, AbstractNode> filteredNodes) { + out.print("Reviving at "); + out.print(location); + out.print(": input='"); + for (int i=0; i<8 && location+i < input.length; i++) { + out.print((char) input[location+i]); + } + out.print("', unexpandable="); + + boolean first = true; + for (int i=0; i 0) { + out.print(", unmatchableLeafNodes="); + out.print(unmatchableLeafNodes.getSize()); + } + + if (unmatchableMidProductionNodes.getSize() > 0) { + out.print(", unmatchableMidProductionNodes="); + out.print(unmatchableMidProductionNodes.toString()); + } + + if (filteredNodes.getSize() > 0) { + out.print(", filteredNodes="); + out.print(filteredNodes.getSize()); + } + + out.println(); + } + + @Override + public void revived(DoubleArrayList, AbstractNode> recoveredNodes) { + out.println("Revived nodes:"); + for (int i=0; i { private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; private IdDispenser stackNodeIdDispenser; - + + public ToNextWhitespaceRecoverer(IdDispenser stackNodeIdDispenser) { + this.stackNodeIdDispenser = stackNodeIdDispenser; + } + + @Override + public DoubleArrayList, AbstractNode> reviveStacks(int[] input, + int location, + Stack> unexpandableNodes, + Stack> unmatchableLeafNodes, + DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, + DoubleStack, AbstractNode> filteredNodes) { + + ArrayList> failedNodes = new ArrayList>(); + collectUnexpandableNodes(unexpandableNodes, failedNodes); + collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); + //collectFilteredNodes(filteredNodes, failedNodes); + + return reviveFailedNodes(input, location, failedNodes); + } + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); - for (int i = recoveryNodes.size() - 1; i >= 0; --i) { + // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { + // But this caused problems because recovery nodes with a later position + // where queued before nodes with an earlier position which the parser cannot handle. + for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); @@ -82,6 +105,11 @@ private static void collectUnexpandableNodes(Stack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); @@ -119,7 +147,7 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr ArrayList recoveryProductions = new ArrayList(); collectProductions(node, recoveryProductions); if (recoveryProductions.size() > 0) { - recoveryNodes.add(node, recoveryProductions); + recoveryNodes.add(node, recoveryProductions); } IntegerObjectList> edges = node.getEdges(); @@ -176,22 +204,5 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } - public ToNextWhitespaceRecoverer(IdDispenser stackNodeIdDispenser) { - this.stackNodeIdDispenser = stackNodeIdDispenser; - } - @Override - public DoubleArrayList, AbstractNode> reviveStacks(int[] input, - int location, - Stack> unexpandableNodes, - Stack> unmatchableLeafNodes, - DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, - DoubleStack, AbstractNode> filteredNodes) { - ArrayList> failedNodes = new ArrayList>(); - collectUnexpandableNodes(unexpandableNodes, failedNodes); - collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); - //collectFilteredNodes(filteredNodes, failedNodes); - - return reviveFailedNodes(input, location, failedNodes); - } } diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java new file mode 100644 index 00000000000..8f0391caafa --- /dev/null +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -0,0 +1,39 @@ +package org.rascalmpl.parser.util; + +import java.util.Iterator; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IList; +import io.usethesource.vallang.IValue; + +public class DebugUtil { + /** + * Turn a production IConstructor into a string of the form "S -> E1 E2 ..." + */ + public static String prodToString(IConstructor prod) { + StringBuilder builder = new StringBuilder("'"); + + IConstructor sort = (IConstructor) prod.get(0); + builder.append(stripQuotes(String.valueOf(sort.get(0)))); + + builder.append(" ->"); + + IList children = (IList) prod.get(1); + for (IValue child : children) { + builder.append(" "); + IConstructor conChild = (IConstructor) child; + builder.append(stripQuotes(String.valueOf((conChild).get(0)))); + } + builder.append("'"); + + return builder.toString(); + } + + private static String stripQuotes(String s) { + if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"') { + return s.substring(1, s.length()-1); + } + + return s; + } +} diff --git a/src/org/rascalmpl/unicode/UnicodeConverter.java b/src/org/rascalmpl/unicode/UnicodeConverter.java new file mode 100644 index 00000000000..b98980301d7 --- /dev/null +++ b/src/org/rascalmpl/unicode/UnicodeConverter.java @@ -0,0 +1,12 @@ +package org.rascalmpl.unicode; + +public class UnicodeConverter { + public static String unicodeArrayToString(int[] chars) { + StringBuilder builder = new StringBuilder(); + for (int c : chars) { + builder.appendCodePoint(c); + } + + return builder.toString(); + } +} diff --git a/test/org/rascalmpl/test/parser/IParserTest.java b/test/org/rascalmpl/test/parser/IParserTest.java index a9598a8239e..ccf5cd8c5bb 100644 --- a/test/org/rascalmpl/test/parser/IParserTest.java +++ b/test/org/rascalmpl/test/parser/IParserTest.java @@ -38,7 +38,7 @@ public static AbstractStackNode[] createExpectArray(IConstructor p expectArray[index-1].setAlternativeProduction(prod); - return expectArray; + return (AbstractStackNode[]) new AbstractStackNode[]{expectArray[0]}; } ITree executeParser(); diff --git a/test/org/rascalmpl/test/parser/RecoveryTests2.java b/test/org/rascalmpl/test/parser/RecoveryTests2.java new file mode 100644 index 00000000000..6156cdc8529 --- /dev/null +++ b/test/org/rascalmpl/test/parser/RecoveryTests2.java @@ -0,0 +1,148 @@ +package org.rascalmpl.test.parser; + +import java.io.IOException; +import java.io.PrintWriter; +import java.io.StringReader; + +import org.junit.Assert; +import org.junit.Test; +import org.rascalmpl.parser.gtd.SGTDBF; +import org.rascalmpl.parser.gtd.recovery.IRecoverer; +import org.rascalmpl.parser.gtd.result.out.DefaultNodeFlattener; +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; +import org.rascalmpl.parser.gtd.stack.LiteralStackNode; +import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; +import org.rascalmpl.parser.uptr.UPTRNodeFactory; +import org.rascalmpl.parser.uptr.debug.DebugLogger; +import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; +import org.rascalmpl.values.RascalValueFactory; +import org.rascalmpl.values.ValueFactoryFactory; +import org.rascalmpl.values.parsetrees.ITree; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.ISourceLocation; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.io.StandardTextReader; + +/** + * S -> A ws B ws C + * A -> [a] + * B -> [b] ws [b] + * C -> [c] + * + * ws -> [\ ] + */ +public class RecoveryTests2 extends SGTDBF implements IParserTest{ + private final static IConstructor SYMBOL_START_S = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("S")); + + private final static IConstructor SYMBOL_ws = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("ws")); + private final static IConstructor SYMBOL_char_space = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(32)))); + + private final static IConstructor SYMBOL_A = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("A")); + private final static IConstructor SYMBOL_a = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("a")); + private final static IConstructor SYMBOL_char_a = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(97)))); + private final static IConstructor PROD_A_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_A, VF.list(SYMBOL_a), VF.set()); + + private final static IConstructor SYMBOL_B = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("B")); + private final static IConstructor SYMBOL_b = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("b")); + private final static IConstructor SYMBOL_char_b = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(98)))); + private final static IConstructor PROD_B_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_B, VF.list(SYMBOL_b, SYMBOL_ws, SYMBOL_b), VF.set()); + + private final static IConstructor SYMBOL_C = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("C")); + private final static IConstructor SYMBOL_c = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("c")); + private final static IConstructor SYMBOL_char_c = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(99)))); + private final static IConstructor PROD_C_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_C, VF.list(SYMBOL_c), VF.set()); + + private final static IConstructor PROD_S_A_B_C = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_START_S, VF.list(SYMBOL_A, SYMBOL_ws, SYMBOL_B, SYMBOL_ws, SYMBOL_C), VF.set()); + + private final static IConstructor PROD_ws = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_ws, VF.list(SYMBOL_char_space), VF.set()); + private final static IConstructor PROD_a_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_a, VF.list(SYMBOL_char_a), VF.set()); + private final static IConstructor PROD_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_b, VF.list(SYMBOL_char_b), VF.set()); + private final static IConstructor PROD_c_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_c, VF.list(SYMBOL_char_c), VF.set()); + + public AbstractStackNode[] S(){ + return IParserTest.createExpectArray(PROD_S_A_B_C, + new NonTerminalStackNode(1, 0, "A"), + new LiteralStackNode(2, 1, PROD_ws, new int[]{' '}), + new NonTerminalStackNode(3, 2, "B"), + new LiteralStackNode(4, 3, PROD_ws, new int[]{' '}), + new NonTerminalStackNode(5, 4, "C") + ); + } + + public AbstractStackNode[] A(){ + return IParserTest.createExpectArray(PROD_A_a, + new LiteralStackNode(6, 0, PROD_a_a, new int[]{'a'}) + ); + } + + public AbstractStackNode[] B(){ + return IParserTest.createExpectArray(PROD_B_b_b, + new LiteralStackNode(9, 0, PROD_b_b, new int[]{'b'}), + new LiteralStackNode(10, 1, PROD_ws, new int[]{' '}), + new LiteralStackNode(11, 2, PROD_b_b, new int[]{'b'}) + ); + } + + public AbstractStackNode[] C(){ + return IParserTest.createExpectArray(PROD_C_c, + new LiteralStackNode(12, 0, PROD_c_c, new int[]{'c'}) + ); + } + + private int nextFreeStackNodeId = 100; + + @Override + protected int getFreeStackNodeId() { + return nextFreeStackNodeId++; + } + + private ITree parse(String s) { + DebugLogger debugLogger = new DebugLogger(new PrintWriter(System.out)); + IRecoverer recoverer = new ToNextWhitespaceRecoverer(() -> nextFreeStackNodeId++); + return parse("S" /* NONTERMINAL_START_S */, null, s.toCharArray(), + new DefaultNodeFlattener(), new UPTRNodeFactory(false), recoverer, debugLogger); + } + + private ITree parseB(String s) { + return parse("B", null, s.toCharArray(), + new DefaultNodeFlattener(), new UPTRNodeFactory(false), null, null); + } + + + private ITree toTree(String s) { + try { + return (ITree) new StandardTextReader().read(ValueFactoryFactory.getValueFactory(), RascalValueFactory.uptr, RascalValueFactory.Tree, new StringReader(s)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public ITree executeParser() { + return parse("a b b c"); + } + + @Override + public IValue getExpectedResult() { + return toTree("appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"); + } + + @Test + public void testOk() { + Assert.assertEquals(getExpectedResult(), executeParser()); + } + + @Test + public void testB() { + String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; + Assert.assertEquals(toTree(expected), parseB("b")); + } + + @Test + public void testMissingCharRecovery() { + String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(skipped(\\char-class([range(1,1114111)])),[char(120)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; + Assert.assertEquals(toTree(expected), parse("a b x c")); + } + +} From 13ffcefa1c7e62e62e80f63fe0eff1b26c67b154 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 18 Jun 2024 11:45:51 +0200 Subject: [PATCH 29/87] Added 'allowRecovery` keyword parameter in parsing functions. --- src/org/rascalmpl/interpreter/Evaluator.java | 2 +- src/org/rascalmpl/library/ParseTree.rsc | 29 +++++----- src/org/rascalmpl/library/Prelude.java | 24 ++++---- src/org/rascalmpl/parser/gtd/SGTDBF.java | 7 ++- .../parser/gtd/util/IdDispenser.java | 2 +- .../util/ReflectiveStackNodeIdDispenser.java | 53 +++++++++++++++++ .../semantics/dynamic/Expression.java | 14 +++-- .../rascalmpl/semantics/dynamic/Import.java | 4 +- .../rascalmpl/values/IRascalValueFactory.java | 8 +-- .../values/RascalFunctionValueFactory.java | 58 +++++++++++-------- 10 files changed, 133 insertions(+), 68 deletions(-) create mode 100644 src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java diff --git a/src/org/rascalmpl/interpreter/Evaluator.java b/src/org/rascalmpl/interpreter/Evaluator.java index 4ed8cd191ce..5f662043c28 100755 --- a/src/org/rascalmpl/interpreter/Evaluator.java +++ b/src/org/rascalmpl/interpreter/Evaluator.java @@ -990,7 +990,7 @@ private IFunction parserForCurrentModule(RascalFunctionValueFactory vf, ModuleEn IMap syntaxDefinition = curMod.getSyntaxDefinition(); IMap grammar = (IMap) getParserGenerator().getGrammarFromModules(getMonitor(), curMod.getName(), syntaxDefinition).get("rules"); IConstructor reifiedType = vf.reifiedType(dummy, grammar); - return vf.parsers(reifiedType, vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); + return vf.parsers(reifiedType, vf.bool(false), vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); } private Result evalMore(String command, ISourceLocation location) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 2813f58b661..733680858ed 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -393,14 +393,14 @@ catch ParseError(loc l): { } ``` } -&T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, |unknown:///|); +&T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, allowRecovery=allowRecovery, hasSideEffects=hasSideEffects, filters=filters)(input, |unknown:///|); -&T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, origin); +&T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, allowRecovery=allowRecovery, hasSideEffects=hasSideEffects, filters=filters)(input, origin); -&T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) - = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, input); +&T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) + = parser(begin, allowAmbiguity=allowAmbiguity, allowRecovery=allowRecovery, hasSideEffects=hasSideEffects, filters=filters)(input, input); @synopsis{Generates a parser from an input grammar.} @@ -416,15 +416,16 @@ So the parse function reads either directly from a str or via the contents of a which leads to the prefix of the `src` fields of the resulting tree. The parse function behaves differently depending of the given keyword parameters: - * `allowAmbiguity`: if true then no exception is thrown in case of ambiguity and a parse forest is returned. if false, + * `allowAmbiguity`: if true then no exception is thrown in case of ambiguity and a parse forest is returned. if false, the parser throws an exception during tree building and produces only the first ambiguous subtree in its message. if set to `false`, the parse constructs trees in linear time. if set to `true` the parser constructs trees in polynomial time. - * + * 'allowRecovery`: ***experimental*** if true, the parser tries to recover on a parse error. if a parse error is encountered that can be recovered from, special `skipped` nodes + are included in the resulting parse tree. More documentation will be added here when this feature matures. * `hasSideEffects`: if false then the parser is a lot faster when constructing trees, since it does not execute the parse _actions_ in an interpreted environment to make side effects (like a symbol table) and it can share more intermediate results as a result. } @javaClass{org.rascalmpl.library.Prelude} -java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @javaClass{org.rascalmpl.library.Prelude} @synopsis{Generates a parser function that can be used to find the left-most deepest ambiguous sub-sentence.} @@ -437,7 +438,7 @@ the tree that exhibits ambiguity. This can be done very quickly, while the whole * The returned sub-tree usually has a different type than the parameter of the type[] symbol that was passed in. The reason is that sub-trees typically have a different non-terminal than the start non-terminal of a grammar. } -java Tree (value input, loc origin) firstAmbiguityFinder(type[Tree] grammar, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java Tree (value input, loc origin) firstAmbiguityFinder(type[Tree] grammar, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @synopsis{Generates parsers from a grammar (reified type), where all non-terminals in the grammar can be used as start-symbol.} @description{ @@ -445,7 +446,7 @@ This parser generator behaves the same as the `parser` function, but it produces nonterminal parameter. This can be used to select a specific non-terminal from the grammar to use as start-symbol for parsing. } @javaClass{org.rascalmpl.library.Prelude} -java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @javaClass{org.rascalmpl.library.Prelude} @synopsis{Generates a parser function that can be used to find the left-most deepest ambiguous sub-sentence.} @@ -458,7 +459,7 @@ the tree that exhibits ambiguity. This can be done very quickly, while the whole * The returned sub-tree usually has a different type than the parameter of the type[] symbol that was passed in. The reason is that sub-trees typically have a different non-terminal than the start non-terminal of a grammar. } -java Tree (type[Tree] nonterminal, value input, loc origin) firstAmbiguityFinders(type[Tree] grammar, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java Tree (type[Tree] nonterminal, value input, loc origin) firstAmbiguityFinders(type[Tree] grammar, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @synopsis{Parse the input but instead of returning the entire tree, return the trees for the first ambiguous substring.} @description{ @@ -536,7 +537,7 @@ p(type(sort("E"), ()), "e+e", |src:///|); * reifiying types (use of `#`) will trigger the loading of a parser generator anyway. You have to use this notation for types to avoid that: `type(\start(sort("MySort")), ())` to avoid the computation for `#start[A]` } -java &U (type[&U] nonterminal, value input, loc origin) loadParsers(loc savedParsers, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java &U (type[&U] nonterminal, value input, loc origin) loadParsers(loc savedParsers, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @synopsis{Load a previously serialized parser, for a specific non-terminal, from disk for usage} @description{ @@ -544,7 +545,7 @@ This loader behaves just like ((loadParsers)), except that the resulting parser bound to a specific non-terminal. } @javaClass{org.rascalmpl.library.Prelude} -java &U (value input, loc origin) loadParser(type[&U] nonterminal, loc savedParsers, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); +java &U (value input, loc origin) loadParser(type[&U] nonterminal, loc savedParsers, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); @synopsis{Yield the string of characters that form the leafs of the given parse tree.} @description{ diff --git a/src/org/rascalmpl/library/Prelude.java b/src/org/rascalmpl/library/Prelude.java index 33036ee01ab..242137d8aa5 100644 --- a/src/org/rascalmpl/library/Prelude.java +++ b/src/org/rascalmpl/library/Prelude.java @@ -2349,20 +2349,20 @@ public INode arbNode() { protected final TypeReifier tr; - public IFunction parser(IValue start, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) { - return rascalValues.parser(start, allowAmbiguity, hasSideEffects, values.bool(false), filters); + public IFunction parser(IValue start, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, ISet filters) { + return rascalValues.parser(start, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters); } - public IFunction firstAmbiguityFinder(IValue start, IBool hasSideEffects, ISet filters) { - return rascalValues.parser(start, values.bool(true), hasSideEffects, values.bool(true), filters); + public IFunction firstAmbiguityFinder(IValue start, IBool allowRecovery, IBool hasSideEffects, ISet filters) { + return rascalValues.parser(start, values.bool(true), allowRecovery, hasSideEffects, values.bool(true), filters); } - public IFunction parsers(IValue start, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) { - return rascalValues.parsers(start, allowAmbiguity, hasSideEffects, values.bool(false), filters); + public IFunction parsers(IValue start, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, ISet filters) { + return rascalValues.parsers(start, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters); } - public IFunction firstAmbiguityFinders(IValue start, IBool hasSideEffects, ISet filters) { - return rascalValues.parsers(start, values.bool(true), hasSideEffects, values.bool(true), filters); + public IFunction firstAmbiguityFinders(IValue start, IBool allowRecovery, IBool hasSideEffects, ISet filters) { + return rascalValues.parsers(start, values.bool(true), allowRecovery, hasSideEffects, values.bool(true), filters); } public void storeParsers(IValue start, ISourceLocation saveLocation) { @@ -2377,18 +2377,18 @@ public void storeParsers(IValue start, ISourceLocation saveLocation) { } } - public IFunction loadParsers(ISourceLocation savedLocation, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) { + public IFunction loadParsers(ISourceLocation savedLocation, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, ISet filters) { try { - return rascalValues.loadParsers(savedLocation, allowAmbiguity, hasSideEffects, values.bool(false), filters); + return rascalValues.loadParsers(savedLocation, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters); } catch (IOException | ClassNotFoundException e) { throw RuntimeExceptionFactory.io(e.getMessage()); } } - public IFunction loadParser(IValue grammar, ISourceLocation savedLocation, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) { + public IFunction loadParser(IValue grammar, ISourceLocation savedLocation, IBool allowRecovery, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) { try { - return rascalValues.loadParser(grammar, savedLocation, allowAmbiguity, hasSideEffects, values.bool(false), filters); + return rascalValues.loadParser(grammar, savedLocation, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters); } catch (IOException | ClassNotFoundException e) { throw RuntimeExceptionFactory.io(e.getMessage()); diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 458568f5643..15940442b81 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -864,10 +864,11 @@ private boolean findStacksToReduce(){ debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); - debugListener.revived(recoveredNodes); + if (debugListener != null) { + debugListener.revived(recoveredNodes); + } if (recoveredNodes.size() > 0) { - - // for (int i = recoveredNodes.size()-1; i>= 0; i--) { + // was: for (int i = recoveredNodes.size()-1; i>= 0; i--) { for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); diff --git a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java index 3952d7fa3c0..fe9aa091406 100644 --- a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java @@ -1,5 +1,5 @@ package org.rascalmpl.parser.gtd.util; public interface IdDispenser { - int dispenseId(); + int dispenseId(); } diff --git a/src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java b/src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java new file mode 100644 index 00000000000..dfbafab9d84 --- /dev/null +++ b/src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java @@ -0,0 +1,53 @@ +package org.rascalmpl.parser.gtd.util; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +import org.rascalmpl.parser.gtd.IGTD; +import org.rascalmpl.values.parsetrees.ITree; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.ISourceLocation; + +/** + * To offer backwards compatibility for generated parsers that do not yet have the "getFreeStackNodeId" method yet, + * this class uses reflection to find that method and otherwise improvises by just using a ridiculously high starting number. + */ +public class ReflectiveStackNodeIdDispenser implements IdDispenser { + private IGTD parser; + private Method dispenseMethod; + private int nextNodeIdBackup = (Integer.MAX_VALUE/4)*3; + + public ReflectiveStackNodeIdDispenser(IGTD parser) { + try { + dispenseMethod = parser.getClass().getMethod("getFreeStackNodeId"); + } catch (NoSuchMethodException e) { + // Custom IGTB implementation without "getFreeStackNodeId" method. No biggy, we just use nextNodeIdBackup. + } + } + + @Override + public int dispenseId() { + if (dispenseMethod != null) { + try { + return (Integer)dispenseMethod.invoke(parser); + } + catch (InvocationTargetException e) { + if (e.getTargetException() instanceof UnsupportedOperationException) { + // We are dealing with a parser class that has no generated "getFreeStackNodeId" method (yet), + // for backwards compatibility we fall back on "nextNodeIdBackup". + dispenseMethod = null; // No reason to try again. + } else { + throw new RuntimeException(e); + } + } + catch (IllegalAccessException | IllegalArgumentException e) { + throw new RuntimeException(e); + } + } + + return nextNodeIdBackup++; + } + +} + diff --git a/src/org/rascalmpl/semantics/dynamic/Expression.java b/src/org/rascalmpl/semantics/dynamic/Expression.java index ba5f629189d..4d7ed5c1c3f 100644 --- a/src/org/rascalmpl/semantics/dynamic/Expression.java +++ b/src/org/rascalmpl/semantics/dynamic/Expression.java @@ -1070,7 +1070,7 @@ private boolean isBootstrapped(IEvaluatorContext eval) { return false; } - private ITree parseObject(IEvaluatorContext eval, IConstructor grammar, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects) { + private ITree parseObject(IEvaluatorContext eval, IConstructor grammar, ISet filters, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean allowRecovery, boolean hasSideEffects) { RascalFunctionValueFactory vf = eval.getFunctionValueFactory(); IString str = vf.string(new String(input)); @@ -1078,19 +1078,19 @@ private ITree parseObject(IEvaluatorContext eval, IConstructor grammar, ISet fil return (ITree) vf.bootstrapParsers().call(grammar, str, location); } else { - IFunction parser = vf.parser(grammar, vf.bool(allowAmbiguity), vf.bool(hasSideEffects), vf.bool(false), filters); + IFunction parser = vf.parser(grammar, vf.bool(allowAmbiguity), vf.bool(allowRecovery), vf.bool(hasSideEffects), vf.bool(false), filters); return (ITree) parser.call(vf.string(new String(input)), location); } } - private ITree parseObject(IEvaluatorContext eval, IConstructor grammar, ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean hasSideEffects) { + private ITree parseObject(IEvaluatorContext eval, IConstructor grammar, ISet filters, ISourceLocation location, boolean allowAmbiguity, boolean allowRecovery, boolean hasSideEffects) { RascalFunctionValueFactory vf = eval.getFunctionValueFactory(); if (isBootstrapped(eval)) { return (ITree) vf.bootstrapParsers().call(grammar, location, location); } else { - IFunction parser = vf.parser(grammar, vf.bool(allowAmbiguity), vf.bool(hasSideEffects), vf.bool(false), filters); + IFunction parser = vf.parser(grammar, vf.bool(allowAmbiguity), vf.bool(allowRecovery), vf.bool(hasSideEffects), vf.bool(false), filters); return (ITree) parser.call(location, location); } } @@ -1125,11 +1125,13 @@ public Result interpret(IEvaluator> __eval) { IConstructor value = ((IRascalValueFactory) __eval.getValueFactory()).reifiedType(symbol, gr); if (result.getStaticType().isString()) { + // TODO: discuss if we want to allow recovery here tree = parseObject(__eval, value, VF.set(), this.getLocation(), - ((IString) result.getValue()).getValue().toCharArray(), true, false); + ((IString) result.getValue()).getValue().toCharArray(), true, false, false); } else if (result.getStaticType().isSourceLocation()) { - tree = parseObject(__eval, value, VF.set(), (ISourceLocation) result.getValue(), true, false); + // TODO: discuss if we want to allow recovery here + tree = parseObject(__eval, value, VF.set(), (ISourceLocation) result.getValue(), true, false, false); } assert tree != null; // because we checked earlier diff --git a/src/org/rascalmpl/semantics/dynamic/Import.java b/src/org/rascalmpl/semantics/dynamic/Import.java index ebb4656494a..35088776167 100644 --- a/src/org/rascalmpl/semantics/dynamic/Import.java +++ b/src/org/rascalmpl/semantics/dynamic/Import.java @@ -487,7 +487,7 @@ public static ITree parseModuleAndFragments(char[] data, ISourceLocation locatio } else if (reg.exists(parserCacheFile)) { // if we cached a ModuleFile.parsers file, we will use the parser from that (typically after deployment time) - parsers = vf.loadParsers(parserCacheFile, vf.bool(false),vf.bool(false),vf.bool(false), vf.set()); + parsers = vf.loadParsers(parserCacheFile, vf.bool(false), vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); } else { // otherwise we have to generate a fresh parser for this module now @@ -495,7 +495,7 @@ else if (reg.exists(parserCacheFile)) { IMap syntaxDefinition = env.getSyntaxDefinition(); IMap grammar = (IMap) eval.getParserGenerator().getGrammarFromModules(eval.getMonitor(),env.getName(), syntaxDefinition).get("rules"); IConstructor reifiedType = vf.reifiedType(dummy, grammar); - parsers = vf.parsers(reifiedType, vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); + parsers = vf.parsers(reifiedType, vf.bool(false), vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); } try { diff --git a/src/org/rascalmpl/values/IRascalValueFactory.java b/src/org/rascalmpl/values/IRascalValueFactory.java index 4770b97e701..407e88e4972 100644 --- a/src/org/rascalmpl/values/IRascalValueFactory.java +++ b/src/org/rascalmpl/values/IRascalValueFactory.java @@ -94,7 +94,7 @@ default IFunction function(Type functionType, BiFunction getParserGenerator(), this, caller, parser, allowAmbiguity, hasSideEffects, firstAmbiguity, filters)); + return function(functionType, new ParametrizedParseFunction(() -> getParserGenerator(), this, caller, parser, allowAmbiguity, allowRecovery, hasSideEffects, firstAmbiguity, filters)); } @Override @@ -292,7 +294,7 @@ public void storeParsers(IValue reifiedGrammar, ISourceLocation saveLocation) th } @Override - public IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) throws IOException, ClassNotFoundException { + public IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) throws IOException, ClassNotFoundException { RascalTypeFactory rtf = RascalTypeFactory.getInstance(); TypeFactory tf = TypeFactory.getInstance(); @@ -318,12 +320,12 @@ public IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, this, caller, parser, - allowAmbiguity, hasSideEffects, firstAmbiguity, filters) + allowAmbiguity, allowRecovery, hasSideEffects, firstAmbiguity, filters) ); } @Override - public IFunction loadParser(IValue reifiedGrammar, ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) throws IOException, ClassNotFoundException { + public IFunction loadParser(IValue reifiedGrammar, ISourceLocation saveLocation, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) throws IOException, ClassNotFoundException { TypeFactory tf = TypeFactory.getInstance(); Type functionType = tf.functionType(reifiedGrammar.getType().getTypeParameters().getFieldType(0), @@ -347,7 +349,7 @@ public IFunction loadParser(IValue reifiedGrammar, ISourceLocation saveLocation, name = generator.getParserMethodName(startSort); } - return function(functionType, new ParseFunction(ctx.getValueFactory(), caller, parser, name, allowAmbiguity, hasSideEffects, firstAmbiguity, filters)); + return function(functionType, new ParseFunction(ctx.getValueFactory(), caller, parser, name, allowAmbiguity, allowRecovery, hasSideEffects, firstAmbiguity, filters)); } /** @@ -372,7 +374,7 @@ public IFunction bootstrapParsers() { AbstractAST current = ctx.getCurrentAST(); ISourceLocation caller = current != null ? current.getLocation() : URIUtil.rootLocation("unknown"); - return function(functionType, new ParametrizedParseFunction(() -> getParserGenerator(), this, caller, parser, vf.bool(false), vf.bool(false), vf.bool(false), ctx.getValueFactory().set())); + return function(functionType, new ParametrizedParseFunction(() -> getParserGenerator(), this, caller, parser, vf.bool(false), vf.bool(false), vf.bool(false), vf.bool(false), ctx.getValueFactory().set())); } public IString createHole(ITree part, IInteger index) { @@ -417,19 +419,21 @@ static private class ParseFunction implements BiFunction> parser; protected final String methodName; protected final ISourceLocation caller; - public ParseFunction(IValueFactory vf, ISourceLocation caller, Class> parser, String methodName, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { + public ParseFunction(IValueFactory vf, ISourceLocation caller, Class> parser, String methodName, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { this.vf = vf; this.caller = caller; this.parser = parser; this.methodName = methodName; this.filters = filters; this.allowAmbiguity = allowAmbiguity.getValue() || firstAmbiguity.getValue(); + this.allowRecovery = allowRecovery.getValue(); this.hasSideEffects = hasSideEffects.getValue(); this.firstAmbiguity = firstAmbiguity.getValue(); } @@ -454,10 +458,10 @@ else if (parameters[0].getType().isSourceLocation()) { } if (parameters[0].getType().isString()) { - return parse(methodName, filters, (IString) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects); + return parse(methodName, filters, (IString) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, allowRecovery, hasSideEffects); } else if (parameters[0].getType().isSourceLocation()) { - return parse(methodName, filters, (ISourceLocation) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, hasSideEffects); + return parse(methodName, filters, (ISourceLocation) parameters[0], (ISourceLocation) parameters[1], allowAmbiguity, allowRecovery, hasSideEffects); } } @@ -477,13 +481,13 @@ private IGTD getParser() { } } - protected IValue parse(String methodName, ISet filters, IString input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects) { + protected IValue parse(String methodName, ISet filters, IString input, ISourceLocation origin, boolean allowAmbiguity, boolean allowRecovery, boolean hasSideEffects) { try { if (origin == null) { origin = URIUtil.rootLocation("unknown"); } - return parseObject(methodName, origin, input.getValue().toCharArray(), allowAmbiguity, hasSideEffects, filters); + return parseObject(methodName, origin, input.getValue().toCharArray(), allowAmbiguity, allowRecovery, hasSideEffects, filters); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -500,7 +504,7 @@ protected IValue parse(String methodName, ISet filters, IString input, ISourceL protected IValue firstAmbiguity(String methodName, IString input) { try { - return parseObject(methodName, URIUtil.invalidLocation(), input.getValue().toCharArray(), false, false, vf.set()); + return parseObject(methodName, URIUtil.invalidLocation(), input.getValue().toCharArray(), false, false, false, vf.set()); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -516,7 +520,7 @@ protected IValue firstAmbiguity(String methodName, IString input) { protected IValue firstAmbiguity(String methodName, ISourceLocation input) { try { - return parseObject(methodName, input, readAll(input), false, false, vf.set()); + return parseObject(methodName, input, readAll(input), false, false, false, vf.set()); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -543,13 +547,13 @@ private IString printSymbol(IConstructor symbol) { return vf.string(SymbolAdapter.toString(symbol, false)); } - protected IValue parse(String methodName, ISet filters, ISourceLocation input, ISourceLocation origin, boolean allowAmbiguity, boolean hasSideEffects) { + protected IValue parse(String methodName, ISet filters, ISourceLocation input, ISourceLocation origin, boolean allowAmbiguity, boolean allowRecovery, boolean hasSideEffects) { if (origin == null) { origin = input; } try { - return parseObject(methodName, input, readAll(input), allowAmbiguity, hasSideEffects, filters); + return parseObject(methodName, input, readAll(input), allowAmbiguity, allowRecovery, hasSideEffects, filters); } catch (ParseError pe) { ISourceLocation errorLoc = pe.getLocation(); @@ -567,10 +571,14 @@ protected IValue parse(String methodName, ISet filters, ISourceLocation input, I } } - private ITree parseObject(String methodName, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean hasSideEffects, ISet filters) { + private ITree parseObject(String methodName, ISourceLocation location, char[] input, boolean allowAmbiguity, boolean allowRecovery, boolean hasSideEffects, ISet filters) { IActionExecutor exec = filters.isEmpty() ? new NoActionExecutor() : new RascalFunctionActionExecutor(filters, !hasSideEffects); - - return (ITree) getParser().parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener(), new UPTRNodeFactory(allowAmbiguity), (IRecoverer) null); + IGTD parserInstance = getParser(); + IRecoverer recoverer = null; + if (allowRecovery) { + recoverer = new ToNextWhitespaceRecoverer(new ReflectiveStackNodeIdDispenser(parserInstance)); + } + return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer); } } @@ -585,8 +593,8 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in static private class ParametrizedParseFunction extends ParseFunction { private Supplier generator; - public ParametrizedParseFunction(Supplier generator, IValueFactory vf, ISourceLocation caller, Class> parser, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { - super(vf, caller, parser, null, allowAmbiguity, hasSideEffects, firstAmbiguity, filters); + public ParametrizedParseFunction(Supplier generator, IValueFactory vf, ISourceLocation caller, Class> parser, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { + super(vf, caller, parser, null, allowAmbiguity, allowRecovery, hasSideEffects, firstAmbiguity, filters); this.generator = generator; } @@ -621,10 +629,10 @@ else if (parameters[1].getType().isSourceLocation()) { } if (parameters[1].getType().isString()) { - return parse(name, filters, (IString) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects); + return parse(name, filters, (IString) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, allowRecovery, hasSideEffects); } else if (parameters[1].getType().isSourceLocation()) { - return parse(name, filters, (ISourceLocation) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, hasSideEffects); + return parse(name, filters, (ISourceLocation) parameters[1], (ISourceLocation) parameters[2], allowAmbiguity, allowRecovery, hasSideEffects); } } From b8eb72e807ce6b8ed93081809f8f6b050e809adb Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 18 Jun 2024 13:46:51 +0200 Subject: [PATCH 30/87] Removed 'Reflective' prefix from StackNodeIdDispenser class name Reflective is just the way the dispenser is implemented. No need for users of this class to be aware of this. --- ...iveStackNodeIdDispenser.java => StackNodeIdDispenser.java} | 4 ++-- .../parser/uptr/recovery/ToNextWhitespaceRecoverer.java | 2 +- src/org/rascalmpl/values/RascalFunctionValueFactory.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/org/rascalmpl/parser/gtd/util/{ReflectiveStackNodeIdDispenser.java => StackNodeIdDispenser.java} (91%) diff --git a/src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java similarity index 91% rename from src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java rename to src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java index dfbafab9d84..403bfbe9247 100644 --- a/src/org/rascalmpl/parser/gtd/util/ReflectiveStackNodeIdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java @@ -13,12 +13,12 @@ * To offer backwards compatibility for generated parsers that do not yet have the "getFreeStackNodeId" method yet, * this class uses reflection to find that method and otherwise improvises by just using a ridiculously high starting number. */ -public class ReflectiveStackNodeIdDispenser implements IdDispenser { +public class StackNodeIdDispenser implements IdDispenser { private IGTD parser; private Method dispenseMethod; private int nextNodeIdBackup = (Integer.MAX_VALUE/4)*3; - public ReflectiveStackNodeIdDispenser(IGTD parser) { + public StackNodeIdDispenser(IGTD parser) { try { dispenseMethod = parser.getClass().getMethod("getFreeStackNodeId"); } catch (NoSuchMethodException e) { diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 382662ffc4c..daebd1eca8e 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -63,7 +63,7 @@ private DoubleArrayList, AbstractNode> reviveNod for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); - + // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 918148c87e8..8278d4dce93 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -47,7 +47,7 @@ import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.action.IActionExecutor; import org.rascalmpl.parser.gtd.result.out.DefaultNodeFlattener; -import org.rascalmpl.parser.gtd.util.ReflectiveStackNodeIdDispenser; +import org.rascalmpl.parser.gtd.util.StackNodeIdDispenser; import org.rascalmpl.parser.uptr.UPTRNodeFactory; import org.rascalmpl.parser.uptr.action.NoActionExecutor; import org.rascalmpl.parser.uptr.action.RascalFunctionActionExecutor; @@ -576,7 +576,7 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IGTD parserInstance = getParser(); IRecoverer recoverer = null; if (allowRecovery) { - recoverer = new ToNextWhitespaceRecoverer(new ReflectiveStackNodeIdDispenser(parserInstance)); + recoverer = new ToNextWhitespaceRecoverer(new StackNodeIdDispenser(parserInstance)); } return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer); } From f82006c9a0bf89c772dcea677b6115bf324c0b11 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 27 Jun 2024 15:02:08 +0200 Subject: [PATCH 31/87] WIP: recovery starting to work --- .../tests/recovery/BasicRecoveryTests.rsc | 18 +++ src/org/rascalmpl/parser/gtd/SGTDBF.java | 19 +-- .../parser/gtd/result/SkippedNode.java | 19 ++- .../result/out/INodeConstructorFactory.java | 4 +- .../gtd/result/out/RecoveryNodeFlattener.java | 3 +- .../parser/gtd/stack/AbstractStackNode.java | 2 + .../parser/gtd/stack/SkippingStackNode.java | 10 +- .../parser/gtd/util/DoubleArrayList.java | 19 +++ .../parser/uptr/UPTRNodeFactory.java | 22 ++- .../recovery/ToNextWhitespaceRecoverer.java | 22 ++- .../rascalmpl/values/RascalValueFactory.java | 2 +- .../rascalmpl/test/parser/RecoveryTests.java | 35 ++--- .../rascalmpl/test/parser/RecoveryTests2.java | 148 ------------------ .../test/recovery/ErrorRecoveryModules.java | 11 ++ 14 files changed, 133 insertions(+), 201 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc delete mode 100644 test/org/rascalmpl/test/parser/RecoveryTests2.java create mode 100644 test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc new file mode 100644 index 00000000000..de6194dcc99 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -0,0 +1,18 @@ +module lang::rascal::tests::recovery::BasicRecoveryTests + + +import ParseTree; +import IO; + +layout Layout = [\ ]* !>> [\ ]; + +syntax S = A B C; +syntax A = "a"; +syntax B = "b" "b"; +syntax C = "c"; + +test bool parseTest() { + Tree t = parse(#S, "a b x c", allowRecovery=true); + iprintln(t); + return true; +} diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 15940442b81..f7a4ddb5588 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -919,6 +919,8 @@ private void queueMatchableNode(AbstractStackNode

node, int length, AbstractN * Inserts a recovery node into the todo-list, and possibly * rewinds the parser to an earlier location in the input */ + // TODO: look for original code: + // - queueMatchableNode (just above here) @SuppressWarnings("unchecked") private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int length, AbstractNode result){ assert result != null; @@ -937,20 +939,19 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int DoubleStack, AbstractNode>[] oldTodoLists = todoLists; todoLists = new DoubleStack[negativeOffset + Math.max(queueDepth, length) + 1]; - System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset + 1, queueDepth - queueIndex); - System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset + 1, queueIndex); + System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset, queueDepth - queueIndex); + System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset , queueIndex); // reset the parser! queueIndex = 0; - location = startPosition + 1; + location = startPosition; - DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; - if (terminalsTodo == null){ + // was: DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; + DoubleStack, AbstractNode> terminalsTodo = todoLists[length]; + if (terminalsTodo == null) { terminalsTodo = new DoubleStack, AbstractNode>(); - todoLists[queueIndex + 1] = terminalsTodo; // Why the +1 and not length? To get the recovered node to be processed first? - } - else { - assert false: "this should never happen"; + // was: todoLists[queueIndex + 1] = terminalsTodo; // Why the +1 and not length? To get the recovered node to be processed first? + todoLists[length] = terminalsTodo; } terminalsTodo.push(node, result); diff --git a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java index 8fd8adacfc2..48882533b2b 100644 --- a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java +++ b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java @@ -11,8 +11,6 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.result; -import java.util.Arrays; - import org.rascalmpl.unicode.UnicodeConverter; /** @@ -21,13 +19,16 @@ public class SkippedNode extends AbstractNode { public final static int ID = 9; + private final Object production; + private final int dot; private final int[] skippedChars; - private final int offset; - public SkippedNode(int[] skippedChars, int offset){ + public SkippedNode(Object production, int dot, int[] skippedChars, int offset) { super(); + this.production = production; + this.dot = dot; this.skippedChars = skippedChars; this.offset = offset; } @@ -35,7 +36,15 @@ public SkippedNode(int[] skippedChars, int offset){ public int getTypeIdentifier(){ return ID; } - + + public Object getProduction() { + return production; + } + + public int getDot() { + return dot; + } + public int[] getSkippedChars(){ return skippedChars; } diff --git a/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java b/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java index e79ae43757b..5a94f7c3b64 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java +++ b/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java @@ -38,8 +38,8 @@ public interface INodeConstructorFactory { T createListAmbiguityNode(ArrayList alternatives); - T createRecoveryNode(int[] characters); - + T createRecoveryNode(int dot, ArrayList recognizedPrefix, int[] unrecognizedCharacters, Object production); + ArrayList getChildren(T node); P createPositionInformation(URI input, int offset, int endOffset, PositionStore positionStore); diff --git a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java index 833da06dcd4..b7a2605bfa9 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java @@ -12,6 +12,7 @@ package org.rascalmpl.parser.gtd.result.out; import org.rascalmpl.parser.gtd.result.SkippedNode; +import org.rascalmpl.parser.gtd.util.ArrayList; /** * A converter for result nodes that contain skipped characters for error recovery @@ -23,6 +24,6 @@ public RecoveryNodeFlattener(){ } public T convertToUPTR(INodeConstructorFactory nodeConstructorFactory, SkippedNode node){ - return nodeConstructorFactory.createRecoveryNode(node.getSkippedChars()); + return nodeConstructorFactory.createRecoveryNode(node.getDot(), new ArrayList<>(), node.getSkippedChars(), node.getProduction()); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index f0df8513e42..4f8259690df 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -34,7 +34,9 @@ public abstract class AbstractStackNode

{ protected AbstractStackNode

[] production; protected AbstractStackNode

[][] alternateProductions; + // Our edges protected IntegerObjectList> edgesMap; // : key=startLocation, value=EdgesSet a that location + // Edges of our children protected ArrayList[] prefixesMap; protected EdgesSet

incomingEdges; diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 84590af3d54..2ef909c5684 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -17,10 +17,10 @@ public final class SkippingStackNode

extends AbstractMatchableStackNode

{ private final SkippedNode result; - public SkippingStackNode(int id, int[] until, int[] input, int startLocation, P parentProduction){ + public SkippingStackNode(int id, int[] until, int[] input, int startLocation, P parentProduction, int dot){ super(id, 0); - this.result = buildResult(input, until, startLocation); + this.result = buildResult(input, until, startLocation, parentProduction, dot); setAlternativeProduction(parentProduction); } @@ -36,7 +36,7 @@ private SkippingStackNode(SkippingStackNode

original, SkippedNode result, int this.result = result; } - private static SkippedNode buildResult(int[] input, int[] until, int startLocation){ + private SkippedNode buildResult(int[] input, int[] until, int startLocation, P production, int dot){ for (int to = startLocation ; to < input.length; ++to) { for (int i = 0; i < until.length; ++i) { if (input[to] == until[i]) { @@ -44,12 +44,12 @@ private static SkippedNode buildResult(int[] input, int[] until, int startLocati int[] chars = new int[length]; System.arraycopy(input, startLocation, chars, 0, length); - return new SkippedNode(chars, startLocation); + return new SkippedNode(production, dot, chars, startLocation); } } } - return new SkippedNode(new int[0], startLocation); + return new SkippedNode(production, dot, new int[0], startLocation); } public boolean isEmptyLeafNode(){ diff --git a/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java b/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java index 69b9e5ef0ef..655adb7a879 100644 --- a/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java +++ b/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java @@ -11,6 +11,10 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.util; +import java.util.Comparator; + +import org.apache.commons.lang3.tuple.Pair; + @SuppressWarnings("unchecked") public class DoubleArrayList{ private final static int DEFAULT_SIZE = 8; @@ -103,6 +107,21 @@ public int size(){ return size; } + public void sort(Comparator> comparator) { + java.util.List> elems = new java.util.ArrayList<>(size); + for (int i=0; i elem = elems.get(i); + first[i] = elem.getLeft(); + second[i] = elem.getRight(); + } + } + @Override public String toString() { StringBuilder builder = new StringBuilder("list["); diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index 5988650baaa..b971764ccfd 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -1,7 +1,6 @@ package org.rascalmpl.parser.uptr; import java.net.URI; -import java.util.Arrays; import java.util.IdentityHashMap; import java.util.Map; @@ -14,6 +13,7 @@ import io.usethesource.vallang.IListWriter; import io.usethesource.vallang.ISetWriter; import io.usethesource.vallang.ISourceLocation; +import io.usethesource.vallang.IValue; import org.rascalmpl.values.RascalValueFactory; import org.rascalmpl.values.ValueFactoryFactory; @@ -22,8 +22,7 @@ import org.rascalmpl.values.parsetrees.TreeAdapter; public class UPTRNodeFactory implements INodeConstructorFactory{ - private final static RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); - private final static IConstructor SKIPPED = VF.constructor(RascalValueFactory.Production_Skipped, VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Range, VF.integer(1), VF.integer(Character.MAX_CODE_POINT))))); + private static final RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); private boolean allowAmb; public UPTRNodeFactory(boolean allowAmbiguity){ @@ -143,8 +142,19 @@ public Object getProductionFromNode(ITree node){ } @Override - public ITree createRecoveryNode(int[] characters) { - IList chars = Arrays.stream(characters).mapToObj(ch -> VF.character(ch)).collect(VF.listWriter()); - return VF.appl(SKIPPED, chars); + public ITree createRecoveryNode(int dot, ArrayList recognizedPrefix, int[] unrecognizedCharacters, Object production) { + IListWriter args = VF.listWriter(); + for (int i=0; i, AbstractNode> reviveStac ArrayList> failedNodes = new ArrayList>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); + // TODO: handle unmatchableLeafNodes //collectFilteredNodes(filteredNodes, failedNodes); return reviveFailedNodes(input, location, failedNodes); @@ -60,22 +62,28 @@ private DoubleArrayList, AbstractNode> reviveNod // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { // But this caused problems because recovery nodes with a later position // where queued before nodes with an earlier position which the parser cannot handle. - for (int i = 0; i Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); + + for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); + //Pair, ArrayList> elem = elems.get(i); + //AbstractStackNode recoveryNode = elem.getLeft(); + //ArrayList prods = elem.getRight(); // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); - AbstractStackNode continuer = new RecoveryPointStackNode(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); + AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); int startLocation = recoveryNode.getStartLocation(); - AbstractStackNode recoverLiteral = new SkippingStackNode(stackNodeIdDispenser.dispenseId(), WHITESPACE, input, startLocation, prod); + AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), WHITESPACE, input, startLocation, prod, recoveryNode.getDot()); recoverLiteral = recoverLiteral.getCleanCopy(startLocation); recoverLiteral.initEdges(); - EdgesSet edges = new EdgesSet(1); + EdgesSet edges = new EdgesSet<>(1); edges.add(continuer); recoverLiteral.addEdges(edges, startLocation); @@ -106,9 +114,11 @@ private static void collectUnexpandableNodes(Stack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { @@ -152,7 +162,7 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr IntegerObjectList> edges = node.getEdges(); - for (int i = edges.size() - 1; i >= 0; --i) { + for (int i = edges.size() - 1; i >= 0; --i) { // Rewind EdgesSet edgesList = edges.getValue(i); if (edgesList != null) { diff --git a/src/org/rascalmpl/values/RascalValueFactory.java b/src/org/rascalmpl/values/RascalValueFactory.java index cbfba6611b4..ad3b5f4d668 100644 --- a/src/org/rascalmpl/values/RascalValueFactory.java +++ b/src/org/rascalmpl/values/RascalValueFactory.java @@ -152,7 +152,7 @@ public class RascalValueFactory extends AbstractValueFactoryAdapter implements I public static final Type Production_Priority = tf.constructor(uptr, Production, "priority", Symbol, "def", tf.listType(Production), "choices"); public static final Type Production_Composition = tf.constructor(uptr, Production, "composition", Production, "lhs", Production, "rhs"); public static final Type Production_Associativity = tf.constructor(uptr, Production, "associativity", Symbol, "def", Associativity, "assoc", tf.setType(Production), "alternatives"); - public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped", Symbol, "def"); + public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped", Symbol, "def", Production, "prod", tf.integerType(), "dot"); /* Constructors for Attr */ public static final Type Attr_Assoc = tf.constructor(uptr, Attr, "assoc", Associativity, "assoc"); diff --git a/test/org/rascalmpl/test/parser/RecoveryTests.java b/test/org/rascalmpl/test/parser/RecoveryTests.java index 869b74bca6e..ce3030b8223 100644 --- a/test/org/rascalmpl/test/parser/RecoveryTests.java +++ b/test/org/rascalmpl/test/parser/RecoveryTests.java @@ -1,6 +1,7 @@ package org.rascalmpl.test.parser; import java.io.IOException; +import java.io.PrintWriter; import java.io.StringReader; import org.junit.Assert; @@ -12,6 +13,7 @@ import org.rascalmpl.parser.gtd.stack.LiteralStackNode; import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; import org.rascalmpl.parser.uptr.UPTRNodeFactory; +import org.rascalmpl.parser.uptr.debug.DebugLogger; import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; import org.rascalmpl.values.RascalValueFactory; import org.rascalmpl.values.ValueFactoryFactory; @@ -24,9 +26,9 @@ /** * S -> A ws B ws C - * A -> [a] ws [a] + * A -> [a] * B -> [b] ws [b] - * C -> [c] ws [c] + * C -> [c] * * ws -> [\ ] */ @@ -39,7 +41,7 @@ public class RecoveryTests extends SGTDBF private final static IConstructor SYMBOL_A = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("A")); private final static IConstructor SYMBOL_a = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("a")); private final static IConstructor SYMBOL_char_a = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(97)))); - private final static IConstructor PROD_A_a_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_A, VF.list(SYMBOL_a, SYMBOL_ws, SYMBOL_a), VF.set()); + private final static IConstructor PROD_A_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_A, VF.list(SYMBOL_a), VF.set()); private final static IConstructor SYMBOL_B = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("B")); private final static IConstructor SYMBOL_b = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("b")); @@ -49,7 +51,7 @@ public class RecoveryTests extends SGTDBF private final static IConstructor SYMBOL_C = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("C")); private final static IConstructor SYMBOL_c = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("c")); private final static IConstructor SYMBOL_char_c = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(99)))); - private final static IConstructor PROD_C_c_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_C, VF.list(SYMBOL_c, SYMBOL_ws, SYMBOL_c), VF.set()); + private final static IConstructor PROD_C_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_C, VF.list(SYMBOL_c), VF.set()); private final static IConstructor PROD_S_A_B_C = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_START_S, VF.list(SYMBOL_A, SYMBOL_ws, SYMBOL_B, SYMBOL_ws, SYMBOL_C), VF.set()); @@ -69,10 +71,8 @@ public AbstractStackNode[] S(){ } public AbstractStackNode[] A(){ - return IParserTest.createExpectArray(PROD_A_a_a, - new LiteralStackNode(6, 0, PROD_a_a, new int[]{'a'}), - new LiteralStackNode(7, 1, PROD_ws, new int[]{' '}), - new LiteralStackNode(8, 2, PROD_a_a, new int[]{'a'}) + return IParserTest.createExpectArray(PROD_A_a, + new LiteralStackNode(6, 0, PROD_a_a, new int[]{'a'}) ); } @@ -85,10 +85,8 @@ public AbstractStackNode[] B(){ } public AbstractStackNode[] C(){ - return IParserTest.createExpectArray(PROD_C_c_c, - new LiteralStackNode(12, 0, PROD_c_c, new int[]{'c'}), - new LiteralStackNode(13, 1, PROD_ws, new int[]{' '}), - new LiteralStackNode(14, 2, PROD_c_c, new int[]{'c'}) + return IParserTest.createExpectArray(PROD_C_c, + new LiteralStackNode(12, 0, PROD_c_c, new int[]{'c'}) ); } @@ -100,9 +98,10 @@ protected int getFreeStackNodeId() { } private ITree parse(String s) { + DebugLogger debugLogger = new DebugLogger(new PrintWriter(System.out)); IRecoverer recoverer = new ToNextWhitespaceRecoverer(() -> nextFreeStackNodeId++); return parse("S" /* NONTERMINAL_START_S */, null, s.toCharArray(), - new DefaultNodeFlattener(), new UPTRNodeFactory(false), recoverer, null); + new DefaultNodeFlattener(), new UPTRNodeFactory(false), recoverer, debugLogger); } private ITree toTree(String s) { @@ -115,12 +114,12 @@ private ITree toTree(String s) { @Override public ITree executeParser() { - return parse("a a b b c c"); + return parse("a b b c"); } @Override public IValue getExpectedResult() { - return toTree("appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"); + return toTree("appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"); } @Test @@ -129,9 +128,9 @@ public void testOk() { } @Test - public void testMissingCharRecovery() { - String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; - Assert.assertEquals(toTree(expected), parse("a a b c c")); + public void testIncorrectCharacter() { + String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(skipped(sort(\"B\"),prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),2),[char(120)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; + Assert.assertEquals(toTree(expected), parse("a b x c")); } } diff --git a/test/org/rascalmpl/test/parser/RecoveryTests2.java b/test/org/rascalmpl/test/parser/RecoveryTests2.java deleted file mode 100644 index 6156cdc8529..00000000000 --- a/test/org/rascalmpl/test/parser/RecoveryTests2.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.rascalmpl.test.parser; - -import java.io.IOException; -import java.io.PrintWriter; -import java.io.StringReader; - -import org.junit.Assert; -import org.junit.Test; -import org.rascalmpl.parser.gtd.SGTDBF; -import org.rascalmpl.parser.gtd.recovery.IRecoverer; -import org.rascalmpl.parser.gtd.result.out.DefaultNodeFlattener; -import org.rascalmpl.parser.gtd.stack.AbstractStackNode; -import org.rascalmpl.parser.gtd.stack.LiteralStackNode; -import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; -import org.rascalmpl.parser.uptr.UPTRNodeFactory; -import org.rascalmpl.parser.uptr.debug.DebugLogger; -import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; -import org.rascalmpl.values.RascalValueFactory; -import org.rascalmpl.values.ValueFactoryFactory; -import org.rascalmpl.values.parsetrees.ITree; - -import io.usethesource.vallang.IConstructor; -import io.usethesource.vallang.ISourceLocation; -import io.usethesource.vallang.IValue; -import io.usethesource.vallang.io.StandardTextReader; - -/** - * S -> A ws B ws C - * A -> [a] - * B -> [b] ws [b] - * C -> [c] - * - * ws -> [\ ] - */ -public class RecoveryTests2 extends SGTDBF implements IParserTest{ - private final static IConstructor SYMBOL_START_S = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("S")); - - private final static IConstructor SYMBOL_ws = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("ws")); - private final static IConstructor SYMBOL_char_space = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(32)))); - - private final static IConstructor SYMBOL_A = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("A")); - private final static IConstructor SYMBOL_a = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("a")); - private final static IConstructor SYMBOL_char_a = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(97)))); - private final static IConstructor PROD_A_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_A, VF.list(SYMBOL_a), VF.set()); - - private final static IConstructor SYMBOL_B = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("B")); - private final static IConstructor SYMBOL_b = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("b")); - private final static IConstructor SYMBOL_char_b = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(98)))); - private final static IConstructor PROD_B_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_B, VF.list(SYMBOL_b, SYMBOL_ws, SYMBOL_b), VF.set()); - - private final static IConstructor SYMBOL_C = VF.constructor(RascalValueFactory.Symbol_Sort, VF.string("C")); - private final static IConstructor SYMBOL_c = VF.constructor(RascalValueFactory.Symbol_Lit, VF.string("c")); - private final static IConstructor SYMBOL_char_c = VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Single, VF.integer(99)))); - private final static IConstructor PROD_C_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_C, VF.list(SYMBOL_c), VF.set()); - - private final static IConstructor PROD_S_A_B_C = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_START_S, VF.list(SYMBOL_A, SYMBOL_ws, SYMBOL_B, SYMBOL_ws, SYMBOL_C), VF.set()); - - private final static IConstructor PROD_ws = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_ws, VF.list(SYMBOL_char_space), VF.set()); - private final static IConstructor PROD_a_a = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_a, VF.list(SYMBOL_char_a), VF.set()); - private final static IConstructor PROD_b_b = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_b, VF.list(SYMBOL_char_b), VF.set()); - private final static IConstructor PROD_c_c = VF.constructor(RascalValueFactory.Production_Default, SYMBOL_c, VF.list(SYMBOL_char_c), VF.set()); - - public AbstractStackNode[] S(){ - return IParserTest.createExpectArray(PROD_S_A_B_C, - new NonTerminalStackNode(1, 0, "A"), - new LiteralStackNode(2, 1, PROD_ws, new int[]{' '}), - new NonTerminalStackNode(3, 2, "B"), - new LiteralStackNode(4, 3, PROD_ws, new int[]{' '}), - new NonTerminalStackNode(5, 4, "C") - ); - } - - public AbstractStackNode[] A(){ - return IParserTest.createExpectArray(PROD_A_a, - new LiteralStackNode(6, 0, PROD_a_a, new int[]{'a'}) - ); - } - - public AbstractStackNode[] B(){ - return IParserTest.createExpectArray(PROD_B_b_b, - new LiteralStackNode(9, 0, PROD_b_b, new int[]{'b'}), - new LiteralStackNode(10, 1, PROD_ws, new int[]{' '}), - new LiteralStackNode(11, 2, PROD_b_b, new int[]{'b'}) - ); - } - - public AbstractStackNode[] C(){ - return IParserTest.createExpectArray(PROD_C_c, - new LiteralStackNode(12, 0, PROD_c_c, new int[]{'c'}) - ); - } - - private int nextFreeStackNodeId = 100; - - @Override - protected int getFreeStackNodeId() { - return nextFreeStackNodeId++; - } - - private ITree parse(String s) { - DebugLogger debugLogger = new DebugLogger(new PrintWriter(System.out)); - IRecoverer recoverer = new ToNextWhitespaceRecoverer(() -> nextFreeStackNodeId++); - return parse("S" /* NONTERMINAL_START_S */, null, s.toCharArray(), - new DefaultNodeFlattener(), new UPTRNodeFactory(false), recoverer, debugLogger); - } - - private ITree parseB(String s) { - return parse("B", null, s.toCharArray(), - new DefaultNodeFlattener(), new UPTRNodeFactory(false), null, null); - } - - - private ITree toTree(String s) { - try { - return (ITree) new StandardTextReader().read(ValueFactoryFactory.getValueFactory(), RascalValueFactory.uptr, RascalValueFactory.Tree, new StringReader(s)); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public ITree executeParser() { - return parse("a b b c"); - } - - @Override - public IValue getExpectedResult() { - return toTree("appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"); - } - - @Test - public void testOk() { - Assert.assertEquals(getExpectedResult(), executeParser()); - } - - @Test - public void testB() { - String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\"),lit(\"ws\"),lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\"),lit(\"ws\"),lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; - Assert.assertEquals(toTree(expected), parseB("b")); - } - - @Test - public void testMissingCharRecovery() { - String expected = "appl(prod(sort(\"S\"),[sort(\"A\"),lit(\"ws\"),sort(\"B\"),lit(\"ws\"),sort(\"C\")],{}),[appl(prod(sort(\"A\"),[lit(\"a\")],{}),[appl(prod(lit(\"a\"),[\\char-class([single(97)])],{}),[char(97)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"B\"),[lit(\"b\"),lit(\"ws\"),lit(\"b\")],{}),[appl(prod(lit(\"b\"),[\\char-class([single(98)])],{}),[char(98)]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(skipped(\\char-class([range(1,1114111)])),[char(120)])]),appl(prod(lit(\"ws\"),[\\char-class([single(32)])],{}),[char(32)]),appl(prod(sort(\"C\"),[lit(\"c\")],{}),[appl(prod(lit(\"c\"),[\\char-class([single(99)])],{}),[char(99)])])])"; - Assert.assertEquals(toTree(expected), parse("a b x c")); - } - -} diff --git a/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java b/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java new file mode 100644 index 00000000000..abf446cf843 --- /dev/null +++ b/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java @@ -0,0 +1,11 @@ +package org.rascalmpl.test.recovery; + +import org.junit.runner.RunWith; +import org.rascalmpl.test.infrastructure.RascalJUnitTestPrefix; +import org.rascalmpl.test.infrastructure.RascalJUnitTestRunner; + +@RunWith(RascalJUnitTestRunner.class) +@RascalJUnitTestPrefix("lang::rascal::tests::recovery") +public class ErrorRecoveryModules { + +} From 88cddc347c2d8b0d7f0623b3770b6068517eba2d Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 10 Jul 2024 12:21:28 +0200 Subject: [PATCH 32/87] Added some methods to analyze recovered parse errors in parse trees --- src/org/rascalmpl/library/ParseTree.rsc | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 733680858ed..010655cf502 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -189,7 +189,7 @@ data Production | \reference(Symbol def, str cons) // <5> ; -data Production = skipped(Symbol def, Production prod, int validPrefix); +data Production = skipped(Symbol def, Production prod, int dot); @synopsis{Attributes in productions.} @description{ @@ -766,3 +766,25 @@ bool isNonTerminalType(Symbol::\parameterized-sort(str _, list[Symbol] _)) = tru bool isNonTerminalType(Symbol::\parameterized-lex(str _, list[Symbol] _)) = true; bool isNonTerminalType(Symbol::\start(Symbol s)) = isNonTerminalType(s); default bool isNonTerminalType(Symbol s) = false; + +@synopsis{Check if a parse tree contains any skipped nodes, the result of error recovery.} +bool hasErrors(Tree tree) = /appl(skipped(_, _, _), _) := tree; + +@synopsis{Find all skipped nodes in a parse tree.} +list[Tree] findAllErrors(Tree tree) { + return for (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) append(error); +} + +@synopsis{Find the first skipped node in a parse tree.} +Tree findFirstError(Tree tree) { + if (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) return error; + fail; +} + +Symbol getErrorSymbol(appl(skipped(Symbol symbol, _, _), _)) = symbol; + +Production getErrorProduction(appl(skipped(_, Production production, _), _)) = production; + +int getErrorPosition(appl(skipped(_, _, int dot), _)) = dot; + +str getErrorString(appl(skipped(_, _, _), list chars)) = ""; From 6d192810a989ab1589398b444c0248b241ca80b2 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 10 Jul 2024 12:26:26 +0200 Subject: [PATCH 33/87] Added comment and removed unused import --- .../parser/uptr/recovery/ToNextWhitespaceRecoverer.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 97c4941ca4c..24566530f12 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -12,7 +12,6 @@ *******************************************************************************/ package org.rascalmpl.parser.uptr.recovery; -import org.apache.commons.lang3.tuple.Pair; import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; @@ -79,9 +78,8 @@ private DoubleArrayList, AbstractNode> reviveNod AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); int startLocation = recoveryNode.getStartLocation(); - AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), WHITESPACE, input, startLocation, prod, recoveryNode.getDot()); - recoverLiteral = recoverLiteral.getCleanCopy(startLocation); + recoverLiteral = recoverLiteral.getCleanCopy(startLocation); // This copy might not be needed recoverLiteral.initEdges(); EdgesSet edges = new EdgesSet<>(1); edges.add(continuer); From bcd59147860023708de6e39e556346d7c6d7c288 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 10 Jul 2024 12:26:59 +0200 Subject: [PATCH 34/87] Enabled logging when error recovery is in effect --- src/org/rascalmpl/values/RascalFunctionValueFactory.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 8278d4dce93..0a4cffc46a5 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -13,6 +13,7 @@ package org.rascalmpl.values; import java.io.IOException; +import java.io.PrintWriter; import java.io.Reader; import java.lang.reflect.InvocationTargetException; import java.util.Arrays; @@ -41,6 +42,7 @@ import org.rascalmpl.library.lang.rascal.syntax.RascalParser; import org.rascalmpl.parser.ParserGenerator; import org.rascalmpl.parser.gtd.IGTD; +import org.rascalmpl.parser.gtd.debug.IDebugListener; import org.rascalmpl.parser.gtd.exception.ParseError; import org.rascalmpl.parser.gtd.exception.UndeclaredNonTerminalException; import org.rascalmpl.parser.gtd.io.InputConverter; @@ -51,6 +53,7 @@ import org.rascalmpl.parser.uptr.UPTRNodeFactory; import org.rascalmpl.parser.uptr.action.NoActionExecutor; import org.rascalmpl.parser.uptr.action.RascalFunctionActionExecutor; +import org.rascalmpl.parser.uptr.debug.DebugLogger; import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; import org.rascalmpl.types.NonTerminalType; import org.rascalmpl.types.RascalTypeFactory; @@ -575,10 +578,12 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IActionExecutor exec = filters.isEmpty() ? new NoActionExecutor() : new RascalFunctionActionExecutor(filters, !hasSideEffects); IGTD parserInstance = getParser(); IRecoverer recoverer = null; + IDebugListener debugListener = null; if (allowRecovery) { recoverer = new ToNextWhitespaceRecoverer(new StackNodeIdDispenser(parserInstance)); + debugListener = new DebugLogger(new PrintWriter(System.out, true)); } - return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer); + return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); } } From fb6ffe56484f43111c04fb2b387f04ce40ba4965 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 10 Jul 2024 12:39:15 +0200 Subject: [PATCH 35/87] Made printing of productions more robust (it crashed on 'regular' productions) --- src/org/rascalmpl/parser/util/DebugUtil.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index 8f0391caafa..a99a0fa4606 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -1,7 +1,5 @@ package org.rascalmpl.parser.util; -import java.util.Iterator; - import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IList; import io.usethesource.vallang.IValue; @@ -18,12 +16,18 @@ public static String prodToString(IConstructor prod) { builder.append(" ->"); - IList children = (IList) prod.get(1); - for (IValue child : children) { + if (prod.getName() == "prod") { + IList children = (IList) prod.get(1); + for (IValue child : children) { + builder.append(" "); + IConstructor conChild = (IConstructor) child; + builder.append(stripQuotes(String.valueOf((conChild).get(0)))); + } + } else { builder.append(" "); - IConstructor conChild = (IConstructor) child; - builder.append(stripQuotes(String.valueOf((conChild).get(0)))); + builder.append(prod.toString()); } + builder.append("'"); return builder.toString(); From 27f1afce2e41adb79c78a92e6d48c8903cad1957 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 10 Jul 2024 12:40:38 +0200 Subject: [PATCH 36/87] Added some basic recovery tests --- .../tests/recovery/BasicRecoveryTests.rsc | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index de6194dcc99..9370fe68cdb 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -6,13 +6,15 @@ import IO; layout Layout = [\ ]* !>> [\ ]; -syntax S = A B C; -syntax A = "a"; -syntax B = "b" "b"; -syntax C = "c"; +syntax S = A End; +syntax A = "1" "2" "3"; +syntax End = "$"; -test bool parseTest() { - Tree t = parse(#S, "a b x c", allowRecovery=true); - iprintln(t); - return true; +test bool parseOk() { + return !hasErrors(parse(#S, "1 2 3 $", allowRecovery=true)); +} + +test bool simpleRecovery() { + Tree t = parse(#S, "1 2 x $", allowRecovery=true); + return hasErrors(t) && size(findAllErrors(t)) == 1; } From 15fedbf07a6acdfe15f4e95805b9691f065c3b80 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 18 Jul 2024 08:50:57 +0200 Subject: [PATCH 37/87] Implemented parser datastructure visualization using graphviz --- .../tests/recovery/BasicRecoveryTests.rsc | 26 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 136 ++++- .../parser/gtd/stack/AbstractStackNode.java | 11 +- .../gtd/stack/AlternativeStackNode.java | 2 +- .../parser/gtd/stack/CharStackNode.java | 9 +- .../parser/gtd/stack/LiteralStackNode.java | 5 + .../recovery/ToNextWhitespaceRecoverer.java | 7 +- .../util/visualize/DebugVisualizer.java | 521 ++++++++++++++++++ .../visualize/dot/AttributeStatement.java | 28 + .../util/visualize/dot/CompassPoint.java | 5 + .../util/visualize/dot/DotAttribute.java | 104 ++++ .../rascalmpl/util/visualize/dot/DotEdge.java | 66 +++ .../util/visualize/dot/DotField.java | 23 + .../util/visualize/dot/DotGraph.java | 107 ++++ .../rascalmpl/util/visualize/dot/DotNode.java | 69 +++ .../util/visualize/dot/DotRecord.java | 37 ++ .../util/visualize/dot/DotRecordEntry.java | 7 + .../util/visualize/dot/DotStatement.java | 7 + .../util/visualize/dot/DotSubgraph.java | 40 ++ .../rascalmpl/util/visualize/dot/NodeId.java | 126 +++++ 20 files changed, 1300 insertions(+), 36 deletions(-) create mode 100644 src/org/rascalmpl/util/visualize/DebugVisualizer.java create mode 100644 src/org/rascalmpl/util/visualize/dot/AttributeStatement.java create mode 100644 src/org/rascalmpl/util/visualize/dot/CompassPoint.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotAttribute.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotEdge.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotField.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotGraph.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotNode.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotRecord.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotStatement.java create mode 100644 src/org/rascalmpl/util/visualize/dot/DotSubgraph.java create mode 100644 src/org/rascalmpl/util/visualize/dot/NodeId.java diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index 9370fe68cdb..ec7ff7270a1 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -1,20 +1,28 @@ module lang::rascal::tests::recovery::BasicRecoveryTests - import ParseTree; -import IO; -layout Layout = [\ ]* !>> [\ ]; +layout Layout = [\ ]; //* !>> [\ ]; -syntax S = A End; -syntax A = "1" "2" "3"; +syntax S = ABC End; +syntax ABC = "a" "b" "c"; syntax End = "$"; -test bool parseOk() { - return !hasErrors(parse(#S, "1 2 3 $", allowRecovery=true)); +test bool ok() { + return !hasErrors(parse(#S, "a b c $", allowRecovery=true)); +} + +test bool abx() { + Tree t = parse(#S, "a b x $", allowRecovery=true); + return hasErrors(t) && size(findAllErrors(t)) == 1; +} + +test bool axc() { + Tree t = parse(#S, "a x c $", allowRecovery=true); + return hasErrors(t) && size(findAllErrors(t)) == 1; } -test bool simpleRecovery() { - Tree t = parse(#S, "1 2 x $", allowRecovery=true); +test bool ax() { + Tree t = parse(#S, "a x $", allowRecovery=true); return hasErrors(t) && size(findAllErrors(t)) == 1; } diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index f7a4ddb5588..b7f7aaa7a5d 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -34,6 +34,7 @@ import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.EpsilonStackNode; import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; +import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.stack.filter.ICompletionFilter; import org.rascalmpl.parser.gtd.stack.filter.IEnterFilter; @@ -45,6 +46,8 @@ import org.rascalmpl.parser.gtd.util.IntegerList; import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.util.visualize.DebugVisualizer; +import org.rascalmpl.util.visualize.dot.NodeId; /** * This is the core of the parser; it drives the parse process. @@ -122,14 +125,16 @@ public abstract class SGTDBF implements IGTD{ // Debugging private IDebugListener

debugListener; + private DebugVisualizer visualizer; // Temporary instrumentation for accurate profiling private long timestamp; private boolean printTimes = false; + public SGTDBF(){ super(); - + positionStore = new PositionStore(); stacksToExpand = new Stack>(); @@ -149,6 +154,8 @@ public SGTDBF(){ unmatchableLeafNodes = new Stack>(); unmatchableMidProductionNodes = new DoubleStack, AbstractNode>, AbstractStackNode

>(); filteredNodes = new DoubleStack, AbstractNode>(); + + visualizer = new DebugVisualizer("Parser"); } /** @@ -758,7 +765,9 @@ private void moveToNext(AbstractStackNode

node, AbstractNode result){ */ private void move(AbstractStackNode

node, AbstractNode result){ if(debugListener != null) debugListener.moving(node, result); - + if (node instanceof RecoveryPointStackNode) { + opportunityToBreak(); + } // Handle filtering. ICompletionFilter[] completionFilters = node.getCompletionFilters(); if(completionFilters != null){ @@ -790,13 +799,17 @@ private void move(AbstractStackNode

node, AbstractNode result){ /** * Initiate the handling of stacks. */ - private void reduce(){ + private void reduceTerminals() { // Reduce terminals + visualize("Reducing terminals", DebugVisualizer.TERMINALS_TO_REDUCE_ID); while(!stacksWithTerminalsToReduce.isEmpty()){ move(stacksWithTerminalsToReduce.peekFirst(), stacksWithTerminalsToReduce.popSecond()); } - + } + + private void reduceNonTerminals() { // Reduce non-terminals + visualize("Reducing non-terminals", DebugVisualizer.NON_TERMINALS_TO_REDUCE_ID); while(!stacksWithNonTerminalsToReduce.isEmpty()){ move(stacksWithNonTerminalsToReduce.peekFirst(), stacksWithNonTerminalsToReduce.popSecond()); } @@ -822,6 +835,7 @@ private boolean findFirstStacksToReduce(){ if (recoverer != null) { if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); } DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { @@ -845,12 +859,18 @@ private boolean findFirstStacksToReduce(){ * Locates the set of stacks that is queued for handling, for which the least amount of characters needs to be shifted. */ private boolean findStacksToReduce(){ + visualize("Finding stacks to reduce", DebugVisualizer.TODO_LISTS_ID); int queueDepth = todoLists.length; for(int i = 1; i < queueDepth; ++i){ queueIndex = (queueIndex + 1) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; if(!(terminalsTodo == null || terminalsTodo.isEmpty())){ + if (debugListener != null) { + NodeId reduceNodeId = new NodeId("todo-" + i); + visualize("Found stack to reduce", reduceNodeId); + } + stacksWithTerminalsToReduce = terminalsTodo; location += i; @@ -861,19 +881,29 @@ private boolean findStacksToReduce(){ if (recoverer != null && location < input.length) { if (debugListener != null) { + visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); + // Visualize state and include recovered nodes + visualizer.createGraph(this, "Reviving"); + visualizer.addRecoveredNodes(recoveredNodes); + visualizer.writeGraph(); + opportunityToBreak(); } if (recoveredNodes.size() > 0) { // was: for (int i = recoveredNodes.size()-1; i>= 0; i--) { for (int i = 0; i < recoveredNodes.size(); i++) { - AbstractStackNode

recovered = recoveredNodes.getFirst(i); + AbstractStackNode

recovered = recoveredNodes.getFirst(i); // int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation()); + if (debugListener != null) { + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + visualize("Queue recovery node", DebugVisualizer.getNodeId(recovered)); + } queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), recoveredNodes.getSecond(i)); } return findStacksToReduce(); @@ -884,7 +914,7 @@ private boolean findStacksToReduce(){ return false; } - + public boolean parseErrorHasOccurred(){ return parseErrorOccured; } @@ -1201,6 +1231,7 @@ private void expandStack(AbstractStackNode

stack){ * Initiate stack expansion for all queued stacks. */ private void expand(){ + visualize("Expanding", DebugVisualizer.STACKS_TO_EXPAND_ID); while(!stacksToExpand.isEmpty()){ expandStack(stacksToExpand.pop()); } @@ -1244,14 +1275,16 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] stacksToExpand.push(rootNode); lookAheadChar = (input.length > 0) ? input[0] : 0; - if(debugListener != null) debugListener.shifting(location, input, positionStore); + if(debugListener != null) { + debugListener.shifting(location, input, positionStore); + } expand(); if(findFirstStacksToReduce()){ boolean shiftedLevel = (location != 0); - do{ + do { lookAheadChar = (location < input.length) ? input[location] : 0; if(shiftedLevel){ // Nullable fix for the first level. sharedNextNodes.clear(); @@ -1266,18 +1299,22 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] } // Reduce-expand loop. - do{ + do { if(debugListener != null) debugListener.iterating(); + + reduceTerminals(); - reduce(); + reduceNonTerminals(); expand(); - }while(!stacksWithNonTerminalsToReduce.isEmpty() || !stacksWithTerminalsToReduce.isEmpty()); + } while(!stacksWithNonTerminalsToReduce.isEmpty() || !stacksWithTerminalsToReduce.isEmpty()); shiftedLevel = true; - }while(findStacksToReduce()); + } while(findStacksToReduce()); } + visualize("Done", DebugVisualizer.PARSER_ID); + // Check if we were successful. if(location == input.length){ EdgesSet

startNodeEdgesSet = startNode.getIncomingEdges(); @@ -1424,4 +1461,79 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo checkTime("Unbinarizing, post-parse filtering, and mapping to UPTR"); } } + + /** + * Datastructure visualization for debugging purposes + */ + + private void visualize(String step, NodeId highlight) { + // Only visualize when debugging + if (debugListener != null) { + visualizer.createGraph(this, step); + if (highlight != null) { + visualizer.highlight(highlight); + } + visualizer.writeGraph(); + + opportunityToBreak(); + } + } + + private void opportunityToBreak() { + } + + + /** + * Getters used for graph generation/debugging + */ + + public int[] getInput() { + return input; + } + + public int getLocation() { + return location; + } + + public int getLookAheadChar() { + return lookAheadChar; + } + + public DoubleStack, AbstractNode>[] getTodoLists() { + return todoLists; + } + + public int getQueueIndex() { + return queueIndex; + } + + public Stack> getStacksToExpand() { + return stacksToExpand; + } + + public DoubleStack, AbstractContainerNode

> getStacksWithNonTerminalsToReduce() { + return stacksWithNonTerminalsToReduce; + } + + public DoubleStack, AbstractNode> getStacksWithTerminalsToReduce() { + return stacksWithTerminalsToReduce; + } + + public Stack> getUnexpandableNodes() { + return unexpandableNodes; + } + + public Stack> getUnmatchableLeafNodes() { + return unmatchableLeafNodes; + } + + public DoubleStack, AbstractNode>, AbstractStackNode

> getUnmatchableMidProductionNodes() { + return unmatchableMidProductionNodes; + } + + public DoubleStack, AbstractNode> getFilteredNodes() { + return filteredNodes; + } + + } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index 005b1e99455..e431402f300 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -35,7 +35,7 @@ public abstract class AbstractStackNode

{ protected AbstractStackNode

[][] alternateProductions; // Our edges - protected IntegerObjectList> edgesMap; // : key=startLocation, value=EdgesSet a that location + protected IntegerObjectList> edgesMap; // : key=startLocation, value=EdgesSet at that location // Edges of our children protected ArrayList[] prefixesMap; @@ -187,7 +187,7 @@ public boolean isExpandable(){ * Returns the name associated with the symbol in this node (optional operation). */ public abstract String getName(); - + /** * Check whether of this this node is equal to the given node. */ @@ -745,7 +745,7 @@ public IntegerList getPropagatedReductions(){ } public String toShortString() { - return "id=" + id + ",dot=" + dot + ",start=" + startLocation; + return "." + dot + "@" + startLocation; } @Override @@ -765,7 +765,7 @@ public String toString() { } else { builder.append(","); } - builder.append(prodElem.getId()); + builder.append(prodElem.toShortString()); } builder.append("]"); } @@ -782,9 +782,10 @@ public String toString() { if (alternateProductions != null && alternateProductions.length != 0) { builder.append(",alternateProductions=" + Arrays.toString(alternateProductions)); } + /* if (edgesMap != null && edgesMap.size() != 0) { builder.append(",edges=" + edgesMap); - } + }*/ if (prefixesMap != null && prefixesMap.length != 0) { builder.append(",prefixes=" + Arrays.toString(prefixesMap)); } diff --git a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java index 041e9273ce9..6412f806d25 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java @@ -47,7 +47,7 @@ private AlternativeStackNode(AlternativeStackNode

original, int startLocation children = original.children; } - + /** * Generates and initializes the alternatives for this alternative. */ diff --git a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java index ff2d8b2e326..94274125fc2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java @@ -89,7 +89,7 @@ public AbstractNode getResult(){ public String toString(){ StringBuilder sb = new StringBuilder(); - sb.append('['); + sb.append("CharStackNode[class="); int[] range = ranges[0]; sb.append(range[0]); sb.append('-'); @@ -101,13 +101,10 @@ public String toString(){ sb.append('-'); sb.append(range[1]); } + sb.append(","); + sb.append(super.toString()); sb.append(']'); - sb.append(getId()); - sb.append('('); - sb.append(startLocation); - sb.append(')'); - return sb.toString(); } diff --git a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java index cd0db6bf2f2..f83bd73703b 100644 --- a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java @@ -77,6 +77,11 @@ public int getLength(){ public AbstractNode getResult(){ return result; } + + @Override + public String toShortString() { + return "'" + UnicodeConverter.unicodeArrayToString(literal) + "'"; + } public String toString(){ StringBuilder sb = new StringBuilder("lit['"); diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 24566530f12..65add6461bf 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -25,6 +25,7 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; @@ -67,9 +68,6 @@ private DoubleArrayList, AbstractNode> reviveNod for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); - //Pair, ArrayList> elem = elems.get(i); - //AbstractStackNode recoveryNode = elem.getLeft(); - //ArrayList prods = elem.getRight(); // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). for (int j = prods.size() - 1; j >= 0; --j) { @@ -183,10 +181,12 @@ private void collectProductions(AbstractStackNode node, ArrayList< int dot = node.getDot(); + System.err.println("collect productions for node: " + node); if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)){ productions.add(parentProduction); + System.err.println("adding production: " + parentProduction); if (ProductionAdapter.isList(parentProduction)) { return; // Don't follow productions in lists productions, since they are 'cyclic'. @@ -200,6 +200,7 @@ private void collectProductions(AbstractStackNode node, ArrayList< IConstructor parentProduction = currentNode.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)) { productions.add(parentProduction); + System.err.println("adding production at " + i + ": " + parentProduction); } } diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java new file mode 100644 index 00000000000..b83f12976d0 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -0,0 +1,521 @@ +package org.rascalmpl.util.visualize; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.Consumer; + +import org.apache.commons.io.FileUtils; +import org.rascalmpl.parser.gtd.SGTDBF; +import org.rascalmpl.parser.gtd.result.AbstractNode; +import org.rascalmpl.parser.gtd.result.CharNode; +import org.rascalmpl.parser.gtd.result.EpsilonNode; +import org.rascalmpl.parser.gtd.result.LiteralNode; +import org.rascalmpl.parser.gtd.result.RecoveredNode; +import org.rascalmpl.parser.gtd.result.SkippedNode; +import org.rascalmpl.parser.gtd.result.SortContainerNode; +import org.rascalmpl.parser.gtd.result.struct.Link; +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; +import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; +import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; +import org.rascalmpl.parser.gtd.util.DoubleArrayList; +import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.IntegerObjectList; +import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.parser.util.DebugUtil; +import org.rascalmpl.unicode.UnicodeConverter; +import org.rascalmpl.util.visualize.dot.CompassPoint; +import org.rascalmpl.util.visualize.dot.DotAttribute; +import org.rascalmpl.util.visualize.dot.DotEdge; +import org.rascalmpl.util.visualize.dot.DotField; +import org.rascalmpl.util.visualize.dot.DotGraph; +import org.rascalmpl.util.visualize.dot.DotNode; +import org.rascalmpl.util.visualize.dot.DotRecord; +import org.rascalmpl.util.visualize.dot.NodeId; + +import io.usethesource.vallang.IConstructor; + +public class DebugVisualizer { + static final String BASE_DIR = "D:/debug/parser-traces/docs/"; + private static final boolean VISUALIZATION_ENABLED = true; + + public static final NodeId PARSER_ID = new NodeId("Parser"); + public static final NodeId TODO_LISTS_ID= new NodeId("todoLists"); + public static final NodeId STACKS_TO_EXPAND_ID = new NodeId("stacksToExpand"); + public static final NodeId TERMINALS_TO_REDUCE_ID = new NodeId("terminalsToReduce"); + public static final NodeId NON_TERMINALS_TO_REDUCE_ID = new NodeId("nonTerminalsToReduce"); + + public static final NodeId ERROR_TRACKING_ID = new NodeId("error"); + public static final NodeId UNEXPANDABLE_NODES_ID = new NodeId("unexpandableNodes"); + public static final NodeId UNMATCHABLE_LEAF_NODES_ID = new NodeId("unmatchableLeafNodes"); + public static final NodeId UNMATCHABLE_MID_PRODUCTION_NODES_ID = new NodeId("unmatchableMidProductionNodes"); + public static final NodeId FILTERED_NODES_ID = new NodeId("filteredNodes"); + + private static final NodeId RECOVERED_NODES_ID = new NodeId("recoveredNodes"); + + /*static public class GraphObject { + static public class Kind { + private boolean dotGraph = true; + } + + private Kind kind = new Kind(); + private String text = "\ndigraph G {\n a -> b; \n}\n"; + }*/ + + private static class StreamGobbler implements Runnable { + private InputStream inputStream; + private Consumer consumer; + + public StreamGobbler(InputStream inputStream, Consumer consumer) { + this.inputStream = inputStream; + this.consumer = consumer; + } + + @Override + public void run() { + new BufferedReader(new InputStreamReader(inputStream)).lines() + .forEach(consumer); + } + } + + private String name; + private Map stackNodeNodes; + private DotGraph graph; + private int frame; + + public DebugVisualizer(String name) { + this.name = name; + stackNodeNodes = new HashMap<>(); + File frameDir = new File(BASE_DIR + "/frames/" + name); + if (frameDir.exists()) { + try { + FileUtils.deleteDirectory(frameDir); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + frameDir.mkdirs(); + } + + public void visualize(AbstractStackNode node) { + if (VISUALIZATION_ENABLED) { + writeGraph(createGraph(node)); + } + } + + public int getFrame() { + return frame; + } + + private void reset() { + stackNodeNodes.clear(); + graph = null; + frame++; + } + + public void highlight(NodeId id) { + if (graph != null) { + graph.highlight(id); + } + } + + public void highlightStack(AbstractStackNode stack) { + if (graph != null) { + DotNode dotNode = stackNodeNodes.get(stack.getId()); + highlight(dotNode.getId()); + } + } + + private synchronized DotGraph createGraph(AbstractStackNode stackNode) { + reset(); + graph = new DotGraph(name, true); + addStack(graph, stackNode); + return graph; + } + + public synchronized

void addRecoveredNodes(DoubleArrayList, AbstractNode> recoveredNodes) { + addStackAndNodeDoubleList(graph, RECOVERED_NODES_ID, recoveredNodes); + graph.addEdge(ERROR_TRACKING_ID, RECOVERED_NODES_ID, "Nodes to revive"); + highlight(RECOVERED_NODES_ID); + } + + private

DotNode addStack(DotGraph graph, AbstractStackNode

stackNode) { + DotNode node = stackNodeNodes.get(stackNode.getId()); + if (node != null) { + return node; + } + + node = createDotNode(stackNode); + + stackNodeNodes.put(stackNode.getId(), node); + + graph.addNode(node); + + IntegerObjectList> edges = stackNode.getEdges(); + if (edges != null) { + for (int i = edges.size() - 1; i >= 0; --i) { + EdgesSet

edgesList = edges.getValue(i); + if (edgesList != null) { + for (int j = edgesList.size() - 1; j >= 0; --j) { + AbstractStackNode

parentStackNode = edgesList.get(j); + DotNode parentDotNode = addStack(graph, parentStackNode); + graph.addEdge(node.getId(), parentDotNode.getId()); + } + } + } + } + + return node; + } + + private

DotNode createDotNode(AbstractStackNode

stackNode) { + String type = stackNode.getClass().getSimpleName(); + if (type.endsWith("StackNode")) { + type = type.substring(0, type.length() - "StackNode".length()); + } + + String nodeName; + + if (stackNode instanceof RecoveryPointStackNode) { + RecoveryPointStackNode

recoveryNode = (RecoveryPointStackNode

) stackNode; + nodeName = String.valueOf(recoveryNode.getId()); + } else { + try { + nodeName = stackNode.getName(); + } catch (UnsupportedOperationException e) { + nodeName = ""; + } + + if (nodeName.startsWith("layouts_")) { + nodeName = nodeName.substring("layouts_".length()); + } + } + + DotNode node = new DotNode(getNodeId(stackNode)); + String label = String.format("%s: %s\n.%d@%d", + type, nodeName, stackNode.getDot(), stackNode.getStartLocation()); + P parentProduction = stackNode.getParentProduction(); + if (parentProduction instanceof IConstructor) { + label += "\nin: " + DebugUtil.prodToString((IConstructor) parentProduction); + } + node.addAttribute(DotAttribute.ATTR_LABEL, label); + + // TODO: add prefixes + + return node; + } + + private void addParserNode(DotGraph graph, AbstractNode parserNode) { + NodeId id = getNodeId(parserNode); + DotNode dotNode = new DotNode(id); + dotNode.addAttribute(DotAttribute.ATTR_NODE_SHAPE, "octagon"); + + String nodeName = parserNode.getClass().getSimpleName(); + if (nodeName.endsWith("Node")) { + nodeName = nodeName.substring(0, nodeName.length() - "Node".length()); + } + + dotNode.addAttribute(DotAttribute.ATTR_LABEL, nodeName); + + switch (parserNode.getTypeIdentifier()) { + case EpsilonNode.ID: + break; + case CharNode.ID: + enrichCharNode(dotNode, (CharNode) parserNode); + break; + case LiteralNode.ID: + enrichLiteralNode(dotNode, (LiteralNode) parserNode); + break; + case SortContainerNode.ID: + case RecoveredNode.ID: + enrichSortContainerNode(dotNode, (SortContainerNode) parserNode); + break; + case SkippedNode.ID: + enrichSkippedNode(dotNode, (SkippedNode) parserNode); + break; + default: + enrichUnknownParserNode(dotNode, parserNode); + break; + } + + graph.addNode(dotNode); + } + + private void enrichCharNode(DotNode dotNode, CharNode charNode) { + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); + int c = charNode.getCharacter(); + label += "\nchar=" + c + "('" + (char) c + "')"; + dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); + } + + private void enrichLiteralNode(DotNode dotNode, LiteralNode literalNode) { + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); + int[] content = literalNode.getContent(); + label += " \"" + UnicodeConverter.unicodeArrayToString(content) + "\""; + /* Maybe include production? + label += "\nprod=" + DebugUtil.prodToString((IConstructor)literalNode.getProduction()); + */ + dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); + } + + private void enrichSkippedNode(DotNode dotNode, SkippedNode skippedNode) { + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); + label += "\n." + skippedNode.getDot() + "@" + skippedNode.getOffset() + ": " + " \"" + UnicodeConverter.unicodeArrayToString(skippedNode.getSkippedChars()) + "\""; + label += "\nin: " + DebugUtil.prodToString((IConstructor) skippedNode.getProduction()); + + dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); + } + + private void enrichSortContainerNode(DotNode dotNode, SortContainerNode sortNode) { + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); + label += " " + sortNode.getOffset() + "-" + sortNode.getEndOffset(); + label += "\n" + DebugUtil.prodToString(sortNode.getFirstProduction()); + dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); + + // TODO: add links + } + + private void enrichUnknownParserNode(DotNode dotNode, AbstractNode parserNode) { + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); + label += "\ntype=" + parserNode.getTypeIdentifier(); + dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); + } + + public static

NodeId getNodeId(AbstractStackNode

stackNode) { + return new NodeId(String.valueOf(stackNode.getId())); + } + + private static NodeId getNodeId(Object node) { + return new NodeId(String.valueOf(System.identityHashCode(node))); + } + + public void writeGraph() { + if (graph != null) { + writeGraph(graph); + } + } + + public void createGraph(SGTDBF parser, String step) { + if (!VISUALIZATION_ENABLED) { + return; + } + reset(); + + graph = new DotGraph(name, true); + + int location = parser.getLocation(); + + DotNode parserNode = new DotNode(PARSER_ID); + + String input = UnicodeConverter.unicodeArrayToString(parser.getInput()); + + char lookahead = (char) parser.getLookAheadChar(); + if (lookahead == '\0') { + lookahead = '$'; + } + + String label = String.format("Parser\nInput: \"%s\"\nLocation: %d ('%c')\nStep %d: %s", + input, location, lookahead, frame, step); + parserNode.setAttribute(DotAttribute.ATTR_LABEL, label); + graph.addNode(parserNode); + + addTodoLists(parser, graph); + addStacksToExpand(parser, graph); + addTerminalsToReduce(parser, graph); + addNonTerminalsToReduce(parser, graph); + + addErrorNodes(parser, graph); + } + + private void addTodoLists(SGTDBF parser, DotGraph graph) { + DoubleStack, AbstractNode>[] todoLists = parser.getTodoLists(); + int start = parser.getQueueIndex(); + + DotNode todoListsNode = DotNode.createArrayNode(TODO_LISTS_ID, todoLists.length); + + for (int tokenLength=1; tokenLength<=todoLists.length; tokenLength++) { + int index = (start + tokenLength - 1) % todoLists.length; + DoubleStack, AbstractNode> todoList = todoLists[index]; + if (todoList != null && !todoList.isEmpty()) { + NodeId todoListId = new NodeId("todo-" + tokenLength); + addStackAndNodeDoubleStack(graph, todoListId, todoList); + graph.addEdge(DotEdge.createArrayEdge(TODO_LISTS_ID, tokenLength, todoListId)); + } + } + + graph.addNode(todoListsNode); + graph.addEdge(PARSER_ID, TODO_LISTS_ID, "todo lists"); + } + + private void addStacksToExpand(SGTDBF parser, DotGraph graph) { + Stack> stacksToExpand = parser.getStacksToExpand(); + addStackNodeStack(graph, STACKS_TO_EXPAND_ID, stacksToExpand); + graph.addEdge(PARSER_ID, STACKS_TO_EXPAND_ID, "stacks to expand"); + } + + private void addTerminalsToReduce(SGTDBF parser, DotGraph graph) { + addStackAndNodeDoubleStack(graph, TERMINALS_TO_REDUCE_ID, parser.getStacksWithTerminalsToReduce()); + graph.addEdge(PARSER_ID, TERMINALS_TO_REDUCE_ID, "terminals to reduce"); + } + + private void addNonTerminalsToReduce(SGTDBF parser, DotGraph graph) { + addStackAndNodeDoubleStack(graph, NON_TERMINALS_TO_REDUCE_ID, parser.getStacksWithNonTerminalsToReduce()); + graph.addEdge(PARSER_ID, NON_TERMINALS_TO_REDUCE_ID, "non-terminals to reduce"); + } + + private void addErrorNodes(SGTDBF parser, DotGraph graph) { + addUnexpandableNodes(parser, graph); + addUnmatchableLeafNodes(parser, graph); + addUnmatchableMidProductionNodes(parser, graph); + addFilteredNodes(parser, graph); + + graph.addNode(ERROR_TRACKING_ID, "Errors"); + + graph.addEdge(PARSER_ID, ERROR_TRACKING_ID, "error tracking"); + graph.addEdge(ERROR_TRACKING_ID, UNEXPANDABLE_NODES_ID, "unexpandable"); + graph.addEdge(ERROR_TRACKING_ID, UNMATCHABLE_LEAF_NODES_ID, "unmatchable leafs"); + graph.addEdge(ERROR_TRACKING_ID, UNMATCHABLE_MID_PRODUCTION_NODES_ID, "unmatchable mid-prod"); + graph.addEdge(ERROR_TRACKING_ID, FILTERED_NODES_ID, "filtered"); + } + + private void addUnexpandableNodes(SGTDBF parser, DotGraph graph) { + addStackNodeStack(graph, UNEXPANDABLE_NODES_ID, parser.getUnexpandableNodes()); + } + + private void addUnmatchableLeafNodes(SGTDBF parser, DotGraph graph) { + addStackNodeStack(graph, UNMATCHABLE_LEAF_NODES_ID, parser.getUnmatchableLeafNodes()); + } + + private void addUnmatchableMidProductionNodes(SGTDBF parser, DotGraph graph) { + DoubleStack, AbstractNode>, AbstractStackNode

> unmatchableMidProductionNodes = parser.getUnmatchableMidProductionNodes(); + + graph.addArrayNode(UNMATCHABLE_MID_PRODUCTION_NODES_ID, unmatchableMidProductionNodes.getSize()); + for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { + NodeId failureId = new NodeId("unmatchable-mid-production-" + i); + DotRecord failureRecord = new DotRecord(); + failureRecord.addEntry(new DotField("Failed Node", "failedNode")); + failureRecord.addEntry(new DotField("Predecessors", "predecessors")); + graph.addRecordNode(failureId, failureRecord); + graph.addEdge(new NodeId(UNMATCHABLE_MID_PRODUCTION_NODES_ID, String.valueOf(i)), failureId); + + DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); + AbstractStackNode

failedNode = unmatchableMidProductionNodes.getSecond(i); + + DotNode node = addStack(graph, failedNode); + NodeId predecessorsId = new NodeId("unmatchable-mid-production-predecessors-" + i); + addStackAndNodeDoubleList(graph, predecessorsId, failedNodePredecessors); + + graph.addEdge(new NodeId(failureId, "failedNode"), node.getId()); + graph.addEdge(new NodeId(failureId, "predecessors"), predecessorsId); + } + } + + private void addFilteredNodes(SGTDBF parser, DotGraph graph) { + addStackAndNodeDoubleStack(graph, FILTERED_NODES_ID, parser.getFilteredNodes()); + } + /* + * private final Stack> unexpandableNodes; + private final Stack> unmatchableLeafNodes; // Leaf nodes (for instance literals) that failed to match + private final DoubleStack, AbstractNode>, AbstractStackNode

> unmatchableMidProductionNodes; + private final DoubleStack, AbstractNode> filteredNodes; + + */ + + private void addStackAndNodeDoubleStack(DotGraph graph, NodeId nodeId, DoubleStack, N> doubleStack) { + DotNode arrayNode = DotNode.createArrayNode(nodeId, doubleStack == null ? 0 : doubleStack.getSize()); + graph.addNode(arrayNode); + + if (doubleStack == null) { + return; + } + + for (int j=0; j stack = doubleStack.getFirst(j); + DotNode stackDotNode = addStack(graph, stack); + graph.addEdge(new NodeId(nodeId, String.valueOf(j), CompassPoint.SW), stackDotNode.getId(), "Stack"); + + AbstractNode node = doubleStack.getSecond(j); + addParserNode(graph, node); + graph.addEdge(new NodeId(nodeId, String.valueOf(j), CompassPoint.SE), getNodeId(node), "Node"); + } + } + + private void addStackAndNodeDoubleList(DotGraph graph, NodeId nodeId, DoubleArrayList, N> doubleList) { + DotNode arrayNode = DotNode.createArrayNode(nodeId, doubleList.size()); + graph.addNode(arrayNode); + + for (int i=0; i stack = doubleList.getFirst(i); + DotNode stackDotNode = addStack(graph, stack); + graph.addEdge(new NodeId(nodeId, String.valueOf(i), CompassPoint.SW), stackDotNode.getId(), "Stack"); + + AbstractNode node = doubleList.getSecond(i); + addParserNode(graph, node); + graph.addEdge(new NodeId(nodeId, String.valueOf(i), CompassPoint.SE), getNodeId(node), "Node"); + } + } + + private

void addStackNodeStack(DotGraph graph, NodeId nodeId, Stack> stack) { + if (stack == null) { + return; + } + + DotNode arrayNode = DotNode.createArrayNode(nodeId, stack.getSize()); + + for (int j=0; j stackNode = stack.get(j); + addStack(graph, stackNode); + + graph.addEdge(DotEdge.createArrayEdge(nodeId, j, getNodeId(stackNode))); + } + + graph.addNode(arrayNode); + } + + private void writeGraph(DotGraph graph) { + try { + String dotFile = BASE_DIR + name + ".dot"; + String svgFile = BASE_DIR + name + ".svg"; + String frameDir = BASE_DIR + "/frames/" + name + "/"; + String frameSvgFile = frameDir + String.format("%04d", frame) + ".svg"; + String frameDotFile = frameDir + String.format("%04d", frame) + ".dot"; + FileWriter writer = new FileWriter(dotFile); + writer.write(graph.toString()); + writer.close(); + + String cmd = String.format("dot -Tsvg %s -o %s", dotFile, svgFile); + Process process = Runtime.getRuntime().exec(cmd); + StreamGobbler streamGobbler = new StreamGobbler(process.getInputStream(), System.out::println); + ExecutorService executorService = Executors.newSingleThreadExecutor(); + Future future = executorService.submit(streamGobbler); + + process.waitFor(); + future.get(10, TimeUnit.SECONDS); + + Path svgPath = Paths.get(svgFile); + Path dotPath = Paths.get(dotFile); + Path frameSvgPath = Paths.get(frameSvgFile); + Path frameDotPath = Paths.get(frameDotFile); + Files.copy(svgPath, frameSvgPath, StandardCopyOption.REPLACE_EXISTING); + Files.copy(dotPath, frameDotPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException | InterruptedException | ExecutionException | TimeoutException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java b/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java new file mode 100644 index 00000000000..87d745a5e5c --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java @@ -0,0 +1,28 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; + +public class AttributeStatement { + enum Scope { GRAPH, NODE, EDGE } + + private Scope scope; + private DotAttribute attribute; + + void writeSource(PrintWriter writer) { + if (scope != null) { + switch (scope) { + case GRAPH: + writer.write("graph "); + break; + case NODE: + writer.write("node "); + break; + case EDGE: + writer.write("edge "); + break; + } + } + + attribute.writeSource(writer); + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/CompassPoint.java b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java new file mode 100644 index 00000000000..3b31af24fd1 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java @@ -0,0 +1,5 @@ +package org.rascalmpl.util.visualize.dot; + +public enum CompassPoint { + N, NE, E, SE, S, SW, W, NW, C, X; +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotAttribute.java b/src/org/rascalmpl/util/visualize/dot/DotAttribute.java new file mode 100644 index 00000000000..761218c6e29 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotAttribute.java @@ -0,0 +1,104 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.List; + +public class DotAttribute { + public static final String ATTR_LABEL = "label"; + public static final String ATTR_COLOR = "color"; + public static final String ATTR_NODE_SHAPE = "shape"; + public static final String NODE_SHAPE_RECORD = "record"; + + String property; + String value; + + public DotAttribute(String property, String value) { + this.property = property; + this.value = value; + } + + public String getProperty() { + return property; + } + + public String getValue() { + return value; + } + + void setValue(String value) { + this.value = value; + } + + void writeSource(PrintWriter writer) { + NodeId.writeId(writer, property); + writer.write("="); + NodeId.writeId(writer, value); + } + + static void writeAttributes(PrintWriter writer, List attributes) { + if (!attributes.isEmpty()) { + writer.write("["); + boolean first = true; + for (DotAttribute attribute : attributes) { + if (first) { + first = false; + } else { + writer.write(", "); + } + attribute.writeSource(writer); + } + writer.write("]"); + } + } + + public static DotAttribute createRecordLabel(List elements) { + StringBuilder value = new StringBuilder(); + + boolean first = true; + for (String element : elements) { + if (first) { + first = false; + } else { + value.append("| "); + } + + value.append('<'); + value.append(element); + value.append('>'); + value.append(' '); + value.append(element); + } + + return new DotAttribute(ATTR_LABEL, value.toString()); + } + + public static DotAttribute createArrayLabel(int size) { + StringBuilder value = new StringBuilder(); + + boolean first = true; + for (int i=0; i'); + value.append(' '); + + value.append(i); + } + + return new DotAttribute(ATTR_LABEL, value.toString()); + } + + public static DotAttribute createRecordLabel(DotRecord rec) { + StringWriter writer = new StringWriter(); + rec.writeSource(new PrintWriter(writer, true), true); + return new DotAttribute(ATTR_LABEL, writer.toString()); + } + +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotEdge.java b/src/org/rascalmpl/util/visualize/dot/DotEdge.java new file mode 100644 index 00000000000..7faf1ee1259 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotEdge.java @@ -0,0 +1,66 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +public class DotEdge implements DotStatement { + private NodeId from; + private List to; + private boolean directed; + private List attributes; + + public DotEdge(NodeId from) { + this(from, true); + } + + public DotEdge(NodeId from, boolean directed) { + this.from = from; + this.directed = directed; + to = new ArrayList<>(); + attributes = new ArrayList<>(); + } + + public DotEdge(NodeId from, NodeId... to) { + this(from, true, to); + } + + public DotEdge(NodeId from, boolean directed, NodeId... to) { + this(from, directed); + for (NodeId node : to) { + addTo(node); + } + } + + public void addTo(NodeId node) { + to.add(node); + } + + public void addAttribute(String property, String value) { + attributes.add(new DotAttribute(property, value)); + } + + @Override + public void writeSource(PrintWriter writer) { + from.writeSource(writer); + for (NodeId node : to) { + if (directed) { + writer.write(" -> "); + } else { + writer.write(" -- "); + } + + node.writeSource(writer); + } + DotAttribute.writeAttributes(writer, attributes); + } + + public static DotEdge createArrayEdge(NodeId array, int index, NodeId element) { + return new DotEdge(new NodeId(array, String.valueOf(index)), true, element); + } + + public static DotEdge createArrayEdge(NodeId array, int index, CompassPoint direction, NodeId element) { + return new DotEdge(new NodeId(array.getId(), String.valueOf(index), direction), true, element); + } +} + diff --git a/src/org/rascalmpl/util/visualize/dot/DotField.java b/src/org/rascalmpl/util/visualize/dot/DotField.java new file mode 100644 index 00000000000..70a25aad7b8 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotField.java @@ -0,0 +1,23 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; + +public class DotField implements DotRecordEntry { + private String label; + private String portId; + + public DotField(String label, String portId) { + this.label = label; + this.portId = portId; + } + + @Override + public void writeSource(PrintWriter writer, boolean topLevel) { + if (portId != null) { + writer.write('<'); + writer.write(portId); + writer.write("> "); + } + NodeId.writeString(writer, label); + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotGraph.java b/src/org/rascalmpl/util/visualize/dot/DotGraph.java new file mode 100644 index 00000000000..27293e2f1e5 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotGraph.java @@ -0,0 +1,107 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/* +Loosely follow (a subset of) the DotGraph syntax definition: +https://graphviz.org/doc/info/lang.html +*/ + +public class DotGraph { + private static final String COLOR_HIGHLIGHT = "red"; + + private String id; + private boolean digraph; + private List statements; + private Map nodes; + + public DotGraph(String id, boolean digraph) { + this.id = id; + this.digraph = digraph; + statements = new ArrayList<>(); + nodes = new HashMap<>(); + } + + private void addStatement(DotStatement statement) { + statements.add(statement); + } + + public void addNode(DotNode node) { + addStatement(node); + nodes.put(node.getId(), node); + } + + public void addNode(String id, String label) { + addNode(new NodeId(id), label); + } + + public void addNode(NodeId id, String label) { + DotNode node = new DotNode(id); + node.addAttribute(DotAttribute.ATTR_LABEL, label); + addNode(node); + } + + public void addArrayNode(NodeId id, int size) { + DotNode node = new DotNode(id); + node.addAttribute(DotAttribute.ATTR_NODE_SHAPE, DotAttribute.NODE_SHAPE_RECORD); + node.addAttribute(DotAttribute.createArrayLabel(size)); + addNode(node); + } + + public void addRecordNode(NodeId id, DotRecord dotRecord) { + DotNode node = new DotNode(id); + node.addAttribute(DotAttribute.ATTR_NODE_SHAPE, DotAttribute.NODE_SHAPE_RECORD); + node.addAttribute(DotAttribute.createRecordLabel(dotRecord)); + addNode(node); + } + + public void highlight(NodeId id) { + DotNode node = nodes.get(id); + if (node != null) { + node.addAttribute(DotAttribute.ATTR_COLOR, COLOR_HIGHLIGHT); + } + } + + public void addEdge(String from, String to) { + addEdge(new NodeId(from), new NodeId(to)); + } + + public void addEdge(NodeId from, NodeId to) { + addEdge(new DotEdge(from, true, to)); + } + + public void addEdge(NodeId from, NodeId to, String label) { + DotEdge edge = new DotEdge(from, true, to); + edge.addAttribute(DotAttribute.ATTR_LABEL, label); + addEdge(edge); + } + + public void addEdge(DotEdge edge) { + addStatement(edge); + } + + public void writeSource(PrintWriter writer) { + writer.write(digraph ? "digraph" : "graph"); + if (id != null) { + writer.write(" "); + writer.write(id); + } + writer.println(" {"); + for (DotStatement statement : statements) { + statement.writeSource(writer); + writer.println(";"); + } + writer.println("}"); + } + + public String toString() { + StringWriter writer = new StringWriter(); + writeSource(new PrintWriter(writer)); + return writer.toString(); + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotNode.java b/src/org/rascalmpl/util/visualize/dot/DotNode.java new file mode 100644 index 00000000000..118d1c00c96 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotNode.java @@ -0,0 +1,69 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +public class DotNode implements DotStatement { + public static DotNode createArrayNode(NodeId id, int size) { + DotNode node = new DotNode(id); + node.addAttribute(DotAttribute.createArrayLabel(size)); + node.addAttribute(DotAttribute.ATTR_NODE_SHAPE, DotAttribute.NODE_SHAPE_RECORD); + return node; + } + + private NodeId id; + private List attributes; + + public DotNode(String id) { + this(new NodeId(id)); + } + + public DotNode(NodeId id) { + this.id = id; + attributes = new ArrayList<>(); + } + + public NodeId getId() { + return id; + } + + public void addAttribute(String property, String value) { + addAttribute(new DotAttribute(property, value)); + } + + public void addAttribute(DotAttribute attribute) { + attributes.add(attribute); + } + + public void setAttribute(String property, String value) { + for (DotAttribute attribute : attributes) { + if (attribute.getProperty().equals(property)) { + attribute.setValue(value); + return; + } + } + + addAttribute(property, value); + } + + public String getAttributeValue(String property) { + for (DotAttribute attribute : attributes) { + if (attribute.getProperty().equals(property)) { + return attribute.getValue(); + } + } + + return null; + } + + public void setLabel(String label) { + setAttribute(DotAttribute.ATTR_LABEL, label); + } + + @Override + public void writeSource(PrintWriter writer) { + id.writeSource(writer); + DotAttribute.writeAttributes(writer, attributes); + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotRecord.java b/src/org/rascalmpl/util/visualize/dot/DotRecord.java new file mode 100644 index 00000000000..73a5cdd73b8 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotRecord.java @@ -0,0 +1,37 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +public class DotRecord implements DotRecordEntry { + private List entries; + + public DotRecord() { + entries = new ArrayList<>(); + } + + public void addEntry(DotRecordEntry entry) { + entries.add(entry); + } + + @Override + public void writeSource(PrintWriter writer, boolean topLevel) { + if (!topLevel) { + writer.write("{ "); + } + boolean first = true; + for (DotRecordEntry entry : entries) { + if (first) { + first = false; + } else { + writer.write(" | "); + } + + entry.writeSource(writer, false); + } + if (!topLevel) { + writer.write("} "); + } + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java b/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java new file mode 100644 index 00000000000..a0ed741e9f0 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java @@ -0,0 +1,7 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; + +public interface DotRecordEntry { + void writeSource(PrintWriter writer, boolean topLevel); +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotStatement.java b/src/org/rascalmpl/util/visualize/dot/DotStatement.java new file mode 100644 index 00000000000..83919814059 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotStatement.java @@ -0,0 +1,7 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; + +public interface DotStatement { + void writeSource(PrintWriter writer); +} diff --git a/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java b/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java new file mode 100644 index 00000000000..23e7fb6b6a6 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java @@ -0,0 +1,40 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.List; + +public class DotSubgraph implements DotStatement { + private String id; + private List statements; + + public DotSubgraph(String id) { + this.id = id; + statements = new ArrayList<>(); + } + + public void addStatement(DotStatement statement) { + statements.add(statement); + } + + public String getId() { + return id; + } + + @Override + public void writeSource(PrintWriter writer) { + writer.write("subgraph"); + if (id != null) { + NodeId.writeId(writer, id); + writer.write(" "); + } + writer.println(" {"); + for (DotStatement statement : statements) { + writer.write(" "); + statement.writeSource(writer); + writer.println(";"); + } + + writer.println(" }"); + } +} diff --git a/src/org/rascalmpl/util/visualize/dot/NodeId.java b/src/org/rascalmpl/util/visualize/dot/NodeId.java new file mode 100644 index 00000000000..fe2c3849ea3 --- /dev/null +++ b/src/org/rascalmpl/util/visualize/dot/NodeId.java @@ -0,0 +1,126 @@ +package org.rascalmpl.util.visualize.dot; + +import java.io.PrintWriter; + +public class NodeId { + public static void writeId(PrintWriter stream, String id) { + stream.write("\""); + writeString(stream, id); + stream.write("\""); + } + + public static void writeString(PrintWriter stream, String s) { + for (int i=0; i Date: Thu, 18 Jul 2024 13:01:46 +0200 Subject: [PATCH 38/87] Implemented skipping of remaining characters when top-level production is recovered --- .../tests/recovery/BasicRecoveryTests.rsc | 11 ++++- src/org/rascalmpl/parser/gtd/SGTDBF.java | 4 +- .../parser/gtd/stack/SkippingStackNode.java | 46 +++++++++++-------- .../recovery/ToNextWhitespaceRecoverer.java | 41 ++++++++++++++++- 4 files changed, 79 insertions(+), 23 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index ec7ff7270a1..2dc8c2fbdec 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -1,8 +1,9 @@ module lang::rascal::tests::recovery::BasicRecoveryTests import ParseTree; +import IO; -layout Layout = [\ ]; //* !>> [\ ]; +layout Layout = [\ ]* !>> [\ ]; syntax S = ABC End; syntax ABC = "a" "b" "c"; @@ -13,7 +14,8 @@ test bool ok() { } test bool abx() { - Tree t = parse(#S, "a b x $", allowRecovery=true); + Tree t = parse(#S, "a b x $", allowRecovery=true, allowAmbiguity=true); + iprintln(t); return hasErrors(t) && size(findAllErrors(t)) == 1; } @@ -26,3 +28,8 @@ test bool ax() { Tree t = parse(#S, "a x $", allowRecovery=true); return hasErrors(t) && size(findAllErrors(t)) == 1; } + +test bool missingEnd() { + Tree t = parse(#S, "a b c", allowRecovery=true); + return hasErrors(t) && size(findAllErrors(t)) == 1; +} diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index b7f7aaa7a5d..f3094d2e4e0 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -861,7 +861,7 @@ private boolean findFirstStacksToReduce(){ private boolean findStacksToReduce(){ visualize("Finding stacks to reduce", DebugVisualizer.TODO_LISTS_ID); int queueDepth = todoLists.length; - for(int i = 1; i < queueDepth; ++i){ + for(int i = 1; i < queueDepth-1; ++i){ queueIndex = (queueIndex + 1) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; @@ -879,7 +879,7 @@ private boolean findStacksToReduce(){ } } - if (recoverer != null && location < input.length) { + if (recoverer != null) { if (debugListener != null) { visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 2ef909c5684..7030149d130 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -14,13 +14,39 @@ import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.result.SkippedNode; +import io.usethesource.vallang.IConstructor; + public final class SkippingStackNode

extends AbstractMatchableStackNode

{ private final SkippedNode result; - public SkippingStackNode(int id, int[] until, int[] input, int startLocation, P parentProduction, int dot){ + public static SkippedNode createResultUntilCharClass(int[] until, int[] input, int startLocation, IConstructor production, int dot) { + for (int to = startLocation ; to < input.length; ++to) { + for (int i = 0; i < until.length; ++i) { + if (input[to] == until[i]) { + int length = to - startLocation; + return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); + } + } + } + + return new SkippedNode(production, dot, new int[0], startLocation); + } + + public static SkippedNode createResultUntilEndOfInput(int[] input, int startLocation, IConstructor production, int dot) { + int length = input.length - startLocation; + return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); + } + + private static int[] createSkippedToken(int[] input, int startLocation, int length) { + int[] token = new int[length]; + System.arraycopy(input, startLocation, token, 0, length); + return token; + } + + public SkippingStackNode(int id, P parentProduction, SkippedNode result) { super(id, 0); - this.result = buildResult(input, until, startLocation, parentProduction, dot); + this.result = result; setAlternativeProduction(parentProduction); } @@ -35,23 +61,7 @@ private SkippingStackNode(SkippingStackNode

original, SkippedNode result, int this.result = result; } - - private SkippedNode buildResult(int[] input, int[] until, int startLocation, P production, int dot){ - for (int to = startLocation ; to < input.length; ++to) { - for (int i = 0; i < until.length; ++i) { - if (input[to] == until[i]) { - int length = to - startLocation; - int[] chars = new int[length]; - System.arraycopy(input, startLocation, chars, 0, length); - - return new SkippedNode(production, dot, chars, startLocation); - } - } - } - return new SkippedNode(production, dot, new int[0], startLocation); - } - public boolean isEmptyLeafNode(){ return result.isEmpty(); } diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 65add6461bf..e799a05b42e 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -14,6 +14,7 @@ import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; +import org.rascalmpl.parser.gtd.result.SkippedNode; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; import org.rascalmpl.parser.gtd.stack.SkippingStackNode; @@ -75,8 +76,17 @@ private DoubleArrayList, AbstractNode> reviveNod AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); + int dot = recoveryNode.getDot(); int startLocation = recoveryNode.getStartLocation(); - AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), WHITESPACE, input, startLocation, prod, recoveryNode.getDot()); + + SkippedNode result; + if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { + result = SkippingStackNode.createResultUntilEndOfInput(input, startLocation, prod, dot); + } else { + result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); + } + + AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result); recoverLiteral = recoverLiteral.getCleanCopy(startLocation); // This copy might not be needed recoverLiteral.initEdges(); EdgesSet edges = new EdgesSet<>(1); @@ -92,6 +102,35 @@ private DoubleArrayList, AbstractNode> reviveNod return recoveredNodes; } + + // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) + // As this is experimental code, this method is extremely conservative. + // Any sharing will result in returning 'false'. + // We will need to change this strategy in the future to improve error recovery. + private boolean isTopLevelProduction(AbstractStackNode node) { + while (node != null && node.getDot() != 0) { + node = getSinglePredecessor(node); + } + + if (node != null) { + node = getSinglePredecessor(node); + return node != null && node.getStartLocation() == -1; + } + + return false; + } + + private AbstractStackNode getSinglePredecessor(AbstractStackNode node) { + IntegerObjectList> edgeMap = node.getEdges(); + if (edgeMap.size() == 1) { + EdgesSet edges = edgeMap.getValue(0); + if (edges.size() == 1) { + return edges.get(0); + } + } + + return null; + } private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); From 2b044fdd0ca49eb0785c23c351a6ff6066351c4b Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 18 Jul 2024 13:25:56 +0200 Subject: [PATCH 39/87] Reinstated end-of-input check before recovering (for now) --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index f3094d2e4e0..6743876f9b2 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -879,7 +879,7 @@ private boolean findStacksToReduce(){ } } - if (recoverer != null) { + if (recoverer != null && location < input.length) { if (debugListener != null) { visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); From d5dbe8c2cf92543fdddb558e827eca1677b029a0 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 6 Aug 2024 09:32:21 +0200 Subject: [PATCH 40/87] Minor comment and test changes --- .../library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc | 4 ++++ src/org/rascalmpl/parser/gtd/SGTDBF.java | 2 -- .../parser/uptr/recovery/ToNextWhitespaceRecoverer.java | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index 2dc8c2fbdec..c9078463389 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -21,15 +21,19 @@ test bool abx() { test bool axc() { Tree t = parse(#S, "a x c $", allowRecovery=true); + iprintln(t); return hasErrors(t) && size(findAllErrors(t)) == 1; } test bool ax() { Tree t = parse(#S, "a x $", allowRecovery=true); + iprintln(t); return hasErrors(t) && size(findAllErrors(t)) == 1; } +/* test bool missingEnd() { Tree t = parse(#S, "a b c", allowRecovery=true); return hasErrors(t) && size(findAllErrors(t)) == 1; } +*/ diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 6743876f9b2..2bf7774ed65 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -949,8 +949,6 @@ private void queueMatchableNode(AbstractStackNode

node, int length, AbstractN * Inserts a recovery node into the todo-list, and possibly * rewinds the parser to an earlier location in the input */ - // TODO: look for original code: - // - queueMatchableNode (just above here) @SuppressWarnings("unchecked") private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int length, AbstractNode result){ assert result != null; diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index e799a05b42e..7ca1a20a46f 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -26,7 +26,6 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; -import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; From 2246e06b8f33e013bbcb26008f6f72f4371666bf Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 8 Aug 2024 10:34:20 +0200 Subject: [PATCH 41/87] In addition to using whitespace we now also skip until after the last token in the production --- src/org/rascalmpl/library/ParseTree.rsc | 30 +- .../tests/recovery/BasicRecoveryTests2.rsc | 40 +++ .../gtd/stack/AbstractMatchableStackNode.java | 6 +- .../parser/gtd/stack/AbstractStackNode.java | 10 +- .../parser/gtd/stack/SkippingStackNode.java | 31 +- .../uptr/recovery/ToTokenRecoverer.java | 305 ++++++++++++++++++ .../util/visualize/DebugVisualizer.java | 19 +- .../values/RascalFunctionValueFactory.java | 3 +- 8 files changed, 430 insertions(+), 14 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc create mode 100644 src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 010655cf502..1f88edb3ef1 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -143,6 +143,7 @@ extend Type; extend Message; extend List; +import String; @synopsis{The Tree data type as produced by the parser.} @description{ @@ -770,21 +771,38 @@ default bool isNonTerminalType(Symbol s) = false; @synopsis{Check if a parse tree contains any skipped nodes, the result of error recovery.} bool hasErrors(Tree tree) = /appl(skipped(_, _, _), _) := tree; -@synopsis{Find all skipped nodes in a parse tree.} +@synopsis{Find all error productions in a parse tree.} list[Tree] findAllErrors(Tree tree) { return for (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) append(error); } -@synopsis{Find the first skipped node in a parse tree.} +@synopsis{Find the first production containing an error.} Tree findFirstError(Tree tree) { if (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) return error; fail; } -Symbol getErrorSymbol(appl(skipped(Symbol symbol, _, _), _)) = symbol; +private Production getSkipped(appl(_, [*_, appl(skip:skipped(_, _, _), _)])) = skip; -Production getErrorProduction(appl(skipped(_, Production production, _), _)) = production; +Symbol getErrorSymbol(Tree error) { + if (skipped(Symbol s, _, _) := getSkipped(error)) { + return s; + } + fail; +} + +Production getErrorProduction(Tree error) { + if (skipped(_, Production p, _) := getSkipped(error)) { + return p; + } + fail; +} -int getErrorPosition(appl(skipped(_, _, int dot), _)) = dot; +int getErrorPosition(Tree error) { + if (skipped(_, _, int dot) := getSkipped(error)) { + return dot; + } + fail; +} -str getErrorString(appl(skipped(_, _, _), list chars)) = ""; +str getErrorText(appl(_, [*_, appl(skipped(_, _, _), chars)])) = stringChars([c | ch <- chars, char(c) := ch]); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc new file mode 100644 index 00000000000..5a08df51ec7 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc @@ -0,0 +1,40 @@ +module lang::rascal::tests::recovery::BasicRecoveryTests2 + +import ParseTree; +import IO; + +layout Layout = [\ ]* !>> [\ ]; + +syntax S = T; + +syntax T = ABC End; +syntax ABC = "a" "b" "c"; +syntax End = "$"; + +test bool ok() { + return !hasErrors(parse(#S, "a b c $", allowRecovery=true)); +} + +test bool abx() { + Tree t = parse(#S, "a b x $", allowRecovery=true); + return getErrorText(findFirstError(t)) == "x"; +} + +test bool axc() { + Tree t = parse(#S, "a x c $", allowRecovery=true); + iprintln(t); + return getErrorText(findFirstError(t)) == "x c"; +} + +test bool ax() { + Tree t = parse(#S, "a x $", allowRecovery=true); + return getErrorText(findFirstError(t)) == "x"; +} + +/* +test bool missingEnd() { + Tree t = parse(#S, "a b c", allowRecovery=true); + println("Error text: "); + return hasErrors(t) && size(findAllErrors(t)) == 1; +} +*/ diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java index 3afee7e1894..3943c6dca5f 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java @@ -25,7 +25,11 @@ public abstract class AbstractMatchableStackNode

extends AbstractStackNode

protected AbstractMatchableStackNode(int id, int dot){ super(id, dot); } - + + protected AbstractMatchableStackNode(int id, int dot, int startLocation){ + super(id, dot, startLocation); + } + protected AbstractMatchableStackNode(int id, int dot, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index e431402f300..b477301e954 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -63,17 +63,21 @@ public abstract class AbstractStackNode

{ private IntegerList propagatedReductions; protected AbstractStackNode(int id, int dot){ + this(id, dot, DEFAULT_START_LOCATION); + } + + protected AbstractStackNode(int id, int dot, int startLocation) { super(); this.id = id; this.dot = dot; - this.startLocation = DEFAULT_START_LOCATION; + this.startLocation = startLocation; this.enterFilters = null; this.completionFilters = null; } - + protected AbstractStackNode(int id, int dot, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(); @@ -745,7 +749,7 @@ public IntegerList getPropagatedReductions(){ } public String toShortString() { - return "." + dot + "@" + startLocation; + return null; } @Override diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 7030149d130..ae0bcf41be0 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -37,6 +37,28 @@ public static SkippedNode createResultUntilEndOfInput(int[] input, int startLoca return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); } + public static SkippedNode createResultUntilToken(String token, int[] input, int startLocation, IConstructor production, int dot) { + int length = token.length(); + for (int start=startLocation; start+length < input.length; start++) { + boolean match = true; + for (int j=0; j original, int startLocation){ super(original, startLocation); diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java new file mode 100644 index 00000000000..858f2d7cdf3 --- /dev/null +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -0,0 +1,305 @@ +/******************************************************************************* + * Copyright (c) 2009-2022 NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + * + * Contributors: + + * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI + * * Arnold Lankamp - Arnold.Lankamp@cwi.nl +*******************************************************************************/ +package org.rascalmpl.parser.uptr.recovery; + +import java.util.List; + +import org.rascalmpl.parser.gtd.recovery.IRecoverer; +import org.rascalmpl.parser.gtd.result.AbstractNode; +import org.rascalmpl.parser.gtd.result.SkippedNode; +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; +import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; +import org.rascalmpl.parser.gtd.stack.SkippingStackNode; +import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; +import org.rascalmpl.parser.gtd.util.ArrayList; +import org.rascalmpl.parser.gtd.util.DoubleArrayList; +import org.rascalmpl.parser.gtd.util.DoubleStack; +import org.rascalmpl.parser.gtd.util.IdDispenser; +import org.rascalmpl.parser.gtd.util.IntegerObjectList; +import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; +import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.values.RascalValueFactory; +import org.rascalmpl.values.parsetrees.ProductionAdapter; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IList; +import io.usethesource.vallang.IString; +import io.usethesource.vallang.IValue; + +public class ToTokenRecoverer implements IRecoverer { + private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; + + private IdDispenser stackNodeIdDispenser; + + public ToTokenRecoverer(IdDispenser stackNodeIdDispenser) { + this.stackNodeIdDispenser = stackNodeIdDispenser; + } + + @Override + public DoubleArrayList, AbstractNode> reviveStacks(int[] input, + int location, + Stack> unexpandableNodes, + Stack> unmatchableLeafNodes, + DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, + DoubleStack, AbstractNode> filteredNodes) { + + ArrayList> failedNodes = new ArrayList>(); + collectUnexpandableNodes(unexpandableNodes, failedNodes); + collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); + // TODO: handle unmatchableLeafNodes + //collectFilteredNodes(filteredNodes, failedNodes); + + return reviveFailedNodes(input, location, failedNodes); + } + + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ + DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); + + // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { + // But this caused problems because recovery nodes with a later position + // where queued before nodes with an earlier position which the parser cannot handle. + + recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); + + for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); + ArrayList prods = recoveryNodes.getSecond(i); + + int dot = recoveryNode.getDot(); + int startLocation = recoveryNode.getStartLocation(); + + // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). + for (int j = prods.size() - 1; j >= 0; --j) { + IConstructor prod = prods.get(j); + + List> skippingNodes = findSkippingNodes(input, recoveryNode, prod, dot, startLocation); + for (SkippingStackNode skippingNode : skippingNodes) { + AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); + + EdgesSet edges = new EdgesSet<>(1); + edges.add(continuer); + + continuer.setIncomingEdges(edges); + + skippingNode.initEdges(); + skippingNode.addEdges(edges, startLocation); + recoveredNodes.add(skippingNode, skippingNode.getResult()); + } + } + } + + return recoveredNodes; + } + + List> findSkippingNodes(int[] input, AbstractStackNode recoveryNode, IConstructor prod, int dot, int startLocation) { + List> nodes = new java.util.ArrayList<>(); + + SkippedNode result; + + // If we are the top-level node, just skip the rest of the input + if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { + result = SkippingStackNode.createResultUntilEndOfInput(input, startLocation, prod, dot); + nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); + return nodes; // No other nodes would be useful + } + + // Try to find whitespace to skip to + result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); + if (result != null) { + nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); + } + + // Do something more clever: find the last token of this production and skip until after that + String lastLiteral = findLastLiteral(prod); + if (lastLiteral != null) { + result = SkippingStackNode.createResultUntilToken(lastLiteral, input, startLocation, prod, dot); + if (result != null) { + nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); + } + } + + // TODO: or find the first token of the next production and skip until before that + + return nodes; + } + + String findLastLiteral(IConstructor prod) { + IList args = (IList) prod.get(1); + if (args.isEmpty()) { + return null; + } + + IValue last = args.get(args.length()-1); + if (last instanceof IConstructor) { + IConstructor cons = (IConstructor) last; + if (cons.getConstructorType() == RascalValueFactory.Symbol_Lit) { + return ((IString) cons.get(0)).getValue(); + } + } + + return null; + } + + // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) + // As this is experimental code, this method is extremely conservative. + // Any sharing will result in returning 'false'. + // We will need to change this strategy in the future to improve error recovery. + private boolean isTopLevelProduction(AbstractStackNode node) { + while (node != null && node.getDot() != 0) { + node = getSinglePredecessor(node); + } + + if (node != null) { + node = getSinglePredecessor(node); + return node != null && node.getStartLocation() == -1; + } + + return false; + } + + private AbstractStackNode getSinglePredecessor(AbstractStackNode node) { + IntegerObjectList> edgeMap = node.getEdges(); + if (edgeMap.size() == 1) { + EdgesSet edges = edgeMap.getValue(0); + if (edges.size() == 1) { + return edges.get(0); + } + } + + return null; + } + + private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { + DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); + + for (int i = failedNodes.size() - 1; i >= 0; --i) { + findRecoveryNodes(failedNodes.get(i), recoveryNodes); + } + + return reviveNodes(input, location, recoveryNodes); + } + + private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { + for (int i = unexpandableNodes.getSize() - 1; i >= 0; --i) { + failedNodes.add(unexpandableNodes.get(i)); + } + } + + /** + * Make a fresh copy of each unmatchable mid-production node and link in the predecessors of the original node. + * The new copies are added to `failedNodes` + * @param location the location where the failure occurs + * @param unmatchableMidProductionNodes each pair consists of a list of predecessors and a node that failed to match + * @param failedNodes the list to which failed nodes must be added + */ + private static void collectUnmatchableMidProductionNodes(int location, DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ + for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { + DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); + AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version + + // Merge the information on the predecessors into the failed node. + for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { + AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); + AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); + failedNode.updateNode(predecessor, predecessorResult); + } + + failedNodes.add(failedNode); + } + } + + /** + * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. + */ + private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { + ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap>(); + Stack> todo = new Stack>(); + + todo.push(failer); + + while (!todo.isEmpty()) { + AbstractStackNode node = todo.pop(); + + if (visited.contains(node)) { + continue; // Don't follow cycles + } + + visited.put(node, 0); + + if (node == failer || node.getDot() == 0) { + ArrayList recoveryProductions = new ArrayList(); + collectProductions(node, recoveryProductions); + if (recoveryProductions.size() > 0) { + recoveryNodes.add(node, recoveryProductions); + } + } + + IntegerObjectList> edges = node.getEdges(); + + for (int i = edges.size() - 1; i >= 0; --i) { // Rewind + EdgesSet edgesList = edges.getValue(i); + + if (edgesList != null) { + for (int j = edgesList.size() - 1; j >= 0; --j) { + AbstractStackNode parent = edgesList.get(j); + + todo.push(parent); + } + } + } + } + } + + // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) + private void collectProductions(AbstractStackNode node, ArrayList productions) { + AbstractStackNode[] production = node.getProduction(); + if (production == null) { + return; // The root node does not have a production, so ignore it. + } + + int dot = node.getDot(); + + System.err.println("collect productions for node: " + node); + if (node.isEndNode()) { + IConstructor parentProduction = node.getParentProduction(); + if (ProductionAdapter.isContextFree(parentProduction)){ + productions.add(parentProduction); + System.err.println("adding production: " + parentProduction); + + if (ProductionAdapter.isList(parentProduction)) { + return; // Don't follow productions in lists productions, since they are 'cyclic'. + } + } + } + + for (int i = dot + 1; i < production.length; ++i) { + AbstractStackNode currentNode = production[i]; + if (currentNode.isEndNode()) { + IConstructor parentProduction = currentNode.getParentProduction(); + if (ProductionAdapter.isContextFree(parentProduction)) { + productions.add(parentProduction); + System.err.println("adding production at " + i + ": " + parentProduction); + } + } + + AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); + if (alternateProductions != null) { + for (int j = alternateProductions.length - 1; j >= 0; --j) { + collectProductions(alternateProductions[j][i], productions); + } + } + } + } + + +} diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java index b83f12976d0..b0fb64f41ca 100644 --- a/src/org/rascalmpl/util/visualize/DebugVisualizer.java +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -210,6 +210,12 @@ private

DotNode createDotNode(AbstractStackNode

stackNode) { DotNode node = new DotNode(getNodeId(stackNode)); String label = String.format("%s: %s\n.%d@%d", type, nodeName, stackNode.getDot(), stackNode.getStartLocation()); + + String shortString = stackNode.toShortString(); + if (shortString != null) { + label += "\n" + shortString; + } + P parentProduction = stackNode.getParentProduction(); if (parentProduction instanceof IConstructor) { label += "\nin: " + DebugUtil.prodToString((IConstructor) parentProduction); @@ -461,13 +467,22 @@ private void addStackAndNodeDoubleList(DotGraph grap graph.addNode(arrayNode); for (int i=0; i stack = doubleList.getFirst(i); DotNode stackDotNode = addStack(graph, stack); - graph.addEdge(new NodeId(nodeId, String.valueOf(i), CompassPoint.SW), stackDotNode.getId(), "Stack"); + graph.addEdge(new NodeId(entryId, "stack", CompassPoint.SW), stackDotNode.getId(), "Stack"); AbstractNode node = doubleList.getSecond(i); addParserNode(graph, node); - graph.addEdge(new NodeId(nodeId, String.valueOf(i), CompassPoint.SE), getNodeId(node), "Node"); + graph.addEdge(new NodeId(entryId, "node", CompassPoint.SE), getNodeId(node), "Node"); + + + graph.addEdge(new NodeId(nodeId, String.valueOf(i)), entryId); } } diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 0a4cffc46a5..ca7d5ed073a 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -55,6 +55,7 @@ import org.rascalmpl.parser.uptr.action.RascalFunctionActionExecutor; import org.rascalmpl.parser.uptr.debug.DebugLogger; import org.rascalmpl.parser.uptr.recovery.ToNextWhitespaceRecoverer; +import org.rascalmpl.parser.uptr.recovery.ToTokenRecoverer; import org.rascalmpl.types.NonTerminalType; import org.rascalmpl.types.RascalTypeFactory; import org.rascalmpl.types.ReifiedType; @@ -580,7 +581,7 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IRecoverer recoverer = null; IDebugListener debugListener = null; if (allowRecovery) { - recoverer = new ToNextWhitespaceRecoverer(new StackNodeIdDispenser(parserInstance)); + recoverer = new ToTokenRecoverer(new StackNodeIdDispenser(parserInstance)); debugListener = new DebugLogger(new PrintWriter(System.out, true)); } return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); From 4b11450c03f676c12d32048c94a23e639484a6f1 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 19 Aug 2024 10:32:54 +0200 Subject: [PATCH 42/87] Implemented dynamic last/follow calculations --- .vscode/launch.json | 9 + .../tests/recovery/BasicRecoveryTests.rsc | 34 ++-- .../tests/recovery/BasicRecoveryTests2.rsc | 40 ----- .../tests/recovery/ListRecoveryTests.rsc | 32 ++++ .../tests/recovery/NestedRecoveryTests.rsc | 28 ++++ .../tests/recovery/PicoRecoveryTests.rsc | 122 ++++++++++++++ .../tests/recovery/RunRecoveryTests.rsc | 9 + src/org/rascalmpl/parser/gtd/SGTDBF.java | 20 ++- .../CaseInsensitiveLiteralStackNode.java | 13 ++ .../parser/gtd/stack/CharStackNode.java | 30 +++- .../parser/gtd/stack/LiteralStackNode.java | 4 + .../gtd/stack/SeparatedListStackNode.java | 5 +- .../parser/gtd/stack/SkippingStackNode.java | 4 +- .../CaseInsensitiveLiteralMatcher.java | 66 ++++++++ .../parser/uptr/recovery/FailingMatcher.java | 8 + .../parser/uptr/recovery/InputMatcher.java | 66 ++++++++ .../parser/uptr/recovery/LiteralMatcher.java | 41 +++++ .../uptr/recovery/ToTokenRecoverer.java | 158 +++++++++++++++--- src/org/rascalmpl/parser/util/DebugUtil.java | 4 +- .../util/visualize/DebugVisualizer.java | 112 ++++++++++++- .../util/visualize/dot/DotGraph.java | 6 +- .../values/RascalFunctionValueFactory.java | 2 +- 22 files changed, 716 insertions(+), 97 deletions(-) delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc create mode 100644 src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java create mode 100644 src/org/rascalmpl/parser/uptr/recovery/FailingMatcher.java create mode 100644 src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java create mode 100644 src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java diff --git a/.vscode/launch.json b/.vscode/launch.json index 8d7a2f6a52c..e16d6d8230e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -39,6 +39,15 @@ "projectName": "rascal", "vmArgs": "-Xss80m -Xmx2g -ea" }, + { + "type": "java", + "name": "Recovery tests", + "request": "launch", + "mainClass": "org.rascalmpl.shell.RascalShell", + "projectName": "rascal", + "vmArgs": "-Xss80m -Xmx2g -ea", + "args": "lang::rascal::tests::recovery::RunRecoveryTests" + }, { "type": "java", "name": "Launch RascalShell Tutor", diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index c9078463389..e84ba513118 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -5,35 +5,35 @@ import IO; layout Layout = [\ ]* !>> [\ ]; -syntax S = ABC End; -syntax ABC = "a" "b" "c"; +syntax S = T; + +syntax T = ABC End; +syntax ABC = 'a' 'b' 'c'; syntax End = "$"; -test bool ok() { - return !hasErrors(parse(#S, "a b c $", allowRecovery=true)); +private Tree parseS(str input, bool visualize=false) + = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + +test bool basicOk() { + return !hasErrors(parseS("a b c $", visualize=true)); } test bool abx() { - Tree t = parse(#S, "a b x $", allowRecovery=true, allowAmbiguity=true); - iprintln(t); - return hasErrors(t) && size(findAllErrors(t)) == 1; + Tree t = parseS("a b x $", visualize=true); + return getErrorText(findFirstError(t)) == "x"; } test bool axc() { - Tree t = parse(#S, "a x c $", allowRecovery=true); + Tree t = parseS("a x c $", visualize=true); iprintln(t); - return hasErrors(t) && size(findAllErrors(t)) == 1; + return getErrorText(findFirstError(t)) == "x c"; } test bool ax() { - Tree t = parse(#S, "a x $", allowRecovery=true); - iprintln(t); - return hasErrors(t) && size(findAllErrors(t)) == 1; + Tree t = parseS("a x $", visualize=true); + return getErrorText(findFirstError(t)) = "x"; } -/* -test bool missingEnd() { - Tree t = parse(#S, "a b c", allowRecovery=true); - return hasErrors(t) && size(findAllErrors(t)) == 1; +int main(list[str] args){ + startRepl(REPL()); } -*/ diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc deleted file mode 100644 index 5a08df51ec7..00000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests2.rsc +++ /dev/null @@ -1,40 +0,0 @@ -module lang::rascal::tests::recovery::BasicRecoveryTests2 - -import ParseTree; -import IO; - -layout Layout = [\ ]* !>> [\ ]; - -syntax S = T; - -syntax T = ABC End; -syntax ABC = "a" "b" "c"; -syntax End = "$"; - -test bool ok() { - return !hasErrors(parse(#S, "a b c $", allowRecovery=true)); -} - -test bool abx() { - Tree t = parse(#S, "a b x $", allowRecovery=true); - return getErrorText(findFirstError(t)) == "x"; -} - -test bool axc() { - Tree t = parse(#S, "a x c $", allowRecovery=true); - iprintln(t); - return getErrorText(findFirstError(t)) == "x c"; -} - -test bool ax() { - Tree t = parse(#S, "a x $", allowRecovery=true); - return getErrorText(findFirstError(t)) == "x"; -} - -/* -test bool missingEnd() { - Tree t = parse(#S, "a b c", allowRecovery=true); - println("Error text: "); - return hasErrors(t) && size(findAllErrors(t)) == 1; -} -*/ diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc new file mode 100644 index 00000000000..184fd2b2eb8 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc @@ -0,0 +1,32 @@ +module lang::rascal::tests::recovery::ListRecoveryTests + +import ParseTree; +import IO; + +layout Layout = [\ ]* !>> [\ ]; + +syntax S = T End; + +syntax T = { AB "," }*; +syntax AB = "a" "b"; +syntax End = "$"; + +Tree parseList(str s, bool visualize=false) { + return parser(#S, allowRecovery=true, allowAmbiguity=true)(s, |unknown:///?visualize=<"">|); +} + +test bool listOk() { + return !hasErrors(parseList("a b , a b , a b $", visualize=true)); +} + +test bool listTypo() { + Tree t = parseList("a b, a x, ab $", visualize=true); + iprintln(t); + return hasErrors(t); +} + +test bool listTypoWs() { + Tree t = parseList("a b , a x , a b $", visualize=true); + iprintln(t); + return hasErrors(t); +} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc new file mode 100644 index 00000000000..292043f00a4 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc @@ -0,0 +1,28 @@ +module lang::rascal::tests::recovery::NestedRecoveryTests + +import ParseTree; +import IO; + +layout Layout = [\ ]* !>> [\ ]; + +syntax S = T; + +syntax T = A B "c"; + +syntax A = "a"; +syntax B = "b" "b"; +//syntax C = "c"; + +private Tree parseS(str input, bool visualize=false) + = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + +test bool nestedOk() { + return !hasErrors(parseS("a b b c", visualize=true)); +} + +test bool nestedTypo() { + Tree t = parseS("a b x c", visualize=true); + iprintln(t); + println("Error text: \'\'"); + return getErrorText(findFirstError(t)) == "x "; +} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc new file mode 100644 index 00000000000..c4683085179 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -0,0 +1,122 @@ +module lang::rascal::tests::recovery::PicoRecoveryTests + +import lang::pico::\syntax::Main; + +import ParseTree; +import IO; + +private Tree parsePico(str input, bool visualize=false) + = parser(#Program, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + +test bool picoOk() { + t = parsePico("begin declare input : natural, + output : natural, + repnr : natural, + rep : natural; + input := 14; + output := 0; + while input - 1 do + rep := output; + repnr := input; + while repnr - 1 do + output := output + rep; + repnr := repnr - 1 + od; + input := input - 1 + od +end"); + return !hasErrors(t); +} + +test bool picoTypo() { + t = parsePico("begin declare input : natural, + output : natural, + repnr : natural, + rep : natural; + input := 14; + output := 0; + while input - 1 do + rep := output; + repnr := input; + while repnr - 1 do + output := output x rep; + repnr := repnr - 1 + od; + input := input - 1 + od +end"); + iprintln(findFirstError(t)); + return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "x rep; + repnr := repnr - 1 + od"; +} + +test bool picoMissingSemi() { + t = parsePico("begin declare input : natural, + output : natural, + repnr : natural, + rep : natural; + input := 14; + output := 0; + while input - 1 do + rep := output; + repnr := input; + while repnr - 1 do + output := output + rep; + repnr := repnr - 1 + od + input := input - 1 + od +end"); + str errorText = getErrorText(findFirstError(t)); + println("error count: "); + println("error text: "); + + for (error <- findAllErrors(t)) { + println(" error: "); + } + + return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 + od"; +} + +test bool picoMissingTypoMinimal() { + t = parsePico( +"begin declare; + while input do + input x= 14; + output := 0 + od +end", visualize=true); + + for (error <- findAllErrors(t)) { + println(" error: "); + } + +return !hasErrors(t); + /*str errorText = getErrorText(findFirstError(t)); + println("error text: "); + return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 + od"; + */ +} +test bool picoMissingSemiMinimal() { + t = parsePico( +"begin declare; + while input do + input := 14 + output := 0 + od +end", visualize=true); + + for (error <- findAllErrors(t)) { + println(" error: "); + } + +return !hasErrors(t); + /*str errorText = getErrorText(findFirstError(t)); + println("error text: "); + return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 + od"; + */ +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc new file mode 100644 index 00000000000..3a5e589a2dd --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc @@ -0,0 +1,9 @@ +module lang::rascal::tests::recovery::RunRecoveryTests + +import lang::rascal::tests::recovery::BasicRecoveryTests; +import util::REPL; + +int main(list[str] args){ + Terminal terminal = newREPL(repl()); + return 0; +} diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 2bf7774ed65..c8e37cf7bf4 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -154,8 +154,6 @@ public SGTDBF(){ unmatchableLeafNodes = new Stack>(); unmatchableMidProductionNodes = new DoubleStack, AbstractNode>, AbstractStackNode

>(); filteredNodes = new DoubleStack, AbstractNode>(); - - visualizer = new DebugVisualizer("Parser"); } /** @@ -835,8 +833,8 @@ private boolean findFirstStacksToReduce(){ if (recoverer != null) { if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); - visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); } + visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); @@ -866,7 +864,7 @@ private boolean findStacksToReduce(){ DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; if(!(terminalsTodo == null || terminalsTodo.isEmpty())){ - if (debugListener != null) { + if (DebugVisualizer.VISUALIZATION_ENABLED) { NodeId reduceNodeId = new NodeId("todo-" + i); visualize("Found stack to reduce", reduceNodeId); } @@ -881,18 +879,21 @@ private boolean findStacksToReduce(){ if (recoverer != null && location < input.length) { if (debugListener != null) { - visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } + visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); + } + if (DebugVisualizer.VISUALIZATION_ENABLED && visualizer != null) { // Visualize state and include recovered nodes visualizer.createGraph(this, "Reviving"); visualizer.addRecoveredNodes(recoveredNodes); visualizer.writeGraph(); opportunityToBreak(); } + if (recoveredNodes.size() > 0) { // was: for (int i = recoveredNodes.size()-1; i>= 0; i--) { for (int i = 0; i < recoveredNodes.size(); i++) { @@ -902,8 +903,8 @@ private boolean findStacksToReduce(){ if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); - visualize("Queue recovery node", DebugVisualizer.getNodeId(recovered)); } + visualize("Queue recovery node", DebugVisualizer.getNodeId(recovered)); queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), recoveredNodes.getSecond(i)); } return findStacksToReduce(); @@ -1261,6 +1262,9 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] this.recoverer = recoverer; this.debugListener = debugListener; + + String query = inputURI.getQuery(); + visualizer = query != null && query.contains("visualize=true") ? new DebugVisualizer("Parser") : null; // Initialzed the position store. positionStore.index(input); @@ -1466,7 +1470,7 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo private void visualize(String step, NodeId highlight) { // Only visualize when debugging - if (debugListener != null) { + if (DebugVisualizer.VISUALIZATION_ENABLED && visualizer != null) { visualizer.createGraph(this, step); if (highlight != null) { visualizer.highlight(highlight); @@ -1477,7 +1481,7 @@ private void visualize(String step, NodeId highlight) { } } - private void opportunityToBreak() { + public static void opportunityToBreak() { } diff --git a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java index 9bf625a33fb..169a5e2fd66 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java @@ -117,7 +117,20 @@ public int getLength(){ public AbstractNode getResult(){ return result; } + + public int[][] getLiteral() { + return ciLiteral; + } + @Override + public String toShortString() { + int[] codePoints = new int[ciLiteral.length]; + for (int i=0; i extends AbstractMatchableStackNode

{ private final int[][] ranges; @@ -86,6 +87,33 @@ public AbstractNode getResult(){ return result; } + @Override + public String toShortString() { + StringBuilder sb = new StringBuilder(); + for (int i=0; i 0) { + sb.append(','); + } + + int[] range = ranges[i]; + sb.append(codePointToString(range[0])); + if (range[0] != range[1]) { + sb.append('-'); + sb.append(codePointToString(range[1])); + } + } + + return sb.toString(); + } + + private String codePointToString(int codePoint) { + if (Character.isLetterOrDigit(codePoint)) { + return new String(Character.toChars(codePoint)); + } + + return String.valueOf(codePoint); + } + public String toString(){ StringBuilder sb = new StringBuilder(); @@ -94,7 +122,7 @@ public String toString(){ sb.append(range[0]); sb.append('-'); sb.append(range[1]); - for(int i = ranges.length - 2; i >= 0; --i){ + for(int i = 1; i original, int startLocation){ super(original, startLocation); diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index 38adc9cf619..15418e849c5 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -31,8 +31,11 @@ public SeparatedListStackNode(int id, int dot, P production, AbstractStackNode

child, AbstractStackNode

[] separators, boolean isPlusList, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index ae0bcf41be0..e6edd477d0e 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -55,8 +55,8 @@ public static SkippedNode createResultUntilToken(String token, int[] input, int return null; } - public static SkippedNode createResultUntilChar(int[] input, int startLocation, int length, IConstructor production, int dot) { - return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); + public static SkippedNode createResultUntilChar(int[] input, int startLocation, int endLocation, IConstructor production, int dot) { + return new SkippedNode(production, dot, createSkippedToken(input, startLocation, endLocation - startLocation), startLocation); } private static int[] createSkippedToken(int[] input, int startLocation, int length) { diff --git a/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java new file mode 100644 index 00000000000..587bd419748 --- /dev/null +++ b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java @@ -0,0 +1,66 @@ +package org.rascalmpl.parser.uptr.recovery; + +public class CaseInsensitiveLiteralMatcher implements InputMatcher { + private final int[] chars; + private final int[] altChars; + + public CaseInsensitiveLiteralMatcher(String literal) { + int length = literal.length(); + chars = new int[length]; + altChars = new int[length]; + for(int i = 0; i InputMatcher createMatcher(AbstractStackNode

stackNode) { + if (stackNode instanceof LiteralStackNode) { + return new LiteralMatcher(((LiteralStackNode

) stackNode).getLiteral()); + } + + if (stackNode instanceof CaseInsensitiveLiteralStackNode) { + return new CaseInsensitiveLiteralMatcher(((CaseInsensitiveLiteralStackNode

) stackNode).getLiteral()); + } + + return null; + } +} diff --git a/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java new file mode 100644 index 00000000000..b5531cff372 --- /dev/null +++ b/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java @@ -0,0 +1,41 @@ +package org.rascalmpl.parser.uptr.recovery; + +public class LiteralMatcher implements InputMatcher { + private int[] chars; + + public LiteralMatcher(String literal) { + chars = new int[literal.length()]; + for (int i=0; i { - private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; - private IdDispenser stackNodeIdDispenser; public ToTokenRecoverer(IdDispenser stackNodeIdDispenser) { @@ -71,6 +78,9 @@ private DoubleArrayList, AbstractNode> reviveNod recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); + DebugVisualizer visualizer = new DebugVisualizer("Recovery"); + visualizer.visualizeRecoveryNodes(recoveryNodes); + for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); @@ -114,42 +124,130 @@ List> findSkippingNodes(int[] input, AbstractSta } // Try to find whitespace to skip to + /* result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); if (result != null) { nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); - } + }*/ // Do something more clever: find the last token of this production and skip until after that - String lastLiteral = findLastLiteral(prod); - if (lastLiteral != null) { - result = SkippingStackNode.createResultUntilToken(lastLiteral, input, startLocation, prod, dot); - if (result != null) { + List endMatchers = findEndMatchers(prod); + for (InputMatcher endMatcher : endMatchers) { + MatchResult endMatch = endMatcher.findMatch(input, startLocation); + if (endMatch != null) { + result = SkippingStackNode.createResultUntilChar(input, startLocation, endMatch.getEnd(), prod, dot); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); } } - // TODO: or find the first token of the next production and skip until before that + // Find the first token of the next production and skip until before that + List nextMatchers = findNextMatchers(recoveryNode); + for (InputMatcher nextMatcher : nextMatchers) { + MatchResult nextMatch = nextMatcher.findMatch(input, startLocation); + if (nextMatch != null) { + result = SkippingStackNode.createResultUntilChar(input, startLocation, nextMatch.getStart(), prod, dot); + nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); + } + } return nodes; } - String findLastLiteral(IConstructor prod) { + // Gather matchers for the last token of a production + List findEndMatchers(IConstructor prod) { IList args = (IList) prod.get(1); if (args.isEmpty()) { - return null; + return Collections.emptyList(); } IValue last = args.get(args.length()-1); if (last instanceof IConstructor) { - IConstructor cons = (IConstructor) last; - if (cons.getConstructorType() == RascalValueFactory.Symbol_Lit) { - return ((IString) cons.get(0)).getValue(); + return Collections.singletonList(InputMatcher.createMatcher((IConstructor) last)); + } + + return Collections.emptyList(); + } + + void forAllParents(AbstractStackNode stackNode, Consumer> consumer) { + IntegerObjectList> edges = stackNode.getEdges(); + if (edges != null) { + for (int i = edges.size() - 1; i >= 0; --i) { + EdgesSet edgesList = edges.getValue(i); + if (edgesList != null) { + for (int j = edgesList.size() - 1; j >= 0; --j) { + consumer.accept(edgesList.get(j)); + } + } + } + } + } + + AbstractStackNode getSingleParentStack(AbstractStackNode stackNode) { + if (stackNode == null) { + return null; + } + + IntegerObjectList> edges = stackNode.getEdges(); + if (edges != null) { + EdgesSet edgesList = edges.getValue(0); + if (edgesList != null) { + return edgesList.get(0); } } return null; } + // Find matchers for the first token after the current stack node + List findNextMatchers(AbstractStackNode stackNode) { + DebugVisualizer visualizer = new DebugVisualizer("findNextMatcher"); + visualizer.visualize(stackNode); + + final List matchers = new java.util.ArrayList<>(); + + AbstractStackNode parent = getSingleParentStack(stackNode); + if (parent == null) { + return matchers; + } + + AbstractStackNode[] prod = parent.getProduction(); + if (prod == null) { + return matchers; + } + + int nextDot = parent.getDot() + 1; + if (nextDot >= prod.length) { + return matchers; + } + + AbstractStackNode next = prod[nextDot]; + if (next instanceof NonTerminalStackNode && next.getName().startsWith("layouts_")) { + // Look "through" layout for now, should be more general to look through any node that can be empty + nextDot++; + if (nextDot >= prod.length) { + return matchers; + } + next = prod[nextDot]; + } + + if (next instanceof LiteralStackNode) { + LiteralStackNode nextLiteral = (LiteralStackNode) next; + matchers.add(new LiteralMatcher(nextLiteral.getLiteral())); + } + + if (next instanceof CaseInsensitiveLiteralStackNode) { + CaseInsensitiveLiteralStackNode nextLiteral = (CaseInsensitiveLiteralStackNode) next; + matchers.add(new CaseInsensitiveLiteralMatcher(nextLiteral.getLiteral())); + } + + if (next instanceof NonTerminalStackNode) { + NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; + SGTDBF.opportunityToBreak(); + } + + return matchers; + } + // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) // As this is experimental code, this method is extremely conservative. // Any sharing will result in returning 'false'. @@ -227,6 +325,7 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr todo.push(failer); + boolean useNext = true; // Use this to only use the last active element from a production while (!todo.isEmpty()) { AbstractStackNode node = todo.pop(); @@ -236,13 +335,14 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr visited.put(node, 0); - if (node == failer || node.getDot() == 0) { + if (useNext) { ArrayList recoveryProductions = new ArrayList(); collectProductions(node, recoveryProductions); if (recoveryProductions.size() > 0) { - recoveryNodes.add(node, recoveryProductions); + addRecoveryNode(node, recoveryProductions, recoveryNodes); } } + useNext = node.getDot() == 0; IntegerObjectList> edges = node.getEdges(); @@ -259,6 +359,28 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr } } } + + // Only add recovery nodes that are not already present. + private void addRecoveryNode(AbstractStackNode node, ArrayList productions, DoubleArrayList, ArrayList> recoveryNodes) { + for (int i=0; i prods = recoveryNodes.getSecond(i); + if (prods.size() == productions.size()) { + boolean equal = true; + for (int j=0; equal && j node, ArrayList productions) { @@ -269,12 +391,10 @@ private void collectProductions(AbstractStackNode node, ArrayList< int dot = node.getDot(); - System.err.println("collect productions for node: " + node); if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)){ productions.add(parentProduction); - System.err.println("adding production: " + parentProduction); if (ProductionAdapter.isList(parentProduction)) { return; // Don't follow productions in lists productions, since they are 'cyclic'. diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index a99a0fa4606..11a3808482f 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -16,7 +16,7 @@ public static String prodToString(IConstructor prod) { builder.append(" ->"); - if (prod.getName() == "prod") { + if (prod.getName().equals("prod")) { IList children = (IList) prod.get(1); for (IValue child : children) { builder.append(" "); @@ -35,7 +35,7 @@ public static String prodToString(IConstructor prod) { private static String stripQuotes(String s) { if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"') { - return s.substring(1, s.length()-1); + return s.substring(1, s.length()-1).replace("\\", ""); } return s; diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java index b0fb64f41ca..c944242d5a3 100644 --- a/src/org/rascalmpl/util/visualize/DebugVisualizer.java +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -12,7 +12,9 @@ import java.nio.file.StandardCopyOption; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -30,10 +32,10 @@ import org.rascalmpl.parser.gtd.result.RecoveredNode; import org.rascalmpl.parser.gtd.result.SkippedNode; import org.rascalmpl.parser.gtd.result.SortContainerNode; -import org.rascalmpl.parser.gtd.result.struct.Link; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; +import org.rascalmpl.parser.gtd.util.ArrayList; import org.rascalmpl.parser.gtd.util.DoubleArrayList; import org.rascalmpl.parser.gtd.util.DoubleStack; import org.rascalmpl.parser.gtd.util.IntegerObjectList; @@ -53,7 +55,8 @@ public class DebugVisualizer { static final String BASE_DIR = "D:/debug/parser-traces/docs/"; - private static final boolean VISUALIZATION_ENABLED = true; + public static final boolean VISUALIZATION_ENABLED = true; + private static final boolean INCLUDE_PRODUCTIONS = false; public static final NodeId PARSER_ID = new NodeId("Parser"); public static final NodeId TODO_LISTS_ID= new NodeId("todoLists"); @@ -119,6 +122,14 @@ public void visualize(AbstractStackNode node) { } } + public void visualizeRecoveryNodes(DoubleArrayList, ArrayList> recoveryNodes) { + writeGraph(createGraph(recoveryNodes)); + } + + public void visualizeProductionTrees(AbstractStackNode[] nodes) { + writeGraph(createProductionGraph(nodes)); + } + public int getFrame() { return frame; } @@ -149,6 +160,76 @@ private synchronized DotGraph createGraph(AbstractStackNode stackN return graph; } + private DotGraph createGraph(DoubleArrayList, ArrayList> recoveryNodes) { + reset(); + graph = new DotGraph(name, true); + final NodeId RECOVERY_NODES_ID = new NodeId("recovery-nodes"); + + DotNode arrayNode = DotNode.createArrayNode(RECOVERY_NODES_ID, recoveryNodes.size()); + graph.addNode(arrayNode); + + for (int i=0; i[] stackNodes) { + reset(); + graph = new DotGraph(name, true); + for (AbstractStackNode stackNode : stackNodes) { + addProductionNodes(graph, stackNode); + } + return graph; + } + + private

NodeId addProductionNodes(DotGraph graph, AbstractStackNode

stackNode) { + DotNode node = createDotNode(stackNode); + graph.addNode(node); + + AbstractStackNode

[] prods = stackNode.getProduction(); + if (prods != null) { + NodeId prodArrayId = new NodeId(node.getId() + "-prod"); + graph.addArrayNode(prodArrayId, prods.length); + for (int i=0; i child = prods[i]; + DotNode childNode = createDotNode(child); + graph.addNode(childNode); + graph.addEdge(new NodeId(prodArrayId, String.valueOf(i)), childNode.getId()); + } + + graph.addEdge(node.getId(), prodArrayId, "Production"); + } + + return node.getId(); + } + + private void addProductionArray(DotGraph graph, NodeId nodeId, ArrayList productions) { + DotNode arrayNode = DotNode.createArrayNode(nodeId, productions.size()); + graph.addNode(arrayNode); + for (int i=0; i void addRecoveredNodes(DoubleArrayList, AbstractNode> recoveredNodes) { addStackAndNodeDoubleList(graph, RECOVERED_NODES_ID, recoveredNodes); graph.addEdge(ERROR_TRACKING_ID, RECOVERED_NODES_ID, "Nodes to revive"); @@ -167,6 +248,10 @@ private

DotNode addStack(DotGraph graph, AbstractStackNode

stackNode) { graph.addNode(node); + if (INCLUDE_PRODUCTIONS) { + addProductionNodes(graph, stackNode); + } + IntegerObjectList> edges = stackNode.getEdges(); if (edges != null) { for (int i = edges.size() - 1; i >= 0; --i) { @@ -207,9 +292,28 @@ private

DotNode createDotNode(AbstractStackNode

stackNode) { } } + int dot = stackNode.getDot(); + + String extraInfo = ""; + if (stackNode.isMatchable()) { + extraInfo += ",matchable"; + } + if (stackNode.isSeparator()) { + extraInfo += ",sep"; + } + if (stackNode.isExpandable()) { + extraInfo += ",expandable"; + } + if (stackNode.isLayout()) { + extraInfo += ",layout"; + } + if (stackNode.isEndNode()) { + extraInfo += ",end"; + } + DotNode node = new DotNode(getNodeId(stackNode)); - String label = String.format("%s: %s\n.%d@%d", - type, nodeName, stackNode.getDot(), stackNode.getStartLocation()); + String label = String.format("%s: %s\n.%d@%d %s", + type, nodeName, dot, stackNode.getStartLocation(), extraInfo); String shortString = stackNode.toShortString(); if (shortString != null) { diff --git a/src/org/rascalmpl/util/visualize/dot/DotGraph.java b/src/org/rascalmpl/util/visualize/dot/DotGraph.java index 27293e2f1e5..dc11db9202d 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotGraph.java +++ b/src/org/rascalmpl/util/visualize/dot/DotGraph.java @@ -32,8 +32,10 @@ private void addStatement(DotStatement statement) { } public void addNode(DotNode node) { - addStatement(node); - nodes.put(node.getId(), node); + if (!nodes.containsKey(node.getId())) { + addStatement(node); + nodes.put(node.getId(), node); + } } public void addNode(String id, String label) { diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index ca7d5ed073a..bac01801576 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -582,7 +582,7 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IDebugListener debugListener = null; if (allowRecovery) { recoverer = new ToTokenRecoverer(new StackNodeIdDispenser(parserInstance)); - debugListener = new DebugLogger(new PrintWriter(System.out, true)); + //debugListener = new DebugLogger(new PrintWriter(System.out, true)); } return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); } From 8ef03b400b659c32ca727ee73e3ca5f18f1f26e7 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Tue, 20 Aug 2024 08:55:32 +0200 Subject: [PATCH 43/87] Minor cleanup of parser tests and debug support --- .../rascal/tests/recovery/PicoRecoveryTests.rsc | 4 ++-- src/org/rascalmpl/parser/gtd/SGTDBF.java | 14 +++++--------- src/org/rascalmpl/parser/util/DebugUtil.java | 7 +++++++ 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc index c4683085179..7c2b4aad77e 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -93,7 +93,7 @@ end", visualize=true); println(" error: "); } -return !hasErrors(t); +return hasErrors(t); /*str errorText = getErrorText(findFirstError(t)); println("error text: "); return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 @@ -113,7 +113,7 @@ end", visualize=true); println(" error: "); } -return !hasErrors(t); +return hasErrors(t); /*str errorText = getErrorText(findFirstError(t)); println("error text: "); return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index c8e37cf7bf4..a3dd982bbc5 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -46,6 +46,7 @@ import org.rascalmpl.parser.gtd.util.IntegerList; import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.Stack; +import org.rascalmpl.parser.util.DebugUtil; import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.util.visualize.dot.NodeId; @@ -763,9 +764,7 @@ private void moveToNext(AbstractStackNode

node, AbstractNode result){ */ private void move(AbstractStackNode

node, AbstractNode result){ if(debugListener != null) debugListener.moving(node, result); - if (node instanceof RecoveryPointStackNode) { - opportunityToBreak(); - } + // Handle filtering. ICompletionFilter[] completionFilters = node.getCompletionFilters(); if(completionFilters != null){ @@ -891,7 +890,7 @@ private boolean findStacksToReduce(){ visualizer.createGraph(this, "Reviving"); visualizer.addRecoveredNodes(recoveredNodes); visualizer.writeGraph(); - opportunityToBreak(); + DebugUtil.opportunityToBreak(); } if (recoveredNodes.size() > 0) { @@ -1477,14 +1476,11 @@ private void visualize(String step, NodeId highlight) { } visualizer.writeGraph(); - opportunityToBreak(); + DebugUtil.opportunityToBreak(); } } - public static void opportunityToBreak() { - } - - + /** * Getters used for graph generation/debugging */ diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index 11a3808482f..4e821916fea 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -40,4 +40,11 @@ private static String stripQuotes(String s) { return s; } + + /* + * Nop method that allows breakpoints to be set at the call site even if originally there is no code to break on + */ + public static void opportunityToBreak() { + } + } From 7d82f20aed3ae4fe71d3d706071ad42d6421c02c Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 21 Aug 2024 09:08:31 +0200 Subject: [PATCH 44/87] Implemented error tree fixup (including source locations) --- src/org/rascalmpl/library/ParseTree.rsc | 43 +++---- .../tests/recovery/BasicRecoveryTests.rsc | 8 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 107 +++++++++++++++++- .../parser/gtd/result/SkippedNode.java | 13 ++- .../gtd/result/out/DefaultNodeFlattener.java | 2 +- .../result/out/INodeConstructorFactory.java | 4 +- .../gtd/result/out/RecoveryNodeFlattener.java | 14 ++- .../parser/gtd/stack/SkippingStackNode.java | 20 ++-- .../parser/uptr/UPTRNodeFactory.java | 21 +++- .../recovery/ToNextWhitespaceRecoverer.java | 4 +- .../uptr/recovery/ToTokenRecoverer.java | 44 ++++--- src/org/rascalmpl/parser/util/DebugUtil.java | 6 +- .../values/RascalFunctionValueFactory.java | 6 +- .../rascalmpl/values/RascalValueFactory.java | 3 +- 14 files changed, 211 insertions(+), 84 deletions(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 1f88edb3ef1..efb1b386534 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -190,7 +190,8 @@ data Production | \reference(Symbol def, str cons) // <5> ; -data Production = skipped(Symbol def, Production prod, int dot); +data Production = error(Symbol def, Production prod, int dot) + | skipped(Symbol symbol); @synopsis{Attributes in productions.} @description{ @@ -769,40 +770,32 @@ bool isNonTerminalType(Symbol::\start(Symbol s)) = isNonTerminalType(s); default bool isNonTerminalType(Symbol s) = false; @synopsis{Check if a parse tree contains any skipped nodes, the result of error recovery.} -bool hasErrors(Tree tree) = /appl(skipped(_, _, _), _) := tree; +bool hasErrors(Tree tree) = /appl(error(_, _, _), _) := tree; @synopsis{Find all error productions in a parse tree.} -list[Tree] findAllErrors(Tree tree) { - return for (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) append(error); -} +list[Tree] findAllErrors(Tree tree) = [err | /err:appl(error(_, _, _), _) := tree]; @synopsis{Find the first production containing an error.} Tree findFirstError(Tree tree) { - if (/error:appl(_, [*_, appl(skipped(_, _, _), _)]) := tree) return error; + if (/err:appl(error(_, _, _), _) := tree) return err; fail; } -private Production getSkipped(appl(_, [*_, appl(skip:skipped(_, _, _), _)])) = skip; +@synopsis{Get the symbol (sort) of the failing production} +Symbol getErrorSymbol(appl(error(Symbol sym, _, _), _)) = sym; -Symbol getErrorSymbol(Tree error) { - if (skipped(Symbol s, _, _) := getSkipped(error)) { - return s; - } - fail; -} +@synopsis{Get the production that failed} +Production getErrorProduction(appl(error(_, Production prod, _), _)) = prod; -Production getErrorProduction(Tree error) { - if (skipped(_, Production p, _) := getSkipped(error)) { - return p; - } - fail; -} +@synopsis{Get the dot (position in the production) of the failing element in a production} +int getErrorDot(appl(error(_, _, int dot), _)) = dot; -int getErrorPosition(Tree error) { - if (skipped(_, _, int dot) := getSkipped(error)) { - return dot; - } - fail; +@synopsis{Get the skipped tree} +Tree getSkipped(appl(error(_, _, _), [*_, skip:appl(skipped(_), _)])) { + return skip; } -str getErrorText(appl(_, [*_, appl(skipped(_, _, _), chars)])) = stringChars([c | ch <- chars, char(c) := ch]); +@synopsis{Get the text that failed to parse} +str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) { + return stringChars([c | ch <- chars, char(c) := ch]); +} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index e84ba513118..8be50b9704a 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -25,15 +25,13 @@ test bool abx() { test bool axc() { Tree t = parseS("a x c $", visualize=true); - iprintln(t); + iprintln(getSkipped(findFirstError(t))@\loc); return getErrorText(findFirstError(t)) == "x c"; } +/* We need to look through non-terminals to make this test work test bool ax() { Tree t = parseS("a x $", visualize=true); return getErrorText(findFirstError(t)) = "x"; } - -int main(list[str] args){ - startRepl(REPL()); -} +*/ diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index a3dd982bbc5..7d5a2659227 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -13,6 +13,7 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.net.URI; +import java.util.concurrent.atomic.AtomicBoolean; import org.rascalmpl.parser.gtd.debug.IDebugListener; import org.rascalmpl.parser.gtd.exception.ParseError; @@ -34,7 +35,6 @@ import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.EpsilonStackNode; import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; -import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.stack.filter.ICompletionFilter; import org.rascalmpl.parser.gtd.stack.filter.IEnterFilter; @@ -49,6 +49,13 @@ import org.rascalmpl.parser.util.DebugUtil; import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.util.visualize.dot.NodeId; +import org.rascalmpl.values.RascalValueFactory; + +import io.usethesource.vallang.IConstructor; +import io.usethesource.vallang.IList; +import io.usethesource.vallang.ISet; +import io.usethesource.vallang.IValue; +import io.usethesource.vallang.type.Type; /** * This is the core of the parser; it drives the parse process. @@ -1445,8 +1452,11 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo finally { actionExecutor.completed(rootEnvironment, (parseResult == null)); } - if(parseResult != null){ - return parseResult; // Success. + if(parseResult != null) { + if (recoverer != null) { + parseResult = fixErrorNodes(parseResult, nodeConstructorFactory); + } + return parseResult; // Success. } int offset = filteringTracker.getOffset(); @@ -1463,6 +1473,97 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo } } + /** + * Error nodes end up in a n inconvenient form because of the parser algorithm. + * This post-processing step transforms the original tree into a more useful form. + * In essence, error subtrees look like this after parsing: + * `appl(prod(S,[]), [,,...,appl(skipped(S,prod(S,[]),),[)]])` + * This method pulls up the skipped node, transforming these trees into: + * `appl(skipped(S,prod(S,[]),), [,,...,[])])` + * This means productions that failed to parse can be recognized at the top level. + * Note that this can only be done when we know the actual type of T is IConstructor. + */ + @SuppressWarnings("unchecked") + T fixErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { + if (!(tree instanceof IConstructor)) { + return tree; + } + + return (T) fixErrorNodes((IConstructor) tree, (INodeConstructorFactory) nodeConstructorFactory); + } + + IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { + IConstructor result; + Type type = tree.getConstructorType(); + if (type == RascalValueFactory.Tree_Appl) { + IValue prod = tree.get(0); + IList childList = (IList) tree.get(1); + int childCount = childList.length(); + + ArrayList children = new ArrayList<>(childCount); + boolean anyChanges = false; + boolean errorTree = false; + for (int i=0; i alternatives = new ArrayList<>(alternativeSet.size()); + final AtomicBoolean anyChanges = new AtomicBoolean(false); + alternativeSet.forEach(alt -> { + IConstructor newAlt = fixErrorNodes((IConstructor) alt, nodeConstructorFactory); + if (newAlt != alt) { + anyChanges.setPlain(true); + } + alternatives.add(newAlt); + }); + if (anyChanges.getPlain()) { + result = nodeConstructorFactory.createAmbiguityNode(alternatives); + } else { + result = tree; + } + } else if (type == RascalValueFactory.Tree_Cycle) { + result = tree; + } else { + throw new RuntimeException("Unrecognized tree type: " + type); + } + + if (result != tree) { + IValue loc = tree.asWithKeywordParameters().getParameter(RascalValueFactory.Location); + result = result.asWithKeywordParameters().setParameter(RascalValueFactory.Location, loc); + } + + return result; + } + /** * Datastructure visualization for debugging purposes */ diff --git a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java index 48882533b2b..53578520ce2 100644 --- a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java +++ b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java @@ -11,22 +11,27 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.result; +import java.net.URI; + import org.rascalmpl.unicode.UnicodeConverter; /** * Result tree node that represents a skipped portion of the input sentence. */ public class SkippedNode extends AbstractNode { - public final static int ID = 9; + public static final int ID = 9; + // TODO: only "skippedChars" is actually needed + private final URI input; private final Object production; private final int dot; private final int[] skippedChars; private final int offset; - public SkippedNode(Object production, int dot, int[] skippedChars, int offset) { + public SkippedNode(URI input, Object production, int dot, int[] skippedChars, int offset) { super(); + this.input = input; this.production = production; this.dot = dot; this.skippedChars = skippedChars; @@ -37,6 +42,10 @@ public int getTypeIdentifier(){ return ID; } + public URI getInput() { + return input; + } + public Object getProduction() { return production; } diff --git a/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java index ad16a892f8a..4f1655d800f 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java @@ -66,7 +66,7 @@ public T convert(INodeConstructorFactory nodeConstructorFactory, AbstractN case RecoveredNode.ID: return convert(nodeConstructorFactory, ((SortContainerNode) node).getFirstAlternative().getNode(), stack, depth, cycleMark, positionStore, filteringTracker, actionExecutor, environment); case SkippedNode.ID: - return recoveryNodeConverter.convertToUPTR(nodeConstructorFactory, (SkippedNode) node); + return recoveryNodeConverter.convertToUPTR(nodeConstructorFactory, (SkippedNode) node, positionStore); default: throw new RuntimeException("Incorrect result node id: "+node.getTypeIdentifier()); } diff --git a/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java b/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java index 5a94f7c3b64..1de4b441a58 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java +++ b/src/org/rascalmpl/parser/gtd/result/out/INodeConstructorFactory.java @@ -38,7 +38,9 @@ public interface INodeConstructorFactory { T createListAmbiguityNode(ArrayList alternatives); - T createRecoveryNode(int dot, ArrayList recognizedPrefix, int[] unrecognizedCharacters, Object production); + T createSkippedNode(int[] unrecognizedCharacters); + + T createErrorNode(ArrayList children, Object production); ArrayList getChildren(T node); diff --git a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java index b7a2605bfa9..973e39e120c 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java @@ -11,8 +11,8 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.result.out; +import org.rascalmpl.parser.gtd.location.PositionStore; import org.rascalmpl.parser.gtd.result.SkippedNode; -import org.rascalmpl.parser.gtd.util.ArrayList; /** * A converter for result nodes that contain skipped characters for error recovery @@ -23,7 +23,15 @@ public RecoveryNodeFlattener(){ super(); } - public T convertToUPTR(INodeConstructorFactory nodeConstructorFactory, SkippedNode node){ - return nodeConstructorFactory.createRecoveryNode(node.getDot(), new ArrayList<>(), node.getSkippedChars(), node.getProduction()); + public T convertToUPTR(INodeConstructorFactory nodeConstructorFactory, SkippedNode node, PositionStore positionStore){ + T result = nodeConstructorFactory.createSkippedNode(node.getSkippedChars()); + + // Add source location + int startOffset = node.getOffset(); + int endOffset = startOffset + node.getLength(); + P sourceLocation = nodeConstructorFactory.createPositionInformation(node.getInput(), startOffset, endOffset, positionStore); + result = nodeConstructorFactory.addPositionInformation(result, sourceLocation); + + return result; } } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index e6edd477d0e..b4ddf1e8ec3 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -11,6 +11,8 @@ *******************************************************************************/ package org.rascalmpl.parser.gtd.stack; +import java.net.URI; + import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.result.SkippedNode; @@ -19,25 +21,25 @@ public final class SkippingStackNode

extends AbstractMatchableStackNode

{ private final SkippedNode result; - public static SkippedNode createResultUntilCharClass(int[] until, int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilCharClass(URI uri, int[] until, int[] input, int startLocation, IConstructor production, int dot) { for (int to = startLocation ; to < input.length; ++to) { for (int i = 0; i < until.length; ++i) { if (input[to] == until[i]) { int length = to - startLocation; - return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); + return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, length), startLocation); } } } - return new SkippedNode(production, dot, new int[0], startLocation); + return new SkippedNode(uri, production, dot, new int[0], startLocation); } - public static SkippedNode createResultUntilEndOfInput(int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilEndOfInput(URI uri, int[] input, int startLocation, IConstructor production, int dot) { int length = input.length - startLocation; - return new SkippedNode(production, dot, createSkippedToken(input, startLocation, length), startLocation); + return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, length), startLocation); } - public static SkippedNode createResultUntilToken(String token, int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilToken(URI uri, String token, int[] input, int startLocation, IConstructor production, int dot) { int length = token.length(); for (int start=startLocation; start+length < input.length; start++) { boolean match = true; @@ -47,7 +49,7 @@ public static SkippedNode createResultUntilToken(String token, int[] input, int } if (match) { - return createResultUntilChar(input, startLocation, start+length-startLocation, production, dot); + return createResultUntilChar(uri, input, startLocation, start+length-startLocation, production, dot); } } } @@ -55,8 +57,8 @@ public static SkippedNode createResultUntilToken(String token, int[] input, int return null; } - public static SkippedNode createResultUntilChar(int[] input, int startLocation, int endLocation, IConstructor production, int dot) { - return new SkippedNode(production, dot, createSkippedToken(input, startLocation, endLocation - startLocation), startLocation); + public static SkippedNode createResultUntilChar(URI uri, int[] input, int startLocation, int endLocation, IConstructor production, int dot) { + return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, endLocation - startLocation), startLocation); } private static int[] createSkippedToken(int[] input, int startLocation, int length) { diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index b971764ccfd..d3aa2a63a7e 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -1,6 +1,7 @@ package org.rascalmpl.parser.uptr; import java.net.URI; +import java.util.Arrays; import java.util.IdentityHashMap; import java.util.Map; @@ -13,7 +14,6 @@ import io.usethesource.vallang.IListWriter; import io.usethesource.vallang.ISetWriter; import io.usethesource.vallang.ISourceLocation; -import io.usethesource.vallang.IValue; import org.rascalmpl.values.RascalValueFactory; import org.rascalmpl.values.ValueFactoryFactory; @@ -23,6 +23,8 @@ public class UPTRNodeFactory implements INodeConstructorFactory{ private static final RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory(); + private static final IConstructor SKIPPED = VF.constructor(RascalValueFactory.Production_Skipped, VF.constructor(RascalValueFactory.Symbol_IterStar, VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Range, VF.integer(1), VF.integer(Character.MAX_CODE_POINT)))))); + private boolean allowAmb; public UPTRNodeFactory(boolean allowAmbiguity){ @@ -142,7 +144,21 @@ public Object getProductionFromNode(ITree node){ } @Override - public ITree createRecoveryNode(int dot, ArrayList recognizedPrefix, int[] unrecognizedCharacters, Object production) { + public ITree createSkippedNode(int[] characters) { + IList chars = Arrays.stream(characters).mapToObj(VF::character).collect(VF.listWriter()); + return VF.appl(SKIPPED, chars); + } + + public ITree createErrorNode(ArrayList children, Object production) { + IConstructor prod = (IConstructor) production; + IConstructor errorProd = VF.constructor(RascalValueFactory.Production_Error, prod.get(0), prod, VF.integer(children.size()-1)); + return buildAppl(children, errorProd); + } + + /* + @Override + public ITree createRecoveryNode(int[] characters) { + } IListWriter args = VF.listWriter(); for (int i=0; i recognizedPrefix, int[ IConstructor skipped = VF.constructor(RascalValueFactory.Production_Skipped, prod.get(0), (IValue) production, VF.integer(dot)); return VF.appl(skipped, args.done()); } + */ } diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 7ca1a20a46f..7b1e9439af8 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -80,9 +80,9 @@ private DoubleArrayList, AbstractNode> reviveNod SkippedNode result; if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { - result = SkippingStackNode.createResultUntilEndOfInput(input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilEndOfInput(null, input, startLocation, prod, dot); } else { - result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilCharClass(null, WHITESPACE, input, startLocation, prod, dot); } AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result); diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 2c779efdef1..1815b5d4448 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -12,13 +12,11 @@ *******************************************************************************/ package org.rascalmpl.parser.uptr.recovery; +import java.net.URI; import java.util.Collections; import java.util.List; -import java.util.Set; -import java.util.TreeSet; import java.util.function.Consumer; -import org.rascalmpl.parser.gtd.SGTDBF; import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.result.SkippedNode; @@ -27,7 +25,6 @@ import org.rascalmpl.parser.gtd.stack.LiteralStackNode; import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; -import org.rascalmpl.parser.gtd.stack.SeparatedListStackNode; import org.rascalmpl.parser.gtd.stack.SkippingStackNode; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.util.ArrayList; @@ -38,6 +35,7 @@ import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; import org.rascalmpl.parser.uptr.recovery.InputMatcher.MatchResult; +import org.rascalmpl.parser.util.DebugUtil; import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.values.parsetrees.ProductionAdapter; @@ -46,9 +44,11 @@ import io.usethesource.vallang.IValue; public class ToTokenRecoverer implements IRecoverer { + private URI uri; private IdDispenser stackNodeIdDispenser; - public ToTokenRecoverer(IdDispenser stackNodeIdDispenser) { + public ToTokenRecoverer(URI uri, IdDispenser stackNodeIdDispenser) { + this.uri = uri; this.stackNodeIdDispenser = stackNodeIdDispenser; } @@ -60,17 +60,17 @@ public DoubleArrayList, AbstractNode> reviveStac DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, DoubleStack, AbstractNode> filteredNodes) { - ArrayList> failedNodes = new ArrayList>(); + ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); - // TODO: handle unmatchableLeafNodes - //collectFilteredNodes(filteredNodes, failedNodes); + // handle unmatchableLeafNodes + // collectFilteredNodes(filteredNodes, failedNodes); return reviveFailedNodes(input, location, failedNodes); } private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ - DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); + DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { // But this caused problems because recovery nodes with a later position @@ -118,7 +118,7 @@ List> findSkippingNodes(int[] input, AbstractSta // If we are the top-level node, just skip the rest of the input if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { - result = SkippingStackNode.createResultUntilEndOfInput(input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilEndOfInput(uri, input, startLocation, prod, dot); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); return nodes; // No other nodes would be useful } @@ -135,7 +135,7 @@ List> findSkippingNodes(int[] input, AbstractSta for (InputMatcher endMatcher : endMatchers) { MatchResult endMatch = endMatcher.findMatch(input, startLocation); if (endMatch != null) { - result = SkippingStackNode.createResultUntilChar(input, startLocation, endMatch.getEnd(), prod, dot); + result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, endMatch.getEnd(), prod, dot); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); } } @@ -145,7 +145,7 @@ List> findSkippingNodes(int[] input, AbstractSta for (InputMatcher nextMatcher : nextMatchers) { MatchResult nextMatch = nextMatcher.findMatch(input, startLocation); if (nextMatch != null) { - result = SkippingStackNode.createResultUntilChar(input, startLocation, nextMatch.getStart(), prod, dot); + result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart(), prod, dot); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); } } @@ -241,8 +241,8 @@ List findNextMatchers(AbstractStackNode stackNode) { } if (next instanceof NonTerminalStackNode) { - NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; - SGTDBF.opportunityToBreak(); + //NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; + DebugUtil.opportunityToBreak(); } return matchers; @@ -278,7 +278,7 @@ private AbstractStackNode getSinglePredecessor(AbstractStackNode, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { - DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); + DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList<>(); for (int i = failedNodes.size() - 1; i >= 0; --i) { findRecoveryNodes(failedNodes.get(i), recoveryNodes); @@ -320,8 +320,8 @@ private static void collectUnmatchableMidProductionNodes(int location, DoubleSta * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. */ private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { - ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap>(); - Stack> todo = new Stack>(); + ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap<>(); + Stack> todo = new Stack<>(); todo.push(failer); @@ -336,7 +336,7 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr visited.put(node, 0); if (useNext) { - ArrayList recoveryProductions = new ArrayList(); + ArrayList recoveryProductions = new ArrayList<>(); collectProductions(node, recoveryProductions); if (recoveryProductions.size() > 0) { addRecoveryNode(node, recoveryProductions, recoveryNodes); @@ -389,8 +389,6 @@ private void collectProductions(AbstractStackNode node, ArrayList< return; // The root node does not have a production, so ignore it. } - int dot = node.getDot(); - if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)){ @@ -402,13 +400,13 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } + int dot = node.getDot(); for (int i = dot + 1; i < production.length; ++i) { AbstractStackNode currentNode = production[i]; if (currentNode.isEndNode()) { IConstructor parentProduction = currentNode.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)) { productions.add(parentProduction); - System.err.println("adding production at " + i + ": " + parentProduction); } } @@ -419,7 +417,5 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } } - } - - + } } diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index 4e821916fea..e1c29c6b084 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -41,10 +41,8 @@ private static String stripQuotes(String s) { return s; } - /* - * Nop method that allows breakpoints to be set at the call site even if originally there is no code to break on - */ public static void opportunityToBreak() { + // Nop method that allows breakpoints to be set at the call site even if originally there is no code to break on } - + } diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index bac01801576..b9fa5e44989 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -16,6 +16,7 @@ import java.io.PrintWriter; import java.io.Reader; import java.lang.reflect.InvocationTargetException; +import java.net.URI; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -580,11 +581,12 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IGTD parserInstance = getParser(); IRecoverer recoverer = null; IDebugListener debugListener = null; + URI uri = location.getURI(); if (allowRecovery) { - recoverer = new ToTokenRecoverer(new StackNodeIdDispenser(parserInstance)); + recoverer = new ToTokenRecoverer(uri, new StackNodeIdDispenser(parserInstance)); //debugListener = new DebugLogger(new PrintWriter(System.out, true)); } - return (ITree) parserInstance.parse(methodName, location.getURI(), input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); + return (ITree) parserInstance.parse(methodName, uri, input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); } } diff --git a/src/org/rascalmpl/values/RascalValueFactory.java b/src/org/rascalmpl/values/RascalValueFactory.java index ad3b5f4d668..3c4a7218b5d 100644 --- a/src/org/rascalmpl/values/RascalValueFactory.java +++ b/src/org/rascalmpl/values/RascalValueFactory.java @@ -152,7 +152,8 @@ public class RascalValueFactory extends AbstractValueFactoryAdapter implements I public static final Type Production_Priority = tf.constructor(uptr, Production, "priority", Symbol, "def", tf.listType(Production), "choices"); public static final Type Production_Composition = tf.constructor(uptr, Production, "composition", Production, "lhs", Production, "rhs"); public static final Type Production_Associativity = tf.constructor(uptr, Production, "associativity", Symbol, "def", Associativity, "assoc", tf.setType(Production), "alternatives"); - public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped", Symbol, "def", Production, "prod", tf.integerType(), "dot"); + public static final Type Production_Error = tf.constructor(uptr, Production, "error", Symbol, "def", Production, "prod", tf.integerType(), "dot"); + public static final Type Production_Skipped = tf.constructor(uptr, Production, "skipped", Symbol, "def"); /* Constructors for Attr */ public static final Type Attr_Assoc = tf.constructor(uptr, Attr, "assoc", Associativity, "assoc"); From 706662bb2af4bee91e81f67dfccc01bb15392a92 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 21 Aug 2024 09:19:42 +0200 Subject: [PATCH 45/87] Renamed RecoveryNodeFlattener to SkippedNodeFlattener --- .../lang/rascal/tests/recovery/BasicRecoveryTests.rsc | 2 +- .../parser/gtd/result/out/DefaultNodeFlattener.java | 10 +++++----- ...eryNodeFlattener.java => SkippedNodeFlattener.java} | 5 ++--- 3 files changed, 8 insertions(+), 9 deletions(-) rename src/org/rascalmpl/parser/gtd/result/out/{RecoveryNodeFlattener.java => SkippedNodeFlattener.java} (94%) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index 8be50b9704a..bfc309359ce 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -25,7 +25,7 @@ test bool abx() { test bool axc() { Tree t = parseS("a x c $", visualize=true); - iprintln(getSkipped(findFirstError(t))@\loc); + iprintln(getSkipped(findFirstError(t)).src); return getErrorText(findFirstError(t)) == "x c"; } diff --git a/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java index 4f1655d800f..12ca785fb41 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/DefaultNodeFlattener.java @@ -25,13 +25,12 @@ /** * Converter for parse trees that produces trees in UPTR format. */ -@SuppressWarnings("unchecked") public class DefaultNodeFlattener implements INodeFlattener{ private final CharNodeFlattener charNodeConverter; private final LiteralNodeFlattener literalNodeConverter; private final SortContainerNodeFlattener sortContainerNodeConverter; private final ListContainerNodeFlattener listContainerNodeConverter; - private final RecoveryNodeFlattener recoveryNodeConverter; + private final SkippedNodeFlattener skippedNodeConverter; public DefaultNodeFlattener(){ super(); @@ -40,7 +39,7 @@ public DefaultNodeFlattener(){ literalNodeConverter = new LiteralNodeFlattener(); sortContainerNodeConverter = new SortContainerNodeFlattener(); listContainerNodeConverter = new ListContainerNodeFlattener(); - recoveryNodeConverter = new RecoveryNodeFlattener(); + skippedNodeConverter = new SkippedNodeFlattener(); } /** @@ -53,6 +52,7 @@ protected static class IsInError{ /** * Convert the given node. */ + @SuppressWarnings("unchecked") public T convert(INodeConstructorFactory nodeConstructorFactory, AbstractNode node, IndexedStack stack, int depth, CycleMark cycleMark, PositionStore positionStore, FilteringTracker filteringTracker, IActionExecutor actionExecutor, Object environment){ switch(node.getTypeIdentifier()){ case CharNode.ID: @@ -66,7 +66,7 @@ public T convert(INodeConstructorFactory nodeConstructorFactory, AbstractN case RecoveredNode.ID: return convert(nodeConstructorFactory, ((SortContainerNode) node).getFirstAlternative().getNode(), stack, depth, cycleMark, positionStore, filteringTracker, actionExecutor, environment); case SkippedNode.ID: - return recoveryNodeConverter.convertToUPTR(nodeConstructorFactory, (SkippedNode) node, positionStore); + return skippedNodeConverter.convertToUPTR(nodeConstructorFactory, (SkippedNode) node, positionStore); default: throw new RuntimeException("Incorrect result node id: "+node.getTypeIdentifier()); } @@ -76,6 +76,6 @@ public T convert(INodeConstructorFactory nodeConstructorFactory, AbstractN * Converts the given parse tree to a tree in UPTR format. */ public T convert(INodeConstructorFactory nodeConstructorFactory, AbstractNode parseTree, PositionStore positionStore, FilteringTracker filteringTracker, IActionExecutor actionExecutor, Object rootEnvironment){ - return convert(nodeConstructorFactory, parseTree, new IndexedStack(), 0, new CycleMark(), positionStore, filteringTracker, actionExecutor, rootEnvironment); + return convert(nodeConstructorFactory, parseTree, new IndexedStack<>(), 0, new CycleMark(), positionStore, filteringTracker, actionExecutor, rootEnvironment); } } diff --git a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java similarity index 94% rename from src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java rename to src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java index 973e39e120c..d0b5877e7c8 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/RecoveryNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java @@ -17,9 +17,8 @@ /** * A converter for result nodes that contain skipped characters for error recovery */ -public class RecoveryNodeFlattener{ - - public RecoveryNodeFlattener(){ +public class SkippedNodeFlattener{ + public SkippedNodeFlattener(){ super(); } From 7d6812965d2761e9626285003603f8ca95b9a71f Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 22 Aug 2024 10:36:00 +0200 Subject: [PATCH 46/87] Implemented basic error disambiguation --- src/org/rascalmpl/library/ParseTree.rsc | 63 +++++++++++++++++++ .../tests/recovery/BasicRecoveryTests.rsc | 4 +- .../tests/recovery/PicoRecoveryTests.rsc | 14 ++++- 3 files changed, 77 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index efb1b386534..74ab80f4a7c 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -142,8 +142,11 @@ module ParseTree extend Type; extend Message; extend List; +extend Set; import String; +import IO; +import Node; @synopsis{The Tree data type as produced by the parser.} @description{ @@ -178,6 +181,8 @@ construct ordered and un-ordered compositions, and associativity groups. <4> `assoc` means all alternatives are acceptable, but nested on the declared side; <5> `reference` means a reference to another production rule which should be substituted there, for extending priority chains and such. +<6> `error` means a node produced by error recovery. +<7> `skipped` means characters skipped during error recovery, always the last child of an `appl` with a `error` production. } data Production = prod(Symbol def, list[Symbol] symbols, set[Attr] attributes) // <1> @@ -799,3 +804,61 @@ Tree getSkipped(appl(error(_, _, _), [*_, skip:appl(skipped(_), _)])) { str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) { return stringChars([c | ch <- chars, char(c) := ch]); } + +@synopsis{Error recovery often produces ambiguous trees where errors can be recovered in multiple ways. +This filter removes error trees until no ambiguities caused by error recovery are left. +Note that regular ambiguous trees remain in the parse tree. +} +Tree defaultErrorDisambiguationFilter(t: appl(Production prod, args)) { + Tree result = appl(prod, [defaultErrorDisambiguationFilter(arg) | arg <- args]); + return setKeywordParameters(result, getKeywordParameters(t)); +} + +Tree defaultErrorDisambiguationFilter(amb(set[Tree] alternatives)) { + // Go depth-first + set[Tree] disambiguatedAlts = { defaultErrorDisambiguationFilter(alt) | Tree alt <- alternatives }; + + set[Tree] errorTrees = { alt | Tree alt <- disambiguatedAlts, appl(error(_,_,_), _) := alt }; + set[Tree] nonErrorTrees = { alt | Tree alt <- disambiguatedAlts, appl(error(_,_,_), _) !:= alt }; + + if (nonErrorTrees == {}) { + return getBestErrorTree(errorTrees); + } else if ({Tree single} := nonErrorTrees) { + // One ambiguity left, no ambiguity concerns here + return single; + } + + // Multiple non-error trees left, return an ambiguity node with just the non-error trees + return amb(nonErrorTrees); +} + +private Tree getBestErrorTree(set[Tree] trees) { + Tree best = char(0); + int bestErrorCount = -1; + int bestErrorLength = 0; + + for (tree <- trees) { + list[Tree] errors = findAllErrors(tree); + int errorCount = size(errors); + int errorLength = 0; + for (err <- errors) { + errorLength += getSkipped(err).src.length; + } + + if (bestErrorCount == -1 || errorCount < bestErrorCount || (errorCount == bestErrorCount && errorLength < bestErrorLength)) { + best = tree; + bestErrorCount = errorCount; + bestErrorLength = errorLength; + } + } + + if (bestErrorCount != -1) { + return best; + } + + // trees must have been empty + fail; +} + +// Handle char and cycle nodes +default Tree defaultErrorDisambiguationFilter(Tree t) = t; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index bfc309359ce..f2e1e129787 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -25,7 +25,9 @@ test bool abx() { test bool axc() { Tree t = parseS("a x c $", visualize=true); - iprintln(getSkipped(findFirstError(t)).src); + iprintln(t); + println("after disambiguation:"); + iprintln(defaultErrorDisambiguationFilter(t)); return getErrorText(findFirstError(t)) == "x c"; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc index 7c2b4aad77e..5d3236ba653 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -5,7 +5,7 @@ import lang::pico::\syntax::Main; import ParseTree; import IO; -private Tree parsePico(str input, bool visualize=false) +Tree parsePico(str input, bool visualize=false) = parser(#Program, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool picoOk() { @@ -80,19 +80,27 @@ end"); od"; } -test bool picoMissingTypoMinimal() { +test bool picoTypoMinimal() { t = parsePico( "begin declare; while input do input x= 14; output := 0 od -end", visualize=true); +end", visualize=false); + + iprintln(t); for (error <- findAllErrors(t)) { println(" error: "); } + disambiguated = defaultErrorDisambiguationFilter(t); + println("after disambiguation:"); + for (error <- findAllErrors(disambiguated)) { + println(" error: "); + } + return hasErrors(t); /*str errorText = getErrorText(findFirstError(t)); println("error text: "); From 6b2939c447fbae9bb76b12d92ed5f03db741c081 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Thu, 22 Aug 2024 11:31:22 +0200 Subject: [PATCH 47/87] Added support for error and skipped productions in vis::Text --- src/org/rascalmpl/library/vis/Text.rsc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/org/rascalmpl/library/vis/Text.rsc b/src/org/rascalmpl/library/vis/Text.rsc index 566a229c92d..5c8c6846925 100644 --- a/src/org/rascalmpl/library/vis/Text.rsc +++ b/src/org/rascalmpl/library/vis/Text.rsc @@ -41,6 +41,8 @@ str prettyTree(Tree t, bool src=false, bool characters=true, bool \layout=false, str nodeLabel(appl(prod(label(str l, Symbol nt), _, _), _)) = " = : "; str nodeLabel(appl(prod(Symbol nt, as, _), _)) = " = <}>"; + str nodeLabel(appl(error(Symbol nt, Production p, int dot), _)) = "!error dot=: "; + str nodeLabel(appl(skipped(Symbol s), chars)) = "skipped"; str nodeLabel(appl(regular(Symbol nt), _)) = ""; str nodeLabel(char(32)) = "⎵"; str nodeLabel(char(10)) = "\\r"; From ff350caf1c06fcde38de223b9236e4914d77486a Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Mon, 26 Aug 2024 13:50:58 +0200 Subject: [PATCH 48/87] Look "through" non-terminals when finding end/next matchers --- src/org/rascalmpl/library/ParseTree.rsc | 4 +- .../tests/recovery/RascalRecoveryTests.rsc | 163 ++++++++++++++++++ .../lang/rascal/tests/recovery/Test.rsc | 1 + .../lang/rascal/tests/recovery/ToyRascal.rsc | 17 ++ .../tests/recovery/ToyRascalRecoveryTests.rsc | 37 ++++ .../rascalmpl/parser/gtd/ExpectsProvider.java | 7 + src/org/rascalmpl/parser/gtd/IGTD.java | 2 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 39 +++-- .../gtd/stack/AlternativeStackNode.java | 4 + .../parser/gtd/stack/EmptyStackNode.java | 4 + .../parser/gtd/stack/ListStackNode.java | 4 + .../gtd/stack/MultiCharacterStackNode.java | 4 + .../gtd/stack/NonTerminalStackNode.java | 4 + .../parser/gtd/stack/OptionalStackNode.java | 4 + .../gtd/stack/SeparatedListStackNode.java | 4 + .../parser/gtd/stack/SequenceStackNode.java | 4 + .../parser/gtd/stack/SkippingStackNode.java | 4 + .../uptr/recovery/ToTokenRecoverer.java | 142 ++++++++++----- .../values/RascalFunctionValueFactory.java | 2 +- 19 files changed, 383 insertions(+), 67 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc create mode 100644 src/org/rascalmpl/parser/gtd/ExpectsProvider.java diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 74ab80f4a7c..2e4c22af8a1 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -800,7 +800,9 @@ Tree getSkipped(appl(error(_, _, _), [*_, skip:appl(skipped(_), _)])) { return skip; } -@synopsis{Get the text that failed to parse} +@synopsis{Get the text that failed to parse. This is only the text of the part that has been skipped to be able to continue parsing. +If you want the text of the whole error tree, you can just use string interpolation: "". +} str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) { return stringChars([c | ch <- chars, char(c) := ch]); } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc new file mode 100644 index 00000000000..7c17abd899d --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc @@ -0,0 +1,163 @@ +module lang::rascal::tests::recovery::RascalRecoveryTests + +import lang::rascal::\syntax::Rascal; + +import ParseTree; +import IO; + +Tree parseRascal(str input, bool visualize=false) { + Tree result = parser(#start[Module], allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + list[Tree] errors = findAllErrors(result); + if (errors != []) { + println("Tree has errors"); + for (error <- errors) { + println("- "); + } + + Tree disambiguated = defaultErrorDisambiguationFilter(result); + println("Best error: "); + } + + return result; +} + +Tree parseFunctionDeclaration(str input, bool visualize=false) { + Tree result = parser(#FunctionDeclaration, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + list[Tree] errors = findAllErrors(result); + if (errors != []) { + println("Tree has errors"); + for (error <- errors) { + println("- "); + } + + Tree disambiguated = defaultErrorDisambiguationFilter(result); + println("Best error: "); + } + + return result; +} + +test bool rascalOk() { + Tree t = parseRascal(" + module A + + int inc(int i) { + return i+1; + } + "); + return !hasErrors(t); +} + +test bool rascalFunctionDeclarationOk() { + Tree t = parseFunctionDeclaration("void f(){}"); + return !hasErrors(t); +} + + +test bool rascalModuleFollowedBySemi() { + Tree t = parseRascal(" + module A + ; + "); + + // There are a lot of productions in Rascal that have a ; as terminator. + // The parser assumes the user has only entered the ; on one of them, + // so the error list contains them all. + list[Tree] errors = findAllErrors(t); + assert size(errors) == 10; + + return getErrorText(findFirstError(t)) == ";"; +} + +test bool rascalOperatorTypo() { + Tree t = parseRascal(" + module A + + int f() = 1 x 1; + "); + + println("error text: "); + return getErrorText(findFirstError(t)) == "x 1"; +} + +test bool rascalIllegalStatement() { + Tree t = parseRascal("module A void f(){a}"); + + println("error text: "); + return getErrorText(findFirstError(t)) == "a}"; +} + +test bool rascalMissingCloseParen() { + Tree t = parseRascal("module A void f({} void g(){}"); + + assert getErrorText(findFirstError(t)) == "void g("; + assert getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "("; + + return true; +} + +test bool rascalFunctionDeclarationMissingCloseParen() { + Tree t = parseFunctionDeclaration("void f({} void g() {}", visualize=false); + + assert getErrorText(findFirstError(t)) == "void g("; + + Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); + assert getErrorText(error) == "("; + Tree skipped = getSkipped(error); + loc location = getSkipped(error).src; + assert location.begin.column == 16 && location.length == 1; + + return true; +} + +// Not working yet: +/* +test bool rascalMissingOpeningParen() { + Tree t = parseRascal("module A void f){} void g() { }"); + + println("error text: "); + return getErrorText(findFirstError(t)) == "a}"; +} + +test bool rascalFunFunMissingCloseParen() { + Tree t = parseRascal("module A void f(){void g({}} void h(){}"); + + println("error text: "); + return getErrorText(findFirstError(t)) == "a}"; +} + +test bool rascalIfMissingExpr() { + Tree t = parseRascal("module A void f(){if(){}} 1;", visualize=false); + + println("error text: "); + return getErrorText(findFirstError(t)) == ";"; +} + +test bool rascalIfMissingOpeningParen() { + Tree t = parseRascal("module A void f(){if 1){}}", visualize=false); + + println("error text: "); + return getErrorText(findFirstError(t)) == ";"; +} + +test bool rascalIfMissingCloseParen() { + Tree t = parseRascal("module A void f(){if(1{}}", visualize=false); + + println("error text: "); + return getErrorText(findFirstError(t)) == ";"; +} + +test bool rascalIfEmptyBody() { + Tree t = parseRascal("module A void f(){if(1){}} 1;"); + + println("error text: "); + return getErrorText(findFirstError(t)) == ";"; +} + +test bool rascalIfMissingSemi() { + Tree t = parseRascal("module A void f(){if (true) {a}}"); + + println("error text: "); + return getErrorText(findFirstError(t)) == ";"; +} +*/ diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc new file mode 100644 index 00000000000..455e3661677 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc @@ -0,0 +1 @@ +module A void f(){ void g() {}} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc new file mode 100644 index 00000000000..81a9fd1c5b2 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc @@ -0,0 +1,17 @@ +module lang::rascal::tests::recovery::ToyRascal + +start syntax FunctionDeclaration = Signature FunctionBody; + +syntax Signature = Name Parameters; + +syntax Name = "f" | "g"; + +syntax Parameters = "(" ")"; + +syntax FunctionBody = "{" Statement* statements "}" ; + +syntax Statement + = "{" Statement+ statements "}" + | "s" ";"; + +layout Layout = [ \n\r\t]*; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc new file mode 100644 index 00000000000..455197c1f35 --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc @@ -0,0 +1,37 @@ +module lang::rascal::tests::recovery::ToyRascalRecoveryTests + +import lang::rascal::tests::recovery::ToyRascal; + +import ParseTree; +import IO; + +Tree parseToyRascal(str input, bool visualize=false) { + Tree result = parser(#start[FunctionDeclaration], allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); + list[Tree] errors = findAllErrors(result); + if (errors != []) { + println("Tree has errors"); + for (error <- errors) { + println("- "); + } + + Tree disambiguated = defaultErrorDisambiguationFilter(result); + println("Best error: "); + } + + return result; +} + +test bool toyRascalOk() { + Tree t = parseToyRascal("f(){s;}"); + return !hasErrors(t); +} + +test bool toyRascalMissingOpenParen() { + Tree t = parseToyRascal("f){}", visualize=true); + return hasErrors(t) && getErrorText(findFirstError(t)) == ")"; +} + +test bool toyRascalMissingCloseParen() { + Tree t = parseToyRascal("f({}", visualize=true); + return hasErrors(t) && getErrorText(findFirstError(t)) == ")"; +} diff --git a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java new file mode 100644 index 00000000000..18266a7d10d --- /dev/null +++ b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java @@ -0,0 +1,7 @@ +package org.rascalmpl.parser.gtd; + +import org.rascalmpl.parser.gtd.stack.AbstractStackNode; + +public interface ExpectsProvider

{ + AbstractStackNode

[] getExpects(String nonTerminal); +} diff --git a/src/org/rascalmpl/parser/gtd/IGTD.java b/src/org/rascalmpl/parser/gtd/IGTD.java index bb4e9b5c656..0e61d06b213 100644 --- a/src/org/rascalmpl/parser/gtd/IGTD.java +++ b/src/org/rascalmpl/parser/gtd/IGTD.java @@ -22,7 +22,7 @@ /** * Parser interface. */ -public interface IGTD{ +public interface IGTD extends ExpectsProvider

{ /** * Parse the input string, using the given non-terminal as start node. If * the parse process successfully completes a result will be constructed diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 7d5a2659227..1ed1b597447 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -60,7 +60,7 @@ /** * This is the core of the parser; it drives the parse process. */ -public abstract class SGTDBF implements IGTD{ +public abstract class SGTDBF implements IGTD { private final static int DEFAULT_TODOLIST_CAPACITY = 16; private URI inputURI; @@ -173,35 +173,38 @@ protected int getFreeStackNodeId() { throw new UnsupportedOperationException(); } - /** - * Triggers the gathering of alternatives for the given non-terminal. - */ @SuppressWarnings("unchecked") - protected AbstractStackNode

[] invokeExpects(AbstractStackNode

nonTerminal){ - String name = nonTerminal.getName(); - AbstractStackNode

[] expects = expectCache.get(name); - if(expects == null){ + public AbstractStackNode

[] getExpects(String nonTerminal) { + AbstractStackNode

[] expects = expectCache.get(nonTerminal); + if (expects == null) { try{ - Method method = getClass().getMethod(name); - try{ + Method method = getClass().getMethod(nonTerminal); + try { method.setAccessible(true); // Try to bypass the 'isAccessible' check to save time. - }catch(SecurityException sex){ + } catch (SecurityException sex) { // Ignore this if it happens. } expects = (AbstractStackNode

[]) method.invoke(this); - }catch(NoSuchMethodException nsmex){ - throw new UndeclaredNonTerminalException(name, getClass()); - }catch(IllegalAccessException iaex){ + } catch (NoSuchMethodException nsmex) { + throw new UndeclaredNonTerminalException(nonTerminal, getClass()); + } catch (IllegalAccessException iaex) { throw new RuntimeException(iaex); - }catch(InvocationTargetException itex){ + } catch (InvocationTargetException itex) { throw new RuntimeException(itex.getTargetException()); } - expectCache.putUnsafe(name, expects); + expectCache.putUnsafe(nonTerminal, expects); } - - return expects; + + return expects; + } + + /** + * Triggers the gathering of alternatives for the given non-terminal. + */ + protected AbstractStackNode

[] invokeExpects(AbstractStackNode

nonTerminal){ + return getExpects(nonTerminal.getName()); } /** diff --git a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java index 6412f806d25..dceb05517ec 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java @@ -88,6 +88,10 @@ public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append("alt"); diff --git a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java index 79297837d49..685c839065d 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java @@ -80,6 +80,10 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); diff --git a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java index 7f5734b2a10..a513a9472c9 100644 --- a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java @@ -93,6 +93,10 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); diff --git a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java index 6543b14f9e1..3b0ed831748 100644 --- a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java @@ -104,6 +104,10 @@ public AbstractNode getResult(){ return result; } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index 247562026f7..3114bc97baf 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -76,6 +76,10 @@ public AbstractNode getResult(){ throw new UnsupportedOperationException(); } + public String toShortString() { + return expectIdentifier; + } + public String toString(){ StringBuilder sb = new StringBuilder("NonTerminal["); sb.append(expectIdentifier); diff --git a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java index 4b916f2a3be..c4694769941 100644 --- a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java @@ -92,6 +92,10 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index 15418e849c5..3310d11076b 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -110,6 +110,10 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); diff --git a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java index 638d978d00b..6d25c37750e 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java @@ -86,6 +86,10 @@ public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } + public String toShortString() { + return toString(); + } + public String toString(){ StringBuilder sb = new StringBuilder(); sb.append("seq"); diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index b4ddf1e8ec3..4acefaf19d3 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -127,6 +127,10 @@ public AbstractNode getResult(){ return sb.toString(); }*/ + public String toShortString() { + return "skip(" + result.toString() + ")"; + } + @Override public String toString() { return "SkippingStackNode[result=" + result + "," + super.toString() + "]"; diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 1815b5d4448..9ef73497c44 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -9,14 +9,18 @@ * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI * * Arnold Lankamp - Arnold.Lankamp@cwi.nl + * * Pieter Olviier - Pieter.Olivier@swat.engineering *******************************************************************************/ package org.rascalmpl.parser.uptr.recovery; import java.net.URI; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.Consumer; +import org.rascalmpl.parser.gtd.ExpectsProvider; import org.rascalmpl.parser.gtd.recovery.IRecoverer; import org.rascalmpl.parser.gtd.result.AbstractNode; import org.rascalmpl.parser.gtd.result.SkippedNode; @@ -35,7 +39,6 @@ import org.rascalmpl.parser.gtd.util.ObjectKeyedIntegerMap; import org.rascalmpl.parser.gtd.util.Stack; import org.rascalmpl.parser.uptr.recovery.InputMatcher.MatchResult; -import org.rascalmpl.parser.util.DebugUtil; import org.rascalmpl.util.visualize.DebugVisualizer; import org.rascalmpl.values.parsetrees.ProductionAdapter; @@ -46,9 +49,11 @@ public class ToTokenRecoverer implements IRecoverer { private URI uri; private IdDispenser stackNodeIdDispenser; + private ExpectsProvider expectsProvider; - public ToTokenRecoverer(URI uri, IdDispenser stackNodeIdDispenser) { + public ToTokenRecoverer(URI uri, ExpectsProvider expectsProvider, IdDispenser stackNodeIdDispenser) { this.uri = uri; + this.expectsProvider = expectsProvider; this.stackNodeIdDispenser = stackNodeIdDispenser; } @@ -60,22 +65,19 @@ public DoubleArrayList, AbstractNode> reviveStac DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, DoubleStack, AbstractNode> filteredNodes) { + // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to improve error recovery. + ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); - // handle unmatchableLeafNodes - // collectFilteredNodes(filteredNodes, failedNodes); - return reviveFailedNodes(input, location, failedNodes); + return reviveFailedNodes(input, failedNodes); } - private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, DoubleArrayList, ArrayList> recoveryNodes){ DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); - - // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { - // But this caused problems because recovery nodes with a later position - // where queued before nodes with an earlier position which the parser cannot handle. + // Sort nodes by start location recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); DebugVisualizer visualizer = new DebugVisualizer("Recovery"); @@ -124,14 +126,16 @@ List> findSkippingNodes(int[] input, AbstractSta } // Try to find whitespace to skip to + // This often creates hopeless recovery attempts, but it might help in some cases. + // Further experimentation should quantify this statement. /* result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); if (result != null) { nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); }*/ - // Do something more clever: find the last token of this production and skip until after that - List endMatchers = findEndMatchers(prod); + // Find the last token of this production and skip until after that + List endMatchers = findEndMatchers(recoveryNode); for (InputMatcher endMatcher : endMatchers) { MatchResult endMatch = endMatcher.findMatch(input, startLocation); if (endMatch != null) { @@ -143,7 +147,7 @@ List> findSkippingNodes(int[] input, AbstractSta // Find the first token of the next production and skip until before that List nextMatchers = findNextMatchers(recoveryNode); for (InputMatcher nextMatcher : nextMatchers) { - MatchResult nextMatch = nextMatcher.findMatch(input, startLocation); + MatchResult nextMatch = nextMatcher.findMatch(input, startLocation+1); if (nextMatch != null) { result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart(), prod, dot); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); @@ -154,7 +158,7 @@ List> findSkippingNodes(int[] input, AbstractSta } // Gather matchers for the last token of a production - List findEndMatchers(IConstructor prod) { + List findEndMatchersBasedOnProduction(IConstructor prod) { IList args = (IList) prod.get(1); if (args.isEmpty()) { return Collections.emptyList(); @@ -168,6 +172,50 @@ List findEndMatchers(IConstructor prod) { return Collections.emptyList(); } + // Find matchers for the last token of the current stack node + List findEndMatchers(AbstractStackNode stackNode) { + final List matchers = new java.util.ArrayList<>(); + + AbstractStackNode[] prod = stackNode.getProduction(); + addEndMatchers(prod, prod.length-1, matchers, new HashSet<>()); + + return matchers; + } + + void addEndMatchers(AbstractStackNode[] prod, int dot, List matchers, Set visitedNodes) { + if (prod == null || dot < 0) { + return; + } + + AbstractStackNode last = prod[dot]; + if (visitedNodes.contains(last.getId())) { + return; + } + visitedNodes.add(last.getId()); + + // Future improvement: while (isNullable(last) addEndMatchers(prod, dot-1, matchers); + + if (last instanceof LiteralStackNode) { + LiteralStackNode lastLiteral = (LiteralStackNode) last; + matchers.add(new LiteralMatcher(lastLiteral.getLiteral())); + } + + if (last instanceof CaseInsensitiveLiteralStackNode) { + CaseInsensitiveLiteralStackNode lastLiteral = (CaseInsensitiveLiteralStackNode) last; + matchers.add(new CaseInsensitiveLiteralMatcher(lastLiteral.getLiteral())); + } + + if (last instanceof NonTerminalStackNode) { + NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) last; + String name = nextNonTerminal.getName(); + AbstractStackNode[] alternatives = expectsProvider.getExpects(name); + for (AbstractStackNode alternative : alternatives) { + // In the future we might want to use all prefix-sharing alternatives here + addEndMatchers(alternative.getProduction(), 0, matchers, visitedNodes); + } + } + } + void forAllParents(AbstractStackNode stackNode, Consumer> consumer) { IntegerObjectList> edges = stackNode.getEdges(); if (edges != null) { @@ -182,7 +230,7 @@ void forAllParents(AbstractStackNode stackNode, Consumer getSingleParentStack(AbstractStackNode stackNode) { + private AbstractStackNode getSingleParentStack(AbstractStackNode stackNode) { if (stackNode == null) { return null; } @@ -200,35 +248,39 @@ AbstractStackNode getSingleParentStack(AbstractStackNode findNextMatchers(AbstractStackNode stackNode) { - DebugVisualizer visualizer = new DebugVisualizer("findNextMatcher"); - visualizer.visualize(stackNode); - final List matchers = new java.util.ArrayList<>(); + // Future improvement: use all parents instead of just one AbstractStackNode parent = getSingleParentStack(stackNode); if (parent == null) { return matchers; } - AbstractStackNode[] prod = parent.getProduction(); - if (prod == null) { - return matchers; - } + addNextMatchers(parent.getProduction(), parent.getDot()+1, matchers, new HashSet<>()); - int nextDot = parent.getDot() + 1; - if (nextDot >= prod.length) { - return matchers; + return matchers; + } + + private void addNextMatchers(AbstractStackNode[] prod, int dot, List matchers, Set visitedNodes) { + if (prod == null || dot >= prod.length) { + return; } - AbstractStackNode next = prod[nextDot]; - if (next instanceof NonTerminalStackNode && next.getName().startsWith("layouts_")) { - // Look "through" layout for now, should be more general to look through any node that can be empty - nextDot++; - if (nextDot >= prod.length) { - return matchers; + AbstractStackNode next = prod[dot]; + while (next instanceof NonTerminalStackNode && next.getName().startsWith("layouts_")) { + // Look "through" layout for now, this should really be more general and look through any node that can be empty + // When a node can be empty, we should also consider all prefix-shared alternatives. + dot++; + if (dot >= prod.length) { + return; } - next = prod[nextDot]; + next = prod[dot]; + } + + if (visitedNodes.contains(next.getId())) { + return; } + visitedNodes.add(next.getId()); if (next instanceof LiteralStackNode) { LiteralStackNode nextLiteral = (LiteralStackNode) next; @@ -241,11 +293,13 @@ List findNextMatchers(AbstractStackNode stackNode) { } if (next instanceof NonTerminalStackNode) { - //NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; - DebugUtil.opportunityToBreak(); + NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; + String name = nextNonTerminal.getName(); + AbstractStackNode[] alternatives = expectsProvider.getExpects(name); + for (AbstractStackNode alternative : alternatives) { + addNextMatchers(alternative.getProduction(), 0, matchers, visitedNodes); + } } - - return matchers; } // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) @@ -277,14 +331,14 @@ private AbstractStackNode getSinglePredecessor(AbstractStackNode, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { + private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, ArrayList> failedNodes) { DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList<>(); for (int i = failedNodes.size() - 1; i >= 0; --i) { findRecoveryNodes(failedNodes.get(i), recoveryNodes); } - return reviveNodes(input, location, recoveryNodes); + return reviveNodes(input, recoveryNodes); } private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { @@ -325,7 +379,6 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr todo.push(failer); - boolean useNext = true; // Use this to only use the last active element from a production while (!todo.isEmpty()) { AbstractStackNode node = todo.pop(); @@ -335,14 +388,11 @@ private void findRecoveryNodes(AbstractStackNode failer, DoubleArr visited.put(node, 0); - if (useNext) { - ArrayList recoveryProductions = new ArrayList<>(); - collectProductions(node, recoveryProductions); - if (recoveryProductions.size() > 0) { - addRecoveryNode(node, recoveryProductions, recoveryNodes); - } + ArrayList recoveryProductions = new ArrayList<>(); + collectProductions(node, recoveryProductions); + if (recoveryProductions.size() > 0) { + addRecoveryNode(node, recoveryProductions, recoveryNodes); } - useNext = node.getDot() == 0; IntegerObjectList> edges = node.getEdges(); diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index b9fa5e44989..f7ff0997972 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -583,7 +583,7 @@ private ITree parseObject(String methodName, ISourceLocation location, char[] in IDebugListener debugListener = null; URI uri = location.getURI(); if (allowRecovery) { - recoverer = new ToTokenRecoverer(uri, new StackNodeIdDispenser(parserInstance)); + recoverer = new ToTokenRecoverer(uri, parserInstance, new StackNodeIdDispenser(parserInstance)); //debugListener = new DebugLogger(new PrintWriter(System.out, true)); } return (ITree) parserInstance.parse(methodName, uri, input, exec, new DefaultNodeFlattener<>(), new UPTRNodeFactory(allowAmbiguity), recoverer, debugListener); From 2b6a69fa648246cc841250e9a4780aab90503fab Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 07:45:20 +0200 Subject: [PATCH 49/87] Miscellaneous cleanup of error recovery code --- .vscode/launch.json | 16 ---- src/org/rascalmpl/library/ParseTree.rsc | 4 +- .../tests/recovery/BasicRecoveryTests.rsc | 10 +-- .../tests/recovery/NestedRecoveryTests.rsc | 14 ++-- .../tests/recovery/PicoRecoveryTests.rsc | 73 +++++-------------- .../tests/recovery/RunRecoveryTests.rsc | 9 --- src/org/rascalmpl/library/util/TermREPL.java | 2 - src/org/rascalmpl/parser/gtd/SGTDBF.java | 43 +++++------ .../parser/gtd/recovery/IRecoverer.java | 2 +- .../parser/gtd/stack/SkippingStackNode.java | 40 ++++------ .../parser/gtd/stack/edge/EdgesSet.java | 58 +++++++-------- .../parser/gtd/util/StackNodeIdDispenser.java | 2 +- 12 files changed, 97 insertions(+), 176 deletions(-) delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc diff --git a/.vscode/launch.json b/.vscode/launch.json index e16d6d8230e..833fb54b4da 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -4,13 +4,6 @@ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ - { - "type": "java", - "name": "Simple2", - "request": "launch", - "mainClass": "org.rascalmpl.test.parser.Simple2", - "projectName": "rascal" - }, { "type": "java", "name": "Launch DocRunner", @@ -39,15 +32,6 @@ "projectName": "rascal", "vmArgs": "-Xss80m -Xmx2g -ea" }, - { - "type": "java", - "name": "Recovery tests", - "request": "launch", - "mainClass": "org.rascalmpl.shell.RascalShell", - "projectName": "rascal", - "vmArgs": "-Xss80m -Xmx2g -ea", - "args": "lang::rascal::tests::recovery::RunRecoveryTests" - }, { "type": "java", "name": "Launch RascalShell Tutor", diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 2e4c22af8a1..3cf13604f35 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -829,7 +829,7 @@ Tree defaultErrorDisambiguationFilter(amb(set[Tree] alternatives)) { // One ambiguity left, no ambiguity concerns here return single; } - + // Multiple non-error trees left, return an ambiguity node with just the non-error trees return amb(nonErrorTrees); } @@ -847,7 +847,7 @@ private Tree getBestErrorTree(set[Tree] trees) { errorLength += getSkipped(err).src.length; } - if (bestErrorCount == -1 || errorCount < bestErrorCount || (errorCount == bestErrorCount && errorLength < bestErrorLength)) { + if (bestErrorCount == -1 || errorCount < bestErrorCount || (errorCount == bestErrorCount && errorLength < bestErrorLength)) { best = tree; bestErrorCount = errorCount; bestErrorLength = errorLength; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index f2e1e129787..82a76e18b51 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -11,7 +11,7 @@ syntax T = ABC End; syntax ABC = 'a' 'b' 'c'; syntax End = "$"; -private Tree parseS(str input, bool visualize=false) +private Tree parseS(str input, bool visualize=false) = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool basicOk() { @@ -20,7 +20,7 @@ test bool basicOk() { test bool abx() { Tree t = parseS("a b x $", visualize=true); - return getErrorText(findFirstError(t)) == "x"; + return getErrorText(findFirstError(t)) == "x"; } test bool axc() { @@ -28,12 +28,10 @@ test bool axc() { iprintln(t); println("after disambiguation:"); iprintln(defaultErrorDisambiguationFilter(t)); - return getErrorText(findFirstError(t)) == "x c"; + return getErrorText(findFirstError(t)) == "x c"; } -/* We need to look through non-terminals to make this test work test bool ax() { Tree t = parseS("a x $", visualize=true); - return getErrorText(findFirstError(t)) = "x"; + return getErrorText(findFirstError(t)) == "x "; } -*/ diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc index 292043f00a4..638d25518ec 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc @@ -7,22 +7,20 @@ layout Layout = [\ ]* !>> [\ ]; syntax S = T; -syntax T = A B "c"; +syntax T = A B C; syntax A = "a"; syntax B = "b" "b"; -//syntax C = "c"; +syntax C = "c"; -private Tree parseS(str input, bool visualize=false) +private Tree parseS(str input, bool visualize=false) = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool nestedOk() { - return !hasErrors(parseS("a b b c", visualize=true)); + return !hasErrors(parseS("a b b c")); } test bool nestedTypo() { - Tree t = parseS("a b x c", visualize=true); - iprintln(t); - println("Error text: \'\'"); - return getErrorText(findFirstError(t)) == "x "; + Tree t = parseS("a b x c"); + return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x "; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc index 5d3236ba653..6abed349fac 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -5,17 +5,17 @@ import lang::pico::\syntax::Main; import ParseTree; import IO; -Tree parsePico(str input, bool visualize=false) +Tree parsePico(str input, bool visualize=false) = parser(#Program, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool picoOk() { - t = parsePico("begin declare input : natural, - output : natural, + t = parsePico("begin declare input : natural, + output : natural, repnr : natural, rep : natural; input := 14; output := 0; - while input - 1 do + while input - 1 do rep := output; repnr := input; while repnr - 1 do @@ -45,20 +45,18 @@ test bool picoTypo() { input := input - 1 od end"); - iprintln(findFirstError(t)); - return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "x rep; - repnr := repnr - 1 - od"; + Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); + return getErrorText(error) == "output x rep"; } test bool picoMissingSemi() { - t = parsePico("begin declare input : natural, - output : natural, + t = parsePico("begin declare input : natural, + output : natural, repnr : natural, rep : natural; input := 14; output := 0; - while input - 1 do + while input - 1 do rep := output; repnr := input; while repnr - 1 do @@ -68,63 +66,32 @@ test bool picoMissingSemi() { input := input - 1 od end"); - str errorText = getErrorText(findFirstError(t)); - println("error count: "); - println("error text: "); - - for (error <- findAllErrors(t)) { - println(" error: "); - } - - return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 - od"; + Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); + return getErrorText(error) == "input := input - 1\n od"; } -test bool picoTypoMinimal() { +test bool picoTypoSmall() { t = parsePico( "begin declare; while input do input x= 14; output := 0 od -end", visualize=false); - - iprintln(t); - - for (error <- findAllErrors(t)) { - println(" error: "); - } - - disambiguated = defaultErrorDisambiguationFilter(t); - println("after disambiguation:"); - for (error <- findAllErrors(disambiguated)) { - println(" error: "); - } +end"); -return hasErrors(t); - /*str errorText = getErrorText(findFirstError(t)); - println("error text: "); - return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 - od"; - */ + Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); + return getErrorText(error) == "x= 14"; } -test bool picoMissingSemiMinimal() { + +test bool picoMissingSemiSmall() { t = parsePico( "begin declare; while input do input := 14 output := 0 od -end", visualize=true); - - for (error <- findAllErrors(t)) { - println(" error: "); - } +end"); -return hasErrors(t); - /*str errorText = getErrorText(findFirstError(t)); - println("error text: "); - return hasErrors(t) && size(findAllErrors(t)) == 1 && getErrorText(findFirstError(t)) == "input := input - 1 - od"; - */ + Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); + return getErrorText(error) == "output := 0\n od"; } \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc deleted file mode 100644 index 3a5e589a2dd..00000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RunRecoveryTests.rsc +++ /dev/null @@ -1,9 +0,0 @@ -module lang::rascal::tests::recovery::RunRecoveryTests - -import lang::rascal::tests::recovery::BasicRecoveryTests; -import util::REPL; - -int main(list[str] args){ - Terminal terminal = newREPL(repl()); - return 0; -} diff --git a/src/org/rascalmpl/library/util/TermREPL.java b/src/org/rascalmpl/library/util/TermREPL.java index 75ab0f0398b..f149bb36340 100644 --- a/src/org/rascalmpl/library/util/TermREPL.java +++ b/src/org/rascalmpl/library/util/TermREPL.java @@ -103,7 +103,6 @@ public static class TheREPL implements ILanguageProtocol { private final AbstractFunction completor; private final IValueFactory vf; private final AbstractFunction stacktrace; - private IDEServices services; public TheREPL(IValueFactory vf, IString title, IString welcome, IString prompt, IString quit, ISourceLocation history, IFunction handler, IFunction completor, IValue stacktrace, InputStream input, OutputStream stderr, OutputStream stdout) { @@ -150,7 +149,6 @@ public void initialize(InputStream input, OutputStream stdout, OutputStream stde this.stdout = stdout; this.stderr = stderr; this.input = input; - this.services = services; } @Override diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 1ed1b597447..7357552ffb8 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -111,7 +111,7 @@ public abstract class SGTDBF implements IGTD { // Reflection is used to get the expects for each non-terminal. - // This cache is used so the reflection call is only needed once. + // This cache is used so the reflection call is only needed once per non-terminal. private final HashMap[]> expectCache; private final IntegerObjectList> sharedLastExpects; @@ -877,7 +877,7 @@ private boolean findStacksToReduce(){ NodeId reduceNodeId = new NodeId("todo-" + i); visualize("Found stack to reduce", reduceNodeId); } - + stacksWithTerminalsToReduce = terminalsTodo; location += i; @@ -935,7 +935,7 @@ public boolean parseErrorHasOccurred(){ @SuppressWarnings("unchecked") private void queueMatchableNode(AbstractStackNode

node, int length, AbstractNode result){ assert result != null; - + int queueDepth = todoLists.length; if(length >= queueDepth){ DoubleStack, AbstractNode>[] oldTodoLists = todoLists; @@ -972,23 +972,21 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int // However, we may assume that the queue before the current index is // done, based on the way we cycle the queue now. The queue is // looking forward to the future and we never re-use past entries. - + int negativeOffset = location - startPosition; - + DoubleStack, AbstractNode>[] oldTodoLists = todoLists; todoLists = new DoubleStack[negativeOffset + Math.max(queueDepth, length) + 1]; System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset, queueDepth - queueIndex); System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset , queueIndex); - + // reset the parser! queueIndex = 0; location = startPosition; - - // was: DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex + 1]; + DoubleStack, AbstractNode> terminalsTodo = todoLists[length]; if (terminalsTodo == null) { terminalsTodo = new DoubleStack, AbstractNode>(); - // was: todoLists[queueIndex + 1] = terminalsTodo; // Why the +1 and not length? To get the recovered node to be processed first? todoLists[length] = terminalsTodo; } @@ -997,7 +995,7 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int else if (startPosition == location) { // this is the normal case where new matchable nodes are discovered // for the current parsing location, so reuse the code for queuing - queueMatchableNode(node, length, result); + queueMatchableNode(node, length, result); } else { // This would mean we have discovered a recovery node for a location @@ -1006,9 +1004,7 @@ else if (startPosition == location) { throw new RuntimeException("discovered a future recovery? " + node); } } - - - + /** * Handles the retrieved alternatives for the given stack. */ @@ -1271,7 +1267,7 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] this.recoverer = recoverer; this.debugListener = debugListener; - + String query = inputURI.getQuery(); visualizer = query != null && query.contains("visualize=true") ? new DebugVisualizer("Parser") : null; @@ -1312,7 +1308,7 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] // Reduce-expand loop. do { if(debugListener != null) debugListener.iterating(); - + reduceTerminals(); reduceNonTerminals(); @@ -1477,17 +1473,17 @@ protected T buildResult(AbstractNode result, INodeFlattener converter, INo } /** - * Error nodes end up in a n inconvenient form because of the parser algorithm. + * Error nodes end up in an inconvenient form because of the parser algorithm. * This post-processing step transforms the original tree into a more useful form. * In essence, error subtrees look like this after parsing: - * `appl(prod(S,[]), [,,...,appl(skipped(S,prod(S,[]),),[)]])` - * This method pulls up the skipped node, transforming these trees into: - * `appl(skipped(S,prod(S,[]),), [,,...,[])])` + * `appl(prod(S,[]), [,,...,appl(skipped([]))])` + * This method transforms these trees into: + * `appl(error(S,prod(S,[]),), [,,...,appl(skipped([]))])` * This means productions that failed to parse can be recognized at the top level. * Note that this can only be done when we know the actual type of T is IConstructor. */ @SuppressWarnings("unchecked") - T fixErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { + private T fixErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { if (!(tree instanceof IConstructor)) { return tree; } @@ -1495,7 +1491,7 @@ T fixErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { return (T) fixErrorNodes((IConstructor) tree, (INodeConstructorFactory) nodeConstructorFactory); } - IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { + private IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { IConstructor result; Type type = tree.getConstructorType(); if (type == RascalValueFactory.Tree_Appl) { @@ -1584,9 +1580,8 @@ private void visualize(String step, NodeId highlight) { } } - /** - * Getters used for graph generation/debugging + * Getters used for graph generation for debugging (see DebugVisualizer) */ public int[] getInput() { @@ -1636,6 +1631,4 @@ public DoubleStack, AbstractNode>, Abstract public DoubleStack, AbstractNode> getFilteredNodes() { return filteredNodes; } - - } diff --git a/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java b/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java index 40dc4d52257..f4e7522b1d6 100644 --- a/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java +++ b/src/org/rascalmpl/parser/gtd/recovery/IRecoverer.java @@ -24,7 +24,7 @@ public interface IRecoverer

{ * and some of its history. With this information new stack nodes may be generated for the parser * to continue with. It is up to the reviveStacks method to make sure the parser still generates * derivation trees that cover the entire input. - * + * * @param input the 24-bit unicode input character array * @param location the current character offset in the input that the parser got stuck on * @param unexpandableNodes these are non-terminals that were predicted at this location but did not fly diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 4acefaf19d3..062f640694c 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -8,6 +8,7 @@ * Contributors: * * Arnold Lankamp - Arnold.Lankamp@cwi.nl + * * Pieter Olivier - Pieter.Olivier@swat.engineering *******************************************************************************/ package org.rascalmpl.parser.gtd.stack; @@ -20,7 +21,7 @@ public final class SkippingStackNode

extends AbstractMatchableStackNode

{ private final SkippedNode result; - + public static SkippedNode createResultUntilCharClass(URI uri, int[] until, int[] input, int startLocation, IConstructor production, int dot) { for (int to = startLocation ; to < input.length; ++to) { for (int i = 0; i < until.length; ++i) { @@ -69,34 +70,34 @@ private static int[] createSkippedToken(int[] input, int startLocation, int leng public SkippingStackNode(int id, P parentProduction, SkippedNode result) { super(id, 0); - + this.result = result; setAlternativeProduction(parentProduction); } public SkippingStackNode(int id, P parentProduction, SkippedNode result, int startLocation) { super(id, 0, startLocation); - + this.result = result; setAlternativeProduction(parentProduction); } private SkippingStackNode(SkippingStackNode

original, int startLocation){ super(original, startLocation); - + this.result = original.result; } - + private SkippingStackNode(SkippingStackNode

original, SkippedNode result, int startLocation){ super(original, startLocation); - + this.result = result; } - + public boolean isEmptyLeafNode(){ return result.isEmpty(); } - + public AbstractNode match(int[] input, int location){ return result; } @@ -104,29 +105,20 @@ public AbstractNode match(int[] input, int location){ public AbstractStackNode

getCleanCopy(int startLocation){ return new SkippingStackNode

(this, startLocation); } - + public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ return new SkippingStackNode

(this, (SkippedNode) result, startLocation); } - + public int getLength(){ return result.getLength(); } - + public AbstractNode getResult(){ return result; } - - /*Original: public String toString(){ - StringBuilder sb = new StringBuilder(); - sb.append(getId()); - sb.append('('); - sb.append(startLocation); - sb.append(')'); - - return sb.toString(); - }*/ - + + @Override public String toShortString() { return "skip(" + result.toString() + ")"; } @@ -155,9 +147,9 @@ public boolean isEqual(AbstractStackNode

stackNode){ if ( !(stackNode instanceof SkippingStackNode)) { return false; } - + SkippingStackNode

otherNode = (SkippingStackNode

) stackNode; - + return otherNode.id == id; } } diff --git a/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java b/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java index f95daf557c7..56f1b3d3419 100644 --- a/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java +++ b/src/org/rascalmpl/parser/gtd/stack/edge/EdgesSet.java @@ -22,62 +22,62 @@ @SuppressWarnings({"unchecked", "cast"}) public class EdgesSet

{ - public final static int DEFAULT_RESULT_STORE_ID = -1; - - private final static int DEFAULT_SIZE = 8; - + public static final int DEFAULT_RESULT_STORE_ID = -1; + + private static final int DEFAULT_SIZE = 8; + private AbstractStackNode

[] edges; private int size; private int lastVisitedLevel = -1; private IntegerMap lastVisitedFilteredLevel; - + private AbstractContainerNode

lastResults; // Indexed by `resultStoreId`. private IntegerObjectList> lastFilteredResults; - + public EdgesSet(){ super(); - + edges = (AbstractStackNode

[]) new AbstractStackNode[DEFAULT_SIZE]; size = 0; } - + public EdgesSet(int initialSize){ super(); - + edges = (AbstractStackNode

[]) new AbstractStackNode[initialSize]; size = 0; } - + private void enlarge(){ edges = Arrays.copyOf(edges, size << 1, edges.getClass()); } - + public void add(AbstractStackNode

edge){ - while(size >= edges.length){ // While instead of if to enable the JIT to eliminate the bounds check on the edges array + while(size >= edges.length){ // While instead of if to enable the JIT to eliminate the bounds check on the edges array enlarge(); } - + edges[size++] = edge; } - + public boolean contains(AbstractStackNode

node){ for(int i = size - 1; i >= 0; --i){ if(edges[i] == node) return true; } return false; } - + public boolean containsBefore(AbstractStackNode

node, int limit){ for(int i = limit - 1; i >= 0; --i){ if(edges[i] == node) return true; } return false; } - + public boolean containsAfter(AbstractStackNode

node, int limit){ if(limit >= 0){ // Bounds check elimination helper. for(int i = size - 1; i >= limit; --i){ @@ -86,11 +86,11 @@ public boolean containsAfter(AbstractStackNode

node, int limit){ } return false; } - + public AbstractStackNode

get(int index){ return edges[index]; } - + public void setLastVisitedLevel(int level, int resultStoreId){ if(resultStoreId == DEFAULT_RESULT_STORE_ID){ lastVisitedLevel = level; @@ -98,21 +98,21 @@ public void setLastVisitedLevel(int level, int resultStoreId){ if(lastVisitedFilteredLevel == null){ lastVisitedFilteredLevel = new IntegerMap(); } - + lastVisitedFilteredLevel.put(resultStoreId, level); } } - + public int getLastVisitedLevel(int resultStoreId){ if(resultStoreId == DEFAULT_RESULT_STORE_ID) return lastVisitedLevel; - + if(lastVisitedFilteredLevel == null){ lastVisitedFilteredLevel = new IntegerMap(); return -1; } return lastVisitedFilteredLevel.get(resultStoreId); } - + public void setLastResult(AbstractContainerNode

lastResult, int resultStoreId){ if(resultStoreId == DEFAULT_RESULT_STORE_ID){ lastResults = lastResult; @@ -120,25 +120,25 @@ public void setLastResult(AbstractContainerNode

lastResult, int resultStoreId if(lastFilteredResults == null){ lastFilteredResults = new IntegerObjectList>(DEFAULT_SIZE); } - + lastFilteredResults.add(resultStoreId, lastResult); } } - + public AbstractContainerNode

getLastResult(int resultStoreId){ if(resultStoreId == DEFAULT_RESULT_STORE_ID) return lastResults; - + if(lastFilteredResults == null){ lastFilteredResults = new IntegerObjectList>(DEFAULT_SIZE); return null; } return lastFilteredResults.findValue(resultStoreId); } - + public int size(){ return size; } - + public void clear(){ size = 0; } @@ -146,7 +146,7 @@ public void clear(){ @Override public String toString() { StringBuilder builder = new StringBuilder("EdgesSet[{"); - + for (int i=0; i 0) { builder.append(","); @@ -170,5 +170,5 @@ public String toString() { return builder.toString(); } - + } diff --git a/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java index 403bfbe9247..752813d5021 100644 --- a/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java @@ -25,7 +25,7 @@ public StackNodeIdDispenser(IGTD parser) { // Custom IGTB implementation without "getFreeStackNodeId" method. No biggy, we just use nextNodeIdBackup. } } - + @Override public int dispenseId() { if (dispenseMethod != null) { From 7ee16aafffe4610fae60b2c280d53b7d6c8adc12 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 09:09:03 +0200 Subject: [PATCH 50/87] Removed unused fields from SkippedNode --- .../parser/gtd/result/SkippedNode.java | 30 ++++++------------- .../parser/gtd/stack/SkippingStackNode.java | 2 +- .../util/visualize/DebugVisualizer.java | 24 ++++++--------- 3 files changed, 19 insertions(+), 37 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java index 53578520ce2..2a376fda673 100644 --- a/src/org/rascalmpl/parser/gtd/result/SkippedNode.java +++ b/src/org/rascalmpl/parser/gtd/result/SkippedNode.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2009-2013 CWI + * Copyright (c) 2009-2024 CWI * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -8,6 +8,7 @@ * Contributors: * * Arnold Lankamp - Arnold.Lankamp@cwi.nl + * * Pieter Olivier - Pieter.Olivier@swat.engineering *******************************************************************************/ package org.rascalmpl.parser.gtd.result; @@ -20,24 +21,19 @@ */ public class SkippedNode extends AbstractNode { public static final int ID = 9; - - // TODO: only "skippedChars" is actually needed + private final URI input; - private final Object production; - private final int dot; private final int[] skippedChars; private final int offset; - public SkippedNode(URI input, Object production, int dot, int[] skippedChars, int offset) { + public SkippedNode(URI input, int[] skippedChars, int offset) { super(); - + this.input = input; - this.production = production; - this.dot = dot; this.skippedChars = skippedChars; this.offset = offset; } - + public int getTypeIdentifier(){ return ID; } @@ -46,29 +42,21 @@ public URI getInput() { return input; } - public Object getProduction() { - return production; - } - - public int getDot() { - return dot; - } - public int[] getSkippedChars(){ return skippedChars; } - + /** * Returns the offset in the input string this node starts at. */ public int getOffset(){ return offset; } - + public boolean isEmpty() { return (skippedChars.length == 0); } - + public boolean isNonterminalSeparator() { return false; } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 062f640694c..f903fc88b4b 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2009-2013 CWI + * Copyright (c) 2009-2024 CWI * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java index c944242d5a3..91398517acf 100644 --- a/src/org/rascalmpl/util/visualize/DebugVisualizer.java +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -10,11 +10,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; -import java.util.Arrays; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -185,7 +182,7 @@ private DotGraph createGraph(DoubleArrayList, Ar addProductionArray(graph, productionsId, recoveryNodes.getSecond(i)); graph.addEdge(new NodeId(pairId, "productions"), productionsId); } - + return graph; } @@ -241,7 +238,7 @@ private

DotNode addStack(DotGraph graph, AbstractStackNode

stackNode) { if (node != null) { return node; } - + node = createDotNode(stackNode); stackNodeNodes.put(stackNode.getId(), node); @@ -312,7 +309,7 @@ private

DotNode createDotNode(AbstractStackNode

stackNode) { } DotNode node = new DotNode(getNodeId(stackNode)); - String label = String.format("%s: %s\n.%d@%d %s", + String label = String.format("%s: %s\n.%d@%d %s", type, nodeName, dot, stackNode.getStartLocation(), extraInfo); String shortString = stackNode.toShortString(); @@ -378,7 +375,7 @@ private void enrichLiteralNode(DotNode dotNode, LiteralNode literalNode) { String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); int[] content = literalNode.getContent(); label += " \"" + UnicodeConverter.unicodeArrayToString(content) + "\""; - /* Maybe include production? + /* Maybe include production? label += "\nprod=" + DebugUtil.prodToString((IConstructor)literalNode.getProduction()); */ dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); @@ -386,8 +383,7 @@ private void enrichLiteralNode(DotNode dotNode, LiteralNode literalNode) { private void enrichSkippedNode(DotNode dotNode, SkippedNode skippedNode) { String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); - label += "\n." + skippedNode.getDot() + "@" + skippedNode.getOffset() + ": " + " \"" + UnicodeConverter.unicodeArrayToString(skippedNode.getSkippedChars()) + "\""; - label += "\nin: " + DebugUtil.prodToString((IConstructor) skippedNode.getProduction()); + label += "\n@" + skippedNode.getOffset() + ": " + " \"" + UnicodeConverter.unicodeArrayToString(skippedNode.getSkippedChars()) + "\""; dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); } @@ -397,10 +393,8 @@ private void enrichSortContainerNode(DotNode dotNode, SortContainerNode void writeGraph() { + public void writeGraph() { if (graph != null) { writeGraph(graph); } @@ -440,7 +434,7 @@ public void createGraph(SGTDBF parser, String step) { lookahead = '$'; } - String label = String.format("Parser\nInput: \"%s\"\nLocation: %d ('%c')\nStep %d: %s", + String label = String.format("Parser\nInput: \"%s\"\nLocation: %d ('%c')\nStep %d: %s", input, location, lookahead, frame, step); parserNode.setAttribute(DotAttribute.ATTR_LABEL, label); graph.addNode(parserNode); @@ -535,7 +529,7 @@ private void addUnmatchableMidProductionNodes(SGTDBF parser, graph.addEdge(new NodeId(failureId, "predecessors"), predecessorsId); } } - + private void addFilteredNodes(SGTDBF parser, DotGraph graph) { addStackAndNodeDoubleStack(graph, FILTERED_NODES_ID, parser.getFilteredNodes()); } From 002e7a6db7f7c7f0224eaecd457d602d6871b2f6 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 09:59:34 +0200 Subject: [PATCH 51/87] Switched from hardcoding to environment variable for the parser visualization path --- .../lang/rascal/tests/recovery/ToyRascal.rsc | 2 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 464 +++++++++--------- .../parser/gtd/stack/SkippingStackNode.java | 24 +- .../recovery/ToNextWhitespaceRecoverer.java | 84 ++-- .../uptr/recovery/ToTokenRecoverer.java | 61 ++- .../util/visualize/DebugVisualizer.java | 54 +- src/org/rascalmpl/util/visualize/replay.html | 36 ++ 7 files changed, 377 insertions(+), 348 deletions(-) create mode 100644 src/org/rascalmpl/util/visualize/replay.html diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc index 81a9fd1c5b2..3f6bfb736bb 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc @@ -10,7 +10,7 @@ syntax Parameters = "(" ")"; syntax FunctionBody = "{" Statement* statements "}" ; -syntax Statement +syntax Statement = "{" Statement+ statements "}" | "s" ";"; diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 7357552ffb8..2a989a704c8 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -62,7 +62,7 @@ */ public abstract class SGTDBF implements IGTD { private final static int DEFAULT_TODOLIST_CAPACITY = 16; - + private URI inputURI; private int[] input; private int location; @@ -70,14 +70,14 @@ public abstract class SGTDBF implements IGTD { // A mapping between character location and line/column. private final PositionStore positionStore; - + // Terminals that matched. Circular buffer indexed by length of the terminal. Each entry contains the node to reduce and the result node // This is a circular buffer where queueIndex determines the start of the buffer. // At each position, a stack is maintained of all terminals to reduce of a certain length. // So at queueIndex+3, all terminals of length 3 that need reducing are stored. private DoubleStack, AbstractNode>[] todoLists; private int queueIndex; - + // Stack of non-terminal nodes to expand // - Nodes are removed in expand, which pops and expands all stack nodes on this stack // - Nodes are added in: @@ -104,60 +104,60 @@ public abstract class SGTDBF implements IGTD { // - findStacksToReduce: again the first non-empty `todoList` is assigned to this variable // - parse: variable is used in main reduce/expand loop to determine when it is time to look for more `stacksToReduce`. private DoubleStack, AbstractNode> stacksWithTerminalsToReduce; - + private final HashMap> cachedEdgesForExpect; - + private final IntegerKeyedDoubleValueHashMap, DoubleArrayList, AbstractNode>> sharedNextNodes; - - + + // Reflection is used to get the expects for each non-terminal. // This cache is used so the reflection call is only needed once per non-terminal. private final HashMap[]> expectCache; - + private final IntegerObjectList> sharedLastExpects; - + // Guard private boolean invoked; - + // Error reporting private final Stack> unexpandableNodes; private final Stack> unmatchableLeafNodes; // Leaf nodes (for instance literals) that failed to match private final DoubleStack, AbstractNode>, AbstractStackNode

> unmatchableMidProductionNodes; private final DoubleStack, AbstractNode> filteredNodes; - + // Error reporting guards private boolean parseErrorOccured; - + // Error recovery private IRecoverer

recoverer; - + // Debugging private IDebugListener

debugListener; private DebugVisualizer visualizer; - + // Temporary instrumentation for accurate profiling private long timestamp; private boolean printTimes = false; - + public SGTDBF(){ super(); positionStore = new PositionStore(); - + stacksToExpand = new Stack>(); stacksWithNonTerminalsToReduce = new DoubleStack, AbstractContainerNode

>(); - + cachedEdgesForExpect = new HashMap>(); - + sharedNextNodes = new IntegerKeyedDoubleValueHashMap, DoubleArrayList, AbstractNode>>(); - + location = 0; - + expectCache = new HashMap[]>(); - + sharedLastExpects = new IntegerObjectList>(); - + unexpandableNodes = new Stack>(); unmatchableLeafNodes = new Stack>(); unmatchableMidProductionNodes = new DoubleStack, AbstractNode>, AbstractStackNode

>(); @@ -165,14 +165,14 @@ public SGTDBF(){ } /** - * Return a stack node id that is guaranteed not to be in use. + * Return a stack node id that is guaranteed not to be in use. * The parser generator generates an override for this method as it knows which ids have been dispensed. * Tests that need this should override this method, probably using a common base class. */ protected int getFreeStackNodeId() { throw new UnsupportedOperationException(); } - + @SuppressWarnings("unchecked") public AbstractStackNode

[] getExpects(String nonTerminal) { AbstractStackNode

[] expects = expectCache.get(nonTerminal); @@ -184,7 +184,7 @@ public AbstractStackNode

[] getExpects(String nonTerminal) { } catch (SecurityException sex) { // Ignore this if it happens. } - + expects = (AbstractStackNode

[]) method.invoke(this); } catch (NoSuchMethodException nsmex) { throw new UndeclaredNonTerminalException(nonTerminal, getClass()); @@ -193,7 +193,7 @@ public AbstractStackNode

[] getExpects(String nonTerminal) { } catch (InvocationTargetException itex) { throw new RuntimeException(itex.getTargetException()); } - + expectCache.putUnsafe(nonTerminal, expects); } @@ -206,7 +206,7 @@ public AbstractStackNode

[] getExpects(String nonTerminal) { protected AbstractStackNode

[] invokeExpects(AbstractStackNode

nonTerminal){ return getExpects(nonTerminal.getName()); } - + /** * Moves to the next symbol in the production. */ @@ -218,7 +218,7 @@ private AbstractStackNode

updateNextNode(AbstractStackNode

next, AbstractS predecessors.add(node, result); return null; } - + AbstractStackNode

alternative = alternativeEntry.value1; if(alternative.getStartLocation() == location){ if(alternative.isMatchable()){ @@ -245,52 +245,52 @@ private AbstractStackNode

updateNextNode(AbstractStackNode

next, AbstractS } } } - + alternative.updateNode(node, result); - + if(debugListener != null) debugListener.progressed(node, result, alternative); - + return alternative; } - + if(next.isMatchable()){ // Eager matching optimization. if((location + next.getLength()) > input.length) return null; - + AbstractNode nextResult = next.match(input, location); if(nextResult == null){ // Push the node including it's predecessor to the appropriate error tracking collection (and take care of merging when necessary). DoubleArrayList, AbstractNode> predecessors = new DoubleArrayList, AbstractNode>(); predecessors.add(node, result); unmatchableMidProductionNodes.push(predecessors, next); - + sharedNextNodes.putUnsafe(next.getId(), next, predecessors); - + if(debugListener != null) debugListener.failedToMatch(next); - + return null; } - + if(debugListener != null) debugListener.matched(next, nextResult); - + next = next.getCleanCopyWithResult(location, nextResult); }else{ next = next.getCleanCopy(location); } - + if(!node.isMatchable() || result.isEmpty()){ next.updateNode(node, result); }else{ // Non-nullable terminal specific edge set sharing optimization. next.updateNodeAfterNonEmptyMatchable(node, result); } - + if(debugListener != null) debugListener.progressed(node, result, next); - + sharedNextNodes.putUnsafe(next.getId(), next, null); stacksToExpand.push(next); - + return next; } - + /** * Moves to the next symbol in an alternative continuation of a prefix-shared production. */ @@ -303,7 +303,7 @@ private boolean updateAlternativeNextNode(AbstractStackNode

next, AbstractSta predecessors.add(node, result); return false; } - + AbstractStackNode

alternative = alternativeEntry.value1; if(result.isEmpty()){ if(alternative.isMatchable()){ @@ -323,84 +323,84 @@ private boolean updateAlternativeNextNode(AbstractStackNode

next, AbstractSta } } } - + alternative.updatePrefixSharedNode(edgesMap, prefixesMap); // Prevent unnecessary overhead; share whenever possible. - + if(debugListener != null) debugListener.progressed(node, result, alternative); - + return true; } - + if(next.isMatchable()){ // Eager matching optimization. if((location + next.getLength()) > input.length) return false; - + AbstractNode nextResult = next.match(input, location); if(nextResult == null){ // Push the node including it's predecessor to the appropriate error tracking collection (and take care of merging when necessary). DoubleArrayList, AbstractNode> predecessors = new DoubleArrayList, AbstractNode>(); predecessors.add(node, result); unmatchableMidProductionNodes.push(predecessors, next); - + sharedNextNodes.putUnsafe(id, next, predecessors); - + if(debugListener != null) debugListener.failedToMatch(next); - + return false; } - + if(debugListener != null) debugListener.matched(next, nextResult); - + next = next.getCleanCopyWithResult(location, nextResult); }else{ next = next.getCleanCopy(location); } - + next.updatePrefixSharedNode(edgesMap, prefixesMap); // Prevent unnecessary overhead; share whenever possible. if(debugListener != null) debugListener.progressed(node, result, next); - + sharedNextNodes.putUnsafe(id, next, null); stacksToExpand.push(next); - + return true; } - + /** * Part of the hidden-right-recursion fix. * Executes absent reductions. */ private void propagateReductions(AbstractStackNode

node, AbstractNode nodeResultStore, AbstractStackNode

next, AbstractNode nextResultStore, int potentialNewEdges){ IntegerList propagatedReductions = next.getPropagatedReductions(); - + IntegerObjectList> edgesMap = node.getEdges(); ArrayList[] prefixes = node.getPrefixesMap(); - + P production = next.getParentProduction(); String name = edgesMap.getValue(0).get(0).getName(); - + boolean hasNestingRestrictions = hasNestingRestrictions(name); IntegerList filteredParents = null; if(hasNestingRestrictions){ filteredParents = getFilteredParents(next.getId()); } - + int fromIndex = edgesMap.size() - potentialNewEdges; for(int i = edgesMap.size() - 1; i >= fromIndex; --i){ int startLocation = edgesMap.getKey(i); - + // We know we haven't been here before. propagatedReductions.add(startLocation); - + ArrayList edgePrefixes = new ArrayList(); Link prefix = (prefixes != null) ? new Link(prefixes[i], nodeResultStore) : new Link(null, nodeResultStore); edgePrefixes.add(prefix); - + Link resultLink = new Link(edgePrefixes, nextResultStore); - + EdgesSet

edgeSet = edgesMap.getValue(i); - + if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet); - + if(!hasNestingRestrictions){ handleEdgeList(edgeSet, name, production, resultLink, startLocation); }else{ @@ -408,7 +408,7 @@ private void propagateReductions(AbstractStackNode

node, AbstractNode nodeRes } } } - + /** * Part of the hidden-right-recursion fix. * Propagates absent prefixes. @@ -429,7 +429,7 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, nextNextAlternative.getResult(), nrOfAddedEdges); }else{ nextNextAlternative.updateNode(next, nextResult); - + if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative); } }else{ @@ -439,7 +439,7 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, nextNextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges); }else{ nextNextAlternative.updateNode(next, nextResult); - + if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative); } } @@ -447,7 +447,7 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul predecessors.add(next, nextResult); } } - + // Handle alternative continuations (related to prefix sharing). AbstractStackNode

[][] alternateProds = next.getAlternateProductions(); if(alternateProds != null){ @@ -455,18 +455,18 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul if(!nextNext.isMatchable()) return; // Matchable, abort. nextNextAlternative = nextNext.getCleanCopy(location); nextNextAlternative.updateNode(next, nextResult); - + if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative); } - + IntegerObjectList> nextNextEdgesMap = nextNextAlternative.getEdges(); ArrayList[] nextNextPrefixesMap = nextNextAlternative.getPrefixesMap(); - + for(int i = alternateProds.length - 1; i >= 0; --i){ prod = alternateProds[i]; if(nextDot == prod.length) continue; AbstractStackNode

alternativeNextNext = prod[nextDot]; - + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> nextNextAltAlternativeEntry = sharedNextNodes.get(alternativeNextNext.getId()); AbstractStackNode

nextNextAltAlternative = null; if(nextNextAltAlternativeEntry != null){ @@ -478,7 +478,7 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative, nextNextAltAlternative.getResult(), nrOfAddedEdges, nextNextEdgesMap, nextNextPrefixesMap); }else{ nextNextAltAlternative.updatePrefixSharedNode(nextNextEdgesMap, nextNextPrefixesMap); - + if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAltAlternative); } }else{ @@ -488,7 +488,7 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative, nextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges, nextNextEdgesMap, nextNextPrefixesMap); }else{ nextNextAltAlternative.updatePrefixSharedNode(nextNextEdgesMap, nextNextPrefixesMap); - + if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAltAlternative); } } @@ -499,7 +499,7 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul } } } - + /** * Part of the hidden-right-recursion fix. * Inserts missing prefixes and triggers reductions where necessary. @@ -508,18 +508,18 @@ private void propagateEdgesAndPrefixes(AbstractStackNode

node, AbstractNode n int nrOfAddedEdges = next.updateOvertakenNode(node, nodeResult); if(debugListener != null) debugListener.propagated(node, nodeResult, next); - + if(nrOfAddedEdges == 0) return; - + if(next.isEndNode()){ propagateReductions(node, nodeResult, next, nextResult, nrOfAddedEdges); } - + if(next.hasNext()){ propagatePrefixes(next, nextResult, nrOfAddedEdges); } } - + /** * Part of the hidden-right-recursion fix. * Inserts missing prefixes and triggers reductions where necessary (specific for nullable nodes). @@ -528,62 +528,62 @@ private void propagateEdgesAndPrefixesForNullable(AbstractStackNode

node, Abs int nrOfAddedEdges = next.updateOvertakenNullableNode(node, nodeResult, potentialNewEdges); if(debugListener != null) debugListener.propagated(node, nodeResult, next); - + if(nrOfAddedEdges == 0) return; - + if(next.isEndNode()){ propagateReductions(node, nodeResult, next, nextResult, nrOfAddedEdges); } - + if(next.hasNext()){ propagatePrefixes(next, nextResult, nrOfAddedEdges); } } - + /** * Part of the hidden-right-recursion fix. * Inserts missing prefixes and triggers reductions where necessary (specifically for alternative continuations of prefix-shared productions). */ private void propagateAlternativeEdgesAndPrefixes(AbstractStackNode

node, AbstractNode nodeResult, AbstractStackNode

next, AbstractNode nextResult, int potentialNewEdges, IntegerObjectList> edgesMap, ArrayList[] prefixesMap){ next.updatePrefixSharedNode(edgesMap, prefixesMap); - + if(debugListener != null) debugListener.propagated(node, nodeResult, next); - + if(potentialNewEdges == 0) return; - + if(next.isEndNode()){ propagateReductions(node, nodeResult, next, nextResult, potentialNewEdges); } - + if(next.hasNext()){ propagatePrefixes(next, nextResult, potentialNewEdges); } } - + /** * Initiates the handling of reductions. */ private void updateEdges(AbstractStackNode

node, AbstractNode result){ IntegerObjectList> edgesMap = node.getEdges(); ArrayList[] prefixesMap = node.getPrefixesMap(); - + P production = node.getParentProduction(); String name = edgesMap.getValue(0).get(0).getName(); - + // Check for nesting restrictions. boolean hasNestingRestrictions = hasNestingRestrictions(name); IntegerList filteredParents = null; if(hasNestingRestrictions){ filteredParents = getFilteredParents(node.getId()); } - + for(int i = edgesMap.size() - 1; i >= 0; --i){ Link resultLink = new Link((prefixesMap != null) ? prefixesMap[i] : null, result); - + EdgesSet

edgeSet = edgesMap.getValue(i); - + if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet); - + if(!hasNestingRestrictions){ // Select the optimized path for handling edge sets that don't have nesting restrictions associated with them. handleEdgeList(edgeSet, name, production, resultLink, edgesMap.getKey(i)); }else{ @@ -591,40 +591,40 @@ private void updateEdges(AbstractStackNode

node, AbstractNode result){ } } } - + /** * Initiates the handling of reductions for nullable symbols. */ private void updateNullableEdges(AbstractStackNode

node, AbstractNode result){ IntegerList propagatedReductions = node.getPropagatedReductions(); - + int initialSize = propagatedReductions.size(); - + IntegerObjectList> edgesMap = node.getEdges(); ArrayList[] prefixesMap = node.getPrefixesMap(); - + P production = node.getParentProduction(); String name = edgesMap.getValue(0).get(0).getName(); - + // Check for nesting restrictions. boolean hasNestingRestrictions = hasNestingRestrictions(name); IntegerList filteredParents = null; if(hasNestingRestrictions){ filteredParents = getFilteredParents(node.getId()); } - + for(int i = edgesMap.size() - 1; i >= 0; --i){ int startLocation = edgesMap.getKey(i); - + if(propagatedReductions.containsBefore(startLocation, initialSize)) continue; // Prevent duplicate reductions (artifact of the hidden-right-recursion fix). propagatedReductions.add(startLocation); - + Link resultLink = new Link((prefixesMap != null) ? prefixesMap[i] : null, result); - + EdgesSet

edgeSet = edgesMap.getValue(i); - + if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet); - + if(!hasNestingRestrictions){ // Select the optimized path for handling edge sets that don't have nesting restrictions associated with them. handleEdgeList(edgeSet, name, production, resultLink, startLocation); }else{ @@ -632,7 +632,7 @@ private void updateNullableEdges(AbstractStackNode

node, AbstractNode result) } } } - + /** * Handles reductions. */ @@ -641,7 +641,7 @@ private void handleEdgeList(EdgesSet

edgeSet, String name, P production, Link int resultStoreId = EdgesSet.DEFAULT_RESULT_STORE_ID; if(edgeSet.getLastVisitedLevel(resultStoreId) != location){ AbstractStackNode

edge = edgeSet.get(0); - + if(edge.isRecovered()){ resultStore = new RecoveredNode

(inputURI, startLocation, location); }else if(edge.isExpandable()){ @@ -649,27 +649,27 @@ private void handleEdgeList(EdgesSet

edgeSet, String name, P production, Link }else{ resultStore = new SortContainerNode

(inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout()); } - + stacksWithNonTerminalsToReduce.push(edge, resultStore); - + if(debugListener != null) debugListener.reduced(edge); - + for(int j = edgeSet.size() - 1; j >= 1; --j){ edge = edgeSet.get(j); stacksWithNonTerminalsToReduce.push(edge, resultStore); - + if(debugListener != null) debugListener.reduced(edge); } - + edgeSet.setLastVisitedLevel(location, resultStoreId); edgeSet.setLastResult(resultStore, resultStoreId); }else{ resultStore = edgeSet.getLastResult(resultStoreId); } - + resultStore.addAlternative(production, resultLink); } - + // Reuse these structures. private final IntegerList firstTimeRegistration = new IntegerList(); private final IntegerList firstTimeReductions = new IntegerList(); @@ -685,15 +685,15 @@ private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P for(int j = edgeSet.size() - 1; j >= 0; --j){ AbstractStackNode

edge = edgeSet.get(j); int resultStoreId = getResultStoreId(edge.getId()); - + if(!firstTimeReductions.contains(resultStoreId)){ if(firstTimeRegistration.contains(resultStoreId)){ if(debugListener != null) debugListener.filteredByNestingRestriction(edge); - + continue; } firstTimeRegistration.add(resultStoreId); - + // Check whether or not the nesting is allowed. if(filteredParents == null || !filteredParents.contains(edge.getId())){ AbstractContainerNode

resultStore = null; @@ -708,16 +708,16 @@ private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P }else { resultStore = new SortContainerNode

(inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout()); } - + edgeSet.setLastVisitedLevel(location, resultStoreId); edgeSet.setLastResult(resultStore, resultStoreId); - + stacksWithNonTerminalsToReduce.push(edge, resultStore); firstTimeReductions.add(resultStoreId); } - + resultStore.addAlternative(production, resultLink); - + if(debugListener != null) debugListener.reduced(edge); }else{ if(debugListener != null) debugListener.filteredByNestingRestriction(edge); @@ -728,7 +728,7 @@ private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P } } } - + /** * Move to the next symbol(s) in the production. */ @@ -738,7 +738,7 @@ private void moveToNext(AbstractStackNode

node, AbstractNode result){ AbstractStackNode

[] prod = node.getProduction(); AbstractStackNode

newNext = prod[nextDot]; AbstractStackNode

next = updateNextNode(newNext, node, result); - + // Handle alternative continuations of this production (related to prefix-sharing). AbstractStackNode

[][] alternateProds = node.getAlternateProductions(); if(alternateProds != null){ @@ -748,17 +748,17 @@ private void moveToNext(AbstractStackNode

node, AbstractNode result){ edgesMap = next.getEdges(); prefixesMap = next.getPrefixesMap(); } - + for(int i = alternateProds.length - 1; i >= 0; --i){ prod = alternateProds[i]; if(nextDot == prod.length) continue; AbstractStackNode

newAlternativeNext = prod[nextDot]; - + if(edgesMap != null){ updateAlternativeNextNode(newAlternativeNext, node, result, edgesMap, prefixesMap); }else{ AbstractStackNode

alternativeNext = updateNextNode(newAlternativeNext, node, result); - + if(alternativeNext != null){ edgesMap = alternativeNext.getEdges(); prefixesMap = alternativeNext.getPrefixesMap(); @@ -767,7 +767,7 @@ private void moveToNext(AbstractStackNode

node, AbstractNode result){ } } } - + /** * Progress to the next 'states' associated with the given node. * I.e. move to the next symbol(s) in the production (if available) and executed reductions if necessary. @@ -782,14 +782,14 @@ private void move(AbstractStackNode

node, AbstractNode result){ for(int i = completionFilters.length - 1; i >= 0; --i){ if(completionFilters[i].isFiltered(input, startLocation, location, positionStore)){ filteredNodes.push(node, result); - + if(debugListener != null) debugListener.filteredByCompletionFilter(node, result); - + return; } } } - + if(node.isEndNode()){ if(!result.isEmpty() || node.getId() == AbstractExpandableStackNode.DEFAULT_LIST_EPSILON_ID){ // Only go into the nullable fix path for nullables (special list epsilons can be ignored as well). updateEdges(node, result); @@ -797,12 +797,12 @@ private void move(AbstractStackNode

node, AbstractNode result){ updateNullableEdges(node, result); } } - + if(node.hasNext()){ moveToNext(node, result); } } - + /** * Initiate the handling of stacks. */ @@ -821,7 +821,7 @@ private void reduceNonTerminals() { move(stacksWithNonTerminalsToReduce.peekFirst(), stacksWithNonTerminalsToReduce.popSecond()); } } - + /** * Locates the initial set of stacks that is queued for handling, for which the least amount of characters needs to be shifted. */ @@ -830,15 +830,15 @@ private boolean findFirstStacksToReduce(){ DoubleStack, AbstractNode> terminalsTodo = todoLists[i]; if(!(terminalsTodo == null || terminalsTodo.isEmpty())){ stacksWithTerminalsToReduce = terminalsTodo; - + location += i; - + queueIndex = i; - + return true; } } - + if (recoverer != null) { if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); @@ -855,13 +855,13 @@ private boolean findFirstStacksToReduce(){ } return findStacksToReduce(); } - + parseErrorOccured = true; } - + return false; } - + /** * Locates the set of stacks that is queued for handling, for which the least amount of characters needs to be shifted. */ @@ -870,7 +870,7 @@ private boolean findStacksToReduce(){ int queueDepth = todoLists.length; for(int i = 1; i < queueDepth-1; ++i){ queueIndex = (queueIndex + 1) % queueDepth; - + DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; if(!(terminalsTodo == null || terminalsTodo.isEmpty())){ if (DebugVisualizer.VISUALIZATION_ENABLED) { @@ -879,13 +879,13 @@ private boolean findStacksToReduce(){ } stacksWithTerminalsToReduce = terminalsTodo; - + location += i; - + return true; } } - + if (recoverer != null && location < input.length) { if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); @@ -902,14 +902,14 @@ private boolean findStacksToReduce(){ visualizer.writeGraph(); DebugUtil.opportunityToBreak(); } - + if (recoveredNodes.size() > 0) { // was: for (int i = recoveredNodes.size()-1; i>= 0; i--) { for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); - + // int levelsFromHere = recovered.getLength() - (location - recovered.getStartLocation()); - + if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } @@ -918,17 +918,17 @@ private boolean findStacksToReduce(){ } return findStacksToReduce(); } - + parseErrorOccured = true; } - + return false; } public boolean parseErrorHasOccurred(){ return parseErrorOccured; } - + /** * Inserts a stack bottom into the todo-list. */ @@ -945,7 +945,7 @@ private void queueMatchableNode(AbstractStackNode

node, int length, AbstractN queueDepth = length + 1; queueIndex = 0; } - + int insertLocation = (queueIndex + length) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[insertLocation]; if(terminalsTodo == null){ @@ -954,7 +954,7 @@ private void queueMatchableNode(AbstractStackNode

node, int length, AbstractN } terminalsTodo.push(node, result); } - + /** * Inserts a recovery node into the todo-list, and possibly * rewinds the parser to an earlier location in the input @@ -962,15 +962,15 @@ private void queueMatchableNode(AbstractStackNode

node, int length, AbstractN @SuppressWarnings("unchecked") private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int length, AbstractNode result){ assert result != null; - + int queueDepth = todoLists.length; if (startPosition < location) { // Have to reset the parser to an earlier location to at least - // be able to process the new node. Cannot throw away the queue, + // be able to process the new node. Cannot throw away the queue, // because there are possibly already other recovery tokens in the queue. // However, we may assume that the queue before the current index is - // done, based on the way we cycle the queue now. The queue is + // done, based on the way we cycle the queue now. The queue is // looking forward to the future and we never re-use past entries. int negativeOffset = location - startPosition; @@ -989,7 +989,7 @@ private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int terminalsTodo = new DoubleStack, AbstractNode>(); todoLists[length] = terminalsTodo; } - + terminalsTodo.push(node, result); } else if (startPosition == location) { @@ -1010,102 +1010,102 @@ else if (startPosition == location) { */ private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cachedEdges, AbstractStackNode

stackBeingWorkedOn){ boolean hasValidAlternatives = false; - + sharedLastExpects.dirtyClear(); - + EXPECTS: for(int i = expects.length - 1; i >= 0; --i){ AbstractStackNode

first = expects[i]; - + if(first.isMatchable()){ // Eager matching optimization. int length = first.getLength(); int endLocation = location + length; if(endLocation > input.length) continue; - + AbstractNode result = first.match(input, location); if(result == null){ unmatchableLeafNodes.push(first); - + if(debugListener != null) debugListener.failedToMatch(first); - + continue; } - + if(debugListener != null) debugListener.matched(first, result); - + // Handle filtering. IEnterFilter[] enterFilters = first.getEnterFilters(); if(enterFilters != null){ for(int j = enterFilters.length - 1; j >= 0; --j){ if(enterFilters[j].isFiltered(input, location, positionStore)){ if(debugListener != null) debugListener.filteredByEnterFilter(first); - + continue EXPECTS; } } } - + first = first.getCleanCopyWithResult(location, result); - + queueMatchableNode(first, length, result); }else{ first = first.getCleanCopy(location); stacksToExpand.push(first); } - + first.initEdges(); first.addEdges(cachedEdges, location); - + sharedLastExpects.add(first.getId(), first); - + hasValidAlternatives = true; - + if(debugListener != null) debugListener.expanded(stackBeingWorkedOn, first); } - + return hasValidAlternatives; } - + /** * Check whether or not the given sort name has nesting restrictions associated with it. */ protected boolean hasNestingRestrictions(String name){ return false; // Priority and associativity filtering is off by default. } - + /** * Retrieves the set of disallowed parents for the given child. */ protected IntegerList getFilteredParents(int childId){ return null; // Default implementation; intended to be overwritten in sub-classes. } - + /** * Retrieves the resultstore id associated with the given id. */ protected int getResultStoreId(int id){ return EdgesSet.DEFAULT_RESULT_STORE_ID; // Default implementation; intended to be overwritten in sub-classes. } - + /** * Expands the given stack node. */ - private void expandStack(AbstractStackNode

stack){ + private void expandStack(AbstractStackNode

stack){ if(debugListener != null) debugListener.expanding(stack); - + // Handle filtering. IEnterFilter[] enterFilters = stack.getEnterFilters(); if(enterFilters != null){ for(int i = enterFilters.length - 1; i >= 0; --i){ if(enterFilters[i].isFiltered(input, location, positionStore)){ unexpandableNodes.push(stack); - + if(debugListener != null) debugListener.filteredByEnterFilter(stack); - + return; } } } - + if(stack.isMatchable()){ // Eager matching optimization related. queueMatchableNode(stack, stack.getLength(), stack.getResult()); }else if(!stack.isExpandable()){ // A 'normal' non-terminal. @@ -1113,13 +1113,13 @@ private void expandStack(AbstractStackNode

stack){ if(cachedEdges == null){ cachedEdges = new EdgesSet

(1); cachedEdgesForExpect.put(stack.getName(), cachedEdges); - + AbstractStackNode

[] expects = invokeExpects(stack); if(expects == null){ unexpandableNodes.push(stack); return; } - + if(!handleExpects(expects, cachedEdges, stack)){ unexpandableNodes.push(stack); return; @@ -1128,28 +1128,28 @@ private void expandStack(AbstractStackNode

stack){ int resultStoreId = getResultStoreId(stack.getId()); if(cachedEdges.getLastVisitedLevel(resultStoreId) == location){ // Is nullable, add the known results. stacksWithNonTerminalsToReduce.push(stack, cachedEdges.getLastResult(resultStoreId)); - + if(debugListener != null) debugListener.foundIterationCachedNullableResult(stack); } } - + cachedEdges.add(stack); - + stack.setIncomingEdges(cachedEdges); }else{ // Expandable EdgesSet

cachedEdges = cachedEdgesForExpect.get(stack.getName()); if(cachedEdges == null){ boolean expanded = false; - + cachedEdges = new EdgesSet

(); cachedEdgesForExpect.put(stack.getName(), cachedEdges); - + AbstractStackNode

[] listChildren = stack.getChildren(); - + CHILDREN : for(int i = listChildren.length - 1; i >= 0; --i){ AbstractStackNode

child = listChildren[i]; int childId = child.getId(); - + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> sharedChildEntry = sharedNextNodes.get(childId); if(sharedChildEntry != null && sharedChildEntry.value2 == null){ AbstractStackNode

sharedChild = sharedChildEntry.value1; @@ -1159,60 +1159,60 @@ private void expandStack(AbstractStackNode

stack){ int length = child.getLength(); int endLocation = location + length; if(endLocation > input.length) continue; - + AbstractNode result = child.match(input, location); if(result == null){ unmatchableLeafNodes.push(child); - + if(debugListener != null) debugListener.failedToMatch(child); - + continue; } - + if(debugListener != null) debugListener.matched(child, result); - + // Handle filtering IEnterFilter[] childEnterFilters = child.getEnterFilters(); if(childEnterFilters != null){ for(int j = childEnterFilters.length - 1; j >= 0; --j){ - if(childEnterFilters[j].isFiltered(input, location, positionStore)) { + if(childEnterFilters[j].isFiltered(input, location, positionStore)) { if(debugListener != null) debugListener.filteredByEnterFilter(child); - + continue CHILDREN; } } } - + child = child.getCleanCopyWithResult(location, result); queueMatchableNode(child, length, result); }else{ child = child.getCleanCopy(location); stacksToExpand.push(child); } - + child.initEdges(); child.setEdgesSetWithPrefix(cachedEdges, null, location); - + sharedNextNodes.putUnsafe(childId, child, null); - + if(debugListener != null) debugListener.expanded(stack, child); } - + expanded = true; } - + if(stack.canBeEmpty()){ // Star list or optional. AbstractStackNode

empty = stack.getEmptyChild().getCleanCopyWithResult(location, EpsilonStackNode.EPSILON_RESULT); empty.initEdges(); empty.addEdges(cachedEdges, location); - + stacksToExpand.push(empty); - + if(debugListener != null) debugListener.expanded(stack, empty); - + expanded = true; } - + if(!expanded){ unexpandableNodes.push(stack); } @@ -1224,13 +1224,13 @@ private void expandStack(AbstractStackNode

stack){ if(debugListener != null) debugListener.foundIterationCachedNullableResult(stack); } - + cachedEdges.add(stack); - + stack.setIncomingEdges(cachedEdges); } } - + /** * Initiate stack expansion for all queued stacks. */ @@ -1240,11 +1240,11 @@ private void expand(){ expandStack(stacksToExpand.pop()); } } - + protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] input){ return parse(startNode, inputURI, input, (IRecoverer

) null, (IDebugListener

) null); } - + /** * Initiates parsing. */ @@ -1339,7 +1339,7 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] try { // A parse error occured, and recovery failed as well parseErrorOccured = true; - + int errorLocation = (location == Integer.MAX_VALUE ? 0 : location); int line = positionStore.findLine(errorLocation); int column = positionStore.getColumn(errorLocation, line); @@ -1356,12 +1356,12 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] private void initTime() { timestamp = System.nanoTime(); } - + private void checkTime(String msg) { long newStamp = System.nanoTime(); long duration = newStamp - timestamp; timestamp = newStamp; - + if (printTimes) { System.err.println(msg + ": " + duration / (1000 * 1000)); } @@ -1370,13 +1370,13 @@ private void checkTime(String msg) { private static int[] charsToInts(char[] input){ int[] result = new int[Character.codePointCount(input, 0, input.length)]; int j = 0; - + for(int i = 0; i < input.length; i++){ if (!Character.isLowSurrogate(input[i])) { result[j++] = Character.codePointAt(input, i); } } - + return result; } @@ -1387,15 +1387,15 @@ private T parse(String nonterminal, URI inputURI, int[] input, IActionExecutor(AbstractStackNode.START_SYMBOL_ID, 0, nonterminal), inputURI, input, recoverer, debugListener); return buildResult(result, converter, nodeConstructorFactory, actionExecutor); } - + public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, IDebugListener

debugListener){ return parse(nonterminal, inputURI, charsToInts(input), actionExecutor, converter, nodeConstructorFactory, recoverer, debugListener); } - + public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer){ return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, recoverer, null); } - + public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IDebugListener

debugListener){ return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, null, debugListener); } @@ -1411,11 +1411,11 @@ private T parse(String nonterminal, URI inputURI, int[] input, INodeFlattener(AbstractStackNode.START_SYMBOL_ID, 0, nonterminal), inputURI, input, recoverer, debugListener); return buildResult(result, converter, nodeConstructorFactory, new VoidActionExecutor()); } - + public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, IDebugListener

debugListener){ return parse(nonterminal, inputURI, charsToInts(input), converter, nodeConstructorFactory, recoverer, debugListener); } - + public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer){ return parse(nonterminal, inputURI, input, converter, nodeConstructorFactory, recoverer, null); } @@ -1423,18 +1423,18 @@ public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IDebugListener

debugListener){ return parse(nonterminal, inputURI, input, converter, nodeConstructorFactory, null, debugListener); } - + public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory) { return parse(nonterminal, inputURI, charsToInts(input), converter, nodeConstructorFactory, null, null); } - + protected T parse(AbstractStackNode

startNode, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory) { - + AbstractNode result = parse(startNode, inputURI, charsToInts(input), null, null); - + return buildResult(result, converter, nodeConstructorFactory, new VoidActionExecutor()); } - + /** * Constructed the final parse result using the given converter. */ @@ -1542,7 +1542,7 @@ private IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory extends AbstractMatchableStackNode

{ private final SkippedNode result; - public static SkippedNode createResultUntilCharClass(URI uri, int[] until, int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilCharClass(URI uri, int[] until, int[] input, int startLocation) { for (int to = startLocation ; to < input.length; ++to) { for (int i = 0; i < until.length; ++i) { if (input[to] == until[i]) { int length = to - startLocation; - return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, length), startLocation); + return new SkippedNode(uri, createSkippedToken(input, startLocation, length), startLocation); } } } - return new SkippedNode(uri, production, dot, new int[0], startLocation); + return new SkippedNode(uri, new int[0], startLocation); } - public static SkippedNode createResultUntilEndOfInput(URI uri, int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilEndOfInput(URI uri, int[] input, int startLocation) { int length = input.length - startLocation; - return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, length), startLocation); + return new SkippedNode(uri, createSkippedToken(input, startLocation, length), startLocation); } - public static SkippedNode createResultUntilToken(URI uri, String token, int[] input, int startLocation, IConstructor production, int dot) { + public static SkippedNode createResultUntilToken(URI uri, String token, int[] input, int startLocation) { int length = token.length(); for (int start=startLocation; start+length < input.length; start++) { boolean match = true; @@ -50,7 +48,7 @@ public static SkippedNode createResultUntilToken(URI uri, String token, int[] in } if (match) { - return createResultUntilChar(uri, input, startLocation, start+length-startLocation, production, dot); + return createResultUntilChar(uri, input, startLocation, start+length-startLocation); } } } @@ -58,8 +56,8 @@ public static SkippedNode createResultUntilToken(URI uri, String token, int[] in return null; } - public static SkippedNode createResultUntilChar(URI uri, int[] input, int startLocation, int endLocation, IConstructor production, int dot) { - return new SkippedNode(uri, production, dot, createSkippedToken(input, startLocation, endLocation - startLocation), startLocation); + public static SkippedNode createResultUntilChar(URI uri, int[] input, int startLocation, int endLocation) { + return new SkippedNode(uri, createSkippedToken(input, startLocation, endLocation - startLocation), startLocation); } private static int[] createSkippedToken(int[] input, int startLocation, int length) { @@ -103,11 +101,11 @@ public AbstractNode match(int[] input, int location){ } public AbstractStackNode

getCleanCopy(int startLocation){ - return new SkippingStackNode

(this, startLocation); + return new SkippingStackNode<>(this, startLocation); } public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ - return new SkippingStackNode

(this, (SkippedNode) result, startLocation); + return new SkippingStackNode<>(this, (SkippedNode) result, startLocation); } public int getLength(){ diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java index 7b1e9439af8..cc0f4a715dd 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java @@ -32,7 +32,7 @@ public class ToNextWhitespaceRecoverer implements IRecoverer { private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; - + private IdDispenser stackNodeIdDispenser; public ToNextWhitespaceRecoverer(IdDispenser stackNodeIdDispenser) { @@ -47,21 +47,17 @@ public DoubleArrayList, AbstractNode> reviveStac DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, DoubleStack, AbstractNode> filteredNodes) { - ArrayList> failedNodes = new ArrayList>(); + ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); - // TODO: handle unmatchableLeafNodes - //collectFilteredNodes(filteredNodes, failedNodes); + // handle unmatchableLeafNodes? + // collectFilteredNodes(filteredNodes, failedNodes); - return reviveFailedNodes(input, location, failedNodes); + return reviveFailedNodes(input, failedNodes); } - private DoubleArrayList, AbstractNode> reviveNodes(int[] input, int location, DoubleArrayList, ArrayList> recoveryNodes){ - DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList, AbstractNode>(); - - // original: for (int i = recoveryNodes.size() - 1; i >= 0; --i) { - // But this caused problems because recovery nodes with a later position - // where queued before nodes with an earlier position which the parser cannot handle. + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, DoubleArrayList, ArrayList> recoveryNodes){ + DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); @@ -72,17 +68,16 @@ private DoubleArrayList, AbstractNode> reviveNod // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); - + AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); - - int dot = recoveryNode.getDot(); + int startLocation = recoveryNode.getStartLocation(); SkippedNode result; if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { - result = SkippingStackNode.createResultUntilEndOfInput(null, input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilEndOfInput(null, input, startLocation); } else { - result = SkippingStackNode.createResultUntilCharClass(null, WHITESPACE, input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilCharClass(null, WHITESPACE, input, startLocation); } AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result); @@ -90,15 +85,15 @@ private DoubleArrayList, AbstractNode> reviveNod recoverLiteral.initEdges(); EdgesSet edges = new EdgesSet<>(1); edges.add(continuer); - + recoverLiteral.addEdges(edges, startLocation); - + continuer.setIncomingEdges(edges); - + recoveredNodes.add(recoverLiteral, recoverLiteral.getResult()); } } - + return recoveredNodes; } @@ -130,23 +125,23 @@ private AbstractStackNode getSinglePredecessor(AbstractStackNode, AbstractNode> reviveFailedNodes(int[] input, int location, ArrayList> failedNodes) { - DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList, ArrayList>(); - + + private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, ArrayList> failedNodes) { + DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList<>(); + for (int i = failedNodes.size() - 1; i >= 0; --i) { findRecoveryNodes(failedNodes.get(i), recoveryNodes); } - - return reviveNodes(input, location, recoveryNodes); + + return reviveNodes(input, recoveryNodes); } - + private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { for (int i = unexpandableNodes.getSize() - 1; i >= 0; --i) { failedNodes.add(unexpandableNodes.get(i)); } } - + /** * Make a fresh copy of each unmatchable mid-production node and link in the predecessors of the original node. * The new copies are added to `failedNodes` @@ -158,14 +153,14 @@ private static void collectUnmatchableMidProductionNodes(int location, DoubleSta for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version - + // Merge the information on the predecessors into the failed node. for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); failedNode.updateNode(predecessor, predecessorResult); } - + failedNodes.add(failedNode); } } @@ -174,42 +169,42 @@ private static void collectUnmatchableMidProductionNodes(int location, DoubleSta * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. */ private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { - ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap>(); - Stack> todo = new Stack>(); - + ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap<>(); + Stack> todo = new Stack<>(); + todo.push(failer); - + while (!todo.isEmpty()) { AbstractStackNode node = todo.pop(); - + if (visited.contains(node)) { continue; // Don't follow cycles } - + visited.put(node, 0); - - ArrayList recoveryProductions = new ArrayList(); + + ArrayList recoveryProductions = new ArrayList<>(); collectProductions(node, recoveryProductions); if (recoveryProductions.size() > 0) { recoveryNodes.add(node, recoveryProductions); } - + IntegerObjectList> edges = node.getEdges(); - + for (int i = edges.size() - 1; i >= 0; --i) { // Rewind EdgesSet edgesList = edges.getValue(i); if (edgesList != null) { for (int j = edgesList.size() - 1; j >= 0; --j) { AbstractStackNode parent = edgesList.get(j); - + todo.push(parent); } } } } } - + // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) private void collectProductions(AbstractStackNode node, ArrayList productions) { AbstractStackNode[] production = node.getProduction(); @@ -219,12 +214,10 @@ private void collectProductions(AbstractStackNode node, ArrayList< int dot = node.getDot(); - System.err.println("collect productions for node: " + node); if (node.isEndNode()) { IConstructor parentProduction = node.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)){ productions.add(parentProduction); - System.err.println("adding production: " + parentProduction); if (ProductionAdapter.isList(parentProduction)) { return; // Don't follow productions in lists productions, since they are 'cyclic'. @@ -238,7 +231,6 @@ private void collectProductions(AbstractStackNode node, ArrayList< IConstructor parentProduction = currentNode.getParentProduction(); if (ProductionAdapter.isContextFree(parentProduction)) { productions.add(parentProduction); - System.err.println("adding production at " + i + ": " + parentProduction); } } @@ -250,6 +242,4 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } } - - } diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 9ef73497c44..f12f88ea3f2 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2009-2022 NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2009-2024 NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -66,7 +66,7 @@ public DoubleArrayList, AbstractNode> reviveStac DoubleStack, AbstractNode> filteredNodes) { // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to improve error recovery. - + ArrayList> failedNodes = new ArrayList<>(); collectUnexpandableNodes(unexpandableNodes, failedNodes); collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); @@ -82,25 +82,24 @@ private DoubleArrayList, AbstractNode> reviveNod DebugVisualizer visualizer = new DebugVisualizer("Recovery"); visualizer.visualizeRecoveryNodes(recoveryNodes); - + for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); ArrayList prods = recoveryNodes.getSecond(i); - int dot = recoveryNode.getDot(); int startLocation = recoveryNode.getStartLocation(); // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). for (int j = prods.size() - 1; j >= 0; --j) { IConstructor prod = prods.get(j); - - List> skippingNodes = findSkippingNodes(input, recoveryNode, prod, dot, startLocation); + + List> skippingNodes = findSkippingNodes(input, recoveryNode, prod, startLocation); for (SkippingStackNode skippingNode : skippingNodes) { AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); - + EdgesSet edges = new EdgesSet<>(1); edges.add(continuer); - + continuer.setIncomingEdges(edges); skippingNode.initEdges(); @@ -109,18 +108,18 @@ private DoubleArrayList, AbstractNode> reviveNod } } } - + return recoveredNodes; } - List> findSkippingNodes(int[] input, AbstractStackNode recoveryNode, IConstructor prod, int dot, int startLocation) { + List> findSkippingNodes(int[] input, AbstractStackNode recoveryNode, IConstructor prod, int startLocation) { List> nodes = new java.util.ArrayList<>(); SkippedNode result; // If we are the top-level node, just skip the rest of the input if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { - result = SkippingStackNode.createResultUntilEndOfInput(uri, input, startLocation, prod, dot); + result = SkippingStackNode.createResultUntilEndOfInput(uri, input, startLocation); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); return nodes; // No other nodes would be useful } @@ -139,7 +138,7 @@ List> findSkippingNodes(int[] input, AbstractSta for (InputMatcher endMatcher : endMatchers) { MatchResult endMatch = endMatcher.findMatch(input, startLocation); if (endMatch != null) { - result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, endMatch.getEnd(), prod, dot); + result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, endMatch.getEnd()); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); } } @@ -149,7 +148,7 @@ List> findSkippingNodes(int[] input, AbstractSta for (InputMatcher nextMatcher : nextMatchers) { MatchResult nextMatch = nextMatcher.findMatch(input, startLocation+1); if (nextMatch != null) { - result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart(), prod, dot); + result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart()); nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); } } @@ -181,7 +180,7 @@ List findEndMatchers(AbstractStackNode stackNode) { return matchers; } - + void addEndMatchers(AbstractStackNode[] prod, int dot, List matchers, Set visitedNodes) { if (prod == null || dot < 0) { return; @@ -330,23 +329,23 @@ private AbstractStackNode getSinglePredecessor(AbstractStackNode, AbstractNode> reviveFailedNodes(int[] input, ArrayList> failedNodes) { DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList<>(); - + for (int i = failedNodes.size() - 1; i >= 0; --i) { findRecoveryNodes(failedNodes.get(i), recoveryNodes); } - + return reviveNodes(input, recoveryNodes); } - + private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { for (int i = unexpandableNodes.getSize() - 1; i >= 0; --i) { failedNodes.add(unexpandableNodes.get(i)); } } - + /** * Make a fresh copy of each unmatchable mid-production node and link in the predecessors of the original node. * The new copies are added to `failedNodes` @@ -358,14 +357,14 @@ private static void collectUnmatchableMidProductionNodes(int location, DoubleSta for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version - + // Merge the information on the predecessors into the failed node. for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); failedNode.updateNode(predecessor, predecessorResult); } - + failedNodes.add(failedNode); } } @@ -376,33 +375,33 @@ private static void collectUnmatchableMidProductionNodes(int location, DoubleSta private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap<>(); Stack> todo = new Stack<>(); - + todo.push(failer); - + while (!todo.isEmpty()) { AbstractStackNode node = todo.pop(); - + if (visited.contains(node)) { continue; // Don't follow cycles } - + visited.put(node, 0); - + ArrayList recoveryProductions = new ArrayList<>(); collectProductions(node, recoveryProductions); if (recoveryProductions.size() > 0) { addRecoveryNode(node, recoveryProductions, recoveryNodes); } - + IntegerObjectList> edges = node.getEdges(); - + for (int i = edges.size() - 1; i >= 0; --i) { // Rewind EdgesSet edgesList = edges.getValue(i); if (edgesList != null) { for (int j = edgesList.size() - 1; j >= 0; --j) { AbstractStackNode parent = edgesList.get(j); - + todo.push(parent); } } @@ -431,7 +430,7 @@ private void addRecoveryNode(AbstractStackNode node, ArrayList node, ArrayList productions) { AbstractStackNode[] production = node.getProduction(); @@ -467,5 +466,5 @@ private void collectProductions(AbstractStackNode node, ArrayList< } } } - } + } } diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java index 91398517acf..820ba2f39b4 100644 --- a/src/org/rascalmpl/util/visualize/DebugVisualizer.java +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -7,8 +7,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.HashMap; import java.util.Map; @@ -50,9 +48,20 @@ import io.usethesource.vallang.IConstructor; +/** + * The parser uses quite complex datastructures. + * In order to understand what is going on when parsing, this class can generate graphs (as dot files) + * representing the internal datastructurs of the parser. + * + * These graphs are written to files that are relative to a directory specified in the environment + * variable PARSER_VISUALIZATION_PATH. + * + * The parser can generate a large number of snapshots of the parser state during a single parse. + * The file 'replay.html' contains an simple example of a html file to navigate through these snapshots. + */ public class DebugVisualizer { - static final String BASE_DIR = "D:/debug/parser-traces/docs/"; public static final boolean VISUALIZATION_ENABLED = true; + private static final String PARSER_VISUALIZATION_PATH_ENV = "PARSER_VISUALIZATION_PATH"; private static final boolean INCLUDE_PRODUCTIONS = false; public static final NodeId PARSER_ID = new NodeId("Parser"); @@ -69,15 +78,6 @@ public class DebugVisualizer { private static final NodeId RECOVERED_NODES_ID = new NodeId("recoveredNodes"); - /*static public class GraphObject { - static public class Kind { - private boolean dotGraph = true; - } - - private Kind kind = new Kind(); - private String text = "\ndigraph G {\n a -> b; \n}\n"; - }*/ - private static class StreamGobbler implements Runnable { private InputStream inputStream; private Consumer consumer; @@ -95,14 +95,24 @@ public void run() { } private String name; + private File basePath; + private File frameDir; private Map stackNodeNodes; private DotGraph graph; private int frame; public DebugVisualizer(String name) { + // In the future we might want to offer some way to control the path from within Rascal. + String path = System.getenv(PARSER_VISUALIZATION_PATH_ENV); + if (path == null) { + throw new RuntimeException("The environment variable '" + PARSER_VISUALIZATION_PATH_ENV + "' is not set."); + } + basePath = new File(System.getenv(PARSER_VISUALIZATION_PATH_ENV)); + this.name = name; stackNodeNodes = new HashMap<>(); - File frameDir = new File(BASE_DIR + "/frames/" + name); + + frameDir = new File(new File(basePath, "frames"), name); if (frameDir.exists()) { try { FileUtils.deleteDirectory(frameDir); @@ -603,11 +613,11 @@ private

void addStackNodeStack(DotGraph graph, NodeId nodeId, Stack + + + + From 238e4c2bbcc0b801308fd89cba2a9a78fafdd033 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 10:03:01 +0200 Subject: [PATCH 52/87] Fixed some compiler warnings in the DebugVisualizer --- .../util/visualize/DebugVisualizer.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/org/rascalmpl/util/visualize/DebugVisualizer.java b/src/org/rascalmpl/util/visualize/DebugVisualizer.java index 820ba2f39b4..0a564e31748 100644 --- a/src/org/rascalmpl/util/visualize/DebugVisualizer.java +++ b/src/org/rascalmpl/util/visualize/DebugVisualizer.java @@ -170,9 +170,9 @@ private synchronized DotGraph createGraph(AbstractStackNode stackN private DotGraph createGraph(DoubleArrayList, ArrayList> recoveryNodes) { reset(); graph = new DotGraph(name, true); - final NodeId RECOVERY_NODES_ID = new NodeId("recovery-nodes"); + final NodeId recoveryNodesId = new NodeId("recovery-nodes"); - DotNode arrayNode = DotNode.createArrayNode(RECOVERY_NODES_ID, recoveryNodes.size()); + DotNode arrayNode = DotNode.createArrayNode(recoveryNodesId, recoveryNodes.size()); graph.addNode(arrayNode); for (int i=0; i, Ar recoveryRecord.addEntry(new DotField("Productions", "productions")); graph.addRecordNode(pairId, recoveryRecord); - graph.addEdge(new NodeId(RECOVERY_NODES_ID, String.valueOf(i)), pairId); + graph.addEdge(new NodeId(recoveryNodesId, String.valueOf(i)), pairId); DotNode node = addStack(graph, recoveryNodes.getFirst(i)); @@ -333,11 +333,10 @@ private

DotNode createDotNode(AbstractStackNode

stackNode) { } node.addAttribute(DotAttribute.ATTR_LABEL, label); - // TODO: add prefixes - return node; } + @SuppressWarnings("unchecked") private void addParserNode(DotGraph graph, AbstractNode parserNode) { NodeId id = getNodeId(parserNode); DotNode dotNode = new DotNode(id); @@ -543,13 +542,6 @@ private void addUnmatchableMidProductionNodes(SGTDBF parser, private void addFilteredNodes(SGTDBF parser, DotGraph graph) { addStackAndNodeDoubleStack(graph, FILTERED_NODES_ID, parser.getFilteredNodes()); } - /* - * private final Stack> unexpandableNodes; - private final Stack> unmatchableLeafNodes; // Leaf nodes (for instance literals) that failed to match - private final DoubleStack, AbstractNode>, AbstractStackNode

> unmatchableMidProductionNodes; - private final DoubleStack, AbstractNode> filteredNodes; - - */ private void addStackAndNodeDoubleStack(DotGraph graph, NodeId nodeId, DoubleStack, N> doubleStack) { DotNode arrayNode = DotNode.createArrayNode(nodeId, doubleStack == null ? 0 : doubleStack.getSize()); From 704726bba54372dd22ec1bd4584b524fdde92c4a Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:16:31 +0200 Subject: [PATCH 53/87] Fixed some compiler warnings --- .../gtd/stack/AbstractMatchableStackNode.java | 21 +- .../parser/gtd/stack/AbstractStackNode.java | 232 +++++++++--------- .../gtd/stack/AlternativeStackNode.java | 57 +++-- .../CaseInsensitiveLiteralStackNode.java | 83 ++++--- .../parser/gtd/stack/CharStackNode.java | 76 +++--- .../parser/gtd/stack/EmptyStackNode.java | 51 ++-- .../parser/gtd/stack/EpsilonStackNode.java | 56 +++-- .../parser/gtd/stack/ListStackNode.java | 57 +++-- .../parser/gtd/stack/LiteralStackNode.java | 60 +++-- .../gtd/stack/MultiCharacterStackNode.java | 95 +++---- .../gtd/stack/NonTerminalStackNode.java | 47 ++-- .../parser/gtd/stack/OptionalStackNode.java | 51 ++-- .../gtd/stack/RecoveryPointStackNode.java | 41 ++-- .../gtd/stack/SeparatedListStackNode.java | 55 +++-- .../parser/gtd/stack/SequenceStackNode.java | 53 ++-- .../parser/gtd/stack/SkippingStackNode.java | 4 + 16 files changed, 586 insertions(+), 453 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java index 3943c6dca5f..2c165cbe3d0 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractMatchableStackNode.java @@ -21,7 +21,7 @@ * Literals and characters are examples of matchable nodes. */ public abstract class AbstractMatchableStackNode

extends AbstractStackNode

{ - + protected AbstractMatchableStackNode(int id, int dot){ super(id, dot); } @@ -33,39 +33,40 @@ protected AbstractMatchableStackNode(int id, int dot, int startLocation){ protected AbstractMatchableStackNode(int id, int dot, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); } - + protected AbstractMatchableStackNode(AbstractMatchableStackNode

original, int startLocation){ super(original, startLocation); } - + /** * Matches the node to the input string and the indicated location and * constructs the result in case the match was successful. Null will - * be returned otherwise. + * be returned otherwise. */ public abstract AbstractNode match(int[] input, int location); - + /** * Returns the length (in number of characters) of the matchable. */ public abstract int getLength(); - + public String getName(){ throw new UnsupportedOperationException(); } - + public AbstractStackNode

[] getChildren(){ throw new UnsupportedOperationException(); } - + public boolean canBeEmpty(){ throw new UnsupportedOperationException(); } - + public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } - + + @Override public final boolean isMatchable(){ return true; } diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index b477301e954..73a8ab793b9 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -30,7 +30,7 @@ public abstract class AbstractStackNode

{ public static final int START_SYMBOL_ID = -1; public static final int DEFAULT_START_LOCATION = -1; - + protected AbstractStackNode

[] production; protected AbstractStackNode

[][] alternateProductions; @@ -38,82 +38,82 @@ public abstract class AbstractStackNode

{ protected IntegerObjectList> edgesMap; // : key=startLocation, value=EdgesSet at that location // Edges of our children protected ArrayList[] prefixesMap; - + protected EdgesSet

incomingEdges; - + protected final int id; protected final int dot; - + protected final int startLocation; - + // Flags private boolean isEndNode; private boolean isSeparator; private boolean isLayout; - + // Filters private final IEnterFilter[] enterFilters; private final ICompletionFilter[] completionFilters; - + // The production (specific to end nodes only) private P alternativeProduction; - + // Hidden-right-recursion related private BitSet propagatedPrefixes; private IntegerList propagatedReductions; - + protected AbstractStackNode(int id, int dot){ this(id, dot, DEFAULT_START_LOCATION); } protected AbstractStackNode(int id, int dot, int startLocation) { super(); - + this.id = id; this.dot = dot; - + this.startLocation = startLocation; - + this.enterFilters = null; this.completionFilters = null; } protected AbstractStackNode(int id, int dot, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(); - + this.id = id; this.dot = dot; - + this.startLocation = DEFAULT_START_LOCATION; - + this.enterFilters = enterFilters; this.completionFilters = completionFilters; } - + protected AbstractStackNode(AbstractStackNode

original, int startLocation){ this(original.id, original, startLocation); } - + protected AbstractStackNode(int id, AbstractStackNode

original, int startLocation){ super(); - + this.id = id; dot = original.dot; - + production = original.production; alternateProductions = original.alternateProductions; - + this.startLocation = startLocation; - + isEndNode = original.isEndNode; isSeparator = original.isSeparator; isLayout = original.isLayout; - + alternativeProduction = original.alternativeProduction; enterFilters = original.enterFilters; completionFilters = original.completionFilters; } - + // General. /** * Returns the id of the node. @@ -121,14 +121,14 @@ protected AbstractStackNode(int id, AbstractStackNode

original, int startLoca public int getId(){ return id; } - + /** * Returns the start location of this node. */ public int getStartLocation(){ return startLocation; } - + /** * Checks whether or not this node is the last node in the production. * Note that while this may be an end node alternative continuations of this production may be possible. @@ -136,46 +136,46 @@ public int getStartLocation(){ public boolean isEndNode(){ return isEndNode; } - + public boolean isRecovered() { return false; } - + /** * Mark this node as being a separator. */ public void markAsSeparator(){ isSeparator = true; } - + /** * Checks whether or not this nodes is a separator. */ public boolean isSeparator(){ return isSeparator; } - + /** * Marks this node as representing layout. */ public void markAsLayout(){ isLayout = true; } - + /** * Checks whether or not this node represents layout. */ public boolean isLayout(){ return isLayout; } - + /** * Checks whether this node represents a 'matchable' leaf node. */ public boolean isMatchable(){ return false; } - + /** * Check wheterh this node represents a node which needs to be expanded in-place. */ @@ -186,7 +186,7 @@ public boolean isExpandable(){ * Checks if this node represents a nullable leaf node. */ public abstract boolean isEmptyLeafNode(); - + /** * Returns the name associated with the symbol in this node (optional operation). */ @@ -196,7 +196,7 @@ public boolean isExpandable(){ * Check whether of this this node is equal to the given node. */ public abstract boolean isEqual(AbstractStackNode

stackNode); - + // Last node specific stuff. /** * Associates a production with this node, indicating that this is the last node in the production. @@ -206,7 +206,7 @@ public void setAlternativeProduction(P parentProduction){ this.alternativeProduction = parentProduction; isEndNode = true; } - + /** * Retrieves the production associated with the alternative this node is a part of. * Only the last node in the alternative will have this production on it. @@ -214,7 +214,7 @@ public void setAlternativeProduction(P parentProduction){ public P getParentProduction(){ return alternativeProduction; } - + /** * Retrieves the enter filters associated with the symbol in this node. * The returned value may be null if no enter filters are present. @@ -222,7 +222,7 @@ public P getParentProduction(){ public IEnterFilter[] getEnterFilters(){ return enterFilters; } - + /** * Retrieves the completion filters associated with the symbol in this node. * The returned value may be null if no completion filters are present. @@ -230,7 +230,7 @@ public IEnterFilter[] getEnterFilters(){ public ICompletionFilter[] getCompletionFilters(){ return completionFilters; } - + /** * Checks whether or not the filters that are associated with this nodes symbol are equal to those of the given node. */ @@ -251,13 +251,13 @@ public boolean hasEqualFilters(AbstractStackNode

otherNode){ }else if(enterFilters != null){ return false; } - + ICompletionFilter[] otherCompletionFilters = otherNode.completionFilters; if(otherCompletionFilters != null){ if(completionFilters == null || completionFilters.length != otherCompletionFilters.length) { return false; } - + OUTER: for(int i = completionFilters.length - 1; i >= 0; --i){ ICompletionFilter completionFilter = completionFilters[i]; for(int j = otherCompletionFilters.length - 1; j >= 0; --j){ @@ -268,15 +268,15 @@ public boolean hasEqualFilters(AbstractStackNode

otherNode){ }else if(completionFilters != null){ return false; } - + return true; } - + /** * Children must override the hash code method (it can't be enforce, but I'm putting it in here anyway). */ public abstract int hashCode(); - + /** * Checks equality. */ @@ -286,20 +286,20 @@ public boolean equals(Object o){ } return false; } - + // Creation and sharing. /** * Creates a new copy of this node for the indicated position. */ public abstract AbstractStackNode

getCleanCopy(int startLocation); - + /** * Creates a new copy of this node for the indicated position and associated the given result with it. * This method has the same effect as the one above, but is exclusively used for matchable nodes; * as their results are constructed before the node is created. */ public abstract AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result); - + /** * Checks whether or not this node has the same id as the given one. * This method is used when determining whether or not stacks should to be merged. @@ -307,7 +307,7 @@ public boolean equals(Object o){ public boolean isSimilar(AbstractStackNode

node){ return (node.id == id); } - + // Node continuations. /** * Indicates the current location in the alternative this node is a part of. @@ -315,14 +315,14 @@ public boolean isSimilar(AbstractStackNode

node){ public int getDot(){ return dot; } - + /** * Sets the main alternative this node is a part of. */ public void setProduction(AbstractStackNode

[] production){ this.production = production; } - + /** * Adds an additional alternative this node is a part of. * This can be the case if the symbol this node is associated with is located in a shared prefix of more then one alternative. @@ -342,28 +342,28 @@ public void addProduction(AbstractStackNode

[] production){ } } } - + /** * Checks if this node has possible continuations. */ public boolean hasNext(){ return !((production == null) || ((dot + 1) == production.length)); } - + /** * Retrieves the main alternative the symbol this node is associated with is a part of. */ public AbstractStackNode

[] getProduction(){ return production; } - + /** * Retrieves the alternative continuations the symbols this node is associated with is a part of. */ public AbstractStackNode

[][] getAlternateProductions(){ return alternateProductions; } - + // Edges & prefixes. /** * Initializes the set of edges of this node. @@ -371,14 +371,14 @@ public AbstractStackNode

[][] getAlternateProductions(){ public void initEdges(){ edgesMap = new IntegerObjectList<>(); } - + /** * Assigns the set of incoming edges to this stack node. Because of sharing there will always be only one of these sets. */ public void setIncomingEdges(EdgesSet

incomingEdges){ this.incomingEdges = incomingEdges; } - + /** * Adds the given edge to the set of edges for the indicated location. */ @@ -388,22 +388,22 @@ public EdgesSet

addEdge(AbstractStackNode

edge, int startLocation){ edges = new EdgesSet<>(1); edgesMap.add(startLocation, edges); } - + edges.add(edge); - + return edges; } - + /** * Adds the given set of edges to the edge map for the indicated location. */ public void addEdges(EdgesSet

edges, int startLocation){ edgesMap.add(startLocation, edges); } - + /** * Sets the given edges set for the indicated location and associated the given prefix with this node. - * + * * This method is used by children of expandable nodes; * expandable nodes contain dynamically unfolding alternatives, which can be cyclic. * A 'null' prefix, for example, indicates that the current node starts the alternative. @@ -422,9 +422,9 @@ public void setEdgesSetWithPrefix(EdgesSet

edges, Link prefix, int startLocat System.arraycopy(oldPrefixesMap, 0, prefixesMap, 0, edgesMapSize); } } - + edgesMap.add(startLocation, edges); - + ArrayList prefixes = prefixesMap[edgesMapSize]; if(prefixes == null){ prefixes = new ArrayList<>(1); @@ -432,7 +432,7 @@ public void setEdgesSetWithPrefix(EdgesSet

edges, Link prefix, int startLocat } prefixes.add(prefix); } - + /** * Add the given prefix at the indicated index. * This index is directly linked to a certain set of edges in the edge map. @@ -443,29 +443,29 @@ private void addPrefix(Link prefix, int index){ prefixes = new ArrayList<>(1); prefixesMap[index] = prefixes; } - + prefixes.add(prefix); } - + /** * Updates this node with information from its predecessor in the alternative. * The sets of edges on this predecessor will be transferred to this node and * the required prefixes for this node will be constructed, using the predecessor's * prefixes in combination with its given result. - * + * * This method also takes care of stack merges in the process of doing this. */ public void updateNode(AbstractStackNode

predecessor, AbstractNode predecessorResult){ IntegerObjectList> edgesMapToAdd = predecessor.edgesMap; ArrayList[] prefixesMapToAdd = predecessor.prefixesMap; - + if(edgesMap == null){ // Clean node, no stack merge occurred. // Initialize the edges map by cloning the one of the predecessor, as we need to transfer all these edges to this node. edgesMap = new IntegerObjectList<>(edgesMapToAdd); - + // Initialize the prefixes map. prefixesMap = new ArrayList[edgesMap.size()]; - + if(prefixesMapToAdd == null){ // The predecessor was the first node in the alternative, so the prefix of this node is just the predecessor's result. int index = edgesMap.findKey(predecessor.getStartLocation()); addPrefix(new Link(null, predecessorResult), index); @@ -477,7 +477,7 @@ public void updateNode(AbstractStackNode

predecessor, AbstractNode predecesso prefixes = new ArrayList<>(1); prefixesMap[i] = prefixes; } - + prefixes.add(new Link(prefixesMapToAdd[i], predecessorResult)); } } @@ -517,7 +517,7 @@ private void handleStackMergeForNonCyclicProduction(AbstractNode predecessorResu System.arraycopy(oldPrefixesMap, 0, prefixesMap, 0, edgesMapSize); } } - + if(prefixesMapToAdd == null){ // The predecessor was the first node in the alternative, so the prefix of this node is just the predecessor's result. addPrefix(new Link(null, predecessorResult), edgesMapSize); edgesMap.add(edgesMapToAdd.getKey(0), edgesMapToAdd.getValue(0)); @@ -529,38 +529,38 @@ private void handleStackMergeForNonCyclicProduction(AbstractNode predecessorResu if(index == -1){ // No prefix set for the given start location is present yet. index = edgesMap.size(); edgesMap.add(locationStart, edgesMapToAdd.getValue(i)); - + prefixes = new ArrayList<>(1); prefixesMap[index] = prefixes; }else{ // A prefix set for the given start location is present. prefixes = prefixesMap[index]; } - + // Add the prefix to the appropriate prefix set. prefixes.add(new Link(prefixesMapToAdd[i], predecessorResult)); } } } - + /** * This method is a specialized version of 'updateNode'. - * + * * In essence its function is the same, with the difference that stack merges are not handled * and the edges map is simply reused. - * + * * The is part of the 'selective edge sharing' optimization. * Since it is guaranteed that stack merges can never occur after non-nullable terminals * in a production it is save to use this assumption to improve efficiency. */ public void updateNodeAfterNonEmptyMatchable(AbstractStackNode

predecessor, AbstractNode result){ ArrayList[] prefixesMapToAdd = predecessor.prefixesMap; - + // Set the edges map. edgesMap = predecessor.edgesMap; - + // Initialize the prefixes map. prefixesMap = new ArrayList[edgesMap.size()]; - + if(prefixesMapToAdd == null){ // The predecessor was the first node in the alternative, so the prefix of this node is just the predecessor's result. addPrefix(new Link(null, result), edgesMap.findKey(predecessor.getStartLocation())); }else{ // The predecessor has prefixes. @@ -573,13 +573,13 @@ public void updateNodeAfterNonEmptyMatchable(AbstractStackNode

predecessor, A } } } - + /** * This method is a specialized version of 'updateNode'. - * + * * Specifically, this is a part of the hidden-right-recursion fix * (for non-nullable predecessor nodes). - * + * * It merges stacks and keeps track of which ones get merged * (the returned number indicated how many edge sets were added and * may need to be propagated forward). @@ -587,7 +587,7 @@ public void updateNodeAfterNonEmptyMatchable(AbstractStackNode

predecessor, A public int updateOvertakenNode(AbstractStackNode

predecessor, AbstractNode result){ IntegerObjectList> edgesMapToAdd = predecessor.edgesMap; ArrayList[] prefixesMapToAdd = predecessor.prefixesMap; - + // Initialize the prefixes map. int edgesMapSize = edgesMap.size(); int possibleMaxSize = edgesMapSize + edgesMapSize; @@ -600,7 +600,7 @@ public int updateOvertakenNode(AbstractStackNode

predecessor, AbstractNode re System.arraycopy(oldPrefixesMap, 0, prefixesMap, 0, edgesMapSize); } } - + int nrOfAddedEdges = 0; if(prefixesMapToAdd == null){ // The predecessor was the first node in the alternative, so the prefix of this node is just the predecessor's result. addPrefix(new Link(null, result), edgesMapSize); @@ -609,36 +609,36 @@ public int updateOvertakenNode(AbstractStackNode

predecessor, AbstractNode re }else{ // The predecessor has prefixes. for(int i = edgesMapToAdd.size() - 1; i >= 0; --i){ int locationStart = edgesMapToAdd.getKey(i); - + // Prefix not present, add it. int index = edgesMap.findKey(locationStart); ArrayList prefixes; if(index == -1){ // No prefix set for the given start location is present yet. index = edgesMap.size(); edgesMap.add(locationStart, edgesMapToAdd.getValue(i)); - + prefixes = new ArrayList<>(1); prefixesMap[index] = prefixes; - + ++nrOfAddedEdges; }else{ // A prefix set for the given start location is present. prefixes = prefixesMap[index]; } - + // Add the prefix to the prefix set. prefixes.add(new Link(prefixesMapToAdd[i], result)); } } - + return nrOfAddedEdges; } - + /** * This method is a specialized version of 'updateNode'. - * + * * Specifically, this is a part of the hidden-right-recursion fix * (for nullable predecessor nodes). - * + * * It merges stacks and keeps track of which ones get merged * (the returned number indicated how many edge sets were added and * may need to be propagated forward). @@ -648,13 +648,13 @@ public int updateOvertakenNode(AbstractStackNode

predecessor, AbstractNode re public int updateOvertakenNullableNode(AbstractStackNode

predecessor, AbstractNode result, int potentialNewEdges){ IntegerObjectList> edgesMapToAdd = predecessor.edgesMap; ArrayList[] prefixesMapToAdd = predecessor.prefixesMap; - + // Initialize the prefixes map. int edgesMapSize = edgesMap.size(); int possibleMaxSize = edgesMapSize + potentialNewEdges; if(prefixesMap == null){ prefixesMap = new ArrayList[possibleMaxSize]; - + propagatedPrefixes = new BitSet(edgesMapSize); }else{ if(prefixesMap.length < possibleMaxSize){ @@ -662,14 +662,14 @@ public int updateOvertakenNullableNode(AbstractStackNode

predecessor, Abstrac prefixesMap = new ArrayList[possibleMaxSize]; System.arraycopy(oldPrefixesMap, 0, prefixesMap, 0, edgesMapSize); } - + if(propagatedPrefixes == null){ propagatedPrefixes = new BitSet(edgesMapSize); }else{ propagatedPrefixes.enlargeTo(possibleMaxSize); } } - + int nrOfAddedEdges = 0; if(prefixesMapToAdd == null){ // The predecessor was the first node in the alternative, so the prefix of this node is just the predecessor's result. // A prefix is not yet present, so add it. As it's the first and only possible nullable prefix, it's guaranteed that there aren't any prefixes for the current start location present yet. @@ -679,72 +679,72 @@ public int updateOvertakenNullableNode(AbstractStackNode

predecessor, Abstrac }else{ // The predecessor has prefixes. for(int i = edgesMapToAdd.size() - 1; i >= 0; --i){ int locationStart = edgesMapToAdd.getKey(i); - + // Prefix not present, add it. int index = edgesMap.findKey(locationStart); - + ArrayList prefixes; if(index == -1){ // No prefix set for the given start location is present yet. index = edgesMap.size(); edgesMap.add(locationStart, edgesMapToAdd.getValue(i)); propagatedPrefixes.set(index); - + prefixes = new ArrayList<>(1); prefixesMap[index] = prefixes; - + ++nrOfAddedEdges; }else{ // A prefix set for the given start location is present. if(propagatedPrefixes.isSet(index)) continue; // Prefix present, abort. - + prefixes = prefixesMap[index]; } - + // Add the prefix to the prefix set. prefixes.add(new Link(prefixesMapToAdd[i], result)); } } - + return nrOfAddedEdges; } - + /** * This method is a specialized version of 'updateNode'. - * + * * When alternatives have a shared prefix, their successors can shared the same edges and prefixes maps. */ public void updatePrefixSharedNode(IntegerObjectList> edgesMap, ArrayList[] prefixesMap){ this.edgesMap = edgesMap; this.prefixesMap = prefixesMap; } - + /** * Returns the edges map. */ public IntegerObjectList> getEdges(){ return edgesMap; } - + /** * Returns the incoming edges set. */ public EdgesSet

getIncomingEdges(){ return incomingEdges; } - + /** * Returns the prefixes map. */ public ArrayList[] getPrefixesMap(){ return prefixesMap; } - + /** * Returns the list of start location for which the results have already * been propagated (hidden-right-recursion specific). */ public IntegerList getPropagatedReductions(){ if(propagatedReductions == null) propagatedReductions = new IntegerList(); - + return propagatedReductions; } @@ -811,34 +811,36 @@ public String toString() { return builder.toString(); } + abstract void accept(StackNodeVisitor

visitor); + // Matchables. /** * Matches the symbol associated with this node to the input at the specified location. * This operation is only available on matchable nodes. */ public abstract AbstractNode match(int[] input, int location); - + /** * Returns the length of this node (only applies to matchables). */ public abstract int getLength(); - + /** * Returns the result associated with this node (only applies to matchables, after matching). */ public abstract AbstractNode getResult(); - + // Expandables. /** * Returns the set of children of this node (only applies to expandables). */ public abstract AbstractStackNode

[] getChildren(); - + /** * Returns whether or not this node can be nullable (only applies to expandables). */ public abstract boolean canBeEmpty(); - + /** * Returns the empty child of this node (only applies to expandables that are nullable). */ diff --git a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java index dceb05517ec..4a4570b0403 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java @@ -18,30 +18,30 @@ public class AlternativeStackNode

extends AbstractExpandableStackNode

{ private final P production; private final String name; - + private final AbstractStackNode

[] children; - + public AlternativeStackNode(int id, int dot, P production, AbstractStackNode

[] alternatives){ super(id, dot); - + this.production = production; this.name = String.valueOf(id); - + this.children = generateAlternatives(alternatives); } - + public AlternativeStackNode(int id, int dot, P production, AbstractStackNode

[] alternatives, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); - + this.production = production; this.name = String.valueOf(id); - + this.children = generateAlternatives(alternatives); } - + private AlternativeStackNode(AlternativeStackNode

original, int startLocation){ super(original, startLocation); - + production = original.production; name = original.name; @@ -54,44 +54,46 @@ private AlternativeStackNode(AlternativeStackNode

original, int startLocation @SuppressWarnings("unchecked") private AbstractStackNode

[] generateAlternatives(AbstractStackNode

[] alternatives){ AbstractStackNode

[] children = (AbstractStackNode

[]) new AbstractStackNode[alternatives.length]; - + for(int i = alternatives.length - 1; i >= 0; --i){ AbstractStackNode

child = alternatives[i].getCleanCopy(DEFAULT_START_LOCATION); AbstractStackNode

[] prod = (AbstractStackNode

[]) new AbstractStackNode[]{child}; child.setProduction(prod); child.setAlternativeProduction(production); - + children[i] = child; } - + return children; } - + public String getName(){ return name; } - + public AbstractStackNode

getCleanCopy(int startLocation){ return new AlternativeStackNode

(this, startLocation); } - + public AbstractStackNode

[] getChildren(){ return children; } - + public boolean canBeEmpty(){ return false; } - + public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append("alt"); @@ -99,21 +101,32 @@ public String toString(){ sb.append('('); sb.append(startLocation); sb.append(')'); - + return sb.toString(); } - + + @Override public int hashCode(){ return production.hashCode(); } - + + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof AlternativeStackNode)) return false; - + AlternativeStackNode

otherNode = (AlternativeStackNode

) stackNode; if(!production.equals(otherNode.production)) return false; - + return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java index 169a5e2fd66..5fe0f1b4c0a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java @@ -18,51 +18,51 @@ public final class CaseInsensitiveLiteralStackNode

extends AbstractMatchableStackNode

{ private final Object production; - + private final int[][] ciLiteral; - + private final AbstractNode result; - + public CaseInsensitiveLiteralStackNode(int id, int dot, Object production, int[] ciLiteral){ super(id, dot); - + this.production = production; - + this.ciLiteral = fill(ciLiteral); - + result = null; } - + public CaseInsensitiveLiteralStackNode(int id, int dot, P production, int[] ciLiteral, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); - + this.production = production; - + this.ciLiteral = fill(ciLiteral); - + result = null; } - + private CaseInsensitiveLiteralStackNode(CaseInsensitiveLiteralStackNode

original, int startLocation){ super(original, startLocation); - + production = original.production; - + ciLiteral = original.ciLiteral; - + result = null; } - + private CaseInsensitiveLiteralStackNode(CaseInsensitiveLiteralStackNode

original, int startLocation, AbstractNode result){ super(original, startLocation); - + this.production = original.production; - + this.ciLiteral = original.ciLiteral; - + this.result = result; } - + private static int[][] fill(int[] ciLiteral){ int nrOfCharacters = ciLiteral.length; int[][] ciLiteralResult = new int[nrOfCharacters][]; @@ -79,11 +79,11 @@ private static int[][] fill(int[] ciLiteral){ } return ciLiteralResult; } - + public boolean isEmptyLeafNode(){ return false; } - + public AbstractNode match(int[] input, int location){ int literalLength = ciLiteral.length; int[] resultLiteral = new int[literalLength]; @@ -98,22 +98,22 @@ public AbstractNode match(int[] input, int location){ } return null; // Did not match. } - + return new LiteralNode(production, resultLiteral); } - + public AbstractStackNode

getCleanCopy(int startLocation){ - return new CaseInsensitiveLiteralStackNode

(this, startLocation); + return new CaseInsensitiveLiteralStackNode<>(this, startLocation); } - + public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ - return new CaseInsensitiveLiteralStackNode

(this, startLocation, result); + return new CaseInsensitiveLiteralStackNode<>(this, startLocation, result); } - + public int getLength(){ return ciLiteral.length; } - + public AbstractNode getResult(){ return result; } @@ -121,7 +121,7 @@ public AbstractNode getResult(){ public int[][] getLiteral() { return ciLiteral; } - + @Override public String toShortString() { int[] codePoints = new int[ciLiteral.length]; @@ -131,6 +131,7 @@ public String toShortString() { return new String(codePoints, 0, codePoints.length); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); for(int i = 0; i < ciLiteral.length; ++i){ @@ -140,21 +141,33 @@ public String toString(){ sb.append('('); sb.append(startLocation); sb.append(')'); - + return sb.toString(); } - + + @Override public int hashCode(){ return production.hashCode(); } - + + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + + @Override public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof CaseInsensitiveLiteralStackNode)) return false; - + CaseInsensitiveLiteralStackNode

otherNode = (CaseInsensitiveLiteralStackNode

) stackNode; - + if(!production.equals(otherNode.production)) return false; - + return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java index 8f2bd794944..d9a0689cec5 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java @@ -15,78 +15,77 @@ import org.rascalmpl.parser.gtd.result.CharNode; import org.rascalmpl.parser.gtd.stack.filter.ICompletionFilter; import org.rascalmpl.parser.gtd.stack.filter.IEnterFilter; -import org.rascalmpl.unicode.UnicodeConverter; public final class CharStackNode

extends AbstractMatchableStackNode

{ private final int[][] ranges; - + private final AbstractNode result; - + public CharStackNode(int id, int dot, int[][] ranges){ super(id, dot); this.ranges = ranges; - + result = null; } - + public CharStackNode(int id, int dot, int[][] ranges, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); this.ranges = ranges; - + result = null; } - + private CharStackNode(CharStackNode

original, int startLocation){ super(original, startLocation); - + ranges = original.ranges; - + result = null; } - + private CharStackNode(CharStackNode

original, int startLocation, AbstractNode result){ super(original, startLocation); - + this.ranges = original.ranges; - + this.result = result; } - + public boolean isEmptyLeafNode(){ return false; } - + public AbstractNode match(int[] input, int location){ int next = input[location]; - + for(int i = ranges.length - 1; i >= 0; --i){ int[] range = ranges[i]; if(next >= range[0] && next <= range[1]){ return CharNode.createCharNode(next); } } - + return null; } - + public AbstractStackNode

getCleanCopy(int startLocation){ - return new CharStackNode

(this, startLocation); + return new CharStackNode<>(this, startLocation); } - + public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ - return new CharStackNode

(this, startLocation, result); + return new CharStackNode<>(this, startLocation, result); } - + public int getLength(){ return 1; } - + public AbstractNode getResult(){ return result; } - + @Override public String toShortString() { StringBuilder sb = new StringBuilder(); @@ -114,9 +113,10 @@ private String codePointToString(int codePoint) { return String.valueOf(codePoint); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); - + sb.append("CharStackNode[class="); int[] range = ranges[0]; sb.append(range[0]); @@ -132,30 +132,35 @@ public String toString(){ sb.append(","); sb.append(super.toString()); sb.append(']'); - + return sb.toString(); } - + public int hashCode(){ int hash = 0; - + for(int i = ranges.length - 1; i >= 0; --i){ int[] range = ranges[i]; hash = hash << 3 + hash >> 5; hash ^= range[0] + (range[1] << 2); } - + return hash; } - + + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof CharStackNode)) return false; - + CharStackNode

otherNode = (CharStackNode

) stackNode; - + int[][] otherRanges = otherNode.ranges; if(ranges.length != otherRanges.length) return false; - + OUTER: for(int i = ranges.length - 1; i >= 0; --i){ int[] range = ranges[i]; for(int j = otherRanges.length - 1; j >= 0; --j){ @@ -165,7 +170,12 @@ public boolean isEqual(AbstractStackNode

stackNode){ return false; // Could not find a certain range. } // Found all ranges. - + return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java index 685c839065d..65c1711b383 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java @@ -25,82 +25,97 @@ public final class EmptyStackNode

extends AbstractExpandableStackNode

{ private final AbstractStackNode

emptyChild; private static final AbstractStackNode[] children = new AbstractStackNode[0]; - + public EmptyStackNode(int id, int dot, P production){ super(id, dot); - + this.production = production; this.name = "empty"+id; // Add the id to make it unique. - + this.emptyChild = generateEmptyChild(); } - + public EmptyStackNode(int id, int dot, P production, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters) { super(id, dot, enterFilters, completionFilters); - + this.production = production; this.name = "empty"+id; - this.emptyChild = generateEmptyChild(); + this.emptyChild = generateEmptyChild(); } - + private EmptyStackNode(EmptyStackNode

original, int startLocation){ super(original, startLocation); - + production = original.production; name = original.name; emptyChild = original.emptyChild; } - + @SuppressWarnings("unchecked") private AbstractStackNode

generateEmptyChild(){ AbstractStackNode

empty = (AbstractStackNode

) EMPTY.getCleanCopy(DEFAULT_START_LOCATION); empty.setAlternativeProduction(production); return empty; } - + public String getName(){ return name; } - + public AbstractStackNode

getCleanCopy(int startLocation){ return new EmptyStackNode

(this, startLocation); } - + @SuppressWarnings("unchecked") public AbstractStackNode

[] getChildren(){ return (AbstractStackNode

[]) children; } - + public boolean canBeEmpty(){ return true; } - + public AbstractStackNode

getEmptyChild(){ return emptyChild; } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); sb.append('('); sb.append(startLocation); sb.append(')'); - + return sb.toString(); } - + + @Override public int hashCode(){ return 1; } - + + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + + @Override public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof EmptyStackNode)) return false; - + return hasEqualFilters(stackNode); } + + @Override + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java index dfa16c05fed..0e05002d9a1 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java @@ -18,73 +18,85 @@ public final class EpsilonStackNode

extends AbstractMatchableStackNode

{ public final static EpsilonNode EPSILON_RESULT = new EpsilonNode(); - + private final AbstractNode result; - + public EpsilonStackNode(int id, int dot){ super(id, dot); - + result = null; } - + public EpsilonStackNode(int id, int dot, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); - + result = null; } - + private EpsilonStackNode(EpsilonStackNode

original, int startLocation){ super(original, startLocation); - + result = null; } - + private EpsilonStackNode(EpsilonStackNode

original, int startLocation, AbstractNode result){ super(original, startLocation); - + this.result = result; } - + public boolean isEmptyLeafNode(){ return true; } - + public AbstractNode match(int[] input, int location){ return EPSILON_RESULT; } - + public AbstractStackNode

getCleanCopy(int startLocation){ - return new EpsilonStackNode

(this, startLocation); + return new EpsilonStackNode<>(this, startLocation); } - + public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ - return new EpsilonStackNode

(this, startLocation, result); + return new EpsilonStackNode<>(this, startLocation, result); } - + public int getLength(){ return 0; } - + public AbstractNode getResult(){ return result; } - + + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append('('); sb.append(startLocation); sb.append(')'); - + return sb.toString(); } - + + @Override public int hashCode(){ return 0; } - + + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof EpsilonStackNode)) return false; - + return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java index a513a9472c9..5f15b9f7af2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java @@ -21,37 +21,37 @@ public final class ListStackNode

extends AbstractExpandableStackNode

{ private final AbstractStackNode

[] children; private final AbstractStackNode

emptyChild; - + public ListStackNode(int id, int dot, P production, AbstractStackNode

child, boolean isPlusList){ super(id, dot); - + this.production = production; this.name = String.valueOf(id); // Add the id to make it unique. - + this.children = generateChildren(child); this.emptyChild = isPlusList ? null : generateEmptyChild(); } - + public ListStackNode(int id, int dot, P production, AbstractStackNode

child, boolean isPlusList, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); - + this.production = production; this.name = String.valueOf(id); // Add the id to make it unique. - + this.children = generateChildren(child); this.emptyChild = isPlusList ? null : generateEmptyChild(); } - + private ListStackNode(ListStackNode

original, int startLocation){ super(original, startLocation); - + production = original.production; name = original.name; children = original.children; emptyChild = original.emptyChild; } - + /** * Generates and initializes the alternative for this list. */ @@ -62,7 +62,7 @@ private AbstractStackNode

[] generateChildren(AbstractStackNode

child){ listNode.setProduction((AbstractStackNode

[]) new AbstractStackNode[]{listNode, listNode}); return (AbstractStackNode

[]) new AbstractStackNode[]{listNode}; } - + /** * Generates and initializes the empty child for this list (in case this is a star list). */ @@ -72,52 +72,65 @@ private AbstractStackNode

generateEmptyChild(){ empty.setAlternativeProduction(production); return empty; } - + public String getName(){ return name; } - + public AbstractStackNode

getCleanCopy(int startLocation){ - return new ListStackNode

(this, startLocation); + return new ListStackNode<>(this, startLocation); } - + public AbstractStackNode

[] getChildren(){ return children; } - + public boolean canBeEmpty(){ return emptyChild != null; } - + public AbstractStackNode

getEmptyChild(){ return emptyChild; } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); sb.append('('); sb.append(startLocation); sb.append(')'); - + return sb.toString(); } - + + @Override public int hashCode(){ return production.hashCode(); } - + + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof ListStackNode)) return false; - + ListStackNode

otherNode = (ListStackNode

) stackNode; - + if(!production.equals(otherNode.production)) return false; - + return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java index 770a84a8a2e..c6da6c35fca 100644 --- a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java @@ -20,64 +20,64 @@ public final class LiteralStackNode

extends AbstractMatchableStackNode

{ private final int[] literal; private final P production; - + private final LiteralNode result; - + public LiteralStackNode(int id, int dot, P production, int[] literal){ super(id, dot); - + this.literal = literal; this.production = production; - + result = new LiteralNode(production, literal); } - + public LiteralStackNode(int id, int dot, P production, int[] literal, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); - + this.literal = literal; this.production = production; - + result = new LiteralNode(production, literal); } public int[] getLiteral() { return literal; } - + private LiteralStackNode(LiteralStackNode

original, int startLocation){ super(original, startLocation); - + literal = original.literal; production = original.production; - + result = original.result; } - + public boolean isEmptyLeafNode(){ return false; } - + public AbstractNode match(int[] input, int location){ for(int i = literal.length - 1; i >= 0; --i){ if(literal[i] != input[location + i]) return null; // Did not match. } - + return result; } - + public AbstractStackNode

getCleanCopy(int startLocation){ return new LiteralStackNode

(this, startLocation); } - + public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ return new LiteralStackNode

(this, startLocation); } - + public int getLength(){ return literal.length; } - + public AbstractNode getResult(){ return result; } @@ -86,29 +86,41 @@ public AbstractNode getResult(){ public String toShortString() { return "'" + UnicodeConverter.unicodeArrayToString(literal) + "'"; } - + + @Override public String toString(){ StringBuilder sb = new StringBuilder("lit['"); sb.append(UnicodeConverter.unicodeArrayToString(literal)); sb.append("',"); sb.append(super.toString()); - + sb.append(']'); return sb.toString(); } - + + @Override public int hashCode(){ return production.hashCode(); } - + + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof LiteralStackNode)) return false; - + LiteralStackNode

otherNode = (LiteralStackNode

) stackNode; - + if(!production.equals(otherNode.production)) return false; - + return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java index 3b0ed831748..31b79f8a74a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java @@ -18,62 +18,62 @@ public class MultiCharacterStackNode

extends AbstractMatchableStackNode

{ private final P production; - + private final int[][] characters; - + private final AbstractNode result; - + public MultiCharacterStackNode(int id, int dot, P production, int[][] characters){ super(id, dot); - + this.production = production; - + this.characters = characters; - + result = null; } - + public MultiCharacterStackNode(int id, int dot, P production, int[][] characters, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); - + this.production = production; - + this.characters = characters; - + result = null; } - + private MultiCharacterStackNode(MultiCharacterStackNode

original, int startLocation){ super(original, startLocation); - + production = original.production; - + characters = original.characters; - + result = null; } - + private MultiCharacterStackNode(MultiCharacterStackNode

original, int startLocation, AbstractNode result){ super(original, startLocation); - + this.production = original.production; - + this.characters = original.characters; - + this.result = result; } - + public boolean isEmptyLeafNode(){ return false; } - + public AbstractNode match(int[] input, int location){ int nrOfCharacters = characters.length; int[] resultArray = new int[nrOfCharacters]; - + OUTER : for(int i = nrOfCharacters - 1; i >= 0; --i){ int next = input[location + i]; - + int[] alternatives = characters[i]; for(int j = alternatives.length - 1; j >= 0; --j){ int alternative = alternatives[j]; @@ -84,33 +84,35 @@ public AbstractNode match(int[] input, int location){ } return null; } - + return new LiteralNode(production, resultArray); } - + public AbstractStackNode

getCleanCopy(int startLocation){ return new MultiCharacterStackNode

(this, startLocation); } - + public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ return new MultiCharacterStackNode

(this, startLocation, result); } - + public int getLength(){ return 1; } - + public AbstractNode getResult(){ return result; } - + + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); - + sb.append('['); for(int i = characters.length - 1; i >= 0; --i){ int[] range = characters[i]; @@ -119,42 +121,48 @@ public String toString(){ sb.append(range[1]); } sb.append(']'); - + sb.append(getId()); sb.append('('); sb.append(startLocation); sb.append(')'); - + return sb.toString(); } - + + @Override public int hashCode(){ int hash = 0; - + for(int i = characters.length - 1; i >= 0; --i){ int[] chars = characters[i]; - for(int j = chars.length - 1; j <= 0; --j){ + for(int j = chars.length - 1; j >= 0; --j){ hash = hash << 3 + hash >> 5; hash ^= chars[0] + (chars[1] << 2); } } - + return hash; } - + + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof MultiCharacterStackNode)) return false; - + MultiCharacterStackNode

otherNode = (MultiCharacterStackNode

) stackNode; - + int[][] otherCharacters = otherNode.characters; if(characters.length != otherCharacters.length) return false; - + for(int i = characters.length - 1; i >= 0; --i){ int[] chars = characters[i]; int[] otherChars = otherCharacters[i]; if(chars.length != otherChars.length) return false; - + POS: for(int j = chars.length - 1; j <= 0; --j){ int c = chars[j]; for(int k = otherChars.length - 1; k <= 0; --k){ @@ -164,7 +172,12 @@ public boolean isEqual(AbstractStackNode

stackNode){ } } // Found all characters. - + return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index 3114bc97baf..5734d315a4e 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -17,61 +17,61 @@ public final class NonTerminalStackNode

extends AbstractStackNode

{ private final String expectIdentifier; - + public NonTerminalStackNode(int id, int dot, String expectIdentifier){ super(id, dot); - + this.expectIdentifier = expectIdentifier; } - + public NonTerminalStackNode(int id, int dot, String expectIdentifier, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); - + this.expectIdentifier = expectIdentifier; } - + private NonTerminalStackNode(NonTerminalStackNode

original, int startLocation){ super(original, startLocation); - + expectIdentifier = original.expectIdentifier; } - + public boolean isEmptyLeafNode(){ return false; } - + public String getName(){ return expectIdentifier; } - + public AbstractNode match(int[] input, int location){ throw new UnsupportedOperationException(); } - + public AbstractStackNode

getCleanCopy(int startLocation){ return new NonTerminalStackNode

(this, startLocation); } - + public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ throw new UnsupportedOperationException(); } - + public int getLength(){ throw new UnsupportedOperationException(); } - + public AbstractStackNode

[] getChildren(){ throw new UnsupportedOperationException(); } - + public boolean canBeEmpty(){ throw new UnsupportedOperationException(); } - + public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } - + public AbstractNode getResult(){ throw new UnsupportedOperationException(); } @@ -86,21 +86,26 @@ public String toString(){ sb.append(","); sb.append(super.toString()); sb.append("]"); - + return sb.toString(); } - + public int hashCode(){ return expectIdentifier.hashCode(); } - + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof NonTerminalStackNode)) return false; - + NonTerminalStackNode

otherNode = (NonTerminalStackNode

) stackNode; if(!expectIdentifier.equals(otherNode.expectIdentifier)) return false; - + return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java index c4694769941..8728f58fd51 100644 --- a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java @@ -18,40 +18,40 @@ public final class OptionalStackNode

extends AbstractExpandableStackNode

{ private final P production; private final String name; - + private final AbstractStackNode

[] children; private final AbstractStackNode

emptyChild; - + public OptionalStackNode(int id, int dot, P production, AbstractStackNode

optional){ super(id, dot); - + this.production = production; this.name = String.valueOf(id); // Add the id to make it unique. - + this.children = generateChildren(optional); this.emptyChild = generateEmptyChild(); } - + public OptionalStackNode(int id, int dot, P production, AbstractStackNode

optional, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); - + this.production = production; this.name = String.valueOf(id); // Add the id to make it unique. - + this.children = generateChildren(optional); this.emptyChild = generateEmptyChild(); } - + private OptionalStackNode(OptionalStackNode

original, int startLocation){ super(original, startLocation); - + production = original.production; name = original.name; - + children = original.children; emptyChild = original.emptyChild; } - + /** * Generates and initializes the alternative for this optional. */ @@ -61,7 +61,7 @@ private AbstractStackNode

[] generateChildren(AbstractStackNode

optional){ child.setAlternativeProduction(production); return (AbstractStackNode

[]) new AbstractStackNode[]{child}; } - + /** * Generates and initializes the empty child for this optional. */ @@ -71,23 +71,23 @@ private AbstractStackNode

generateEmptyChild(){ empty.setAlternativeProduction(production); return empty; } - + public String getName(){ return name; } - + public AbstractStackNode

getCleanCopy(int startLocation){ return new OptionalStackNode

(this, startLocation); } - + public AbstractStackNode

[] getChildren(){ return children; } - + public boolean canBeEmpty(){ return true; } - + public AbstractStackNode

getEmptyChild(){ return emptyChild; } @@ -102,21 +102,26 @@ public String toString(){ sb.append('('); sb.append(startLocation); sb.append(')'); - + return sb.toString(); } - + public int hashCode(){ return production.hashCode(); } - + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof OptionalStackNode)) return false; - + OptionalStackNode

otherNode = (OptionalStackNode

) stackNode; - + if(!production.equals(otherNode.production)) return false; - + return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java index 60daa28bc93..d685bb6001e 100644 --- a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java @@ -21,7 +21,7 @@ public class RecoveryPointStackNode

extends AbstractStackNode

{ public RecoveryPointStackNode(int id, P parent, AbstractStackNode

robustNode){ super(id, robustNode, robustNode.startLocation); this.prefixesMap = robustNode.prefixesMap; - + // Modify the production, so it ends 'here'. int productionLength = robustNode.dot + 1; if(productionLength == robustNode.production.length){ @@ -30,27 +30,27 @@ public RecoveryPointStackNode(int id, P parent, AbstractStackNode

robustNode) this.production = (AbstractStackNode

[]) new AbstractStackNode[productionLength]; System.arraycopy(robustNode.production, 0, this.production, 0, productionLength); } - + this.parent = parent; this.name = "recovery " + id; this.edgesMap = robustNode.edgesMap; } - + @Override public P getParentProduction() { return parent; } - + @Override public boolean isRecovered() { return true; } - + @Override public boolean isEmptyLeafNode(){ return false; } - + @Override public boolean isEndNode() { return true; @@ -60,35 +60,35 @@ public boolean isEndNode() { public String getName(){ return "***robust:" + name + "***"; } - + public AbstractNode match(int[] input, int location){ throw new UnsupportedOperationException(); } - + public AbstractStackNode

getCleanCopy(int startLocation){ throw new UnsupportedOperationException(); } - + public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ throw new UnsupportedOperationException(); } - + public int getLength(){ throw new UnsupportedOperationException(); } - + public AbstractStackNode

[] getChildren(){ throw new UnsupportedOperationException(); } - + public boolean canBeEmpty(){ throw new UnsupportedOperationException(); } - + public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } - + public AbstractNode getResult(){ throw new UnsupportedOperationException(); } @@ -101,19 +101,24 @@ public String toString(){ sb.append('('); sb.append(startLocation); sb.append(')'); - + return sb.toString(); } - + public int hashCode(){ return getName().hashCode(); } - + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof RecoveryPointStackNode)) return false; - + RecoveryPointStackNode

otherNode = (RecoveryPointStackNode

) stackNode; return otherNode.name.equals(name) && otherNode.startLocation == startLocation; } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index 3310d11076b..a34c98e96f2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -21,13 +21,13 @@ public final class SeparatedListStackNode

extends AbstractExpandableStackNode private final AbstractStackNode

[] children; private final AbstractStackNode

emptyChild; - + public SeparatedListStackNode(int id, int dot, P production, AbstractStackNode

child, AbstractStackNode

[] separators, boolean isPlusList){ super(id, dot); - + this.production = production; this.name = String.valueOf(id); // Add the id to make it unique. - + this.children = generateChildren(child, separators); this.emptyChild = isPlusList ? null : generateEmptyChild(); } @@ -35,27 +35,27 @@ public SeparatedListStackNode(int id, int dot, P production, AbstractStackNode

child, AbstractStackNode

[] separators, boolean isPlusList, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); - + this.production = production; this.name = String.valueOf(id); // Add the id to make it unique. - + this.children = generateChildren(child, separators); this.emptyChild = isPlusList ? null : generateEmptyChild(); } - + private SeparatedListStackNode(SeparatedListStackNode

original, int startLocation){ super(original, startLocation); - + production = original.production; name = original.name; - + children = original.children; emptyChild = original.emptyChild; } - + /** * Generates and initializes the alternative for this separated list. */ @@ -63,10 +63,10 @@ private SeparatedListStackNode(SeparatedListStackNode

original, int startLoca private AbstractStackNode

[] generateChildren(AbstractStackNode

child, AbstractStackNode

[] separators){ AbstractStackNode

listNode = child.getCleanCopy(DEFAULT_START_LOCATION); listNode.setAlternativeProduction(production); - + int numberOfSeparators = separators.length; AbstractStackNode

[] prod = (AbstractStackNode

[]) new AbstractStackNode[numberOfSeparators + 2]; - + listNode.setProduction(prod); prod[0] = listNode; // Start for(int i = numberOfSeparators - 1; i >= 0; --i){ @@ -76,10 +76,10 @@ private AbstractStackNode

[] generateChildren(AbstractStackNode

child, Abst prod[i + 1] = separator; } prod[numberOfSeparators + 1] = listNode; // End - + return (AbstractStackNode

[]) new AbstractStackNode[]{listNode}; } - + /** * Generates and initializes the empty child for this separated list (if it's a star list). */ @@ -89,23 +89,23 @@ private AbstractStackNode

generateEmptyChild(){ empty.setAlternativeProduction(production); return empty; } - + public String getName(){ return name; } - + public AbstractStackNode

getCleanCopy(int startLocation){ return new SeparatedListStackNode

(this, startLocation); } - + public AbstractStackNode

[] getChildren(){ return children; } - + public boolean canBeEmpty(){ return emptyChild != null; } - + public AbstractStackNode

getEmptyChild(){ return emptyChild; } @@ -120,21 +120,26 @@ public String toString(){ sb.append('('); sb.append(startLocation); sb.append(')'); - + return sb.toString(); } - + public int hashCode(){ return production.hashCode(); } - + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof SeparatedListStackNode)) return false; - + SeparatedListStackNode

otherNode = (SeparatedListStackNode

) stackNode; - + if(!production.equals(otherNode.production)) return false; - + return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java index 6d25c37750e..aede3e93b21 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java @@ -18,70 +18,70 @@ public class SequenceStackNode

extends AbstractExpandableStackNode

{ private final P production; private final String name; - + private final AbstractStackNode

[] children; - + public SequenceStackNode(int id, int dot, P production, AbstractStackNode

[] children){ super(id, dot); - + this.production = production; this.name = String.valueOf(id); - + this.children = generateChildren(children); } - + public SequenceStackNode(int id, int dot, P production, AbstractStackNode

[] children, IEnterFilter[] enterFilters, ICompletionFilter[] completionFilters){ super(id, dot, enterFilters, completionFilters); - + this.production = production; this.name = String.valueOf(id); - + this.children = generateChildren(children); } - + private SequenceStackNode(SequenceStackNode

original, int startLocation){ super(original, startLocation); - + production = original.production; name = original.name; children = original.children; } - + /** * Generates and initializes the alternatives for this sequence. */ @SuppressWarnings("unchecked") private AbstractStackNode

[] generateChildren(AbstractStackNode

[] children){ AbstractStackNode

[] prod = (AbstractStackNode

[]) new AbstractStackNode[children.length]; - + for(int i = children.length - 1; i >= 0; --i){ AbstractStackNode

child = children[i].getCleanCopy(DEFAULT_START_LOCATION); child.setProduction(prod); prod[i] = child; } - + prod[prod.length - 1].setAlternativeProduction(production); - + return (AbstractStackNode

[]) new AbstractStackNode[]{prod[0]}; } - + public String getName(){ return name; } - + public AbstractStackNode

getCleanCopy(int startLocation){ return new SequenceStackNode

(this, startLocation); } - + public AbstractStackNode

[] getChildren(){ return children; } - + public boolean canBeEmpty(){ return false; } - + public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } @@ -97,21 +97,26 @@ public String toString(){ sb.append('('); sb.append(startLocation); sb.append(')'); - + return sb.toString(); } - + public int hashCode(){ return production.hashCode(); } - + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof SequenceStackNode)) return false; - + SequenceStackNode

otherNode = (SequenceStackNode

) stackNode; - + if(!production.equals(otherNode.production)) return false; - + return hasEqualFilters(stackNode); } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } + } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 05213cc85b7..3c184cec4a4 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -150,4 +150,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return otherNode.id == id; } + + void accept(StackNodeVisitor

visitor) { + visitor.visit(this); + } } From af7b5d5646740f0dc983d77596cfcbf3da21968e Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:17:40 +0200 Subject: [PATCH 54/87] INtroduced StackNodeVisitor --- .../parser/gtd/stack/StackNodeVisitor.java | 18 +++++ .../gtd/stack/StackNodeVisitorAdapter.java | 69 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java create mode 100644 src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java new file mode 100644 index 00000000000..1dd2620c983 --- /dev/null +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java @@ -0,0 +1,18 @@ +package org.rascalmpl.parser.gtd.stack; + +public interface StackNodeVisitor

{ + void visit(AlternativeStackNode

node); + void visit(CaseInsensitiveLiteralStackNode

node); + void visit(CharStackNode

node); + void visit(EmptyStackNode

node); + void visit(EpsilonStackNode

node); + void visit(ListStackNode

node); + void visit(LiteralStackNode

node); + void visit(MultiCharacterStackNode

node); + void visit(NonTerminalStackNode

node); + void visit(OptionalStackNode

node); + void visit(RecoveryPointStackNode

node); + void visit(SeparatedListStackNode

node); + void visit(SequenceStackNode

node); + void visit(SkippingStackNode

node); +} diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java new file mode 100644 index 00000000000..d4e10e8874f --- /dev/null +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java @@ -0,0 +1,69 @@ +package org.rascalmpl.parser.gtd.stack; + +public class StackNodeVisitorAdapter

implements StackNodeVisitor

{ + + @Override + public void visit(AlternativeStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(CaseInsensitiveLiteralStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(CharStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(EmptyStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(EpsilonStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(ListStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(LiteralStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(MultiCharacterStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(NonTerminalStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(OptionalStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(SeparatedListStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(SequenceStackNode

node) { + // Do nothing fallback + } + + @Override + public void visit(SkippingStackNode

node) { + // Do nothing fallback + } +} From 2d3fbb1868e6df6e32b4be66cc14b5f6f04f38ae Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:20:55 +0200 Subject: [PATCH 55/87] Fixed more warnings --- .../parser/gtd/stack/NonTerminalStackNode.java | 10 +++++++++- .../rascalmpl/parser/gtd/stack/OptionalStackNode.java | 8 ++++++++ .../parser/gtd/stack/RecoveryPointStackNode.java | 7 +++++++ .../parser/gtd/stack/SeparatedListStackNode.java | 8 ++++++++ .../rascalmpl/parser/gtd/stack/SequenceStackNode.java | 10 +++++++++- .../rascalmpl/parser/gtd/stack/SkippingStackNode.java | 7 +------ 6 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index 5734d315a4e..236c930e9c3 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -49,7 +49,7 @@ public AbstractNode match(int[] input, int location){ } public AbstractStackNode

getCleanCopy(int startLocation){ - return new NonTerminalStackNode

(this, startLocation); + return new NonTerminalStackNode<>(this, startLocation); } public AbstractStackNode

getCleanCopyWithResult(int startLocation, AbstractNode result){ @@ -76,10 +76,12 @@ public AbstractNode getResult(){ throw new UnsupportedOperationException(); } + @Override public String toShortString() { return expectIdentifier; } + @Override public String toString(){ StringBuilder sb = new StringBuilder("NonTerminal["); sb.append(expectIdentifier); @@ -90,10 +92,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return expectIdentifier.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof NonTerminalStackNode)) return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java index 8728f58fd51..3071da38d51 100644 --- a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java @@ -92,10 +92,12 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); @@ -106,10 +108,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return production.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof OptionalStackNode)) return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java index d685bb6001e..e320a8a5ff1 100644 --- a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java @@ -93,6 +93,7 @@ public AbstractNode getResult(){ throw new UnsupportedOperationException(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(getName()); @@ -105,10 +106,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return getName().hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof RecoveryPointStackNode)) return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index a34c98e96f2..7610dabe764 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -110,10 +110,12 @@ public AbstractStackNode

getEmptyChild(){ return emptyChild; } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append(name); @@ -124,10 +126,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return production.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof SeparatedListStackNode)) return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java index aede3e93b21..a60eacd05c8 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java @@ -71,7 +71,7 @@ public String getName(){ } public AbstractStackNode

getCleanCopy(int startLocation){ - return new SequenceStackNode

(this, startLocation); + return new SequenceStackNode<>(this, startLocation); } public AbstractStackNode

[] getChildren(){ @@ -86,10 +86,12 @@ public AbstractStackNode

getEmptyChild(){ throw new UnsupportedOperationException(); } + @Override public String toShortString() { return toString(); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append("seq"); @@ -101,10 +103,16 @@ public String toString(){ return sb.toString(); } + @Override public int hashCode(){ return production.hashCode(); } + @Override + public boolean equals(Object peer) { + return super.equals(peer); + } + public boolean isEqual(AbstractStackNode

stackNode){ if(!(stackNode instanceof SequenceStackNode)) return false; diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index 3c184cec4a4..a3ed7735223 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -131,14 +131,9 @@ public int hashCode(){ return getParentProduction().hashCode(); } - @SuppressWarnings("unchecked") @Override public boolean equals(Object rhs) { - if (rhs instanceof AbstractStackNode) { - return isEqual((AbstractStackNode

)rhs); - } - - return false; + return super.equals(rhs); } public boolean isEqual(AbstractStackNode

stackNode){ From e95b1aa36bb9d255491a42e6c025f7ef1429c590 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:24:27 +0200 Subject: [PATCH 56/87] Madde AbstractStackNode.visit public --- src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java | 2 +- src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java | 3 ++- .../parser/gtd/stack/CaseInsensitiveLiteralStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/CharStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java | 2 +- src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/ListStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java | 3 ++- .../rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java | 3 ++- .../rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java | 3 ++- .../rascalmpl/parser/gtd/stack/SeparatedListStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java | 3 ++- src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java | 3 ++- .../rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java | 5 +++++ 16 files changed, 33 insertions(+), 15 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index 73a8ab793b9..4099f96f14e 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -811,7 +811,7 @@ public String toString() { return builder.toString(); } - abstract void accept(StackNodeVisitor

visitor); + public abstract void accept(StackNodeVisitor

visitor); // Matchables. /** diff --git a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java index 4a4570b0403..daabf1024a2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java @@ -125,7 +125,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java index 5fe0f1b4c0a..703e83b106a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java @@ -166,7 +166,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java index d9a0689cec5..6bdd7c31a8c 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java @@ -174,7 +174,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java index 65c1711b383..f64652030bb 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java @@ -114,7 +114,7 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - void accept(StackNodeVisitor

visitor) { + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java index 0e05002d9a1..2d79ea86fe4 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java @@ -95,7 +95,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java index 5f15b9f7af2..21bdb8e99aa 100644 --- a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java @@ -129,7 +129,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java index c6da6c35fca..f95efe128cc 100644 --- a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java @@ -119,7 +119,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java index 31b79f8a74a..b7698066ee4 100644 --- a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java @@ -176,7 +176,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index 236c930e9c3..b88010f5445 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -112,7 +112,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java index 3071da38d51..a0255cc11a0 100644 --- a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java @@ -128,7 +128,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java index e320a8a5ff1..7751f33473a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java @@ -124,7 +124,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return otherNode.name.equals(name) && otherNode.startLocation == startLocation; } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index 7610dabe764..96bffbff7e8 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -146,7 +146,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java index a60eacd05c8..09e9b769efa 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java @@ -123,7 +123,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return hasEqualFilters(stackNode); } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index a3ed7735223..ee6ccad186f 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -146,7 +146,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ return otherNode.id == id; } - void accept(StackNodeVisitor

visitor) { + @Override + public void accept(StackNodeVisitor

visitor) { visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java index d4e10e8874f..73d14ad3da7 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java @@ -52,6 +52,11 @@ public void visit(OptionalStackNode

node) { // Do nothing fallback } + @Override + public void visit(RecoveryPointStackNode

node) { + // Do nothing fallback + } + @Override public void visit(SeparatedListStackNode

node) { // Do nothing fallback From c06dc6dc1d49f28a7f37188957815aaa50df03b4 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:35:07 +0200 Subject: [PATCH 57/87] Use visitor pattern instead of instanceof in ToTokenRecoverer --- .../uptr/recovery/ToTokenRecoverer.java | 66 ++++++++++--------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index f12f88ea3f2..99fbfd08a16 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -30,6 +30,7 @@ import org.rascalmpl.parser.gtd.stack.NonTerminalStackNode; import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; import org.rascalmpl.parser.gtd.stack.SkippingStackNode; +import org.rascalmpl.parser.gtd.stack.StackNodeVisitorAdapter; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.util.ArrayList; import org.rascalmpl.parser.gtd.util.DoubleArrayList; @@ -194,25 +195,26 @@ void addEndMatchers(AbstractStackNode[] prod, int dot, List lastLiteral = (LiteralStackNode) last; - matchers.add(new LiteralMatcher(lastLiteral.getLiteral())); - } + last.accept(new StackNodeVisitorAdapter() { + @Override + public void visit(LiteralStackNode literal) { + matchers.add(new LiteralMatcher(literal.getLiteral())); + } - if (last instanceof CaseInsensitiveLiteralStackNode) { - CaseInsensitiveLiteralStackNode lastLiteral = (CaseInsensitiveLiteralStackNode) last; - matchers.add(new CaseInsensitiveLiteralMatcher(lastLiteral.getLiteral())); - } + @Override + public void visit(CaseInsensitiveLiteralStackNode literal) { + matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); + } - if (last instanceof NonTerminalStackNode) { - NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) last; - String name = nextNonTerminal.getName(); - AbstractStackNode[] alternatives = expectsProvider.getExpects(name); - for (AbstractStackNode alternative : alternatives) { - // In the future we might want to use all prefix-sharing alternatives here - addEndMatchers(alternative.getProduction(), 0, matchers, visitedNodes); + @Override + public void visit(NonTerminalStackNode nonTerminal) { + String name = nonTerminal.getName(); + AbstractStackNode[] alternatives = expectsProvider.getExpects(name); + for (AbstractStackNode alternative : alternatives) { + addEndMatchers(alternative.getProduction(), 0, matchers, visitedNodes); + } } - } + }); } void forAllParents(AbstractStackNode stackNode, Consumer> consumer) { @@ -281,24 +283,26 @@ private void addNextMatchers(AbstractStackNode[] prod, int dot, Li } visitedNodes.add(next.getId()); - if (next instanceof LiteralStackNode) { - LiteralStackNode nextLiteral = (LiteralStackNode) next; - matchers.add(new LiteralMatcher(nextLiteral.getLiteral())); - } + next.accept(new StackNodeVisitorAdapter() { + @Override + public void visit(LiteralStackNode literal) { + matchers.add(new LiteralMatcher(literal.getLiteral())); + } - if (next instanceof CaseInsensitiveLiteralStackNode) { - CaseInsensitiveLiteralStackNode nextLiteral = (CaseInsensitiveLiteralStackNode) next; - matchers.add(new CaseInsensitiveLiteralMatcher(nextLiteral.getLiteral())); - } + @Override + public void visit(CaseInsensitiveLiteralStackNode literal) { + matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); + } - if (next instanceof NonTerminalStackNode) { - NonTerminalStackNode nextNonTerminal = (NonTerminalStackNode) next; - String name = nextNonTerminal.getName(); - AbstractStackNode[] alternatives = expectsProvider.getExpects(name); - for (AbstractStackNode alternative : alternatives) { - addNextMatchers(alternative.getProduction(), 0, matchers, visitedNodes); + @Override + public void visit(NonTerminalStackNode nonTerminal) { + String name = nonTerminal.getName(); + AbstractStackNode[] alternatives = expectsProvider.getExpects(name); + for (AbstractStackNode alternative : alternatives) { + addNextMatchers(alternative.getProduction(), 0, matchers, visitedNodes); + } } - } + }); } // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) From d584dfee2ef053bc3f6a56de161a18b58b3585dd Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 11:49:34 +0200 Subject: [PATCH 58/87] Renamed DebugVisualizer to ParseStateVisualizer --- src/org/rascalmpl/parser/gtd/SGTDBF.java | 28 +++++++++---------- .../gtd/stack/RecoveryPointStackNode.java | 2 +- .../uptr/recovery/ToTokenRecoverer.java | 5 ++-- ...ualizer.java => ParseStateVisualizer.java} | 24 +++++++--------- 4 files changed, 27 insertions(+), 32 deletions(-) rename src/org/rascalmpl/util/visualize/{DebugVisualizer.java => ParseStateVisualizer.java} (97%) diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 2a989a704c8..efe9928177c 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -47,7 +47,7 @@ import org.rascalmpl.parser.gtd.util.IntegerObjectList; import org.rascalmpl.parser.gtd.util.Stack; import org.rascalmpl.parser.util.DebugUtil; -import org.rascalmpl.util.visualize.DebugVisualizer; +import org.rascalmpl.util.visualize.ParseStateVisualizer; import org.rascalmpl.util.visualize.dot.NodeId; import org.rascalmpl.values.RascalValueFactory; @@ -133,7 +133,7 @@ public abstract class SGTDBF implements IGTD { // Debugging private IDebugListener

debugListener; - private DebugVisualizer visualizer; + private ParseStateVisualizer visualizer; // Temporary instrumentation for accurate profiling private long timestamp; @@ -808,7 +808,7 @@ private void move(AbstractStackNode

node, AbstractNode result){ */ private void reduceTerminals() { // Reduce terminals - visualize("Reducing terminals", DebugVisualizer.TERMINALS_TO_REDUCE_ID); + visualize("Reducing terminals", ParseStateVisualizer.TERMINALS_TO_REDUCE_ID); while(!stacksWithTerminalsToReduce.isEmpty()){ move(stacksWithTerminalsToReduce.peekFirst(), stacksWithTerminalsToReduce.popSecond()); } @@ -816,7 +816,7 @@ private void reduceTerminals() { private void reduceNonTerminals() { // Reduce non-terminals - visualize("Reducing non-terminals", DebugVisualizer.NON_TERMINALS_TO_REDUCE_ID); + visualize("Reducing non-terminals", ParseStateVisualizer.NON_TERMINALS_TO_REDUCE_ID); while(!stacksWithNonTerminalsToReduce.isEmpty()){ move(stacksWithNonTerminalsToReduce.peekFirst(), stacksWithNonTerminalsToReduce.popSecond()); } @@ -843,7 +843,7 @@ private boolean findFirstStacksToReduce(){ if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } - visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); + visualize("Recovering", ParseStateVisualizer.ERROR_TRACKING_ID); DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); @@ -866,14 +866,14 @@ private boolean findFirstStacksToReduce(){ * Locates the set of stacks that is queued for handling, for which the least amount of characters needs to be shifted. */ private boolean findStacksToReduce(){ - visualize("Finding stacks to reduce", DebugVisualizer.TODO_LISTS_ID); + visualize("Finding stacks to reduce", ParseStateVisualizer.TODO_LISTS_ID); int queueDepth = todoLists.length; for(int i = 1; i < queueDepth-1; ++i){ queueIndex = (queueIndex + 1) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; if(!(terminalsTodo == null || terminalsTodo.isEmpty())){ - if (DebugVisualizer.VISUALIZATION_ENABLED) { + if (ParseStateVisualizer.VISUALIZATION_ENABLED) { NodeId reduceNodeId = new NodeId("todo-" + i); visualize("Found stack to reduce", reduceNodeId); } @@ -890,12 +890,12 @@ private boolean findStacksToReduce(){ if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } - visualize("Recovering", DebugVisualizer.ERROR_TRACKING_ID); + visualize("Recovering", ParseStateVisualizer.ERROR_TRACKING_ID); DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); } - if (DebugVisualizer.VISUALIZATION_ENABLED && visualizer != null) { + if (ParseStateVisualizer.VISUALIZATION_ENABLED && visualizer != null) { // Visualize state and include recovered nodes visualizer.createGraph(this, "Reviving"); visualizer.addRecoveredNodes(recoveredNodes); @@ -913,7 +913,7 @@ private boolean findStacksToReduce(){ if (debugListener != null) { debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); } - visualize("Queue recovery node", DebugVisualizer.getNodeId(recovered)); + visualize("Queue recovery node", ParseStateVisualizer.getNodeId(recovered)); queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), recoveredNodes.getSecond(i)); } return findStacksToReduce(); @@ -1235,7 +1235,7 @@ private void expandStack(AbstractStackNode

stack){ * Initiate stack expansion for all queued stacks. */ private void expand(){ - visualize("Expanding", DebugVisualizer.STACKS_TO_EXPAND_ID); + visualize("Expanding", ParseStateVisualizer.STACKS_TO_EXPAND_ID); while(!stacksToExpand.isEmpty()){ expandStack(stacksToExpand.pop()); } @@ -1269,7 +1269,7 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] this.debugListener = debugListener; String query = inputURI.getQuery(); - visualizer = query != null && query.contains("visualize=true") ? new DebugVisualizer("Parser") : null; + visualizer = query != null && query.contains("visualize=true") ? new ParseStateVisualizer("Parser") : null; // Initialzed the position store. positionStore.index(input); @@ -1320,7 +1320,7 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] } while(findStacksToReduce()); } - visualize("Done", DebugVisualizer.PARSER_ID); + visualize("Done", ParseStateVisualizer.PARSER_ID); // Check if we were successful. if(location == input.length){ @@ -1569,7 +1569,7 @@ private IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory, AbstractNode> reviveNod // Sort nodes by start location recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); - DebugVisualizer visualizer = new DebugVisualizer("Recovery"); + ParseStateVisualizer visualizer = new ParseStateVisualizer("Recovery"); visualizer.visualizeRecoveryNodes(recoveryNodes); for (int i = 0; i DotNode createDotNode(AbstractStackNode

stackNode) { String nodeName; - if (stackNode instanceof RecoveryPointStackNode) { - RecoveryPointStackNode

recoveryNode = (RecoveryPointStackNode

) stackNode; - nodeName = String.valueOf(recoveryNode.getId()); - } else { - try { - nodeName = stackNode.getName(); - } catch (UnsupportedOperationException e) { - nodeName = ""; - } + try { + nodeName = stackNode.getName(); + } catch (UnsupportedOperationException e) { + nodeName = ""; + } - if (nodeName.startsWith("layouts_")) { - nodeName = nodeName.substring("layouts_".length()); - } + if (nodeName.startsWith("layouts_")) { + nodeName = nodeName.substring("layouts_".length()); } int dot = stackNode.getDot(); From 41ea6e888cda5ac3b3cb3b244acd793254557f8c Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 12:11:18 +0200 Subject: [PATCH 59/87] Generalized AbstractStackNode visitor to be able to return values --- .../parser/gtd/stack/AbstractStackNode.java | 2 +- .../gtd/stack/AlternativeStackNode.java | 4 +- .../CaseInsensitiveLiteralStackNode.java | 4 +- .../parser/gtd/stack/CharStackNode.java | 4 +- .../parser/gtd/stack/EmptyStackNode.java | 4 +- .../parser/gtd/stack/EpsilonStackNode.java | 4 +- .../parser/gtd/stack/ListStackNode.java | 4 +- .../parser/gtd/stack/LiteralStackNode.java | 4 +- .../gtd/stack/MultiCharacterStackNode.java | 4 +- .../gtd/stack/NonTerminalStackNode.java | 4 +- .../parser/gtd/stack/OptionalStackNode.java | 4 +- .../gtd/stack/RecoveryPointStackNode.java | 4 +- .../gtd/stack/SeparatedListStackNode.java | 4 +- .../parser/gtd/stack/SequenceStackNode.java | 4 +- .../parser/gtd/stack/SkippingStackNode.java | 4 +- .../parser/gtd/stack/StackNodeVisitor.java | 30 ++++++------- .../gtd/stack/StackNodeVisitorAdapter.java | 44 ++++++++++++------- .../parser/uptr/recovery/InputMatcher.java | 23 +++++----- .../uptr/recovery/ToTokenRecoverer.java | 22 ++++++---- 19 files changed, 100 insertions(+), 77 deletions(-) diff --git a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java index 4099f96f14e..24219cb44d2 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AbstractStackNode.java @@ -811,7 +811,7 @@ public String toString() { return builder.toString(); } - public abstract void accept(StackNodeVisitor

visitor); + public abstract R accept(StackNodeVisitor visitor); // Matchables. /** diff --git a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java index daabf1024a2..b28c5e8fefd 100644 --- a/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/AlternativeStackNode.java @@ -126,8 +126,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java index 703e83b106a..087bd26776a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CaseInsensitiveLiteralStackNode.java @@ -167,8 +167,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java index 6bdd7c31a8c..73aeca2258c 100644 --- a/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/CharStackNode.java @@ -175,8 +175,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java index f64652030bb..7e02791339f 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EmptyStackNode.java @@ -114,8 +114,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java index 2d79ea86fe4..da95027c47d 100644 --- a/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/EpsilonStackNode.java @@ -96,8 +96,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java index 21bdb8e99aa..d03bb50eb4e 100644 --- a/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/ListStackNode.java @@ -130,8 +130,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java index f95efe128cc..49dbdbeacc1 100644 --- a/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/LiteralStackNode.java @@ -120,8 +120,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java index b7698066ee4..4e9640dfa40 100644 --- a/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/MultiCharacterStackNode.java @@ -177,8 +177,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java index b88010f5445..db81fd68134 100644 --- a/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/NonTerminalStackNode.java @@ -113,8 +113,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java index a0255cc11a0..508b552cd12 100644 --- a/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/OptionalStackNode.java @@ -129,8 +129,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java index 9874ab19c80..11a8cdd8d58 100644 --- a/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/RecoveryPointStackNode.java @@ -125,8 +125,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java index 96bffbff7e8..28ceca7d995 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SeparatedListStackNode.java @@ -147,8 +147,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java index 09e9b769efa..4a0e08cc27a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SequenceStackNode.java @@ -124,8 +124,8 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java index ee6ccad186f..fd01a665698 100644 --- a/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java +++ b/src/org/rascalmpl/parser/gtd/stack/SkippingStackNode.java @@ -147,7 +147,7 @@ public boolean isEqual(AbstractStackNode

stackNode){ } @Override - public void accept(StackNodeVisitor

visitor) { - visitor.visit(this); + public R accept(StackNodeVisitor visitor) { + return visitor.visit(this); } } diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java index 1dd2620c983..1171c62fd0b 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java @@ -1,18 +1,18 @@ package org.rascalmpl.parser.gtd.stack; -public interface StackNodeVisitor

{ - void visit(AlternativeStackNode

node); - void visit(CaseInsensitiveLiteralStackNode

node); - void visit(CharStackNode

node); - void visit(EmptyStackNode

node); - void visit(EpsilonStackNode

node); - void visit(ListStackNode

node); - void visit(LiteralStackNode

node); - void visit(MultiCharacterStackNode

node); - void visit(NonTerminalStackNode

node); - void visit(OptionalStackNode

node); - void visit(RecoveryPointStackNode

node); - void visit(SeparatedListStackNode

node); - void visit(SequenceStackNode

node); - void visit(SkippingStackNode

node); +public interface StackNodeVisitor { + R visit(AlternativeStackNode

node); + R visit(CaseInsensitiveLiteralStackNode

node); + R visit(CharStackNode

node); + R visit(EmptyStackNode

node); + R visit(EpsilonStackNode

node); + R visit(ListStackNode

node); + R visit(LiteralStackNode

node); + R visit(MultiCharacterStackNode

node); + R visit(NonTerminalStackNode

node); + R visit(OptionalStackNode

node); + R visit(RecoveryPointStackNode

node); + R visit(SeparatedListStackNode

node); + R visit(SequenceStackNode

node); + R visit(SkippingStackNode

node); } diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java index 73d14ad3da7..bcd6dd2fec9 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java @@ -1,74 +1,88 @@ package org.rascalmpl.parser.gtd.stack; -public class StackNodeVisitorAdapter

implements StackNodeVisitor

{ +public class StackNodeVisitorAdapter implements StackNodeVisitor { @Override - public void visit(AlternativeStackNode

node) { + public R visit(AlternativeStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(CaseInsensitiveLiteralStackNode

node) { + public R visit(CaseInsensitiveLiteralStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(CharStackNode

node) { + public R visit(CharStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(EmptyStackNode

node) { + public R visit(EmptyStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(EpsilonStackNode

node) { + public R visit(EpsilonStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(ListStackNode

node) { + public R visit(ListStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(LiteralStackNode

node) { + public R visit(LiteralStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(MultiCharacterStackNode

node) { + public R visit(MultiCharacterStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(NonTerminalStackNode

node) { + public R visit(NonTerminalStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(OptionalStackNode

node) { + public R visit(OptionalStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(RecoveryPointStackNode

node) { + public R visit(RecoveryPointStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(SeparatedListStackNode

node) { + public R visit(SeparatedListStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(SequenceStackNode

node) { + public R visit(SequenceStackNode

node) { // Do nothing fallback + return null; } @Override - public void visit(SkippingStackNode

node) { + public R visit(SkippingStackNode

node) { // Do nothing fallback + return null; } } diff --git a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java index 614df9acd32..3dbce46f9e7 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java @@ -3,6 +3,7 @@ import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.CaseInsensitiveLiteralStackNode; import org.rascalmpl.parser.gtd.stack.LiteralStackNode; +import org.rascalmpl.parser.gtd.stack.StackNodeVisitorAdapter; import org.rascalmpl.values.RascalValueFactory; import io.usethesource.vallang.IConstructor; @@ -37,7 +38,7 @@ public int getEnd() { @Override public String toString() { return "MatchResult [start=" + start + ", length=" + length + "]"; - } + } } public static InputMatcher createMatcher(IConstructor constructor) { @@ -53,14 +54,16 @@ public static InputMatcher createMatcher(IConstructor constructor) { } public static

InputMatcher createMatcher(AbstractStackNode

stackNode) { - if (stackNode instanceof LiteralStackNode) { - return new LiteralMatcher(((LiteralStackNode

) stackNode).getLiteral()); - } - - if (stackNode instanceof CaseInsensitiveLiteralStackNode) { - return new CaseInsensitiveLiteralMatcher(((CaseInsensitiveLiteralStackNode

) stackNode).getLiteral()); - } - - return null; + return stackNode.accept(new StackNodeVisitorAdapter() { + @Override + public InputMatcher visit(LiteralStackNode

literal) { + return new LiteralMatcher(literal.getLiteral()); + } + + @Override + public InputMatcher visit(CaseInsensitiveLiteralStackNode

literal) { + return new CaseInsensitiveLiteralMatcher(literal.getLiteral()); + } + }); } } diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 5e271eefd40..2c8a5f4bfee 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -194,24 +194,27 @@ void addEndMatchers(AbstractStackNode[] prod, int dot, List() { + last.accept(new StackNodeVisitorAdapter() { @Override - public void visit(LiteralStackNode literal) { + public Void visit(LiteralStackNode literal) { matchers.add(new LiteralMatcher(literal.getLiteral())); + return null; } @Override - public void visit(CaseInsensitiveLiteralStackNode literal) { + public Void visit(CaseInsensitiveLiteralStackNode literal) { matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); + return null; } @Override - public void visit(NonTerminalStackNode nonTerminal) { + public Void visit(NonTerminalStackNode nonTerminal) { String name = nonTerminal.getName(); AbstractStackNode[] alternatives = expectsProvider.getExpects(name); for (AbstractStackNode alternative : alternatives) { addEndMatchers(alternative.getProduction(), 0, matchers, visitedNodes); } + return null; } }); } @@ -282,24 +285,27 @@ private void addNextMatchers(AbstractStackNode[] prod, int dot, Li } visitedNodes.add(next.getId()); - next.accept(new StackNodeVisitorAdapter() { + next.accept(new StackNodeVisitorAdapter() { @Override - public void visit(LiteralStackNode literal) { + public Void visit(LiteralStackNode literal) { matchers.add(new LiteralMatcher(literal.getLiteral())); + return null; } @Override - public void visit(CaseInsensitiveLiteralStackNode literal) { + public Void visit(CaseInsensitiveLiteralStackNode literal) { matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); + return null; } @Override - public void visit(NonTerminalStackNode nonTerminal) { + public Void visit(NonTerminalStackNode nonTerminal) { String name = nonTerminal.getName(); AbstractStackNode[] alternatives = expectsProvider.getExpects(name); for (AbstractStackNode alternative : alternatives) { addNextMatchers(alternative.getProduction(), 0, matchers, visitedNodes); } + return null; } }); } From 82c156d8d03a700cf2a9741b056bcb8e4decdaa7 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 12:12:48 +0200 Subject: [PATCH 60/87] Removed debug print --- .../library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index 82a76e18b51..6dc9a0ff937 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -25,9 +25,6 @@ test bool abx() { test bool axc() { Tree t = parseS("a x c $", visualize=true); - iprintln(t); - println("after disambiguation:"); - iprintln(defaultErrorDisambiguationFilter(t)); return getErrorText(findFirstError(t)) == "x c"; } From c406e2b13c4700d8b7dc0ff6116d2322106de749 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 13:29:23 +0200 Subject: [PATCH 61/87] Added license header to files introduced for error recovery --- .../tests/recovery/BasicRecoveryTests.rsc | 15 +++++++++++++- .../tests/recovery/ListRecoveryTests.rsc | 14 +++++++++++++ .../tests/recovery/NestedRecoveryTests.rsc | 14 +++++++++++++ .../tests/recovery/PicoRecoveryTests.rsc | 20 ++++++++++++++++--- .../tests/recovery/RascalRecoveryTests.rsc | 14 +++++++++++++ .../lang/rascal/tests/recovery/Test.rsc | 1 - .../lang/rascal/tests/recovery/ToyRascal.rsc | 14 +++++++++++++ .../tests/recovery/ToyRascalRecoveryTests.rsc | 14 +++++++++++++ .../rascalmpl/parser/gtd/ExpectsProvider.java | 16 ++++++++++++++- .../parser/gtd/stack/StackNodeVisitor.java | 16 ++++++++++++++- .../gtd/stack/StackNodeVisitorAdapter.java | 16 ++++++++++++++- .../parser/gtd/util/IdDispenser.java | 16 ++++++++++++++- .../parser/gtd/util/StackNodeIdDispenser.java | 16 ++++++++++++++- .../CaseInsensitiveLiteralMatcher.java | 16 ++++++++++++++- .../parser/uptr/recovery/FailingMatcher.java | 14 +++++++++++++ .../parser/uptr/recovery/InputMatcher.java | 16 ++++++++++++++- .../parser/uptr/recovery/LiteralMatcher.java | 16 ++++++++++++++- src/org/rascalmpl/parser/util/DebugUtil.java | 20 ++++++++++++++++++- .../util/visualize/ParseStateVisualizer.java | 16 +++++++++++++-- src/org/rascalmpl/util/visualize/replay.html | 12 +++++++++++ .../test/recovery/ErrorRecoveryModules.java | 18 +++++++++++++++-- 21 files changed, 296 insertions(+), 18 deletions(-) delete mode 100644 src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index 6dc9a0ff937..4a3dfdb45d7 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -1,7 +1,20 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::BasicRecoveryTests import ParseTree; -import IO; layout Layout = [\ ]* !>> [\ ]; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc index 184fd2b2eb8..1ef55f7172d 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::ListRecoveryTests import ParseTree; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc index 638d25518ec..8cb23ab160a 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::NestedRecoveryTests import ParseTree; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc index 6abed349fac..3b2adbb072d 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::PicoRecoveryTests import lang::pico::\syntax::Main; @@ -29,13 +43,13 @@ end"); } test bool picoTypo() { - t = parsePico("begin declare input : natural, - output : natural, + t = parsePico("begin declare input : natural, + output : natural, repnr : natural, rep : natural; input := 14; output := 0; - while input - 1 do + while input - 1 do rep := output; repnr := input; while repnr - 1 do diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc index 7c17abd899d..82e94caedae 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::RascalRecoveryTests import lang::rascal::\syntax::Rascal; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc deleted file mode 100644 index 455e3661677..00000000000 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/Test.rsc +++ /dev/null @@ -1 +0,0 @@ -module A void f(){ void g() {}} diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc index 3f6bfb736bb..24e0ac2d7d7 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::ToyRascal start syntax FunctionDeclaration = Signature FunctionBody; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc index 455197c1f35..d3900bda115 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + module lang::rascal::tests::recovery::ToyRascalRecoveryTests import lang::rascal::tests::recovery::ToyRascal; diff --git a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java index 18266a7d10d..6cccdd21b60 100644 --- a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java +++ b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java @@ -1,4 +1,18 @@ -package org.rascalmpl.parser.gtd; +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.parser.gtd; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java index 1171c62fd0b..03231e4c41e 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java @@ -1,4 +1,18 @@ -package org.rascalmpl.parser.gtd.stack; +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.parser.gtd.stack; public interface StackNodeVisitor { R visit(AlternativeStackNode

node); diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java index bcd6dd2fec9..92c8835d86a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java @@ -1,4 +1,18 @@ -package org.rascalmpl.parser.gtd.stack; +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.parser.gtd.stack; public class StackNodeVisitorAdapter implements StackNodeVisitor { diff --git a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java index fe9aa091406..f1c5de57008 100644 --- a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java @@ -1,4 +1,18 @@ -package org.rascalmpl.parser.gtd.util; +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.parser.gtd.util; public interface IdDispenser { int dispenseId(); diff --git a/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java index 752813d5021..3068daeac24 100644 --- a/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java @@ -1,4 +1,18 @@ -package org.rascalmpl.parser.gtd.util; +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.parser.gtd.util; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; diff --git a/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java index 587bd419748..77d9a4dd53b 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/CaseInsensitiveLiteralMatcher.java @@ -1,4 +1,18 @@ -package org.rascalmpl.parser.uptr.recovery; +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.parser.uptr.recovery; public class CaseInsensitiveLiteralMatcher implements InputMatcher { private final int[] chars; diff --git a/src/org/rascalmpl/parser/uptr/recovery/FailingMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/FailingMatcher.java index 1e4eb6b8b95..f69dac24ad8 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/FailingMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/FailingMatcher.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.parser.uptr.recovery; public class FailingMatcher implements InputMatcher { diff --git a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java index 3dbce46f9e7..c56a50975c5 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java @@ -1,4 +1,18 @@ -package org.rascalmpl.parser.uptr.recovery; +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.parser.uptr.recovery; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; import org.rascalmpl.parser.gtd.stack.CaseInsensitiveLiteralStackNode; diff --git a/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java index b5531cff372..2461d0c8f3c 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/LiteralMatcher.java @@ -1,4 +1,18 @@ -package org.rascalmpl.parser.uptr.recovery; +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.parser.uptr.recovery; public class LiteralMatcher implements InputMatcher { private int[] chars; diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index e1c29c6b084..02971ef4ba4 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -1,4 +1,18 @@ -package org.rascalmpl.parser.util; +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.parser.util; import io.usethesource.vallang.IConstructor; import io.usethesource.vallang.IList; @@ -8,6 +22,10 @@ public class DebugUtil { /** * Turn a production IConstructor into a string of the form "S -> E1 E2 ..." */ + + private DebugUtil() { + } + public static String prodToString(IConstructor prod) { StringBuilder builder = new StringBuilder("'"); diff --git a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java index a9040131dba..7086cd1654e 100644 --- a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java +++ b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java @@ -1,3 +1,17 @@ +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + package org.rascalmpl.util.visualize; import java.io.BufferedReader; @@ -28,8 +42,6 @@ import org.rascalmpl.parser.gtd.result.SkippedNode; import org.rascalmpl.parser.gtd.result.SortContainerNode; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; -import org.rascalmpl.parser.gtd.stack.RecoveryPointStackNode; -import org.rascalmpl.parser.gtd.stack.StackNodeVisitorAdapter; import org.rascalmpl.parser.gtd.stack.edge.EdgesSet; import org.rascalmpl.parser.gtd.util.ArrayList; import org.rascalmpl.parser.gtd.util.DoubleArrayList; diff --git a/src/org/rascalmpl/util/visualize/replay.html b/src/org/rascalmpl/util/visualize/replay.html index ff8c16daeed..409874049aa 100644 --- a/src/org/rascalmpl/util/visualize/replay.html +++ b/src/org/rascalmpl/util/visualize/replay.html @@ -1,3 +1,15 @@ + diff --git a/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java b/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java index abf446cf843..8f96ea6a521 100644 --- a/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java +++ b/test/org/rascalmpl/test/recovery/ErrorRecoveryModules.java @@ -1,4 +1,18 @@ -package org.rascalmpl.test.recovery; +/** + * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.test.recovery; import org.junit.runner.RunWith; import org.rascalmpl.test.infrastructure.RascalJUnitTestPrefix; @@ -7,5 +21,5 @@ @RunWith(RascalJUnitTestRunner.class) @RascalJUnitTestPrefix("lang::rascal::tests::recovery") public class ErrorRecoveryModules { - + } From c884ceb37ee4d5653df37ba4399214a9f89e875d Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Wed, 28 Aug 2024 13:49:55 +0200 Subject: [PATCH 62/87] Added license header to more files and fixed years. --- .../tests/recovery/BasicRecoveryTests.rsc | 2 +- .../tests/recovery/ListRecoveryTests.rsc | 2 +- .../tests/recovery/NestedRecoveryTests.rsc | 2 +- .../tests/recovery/PicoRecoveryTests.rsc | 2 +- .../tests/recovery/RascalRecoveryTests.rsc | 2 +- .../lang/rascal/tests/recovery/ToyRascal.rsc | 2 +- .../tests/recovery/ToyRascalRecoveryTests.rsc | 2 +- src/org/rascalmpl/library/util/TermREPL.java | 41 ++++++++++++------- .../rascalmpl/parser/gtd/ExpectsProvider.java | 2 +- .../parser/gtd/debug/IDebugListener.java | 8 ++-- .../gtd/result/out/SkippedNodeFlattener.java | 4 +- .../parser/gtd/stack/StackNodeVisitor.java | 2 +- .../parser/gtd/util/StackNodeIdDispenser.java | 2 +- .../parser/uptr/recovery/InputMatcher.java | 2 +- src/org/rascalmpl/parser/util/DebugUtil.java | 2 +- .../util/visualize/ParseStateVisualizer.java | 2 +- .../visualize/dot/AttributeStatement.java | 13 ++++++ .../util/visualize/dot/CompassPoint.java | 16 +++++++- .../util/visualize/dot/DotAttribute.java | 16 +++++++- .../rascalmpl/util/visualize/dot/DotEdge.java | 22 ++++++++-- .../util/visualize/dot/DotField.java | 16 +++++++- .../util/visualize/dot/DotGraph.java | 18 +++++++- .../rascalmpl/util/visualize/dot/DotNode.java | 16 +++++++- .../util/visualize/dot/DotRecord.java | 16 +++++++- .../util/visualize/dot/DotRecordEntry.java | 16 +++++++- .../util/visualize/dot/DotStatement.java | 16 +++++++- .../util/visualize/dot/DotSubgraph.java | 22 ++++++++-- .../rascalmpl/util/visualize/dot/NodeId.java | 18 +++++++- src/org/rascalmpl/util/visualize/replay.html | 2 +- .../rascalmpl/test/parser/RecoveryTests.java | 28 +++++++++---- .../rascalmpl/test/parser/StackNodeTest.java | 17 +++++++- 31 files changed, 269 insertions(+), 62 deletions(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index 4a3dfdb45d7..43a38024daa 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc index 1ef55f7172d..9e4ca7e12f2 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ListRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc index 8cb23ab160a..25aa4be006a 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/NestedRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc index 3b2adbb072d..a2d7be0088f 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/PicoRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc index 82e94caedae..ccc2918d1d8 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc index 24e0ac2d7d7..902691170db 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascal.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc index d3900bda115..7e56e3369a4 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/ToyRascalRecoveryTests.rsc @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/library/util/TermREPL.java b/src/org/rascalmpl/library/util/TermREPL.java index f149bb36340..5acb2776393 100644 --- a/src/org/rascalmpl/library/util/TermREPL.java +++ b/src/org/rascalmpl/library/util/TermREPL.java @@ -1,3 +1,16 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ package org.rascalmpl.library.util; import java.io.ByteArrayInputStream; @@ -71,7 +84,7 @@ public ITuple newREPL(IConstructor repl, IString title, IString welcome, IString } TypeFactory tf = TypeFactory.getInstance(); - IFunction run = vf.function(tf.functionType(tf.voidType(), tf.tupleEmpty(), tf.tupleEmpty()), + IFunction run = vf.function(tf.functionType(tf.voidType(), tf.tupleEmpty(), tf.tupleEmpty()), (args, kwargs) -> { try { baseRepl.run(); @@ -80,14 +93,14 @@ public ITuple newREPL(IConstructor repl, IString title, IString welcome, IString throw RuntimeExceptionFactory.io(e.getMessage()); } return vf.tuple(); - }); + }); - IFunction send = vf.function(tf.functionType(tf.voidType(), tf.tupleType(tf.stringType()), tf.tupleEmpty()), + IFunction send = vf.function(tf.functionType(tf.voidType(), tf.tupleType(tf.stringType()), tf.tupleEmpty()), (args, kwargs) -> { baseRepl.queueCommand(((IString)args[0]).getValue()); return vf.tuple(); }); - + return vf.tuple(run, send); } @@ -110,7 +123,7 @@ public TheREPL(IValueFactory vf, IString title, IString welcome, IString prompt, this.input = input; this.stderr = stderr; this.stdout = stdout; - + // TODO: these casts mean that TheRepl only works with functions produced by the // interpreter for now. The reason is that the REPL needs access to environment configuration // parameters of these functions such as stdout, stdin, etc. @@ -192,32 +205,32 @@ public void handleInput(String line, Map output, Map output, Map metadata, IConstructor content) throws IOException, UnsupportedEncodingException { String id = ((IString) content.get("id")).getValue(); Function callback = liftProviderFunction(content.get("callback")); REPLContentServer server = contentManager.addServer(id, callback); - + String URL = "http://localhost:" + server.getListeningPort(); - + produceHTMLResponse(id, URL, output, metadata); } - + private void produceHTMLResponse(String id, String URL, Map output, Map metadata) throws UnsupportedEncodingException{ String html; if (metadata.containsKey("origin") && metadata.get("origin").equals("notebook")) html = " \n

\n
"; else html = ""; - + metadata.put("url", URL); output.put("text/html", new ByteArrayInputStream(html.getBytes("UTF8"))); - + String message = "Serving visual content at |" + URL + "|"; output.put("text/plain", new ByteArrayInputStream(message.getBytes("UTF8"))); - + } private Function liftProviderFunction(IValue callback) { @@ -238,7 +251,7 @@ private void handleJSONResponse(Map output, IConstructor re IValue dtf = kws.getParameter("dateTimeFormat"); IValue dai = kws.getParameter("dateTimeAsInt"); - + JsonValueWriter writer = new JsonValueWriter() .setCalendarFormat(dtf != null ? ((IString) dtf).getValue() : "yyyy-MM-dd\'T\'HH:mm:ss\'Z\'") .setDatesAsInt(dai != null ? ((IBool) dai).getValue() : true); @@ -312,7 +325,7 @@ private IValue call(IFunction f, Type[] types, IValue[] args) { @Override public CompletionResult completeFragment(String line, int cursor) { ITuple result = (ITuple)call(completor, new Type[] { tf.stringType(), tf.integerType() }, - new IValue[] { vf.string(line), vf.integer(cursor) }); + new IValue[] { vf.string(line), vf.integer(cursor) }); List suggestions = new ArrayList<>(); diff --git a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java index 6cccdd21b60..39c627293b8 100644 --- a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java +++ b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java b/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java index fc6a8a2bec3..550987600f8 100644 --- a/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java +++ b/src/org/rascalmpl/parser/gtd/debug/IDebugListener.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2012-2013 CWI + * Copyright (c) 2012-2024 CWI * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -22,11 +22,11 @@ public interface IDebugListener

{ void shifting(int offset, int[] input, PositionStore positionStore); - + void iterating(); - + void matched(AbstractStackNode

node, AbstractNode result); - + void failedToMatch(AbstractStackNode

node); void expanding(AbstractStackNode

node); diff --git a/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java b/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java index d0b5877e7c8..92e468a3266 100644 --- a/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java +++ b/src/org/rascalmpl/parser/gtd/result/out/SkippedNodeFlattener.java @@ -1,5 +1,5 @@ /******************************************************************************* - * Copyright (c) 2009-2013 CWI + * Copyright (c) 2009-2024 CWI * All rights reserved. This program and the accompanying materials * are made available under the terms of the Eclipse Public License v1.0 * which accompanies this distribution, and is available at @@ -21,7 +21,7 @@ public class SkippedNodeFlattener{ public SkippedNodeFlattener(){ super(); } - + public T convertToUPTR(INodeConstructorFactory nodeConstructorFactory, SkippedNode node, PositionStore positionStore){ T result = nodeConstructorFactory.createSkippedNode(node.getSkippedChars()); diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java index 03231e4c41e..c3286e28606 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitor.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java index 3068daeac24..529f5db087d 100644 --- a/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/StackNodeIdDispenser.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java index c56a50975c5..6cf59a47187 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java +++ b/src/org/rascalmpl/parser/uptr/recovery/InputMatcher.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index 02971ef4ba4..b58ad8e876e 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java index 7086cd1654e..faf6e762a0b 100644 --- a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java +++ b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java b/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java index 87d745a5e5c..34cd55e08ab 100644 --- a/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java +++ b/src/org/rascalmpl/util/visualize/dot/AttributeStatement.java @@ -1,3 +1,16 @@ +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/CompassPoint.java b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java index 3b31af24fd1..60ef4da2295 100644 --- a/src/org/rascalmpl/util/visualize/dot/CompassPoint.java +++ b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java @@ -1,4 +1,18 @@ -package org.rascalmpl.util.visualize.dot; +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.util.visualize.dot; public enum CompassPoint { N, NE, E, SE, S, SW, W, NW, C, X; diff --git a/src/org/rascalmpl/util/visualize/dot/DotAttribute.java b/src/org/rascalmpl/util/visualize/dot/DotAttribute.java index 761218c6e29..052f8d33c94 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotAttribute.java +++ b/src/org/rascalmpl/util/visualize/dot/DotAttribute.java @@ -1,4 +1,18 @@ -package org.rascalmpl.util.visualize.dot; +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; import java.io.StringWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotEdge.java b/src/org/rascalmpl/util/visualize/dot/DotEdge.java index 7faf1ee1259..e8a7cb158d9 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotEdge.java +++ b/src/org/rascalmpl/util/visualize/dot/DotEdge.java @@ -1,4 +1,18 @@ -package org.rascalmpl.util.visualize.dot; +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; import java.util.ArrayList; @@ -6,7 +20,7 @@ public class DotEdge implements DotStatement { private NodeId from; - private List to; + private List to; private boolean directed; private List attributes; @@ -56,11 +70,11 @@ public void writeSource(PrintWriter writer) { } public static DotEdge createArrayEdge(NodeId array, int index, NodeId element) { - return new DotEdge(new NodeId(array, String.valueOf(index)), true, element); + return new DotEdge(new NodeId(array, String.valueOf(index)), true, element); } public static DotEdge createArrayEdge(NodeId array, int index, CompassPoint direction, NodeId element) { - return new DotEdge(new NodeId(array.getId(), String.valueOf(index), direction), true, element); + return new DotEdge(new NodeId(array.getId(), String.valueOf(index), direction), true, element); } } diff --git a/src/org/rascalmpl/util/visualize/dot/DotField.java b/src/org/rascalmpl/util/visualize/dot/DotField.java index 70a25aad7b8..86cf5f5eeae 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotField.java +++ b/src/org/rascalmpl/util/visualize/dot/DotField.java @@ -1,4 +1,18 @@ -package org.rascalmpl.util.visualize.dot; +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotGraph.java b/src/org/rascalmpl/util/visualize/dot/DotGraph.java index dc11db9202d..9e14c448ee8 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotGraph.java +++ b/src/org/rascalmpl/util/visualize/dot/DotGraph.java @@ -1,4 +1,18 @@ -package org.rascalmpl.util.visualize.dot; +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; import java.io.StringWriter; @@ -19,7 +33,7 @@ public class DotGraph { private boolean digraph; private List statements; private Map nodes; - + public DotGraph(String id, boolean digraph) { this.id = id; this.digraph = digraph; diff --git a/src/org/rascalmpl/util/visualize/dot/DotNode.java b/src/org/rascalmpl/util/visualize/dot/DotNode.java index 118d1c00c96..ae791e83633 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotNode.java +++ b/src/org/rascalmpl/util/visualize/dot/DotNode.java @@ -1,4 +1,18 @@ -package org.rascalmpl.util.visualize.dot; +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; import java.util.ArrayList; diff --git a/src/org/rascalmpl/util/visualize/dot/DotRecord.java b/src/org/rascalmpl/util/visualize/dot/DotRecord.java index 73a5cdd73b8..b503636539a 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotRecord.java +++ b/src/org/rascalmpl/util/visualize/dot/DotRecord.java @@ -1,4 +1,18 @@ -package org.rascalmpl.util.visualize.dot; +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; import java.util.ArrayList; diff --git a/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java b/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java index a0ed741e9f0..f54127c1bde 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java +++ b/src/org/rascalmpl/util/visualize/dot/DotRecordEntry.java @@ -1,4 +1,18 @@ -package org.rascalmpl.util.visualize.dot; +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotStatement.java b/src/org/rascalmpl/util/visualize/dot/DotStatement.java index 83919814059..68af11f4749 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotStatement.java +++ b/src/org/rascalmpl/util/visualize/dot/DotStatement.java @@ -1,4 +1,18 @@ -package org.rascalmpl.util.visualize.dot; +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; diff --git a/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java b/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java index 23e7fb6b6a6..ac19707e6ac 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java +++ b/src/org/rascalmpl/util/visualize/dot/DotSubgraph.java @@ -1,4 +1,18 @@ -package org.rascalmpl.util.visualize.dot; +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; import java.util.ArrayList; @@ -12,11 +26,11 @@ public DotSubgraph(String id) { this.id = id; statements = new ArrayList<>(); } - + public void addStatement(DotStatement statement) { statements.add(statement); } - + public String getId() { return id; } @@ -34,7 +48,7 @@ public void writeSource(PrintWriter writer) { statement.writeSource(writer); writer.println(";"); } - + writer.println(" }"); } } diff --git a/src/org/rascalmpl/util/visualize/dot/NodeId.java b/src/org/rascalmpl/util/visualize/dot/NodeId.java index fe2c3849ea3..180e15e7e54 100644 --- a/src/org/rascalmpl/util/visualize/dot/NodeId.java +++ b/src/org/rascalmpl/util/visualize/dot/NodeId.java @@ -1,4 +1,18 @@ -package org.rascalmpl.util.visualize.dot; +/** + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + * + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + **/ + + package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; @@ -122,5 +136,5 @@ public String toString() { return result; } - + } diff --git a/src/org/rascalmpl/util/visualize/replay.html b/src/org/rascalmpl/util/visualize/replay.html index 409874049aa..891778454b5 100644 --- a/src/org/rascalmpl/util/visualize/replay.html +++ b/src/org/rascalmpl/util/visualize/replay.html @@ -1,5 +1,5 @@ From 96fc6045c3244bbc0b98ce36c5a7f80b93fa15e0 Mon Sep 17 00:00:00 2001 From: Pieter Olivier Date: Fri, 6 Sep 2024 11:21:59 +0200 Subject: [PATCH 77/87] Various changes based on an excellent code review by Sung. --- src/org/rascalmpl/library/ParseTree.rsc | 29 +- .../tests/recovery/BasicRecoveryTests.rsc | 8 +- .../tests/recovery/RascalRecoveryTests.rsc | 3 +- .../rascalmpl/parser/gtd/ExpectsProvider.java | 2 +- src/org/rascalmpl/parser/gtd/SGTDBF.java | 1193 ++++++++++------- .../parser/gtd/stack/AbstractStackNode.java | 8 +- .../parser/gtd/stack/EpsilonStackNode.java | 5 + .../gtd/stack/RecoveryPointStackNode.java | 5 + .../parser/gtd/stack/SkippingStackNode.java | 18 - .../parser/gtd/stack/StackNodeVisitor.java | 2 +- .../gtd/stack/StackNodeVisitorAdapter.java | 4 +- .../parser/gtd/util/DoubleArrayList.java | 15 +- .../parser/gtd/util/IdDispenser.java | 4 +- .../parser/uptr/UPTRNodeFactory.java | 19 - .../CaseInsensitiveLiteralMatcher.java | 26 +- .../parser/uptr/recovery/InputMatcher.java | 5 +- .../recovery/ToNextWhitespaceRecoverer.java | 245 ---- .../uptr/recovery/ToTokenRecoverer.java | 878 ++++++------ src/org/rascalmpl/parser/util/DebugUtil.java | 8 +- .../util/visualize/ParseStateVisualizer.java | 33 +- .../util/visualize/dot/CompassPoint.java | 2 + .../util/visualize/dot/DotAttribute.java | 2 +- .../rascalmpl/util/visualize/dot/NodeId.java | 20 +- src/org/rascalmpl/util/visualize/replay.html | 20 +- .../values/RascalFunctionValueFactory.java | 3 - .../values/parsetrees/ProductionAdapter.java | 8 +- .../rascalmpl/test/parser/RecoveryTests.java | 150 --- 27 files changed, 1237 insertions(+), 1478 deletions(-) delete mode 100644 src/org/rascalmpl/parser/uptr/recovery/ToNextWhitespaceRecoverer.java delete mode 100644 test/org/rascalmpl/test/parser/RecoveryTests.java diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 7027f02cdb9..0dbf69684d8 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -195,8 +195,8 @@ data Production ; data Production - = error(Symbol def, Production prod, int dot) - | skipped(Symbol symbol); + = \error(Symbol def, Production prod, int dot) + | \skipped(Symbol symbol); @synopsis{Attributes in productions.} @description{ @@ -775,22 +775,17 @@ bool isNonTerminalType(Symbol::\parameterized-lex(str _, list[Symbol] _)) = true bool isNonTerminalType(Symbol::\start(Symbol s)) = isNonTerminalType(s); default bool isNonTerminalType(Symbol s) = false; -@synopsis{Check if a parse tree contains any skipped nodes, the result of error recovery.} +@synopsis{Check if a parse tree contains any error nodes, the result of error recovery.} bool hasErrors(Tree tree) = /appl(error(_, _, _), _) := tree; @synopsis{Find all error productions in a parse tree.} list[Tree] findAllErrors(Tree tree) = [err | /err:appl(error(_, _, _), _) := tree]; @synopsis{Find the first production containing an error.} -Tree findFirstError(Tree tree) { - if (/err:appl(error(_, _, _), _) := tree) return err; - fail; -} +Tree findFirstError(/err:appl(error(_, _, _), _)) = err; -@synopsis{Find the best error from a tree containing errors.} -Tree findBestError(Tree tree) { - return findFirstError(defaultErrorDisambiguationFilter(tree)); -} +@synopsis{Find the best error from a tree containing errors. This function will fail if `tree` does not contain an error.} +Tree findBestError(Tree tree) = findFirstError(defaultErrorDisambiguationFilter(tree)); @synopsis{Get the symbol (sort) of the failing production} Symbol getErrorSymbol(appl(error(Symbol sym, _, _), _)) = sym; @@ -802,16 +797,12 @@ Production getErrorProduction(appl(error(_, Production prod, _), _)) = prod; int getErrorDot(appl(error(_, _, int dot), _)) = dot; @synopsis{Get the skipped tree} -Tree getSkipped(appl(error(_, _, _), [*_, skip:appl(skipped(_), _)])) { - return skip; -} +Tree getSkipped(appl(error(_, _, _), [*_, skip:appl(skipped(_), _)])) = skip; @synopsis{Get the text that failed to parse. This is only the text of the part that has been skipped to be able to continue parsing. If you want the text of the whole error tree, you can just use string interpolation: "". } -str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) { - return stringChars([c | ch <- chars, char(c) := ch]); -} +str getErrorText(appl(error(_, _, _), [*_, appl(skipped(_), chars)])) = stringChars([c | char(c) <- chars]); @synopsis{Error recovery often produces ambiguous trees where errors can be recovered in multiple ways. This filter removes error trees until no ambiguities caused by error recovery are left. @@ -831,7 +822,9 @@ Tree defaultErrorDisambiguationFilter(amb(set[Tree] alternatives)) { if (nonErrorTrees == {}) { return getBestErrorTree(errorTrees); - } else if ({Tree single} := nonErrorTrees) { + } + + if ({Tree single} := nonErrorTrees) { // One ambiguity left, no ambiguity concerns here return single; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc index b82245f27c6..3c3b32d7f3c 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/BasicRecoveryTests.rsc @@ -28,20 +28,20 @@ private Tree parseS(str input, bool visualize=false) = parser(#S, allowRecovery=true, allowAmbiguity=true)(input, |unknown:///?visualize=<"">|); test bool basicOk() { - return !hasErrors(parseS("a b c $", visualize=true)); + return !hasErrors(parseS("a b c $")); } test bool abx() { - Tree t = parseS("a b x $", visualize=true); + Tree t = parseS("a b x $"); return getErrorText(findFirstError(defaultErrorDisambiguationFilter(t))) == "x "; } test bool axc() { - Tree t = parseS("a x c $", visualize=true); + Tree t = parseS("a x c $"); return getErrorText(findFirstError(t)) == "x c"; } test bool ax() { - Tree t = parseS("a x $", visualize=true); + Tree t = parseS("a x $"); return getErrorText(findFirstError(t)) == "x "; } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc index 3c71c2f7e80..8a461911cfb 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/recovery/RascalRecoveryTests.rsc @@ -108,13 +108,12 @@ test bool rascalMissingCloseParen() { } test bool rascalFunctionDeclarationMissingCloseParen() { - Tree t = parseFunctionDeclaration("void f({} void g() {}", visualize=false); + Tree t = parseFunctionDeclaration("void f({} void g() {}"); assert getErrorText(findFirstError(t)) == "void g("; Tree error = findFirstError(defaultErrorDisambiguationFilter(t)); assert getErrorText(error) == "("; - Tree skipped = getSkipped(error); loc location = getSkipped(error).src; assert location.begin.column == 16 && location.length == 1; diff --git a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java index 39c627293b8..fbbd7970c4e 100644 --- a/src/org/rascalmpl/parser/gtd/ExpectsProvider.java +++ b/src/org/rascalmpl/parser/gtd/ExpectsProvider.java @@ -12,7 +12,7 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **/ - package org.rascalmpl.parser.gtd; +package org.rascalmpl.parser.gtd; import org.rascalmpl.parser.gtd.stack.AbstractStackNode; diff --git a/src/org/rascalmpl/parser/gtd/SGTDBF.java b/src/org/rascalmpl/parser/gtd/SGTDBF.java index 333eff83004..c705e424f6d 100755 --- a/src/org/rascalmpl/parser/gtd/SGTDBF.java +++ b/src/org/rascalmpl/parser/gtd/SGTDBF.java @@ -1,13 +1,10 @@ /******************************************************************************* - * Copyright (c) 2009-2013 CWI - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at - * http://www.eclipse.org/legal/epl-v10.html + * Copyright (c) 2009-2013 CWI All rights reserved. This program and the accompanying materials are + * made available under the terms of the Eclipse Public License v1.0 which accompanies this + * distribution, and is available at http://www.eclipse.org/legal/epl-v10.html * - * Contributors: - * * Arnold Lankamp - Arnold.Lankamp@cwi.nl -*******************************************************************************/ + * Contributors: * Arnold Lankamp - Arnold.Lankamp@cwi.nl + *******************************************************************************/ package org.rascalmpl.parser.gtd; import java.lang.reflect.InvocationTargetException; @@ -71,7 +68,8 @@ public abstract class SGTDBF implements IGTD { // A mapping between character location and line/column. private final PositionStore positionStore; - // Terminals that matched. Circular buffer indexed by length of the terminal. Each entry contains the node to reduce and the result node + // Terminals that matched. Circular buffer indexed by length of the terminal. Each entry contains + // the node to reduce and the result node // This is a circular buffer where queueIndex determines the start of the buffer. // At each position, a stack is maintained of all terminals to reduce of a certain length. // So at queueIndex+3, all terminals of length 3 that need reducing are stored. @@ -81,28 +79,36 @@ public abstract class SGTDBF implements IGTD { // Stack of non-terminal nodes to expand // - Nodes are removed in expand, which pops and expands all stack nodes on this stack // - Nodes are added in: - // - parse: the root node is pushed - // - updateNextNode: next node of the production is pushed - // - updateAlternativeNextNode: next node of a prefix-shared production is pushed - // - handleExpects: non-matchable first element of each alternative is pushed - // - expandStack: when an expandable stack is expanded, all non-matchable children are pushed + // - parse: the root node is pushed + // - updateNextNode: next node of the production is pushed + // - updateAlternativeNextNode: next node of a prefix-shared production is pushed + // - handleExpects: non-matchable first element of each alternative is pushed + // - expandStack: when an expandable stack is expanded, all non-matchable children are pushed private final Stack> stacksToExpand; - // The current stack of non-terminals to reduce. Each stack has a container node to accumulate results. - // - Nodes are removed in `reduce` where all productions are advanced one dot past the non-terminal node + // The current stack of non-terminals to reduce. Each stack has a container node to accumulate + // results. + // - Nodes are removed in `reduceNonTerminals` where all productions are advanced one dot past the + // non-terminal node // - Nodes are added in: - // - handleEdgeList: result container node is created and all edges are pushed with the same result container node - // - handleEdgeListWithRestrictions: result container node is created and all edges are pushed with the same result container node - // - expandStack: non-matchable, non-expandable nodes (and their existing result container node) are added if their name can be found in `cachedEdgesForExpect`. - // - expandStack: expandable nodes that are nullable? Might be a cycle thing + // - handleEdgeList: result container node is created and all edges are pushed with the same result + // container node + // - handleEdgeListWithRestrictions: result container node is created and all edges are pushed with + // the same result container node + // - expandStack: non-matchable, non-expandable nodes (and their existing result container node) are + // added if their name can be found in `cachedEdgesForExpect`. + // - expandStack: expandable nodes that are nullable? Might be a cycle thing private final DoubleStack, AbstractContainerNode

> stacksWithNonTerminalsToReduce; - // The current stack of non-terminals to reduce: it contains the matchable node with the smallest length from todoLists. - // - Nodes are removed in `reduce` where all productions are advanced one dot past the matchable node + // The current stack of non-terminals to reduce: it contains the matchable node with the smallest + // length from todoLists. + // - Nodes are removed in `reduceTerminals` where all productions are advanced one dot past the + // matchable node // - Variable is assigned in: - // - findFirstStacksToReduce: the first non-empty `todoList` is assigned to this variable - // - findStacksToReduce: again the first non-empty `todoList` is assigned to this variable - // - parse: variable is used in main reduce/expand loop to determine when it is time to look for more `stacksToReduce`. + // - findFirstStacksToReduce: the first non-empty `todoList` is assigned to this variable + // - findStacksToReduce: again the first non-empty `todoList` is assigned to this variable + // - parse: variable is used in main reduce/expand loop to determine when it is time to look for + // more `stacksToReduce`. private DoubleStack, AbstractNode> stacksWithTerminalsToReduce; private final HashMap> cachedEdgesForExpect; @@ -121,7 +127,8 @@ public abstract class SGTDBF implements IGTD { // Error reporting private final Stack> unexpandableNodes; - private final Stack> unmatchableLeafNodes; // Leaf nodes (for instance literals) that failed to match + private final Stack> unmatchableLeafNodes; // Leaf nodes (for instance literals) that failed to + // match private final DoubleStack, AbstractNode>, AbstractStackNode

> unmatchableMidProductionNodes; private final DoubleStack, AbstractNode> filteredNodes; @@ -140,7 +147,7 @@ public abstract class SGTDBF implements IGTD { private boolean printTimes = false; - public SGTDBF(){ + public SGTDBF() { super(); positionStore = new PositionStore(); @@ -150,7 +157,8 @@ public SGTDBF(){ cachedEdgesForExpect = new HashMap>(); - sharedNextNodes = new IntegerKeyedDoubleValueHashMap, DoubleArrayList, AbstractNode>>(); + sharedNextNodes = + new IntegerKeyedDoubleValueHashMap, DoubleArrayList, AbstractNode>>(); location = 0; @@ -160,14 +168,15 @@ public SGTDBF(){ unexpandableNodes = new Stack>(); unmatchableLeafNodes = new Stack>(); - unmatchableMidProductionNodes = new DoubleStack, AbstractNode>, AbstractStackNode

>(); + unmatchableMidProductionNodes = + new DoubleStack, AbstractNode>, AbstractStackNode

>(); filteredNodes = new DoubleStack, AbstractNode>(); } /** - * Return a stack node id that is guaranteed not to be in use. - * The parser generator generates an override for this method as it knows which ids have been dispensed. - * Tests that need this should override this method, probably using a common base class. + * Return a stack node id that is guaranteed not to be in use. The parser generator generates an + * override for this method as it knows which ids have been dispensed. Tests that need this should + * override this method, probably using a common base class. */ protected int getFreeStackNodeId() { throw new UnsupportedOperationException(); @@ -177,20 +186,24 @@ protected int getFreeStackNodeId() { public AbstractStackNode

[] getExpects(String nonTerminal) { AbstractStackNode

[] expects = expectCache.get(nonTerminal); if (expects == null) { - try{ + try { Method method = getClass().getMethod(nonTerminal); try { method.setAccessible(true); // Try to bypass the 'isAccessible' check to save time. - } catch (SecurityException sex) { + } + catch (SecurityException sex) { // Ignore this if it happens. } expects = (AbstractStackNode

[]) method.invoke(this); - } catch (NoSuchMethodException nsmex) { + } + catch (NoSuchMethodException nsmex) { throw new UndeclaredNonTerminalException(nonTerminal, getClass()); - } catch (IllegalAccessException iaex) { + } + catch (IllegalAccessException iaex) { throw new RuntimeException(iaex); - } catch (InvocationTargetException itex) { + } + catch (InvocationTargetException itex) { throw new RuntimeException(itex.getTargetException()); } @@ -203,43 +216,52 @@ public AbstractStackNode

[] getExpects(String nonTerminal) { /** * Triggers the gathering of alternatives for the given non-terminal. */ - protected AbstractStackNode

[] invokeExpects(AbstractStackNode

nonTerminal){ + protected AbstractStackNode

[] invokeExpects(AbstractStackNode

nonTerminal) { return getExpects(nonTerminal.getName()); } /** * Moves to the next symbol in the production. */ - private AbstractStackNode

updateNextNode(AbstractStackNode

next, AbstractStackNode

node, AbstractNode result){ - IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> alternativeEntry = sharedNextNodes.get(next.getId()); - if(alternativeEntry != null){ // Sharing check. + private AbstractStackNode

updateNextNode(AbstractStackNode

next, AbstractStackNode

node, + AbstractNode result) { + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> alternativeEntry = + sharedNextNodes.get(next.getId()); + if (alternativeEntry != null) { // Sharing check. DoubleArrayList, AbstractNode> predecessors = alternativeEntry.value2; - if(predecessors != null){ + if (predecessors != null) { predecessors.add(node, result); return null; } AbstractStackNode

alternative = alternativeEntry.value1; - if(alternative.getStartLocation() == location){ - if(alternative.isMatchable()){ - if(alternative.isEmptyLeafNode()){ + if (alternative.getStartLocation() == location) { + if (alternative.isMatchable()) { + if (alternative.isEmptyLeafNode()) { // Encountered a possible stack 'overtake'. - if(node.getStartLocation() != location){ + if (node.getStartLocation() != location) { propagateEdgesAndPrefixes(node, result, alternative, alternative.getResult()); - }else{ - propagateEdgesAndPrefixesForNullable(node, result, alternative, alternative.getResult(), node.getEdges().size()); + } + else { + propagateEdgesAndPrefixesForNullable(node, result, alternative, alternative.getResult(), + node.getEdges().size()); } return alternative; } - }else{ + } + else { EdgesSet

alternativeEdgesSet = alternative.getIncomingEdges(); int resultStoreId = getResultStoreId(alternative.getId()); - if(alternativeEdgesSet != null && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){ + if (alternativeEdgesSet != null + && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) { // Encountered a possible stack 'overtake'. - if(node.getStartLocation() != location){ - propagateEdgesAndPrefixes(node, result, alternative, alternativeEdgesSet.getLastResult(resultStoreId)); - }else{ - propagateEdgesAndPrefixesForNullable(node, result, alternative, alternativeEdgesSet.getLastResult(resultStoreId), node.getEdges().size()); + if (node.getStartLocation() != location) { + propagateEdgesAndPrefixes(node, result, alternative, + alternativeEdgesSet.getLastResult(resultStoreId)); + } + else { + propagateEdgesAndPrefixesForNullable(node, result, alternative, + alternativeEdgesSet.getLastResult(resultStoreId), node.getEdges().size()); } return alternative; } @@ -248,42 +270,51 @@ private AbstractStackNode

updateNextNode(AbstractStackNode

next, AbstractS alternative.updateNode(node, result); - if(debugListener != null) debugListener.progressed(node, result, alternative); + if (debugListener != null) + debugListener.progressed(node, result, alternative); return alternative; } - if(next.isMatchable()){ // Eager matching optimization. - if((location + next.getLength()) > input.length) return null; + if (next.isMatchable()) { // Eager matching optimization. + if ((location + next.getLength()) > input.length) + return null; AbstractNode nextResult = next.match(input, location); - if(nextResult == null){ - // Push the node including it's predecessor to the appropriate error tracking collection (and take care of merging when necessary). - DoubleArrayList, AbstractNode> predecessors = new DoubleArrayList, AbstractNode>(); + if (nextResult == null) { + // Push the node including it's predecessor to the appropriate error tracking collection (and take + // care of merging when necessary). + DoubleArrayList, AbstractNode> predecessors = + new DoubleArrayList, AbstractNode>(); predecessors.add(node, result); unmatchableMidProductionNodes.push(predecessors, next); sharedNextNodes.putUnsafe(next.getId(), next, predecessors); - if(debugListener != null) debugListener.failedToMatch(next); + if (debugListener != null) + debugListener.failedToMatch(next); return null; } - if(debugListener != null) debugListener.matched(next, nextResult); + if (debugListener != null) + debugListener.matched(next, nextResult); next = next.getCleanCopyWithResult(location, nextResult); - }else{ + } + else { next = next.getCleanCopy(location); } - if(!node.isMatchable() || result.isEmpty()){ + if (!node.isMatchable() || result.isEmpty()) { next.updateNode(node, result); - }else{ // Non-nullable terminal specific edge set sharing optimization. + } + else { // Non-nullable terminal specific edge set sharing optimization. next.updateNodeAfterNonEmptyMatchable(node, result); } - if(debugListener != null) debugListener.progressed(node, result, next); + if (debugListener != null) + debugListener.progressed(node, result, next); sharedNextNodes.putUnsafe(next.getId(), next, null); stacksToExpand.push(next); @@ -294,70 +325,85 @@ private AbstractStackNode

updateNextNode(AbstractStackNode

next, AbstractS /** * Moves to the next symbol in an alternative continuation of a prefix-shared production. */ - private boolean updateAlternativeNextNode(AbstractStackNode

next, AbstractStackNode

node, AbstractNode result, IntegerObjectList> edgesMap, ArrayList[] prefixesMap){ + private boolean updateAlternativeNextNode(AbstractStackNode

next, AbstractStackNode

node, AbstractNode result, + IntegerObjectList> edgesMap, ArrayList[] prefixesMap) { int id = next.getId(); - IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> alternativeEntry = sharedNextNodes.get(id); - if(alternativeEntry != null){ // Sharing check. + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> alternativeEntry = + sharedNextNodes.get(id); + if (alternativeEntry != null) { // Sharing check. DoubleArrayList, AbstractNode> predecessors = alternativeEntry.value2; - if(predecessors != null){ + if (predecessors != null) { predecessors.add(node, result); return false; } AbstractStackNode

alternative = alternativeEntry.value1; - if(result.isEmpty()){ - if(alternative.isMatchable()){ - if(alternative.isEmptyLeafNode()){ + if (result.isEmpty()) { + if (alternative.isMatchable()) { + if (alternative.isEmptyLeafNode()) { // Encountered a possible stack 'overtake'. - propagateAlternativeEdgesAndPrefixes(node, result, alternative, alternative.getResult(), node.getEdges().size(), edgesMap, prefixesMap); + propagateAlternativeEdgesAndPrefixes(node, result, alternative, alternative.getResult(), + node.getEdges().size(), edgesMap, prefixesMap); return true; } - }else{ + } + else { EdgesSet

alternativeEdgesSet = alternative.getIncomingEdges(); int resultStoreId = getResultStoreId(alternative.getId()); - if(alternativeEdgesSet != null && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){ + if (alternativeEdgesSet != null + && alternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) { AbstractContainerNode

nextResult = alternativeEdgesSet.getLastResult(resultStoreId); // Encountered a possible stack 'overtake'. - propagateAlternativeEdgesAndPrefixes(node, result, alternative, nextResult, node.getEdges().size(), edgesMap, prefixesMap); + propagateAlternativeEdgesAndPrefixes(node, result, alternative, nextResult, + node.getEdges().size(), edgesMap, prefixesMap); return true; } } } - alternative.updatePrefixSharedNode(edgesMap, prefixesMap); // Prevent unnecessary overhead; share whenever possible. + alternative.updatePrefixSharedNode(edgesMap, prefixesMap); // Prevent unnecessary overhead; share whenever + // possible. - if(debugListener != null) debugListener.progressed(node, result, alternative); + if (debugListener != null) + debugListener.progressed(node, result, alternative); return true; } - if(next.isMatchable()){ // Eager matching optimization. - if((location + next.getLength()) > input.length) return false; + if (next.isMatchable()) { // Eager matching optimization. + if ((location + next.getLength()) > input.length) + return false; AbstractNode nextResult = next.match(input, location); - if(nextResult == null){ - // Push the node including it's predecessor to the appropriate error tracking collection (and take care of merging when necessary). - DoubleArrayList, AbstractNode> predecessors = new DoubleArrayList, AbstractNode>(); + if (nextResult == null) { + // Push the node including it's predecessor to the appropriate error tracking collection (and take + // care of merging when necessary). + DoubleArrayList, AbstractNode> predecessors = + new DoubleArrayList, AbstractNode>(); predecessors.add(node, result); unmatchableMidProductionNodes.push(predecessors, next); sharedNextNodes.putUnsafe(id, next, predecessors); - if(debugListener != null) debugListener.failedToMatch(next); + if (debugListener != null) + debugListener.failedToMatch(next); return false; } - if(debugListener != null) debugListener.matched(next, nextResult); + if (debugListener != null) + debugListener.matched(next, nextResult); next = next.getCleanCopyWithResult(location, nextResult); - }else{ + } + else { next = next.getCleanCopy(location); } next.updatePrefixSharedNode(edgesMap, prefixesMap); // Prevent unnecessary overhead; share whenever possible. - if(debugListener != null) debugListener.progressed(node, result, next); + if (debugListener != null) + debugListener.progressed(node, result, next); sharedNextNodes.putUnsafe(id, next, null); stacksToExpand.push(next); @@ -366,10 +412,10 @@ private boolean updateAlternativeNextNode(AbstractStackNode

next, AbstractSta } /** - * Part of the hidden-right-recursion fix. - * Executes absent reductions. + * Part of the hidden-right-recursion fix. Executes absent reductions. */ - private void propagateReductions(AbstractStackNode

node, AbstractNode nodeResultStore, AbstractStackNode

next, AbstractNode nextResultStore, int potentialNewEdges){ + private void propagateReductions(AbstractStackNode

node, AbstractNode nodeResultStore, AbstractStackNode

next, + AbstractNode nextResultStore, int potentialNewEdges) { IntegerList propagatedReductions = next.getPropagatedReductions(); IntegerObjectList> edgesMap = node.getEdges(); @@ -380,12 +426,12 @@ private void propagateReductions(AbstractStackNode

node, AbstractNode nodeRes boolean hasNestingRestrictions = hasNestingRestrictions(name); IntegerList filteredParents = null; - if(hasNestingRestrictions){ + if (hasNestingRestrictions) { filteredParents = getFilteredParents(next.getId()); } int fromIndex = edgesMap.size() - potentialNewEdges; - for(int i = edgesMap.size() - 1; i >= fromIndex; --i){ + for (int i = edgesMap.size() - 1; i >= fromIndex; --i) { int startLocation = edgesMap.getKey(i); // We know we haven't been here before. @@ -399,100 +445,129 @@ private void propagateReductions(AbstractStackNode

node, AbstractNode nodeRes EdgesSet

edgeSet = edgesMap.getValue(i); - if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet); + if (debugListener != null) + debugListener.reducing(node, resultLink, edgeSet); - if(!hasNestingRestrictions){ + if (!hasNestingRestrictions) { handleEdgeList(edgeSet, name, production, resultLink, startLocation); - }else{ + } + else { handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, startLocation, filteredParents); } } } /** - * Part of the hidden-right-recursion fix. - * Propagates absent prefixes. + * Part of the hidden-right-recursion fix. Propagates absent prefixes. */ - private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResult, int nrOfAddedEdges){ + private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResult, int nrOfAddedEdges) { // Proceed with the tail of the production. int nextDot = next.getDot() + 1; AbstractStackNode

[] prod = next.getProduction(); AbstractStackNode

nextNext = prod[nextDot]; - IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> nextNextAlternativeEntry = sharedNextNodes.get(nextNext.getId()); + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> nextNextAlternativeEntry = + sharedNextNodes.get(nextNext.getId()); AbstractStackNode

nextNextAlternative = null; - if(nextNextAlternativeEntry != null){ // Valid continuation. + if (nextNextAlternativeEntry != null) { // Valid continuation. DoubleArrayList, AbstractNode> predecessors = nextNextAlternativeEntry.value2; - if(predecessors == null){ + if (predecessors == null) { nextNextAlternative = nextNextAlternativeEntry.value1; - if(nextNextAlternative.isMatchable()){ - if(nextNextAlternative.isEmptyLeafNode()){ - propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, nextNextAlternative.getResult(), nrOfAddedEdges); - }else{ + if (nextNextAlternative.isMatchable()) { + if (nextNextAlternative.isEmptyLeafNode()) { + propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, + nextNextAlternative.getResult(), nrOfAddedEdges); + } + else { nextNextAlternative.updateNode(next, nextResult); - if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative); + if (debugListener != null) + debugListener.propagated(next, nextResult, nextNextAlternative); } - }else{ + } + else { EdgesSet

nextNextAlternativeEdgesSet = nextNextAlternative.getIncomingEdges(); int resultStoreId = getResultStoreId(nextNextAlternative.getId()); - if(nextNextAlternativeEdgesSet != null && nextNextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){ - propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, nextNextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges); - }else{ + if (nextNextAlternativeEdgesSet != null + && nextNextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) { + propagateEdgesAndPrefixesForNullable(next, nextResult, nextNextAlternative, + nextNextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges); + } + else { nextNextAlternative.updateNode(next, nextResult); - if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative); + if (debugListener != null) + debugListener.propagated(next, nextResult, nextNextAlternative); } } - }else{ + } + else { predecessors.add(next, nextResult); } } // Handle alternative continuations (related to prefix sharing). AbstractStackNode

[][] alternateProds = next.getAlternateProductions(); - if(alternateProds != null){ - if(nextNextAlternative == null){ // If the first continuation has not been initialized yet (it may be a matchable that didn't match), create a dummy version to construct the necessary edges and prefixes. - if(!nextNext.isMatchable()) return; // Matchable, abort. + if (alternateProds != null) { + if (nextNextAlternative == null) { // If the first continuation has not been initialized yet (it may be a + // matchable that didn't match), create a dummy version to construct the + // necessary edges and prefixes. + if (!nextNext.isMatchable()) + return; // Matchable, abort. nextNextAlternative = nextNext.getCleanCopy(location); nextNextAlternative.updateNode(next, nextResult); - if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAlternative); + if (debugListener != null) + debugListener.propagated(next, nextResult, nextNextAlternative); } IntegerObjectList> nextNextEdgesMap = nextNextAlternative.getEdges(); ArrayList[] nextNextPrefixesMap = nextNextAlternative.getPrefixesMap(); - for(int i = alternateProds.length - 1; i >= 0; --i){ + for (int i = alternateProds.length - 1; i >= 0; --i) { prod = alternateProds[i]; - if(nextDot == prod.length) continue; + if (nextDot == prod.length) + continue; AbstractStackNode

alternativeNextNext = prod[nextDot]; - IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> nextNextAltAlternativeEntry = sharedNextNodes.get(alternativeNextNext.getId()); + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> nextNextAltAlternativeEntry = + sharedNextNodes.get(alternativeNextNext.getId()); AbstractStackNode

nextNextAltAlternative = null; - if(nextNextAltAlternativeEntry != null){ - DoubleArrayList, AbstractNode> predecessors = nextNextAltAlternativeEntry.value2; - if(predecessors == null){ + if (nextNextAltAlternativeEntry != null) { + DoubleArrayList, AbstractNode> predecessors = + nextNextAltAlternativeEntry.value2; + if (predecessors == null) { nextNextAltAlternative = nextNextAltAlternativeEntry.value1; - if(nextNextAltAlternative.isMatchable()){ - if(nextNextAltAlternative.isEmptyLeafNode()){ - propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative, nextNextAltAlternative.getResult(), nrOfAddedEdges, nextNextEdgesMap, nextNextPrefixesMap); - }else{ + if (nextNextAltAlternative.isMatchable()) { + if (nextNextAltAlternative.isEmptyLeafNode()) { + propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative, + nextNextAltAlternative.getResult(), nrOfAddedEdges, nextNextEdgesMap, + nextNextPrefixesMap); + } + else { nextNextAltAlternative.updatePrefixSharedNode(nextNextEdgesMap, nextNextPrefixesMap); - if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAltAlternative); + if (debugListener != null) + debugListener.propagated(next, nextResult, nextNextAltAlternative); } - }else{ + } + else { EdgesSet

nextAlternativeEdgesSet = nextNextAltAlternative.getIncomingEdges(); int resultStoreId = getResultStoreId(nextNextAltAlternative.getId()); - if(nextAlternativeEdgesSet != null && nextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location){ - propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative, nextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges, nextNextEdgesMap, nextNextPrefixesMap); - }else{ + if (nextAlternativeEdgesSet != null + && nextAlternativeEdgesSet.getLastVisitedLevel(resultStoreId) == location) { + propagateAlternativeEdgesAndPrefixes(next, nextResult, nextNextAltAlternative, + nextAlternativeEdgesSet.getLastResult(resultStoreId), nrOfAddedEdges, + nextNextEdgesMap, nextNextPrefixesMap); + } + else { nextNextAltAlternative.updatePrefixSharedNode(nextNextEdgesMap, nextNextPrefixesMap); - if(debugListener != null) debugListener.propagated(next, nextResult, nextNextAltAlternative); + if (debugListener != null) + debugListener.propagated(next, nextResult, nextNextAltAlternative); } } - }else{ + } + else { predecessors.add(next, nextResult); } } @@ -501,61 +576,71 @@ private void propagatePrefixes(AbstractStackNode

next, AbstractNode nextResul } /** - * Part of the hidden-right-recursion fix. - * Inserts missing prefixes and triggers reductions where necessary. + * Part of the hidden-right-recursion fix. Inserts missing prefixes and triggers reductions where + * necessary. */ - private void propagateEdgesAndPrefixes(AbstractStackNode

node, AbstractNode nodeResult, AbstractStackNode

next, AbstractNode nextResult){ + private void propagateEdgesAndPrefixes(AbstractStackNode

node, AbstractNode nodeResult, + AbstractStackNode

next, AbstractNode nextResult) { int nrOfAddedEdges = next.updateOvertakenNode(node, nodeResult); - if(debugListener != null) debugListener.propagated(node, nodeResult, next); + if (debugListener != null) + debugListener.propagated(node, nodeResult, next); - if(nrOfAddedEdges == 0) return; + if (nrOfAddedEdges == 0) + return; - if(next.isEndNode()){ + if (next.isEndNode()) { propagateReductions(node, nodeResult, next, nextResult, nrOfAddedEdges); } - if(next.hasNext()){ + if (next.hasNext()) { propagatePrefixes(next, nextResult, nrOfAddedEdges); } } /** - * Part of the hidden-right-recursion fix. - * Inserts missing prefixes and triggers reductions where necessary (specific for nullable nodes). + * Part of the hidden-right-recursion fix. Inserts missing prefixes and triggers reductions where + * necessary (specific for nullable nodes). */ - private void propagateEdgesAndPrefixesForNullable(AbstractStackNode

node, AbstractNode nodeResult, AbstractStackNode

next, AbstractNode nextResult, int potentialNewEdges){ + private void propagateEdgesAndPrefixesForNullable(AbstractStackNode

node, AbstractNode nodeResult, + AbstractStackNode

next, AbstractNode nextResult, int potentialNewEdges) { int nrOfAddedEdges = next.updateOvertakenNullableNode(node, nodeResult, potentialNewEdges); - if(debugListener != null) debugListener.propagated(node, nodeResult, next); + if (debugListener != null) + debugListener.propagated(node, nodeResult, next); - if(nrOfAddedEdges == 0) return; + if (nrOfAddedEdges == 0) + return; - if(next.isEndNode()){ + if (next.isEndNode()) { propagateReductions(node, nodeResult, next, nextResult, nrOfAddedEdges); } - if(next.hasNext()){ + if (next.hasNext()) { propagatePrefixes(next, nextResult, nrOfAddedEdges); } } /** - * Part of the hidden-right-recursion fix. - * Inserts missing prefixes and triggers reductions where necessary (specifically for alternative continuations of prefix-shared productions). + * Part of the hidden-right-recursion fix. Inserts missing prefixes and triggers reductions where + * necessary (specifically for alternative continuations of prefix-shared productions). */ - private void propagateAlternativeEdgesAndPrefixes(AbstractStackNode

node, AbstractNode nodeResult, AbstractStackNode

next, AbstractNode nextResult, int potentialNewEdges, IntegerObjectList> edgesMap, ArrayList[] prefixesMap){ + private void propagateAlternativeEdgesAndPrefixes(AbstractStackNode

node, AbstractNode nodeResult, + AbstractStackNode

next, AbstractNode nextResult, int potentialNewEdges, + IntegerObjectList> edgesMap, ArrayList[] prefixesMap) { next.updatePrefixSharedNode(edgesMap, prefixesMap); - if(debugListener != null) debugListener.propagated(node, nodeResult, next); + if (debugListener != null) + debugListener.propagated(node, nodeResult, next); - if(potentialNewEdges == 0) return; + if (potentialNewEdges == 0) + return; - if(next.isEndNode()){ + if (next.isEndNode()) { propagateReductions(node, nodeResult, next, nextResult, potentialNewEdges); } - if(next.hasNext()){ + if (next.hasNext()) { propagatePrefixes(next, nextResult, potentialNewEdges); } } @@ -563,7 +648,7 @@ private void propagateAlternativeEdgesAndPrefixes(AbstractStackNode

node, Abs /** * Initiates the handling of reductions. */ - private void updateEdges(AbstractStackNode

node, AbstractNode result){ + private void updateEdges(AbstractStackNode

node, AbstractNode result) { IntegerObjectList> edgesMap = node.getEdges(); ArrayList[] prefixesMap = node.getPrefixesMap(); @@ -573,21 +658,25 @@ private void updateEdges(AbstractStackNode

node, AbstractNode result){ // Check for nesting restrictions. boolean hasNestingRestrictions = hasNestingRestrictions(name); IntegerList filteredParents = null; - if(hasNestingRestrictions){ + if (hasNestingRestrictions) { filteredParents = getFilteredParents(node.getId()); } - for(int i = edgesMap.size() - 1; i >= 0; --i){ + for (int i = edgesMap.size() - 1; i >= 0; --i) { Link resultLink = new Link((prefixesMap != null) ? prefixesMap[i] : null, result); EdgesSet

edgeSet = edgesMap.getValue(i); - if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet); + if (debugListener != null) + debugListener.reducing(node, resultLink, edgeSet); - if(!hasNestingRestrictions){ // Select the optimized path for handling edge sets that don't have nesting restrictions associated with them. + if (!hasNestingRestrictions) { // Select the optimized path for handling edge sets that don't have nesting + // restrictions associated with them. handleEdgeList(edgeSet, name, production, resultLink, edgesMap.getKey(i)); - }else{ - handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, edgesMap.getKey(i), filteredParents); + } + else { + handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, edgesMap.getKey(i), + filteredParents); } } } @@ -595,7 +684,7 @@ private void updateEdges(AbstractStackNode

node, AbstractNode result){ /** * Initiates the handling of reductions for nullable symbols. */ - private void updateNullableEdges(AbstractStackNode

node, AbstractNode result){ + private void updateNullableEdges(AbstractStackNode

node, AbstractNode result) { IntegerList propagatedReductions = node.getPropagatedReductions(); int initialSize = propagatedReductions.size(); @@ -609,25 +698,29 @@ private void updateNullableEdges(AbstractStackNode

node, AbstractNode result) // Check for nesting restrictions. boolean hasNestingRestrictions = hasNestingRestrictions(name); IntegerList filteredParents = null; - if(hasNestingRestrictions){ + if (hasNestingRestrictions) { filteredParents = getFilteredParents(node.getId()); } - for(int i = edgesMap.size() - 1; i >= 0; --i){ + for (int i = edgesMap.size() - 1; i >= 0; --i) { int startLocation = edgesMap.getKey(i); - if(propagatedReductions.containsBefore(startLocation, initialSize)) continue; // Prevent duplicate reductions (artifact of the hidden-right-recursion fix). + if (propagatedReductions.containsBefore(startLocation, initialSize)) + continue; // Prevent duplicate reductions (artifact of the hidden-right-recursion fix). propagatedReductions.add(startLocation); Link resultLink = new Link((prefixesMap != null) ? prefixesMap[i] : null, result); EdgesSet

edgeSet = edgesMap.getValue(i); - if(debugListener != null) debugListener.reducing(node, resultLink, edgeSet); + if (debugListener != null) + debugListener.reducing(node, resultLink, edgeSet); - if(!hasNestingRestrictions){ // Select the optimized path for handling edge sets that don't have nesting restrictions associated with them. + if (!hasNestingRestrictions) { // Select the optimized path for handling edge sets that don't have nesting + // restrictions associated with them. handleEdgeList(edgeSet, name, production, resultLink, startLocation); - }else{ + } + else { handleEdgeListWithRestrictions(edgeSet, name, production, resultLink, startLocation, filteredParents); } } @@ -636,34 +729,41 @@ private void updateNullableEdges(AbstractStackNode

node, AbstractNode result) /** * Handles reductions. */ - private void handleEdgeList(EdgesSet

edgeSet, String name, P production, Link resultLink, int startLocation){ + private void handleEdgeList(EdgesSet

edgeSet, String name, P production, Link resultLink, int startLocation) { AbstractContainerNode

resultStore = null; int resultStoreId = EdgesSet.DEFAULT_RESULT_STORE_ID; - if(edgeSet.getLastVisitedLevel(resultStoreId) != location){ + if (edgeSet.getLastVisitedLevel(resultStoreId) != location) { AbstractStackNode

edge = edgeSet.get(0); - if(edge.isRecovered()){ + if (edge.isRecovered()) { resultStore = new RecoveredNode

(inputURI, startLocation, location); - }else if(edge.isExpandable()){ - resultStore = new ExpandableContainerNode

(inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout()); - }else{ - resultStore = new SortContainerNode

(inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout()); + } + else if (edge.isExpandable()) { + resultStore = new ExpandableContainerNode

(inputURI, startLocation, location, + startLocation == location, edge.isSeparator(), edge.isLayout()); + } + else { + resultStore = new SortContainerNode

(inputURI, startLocation, location, startLocation == location, + edge.isSeparator(), edge.isLayout()); } stacksWithNonTerminalsToReduce.push(edge, resultStore); - if(debugListener != null) debugListener.reduced(edge); + if (debugListener != null) + debugListener.reduced(edge); - for(int j = edgeSet.size() - 1; j >= 1; --j){ + for (int j = edgeSet.size() - 1; j >= 1; --j) { edge = edgeSet.get(j); stacksWithNonTerminalsToReduce.push(edge, resultStore); - if(debugListener != null) debugListener.reduced(edge); + if (debugListener != null) + debugListener.reduced(edge); } edgeSet.setLastVisitedLevel(location, resultStoreId); edgeSet.setLastResult(resultStore, resultStoreId); - }else{ + } + else { resultStore = edgeSet.getLastResult(resultStoreId); } @@ -677,36 +777,43 @@ private void handleEdgeList(EdgesSet

edgeSet, String name, P production, Link /** * Handles reductions which may be associated with nesting restrictions. */ - private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P production, Link resultLink, int startLocation, IntegerList filteredParents){ + private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P production, Link resultLink, + int startLocation, IntegerList filteredParents) { // Only add the result to each resultstore once. - // Make sure each edge only gets added to the non-terminal reduction list once per level, by keeping track of them. + // Make sure each edge only gets added to the non-terminal reduction list once per level, by keeping + // track of them. firstTimeRegistration.clear(); firstTimeReductions.clear(); - for(int j = edgeSet.size() - 1; j >= 0; --j){ + for (int j = edgeSet.size() - 1; j >= 0; --j) { AbstractStackNode

edge = edgeSet.get(j); int resultStoreId = getResultStoreId(edge.getId()); - if(!firstTimeReductions.contains(resultStoreId)){ - if(firstTimeRegistration.contains(resultStoreId)){ - if(debugListener != null) debugListener.filteredByNestingRestriction(edge); + if (!firstTimeReductions.contains(resultStoreId)) { + if (firstTimeRegistration.contains(resultStoreId)) { + if (debugListener != null) + debugListener.filteredByNestingRestriction(edge); continue; } firstTimeRegistration.add(resultStoreId); // Check whether or not the nesting is allowed. - if(filteredParents == null || !filteredParents.contains(edge.getId())){ + if (filteredParents == null || !filteredParents.contains(edge.getId())) { AbstractContainerNode

resultStore = null; - if(edgeSet.getLastVisitedLevel(resultStoreId) == location){ + if (edgeSet.getLastVisitedLevel(resultStoreId) == location) { resultStore = edgeSet.getLastResult(resultStoreId); } - if(resultStore == null){ + if (resultStore == null) { if (edge.isRecovered()) { resultStore = new RecoveredNode

(inputURI, startLocation, location); - }else if (edge.isExpandable()) { - resultStore = new ExpandableContainerNode

(inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout()); - }else { - resultStore = new SortContainerNode

(inputURI, startLocation, location, startLocation == location, edge.isSeparator(), edge.isLayout()); + } + else if (edge.isExpandable()) { + resultStore = new ExpandableContainerNode

(inputURI, startLocation, location, + startLocation == location, edge.isSeparator(), edge.isLayout()); + } + else { + resultStore = new SortContainerNode

(inputURI, startLocation, location, + startLocation == location, edge.isSeparator(), edge.isLayout()); } edgeSet.setLastVisitedLevel(location, resultStoreId); @@ -718,11 +825,15 @@ private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P resultStore.addAlternative(production, resultLink); - if(debugListener != null) debugListener.reduced(edge); - }else{ - if(debugListener != null) debugListener.filteredByNestingRestriction(edge); + if (debugListener != null) + debugListener.reduced(edge); + } + else { + if (debugListener != null) + debugListener.filteredByNestingRestriction(edge); } - }else{ + } + else { AbstractContainerNode

resultStore = edgeSet.getLastResult(resultStoreId); stacksWithNonTerminalsToReduce.push(edge, resultStore); } @@ -732,7 +843,7 @@ private void handleEdgeListWithRestrictions(EdgesSet

edgeSet, String name, P /** * Move to the next symbol(s) in the production. */ - private void moveToNext(AbstractStackNode

node, AbstractNode result){ + private void moveToNext(AbstractStackNode

node, AbstractNode result) { int nextDot = node.getDot() + 1; AbstractStackNode

[] prod = node.getProduction(); @@ -741,25 +852,27 @@ private void moveToNext(AbstractStackNode

node, AbstractNode result){ // Handle alternative continuations of this production (related to prefix-sharing). AbstractStackNode

[][] alternateProds = node.getAlternateProductions(); - if(alternateProds != null){ + if (alternateProds != null) { IntegerObjectList> edgesMap = null; ArrayList[] prefixesMap = null; - if(next != null){ + if (next != null) { edgesMap = next.getEdges(); prefixesMap = next.getPrefixesMap(); } - for(int i = alternateProds.length - 1; i >= 0; --i){ + for (int i = alternateProds.length - 1; i >= 0; --i) { prod = alternateProds[i]; - if(nextDot == prod.length) continue; + if (nextDot == prod.length) + continue; AbstractStackNode

newAlternativeNext = prod[nextDot]; - if(edgesMap != null){ + if (edgesMap != null) { updateAlternativeNextNode(newAlternativeNext, node, result, edgesMap, prefixesMap); - }else{ + } + else { AbstractStackNode

alternativeNext = updateNextNode(newAlternativeNext, node, result); - if(alternativeNext != null){ + if (alternativeNext != null) { edgesMap = alternativeNext.getEdges(); prefixesMap = alternativeNext.getPrefixesMap(); } @@ -769,36 +882,50 @@ private void moveToNext(AbstractStackNode

node, AbstractNode result){ } /** - * Progress to the next 'states' associated with the given node. - * I.e. move to the next symbol(s) in the production (if available) and executed reductions if necessary. + * Progress to the next 'states' associated with the given node. I.e. move to the next symbol(s) in + * the production (if available) and executed reductions if necessary. */ - private void move(AbstractStackNode

node, AbstractNode result){ - if(debugListener != null) debugListener.moving(node, result); + private void move(AbstractStackNode

node, AbstractNode result) { + if (debugListener != null) + debugListener.moving(node, result); // Handle filtering. ICompletionFilter[] completionFilters = node.getCompletionFilters(); - if(completionFilters != null){ + if (completionFilters != null) { int startLocation = node.getStartLocation(); - for(int i = completionFilters.length - 1; i >= 0; --i){ - if(completionFilters[i].isFiltered(input, startLocation, location, positionStore)){ + for (int i = completionFilters.length - 1; i >= 0; --i) { + if (completionFilters[i].isFiltered(input, startLocation, location, positionStore)) { filteredNodes.push(node, result); - if(debugListener != null) debugListener.filteredByCompletionFilter(node, result); + if (debugListener != null) + debugListener.filteredByCompletionFilter(node, result); return; } } } - if(node.isEndNode()){ - if(!result.isEmpty() || node.getId() == AbstractExpandableStackNode.DEFAULT_LIST_EPSILON_ID){ // Only go into the nullable fix path for nullables (special list epsilons can be ignored as well). + if (node.isEndNode()) { + if (!result.isEmpty() || node.getId() == AbstractExpandableStackNode.DEFAULT_LIST_EPSILON_ID) { // Only go + // into the + // nullable + // fix path + // for + // nullables + // (special + // list + // epsilons + // can be + // ignored + // as well). updateEdges(node, result); - }else{ + } + else { updateNullableEdges(node, result); } } - if(node.hasNext()){ + if (node.hasNext()) { moveToNext(node, result); } } @@ -809,7 +936,7 @@ private void move(AbstractStackNode

node, AbstractNode result){ private void reduceTerminals() { // Reduce terminals visualize("Reducing terminals", ParseStateVisualizer.TERMINALS_TO_REDUCE_ID); - while(!stacksWithTerminalsToReduce.isEmpty()){ + while (!stacksWithTerminalsToReduce.isEmpty()) { move(stacksWithTerminalsToReduce.peekFirst(), stacksWithTerminalsToReduce.popSecond()); } } @@ -817,18 +944,19 @@ private void reduceTerminals() { private void reduceNonTerminals() { // Reduce non-terminals visualize("Reducing non-terminals", ParseStateVisualizer.NON_TERMINALS_TO_REDUCE_ID); - while(!stacksWithNonTerminalsToReduce.isEmpty()){ + while (!stacksWithNonTerminalsToReduce.isEmpty()) { move(stacksWithNonTerminalsToReduce.peekFirst(), stacksWithNonTerminalsToReduce.popSecond()); } } /** - * Locates the initial set of stacks that is queued for handling, for which the least amount of characters needs to be shifted. + * Locates the initial set of stacks that is queued for handling, for which the least amount of + * characters needs to be shifted. */ - private boolean findFirstStacksToReduce(){ - for(int i = 0; i < todoLists.length; ++i){ + private boolean findFirstStacksToReduce() { + for (int i = 0; i < todoLists.length; ++i) { DoubleStack, AbstractNode> terminalsTodo = todoLists[i]; - if(!(terminalsTodo == null || terminalsTodo.isEmpty())){ + if (!(terminalsTodo == null || terminalsTodo.isEmpty())) { stacksWithTerminalsToReduce = terminalsTodo; location += i; @@ -841,14 +969,17 @@ private boolean findFirstStacksToReduce(){ if (recoverer != null) { if (debugListener != null) { - debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, + unmatchableMidProductionNodes, filteredNodes); } visualize("Recovering", ParseStateVisualizer.ERROR_TRACKING_ID); - DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, + unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); } - if (recoveredNodes.size() > 0) { // TODO Do something with the revived node. Is this the right location to do this? + if (recoveredNodes.size() > 0) { // TODO Do something with the revived node. Is this the right location to + // do this? for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); queueMatchableNode(recovered, recovered.getLength(), recoveredNodes.getSecond(i)); @@ -863,16 +994,17 @@ private boolean findFirstStacksToReduce(){ } /** - * Locates the set of stacks that is queued for handling, for which the least amount of characters needs to be shifted. + * Locates the set of stacks that is queued for handling, for which the least amount of characters + * needs to be shifted. */ - private boolean findStacksToReduce(){ + private boolean findStacksToReduce() { visualize("Finding stacks to reduce", ParseStateVisualizer.TODO_LISTS_ID); int queueDepth = todoLists.length; - for(int i = 1; i < queueDepth; ++i){ + for (int i = 1; i < queueDepth; ++i) { queueIndex = (queueIndex + 1) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[queueIndex]; - if(!(terminalsTodo == null || terminalsTodo.isEmpty())){ + if (!(terminalsTodo == null || terminalsTodo.isEmpty())) { if (ParseStateVisualizer.VISUALIZATION_ENABLED) { NodeId reduceNodeId = new NodeId("todo-" + i); visualize("Found stack to reduce", reduceNodeId); @@ -888,10 +1020,12 @@ private boolean findStacksToReduce(){ if (recoverer != null && location < input.length) { if (debugListener != null) { - debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, + unmatchableMidProductionNodes, filteredNodes); } visualize("Recovering", ParseStateVisualizer.ERROR_TRACKING_ID); - DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + DoubleArrayList, AbstractNode> recoveredNodes = recoverer.reviveStacks(input, location, + unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); if (debugListener != null) { debugListener.revived(recoveredNodes); } @@ -907,10 +1041,12 @@ private boolean findStacksToReduce(){ for (int i = 0; i < recoveredNodes.size(); i++) { AbstractStackNode

recovered = recoveredNodes.getFirst(i); if (debugListener != null) { - debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, unmatchableMidProductionNodes, filteredNodes); + debugListener.reviving(input, location, unexpandableNodes, unmatchableLeafNodes, + unmatchableMidProductionNodes, filteredNodes); } visualize("Queue recovery node", ParseStateVisualizer.getNodeId(recovered)); - queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), recoveredNodes.getSecond(i)); + queueRecoveryNode(recovered, recovered.getStartLocation(), recovered.getLength(), + recoveredNodes.getSecond(i)); } return findStacksToReduce(); } @@ -921,7 +1057,7 @@ private boolean findStacksToReduce(){ return false; } - public boolean parseErrorHasOccurred(){ + public boolean parseErrorHasOccurred() { return parseErrorOccured; } @@ -929,11 +1065,11 @@ public boolean parseErrorHasOccurred(){ * Inserts a stack bottom into the todoList. */ @SuppressWarnings("unchecked") - private void queueMatchableNode(AbstractStackNode

node, int length, AbstractNode result){ + private void queueMatchableNode(AbstractStackNode

node, int length, AbstractNode result) { assert result != null; int queueDepth = todoLists.length; - if(length >= queueDepth){ + if (length >= queueDepth) { DoubleStack, AbstractNode>[] oldTodoLists = todoLists; todoLists = new DoubleStack[length + 1]; System.arraycopy(oldTodoLists, queueIndex, todoLists, 0, queueDepth - queueIndex); @@ -944,96 +1080,101 @@ private void queueMatchableNode(AbstractStackNode

node, int length, AbstractN int insertLocation = (queueIndex + length) % queueDepth; DoubleStack, AbstractNode> terminalsTodo = todoLists[insertLocation]; - if(terminalsTodo == null){ + if (terminalsTodo == null) { terminalsTodo = new DoubleStack<>(); todoLists[insertLocation] = terminalsTodo; } terminalsTodo.push(node, result); } - /** - * Inserts a recovery node into the todoList, and possibly - * rewinds the parser to an earlier location in the input - */ - @SuppressWarnings("unchecked") - private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int length, AbstractNode result){ - assert result != null; + /** + * Inserts a recovery node into the todoList, and possibly rewinds the parser to an earlier location + * in the input + */ + @SuppressWarnings("unchecked") + private void queueRecoveryNode(AbstractStackNode

node, int startPosition, int length, AbstractNode result) { + assert result != null; - int queueDepth = todoLists.length; + int queueDepth = todoLists.length; - if (startPosition < location) { - // Have to reset the parser to an earlier location to at least - // be able to process the new node. Cannot throw away the queue, - // because there are possibly already other recovery tokens in the queue. - // However, we may assume that the queue before the current index is - // done, based on the way we cycle the queue now. The queue is - // looking forward to the future and we never re-use past entries. + if (startPosition < location) { + // Have to reset the parser to an earlier location to at least + // be able to process the new node. Cannot throw away the queue, + // because there are possibly already other recovery tokens in the queue. + // However, we may assume that the queue before the current index is + // done, based on the way we cycle the queue now. The queue is + // looking forward to the future and we never re-use past entries. - int negativeOffset = location - startPosition; + int negativeOffset = location - startPosition; - DoubleStack, AbstractNode>[] oldTodoLists = todoLists; - todoLists = new DoubleStack[negativeOffset + Math.max(queueDepth, length) + 1]; - System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset, queueDepth - queueIndex); - System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset , queueIndex); + DoubleStack, AbstractNode>[] oldTodoLists = todoLists; + todoLists = new DoubleStack[negativeOffset + Math.max(queueDepth, length) + 1]; + System.arraycopy(oldTodoLists, queueIndex, todoLists, negativeOffset, queueDepth - queueIndex); + System.arraycopy(oldTodoLists, 0, todoLists, queueDepth - queueIndex + negativeOffset, queueIndex); - // reset the parser! - queueIndex = 0; - location = startPosition; + // reset the parser! + queueIndex = 0; + location = startPosition; DoubleStack, AbstractNode> terminalsTodo = todoLists[length]; - if (terminalsTodo == null) { - terminalsTodo = new DoubleStack<>(); + if (terminalsTodo == null) { + terminalsTodo = new DoubleStack<>(); todoLists[length] = terminalsTodo; - } - - terminalsTodo.push(node, result); - } - else if (startPosition == location) { - // this is the normal case where new matchable nodes are discovered - // for the current parsing location, so reuse the code for queuing - queueMatchableNode(node, length, result); - } - else { - // This would mean we have discovered a recovery node for a location - // we have not been yet. That would be odd because then there would - // not have been a parse error and we wouldn't need recovery... - throw new RuntimeException("discovered a future recovery? " + node); - } - } + } + + terminalsTodo.push(node, result); + } + else if (startPosition == location) { + // this is the normal case where new matchable nodes are discovered + // for the current parsing location, so reuse the code for queuing + queueMatchableNode(node, length, result); + } + else { + // This would mean we have discovered a recovery node for a location + // we have not been yet. That would be odd because then there would + // not have been a parse error and we wouldn't need recovery... + throw new RuntimeException("discovered a future recovery? " + node); + } + } /** * Handles the retrieved alternatives for the given stack. */ - private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cachedEdges, AbstractStackNode

stackBeingWorkedOn){ + private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cachedEdges, + AbstractStackNode

stackBeingWorkedOn) { boolean hasValidAlternatives = false; sharedLastExpects.dirtyClear(); - EXPECTS: for(int i = expects.length - 1; i >= 0; --i){ + EXPECTS: for (int i = expects.length - 1; i >= 0; --i) { AbstractStackNode

first = expects[i]; - if(first.isMatchable()){ // Eager matching optimization. + if (first.isMatchable()) { // Eager matching optimization. int length = first.getLength(); int endLocation = location + length; - if(endLocation > input.length) continue; + if (endLocation > input.length) + continue; AbstractNode result = first.match(input, location); - if(result == null){ + if (result == null) { unmatchableLeafNodes.push(first); - if(debugListener != null) debugListener.failedToMatch(first); + if (debugListener != null) + debugListener.failedToMatch(first); continue; } - if(debugListener != null) debugListener.matched(first, result); + if (debugListener != null) + debugListener.matched(first, result); // Handle filtering. IEnterFilter[] enterFilters = first.getEnterFilters(); - if(enterFilters != null){ - for(int j = enterFilters.length - 1; j >= 0; --j){ - if(enterFilters[j].isFiltered(input, location, positionStore)){ - if(debugListener != null) debugListener.filteredByEnterFilter(first); + if (enterFilters != null) { + for (int j = enterFilters.length - 1; j >= 0; --j) { + if (enterFilters[j].isFiltered(input, location, positionStore)) { + if (debugListener != null) + debugListener.filteredByEnterFilter(first); continue EXPECTS; } @@ -1043,7 +1184,8 @@ private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cached first = first.getCleanCopyWithResult(location, result); queueMatchableNode(first, length, result); - } else { + } + else { first = first.getCleanCopy(location); stacksToExpand.push(first); } @@ -1055,7 +1197,8 @@ private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cached hasValidAlternatives = true; - if(debugListener != null) debugListener.expanded(stackBeingWorkedOn, first); + if (debugListener != null) + debugListener.expanded(stackBeingWorkedOn, first); } return hasValidAlternatives; @@ -1064,77 +1207,83 @@ private boolean handleExpects(AbstractStackNode

[] expects, EdgesSet

cached /** * Check whether or not the given sort name has nesting restrictions associated with it. */ - protected boolean hasNestingRestrictions(String name){ + protected boolean hasNestingRestrictions(String name) { return false; // Priority and associativity filtering is off by default. } /** * Retrieves the set of disallowed parents for the given child. */ - protected IntegerList getFilteredParents(int childId){ + protected IntegerList getFilteredParents(int childId) { return null; // Default implementation; intended to be overwritten in sub-classes. } /** * Retrieves the resultstore id associated with the given id. */ - protected int getResultStoreId(int id){ + protected int getResultStoreId(int id) { return EdgesSet.DEFAULT_RESULT_STORE_ID; // Default implementation; intended to be overwritten in sub-classes. } /** * Expands the given stack node. */ - private void expandStack(AbstractStackNode

stack){ - if(debugListener != null) debugListener.expanding(stack); + private void expandStack(AbstractStackNode

stack) { + if (debugListener != null) + debugListener.expanding(stack); // Handle filtering. IEnterFilter[] enterFilters = stack.getEnterFilters(); - if(enterFilters != null){ - for(int i = enterFilters.length - 1; i >= 0; --i){ - if(enterFilters[i].isFiltered(input, location, positionStore)){ + if (enterFilters != null) { + for (int i = enterFilters.length - 1; i >= 0; --i) { + if (enterFilters[i].isFiltered(input, location, positionStore)) { unexpandableNodes.push(stack); - if(debugListener != null) debugListener.filteredByEnterFilter(stack); + if (debugListener != null) + debugListener.filteredByEnterFilter(stack); return; } } } - if(stack.isMatchable()){ // Eager matching optimization related. + if (stack.isMatchable()) { // Eager matching optimization related. queueMatchableNode(stack, stack.getLength(), stack.getResult()); - }else if(!stack.isExpandable()){ // A 'normal' non-terminal. + } + else if (!stack.isExpandable()) { // A 'normal' non-terminal. EdgesSet

cachedEdges = cachedEdgesForExpect.get(stack.getName()); - if(cachedEdges == null){ + if (cachedEdges == null) { cachedEdges = new EdgesSet

(1); cachedEdgesForExpect.put(stack.getName(), cachedEdges); AbstractStackNode

[] expects = invokeExpects(stack); - if(expects == null){ + if (expects == null) { unexpandableNodes.push(stack); return; } - if(!handleExpects(expects, cachedEdges, stack)){ + if (!handleExpects(expects, cachedEdges, stack)) { unexpandableNodes.push(stack); return; } - }else{ + } + else { int resultStoreId = getResultStoreId(stack.getId()); - if(cachedEdges.getLastVisitedLevel(resultStoreId) == location){ // Is nullable, add the known results. + if (cachedEdges.getLastVisitedLevel(resultStoreId) == location) { // Is nullable, add the known results. stacksWithNonTerminalsToReduce.push(stack, cachedEdges.getLastResult(resultStoreId)); - if(debugListener != null) debugListener.foundIterationCachedNullableResult(stack); + if (debugListener != null) + debugListener.foundIterationCachedNullableResult(stack); } } cachedEdges.add(stack); stack.setIncomingEdges(cachedEdges); - }else{ // Expandable + } + else { // Expandable EdgesSet

cachedEdges = cachedEdgesForExpect.get(stack.getName()); - if(cachedEdges == null){ + if (cachedEdges == null) { boolean expanded = false; cachedEdges = new EdgesSet

(); @@ -1142,37 +1291,43 @@ private void expandStack(AbstractStackNode

stack){ AbstractStackNode

[] listChildren = stack.getChildren(); - CHILDREN : for(int i = listChildren.length - 1; i >= 0; --i){ + CHILDREN: for (int i = listChildren.length - 1; i >= 0; --i) { AbstractStackNode

child = listChildren[i]; int childId = child.getId(); - IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> sharedChildEntry = sharedNextNodes.get(childId); - if(sharedChildEntry != null && sharedChildEntry.value2 == null){ + IntegerKeyedDoubleValueHashMap.Entry, DoubleArrayList, AbstractNode>> sharedChildEntry = + sharedNextNodes.get(childId); + if (sharedChildEntry != null && sharedChildEntry.value2 == null) { AbstractStackNode

sharedChild = sharedChildEntry.value1; sharedChild.setEdgesSetWithPrefix(cachedEdges, null, location); - }else{ - if(child.isMatchable()){ + } + else { + if (child.isMatchable()) { int length = child.getLength(); int endLocation = location + length; - if(endLocation > input.length) continue; + if (endLocation > input.length) + continue; AbstractNode result = child.match(input, location); - if(result == null){ + if (result == null) { unmatchableLeafNodes.push(child); - if(debugListener != null) debugListener.failedToMatch(child); + if (debugListener != null) + debugListener.failedToMatch(child); continue; } - if(debugListener != null) debugListener.matched(child, result); + if (debugListener != null) + debugListener.matched(child, result); // Handle filtering IEnterFilter[] childEnterFilters = child.getEnterFilters(); - if(childEnterFilters != null){ - for(int j = childEnterFilters.length - 1; j >= 0; --j){ - if(childEnterFilters[j].isFiltered(input, location, positionStore)) { - if(debugListener != null) debugListener.filteredByEnterFilter(child); + if (childEnterFilters != null) { + for (int j = childEnterFilters.length - 1; j >= 0; --j) { + if (childEnterFilters[j].isFiltered(input, location, positionStore)) { + if (debugListener != null) + debugListener.filteredByEnterFilter(child); continue CHILDREN; } @@ -1181,7 +1336,8 @@ private void expandStack(AbstractStackNode

stack){ child = child.getCleanCopyWithResult(location, result); queueMatchableNode(child, length, result); - }else{ + } + else { child = child.getCleanCopy(location); stacksToExpand.push(child); } @@ -1191,34 +1347,38 @@ private void expandStack(AbstractStackNode

stack){ sharedNextNodes.putUnsafe(childId, child, null); - if(debugListener != null) debugListener.expanded(stack, child); + if (debugListener != null) + debugListener.expanded(stack, child); } expanded = true; } - if(stack.canBeEmpty()){ // Star list or optional. - AbstractStackNode

empty = stack.getEmptyChild().getCleanCopyWithResult(location, EpsilonStackNode.EPSILON_RESULT); + if (stack.canBeEmpty()) { // Star list or optional. + AbstractStackNode

empty = + stack.getEmptyChild().getCleanCopyWithResult(location, EpsilonStackNode.EPSILON_RESULT); empty.initEdges(); empty.addEdges(cachedEdges, location); stacksToExpand.push(empty); - if(debugListener != null) debugListener.expanded(stack, empty); + if (debugListener != null) + debugListener.expanded(stack, empty); expanded = true; } - if(!expanded){ + if (!expanded) { unexpandableNodes.push(stack); } } int resultStoreId = getResultStoreId(stack.getId()); - if(cachedEdges.getLastVisitedLevel(resultStoreId) == location){ // Is nullable, add the known results. + if (cachedEdges.getLastVisitedLevel(resultStoreId) == location) { // Is nullable, add the known results. stacksWithNonTerminalsToReduce.push(stack, cachedEdges.getLastResult(resultStoreId)); - if(debugListener != null) debugListener.foundIterationCachedNullableResult(stack); + if (debugListener != null) + debugListener.foundIterationCachedNullableResult(stack); } cachedEdges.add(stack); @@ -1230,14 +1390,14 @@ private void expandStack(AbstractStackNode

stack){ /** * Initiate stack expansion for all queued stacks. */ - private void expand(){ + private void expand() { visualize("Expanding", ParseStateVisualizer.STACKS_TO_EXPAND_ID); - while(!stacksToExpand.isEmpty()){ + while (!stacksToExpand.isEmpty()) { expandStack(stacksToExpand.pop()); } } - protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] input){ + protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] input) { return parse(startNode, inputURI, input, (IRecoverer

) null, (IDebugListener

) null); } @@ -1245,131 +1405,146 @@ protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] * Initiates parsing. */ @SuppressWarnings("unchecked") - protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] input, IRecoverer

recoverer, IDebugListener

debugListener){ - initTime(); + protected AbstractNode parse(AbstractStackNode

startNode, URI inputURI, int[] input, IRecoverer

recoverer, + IDebugListener

debugListener) { + initTime(); - try { + try { - if(invoked){ - throw new RuntimeException("Can only invoke 'parse' once."); - } + if (invoked) { + throw new RuntimeException("Can only invoke 'parse' once."); + } - invoked = true; + invoked = true; - // Initialize. - this.inputURI = inputURI; - this.input = input; + // Initialize. + this.inputURI = inputURI; + this.input = input; - this.recoverer = recoverer; - this.debugListener = debugListener; + this.recoverer = recoverer; + this.debugListener = debugListener; - if (inputURI != null) { - String query = inputURI.getQuery(); - visualizer = query != null && query.contains("visualize=true") ? new ParseStateVisualizer("Parser") : null; - } + if (ParseStateVisualizer.VISUALIZATION_ENABLED && inputURI != null) { + String query = inputURI.getQuery(); + visualizer = + query != null && query.contains("visualize=true") ? new ParseStateVisualizer("Parser") : null; + } - // Initialzed the position store. - positionStore.index(input); + // Initialzed the position store. + positionStore.index(input); - todoLists = new DoubleStack[DEFAULT_TODOLIST_CAPACITY]; + todoLists = new DoubleStack[DEFAULT_TODOLIST_CAPACITY]; - // Handle the initial expansion of the root node. - AbstractStackNode

rootNode = startNode; - rootNode.initEdges(); - stacksToExpand.push(rootNode); - lookAheadChar = (input.length > 0) ? input[0] : 0; + // Handle the initial expansion of the root node. + AbstractStackNode

rootNode = startNode; + rootNode.initEdges(); + stacksToExpand.push(rootNode); + lookAheadChar = (input.length > 0) ? input[0] : 0; - if(debugListener != null) { - debugListener.shifting(location, input, positionStore); - } + if (debugListener != null) { + debugListener.shifting(location, input, positionStore); + } - expand(); + expand(); - if(findFirstStacksToReduce()){ - boolean shiftedLevel = (location != 0); + if (findFirstStacksToReduce()) { + boolean shiftedLevel = (location != 0); - do { - lookAheadChar = (location < input.length) ? input[location] : 0; - if(shiftedLevel){ // Nullable fix for the first level. - sharedNextNodes.clear(); - cachedEdgesForExpect.clear(); + do { + lookAheadChar = (location < input.length) ? input[location] : 0; + if (shiftedLevel) { // Nullable fix for the first level. + sharedNextNodes.clear(); + cachedEdgesForExpect.clear(); - unexpandableNodes.dirtyClear(); - unmatchableLeafNodes.dirtyClear(); - unmatchableMidProductionNodes.dirtyClear(); - filteredNodes.dirtyClear(); + unexpandableNodes.dirtyClear(); + unmatchableLeafNodes.dirtyClear(); + unmatchableMidProductionNodes.dirtyClear(); + filteredNodes.dirtyClear(); - if(debugListener != null) debugListener.shifting(location, input, positionStore); - } + if (debugListener != null) + debugListener.shifting(location, input, positionStore); + } - // Reduce-expand loop. - do { - if(debugListener != null) debugListener.iterating(); + // Reduce-expand loop. + do { + if (debugListener != null) { + debugListener.iterating(); + } - reduceTerminals(); + reduceTerminals(); - reduceNonTerminals(); + reduceNonTerminals(); - expand(); - } while(!stacksWithNonTerminalsToReduce.isEmpty() || !stacksWithTerminalsToReduce.isEmpty()); + expand(); + } + while (!stacksWithNonTerminalsToReduce.isEmpty() || !stacksWithTerminalsToReduce.isEmpty()); - shiftedLevel = true; - } while(findStacksToReduce()); - } + shiftedLevel = true; + } + while (findStacksToReduce()); + } - visualize("Done", ParseStateVisualizer.PARSER_ID); + visualize("Done", ParseStateVisualizer.PARSER_ID); - // Check if we were successful. - if(location == input.length){ - EdgesSet

startNodeEdgesSet = startNode.getIncomingEdges(); - int resultStoreId = getResultStoreId(startNode.getId()); - if(startNodeEdgesSet != null && startNodeEdgesSet.getLastVisitedLevel(resultStoreId) == input.length){ - // Parsing succeeded. - return startNodeEdgesSet.getLastResult(resultStoreId); // Success. - } - } - } - finally { - checkTime("Parsing"); - } + // Check if we were successful. + if (location == input.length) { + EdgesSet

startNodeEdgesSet = startNode.getIncomingEdges(); + int resultStoreId = getResultStoreId(startNode.getId()); + if (startNodeEdgesSet != null && startNodeEdgesSet.getLastVisitedLevel(resultStoreId) == input.length) { + // Parsing succeeded. + return startNodeEdgesSet.getLastResult(resultStoreId); // Success. + } + } + } + finally { + checkTime("Parsing"); + } - try { - // A parse error occured, and recovery failed as well - parseErrorOccured = true; + try { + // A parse error occured, and recovery failed as well + parseErrorOccured = true; - int errorLocation = (location == Integer.MAX_VALUE ? 0 : location); - int line = positionStore.findLine(errorLocation); - int column = positionStore.getColumn(errorLocation, line); - if (location == input.length) { - throw new ParseError("Parse error", inputURI, errorLocation, 0, line + 1, line + 1, column, column, (Stack>) (Stack) unexpandableNodes, (Stack>) (Stack) unmatchableLeafNodes, (DoubleStack>, AbstractStackNode>) (DoubleStack) unmatchableMidProductionNodes, (DoubleStack, AbstractNode>) (DoubleStack) filteredNodes); - } - throw new ParseError("Parse error", inputURI, errorLocation, 1, line + 1, line + 1, column, column + 1, (Stack>) (Stack) unexpandableNodes, (Stack>) (Stack) unmatchableLeafNodes, (DoubleStack>, AbstractStackNode>) (DoubleStack) unmatchableMidProductionNodes, (DoubleStack, AbstractNode>) (DoubleStack) filteredNodes); - } - finally { - checkTime("Error handling"); - } + int errorLocation = (location == Integer.MAX_VALUE ? 0 : location); + int line = positionStore.findLine(errorLocation); + int column = positionStore.getColumn(errorLocation, line); + if (location == input.length) { + throw new ParseError("Parse error", inputURI, errorLocation, 0, line + 1, line + 1, column, column, + (Stack>) (Stack) unexpandableNodes, + (Stack>) (Stack) unmatchableLeafNodes, + (DoubleStack>, AbstractStackNode>) (DoubleStack) unmatchableMidProductionNodes, + (DoubleStack, AbstractNode>) (DoubleStack) filteredNodes); + } + throw new ParseError("Parse error", inputURI, errorLocation, 1, line + 1, line + 1, column, column + 1, + (Stack>) (Stack) unexpandableNodes, + (Stack>) (Stack) unmatchableLeafNodes, + (DoubleStack>, AbstractStackNode>) (DoubleStack) unmatchableMidProductionNodes, + (DoubleStack, AbstractNode>) (DoubleStack) filteredNodes); + } + finally { + checkTime("Error handling"); + } } private void initTime() { - timestamp = System.nanoTime(); + timestamp = System.nanoTime(); } - private void checkTime(String msg) { - long newStamp = System.nanoTime(); + private void checkTime(String msg) { + long newStamp = System.nanoTime(); long duration = newStamp - timestamp; timestamp = newStamp; if (printTimes) { - System.err.println(msg + ": " + duration / (1000 * 1000)); + System.err.println(msg + ": " + duration / (1000 * 1000)); } - } + } - private static int[] charsToInts(char[] input){ + private static int[] charsToInts(char[] input) { int[] result = new int[Character.codePointCount(input, 0, input.length)]; int j = 0; - for(int i = 0; i < input.length; i++){ + for (int i = 0; i < input.length; i++) { if (!Character.isLowSurrogate(input[i])) { result[j++] = Character.codePointAt(input, i); } @@ -1381,52 +1556,73 @@ private static int[] charsToInts(char[] input){ /** * Parses with post parse filtering. */ - private T parse(String nonterminal, URI inputURI, int[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, IDebugListener

debugListener){ - AbstractNode result = parse(new NonTerminalStackNode

(AbstractStackNode.START_SYMBOL_ID, 0, nonterminal), inputURI, input, recoverer, debugListener); + private T parse(String nonterminal, URI inputURI, int[] input, IActionExecutor actionExecutor, + INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, + IDebugListener

debugListener) { + AbstractNode result = parse(new NonTerminalStackNode

(AbstractStackNode.START_SYMBOL_ID, 0, nonterminal), + inputURI, input, recoverer, debugListener); return buildResult(result, converter, nodeConstructorFactory, actionExecutor); } - public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, IDebugListener

debugListener){ - return parse(nonterminal, inputURI, charsToInts(input), actionExecutor, converter, nodeConstructorFactory, recoverer, debugListener); + public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, + INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, + IDebugListener

debugListener) { + return parse(nonterminal, inputURI, charsToInts(input), actionExecutor, converter, nodeConstructorFactory, + recoverer, debugListener); } - public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer){ + public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, + INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer) { return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, recoverer, null); } - public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IDebugListener

debugListener){ - return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, null, debugListener); + public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, + INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, + IDebugListener

debugListener) { + return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, null, + debugListener); } - public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory){ + public T parse(String nonterminal, URI inputURI, char[] input, IActionExecutor actionExecutor, + INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory) { return parse(nonterminal, inputURI, input, actionExecutor, converter, nodeConstructorFactory, null, null); } /** * Parses without post parse filtering. */ - private T parse(String nonterminal, URI inputURI, int[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, IDebugListener

debugListener){ - AbstractNode result = parse(new NonTerminalStackNode

(AbstractStackNode.START_SYMBOL_ID, 0, nonterminal), inputURI, input, recoverer, debugListener); + private T parse(String nonterminal, URI inputURI, int[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, + IDebugListener

debugListener) { + AbstractNode result = parse(new NonTerminalStackNode

(AbstractStackNode.START_SYMBOL_ID, 0, nonterminal), + inputURI, input, recoverer, debugListener); return buildResult(result, converter, nodeConstructorFactory, new VoidActionExecutor()); } - public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, IDebugListener

debugListener){ - return parse(nonterminal, inputURI, charsToInts(input), converter, nodeConstructorFactory, recoverer, debugListener); + public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer, + IDebugListener

debugListener) { + return parse(nonterminal, inputURI, charsToInts(input), converter, nodeConstructorFactory, recoverer, + debugListener); } - public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer){ + public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory, IRecoverer

recoverer) { return parse(nonterminal, inputURI, input, converter, nodeConstructorFactory, recoverer, null); } - public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IDebugListener

debugListener){ + public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory, IDebugListener

debugListener) { return parse(nonterminal, inputURI, input, converter, nodeConstructorFactory, null, debugListener); } - public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory) { + public T parse(String nonterminal, URI inputURI, char[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory) { return parse(nonterminal, inputURI, charsToInts(input), converter, nodeConstructorFactory, null, null); } - protected T parse(AbstractStackNode

startNode, URI inputURI, char[] input, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory) { + protected T parse(AbstractStackNode

startNode, URI inputURI, char[] input, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory) { AbstractNode result = parse(startNode, inputURI, charsToInts(input), null, null); @@ -1436,71 +1632,79 @@ protected T parse(AbstractStackNode

startNode, URI inputURI, char[] input, IN /** * Constructed the final parse result using the given converter. */ - protected T buildResult(AbstractNode result, INodeFlattener converter, INodeConstructorFactory nodeConstructorFactory, IActionExecutor actionExecutor){ - initTime(); - try { - FilteringTracker filteringTracker = new FilteringTracker(); - // Invoke the forest flattener, a.k.a. "the bulldozer". - Object rootEnvironment = actionExecutor != null ? actionExecutor.createRootEnvironment() : null; - T parseResult = null; - try { - parseResult = converter.convert(nodeConstructorFactory, result, positionStore, filteringTracker, actionExecutor, rootEnvironment); - } - finally { - actionExecutor.completed(rootEnvironment, (parseResult == null)); - } - if (parseResult != null) { - if (recoverer != null) { - parseResult = fixErrorNodes(parseResult, nodeConstructorFactory); + protected T buildResult(AbstractNode result, INodeFlattener converter, + INodeConstructorFactory nodeConstructorFactory, IActionExecutor actionExecutor) { + initTime(); + try { + FilteringTracker filteringTracker = new FilteringTracker(); + // Invoke the forest flattener, a.k.a. "the bulldozer". + Object rootEnvironment = actionExecutor != null ? actionExecutor.createRootEnvironment() : null; + T parseResult = null; + try { + parseResult = converter.convert(nodeConstructorFactory, result, positionStore, filteringTracker, + actionExecutor, rootEnvironment); + } + finally { + actionExecutor.completed(rootEnvironment, (parseResult == null)); + } + if (parseResult != null) { + if (recoverer != null) { + parseResult = introduceErrorNodes(parseResult, nodeConstructorFactory); + } + return parseResult; // Success. } - return parseResult; // Success. - } - - int offset = filteringTracker.getOffset(); - int endOffset = filteringTracker.getEndOffset(); - int length = endOffset - offset; - int beginLine = positionStore.findLine(offset); - int beginColumn = positionStore.getColumn(offset, beginLine); - int endLine = positionStore.findLine(endOffset); - int endColumn = positionStore.getColumn(endOffset, endLine); - throw new ParseError("All results were filtered", inputURI, offset, length, beginLine + 1, endLine + 1, beginColumn, endColumn); - } - finally { - checkTime("Unbinarizing, post-parse filtering, and mapping to UPTR"); - } + + int offset = filteringTracker.getOffset(); + int endOffset = filteringTracker.getEndOffset(); + int length = endOffset - offset; + int beginLine = positionStore.findLine(offset); + int beginColumn = positionStore.getColumn(offset, beginLine); + int endLine = positionStore.findLine(endOffset); + int endColumn = positionStore.getColumn(endOffset, endLine); + throw new ParseError("All results were filtered", inputURI, offset, length, beginLine + 1, endLine + 1, + beginColumn, endColumn); + } + finally { + checkTime("Unbinarizing, post-parse filtering, and mapping to UPTR"); + } } /** - * Error nodes end up in an inconvenient form because of the parser algorithm. - * This post-processing step transforms the original tree into a more useful form. - * In essence, error subtrees look like this after parsing: - * `appl(prod(S,[]), [,,...,appl(skipped([]))])` - * This method transforms these trees into: + * After parsing, parse trees will only contain `skipped` nodes. This post-processing step + * transforms the original tree into a more useful form. In essence, subtrees containing errors look + * like this after parsing: `appl(prod(S,[]), + * [,,...,appl(skipped([]))])` This method transforms these trees into: * `appl(error(S,prod(S,[]),), [,,...,appl(skipped([]))])` - * This means productions that failed to parse can be recognized at the top level. - * Note that this can only be done when we know the actual type of T is IConstructor. + * This means productions that failed to parse can be recognized at the top level. Note that this + * can only be done when we know the actual type of T is IConstructor. */ @SuppressWarnings("unchecked") - private T fixErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { + private T introduceErrorNodes(T tree, INodeConstructorFactory nodeConstructorFactory) { if (!(tree instanceof IConstructor)) { return tree; } - return (T) fixErrorNodes((IConstructor) tree, (INodeConstructorFactory) nodeConstructorFactory); + return (T) introduceErrorNodes((IConstructor) tree, + (INodeConstructorFactory) nodeConstructorFactory); } - private IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { + private IConstructor introduceErrorNodes(IConstructor tree, + INodeConstructorFactory nodeConstructorFactory) { IConstructor result; Type type = tree.getConstructorType(); if (type == RascalValueFactory.Tree_Appl) { result = fixErrorAppl(tree, nodeConstructorFactory); - } else if (type == RascalValueFactory.Tree_Char) { + } + else if (type == RascalValueFactory.Tree_Char) { result = tree; - } else if (type == RascalValueFactory.Tree_Amb) { + } + else if (type == RascalValueFactory.Tree_Amb) { result = fixErrorAmb(tree, nodeConstructorFactory); - } else if (type == RascalValueFactory.Tree_Cycle) { + } + else if (type == RascalValueFactory.Tree_Cycle) { result = tree; - } else { + } + else { throw new RuntimeException("Unrecognized tree type: " + type); } @@ -1512,7 +1716,8 @@ private IConstructor fixErrorNodes(IConstructor tree, INodeConstructorFactory nodeConstructorFactory) { + private IConstructor fixErrorAppl(IConstructor tree, + INodeConstructorFactory nodeConstructorFactory) { IValue prod = tree.get(0); IList childList = (IList) tree.get(1); int childCount = childList.length(); @@ -1520,10 +1725,10 @@ private IConstructor fixErrorAppl(IConstructor tree, INodeConstructorFactory children = new ArrayList<>(childCount); boolean anyChanges = false; boolean errorTree = false; - for (int i=0; i nodeConstructorFactory) { + private IConstructor fixErrorAmb(IConstructor tree, + INodeConstructorFactory nodeConstructorFactory) { ISet alternativeSet = (ISet) tree.get(0); ArrayList alternatives = new ArrayList<>(alternativeSet.size()); final AtomicBoolean anyChanges = new AtomicBoolean(false); alternativeSet.forEach(alt -> { - IConstructor newAlt = fixErrorNodes((IConstructor) alt, nodeConstructorFactory); + IConstructor newAlt = introduceErrorNodes((IConstructor) alt, nodeConstructorFactory); if (newAlt != alt) { anyChanges.setPlain(true); } @@ -1568,7 +1775,7 @@ private IConstructor fixErrorAmb(IConstructor tree, INodeConstructorFactory { R visit(AlternativeStackNode

node); diff --git a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java index 92c8835d86a..68094cc199a 100644 --- a/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java +++ b/src/org/rascalmpl/parser/gtd/stack/StackNodeVisitorAdapter.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -12,7 +12,7 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **/ - package org.rascalmpl.parser.gtd.stack; +package org.rascalmpl.parser.gtd.stack; public class StackNodeVisitorAdapter implements StackNodeVisitor { diff --git a/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java b/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java index 655adb7a879..4eb156c4a98 100644 --- a/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java +++ b/src/org/rascalmpl/parser/gtd/util/DoubleArrayList.java @@ -107,16 +107,21 @@ public int size(){ return size; } - public void sort(Comparator> comparator) { - java.util.List> elems = new java.util.ArrayList<>(size); + private java.util.List> zip() { + java.util.List> entries = new java.util.ArrayList<>(size); for (int i=0; i> comparator) { + java.util.List> entries = zip(); - elems.sort(comparator); + entries.sort(comparator); for (int i=0; i elem = elems.get(i); + Pair elem = entries.get(i); first[i] = elem.getLeft(); second[i] = elem.getRight(); } diff --git a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java index f1c5de57008..f83fe0c4bf6 100644 --- a/src/org/rascalmpl/parser/gtd/util/IdDispenser.java +++ b/src/org/rascalmpl/parser/gtd/util/IdDispenser.java @@ -1,5 +1,5 @@ /** - * Copyright (c) 2022, NWO-I Centrum Wiskunde & Informatica (CWI) + * Copyright (c) 2024, NWO-I Centrum Wiskunde & Informatica (CWI) * All rights reserved. * * Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -12,7 +12,7 @@ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. **/ - package org.rascalmpl.parser.gtd.util; +package org.rascalmpl.parser.gtd.util; public interface IdDispenser { int dispenseId(); diff --git a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java index d3aa2a63a7e..3b5e8a84c68 100644 --- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java +++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java @@ -155,23 +155,4 @@ public ITree createErrorNode(ArrayList children, Object production) { return buildAppl(children, errorProd); } - /* - @Override - public ITree createRecoveryNode(int[] characters) { - } - IListWriter args = VF.listWriter(); - for (int i=0; i { - private static final int[] WHITESPACE = {' ', '\r', '\n', '\t' }; - - private IdDispenser stackNodeIdDispenser; - - public ToNextWhitespaceRecoverer(IdDispenser stackNodeIdDispenser) { - this.stackNodeIdDispenser = stackNodeIdDispenser; - } - - @Override - public DoubleArrayList, AbstractNode> reviveStacks(int[] input, - int location, - Stack> unexpandableNodes, - Stack> unmatchableLeafNodes, - DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, - DoubleStack, AbstractNode> filteredNodes) { - - ArrayList> failedNodes = new ArrayList<>(); - collectUnexpandableNodes(unexpandableNodes, failedNodes); - collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); - // handle unmatchableLeafNodes? - // collectFilteredNodes(filteredNodes, failedNodes); - - return reviveFailedNodes(input, failedNodes); - } - - private DoubleArrayList, AbstractNode> reviveNodes(int[] input, DoubleArrayList, ArrayList> recoveryNodes){ - DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); - - recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); - - for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); - ArrayList prods = recoveryNodes.getSecond(i); - - // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). - for (int j = prods.size() - 1; j >= 0; --j) { - IConstructor prod = prods.get(j); - - AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); - - int startLocation = recoveryNode.getStartLocation(); - - SkippedNode result; - if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { - result = SkippingStackNode.createResultUntilEndOfInput(null, input, startLocation); - } else { - result = SkippingStackNode.createResultUntilCharClass(null, WHITESPACE, input, startLocation); - } - - AbstractStackNode recoverLiteral = new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result); - recoverLiteral = recoverLiteral.getCleanCopy(startLocation); // This copy might not be needed - recoverLiteral.initEdges(); - EdgesSet edges = new EdgesSet<>(1); - edges.add(continuer); - - recoverLiteral.addEdges(edges, startLocation); - - continuer.setIncomingEdges(edges); - - recoveredNodes.add(recoverLiteral, recoverLiteral.getResult()); - } - } - - return recoveredNodes; - } - - // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) - // As this is experimental code, this method is extremely conservative. - // Any sharing will result in returning 'false'. - // We will need to change this strategy in the future to improve error recovery. - private boolean isTopLevelProduction(AbstractStackNode node) { - while (node != null && node.getDot() != 0) { - node = getSinglePredecessor(node); - } - - if (node != null) { - node = getSinglePredecessor(node); - return node != null && node.getStartLocation() == -1; - } - - return false; - } - - private AbstractStackNode getSinglePredecessor(AbstractStackNode node) { - IntegerObjectList> edgeMap = node.getEdges(); - if (edgeMap.size() == 1) { - EdgesSet edges = edgeMap.getValue(0); - if (edges.size() == 1) { - return edges.get(0); - } - } - - return null; - } - - private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, ArrayList> failedNodes) { - DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList<>(); - - for (int i = failedNodes.size() - 1; i >= 0; --i) { - findRecoveryNodes(failedNodes.get(i), recoveryNodes); - } - - return reviveNodes(input, recoveryNodes); - } - - private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { - for (int i = unexpandableNodes.getSize() - 1; i >= 0; --i) { - failedNodes.add(unexpandableNodes.get(i)); - } - } - - /** - * Make a fresh copy of each unmatchable mid-production node and link in the predecessors of the original node. - * The new copies are added to `failedNodes` - * @param location the location where the failure occurs - * @param unmatchableMidProductionNodes each pair consists of a list of predecessors and a node that failed to match - * @param failedNodes the list to which failed nodes must be added - */ - private static void collectUnmatchableMidProductionNodes(int location, DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ - for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { - DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); - AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version - - // Merge the information on the predecessors into the failed node. - for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { - AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); - AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); - failedNode.updateNode(predecessor, predecessorResult); - } - - failedNodes.add(failedNode); - } - } - - /** - * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. - */ - private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { - ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap<>(); - Stack> todo = new Stack<>(); - - todo.push(failer); - - while (!todo.isEmpty()) { - AbstractStackNode node = todo.pop(); - - if (visited.contains(node)) { - continue; // Don't follow cycles - } - - visited.put(node, 0); - - ArrayList recoveryProductions = new ArrayList<>(); - collectProductions(node, recoveryProductions); - if (recoveryProductions.size() > 0) { - recoveryNodes.add(node, recoveryProductions); - } - - IntegerObjectList> edges = node.getEdges(); - - for (int i = edges.size() - 1; i >= 0; --i) { // Rewind - EdgesSet edgesList = edges.getValue(i); - - if (edgesList != null) { - for (int j = edgesList.size() - 1; j >= 0; --j) { - AbstractStackNode parent = edgesList.get(j); - - todo.push(parent); - } - } - } - } - } - - // Gathers all productions that are marked for recovery (the given node can be part of a prefix shared production) - private void collectProductions(AbstractStackNode node, ArrayList productions) { - AbstractStackNode[] production = node.getProduction(); - if (production == null) { - return; // The root node does not have a production, so ignore it. - } - - int dot = node.getDot(); - - if (node.isEndNode()) { - IConstructor parentProduction = node.getParentProduction(); - if (ProductionAdapter.isContextFree(parentProduction)){ - productions.add(parentProduction); - - if (ProductionAdapter.isList(parentProduction)) { - return; // Don't follow productions in lists productions, since they are 'cyclic'. - } - } - } - - for (int i = dot + 1; i < production.length; ++i) { - AbstractStackNode currentNode = production[i]; - if (currentNode.isEndNode()) { - IConstructor parentProduction = currentNode.getParentProduction(); - if (ProductionAdapter.isContextFree(parentProduction)) { - productions.add(parentProduction); - } - } - - AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); - if (alternateProductions != null) { - for (int j = alternateProductions.length - 1; j >= 0; --j) { - collectProductions(alternateProductions[j][i], productions); - } - } - } - } -} diff --git a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java index 2c8a5f4bfee..8e46000bb7c 100644 --- a/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java +++ b/src/org/rascalmpl/parser/uptr/recovery/ToTokenRecoverer.java @@ -1,23 +1,19 @@ /******************************************************************************* - * Copyright (c) 2009-2024 NWO-I Centrum Wiskunde & Informatica (CWI) - * All rights reserved. This program and the accompanying materials - * are made available under the terms of the Eclipse Public License v1.0 - * which accompanies this distribution, and is available at + * Copyright (c) 2009-2024 NWO-I Centrum Wiskunde & Informatica (CWI) All rights reserved. This + * program and the accompanying materials are made available under the terms of the Eclipse Public + * License v1.0 which accompanies this distribution, and is available at * http://www.eclipse.org/legal/epl-v10.html * * Contributors: - - * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI - * * Arnold Lankamp - Arnold.Lankamp@cwi.nl -*******************************************************************************/ + * + * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI * Arnold Lankamp - Arnold.Lankamp@cwi.nl + *******************************************************************************/ package org.rascalmpl.parser.uptr.recovery; import java.net.URI; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.function.Consumer; import org.rascalmpl.parser.gtd.ExpectsProvider; import org.rascalmpl.parser.gtd.recovery.IRecoverer; @@ -43,437 +39,439 @@ import org.rascalmpl.values.parsetrees.ProductionAdapter; import io.usethesource.vallang.IConstructor; -import io.usethesource.vallang.IList; -import io.usethesource.vallang.IValue; public class ToTokenRecoverer implements IRecoverer { - private URI uri; - private IdDispenser stackNodeIdDispenser; - private ExpectsProvider expectsProvider; - - public ToTokenRecoverer(URI uri, ExpectsProvider expectsProvider, IdDispenser stackNodeIdDispenser) { - this.uri = uri; - this.expectsProvider = expectsProvider; - this.stackNodeIdDispenser = stackNodeIdDispenser; - } - - @Override - public DoubleArrayList, AbstractNode> reviveStacks(int[] input, - int location, - Stack> unexpandableNodes, - Stack> unmatchableLeafNodes, - DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, - DoubleStack, AbstractNode> filteredNodes) { - - // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to improve error recovery. - - ArrayList> failedNodes = new ArrayList<>(); - collectUnexpandableNodes(unexpandableNodes, failedNodes); - collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); - - return reviveFailedNodes(input, failedNodes); - } - - private DoubleArrayList, AbstractNode> reviveNodes(int[] input, DoubleArrayList, ArrayList> recoveryNodes){ - DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); - - // Sort nodes by start location - recoveryNodes.sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); - - ParseStateVisualizer visualizer = new ParseStateVisualizer("Recovery"); - visualizer.visualizeRecoveryNodes(recoveryNodes); - - for (int i = 0; i recoveryNode = recoveryNodes.getFirst(i); - ArrayList prods = recoveryNodes.getSecond(i); - - int startLocation = recoveryNode.getStartLocation(); - - // Handle every possible continuation associated with the recovery node (there can be more then one because of prefix-sharing). - for (int j = prods.size() - 1; j >= 0; --j) { - IConstructor prod = prods.get(j); - - List> skippingNodes = findSkippingNodes(input, recoveryNode, prod, startLocation); - for (SkippingStackNode skippingNode : skippingNodes) { - AbstractStackNode continuer = new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); - - EdgesSet edges = new EdgesSet<>(1); - edges.add(continuer); - - continuer.setIncomingEdges(edges); - - skippingNode.initEdges(); - skippingNode.addEdges(edges, startLocation); - recoveredNodes.add(skippingNode, skippingNode.getResult()); - } - } - } - - return recoveredNodes; - } - - List> findSkippingNodes(int[] input, AbstractStackNode recoveryNode, IConstructor prod, int startLocation) { - List> nodes = new java.util.ArrayList<>(); - - SkippedNode result; - - // If we are the top-level node, just skip the rest of the input - if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { - result = SkippingStackNode.createResultUntilEndOfInput(uri, input, startLocation); - nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); - return nodes; // No other nodes would be useful - } - - // Try to find whitespace to skip to - // This often creates hopeless recovery attempts, but it might help in some cases. - // Further experimentation should quantify this statement. - /* - result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, dot); - if (result != null) { - nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); - }*/ - - // Find the last token of this production and skip until after that - List endMatchers = findEndMatchers(recoveryNode); - for (InputMatcher endMatcher : endMatchers) { - MatchResult endMatch = endMatcher.findMatch(input, startLocation); - if (endMatch != null) { - result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, endMatch.getEnd()); - nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); - } - } - - // Find the first token of the next production and skip until before that - List nextMatchers = findNextMatchers(recoveryNode); - for (InputMatcher nextMatcher : nextMatchers) { - MatchResult nextMatch = nextMatcher.findMatch(input, startLocation+1); - if (nextMatch != null) { - result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart()); - nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); - } - } - - return nodes; - } - - // Gather matchers for the last token of a production - List findEndMatchersBasedOnProduction(IConstructor prod) { - IList args = (IList) prod.get(1); - if (args.isEmpty()) { - return Collections.emptyList(); - } - - IValue last = args.get(args.length()-1); - if (last instanceof IConstructor) { - return Collections.singletonList(InputMatcher.createMatcher((IConstructor) last)); - } - - return Collections.emptyList(); - } - - // Find matchers for the last token of the current stack node - List findEndMatchers(AbstractStackNode stackNode) { - final List matchers = new java.util.ArrayList<>(); - - AbstractStackNode[] prod = stackNode.getProduction(); - addEndMatchers(prod, prod.length-1, matchers, new HashSet<>()); - - return matchers; - } - - void addEndMatchers(AbstractStackNode[] prod, int dot, List matchers, Set visitedNodes) { - if (prod == null || dot < 0) { - return; - } - - AbstractStackNode last = prod[dot]; - if (visitedNodes.contains(last.getId())) { - return; - } - visitedNodes.add(last.getId()); - - // Future improvement: while (isNullable(last) addEndMatchers(prod, dot-1, matchers); - - last.accept(new StackNodeVisitorAdapter() { - @Override - public Void visit(LiteralStackNode literal) { - matchers.add(new LiteralMatcher(literal.getLiteral())); - return null; - } - - @Override - public Void visit(CaseInsensitiveLiteralStackNode literal) { - matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); - return null; - } - - @Override - public Void visit(NonTerminalStackNode nonTerminal) { - String name = nonTerminal.getName(); - AbstractStackNode[] alternatives = expectsProvider.getExpects(name); - for (AbstractStackNode alternative : alternatives) { - addEndMatchers(alternative.getProduction(), 0, matchers, visitedNodes); - } - return null; - } - }); - } - - void forAllParents(AbstractStackNode stackNode, Consumer> consumer) { - IntegerObjectList> edges = stackNode.getEdges(); + private URI uri; + private IdDispenser stackNodeIdDispenser; + private ExpectsProvider expectsProvider; + + public ToTokenRecoverer(URI uri, ExpectsProvider expectsProvider, IdDispenser stackNodeIdDispenser) { + this.uri = uri; + this.expectsProvider = expectsProvider; + this.stackNodeIdDispenser = stackNodeIdDispenser; + } + + @Override + public DoubleArrayList, AbstractNode> reviveStacks(int[] input, int location, + Stack> unexpandableNodes, + Stack> unmatchableLeafNodes, + DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, + DoubleStack, AbstractNode> filteredNodes) { + + // For now we ignore unmatchable leaf nodes and filtered nodes. At some point we might use those to + // improve error recovery. + + ArrayList> failedNodes = new ArrayList<>(); + collectUnexpandableNodes(unexpandableNodes, failedNodes); + collectUnmatchableMidProductionNodes(location, unmatchableMidProductionNodes, failedNodes); + + return reviveFailedNodes(input, failedNodes); + } + + private DoubleArrayList, AbstractNode> reviveNodes(int[] input, + DoubleArrayList, ArrayList> recoveryNodes) { + DoubleArrayList, AbstractNode> recoveredNodes = new DoubleArrayList<>(); + + // Sort nodes by start location + recoveryNodes + .sort((e1, e2) -> Integer.compare(e2.getLeft().getStartLocation(), e1.getLeft().getStartLocation())); + + ParseStateVisualizer visualizer = new ParseStateVisualizer("Recovery"); + visualizer.visualizeRecoveryNodes(recoveryNodes); + + for (int i = 0; i < recoveryNodes.size(); i++) { + AbstractStackNode recoveryNode = recoveryNodes.getFirst(i); + ArrayList prods = recoveryNodes.getSecond(i); + + int startLocation = recoveryNode.getStartLocation(); + + // Handle every possible continuation associated with the recovery node (there can be more then one + // because of prefix-sharing). + for (int j = prods.size() - 1; j >= 0; --j) { + IConstructor prod = prods.get(j); + + List> skippingNodes = + findSkippingNodes(input, recoveryNode, prod, startLocation); + for (SkippingStackNode skippingNode : skippingNodes) { + AbstractStackNode continuer = + new RecoveryPointStackNode<>(stackNodeIdDispenser.dispenseId(), prod, recoveryNode); + + EdgesSet edges = new EdgesSet<>(1); + edges.add(continuer); + + continuer.setIncomingEdges(edges); + + skippingNode.initEdges(); + skippingNode.addEdges(edges, startLocation); + recoveredNodes.add(skippingNode, skippingNode.getResult()); + } + } + } + + return recoveredNodes; + } + + private List> findSkippingNodes(int[] input, + AbstractStackNode recoveryNode, IConstructor prod, int startLocation) { + List> nodes = new java.util.ArrayList<>(); + + SkippedNode result; + + // If we are the top-level node, just skip the rest of the input + if (!recoveryNode.isEndNode() && isTopLevelProduction(recoveryNode)) { + result = SkippingStackNode.createResultUntilEndOfInput(uri, input, startLocation); + nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); + return nodes; // No other nodes would be useful + } + + // Try to find whitespace to skip to + // This often creates hopeless recovery attempts, but it might help in some cases. + // Further experimentation should quantify this statement. + /* + * result = SkippingStackNode.createResultUntilCharClass(WHITESPACE, input, startLocation, prod, + * dot); if (result != null) { nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), + * prod, result, startLocation)); } + */ + + // Find the last token of this production and skip until after that + List endMatchers = findEndMatchers(recoveryNode); + for (InputMatcher endMatcher : endMatchers) { + MatchResult endMatch = endMatcher.findMatch(input, startLocation); + if (endMatch != null) { + result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, endMatch.getEnd()); + nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); + } + } + + // Find the first token of the next production and skip until before that + List nextMatchers = findNextMatchers(recoveryNode); + for (InputMatcher nextMatcher : nextMatchers) { + MatchResult nextMatch = nextMatcher.findMatch(input, startLocation + 1); + if (nextMatch != null) { + result = SkippingStackNode.createResultUntilChar(uri, input, startLocation, nextMatch.getStart()); + nodes.add(new SkippingStackNode<>(stackNodeIdDispenser.dispenseId(), prod, result, startLocation)); + } + } + + return nodes; + } + + // Find matchers for the last token of the current stack node + private List findEndMatchers(AbstractStackNode stackNode) { + final List matchers = new java.util.ArrayList<>(); + + AbstractStackNode[] prod = stackNode.getProduction(); + addEndMatchers(prod, prod.length - 1, matchers, new HashSet<>()); + + return matchers; + } + + private void addEndMatchers(AbstractStackNode[] prod, int dot, List matchers, + Set visitedNodes) { + if (prod == null || dot < 0 || dot >= prod.length) { + return; + } + + AbstractStackNode last = prod[dot]; + if (visitedNodes.contains(last.getId())) { + return; + } + visitedNodes.add(last.getId()); + + // Future improvement: while (isNullable(last) addEndMatchers(prod, dot-1, matchers); + + last.accept(new StackNodeVisitorAdapter() { + @Override + public Void visit(LiteralStackNode literal) { + matchers.add(new LiteralMatcher(literal.getLiteral())); + return null; + } + + @Override + public Void visit(CaseInsensitiveLiteralStackNode literal) { + matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); + return null; + } + + @Override + public Void visit(NonTerminalStackNode nonTerminal) { + String name = nonTerminal.getName(); + AbstractStackNode[] alternatives = expectsProvider.getExpects(name); + for (AbstractStackNode alternative : alternatives) { + addEndMatchers(alternative.getProduction(), 0, matchers, visitedNodes); + } + return null; + } + }); + } + + private AbstractStackNode getSingleParentStack(AbstractStackNode stackNode) { + if (stackNode == null) { + return null; + } + + IntegerObjectList> edges = stackNode.getEdges(); if (edges != null) { - for (int i = edges.size() - 1; i >= 0; --i) { - EdgesSet edgesList = edges.getValue(i); - if (edgesList != null) { - for (int j = edgesList.size() - 1; j >= 0; --j) { - consumer.accept(edgesList.get(j)); - } - } - } - } - } - - private AbstractStackNode getSingleParentStack(AbstractStackNode stackNode) { - if (stackNode == null) { - return null; - } - - IntegerObjectList> edges = stackNode.getEdges(); - if (edges != null) { - EdgesSet edgesList = edges.getValue(0); - if (edgesList != null) { - return edgesList.get(0); - } - } - - return null; - } - - // Find matchers for the first token after the current stack node - List findNextMatchers(AbstractStackNode stackNode) { - final List matchers = new java.util.ArrayList<>(); - - // Future improvement: use all parents instead of just one - AbstractStackNode parent = getSingleParentStack(stackNode); - if (parent == null) { - return matchers; - } - - addNextMatchers(parent.getProduction(), parent.getDot()+1, matchers, new HashSet<>()); - - return matchers; - } - - private void addNextMatchers(AbstractStackNode[] prod, int dot, List matchers, Set visitedNodes) { - if (prod == null || dot >= prod.length) { - return; - } - - AbstractStackNode next = prod[dot]; - while (next instanceof NonTerminalStackNode && next.getName().startsWith("layouts_")) { - // Look "through" layout for now, this should really be more general and look through any node that can be empty - // When a node can be empty, we should also consider all prefix-shared alternatives. - dot++; - if (dot >= prod.length) { - return; - } - next = prod[dot]; - } - - if (visitedNodes.contains(next.getId())) { - return; - } - visitedNodes.add(next.getId()); - - next.accept(new StackNodeVisitorAdapter() { - @Override - public Void visit(LiteralStackNode literal) { - matchers.add(new LiteralMatcher(literal.getLiteral())); - return null; - } - - @Override - public Void visit(CaseInsensitiveLiteralStackNode literal) { - matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); - return null; - } - - @Override - public Void visit(NonTerminalStackNode nonTerminal) { - String name = nonTerminal.getName(); - AbstractStackNode[] alternatives = expectsProvider.getExpects(name); - for (AbstractStackNode alternative : alternatives) { - addNextMatchers(alternative.getProduction(), 0, matchers, visitedNodes); - } - return null; - } - }); - } - - // Check if a node is a top-level production (i.e., its parent production node has no parents and starts at position -1) - // As this is experimental code, this method is extremely conservative. - // Any sharing will result in returning 'false'. - // We will need to change this strategy in the future to improve error recovery. - private boolean isTopLevelProduction(AbstractStackNode node) { - while (node != null && node.getDot() != 0) { - node = getSinglePredecessor(node); - } - - if (node != null) { - node = getSinglePredecessor(node); - return node != null && node.getStartLocation() == -1; - } - - return false; - } - - private AbstractStackNode getSinglePredecessor(AbstractStackNode node) { - IntegerObjectList> edgeMap = node.getEdges(); - if (edgeMap.size() == 1) { - EdgesSet edges = edgeMap.getValue(0); - if (edges.size() == 1) { - return edges.get(0); - } - } - - return null; - } - - private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, ArrayList> failedNodes) { - DoubleArrayList, ArrayList> recoveryNodes = new DoubleArrayList<>(); - - for (int i = failedNodes.size() - 1; i >= 0; --i) { - findRecoveryNodes(failedNodes.get(i), recoveryNodes); - } - - return reviveNodes(input, recoveryNodes); - } - - private static void collectUnexpandableNodes(Stack> unexpandableNodes, ArrayList> failedNodes) { - for (int i = unexpandableNodes.getSize() - 1; i >= 0; --i) { - failedNodes.add(unexpandableNodes.get(i)); - } - } - - /** - * Make a fresh copy of each unmatchable mid-production node and link in the predecessors of the original node. - * The new copies are added to `failedNodes` - * @param location the location where the failure occurs - * @param unmatchableMidProductionNodes each pair consists of a list of predecessors and a node that failed to match - * @param failedNodes the list to which failed nodes must be added - */ - private static void collectUnmatchableMidProductionNodes(int location, DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, ArrayList> failedNodes){ - for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { - DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); - AbstractStackNode failedNode = unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference updates of the static version - - // Merge the information on the predecessors into the failed node. - for(int j = failedNodePredecessors.size() - 1; j >= 0; --j) { - AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); - AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); - failedNode.updateNode(predecessor, predecessorResult); - } - - failedNodes.add(failedNode); - } - } - - /** - * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. - */ - private void findRecoveryNodes(AbstractStackNode failer, DoubleArrayList, ArrayList> recoveryNodes) { - ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap<>(); - Stack> todo = new Stack<>(); - - todo.push(failer); - - while (!todo.isEmpty()) { - AbstractStackNode node = todo.pop(); - - if (visited.contains(node)) { - continue; // Don't follow cycles - } - - visited.put(node, 0); - - ArrayList recoveryProductions = new ArrayList<>(); - collectProductions(node, recoveryProductions); - if (recoveryProductions.size() > 0) { - addRecoveryNode(node, recoveryProductions, recoveryNodes); - } - - IntegerObjectList> edges = node.getEdges(); - - for (int i = edges.size() - 1; i >= 0; --i) { // Rewind - EdgesSet edgesList = edges.getValue(i); - - if (edgesList != null) { - for (int j = edgesList.size() - 1; j >= 0; --j) { - AbstractStackNode parent = edgesList.get(j); - - todo.push(parent); - } - } - } - } - } - - // Only add recovery nodes that are not already present. - private void addRecoveryNode(AbstractStackNode node, ArrayList productions, DoubleArrayList, ArrayList> recoveryNodes) { - for (int i=0; i prods = recoveryNodes.getSecond(i); - if (prods.size() == productions.size()) { - boolean equal = true; - for (int j=0; equal && j node, ArrayList productions) { - AbstractStackNode[] production = node.getProduction(); - if (production == null) { - return; // The root node does not have a production, so ignore it. - } - - if (node.isEndNode()) { - IConstructor parentProduction = node.getParentProduction(); - if (ProductionAdapter.isContextFree(parentProduction)){ - productions.add(parentProduction); - - if (ProductionAdapter.isList(parentProduction)) { - return; // Don't follow productions in lists productions, since they are 'cyclic'. - } - } - } - - int dot = node.getDot(); - for (int i = dot + 1; i < production.length; ++i) { - AbstractStackNode currentNode = production[i]; - if (currentNode.isEndNode()) { - IConstructor parentProduction = currentNode.getParentProduction(); - if (ProductionAdapter.isContextFree(parentProduction)) { - productions.add(parentProduction); - } - } - - AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); - if (alternateProductions != null) { - for (int j = alternateProductions.length - 1; j >= 0; --j) { - collectProductions(alternateProductions[j][i], productions); - } - } - } - } + EdgesSet edgesList = edges.getValue(0); + if (edgesList != null) { + return edgesList.get(0); + } + } + + return null; + } + + // Find matchers for the first token after the current stack node + private List findNextMatchers(AbstractStackNode stackNode) { + final List matchers = new java.util.ArrayList<>(); + + // Future improvement: use all parents instead of just one + AbstractStackNode parent = getSingleParentStack(stackNode); + if (parent == null) { + return matchers; + } + + addNextMatchers(parent.getProduction(), parent.getDot() + 1, matchers, new HashSet<>()); + + return matchers; + } + + private void addNextMatchers(AbstractStackNode[] prod, int dot, List matchers, + Set visitedNodes) { + if (prod == null || dot < 0 || dot >= prod.length) { + return; + } + + AbstractStackNode next = prod[dot]; + while (next instanceof NonTerminalStackNode && next.getName().startsWith("layouts_")) { + // Look "through" layout for now, this should really be more general and look through any node that + // can be empty + // When a node can be empty, we should also consider all prefix-shared alternatives. + dot++; + if (dot >= prod.length) { + return; + } + next = prod[dot]; + } + + if (visitedNodes.contains(next.getId())) { + return; + } + visitedNodes.add(next.getId()); + + next.accept(new StackNodeVisitorAdapter() { + @Override + public Void visit(LiteralStackNode literal) { + matchers.add(new LiteralMatcher(literal.getLiteral())); + return null; + } + + + @Override + + public Void visit(CaseInsensitiveLiteralStackNode literal) { + matchers.add(new CaseInsensitiveLiteralMatcher(literal.getLiteral())); + return null; + } + + @Override + + public Void visit(NonTerminalStackNode nonTerminal) { + String name = nonTerminal.getName(); + AbstractStackNode[] alternatives = expectsProvider.getExpects(name); + for (AbstractStackNode alternative : alternatives) { + addNextMatchers(alternative.getProduction(), 0, matchers, visitedNodes); + } + + return null; + } + + }); + } + + // Check if a node is a top-level production (i.e., its parent production node has no parents and + // starts at position -1) + // As this is experimental code, this method is extremely conservative. + // Any sharing will result in returning 'false'. + // We will need to change this strategy in the future to improve error recovery. + private boolean isTopLevelProduction(AbstractStackNode node) { + + while (node != null && node.getDot() != 0) { + node = getSinglePredecessor(node); + } + + if (node != null) { + node = getSinglePredecessor(node); + return node != null && node.getStartLocation() == -1; + } + + return false; + } + + private AbstractStackNode getSinglePredecessor(AbstractStackNode node) { + IntegerObjectList> edgeMap = node.getEdges(); + if (edgeMap.size() == 1) { + EdgesSet edges = edgeMap.getValue(0); + if (edges.size() == 1) { + return edges.get(0); + } + } + + return null; + } + + + private DoubleArrayList, AbstractNode> reviveFailedNodes(int[] input, + ArrayList> failedNodes) { + DoubleArrayList, ArrayList> recoveryNodes = + new DoubleArrayList<>(); + + for (int i = failedNodes.size() - 1; i >= 0; --i) { + findRecoveryNodes(failedNodes.get(i), recoveryNodes); + } + + return reviveNodes(input, recoveryNodes); + } + + private static void collectUnexpandableNodes(Stack> unexpandableNodes, + ArrayList> failedNodes) { + for (int i = unexpandableNodes.getSize() - 1; i >= 0; --i) { + failedNodes.add(unexpandableNodes.get(i)); + } + } + + /** + * Make a fresh copy of each unmatchable mid-production node and link in the predecessors of the + * original node. The new copies are added to `failedNodes` + * + * @param location the location where the failure occurs + * @param unmatchableMidProductionNodes each pair consists of a list of predecessors and a node that + * failed to match + * @param failedNodes the list to which failed nodes must be added + */ + private static void collectUnmatchableMidProductionNodes(int location, + DoubleStack, AbstractNode>, AbstractStackNode> unmatchableMidProductionNodes, + ArrayList> failedNodes) { + for (int i = unmatchableMidProductionNodes.getSize() - 1; i >= 0; --i) { + DoubleArrayList, AbstractNode> failedNodePredecessors = + unmatchableMidProductionNodes.getFirst(i); + AbstractStackNode failedNode = + unmatchableMidProductionNodes.getSecond(i).getCleanCopy(location); // Clone it to prevent by-reference + // updates of the static version + + // Merge the information on the predecessors into the failed node. + for (int j = failedNodePredecessors.size() - 1; j >= 0; --j) { + AbstractStackNode predecessor = failedNodePredecessors.getFirst(j); + AbstractNode predecessorResult = failedNodePredecessors.getSecond(j); + failedNode.updateNode(predecessor, predecessorResult); + } + + failedNodes.add(failedNode); + } + } + + /** + * Travels up the parse graph in an attempt to find the closest recoverable parent nodes. + */ + private void findRecoveryNodes(AbstractStackNode failer, + DoubleArrayList, ArrayList> recoveryNodes) { + ObjectKeyedIntegerMap> visited = new ObjectKeyedIntegerMap<>(); + Stack> todo = new Stack<>(); + + todo.push(failer); + + while (!todo.isEmpty()) { + AbstractStackNode node = todo.pop(); + + if (visited.contains(node)) { + continue; // Don't follow cycles + } + + visited.put(node, 0); + + ArrayList recoveryProductions = new ArrayList<>(); + collectProductions(node, recoveryProductions); + if (recoveryProductions.size() > 0) { + addRecoveryNode(node, recoveryProductions, recoveryNodes); + } + + IntegerObjectList> edges = node.getEdges(); + + for (int i = edges.size() - 1; i >= 0; --i) { // Rewind + EdgesSet edgesList = edges.getValue(i); + + if (edgesList != null) { + for (int j = edgesList.size() - 1; j >= 0; --j) { + AbstractStackNode parent = edgesList.get(j); + + todo.push(parent); + } + } + } + } + } + + // Only add recovery nodes that are not already present. + private void addRecoveryNode(AbstractStackNode node, ArrayList productions, + DoubleArrayList, ArrayList> recoveryNodes) { + for (int i = 0; i < recoveryNodes.size(); i++) { + if (recoveryNodes.getFirst(i) == node && equalProductions(productions, recoveryNodes.getSecond(i))) { + return; + } + } + + recoveryNodes.add(node, productions); + } + + private boolean equalProductions(ArrayList prods1, ArrayList prods2) { + if (prods1.size() != prods2.size()) { + return false; + } + + for (int j = 0; j < prods1.size(); j++) { + if (prods1.get(j) != prods2.get(j)) { + return false; + } + } + + return true; + } + + // Gathers all productions that are marked for recovery (the given node can be part of a prefix + // shared production) + private void collectProductions(AbstractStackNode node, ArrayList productions) { + AbstractStackNode[] production = node.getProduction(); + if (production == null) { + return; // The root node does not have a production, so ignore it. + } + + if (node.isEndNode()) { + IConstructor parentProduction = node.getParentProduction(); + if (ProductionAdapter.isContextFree(parentProduction)) { + productions.add(parentProduction); + + if (ProductionAdapter.isList(parentProduction)) { + return; // Don't follow productions in lists productions, since they are 'cyclic'. + } + } + } + + int dot = node.getDot(); + for (int i = dot + 1; i < production.length; ++i) { + AbstractStackNode currentNode = production[i]; + if (currentNode.isEndNode()) { + IConstructor parentProduction = currentNode.getParentProduction(); + if (ProductionAdapter.isContextFree(parentProduction)) { + productions.add(parentProduction); + } + } + + AbstractStackNode[][] alternateProductions = currentNode.getAlternateProductions(); + if (alternateProductions != null) { + for (int j = alternateProductions.length - 1; j >= 0; --j) { + collectProductions(alternateProductions[j][i], productions); + } + } + } + } } diff --git a/src/org/rascalmpl/parser/util/DebugUtil.java b/src/org/rascalmpl/parser/util/DebugUtil.java index b58ad8e876e..0f31c246fed 100644 --- a/src/org/rascalmpl/parser/util/DebugUtil.java +++ b/src/org/rascalmpl/parser/util/DebugUtil.java @@ -30,7 +30,7 @@ public static String prodToString(IConstructor prod) { StringBuilder builder = new StringBuilder("'"); IConstructor sort = (IConstructor) prod.get(0); - builder.append(stripQuotes(String.valueOf(sort.get(0)))); + builder.append(quotedStringToPlain(String.valueOf(sort.get(0)))); builder.append(" ->"); @@ -39,7 +39,7 @@ public static String prodToString(IConstructor prod) { for (IValue child : children) { builder.append(" "); IConstructor conChild = (IConstructor) child; - builder.append(stripQuotes(String.valueOf((conChild).get(0)))); + builder.append(quotedStringToPlain(String.valueOf((conChild).get(0)))); } } else { builder.append(" "); @@ -51,7 +51,7 @@ public static String prodToString(IConstructor prod) { return builder.toString(); } - private static String stripQuotes(String s) { + private static String quotedStringToPlain(String s) { if (s.charAt(0) == '"' && s.charAt(s.length()-1) == '"') { return s.substring(1, s.length()-1).replace("\\", ""); } @@ -59,7 +59,7 @@ private static String stripQuotes(String s) { return s; } - public static void opportunityToBreak() { + public static void opportunityToBreak() { // Nop method that allows breakpoints to be set at the call site even if originally there is no code to break on } diff --git a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java index faf6e762a0b..8fc1e230dbb 100644 --- a/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java +++ b/src/org/rascalmpl/util/visualize/ParseStateVisualizer.java @@ -107,10 +107,10 @@ public void run() { } } - private String name; - private File basePath; - private File frameDir; - private Map stackNodeNodes; + private final String name; + private final File basePath; + private final File frameDir; + private final Map stackNodeNodes; private DotGraph graph; private int frame; @@ -274,14 +274,14 @@ private

DotNode addStack(DotGraph graph, AbstractStackNode

stackNode) { IntegerObjectList> edges = stackNode.getEdges(); if (edges != null) { - for (int i = edges.size() - 1; i >= 0; --i) { - EdgesSet

edgesList = edges.getValue(i); - if (edgesList != null) { - for (int j = edgesList.size() - 1; j >= 0; --j) { - AbstractStackNode

parentStackNode = edgesList.get(j); + for (int i = edges.size() - 1; i >= 0; --i) { + EdgesSet

edgesList = edges.getValue(i); + if (edgesList != null) { + for (int j = edgesList.size() - 1; j >= 0; --j) { + AbstractStackNode

parentStackNode = edgesList.get(j); DotNode parentDotNode = addStack(graph, parentStackNode); graph.addEdge(node.getId(), parentDotNode.getId()); - } + } } } } @@ -382,19 +382,14 @@ private void addParserNode(DotGraph graph, AbstractNode parserNode) { } private void enrichCharNode(DotNode dotNode, CharNode charNode) { - String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); int c = charNode.getCharacter(); - label += "\nchar=" + c + "('" + (char) c + "')"; + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL) + "\nchar=" + c + "('" + (char) c + "')"; dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); } private void enrichLiteralNode(DotNode dotNode, LiteralNode literalNode) { - String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL); int[] content = literalNode.getContent(); - label += " \"" + UnicodeConverter.unicodeArrayToString(content) + "\""; - /* Maybe include production? - label += "\nprod=" + DebugUtil.prodToString((IConstructor)literalNode.getProduction()); - */ + String label = dotNode.getAttributeValue(DotAttribute.ATTR_LABEL) + " \"" + UnicodeConverter.unicodeArrayToString(content) + "\""; dotNode.setAttribute(DotAttribute.ATTR_LABEL, label); } @@ -535,8 +530,8 @@ private void addUnmatchableMidProductionNodes(SGTDBF parser, graph.addRecordNode(failureId, failureRecord); graph.addEdge(new NodeId(UNMATCHABLE_MID_PRODUCTION_NODES_ID, String.valueOf(i)), failureId); - DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); - AbstractStackNode

failedNode = unmatchableMidProductionNodes.getSecond(i); + DoubleArrayList, AbstractNode> failedNodePredecessors = unmatchableMidProductionNodes.getFirst(i); + AbstractStackNode

failedNode = unmatchableMidProductionNodes.getSecond(i); DotNode node = addStack(graph, failedNode); NodeId predecessorsId = new NodeId("unmatchable-mid-production-predecessors-" + i); diff --git a/src/org/rascalmpl/util/visualize/dot/CompassPoint.java b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java index 60ef4da2295..47e1c720ac1 100644 --- a/src/org/rascalmpl/util/visualize/dot/CompassPoint.java +++ b/src/org/rascalmpl/util/visualize/dot/CompassPoint.java @@ -14,6 +14,8 @@ package org.rascalmpl.util.visualize.dot; + + // Note: C means Center, X means unspecified. public enum CompassPoint { N, NE, E, SE, S, SW, W, NW, C, X; } diff --git a/src/org/rascalmpl/util/visualize/dot/DotAttribute.java b/src/org/rascalmpl/util/visualize/dot/DotAttribute.java index 052f8d33c94..ba8f4b438c8 100644 --- a/src/org/rascalmpl/util/visualize/dot/DotAttribute.java +++ b/src/org/rascalmpl/util/visualize/dot/DotAttribute.java @@ -74,7 +74,7 @@ public static DotAttribute createRecordLabel(List elements) { if (first) { first = false; } else { - value.append("| "); + value.append(" | "); } value.append('<'); diff --git a/src/org/rascalmpl/util/visualize/dot/NodeId.java b/src/org/rascalmpl/util/visualize/dot/NodeId.java index 180e15e7e54..8b4cbf5cb9f 100644 --- a/src/org/rascalmpl/util/visualize/dot/NodeId.java +++ b/src/org/rascalmpl/util/visualize/dot/NodeId.java @@ -15,6 +15,7 @@ package org.rascalmpl.util.visualize.dot; import java.io.PrintWriter; +import java.util.Objects; public class NodeId { public static void writeId(PrintWriter stream, String id) { @@ -104,21 +105,10 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; NodeId other = (NodeId) obj; - if (id == null) { - if (other.id != null) - return false; - } - else if (!id.equals(other.id)) - return false; - if (portId == null) { - if (other.portId != null) - return false; - } - else if (!portId.equals(other.portId)) - return false; - if (direction != other.direction) - return false; - return true; + + return Objects.equals(id, other.id) + && Objects.equals(portId, other.portId) + && direction == other.direction; } @Override diff --git a/src/org/rascalmpl/util/visualize/replay.html b/src/org/rascalmpl/util/visualize/replay.html index 891778454b5..23e9f700ed0 100644 --- a/src/org/rascalmpl/util/visualize/replay.html +++ b/src/org/rascalmpl/util/visualize/replay.html @@ -11,7 +11,7 @@ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --> - +