Error recovery for iggy

src/org/iguana/grammar/Grammar.java

@@ -279,7 +279,10 @@ public int hashCode() {
     public String toString() {
         StringBuilder sb = new StringBuilder();
         for (Rule rule : rules) {
-            sb.append(rule).append("\n");
+            sb.append(rule)
+              .append("  ;")
+              .append("\n");
+            sb.append("\n");
         }
         sb.delete(sb.length() - 1, sb.length());
         return sb.toString();

src/org/iguana/grammar/condition/Conditions.java

@@ -38,43 +38,46 @@
 
 public interface Conditions {
 
+    <T extends Result> boolean execute(
+            Input input,
+            BodyGrammarSlot slot,
+            GSSNode<T> u,
+            int leftExtent,
+            int rightExtent,
+            T result,
+            IEvaluatorContext ctx,
+            IguanaRuntime<T> runtime);
+
     default <T extends Result> boolean execute(
             Input input,
             BodyGrammarSlot slot,
             GSSNode<T> u,
             int leftExtent,
             int rightExtent,
+            T result,
             IguanaRuntime<T> runtime) {
-        return execute(input, slot, u, leftExtent, rightExtent, GLLEvaluator.getDefaultEvaluatorContext(), runtime);
+        return execute(input, slot, u, leftExtent, rightExtent, result, GLLEvaluator.getDefaultEvaluatorContext(),
+                       runtime);
     }
 
-    <T extends Result> boolean execute(
-        Input input,
-        BodyGrammarSlot slot,
-        GSSNode<T> u,
-        int leftExtent,
-        int rightExtent,
-        IEvaluatorContext ctx,
-        IguanaRuntime<T> runtime
-    );
-
     default <T extends Result> boolean execute(
             Input input,
             BodyGrammarSlot slot,
             GSSNode<T> u,
             int inputIndex,
+            T result,
+            IEvaluatorContext ctx,
             IguanaRuntime<T> runtime) {
-        return execute(input, slot, u, inputIndex, GLLEvaluator.getDefaultEvaluatorContext(), runtime);
+        return execute(input, slot, u, inputIndex, inputIndex, result, ctx, runtime);
     }
 
     default <T extends Result> boolean execute(
             Input input,
             BodyGrammarSlot slot,
             GSSNode<T> u,
             int inputIndex,
-            IEvaluatorContext ctx,
+            T result,
             IguanaRuntime<T> runtime) {
-        return execute(input, slot, u, inputIndex, inputIndex, ctx, runtime);
+        return execute(input, slot, u, inputIndex, result, GLLEvaluator.getDefaultEvaluatorContext(), runtime);
     }
-
 }

src/org/iguana/grammar/condition/ConditionsFactory.java

@@ -54,6 +54,7 @@ public <T extends Result> boolean execute(
                 GSSNode<T> u,
                 int leftExtent,
                 int rightExtent,
+                T result,
                 IEvaluatorContext ctx,
                 IguanaRuntime<T> runtime) {
             return false;
@@ -90,12 +91,14 @@ public <T extends Result> boolean execute(
                         GSSNode<T> gssNode,
                         int lefExtent,
                         int rightExtent,
+                        T result,
                         IEvaluatorContext ctx,
                         IguanaRuntime<T> runtime) {
                     for (int j = 0; j < actions.size(); j++) {
                         SlotAction slotAction = actions.get(j);
                         if (slotAction.execute(input, slot, gssNode, lefExtent, rightExtent, ctx)) {
-                            runtime.recordParseError(rightExtent, input, slot, gssNode, slotAction.toString());
+                            runtime.recordParseError(rightExtent, input, slot, gssNode, result,
+                                                     ctx.getEmptyEnvironment(), slotAction.toString());
                             return true;
                         }
                     }
@@ -119,12 +122,14 @@ public <T extends Result> boolean execute(
                     GSSNode<T> gssNode,
                     int leftExtent,
                     int rightExtent,
+                    T result,
                     IEvaluatorContext ctx,
                     IguanaRuntime<T> runtime) {
                 for (int j = 0; j < actions.size(); j++) {
                     SlotAction slotAction = actions.get(j);
                     if (slotAction.execute(input, slot, gssNode, leftExtent, rightExtent, ctx)) {
-                        runtime.recordParseError(rightExtent, input, slot, gssNode, slotAction.toString());
+                        runtime.recordParseError(rightExtent, input, slot, gssNode, result, ctx.getEnvironment(),
+                                                 slotAction.toString());
                         return true;
                     }
                 }

src/org/iguana/grammar/operations/FirstFollowSets.java

@@ -37,6 +37,7 @@
 import org.iguana.grammar.symbol.Return;
 import org.iguana.grammar.symbol.Symbol;
 import org.iguana.grammar.symbol.Terminal;
+import org.iguana.regex.Char;
 import org.iguana.regex.CharRange;
 import org.iguana.regex.EOF;
 import org.iguana.regex.Epsilon;
@@ -110,8 +111,9 @@ public Set<Nonterminal> getNullableNonterminals() {
 
     public Set<CharRange> getFirstSet(Nonterminal nonterminal) {
         Set<CharRange> firstSet = new HashSet<>(firstSets.get(nonterminal));
-        if (isNullable(nonterminal))
+        if (isNullable(nonterminal)) {
             firstSet.addAll(Epsilon.getInstance().getFirstSet());
+        }
         return firstSet;
     }
 
@@ -384,7 +386,8 @@ private static class FirstSymbolVisitor extends AbstractGrammarGraphSymbolVisito
 
         @Override
         public Set<CharRange> visit(Error error) {
-            return Collections.emptySet();
+            return Collections.singleton(CharRange.in(Char.MIN_VALUE, Char.MAX_VALUE));
+//            return Collections.emptySet();
         }
 
         @Override

src/org/iguana/grammar/runtime/RuntimeGrammar.java

@@ -240,6 +240,7 @@ public Builder(RuntimeGrammar grammar) {
             ebnfRights = new HashMap<>(grammar.ebnfRights);
             startSymbols = new ArrayList<>(grammar.startSymbols);
             regularExpressionDefinitions = new HashMap<>(grammar.getRegularExpressionDefinitions());
+            literals = new HashMap<>(grammar.literals);
             globals = new HashMap<>(grammar.globals);
             name = grammar.name;
         }

src/org/iguana/grammar/slot/BodyGrammarSlot.java

@@ -43,7 +43,7 @@
 import java.util.Map;
 import java.util.Set;
 
-public class BodyGrammarSlot implements GrammarSlot {
+public class BodyGrammarSlot implements GrammarSlot, FollowTest {
 
     protected final Position position;
 
@@ -266,4 +266,9 @@ public boolean isEnd() {
     public FollowTest getFollowTest() {
         return followTest;
     }
+
+    @Override
+    public boolean test(int v) {
+        return followTest.test(v);
+    }
 }

src/org/iguana/grammar/slot/EndGrammarSlot.java

@@ -34,6 +34,7 @@
 import org.iguana.gss.GSSNode;
 import org.iguana.parser.IguanaRuntime;
 import org.iguana.result.Result;
+import org.iguana.sppf.ErrorNode;
 import org.iguana.utils.input.Input;
 
 import java.util.Set;
@@ -102,9 +103,8 @@ public <T extends Result> void execute(
         if (followTest.test(nextChar)) {
             u.pop(input, this, result, value, runtime);
         } else {
-            runtime.recordParseError(rightExtent, input, this, u,
+            runtime.recordParseError(rightExtent, input, this, u, result, runtime.getEnvironment(),
                     "Expected " + followTest + " but was " + (char) nextChar);
         }
     }
-
 }

src/org/iguana/grammar/slot/EpsilonGrammarSlot.java

@@ -74,7 +74,8 @@ public <T extends Result> void execute(
         if (followTest.test(nextChar)) {
             u.pop(input, this, epsilonSlot.getResult(input, i, runtime), value, runtime);
         } else {
-            runtime.recordParseError(i, input, this, u, "Expected " + followTest + " but was " + (char) nextChar);
+            String description = "Expected " + followTest + " but was " + (char) nextChar;
+            runtime.recordParseError(i, input, this, u, result, runtime.getEnvironment(), description);
         }
     }
 

src/org/iguana/grammar/slot/EpsilonTransition.java

@@ -109,22 +109,22 @@ public <T extends Result> void execute(
         switch (type) {
 
             case DUMMY:
-                if (conditions.execute(input, origin, u, i, runtime.getEvaluatorContext(), runtime))
+                if (conditions.execute(input, origin, u, i, result, runtime.getEvaluatorContext(), runtime))
                     return;
                 break;
 
             case CLEAR_LABEL: // TODO: Decide if this case is needed
                 break;
 
             case OPEN:
-                if (conditions.execute(input, origin, u, i, runtime.getEvaluatorContext(), runtime))
+                if (conditions.execute(input, origin, u, i, result, runtime.getEvaluatorContext(), runtime))
                     return;
                 runtime.getEvaluatorContext().pushEnvironment();
                 break;
 
             case CLOSE:
                 runtime.getEvaluatorContext().popEnvironment();
-                if (conditions.execute(input, origin, u, i, runtime.getEvaluatorContext(), runtime))
+                if (conditions.execute(input, origin, u, i, result, runtime.getEvaluatorContext(), runtime))
                     return;
                 break;
 
@@ -133,7 +133,7 @@ public <T extends Result> void execute(
                 runtime.getEvaluatorContext().declareVariable(
                         String.format(Expression.LeftExtent.format, label), Tuple.of(i, -1));
 
-                if (conditions.execute(input, origin, u, i, runtime.getEvaluatorContext(), runtime))
+                if (conditions.execute(input, origin, u, i, result, runtime.getEvaluatorContext(), runtime))
                     return;
                 break;
 
@@ -143,14 +143,14 @@ public <T extends Result> void execute(
 
                 Integer lhs;
                 if (!(value instanceof Tuple)) {
-                    lhs = (Integer) ((Tuple<?, ?>) value).getFirst();
+                    lhs = (Integer) ((Tuple) value).getFirst();
                 } else {
                     throw new UnexpectedRuntimeTypeException(AST.var(label));
                 }
 
                 runtime.getEvaluatorContext().storeVariable(label, Tuple.of(lhs, i));
 
-                if (conditions.execute(input, origin, u, i, runtime.getEvaluatorContext(), runtime))
+                if (conditions.execute(input, origin, u, i, result, runtime.getEvaluatorContext(), runtime))
                     return;
                 break;
         }

src/org/iguana/grammar/slot/ErrorTransition.java

@@ -1,11 +1,19 @@
 package org.iguana.grammar.slot;
 
 import org.iguana.datadependent.env.Environment;
+import org.iguana.grammar.symbol.Terminal;
 import org.iguana.gss.GSSNode;
 import org.iguana.parser.IguanaRuntime;
+import org.iguana.regex.EOF;
+import org.iguana.regex.IguanaTokenizer;
+import org.iguana.regex.Token;
 import org.iguana.result.Result;
+import org.iguana.utils.collections.CollectionsUtil;
 import org.iguana.utils.input.Input;
 
+import java.util.ArrayList;
+import java.util.List;
+
 public class ErrorTransition extends AbstractTransition {
 
     public ErrorTransition(BodyGrammarSlot origin, BodyGrammarSlot dest) {
@@ -24,17 +32,33 @@ public <T extends Result> void handleError(
             Environment env,
             IguanaRuntime<T> runtime) {
         int rightExtent = result.isDummy() ? u.getInputIndex() : result.getRightExtent();
-        int i = rightExtent;
-        while (i < input.length() && !dest.testFollow(input.charAt(i))) {
-            i++;
+        IguanaTokenizer tokenizer = runtime.getTokenizer();
+        tokenizer.prepare(input, rightExtent);
+
+        List<Token> errorTokens = new ArrayList<>();
+        for (Token token = tokenizer.nextToken(dest.getFollowTest()); token != null && token.getRegularExpression() != EOF.getInstance(); token = tokenizer.nextToken(dest.getFollowTest())) {
+            errorTokens.add(token);
         }
-        if (i < input.length()) {
-            T cr = runtime.getResultOps().error(this.dest, rightExtent, i);
-            T n = dest.isFirst() ? cr : runtime.getResultOps().merge(null, result, cr, dest);
-            dest.execute(input, u, n, env, runtime);
-        } else {
+        if (errorTokens.isEmpty()) {
             System.out.println("Warning: could not recover from the parse error: " + origin);
+            return;
         }
+        T cr;
+        if (errorTokens.size() == 1) {
+            Token token = errorTokens.get(0);
+            cr = runtime.getResultOps().error(this.dest, token.getStart(), token.getEnd());
+        } else {
+            List<T> children = new ArrayList<>();
+            for (Token errorToken : errorTokens) {
+                Terminal terminal = Terminal.from(errorToken.getRegularExpression());
+                children.add(runtime.getResultOps().base(terminal, errorToken.getStart(), errorToken.getEnd()));
+            }
+            int start = CollectionsUtil.first(errorTokens).getStart();
+            int end = CollectionsUtil.last(errorTokens).getEnd();
+            cr = runtime.getResultOps().error(this.dest, start, end, children);
+        }
+        T n = dest.isFirst() ? cr : runtime.getResultOps().merge(null, result, cr, dest);
+        dest.execute(input, u, n, env, runtime);
     }
 
     @Override
@@ -44,6 +68,13 @@ public <T extends Result> void execute(
             T result,
             Environment env,
             IguanaRuntime<T> runtime) {
-        dest.execute(input, u, result, env, runtime);
+        int inputIndex = result.getRightExtent();
+        // Error transitions do not match any input, therefore, it is important to do a follow check
+        // before moving the destination slot to catch the error before the error slot.
+        if (dest.getFollowTest().test(input.charAt(inputIndex))) {
+            dest.execute(input, u, result, env, runtime);
+        } else {
+            runtime.recordParseError(inputIndex, input, origin, u, result, env, "Expected " + dest.getFollowTest().toString() + " but seen " + input.charAt(inputIndex));
+        }
     }
 }

src/org/iguana/grammar/slot/GrammarSlotUtil.java

@@ -0,0 +1,17 @@
+package org.iguana.grammar.slot;
+
+public class GrammarSlotUtil {
+    public static EndGrammarSlot getEndSlot(BodyGrammarSlot slot) {
+        BodyGrammarSlot curr = slot;
+        BodyGrammarSlot end = slot;
+        while (curr != null) {
+            end = curr;
+            if (curr.getOutTransition() == null) {
+                curr = null;
+            } else {
+                curr = curr.getOutTransition().destination();
+            }
+        }
+        return (EndGrammarSlot) end;
+    }
+}

src/org/iguana/grammar/slot/NonterminalGrammarSlot.java

@@ -173,9 +173,9 @@ public <T extends Result> void create(
         }
 
         if (gssNode == null) {
-
             List<BodyGrammarSlot> firstSlots = getFirstSlots(input.charAt(i));
             if (firstSlots == null || firstSlots.isEmpty()) {
+                runtime.recordParseError(i, input, returnSlot, u, result, env, "Prediction failed");
                 return;
             }
 
@@ -205,8 +205,8 @@ public <T extends Result> void create(
                             String.format(Expression.LeftExtent.format, slot.getLabel()), i);
 
                 int inputIndex = result.isDummy() ? gssNode.getInputIndex() : result.getRightExtent();
-                if (!slot.getConditions().execute(input, returnSlot, gssNode, inputIndex, runtime.getEvaluatorContext(),
-                                                  runtime)) {
+                if (!slot.getConditions().execute(input, returnSlot, gssNode, inputIndex, result,
+                                                  runtime.getEvaluatorContext(), runtime)) {
                     runtime.scheduleDescriptor(slot, gssNode, runtime.getResultOps().dummy(), runtime.getEnvironment());
                 }
 
@@ -221,5 +221,4 @@ public <T extends Result> void create(
             gssNode.addGSSEdge(input, returnSlot, i, u, result, env, runtime);
         }
     }
-
 }

src/org/iguana/grammar/slot/NonterminalTransition.java

@@ -85,7 +85,7 @@ public <T extends Result> void execute(
 
         runtime.setEnvironment(env);
 
-        if (preConditions.execute(input, origin, u, i, runtime.getEvaluatorContext(), runtime))
+        if (preConditions.execute(input, origin, u, i, result, runtime.getEvaluatorContext(), runtime))
             return;
 
         nonterminal.create(input, dest, u, result, arguments, runtime.getEnvironment(), runtime);

src/org/iguana/grammar/slot/TerminalGrammarSlot.java

@@ -64,7 +64,7 @@ public <T extends Result> T getResult(Input input, int i, IguanaRuntime<T> runti
             if (length < 0) {
                 terminalNodes.put(i, failure);
             } else {
-                terminalNode = runtime.getResultOps().base(this, i, i + length);
+                terminalNode = runtime.getResultOps().base(this.terminal, i, i + length);
                 terminalNodes.put(i, terminalNode);
             }
         }