[Hydra] Improve error message in parse_overrides

hydra/core/override_parser/overrides_parser.py

@@ -2,11 +2,6 @@
 import sys
 from typing import Any, List, Optional
 
-from omegaconf.vendor.antlr4.error.Errors import (
-    LexerNoViableAltException,
-    RecognitionException,
-)
-
 from hydra._internal.grammar import grammar_functions
 from hydra._internal.grammar.functions import Functions
 from hydra.core.config_loader import ConfigLoader
@@ -17,6 +12,11 @@
 from hydra.core.override_parser.types import Override
 from hydra.errors import HydraException, OverrideParseException
 
+from omegaconf.vendor.antlr4.error.Errors import (
+    LexerNoViableAltException,
+    RecognitionException,
+)
+
 try:
     from hydra.grammar.gen.OverrideLexer import (  # type: ignore[attr-defined]
         CommonTokenStream,
@@ -80,7 +80,7 @@ def parse_override(self, s: str) -> Override:
 
     def parse_overrides(self, overrides: List[str]) -> List[Override]:
         ret: List[Override] = []
-        for override in overrides:
+        for idx, override in enumerate(overrides):
             try:
                 parsed = self.parse_rule(override, "override")
             except HydraException as e:
@@ -98,7 +98,8 @@ def parse_overrides(self, overrides: List[str]) -> List[Override]:
                     msg = f"Error parsing override '{override}'" f"\n{e}"
                 raise OverrideParseException(
                     override=override,
-                    message=f"{msg}"
+                    message=f"Error when parsing index: {idx}, string: {override} out of {overrides}."
+                    f"\n{msg}"
                     f"\nSee https://hydra.cc/docs/1.2/advanced/override_grammar/basic for details",
                 ) from e.__cause__
             assert isinstance(parsed, Override)

tests/test_hydra_cli_errors.py

@@ -3,56 +3,63 @@
 from pathlib import Path
 from typing import Any
 
-from pytest import mark, param
-
 from hydra.test_utils.test_utils import (
     chdir_hydra_root,
     normalize_newlines,
     run_with_error,
 )
 
+from pytest import mark, param
+
 chdir_hydra_root()
 
 
 @mark.parametrize(
-    "override,expected",
+    "override,expected_substring_1,expected_substring_2",
     [
         param(
             "+key=int(",
+            "Error when parsing index: 1, string: +key=int( out of [",
             "no viable alternative at input 'int('",
             id="parse_error_in_function",
         ),
         param(
             "+key=sort()",
+            """Error when parsing index: 1, string: +key=sort() out of [""",
             """Error parsing override '+key=sort()'
 ValueError while evaluating 'sort()': empty sort input""",
             id="empty_sort",
         ),
         param(
             "key=sort(interval(1,10))",
+            """Error when parsing index: 1, string: key=sort(interval(1,10)) out of [""",
             """Error parsing override 'key=sort(interval(1,10))'
 TypeError while evaluating 'sort(interval(1,10))': mismatch type argument args[0]""",
             id="sort_interval",
         ),
         param(
             "+key=choice()",
+            """Error when parsing index: 1, string: +key=choice() out of [""",
             """Error parsing override '+key=choice()'
 ValueError while evaluating 'choice()': empty choice is not legal""",
             id="empty choice",
         ),
         param(
             "+key=extend_list(1, 2, 3)",
+            """Error when parsing index: 1, string: +key=extend_list(1, 2, 3) out of [""",
             """Error parsing override '+key=extend_list(1, 2, 3)'
 Trying to use override symbols when extending a list""",
             id="plus key extend_list",
         ),
         param(
             "key={inner_key=extend_list(1, 2, 3)}",
+            """Error when parsing index: 1, string: key={inner_key=extend_list(1, 2, 3)} out of [""",
             "no viable alternative at input '{inner_key='",
             id="embedded extend_list",
         ),
         param(
             ["+key=choice(choice(a,b))", "-m"],
+            """Error when parsing index: 1, string: +key=choice(choice(a,b)) out of [""",
             """Error parsing override '+key=choice(choice(a,b))'
 ValueError while evaluating 'choice(choice(a,b))': nesting choices is not supported
 See https://hydra.cc/docs/1.2/advanced/override_grammar/basic for details
@@ -67,26 +74,36 @@
 
 Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
 """,
+            "",
             id="config_dir_not_found",
         ),
     ],
 )
-def test_cli_error(tmpdir: Any, monkeypatch: Any, override: Any, expected: str) -> None:
+def test_cli_error(
+    tmpdir: Any,
+    monkeypatch: Any,
+    override: Any,
+    expected_substring_1: str,
+    expected_substring_2: str,
+) -> None:
     monkeypatch.chdir("tests/test_apps/app_without_config/")
     if isinstance(override, str):
         override = [override]
     cmd = ["my_app.py", "hydra.sweep.dir=" + str(tmpdir)] + override
     ret = normalize_newlines(run_with_error(cmd))
     assert (
-        re.search("^" + re.escape(normalize_newlines(expected.strip())), ret)
-        is not None
+        expected_substring_1.strip() in ret and expected_substring_2.strip() in ret
     ), (
         f"Result:"
         f"\n---"
         f"\n{ret}"
         f"\n---"
-        f"\nDid not match expected:"
+        f"\nDoes not contain expected_substring_1:"
+        f"\n---"
+        f"\n{expected_substring_1}"
+        f"\n---"
+        f"\nand expected_substring_2:"
         f"\n---"
-        f"\n{expected}"
+        f"\n{expected_substring_2}"
         f"\n---"
     )