[ATO-797] Rasa data validate does not catch domain loading as an error (

#12521)

* init fix

* add tests

* add domain path validation

* fix default values and tests

* fix tests

* add changelog

changelog/12521.bugfix.md

@@ -0,0 +1 @@
+Introduce a validation step in `rasa data validate` and `rasa train` commands to identify non-existent paths and empty domains.

rasa/cli/data.py

@@ -9,7 +9,11 @@
 from rasa.cli.arguments import data as arguments
 from rasa.cli.arguments import default_arguments
 import rasa.cli.utils
-from rasa.shared.constants import DEFAULT_DATA_PATH, DEFAULT_CONFIG_PATH
+from rasa.shared.constants import (
+    DEFAULT_DATA_PATH,
+    DEFAULT_CONFIG_PATH,
+    DEFAULT_DOMAIN_PATH,
+)
 import rasa.shared.data
 from rasa.shared.importers.importer import TrainingDataImporter
 import rasa.shared.nlu.training_data.loading
@@ -146,8 +150,13 @@ def _build_training_data_importer(args: argparse.Namespace) -> "TrainingDataImpo
         args.config, "config", DEFAULT_CONFIG_PATH, none_is_valid=True
     )
 
+    # Exit the validation if the domain path is invalid
+    domain = rasa.cli.utils.get_validated_path(
+        args.domain, "domain", DEFAULT_DOMAIN_PATH, none_is_valid=False
+    )
+
     return TrainingDataImporter.load_from_config(
-        domain_path=args.domain, training_data_paths=args.data, config_path=config
+        domain_path=domain, training_data_paths=args.data, config_path=config
     )
 
 

rasa/cli/utils.py

@@ -233,12 +233,19 @@ def validate_files(
     if stories_only:
         all_good = _validate_story_structure(validator, max_history, fail_on_warnings)
     else:
-        all_good = (
-            _validate_domain(validator)
-            and _validate_nlu(validator, fail_on_warnings)
-            and _validate_story_structure(validator, max_history, fail_on_warnings)
+        if importer.get_domain().is_empty():
+            rasa.shared.utils.cli.print_error_and_exit(
+                "Encountered empty domain during validation."
+            )
+
+        valid_domain = _validate_domain(validator)
+        valid_nlu = _validate_nlu(validator, fail_on_warnings)
+        valid_stories = _validate_story_structure(
+            validator, max_history, fail_on_warnings
         )
 
+        all_good = valid_domain and valid_nlu and valid_stories
+
     validator.warn_if_config_mandatory_keys_are_not_set()
 
     telemetry.track_validate_files(all_good)
@@ -249,12 +256,17 @@ def validate_files(
 
 
 def _validate_domain(validator: "Validator") -> bool:
+    valid_domain_validity = validator.verify_domain_validity()
+    valid_actions_in_stories_rules = validator.verify_actions_in_stories_rules()
+    valid_forms_in_stories_rules = validator.verify_forms_in_stories_rules()
+    valid_form_slots = validator.verify_form_slots()
+    valid_slot_mappings = validator.verify_slot_mappings()
     return (
-        validator.verify_domain_validity()
-        and validator.verify_actions_in_stories_rules()
-        and validator.verify_forms_in_stories_rules()
-        and validator.verify_form_slots()
-        and validator.verify_slot_mappings()
+        valid_domain_validity
+        and valid_actions_in_stories_rules
+        and valid_forms_in_stories_rules
+        and valid_form_slots
+        and valid_slot_mappings
     )
 
 

tests/cli/conftest.py

@@ -110,6 +110,20 @@ def do_run(*args):
     return do_run
 
 
+@pytest.fixture
+def run_in_simple_project_with_no_domain(testdir: Testdir) -> Callable[..., RunResult]:
+    os.environ["LOG_LEVEL"] = "WARNING"
+
+    create_simple_project(testdir.tmpdir)
+    Path(testdir.tmpdir / "domain.yml").unlink()
+
+    def do_run(*args):
+        args = [shutil.which(RASA_EXE)] + list(args)
+        return testdir.run(*args)
+
+    return do_run
+
+
 @pytest.fixture
 def run_in_simple_project_with_model(
     testdir: Testdir, trained_simple_project: Text

tests/cli/test_rasa_data.py

@@ -209,6 +209,20 @@ def test_data_validate_not_used_warning(
         assert warning in str(result.stderr)
 
 
+def test_data_validate_failed_to_load_domain(
+    run_in_simple_project_with_no_domain: Callable[..., RunResult]
+):
+    result = run_in_simple_project_with_no_domain(
+        "data",
+        "validate",
+        "--domain",
+        "not-existing-domain.yml",
+    )
+
+    assert "The path 'not-existing-domain.yml' does not exist." in str(result.outlines)
+    assert result.ret == 1
+
+
 def test_data_split_stories(run_in_simple_project: Callable[..., RunResult]):
     stories_yml = (
         "stories:\n"

tests/cli/test_rasa_train.py

@@ -541,7 +541,7 @@ def test_train_validation_fail_on_warnings(
     request: pytest.FixtureRequest,
 ):
     test_data_dir = Path(request.config.rootdir, "data", "test_moodbot", "data")
-    test_domain = Path(request.config.rootdir, "data", "test_domains", "domain.yml")
+    test_domain = Path(request.config.rootdir, "data", "test_domains", "default.yml")
 
     result = run_in_simple_project_with_warnings(
         "train",
@@ -558,6 +558,19 @@ def test_train_validation_fail_on_warnings(
     assert result.ret == 1
 
 
+def test_train_validation_fail_to_load_domain(
+    run_in_simple_project: Callable[..., RunResult],
+):
+    result = run_in_simple_project(
+        "train",
+        "--domain",
+        "not_existing_domain.yml",
+    )
+
+    assert "Encountered empty domain during validation." in str(result.outlines)
+    assert result.ret == 1
+
+
 def test_train_validation_max_history_1(
     run_in_simple_project_with_warnings: Callable[..., RunResult],
     request: pytest.FixtureRequest,