diff --git a/causal_testing/json_front/json_class.py b/causal_testing/json_front/json_class.py index b0f0d806..954d986b 100644 --- a/causal_testing/json_front/json_class.py +++ b/causal_testing/json_front/json_class.py @@ -20,6 +20,7 @@ from causal_testing.specification.causal_specification import CausalSpecification from causal_testing.specification.scenario import Scenario from causal_testing.specification.variable import Input, Meta, Output +from causal_testing.testing.base_test_case import BaseTestCase from causal_testing.testing.causal_test_case import CausalTestCase from causal_testing.testing.causal_test_engine import CausalTestEngine from causal_testing.testing.estimators import Estimator @@ -46,7 +47,7 @@ class JsonUtility: def __init__(self, output_path: str, output_overwrite: bool = False): self.input_paths = None - self.variables = None + self.variables = {"inputs": {}, "outputs": {}, "metas": {}} self.data = [] self.test_plan = None self.scenario = None @@ -66,6 +67,7 @@ def set_paths(self, json_path: str, dag_path: str, data_paths: str): def setup(self, scenario: Scenario): """Function to populate all the necessary parts of the json_class needed to execute tests""" self.scenario = scenario + self._get_scenario_variables() self.scenario.setup_treatment_variables() self.causal_specification = CausalSpecification( scenario=self.scenario, causal_dag=CausalDAG(self.input_paths.dag_path) @@ -73,6 +75,63 @@ def setup(self, scenario: Scenario): self._json_parse() self._populate_metas() + def run_json_tests(self, effects: dict, estimators: dict, f_flag: bool = False, mutates: dict = None): + """Runs and evaluates each test case specified in the JSON input + + :param effects: Dictionary mapping effect class instances to string representations. + :param mutates: Dictionary mapping mutation functions to string representations. + :param estimators: Dictionary mapping estimator classes to string representations. + :param f_flag: Failure flag that if True the script will stop executing when a test fails. + """ + failures = 0 + for test in self.test_plan["tests"]: + if "skip" in test and test["skip"]: + continue + test["estimator"] = estimators[test["estimator"]] + if "mutations" in test: + abstract_test = self._create_abstract_test_case(test, mutates, effects) + + concrete_tests, dummy = abstract_test.generate_concrete_tests(5, 0.05) + failures = self._execute_tests(concrete_tests, test, f_flag) + msg = ( + f"Executing test: {test['name']}\n" + + "abstract_test\n" + + f"{abstract_test}\n" + + f"{abstract_test.treatment_variable.name},{abstract_test.treatment_variable.distribution}\n" + + f"Number of concrete tests for test case: {str(len(concrete_tests))}\n" + + f"{failures}/{len(concrete_tests)} failed for {test['name']}" + ) + self._append_to_file(msg, logging.INFO) + else: + outcome_variable = next( + iter(test["expected_effect"]) + ) # Take first key from dictionary of expected effect + base_test_case = BaseTestCase( + treatment_variable=self.variables["inputs"][test["treatment_variable"]], + outcome_variable=self.variables["outputs"][outcome_variable], + ) + + causal_test_case = CausalTestCase( + base_test_case=base_test_case, + expected_causal_effect=effects[test["expected_effect"][outcome_variable]], + control_value=test["control_value"], + treatment_value=test["treatment_value"], + estimate_type=test["estimate_type"], + ) + if self._execute_test_case(causal_test_case=causal_test_case, test=test, f_flag=f_flag): + result = "failed" + else: + result = "passed" + + msg = ( + f"Executing concrete test: {test['name']} \n" + + f"treatment variable: {test['treatment_variable']} \n" + + f"outcome_variable = {outcome_variable} \n" + + f"control value = {test['control_value']}, treatment value = {test['treatment_value']} \n" + + f"result - {result}" + ) + self._append_to_file(msg, logging.INFO) + def _create_abstract_test_case(self, test, mutates, effects): assert len(test["mutations"]) == 1 abstract_test = AbstractCausalTestCase( @@ -81,7 +140,7 @@ def _create_abstract_test_case(self, test, mutates, effects): treatment_variable=next(self.scenario.variables[v] for v in test["mutations"]), expected_causal_effect={ self.scenario.variables[variable]: effects[effect] - for variable, effect in test["expectedEffect"].items() + for variable, effect in test["expected_effect"].items() }, effect_modifiers={self.scenario.variables[v] for v in test["effect_modifiers"]} if "effect_modifiers" in test @@ -91,35 +150,8 @@ def _create_abstract_test_case(self, test, mutates, effects): ) return abstract_test - def generate_tests(self, effects: dict, mutates: dict, estimators: dict, f_flag: bool): - """Runs and evaluates each test case specified in the JSON input - - :param effects: Dictionary mapping effect class instances to string representations. - :param mutates: Dictionary mapping mutation functions to string representations. - :param estimators: Dictionary mapping estimator classes to string representations. - :param f_flag: Failure flag that if True the script will stop executing when a test fails. - """ + def _execute_tests(self, concrete_tests, test, f_flag): failures = 0 - for test in self.test_plan["tests"]: - if "skip" in test and test["skip"]: - continue - abstract_test = self._create_abstract_test_case(test, mutates, effects) - - concrete_tests, dummy = abstract_test.generate_concrete_tests(5, 0.05) - failures = self._execute_tests(concrete_tests, estimators, test, f_flag) - msg = ( - f"Executing test: {test['name']} \n" - + "abstract_test \n" - + f"{abstract_test} \n" - + f"{abstract_test.treatment_variable.name},{abstract_test.treatment_variable.distribution} \n" - + f"Number of concrete tests for test case: {str(len(concrete_tests))} \n" - + f"{failures}/{len(concrete_tests)} failed for {test['name']}" - ) - self._append_to_file(msg, logging.INFO) - - def _execute_tests(self, concrete_tests, estimators, test, f_flag): - failures = 0 - test["estimator"] = estimators[test["estimator"]] if "formula" in test: self._append_to_file(f"Estimator formula used for test: {test['formula']}") for concrete_test in concrete_tests: @@ -161,7 +193,6 @@ def _execute_test_case(self, causal_test_case: CausalTestCase, test: Iterable[Ma :rtype: bool """ failed = False - causal_test_engine, estimation_model = self._setup_test(causal_test_case, test) causal_test_result = causal_test_engine.execute_test( estimation_model, causal_test_case, estimate_type=causal_test_case.estimate_type @@ -169,7 +200,6 @@ def _execute_test_case(self, causal_test_case: CausalTestCase, test: Iterable[Ma test_passes = causal_test_case.expected_causal_effect.apply(causal_test_result) - result_string = str() if causal_test_result.ci_low() and causal_test_result.ci_high(): result_string = ( f"{causal_test_result.ci_low()} < {causal_test_result.test_value.value} < " @@ -214,7 +244,6 @@ def _setup_test(self, causal_test_case: CausalTestCase, test: Mapping) -> tuple[ } if "formula" in test: estimator_kwargs["formula"] = test["formula"] - estimation_model = test["estimator"](**estimator_kwargs) return causal_test_engine, estimation_model @@ -226,10 +255,18 @@ def _append_to_file(self, line: str, log_level: int = None): is possible to use the inbuilt logging level variables such as logging.INFO and logging.WARNING """ with open(self.output_path, "a", encoding="utf-8") as f: - f.write(line) + f.write(line + "\n") if log_level: logger.log(level=log_level, msg=line) + def _get_scenario_variables(self): + for input_var in self.scenario.inputs(): + self.variables["inputs"][input_var.name] = input_var + for output_var in self.scenario.outputs(): + self.variables["outputs"][output_var.name] = output_var + for meta_var in self.scenario.metas(): + self.variables["metas"][meta_var.name] = meta_var + @staticmethod def check_file_exists(output_path: Path, overwrite: bool): """Method that checks if the given path to an output file already exists. If overwrite is true the check is diff --git a/docs/source/frontends/json_front_end.rst b/docs/source/frontends/json_front_end.rst index 7d6b24b7..4d496f5d 100644 --- a/docs/source/frontends/json_front_end.rst +++ b/docs/source/frontends/json_front_end.rst @@ -21,17 +21,28 @@ Use case specific information is also declared here such as the paths to the rel causal_tests.json ----------------- -`examples/poisson/causal_tests.json `_ contains python code written by the user to implement scenario specific features -is the JSON file that allows for the easy specification of multiple causal tests. +`examples/poisson/causal_tests.json `_ contains python code written by the user to implement scenario specific features +is the JSON file that allows for the easy specification of multiple causal tests. Tests can be specified two ways; firstly by specifying a mutation lke in the example tests with the following structure: Each test requires: -1. Test name -2. Mutations -3. Estimator -4. Estimate_type -5. Effect modifiers -6. Expected effects -7. Skip: boolean that if set true the test won't be executed and will be skipped +#. name +#. mutations +#. estimator +#. estimate_type +#. effect_modifiers +#. expected_effects +#. skip: boolean that if set true the test won't be executed and will be skipped + +The second method of specifying a test is to specify the test in a concrete form with the following structure: + +#. name +#. treatment_variable +#. control_value +#. treatment_value +#. estimator +#. estimate_type +#. expected_effect +#. skip Run Commands ------------ diff --git a/examples/poisson/README.md b/examples/poisson/README.md index f4e27a62..1b8fe514 100644 --- a/examples/poisson/README.md +++ b/examples/poisson/README.md @@ -6,6 +6,6 @@ To run this case study: 1. Ensure all project dependencies are installed by running `pip install .` in the top level directory (instructions are provided in the project README). 2. Change directory to `causal_testing/examples/poisson`. -3. Run the command `python test_run_causal_tests.py --data_path data.csv --dag_path dag.dot --json_path causal_tests.json` +3. Run the command `python example_run_causal_tests.py --data_path data.csv --dag_path dag.dot --json_path causal_tests.json` This should print a series of causal test results and produce two CSV files. `intensity_num_shapes_results_random_1000.csv` corresponds to table 1, and `width_num_shapes_results_random_1000.csv` relates to our findings regarding the relationship of width and `P_u`. diff --git a/examples/poisson/causal_tests.json b/examples/poisson/causal_tests.json index 04d96f35..08bf659a 100644 --- a/examples/poisson/causal_tests.json +++ b/examples/poisson/causal_tests.json @@ -6,7 +6,7 @@ "estimator": "WidthHeightEstimator", "estimate_type": "ate", "effect_modifiers": ["intensity", "height"], - "expectedEffect": {"num_lines_abs": "PoissonWidthHeight"}, + "expected_effect": {"num_lines_abs": "PoissonWidthHeight"}, "skip": true }, { @@ -15,7 +15,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": ["intensity", "height"], - "expectedEffect": {"num_shapes_abs": "Positive"}, + "expected_effect": {"num_shapes_abs": "Positive"}, "skip": true }, { @@ -24,7 +24,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": ["intensity", "height"], - "expectedEffect": {"num_lines_unit": "Negative"}, + "expected_effect": {"num_lines_unit": "Negative"}, "skip": true }, { @@ -33,7 +33,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": ["intensity", "height"], - "expectedEffect": {"num_shapes_unit": "Negative"}, + "expected_effect": {"num_shapes_unit": "Negative"}, "skip": true }, { @@ -42,7 +42,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"height": "NoEffect"}, + "expected_effect": {"height": "NoEffect"}, "skip": true }, { @@ -51,7 +51,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"intensity": "NoEffect"}, + "expected_effect": {"intensity": "NoEffect"}, "skip": true }, { @@ -60,7 +60,7 @@ "estimator": "CausalForestEstimator", "estimate_type": "ate", "effect_modifiers": ["intensity", "width"], - "expectedEffect": {"num_shapes_abs": "Positive"}, + "expected_effect": {"num_shapes_abs": "Positive"}, "skip": true }, { @@ -69,7 +69,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"num_lines_unit": "Positive"}, + "expected_effect": {"num_lines_unit": "Positive"}, "skip": true }, { @@ -78,7 +78,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"num_shapes_unit": "NoEffect"}, + "expected_effect": {"num_shapes_unit": "NoEffect"}, "skip": true }, { @@ -87,7 +87,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"num_lines_unit": "NoEffect"}, + "expected_effect": {"num_lines_unit": "NoEffect"}, "skip": true }, { @@ -96,7 +96,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"num_shapes_unit": "Positive"}, + "expected_effect": {"num_shapes_unit": "Positive"}, "skip": true }, { @@ -105,7 +105,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"num_shapes_abs": "NoEffect"}, + "expected_effect": {"num_shapes_abs": "NoEffect"}, "skip": true }, { @@ -114,7 +114,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"num_shapes_unit": "NoEffect"}, + "expected_effect": {"num_shapes_unit": "NoEffect"}, "skip": true }, { @@ -123,7 +123,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"num_lines_unit": "NoEffect"}, + "expected_effect": {"num_lines_unit": "NoEffect"}, "skip": true }, { @@ -132,7 +132,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"width": "NoEffect"}, + "expected_effect": {"width": "NoEffect"}, "skip": true }, { @@ -141,7 +141,7 @@ "estimator": "WidthHeightEstimator", "estimate_type": "ate", "effect_modifiers": ["intensity", "width"], - "expectedEffect": {"num_lines_abs": "PoissonWidthHeight"}, + "expected_effect": {"num_lines_abs": "PoissonWidthHeight"}, "skip": true }, { @@ -150,7 +150,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": ["intensity", "width"], - "expectedEffect": {"num_shapes_abs": "Positive"}, + "expected_effect": {"num_shapes_abs": "Positive"}, "skip": true }, { @@ -159,7 +159,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": ["intensity", "width"], - "expectedEffect": {"num_lines_unit": "Negative"}, + "expected_effect": {"num_lines_unit": "Negative"}, "skip": true }, { @@ -168,7 +168,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": ["intensity", "width"], - "expectedEffect": {"num_shapes_unit": "Negative"}, + "expected_effect": {"num_shapes_unit": "Negative"}, "skip": true }, { @@ -177,7 +177,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"intensity": "NoEffect"}, + "expected_effect": {"intensity": "NoEffect"}, "skip": true }, { @@ -186,7 +186,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"width": "NoEffect"}, + "expected_effect": {"width": "NoEffect"}, "skip": true }, { @@ -195,7 +195,7 @@ "estimator": "WidthHeightEstimator", "effect_modifiers": ["height", "width"], "estimate_type": "ate", - "expectedEffect": {"num_lines_abs": "PoissonIntensity"}, + "expected_effect": {"num_lines_abs": "PoissonIntensity"}, "skip": true }, { @@ -204,7 +204,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": ["height", "width"], - "expectedEffect": {"num_shapes_abs": "Positive"}, + "expected_effect": {"num_shapes_abs": "Positive"}, "skip": true }, { @@ -213,7 +213,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": ["height", "width"], - "expectedEffect": {"num_lines_unit": "NoEffect"}, + "expected_effect": {"num_lines_unit": "NoEffect"}, "skip": true }, { @@ -221,7 +221,7 @@ "mutations": {"intensity": "ChangeByFactor(2)"}, "estimator": "LinearRegressionEstimator", "estimate_type": "risk_ratio", - "expectedEffect": {"num_shapes_unit": "ExactValue4_05"}, + "expected_effect": {"num_shapes_unit": "ExactValue4_05"}, "skip": false }, { @@ -230,7 +230,7 @@ "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"height": "NoEffect"}, + "expected_effect": {"height": "NoEffect"}, "skip": true } ] diff --git a/examples/poisson/example_run_causal_tests.py b/examples/poisson/example_run_causal_tests.py index 3dae849d..78d1b166 100644 --- a/examples/poisson/example_run_causal_tests.py +++ b/examples/poisson/example_run_causal_tests.py @@ -165,7 +165,7 @@ def test_run_causal_tests(): # Load the Causal Variables into the JsonUtility class ready to be used in the tests json_utility.setup(scenario=modelling_scenario) # Sets up all the necessary parts of the json_class needed to execute tests - json_utility.generate_tests(effects, mutates, estimators, False) + json_utility.run_json_tests(effects=effects, mutates=mutates, estimators=estimators, f_flag=False) if __name__ == "__main__": @@ -178,4 +178,4 @@ def test_run_causal_tests(): # Load the Causal Variables into the JsonUtility class ready to be used in the tests json_utility.setup(scenario=modelling_scenario) # Sets up all the necessary parts of the json_class needed to execute tests - json_utility.generate_tests(effects, mutates, estimators, args.f) + json_utility.run_json_tests(effects=effects, mutates=mutates, estimators=estimators, f_flag=args.f) diff --git a/tests/json_front_tests/test_json_class.py b/tests/json_front_tests/test_json_class.py index 05e823a2..b8c8425c 100644 --- a/tests/json_front_tests/test_json_class.py +++ b/tests/json_front_tests/test_json_class.py @@ -83,7 +83,7 @@ def test_f_flag(self): "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"test_output": "NoEffect"}, + "expected_effect": {"test_output": "NoEffect"}, "skip": False, } ] @@ -96,9 +96,9 @@ def test_f_flag(self): } estimators = {"LinearRegressionEstimator": LinearRegressionEstimator} with self.assertRaises(StatisticsError): - self.json_class.generate_tests(effects, mutates, estimators, True) + self.json_class.run_json_tests(effects, estimators, True, mutates) - def test_generate_tests_from_json(self): + def test_run_json_tests_from_json(self): example_test = { "tests": [ { @@ -107,7 +107,7 @@ def test_generate_tests_from_json(self): "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"test_output": "NoEffect"}, + "expected_effect": {"test_output": "NoEffect"}, "skip": False, } ] @@ -120,7 +120,7 @@ def test_generate_tests_from_json(self): } estimators = {"LinearRegressionEstimator": LinearRegressionEstimator} - self.json_class.generate_tests(effects, mutates, estimators, False) + self.json_class.run_json_tests(effects, estimators, False, mutates) # Test that the final log message prints that failed tests are printed, which is expected behaviour for this scenario with open("temp_out.txt", 'r') as reader: @@ -136,7 +136,7 @@ def test_formula_in_json_test(self): "estimator": "LinearRegressionEstimator", "estimate_type": "ate", "effect_modifiers": [], - "expectedEffect": {"test_output": "Positive"}, + "expected_effect": {"test_output": "Positive"}, "skip": False, "formula": "test_output ~ test_input" } @@ -150,11 +150,35 @@ def test_formula_in_json_test(self): } estimators = {"LinearRegressionEstimator": LinearRegressionEstimator} - self.json_class.generate_tests(effects, mutates, estimators, False) + self.json_class.run_json_tests(effects=effects, mutates=mutates, estimators=estimators, f_flag=False) with open("temp_out.txt", 'r') as reader: temp_out = reader.readlines() self.assertIn("test_output ~ test_input", ''.join(temp_out)) + def test_run_concrete_json_testcase(self): + example_test = { + "tests": [ + { + "name": "test1", + "treatment_variable": "test_input", + "control_value": 0, + "treatment_value": 1, + "estimator": "LinearRegressionEstimator", + "estimate_type": "ate", + "expected_effect": {"test_output": "NoEffect"}, + "skip": False, + } + ] + } + self.json_class.test_plan = example_test + effects = {"NoEffect": NoEffect()} + estimators = {"LinearRegressionEstimator": LinearRegressionEstimator} + + self.json_class.run_json_tests(effects=effects, estimators=estimators, f_flag=False) + with open("temp_out.txt", 'r') as reader: + temp_out = reader.readlines() + self.assertIn("failed", temp_out[-1]) + def tearDown(self) -> None: remove_temp_dir_if_existent() diff --git a/tests/resources/data/tests.json b/tests/resources/data/tests.json index 10f8e114..beee4ba5 100644 --- a/tests/resources/data/tests.json +++ b/tests/resources/data/tests.json @@ -1 +1 @@ -{"tests": [{"name": "test1", "mutations": {}, "estimator": null, "estimate_type": null, "effect_modifiers": [], "expectedEffect": {}, "skip": false}]} \ No newline at end of file +{"tests": [{"name": "test1", "mutations": {}, "estimator": null, "estimate_type": null, "effect_modifiers": [], "expected_effect": {}, "skip": false}]} \ No newline at end of file