Skip to content

Commit

Permalink
Merge pull request #226 from CITCOM-project/test-adequacy
Browse files Browse the repository at this point in the history 
Fixed order-dependency bug in independence test DAG adequacy
  • Loading branch information
@jmafoster1
jmafoster1 authored Aug 8, 2023
2 parents bf0bbe3 + 8bb6648 commit 0776b79
Show file tree
Hide file tree
Showing 2 changed files with 121 additions and 13 deletions.
22 changes: 16 additions & 6 deletions causal_testing/testing/causal_test_adequacy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,26 @@ def __init__(
self.test_suite = test_suite
self.tested_pairs = None
self.pairs_to_test = None
self.untested_edges = None
self.untested_pairs = None
self.dag_adequacy = None

def measure_adequacy(self):
"""
Calculate the adequacy measurement, and populate the `dat_adequacy` field.
Calculate the adequacy measurement, and populate the `dag_adequacy` field.
"""
self.tested_pairs = {(t.treatment_variable, t.outcome_variable) for t in self.test_suite}
self.pairs_to_test = set(combinations(self.causal_dag.graph.nodes, 2))
self.untested_edges = self.pairs_to_test.difference(self.tested_pairs)
self.pairs_to_test = set(combinations(self.causal_dag.graph.nodes(), 2))
self.tested_pairs = set()

for n1, n2 in self.pairs_to_test:
if (n1, n2) in self.causal_dag.graph.edges():
if any((t.treatment_variable, t.outcome_variable) == (n1, n2) for t in self.test_suite):
self.tested_pairs.add((n1, n2))
else:
# Causal independences are not order dependent
if any((t.treatment_variable, t.outcome_variable) in {(n1, n2), (n2, n1)} for t in self.test_suite):
self.tested_pairs.add((n1, n2))

self.untested_pairs = self.pairs_to_test.difference(self.tested_pairs)
self.dag_adequacy = len(self.tested_pairs) / len(self.pairs_to_test)

def to_dict(self):
Expand All @@ -45,7 +55,7 @@ def to_dict(self):
"test_suite": self.test_suite,
"tested_pairs": self.tested_pairs,
"pairs_to_test": self.pairs_to_test,
"untested_edges": self.untested_edges,
"untested_pairs": self.untested_pairs,
"dag_adequacy": self.dag_adequacy,
}

Expand Down
112 changes: 105 additions & 7 deletions tests/testing_tests/test_causal_test_adequacy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,10 +107,10 @@ def test_data_adequacy_cateogorical(self):
{"kurtosis": {"test_input_no_dist[T.b]": 0.0}, "bootstrap_size": 100, "passing": 100},
)

def test_dag_adequacy(self):
def test_dag_adequacy_dependent(self):
base_test_case = BaseTestCase(
treatment_variable="test_input",
outcome_variable="test_output",
outcome_variable="B",
effect=None,
)
causal_test_case = CausalTestCase(
Expand All @@ -128,29 +128,127 @@ def test_dag_adequacy(self):
{
"causal_dag": self.json_class.causal_specification.causal_dag,
"test_suite": test_suite,
"tested_pairs": {("test_input", "test_output")},
"tested_pairs": {("test_input", "B")},
"pairs_to_test": {
("B", "C"),
("test_input_no_dist", "test_input"),
("C", "test_output"),
("test_input", "B"),
("test_input_no_dist", "C"),
("test_input_no_dist", "B"),
("test_input", "test_output"),
("test_input", "C"),
("B", "C"),
("test_input_no_dist", "test_output"),
("B", "test_output"),
("test_input_no_dist", "C"),
},
"untested_pairs": {
("B", "C"),
("test_input_no_dist", "test_input"),
("C", "test_output"),
("test_input_no_dist", "B"),
("test_input", "test_output"),
("test_input", "C"),
("test_input_no_dist", "test_output"),
("B", "test_output"),
("test_input_no_dist", "C"),
},
"dag_adequacy": 0.1,
},
)

def test_dag_adequacy_independent(self):
base_test_case = BaseTestCase(
treatment_variable="test_input",
outcome_variable="C",
effect=None,
)
causal_test_case = CausalTestCase(
base_test_case=base_test_case,
expected_causal_effect=None,
estimate_type=None,
)
test_suite = CausalTestSuite()
test_suite.add_test_object(base_test_case, causal_test_case, None, None)
dag_adequacy = DAGAdequacy(self.json_class.causal_specification.causal_dag, test_suite)
dag_adequacy.measure_adequacy()
print(dag_adequacy.to_dict())
self.assertEqual(
dag_adequacy.to_dict(),
{
"causal_dag": self.json_class.causal_specification.causal_dag,
"test_suite": test_suite,
"tested_pairs": {("test_input", "C")},
"pairs_to_test": {
("B", "C"),
("test_input_no_dist", "test_input"),
("C", "test_output"),
("test_input", "B"),
("test_input_no_dist", "B"),
("test_input", "test_output"),
("test_input", "C"),
("test_input_no_dist", "test_output"),
("B", "test_output"),
("test_input_no_dist", "C"),
},
"untested_edges": {
"untested_pairs": {
("B", "C"),
("test_input_no_dist", "test_input"),
("C", "test_output"),
("test_input_no_dist", "B"),
("test_input", "test_output"),
("test_input", "B"),
("test_input_no_dist", "test_output"),
("B", "test_output"),
("test_input_no_dist", "C"),
},
"dag_adequacy": 0.1,
},
)

def test_dag_adequacy_independent_other_way(self):
base_test_case = BaseTestCase(
treatment_variable="C",
outcome_variable="test_input",
effect=None,
)
causal_test_case = CausalTestCase(
base_test_case=base_test_case,
expected_causal_effect=None,
estimate_type=None,
)
test_suite = CausalTestSuite()
test_suite.add_test_object(base_test_case, causal_test_case, None, None)
dag_adequacy = DAGAdequacy(self.json_class.causal_specification.causal_dag, test_suite)
dag_adequacy.measure_adequacy()
print(dag_adequacy.to_dict())
self.assertEqual(
dag_adequacy.to_dict(),
{
"causal_dag": self.json_class.causal_specification.causal_dag,
"test_suite": test_suite,
"tested_pairs": {("test_input", "C")},
"pairs_to_test": {
("B", "C"),
("test_input_no_dist", "test_input"),
("C", "test_output"),
("test_input", "B"),
("test_input_no_dist", "B"),
("test_input", "test_output"),
("test_input", "C"),
("B", "C"),
("test_input_no_dist", "test_output"),
("B", "test_output"),
("test_input_no_dist", "C"),
},
"untested_pairs": {
("B", "C"),
("test_input_no_dist", "test_input"),
("C", "test_output"),
("test_input_no_dist", "B"),
("test_input", "test_output"),
("test_input", "B"),
("test_input_no_dist", "test_output"),
("B", "test_output"),
("test_input_no_dist", "C"),
},
"dag_adequacy": 0.1,
},
Expand Down

0 comments on commit 0776b79

Please sign in to comment.