Skip to content

Commit 445f661

Browse files
jmafoster1jmafoster1
committed
Evaluate dag
1 parent 87d973d commit 445f661

3 files changed

Lines changed: 247 additions & 207 deletions

File tree

‎causal_testing/causal_testing_framework.py‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ def run_tests(self, silent: bool = False, adequacy: bool = False, bootstrap_size
266266
self.df, suppress_estimation_errors=silent, adequacy=adequacy, bootstrap_size=bootstrap_size
267267
)
268268

269-
def evaluate_dag(self, bootstrap_size: bool, alpha: float) -> pd.Series:
269+
def evaluate_dag(self, bootstrap_size: bool = 100, alpha: float = 0.05) -> pd.Series:
270270
"""
271271
Calculate confidence intervals for how well a causal DAG fits a dataset by repeatedly resampling the dataset
272272
and executing the causal tests.
@@ -275,17 +275,22 @@ def evaluate_dag(self, bootstrap_size: bool, alpha: float) -> pd.Series:
275275
:param bootstrap_size: The number of bootstrap samples to use when calculating causal test adequacy
276276
(defaults to 100)
277277
:param alpha: The significance level to use when calculating confidence intervals.
278+
(defaults to 0.05).
278279
"""
279280
self.run_tests(silent=True, adequacy=False)
280281
results = {
281-
test_outcome: len([test for test in self.test_cases if test.result.outcome == test_outcome])
282+
test_outcome.name: len(
283+
[test for test in self.test_cases if test.result and test.result.outcome == test_outcome]
284+
)
282285
for test_outcome in TestOutcome
283286
}
284287

285288
sample_results = []
286289
for sample_index in range(bootstrap_size):
287290
test_outcomes = {test_outcome: 0 for test_outcome in TestOutcome}
288-
for test_case in tqdm(self.test_cases):
291+
for test_case in self.test_cases:
292+
if test_case.skip:
293+
continue
289294
effect_estimate = test_case.estimate_effect(
290295
df=self.df.sample(len(self.df), replace=True, random_state=sample_index)
291296
)
@@ -299,14 +304,16 @@ def evaluate_dag(self, bootstrap_size: bool, alpha: float) -> pd.Series:
299304
sample_results.append(test_outcomes)
300305

301306
sample_results = pd.DataFrame(sample_results)
307+
302308
# Calculate the confidence interval of each column
303-
ci_low_inx = (alpha / 2) * bootstrap_size
304-
ci_high_inx = ((1 - alpha) / 2) * bootstrap_size
309+
ci_low_inx = round((alpha / 2) * bootstrap_size)
310+
ci_high_inx = round(((1 - alpha) / 2) * bootstrap_size)
305311
for outcome in TestOutcome:
306312
data = sorted(sample_results[outcome])
307-
results[f"{outcome}_ci_low"] = data[ci_low_inx]
308-
results[f"{outcome}_ci_high"] = data[ci_high_inx]
313+
results[f"{outcome.name}_ci_low"] = data[ci_low_inx]
314+
results[f"{outcome.name}_ci_high"] = data[ci_high_inx]
309315

316+
print(results)
310317
return pd.Series(results).sort_index()
311318

312319
def save_results(self, output_path) -> list:

‎tests/main_tests/test_ctf.py‎

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
import unittest
2+
from pathlib import Path
3+
import json
4+
import pandas as pd
5+
6+
from causal_testing.causal_testing_framework import CausalTestingFramework
7+
8+
9+
class TestCausalTestingFramework(unittest.TestCase):
10+
def setUp(self):
11+
self.dag_path = "tests/resources/data/dag.dot"
12+
self.data_paths = ["tests/resources/data/data.csv"]
13+
self.test_cases_path = "tests/resources/data/tests.json"
14+
self.output_path = Path("results/results.json")
15+
self.include_edges_path = "tests/resources/data/include_edges.dot"
16+
self.exclude_edges_path = "tests/resources/data/exclude_edges.dot"
17+
self.paths = {
18+
"dag_path": self.dag_path,
19+
"data_paths": self.data_paths,
20+
"test_cases_path": self.test_cases_path,
21+
}
22+
23+
def test_load_data(self):
24+
csv_framework = CausalTestingFramework()
25+
csv_framework.load_data(self.data_paths)
26+
27+
pqt_framework = CausalTestingFramework()
28+
pqt_framework.load_data([path.replace(".csv", ".pqt") for path in self.data_paths])
29+
pd.testing.assert_frame_equal(csv_framework.df, pqt_framework.df)
30+
31+
def test_load_data_query(self):
32+
framework = CausalTestingFramework()
33+
framework.load_data(data_paths=self.data_paths)
34+
self.assertFalse((framework.df["test_input"] > 4).all())
35+
36+
framework.load_data(data_paths=self.data_paths, query="test_input > 4")
37+
self.assertTrue((framework.df["test_input"] > 4).all())
38+
39+
def test_load_data_invalid_extension(self):
40+
framework = CausalTestingFramework()
41+
with self.assertRaises(ValueError):
42+
framework.load_data("data.invalid")
43+
44+
def test_load_dag_missing_node(self):
45+
framework = CausalTestingFramework()
46+
framework.setup(**self.paths)
47+
framework.dag.add_node("missing")
48+
with self.assertRaises(ValueError):
49+
framework.create_variables()
50+
51+
def test_load_tests_before_dag(self):
52+
framework = CausalTestingFramework()
53+
with self.assertRaises(ValueError):
54+
framework.load_test_cases_from_json(self.test_cases_path)
55+
56+
def test_create_base_test_case_missing_treatment(self):
57+
framework = CausalTestingFramework()
58+
framework.setup(**self.paths)
59+
with self.assertRaises(KeyError) as e:
60+
framework.create_base_test(
61+
{"treatment_variable": "missing", "expected_effect": {"test_outcome": "NoEffect"}}
62+
)
63+
self.assertEqual("\"Treatment variable 'missing' not found in inputs or outputs\"", str(e.exception))
64+
65+
def test_create_base_test_case_missing_estimator(self):
66+
framework = CausalTestingFramework()
67+
framework.setup(**self.paths)
68+
with self.assertRaises(ValueError) as e:
69+
framework.create_causal_test(
70+
{"treatment_variable": "test_input", "expected_effect": {"test_output": "NoEffect"}}
71+
)
72+
self.assertEqual("Test configuration must specify an estimator", str(e.exception))
73+
74+
def test_create_test_case_invalid_estimator(self):
75+
framework = CausalTestingFramework()
76+
framework.setup(**self.paths)
77+
with self.assertRaises(ValueError) as e:
78+
framework.create_causal_test(
79+
{
80+
"treatment_variable": "test_input",
81+
"expected_effect": {"test_output": "NoEffect"},
82+
"estimator": "InvalidEstimator",
83+
}
84+
)
85+
self.assertEqual(
86+
f"Unsupported estimator InvalidEstimator. Supported: ['CubicSplineEstimator', 'IPCWEstimator', 'InstrumentalVariableEstimator', 'LinearRegressionEstimator', 'LogisticRegressionEstimator', 'MultinomialRegressionEstimator']. "
87+
"If you have implemented a custom estimator, you will need to add this to your entrypoints via your "
88+
"pyproject.toml file.",
89+
str(e.exception),
90+
)
91+
92+
def test_create_test_case_invalid_effect(self):
93+
framework = CausalTestingFramework()
94+
framework.setup(**self.paths)
95+
test = {
96+
"name": "test1",
97+
"treatment_variable": "test_input",
98+
"estimator": "LinearRegressionEstimator",
99+
"estimate_type": "coefficient",
100+
"expected_effect": {"test_output": "InvalidEffect"},
101+
}
102+
base_test_case = framework.create_base_test(test)
103+
with self.assertRaises(ValueError) as e:
104+
framework.create_causal_test(test)
105+
self.assertEqual(
106+
f"Unsupported causal effect InvalidEffect. Supported: ['ExactValue', 'Negative', 'NoEffect', 'Positive', 'SomeEffect']. "
107+
"If you have implemented a custom causal effect, you will need to add this to your entrypoints via your "
108+
"pyproject.toml file.",
109+
str(e.exception),
110+
)
111+
112+
def test_create_test_case_effect_kwargs(self):
113+
framework = CausalTestingFramework()
114+
framework.setup(**self.paths)
115+
test = {
116+
"name": "test1",
117+
"treatment_variable": "test_input",
118+
"estimator": "LinearRegressionEstimator",
119+
"estimate_type": "coefficient",
120+
"expected_effect": {"test_output": "ExactValue"},
121+
"effect_kwargs": {"value": 4},
122+
}
123+
base_test_case = framework.create_base_test(test)
124+
test_case = framework.create_causal_test(test)
125+
self.assertEqual(test_case.expected_causal_effect.value, 4)
126+
127+
def test_create_test_case_estimator_kwargs(self):
128+
framework = CausalTestingFramework()
129+
framework.setup(**self.paths)
130+
test = {
131+
"name": "test1",
132+
"treatment_variable": "test_input",
133+
"estimator": "InstrumentalVariableEstimator",
134+
"estimate_type": "coefficient",
135+
"expected_effect": {"test_output": "SomeEffect"},
136+
"estimator_kwargs": {"instrument": "instrumental_variable"},
137+
}
138+
base_test_case = framework.create_base_test(test)
139+
test_case = framework.create_causal_test(test)
140+
self.assertEqual(test_case.estimator.instrument, "instrumental_variable")
141+
142+
def test_create_base_test_case_missing_outcome(self):
143+
framework = CausalTestingFramework()
144+
framework.setup(**self.paths)
145+
with self.assertRaises(KeyError) as e:
146+
framework.create_base_test({"treatment_variable": "test_input", "expected_effect": {"missing": "NoEffect"}})
147+
self.assertEqual("\"Outcome variable 'missing' not found in inputs or outputs\"", str(e.exception))
148+
149+
def test_unloaded_tests(self):
150+
framework = CausalTestingFramework()
151+
with self.assertRaises(ValueError) as e:
152+
framework.run_tests()
153+
self.assertEqual("No tests to run.", str(e.exception))
154+
155+
def test_ctf(self):
156+
framework = CausalTestingFramework()
157+
framework.setup(**self.paths)
158+
framework.run_tests()
159+
json_results = framework.save_results(self.output_path)
160+
161+
with open(self.test_cases_path, "r", encoding="utf-8") as f:
162+
test_configs = json.load(f)
163+
164+
self.assertEqual(len(json_results), len(test_configs["tests"]))
165+
166+
result_index = 0
167+
for i, test_config in enumerate(test_configs["tests"]):
168+
result = json_results[i]
169+
170+
if test_config.get("skip", False):
171+
self.assertEqual(result["skip"], True)
172+
self.assertEqual(result["passed"], None)
173+
self.assertEqual(result["result"]["status"], "skipped")
174+
else:
175+
test_case = framework.test_cases[result_index]
176+
result_index += 1
177+
178+
test_passed = (
179+
test_case.expected_causal_effect.apply(test_case.result.effect_estimate)
180+
if test_case.result.effect_estimate is not None
181+
else False
182+
)
183+
self.assertEqual(result["passed"], test_passed)
184+
185+
def test_ctf_exception(self):
186+
framework = CausalTestingFramework(self.paths)
187+
framework.setup(**self.paths, query="test_input < 0")
188+
189+
with self.assertRaises(ValueError):
190+
framework.run_tests()
191+
192+
def test_ctf_exception_silent(self):
193+
framework = CausalTestingFramework(self.paths)
194+
framework.setup(**self.paths, query="test_input < 0")
195+
196+
framework.run_tests(silent=True)
197+
json_results = framework.save_results(self.output_path)
198+
199+
with open(self.test_cases_path, "r", encoding="utf-8") as f:
200+
test_configs = json.load(f)
201+
202+
non_skipped_configs = [t for t in test_configs["tests"] if not t.get("skip", False)]
203+
non_skipped_results = [r for r in json_results if not r.get("skip", False)]
204+
205+
self.assertEqual(len(non_skipped_results), len(non_skipped_configs))
206+
207+
for result in non_skipped_results:
208+
self.assertEqual(result["passed"], False)
209+
210+
def test_ctf_evaluate_dag(self):
211+
framework = CausalTestingFramework(self.paths)
212+
framework.setup(**self.paths)
213+
results = framework.evaluate_dag()
214+
expected = pd.Series(
215+
{
216+
"PASS": 1,
217+
"FAIL": 0,
218+
"INESTIMABLE": 0,
219+
"PASS_ci_low": 0,
220+
"PASS_ci_high": 1,
221+
"FAIL_ci_low": 0,
222+
"FAIL_ci_high": 0,
223+
"INESTIMABLE_ci_low": 0,
224+
"INESTIMABLE_ci_high": 0,
225+
}
226+
).sort_index()
227+
pd.testing.assert_series_equal(results, expected)

0 commit comments

Comments
 (0)