Skip to content

Commit 14da903

Browse files
jmafoster1jmafoster1
committed
Fixed Negative and Positive causal effects so that both imply SomeEffect
1 parent 15de616 commit 14da903

5 files changed

Lines changed: 49 additions & 47 deletions

File tree

‎causal_testing/__main__.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import networkx as nx
1010
import pandas as pd
1111

12-
from causal_testing.causal_testing_framework import CausalTestingFramework
12+
from causal_testing.causal_testing_framework import CausalTestingFramework, read_dataframe
1313
from causal_testing.testing.metamorphic_relation import generate_causal_tests
1414

1515
logger = logging.getLogger(__name__)
@@ -205,9 +205,14 @@ def main() -> None:
205205
logging.info("Discovering causal structure")
206206
# Need to reset index to allow for multiple files having the same index (i.e. starting at zero).
207207
# Otherwise you end up with duplicate indices, which causes problems further down the line
208-
df = pd.concat([pd.read_csv(path) for path in args.data_paths]).reset_index()
208+
df = pd.concat([read_dataframe(path) for path in args.data_paths]).reset_index()
209209
if args.variables:
210210
df = df[args.variables]
211+
# Drop unnamed columns
212+
unnamed_columns = [c for c in df.columns if c.startswith("Unnamed: ")]
213+
if unnamed_columns:
214+
logger.warning(f"Dropping unnamed columns: {unnamed_columns}")
215+
df = df.loc[:, ~df.columns.str.contains("^Unnamed: ")]
211216

212217
discover_class = discover_map[args.technique].load()
213218
discover = discover_class(

‎causal_testing/discovery/abstract_discovery.py‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,13 +145,19 @@ def write_dot(self, individual: CausalDAG, output_file: str):
145145
for _, test in individual.test_results.iterrows():
146146
if (test["treatment"], test["outcome"]) in individual.edges:
147147
individual[test["treatment"]][test["outcome"]]["label"] = test["effect"]
148+
149+
print(test)
150+
148151
if test["result"] == TestResult.PASS:
152+
print(" GREEN")
149153
individual[test["treatment"]][test["outcome"]]["color"] = "green"
150154
individual[test["treatment"]][test["outcome"]]["fontcolor"] = "green"
151155
elif test["result"] == TestResult.INESTIMABLE:
156+
print(" ORANGE")
152157
individual[test["treatment"]][test["outcome"]]["color"] = "orange"
153158
individual[test["treatment"]][test["outcome"]]["fontcolor"] = "orange"
154159
elif test["result"] == TestResult.FAIL:
160+
print(" RED")
155161
individual[test["treatment"]][test["outcome"]]["color"] = "red"
156162
individual[test["treatment"]][test["outcome"]]["fontcolor"] = "red"
157163
else:
@@ -237,4 +243,6 @@ def evaluate_tests(self, causal_dag: CausalDAG) -> pd.DataFrame:
237243
)
238244

239245
causal_dag.test_results = pd.DataFrame(results)
246+
247+
results = pd.DataFrame(results)
240248
return pd.DataFrame(results)

‎causal_testing/testing/causal_effect.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,9 +126,13 @@ def apply(self, res: CausalTestResult) -> bool:
126126
if len(res.effect_estimate.value) > 1:
127127
raise ValueError("Positive Effects are currently only supported on single float datatypes")
128128
if res.effect_estimate.type in {"ate", "coefficient"}:
129-
return bool(res.effect_estimate.value[0] > 0)
129+
return any(
130+
0 < ci_low < ci_high for ci_low, ci_high in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high)
131+
)
130132
if res.effect_estimate.type in ["risk_ratio", "unit_odds_ratio"]:
131-
return bool(res.effect_estimate.value[0] > 1)
133+
return any(
134+
1 < ci_low < ci_high for ci_low, ci_high in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high)
135+
)
132136
raise ValueError(f"Test Value type {res.effect_estimate.type} is not valid for this CausalEffect")
133137

134138

@@ -140,8 +144,12 @@ def apply(self, res: CausalTestResult) -> bool:
140144
if len(res.effect_estimate.value) > 1:
141145
raise ValueError("Negative Effects are currently only supported on single float datatypes")
142146
if res.effect_estimate.type in {"ate", "coefficient"}:
143-
return bool(res.effect_estimate.value[0] < 0)
147+
return any(
148+
ci_low < ci_high < 0 for ci_low, ci_high in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high)
149+
)
144150
if res.effect_estimate.type in ["risk_ratio", "unit_odds_ratio"]:
145-
return bool(res.effect_estimate.value[0] < 1)
151+
return any(
152+
ci_low < ci_high < 1 for ci_low, ci_high in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high)
153+
)
146154
# Dead code but necessary for pylint
147155
raise ValueError(f"Test Value type {res.effect_estimate.type} is not valid for this CausalEffect")

‎tests/discovery_tests/test_abstract_discovery.py‎

Lines changed: 10 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import pandas as pd
77
from tempfile import TemporaryDirectory
88
import os
9+
from numpy import nan
910

1011
from causal_testing.discovery.abstract_discovery import TestResult, Discovery, simple_cycle
1112
from causal_testing.specification.causal_dag import CausalDAG
@@ -56,21 +57,25 @@ def test_simple_cycle_no_cycles(self):
5657
def test_effect_direction_positive(self):
5758
causal_test_case = CausalTestCase(base_test_case=self.base_test_case, expected_causal_effect=None)
5859
causal_test_case.result = CausalTestResult(
59-
effect_estimate=EffectEstimate(type="ate", value=pd.Series(5.05)),
60+
effect_estimate=EffectEstimate(
61+
type="ate", value=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6)
62+
),
6063
)
6164
self.assertEqual(self.abstract_discovery.effect_direction(causal_test_case), "positive")
6265

6366
def test_effect_direction_negative(self):
6467
causal_test_case = CausalTestCase(base_test_case=self.base_test_case, expected_causal_effect=None)
6568
causal_test_case.result = CausalTestResult(
66-
effect_estimate=EffectEstimate(type="ate", value=pd.Series(-5.05)),
69+
effect_estimate=EffectEstimate(
70+
type="ate", value=pd.Series(-5.05), ci_low=pd.Series(-6), ci_high=pd.Series(-5)
71+
),
6772
)
6873
self.assertEqual(self.abstract_discovery.effect_direction(causal_test_case), "negative")
6974

7075
def test_effect_direction_none(self):
7176
causal_test_case = CausalTestCase(base_test_case=self.base_test_case, expected_causal_effect=None)
7277
causal_test_case.result = CausalTestResult(
73-
effect_estimate=EffectEstimate(type="ate", value=pd.Series(0)),
78+
effect_estimate=EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)),
7479
)
7580
self.assertEqual(self.abstract_discovery.effect_direction(causal_test_case), None)
7681

@@ -210,73 +215,64 @@ def test_evaluate_tests_inestimable(self):
210215
"expected_effect": "NoEffect",
211216
"treatment": "length_in",
212217
"outcome": "large_gauge",
213-
"effect": "negative",
214218
},
215219
{
216220
"result": TestResult.PASS,
217221
"expected_effect": "NoEffect",
218222
"treatment": "large_gauge",
219223
"outcome": "length_in",
220-
"effect": "negative",
221224
},
222225
{
223226
"result": TestResult.PASS,
224227
"expected_effect": "NoEffect",
225228
"treatment": "length_in",
226229
"outcome": "color",
227-
"effect": None,
228230
},
229231
{
230232
"result": TestResult.PASS,
231233
"expected_effect": "NoEffect",
232234
"treatment": "color",
233235
"outcome": "length_in",
234-
"effect": None,
235236
},
236237
{
237238
"result": TestResult.FAIL,
238239
"expected_effect": "SomeEffect",
239240
"treatment": "length_in",
240241
"outcome": "completed",
241-
"effect": "positive",
242242
},
243243
{
244244
"result": TestResult.PASS,
245245
"expected_effect": "NoEffect",
246246
"treatment": "large_gauge",
247247
"outcome": "color",
248-
"effect": None,
249248
},
250249
{
251250
"result": TestResult.PASS,
252251
"expected_effect": "NoEffect",
253252
"treatment": "color",
254253
"outcome": "large_gauge",
255-
"effect": None,
256254
},
257255
{
258256
"result": TestResult.FAIL,
259257
"expected_effect": "SomeEffect",
260258
"treatment": "large_gauge",
261259
"outcome": "completed",
262-
"effect": "negative",
263260
},
264261
{
265262
"result": TestResult.INESTIMABLE,
266263
"expected_effect": "NoEffect",
267264
"treatment": "color",
268265
"outcome": "completed",
269-
"effect": None,
270266
},
271267
{
272268
"result": TestResult.INESTIMABLE,
273269
"expected_effect": "NoEffect",
274270
"treatment": "completed",
275271
"outcome": "color",
276-
"effect": None,
277272
},
278273
]
279274
)
275+
expected_results["effect"] = nan
280276
pd.testing.assert_frame_equal(test_results, expected_results)
281277

282278
def test_evaluate_tests(self):
@@ -295,71 +291,62 @@ def test_evaluate_tests(self):
295291
"expected_effect": "NoEffect",
296292
"treatment": "length_in",
297293
"outcome": "large_gauge",
298-
"effect": "positive",
299294
},
300295
{
301296
"result": TestResult.PASS,
302297
"expected_effect": "NoEffect",
303298
"treatment": "large_gauge",
304299
"outcome": "length_in",
305-
"effect": "positive",
306300
},
307301
{
308302
"result": TestResult.PASS,
309303
"expected_effect": "NoEffect",
310304
"treatment": "length_in",
311305
"outcome": "color",
312-
"effect": None,
313306
},
314307
{
315308
"result": TestResult.PASS,
316309
"expected_effect": "NoEffect",
317310
"treatment": "color",
318311
"outcome": "length_in",
319-
"effect": None,
320312
},
321313
{
322314
"result": TestResult.FAIL,
323315
"expected_effect": "SomeEffect",
324316
"treatment": "length_in",
325317
"outcome": "completed",
326-
"effect": "negative",
327318
},
328319
{
329320
"result": TestResult.PASS,
330321
"expected_effect": "NoEffect",
331322
"treatment": "large_gauge",
332323
"outcome": "color",
333-
"effect": None,
334324
},
335325
{
336326
"result": TestResult.PASS,
337327
"expected_effect": "NoEffect",
338328
"treatment": "color",
339329
"outcome": "large_gauge",
340-
"effect": None,
341330
},
342331
{
343332
"result": TestResult.FAIL,
344333
"expected_effect": "SomeEffect",
345334
"treatment": "large_gauge",
346335
"outcome": "completed",
347-
"effect": "positive",
348336
},
349337
{
350338
"result": TestResult.PASS,
351339
"expected_effect": "NoEffect",
352340
"treatment": "color",
353341
"outcome": "completed",
354-
"effect": None,
355342
},
356343
{
357344
"result": TestResult.PASS,
358345
"expected_effect": "NoEffect",
359346
"treatment": "completed",
360347
"outcome": "color",
361-
"effect": None,
362348
},
363349
]
364350
)
351+
expected_results["effect"] = None
365352
pd.testing.assert_frame_equal(test_results, expected_results)

‎tests/testing_tests/test_causal_effect.py‎

Lines changed: 12 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -39,26 +39,23 @@ def test_empty_adjustment_set(self):
3939

4040
def test_Positive_ate_pass(self):
4141
ctr = CausalTestResult(
42-
effect_estimate=EffectEstimate(type="ate", value=pd.Series(5.05)),
42+
effect_estimate=EffectEstimate(
43+
type="ate", value=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6)
44+
),
4345
)
4446
ev = Positive()
4547
self.assertTrue(ev.apply(ctr))
4648

4749
def test_Positive_risk_ratio_pass(self):
4850
ctr = CausalTestResult(
49-
effect_estimate=EffectEstimate(type="risk_ratio", value=pd.Series(5.05)),
51+
effect_estimate=EffectEstimate(
52+
type="risk_ratio", value=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6)
53+
),
5054
)
5155
ev = Positive()
5256
self.assertTrue(ev.apply(ctr))
5357

5458
def test_Positive_fail(self):
55-
ctr = CausalTestResult(
56-
effect_estimate=EffectEstimate(type="ate", value=pd.Series(0)),
57-
)
58-
ev = Positive()
59-
self.assertFalse(ev.apply(ctr))
60-
61-
def test_Positive_fail_ci(self):
6259
ctr = CausalTestResult(
6360
effect_estimate=EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)),
6461
)
@@ -67,26 +64,23 @@ def test_Positive_fail_ci(self):
6764

6865
def test_Negative_ate_pass(self):
6966
ctr = CausalTestResult(
70-
effect_estimate=EffectEstimate(type="ate", value=pd.Series(-5.05)),
67+
effect_estimate=EffectEstimate(
68+
type="ate", value=pd.Series(-5.05), ci_low=pd.Series(-6), ci_high=pd.Series(-5)
69+
),
7170
)
7271
ev = Negative()
7372
self.assertTrue(ev.apply(ctr))
7473

7574
def test_Negative_risk_ratio_pass(self):
7675
ctr = CausalTestResult(
77-
effect_estimate=EffectEstimate(type="risk_ratio", value=pd.Series(0.2)),
76+
effect_estimate=EffectEstimate(
77+
type="risk_ratio", value=pd.Series(0.2), ci_low=pd.Series(0.1), ci_high=pd.Series(0.5)
78+
),
7879
)
7980
ev = Negative()
8081
self.assertTrue(ev.apply(ctr))
8182

8283
def test_Negative_fail(self):
83-
ctr = CausalTestResult(
84-
effect_estimate=EffectEstimate(type="ate", value=pd.Series(0)),
85-
)
86-
ev = Negative()
87-
self.assertFalse(ev.apply(ctr))
88-
89-
def test_Negative_fail_ci(self):
9084
ctr = CausalTestResult(
9185
effect_estimate=EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)),
9286
)

0 commit comments

Comments
 (0)