From 87d973d595f7adb6b1d93c6d542c04c7ff4bdcc3 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Fri, 17 Jul 2026 14:56:12 +0100 Subject: [PATCH 1/7] Causal DAG analysis (and restructuring of CausalTestResult) --- causal_testing/__main__.py | 63 ++++++- causal_testing/causal_testing_framework.py | 50 ++++- .../discovery/abstract_discovery.py | 28 ++- .../discovery/hill_climber_discovery.py | 17 +- causal_testing/testing/causal_effect.py | 82 ++++---- causal_testing/testing/causal_test_case.py | 57 +++--- causal_testing/testing/causal_test_result.py | 7 +- .../covasim_/doubling_beta/example_beta.py | 4 +- .../vaccinating_elderly/example_vaccine.py | 2 +- examples/lr91/example_max_conductances.py | 12 +- .../test_abstract_discovery.py | 60 +++--- .../test_hill_climber_discovery.py | 27 +-- tests/main_tests/test_main.py | 2 +- tests/testing_tests/test_causal_effect.py | 176 ++++++------------ tests/testing_tests/test_causal_test_case.py | 22 +-- 15 files changed, 309 insertions(+), 300 deletions(-) diff --git a/causal_testing/__main__.py b/causal_testing/__main__.py index eebe0a13..75d2f1a8 100644 --- a/causal_testing/__main__.py +++ b/causal_testing/__main__.py @@ -23,6 +23,7 @@ class Command(Enum): TEST = "test" GENERATE = "generate" DISCOVER = "discover" + EVALUATE = "evaluate" def setup_logging(level: str) -> None: @@ -48,6 +49,15 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace: choices=["NONE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], help="Set the logging level (default: WARNING).", ) + main_parser.add_argument( + "-a", + "--alpha", + help=( + "The significance level of the confidence intervals used to determine causality. " + "This should be a value between 0 and 1. Defaults to 0.05 for 95%% confidence intervals." + ), + default=0.05, + ) subparsers = main_parser.add_subparsers( help="The action you want to run - call `causal_testing {action} -h` for further details", dest="command" @@ -108,18 +118,36 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace: default=False, ) + # DAG evaluation + parser_evaluate = subparsers.add_parser( + Command.EVALUATE.value, help="Evaluate how well a causal DAG fits a dataset" + ) + parser_evaluate.add_argument("-D", "--dag-path", help="Path to the DAG file (.dot)", required=True) + parser_evaluate.add_argument("-o", "--output", help="Path for output file (.csv)", required=True) + parser_evaluate.add_argument( + "-i", "--ignore-cycles", help="Ignore cycles in DAG", action="store_true", default=False + ) + parser_evaluate.add_argument("-d", "--data-paths", help="Paths to data files (.csv)", nargs="+", required=True) + parser_evaluate.add_argument("-q", "--query", help="Query string to filter data (e.g. 'age > 18')", type=str) + parser_evaluate.add_argument( + "-b", + "--adequacy-bootstrap-size", + dest="bootstrap_size", + help="Number of bootstrap samples for causal test adequacy. Defaults to 100", + type=int, + default=100, + ) + parser_evaluate.add_argument( + "-s", + "--silent", + action="store_true", + help="Do not crash on error. If set to true, errors are recorded as test results.", + default=False, + ) + # Discovery parser_discover = subparsers.add_parser(Command.DISCOVER.value, help="Discover causal structures from data") parser_discover.add_argument("-d", "--data-paths", help="Paths to data files (.csv)", nargs="+", required=True) - parser_discover.add_argument( - "-a", - "--alpha", - help=( - "The significance level of the confidence intervals used to determine causality. " - "This should be a value between 0 and 1. Defaults to 0.05 for 95%% confidence intervals." - ), - default=0.05, - ) parser_discover.add_argument( "-t", "--technique", @@ -246,6 +274,23 @@ def main() -> None: framework.save_results(args.output) logging.info("Causal testing completed successfully.") + case Command.EVALUATE: + # Create and setup framework + framework = CausalTestingFramework() + + framework.setup( + dag_path=args.dag_path, + data_paths=args.data_paths, + test_cases_path=args.test_config, + query=args.query, + ignore_cycles=args.ignore_cycles, + ) + + logging.info("Running tests on entire dataset") + results = framework.evaluate_dag(alpha=args.alpha, bootstrap_size=args.bootstrap_size) + logging.info("Causal testing completed successfully.") + logging.info("Running tests on bootstrap samples") + results.to_csv(args.output) if __name__ == "__main__": diff --git a/causal_testing/causal_testing_framework.py b/causal_testing/causal_testing_framework.py index a364036b..ef2712ed 100644 --- a/causal_testing/causal_testing_framework.py +++ b/causal_testing/causal_testing_framework.py @@ -14,6 +14,7 @@ from causal_testing.specification.variable import Input, Output from causal_testing.testing.base_test_case import BaseTestCase from causal_testing.testing.causal_test_case import CausalTestCase +from causal_testing.testing.causal_test_result import TestOutcome logger = logging.getLogger(__name__) @@ -265,6 +266,49 @@ def run_tests(self, silent: bool = False, adequacy: bool = False, bootstrap_size self.df, suppress_estimation_errors=silent, adequacy=adequacy, bootstrap_size=bootstrap_size ) + def evaluate_dag(self, bootstrap_size: bool, alpha: float) -> pd.Series: + """ + Calculate confidence intervals for how well a causal DAG fits a dataset by repeatedly resampling the dataset + and executing the causal tests. + Confidence intervals are then calcualted for how many tests pass, fail, or are inestimable. + + :param bootstrap_size: The number of bootstrap samples to use when calculating causal test adequacy + (defaults to 100) + :param alpha: The significance level to use when calculating confidence intervals. + """ + self.run_tests(silent=True, adequacy=False) + results = { + test_outcome: len([test for test in self.test_cases if test.result.outcome == test_outcome]) + for test_outcome in TestOutcome + } + + sample_results = [] + for sample_index in range(bootstrap_size): + test_outcomes = {test_outcome: 0 for test_outcome in TestOutcome} + for test_case in tqdm(self.test_cases): + effect_estimate = test_case.estimate_effect( + df=self.df.sample(len(self.df), replace=True, random_state=sample_index) + ) + if effect_estimate: + if test_case.expected_causal_effect.apply(effect_estimate): + test_outcomes[TestOutcome.PASS] += 1 + else: + test_outcomes[TestOutcome.FAIL] += 1 + else: + test_outcomes[TestOutcome.INESTIMABLE] += 1 + sample_results.append(test_outcomes) + + sample_results = pd.DataFrame(sample_results) + # Calculate the confidence interval of each column + ci_low_inx = (alpha / 2) * bootstrap_size + ci_high_inx = ((1 - alpha) / 2) * bootstrap_size + for outcome in TestOutcome: + data = sorted(sample_results[outcome]) + results[f"{outcome}_ci_low"] = data[ci_low_inx] + results[f"{outcome}_ci_high"] = data[ci_high_inx] + + return pd.Series(results).sort_index() + def save_results(self, output_path) -> list: """Save test results to JSON file in the expected format.""" logger.info(f"Saving results to {output_path}") @@ -301,15 +345,11 @@ def save_results(self, output_path) -> list: result = test_case.result result_index += 1 - test_passed = ( - test_case.expected_causal_effect.apply(result) if result.effect_estimate is not None else False - ) - output = { **base_output, "formula": test_case.estimator.formula if hasattr(test_case.estimator, "formula") else None, "skip": False, - "passed": test_passed, + "passed": test_case.result.outcome == TestOutcome.PASS, "result": ( { "treatment": test_case.estimator.base_test_case.treatment_variable.name, diff --git a/causal_testing/discovery/abstract_discovery.py b/causal_testing/discovery/abstract_discovery.py index ee599ad9..3e10e797 100644 --- a/causal_testing/discovery/abstract_discovery.py +++ b/causal_testing/discovery/abstract_discovery.py @@ -6,7 +6,6 @@ import re import warnings from abc import ABC, abstractmethod -from enum import Enum from itertools import permutations import networkx as nx @@ -19,10 +18,9 @@ from causal_testing.specification.scenario import Scenario from causal_testing.testing.causal_effect import Negative, Positive from causal_testing.testing.causal_test_case import CausalTestCase +from causal_testing.testing.causal_test_result import TestOutcome from causal_testing.testing.metamorphic_relation import generate_metamorphic_relations -TestResult = Enum("TestResult", [("PASS", 2), ("FAIL", 0), ("INESTIMABLE", 1)]) - # Ignore warnings from statsmodels when we try to evaluate test cases warnings.simplefilter("ignore") @@ -111,9 +109,9 @@ def effect_direction(self, test_case: CausalTestCase) -> str: if pd.api.types.is_numeric_dtype( self.df[test_case.base_test_case.treatment_variable.name] ) and pd.api.types.is_numeric_dtype(self.df[test_case.base_test_case.outcome_variable.name]): - if Negative().apply(test_case.result): + if Negative().apply(test_case.result.effect_estimate): return "negative" - if Positive().apply(test_case.result): + if Positive().apply(test_case.result.effect_estimate): return "positive" return None @@ -148,15 +146,15 @@ def write_dot(self, individual: CausalDAG, output_file: str): print(test) - if test["result"] == TestResult.PASS: + if test["result"] == TestOutcome.PASS: print(" GREEN") individual[test["treatment"]][test["outcome"]]["color"] = "green" individual[test["treatment"]][test["outcome"]]["fontcolor"] = "green" - elif test["result"] == TestResult.INESTIMABLE: + elif test["result"] == TestOutcome.INESTIMABLE: print(" ORANGE") individual[test["treatment"]][test["outcome"]]["color"] = "orange" individual[test["treatment"]][test["outcome"]]["fontcolor"] = "orange" - elif test["result"] == TestResult.FAIL: + elif test["result"] == TestOutcome.FAIL: print(" RED") individual[test["treatment"]][test["outcome"]]["color"] = "red" individual[test["treatment"]][test["outcome"]]["fontcolor"] = "red" @@ -166,13 +164,13 @@ def write_dot(self, individual: CausalDAG, output_file: str): individual.add_edge(test["treatment"], test["outcome"], ignore_cycles=True) individual[test["treatment"]][test["outcome"]]["style"] = "dashed" individual[test["treatment"]][test["outcome"]]["label"] = test["effect"] - if test["result"] == TestResult.PASS: + if test["result"] == TestOutcome.PASS: individual[test["treatment"]][test["outcome"]]["style"] = "invis" individual[test["treatment"]][test["outcome"]]["constraint"] = False - elif test["result"] == TestResult.INESTIMABLE: + elif test["result"] == TestOutcome.INESTIMABLE: individual[test["treatment"]][test["outcome"]]["color"] = "orange" individual[test["treatment"]][test["outcome"]]["fontcolor"] = "orange" - elif test["result"] == TestResult.FAIL: + elif test["result"] == TestOutcome.FAIL: individual[test["treatment"]][test["outcome"]]["color"] = "red" individual[test["treatment"]][test["outcome"]]["fontcolor"] = "red" else: @@ -222,9 +220,9 @@ def evaluate_tests(self, causal_dag: CausalDAG) -> pd.DataFrame: results.append( { "result": ( - TestResult.PASS - if test_case.expected_causal_effect.apply(test_case.result) - else TestResult.FAIL + TestOutcome.PASS + if test_case.expected_causal_effect.apply(test_case.result.effect_estimate) + else TestOutcome.FAIL ), "expected_effect": test_case.expected_causal_effect.__class__.__name__, "treatment": test_case.base_test_case.treatment_variable.name, @@ -235,7 +233,7 @@ def evaluate_tests(self, causal_dag: CausalDAG) -> pd.DataFrame: except np.linalg.LinAlgError: results.append( { - "result": TestResult.INESTIMABLE, + "result": TestOutcome.INESTIMABLE, "expected_effect": test_case.expected_causal_effect.__class__.__name__, "treatment": test_case.base_test_case.treatment_variable.name, "outcome": test_case.base_test_case.outcome_variable.name, diff --git a/causal_testing/discovery/hill_climber_discovery.py b/causal_testing/discovery/hill_climber_discovery.py index c9e057bb..3faf506a 100644 --- a/causal_testing/discovery/hill_climber_discovery.py +++ b/causal_testing/discovery/hill_climber_discovery.py @@ -8,8 +8,9 @@ import numpy as np import pandas as pd -from causal_testing.discovery.abstract_discovery import Discovery, TestResult +from causal_testing.discovery.abstract_discovery import Discovery from causal_testing.specification.causal_dag import CausalDAG +from causal_testing.testing.causal_test_result import TestOutcome class HillClimberDiscovery(Discovery): @@ -48,10 +49,10 @@ def sum_test_outcomes(self, test_results: pd.DataFrame) -> dict: axis=1, ) # Ensure every column is initialised - Test outcomes that never occurred won't be in the dataframe otherwise - for col in TestResult: + for col in TestOutcome: if col not in counts.columns: counts[col] = 0 - counts = counts.groupby(["treatment", "outcome"]).sum().reset_index()[list(TestResult)] + counts = counts.groupby(["treatment", "outcome"]).sum().reset_index()[list(TestOutcome)] # The below line normalises by the number of tests *for each edge* # Independence tests X _||_ Y get two tests (X _||_ Y and Y _||_ X) because we don't know which way the # causality flows. We need to normalise this (e.g. if X _||_ Y and Y _||_ X both pass, then the score should be @@ -89,15 +90,15 @@ def evaluate_fitness( ) problem_tests = query_df.groupby(["var1", "var2"]).filter( # Groups are problematic if at least one test fails or no test passes - lambda group: (group["result"] == TestResult.FAIL).any() - or ~(group["result"] == TestResult.PASS).any() + lambda group: (group["result"] == TestOutcome.FAIL).any() + or ~(group["result"] == TestOutcome.PASS).any() ) problem_edges = problem_tests[["treatment", "outcome"]].apply(tuple, axis=1).tolist() fitness_values = ( - counts.get(TestResult.PASS, 0), - -counts.get(TestResult.FAIL, 0), - -counts.get(TestResult.INESTIMABLE, 0), + counts.get(TestOutcome.PASS, 0), + -counts.get(TestOutcome.FAIL, 0), + -counts.get(TestOutcome.INESTIMABLE, 0), ) return fitness_values, problem_edges diff --git a/causal_testing/testing/causal_effect.py b/causal_testing/testing/causal_effect.py index efa34732..4247cce4 100644 --- a/causal_testing/testing/causal_effect.py +++ b/causal_testing/testing/causal_effect.py @@ -7,16 +7,16 @@ import numpy as np -from causal_testing.testing.causal_test_result import CausalTestResult +from causal_testing.estimation.effect_estimate import EffectEstimate class CausalEffect(ABC): """An abstract class representing an expected causal effect.""" @abstractmethod - def apply(self, res: CausalTestResult) -> bool: + def apply(self, effect_estimate: EffectEstimate) -> bool: """Abstract apply method that should return a bool representing if the result meets the outcome - :param res: CausalTestResult to be checked + :param effect_estimate: EffectEstimate to be checked :return: Bool that is true if outcome is met """ @@ -27,21 +27,21 @@ def __str__(self) -> str: class SomeEffect(CausalEffect): """An extension of CausalEffect representing that the expected causal effect should not be zero.""" - def apply(self, res: CausalTestResult) -> bool: - if res.effect_estimate.ci_low is None or res.effect_estimate.ci_high is None: + def apply(self, effect_estimate: EffectEstimate) -> bool: + if effect_estimate.ci_low is None or effect_estimate.ci_high is None: return None - if res.effect_estimate.type in ("risk_ratio", "hazard_ratio", "unit_odds_ratio"): + if effect_estimate.type in ("risk_ratio", "hazard_ratio", "unit_odds_ratio"): return any( 1 < ci_low < ci_high or ci_low < ci_high < 1 - for ci_low, ci_high in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high) + for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high) ) - if res.effect_estimate.type in ("coefficient", "ate"): + if effect_estimate.type in ("coefficient", "ate"): return any( 0 < ci_low < ci_high or ci_low < ci_high < 0 - for ci_low, ci_high in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high) + for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high) ) - raise ValueError(f"Test Value type {res.effect_estimate.type} is not valid for this CausalEffect") + raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect") class NoEffect(CausalEffect): @@ -56,30 +56,26 @@ def __init__(self, atol: float = 1e-10, ctol: float = 0.05): self.atol = atol self.ctol = ctol - def apply(self, res: CausalTestResult) -> bool: - if res.effect_estimate.type in ("risk_ratio", "hazard_ratio", "unit_odds_ratio", "odds_ratio"): + def apply(self, effect_estimate: EffectEstimate) -> bool: + if effect_estimate.type in ("risk_ratio", "hazard_ratio", "unit_odds_ratio", "odds_ratio"): return any( ci_low < 1 < ci_high or np.isclose(value, 1.0, atol=self.atol) for ci_low, ci_high, value in zip( - res.effect_estimate.ci_low, res.effect_estimate.ci_high, res.effect_estimate.value + effect_estimate.ci_low, effect_estimate.ci_high, effect_estimate.value ) ) - if res.effect_estimate.type in ("coefficient", "ate"): - value = ( - res.effect_estimate.value - if isinstance(res.effect_estimate.ci_high, Iterable) - else [res.effect_estimate.value] - ) + if effect_estimate.type in ("coefficient", "ate"): + value = effect_estimate.value if isinstance(effect_estimate.ci_high, Iterable) else [effect_estimate.value] return ( sum( not ((ci_low < 0 < ci_high) or abs(v) < self.atol) - for ci_low, ci_high, v in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high, value) + for ci_low, ci_high, v in zip(effect_estimate.ci_low, effect_estimate.ci_high, value) ) / len(value) < self.ctol ) - raise ValueError(f"Test Value type {res.effect_estimate.type} is not valid for this CausalEffect") + raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect") class ExactValue(CausalEffect): @@ -105,12 +101,12 @@ def __init__(self, value: float, atol: float = None, ci_low: float = None, ci_hi "Try specifying a smaller value of atol." ) - def apply(self, res: CausalTestResult) -> bool: - close = np.isclose(res.effect_estimate.value, self.value, atol=self.atol) - if res.effect_estimate.ci_valid and self.ci_low is not None and self.ci_high is not None: + def apply(self, effect_estimate: EffectEstimate) -> bool: + close = np.isclose(effect_estimate.value, self.value, atol=self.atol) + if effect_estimate.ci_valid and self.ci_low is not None and self.ci_high is not None: return all( close and self.ci_low <= ci_low and self.ci_high >= ci_high - for ci_low, ci_high in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high) + for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high) ) return close @@ -122,34 +118,26 @@ class Positive(SomeEffect): """An extension of CausalEffect representing that the expected causal effect should be positive. Currently only single values are supported for the test value""" - def apply(self, res: CausalTestResult) -> bool: - if len(res.effect_estimate.value) > 1: + def apply(self, effect_estimate: EffectEstimate) -> bool: + if len(effect_estimate.value) > 1: raise ValueError("Positive Effects are currently only supported on single float datatypes") - if res.effect_estimate.type in {"ate", "coefficient"}: - return any( - 0 < ci_low < ci_high for ci_low, ci_high in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high) - ) - if res.effect_estimate.type in ["risk_ratio", "unit_odds_ratio"]: - return any( - 1 < ci_low < ci_high for ci_low, ci_high in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high) - ) - raise ValueError(f"Test Value type {res.effect_estimate.type} is not valid for this CausalEffect") + if effect_estimate.type in {"ate", "coefficient"}: + return any(0 < ci_low < ci_high for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high)) + if effect_estimate.type in ["risk_ratio", "unit_odds_ratio"]: + return any(1 < ci_low < ci_high for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high)) + raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect") class Negative(SomeEffect): """An extension of CausalEffect representing that the expected causal effect should be negative. Currently only single values are supported for the test value""" - def apply(self, res: CausalTestResult) -> bool: - if len(res.effect_estimate.value) > 1: + def apply(self, effect_estimate: EffectEstimate) -> bool: + if len(effect_estimate.value) > 1: raise ValueError("Negative Effects are currently only supported on single float datatypes") - if res.effect_estimate.type in {"ate", "coefficient"}: - return any( - ci_low < ci_high < 0 for ci_low, ci_high in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high) - ) - if res.effect_estimate.type in ["risk_ratio", "unit_odds_ratio"]: - return any( - ci_low < ci_high < 1 for ci_low, ci_high in zip(res.effect_estimate.ci_low, res.effect_estimate.ci_high) - ) + if effect_estimate.type in {"ate", "coefficient"}: + return any(ci_low < ci_high < 0 for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high)) + if effect_estimate.type in ["risk_ratio", "unit_odds_ratio"]: + return any(ci_low < ci_high < 1 for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high)) # Dead code but necessary for pylint - raise ValueError(f"Test Value type {res.effect_estimate.type} is not valid for this CausalEffect") + raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect") diff --git a/causal_testing/testing/causal_test_case.py b/causal_testing/testing/causal_test_case.py index 8b32df32..bc04d73d 100644 --- a/causal_testing/testing/causal_test_case.py +++ b/causal_testing/testing/causal_test_case.py @@ -8,7 +8,7 @@ from causal_testing.estimation.abstract_estimator import Estimator from causal_testing.testing.base_test_case import BaseTestCase from causal_testing.testing.causal_effect import CausalEffect -from causal_testing.testing.causal_test_result import CausalTestResult +from causal_testing.testing.causal_test_result import CausalTestResult, TestOutcome from causal_testing.testing.data_adequacy import DataAdequacy logger = logging.getLogger(__name__) @@ -73,9 +73,9 @@ def measure_adequacy( else: df = df.sample(len(df), replace=True, random_state=i) try: - result = self.estimate_effect(df) - outcomes.append(self.expected_causal_effect.apply(result)) - results.append(result.effect_estimate.to_df()) + effect_estimate = self.estimate_effect(df) + outcomes.append(self.expected_causal_effect.apply(effect_estimate)) + results.append(effect_estimate.to_df()) # Could get a variety of exceptions here due to insufficient/badly formed data # We don't want these to stop execution except Exception: # pylint: disable=W0718 @@ -96,7 +96,6 @@ def measure_adequacy( def execute_test( self, df: pd.DataFrame, - estimate_params: dict[str, any] = None, adequacy: bool = False, suppress_estimation_errors: bool = False, bootstrap_size: int = 100, @@ -106,7 +105,6 @@ def execute_test( Execute a causal test case. :param df: The data to use. - :param estimate_params: Extra parameters for the estimate calculation. :param adequacy: Set to True to calculate the causal test adequacy associated with the effect estimate. :param suppress_estimation_errors: Set to True to suppress estimation errors. (Defaults to False) :param bootstrap_size: The number of bootstrap samples to use. (Defaults to 100) @@ -115,24 +113,33 @@ def execute_test( :return causal_test_result: A CausalTestResult for the executed causal test case. """ if not self.skip: - self.result = self.estimate_effect( - df=df, estimate_params=estimate_params, suppress_estimation_errors=suppress_estimation_errors - ) - if adequacy: - self.result.adequacy = self.measure_adequacy(df=df, bootstrap_size=bootstrap_size, group_by=group_by) - - def estimate_effect( - self, - df: pd.DataFrame, - estimate_params: dict[str, any] = None, - suppress_estimation_errors: bool = False, - ) -> CausalTestResult: + try: + effect_estimate = self.estimate_effect(df=df) + self.result = CausalTestResult( + effect_estimate=effect_estimate, + outcome=( + TestOutcome.PASS + if self.expected_causal_effect.apply(effect_estimate=effect_estimate) + else TestOutcome.FAIL + ), + adequacy=( + self.measure_adequacy(df=df, bootstrap_size=bootstrap_size, group_by=group_by) + if adequacy + else None + ), + ) + except (np.linalg.LinAlgError, ValueError) as e: + if not suppress_estimation_errors: + raise e + self.result = CausalTestResult( + effect_estimate=None, outcome=TestOutcome.INESTIMABLE, error_message=str(e) + ) + + def estimate_effect(self, df: pd.DataFrame) -> CausalTestResult: """ Execute a causal test case and return the causal test result. :param df: The data to use. - :param estimate_params: Extra parameters for the estimate calculation. - :param suppress_estimation_errors: Set to True to suppress estimation errors. (Defaults to False) :return causal_test_result: A CausalTestResult for the executed causal test case. """ if self.query: @@ -140,15 +147,7 @@ def estimate_effect( if not hasattr(self.estimator, f"estimate_{self.estimate_type}"): raise AttributeError(f"{self.estimator.__class__} has no {self.estimate_type} method.") estimate_effect = getattr(self.estimator, f"estimate_{self.estimate_type}") - try: - effect_estimate = estimate_effect(df, **(estimate_params if estimate_params is not None else {})) - return CausalTestResult( - effect_estimate=effect_estimate, - ) - except (np.linalg.LinAlgError, ValueError) as e: - if not suppress_estimation_errors: - raise e - return CausalTestResult(effect_estimate=None, error_message=str(e)) + return estimate_effect(df) def __str__(self): treatment_config = {self.treatment_variable.name: self.estimator.treatment_value} diff --git a/causal_testing/testing/causal_test_result.py b/causal_testing/testing/causal_test_result.py index 79d17506..9e99894d 100644 --- a/causal_testing/testing/causal_test_result.py +++ b/causal_testing/testing/causal_test_result.py @@ -1,9 +1,12 @@ """This module contains the CausalTestResult class, which is a container for the results of a causal test.""" from dataclasses import dataclass +from enum import Enum from causal_testing.estimation.effect_estimate import EffectEstimate +TestOutcome = Enum("TestOutcome", [("PASS", 2), ("FAIL", 0), ("INESTIMABLE", 1)]) + @dataclass class CausalTestResult: @@ -14,9 +17,11 @@ class CausalTestResult: def __init__( self, effect_estimate: EffectEstimate, + outcome: TestOutcome, adequacy=None, error_message: str = None, ): - self.adequacy = adequacy + self.outcome = outcome self.effect_estimate = effect_estimate + self.adequacy = adequacy self.error_message = error_message diff --git a/examples/covasim_/doubling_beta/example_beta.py b/examples/covasim_/doubling_beta/example_beta.py index 32cb2665..b48923c1 100644 --- a/examples/covasim_/doubling_beta/example_beta.py +++ b/examples/covasim_/doubling_beta/example_beta.py @@ -47,7 +47,7 @@ def doubling_beta_CATE_on_csv( # 6. Create a causal test case causal_test_case = CausalTestCase( base_test_case=base_test_case, - expected_causal_effect=Positive, + expected_causal_effect=Positive(), estimator=LinearRegressionEstimator( base_test_case=base_test_case, treatment_value=0.032, @@ -63,7 +63,7 @@ def doubling_beta_CATE_on_csv( # Repeat for association estimate (no adjustment) causal_test_case = CausalTestCase( base_test_case=base_test_case, - expected_causal_effect=Positive, + expected_causal_effect=Positive(), estimator=LinearRegressionEstimator( base_test_case=base_test_case, treatment_value=0.032, diff --git a/examples/covasim_/vaccinating_elderly/example_vaccine.py b/examples/covasim_/vaccinating_elderly/example_vaccine.py index b877b075..59b9a539 100644 --- a/examples/covasim_/vaccinating_elderly/example_vaccine.py +++ b/examples/covasim_/vaccinating_elderly/example_vaccine.py @@ -70,7 +70,7 @@ def run_test_case(verbose: bool = False): ] results_dict[outcome_variable.name]["test_passes"] = causal_test_case.expected_causal_effect.apply( - causal_test_case.result + causal_test_case.result.effect_estimate ) return results_dict diff --git a/examples/lr91/example_max_conductances.py b/examples/lr91/example_max_conductances.py index ad9d4801..38dc1b40 100644 --- a/examples/lr91/example_max_conductances.py +++ b/examples/lr91/example_max_conductances.py @@ -38,12 +38,12 @@ def test_sensitivity_analysis(): # Read in the 200 model runs and define mean value and expected effect model_runs = pd.read_csv(f"{ROOT}/data/results.csv") conductance_means = { - "G_K": (0.5, Positive), - "G_b": (0.5, Positive), - "G_K1": (0.5, Positive), - "G_si": (0.5, Negative), - "G_Na": (0.5, NoEffect), - "G_Kp": (0.5, NoEffect), + "G_K": (0.5, Positive()), + "G_b": (0.5, Positive()), + "G_K1": (0.5, Positive()), + "G_si": (0.5, Negative()), + "G_Na": (0.5, NoEffect()), + "G_Kp": (0.5, NoEffect()), } # Normalise the inputs as per the original study diff --git a/tests/discovery_tests/test_abstract_discovery.py b/tests/discovery_tests/test_abstract_discovery.py index 1f9d9e34..888f8b29 100644 --- a/tests/discovery_tests/test_abstract_discovery.py +++ b/tests/discovery_tests/test_abstract_discovery.py @@ -7,10 +7,9 @@ from tempfile import TemporaryDirectory import os from numpy import nan - -from causal_testing.discovery.abstract_discovery import TestResult, Discovery, simple_cycle +from causal_testing.discovery.abstract_discovery import Discovery, simple_cycle from causal_testing.specification.causal_dag import CausalDAG -from causal_testing.testing.causal_test_result import CausalTestResult +from causal_testing.testing.causal_test_result import CausalTestResult, TestOutcome from causal_testing.testing.causal_test_case import CausalTestCase from causal_testing.estimation.effect_estimate import EffectEstimate from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator @@ -57,6 +56,7 @@ def test_simple_cycle_no_cycles(self): def test_effect_direction_positive(self): causal_test_case = CausalTestCase(base_test_case=self.base_test_case, expected_causal_effect=None) causal_test_case.result = CausalTestResult( + outcome=None, effect_estimate=EffectEstimate( type="ate", value=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6) ), @@ -66,6 +66,7 @@ def test_effect_direction_positive(self): def test_effect_direction_negative(self): causal_test_case = CausalTestCase(base_test_case=self.base_test_case, expected_causal_effect=None) causal_test_case.result = CausalTestResult( + outcome=None, effect_estimate=EffectEstimate( type="ate", value=pd.Series(-5.05), ci_low=pd.Series(-6), ci_high=pd.Series(-5) ), @@ -75,6 +76,7 @@ def test_effect_direction_negative(self): def test_effect_direction_none(self): causal_test_case = CausalTestCase(base_test_case=self.base_test_case, expected_causal_effect=None) causal_test_case.result = CausalTestResult( + outcome=None, effect_estimate=EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)), ) self.assertEqual(self.abstract_discovery.effect_direction(causal_test_case), None) @@ -146,13 +148,13 @@ def test_write_dot(self): dag.add_edges_from([("A", "B"), ("C", "D"), ("E", "F")]) dag.test_results = pd.DataFrame( [ # Edges - {"treatment": "A", "outcome": "B", "effect": "positive", "result": TestResult.PASS}, - {"treatment": "C", "outcome": "D", "effect": "positive", "result": TestResult.FAIL}, - {"treatment": "E", "outcome": "F", "effect": "None", "result": TestResult.INESTIMABLE}, + {"treatment": "A", "outcome": "B", "effect": "positive", "result": TestOutcome.PASS}, + {"treatment": "C", "outcome": "D", "effect": "positive", "result": TestOutcome.FAIL}, + {"treatment": "E", "outcome": "F", "effect": "None", "result": TestOutcome.INESTIMABLE}, # Independences - {"treatment": "A", "outcome": "C", "effect": None, "result": TestResult.PASS}, - {"treatment": "A", "outcome": "D", "effect": "negative", "result": TestResult.FAIL}, - {"treatment": "A", "outcome": "E", "effect": None, "result": TestResult.INESTIMABLE}, + {"treatment": "A", "outcome": "C", "effect": None, "result": TestOutcome.PASS}, + {"treatment": "A", "outcome": "D", "effect": "negative", "result": TestOutcome.FAIL}, + {"treatment": "A", "outcome": "E", "effect": None, "result": TestOutcome.INESTIMABLE}, ] ) abstract_discovery = AbstractDiscovery(pd.DataFrame()) @@ -211,61 +213,61 @@ def test_evaluate_tests_inestimable(self): expected_results = pd.DataFrame( [ { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "length_in", "outcome": "large_gauge", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "large_gauge", "outcome": "length_in", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "length_in", "outcome": "color", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "color", "outcome": "length_in", }, { - "result": TestResult.FAIL, + "result": TestOutcome.FAIL, "expected_effect": "SomeEffect", "treatment": "length_in", "outcome": "completed", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "large_gauge", "outcome": "color", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "color", "outcome": "large_gauge", }, { - "result": TestResult.FAIL, + "result": TestOutcome.FAIL, "expected_effect": "SomeEffect", "treatment": "large_gauge", "outcome": "completed", }, { - "result": TestResult.INESTIMABLE, + "result": TestOutcome.INESTIMABLE, "expected_effect": "NoEffect", "treatment": "color", "outcome": "completed", }, { - "result": TestResult.INESTIMABLE, + "result": TestOutcome.INESTIMABLE, "expected_effect": "NoEffect", "treatment": "completed", "outcome": "color", @@ -287,61 +289,61 @@ def test_evaluate_tests(self): expected_results = pd.DataFrame( [ { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "length_in", "outcome": "large_gauge", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "large_gauge", "outcome": "length_in", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "length_in", "outcome": "color", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "color", "outcome": "length_in", }, { - "result": TestResult.FAIL, + "result": TestOutcome.FAIL, "expected_effect": "SomeEffect", "treatment": "length_in", "outcome": "completed", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "large_gauge", "outcome": "color", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "color", "outcome": "large_gauge", }, { - "result": TestResult.FAIL, + "result": TestOutcome.FAIL, "expected_effect": "SomeEffect", "treatment": "large_gauge", "outcome": "completed", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "color", "outcome": "completed", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "completed", "outcome": "color", diff --git a/tests/discovery_tests/test_hill_climber_discovery.py b/tests/discovery_tests/test_hill_climber_discovery.py index 9338e42a..7094a4eb 100644 --- a/tests/discovery_tests/test_hill_climber_discovery.py +++ b/tests/discovery_tests/test_hill_climber_discovery.py @@ -6,7 +6,8 @@ import pandas as pd from causal_testing.discovery.hill_climber_discovery import HillClimberDiscovery from causal_testing.specification.causal_dag import CausalDAG -from causal_testing.discovery.abstract_discovery import TestResult, Discovery, simple_cycle +from causal_testing.discovery.abstract_discovery import simple_cycle +from causal_testing.testing.causal_test_result import TestOutcome class TestHillClimber(unittest.TestCase): @@ -15,70 +16,70 @@ def test_sum_test_outcomes(self): test_results = pd.DataFrame( [ { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "length_in", "outcome": "large_gauge", "effect": "positive", }, { - "result": TestResult.INESTIMABLE, + "result": TestOutcome.INESTIMABLE, "expected_effect": "NoEffect", "treatment": "large_gauge", "outcome": "length_in", "effect": None, }, { - "result": TestResult.INESTIMABLE, + "result": TestOutcome.INESTIMABLE, "expected_effect": "NoEffect", "treatment": "length_in", "outcome": "color", "effect": None, }, { - "result": TestResult.INESTIMABLE, + "result": TestOutcome.INESTIMABLE, "expected_effect": "NoEffect", "treatment": "color", "outcome": "length_in", "effect": None, }, { - "result": TestResult.FAIL, + "result": TestOutcome.FAIL, "expected_effect": "SomeEffect", "treatment": "length_in", "outcome": "completed", "effect": "negative", }, { - "result": TestResult.INESTIMABLE, + "result": TestOutcome.INESTIMABLE, "expected_effect": "NoEffect", "treatment": "large_gauge", "outcome": "color", "effect": None, }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "color", "outcome": "large_gauge", "effect": None, }, { - "result": TestResult.FAIL, + "result": TestOutcome.FAIL, "expected_effect": "SomeEffect", "treatment": "large_gauge", "outcome": "completed", "effect": "positive", }, { - "result": TestResult.PASS, + "result": TestOutcome.PASS, "expected_effect": "NoEffect", "treatment": "color", "outcome": "completed", "effect": None, }, { - "result": TestResult.INESTIMABLE, + "result": TestOutcome.INESTIMABLE, "expected_effect": "NoEffect", "treatment": "completed", "outcome": "color", @@ -86,13 +87,13 @@ def test_sum_test_outcomes(self): }, ] ) - expected_results = {TestResult.PASS: 1.5, TestResult.FAIL: 2, TestResult.INESTIMABLE: 2.5} + expected_results = {TestOutcome.PASS: 1.5, TestOutcome.FAIL: 2, TestOutcome.INESTIMABLE: 2.5} hill_climber = HillClimberDiscovery(pd.DataFrame()) self.assertEqual(expected_results, hill_climber.sum_test_outcomes(test_results)) def test_sum_test_outcomes_uninitialised(self): hill_climber = HillClimberDiscovery(pd.DataFrame()) - expected_results = {TestResult.PASS: 0, TestResult.FAIL: 0, TestResult.INESTIMABLE: 0} + expected_results = {TestOutcome.PASS: 0, TestOutcome.FAIL: 0, TestOutcome.INESTIMABLE: 0} self.assertEqual( expected_results, hill_climber.sum_test_outcomes(pd.DataFrame(columns=["treatment", "outcome", "result"])) diff --git a/tests/main_tests/test_main.py b/tests/main_tests/test_main.py index 27b94fb8..3b4e4c37 100644 --- a/tests/main_tests/test_main.py +++ b/tests/main_tests/test_main.py @@ -181,7 +181,7 @@ def test_ctf(self): result_index += 1 test_passed = ( - test_case.expected_causal_effect.apply(test_case.result) + test_case.expected_causal_effect.apply(test_case.result.effect_estimate) if test_case.result.effect_estimate is not None else False ) diff --git a/tests/testing_tests/test_causal_effect.py b/tests/testing_tests/test_causal_effect.py index c0293c00..07e66ce5 100644 --- a/tests/testing_tests/test_causal_effect.py +++ b/tests/testing_tests/test_causal_effect.py @@ -1,7 +1,6 @@ import unittest import pandas as pd from causal_testing.testing.causal_effect import ExactValue, SomeEffect, Positive, Negative, NoEffect -from causal_testing.testing.causal_test_result import CausalTestResult from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator from causal_testing.estimation.effect_estimate import EffectEstimate from causal_testing.utils.validation import CausalValidator @@ -21,112 +20,59 @@ def setUp(self) -> None: adjustment_set={}, ) - def test_None_ci(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate(type="ate", value=pd.Series(0)), - ) - - self.assertIsNone(ctr.effect_estimate.ci_low) - self.assertIsNone(ctr.effect_estimate.ci_high) - - def test_empty_adjustment_set(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate(type="ate", value=pd.Series(0)), - ) - - self.assertIsNone(ctr.effect_estimate.ci_low) - self.assertIsNone(ctr.effect_estimate.ci_high) - def test_Positive_ate_pass(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="ate", value=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6) - ), - ) - ev = Positive() - self.assertTrue(ev.apply(ctr)) + effect_estimate = EffectEstimate(type="ate", value=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6)) + self.assertTrue(Positive().apply(effect_estimate)) def test_Positive_risk_ratio_pass(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="risk_ratio", value=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6) - ), + effect_estimate = EffectEstimate( + type="risk_ratio", value=pd.Series(5.05), ci_low=pd.Series(5), ci_high=pd.Series(6) ) - ev = Positive() - self.assertTrue(ev.apply(ctr)) + self.assertTrue(Positive().apply(effect_estimate)) def test_Positive_fail(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)), - ) - ev = Positive() - self.assertFalse(ev.apply(ctr)) + effect_estimate = EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)) + self.assertFalse(Positive().apply(effect_estimate)) def test_Negative_ate_pass(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="ate", value=pd.Series(-5.05), ci_low=pd.Series(-6), ci_high=pd.Series(-5) - ), + effect_estimate = EffectEstimate( + type="ate", value=pd.Series(-5.05), ci_low=pd.Series(-6), ci_high=pd.Series(-5) ) - ev = Negative() - self.assertTrue(ev.apply(ctr)) + self.assertTrue(Negative().apply(effect_estimate)) def test_Negative_risk_ratio_pass(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="risk_ratio", value=pd.Series(0.2), ci_low=pd.Series(0.1), ci_high=pd.Series(0.5) - ), + effect_estimate = EffectEstimate( + type="risk_ratio", value=pd.Series(0.2), ci_low=pd.Series(0.1), ci_high=pd.Series(0.5) ) - ev = Negative() - self.assertTrue(ev.apply(ctr)) + self.assertTrue(Negative().apply(effect_estimate)) def test_Negative_fail(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)), - ) - ev = Negative() - self.assertFalse(ev.apply(ctr)) + effect_estimate = EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-1), ci_high=pd.Series(1)) + self.assertFalse(Negative().apply(effect_estimate)) def test_exactValue_pass(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate(type="ate", value=pd.Series(5.05)), - ) - ev = ExactValue(5, 0.1) - self.assertTrue(ev.apply(ctr)) + effect_estimate = EffectEstimate(type="ate", value=pd.Series(5.05)) + self.assertTrue(ExactValue(5, 0.1).apply(effect_estimate)) def test_exactValue_pass_ci(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="ate", value=pd.Series(5.05), ci_low=pd.Series(4), ci_high=pd.Series(6) - ), - ) - ev = ExactValue(5, 0.1) - self.assertTrue(ev.apply(ctr)) + effect_estimate = EffectEstimate(type="ate", value=pd.Series(5.05), ci_low=pd.Series(4), ci_high=pd.Series(6)) + self.assertTrue(ExactValue(5, 0.1).apply(effect_estimate)) def test_exactValue_ci_pass_ci(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="ate", value=pd.Series(5.05), ci_low=pd.Series(4.1), ci_high=pd.Series(5.9) - ), + effect_estimate = EffectEstimate( + type="ate", value=pd.Series(5.05), ci_low=pd.Series(4.1), ci_high=pd.Series(5.9) ) - ev = ExactValue(5, ci_low=4, ci_high=6) - self.assertTrue(ev.apply(ctr)) + self.assertTrue(ExactValue(5, ci_low=4, ci_high=6).apply(effect_estimate)) def test_exactValue_ci_fail_ci(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="ate", value=pd.Series(5.05), ci_low=pd.Series(3.9), ci_high=pd.Series(6.1) - ), + effect_estimate = EffectEstimate( + type="ate", value=pd.Series(5.05), ci_low=pd.Series(3.9), ci_high=pd.Series(6.1) ) - ev = ExactValue(5, ci_low=4, ci_high=6) - self.assertFalse(ev.apply(ctr)) + self.assertFalse(ExactValue(5, ci_low=4, ci_high=6).apply(effect_estimate)) def test_exactValue_fail(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate(type="ate", value=pd.Series(0)), - ) - ev = ExactValue(5, 0.1) - self.assertFalse(ev.apply(ctr)) + effect_estimate = EffectEstimate(type="ate", value=pd.Series(0)) + self.assertFalse(ExactValue(5, 0.1).apply(effect_estimate)) def test_invalid_atol(self): with self.assertRaises(ValueError): @@ -149,61 +95,47 @@ def test_invalid_ci_atol(self): ExactValue(1000, ci_low=999, ci_high=1001, atol=50) def test_invalid(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="invalid", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7) - ), + effect_estimate = EffectEstimate( + type="invalid", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7) ) with self.assertRaises(ValueError): - SomeEffect().apply(ctr) + SomeEffect().apply(effect_estimate) with self.assertRaises(ValueError): - NoEffect().apply(ctr) + NoEffect().apply(effect_estimate) with self.assertRaises(ValueError): - Positive().apply(ctr) + Positive().apply(effect_estimate) with self.assertRaises(ValueError): - Negative().apply(ctr) + Negative().apply(effect_estimate) def test_someEffect_pass_coefficient(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="coefficient", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7) - ), + effect_estimate = EffectEstimate( + type="coefficient", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7) ) - self.assertTrue(SomeEffect().apply(ctr)) - self.assertFalse(NoEffect().apply(ctr)) + self.assertTrue(SomeEffect().apply(effect_estimate)) + self.assertFalse(NoEffect().apply(effect_estimate)) def test_someEffect_pass_ate(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="coefficient", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7) - ), + effect_estimate = EffectEstimate( + type="coefficient", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7) ) - self.assertTrue(SomeEffect().apply(ctr)) - self.assertFalse(NoEffect().apply(ctr)) + self.assertTrue(SomeEffect().apply(effect_estimate)) + self.assertFalse(NoEffect().apply(effect_estimate)) def test_someEffect_pass_rr(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="coefficient", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7) - ), + effect_estimate = EffectEstimate( + type="coefficient", value=pd.Series(5.05), ci_low=pd.Series(4.8), ci_high=pd.Series(6.7) ) - self.assertTrue(SomeEffect().apply(ctr)) - self.assertFalse(NoEffect().apply(ctr)) + self.assertTrue(SomeEffect().apply(effect_estimate)) + self.assertFalse(NoEffect().apply(effect_estimate)) def test_someEffect_fail(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate( - type="ate", value=pd.Series(0), ci_low=pd.Series(-0.1), ci_high=pd.Series(0.2) - ), - ) - self.assertFalse(SomeEffect().apply(ctr)) - self.assertTrue(NoEffect().apply(ctr)) + effect_estimate = EffectEstimate(type="ate", value=pd.Series(0), ci_low=pd.Series(-0.1), ci_high=pd.Series(0.2)) + self.assertFalse(SomeEffect().apply(effect_estimate)) + self.assertTrue(NoEffect().apply(effect_estimate)) def test_someEffect_None(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate(type="ate", value=pd.Series(0)), - ) - self.assertEqual(SomeEffect().apply(ctr), None) + effect_estimate = EffectEstimate(type="ate", value=pd.Series(0)) + self.assertEqual(SomeEffect().apply(effect_estimate), None) def test_positive_risk_ratio_e_value(self): cv = CausalValidator() @@ -226,10 +158,8 @@ def test_negative_risk_ratio_e_value_using_ci(self): self.assertEqual(round(e_value, 4), 1.4625) def test_multiple_value_exception_caught(self): - ctr = CausalTestResult( - effect_estimate=EffectEstimate(type="ate", value=pd.Series([0, 1])), - ) + effect_estimate = EffectEstimate(type="ate", value=pd.Series([0, 1])) with self.assertRaises(ValueError): - Positive().apply(ctr) + Positive().apply(effect_estimate) with self.assertRaises(ValueError): - Negative().apply(ctr) + Negative().apply(effect_estimate) diff --git a/tests/testing_tests/test_causal_test_case.py b/tests/testing_tests/test_causal_test_case.py index 53bf0b4e..d475ce79 100644 --- a/tests/testing_tests/test_causal_test_case.py +++ b/tests/testing_tests/test_causal_test_case.py @@ -128,8 +128,8 @@ def test_execute_test_observational_linear_regression_estimator(self): expected_causal_effect=self.expected_causal_effect, estimator=estimation_model, ) - causal_test_result = causal_test_case.estimate_effect(self.df) - pd.testing.assert_series_equal(causal_test_result.effect_estimate.value, pd.Series(4.0), atol=1e-10) + effect_estimate = causal_test_case.estimate_effect(self.df) + pd.testing.assert_series_equal(effect_estimate.value, pd.Series(4.0), atol=1e-10) def test_execute_test_observational_linear_regression_estimator_direct_effect(self): """Check that executing the causal test case returns the correct results for dummy data using a linear @@ -151,8 +151,8 @@ def test_execute_test_observational_linear_regression_estimator_direct_effect(se # 6. Easier to access treatment and outcome values self.treatment_value = 1 self.control_value = 0 - causal_test_result = causal_test_case.estimate_effect(self.df) - pd.testing.assert_series_equal(causal_test_result.effect_estimate.value, pd.Series(4.0), atol=1e-10) + effect_estimate = causal_test_case.estimate_effect(self.df) + pd.testing.assert_series_equal(effect_estimate.value, pd.Series(4.0), atol=1e-10) def test_execute_test_observational_linear_regression_estimator_coefficient(self): """Check that executing the causal test case returns the correct results for dummy data using a linear @@ -169,8 +169,8 @@ def test_execute_test_observational_linear_regression_estimator_coefficient(self estimator=estimation_model, estimate_type="coefficient", ) - causal_test_result = causal_test_case.estimate_effect(self.df) - pd.testing.assert_series_equal(causal_test_result.effect_estimate.value, pd.Series({"D": 0.0}), atol=1e-1) + effect_estimate = causal_test_case.estimate_effect(self.df) + pd.testing.assert_series_equal(effect_estimate.value, pd.Series({"D": 0.0}), atol=1e-1) def test_execute_test_observational_linear_regression_estimator_risk_ratio(self): """Check that executing the causal test case returns the correct results for dummy data using a linear @@ -187,8 +187,8 @@ def test_execute_test_observational_linear_regression_estimator_risk_ratio(self) estimator=estimation_model, estimate_type="risk_ratio", ) - causal_test_result = causal_test_case.estimate_effect(self.df) - pd.testing.assert_series_equal(causal_test_result.effect_estimate.value, pd.Series(0.0), atol=1) + effect_estimate = causal_test_case.estimate_effect(self.df) + pd.testing.assert_series_equal(effect_estimate.value, pd.Series(0.0), atol=1) def test_invalid_estimate_type(self): """Check that executing the causal test case returns the correct results for dummy data using a linear @@ -223,8 +223,8 @@ def test_execute_test_observational_linear_regression_estimator_squared_term(sel expected_causal_effect=self.expected_causal_effect, estimator=estimation_model, ) - causal_test_result = causal_test_case.estimate_effect(self.df) - pd.testing.assert_series_equal(causal_test_result.effect_estimate.value, pd.Series(4.0), atol=1) + effect_estimate = causal_test_case.estimate_effect(self.df) + pd.testing.assert_series_equal(effect_estimate.value, pd.Series(4.0), atol=1) def test_estimate_params_with_formula(self): """Ensure estimate params is handled correctly when a formula is passed into the estimator object""" @@ -245,7 +245,7 @@ def test_estimate_params_with_formula(self): ) self.assertEqual( round( - causal_test_case.estimate_effect(self.df).effect_estimate.value[0], + causal_test_case.estimate_effect(self.df).value[0], 3, ), 1.444, From 445f6617becb90817b437ac0ff1241d7de09e32a Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 20 Jul 2026 08:53:34 +0100 Subject: [PATCH 2/7] Evaluate dag --- causal_testing/causal_testing_framework.py | 21 +- tests/main_tests/test_ctf.py | 227 +++++++++++++++++++++ tests/main_tests/test_main.py | 206 +------------------ 3 files changed, 247 insertions(+), 207 deletions(-) create mode 100644 tests/main_tests/test_ctf.py diff --git a/causal_testing/causal_testing_framework.py b/causal_testing/causal_testing_framework.py index ef2712ed..8aaafee8 100644 --- a/causal_testing/causal_testing_framework.py +++ b/causal_testing/causal_testing_framework.py @@ -266,7 +266,7 @@ def run_tests(self, silent: bool = False, adequacy: bool = False, bootstrap_size self.df, suppress_estimation_errors=silent, adequacy=adequacy, bootstrap_size=bootstrap_size ) - def evaluate_dag(self, bootstrap_size: bool, alpha: float) -> pd.Series: + def evaluate_dag(self, bootstrap_size: bool = 100, alpha: float = 0.05) -> pd.Series: """ Calculate confidence intervals for how well a causal DAG fits a dataset by repeatedly resampling the dataset and executing the causal tests. @@ -275,17 +275,22 @@ def evaluate_dag(self, bootstrap_size: bool, alpha: float) -> pd.Series: :param bootstrap_size: The number of bootstrap samples to use when calculating causal test adequacy (defaults to 100) :param alpha: The significance level to use when calculating confidence intervals. + (defaults to 0.05). """ self.run_tests(silent=True, adequacy=False) results = { - test_outcome: len([test for test in self.test_cases if test.result.outcome == test_outcome]) + test_outcome.name: len( + [test for test in self.test_cases if test.result and test.result.outcome == test_outcome] + ) for test_outcome in TestOutcome } sample_results = [] for sample_index in range(bootstrap_size): test_outcomes = {test_outcome: 0 for test_outcome in TestOutcome} - for test_case in tqdm(self.test_cases): + for test_case in self.test_cases: + if test_case.skip: + continue effect_estimate = test_case.estimate_effect( df=self.df.sample(len(self.df), replace=True, random_state=sample_index) ) @@ -299,14 +304,16 @@ def evaluate_dag(self, bootstrap_size: bool, alpha: float) -> pd.Series: sample_results.append(test_outcomes) sample_results = pd.DataFrame(sample_results) + # Calculate the confidence interval of each column - ci_low_inx = (alpha / 2) * bootstrap_size - ci_high_inx = ((1 - alpha) / 2) * bootstrap_size + ci_low_inx = round((alpha / 2) * bootstrap_size) + ci_high_inx = round(((1 - alpha) / 2) * bootstrap_size) for outcome in TestOutcome: data = sorted(sample_results[outcome]) - results[f"{outcome}_ci_low"] = data[ci_low_inx] - results[f"{outcome}_ci_high"] = data[ci_high_inx] + results[f"{outcome.name}_ci_low"] = data[ci_low_inx] + results[f"{outcome.name}_ci_high"] = data[ci_high_inx] + print(results) return pd.Series(results).sort_index() def save_results(self, output_path) -> list: diff --git a/tests/main_tests/test_ctf.py b/tests/main_tests/test_ctf.py new file mode 100644 index 00000000..1f5f0bd5 --- /dev/null +++ b/tests/main_tests/test_ctf.py @@ -0,0 +1,227 @@ +import unittest +from pathlib import Path +import json +import pandas as pd + +from causal_testing.causal_testing_framework import CausalTestingFramework + + +class TestCausalTestingFramework(unittest.TestCase): + def setUp(self): + self.dag_path = "tests/resources/data/dag.dot" + self.data_paths = ["tests/resources/data/data.csv"] + self.test_cases_path = "tests/resources/data/tests.json" + self.output_path = Path("results/results.json") + self.include_edges_path = "tests/resources/data/include_edges.dot" + self.exclude_edges_path = "tests/resources/data/exclude_edges.dot" + self.paths = { + "dag_path": self.dag_path, + "data_paths": self.data_paths, + "test_cases_path": self.test_cases_path, + } + + def test_load_data(self): + csv_framework = CausalTestingFramework() + csv_framework.load_data(self.data_paths) + + pqt_framework = CausalTestingFramework() + pqt_framework.load_data([path.replace(".csv", ".pqt") for path in self.data_paths]) + pd.testing.assert_frame_equal(csv_framework.df, pqt_framework.df) + + def test_load_data_query(self): + framework = CausalTestingFramework() + framework.load_data(data_paths=self.data_paths) + self.assertFalse((framework.df["test_input"] > 4).all()) + + framework.load_data(data_paths=self.data_paths, query="test_input > 4") + self.assertTrue((framework.df["test_input"] > 4).all()) + + def test_load_data_invalid_extension(self): + framework = CausalTestingFramework() + with self.assertRaises(ValueError): + framework.load_data("data.invalid") + + def test_load_dag_missing_node(self): + framework = CausalTestingFramework() + framework.setup(**self.paths) + framework.dag.add_node("missing") + with self.assertRaises(ValueError): + framework.create_variables() + + def test_load_tests_before_dag(self): + framework = CausalTestingFramework() + with self.assertRaises(ValueError): + framework.load_test_cases_from_json(self.test_cases_path) + + def test_create_base_test_case_missing_treatment(self): + framework = CausalTestingFramework() + framework.setup(**self.paths) + with self.assertRaises(KeyError) as e: + framework.create_base_test( + {"treatment_variable": "missing", "expected_effect": {"test_outcome": "NoEffect"}} + ) + self.assertEqual("\"Treatment variable 'missing' not found in inputs or outputs\"", str(e.exception)) + + def test_create_base_test_case_missing_estimator(self): + framework = CausalTestingFramework() + framework.setup(**self.paths) + with self.assertRaises(ValueError) as e: + framework.create_causal_test( + {"treatment_variable": "test_input", "expected_effect": {"test_output": "NoEffect"}} + ) + self.assertEqual("Test configuration must specify an estimator", str(e.exception)) + + def test_create_test_case_invalid_estimator(self): + framework = CausalTestingFramework() + framework.setup(**self.paths) + with self.assertRaises(ValueError) as e: + framework.create_causal_test( + { + "treatment_variable": "test_input", + "expected_effect": {"test_output": "NoEffect"}, + "estimator": "InvalidEstimator", + } + ) + self.assertEqual( + f"Unsupported estimator InvalidEstimator. Supported: ['CubicSplineEstimator', 'IPCWEstimator', 'InstrumentalVariableEstimator', 'LinearRegressionEstimator', 'LogisticRegressionEstimator', 'MultinomialRegressionEstimator']. " + "If you have implemented a custom estimator, you will need to add this to your entrypoints via your " + "pyproject.toml file.", + str(e.exception), + ) + + def test_create_test_case_invalid_effect(self): + framework = CausalTestingFramework() + framework.setup(**self.paths) + test = { + "name": "test1", + "treatment_variable": "test_input", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "expected_effect": {"test_output": "InvalidEffect"}, + } + base_test_case = framework.create_base_test(test) + with self.assertRaises(ValueError) as e: + framework.create_causal_test(test) + self.assertEqual( + f"Unsupported causal effect InvalidEffect. Supported: ['ExactValue', 'Negative', 'NoEffect', 'Positive', 'SomeEffect']. " + "If you have implemented a custom causal effect, you will need to add this to your entrypoints via your " + "pyproject.toml file.", + str(e.exception), + ) + + def test_create_test_case_effect_kwargs(self): + framework = CausalTestingFramework() + framework.setup(**self.paths) + test = { + "name": "test1", + "treatment_variable": "test_input", + "estimator": "LinearRegressionEstimator", + "estimate_type": "coefficient", + "expected_effect": {"test_output": "ExactValue"}, + "effect_kwargs": {"value": 4}, + } + base_test_case = framework.create_base_test(test) + test_case = framework.create_causal_test(test) + self.assertEqual(test_case.expected_causal_effect.value, 4) + + def test_create_test_case_estimator_kwargs(self): + framework = CausalTestingFramework() + framework.setup(**self.paths) + test = { + "name": "test1", + "treatment_variable": "test_input", + "estimator": "InstrumentalVariableEstimator", + "estimate_type": "coefficient", + "expected_effect": {"test_output": "SomeEffect"}, + "estimator_kwargs": {"instrument": "instrumental_variable"}, + } + base_test_case = framework.create_base_test(test) + test_case = framework.create_causal_test(test) + self.assertEqual(test_case.estimator.instrument, "instrumental_variable") + + def test_create_base_test_case_missing_outcome(self): + framework = CausalTestingFramework() + framework.setup(**self.paths) + with self.assertRaises(KeyError) as e: + framework.create_base_test({"treatment_variable": "test_input", "expected_effect": {"missing": "NoEffect"}}) + self.assertEqual("\"Outcome variable 'missing' not found in inputs or outputs\"", str(e.exception)) + + def test_unloaded_tests(self): + framework = CausalTestingFramework() + with self.assertRaises(ValueError) as e: + framework.run_tests() + self.assertEqual("No tests to run.", str(e.exception)) + + def test_ctf(self): + framework = CausalTestingFramework() + framework.setup(**self.paths) + framework.run_tests() + json_results = framework.save_results(self.output_path) + + with open(self.test_cases_path, "r", encoding="utf-8") as f: + test_configs = json.load(f) + + self.assertEqual(len(json_results), len(test_configs["tests"])) + + result_index = 0 + for i, test_config in enumerate(test_configs["tests"]): + result = json_results[i] + + if test_config.get("skip", False): + self.assertEqual(result["skip"], True) + self.assertEqual(result["passed"], None) + self.assertEqual(result["result"]["status"], "skipped") + else: + test_case = framework.test_cases[result_index] + result_index += 1 + + test_passed = ( + test_case.expected_causal_effect.apply(test_case.result.effect_estimate) + if test_case.result.effect_estimate is not None + else False + ) + self.assertEqual(result["passed"], test_passed) + + def test_ctf_exception(self): + framework = CausalTestingFramework(self.paths) + framework.setup(**self.paths, query="test_input < 0") + + with self.assertRaises(ValueError): + framework.run_tests() + + def test_ctf_exception_silent(self): + framework = CausalTestingFramework(self.paths) + framework.setup(**self.paths, query="test_input < 0") + + framework.run_tests(silent=True) + json_results = framework.save_results(self.output_path) + + with open(self.test_cases_path, "r", encoding="utf-8") as f: + test_configs = json.load(f) + + non_skipped_configs = [t for t in test_configs["tests"] if not t.get("skip", False)] + non_skipped_results = [r for r in json_results if not r.get("skip", False)] + + self.assertEqual(len(non_skipped_results), len(non_skipped_configs)) + + for result in non_skipped_results: + self.assertEqual(result["passed"], False) + + def test_ctf_evaluate_dag(self): + framework = CausalTestingFramework(self.paths) + framework.setup(**self.paths) + results = framework.evaluate_dag() + expected = pd.Series( + { + "PASS": 1, + "FAIL": 0, + "INESTIMABLE": 0, + "PASS_ci_low": 0, + "PASS_ci_high": 1, + "FAIL_ci_low": 0, + "FAIL_ci_high": 0, + "INESTIMABLE_ci_low": 0, + "INESTIMABLE_ci_high": 0, + } + ).sort_index() + pd.testing.assert_series_equal(results, expected) diff --git a/tests/main_tests/test_main.py b/tests/main_tests/test_main.py index 3b4e4c37..dc0b2a94 100644 --- a/tests/main_tests/test_main.py +++ b/tests/main_tests/test_main.py @@ -1,17 +1,15 @@ import unittest -from pathlib import Path import tempfile import os from unittest.mock import patch import shutil import json -import pandas as pd - -from causal_testing.causal_testing_framework import CausalTestingFramework from causal_testing.__main__ import main +from pathlib import Path -class TestCausalTestingFramework(unittest.TestCase): +class TestMain(unittest.TestCase): + def setUp(self): self.dag_path = "tests/resources/data/dag.dot" self.data_paths = ["tests/resources/data/data.csv"] @@ -19,198 +17,6 @@ def setUp(self): self.output_path = Path("results/results.json") self.include_edges_path = "tests/resources/data/include_edges.dot" self.exclude_edges_path = "tests/resources/data/exclude_edges.dot" - self.paths = { - "dag_path": self.dag_path, - "data_paths": self.data_paths, - "test_cases_path": self.test_cases_path, - } - - def test_load_data(self): - csv_framework = CausalTestingFramework() - csv_framework.load_data(self.data_paths) - - pqt_framework = CausalTestingFramework() - pqt_framework.load_data([path.replace(".csv", ".pqt") for path in self.data_paths]) - pd.testing.assert_frame_equal(csv_framework.df, pqt_framework.df) - - def test_load_data_query(self): - framework = CausalTestingFramework() - framework.load_data(data_paths=self.data_paths) - self.assertFalse((framework.df["test_input"] > 4).all()) - - framework.load_data(data_paths=self.data_paths, query="test_input > 4") - self.assertTrue((framework.df["test_input"] > 4).all()) - - def test_load_data_invalid_extension(self): - framework = CausalTestingFramework() - with self.assertRaises(ValueError): - framework.load_data("data.invalid") - - def test_load_dag_missing_node(self): - framework = CausalTestingFramework() - framework.setup(**self.paths) - framework.dag.add_node("missing") - with self.assertRaises(ValueError): - framework.create_variables() - - def test_load_tests_before_dag(self): - framework = CausalTestingFramework() - with self.assertRaises(ValueError): - framework.load_test_cases_from_json(self.test_cases_path) - - def test_create_base_test_case_missing_treatment(self): - framework = CausalTestingFramework() - framework.setup(**self.paths) - with self.assertRaises(KeyError) as e: - framework.create_base_test( - {"treatment_variable": "missing", "expected_effect": {"test_outcome": "NoEffect"}} - ) - self.assertEqual("\"Treatment variable 'missing' not found in inputs or outputs\"", str(e.exception)) - - def test_create_base_test_case_missing_estimator(self): - framework = CausalTestingFramework() - framework.setup(**self.paths) - with self.assertRaises(ValueError) as e: - framework.create_causal_test( - {"treatment_variable": "test_input", "expected_effect": {"test_output": "NoEffect"}} - ) - self.assertEqual("Test configuration must specify an estimator", str(e.exception)) - - def test_create_test_case_invalid_estimator(self): - framework = CausalTestingFramework() - framework.setup(**self.paths) - with self.assertRaises(ValueError) as e: - framework.create_causal_test( - { - "treatment_variable": "test_input", - "expected_effect": {"test_output": "NoEffect"}, - "estimator": "InvalidEstimator", - } - ) - self.assertEqual( - f"Unsupported estimator InvalidEstimator. Supported: ['CubicSplineEstimator', 'IPCWEstimator', 'InstrumentalVariableEstimator', 'LinearRegressionEstimator', 'LogisticRegressionEstimator', 'MultinomialRegressionEstimator']. " - "If you have implemented a custom estimator, you will need to add this to your entrypoints via your " - "pyproject.toml file.", - str(e.exception), - ) - - def test_create_test_case_invalid_effect(self): - framework = CausalTestingFramework() - framework.setup(**self.paths) - test = { - "name": "test1", - "treatment_variable": "test_input", - "estimator": "LinearRegressionEstimator", - "estimate_type": "coefficient", - "expected_effect": {"test_output": "InvalidEffect"}, - } - base_test_case = framework.create_base_test(test) - with self.assertRaises(ValueError) as e: - framework.create_causal_test(test) - self.assertEqual( - f"Unsupported causal effect InvalidEffect. Supported: ['ExactValue', 'Negative', 'NoEffect', 'Positive', 'SomeEffect']. " - "If you have implemented a custom causal effect, you will need to add this to your entrypoints via your " - "pyproject.toml file.", - str(e.exception), - ) - - def test_create_test_case_effect_kwargs(self): - framework = CausalTestingFramework() - framework.setup(**self.paths) - test = { - "name": "test1", - "treatment_variable": "test_input", - "estimator": "LinearRegressionEstimator", - "estimate_type": "coefficient", - "expected_effect": {"test_output": "ExactValue"}, - "effect_kwargs": {"value": 4}, - } - base_test_case = framework.create_base_test(test) - test_case = framework.create_causal_test(test) - self.assertEqual(test_case.expected_causal_effect.value, 4) - - def test_create_test_case_estimator_kwargs(self): - framework = CausalTestingFramework() - framework.setup(**self.paths) - test = { - "name": "test1", - "treatment_variable": "test_input", - "estimator": "InstrumentalVariableEstimator", - "estimate_type": "coefficient", - "expected_effect": {"test_output": "SomeEffect"}, - "estimator_kwargs": {"instrument": "instrumental_variable"}, - } - base_test_case = framework.create_base_test(test) - test_case = framework.create_causal_test(test) - self.assertEqual(test_case.estimator.instrument, "instrumental_variable") - - def test_create_base_test_case_missing_outcome(self): - framework = CausalTestingFramework() - framework.setup(**self.paths) - with self.assertRaises(KeyError) as e: - framework.create_base_test({"treatment_variable": "test_input", "expected_effect": {"missing": "NoEffect"}}) - self.assertEqual("\"Outcome variable 'missing' not found in inputs or outputs\"", str(e.exception)) - - def test_unloaded_tests(self): - framework = CausalTestingFramework() - with self.assertRaises(ValueError) as e: - framework.run_tests() - self.assertEqual("No tests to run.", str(e.exception)) - - def test_ctf(self): - framework = CausalTestingFramework() - framework.setup(**self.paths) - framework.run_tests() - json_results = framework.save_results(self.output_path) - - with open(self.test_cases_path, "r", encoding="utf-8") as f: - test_configs = json.load(f) - - self.assertEqual(len(json_results), len(test_configs["tests"])) - - result_index = 0 - for i, test_config in enumerate(test_configs["tests"]): - result = json_results[i] - - if test_config.get("skip", False): - self.assertEqual(result["skip"], True) - self.assertEqual(result["passed"], None) - self.assertEqual(result["result"]["status"], "skipped") - else: - test_case = framework.test_cases[result_index] - result_index += 1 - - test_passed = ( - test_case.expected_causal_effect.apply(test_case.result.effect_estimate) - if test_case.result.effect_estimate is not None - else False - ) - self.assertEqual(result["passed"], test_passed) - - def test_ctf_exception(self): - framework = CausalTestingFramework(self.paths) - framework.setup(**self.paths, query="test_input < 0") - - with self.assertRaises(ValueError): - framework.run_tests() - - def test_ctf_exception_silent(self): - framework = CausalTestingFramework(self.paths) - framework.setup(**self.paths, query="test_input < 0") - - framework.run_tests(silent=True) - json_results = framework.save_results(self.output_path) - - with open(self.test_cases_path, "r", encoding="utf-8") as f: - test_configs = json.load(f) - - non_skipped_configs = [t for t in test_configs["tests"] if not t.get("skip", False)] - non_skipped_results = [r for r in json_results if not r.get("skip", False)] - - self.assertEqual(len(non_skipped_results), len(non_skipped_configs)) - - for result in non_skipped_results: - self.assertEqual(result["passed"], False) def test_parse_args(self): with patch( @@ -249,7 +55,7 @@ def test_parse_args_adequacy(self): ], ): main() - with open(self.output_path.parent / "main.json") as f: + with open(self.output_path.parent / "main.json", encoding="utf-8") as f: log = json.load(f) executed_tests = [test for test in log if not test.get("skip", False)] assert all(test["result"].get("bootstrap_size", 100) == 100 for test in executed_tests) @@ -273,7 +79,7 @@ def test_parse_args_bootstrap_size(self): ], ): main() - with open(self.output_path.parent / "main.json") as f: + with open(self.output_path.parent / "main.json", encoding="utf-8") as f: log = json.load(f) executed_tests = [test for test in log if not test.get("skip", False)] assert all(test["result"].get("bootstrap_size", 50) == 50 for test in executed_tests) @@ -298,7 +104,7 @@ def test_parse_args_bootstrap_size_explicit_adequacy(self): ], ): main() - with open(self.output_path.parent / "main.json") as f: + with open(self.output_path.parent / "main.json", encoding="utf-8") as f: log = json.load(f) executed_tests = [test for test in log if not test.get("skip", False)] assert all(test["result"].get("bootstrap_size", 50) == 50 for test in executed_tests) From 920f0d6acbdcbe2005c7e99593898593c0ed007c Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 20 Jul 2026 11:16:57 +0100 Subject: [PATCH 3/7] Can now generate causal test cases directly from a DAG --- causal_testing/__main__.py | 78 ++-- causal_testing/causal_testing_framework.py | 2 + .../discovery/abstract_discovery.py | 12 +- .../estimation/abstract_estimator.py | 11 + causal_testing/specification/causal_dag.py | 167 +++++++- causal_testing/testing/causal_test_case.py | 18 + .../testing/metamorphic_relation.py | 305 -------------- tests/main_tests/test_main.py | 28 +- .../test_metamorphic_relations.py | 381 +++++------------- 9 files changed, 330 insertions(+), 672 deletions(-) delete mode 100644 causal_testing/testing/metamorphic_relation.py diff --git a/causal_testing/__main__.py b/causal_testing/__main__.py index 75d2f1a8..1d6853dc 100644 --- a/causal_testing/__main__.py +++ b/causal_testing/__main__.py @@ -1,6 +1,7 @@ """This module contains the main entrypoint functionality to the Causal Testing Framework.""" import argparse +import json import logging from enum import Enum from importlib.metadata import entry_points @@ -10,7 +11,7 @@ import pandas as pd from causal_testing.causal_testing_framework import CausalTestingFramework, read_dataframe -from causal_testing.testing.metamorphic_relation import generate_causal_tests +from causal_testing.specification.causal_dag import CausalDAG logger = logging.getLogger(__name__) @@ -41,24 +42,6 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace: "A causal inference-driven framework for functional black-box testing of complex software.", ) - main_parser.add_argument( - "-l", - "--log_level", - default="WARNING", - type=str.upper, - choices=["NONE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - help="Set the logging level (default: WARNING).", - ) - main_parser.add_argument( - "-a", - "--alpha", - help=( - "The significance level of the confidence intervals used to determine causality. " - "This should be a value between 0 and 1. Defaults to 0.05 for 95%% confidence intervals." - ), - default=0.05, - ) - subparsers = main_parser.add_subparsers( help="The action you want to run - call `causal_testing {action} -h` for further details", dest="command" ) @@ -67,24 +50,6 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace: parser_generate = subparsers.add_parser(Command.GENERATE.value, help="Generate causal tests from a DAG") parser_generate.add_argument("-D", "--dag-path", help="Path to the DAG file (.dot)", required=True) parser_generate.add_argument("-o", "--output", help="Path for output file (.json)", required=True) - parser_generate.add_argument( - "-e", - "--estimator", - help="The name of the estimator class to use when evaluating tests (defaults to LinearRegressionEstimator)", - default="LinearRegressionEstimator", - ) - parser_generate.add_argument( - "-T", - "--effect-type", - help="The effect type to estimate {direct, total}", - default="direct", - ) - parser_generate.add_argument( - "-E", - "--estimate-type", - help="The estimate type to use when evaluating tests (defaults to coefficient)", - default="coefficient", - ) parser_generate.add_argument( "-i", "--ignore-cycles", help="Ignore cycles in DAG", action="store_true", default=False ) @@ -97,11 +62,10 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace: parser_test.add_argument("-D", "--dag-path", help="Path to the DAG file (.dot)", required=True) parser_test.add_argument("-o", "--output", help="Path for output file (.json)", required=True) parser_test.add_argument("-i", "--ignore-cycles", help="Ignore cycles in DAG", action="store_true", default=False) - parser_test.add_argument("-d", "--data-paths", help="Paths to data files (.csv)", nargs="+", required=True) parser_test.add_argument("-t", "--test-config", help="Path to test configuration file (.json)", required=True) parser_test.add_argument("-q", "--query", help="Query string to filter data (e.g. 'age > 18')", type=str) parser_test.add_argument( - "-a", "--adequacy", help="Calculate causal test adequacy for each test case", action="store_true", default=False + "-A", "--adequacy", help="Calculate causal test adequacy for each test case", action="store_true", default=False ) parser_test.add_argument( "-b", @@ -127,7 +91,6 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace: parser_evaluate.add_argument( "-i", "--ignore-cycles", help="Ignore cycles in DAG", action="store_true", default=False ) - parser_evaluate.add_argument("-d", "--data-paths", help="Paths to data files (.csv)", nargs="+", required=True) parser_evaluate.add_argument("-q", "--query", help="Query string to filter data (e.g. 'age > 18')", type=str) parser_evaluate.add_argument( "-b", @@ -147,7 +110,6 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace: # Discovery parser_discover = subparsers.add_parser(Command.DISCOVER.value, help="Discover causal structures from data") - parser_discover.add_argument("-d", "--data-paths", help="Paths to data files (.csv)", nargs="+", required=True) parser_discover.add_argument( "-t", "--technique", @@ -175,6 +137,26 @@ def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace: default=[], ) + for parser in [parser_generate, parser_discover, parser_test, parser_evaluate]: + parser.add_argument( + "-l", + "--log_level", + default="WARNING", + type=str.upper, + choices=["NONE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + help="Set the logging level (default: WARNING).", + ) + parser.add_argument( + "-a", + "--alpha", + help=( + "The significance level of the confidence intervals used to determine causality. " + "This should be a value between 0 and 1. Defaults to 0.05 for 95%% confidence intervals." + ), + default=0.05, + ) + parser.add_argument("-d", "--data-paths", help="Paths to data files (.csv)", nargs="+", required=True) + args = main_parser.parse_args(args) # Assume the user wants test adequacy if they're setting bootstrap_size @@ -203,16 +185,14 @@ def main() -> None: match args.command: case Command.GENERATE: logging.info("Generating causal tests") - generate_causal_tests( - args.dag_path, - args.output, - args.ignore_cycles, - args.threads, - effect_type=args.effect_type, - estimate_type=args.estimate_type, - estimator=args.estimator, + df = pd.concat(read_dataframe(path) for path in args.data_paths) + causal_dag = CausalDAG(args.dag_path, ignore_cycles=args.ignore_cycles, datatypes=df.dtypes) + causal_tests = causal_dag.generate_causal_tests( + threads=args.threads, skip=False, ) + with open(args.output, "w", encoding="utf-8") as f: + json.dump({"tests": [test.to_json() for test in causal_tests]}, f) logging.info("Causal test generation completed successfully.") case Command.DISCOVER: diff --git a/causal_testing/causal_testing_framework.py b/causal_testing/causal_testing_framework.py index 8aaafee8..2b71e7d4 100644 --- a/causal_testing/causal_testing_framework.py +++ b/causal_testing/causal_testing_framework.py @@ -44,6 +44,8 @@ def read_dataframe(file_path: str, **kwargs: dict) -> pd.DataFrame: suffix = Path(file_path).suffix.lower() + print(suffix, type(suffix)) + if suffix in readers: return readers[suffix](file_path, **kwargs) raise ValueError(f"Unsupported file extension: '{suffix}'") diff --git a/causal_testing/discovery/abstract_discovery.py b/causal_testing/discovery/abstract_discovery.py index 3e10e797..7966489d 100644 --- a/causal_testing/discovery/abstract_discovery.py +++ b/causal_testing/discovery/abstract_discovery.py @@ -19,7 +19,6 @@ from causal_testing.testing.causal_effect import Negative, Positive from causal_testing.testing.causal_test_case import CausalTestCase from causal_testing.testing.causal_test_result import TestOutcome -from causal_testing.testing.metamorphic_relation import generate_metamorphic_relations # Ignore warnings from statsmodels when we try to evaluate test cases warnings.simplefilter("ignore") @@ -202,15 +201,8 @@ def evaluate_tests(self, causal_dag: CausalDAG) -> pd.DataFrame: ctf.create_variables() ctf.scenario = Scenario(list(ctf.variables["inputs"].values()) + list(ctf.variables["outputs"].values())) - ctf.test_cases = [ - ctf.create_causal_test( - relation.to_json_stub( - alpha=self.alpha, - **self._json_stub_params(relation.base_test_case.outcome_variable), - ) - ) - for relation in generate_metamorphic_relations(causal_dag) - ] + causal_dag.datatypes = self.df.dtypes + ctf.test_cases = causal_dag.generate_causal_tests() results = [] diff --git a/causal_testing/estimation/abstract_estimator.py b/causal_testing/estimation/abstract_estimator.py index 0ffdadf5..18cb83b7 100644 --- a/causal_testing/estimation/abstract_estimator.py +++ b/causal_testing/estimation/abstract_estimator.py @@ -58,3 +58,14 @@ def add_modelling_assumptions(self): Add modelling assumptions to the estimator. This is a list of strings which list the modelling assumptions that must hold if the resulting causal inference is to be considered valid. """ + + def to_json(self) -> dict: + """ + Convert to a JSON serialisable dict object containing all non-standard parameters to reconstruct. + + :returns: A JSON serialisable dict representing the object. + """ + result = {"estimator": self.__class__.__name__, "estimator_kwargs": {}} + if self.effect_modifiers: + result["estimator_kwargs"]["effect_modifiers"] = self.effect_modifiers + return result diff --git a/causal_testing/specification/causal_dag.py b/causal_testing/specification/causal_dag.py index 35b9c84b..ed13a9a2 100644 --- a/causal_testing/specification/causal_dag.py +++ b/causal_testing/specification/causal_dag.py @@ -3,14 +3,22 @@ from __future__ import annotations import logging +from functools import partial from itertools import combinations +from multiprocessing import Pool from typing import Generator, Set, Union import networkx as nx +import pandas as pd +from causal_testing.estimation.abstract_estimator import Estimator +from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator +from causal_testing.estimation.logistic_regression_estimator import LogisticRegressionEstimator +from causal_testing.estimation.multinomial_regression_estimator import MultinomialRegressionEstimator +from causal_testing.specification.variable import Input, Output, Variable from causal_testing.testing.base_test_case import BaseTestCase - -from .variable import Variable +from causal_testing.testing.causal_effect import NoEffect, SomeEffect +from causal_testing.testing.causal_test_case import CausalTestCase Node = Union[str, int] # Node type hint: A node is a string or an int @@ -123,9 +131,10 @@ class CausalDAG(nx.DiGraph): ensures it is acyclic. A CausalDAG must be specified as a dot file. """ - def __init__(self, file_path: str = None, ignore_cycles: bool = False, **attr): + def __init__(self, file_path: str = None, ignore_cycles: bool = False, datatypes: pd.Series = None, **attr): super().__init__(**attr) self.ignore_cycles = ignore_cycles + self.datatypes = datatypes if file_path: if file_path.endswith(".dot"): graph = nx.DiGraph(nx.nx_pydot.read_dot(file_path)) @@ -533,3 +542,155 @@ def to_dot_string(self) -> str: def __str__(self): return f"Nodes: {self.nodes}\nEdges: {self.edges}" + + def _estimator(self, base_test_case: BaseTestCase, nodes_to_ignore: set) -> Estimator: + treatment_variable = base_test_case.treatment_variable.name + outcome_variable = base_test_case.outcome_variable.name + + if self.datatypes is None or outcome_variable not in self.datatypes: + raise ValueError(f"No datatype specified for {outcome_variable}.") + + adj_sets = self.direct_effect_adjustment_sets( + [treatment_variable], [outcome_variable], nodes_to_ignore=nodes_to_ignore + ) + if not adj_sets: + return None, None + + min_adj_set = sorted(list(map(lambda s: sorted(list(s)), adj_sets)))[0] + + if pd.api.types.is_bool_dtype(self.datatypes[outcome_variable]): + return ( + LogisticRegressionEstimator(base_test_case=base_test_case, adjustment_set=min_adj_set), + "unit_odds_ratio", + ) + if pd.api.types.is_categorical_dtype(self.datatypes[outcome_variable]) or pd.api.types.is_object_dtype( + self.datatypes[outcome_variable] + ): + return ( + MultinomialRegressionEstimator(base_test_case=base_test_case, adjustment_set=min_adj_set), + "unit_odds_ratio", + ) + if pd.api.types.is_numeric_dtype(self.datatypes[outcome_variable]): + return LinearRegressionEstimator(base_test_case=base_test_case, adjustment_set=min_adj_set), "coefficient" + raise ValueError(f"Invalid datatype for {outcome_variable}: {self.datatypes[outcome_variable]}") + + def generate_causal_test( # pylint: disable=R0912 + self, u: str, v: str, nodes_to_ignore: set = None, **kwargs + ) -> list[CausalTestCase]: + """ + Construct a metamorphic relation for a given node pair implied by the Causal DAG, or None if no such relation + can be constructed (e.g. because every valid adjustment set contains a node to ignore). + + :param u: The treatment node. + :param v: The outcome node. + :param nodes_to_ignore: Set of nodes which will be excluded from causal tests. + :param kwargs: Keyword arguments to be passed through to test case generation. + + :return: A list containing ShouldCause and ShouldNotCause metamorphic relations. + """ + + if nodes_to_ignore is None: + nodes_to_ignore = set() + + causal_tests = [] + + # Create a ShouldNotCause relation for each pair of nodes that are not directly connected + if ((u, v) not in self.edges) and ((v, u) not in self.edges): + u_in_ancestors = u in nx.ancestors(self, v) + v_in_ancestors = v in nx.ancestors(self, u) + + # Case 1: U --> ... --> V or U _||_ V + if u_in_ancestors or (not u_in_ancestors and not v_in_ancestors): + base_test_case = BaseTestCase(Input(u, None), Output(v, None)) + estimator, estimate_type = self._estimator(base_test_case, nodes_to_ignore) + if estimator and estimate_type: + causal_tests.append( + CausalTestCase( + base_test_case=base_test_case, + expected_causal_effect=NoEffect(), + estimator=estimator, + estimate_type=estimate_type, + **kwargs, + ), + ) + + # Case 2: V --> ... --> U or U _||_ V + if v in nx.ancestors(self, u) or (not u_in_ancestors and not v_in_ancestors): + base_test_case = BaseTestCase(Input(v, None), Output(u, None)) + estimator, estimate_type = self._estimator(base_test_case, nodes_to_ignore) + if estimator and estimate_type: + causal_tests.append( + CausalTestCase( + base_test_case=base_test_case, + expected_causal_effect=NoEffect(), + estimator=estimator, + estimate_type=estimate_type, + **kwargs, + ), + ) + + # Create a ShouldCause relation for each edge (u, v) or (v, u) + elif (u, v) in self.edges: + base_test_case = BaseTestCase(Input(u, None), Output(v, None)) + estimator, estimate_type = self._estimator(base_test_case, nodes_to_ignore) + if estimator and estimate_type: + causal_tests.append( + CausalTestCase( + base_test_case=base_test_case, + expected_causal_effect=SomeEffect(), + estimator=estimator, + estimate_type=estimate_type, + **kwargs, + ), + ) + else: + base_test_case = BaseTestCase(Input(v, None), Output(u, None)) + estimator, estimate_type = self._estimator(base_test_case, nodes_to_ignore) + if estimator and estimate_type: + causal_tests.append( + CausalTestCase( + base_test_case=base_test_case, + expected_causal_effect=SomeEffect(), + estimator=estimator, + estimate_type=estimate_type, + **kwargs, + ), + ) + return causal_tests + + def generate_causal_tests( + self, nodes_to_ignore: set = None, threads: int = 0, nodes_to_test: set = None, **kwargs: dict + ) -> list[CausalTestCase]: + """ + Construct a list of metamorphic relations implied by the Causal DAG. + This list of metamorphic relations contains a ShouldCause relation for every edge, and a ShouldNotCause + relation for every (minimal) conditional independence relation implied by the structure of the DAG. + + :param nodes_to_ignore: Set of nodes which will be excluded from causal tests. + :param threads: Number of threads to use (if generating in parallel). + :param nodes_to_test: Set of nodes to test the relationships between (defaults to all nodes). + :param kwargs: Keyword arguments to be passed through to test case generation. + + :return: A list containing ShouldCause and ShouldNotCause metamorphic relations. + """ + + if nodes_to_ignore is None: + nodes_to_ignore = set() + nodes_to_ignore = nodes_to_ignore.union(set(self.cycle_nodes())) + + if nodes_to_test is None: + nodes_to_test = self.nodes + + if threads < 2: + causal_tests = [ + self.generate_causal_test(u, v, nodes_to_ignore, **kwargs) + for u, v in combinations(filter(lambda node: node not in nodes_to_ignore, nodes_to_test), 2) + ] + else: + with Pool(threads) as pool: + causal_tests = pool.starmap( + partial(self.generate_causal_test, nodes_to_ignore=nodes_to_ignore, **kwargs), + combinations(filter(lambda node: node not in nodes_to_ignore, nodes_to_test), 2), + ) + + return [item for items in causal_tests for item in items] diff --git a/causal_testing/testing/causal_test_case.py b/causal_testing/testing/causal_test_case.py index bc04d73d..d6f35989 100644 --- a/causal_testing/testing/causal_test_case.py +++ b/causal_testing/testing/causal_test_case.py @@ -157,3 +157,21 @@ def __str__(self): f"Running {treatment_config} instead of {control_config} should cause the following " f"changes to {outcome_variable}: {self.expected_causal_effect}." ) + + def to_json(self): + """ + Convert to a JSON serialisable dict object containing all non-standard parameters to reconstruct. + + :returns: A JSON serialisable dict representing the object. + """ + test_case = { + "name": self.name, + "treatment_variable": self.base_test_case.treatment_variable.name, + "estimate_type": self.estimate_type, + "expected_effect": { + self.base_test_case.outcome_variable.name: self.expected_causal_effect.__class__.__name__ + }, + "skip": self.skip, + "query": self.query, + } | self.estimator.to_json() + return test_case diff --git a/causal_testing/testing/metamorphic_relation.py b/causal_testing/testing/metamorphic_relation.py deleted file mode 100644 index 599c6cb5..00000000 --- a/causal_testing/testing/metamorphic_relation.py +++ /dev/null @@ -1,305 +0,0 @@ -""" -This module contains the ShouldCause and ShouldNotCause metamorphic relations as -defined in our ICST paper [https://eprints.whiterose.ac.uk/195317/]. -""" - -import json -import logging -from dataclasses import dataclass -from itertools import combinations -from multiprocessing import Pool -from typing import Iterable - -import networkx as nx - -from causal_testing.specification.causal_dag import CausalDAG -from causal_testing.testing.base_test_case import BaseTestCase - -logger = logging.getLogger(__name__) - - -@dataclass(order=True) -class MetamorphicRelation: - """Class representing a metamorphic relation.""" - - base_test_case: BaseTestCase - adjustment_vars: Iterable[str] - - def __eq__(self, other): - same_type = self.__class__ == other.__class__ - same_treatment = self.base_test_case.treatment_variable == other.base_test_case.treatment_variable - same_outcome = self.base_test_case.outcome_variable == other.base_test_case.outcome_variable - same_effect = self.base_test_case.effect == other.base_test_case.effect - same_adjustment_set = set(self.adjustment_vars) == set(other.adjustment_vars) - return same_type and same_treatment and same_outcome and same_effect and same_adjustment_set - - def to_json_stub( - self, - skip: bool = False, - estimate_type: str = "coefficient", - effect_type: str = "direct", - estimator: str = "LinearRegressionEstimator", - alpha: float = 0.05, - ) -> dict: - """ - Convert to a JSON frontend stub string for user customisation. - :param skip: Whether to skip the test (default False). - :param effect_type: The type of causal effect to consider (total or direct) - :param estimate_type: The estimate type to use when evaluating tests - :param estimator: The name of the estimator class to use when evaluating the test - :param alpha: The significance level to use when calculating the confidence intervals - """ - if estimator not in [ - "LinearRegressionEstimator", - "LogisticRegressionEstimator", - "MultinomialRegressionEstimator", - ]: - raise ValueError( - f"Unsupported estimator {estimator}. " - "We only support autogeneration using LinearRegressionEstimator or LogisticRegressionEstimator." - "More advanced estimators require careful thought that cannot be easily automated." - ) - return { - "name": str(self), - "estimator": estimator, - "estimate_type": estimate_type, - "effect": effect_type, - "treatment_variable": self.base_test_case.treatment_variable, - "alpha": alpha, - "skip": skip, - "estimator_kwargs": { - "formula": ( - f"{self.base_test_case.outcome_variable} ~ " - f"{' + '.join([self.base_test_case.treatment_variable] + self.adjustment_vars)}" - ), - }, - } - - -class ShouldCause(MetamorphicRelation): - """Class representing a should cause metamorphic relation.""" - - def to_json_stub( - self, - skip: bool = False, - estimate_type: str = "coefficient", - effect_type: str = "direct", - estimator: str = "LinearRegressionEstimator", - alpha: float = 0.05, - ) -> dict: - """ - Convert to a JSON frontend stub string for user customisation. - :param skip: Whether to skip the test (default False). - :param effect_type: The type of causal effect to consider (total or direct) - :param estimate_type: The estimate type to use when evaluating tests - :param estimator: The name of the estimator class to use when evaluating the test - :param alpha: The significance level to use when calculating the confidence intervals - """ - return super().to_json_stub( - skip=skip, estimate_type=estimate_type, effect_type=effect_type, estimator=estimator, alpha=alpha - ) | { - "expected_effect": {self.base_test_case.outcome_variable: "SomeEffect"}, - } - - def __str__(self): - formatted_str = f"{self.base_test_case.treatment_variable} --> {self.base_test_case.outcome_variable}" - if self.adjustment_vars: - formatted_str += f" | {self.adjustment_vars}" - return formatted_str - - -class ShouldNotCause(MetamorphicRelation): - """Class representing a should cause metamorphic relation.""" - - def to_json_stub( - self, - skip: bool = False, - estimate_type: str = "coefficient", - effect_type: str = "direct", - estimator: str = "LinearRegressionEstimator", - alpha: float = 0.05, - ) -> dict: - """ - Convert to a JSON frontend stub string for user customisation. - :param skip: Whether to skip the test (default False). - :param effect_type: The type of causal effect to consider (total or direct) - :param estimate_type: The estimate type to use when evaluating tests - :param estimator: The name of the estimator class to use when evaluating the test - :param alpha: The significance level to use when calculating the confidence intervals - """ - return super().to_json_stub( - skip=skip, estimate_type=estimate_type, effect_type=effect_type, estimator=estimator, alpha=alpha - ) | { - "expected_effect": {self.base_test_case.outcome_variable: "NoEffect"}, - } - - def __str__(self): - formatted_str = f"{self.base_test_case.treatment_variable} _||_ {self.base_test_case.outcome_variable}" - if self.adjustment_vars: - formatted_str += f" | {self.adjustment_vars}" - return formatted_str - - -def min_adj_set(adj_sets: set[set[str]]) -> set[str]: - """ - Given a nonempty set of adjustment sets, return the minimal one. - :param adj_sets: A nonempty set of adjustment sets. - :return: The minimal adjustment set (by alphabetical order if there are multiple sets of the same size). - """ - return sorted(list(map(lambda s: sorted(list(s)), adj_sets)))[0] - - -def generate_metamorphic_relation( # pylint: disable=R0912 - node_pair: tuple[str, str], dag: CausalDAG, nodes_to_ignore: set = None -) -> MetamorphicRelation: - """ - Construct a metamorphic relation for a given node pair implied by the Causal DAG, or None if no such relation can - be constructed (e.g. because every valid adjustment set contains a node to ignore). - - :param node_pair: The pair of nodes to consider. - :param dag: Causal DAG from which the metamorphic relations will be generated. - :param nodes_to_ignore: Set of nodes which will be excluded from causal tests. - - :return: A list containing ShouldCause and ShouldNotCause metamorphic relations. - """ - - if nodes_to_ignore is None: - nodes_to_ignore = set() - - (u, v) = node_pair - metamorphic_relations = [] - - # Create a ShouldNotCause relation for each pair of nodes that are not directly connected - if ((u, v) not in dag.edges) and ((v, u) not in dag.edges): - # Case 1: U --> ... --> V - if u in nx.ancestors(dag, v): - adj_sets = dag.direct_effect_adjustment_sets([u], [v], nodes_to_ignore=nodes_to_ignore) - if adj_sets: - metamorphic_relations.append(ShouldNotCause(BaseTestCase(u, v), min_adj_set(adj_sets))) - - # Case 2: V --> ... --> U - elif v in nx.ancestors(dag, u): - adj_sets = dag.direct_effect_adjustment_sets([v], [u], nodes_to_ignore=nodes_to_ignore) - if adj_sets: - metamorphic_relations.append(ShouldNotCause(BaseTestCase(v, u), min_adj_set(adj_sets))) - - # Case 3: V _||_ U (No directed walk from V to U but there may be a back-door path e.g. U <-- Z --> V). - else: - adj_sets1 = dag.direct_effect_adjustment_sets([u], [v], nodes_to_ignore=nodes_to_ignore) - adj_sets2 = dag.direct_effect_adjustment_sets([v], [u], nodes_to_ignore=nodes_to_ignore) - if adj_sets1: - metamorphic_relations.append(ShouldNotCause(BaseTestCase(u, v), list(adj_sets1[0]))) - if adj_sets2: - metamorphic_relations.append(ShouldNotCause(BaseTestCase(v, u), list(adj_sets2[0]))) - - # Create a ShouldCause relation for each edge (u, v) or (v, u) - elif (u, v) in dag.edges: - adj_sets = dag.direct_effect_adjustment_sets([u], [v], nodes_to_ignore=nodes_to_ignore) - if adj_sets: - metamorphic_relations.append(ShouldCause(BaseTestCase(u, v), min_adj_set(adj_sets))) - else: - adj_sets = dag.direct_effect_adjustment_sets([v], [u], nodes_to_ignore=nodes_to_ignore) - if adj_sets: - metamorphic_relations.append(ShouldCause(BaseTestCase(v, u), min_adj_set(adj_sets))) - return metamorphic_relations - - -def generate_metamorphic_relations( - dag: CausalDAG, nodes_to_ignore: set = None, threads: int = 0, nodes_to_test: set = None -) -> list[MetamorphicRelation]: - """ - Construct a list of metamorphic relations implied by the Causal DAG. - This list of metamorphic relations contains a ShouldCause relation for every edge, and a ShouldNotCause - relation for every (minimal) conditional independence relation implied by the structure of the DAG. - - :param dag: Causal DAG from which the metamorphic relations will be generated. - :param nodes_to_ignore: Set of nodes which will be excluded from causal tests. - :param threads: Number of threads to use (if generating in parallel). - :param nodes_to_test: Set of nodes to test the relationships between (defaults to all nodes). - - :return: A list containing ShouldCause and ShouldNotCause metamorphic relations. - """ - - if nodes_to_ignore is None: - nodes_to_ignore = {} - - if nodes_to_test is None: - nodes_to_test = dag.nodes - - if threads < 2: - metamorphic_relations = [ - generate_metamorphic_relation(node_pair, dag, nodes_to_ignore) - for node_pair in combinations(filter(lambda node: node not in nodes_to_ignore, nodes_to_test), 2) - ] - else: - with Pool(threads) as pool: - metamorphic_relations = pool.starmap( - generate_metamorphic_relation, - map( - lambda node_pair: (node_pair, dag, nodes_to_ignore), - combinations(filter(lambda node: node not in nodes_to_ignore, nodes_to_test), 2), - ), - ) - - return [item for items in metamorphic_relations for item in items] - - -def generate_causal_tests( - dag_path: str, - output_path: str, - ignore_cycles: bool = False, - threads: int = 0, - test_inputs: bool = False, - **json_stub_kargs, -): - """ - Generate and output causal tests for a given DAG. - - :param dag_path: Path to the DOT file that specifies the causal DAG. - :param output_path: Path to save the JSON output. - :param ignore_cycles: Whether to bypass the check that the DAG is actually acyclic. If set to true, tests that - include variables that are part of a cycle as either treatment, outcome, or adjustment will - be omitted from the test set. - :param threads: The number of threads to use to generate tests in parallel. If unspecified, tests are generated in - serial. This is tylically fine unless the number of tests to be generated is >10000. - :param test_inputs: Whether to test independences between inputs (i.e. root nodes in the DAG). Defaults to False - as they will typically be independent by construction. - :param json_stub_kargs: Kwargs to pass into `to_json_stub` (see docstring for details.) - """ - causal_dag = CausalDAG(dag_path, ignore_cycles=ignore_cycles) - - dag_nodes_to_test = [ - node - for node in causal_dag.nodes - if nx.get_node_attributes(causal_dag, "test", default=True)[node] # pylint: disable=E1123 - ] - - if not causal_dag.is_acyclic() and ignore_cycles: - logger.warning( - "Ignoring cycles by removing causal tests that reference any node within a cycle. " - "Your causal test suite WILL NOT BE COMPLETE!" - ) - relations = generate_metamorphic_relations( - causal_dag, - nodes_to_test=dag_nodes_to_test, - nodes_to_ignore=set(causal_dag.cycle_nodes()), - threads=threads, - ) - else: - relations = generate_metamorphic_relations(causal_dag, nodes_to_test=dag_nodes_to_test, threads=threads) - - tests = [ - relation.to_json_stub(**json_stub_kargs) - for relation in relations - if test_inputs or len(list(causal_dag.predecessors(relation.base_test_case.outcome_variable))) > 0 - ] - - logger.warning( - "The skip parameter is hard-coded to False during test generation for better integration with the " - "causal testing component (causal-testing test ...)" - "Please carefully review the generated tests and decide which to skip." - ) - - logger.info(f"Generated {len(tests)} tests. Saving to {output_path}.") - with open(output_path, "w", encoding="utf-8") as f: - json.dump({"tests": tests}, f, indent=2) diff --git a/tests/main_tests/test_main.py b/tests/main_tests/test_main.py index dc0b2a94..df7983a7 100644 --- a/tests/main_tests/test_main.py +++ b/tests/main_tests/test_main.py @@ -51,7 +51,7 @@ def test_parse_args_adequacy(self): str(self.test_cases_path), "--output", str(self.output_path.parent / "main.json"), - "-a", + "-A", ], ): main() @@ -98,7 +98,7 @@ def test_parse_args_bootstrap_size_explicit_adequacy(self): str(self.test_cases_path), "--output", str(self.output_path.parent / "main.json"), - "-a", + "-A", "-b", "50", ], @@ -118,6 +118,8 @@ def test_parse_args_generation(self): "generate", "--dag-path", str(self.dag_path), + "--data-paths", + str(self.data_paths[0]), "--output", os.path.join(tmp, "tests.json"), ], @@ -125,28 +127,6 @@ def test_parse_args_generation(self): main() self.assertTrue(os.path.exists(os.path.join(tmp, "tests.json"))) - def test_parse_args_generation_non_default(self): - with tempfile.TemporaryDirectory() as tmp: - with patch( - "sys.argv", - [ - "causal_testing", - "generate", - "--dag-path", - str(self.dag_path), - "--output", - os.path.join(tmp, "tests_non_default.json"), - "--estimator", - "LogisticRegressionEstimator", - "--estimate-type", - "unit_odds_ratio", - "--effect-type", - "total", - ], - ): - main() - self.assertTrue(os.path.exists(os.path.join(tmp, "tests_non_default.json"))) - def test_parse_args_discover(self): with tempfile.TemporaryDirectory() as tmp: with patch( diff --git a/tests/testing_tests/test_metamorphic_relations.py b/tests/testing_tests/test_metamorphic_relations.py index 4df2bc5c..1663e296 100644 --- a/tests/testing_tests/test_metamorphic_relations.py +++ b/tests/testing_tests/test_metamorphic_relations.py @@ -2,18 +2,18 @@ import os import shutil, tempfile import json +import networkx as nx from causal_testing.specification.causal_dag import CausalDAG from causal_testing.specification.scenario import Scenario -from causal_testing.testing.metamorphic_relation import ( - ShouldCause, - ShouldNotCause, - generate_metamorphic_relations, - generate_metamorphic_relation, - generate_causal_tests, -) from causal_testing.specification.variable import Input, Output from causal_testing.testing.base_test_case import BaseTestCase +from causal_testing.testing.causal_test_case import CausalTestCase +from causal_testing.testing.causal_effect import NoEffect, SomeEffect +from causal_testing.estimation.abstract_estimator import Estimator +from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator +from causal_testing.estimation.logistic_regression_estimator import LogisticRegressionEstimator +from causal_testing.estimation.multinomial_regression_estimator import MultinomialRegressionEstimator class TestMetamorphicRelation(unittest.TestCase): @@ -41,291 +41,110 @@ def setUp(self) -> None: def tearDown(self) -> None: shutil.rmtree(self.temp_dir_path) - def test_json_stub_invalid_estimator(self): - """Test if the ShouldCause MR passes all metamorphic tests where the DAG perfectly represents the program - and there is only a single input.""" - causal_dag = CausalDAG(self.dag_dot_path) - causal_dag.remove_nodes_from(["X2", "X3"]) - adj_set = list(causal_dag.direct_effect_adjustment_sets(["X1"], ["Z"])[0]) - should_not_cause_mr = ShouldNotCause(BaseTestCase("X1", "Z"), adj_set) - with self.assertRaises(ValueError) as e: - should_not_cause_mr.to_json_stub(estimator="InvalidEstimator") - self.assertTrue(e.exception.startswith("Unsupported estimator estimator InvalidEstimator.")) - - def test_should_not_cause_json_stub(self): - """Test if the ShouldCause MR passes all metamorphic tests where the DAG perfectly represents the program - and there is only a single input.""" - causal_dag = CausalDAG(self.dag_dot_path) - causal_dag.remove_nodes_from(["X2", "X3"]) - adj_set = list(causal_dag.direct_effect_adjustment_sets(["X1"], ["Z"])[0]) - should_not_cause_mr = ShouldNotCause(BaseTestCase("X1", "Z"), adj_set) - self.assertEqual( - should_not_cause_mr.to_json_stub(), - { - "effect": "direct", - "estimate_type": "coefficient", - "estimator": "LinearRegressionEstimator", - "expected_effect": {"Z": "NoEffect"}, - "treatment_variable": "X1", - "name": "X1 _||_ Z", - "estimator_kwargs": {"formula": "Z ~ X1"}, - "alpha": 0.05, - "skip": False, - }, - ) - - def test_should_not_cause_logistic_json_stub(self): - """Test if the ShouldCause MR passes all metamorphic tests where the DAG perfectly represents the program - and there is only a single input.""" - causal_dag = CausalDAG(self.dag_dot_path) - causal_dag.remove_nodes_from(["X2", "X3"]) - adj_set = list(causal_dag.direct_effect_adjustment_sets(["X1"], ["Z"])[0]) - should_not_cause_mr = ShouldNotCause(BaseTestCase("X1", "Z"), adj_set) - self.assertEqual( - should_not_cause_mr.to_json_stub( - effect_type="total", estimate_type="unit_odds_ratio", estimator="LogisticRegressionEstimator" - ), - { - "effect": "total", - "estimate_type": "unit_odds_ratio", - "estimator": "LogisticRegressionEstimator", - "expected_effect": {"Z": "NoEffect"}, - "treatment_variable": "X1", - "name": "X1 _||_ Z", - "estimator_kwargs": {"formula": "Z ~ X1"}, - "alpha": 0.05, - "skip": False, - }, - ) - - def test_should_cause_json_stub(self): - """Test if the ShouldCause MR passes all metamorphic tests where the DAG perfectly represents the program - and there is only a single input.""" - causal_dag = CausalDAG(self.dag_dot_path) - causal_dag.remove_nodes_from(["X2", "X3"]) - adj_set = list(causal_dag.direct_effect_adjustment_sets(["X1"], ["Z"])[0]) - should_cause_mr = ShouldCause(BaseTestCase("X1", "Z"), adj_set) - self.assertEqual( - should_cause_mr.to_json_stub(), - { - "effect": "direct", - "estimate_type": "coefficient", - "estimator": "LinearRegressionEstimator", - "expected_effect": {"Z": "SomeEffect"}, - "estimator_kwargs": {"formula": "Z ~ X1"}, - "treatment_variable": "X1", - "name": "X1 --> Z", - "alpha": 0.05, - "skip": False, - }, - ) - - def test_should_cause_logistic_json_stub(self): - """Test if the ShouldCause MR passes all metamorphic tests where the DAG perfectly represents the program - and there is only a single input.""" - causal_dag = CausalDAG(self.dag_dot_path) - causal_dag.remove_nodes_from(["X2", "X3"]) - adj_set = list(causal_dag.direct_effect_adjustment_sets(["X1"], ["Z"])[0]) - should_cause_mr = ShouldCause(BaseTestCase("X1", "Z"), adj_set) - self.assertEqual( - should_cause_mr.to_json_stub( - effect_type="total", - estimate_type="unit_odds_ratio", - estimator="LogisticRegressionEstimator", - skip=False, - ), - { - "effect": "total", - "estimate_type": "unit_odds_ratio", - "estimator": "LogisticRegressionEstimator", - "expected_effect": {"Z": "SomeEffect"}, - "estimator_kwargs": {"formula": "Z ~ X1"}, - "treatment_variable": "X1", - "name": "X1 --> Z", - "alpha": 0.05, - "skip": False, - }, - ) - def test_all_metamorphic_relations_implied_by_dag(self): - dag = CausalDAG(self.dag_dot_path) - print(dag) + dag = CausalDAG(self.dag_dot_path, datatypes={v: float for v in {"X1", "X2", "X3", "Y", "Z", "M"}}) dag.add_edge("Z", "Y") # Add a direct path from Z to Y so M becomes a mediator - metamorphic_relations = generate_metamorphic_relations(dag) - expected_relations = [ - ShouldCause(BaseTestCase("X1", "Z"), []), - ShouldNotCause(BaseTestCase("X1", "M"), ["Z"]), - ShouldNotCause(BaseTestCase("X1", "Y"), ["Z"]), - ShouldNotCause(BaseTestCase("X1", "X2"), []), - ShouldNotCause(BaseTestCase("X2", "X1"), []), - ShouldNotCause(BaseTestCase("X1", "X3"), []), - ShouldNotCause(BaseTestCase("X3", "X1"), []), - ShouldCause(BaseTestCase("Z", "M"), []), - ShouldCause(BaseTestCase("Z", "Y"), ["M"]), - ShouldCause(BaseTestCase("X2", "Z"), []), - ShouldNotCause(BaseTestCase("Z", "X3"), []), - ShouldNotCause(BaseTestCase("X3", "Z"), []), - ShouldCause(BaseTestCase("M", "Y"), ["Z"]), - ShouldNotCause(BaseTestCase("X2", "M"), ["Z"]), - ShouldCause(BaseTestCase("X3", "M"), []), - ShouldNotCause(BaseTestCase("X2", "Y"), ["Z"]), - ShouldNotCause(BaseTestCase("X3", "Y"), ["M", "Z"]), - ShouldNotCause(BaseTestCase("X2", "X3"), []), - ShouldNotCause(BaseTestCase("X3", "X2"), []), - ] + expected_tests = [] + for treatment, outcome in dag.edges: + base_test_case = BaseTestCase(Input(treatment, float), Output(outcome, float)) + expected_tests.append( + CausalTestCase( + base_test_case=base_test_case, + expected_causal_effect=SomeEffect(), + estimate_type="coefficient", + estimator=LinearRegressionEstimator(base_test_case), + name=f"{treatment} -> {outcome}", + skip=False, + ) + ) + for treatment, outcome in [ + ("X1", "M"), + ("X1", "Y"), + ("X1", "X2"), + ("X2", "X1"), + ("X1", "X3"), + ("X3", "X1"), + ("Z", "X3"), + ("X3", "Z"), + ("X2", "M"), + ("X2", "Y"), + ("X3", "Y"), + ("X2", "X3"), + ("X3", "X2"), + ]: + base_test_case = BaseTestCase(Input(treatment, float), Output(outcome, float)) + expected_tests.append( + CausalTestCase( + base_test_case=base_test_case, + expected_causal_effect=NoEffect(), + estimate_type="coefficient", + estimator=LinearRegressionEstimator(base_test_case), + name=f"{treatment} -> {outcome}", + skip=False, + ) + ) - self.assertEqual(expected_relations, metamorphic_relations) + self.assertEqual(sorted(map(str, expected_tests)), sorted(map(str, dag.generate_causal_tests()))) def test_all_metamorphic_relations_implied_by_dag_parallel(self): - dag = CausalDAG(self.dag_dot_path) + dag = CausalDAG(self.dag_dot_path, datatypes={v: float for v in {"X1", "X2", "X3", "Y", "Z", "M"}}) dag.add_edge("Z", "Y") # Add a direct path from Z to Y so M becomes a mediator - metamorphic_relations = generate_metamorphic_relations(dag, threads=2) - - expected_relations = [ - ShouldCause(BaseTestCase("X1", "Z"), []), - ShouldNotCause(BaseTestCase("X1", "M"), ["Z"]), - ShouldNotCause(BaseTestCase("X1", "Y"), ["Z"]), - ShouldNotCause(BaseTestCase("X1", "X2"), []), - ShouldNotCause(BaseTestCase("X2", "X1"), []), - ShouldNotCause(BaseTestCase("X1", "X3"), []), - ShouldNotCause(BaseTestCase("X3", "X1"), []), - ShouldCause(BaseTestCase("Z", "M"), []), - ShouldCause(BaseTestCase("Z", "Y"), ["M"]), - ShouldCause(BaseTestCase("X2", "Z"), []), - ShouldNotCause(BaseTestCase("Z", "X3"), []), - ShouldNotCause(BaseTestCase("X3", "Z"), []), - ShouldCause(BaseTestCase("M", "Y"), ["Z"]), - ShouldNotCause(BaseTestCase("X2", "M"), ["Z"]), - ShouldCause(BaseTestCase("X3", "M"), []), - ShouldNotCause(BaseTestCase("X2", "Y"), ["Z"]), - ShouldNotCause(BaseTestCase("X3", "Y"), ["M", "Z"]), - ShouldNotCause(BaseTestCase("X2", "X3"), []), - ShouldNotCause(BaseTestCase("X3", "X2"), []), - ] - - self.assertEqual(expected_relations, metamorphic_relations) - - def test_all_metamorphic_relations_implied_by_dag_ignore_cycles(self): - dcg = CausalDAG(self.dcg_dot_path, ignore_cycles=True) - metamorphic_relations = generate_metamorphic_relations(dcg, threads=2, nodes_to_ignore=set(dcg.cycle_nodes())) - should_cause_relations = [mr for mr in metamorphic_relations if isinstance(mr, ShouldCause)] - should_not_cause_relations = [mr for mr in metamorphic_relations if isinstance(mr, ShouldNotCause)] - - # Check all ShouldCause relations are present and no extra - self.assertEqual( - should_cause_relations, - [ - ShouldCause(BaseTestCase("a", "b"), []), - ], - ) - self.assertEqual( - should_not_cause_relations, - [], - ) - - def test_generate_metamorphic_relation_(self): - dag = CausalDAG(self.dag_dot_path) - [metamorphic_relation] = generate_metamorphic_relation(("X1", "Z"), dag) - self.assertEqual( - metamorphic_relation, - ShouldCause(BaseTestCase("X1", "Z"), []), - ) - - def test_generate_causal_tests_ignore_cycles(self): - dcg = CausalDAG(self.dcg_dot_path, ignore_cycles=True) - relations = generate_metamorphic_relations(dcg, nodes_to_ignore=set(dcg.cycle_nodes())) - with tempfile.TemporaryDirectory() as tmp: - tests_file = os.path.join(tmp, "causal_tests.json") - generate_causal_tests(self.dcg_dot_path, tests_file, ignore_cycles=True) - with open(tests_file, encoding="utf8") as f: - tests = json.load(f) - expected = list( - map( - lambda x: x.to_json_stub(skip=False), - filter( - lambda relation: len(list(dcg.predecessors(relation.base_test_case.outcome_variable))) > 0, - relations, - ), + expected_tests = [] + for treatment, outcome in dag.edges: + base_test_case = BaseTestCase(Input(treatment, float), Output(outcome, float)) + expected_tests.append( + CausalTestCase( + base_test_case=base_test_case, + expected_causal_effect=SomeEffect(), + estimate_type="coefficient", + estimator=LinearRegressionEstimator(base_test_case), + name=f"{treatment} -> {outcome}", + skip=False, ) ) - self.assertEqual(tests["tests"], expected) - - def test_generate_causal_tests(self): - dag = CausalDAG(self.dag_dot_path) - relations = generate_metamorphic_relations(dag) - with tempfile.TemporaryDirectory() as tmp: - tests_file = os.path.join(tmp, "causal_tests.json") - generate_causal_tests(self.dag_dot_path, tests_file) - with open(tests_file, encoding="utf8") as f: - tests = json.load(f) - expected = list( - map( - lambda x: x.to_json_stub(skip=False), - filter( - lambda relation: len(list(dag.predecessors(relation.base_test_case.outcome_variable))) > 0, - relations, - ), + for treatment, outcome in [ + ("X1", "M"), + ("X1", "Y"), + ("X1", "X2"), + ("X2", "X1"), + ("X1", "X3"), + ("X3", "X1"), + ("Z", "X3"), + ("X3", "Z"), + ("X2", "M"), + ("X2", "Y"), + ("X3", "Y"), + ("X2", "X3"), + ("X3", "X2"), + ]: + base_test_case = BaseTestCase(Input(treatment, float), Output(outcome, float)) + expected_tests.append( + CausalTestCase( + base_test_case=base_test_case, + expected_causal_effect=NoEffect(), + estimate_type="coefficient", + estimator=LinearRegressionEstimator(base_test_case), + name=f"{treatment} -> {outcome}", + skip=False, ) ) - self.assertEqual(tests["tests"], expected) - - def test_generate_causal_tests_test_inputs(self): - dag = CausalDAG(self.dag_dot_path) - relations = generate_metamorphic_relations(dag) - with tempfile.TemporaryDirectory() as tmp: - tests_file = os.path.join(tmp, "causal_tests.json") - generate_causal_tests(self.dag_dot_path, tests_file, test_inputs=True) - with open(tests_file, encoding="utf8") as f: - tests = json.load(f) - expected = list( - map( - lambda x: x.to_json_stub(skip=False), - relations, - ) - ) - self.assertEqual(tests["tests"], expected) - - def test_shoud_cause_string(self): - sc_mr = ShouldCause(BaseTestCase("X", "Y"), ["A", "B", "C"]) - self.assertEqual(str(sc_mr), "X --> Y | ['A', 'B', 'C']") - - def test_shoud_not_cause_string(self): - sc_mr = ShouldNotCause(BaseTestCase("X", "Y"), ["A", "B", "C"]) - self.assertEqual(str(sc_mr), "X _||_ Y | ['A', 'B', 'C']") - - def test_equivalent_metamorphic_relations(self): - sc_mr_a = ShouldCause(BaseTestCase("X", "Y"), ["A", "B", "C"]) - sc_mr_b = ShouldCause(BaseTestCase("X", "Y"), ["A", "B", "C"]) - self.assertEqual(sc_mr_a == sc_mr_b, True) - def test_equivalent_metamorphic_relations_empty_adjustment_set(self): - sc_mr_a = ShouldCause(BaseTestCase("X", "Y"), []) - sc_mr_b = ShouldCause(BaseTestCase("X", "Y"), []) - self.assertEqual(sc_mr_a == sc_mr_b, True) + self.assertEqual(sorted(map(str, expected_tests)), sorted(map(str, dag.generate_causal_tests(threads=2)))) - def test_equivalent_metamorphic_relations_different_order_adjustment_set(self): - sc_mr_a = ShouldCause(BaseTestCase("X", "Y"), ["A", "B", "C"]) - sc_mr_b = ShouldCause(BaseTestCase("X", "Y"), ["C", "A", "B"]) - self.assertEqual(sc_mr_a == sc_mr_b, True) - - def test_different_metamorphic_relations_empty_adjustment_set_different_outcome(self): - sc_mr_a = ShouldCause(BaseTestCase("X", "Z"), []) - sc_mr_b = ShouldCause(BaseTestCase("X", "Y"), []) - self.assertEqual(sc_mr_a == sc_mr_b, False) - - def test_different_metamorphic_relations_empty_adjustment_set_different_treatment(self): - sc_mr_a = ShouldCause(BaseTestCase("X", "Y"), []) - sc_mr_b = ShouldCause(BaseTestCase("Z", "Y"), []) - self.assertEqual(sc_mr_a == sc_mr_b, False) - - def test_different_metamorphic_relations_empty_adjustment_set_adjustment_set(self): - sc_mr_a = ShouldCause(BaseTestCase("X", "Y"), ["A"]) - sc_mr_b = ShouldCause(BaseTestCase("X", "Y"), []) - self.assertEqual(sc_mr_a == sc_mr_b, False) - - def test_different_metamorphic_relations_different_type(self): - sc_mr_a = ShouldCause(BaseTestCase("X", "Y"), []) - sc_mr_b = ShouldNotCause(BaseTestCase("X", "Y"), []) - self.assertEqual(sc_mr_a == sc_mr_b, False) + def test_all_metamorphic_relations_implied_by_dag_ignore_cycles(self): + dcg = CausalDAG(self.dcg_dot_path, ignore_cycles=True, datatypes={v: float for v in {"a", "b", "c", "d"}}) + + base_test_case = BaseTestCase(Input("a", float), Output("b", float)) + expected_tests = [ + CausalTestCase( + base_test_case=base_test_case, + expected_causal_effect=SomeEffect(), + estimate_type="coefficient", + estimator=LinearRegressionEstimator(base_test_case), + name=f"a -> b", + skip=False, + ) + ] + self.assertEqual(sorted(map(str, expected_tests)), sorted(map(str, dcg.generate_causal_tests(threads=2)))) From 175acdaabe69e0878fb9ecaeb69cf97c0e70f573 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Mon, 20 Jul 2026 11:18:19 +0100 Subject: [PATCH 4/7] Removed print --- causal_testing/causal_testing_framework.py | 1 - 1 file changed, 1 deletion(-) diff --git a/causal_testing/causal_testing_framework.py b/causal_testing/causal_testing_framework.py index 2b71e7d4..d88b3cdc 100644 --- a/causal_testing/causal_testing_framework.py +++ b/causal_testing/causal_testing_framework.py @@ -315,7 +315,6 @@ def evaluate_dag(self, bootstrap_size: bool = 100, alpha: float = 0.05) -> pd.Se results[f"{outcome.name}_ci_low"] = data[ci_low_inx] results[f"{outcome.name}_ci_high"] = data[ci_high_inx] - print(results) return pd.Series(results).sort_index() def save_results(self, output_path) -> list: From bcb5427be991919114ca9ad5228f1f53a8cab3a2 Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Tue, 21 Jul 2026 12:01:18 +0100 Subject: [PATCH 5/7] Removed MetamorphicRelation class - now generating CausalTest objects directly --- causal_testing/__main__.py | 2 +- .../estimation/abstract_estimator.py | 20 ++-- .../abstract_regression_estimator.py | 17 +++- .../instrumental_variable_estimator.py | 8 ++ .../estimation/linear_regression_estimator.py | 3 - causal_testing/specification/causal_dag.py | 25 +++-- causal_testing/testing/base_test_case.py | 10 ++ causal_testing/testing/causal_effect.py | 91 +++++++++++-------- causal_testing/testing/causal_test_case.py | 64 ++++++------- causal_testing/testing/causal_test_result.py | 19 +++- causal_testing/testing/data_adequacy.py | 23 +++-- tests/testing_tests/test_causal_effect.py | 8 +- .../test_causal_test_adequacy.py | 10 +- tests/testing_tests/test_causal_test_case.py | 83 +++++++++-------- .../test_metamorphic_relations.py | 47 +++++++--- 15 files changed, 270 insertions(+), 160 deletions(-) diff --git a/causal_testing/__main__.py b/causal_testing/__main__.py index 1d6853dc..5296e612 100644 --- a/causal_testing/__main__.py +++ b/causal_testing/__main__.py @@ -192,7 +192,7 @@ def main() -> None: skip=False, ) with open(args.output, "w", encoding="utf-8") as f: - json.dump({"tests": [test.to_json() for test in causal_tests]}, f) + json.dump({"tests": [test.to_dict() for test in causal_tests]}, f) logging.info("Causal test generation completed successfully.") case Command.DISCOVER: diff --git a/causal_testing/estimation/abstract_estimator.py b/causal_testing/estimation/abstract_estimator.py index 18cb83b7..c3cb61d4 100644 --- a/causal_testing/estimation/abstract_estimator.py +++ b/causal_testing/estimation/abstract_estimator.py @@ -32,9 +32,9 @@ def __init__( # pylint: disable=R0801 self, base_test_case: BaseTestCase, - treatment_value: float, - control_value: float, - adjustment_set: set, + control_value: float = None, + treatment_value: float = None, + adjustment_set: set = None, effect_modifiers: dict[str, Any] = None, alpha: float = 0.05, ): @@ -59,13 +59,17 @@ def add_modelling_assumptions(self): must hold if the resulting causal inference is to be considered valid. """ - def to_json(self) -> dict: + def to_dict(self) -> dict: """ - Convert to a JSON serialisable dict object containing all non-standard parameters to reconstruct. + Convert the estimator to a python dictionary for easy serialisation as JSON or CSV. - :returns: A JSON serialisable dict representing the object. + :returns: A JSON serialisable dict representing the estimator. """ - result = {"estimator": self.__class__.__name__, "estimator_kwargs": {}} + result = {"name": self.__class__.__name__, "alpha": self.alpha, "adjustment_set": sorted(self.adjustment_set)} if self.effect_modifiers: - result["estimator_kwargs"]["effect_modifiers"] = self.effect_modifiers + result["effect_modifiers"] = self.effect_modifiers + if self.control_value is not None: + result["control_value"] = self.control_value + if self.treatment_value is not None: + result["treatment_value"] = self.treatment_value return result diff --git a/causal_testing/estimation/abstract_regression_estimator.py b/causal_testing/estimation/abstract_regression_estimator.py index cebfe727..0e4e95b4 100644 --- a/causal_testing/estimation/abstract_regression_estimator.py +++ b/causal_testing/estimation/abstract_regression_estimator.py @@ -24,8 +24,8 @@ def __init__( # pylint: disable=too-many-arguments self, base_test_case: BaseTestCase, - treatment_value: float = None, control_value: float = None, + treatment_value: float = None, adjustment_set: set = None, effect_modifiers: dict[Variable, Any] = None, adjustment_config: dict[Variable, Any] = None, @@ -35,8 +35,8 @@ def __init__( # pylint: disable=R0801 super().__init__( base_test_case=base_test_case, - treatment_value=treatment_value, control_value=control_value, + treatment_value=treatment_value, adjustment_set=adjustment_set, effect_modifiers=effect_modifiers, alpha=alpha, @@ -144,3 +144,16 @@ def _predict(self, df) -> pd.DataFrame: x = pd.get_dummies(x, columns=[col], drop_first=True) return model.get_prediction(x).summary_frame() + + def to_dict(self) -> dict: + """ + Convert the estimator to a python dictionary for easy serialisation as JSON or CSV. + + :returns: A JSON serialisable dict representing the estimator. + """ + result = super().to_dict() + if self.adjustment_config: + result["adjustment_config"] = self.adjustment_config + if self.formula: + result["formula"] = self.formula + return result diff --git a/causal_testing/estimation/instrumental_variable_estimator.py b/causal_testing/estimation/instrumental_variable_estimator.py index 8ff237ee..20f9d716 100644 --- a/causal_testing/estimation/instrumental_variable_estimator.py +++ b/causal_testing/estimation/instrumental_variable_estimator.py @@ -87,3 +87,11 @@ def estimate_coefficient(self, df: pd.DataFrame) -> EffectEstimate: ci_high = pd.Series(bootstraps[self.bootstrap_size - bound]) return EffectEstimate("coefficient", pd.Series(self.iv_coefficient(df)), ci_low, ci_high) + + def to_dict(self) -> dict: + """ + Convert the estimator to a python dictionary for easy serialisation as JSON or CSV. + + :returns: A JSON serialisable dict representing the estimator. + """ + return super().to_dict() | {"instrument": self.instrument, "bootstrap_size": self.bootstrap_size} diff --git a/causal_testing/estimation/linear_regression_estimator.py b/causal_testing/estimation/linear_regression_estimator.py index daa8a300..55ae1dee 100644 --- a/causal_testing/estimation/linear_regression_estimator.py +++ b/causal_testing/estimation/linear_regression_estimator.py @@ -95,9 +95,6 @@ def estimate_coefficient(self, df: pd.DataFrame) -> EffectEstimate: unit_effect = model.params[treatment] # Unit effect is the coefficient of the treatment [ci_low, ci_high] = self._get_confidence_intervals(model, treatment) - if len(unit_effect) == 0: - unit_effect = pd.Series({self.base_test_case.treatment_variable.name: None}) - return EffectEstimate("coefficient", unit_effect, ci_low, ci_high) def estimate_ate(self, df: pd.DataFrame) -> EffectEstimate: diff --git a/causal_testing/specification/causal_dag.py b/causal_testing/specification/causal_dag.py index ed13a9a2..035d774b 100644 --- a/causal_testing/specification/causal_dag.py +++ b/causal_testing/specification/causal_dag.py @@ -498,12 +498,12 @@ def get_backdoor_graph(self, treatments: list[str]) -> CausalDAG: backdoor_graph.add_edges_from(filter(lambda x: x not in outgoing_edges, self.edges)) return backdoor_graph - def identification(self, base_test_case: BaseTestCase, avoid_variables: set[Variable] = None): + def identification(self, base_test_case: BaseTestCase, nodes_to_ignore: set[Variable] = None): """Identify and return the minimum adjustment set :param base_test_case: A base test case instance containing the outcome_variable and the treatment_variable required for identification. - :param avoid_variables: Variables not to be adjusted for (e.g. hidden variables). + :param nodes_to_ignore: Variables not to be adjusted for (e.g. hidden variables). :return: The smallest set of variables which can be adjusted for to obtain a causal estimate as opposed to a purely associational estimate. """ @@ -517,14 +517,16 @@ def identification(self, base_test_case: BaseTestCase, avoid_variables: set[Vari ) elif base_test_case.effect == "direct": minimal_adjustment_sets = self.direct_effect_adjustment_sets( - [base_test_case.treatment_variable.name], [base_test_case.outcome_variable.name] + [base_test_case.treatment_variable.name], + [base_test_case.outcome_variable.name], + nodes_to_ignore=nodes_to_ignore, ) else: raise ValueError("Causal effect should be 'total' or 'direct'") - if avoid_variables is not None: + if nodes_to_ignore is not None: minimal_adjustment_sets = [ - adj for adj in minimal_adjustment_sets if not {x.name for x in avoid_variables}.intersection(adj) + adj for adj in minimal_adjustment_sets if not {x.name for x in nodes_to_ignore}.intersection(adj) ] minimal_adjustment_set = min(minimal_adjustment_sets, key=len, default=set()) @@ -544,19 +546,12 @@ def __str__(self): return f"Nodes: {self.nodes}\nEdges: {self.edges}" def _estimator(self, base_test_case: BaseTestCase, nodes_to_ignore: set) -> Estimator: - treatment_variable = base_test_case.treatment_variable.name outcome_variable = base_test_case.outcome_variable.name if self.datatypes is None or outcome_variable not in self.datatypes: raise ValueError(f"No datatype specified for {outcome_variable}.") - adj_sets = self.direct_effect_adjustment_sets( - [treatment_variable], [outcome_variable], nodes_to_ignore=nodes_to_ignore - ) - if not adj_sets: - return None, None - - min_adj_set = sorted(list(map(lambda s: sorted(list(s)), adj_sets)))[0] + min_adj_set = self.identification(base_test_case, nodes_to_ignore=nodes_to_ignore) if pd.api.types.is_bool_dtype(self.datatypes[outcome_variable]): return ( @@ -606,6 +601,7 @@ def generate_causal_test( # pylint: disable=R0912 if estimator and estimate_type: causal_tests.append( CausalTestCase( + name=f"{u} _||_ {v}", base_test_case=base_test_case, expected_causal_effect=NoEffect(), estimator=estimator, @@ -621,6 +617,7 @@ def generate_causal_test( # pylint: disable=R0912 if estimator and estimate_type: causal_tests.append( CausalTestCase( + name=f"{v} _||_ {u}", base_test_case=base_test_case, expected_causal_effect=NoEffect(), estimator=estimator, @@ -636,6 +633,7 @@ def generate_causal_test( # pylint: disable=R0912 if estimator and estimate_type: causal_tests.append( CausalTestCase( + name=f"{u} -> {v}", base_test_case=base_test_case, expected_causal_effect=SomeEffect(), estimator=estimator, @@ -649,6 +647,7 @@ def generate_causal_test( # pylint: disable=R0912 if estimator and estimate_type: causal_tests.append( CausalTestCase( + name=f"{v} -> {u}", base_test_case=base_test_case, expected_causal_effect=SomeEffect(), estimator=estimator, diff --git a/causal_testing/testing/base_test_case.py b/causal_testing/testing/base_test_case.py index aca1b376..67d6b914 100644 --- a/causal_testing/testing/base_test_case.py +++ b/causal_testing/testing/base_test_case.py @@ -22,3 +22,13 @@ class BaseTestCase: def __post_init__(self): if self.treatment_variable == self.outcome_variable: raise ValueError(f"Treatment variable {self.treatment_variable} cannot also be the outcome variable.") + + def to_dict(self) -> dict: + """ + :returns: A JSON serialisable dictionary representing the base test case. + """ + return { + "treatment_variable": self.treatment_variable.name, + "outcome_variable": self.outcome_variable.name, + "effect": self.effect, + } diff --git a/causal_testing/testing/causal_effect.py b/causal_testing/testing/causal_effect.py index 4247cce4..29b14eae 100644 --- a/causal_testing/testing/causal_effect.py +++ b/causal_testing/testing/causal_effect.py @@ -3,7 +3,6 @@ ExactValue, Positive, Negative, SomeEffect, NoEffect""" from abc import ABC, abstractmethod -from collections.abc import Iterable import numpy as np @@ -23,25 +22,27 @@ def apply(self, effect_estimate: EffectEstimate) -> bool: def __str__(self) -> str: return type(self).__name__ + def to_dict(self): + """ + Convert the expected effect to a python dictionary for easy serialisation as JSON. + + :returns: A JSON serialisable dict representing the expected effect. + """ + return {"name": self.__class__.__name__} + class SomeEffect(CausalEffect): """An extension of CausalEffect representing that the expected causal effect should not be zero.""" def apply(self, effect_estimate: EffectEstimate) -> bool: - if effect_estimate.ci_low is None or effect_estimate.ci_high is None: - return None - if effect_estimate.type in ("risk_ratio", "hazard_ratio", "unit_odds_ratio"): - return any( - 1 < ci_low < ci_high or ci_low < ci_high < 1 - for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high) - ) - if effect_estimate.type in ("coefficient", "ate"): - return any( - 0 < ci_low < ci_high or ci_low < ci_high < 0 - for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high) - ) + if effect_estimate.type in ("risk_ratio", "hazard_ratio", "unit_odds_ratio", "odds_ratio"): + value_to_check = 1 + elif effect_estimate.type in ("coefficient", "ate"): + value_to_check = 0 + else: + raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect") - raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect") + return (~((effect_estimate.ci_low <= value_to_check) & (value_to_check <= effect_estimate.ci_high))).all() class NoEffect(CausalEffect): @@ -52,30 +53,30 @@ class NoEffect(CausalEffect): :param ctol: Categorical tolerance. The test will pass if this proportion of categories pass. """ - def __init__(self, atol: float = 1e-10, ctol: float = 0.05): + def __init__(self, atol: float = 0, ctol: float = 0.0): self.atol = atol self.ctol = ctol def apply(self, effect_estimate: EffectEstimate) -> bool: if effect_estimate.type in ("risk_ratio", "hazard_ratio", "unit_odds_ratio", "odds_ratio"): - return any( - ci_low < 1 < ci_high or np.isclose(value, 1.0, atol=self.atol) - for ci_low, ci_high, value in zip( - effect_estimate.ci_low, effect_estimate.ci_high, effect_estimate.value - ) - ) - if effect_estimate.type in ("coefficient", "ate"): - value = effect_estimate.value if isinstance(effect_estimate.ci_high, Iterable) else [effect_estimate.value] - return ( - sum( - not ((ci_low < 0 < ci_high) or abs(v) < self.atol) - for ci_low, ci_high, v in zip(effect_estimate.ci_low, effect_estimate.ci_high, value) - ) - / len(value) - < self.ctol - ) + value_to_check = 1 + elif effect_estimate.type in ("coefficient", "ate"): + value_to_check = 0 + else: + raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect") + + return sum( + ((effect_estimate.ci_low <= value_to_check) & (value_to_check <= effect_estimate.ci_high)) + | (np.isclose(effect_estimate.value, value_to_check, atol=self.atol)) + ) / len(effect_estimate.value) >= (1 - self.ctol) + + def to_dict(self): + """ + Convert the expected effect to a python dictionary for easy serialisation as JSON. - raise ValueError(f"Test Value type {effect_estimate.type} is not valid for this CausalEffect") + :returns: A JSON serialisable dict representing the expected effect. + """ + return {"name": self.__class__.__name__, "atol": self.atol, "ctol": self.ctol} class ExactValue(CausalEffect): @@ -98,21 +99,37 @@ def __init__(self, value: float, atol: float = None, ci_low: float = None, ci_hi if self.value - self.atol < self.ci_low or self.value + self.atol > self.ci_high: raise ValueError( "Arithmetic tolerance falls outside the confidence intervals." - "Try specifying a smaller value of atol." + f"Try specifying wider intervals or a value of atol smaller than the current vlaue {self.atol}." ) def apply(self, effect_estimate: EffectEstimate) -> bool: close = np.isclose(effect_estimate.value, self.value, atol=self.atol) if effect_estimate.ci_valid and self.ci_low is not None and self.ci_high is not None: - return all( - close and self.ci_low <= ci_low and self.ci_high >= ci_high - for ci_low, ci_high in zip(effect_estimate.ci_low, effect_estimate.ci_high) + return ( + close.all() + and (self.ci_low <= effect_estimate.ci_low).all() + and (self.ci_high >= effect_estimate.ci_high).all() ) - return close + return close.all() def __str__(self): return f"ExactValue: {self.value}±{self.atol}" + def to_dict(self): + """ + Convert the expected effect to a python dictionary for easy serialisation as JSON or CSV. + + :returns: A JSON serialisable dict representing the expected effect. + """ + effect = {"value": self.value} + if self.ci_low: + effect["ci_low"] = self.ci_low + if self.ci_low: + effect["ci_high"] = self.ci_high + if self.atol: + effect["atol"] = self.atol + return {"name": self.__class__.__name__} | effect + class Positive(SomeEffect): """An extension of CausalEffect representing that the expected causal effect should be positive. diff --git a/causal_testing/testing/causal_test_case.py b/causal_testing/testing/causal_test_case.py index d6f35989..84969e5f 100644 --- a/causal_testing/testing/causal_test_case.py +++ b/causal_testing/testing/causal_test_case.py @@ -69,28 +69,31 @@ def measure_adequacy( if group_by is not None: ids = pd.Series(df[group_by].unique()) ids = ids.sample(len(ids), replace=True, random_state=i) - df = df[df[group_by].isin(ids)] + sample_df = df[df[group_by].isin(ids)] else: - df = df.sample(len(df), replace=True, random_state=i) + sample_df = df.sample(len(df), replace=True, random_state=i) try: - effect_estimate = self.estimate_effect(df) + effect_estimate = self.estimate_effect(sample_df) outcomes.append(self.expected_causal_effect.apply(effect_estimate)) - results.append(effect_estimate.to_df()) - # Could get a variety of exceptions here due to insufficient/badly formed data + results.append( + effect_estimate.to_df().assign( + test_index=i, passed=self.expected_causal_effect.apply(effect_estimate) + ) + ) + # Could get a variety of exceptions here due to insufficient/badly formed data in the sample # We don't want these to stop execution except Exception: # pylint: disable=W0718 - pass + outcomes.append(None) results = pd.concat(results) results["var"] = results.index - results["passed"] = outcomes return DataAdequacy( results=results, kurtosis=results.groupby("var")["effect_estimate"].apply(lambda x: x.kurtosis()), - passing=sum(filter(lambda x: x is not None, outcomes)), - successful=sum(x is not None for x in outcomes), + passing=int(sum(filter(lambda x: x is not None, outcomes))), + successful=int(sum(x is not None for x in outcomes)), ) def execute_test( @@ -149,29 +152,28 @@ def estimate_effect(self, df: pd.DataFrame) -> CausalTestResult: estimate_effect = getattr(self.estimator, f"estimate_{self.estimate_type}") return estimate_effect(df) - def __str__(self): - treatment_config = {self.treatment_variable.name: self.estimator.treatment_value} - control_config = {self.treatment_variable.name: self.estimator.control_value} - outcome_variable = {self.outcome_variable.name} - return ( - f"Running {treatment_config} instead of {control_config} should cause the following " - f"changes to {outcome_variable}: {self.expected_causal_effect}." - ) - - def to_json(self): + def to_dict(self) -> dict: """ - Convert to a JSON serialisable dict object containing all non-standard parameters to reconstruct. + Convert the test case to a python dictionary for easy serialisation as JSON. - :returns: A JSON serialisable dict representing the object. + :returns: A JSON serialisable dict representing the test case. """ - test_case = { - "name": self.name, - "treatment_variable": self.base_test_case.treatment_variable.name, - "estimate_type": self.estimate_type, - "expected_effect": { - self.base_test_case.outcome_variable.name: self.expected_causal_effect.__class__.__name__ - }, - "skip": self.skip, - "query": self.query, - } | self.estimator.to_json() + test_case = ( + {"name": self.name} + | self.base_test_case.to_dict() + | { + "skip": self.skip, + "estimate_type": self.estimate_type, + "query": self.query, + } + ) + + for label, attribute in [ + ("expected_effect", self.expected_causal_effect), + ("estimator", self.estimator), + ("result", self.result), + ]: + if attribute is not None: + test_case[label] = attribute.to_dict() + return test_case diff --git a/causal_testing/testing/causal_test_result.py b/causal_testing/testing/causal_test_result.py index 9e99894d..d6ecc9f5 100644 --- a/causal_testing/testing/causal_test_result.py +++ b/causal_testing/testing/causal_test_result.py @@ -21,7 +21,24 @@ def __init__( adequacy=None, error_message: str = None, ): - self.outcome = outcome self.effect_estimate = effect_estimate + self.outcome = outcome self.adequacy = adequacy self.error_message = error_message + + def to_dict(self): + """ + Convert the result to a python dictionary for easy serialisation as JSON. + + :returns: A JSON serialisable dict representing the test result. + """ + + outcome = {"outcome": self.outcome.name, "passed": self.outcome == TestOutcome.PASS} + if self.error_message: + outcome["error_message"] = self.error_message + + effect_estimate = self.effect_estimate.to_dict() + + adequacy = self.adequacy.to_dict() if self.adequacy else {} + + return outcome | effect_estimate | {"adequacy": adequacy} diff --git a/causal_testing/testing/data_adequacy.py b/causal_testing/testing/data_adequacy.py index 0f9c5105..552f89e4 100644 --- a/causal_testing/testing/data_adequacy.py +++ b/causal_testing/testing/data_adequacy.py @@ -4,6 +4,8 @@ import logging +from pandas import Series + logger = logging.getLogger(__name__) @@ -20,21 +22,26 @@ class DataAdequacy: # pylint: disable=too-many-instance-attributes def __init__( self, - kurtosis=None, - passing=None, - results=None, - successful=None, + kurtosis: Series = None, + passing: int = None, + results: dict = None, + successful: int = None, ): self.kurtosis = kurtosis self.passing = passing self.results = results self.successful = successful - def to_dict(self): - """Returns the adequacy object as a dictionary.""" - return { + def to_dict(self, include_results: bool = False): + """ + :returns: the adequacy object as a dictionary. + :param include_results: Whether to serialise the results. + """ + result = { "kurtosis": self.kurtosis.to_dict(), "passing": self.passing, "successful": self.successful, - "results": self.results.reset_index(drop=True).to_dict(), } + if include_results: + return result | {"results": self.results.reset_index(drop=True).to_dict()} + return result diff --git a/tests/testing_tests/test_causal_effect.py b/tests/testing_tests/test_causal_effect.py index 07e66ce5..cc6de084 100644 --- a/tests/testing_tests/test_causal_effect.py +++ b/tests/testing_tests/test_causal_effect.py @@ -54,6 +54,10 @@ def test_exactValue_pass(self): effect_estimate = EffectEstimate(type="ate", value=pd.Series(5.05)) self.assertTrue(ExactValue(5, 0.1).apply(effect_estimate)) + def test_exactValue_categorical_pass(self): + effect_estimate = EffectEstimate(type="ate", value=pd.Series({"color[T.red]": 5.05, "color[T.blue]": 4.03})) + self.assertTrue(ExactValue(pd.Series({"color[T.red]": 5, "color[T.blue]": 4}), 0.1).apply(effect_estimate)) + def test_exactValue_pass_ci(self): effect_estimate = EffectEstimate(type="ate", value=pd.Series(5.05), ci_low=pd.Series(4), ci_high=pd.Series(6)) self.assertTrue(ExactValue(5, 0.1).apply(effect_estimate)) @@ -133,10 +137,6 @@ def test_someEffect_fail(self): self.assertFalse(SomeEffect().apply(effect_estimate)) self.assertTrue(NoEffect().apply(effect_estimate)) - def test_someEffect_None(self): - effect_estimate = EffectEstimate(type="ate", value=pd.Series(0)) - self.assertEqual(SomeEffect().apply(effect_estimate), None) - def test_positive_risk_ratio_e_value(self): cv = CausalValidator() e_value = cv.estimate_e_value(1.5) diff --git a/tests/testing_tests/test_causal_test_adequacy.py b/tests/testing_tests/test_causal_test_adequacy.py index 36ed256a..233a95f4 100644 --- a/tests/testing_tests/test_causal_test_adequacy.py +++ b/tests/testing_tests/test_causal_test_adequacy.py @@ -53,7 +53,7 @@ def test_data_adequacy_numeric(self): delta=1.0, msg=f"Expected kurtosis near 0, got {adequacy_metric.kurtosis['test_input']}", ) # This adds a numerical tolerance for Pandas - self.assertEqual(adequacy_metric.passing, 19, f"Expected passing 19 not {adequacy_metric.passing}") + self.assertEqual(adequacy_metric.passing, 91, f"Expected passing 91 not {adequacy_metric.passing}") self.assertEqual(adequacy_metric.successful, 100, f"Expected successful 100 not {adequacy_metric.successful}") def test_data_adequacy_categorical(self): @@ -75,7 +75,7 @@ def test_data_adequacy_categorical(self): delta=1.0, msg=f"Expected kurtosis near 0, got {adequacy_metric.kurtosis['test_input_no_dist[T.b]']}", ) - self.assertEqual(adequacy_metric.passing, 100, f"Expected passing 100 not {adequacy_metric.passing}") + self.assertEqual(adequacy_metric.passing, 86, f"Expected passing 86 not {adequacy_metric.passing}") self.assertEqual(adequacy_metric.successful, 100, f"Expected successful 100 not {adequacy_metric.successful}") def test_data_adequacy_group_by(self): @@ -108,11 +108,11 @@ def test_data_adequacy_group_by(self): self.assertEqual( round(adequacy_metric.kurtosis["trtrand"], 3), - -2.739, + -0.857, f"Expected kurtosis not {round(adequacy_metric.kurtosis['trtrand'], 3)}", ) - self.assertEqual(adequacy_metric.passing, 1, f"Expected passing 1 not {adequacy_metric.passing}") - self.assertEqual(adequacy_metric.successful, 5, f"Expected successful 5 not {adequacy_metric.successful}") + self.assertEqual(adequacy_metric.passing, 32, f"Expected passing 32 not {adequacy_metric.passing}") + self.assertEqual(adequacy_metric.successful, 100, f"Expected successful 100 not {adequacy_metric.successful}") def test_dag_adequacy_dependent(self): base_test_case = BaseTestCase( diff --git a/tests/testing_tests/test_causal_test_case.py b/tests/testing_tests/test_causal_test_case.py index d475ce79..17ce6244 100644 --- a/tests/testing_tests/test_causal_test_case.py +++ b/tests/testing_tests/test_causal_test_case.py @@ -9,47 +9,12 @@ from causal_testing.specification.variable import Input, Output from causal_testing.specification.causal_dag import CausalDAG from causal_testing.testing.causal_test_case import CausalTestCase -from causal_testing.testing.causal_effect import ExactValue +from causal_testing.testing.causal_effect import ExactValue, SomeEffect from causal_testing.estimation.linear_regression_estimator import LinearRegressionEstimator from causal_testing.testing.base_test_case import BaseTestCase class TestCausalTestCase(unittest.TestCase): - """Test the CausalTestCase class. - - The base test case is a data class which contains the minimum information - necessary to perform identification. The CausalTestCase class represents - a causal test case. We here test the basic getter methods. - """ - - def setUp(self) -> None: - # 2. Create Scenario and Causal Specification - A = Input("A", float) - C = Output("C", float) - - # 3. Create an intervention and causal test case - self.expected_causal_effect = ExactValue(4) - self.base_test_case = BaseTestCase(A, C) - self.causal_test_case = CausalTestCase( - base_test_case=self.base_test_case, - expected_causal_effect=self.expected_causal_effect, - estimator=LinearRegressionEstimator( - base_test_case=self.base_test_case, - adjustment_set=set(), - control_value=0, - treatment_value=1, - ), - ) - - def test_str(self): - print(str(self.causal_test_case)) - self.assertEqual( - str(self.causal_test_case), - "Running {'A': 1} instead of {'A': 0} should cause the following changes to {'C'}: ExactValue: 4±0.2.", - ) - - -class TestCausalTestExecution(unittest.TestCase): """ Test the causal test execution workflow using observational data. """ @@ -250,3 +215,49 @@ def test_estimate_params_with_formula(self): ), 1.444, ) + + def test_to_dict(self): + estimator = LinearRegressionEstimator( + base_test_case=self.base_test_case_A_C, + adjustment_set=set(), + formula="C ~ A + D", + ) + causal_test_case = CausalTestCase( + name="A |- C", + base_test_case=self.base_test_case_A_C, + expected_causal_effect=ExactValue(4), + estimate_type="coefficient", + estimator=estimator, + ) + causal_test_case.execute_test(self.df, adequacy=True) + + expected = { + "name": "A |- C", + "treatment_variable": "A", + "outcome_variable": "C", + "effect": "total", + "skip": False, + "estimate_type": "coefficient", + "query": None, + "expected_effect": {"name": "ExactValue", "value": 4, "atol": 0.2}, + "estimator": { + "name": "LinearRegressionEstimator", + "alpha": 0.05, + "adjustment_set": [], + "formula": "C ~ A + D", + }, + "result": { + "outcome": "PASS", + "passed": True, + "effect_measure": "coefficient", + "effect_estimate": {"A": 4.0}, + "ci_low": {"A": 4.0}, + "ci_high": {"A": 4.0}, + "adequacy": {"kurtosis": {"A": 0.0}, "passing": 100, "successful": 100}, + }, + } + + # Use json_normalize to avoid rounding errors + pd.testing.assert_frame_equal( + pd.json_normalize(expected).round(2), pd.json_normalize(causal_test_case.to_dict()).round(2) + ) diff --git a/tests/testing_tests/test_metamorphic_relations.py b/tests/testing_tests/test_metamorphic_relations.py index 1663e296..ee712422 100644 --- a/tests/testing_tests/test_metamorphic_relations.py +++ b/tests/testing_tests/test_metamorphic_relations.py @@ -16,6 +16,10 @@ from causal_testing.estimation.multinomial_regression_estimator import MultinomialRegressionEstimator +def sort_test_dict(test: dict): + return test["name"] + + class TestMetamorphicRelation(unittest.TestCase): def setUp(self) -> None: self.temp_dir_path = tempfile.mkdtemp() @@ -53,7 +57,10 @@ def test_all_metamorphic_relations_implied_by_dag(self): base_test_case=base_test_case, expected_causal_effect=SomeEffect(), estimate_type="coefficient", - estimator=LinearRegressionEstimator(base_test_case), + estimator=LinearRegressionEstimator( + base_test_case, + adjustment_set=dag.identification(base_test_case), + ), name=f"{treatment} -> {outcome}", skip=False, ) @@ -79,13 +86,19 @@ def test_all_metamorphic_relations_implied_by_dag(self): base_test_case=base_test_case, expected_causal_effect=NoEffect(), estimate_type="coefficient", - estimator=LinearRegressionEstimator(base_test_case), - name=f"{treatment} -> {outcome}", + estimator=LinearRegressionEstimator( + base_test_case, + adjustment_set=dag.identification(base_test_case), + ), + name=f"{treatment} _||_ {outcome}", skip=False, ) ) - self.assertEqual(sorted(map(str, expected_tests)), sorted(map(str, dag.generate_causal_tests()))) + self.assertEqual( + sorted(map(lambda t: t.to_dict(), expected_tests), key=sort_test_dict), + sorted(map(lambda t: t.to_dict(), dag.generate_causal_tests()), key=sort_test_dict), + ) def test_all_metamorphic_relations_implied_by_dag_parallel(self): dag = CausalDAG(self.dag_dot_path, datatypes={v: float for v in {"X1", "X2", "X3", "Y", "Z", "M"}}) @@ -99,11 +112,15 @@ def test_all_metamorphic_relations_implied_by_dag_parallel(self): base_test_case=base_test_case, expected_causal_effect=SomeEffect(), estimate_type="coefficient", - estimator=LinearRegressionEstimator(base_test_case), + estimator=LinearRegressionEstimator( + base_test_case, adjustment_set=dag.identification(base_test_case) + ), name=f"{treatment} -> {outcome}", skip=False, ) ) + # We can't just do "nx.non_edges" here, since some independences are bidirectional (if there is no path from + # X -> ... -> Y) and some are unidirectional (if X -> Y is not in the DAG but X -> ... -> Y is). for treatment, outcome in [ ("X1", "M"), ("X1", "Y"), @@ -125,13 +142,18 @@ def test_all_metamorphic_relations_implied_by_dag_parallel(self): base_test_case=base_test_case, expected_causal_effect=NoEffect(), estimate_type="coefficient", - estimator=LinearRegressionEstimator(base_test_case), - name=f"{treatment} -> {outcome}", + estimator=LinearRegressionEstimator( + base_test_case, adjustment_set=dag.identification(base_test_case) + ), + name=f"{treatment} _||_ {outcome}", skip=False, ) ) - self.assertEqual(sorted(map(str, expected_tests)), sorted(map(str, dag.generate_causal_tests(threads=2)))) + self.assertEqual( + sorted(map(lambda t: t.to_dict(), expected_tests), key=sort_test_dict), + sorted(map(lambda t: t.to_dict(), dag.generate_causal_tests(threads=2)), key=sort_test_dict), + ) def test_all_metamorphic_relations_implied_by_dag_ignore_cycles(self): dcg = CausalDAG(self.dcg_dot_path, ignore_cycles=True, datatypes={v: float for v in {"a", "b", "c", "d"}}) @@ -142,9 +164,12 @@ def test_all_metamorphic_relations_implied_by_dag_ignore_cycles(self): base_test_case=base_test_case, expected_causal_effect=SomeEffect(), estimate_type="coefficient", - estimator=LinearRegressionEstimator(base_test_case), - name=f"a -> b", + estimator=LinearRegressionEstimator(base_test_case, adjustment_set=dcg.identification(base_test_case)), + name="a -> b", skip=False, ) ] - self.assertEqual(sorted(map(str, expected_tests)), sorted(map(str, dcg.generate_causal_tests(threads=2)))) + self.assertEqual( + sorted(map(lambda t: t.to_dict(), expected_tests), key=sort_test_dict), + sorted(map(lambda t: t.to_dict(), dcg.generate_causal_tests(threads=2)), key=sort_test_dict), + ) From f8e7307e5ffd4b15776bb4379bf354ab8eb351fe Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Wed, 22 Jul 2026 07:59:43 +0100 Subject: [PATCH 6/7] Added small tolerance for rounding errors --- tests/testing_tests/test_causal_test_adequacy.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/testing_tests/test_causal_test_adequacy.py b/tests/testing_tests/test_causal_test_adequacy.py index 233a95f4..c35af7a7 100644 --- a/tests/testing_tests/test_causal_test_adequacy.py +++ b/tests/testing_tests/test_causal_test_adequacy.py @@ -41,7 +41,7 @@ def test_data_adequacy_numeric(self): ) causal_test_case = CausalTestCase( base_test_case=base_test_case, - expected_causal_effect=NoEffect(), + expected_causal_effect=NoEffect(atol=0.05), estimate_type="coefficient", estimator=estimator, ) @@ -53,7 +53,7 @@ def test_data_adequacy_numeric(self): delta=1.0, msg=f"Expected kurtosis near 0, got {adequacy_metric.kurtosis['test_input']}", ) # This adds a numerical tolerance for Pandas - self.assertEqual(adequacy_metric.passing, 91, f"Expected passing 91 not {adequacy_metric.passing}") + self.assertEqual(adequacy_metric.passing, 100, f"Expected passing 100 not {adequacy_metric.passing}") self.assertEqual(adequacy_metric.successful, 100, f"Expected successful 100 not {adequacy_metric.successful}") def test_data_adequacy_categorical(self): @@ -63,7 +63,7 @@ def test_data_adequacy_categorical(self): estimator = LinearRegressionEstimator(base_test_case=base_test_case, adjustment_set={}) causal_test_case = CausalTestCase( base_test_case=base_test_case, - expected_causal_effect=NoEffect(), + expected_causal_effect=NoEffect(atol=1e-10), estimate_type="coefficient", estimator=estimator, ) @@ -75,7 +75,7 @@ def test_data_adequacy_categorical(self): delta=1.0, msg=f"Expected kurtosis near 0, got {adequacy_metric.kurtosis['test_input_no_dist[T.b]']}", ) - self.assertEqual(adequacy_metric.passing, 86, f"Expected passing 86 not {adequacy_metric.passing}") + self.assertEqual(adequacy_metric.passing, 100, f"Expected passing 100 not {adequacy_metric.passing}") self.assertEqual(adequacy_metric.successful, 100, f"Expected successful 100 not {adequacy_metric.successful}") def test_data_adequacy_group_by(self): From 429cf7b9ac9b7d84031663c92ff2f4855b0e995b Mon Sep 17 00:00:00 2001 From: Michael Foster Date: Wed, 22 Jul 2026 08:12:50 +0100 Subject: [PATCH 7/7] Added index column to test data --- causal_testing/causal_testing_framework.py | 2 -- tests/main_tests/test_main.py | 1 - tests/resources/data/data.csv | 10 +++++----- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/causal_testing/causal_testing_framework.py b/causal_testing/causal_testing_framework.py index d88b3cdc..acf6069b 100644 --- a/causal_testing/causal_testing_framework.py +++ b/causal_testing/causal_testing_framework.py @@ -44,8 +44,6 @@ def read_dataframe(file_path: str, **kwargs: dict) -> pd.DataFrame: suffix = Path(file_path).suffix.lower() - print(suffix, type(suffix)) - if suffix in readers: return readers[suffix](file_path, **kwargs) raise ValueError(f"Unsupported file extension: '{suffix}'") diff --git a/tests/main_tests/test_main.py b/tests/main_tests/test_main.py index df7983a7..896eb8f1 100644 --- a/tests/main_tests/test_main.py +++ b/tests/main_tests/test_main.py @@ -138,7 +138,6 @@ def test_parse_args_discover(self): "HillClimberDiscovery", "--data-paths", str(self.data_paths[0]), - str(self.data_paths[0]), "--output", os.path.join(tmp, "discovered_dag.dot"), "--include-edges", diff --git a/tests/resources/data/data.csv b/tests/resources/data/data.csv index ec2d6002..4c1a145e 100644 --- a/tests/resources/data/data.csv +++ b/tests/resources/data/data.csv @@ -1,6 +1,6 @@ test_input,test_input_no_dist,test_output,B,C -1.0,1.1,2.2,0,0 -2.0,1.1,2.8,0,0 -3.0,1.0,1.0,0,0 -4.0,1.2,6.0,0,0 -5.0,0.9,2.5,0,0 +0,1.0,1.1,2.2,0,0 +1,2.0,1.1,2.8,0,0 +2,3.0,1.0,1.0,0,0 +3,4.0,1.2,6.0,0,0 +4,5.0,0.9,2.5,0,0