Skip to content

Commit 355f04d

Browse files
jmafoster1jmafoster1
authored
Merge pull request #395 from CITCOM-project/jmafoster1/388-fix-large-scenarios
Jmafoster1/388 fix large scenarios
2 parents 3b5b62a + 3e6fd99 commit 355f04d

40 files changed

Lines changed: 1362 additions & 1965 deletions

‎causal_testing/__main__.py‎

Lines changed: 166 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,163 @@
11
"""This module contains the main entrypoint functionality to the Causal Testing Framework."""
22

3-
import json
3+
import argparse
44
import logging
5-
import os
6-
import tempfile
5+
from enum import Enum
76
from importlib.metadata import entry_points
8-
from pathlib import Path
7+
from typing import Optional, Sequence
98

109
import networkx as nx
1110
import pandas as pd
1211

12+
from causal_testing.causal_testing_framework import CausalTestingFramework, read_dataframe
1313
from causal_testing.testing.metamorphic_relation import generate_causal_tests
1414

15-
from .main import CausalTestingFramework, CausalTestingPaths, Command, parse_args, setup_logging
15+
logger = logging.getLogger(__name__)
16+
17+
18+
class Command(Enum):
19+
"""
20+
Enum for supported CTF commands.
21+
"""
22+
23+
TEST = "test"
24+
GENERATE = "generate"
25+
DISCOVER = "discover"
26+
27+
28+
def setup_logging(level: str) -> None:
29+
"""Set up logging configuration."""
30+
logging.basicConfig(
31+
level=level, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S"
32+
)
33+
34+
35+
def parse_args(args: Optional[Sequence[str]] = None) -> argparse.Namespace:
36+
"""Parse command line arguments."""
37+
main_parser = argparse.ArgumentParser(
38+
add_help=True,
39+
description="Causal Testing Framework - "
40+
"A causal inference-driven framework for functional black-box testing of complex software.",
41+
)
42+
43+
main_parser.add_argument(
44+
"-l",
45+
"--log_level",
46+
default="WARNING",
47+
type=str.upper,
48+
choices=["NONE", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
49+
help="Set the logging level (default: WARNING).",
50+
)
51+
52+
subparsers = main_parser.add_subparsers(
53+
help="The action you want to run - call `causal_testing {action} -h` for further details", dest="command"
54+
)
55+
56+
# Generation
57+
parser_generate = subparsers.add_parser(Command.GENERATE.value, help="Generate causal tests from a DAG")
58+
parser_generate.add_argument("-D", "--dag-path", help="Path to the DAG file (.dot)", required=True)
59+
parser_generate.add_argument("-o", "--output", help="Path for output file (.json)", required=True)
60+
parser_generate.add_argument(
61+
"-e",
62+
"--estimator",
63+
help="The name of the estimator class to use when evaluating tests (defaults to LinearRegressionEstimator)",
64+
default="LinearRegressionEstimator",
65+
)
66+
parser_generate.add_argument(
67+
"-T",
68+
"--effect-type",
69+
help="The effect type to estimate {direct, total}",
70+
default="direct",
71+
)
72+
parser_generate.add_argument(
73+
"-E",
74+
"--estimate-type",
75+
help="The estimate type to use when evaluating tests (defaults to coefficient)",
76+
default="coefficient",
77+
)
78+
parser_generate.add_argument(
79+
"-i", "--ignore-cycles", help="Ignore cycles in DAG", action="store_true", default=False
80+
)
81+
parser_generate.add_argument(
82+
"--threads", "-t", type=int, help="The number of parallel threads to use.", required=False, default=0
83+
)
84+
85+
# Testing
86+
parser_test = subparsers.add_parser(Command.TEST.value, help="Run causal tests")
87+
parser_test.add_argument("-D", "--dag-path", help="Path to the DAG file (.dot)", required=True)
88+
parser_test.add_argument("-o", "--output", help="Path for output file (.json)", required=True)
89+
parser_test.add_argument("-i", "--ignore-cycles", help="Ignore cycles in DAG", action="store_true", default=False)
90+
parser_test.add_argument("-d", "--data-paths", help="Paths to data files (.csv)", nargs="+", required=True)
91+
parser_test.add_argument("-t", "--test-config", help="Path to test configuration file (.json)", required=True)
92+
parser_test.add_argument("-q", "--query", help="Query string to filter data (e.g. 'age > 18')", type=str)
93+
parser_test.add_argument(
94+
"-a", "--adequacy", help="Calculate causal test adequacy for each test case", action="store_true", default=False
95+
)
96+
parser_test.add_argument(
97+
"-b",
98+
"--adequacy-bootstrap-size",
99+
dest="bootstrap_size",
100+
help="Number of bootstrap samples for causal test adequacy. Defaults to 100",
101+
type=int,
102+
)
103+
parser_test.add_argument(
104+
"-s",
105+
"--silent",
106+
action="store_true",
107+
help="Do not crash on error. If set to true, errors are recorded as test results.",
108+
default=False,
109+
)
110+
111+
# Discovery
112+
parser_discover = subparsers.add_parser(Command.DISCOVER.value, help="Discover causal structures from data")
113+
parser_discover.add_argument("-d", "--data-paths", help="Paths to data files (.csv)", nargs="+", required=True)
114+
parser_discover.add_argument(
115+
"-a",
116+
"--alpha",
117+
help=(
118+
"The significance level of the confidence intervals used to determine causality. "
119+
"This should be a value between 0 and 1. Defaults to 0.05 for 95%% confidence intervals."
120+
),
121+
default=0.05,
122+
)
123+
parser_discover.add_argument(
124+
"-t",
125+
"--technique",
126+
help="The name of the technique to use. Currently supported are 'HillClimberDiscovery' and 'NSGADiscovery'",
127+
required=True,
128+
)
129+
parser_discover.add_argument(
130+
"-V",
131+
"--variables",
132+
help="The subset of variables from the data to consider. Defaults to all.",
133+
nargs="*",
134+
default=[],
135+
)
136+
parser_discover.add_argument("-o", "--output", help="Path for output DAG file (.dot)", required=True)
137+
parser_discover.add_argument(
138+
"-i", "--include-edges", help="Path to file containing edges to include", required=False
139+
)
140+
parser_discover.add_argument(
141+
"-e", "--exclude-edges", help="Path to file containing edges to exclude", required=False
142+
)
143+
parser_discover.add_argument(
144+
"--technique-kwargs",
145+
help="Keywords for the discovery technique. These should be specified as `arg1=value1 arg2=value2...`.",
146+
nargs="*",
147+
default=[],
148+
)
149+
150+
args = main_parser.parse_args(args)
151+
152+
# Assume the user wants test adequacy if they're setting bootstrap_size
153+
if getattr(args, "bootstrap_size", None) is not None:
154+
args.adequacy = True
155+
if getattr(args, "adequacy", False) and getattr(args, "bootstrap_size", None) is None:
156+
# Need this here rather than a default value because otherwise the above always sets adequacy to True
157+
args.bootstrap_size = 100
158+
159+
args.command = Command(args.command)
160+
return args
16161

17162

18163
def main() -> None:
@@ -60,9 +205,14 @@ def main() -> None:
60205
logging.info("Discovering causal structure")
61206
# Need to reset index to allow for multiple files having the same index (i.e. starting at zero).
62207
# Otherwise you end up with duplicate indices, which causes problems further down the line
63-
df = pd.concat([pd.read_csv(path) for path in args.data_paths]).reset_index()
208+
df = pd.concat([read_dataframe(path) for path in args.data_paths]).reset_index()
64209
if args.variables:
65210
df = df[args.variables]
211+
# Drop unnamed columns
212+
unnamed_columns = [c for c in df.columns if c.startswith("Unnamed: ")]
213+
if unnamed_columns:
214+
logger.warning(f"Dropping unnamed columns: {unnamed_columns}")
215+
df = df.drop(unnamed_columns)
66216

67217
discover_class = discover_map[args.technique].load()
68218
discover = discover_class(
@@ -80,55 +230,20 @@ def main() -> None:
80230
discover.write_dot(evolved_dag, args.output)
81231
logging.info("Causal structure discovery completed successfully.")
82232
case Command.TEST:
83-
# Create paths object
84-
paths = CausalTestingPaths(
233+
# Create and setup framework
234+
framework = CausalTestingFramework()
235+
236+
framework.setup(
85237
dag_path=args.dag_path,
86238
data_paths=args.data_paths,
87-
test_config_path=args.test_config,
88-
output_path=args.output,
239+
test_cases_path=args.test_config,
240+
query=args.query,
241+
ignore_cycles=args.ignore_cycles,
89242
)
90243

91-
# Create and setup framework
92-
framework = CausalTestingFramework(paths, ignore_cycles=args.ignore_cycles, query=args.query)
93-
framework.setup()
94-
95-
# Load and run tests
96-
framework.load_tests()
97-
98-
if args.batch_size > 0:
99-
logging.info(f"Running tests in batches of size {args.batch_size}")
100-
with tempfile.TemporaryDirectory() as tmpdir:
101-
output_files = []
102-
for i, results in enumerate(
103-
framework.run_tests_in_batches(
104-
batch_size=args.batch_size,
105-
silent=args.silent,
106-
adequacy=args.adequacy,
107-
bootstrap_size=args.bootstrap_size,
108-
)
109-
):
110-
temp_file_path = os.path.join(tmpdir, f"output_{i}.json")
111-
framework.save_results(results, temp_file_path)
112-
output_files.append(temp_file_path)
113-
del results
114-
115-
# Now stitch the results together from the temporary files
116-
all_results = []
117-
for file_path in output_files:
118-
with open(file_path, "r", encoding="utf-8") as f:
119-
all_results.extend(json.load(f))
120-
121-
output_path = Path(args.output)
122-
output_path.parent.mkdir(parents=True, exist_ok=True)
123-
124-
with open(args.output, "w", encoding="utf-8") as f:
125-
json.dump(all_results, f, indent=4)
126-
else:
127-
logging.info("Running tests in regular mode")
128-
results = framework.run_tests(
129-
silent=args.silent, adequacy=args.adequacy, bootstrap_size=args.bootstrap_size
130-
)
131-
framework.save_results(results)
244+
logging.info("Running tests")
245+
framework.run_tests(silent=args.silent, adequacy=args.adequacy, bootstrap_size=args.bootstrap_size)
246+
framework.save_results(args.output)
132247

133248
logging.info("Causal testing completed successfully.")
134249

0 commit comments

Comments
 (0)