diff --git a/.github/workflows/ci-tests-drafts.yaml b/.github/workflows/ci-tests-drafts.yaml index 7f1dfe80..339f907f 100644 --- a/.github/workflows/ci-tests-drafts.yaml +++ b/.github/workflows/ci-tests-drafts.yaml @@ -25,7 +25,7 @@ jobs: python --version python -m pip install --upgrade pip pip install -e . - pip install -e .[test] + pip install -e .[dev] pip install pytest pytest-cov - name: Register Jupyter Kernel run: | @@ -38,3 +38,7 @@ jobs: with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} + - name: "Build docs" + working-directory: "docs" + run: | + make html diff --git a/.github/workflows/ci-tests.yaml b/.github/workflows/ci-tests.yaml index d74f335f..1f2b16a9 100644 --- a/.github/workflows/ci-tests.yaml +++ b/.github/workflows/ci-tests.yaml @@ -30,7 +30,7 @@ jobs: python --version python -m pip install --upgrade pip pip install -e . - pip install -e .[test] + pip install -e .[dev] pip install pytest pytest-cov - name: Register Jupyter Kernel run: | @@ -43,3 +43,7 @@ jobs: with: fail_ci_if_error: true token: ${{ secrets.CODECOV_TOKEN }} + - name: "Build docs" + working-directory: "docs" + run: | + make html diff --git a/causal_testing/discovery/nsga_discovery.py b/causal_testing/discovery/nsga_discovery.py index 3cb99222..646d20fb 100644 --- a/causal_testing/discovery/nsga_discovery.py +++ b/causal_testing/discovery/nsga_discovery.py @@ -39,7 +39,7 @@ def binary_string_to_causal_dag(self, individual: np.array) -> CausalDAG: Converts a binary string representation of a causal DAG to a CausalDAG object. :param individual: Bitstring of the same length as `possible_edges` such that 1 at position `i` represents - possible_edges[i] being an edge in the graph and 0 represents it not being. + possible_edges[i] being an edge in the graph and 0 represents it not being. :returns: The converted CausalDAG instance. """ causal_dag = CausalDAG() diff --git a/causal_testing/estimation/abstract_regression_estimator.py b/causal_testing/estimation/abstract_regression_estimator.py index cee528b0..fbc58759 100644 --- a/causal_testing/estimation/abstract_regression_estimator.py +++ b/causal_testing/estimation/abstract_regression_estimator.py @@ -64,6 +64,7 @@ def __init__( def _get_adjusted_variables(self, tree: ast.AST) -> set[str]: """ Recursively return variables in an AST. + :returns: Set of all variables not used as part of a function. """ if isinstance(tree, ast.Name) and tree.id != self.treatment_variable: @@ -108,7 +109,9 @@ def _setup_covariates(self, df: pd.DataFrame) -> pd.Series: Parse the formula and set up the covariates from the design matrix so we can use them in the statsmodels array API. This allows us to only parse the formula once, rather than using the formula API, which parses it every time the regression model is fit, which can be a lot if using causal test adequacy. + :param df: The data to use. + :returns: The data and the covariate columns. """ _, covariate_data = dmatrices(self.formula, df, return_type="dataframe") @@ -154,9 +157,10 @@ def treatment_columns(self, model: RegressionResultsWrapper) -> list[str]: This is a workaround for statsmodels mangling the names of categorical variables to include the values. :param model: The fitted model from which to extract the variable names. + :returns: A list of the feature names in the model that represent the treatment. Normally this will just be - [treatment_name], but for categorical treatments, you'll have - [treatment_name[value_1], treatment_name[value_2]]. + [treatment_name], but for categorical treatments, you'll have + [treatment_name[value_1], treatment_name[value_2]]. """ return [ param @@ -170,7 +174,7 @@ def _predict(self, df) -> pd.DataFrame: :param df: The data to use. :param: adjustment_config: The values of the adjustment variables to use. - :return: The estimated outcome under control and treatment, with confidence intervals in the form of a + :returns: The estimated outcome under control and treatment, with confidence intervals in the form of a dataframe with columns "predicted", "se", "ci_lower", and "ci_upper". """ model = self.fit_model(df) diff --git a/causal_testing/testing/causal_test_case.py b/causal_testing/testing/causal_test_case.py index fa6e4c90..ec1de7b8 100644 --- a/causal_testing/testing/causal_test_case.py +++ b/causal_testing/testing/causal_test_case.py @@ -20,6 +20,7 @@ class CausalTestCase: variables, a CausalTestCase stores the values of these variables. Also the outcome variable and value are specified. The goal of a CausalTestCase is to test whether the intervention made to the control via the treatment causes the model-under-test to produce the expected change. + :param base_test_case: A BaseTestCase object consisting of a treatment variable, outcome variable and effect :param expected_causal_effect: The expected causal effect (Positive, Negative, No Effect). :param effect_measure: A string which denotes the type of estimate to return. @@ -70,10 +71,11 @@ def measure_adequacy( ) -> DataAdequacy: """ Calculate the adequacy measurement, and populate the data_adequacy field. + :param df: The original dataset to use. :param bootstrap_size: The number of bootstrap samples to use. (Defaults to 100) :param group_by: For IPCWEstimator - the "id" column to ensure that entire individuals are sampled rather than - random rows. + random rows. """ results = [] outcomes = [] @@ -130,8 +132,7 @@ def execute_test( :param suppress_estimation_errors: Set to True to suppress estimation errors. (Defaults to False) :param bootstrap_size: The number of bootstrap samples to use. (Defaults to 100) :param group_by: For IPCWEstimator - the "id" column to ensure that entire individuals are sampled rather than - random rows. - :return causal_test_result: A CausalTestResult for the executed causal test case. + random rows. """ if not self.skip: try: @@ -161,7 +162,8 @@ def estimate_effect(self, df: pd.DataFrame) -> CausalTestResult: Execute a causal test case and return the causal test result. :param df: The data to use. - :return causal_test_result: A CausalTestResult for the executed causal test case. + + :returns: A CausalTestResult for the executed causal test case. """ if self.query: df = df.query(self.query) diff --git a/images/.gitignore b/docs/source/_static/images/.gitignore similarity index 100% rename from images/.gitignore rename to docs/source/_static/images/.gitignore diff --git a/docs/source/_static/images/schematic-dark.png b/docs/source/_static/images/schematic-dark.png new file mode 100644 index 00000000..ee1f1065 Binary files /dev/null and b/docs/source/_static/images/schematic-dark.png differ diff --git a/docs/source/_static/images/schematic.png b/docs/source/_static/images/schematic.png new file mode 100644 index 00000000..01110fde Binary files /dev/null and b/docs/source/_static/images/schematic.png differ diff --git a/docs/source/_static/images/schematic.tex b/docs/source/_static/images/schematic.tex new file mode 100644 index 00000000..78808899 --- /dev/null +++ b/docs/source/_static/images/schematic.tex @@ -0,0 +1,127 @@ +\documentclass{standalone} + +\usepackage{tikz} +\usetikzlibrary{arrows,positioning,shapes,calc,fit,overlay-beamer-styles, backgrounds} +\usepackage{dsfont,pifont} +\newcommand*{\expe}{\mathds{E}} +\usepackage{amsmath} +\usepackage{booktabs} +\usepackage{fontawesome7} + +\usepackage[default]{FiraSans} +\usepackage[mathrm=sym]{unicode-math} +\setmathfont{Fira Math} + +\newcommand{\indep}{\perp \!\!\! \perp} + +\begin{document} +\tikzset{ + node/.style={circle, draw, minimum size=3ex, inner sep=0.2}, + edge/.style={->,> = latex'}, +} + +\newcommand{\cmark}{\ding{51}}% +\newcommand{\xmark}{\ding{55}}% + +\begin{tikzpicture}[background rectangle/.style={fill=none}, show background rectangle, color=black] + + % Test Case + \begin{scope}[name prefix=test-, local bounding box=test-case] + \node[draw=none, rectangle, anchor=north] (title) at (0, 0) {Causal Test Cases}; + \node[anchor=north,align=center] (tuple) at (title.south) {$I \to_{?} Y_3$\hspace{5mm}$X_2 \indep_? X_2$}; + \node[draw, rectangle] [fit=(title) (tuple)] {}; + \end{scope} + + % ci + \begin{scope}[name prefix=ci-, local bounding box=ci, shift={($(test-test-case.east) + (1, 0)$)}] + \node[draw=none, rectangle, anchor=south west] (title) {Causal Inference}; + \node[draw=none, rectangle, anchor=north, align=center] (brain) at (title.south) {\faIcon{hexagon-nodes-bolt}}; + + \coordinate (top) at ({(0, 0)} |- test-title.north); + \coordinate (bot) at ({(0, 0)} |- test-tuple.south); + + \node[draw, rectangle] [fit=(title) (brain) (top) (bot)] {}; + \end{scope} + + % Estimate + \begin{scope}[name prefix=estimate-, local bounding box=estimate, shift={($(ci-ci.east)+(1, 0)$)}] + \node[draw=none, rectangle, anchor=south west] (title) {Causal Estimate}; + \node[anchor=north] (table) at (title.south) {\faIcon{chart-line}}; + \coordinate (top) at ({(0, 0)} |- test-title.north); + \coordinate (bot) at ({(0, 0)} |- test-tuple.south); + \node[draw, rectangle] [fit=(title) (table) (top) (bot)] {}; + \end{scope} + + % Oracle + \begin{scope}[name prefix=oracle-, local bounding box=test-oracle, shift={($(estimate-estimate.east) + (1, 0)$)}] + \node[draw=none, rectangle, anchor=south west] (title) {Test Oracle}; + \node[draw=none, rectangle, anchor=north] (scale) at (title.south) {\faIcon{scale-balanced}}; + + \coordinate (top) at ({(0, 0)} |- test-title.north); + \coordinate (bot) at ({(0, 0)} |- test-tuple.south); + \node[draw, rectangle] [fit=(title) (scale) (top) (bot)] {}; + \end{scope} + + % Outcome + \begin{scope}[name prefix=outcome-, local bounding box=test-outcome, shift={($(oracle-test-oracle.east) + (1, 0)$)}] + \node[draw=none, rectangle, anchor=south west] (title) at (0,0) {Test Outcomes}; + \node[draw=none, anchor=north] (ok) at (title.south) {\cmark ~ \xmark}; + + \coordinate (top) at ({(0, 0)} |- test-title.north); + \coordinate (bot) at ({(0, 0)} |- test-tuple.south); + \node[draw, rectangle] (test-outcome) [fit=(outcome-title) (outcome-ok) (top) (bot)] {}; + \end{scope} + + + % Causal DAG + \begin{scope}[name prefix=dag-, shift={(0, 2)}] + \node[node] (x1) at (-1, 0) {$X_1$}; + \node[node] (x2) at (-1, 1.4) {$X_2$}; + \node[node] (i) at (0, 0.7) {$I$}; + \node[node] (y1) at (1,0) {$Y_{1}$}; + \node[node] (y2) at (1,0.7) {$Y_2$}; + \node[node] (y3) at (1,1.4) {$Y_3$}; + + \draw[edge] (x1) to (i); + \draw[edge] (x2) to (i); + \draw[edge] (i) to (y1); + \draw[edge] (i) to (y2); + \draw[edge] (i) to (y3); + \draw[edge] (x1) to (y1); + \draw[edge] (x2) to (y3); + \node[draw=none, rectangle] (nodes) [fit=(x1) (x2) (y1) (y2) (y3) (i)] {}; + \node[draw=none, rectangle, anchor=south] (title) at (nodes.north) {Causal DAG}; + \end{scope} + % DAG outline + \node[draw, rectangle] (dag) [fit=(dag-nodes) (dag-title) (dag-title)] {}; + + % Data + \begin{scope}[name prefix=data-, local bounding box=test-data, shift={($(dag) + (5, 1.12)$)}] + \node[draw=none, rectangle] (title) {Test Data}; + \node[anchor=north] (table) at (title.south) { + \begin{tabular}{rrrrrr} + \toprule + $X_1$ & $X_2$ & $I$ & $Y_1$ & $Y_2$ & $Y_3$ \\ + \midrule + 1.2 & ``UK'' & 0.3 & 7.8 & 4 & True \\ + 3.2 & ``UK'' & 0.1 & 7.6 & 8 & False \\ + \multicolumn{6}{c}{$\vdots$} \\ + \bottomrule + \end{tabular} + }; + \node[draw, rectangle] [fit=(title) (table)] {}; + \end{scope} + + + %Information flow + \draw[edge, dashed] (dag.290) -- (ci-ci.160); + \draw[edge, dashed] (dag) -- (test-test-case.north); + \draw[edge, dashed] (test-test-case) -- (ci-ci); + + \draw[edge, dashed] (data-test-data.south) -- (data-test-data |- ci-ci.north); + \draw[edge, dashed] (ci-ci) -- (estimate-estimate); + + \draw[edge, dashed] (estimate-estimate) -- (oracle-test-oracle.west |- estimate-estimate); + \draw[edge, dashed] (oracle-test-oracle.east |- outcome-test-outcome) -- (outcome-test-outcome); +\end{tikzpicture} +\end{document} diff --git a/docs/source/index.rst b/docs/source/index.rst index e0536a98..7283fe13 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -7,7 +7,10 @@ Welcome to the Causal Testing Framework Motivation ---------- -A common problem in computer science is to develop robust and reliable software systems that can perform correctly under various input configurations and maintain consistency across complex, physical scenarios. However, software systems, and more specifically computational models, can be difficult to test: they may contain hundreds of parameters, making testing all possible inputs computationally infeasible; some models may be inherently non-deterministic, producing different outputs for the same inputs due to randomness; or there may exist hidden causal relationships between input-output pairs, causing errors that only appear under specific combinations of input configurations. +From predicting the weather to simulating disease transmission, scientific software plays an increasingly pivotal role in developing scientific understanding that informs our everyday lives. +However, they are also some of the most difficult software systems to properly test. +They have large, complex input spaces, are computationally expensive to run, often rely on stochastic black-box components, and are applied in exploratory contexts where the expected outcomes are not known. +From a practical standpoint, the time and effort that can be dedicated to testing is often limited, especially in an academic context, making it especially important to maximise the efficiency of the limited number of test runs we are able to perform. The Framework ------------- diff --git a/docs/source/modules/estimators.rst b/docs/source/modules/estimators.rst index 20c1a26e..cf286e9b 100644 --- a/docs/source/modules/estimators.rst +++ b/docs/source/modules/estimators.rst @@ -31,7 +31,7 @@ LogisticRegressionEstimator :noindex: MultinomialRegressionEstimator -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ **Recommended use:** For categorical outcomes (e.g. colurs: Red, Green, Blue). @@ -42,19 +42,6 @@ MultinomialRegressionEstimator :show-inheritance: :noindex: -CubicSplineRegressionEstimator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -**Recommended use:** For continuous outcomes with non-linear relationships or changes in behaviour. -Useful when the relationship between treatment and outcome cannot be captured by a linear model. - -.. autoclass:: causal_testing.estimation.cubic_spline_estimator.CubicSplineRegressionEstimator - :members: - :undoc-members: - :show-inheritance: - :noindex: - - InstrumentalVariableEstimator ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/images/schematic-dark.png b/images/schematic-dark.png deleted file mode 100644 index 1be68423..00000000 Binary files a/images/schematic-dark.png and /dev/null differ diff --git a/images/schematic.png b/images/schematic.png deleted file mode 100644 index c1445da4..00000000 Binary files a/images/schematic.png and /dev/null differ diff --git a/images/schematic.tex b/images/schematic.tex deleted file mode 100644 index 7ad9b7c8..00000000 --- a/images/schematic.tex +++ /dev/null @@ -1,154 +0,0 @@ -\documentclass{standalone} - -\usepackage{tikz} -\usetikzlibrary{arrows,positioning,shapes,calc,fit,overlay-beamer-styles, backgrounds} -\usepackage{dsfont,pifont} -\newcommand*{\expe}{\mathds{E}} -\usepackage{amsmath} -\usepackage{booktabs} - -\usepackage[default]{FiraSans} -\usepackage[mathrm=sym]{unicode-math} -\setmathfont{Fira Math} - -\begin{document} -\tikzset{ - node/.style={circle, draw, minimum size=3ex, inner sep=0.2}, - edge/.style={->,> = latex'}, -} - -\newcommand{\cmark}{\ding{51}}% -\newcommand{\xmark}{\ding{55}}% - -\begin{tikzpicture}[background rectangle/.style={fill=none}, show background rectangle, color=black] - - % Test Case - \begin{scope}[name prefix=test-, local bounding box=test-case] - \node[draw=none, rectangle, anchor=north] (title) at (0, 0) {Causal Test Case}; - \node[anchor=north] (tuple) at (title.south) {$(X=i, \Delta=\text{increase}, Y=y_1)$}; - \node[draw, rectangle] [fit=(title) (tuple)] {}; - \end{scope} - - % Estimand - \begin{scope}[name prefix=estimand-, local bounding box=estimand, anchor=south, shift={($(test-test-case.east |- test-tuple.south) + (1, 0)$)}] - \node[anchor=south west] (eqn) at (0,0) { - $\Delta Y=\expe{[I=0 | X_1]} - \expe{[I=1 | X_1]} $ - }; - \node[draw=none, rectangle, anchor=south] (title) at (eqn.north) {Statistical Estimand}; - \node[draw, rectangle] [fit=(estimand-title) (estimand-eqn)] {}; - \end{scope} - - % Estimate - \begin{scope}[name prefix=estimate-, local bounding box=estimate, shift={($(estimand-estimand.east)+(1, 0)$)}] - \node[draw=none, rectangle, anchor=south west] (title) at (0, 0) {Causal Estimate}; - \node[anchor=north] (table) at (title.south) { - $\Delta Y=5$ - }; - \coordinate (top) at ({(0, 0)} |- test-title.north); - \coordinate (bot) at ({(0, 0)} |- estimand-eqn.south); - \node[draw, rectangle] [fit=(title) (table) (top) (bot)] {}; - \end{scope} - - % Oracle - \begin{scope}[name prefix=oracle-, local bounding box=test-oracle, shift={($(estimate-estimate.east) + (1.54, -0.4)$)}] - \begin{scope}[shift={(0,0)}, local bounding box=brain, scale=1.2] - \begin{scope}[shift={(-7.6932,3.5256)}, local bounding box=brain] - \path[draw,line width=0.025cm] (8.162, -2.8955) circle (0.066cm); - \path[draw,line width=0.025cm] (8.0485, -3.2243) circle (0.066cm); - \path[draw,line width=0.025cm] (8.0346, -3.5296) circle (0.066cm); - \path[draw,line width=0.025cm] (8.2166, -3.757) circle (0.066cm); - \path[draw,line width=0.025cm] (7.6556, -3.7827) circle (0.066cm); - \path[draw,line width=0.025cm] (7.6315, -3.5091) circle (0.066cm); - \path[draw,line width=0.025cm] (7.4451, -3.2224) circle (0.066cm); - \path[draw,line width=0.025cm] (7.6247, -2.9461) circle (0.066cm); - - \path[draw,line width=0.025cm,miter limit=4.0] (7.6932, -2.6331) -- (7.3637, -2.8234) -- (7.3637, -3.0567) -- (7.1749, -3.1656) -- (7.1749, -3.5256) -- (7.3341, -3.6175) -- (7.3341, -3.8517) -- (7.6883, -4.0562) -- (7.868, -3.9669) -- (8.0478, -4.0562) -- (8.4019, -3.8517) -- (8.4019, -3.6175) -- (8.5611, -3.5256) -- (8.5611, -3.1656) -- (8.3724, -3.0567) -- (8.3724, -2.8234) -- (8.0429, -2.6331) -- (7.868, -2.7341) -- cycle; - \path[draw,line width=0.025cm,miter limit=4.0] (7.868, -3.9669) -- (7.868, -2.7341); - \path[draw,line width=0.025cm] (7.5588, -2.9461) -- (7.3637, -2.9461); - \path[draw,line width=0.025cm] (7.4451, -3.1565) -- (7.4451, -2.9461); - \path[draw,line width=0.025cm] (7.6316, -3.4431) -- (7.6316, -3.2116) -- (7.868, -3.2116); - \path[draw,line width=0.025cm] (7.5897, -3.7827) -- (7.4177, -3.7827) -- (7.4177, -3.523) -- (7.1749, -3.523); - \path[draw,line width=0.025cm] (8.162, -2.9614) -- (8.162, -3.0534) -- (7.868, -3.0534); - \path[draw,line width=0.025cm] (8.0485, -3.1584) -- (8.0485, -3.0534); - \path[draw,line width=0.025cm] (8.1005, -3.5296) -- (8.313, -3.5296) -- (8.313, -3.3442) -- (8.5611, -3.3442); - \path[draw,line width=0.025cm] (8.1507, -3.757) -- (8.0477, -3.757) -- (8.0477, -4.0561); - \end{scope} - \end{scope} - \node[draw=none, rectangle, anchor=south] (title) at (brain.north) {Test Oracle}; - - \node[draw, rectangle] [fit=(title) (brain)] {}; - \end{scope} - - % Outcome - \begin{scope}[name prefix=outcome-, local bounding box=test-outcome, shift={($(oracle-brain.east |- estimate-estimate.east) + (1, 0)$)}] - \node[draw=none, rectangle, anchor=south west] (title) at (0,0) {Test Outcomes}; - \node[draw=none, anchor=north] (ok) at (title.south) {\cmark ~ \xmark}; - - \coordinate (top) at ({(0, 0)} |- test-title.north); - \coordinate (bot) at ({(0, 0)} |- estimand-eqn.south); - \node[draw, rectangle] (test-outcome) [fit=(outcome-title) (outcome-ok) (top) (bot)] {}; - \end{scope} - - - % Causal DAG - \begin{scope}[name prefix=dag-, shift={($(estimand-estimand.north) + (0, 2)$)}] - \node[node] (x1) at (-1, 0) {$X_1$}; - \node[node] (x2) at (-1, 1.4) {$X_2$}; - \node[node] (i) at (0, 0.7) {$I$}; - \node[node] (y1) at (1,0) {$Y_{1}$}; - \node[node] (y2) at (1,0.7) {$Y_2$}; - \node[node] (y3) at (1,1.4) {$Y_3$}; - - \draw[edge] (x1) to (i); - \draw[edge] (x2) to (i); - \draw[edge] (i) to (y1); - \draw[edge] (i) to (y2); - \draw[edge] (i) to (y3); - \draw[edge] (x1) to (y1); - \draw[edge] (x2) to (y3); - \node[draw=none, rectangle] (nodes) [fit=(x1) (x2) (y1) (y2) (y3) (i)] {}; - \node[draw=none, rectangle, anchor=south] (title) at (nodes.north) {Causal DAG}; - \end{scope} - - % Scenario - \begin{scope}[name prefix=scenario-, shift={($(estimand-estimand.south) + (0, -2)$)}] - \node[draw=none, rectangle] (title) at (0, 0) {Modelling Scenario}; - \node[anchor=north] (constraints) at (title.south) {$\{ x_1 < 5, x_2 = \text{``UK''} \}$}; - \end{scope} - \node[draw, rectangle] (scenario) [fit=(scenario-title) (scenario-constraints)] {}; - - % Data - \begin{scope}[name prefix=data-, local bounding box=test-data] - \node[draw=none, rectangle] (title) at (estimate-estimate |- dag-title) {Test Data}; - \node[anchor=north] (table) at (title.south) { - \begin{tabular}{rrrrrr} - \toprule - $X_1$ & $X_2$ & $I$ & $Y_1$ & $Y_2$ & $Y_3$ \\ - \midrule - 1.2 & ``UK'' & 0.3 & 7.8 & 4 & 100 \\ - 3.2 & ``UK'' & 0.1 & 7.6 & 8 & 95 \\ - \multicolumn{6}{c}{$\vdots$} \\ - \bottomrule - \end{tabular} - }; - \node[draw, rectangle] [fit=(title) (table)] {}; - \end{scope} - - % DAG outline - \node[draw, rectangle] (dag) [fit=(dag-nodes) (dag-title) (dag-title |- data-table.south)] {}; - - %Information flow - \draw[edge, dashed] (dag) -- (estimand-estimand.north); - \draw[edge, dashed] (test-test-case) -- (estimand-estimand); - - \draw[edge, dashed] (scenario.north) -- (estimand-estimand); - \draw[edge, dashed] (scenario.north) -- ([yshift=7.3mm]scenario.north) -- ([yshift=7.3mm]scenario.north -| test-test-case) -- (test-test-case); - \draw[edge, dashed] (scenario.north) -- ([yshift=7.3mm]scenario.north) -- ([yshift=7.3mm]scenario.north -| estimate-estimate) -- (estimate-estimate); - - \draw[edge, dashed] (data-test-data.south) -- (estimate-estimate.north); - \draw[edge, dashed] (estimand-estimand) -- (estimate-estimate); - - \draw[edge, dashed] (estimate-estimate) -- (oracle-test-oracle.west |- estimate-estimate); - \draw[edge, dashed] (oracle-test-oracle.east |- outcome-test-outcome) -- (outcome-test-outcome); -\end{tikzpicture} -\end{document} diff --git a/paper/paper.md b/paper/paper.md index 7781ec4f..c78bdfeb 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -69,7 +69,7 @@ The user may also refine tests to validate the nature of a particular relationsh Next, the user supplies a set of runtime data in the form of a table with each column representing a variable and rows containing the value of each variable for a particular run of the software. Finally, the CTF automatically validates the causal properties by using the causal DAG to identify a statistical estimand [@pearl2009causality] (essentially a set of features in the data which must be controlled for), calculate a causal effect estimate from the supplied data, and validating this against the expected causal relationship. -![Causal Testing workflow.\label{fig:schematic}](../images/schematic.png) +![Causal Testing workflow.\label{fig:schematic}](../docs/_static/images/schematic.png) ## Test Adequacy Because the properties being tested are completely separate from the data used to validate them, traditional coverage-based metrics are not appropriate here. diff --git a/pyproject.toml b/pyproject.toml index f773ba49..c33ced14 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,8 +51,6 @@ dev = [ "pandoc", "pre-commit", "tox", -] -test = [ "nbclient", "nbformat", "ipykernel",