Add files via upload

Author: AmirhosseinHonardoust

app.py

@@ -24,7 +24,11 @@
     format_percent,
     summarize_scored_transactions,
 )
-from src.reason_codes import shap_reason_codes, split_reason_codes  # noqa: E402
+from src.reason_codes import (  # noqa: E402
+    positive_class_shap_values,
+    shap_reason_codes,
+    split_reason_codes,
+)
 from src.score_new_transactions import score_dataframe  # noqa: E402
 from src.validation import (  # noqa: E402
     REQUIRED_FEATURE_COLUMNS,
@@ -154,7 +158,7 @@ def explain_single_transaction(model, df_scored: pd.DataFrame, row_idx: int):
 
     shap_vals = explainer.shap_values(x_for_shap)
 
-    shap_for_fraud_class = shap_vals[1][0] if isinstance(shap_vals, list) else shap_vals[0]
+    shap_for_fraud_class = positive_class_shap_values(shap_vals)[0]
 
     fig = plot_single_shap_bar(shap_for_fraud_class, feature_names)
     reasons = shap_reason_codes(shap_for_fraud_class, feature_names, max_reasons=5)

requirements.txt

@@ -1,8 +1,8 @@
-pandas
-numpy
-scikit-learn
-scipy
-matplotlib
-joblib
-shap
-streamlit
+pandas>=2.1,<4
+numpy>=1.24,<3
+scikit-learn>=1.3,<2
+scipy>=1.10,<2
+matplotlib>=3.7,<4
+joblib>=1.3,<2
+shap>=0.44,<1
+streamlit>=1.30,<2

src/explain.py

@@ -5,6 +5,7 @@
 import shap
 
 from .config import FIGURES_DIR, MODELS_DIR, PROCESSED_DATA_DIR, TARGET_COL
+from .reason_codes import positive_class_shap_values
 
 
 def load_model_and_test():
@@ -46,13 +47,13 @@ def compute_and_plot_global_shap(
     X_for_shap = X_transformed.toarray() if sp.issparse(X_transformed) else X_transformed
 
     explainer = shap.TreeExplainer(clf)
-    shap_values = explainer.shap_values(X_for_shap)
+    shap_values = positive_class_shap_values(explainer.shap_values(X_for_shap))
 
     # SHAP expects names for features; use transformed feature names
     feature_names = preprocessor.get_feature_names_out()
 
     shap.summary_plot(
-        shap_values[1],  # class 1 (fraud)
+        shap_values,  # class 1 (fraud)
         X_for_shap,
         feature_names=feature_names,
         show=False,

src/reason_codes.py

@@ -156,6 +156,28 @@ def humanize_feature_name(feature_name: str) -> str:
     return FRIENDLY_FEATURE_NAMES.get(raw, raw.replace("_", " "))
 
 
+def positive_class_shap_values(shap_values: object) -> np.ndarray:
+    """Return SHAP values for the positive (fraud) class, across shap versions.
+
+    ``TreeExplainer.shap_values`` has returned different shapes over time for
+    binary classifiers:
+
+    - legacy shap: a list ``[class_0_array, class_1_array]``;
+    - modern shap (>= 0.43): a single array shaped
+      ``(n_samples, n_features, n_classes)``.
+
+    This normalizes both (and an already-2-D array) to the class-1 values so the
+    rest of the code can stay version-agnostic.
+    """
+    if isinstance(shap_values, list):
+        return np.asarray(shap_values[1])
+
+    arr = np.asarray(shap_values)
+    if arr.ndim == 3:
+        return arr[..., 1]
+    return arr
+
+
 def shap_reason_codes(
     shap_values: Iterable[float],
     feature_names: Iterable[str],

tests/test_explainability.py

@@ -0,0 +1,99 @@
+from __future__ import annotations
+
+import unittest
+
+import numpy as np
+
+from src.config import TARGET_COL
+from src.generate_synthetic_data import generate_synthetic_fraud_dataset
+from src.reason_codes import positive_class_shap_values
+
+try:
+    import shap  # noqa: F401
+
+    HAS_SHAP = True
+except Exception:  # pragma: no cover - shap is an optional heavy dependency
+    HAS_SHAP = False
+
+
+class PositiveClassShapValuesTests(unittest.TestCase):
+    """Unit tests for the version-robust SHAP normalizer (no shap required)."""
+
+    def test_legacy_list_returns_class_one(self) -> None:
+        class0 = np.zeros((4, 3))
+        class1 = np.ones((4, 3))
+        result = positive_class_shap_values([class0, class1])
+        self.assertEqual(result.shape, (4, 3))
+        self.assertTrue(np.allclose(result, 1.0))
+
+    def test_modern_3d_array_selects_positive_class(self) -> None:
+        # Shape (n_samples, n_features, n_classes); class 1 is all ones.
+        arr = np.zeros((4, 3, 2))
+        arr[..., 1] = 1.0
+        result = positive_class_shap_values(arr)
+        self.assertEqual(result.shape, (4, 3))
+        self.assertTrue(np.allclose(result, 1.0))
+
+    def test_two_dimensional_array_passes_through(self) -> None:
+        arr = np.arange(12, dtype=float).reshape(4, 3)
+        result = positive_class_shap_values(arr)
+        self.assertEqual(result.shape, (4, 3))
+        self.assertTrue(np.allclose(result, arr))
+
+
+@unittest.skipUnless(HAS_SHAP, "shap is not installed")
+class ShapIntegrationTests(unittest.TestCase):
+    """End-to-end SHAP smoke test; skipped cleanly when shap is unavailable."""
+
+    def test_global_shap_runs_and_writes_figure(self) -> None:
+        import matplotlib
+
+        matplotlib.use("Agg")
+
+        from src.config import FIGURES_DIR
+        from src.explain import compute_and_plot_global_shap
+        from src.features import build_pipeline
+
+        df = generate_synthetic_fraud_dataset(n_samples=300, fraud_rate=0.2, seed=7)
+        X = df.drop(columns=[TARGET_COL])
+        y = df[TARGET_COL]
+
+        pipeline = build_pipeline()
+        pipeline.fit(X, y)
+
+        out_path = FIGURES_DIR / "shap_summary.png"
+        if out_path.exists():
+            out_path.unlink()
+
+        # Must not raise; modern shap returns a 3-D array that the old
+        # shap_values[1] indexing mishandled.
+        compute_and_plot_global_shap(pipeline, X, max_samples=100)
+
+        self.assertTrue(out_path.exists())
+
+    def test_single_row_positive_class_shape(self) -> None:
+        import scipy.sparse as sp
+
+        from src.features import build_pipeline
+
+        df = generate_synthetic_fraud_dataset(n_samples=300, fraud_rate=0.2, seed=11)
+        X = df.drop(columns=[TARGET_COL])
+        y = df[TARGET_COL]
+
+        pipeline = build_pipeline()
+        pipeline.fit(X, y)
+
+        preprocessor = pipeline.named_steps["preprocess"]
+        clf = pipeline.named_steps["clf"]
+        x_row = X.iloc[[0]]
+        x_t = preprocessor.transform(x_row)
+        x_t = x_t.toarray() if sp.issparse(x_t) else x_t
+
+        explainer = shap.TreeExplainer(clf)
+        single = positive_class_shap_values(explainer.shap_values(x_t))[0]
+
+        self.assertEqual(single.shape, (x_t.shape[1],))
+
+
+if __name__ == "__main__":
+    unittest.main()