fixed the merged profiler code to the repo

Author: Daniel15568

README.md

@@ -55,6 +55,7 @@ One import, one line. A clean, sorted DataFrame you can read or feed into the ne
 
 | Function | What it answers |
 |---|---|
+| `profile` | What is wrong with this dataset, and how do I fix it? |
 | `change` | Growth as numbers, columns, or a whole series |
 | `vif` | Which features are collinear? |
 | `missing` | How much of each column is missing? |

docs/documentation.md

@@ -45,6 +45,57 @@ polars stays optional: it is only imported when you actually pass a polars objec
 
 ---
 
+## `profile`
+
+The flagship: a one-call **data diagnostician**. Instead of dumping statistics for every column, it runs a battery of checks, ranks the problems worst-first, scores the data's health, and tells you how to fix each issue. It composes the rest of the toolkit (`missing`, `imbalance`, and more) under one entry point.
+
+!!! tip "Similar concept"
+    `ydata-profiling` / `pandas.DataFrame.describe`, but diagnostic (it ranks problems and suggests fixes) rather than descriptive.
+
+**Signature**
+
+```python
+profile(data, target=None)
+```
+
+**Example**
+
+```python
+import pandas as pd
+from percentify import profile
+
+df = pd.DataFrame({
+    "user_id": range(100),                                              # identifier
+    "plan":    ["free"] * 100,                                          # constant
+    "age":     [20 + (i * 37) % 40 for i in range(100)],
+    "spend":   [None if i % 2 == 0 else float((i * 13) % 100) for i in range(100)],
+    "churn":   ["No"] * 96 + ["Yes"] * 4,                              # imbalanced target
+})
+
+report = profile(df, target="churn")
+report.health        # 87
+report.to_frame()
+```
+
+```text
+severity         code   column                          message                           suggestion
+ warning     constant     plan          only one distinct value         drop, carries no information
+ warning high_missing    spend                      50% missing                       impute or drop
+ warning      id_like  user_id identifier-like (100/100 unique)                 drop before modeling
+    info    imbalance <target> class 'Yes' is only 4.0% of rows consider resampling or class weights
+```
+
+The report renders as a compact, color-coded summary in notebooks and terminals, and exposes everything programmatically:
+
+- `report.errors`, `report.warnings`, `report.infos`: findings filtered by severity.
+- `report.to_frame()`: all findings as a tidy DataFrame.
+- `report.health`: a 0 to 100 score (100 is clean).
+- `assert not report.errors`: drop it straight into a CI data-quality gate.
+
+Pass `target=` to also check for **leakage** (features that predict the target almost perfectly) and class imbalance. Accepts pandas or polars input.
+
+---
+
 ## `change`
 
 Percentage change — as two numbers, between two columns, or down a whole series.

docs/index.md

@@ -54,6 +54,7 @@ One import, one line. A clean, sorted DataFrame you can read or feed straight in
 
 | Function | What it answers |
 |---|---|
+| `profile` | What is wrong with this dataset, and how do I fix it? |
 | `change` | How much did a value grow, across numbers, columns, or a whole series? |
 | `vif` | Which features are collinear? |
 | `missing` | How much of each column is missing? |

percentify/__init__.py

@@ -1,9 +1,11 @@
+from .profiling import profile, ProfileReport, Finding
 from .stats import (
     change, vif, missing, cv, outliers, r_squared, pca_variance, imbalance,
     difference, split, display, PercentifyWarning,
 )
 
 __all__ = [
+    "profile", "ProfileReport", "Finding",
     "change", "vif", "missing", "cv", "outliers", "r_squared", "pca_variance", "imbalance",
     "difference", "split", "display", "PercentifyWarning",
 ]

percentify/profiling.py

@@ -26,6 +26,7 @@
 
 from __future__ import annotations
 
+import html
 import re
 import warnings
 from dataclasses import dataclass, field
@@ -34,6 +35,8 @@
 import numpy as np
 import pandas as pd
 
+from .stats import PercentifyWarning, imbalance, missing
+
 __all__ = ["profile", "ProfileReport", "Finding"]
 
 # Severity ordering and the penalty each level deducts from the health score.
@@ -132,18 +135,18 @@ def _encode_target(target: pd.Series) -> tuple[np.ndarray, bool]:
 # --------------------------------------------------------------------------- #
 def check_missing(df: pd.DataFrame, target) -> list[Finding]:
     out: list[Finding] = []
-    n = len(df)
-    if n == 0:
+    if len(df) == 0:
         return out
-    frac = df.isna().mean()
-    for col, f in frac.items():
-        if f >= 1.0:
+    # Reuse the package's missing() so there is a single source of truth.
+    for _, row in missing(df).iterrows():
+        col, pct = row["column"], row["missing_pct"]
+        if pct >= 100.0:
             out.append(Finding(col, "error", "all_missing",
                                "column is entirely missing",
                                "drop the column"))
-        elif f >= 0.4:
+        elif pct >= 40.0:
             out.append(Finding(col, "warning", "high_missing",
-                               f"{f * 100:.0f}% missing",
+                               f"{pct:.0f}% missing",
                                "impute or drop"))
     return out
 
@@ -156,7 +159,7 @@ def check_constant(df: pd.DataFrame, target) -> list[Finding]:
         if df[col].nunique(dropna=True) <= 1:
             out.append(Finding(col, "warning", "constant",
                                "only one distinct value",
-                               "drop — carries no information"))
+                               "drop, carries no information"))
     return out
 
 
@@ -286,7 +289,7 @@ def check_collinearity(df: pd.DataFrame, target) -> list[Finding]:
             if pd.notna(r) and r >= 0.95 and frozenset((a, b)) not in seen:
                 seen.add(frozenset((a, b)))
                 out.append(Finding(f"{a} \u27f7 {b}", "warning", "collinear",
-                                   f"correlation {r:.2f} — redundant pair",
+                                   f"correlation {r:.2f}, redundant pair",
                                    "drop one, or check vif()"))
     return out
 
@@ -298,11 +301,13 @@ def check_leakage(df: pd.DataFrame, target) -> list[Finding]:
     y, y_is_cat = _encode_target(target)
     out: list[Finding] = []
     n = len(df)
+    numeric_cols = set(_numeric_columns(df))
+    text_cols = set(_text_columns(df))
 
     for col in df.columns:
         s = df[col]
         # Numeric feature: near-perfect linear association with the target.
-        if col in _numeric_columns(df):
+        if col in numeric_cols:
             mask = s.notna().to_numpy() & np.isfinite(y)
             if mask.sum() < 10:
                 continue
@@ -314,9 +319,9 @@ def check_leakage(df: pd.DataFrame, target) -> list[Finding]:
             if np.isfinite(r) and r > 0.98:
                 out.append(Finding(col, "error", "leakage",
                                    f"predicts the target (|r|={r:.2f})",
-                                   "likely leakage — confirm it is known at predict time"))
+                                   "likely leakage, confirm it is known at predict time"))
         # Categorical feature: does it almost perfectly determine the target?
-        elif y_is_cat and col in _text_columns(df):
+        elif y_is_cat and col in text_cols:
             nun = s.nunique(dropna=True)
             if nun <= 1 or nun >= 0.5 * n:
                 continue  # constant or id-like handled elsewhere
@@ -331,7 +336,7 @@ def check_leakage(df: pd.DataFrame, target) -> list[Finding]:
                 out.append(Finding(col, "error", "leakage",
                                    f"almost perfectly determines the target "
                                    f"(purity {purity:.2f})",
-                                   "likely leakage — confirm it is known at predict time"))
+                                   "likely leakage, confirm it is known at predict time"))
     return out
 
 
@@ -340,11 +345,16 @@ def check_target_imbalance(df: pd.DataFrame, target) -> list[Finding]:
         return []
     if pd.api.types.is_numeric_dtype(target) and target.nunique(dropna=True) > 20:
         return []  # treat as continuous
-    counts = target.value_counts(normalize=True, dropna=True)
-    if len(counts) >= 2 and counts.min() < 0.05:
-        smallest = counts.idxmin()
+    # Reuse the package's imbalance() rather than re-deriving the class balance.
+    result = imbalance(target)
+    summary = result.attrs.get("summary", {})
+    if summary.get("n_classes", 0) < 2:
+        return []
+    minority = summary["minority_class"]
+    min_pct = dict(zip(result["class"].tolist(), result["pct"].tolist())).get(minority)
+    if min_pct is not None and min_pct < 5.0:
         return [Finding("<target>", "info", "imbalance",
-                        f"class '{smallest}' is only {counts.min() * 100:.1f}% of rows",
+                        f"class '{minority}' is only {min_pct:.1f}% of rows",
                         "consider resampling or class weights")]
     return []
 
@@ -444,12 +454,12 @@ def _repr_html_(self) -> str:
         rows = ""
         for f in self.findings:
             c = colors[f.severity]
-            sug = f" &rarr; {f.suggestion}" if f.suggestion else ""
+            sug = f" &rarr; {html.escape(f.suggestion)}" if f.suggestion else ""
             rows += (
                 f'<tr><td style="color:{c};font-weight:600;padding:2px 10px">'
                 f'{f.severity}</td>'
-                f'<td style="font-family:monospace;padding:2px 10px">{f.column}</td>'
-                f'<td style="padding:2px 10px">{f.message}'
+                f'<td style="font-family:monospace;padding:2px 10px">{html.escape(f.column)}</td>'
+                f'<td style="padding:2px 10px">{html.escape(f.message)}'
                 f'<span style="color:#888">{sug}</span></td></tr>'
             )
         if not rows:
@@ -459,8 +469,8 @@ def _repr_html_(self) -> str:
         for _, r in self.summary.iterrows():
             spark = self._sparklines.get(r["column"], "")
             cols += (
-                f'<tr><td style="font-family:monospace;padding:2px 10px">{r["column"]}</td>'
-                f'<td style="padding:2px 10px;color:#888">{r["dtype"]}</td>'
+                f'<tr><td style="font-family:monospace;padding:2px 10px">{html.escape(str(r["column"]))}</td>'
+                f'<td style="padding:2px 10px;color:#888">{html.escape(str(r["dtype"]))}</td>'
                 f'<td style="font-family:monospace;padding:2px 10px">{spark}</td>'
                 f'<td style="padding:2px 10px">{r["missing_pct"]:.0f}%</td>'
                 f'<td style="padding:2px 10px">{r["cardinality"]:,}</td></tr>'
@@ -504,14 +514,14 @@ def profile(data, target: Optional[str] = None) -> ProfileReport:
             df = df.drop(columns=[target])
         else:
             warnings.warn(f"target {target!r} not found in columns; ignoring",
-                          stacklevel=2)
+                          PercentifyWarning, stacklevel=2)
 
     findings: list[Finding] = []
     for check in CHECKS:
         try:
             findings.extend(check(df, target_series))
         except Exception as exc:  # a broken check must never sink the report
-            warnings.warn(f"{check.__name__} failed: {exc}", stacklevel=2)
+            warnings.warn(f"{check.__name__} failed: {exc}", PercentifyWarning, stacklevel=2)
 
     findings.sort(key=lambda f: (_SEVERITY_ORDER[f.severity], f.column))
 

tests/test_profiling.py

@@ -0,0 +1,89 @@
+import pytest
+import numpy as np
+import pandas as pd
+from percentify import profile, ProfileReport, Finding, PercentifyWarning
+
+
+@pytest.fixture
+def messy_df():
+    np.random.seed(0)
+    base = np.random.randn(200)
+    return pd.DataFrame({
+        "id": range(200),                       # id-like
+        "empty": [None] * 200,                   # entirely missing
+        "const": [7] * 200,                      # constant
+        "x": base,
+        "x_dup": base * 1.0,                     # collinear with x
+        "churn": ["No"] * 195 + ["Yes"] * 5,     # 2.5% minority target
+    })
+
+
+def test_profile_returns_report(messy_df):
+    assert isinstance(profile(messy_df), ProfileReport)
+
+
+def test_profile_all_missing_is_error(messy_df):
+    codes = {(f.column, f.code) for f in profile(messy_df).errors}
+    assert ("empty", "all_missing") in codes
+
+
+def test_profile_flags_constant_and_id(messy_df):
+    warn_codes = {f.code for f in profile(messy_df).warnings}
+    assert "constant" in warn_codes
+    assert "id_like" in warn_codes
+    assert "collinear" in warn_codes
+
+
+def test_profile_target_imbalance(messy_df):
+    infos = {f.code for f in profile(messy_df, target="churn").infos}
+    assert "imbalance" in infos
+
+
+def test_profile_to_frame(messy_df):
+    frame = profile(messy_df).to_frame()
+    assert isinstance(frame, pd.DataFrame)
+    assert list(frame.columns) == ["severity", "code", "column", "message", "suggestion"]
+
+
+def test_profile_findings_are_findings(messy_df):
+    assert all(isinstance(f, Finding) for f in profile(messy_df).findings)
+
+
+def test_profile_clean_data_full_health():
+    clean = pd.DataFrame({
+        "a": [x % 10 for x in range(30)],
+        "b": [x % 7 for x in range(30)],
+    })
+    report = profile(clean)
+    assert report.findings == []
+    assert report.health == 100
+
+
+def test_profile_health_drops_with_issues(messy_df):
+    assert profile(messy_df).health < 100
+
+
+def test_profile_target_not_found_warns(messy_df):
+    with pytest.warns(PercentifyWarning):
+        profile(messy_df, target="does_not_exist")
+
+
+def test_profile_html_escapes_column_names():
+    evil = pd.DataFrame({"<script>": [None] * 200, "ok": range(200)})
+    rendered = profile(evil)._repr_html_()
+    assert "&lt;script&gt;" in rendered
+    assert "<script>" not in rendered
+
+
+def test_profile_composes_missing_high_warning():
+    # 60% missing should surface as a high_missing warning (via missing()).
+    df = pd.DataFrame({"col": [1.0, 2.0] + [None] * 3})
+    codes = {f.code for f in profile(df).warnings}
+    assert "high_missing" in codes
+
+
+def test_profile_polars_input(messy_df):
+    pl = pytest.importorskip("polars")
+    report = profile(pl.from_pandas(messy_df), target="churn")
+    assert isinstance(report, ProfileReport)
+    assert any(f.code == "all_missing" for f in report.errors)