added outliers, co-efficient of variations and easy data profiling

Author: Daniel15568

percentify/__init__.py

@@ -1,5 +1,5 @@
 from .core import percent, change, difference, split, display
-from .stats import vif
+from .stats import vif, missing, cv, outliers
 
-__all__ = ["percent", "change", "difference", "split", "display", "vif"]
+__all__ = ["percent", "change", "difference", "split", "display", "vif", "missing", "cv", "outliers"]
 __version__ = "0.3.2"

percentify/stats.py

@@ -1,4 +1,4 @@
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Sequence, Union
 
 import numpy as np
 import pandas as pd
@@ -70,3 +70,102 @@ def vif(df: pd.DataFrame, decimals: Optional[int] = 2, flag: Optional[float] = N
         result = {k: v for k, v in result.items() if v > flag}
 
     return result
+
+
+def _round(value: float, decimals: Optional[int]) -> float:
+    if decimals is None:
+        return value
+    return round(value, decimals)
+
+
+def missing(df: pd.DataFrame, decimals: Optional[int] = 2) -> Dict[str, float]:
+    """
+    Calculate the percentage of missing values for each column.
+
+    Args:
+        df: DataFrame to profile.
+        decimals: Number of decimal places to round to.
+
+    Returns:
+        dict: Column names mapped to their missing percentage,
+            sorted from highest to lowest.
+    """
+    total = len(df)
+    if total == 0:
+        return {col: 0.0 for col in df.columns}
+
+    result = {}
+    for col in df.columns:
+        pct = df[col].isnull().sum() / total * 100.0
+        result[col] = _round(pct, decimals)
+
+    return dict(sorted(result.items(), key=lambda x: x[1], reverse=True))
+
+
+def cv(data: Union[pd.Series, pd.DataFrame], decimals: Optional[int] = 2) -> Union[float, Dict[str, float]]:
+    """
+    Calculate the coefficient of variation (CV = std / mean * 100).
+
+    Args:
+        data: A Series (single result) or DataFrame (all numeric columns).
+        decimals: Number of decimal places to round to.
+
+    Returns:
+        float if Series, dict if DataFrame.
+
+    Raises:
+        ValueError: If the mean is zero (CV is undefined).
+    """
+    if isinstance(data, pd.Series):
+        mean = data.mean()
+        if mean == 0:
+            raise ValueError("Coefficient of variation is undefined when mean is zero.")
+        val = data.std() / abs(mean) * 100.0
+        return _round(val, decimals)
+
+    numeric = data.select_dtypes(include=[np.number])
+    result = {}
+    for col in numeric.columns:
+        mean = numeric[col].mean()
+        if mean == 0:
+            result[col] = float("inf")
+        else:
+            val = numeric[col].std() / abs(mean) * 100.0
+            result[col] = _round(val, decimals)
+    return result
+
+
+def outliers(data: Union[pd.Series, pd.DataFrame], decimals: Optional[int] = 2, multiplier: float = 1.5) -> Union[float, Dict[str, float]]:
+    """
+    Calculate the percentage of outliers using the IQR method.
+
+    An outlier is any value below Q1 - multiplier*IQR or above Q3 + multiplier*IQR.
+
+    Args:
+        data: A Series (single result) or DataFrame (all numeric columns).
+        decimals: Number of decimal places to round to.
+        multiplier: IQR multiplier for defining outlier bounds (default: 1.5).
+
+    Returns:
+        float if Series, dict if DataFrame.
+    """
+    def _calc(s: pd.Series) -> float:
+        s = s.dropna()
+        if len(s) == 0:
+            return 0.0
+        q1 = s.quantile(0.25)
+        q3 = s.quantile(0.75)
+        iqr = q3 - q1
+        lower = q1 - multiplier * iqr
+        upper = q3 + multiplier * iqr
+        count = ((s < lower) | (s > upper)).sum()
+        return count / len(s) * 100.0
+
+    if isinstance(data, pd.Series):
+        return _round(_calc(data), decimals)
+
+    numeric = data.select_dtypes(include=[np.number])
+    result = {}
+    for col in numeric.columns:
+        result[col] = _round(_calc(numeric[col]), decimals)
+    return result

tests/test_stats.py

@@ -1,9 +1,11 @@
 import pytest
 import numpy as np
 import pandas as pd
-from percentify import vif
+from percentify import vif, missing, cv, outliers
 
 
+# ===== Fixtures =====
+
 @pytest.fixture
 def independent_df():
     np.random.seed(42)
@@ -25,6 +27,8 @@ def collinear_df():
     })
 
 
+# ===== vif =====
+
 def test_vif_returns_all_columns(independent_df):
     result = vif(independent_df)
     assert set(result.keys()) == {"a", "b", "c"}
@@ -77,3 +81,162 @@ def test_vif_custom_decimals(independent_df):
         str_val = str(val)
         if "." in str_val:
             assert len(str_val.split(".")[1]) <= 4
+
+
+# ===== missing =====
+
+def test_missing_basic():
+    df = pd.DataFrame({
+        "a": [1, 2, None, 4, 5],
+        "b": [None, None, 3, 4, 5],
+        "c": [1, 2, 3, 4, 5],
+    })
+    result = missing(df)
+    assert result["b"] == 40.0
+    assert result["a"] == 20.0
+    assert result["c"] == 0.0
+
+
+def test_missing_sorted_descending():
+    df = pd.DataFrame({
+        "a": [1, 2, None, 4, 5],
+        "b": [None, None, 3, 4, 5],
+        "c": [1, 2, 3, 4, 5],
+    })
+    result = missing(df)
+    keys = list(result.keys())
+    assert keys == ["b", "a", "c"]
+
+
+def test_missing_no_nulls():
+    df = pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]})
+    result = missing(df)
+    assert result["a"] == 0.0
+    assert result["b"] == 0.0
+
+
+def test_missing_all_null():
+    df = pd.DataFrame({"a": [None, None, None]})
+    result = missing(df)
+    assert result["a"] == 100.0
+
+
+def test_missing_empty_df():
+    df = pd.DataFrame({"a": [], "b": []})
+    result = missing(df)
+    assert result["a"] == 0.0
+
+
+def test_missing_includes_non_numeric():
+    df = pd.DataFrame({
+        "name": ["Alice", None, "Charlie"],
+        "age": [25, None, 35],
+    })
+    result = missing(df)
+    assert "name" in result
+    assert "age" in result
+    assert result["name"] == 33.33
+    assert result["age"] == 33.33
+
+
+# ===== cv =====
+
+def test_cv_series():
+    s = pd.Series([10, 20, 30, 40, 50])
+    result = cv(s)
+    assert result > 0
+    assert isinstance(result, float)
+
+
+def test_cv_dataframe():
+    df = pd.DataFrame({
+        "a": [10, 20, 30, 40, 50],
+        "b": [100, 100, 100, 100, 100],
+    })
+    result = cv(df)
+    assert isinstance(result, dict)
+    assert result["a"] > 0
+    assert result["b"] == 0.0
+
+
+def test_cv_zero_mean_series():
+    s = pd.Series([-1, 0, 1])
+    with pytest.raises(ValueError):
+        cv(s)
+
+
+def test_cv_zero_mean_dataframe():
+    df = pd.DataFrame({
+        "a": [-1, 0, 1],
+        "b": [10, 20, 30],
+    })
+    result = cv(df)
+    assert result["a"] == float("inf")
+    assert result["b"] > 0
+
+
+def test_cv_ignores_non_numeric():
+    df = pd.DataFrame({
+        "a": [10, 20, 30],
+        "name": ["x", "y", "z"],
+    })
+    result = cv(df)
+    assert "name" not in result
+    assert "a" in result
+
+
+def test_cv_custom_decimals():
+    s = pd.Series([10, 20, 30, 40, 50])
+    result = cv(s, decimals=4)
+    str_val = str(result)
+    if "." in str_val:
+        assert len(str_val.split(".")[1]) <= 4
+
+
+# ===== outliers =====
+
+def test_outliers_series():
+    s = pd.Series([1, 2, 3, 4, 5, 6, 7, 8, 9, 100])
+    result = outliers(s)
+    assert isinstance(result, float)
+    assert result > 0
+
+
+def test_outliers_no_outliers():
+    s = pd.Series([1, 2, 3, 4, 5])
+    result = outliers(s)
+    assert result == 0.0
+
+
+def test_outliers_dataframe():
+    df = pd.DataFrame({
+        "a": [1, 2, 3, 4, 5, 6, 7, 8, 9, 100],
+        "b": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+    })
+    result = outliers(df)
+    assert isinstance(result, dict)
+    assert result["a"] > 0
+    assert result["b"] == 0.0
+
+
+def test_outliers_all_nan():
+    s = pd.Series([None, None, None])
+    result = outliers(s)
+    assert result == 0.0
+
+
+def test_outliers_ignores_non_numeric():
+    df = pd.DataFrame({
+        "a": [1, 2, 3, 4, 100],
+        "name": ["x", "y", "z", "w", "v"],
+    })
+    result = outliers(df)
+    assert "name" not in result
+    assert "a" in result
+
+
+def test_outliers_custom_multiplier():
+    s = pd.Series([1, 2, 3, 4, 5, 6, 7, 8, 9, 15])
+    strict = outliers(s, multiplier=1.0)
+    loose = outliers(s, multiplier=3.0)
+    assert strict >= loose