fix: implement alpha test feedback. Make density=True by default for hist. rework hist implmentation to be identical to matplotlib. rework the demo notebook to be simpler. Add better errors in backend.

Author: ElouenGinat

README.md

@@ -12,6 +12,10 @@ Khisto is a Python library for creating histograms using the **Khiops optimal bi
 
 Documentation is available at **[khiops.github.io/khisto-python](https://khiopsml.github.io/khisto-python/)**.
 
+| Standard Gaussian | Heavy-tailed Pareto |
+| --- | --- |
+| ![Adaptive Gaussian histogram](docs/images/gaussian-quick-start.png) | ![Adaptive Pareto histogram](docs/images/pareto-quick-start.png) |
+
 ## Installation
 
 ```bash
@@ -26,17 +30,26 @@ pip install "khisto[matplotlib]"
 
 ## Quick Start
 
-### NumPy-like API
-
 ```python
 import numpy as np
-from khisto import histogram
+import matplotlib.pyplot as plt
+from khisto.matplotlib import hist
+
+# Generate 10,000 samples from a Pareto distribution
+long_tail_data = np.random.pareto(3, size=10000)
+
+# Plot an adaptive histogram on logarithmic axes.
+n, bins, patches = hist(long_tail_data, density=True)
+plt.xscale("symlog")
+plt.yscale("log")
+plt.show()
 
-# Generate 10,000 samples from a standard Gaussian distribution.
-data = np.random.normal(0, 1, 10000)
+# Generate 10,000 samples from a Normal distribution
+normal_data = np.random.normal(size=10000)
 
-# Compute optimal histogram (drop-in replacement for np.histogram)
-hist, bin_edges = histogram(data)
+# Plot an adaptive histogram
+n, bins, patches = hist(normal_data, density=True)
+plt.show()
 ```
 
 ## Development

docs/demo.ipynb

@@ -80,8 +80,8 @@ Get started
    :hidden:
 
    Histograms <array/histogram/index>
-   Core <core/index>
    Matplotlib <matplotlib/index>
+   Core <core/index>
 
 .. toctree::
    :maxdepth: 2

docs/index.rst

@@ -130,7 +130,7 @@
       "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
       "\u001b[31mValueError\u001b[39m                                Traceback (most recent call last)",
       "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[4]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m      1\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m khisto \u001b[38;5;28;01mimport\u001b[39;00m matplotlib\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m matplotlib.hist([data, [\u001b[32m1\u001b[39m, \u001b[32m2\u001b[39m, \u001b[32m3\u001b[39m], [\u001b[32m2\u001b[39m,\u001b[32m2\u001b[39m,\u001b[32m2\u001b[39m,\u001b[32m2\u001b[39m]], max_bins=\u001b[32m20\u001b[39m, alpha=\u001b[32m0.5\u001b[39m)\n",
-      "\u001b[36mFile \u001b[39m\u001b[32m~/python/khisto-python/src/khisto/matplotlib/hist.py:122\u001b[39m, in \u001b[36mhist\u001b[39m\u001b[34m(x, range, max_bins, density, cumulative, histtype, orientation, log, color, label, ax, edgecolor, linewidth, alpha, **kwargs)\u001b[39m\n\u001b[32m    119\u001b[39m     ax = plt.gca()\n\u001b[32m    121\u001b[39m \u001b[38;5;66;03m# Compute histogram using khisto\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m122\u001b[39m hist_values, bin_edges = \u001b[30;43mkhisto_histogram\u001b[39;49m\u001b[30;43m(\u001b[39;49m\n\u001b[32m    123\u001b[39m \u001b[30;43m    \u001b[39;49m\u001b[30;43mx\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mrange\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mrange\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mmax_bins\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mmax_bins\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mdensity\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mdensity\u001b[39;49m\n\u001b[32m    124\u001b[39m \u001b[30;43m\u001b[39;49m\u001b[30;43m)\u001b[39;49m\n\u001b[32m    125\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m cumulative_mode != \u001b[32m0\u001b[39m:\n\u001b[32m    126\u001b[39m     hist_values = _apply_cumulative(\n\u001b[32m    127\u001b[39m         hist_values,\n\u001b[32m    128\u001b[39m         bin_edges,\n\u001b[32m    129\u001b[39m         density=density,\n\u001b[32m    130\u001b[39m         reverse=cumulative_mode < \u001b[32m0\u001b[39m,\n\u001b[32m    131\u001b[39m     )\n",
+      "\u001b[36mFile \u001b[39m\u001b[32m~/python/khisto-python/src/khisto/matplotlib/hist.py:122\u001b[39m, in \u001b[36mhist\u001b[39m\u001b[34m(x, range, max_bins, density, cumulative, histtype, orientation, log, color, label, ax, edgecolor, linewidth, alpha, **kwargs)\u001b[39m\n\u001b[32m    119\u001b[39m     ax = plt.gca()\n\u001b[32m    121\u001b[39m \u001b[38;5;66;03m# Compute histogram using khisto\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m122\u001b[39m hist_values, bin_edges = \u001b[30;43mkhisto_histogram\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43mx\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mrange\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mrange\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mmax_bins\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mmax_bins\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mdensity\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mdensity\u001b[39;49m\u001b[30;43m)\u001b[39;49m\n\u001b[32m    123\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m cumulative_mode != \u001b[32m0\u001b[39m:\n\u001b[32m    124\u001b[39m     hist_values = _apply_cumulative(\n\u001b[32m    125\u001b[39m         hist_values,\n\u001b[32m    126\u001b[39m         bin_edges,\n\u001b[32m    127\u001b[39m         density=density,\n\u001b[32m    128\u001b[39m         reverse=cumulative_mode < \u001b[32m0\u001b[39m,\n\u001b[32m    129\u001b[39m     )\n",
       "\u001b[36mFile \u001b[39m\u001b[32m~/python/khisto-python/src/khisto/array/histogram/api.py:107\u001b[39m, in \u001b[36mhistogram\u001b[39m\u001b[34m(a, range, max_bins, density)\u001b[39m\n\u001b[32m     53\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mhistogram\u001b[39m(\n\u001b[32m     54\u001b[39m     a: ArrayLike,\n\u001b[32m     55\u001b[39m     \u001b[38;5;28mrange\u001b[39m: Optional[\u001b[38;5;28mtuple\u001b[39m[\u001b[38;5;28mfloat\u001b[39m, \u001b[38;5;28mfloat\u001b[39m]] = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m     56\u001b[39m     max_bins: Optional[\u001b[38;5;28mint\u001b[39m] = \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[32m     57\u001b[39m     density: \u001b[38;5;28mbool\u001b[39m = \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[32m     58\u001b[39m ) -> \u001b[38;5;28mtuple\u001b[39m[NDArray[np.float64], NDArray[np.float64]]:\n\u001b[32m     59\u001b[39m \u001b[38;5;250m    \u001b[39m\u001b[33;03m\"\"\"Compute an optimal histogram using the Khiops binning algorithm.\u001b[39;00m\n\u001b[32m     60\u001b[39m \n\u001b[32m     61\u001b[39m \u001b[33;03m    Parameters\u001b[39;00m\n\u001b[32m   (...)\u001b[39m\u001b[32m    105\u001b[39m \u001b[33;03m       Analysis, 180:0-0, 2023.\u001b[39;00m\n\u001b[32m    106\u001b[39m \u001b[33;03m    \"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m107\u001b[39m     arr = \u001b[30;43mnp\u001b[39;49m\u001b[30;43m.\u001b[39;49m\u001b[30;43masarray\u001b[39;49m\u001b[30;43m(\u001b[39;49m\u001b[30;43ma\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mdtype\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mnp\u001b[39;49m\u001b[30;43m.\u001b[39;49m\u001b[30;43mfloat64\u001b[39;49m\u001b[30;43m)\u001b[39;49m\n\u001b[32m    109\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m arr.ndim != \u001b[32m1\u001b[39m:\n\u001b[32m    110\u001b[39m         \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m    111\u001b[39m             \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mExpected 1-D array, got \u001b[39m\u001b[38;5;132;01m{\u001b[39;00marr.ndim\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m-D array instead. \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    112\u001b[39m             \u001b[33m\"\u001b[39m\u001b[33mReshape your data or flatten it before calling histogram.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    113\u001b[39m         )\n",
       "\u001b[31mValueError\u001b[39m: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions. The detected shape was (3,) + inhomogeneous part."
      ]
@@ -153,7 +153,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": null,
    "id": "23c19584",
    "metadata": {},
    "outputs": [
@@ -174,7 +174,7 @@
        " <a list of 3 BarContainer objects>)"
       ]
      },
-     "execution_count": 5,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     },
@@ -195,7 +195,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": null,
    "id": "2749c664",
    "metadata": {},
    "outputs": [
@@ -219,7 +219,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": null,
    "id": "1cca2392",
    "metadata": {},
    "outputs": [
@@ -241,7 +241,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": null,
    "id": "2ad6d7e5",
    "metadata": {},
    "outputs": [
@@ -273,7 +273,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 5,
    "id": "b6c4ea8c",
    "metadata": {},
    "outputs": [
@@ -297,6 +297,7 @@
    ],
    "source": [
     "from khisto.matplotlib import hist\n",
+    "from khisto.matplotlib.hist import _hist\n",
     "\n",
     "# Basic histogram plot\n",
     "fig, ax = plt.subplots(figsize=(8, 5))\n",
@@ -311,7 +312,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 6,
    "id": "09479225",
    "metadata": {},
    "outputs": [
@@ -339,7 +340,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 8,
    "id": "6c89bf07",
    "metadata": {},
    "outputs": [
@@ -366,7 +367,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 12,
    "id": "25d8d0e5",
    "metadata": {},
    "outputs": [
@@ -403,7 +404,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 15,
    "id": "d985437b",
    "metadata": {},
    "outputs": [
@@ -487,7 +488,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 18,
    "id": "51179a02",
    "metadata": {},
    "outputs": [
@@ -521,7 +522,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 21,
    "id": "e9bbabc9",
    "metadata": {},
    "outputs": [
@@ -558,7 +559,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 22,
    "id": "1190f8aa",
    "metadata": {},
    "outputs": [
@@ -594,7 +595,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 23,
    "id": "bf2ba150",
    "metadata": {},
    "outputs": [

sandbox/khisto_demo.ipynb

@@ -235,13 +235,14 @@ def _process_histogram_file(file_path: Path) -> list[HistogramResult]:
     ]
 
 
-def compute_histograms(x: np.ndarray) -> list[HistogramResult]:
+def compute_histograms(x: NDArray[np.float64]) -> list[HistogramResult]:
     """Compute optimal histogram of an array using khisto CLI binary input.
 
     Parameters
     ----------
-    x : np.ndarray
-        Array of numeric values.
+    x : NDArray[np.float64]
+        Array of numeric values. Only 1-dimensional arrays are supported.
+        Missing values (NaN) are filtered out.
 
     Returns
     -------
@@ -257,10 +258,14 @@ def compute_histograms(x: np.ndarray) -> list[HistogramResult]:
         If input array is empty after filtering.
     """
     x = np.asarray(x, dtype=np.float64)
+
+    if len(x) == 0:
+        raise ValueError("Input array is empty")
+
     x = x[~np.isnan(x)]
 
     if len(x) == 0:
-        raise ValueError("Input array is empty after filtering")
+        raise ValueError("Input array is empty after filtering missing values")
 
     # Use delete=False so the files are closed before the subprocess reads them.
     # On Windows, files keep an exclusive lock while open, whence,