Merge pull request #312 from bbopen/feat/0.10-pandas-json-domain

feat(codec): pandas Arrow fidelity gates + JSON loss preflight

Author: bbopen

examples/living-app/living_app/app.py

@@ -364,4 +364,11 @@ def top_users_by_spend(path: str, top_n: int = 10) -> pd.DataFrame:
     cols = [c for c in cols if c in df.columns]
     if "spend_usd_last_7d" not in df.columns:
         return df[cols].head(top_n)
-    return df[cols].sort_values("spend_usd_last_7d", ascending=False).head(top_n)
+    # The JSON codec requires a default RangeIndex; sort_values().head() keeps
+    # the original row labels, so drop them.
+    return (
+        df[cols]
+        .sort_values("spend_usd_last_7d", ascending=False)
+        .head(top_n)
+        .reset_index(drop=True)
+    )

runtime/tywrap_bridge_core.py

@@ -521,11 +521,42 @@ def serialize_dataframe(obj, *, force_json_markers):
 
 
 def serialize_dataframe_json(obj):
-    """JSON fallback for DataFrame: records orientation."""
+    """JSON fallback for DataFrame values that JavaScript can represent safely."""
+    import pandas as pd  # type: ignore
+
+    _validate_pandas_json_index(obj.index, pd, 'DataFrame')
+    json_columns = [_pandas_json_object_key(column) for column in obj.columns]
+    supported_json_columns = [column for column in json_columns if column is not None]
+    if not obj.columns.is_unique or len(set(supported_json_columns)) != len(
+        supported_json_columns
+    ):
+        raise RuntimeError(
+            'JSON pandas.DataFrame encoding requires column labels to remain unique after '
+            'JSON object-key coercion; rename columns or make them distinct before applying '
+            '.columns.astype(str)'
+        )
+    for column, dtype in obj.dtypes.items():
+        if isinstance(dtype, pd.CategoricalDtype):
+            raise RuntimeError(
+                f'JSON pandas.DataFrame encoding does not support categorical dtype in '
+                f'column {column!r}; use Arrow encoding or convert explicitly '
+                "(e.g. .astype(str))"
+            )
     try:
-        data = obj.to_dict(orient='records')
+        data = (
+            [{} for _ in range(len(obj.index))]
+            if len(obj.columns) == 0
+            else obj.to_dict(orient='records')
+        )
     except Exception as exc:
         raise RuntimeError('JSON fallback failed for pandas.DataFrame') from exc
+    for row_number, row in enumerate(data):
+        for column, value in row.items():
+            row[column] = _normalize_pandas_json_scalar(
+                value,
+                f'DataFrame cell at row {row_number}, column {column!r}',
+                pd,
+            )
     return {
         '__tywrap__': 'dataframe',
         'codecVersion': CODEC_VERSION,
@@ -567,14 +598,23 @@ def serialize_series(obj, *, force_json_markers):
 
 
 def serialize_series_json(obj):
-    """JSON fallback for Series (potentially lossy dtype/NA representation)."""
+    """JSON fallback for Series values that JavaScript can represent safely."""
+    import pandas as pd  # type: ignore
+
+    _validate_pandas_json_index(obj.index, pd, 'Series')
+    if isinstance(obj.dtype, pd.CategoricalDtype):
+        raise RuntimeError(
+            'JSON pandas.Series encoding does not support categorical dtype; use Arrow '
+            "encoding or convert explicitly (e.g. .astype(str))"
+        )
     try:
         data = obj.to_list()  # type: ignore
-    except Exception:
-        try:
-            data = obj.to_dict()  # type: ignore
-        except Exception as exc:
-            raise RuntimeError('JSON fallback failed for pandas.Series') from exc
+    except Exception as exc:
+        raise RuntimeError('JSON fallback failed for pandas.Series') from exc
+    data = [
+        _normalize_pandas_json_scalar(value, f'Series value at position {position}', pd)
+        for position, value in enumerate(data)
+    ]
     return {
         '__tywrap__': 'series',
         'codecVersion': CODEC_VERSION,
@@ -584,6 +624,71 @@ def serialize_series_json(obj):
     }
 
 
+def _validate_pandas_json_index(index, pd, container_name):
+    """Reject index metadata that records/list-oriented JSON would discard."""
+    if isinstance(index, pd.MultiIndex):
+        raise RuntimeError(
+            f'JSON pandas.{container_name} encoding does not support MultiIndex; use Arrow '
+            'encoding or flatten the index explicitly with .reset_index()'
+        )
+    if not (
+        isinstance(index, pd.RangeIndex)
+        and index.start == 0
+        and index.step == 1
+        and index.stop == len(index)
+        and index.name is None
+    ):
+        raise RuntimeError(
+            f'JSON pandas.{container_name} encoding requires an unnamed RangeIndex starting '
+            'at 0 with step 1; use Arrow encoding or normalize explicitly with '
+            '.reset_index(drop=True)'
+        )
+
+
+def _pandas_json_object_key(value):
+    """Return json.dumps' object-key spelling, or None for unsupported keys."""
+    try:
+        encoded = json.dumps({value: None})
+    except (TypeError, ValueError):
+        return None
+    return next(iter(json.loads(encoded)))
+
+
+def _normalize_pandas_json_scalar(value, location, pd):
+    """Normalize pandas nulls and reject values outside the plain JSON domain."""
+    np = sys.modules.get('numpy')
+    if np is not None and isinstance(value, np.generic):
+        value = value.item()
+    if value is None or value is pd.NA or value is pd.NaT:
+        return None
+    if type(value) is bool:
+        return value
+    if type(value) is int:
+        js_safe_integer_max = 2**53 - 1
+        if value < -js_safe_integer_max or value > js_safe_integer_max:
+            raise RuntimeError(
+                f'JSON pandas encoding cannot safely represent {location} integer values '
+                'outside the JavaScript safe integer range; use Arrow encoding or '
+                "cast/encode explicitly (e.g. .astype('float64') or str)"
+            )
+        return value
+    if type(value) is float:
+        if not math.isfinite(value):
+            raise RuntimeError(
+                f'JSON pandas encoding cannot represent non-finite {location} float values '
+                '(NaN or Infinity); use .fillna(...) for intentional missing values or '
+                'Arrow encoding'
+            )
+        return value
+    if type(value) is str:
+        return value
+    raise RuntimeError(
+        f'JSON pandas encoding does not support {location} value of type '
+        f'{type(value).__name__}; use Arrow encoding or convert explicitly '
+        '(e.g. .astype(str))'
+    )
+
+
 def serialize_sparse_matrix(obj):
     """
     Serialize scipy sparse matrices into structured JSON envelopes (json-only;

src/runtime/pyodide-bootstrap-core.generated.ts

@@ -148,10 +148,30 @@ def pandas_nullable_int64() -> object:
     return pd.DataFrame({"value": pd.Series([1, pd.NA], dtype="Int64")})
 
 
+def pandas_named_series() -> object:
+    import pandas as pd
+
+    return pd.Series([1, pd.NA], dtype="Int64", name="original_name")
+
+
 def pandas_categorical() -> object:
     import pandas as pd
 
-    return pd.DataFrame({"value": pd.Categorical(["a", "b", "a"])})
+    return pd.DataFrame(
+        {
+            "value": pd.Categorical(
+                ["a", "b", "a"],
+                categories=["unused", "b", "a"],
+                ordered=True,
+            )
+        }
+    )
+
+
+def pandas_nullable_boolean() -> object:
+    import pandas as pd
+
+    return pd.DataFrame({"value": pd.Series([True, pd.NA, False], dtype="boolean")})
 
 
 def pandas_pyarrow_string() -> object:

test/menagerie/fixtures/library_torture.py

@@ -27,6 +27,16 @@ export type CatalogueExpectation =
       kind: 'table-rows';
       value: readonly object[];
       pandasMetadataIncludes?: readonly string[];
+      pandasMetadataAbsent?: boolean;
+      pandasIndexColumns?: readonly unknown[];
+      fields?: readonly {
+        name: string;
+        type: string;
+        nullCount?: number;
+        validity?: readonly boolean[];
+        dictionaryValues?: readonly unknown[];
+        dictionaryOrdered?: boolean;
+      }[];
     };
 
 export interface CatalogueRow {
@@ -422,17 +432,59 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [
       kind: 'table-rows',
       value: [{ value: 1n }, { value: null }],
       pandasMetadataIncludes: ['"numpy_type": "Int64"'],
+      pandasIndexColumns: [{ kind: 'range', name: null, start: 0, stop: 2, step: 1 }],
+      fields: [{ name: 'value', type: 'Int64', nullCount: 1, validity: [true, false] }],
     },
   }),
   libraryRow({
     id: 'pandas-nullable-int64-json',
     call: 'pandas_nullable_int64()',
     codec: 'json',
     requires: ['pandas'],
-    status: 'KNOWN_LIE',
-    currentBehavior: 'JSON preserves values but drops the nullable integer dtype.',
+    status: 'EXPECTED_OK',
+    currentBehavior:
+      'Values-only JSON preserves nullable integer values and normalizes pd.NA to null.',
     expected: equal([{ value: 1 }, { value: null }]),
   }),
+  libraryRow({
+    id: 'pandas-series-arrow-current-truth',
+    call: 'pandas_named_series()',
+    codec: 'arrow',
+    requires: ['pandas', 'pyarrow'],
+    status: 'KNOWN_LIE',
+    currentBehavior:
+      'Series Arrow output is table-like: values survive, while its name and RangeIndex metadata do not.',
+    expected: {
+      kind: 'table-rows',
+      value: [{ value: 1n }, { value: null }],
+      pandasMetadataAbsent: true,
+      fields: [{ name: 'value', type: 'Int64', nullCount: 1, validity: [true, false] }],
+    },
+    expectedFix: 'Preserve the Series name and RangeIndex metadata through Arrow decoding.',
+  }),
+  libraryRow({
+    id: 'pandas-nullable-boolean-arrow',
+    call: 'pandas_nullable_boolean()',
+    codec: 'arrow',
+    requires: ['pandas', 'pyarrow'],
+    status: 'EXPECTED_OK',
+    currentBehavior: 'Arrow preserves nullable boolean values and their null bitmap.',
+    expected: {
+      kind: 'table-rows',
+      value: [{ value: true }, { value: null }, { value: false }],
+      pandasMetadataIncludes: ['"numpy_type": "boolean"'],
+      fields: [{ name: 'value', type: 'Bool', nullCount: 1, validity: [true, false, true] }],
+    },
+  }),
+  libraryRow({
+    id: 'pandas-nullable-boolean-json',
+    call: 'pandas_nullable_boolean()',
+    codec: 'json',
+    requires: ['pandas'],
+    status: 'EXPECTED_OK',
+    currentBehavior: 'Values-only JSON preserves booleans and normalizes pd.NA to null.',
+    expected: equal([{ value: true }, { value: null }, { value: false }]),
+  }),
   libraryRow({
     id: 'pandas-categorical-arrow',
     call: 'pandas_categorical()',
@@ -443,17 +495,28 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [
     expected: {
       kind: 'table-rows',
       value: [{ value: 'a' }, { value: 'b' }, { value: 'a' }],
-      pandasMetadataIncludes: ['"pandas_type": "categorical"'],
+      pandasMetadataIncludes: [
+        '"pandas_type": "categorical"',
+        '"num_categories": 3, "ordered": true',
+      ],
+      fields: [
+        {
+          name: 'value',
+          type: 'Dictionary<Int8, LargeUtf8>',
+          dictionaryValues: ['unused', 'b', 'a'],
+          dictionaryOrdered: true,
+        },
+      ],
     },
   }),
   libraryRow({
     id: 'pandas-categorical-json',
     call: 'pandas_categorical()',
     codec: 'json',
     requires: ['pandas'],
-    status: 'KNOWN_LIE',
-    currentBehavior: 'JSON preserves values but drops categorical dtype information.',
-    expected: equal([{ value: 'a' }, { value: 'b' }, { value: 'a' }]),
+    status: 'LOUD_FAIL',
+    currentBehavior: 'JSON rejects categorical dtype instead of dropping its vocabulary and order.',
+    expected: error(/categorical dtype.*Arrow.*astype\(str\)/i),
   }),
   libraryRow({
     id: 'pandas-pyarrow-string-arrow',
@@ -467,6 +530,7 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [
       kind: 'table-rows',
       value: [{ value: 'a' }, { value: null }],
       pandasMetadataIncludes: ['"numpy_type": "string"'],
+      fields: [{ name: 'value', type: 'LargeUtf8', nullCount: 1, validity: [true, false] }],
     },
   }),
   libraryRow({
@@ -475,8 +539,8 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [
     codec: 'json',
     requires: ['pandas', 'pyarrow'],
     featureProbe: 'import pandas as pd; pd.Series(["a"], dtype="string[pyarrow]"); print("1")',
-    status: 'KNOWN_LIE',
-    currentBehavior: 'JSON preserves values but drops the Arrow-backed string dtype.',
+    status: 'EXPECTED_OK',
+    currentBehavior: 'Values-only JSON preserves strings and normalizes pd.NA to null.',
     expected: equal([{ value: 'a' }, { value: null }]),
   }),
   libraryRow({
@@ -490,16 +554,18 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [
       kind: 'table-rows',
       value: [{ when: 1704164645000 }],
       pandasMetadataIncludes: ['"timezone": "UTC"'],
+      fields: [{ name: 'when', type: 'Timestamp<MICROSECOND, UTC>' }],
     },
   }),
   libraryRow({
     id: 'pandas-timezone-json',
     call: 'pandas_timezone_aware()',
     codec: 'json',
     requires: ['pandas'],
-    status: 'KNOWN_LIE',
-    currentBehavior: 'JSON delivers an ISO string but loses timestamp dtype and timezone schema.',
-    expected: equal([{ when: '2024-01-02T03:04:05+00:00' }]),
+    status: 'LOUD_FAIL',
+    currentBehavior:
+      'JSON rejects timezone-aware Timestamp cells instead of dropping their dtype schema.',
+    expected: error(/value of type Timestamp.*Arrow.*astype\(str\)/i),
   }),
   libraryRow({
     id: 'pandas-multiindex-arrow',
@@ -519,9 +585,9 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [
     call: 'pandas_multiindex()',
     codec: 'json',
     requires: ['pandas'],
-    status: 'KNOWN_LIE',
-    currentBehavior: 'Records-oriented JSON drops the MultiIndex.',
-    expected: equal([{ value: 3 }]),
+    status: 'LOUD_FAIL',
+    currentBehavior: 'JSON rejects MultiIndex instead of silently dropping its levels.',
+    expected: error(/MultiIndex.*Arrow.*reset_index\(\)/i),
   }),
   libraryRow({
     id: 'pandas-duplicate-labels-arrow',
@@ -537,9 +603,10 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [
     call: 'pandas_duplicate_labels()',
     codec: 'json',
     requires: ['pandas'],
-    status: 'KNOWN_LIE',
-    currentBehavior: 'Records-oriented JSON silently retains only the last duplicate label.',
-    expected: equal([{ value: 2 }]),
+    status: 'LOUD_FAIL',
+    currentBehavior:
+      'JSON rejects duplicate column labels instead of silently retaining the last value.',
+    expected: error(/column labels.*unique after JSON object-key coercion.*astype\(str\)/i),
   }),
   libraryRow({
     id: 'pandas-empty-frame',
@@ -559,8 +626,8 @@ export const RUNTIME_CATALOGUE: readonly CatalogueRow[] = [
     call: 'pandas_empty_frame()',
     codec: 'json',
     requires: ['pandas'],
-    status: 'KNOWN_LIE',
-    currentBehavior: 'Records-oriented JSON loses the empty DataFrame schema.',
+    status: 'EXPECTED_OK',
+    currentBehavior: 'The documented values-only JSON case permits an empty DataFrame.',
     expected: equal([]),
   }),
   libraryRow({