Merge pull request #314 from bbopen/chore/0.10-simplify

refactor(codec): simplification round for the 0.10 series

Author: bbopen

docs/codec-envelopes.md

@@ -40,6 +40,23 @@ Every scientific envelope has these fields:
 - `codecVersion` is `1`. Decoders treat a missing version as legacy version `0`.
 - `encoding` identifies the payload representation.
 
+Scientific values can appear at the root of a result or inside nested Python
+dictionaries, lists, and tuples. The Python bridge replaces each recognized
+value with its envelope. The JavaScript codec then walks plain objects and
+arrays and decodes recognized envelopes within the traversal bounds. A decoded
+envelope is a terminal value, so fields inside its decoded output are not
+scanned again. A `torch.tensor` envelope is the exception because its `value`
+field is a required nested ndarray envelope.
+
+Serialization accepts a maximum nesting depth of 900 and at most 1,000,000
+visited containers. Decoding accepts a maximum depth of 2048 and the same
+1,000,000-container limit. Scientific envelopes count as containers, including
+the ndarray nested inside a tensor. Primitive leaves and the elements of a
+decoded envelope payload do not consume the container limit. The bridge reports
+the result path when a serialization bound is exceeded, and the JavaScript
+codec does the same for decoding bounds. The bridge also rejects circular
+Python containers.
+
 ### SciPy sparse
 
 Supported targets are `scipy.sparse.csr_matrix`, `csc_matrix`, and
@@ -90,7 +107,8 @@ The target is `torch.Tensor`. CPU tensors work by default. Schema sketch:
     "encoding": "arrow" | "json",
     "b64": "...",        // when encoding="arrow"
     "data": [ ... ],     // when encoding="json"
-    "shape": [ ... ]
+    "shape": [ ... ],
+    "dtype": "float32"
   }
 }
 ```
@@ -128,8 +146,24 @@ pyarrow is not available in WASM.
 
 If an Arrow payload arrives without the dependency, decoding fails with an
 installation hint. Set `TYWRAP_CODEC_FALLBACK=json` on the Python side when a
-JSON fallback is acceptable. JSON fallback is lossy for dtype and missing-value
-fidelity.
+JSON representation is acceptable.
+
+An ndarray JSON envelope includes `shape`, `dtype`, and plain JSON `data`. The
+decoder checks that the data matches the declared shape and supported dtype
+domain. It returns plain JavaScript arrays, or a scalar for a zero-dimensional
+array. The `dtype` field records provenance for validation; it does not create a
+typed JavaScript array. JSON encoding rejects ndarray dtypes and values that it
+cannot represent safely, including complex, structured, object, byte-string,
+unicode, datetime, timedelta, big-endian, and unsafe integer cases.
+
+DataFrame and Series JSON envelopes are values-only representations. Before
+encoding, the bridge requires an unnamed `RangeIndex` starting at 0 with step 1.
+It rejects a `MultiIndex`, categorical dtype, unsupported cell values, unsafe
+integers, non-finite floats, duplicate DataFrame column labels, and labels that
+collide after JSON object-key coercion. `None`, `pd.NA`, and `pd.NaT` become
+JSON `null`. Supported booleans, numbers, strings, empty values, and nulls keep
+their value semantics, but pandas dtype and index metadata are not retained as
+they are with Arrow.
 
 ## Size limits and transport framing
 
@@ -154,7 +188,8 @@ their promises resolve. A result that does not match its declared return type
 throws `BridgeValidationError` with the declared type, received shape, and call
 site.
 
-Columnar values retain their decoded provenance. Validation checks the envelope
-marker, dimensions, and dtype for DataFrame, Series, and ndarray returns
-without walking Arrow data. Returns declared as `unknown`, `void`, or `Any` do
-not receive a runtime check.
+Decoded scientific values retain their envelope provenance. Return validation
+recognizes all six markers: `ndarray`, `dataframe`, `series`, `scipy.sparse`,
+`torch.tensor`, and `sklearn.estimator`. It checks the marker and, when present
+in the return contract, dimensions and dtype without walking Arrow data. Returns
+declared as `unknown`, `void`, or `Any` do not receive a runtime check.

runtime/tywrap_bridge_core.py

@@ -60,6 +60,7 @@
 CODEC_VERSION = 1
 MAX_SERIALIZE_DEPTH = 900
 MAX_SERIALIZE_NODES = 1_000_000
+JS_SAFE_INTEGER_MAX = 2**53 - 1
 _SERIALIZE_PATH_IDENTIFIER = re.compile(r'^[A-Za-z_$][\w$]*$', re.ASCII)
 
 
@@ -414,9 +415,8 @@ def serialize_ndarray_json(obj):
     if dtype.kind in ('i', 'u'):
         # JSON numbers become JavaScript Number values. Scan only on this explicit
         # fallback path and reject before tolist() can silently round an integer.
-        js_safe_integer_max = 2**53 - 1
         if obj.size and (
-            (obj < -js_safe_integer_max).any() or (obj > js_safe_integer_max).any()
+            (obj < -JS_SAFE_INTEGER_MAX).any() or (obj > JS_SAFE_INTEGER_MAX).any()
         ):
             raise RuntimeError(
                 f'JSON ndarray encoding cannot safely represent dtype={dtype_label} values '
@@ -525,7 +525,7 @@ def serialize_dataframe_json(obj):
     import pandas as pd  # type: ignore
 
     _validate_pandas_json_index(obj.index, pd, 'DataFrame')
-    json_columns = [_pandas_json_object_key(column) for column in obj.columns]
+    json_columns = [_json_object_key(column) for column in obj.columns]
     supported_json_columns = [column for column in json_columns if column is not None]
     if not obj.columns.is_unique or len(set(supported_json_columns)) != len(
         supported_json_columns
@@ -645,7 +645,7 @@ def _validate_pandas_json_index(index, pd, container_name):
         )
 
 
-def _pandas_json_object_key(value):
+def _json_object_key(value):
     """Return json.dumps' object-key spelling, or None for unsupported keys."""
     try:
         encoded = json.dumps({value: None})
@@ -664,8 +664,7 @@ def _normalize_pandas_json_scalar(value, location, pd):
     if type(value) is bool:
         return value
     if type(value) is int:
-        js_safe_integer_max = 2**53 - 1
-        if value < -js_safe_integer_max or value > js_safe_integer_max:
+        if value < -JS_SAFE_INTEGER_MAX or value > JS_SAFE_INTEGER_MAX:
             raise RuntimeError(
                 f'JSON pandas encoding cannot safely represent {location} integer values '
                 'outside the JavaScript safe integer range; use Arrow encoding or '
@@ -759,8 +758,8 @@ def serialize_torch_tensor(obj, *, force_json_markers, torch_allow_copy=False):
     Rejection order is significant: the categorical rejections (sparse / quantized
     / meta / complex) are checked BEFORE the device/contiguous opt-in branch so
     they fail with a clear, specific message and are NOT bypassable by
-    TYWRAP_TORCH_ALLOW_COPY. The opt-in only governs the lossy-but-lossless device
-    transfer and contiguous copy, never an unrepresentable layout/dtype.
+    TYWRAP_TORCH_ALLOW_COPY. The opt-in only governs the device transfer and
+    contiguous copy, never an unrepresentable layout/dtype.
     """
     import torch  # already importable: is_torch_tensor() gated the dispatch
 
@@ -1053,7 +1052,7 @@ def serialize(obj, *, force_json_markers, torch_allow_copy=False):
                     f'keys must be str, int, float, bool or None, not '
                     f'{type(item_key).__name__} at {invalid_path}'
                 )
-            child_key = next(iter(json.loads(json.dumps({item_key: None}))))
+            child_key = _json_object_key(item_key)
             child_path = _serialize_path(path, child_key)
             if _needs_serialize_visit(item):
                 stack.append(('visit', item, depth + 1, child_path, output, item_key))
@@ -1198,20 +1197,9 @@ def default_encoder(obj):
                 if isinstance(obj, pd.Timedelta):
                     return obj.total_seconds()
 
-        if isinstance(obj, dt.datetime):
-            return obj.isoformat()
-        if isinstance(obj, dt.date):
-            return obj.isoformat()
-        if isinstance(obj, dt.time):
-            return obj.isoformat()
-        if isinstance(obj, dt.timedelta):
-            return obj.total_seconds()
-        if isinstance(obj, decimal.Decimal):
-            return str(obj)
-        if isinstance(obj, uuid.UUID):
-            return str(obj)
-        if isinstance(obj, (Path, PurePath)):
-            return str(obj)
+        stdlib_value = serialize_stdlib(obj)
+        if stdlib_value is not None:
+            return stdlib_value
 
         if isinstance(obj, (bytes, bytearray)):
             return {

src/runtime/bridge-codec.ts

@@ -10,7 +10,11 @@
 
 import { BridgeCodecError, BridgeProtocolError, BridgeExecutionError } from './errors.js';
 import { containsSpecialFloat } from './validators.js';
-import { decodeValueAsync as decodeArrowValue } from '../utils/codec.js';
+import {
+  decodeValueAsync as decodeScientificValue,
+  isScientificMarker,
+  ScientificDecodeError,
+} from '../utils/codec.js';
 import { PROTOCOL_ID } from './transport.js';
 
 // ═══════════════════════════════════════════════════════════════════════════
@@ -91,26 +95,6 @@ function buildPath(basePath: string, key: string | number): string {
   return typeof key === 'number' ? `${basePath}[${key}]` : `${basePath}.${key}`;
 }
 
-const SCIENTIFIC_MARKERS = new Set([
-  'dataframe',
-  'series',
-  'ndarray',
-  'scipy.sparse',
-  'torch.tensor',
-  'sklearn.estimator',
-]);
-
-function scientificMarkerFrom(value: unknown, errorMessage: string): string | undefined {
-  const match = /^(?:Invalid|Unsupported) ([a-z.]+) envelope/.exec(errorMessage);
-  if (match?.[1] && SCIENTIFIC_MARKERS.has(match[1])) {
-    return match[1];
-  }
-  if (isPlainObject(value) && typeof value.__tywrap__ === 'string') {
-    return SCIENTIFIC_MARKERS.has(value.__tywrap__) ? value.__tywrap__ : undefined;
-  }
-  return undefined;
-}
-
 function containsScientificEnvelope(value: unknown): boolean {
   const pending: unknown[] = [value];
   while (pending.length > 0) {
@@ -125,7 +109,7 @@ function containsScientificEnvelope(value: unknown): boolean {
       continue;
     }
     if (typeof current.__tywrap__ === 'string') {
-      return SCIENTIFIC_MARKERS.has(current.__tywrap__);
+      return isScientificMarker(current.__tywrap__);
     }
     for (const item of Object.values(current)) {
       pending.push(item);
@@ -669,8 +653,8 @@ export class BridgeCodec {
   }
 
   /**
-   * Async version that applies Arrow decoders.
-   * Use this when the response may contain encoded DataFrames or ndarrays.
+   * Async version that decodes scientific envelopes.
+   * Use this when the response may contain encoded scientific values.
    *
    * @param payload - The JSON string received from Python
    * @returns Decoded and validated result with Arrow decoding applied
@@ -681,25 +665,31 @@ export class BridgeCodec {
   async decodeResponseAsync<T>(payload: string): Promise<T> {
     const result = this.parseResponseResult(payload);
 
-    // Apply Arrow decoding to the result
+    // Decode scientific envelopes in the result.
     let decoded: unknown;
     try {
-      decoded = await decodeArrowValue(result);
+      decoded = await decodeScientificValue(result);
     } catch (err) {
       const errorMessage = err instanceof Error ? err.message : String(err);
-      const marker = scientificMarkerFrom(result, errorMessage);
-      const genuineArrowError =
-        errorMessage.startsWith('Arrow decode failed:') ||
-        errorMessage.startsWith(
-          'Received an Arrow-encoded payload but no Arrow decoder is available.'
-        );
+      const decodeError = err instanceof ScientificDecodeError ? err : undefined;
+      const marker =
+        decodeError?.marker ??
+        (isPlainObject(result) && isScientificMarker(result.__tywrap__)
+          ? result.__tywrap__
+          : 'unknown');
+      const genuineArrowError = decodeError?.kind === 'arrow';
+      const valueType = genuineArrowError
+        ? 'arrow'
+        : marker === 'unknown'
+          ? 'scientific-envelope'
+          : marker;
       throw new BridgeCodecError(
         genuineArrowError
           ? `Arrow decoding failed: ${errorMessage}`
-          : `Scientific envelope decoding failed (${marker ?? 'unknown'}): ${errorMessage}`,
+          : `Scientific envelope decoding failed (${marker}): ${errorMessage}`,
         {
           codecPhase: 'decode',
-          valueType: genuineArrowError ? 'arrow' : (marker ?? 'scientific-envelope'),
+          valueType,
         }
       );
     }

src/runtime/pyodide-bootstrap-core.generated.ts

@@ -1,4 +1,5 @@
 import { BridgeValidationError } from './errors.js';
+import type { ScientificMarker } from '../utils/codec.js';
 
 /**
  * Pure validation functions for runtime value checking.
@@ -36,27 +37,15 @@ export type ReturnSchema =
   | { kind: 'ref'; name: string }
   | {
       kind: 'marker';
-      marker:
-        | 'dataframe'
-        | 'series'
-        | 'ndarray'
-        | 'scipy.sparse'
-        | 'torch.tensor'
-        | 'sklearn.estimator';
+      marker: ScientificMarker;
       dims?: number;
       dtype?: string;
     };
 
 export type ReturnValidator<T = unknown> = (result: T) => T;
 
 export interface DecodedShapeMetadata {
-  marker:
-    | 'dataframe'
-    | 'series'
-    | 'ndarray'
-    | 'scipy.sparse'
-    | 'torch.tensor'
-    | 'sklearn.estimator';
+  marker: ScientificMarker;
   dims?: number;
   dtype?: string;
 }