docs: clarify def details

Author: peter-gy

marimo/_runtime/_wasm/_duckdb/__init__.py

@@ -1,18 +1,19 @@
 # Copyright 2026 Marimo. All rights reserved.
-"""WASM-only DuckDB fallbacks for remote file scans.
+"""Install WASM-only DuckDB fallbacks for remote file scans.
 
 DuckDB-WASM cannot use ``httpfs``. marimo fetches supported URLs itself,
 materializes supported files as pandas DataFrames, then hands those frames
-back to DuckDB.
+back to DuckDB through replacement scans.
 
 We have two concrete use cases to patch:
 
 * **Direct read methods** — ``duckdb.read_csv/read_parquet/read_json`` are
-  wrapped with :class:`WasmPatchSet`; supported remote URLs are fetched by
+  wrapped with :class:`WasmPatchSet`. Supported remote URLs are fetched by
   marimo and returned as DuckDB relations.
 * **SQL remote scans** — raw DuckDB APIs and marimo's ``mo.sql`` path call the
-  same rewrite helper, which replaces supported URLs with generated
-  replacement-scan tables backed by fetched pandas DataFrames.
+  same sqlglot rewrite helper. It replaces supported URL scans with generated
+  table names and evaluates the original DuckDB call with fetched DataFrames
+  in scope so DuckDB replacement scans can resolve them.
 """
 
 from __future__ import annotations
@@ -212,15 +213,19 @@ def patch_duckdb_query_for_wasm(
     *,
     reserved_names: Sequence[str] = (),
 ) -> WasmDuckDBQueryPatch | None:
-    """Rewrite remote file sources to generated DataFrame names.
+    """Replace supported remote file reads with generated table names.
 
-    Example: ``SELECT * FROM read_csv('https://example.com/cars.csv')``
-    becomes ``SELECT * FROM __marimo_wasm_duckdb_remote_0``, and
-    ``tables["__marimo_wasm_duckdb_remote_0"]`` holds the fetched DataFrame.
+    For example, ``SELECT * FROM read_csv('https://example.com/cars.csv')``
+    becomes ``SELECT * FROM __marimo_wasm_duckdb_remote_0`` when suffix ``0``
+    is free. The returned ``tables`` mapping binds that name to the fetched
+    DataFrame. If the query or ``reserved_names`` already use that identifier,
+    the rewriter uses the next free suffix.
 
-    In Pyodide this raises if sqlglot is unavailable. Returns ``None`` when
-    not running in Pyodide, when the query has no supported remote file
-    source, or when the query cannot be parsed.
+    In Pyodide this raises if sqlglot is unavailable. Returns ``None`` when:
+
+    - marimo is not running in Pyodide;
+    - the query has no supported remote file source;
+    - the query cannot be parsed.
     """
     if not is_pyodide():
         return None
@@ -419,6 +424,8 @@ def _make_sql_api_wrapper(
     query_arg_index: int,
     query_kwarg_names: tuple[str, ...],
 ) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
+    """Wrap DuckDB SQL APIs while preserving the caller's local namespace."""
+
     def _make_wrapper(original: Callable[..., Any]) -> Callable[..., Any]:
         @functools.wraps(original)
         def _wrapper(*args: Any, **kwargs: Any) -> Any:
@@ -451,6 +458,7 @@ def _query_argument(
     query_arg_index: int,
     query_kwarg_names: tuple[str, ...],
 ) -> Any:
+    """Find the query text without assuming the caller used positional args."""
     if len(args) > query_arg_index:
         return args[query_arg_index]
     for name in query_kwarg_names:
@@ -467,6 +475,7 @@ def _replace_query_argument(
     query_arg_index: int,
     query_kwarg_names: tuple[str, ...],
 ) -> tuple[tuple[Any, ...], dict[str, Any]]:
+    """Replace the query at the same call site shape used by the caller."""
     kwargs_dict = dict(kwargs)
     if len(args) > query_arg_index:
         patched_args = list(args)
@@ -490,6 +499,7 @@ def _eval_duckdb_original_call(
     binding_names: _EvalBindingNames,
     extra_locals: Mapping[str, Any] | None = None,
 ) -> Any:
+    """Evaluate the original call where DuckDB can see replacement DataFrames."""
     locals_for_eval = dict(eval_locals)
     if extra_locals is not None:
         locals_for_eval.update(extra_locals)
@@ -551,6 +561,7 @@ def _duckdb_catalog_names(
     original: Callable[..., Any],
     args: tuple[Any, ...],
 ) -> tuple[str, ...]:
+    """Reserve existing DuckDB table names before generating replacements."""
     try:
         relation = _show_duckdb_tables(original, args)
         rows = relation.fetchall()
@@ -563,6 +574,7 @@ def _show_duckdb_tables(
     original: Callable[..., Any],
     args: tuple[Any, ...],
 ) -> Any:
+    """Run ``SHOW TABLES`` through the same DuckDB entry point being patched."""
     import duckdb
 
     original_call = inspect.unwrap(original)
@@ -614,6 +626,7 @@ def _direct_reader_source(
     *,
     call_spec: _DirectReaderCallSpec,
 ) -> tuple[RemoteFileSource, Any] | None:
+    """Return a remote source only for direct reader calls we can emulate."""
     options = dict(kwargs)
     try:
         source, rest_args = _pop_source_argument(
@@ -646,6 +659,7 @@ def _pop_source_argument(
     *,
     call_spec: _DirectReaderCallSpec,
 ) -> tuple[Any, tuple[Any, ...]]:
+    """Remove the source argument so remaining kwargs are pure reader options."""
     source_positional_index = call_spec.source_positional_index
     if len(args) > source_positional_index:
         return (
@@ -688,6 +702,7 @@ def _replace_remote_sources(
     statements: Sequence[exp.Expression],
     table_names: _RemoteTableNames,
 ) -> list[exp.Expression]:
+    """Replace supported remote table nodes while preserving aliases."""
     from sqlglot import exp
 
     def replace_table(node: exp.Expression) -> exp.Expression:
@@ -715,6 +730,7 @@ def replace_table(node: exp.Expression) -> exp.Expression:
 def _reserved_sql_names(
     statements: Sequence[exp.Expression],
 ) -> tuple[str, ...]:
+    """Collect SQL identifiers that generated table names must not shadow."""
     from sqlglot import exp
 
     names: set[str] = set()
@@ -731,6 +747,7 @@ def _reserved_sql_names(
 def _format_duckdb_query(
     statements: Sequence[exp.Expression], *, original_query: str
 ) -> str:
+    """Serialize sqlglot statements without dropping a trailing semicolon."""
     patched_query = "; ".join(
         statement.sql(dialect="duckdb") for statement in statements
     )

marimo/_runtime/_wasm/_duckdb/dataframe.py

@@ -1,5 +1,11 @@
 # Copyright 2026 Marimo. All rights reserved.
-"""Decode fetched DuckDB file bytes into pandas DataFrames."""
+"""Decode fetched DuckDB file bytes into pandas DataFrames.
+
+The WASM patch fetches remote bytes in Python, but DuckDB's Python readers
+still expect local paths for CSV, JSON, and parquet parsing. This module
+materializes fetched bytes through short-lived temp files where DuckDB parsing
+is needed and synthesizes DataFrames for ``read_text`` and ``read_blob``.
+"""
 
 from __future__ import annotations
 
@@ -29,6 +35,7 @@
 def read_csv_dataframe(
     data: bytes, options: Mapping[str, Any], *, url: str
 ) -> pd.DataFrame:
+    """Read CSV/TSV bytes with a suffix that preserves compression hints."""
     import duckdb
 
     return _read_temp_dataframe(
@@ -71,6 +78,7 @@ def read_json_dataframe(
 def read_json_objects_dataframe(
     data: bytes, options: Mapping[str, Any], *, url: str
 ) -> pd.DataFrame:
+    """Read JSON-object bytes through DuckDB's SQL-only table function."""
     return _read_temp_dataframe(
         data,
         suffix=_temp_suffix(
@@ -99,6 +107,7 @@ def _read_json_objects_path(
 
 
 def read_text_dataframe(data: bytes, url: str) -> pd.DataFrame:
+    """Match DuckDB's ``read_text`` shape for an already-fetched object."""
     import pandas as pd
 
     return pd.DataFrame(
@@ -112,6 +121,7 @@ def read_text_dataframe(data: bytes, url: str) -> pd.DataFrame:
 
 
 def read_blob_dataframe(data: bytes, url: str) -> pd.DataFrame:
+    """Match DuckDB's ``read_blob`` shape for an already-fetched object."""
     import pandas as pd
 
     return pd.DataFrame(
@@ -127,6 +137,7 @@ def read_blob_dataframe(data: bytes, url: str) -> pd.DataFrame:
 def append_filename_column(
     df: pd.DataFrame, url: str, column_name: str
 ) -> pd.DataFrame:
+    """Apply DuckDB's filename option after bytes have been decoded."""
     if column_name in df.columns:
         raise ValueError(
             f'Option filename adds column "{column_name}", but a column with this '
@@ -158,6 +169,7 @@ def _read_temp_dataframe(
 
 
 def _temp_suffix(url: str, *, suffixes: tuple[str, ...], default: str) -> str:
+    """Preserve file extensions so DuckDB can infer format details."""
     path = urlparse(url).path.lower()
     for suffix in suffixes:
         if path.endswith(suffix):

marimo/_runtime/_wasm/_duckdb/io.py

@@ -1,5 +1,13 @@
 # Copyright 2026 Marimo. All rights reserved.
-"""Map DuckDB remote files to readers."""
+"""Resolve DuckDB remote file reads into fetch and DataFrame reader steps.
+
+DuckDB's native network scanner is unavailable in Pyodide. This module is the
+compatibility layer that recognizes supported URL shapes, validates the reader
+options we can reproduce, fetches bytes through WASM fetch shim, and
+dispatches to a local DataFrame reader. Unsupported readers or option
+combinations return ``None`` so callers can use the unpatched DuckDB path
+and surface the underlying error.
+"""
 
 from __future__ import annotations
 
@@ -40,6 +48,7 @@ class RemoteFile:
     fetcher_name: str
 
     def fetch(self) -> _FetchedBytes:
+        """Fetch through a named strategy so sources stay hashable."""
         return _fetcher_by_name(self.fetcher_name).fetch(self.url)
 
 
@@ -265,9 +274,11 @@ class RemoteFileSource:
     options: tuple[tuple[str, Any], ...] = ()
 
     def read_options(self) -> dict[str, Any]:
+        """Expose sorted, hashable options as regular reader kwargs."""
         return dict(self.options)
 
     def read_dataframe(self) -> pd.DataFrame:
+        """Read one or more remote files using DuckDB-compatible concat rules."""
         frames = [self._read_file_dataframe(file) for file in self.files]
         if len(frames) == 1:
             return frames[0]
@@ -287,6 +298,7 @@ def read_dataframe(self) -> pd.DataFrame:
         return pd.concat(frames, ignore_index=True)
 
     def _read_file_dataframe(self, file: RemoteFile) -> pd.DataFrame:
+        """Fetch bytes, decode them, then apply DuckDB's filename option."""
         fetched = file.fetch()
         options = self.read_options()
         reader = _reader_by_name(self.reader_name)
@@ -300,6 +312,7 @@ def _read_file_dataframe(self, file: RemoteFile) -> pd.DataFrame:
 
 
 def remote_file_from_url(url: str) -> RemoteFile | None:
+    """Return a fetchable remote file only for URL schemes marimo supports."""
     fetcher = _fetcher_for_url(url)
     if fetcher is None:
         return None
@@ -311,6 +324,7 @@ def remote_file_source_from_reader_args(
     source: Any,
     raw_options: Mapping[str, Any],
 ) -> RemoteFileSource | None:
+    """Map a DuckDB reader call to a reproducible remote DataFrame source."""
     reader = reader_for_function(function_name)
     if reader is None:
         return None
@@ -328,6 +342,7 @@ def remote_file_source_from_reader_args(
 def _remote_files_from_source_arg(
     source: Any,
 ) -> tuple[RemoteFile, ...] | None:
+    """Accept DuckDB source shapes that are static URL strings or URL lists."""
     if isinstance(source, str):
         file = remote_file_from_url(source)
         return (file,) if file is not None else None
@@ -372,6 +387,7 @@ def _reader_by_name(
 
 
 def reader_for_url(url: str) -> _DataFrameReader | None:
+    """Infer a reader from direct URL table syntax such as ``FROM 'x.csv'``."""
     path = urlparse(url).path.lower()
     return next(
         (
@@ -386,6 +402,7 @@ def reader_for_url(url: str) -> _DataFrameReader | None:
 def reader_for_function(
     function_name: str,
 ) -> _DataFrameReader | None:
+    """Resolve DuckDB table-function names to marimo's fallback readers."""
     return next(
         (
             reader
@@ -397,6 +414,7 @@ def reader_for_function(
 
 
 def _csv_reader_options(options: Mapping[str, Any]) -> dict[str, Any]:
+    """Drop options implemented outside DuckDB's CSV reader call."""
     return {
         key: value
         for key, value in options.items()
@@ -405,6 +423,7 @@ def _csv_reader_options(options: Mapping[str, Any]) -> dict[str, Any]:
 
 
 def _json_reader_options(options: Mapping[str, Any]) -> dict[str, Any]:
+    """Translate DuckDB JSON option spelling to DuckDB Python API spelling."""
     return {
         key: _normalize_json_reader_option(key, value)
         for key, value in options.items()
@@ -413,12 +432,14 @@ def _json_reader_options(options: Mapping[str, Any]) -> dict[str, Any]:
 
 
 def _normalize_delimiter(value: Any) -> Any:
+    """Convert SQL's escaped tab literal to the byte delimiter DuckDB expects."""
     if value == r"\t":
         return "\t"
     return value
 
 
 def _normalize_json_reader_option(key: str, value: Any) -> Any:
+    """Normalize JSON values whose SQL names differ from Python reader values."""
     if key == "compression":
         return _normalize_json_compression(value)
     if key == "format":
@@ -427,6 +448,7 @@ def _normalize_json_reader_option(key: str, value: Any) -> Any:
 
 
 def _normalize_json_compression(value: Any) -> str:
+    """Map SQL compression aliases to DuckDB Python JSON reader values."""
     compression = str(value).lower()
     if compression == "auto":
         return "auto_detect"
@@ -436,6 +458,7 @@ def _normalize_json_compression(value: Any) -> str:
 
 
 def _normalize_json_format(value: Any) -> str:
+    """Map DuckDB SQL JSON format aliases to Python reader values."""
     fmt = str(value).lower()
     if fmt == "ndjson":
         return "newline_delimited"
@@ -445,6 +468,7 @@ def _normalize_json_format(value: Any) -> str:
 
 
 def _apply_json_option(options: dict[str, Any], key: str, value: Any) -> bool:
+    """Keep JSON options only when the fallback can safely pass them through."""
     if key == "format":
         options["format"] = _normalize_json_format(value)
         return True
@@ -457,12 +481,14 @@ def _apply_json_option(options: dict[str, Any], key: str, value: Any) -> bool:
 
 
 def _is_safe_reader_option_name(key: str) -> bool:
+    """Reject option names that cannot be passed as Python reader kwargs."""
     return key.isidentifier()
 
 
 def _apply_common_table_option(
     options: dict[str, Any], key: str, value: Any
 ) -> bool:
+    """Handle options marimo applies after per-file reads are decoded."""
     if key == "filename" and isinstance(value, bool | str):
         options["filename"] = value
         return True
@@ -475,6 +501,7 @@ def _apply_common_table_option(
 def _apply_compression_option(
     options: dict[str, Any], key: str, value: Any
 ) -> bool:
+    """Accept only compression modes supported by the byte-fetch fallback."""
     if key == "compression" and _is_supported_compression(value):
         options["compression"] = str(value).lower()
         return True
@@ -484,10 +511,12 @@ def _apply_compression_option(
 def _apply_shared_source_option(
     options: dict[str, Any], key: str, value: Any
 ) -> bool:
+    """Apply source options shared by CSV, parquet, and JSON fallbacks."""
     return _apply_compression_option(
         options, key, value
     ) or _apply_common_table_option(options, key, value)
 
 
 def _is_supported_compression(value: Any) -> bool:
+    """Limit compression to modes the fallback knows how to preserve."""
     return str(value).lower() in {"auto", "none", "gzip"}