Update kerchunk writer refs_to_dataframe to deal with ArrowStringArray from new ZarrParser deps. (zarr-developers#907)

* adds new zarr parser deps and fix to acccessor

* adds pandas dep to kerchunk writer

Author: norlandrhagen

pyproject.toml

@@ -82,7 +82,9 @@ all_parsers = [
 icechunk = [
     "icechunk>=1.1.2",
 ]
-kerchunk = ["fastparquet"]
+zarr = ["arro3-core", "pyarrow"]
+
+kerchunk = ["fastparquet", "pandas"]
 
 all_writers = [
     "virtualizarr[icechunk]",

virtualizarr/accessor.py

@@ -192,21 +192,27 @@ def to_kerchunk(
 
             return None
         elif format == "parquet":
+            import pandas as pd
             from kerchunk.df import refs_to_dataframe
 
             if isinstance(filepath, Path):
                 url = str(filepath)
             elif isinstance(filepath, str):
                 url = filepath
 
-            # refs_to_dataframe is responsible for writing to parquet.
-            # at no point does it create a full in-memory dataframe.
-            refs_to_dataframe(
-                refs,
-                url=url,
-                record_size=record_size,
-                categorical_threshold=categorical_threshold,
-            )
+            # The zarr-parser performance update PR #892 adds pyarrow and arro3-core as deps.
+            # These break the `kerchunk` refs_to_dataframe behavior.
+            # It seems like pyarrow makes pandas default to an ArrowStringArray
+            # which fastparquet cannot zero-copy encode.
+            # TODO: remove once fastparquet or kerchunk handle ArrowStringArray.
+
+            with pd.option_context("future.infer_string", False):
+                refs_to_dataframe(
+                    refs,
+                    url=url,
+                    record_size=record_size,
+                    categorical_threshold=categorical_threshold,
+                )
             return None
         else:
             raise ValueError(f"Unrecognized output format: {format}")