polars lazyframe support added to get_dataframe (#33)

Author: cdc-ap66

cfa/dataops/catalog.py

@@ -301,15 +301,17 @@ def _get_version_blobs(self, version: str = "latest") -> list:
         )
 
     def get_dataframe(
-        self, output="pandas", version="latest"
+        self, output="pandas", version="latest", pl_lazy: bool = False
     ) -> pd.DataFrame | pl.DataFrame:
         """Get the data as a pandas or polars dataframe
 
         Args:
             output (str, optional): the type of dataframe to return,
-            either 'pandas' or 'polars'. Defaults to "pandas".
+                either 'pandas' or 'polars'. Defaults to "pandas".
             version (str, optional): the version of the data to get.
-            Defaults to "latest".
+                Defaults to "latest".
+            pl_lazy (bool, optional): whether to return a lazy polars dataframe.
+                Defaults to False.
 
         Raises:
             ValueError: if output is not 'pandas' or 'polars'
@@ -334,6 +336,8 @@ def get_dataframe(
                     [pl.read_csv(blob) for blob in blob_files],
                     how="vertical_relaxed",
                 )
+                if pl_lazy:
+                    df = df.lazy()
             return df
         elif file_ext == "json":
             if output in ["pandas", "pd"]:
@@ -343,6 +347,8 @@ def get_dataframe(
                 df = pl.concat(
                     [pl.read_json(blob) for blob in blob_files],
                 )
+                if pl_lazy:
+                    df = df.lazy()
             return df
         elif file_ext == "jsonl":
             if output in ["pandas", "pd"]:
@@ -354,6 +360,8 @@ def get_dataframe(
                 df = pl.concat(
                     [pl.read_ndjson(blob) for blob in blob_files],
                 )
+                if pl_lazy:
+                    df = df.lazy()
             return df
         elif file_ext == "parquet" or file_ext == "parq":
             if output in ["pandas", "pd"]:
@@ -366,6 +374,8 @@ def get_dataframe(
                     [pl.read_parquet(pq_file) for pq_file in blob_files],
                     how="vertical_relaxed",
                 )
+                if pl_lazy:
+                    df = df.lazy()
             return df
 
     def ledger_entry(self, action: str) -> None:

changelog.md

@@ -8,6 +8,12 @@ The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release})
 
 ---
 
+## [2025.10.14.0a]
+
+### Added
+
+- Support for getting polars LazyFrames
+
 ## [2025.10.03.0a]
 
 ### Added

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cfa.dataops"
-version = "2025.10.03.0a"
+version = "2025.10.14.0a"
 description = "Data cataloging, ETL, modeling, verification, and validation for CFA"
 authors = [
     { name = "Phil Rogers", email = "ap66@cdc.gov" },