diff --git a/cfa/dataops/catalog.py b/cfa/dataops/catalog.py index dbdf87d..01d4193 100644 --- a/cfa/dataops/catalog.py +++ b/cfa/dataops/catalog.py @@ -422,7 +422,10 @@ def get_dataframe( df.reset_index(inplace=True, drop=True) else: df = pl.concat( - [pl.read_csv(blob) for blob in blob_files], + [ + pl.read_csv(blob, infer_schema_length=None) + for blob in blob_files + ], how="diagonal", ) if pl_lazy: @@ -434,7 +437,11 @@ def get_dataframe( df.reset_index(inplace=True, drop=True) else: df = pl.concat( - [pl.read_json(blob) for blob in blob_files], + [ + pl.read_json(blob, infer_schema_length=None) + for blob in blob_files + ], + how="diagonal", ) if pl_lazy: df = df.lazy() @@ -448,6 +455,7 @@ def get_dataframe( else: df = pl.concat( [pl.read_ndjson(blob) for blob in blob_files], + how="diagonal", ) if pl_lazy: df = df.lazy() diff --git a/changelog.md b/changelog.md index 49512da..da27e2a 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,12 @@ The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release}) --- +## [2025.12.08.1a] + +### Fixed + +- Polars dataframe loading+concat from csv/json files that contain varying number of columns (more updates) + ## [2025.12.08.0a] ### Fixed diff --git a/pyproject.toml b/pyproject.toml index c397023..655a455 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cfa.dataops" -version = "2025.12.08.0a" +version = "2025.12.08.1a" description = "Data cataloging, ETL, modeling, verification, and validation for CFA" authors = [ { name = "Phil Rogers", email = "ap66@cdc.gov" },