From fbffe3ff4f7ab2511302a9ec51777a55397f93d9 Mon Sep 17 00:00:00 2001
From: cdc-ap66 <ap66@cdc.gov>
Date: Mon, 8 Dec 2025 18:48:01 +0000
Subject: [PATCH 1/2] resolving schema inference issue

---
 cfa/dataops/catalog.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/cfa/dataops/catalog.py b/cfa/dataops/catalog.py
index dbdf87d..01d4193 100644
--- a/cfa/dataops/catalog.py
+++ b/cfa/dataops/catalog.py
@@ -422,7 +422,10 @@ def get_dataframe(
                 df.reset_index(inplace=True, drop=True)
             else:
                 df = pl.concat(
-                    [pl.read_csv(blob) for blob in blob_files],
+                    [
+                        pl.read_csv(blob, infer_schema_length=None)
+                        for blob in blob_files
+                    ],
                     how="diagonal",
                 )
                 if pl_lazy:
@@ -434,7 +437,11 @@ def get_dataframe(
                 df.reset_index(inplace=True, drop=True)
             else:
                 df = pl.concat(
-                    [pl.read_json(blob) for blob in blob_files],
+                    [
+                        pl.read_json(blob, infer_schema_length=None)
+                        for blob in blob_files
+                    ],
+                    how="diagonal",
                 )
                 if pl_lazy:
                     df = df.lazy()
@@ -448,6 +455,7 @@ def get_dataframe(
             else:
                 df = pl.concat(
                     [pl.read_ndjson(blob) for blob in blob_files],
+                    how="diagonal",
                 )
                 if pl_lazy:
                     df = df.lazy()

From 400fd6bc7b7f37be15d8be3af9b4cc68c2c56142 Mon Sep 17 00:00:00 2001
From: cdc-ap66 <ap66@cdc.gov>
Date: Mon, 8 Dec 2025 21:07:46 +0000
Subject: [PATCH 2/2] version bump

---
 changelog.md   | 6 ++++++
 pyproject.toml | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/changelog.md b/changelog.md
index 49512da..da27e2a 100644
--- a/changelog.md
+++ b/changelog.md
@@ -8,6 +8,12 @@ The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release})
 
 ---
 
+## [2025.12.08.1a]
+
+### Fixed
+
+- Polars dataframe loading+concat from csv/json files that contain varying number of columns (more updates)
+
 ## [2025.12.08.0a]
 
 ### Fixed
diff --git a/pyproject.toml b/pyproject.toml
index c397023..655a455 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "cfa.dataops"
-version = "2025.12.08.0a"
+version = "2025.12.08.1a"
 description = "Data cataloging, ETL, modeling, verification, and validation for CFA"
 authors = [
     { name = "Phil Rogers", email = "ap66@cdc.gov" },