ap66-fix namespace and pl data loading (#46)

* namespace needs to be higher level if dataops is installed side-by-side

* ran into issue when each json/csv has different number of columns

* version bump

* adding pre-commit to make sure ns package doesn't break

Author: cdc-ap66

.pre-commit-config.yaml

@@ -64,3 +64,9 @@ repos:
       stages: [pre-commit]
       language: system
       entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace
+    - id: remove-init-file-breaking-ns-packaging
+      name: Remove __init__.py files from ./cfa/
+      files: ^cfa/.*__init__\.py$
+      stages: [pre-commit]
+      language: system
+      entry: bash -c 'find ./cfa -maxdepth 1 -name "__init__.py" -delete'

cfa/dataops/catalog.py

@@ -423,7 +423,7 @@ def get_dataframe(
             else:
                 df = pl.concat(
                     [pl.read_csv(blob) for blob in blob_files],
-                    how="vertical_relaxed",
+                    how="diagonal",
                 )
                 if pl_lazy:
                     df = df.lazy()
@@ -461,7 +461,7 @@ def get_dataframe(
             else:
                 df = pl.concat(
                     [pl.read_parquet(pq_file) for pq_file in blob_files],
-                    how="vertical_relaxed",
+                    how="diagonal",
                 )
                 if pl_lazy:
                     df = df.lazy()

cfa/dataops/create_catalog/repo_templates/pyproject.toml.mako

@@ -23,7 +23,8 @@ dependencies = [
 ]
 
 [tool.setuptools.packages.find]
-include = ["${catalog_namespace}*"]
+namespaces = true
+include = ["cfa*"]
 
 [project.optional-dependencies]
 dev = [

changelog.md

@@ -8,6 +8,13 @@ The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release})
 
 ---
 
+## [2025.12.08.0a]
+
+### Fixed
+
+- Polars dataframe loading+concat from csv/json files that contain varying number of columns
+- Namespace package configuration for created catalogs
+
 ## [2025.11.18.0a]
 
 ### Updated

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cfa.dataops"
-version = "2025.11.18.0a"
+version = "2025.12.08.0a"
 description = "Data cataloging, ETL, modeling, verification, and validation for CFA"
 authors = [
     { name = "Phil Rogers", email = "ap66@cdc.gov" },