diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1bf396d..63882f9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -64,3 +64,9 @@ repos: stages: [pre-commit] language: system entry: jupyter nbconvert --ClearOutputPreprocessor.enabled=True --inplace + - id: remove-init-file-breaking-ns-packaging + name: Remove __init__.py files from ./cfa/ + files: ^cfa/.*__init__\.py$ + stages: [pre-commit] + language: system + entry: bash -c 'find ./cfa -maxdepth 1 -name "__init__.py" -delete' diff --git a/cfa/dataops/catalog.py b/cfa/dataops/catalog.py index 538ccf1..dbdf87d 100644 --- a/cfa/dataops/catalog.py +++ b/cfa/dataops/catalog.py @@ -423,7 +423,7 @@ def get_dataframe( else: df = pl.concat( [pl.read_csv(blob) for blob in blob_files], - how="vertical_relaxed", + how="diagonal", ) if pl_lazy: df = df.lazy() @@ -461,7 +461,7 @@ def get_dataframe( else: df = pl.concat( [pl.read_parquet(pq_file) for pq_file in blob_files], - how="vertical_relaxed", + how="diagonal", ) if pl_lazy: df = df.lazy() diff --git a/cfa/dataops/create_catalog/repo_templates/pyproject.toml.mako b/cfa/dataops/create_catalog/repo_templates/pyproject.toml.mako index 0d85906..f8bb0fa 100644 --- a/cfa/dataops/create_catalog/repo_templates/pyproject.toml.mako +++ b/cfa/dataops/create_catalog/repo_templates/pyproject.toml.mako @@ -23,7 +23,8 @@ dependencies = [ ] [tool.setuptools.packages.find] -include = ["${catalog_namespace}*"] +namespaces = true +include = ["cfa*"] [project.optional-dependencies] dev = [ diff --git a/changelog.md b/changelog.md index de25d00..49512da 100644 --- a/changelog.md +++ b/changelog.md @@ -8,6 +8,13 @@ The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release}) --- +## [2025.12.08.0a] + +### Fixed + +- Polars dataframe loading+concat from csv/json files that contain varying number of columns +- Namespace package configuration for created catalogs + ## [2025.11.18.0a] ### Updated diff --git a/pyproject.toml b/pyproject.toml index f4b9ccd..c397023 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cfa.dataops" -version = "2025.11.18.0a" +version = "2025.12.08.0a" description = "Data cataloging, ETL, modeling, verification, and validation for CFA" authors = [ { name = "Phil Rogers", email = "ap66@cdc.gov" },