Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions cfa/dataops/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,17 +220,20 @@ def write_blob(
self.ledger_entry(action="write")
# print(f"file written to: {full_path}")

def read_blobs(self, version: str = "latest", newest: bool = True) -> list[bytes]:
def read_blobs(
self, version: str = "latest", newest: bool = True, print_version: bool = True
) -> list[bytes]:
"""Read a blob in as bytes so it can be loaded into a dataframe

Args:
path_after_prefix (str): The path to the data (e.g.,
{timestamp}/dataset.csv)
version (str, optional): the version of the data to read.
Defaults to "latest".
newest (bool, optional): whether to get the newest matching version. Defaults to True.
print_version (bool, optional): whether to print the version being used. Defaults to True.
"""
Comment thread
ryanraaschCDC marked this conversation as resolved.
blobs = self._get_version_blobs(version, newest=newest)
blobs = self._get_version_blobs(
version, newest=newest, print_version=print_version
)
blob_bytes = [
read_blob_stream(
blob_url=i["name"],
Expand Down Expand Up @@ -473,8 +476,11 @@ def get_dataframe(
return df
else:
raise ValueError(f"Lazy loading not supported for {file_ext} files.")
blobs = self.read_blobs(version, newest=newest)
blob_bytes = [blob.content_as_bytes() for blob in blobs]
blobs = self.read_blobs(version, newest=newest, print_version=False)
blob_bytes = [
blob if isinstance(blob, bytes) else blob.content_as_bytes()
for blob in blobs
]
blob_files = [BytesIO(pq) for pq in blob_bytes]
if file_ext == "csv":
if output in ["pandas", "pd"]:
Expand Down
6 changes: 6 additions & 0 deletions changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ and this project adheres to [Calendar Versioning](https://calver.org/).
The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release})

---
## [2026.04.27.0]

### Fixed

- In some get_dataframe() calls, the version would be printed twice. This has been fixed.

## [2026.04.24.0]

### Updated
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "cfa.dataops"
version = "2026.04.24.0"
version = "2026.04.27.0"
description = "Data cataloging, ETL, modeling, verification, and validation for CFA"
authors = [
{ name = "Phil Rogers", email = "ap66@cdc.gov" },
Expand Down
Loading