Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 12 additions & 10 deletions cfa/dataops/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ def write_blob(
container_name=self.container,
append_blob=append,
)
self.ledger_entry(action="write")
# self.ledger_entry(action="write")
# print(f"file written to: {full_path}")

def read_blobs(
Expand All @@ -242,7 +242,7 @@ def read_blobs(
)
for i in blobs
]
self.ledger_entry(action="read")
# self.ledger_entry(action="read")
return blob_bytes

def read_csv(self, suffix: str) -> pd.DataFrame:
Expand All @@ -252,7 +252,7 @@ def read_csv(self, suffix: str) -> pd.DataFrame:
container_name=self.container,
)
df = pd.read_csv(blob)
self.ledger_entry(action="read")
# self.ledger_entry(action="read")
return df

def get_versions(self) -> list:
Expand Down Expand Up @@ -372,8 +372,8 @@ def download_version_to_local(
with open(local_file_path, "wb") as f:
f.write(file_bytes)
written = True
if written:
self.ledger_entry(action="read")
# if written:
# self.ledger_entry(action="read")
return written

@overload
Expand Down Expand Up @@ -446,7 +446,7 @@ def get_dataframe(
credential=ManagedIdentityCredential()
),
)
self.ledger_entry(action="read")
# self.ledger_entry(action="read")
return df
elif file_ext == "csv":
path = str(PurePosixPath(name).parent / f"*.{file_ext}")
Expand All @@ -459,7 +459,7 @@ def get_dataframe(
credential=ManagedIdentityCredential()
),
)
self.ledger_entry(action="read")
##self.ledger_entry(action="read")
return df
elif file_ext == "ndjson" or file_ext == "jsonl":
path = str(PurePosixPath(name).parent / f"*.{file_ext}")
Expand All @@ -472,7 +472,7 @@ def get_dataframe(
credential=ManagedIdentityCredential()
),
)
self.ledger_entry(action="read")
##self.ledger_entry(action="read")
return df
else:
raise ValueError(f"Lazy loading not supported for {file_ext} files.")
Expand Down Expand Up @@ -544,9 +544,11 @@ def ledger_entry(self, action: str) -> None:
"action": action,
}
log_data = (json.dumps(log_entry) + "\n").encode("utf-8")
write_blob_stream( # TODO: make this a streaming write to a single file (one per day parsed from get_timestamp())
ledger_path = f"{self.ledger_location['prefix']}/{get_date()}.jsonl"

write_blob_stream(
data=log_data,
blob_url=f"{self.ledger_location['prefix']}/{get_date()}.jsonl",
blob_url=ledger_path,
account_name=self.ledger_location["account"],
container_name=self.ledger_location["container"],
append_blob=True,
Expand Down
6 changes: 6 additions & 0 deletions changelog.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,12 @@ and this project adheres to [Calendar Versioning](https://calver.org/).
The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release})

---
## [2026.05.12.0]

### Fixed

- removed the ledger reads and write temporarily

## [2026.04.27.0]

### Fixed
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "cfa.dataops"
version = "2026.04.27.0"
version = "2026.05.12.0"
description = "Data cataloging, ETL, modeling, verification, and validation for CFA"
authors = [
{ name = "Phil Rogers", email = "ap66@cdc.gov" },
Expand Down
Loading