diff --git a/cfa/dataops/catalog.py b/cfa/dataops/catalog.py index 279752a..04b992c 100644 --- a/cfa/dataops/catalog.py +++ b/cfa/dataops/catalog.py @@ -217,7 +217,7 @@ def write_blob( container_name=self.container, append_blob=append, ) - self.ledger_entry(action="write") + # self.ledger_entry(action="write") # print(f"file written to: {full_path}") def read_blobs( @@ -242,7 +242,7 @@ def read_blobs( ) for i in blobs ] - self.ledger_entry(action="read") + # self.ledger_entry(action="read") return blob_bytes def read_csv(self, suffix: str) -> pd.DataFrame: @@ -252,7 +252,7 @@ def read_csv(self, suffix: str) -> pd.DataFrame: container_name=self.container, ) df = pd.read_csv(blob) - self.ledger_entry(action="read") + # self.ledger_entry(action="read") return df def get_versions(self) -> list: @@ -372,8 +372,8 @@ def download_version_to_local( with open(local_file_path, "wb") as f: f.write(file_bytes) written = True - if written: - self.ledger_entry(action="read") + # if written: + # self.ledger_entry(action="read") return written @overload @@ -446,7 +446,7 @@ def get_dataframe( credential=ManagedIdentityCredential() ), ) - self.ledger_entry(action="read") + # self.ledger_entry(action="read") return df elif file_ext == "csv": path = str(PurePosixPath(name).parent / f"*.{file_ext}") @@ -459,7 +459,7 @@ def get_dataframe( credential=ManagedIdentityCredential() ), ) - self.ledger_entry(action="read") + ##self.ledger_entry(action="read") return df elif file_ext == "ndjson" or file_ext == "jsonl": path = str(PurePosixPath(name).parent / f"*.{file_ext}") @@ -472,7 +472,7 @@ def get_dataframe( credential=ManagedIdentityCredential() ), ) - self.ledger_entry(action="read") + ##self.ledger_entry(action="read") return df else: raise ValueError(f"Lazy loading not supported for {file_ext} files.") @@ -544,9 +544,11 @@ def ledger_entry(self, action: str) -> None: "action": action, } log_data = (json.dumps(log_entry) + "\n").encode("utf-8") - write_blob_stream( # TODO: make this a streaming write to a single file (one per day parsed from get_timestamp()) + ledger_path = f"{self.ledger_location['prefix']}/{get_date()}.jsonl" + + write_blob_stream( data=log_data, - blob_url=f"{self.ledger_location['prefix']}/{get_date()}.jsonl", + blob_url=ledger_path, account_name=self.ledger_location["account"], container_name=self.ledger_location["container"], append_blob=True, diff --git a/changelog.md b/changelog.md index 2445091..71e6999 100644 --- a/changelog.md +++ b/changelog.md @@ -7,6 +7,12 @@ and this project adheres to [Calendar Versioning](https://calver.org/). The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release}) --- +## [2026.05.12.0] + +### Fixed + +- removed the ledger reads and write temporarily + ## [2026.04.27.0] ### Fixed diff --git a/pyproject.toml b/pyproject.toml index afb80c2..5d5705f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "cfa.dataops" -version = "2026.04.27.0" +version = "2026.05.12.0" description = "Data cataloging, ETL, modeling, verification, and validation for CFA" authors = [ { name = "Phil Rogers", email = "ap66@cdc.gov" },