Ap66-issue42-mocking_blob_access (#50)

* updates for testing

* more catalog testing

* new testing sub-module

* adding first dataset testing

* catalog functionality tested

* fixing path and adding doctests

* doctest for simple stuff

* version pump

* heading fix

* Update coverage badge

---------

Co-authored-by: cdc-ap66 <cdc-ap66@users.noreply.github.com>

Author: cdc-ap66

cfa/scenarios/dataops/datasets/catalog.py

@@ -260,4 +260,16 @@ def get_data(
 
 
 def list_datasets() -> list[str]:
+    """
+    Lists all available datasets in the catalog
+
+    Returns:
+        list[str]: list of dataset names
+
+    Examples:
+        >>> datasets = list_datasets()
+        >>> 'covid19vax_trends' in datasets
+        True
+
+    """
     return [x for x in datasets.__dict__.keys()]

cfa/scenarios/dataops/datasets/schemas/covid19vax_trends.py

@@ -46,30 +46,42 @@ def __repr__(self) -> str:
             str,
             checks=[
                 pa.Check.str_matches(
-                    r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}$"
+                    r"^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})?$"
                 )
             ],
         ),
         "location": pa.Column(
             str, checks=[pa.Check.str_matches(r"^[A-Z]{2}$")]
         ),
         "demographic_category": pa.Column(
-            str, checks=[pa.Check.str_matches(r"^[A-Za-z0-9\_\-]+$")]
+            str, checks=[pa.Check.str_matches(r"^[A-Za-z0-9\_\-\+\<\>]+$")]
         ),
         "census": pa.Column(
             int, checks=[pa.Check.in_range(0, 10_000_000_000)], nullable=True
         ),
         "administered_dose1": pa.Column(
-            int, checks=[pa.Check.in_range(0, 10_000_000_000)], nullable=True
+            int,
+            checks=[pa.Check.in_range(0, 10_000_000_000)],
+            nullable=True,
+            coerce=True,
         ),
         "series_complete_yes": pa.Column(
-            int, checks=[pa.Check.in_range(0, 10_000_000_000)], nullable=True
+            int,
+            checks=[pa.Check.in_range(0, 10_000_000_000)],
+            nullable=True,
+            coerce=True,
         ),
         "booster_doses": pa.Column(
-            int, checks=[pa.Check.in_range(0, 10_000_000_000)], nullable=True
+            int,
+            checks=[pa.Check.in_range(0, 10_000_000_000)],
+            nullable=True,
+            coerce=True,
         ),
         "second_booster": pa.Column(
-            int, checks=[pa.Check.in_range(0, 10_000_000_000)], nullable=True
+            int,
+            checks=[pa.Check.in_range(0, 10_000_000_000)],
+            nullable=True,
+            coerce=True,
         ),
         "administered_dose1_pct_agegroup": pa.Column(
             float, checks=[pa.Check.in_range(0, 100)], nullable=True
@@ -92,7 +104,7 @@ def __repr__(self) -> str:
             str,
             checks=[
                 pa.Check.str_matches(
-                    r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}$"
+                    r"^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3})?$"
                 )
             ],
         ),

changelog.md

@@ -8,6 +8,18 @@ The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release})
 
 ---
 
+## [2025.06.13.0a]
+
+### Added
+
+- blob mocking and testing using synthetic data to increase coverage of dataset catalog
+- tests for covid19_vax_trends dataset and transform that can be duplicated for other tests
+
+### Updated
+
+- `cfa/scenarios/dataops/datasets/schemas/covid19vax_trends.py` to fix some errors with schema validation in synthetic data
+- `tests/conftest.py` to include blob write function mocking fixture so it can be reused
+
 ## [2025.06.10.0a]
 
 ### Added

docs/assets/badges/coverage.svg

@@ -1,6 +1,6 @@
 [project]
 name = "cfa.scenarios.dataops"
-version = "2025.06.10.0a"
+version = "2025.06.13.0a"
 description = "Data cataloging, ETL, modeling, verification, and validation for the CFA Scenarios Team"
 authors = [
     {name = "Phil Rogers", email = "ap66@cdc.gov"}
@@ -48,4 +48,4 @@ build-backend = "poetry.core.masonry.api"
 state_report_pdf = "cfa.scenarios.dataops.workflows.post_processing.state_report_pdf:main"
 
 [tool.pytest.ini_options]
-addopts = "-vv --cov=cfa.scenarios.dataops --cov-report html --cov-report term"
+addopts = "--doctest-modules -vv --cov=cfa --cov-report html --cov-report term"

pyproject.toml

@@ -5,3 +5,16 @@
 def data_dir(tmpdir_factory):
     """Fixture to create a temporary data directory for tests."""
     return tmpdir_factory.mktemp("data")
+
+
+@fixture(scope="session")
+def mock_write_blob_stream():
+    def mock_write_blob_stream(
+        data,
+        blob_url: str,
+        account_name: str,
+        container_name: str,
+    ) -> None:
+        return None
+
+    return mock_write_blob_stream

tests/conftest.py

@@ -0,0 +1,45 @@
+import pandas as pd
+import pandera.pandas as pa
+import pytest
+from pandera.errors import SchemaError
+
+from cfa.scenarios.dataops.datasets.schemas.covid19vax_trends import (
+    extract_schema,
+    load_schema,
+    raw_synth_data,
+    tf_synth_data,
+)
+from cfa.scenarios.dataops.etl.covid19vax_trends import transform
+
+
+def test_covid19_vax_trends_schemas():
+    assert isinstance(extract_schema, pa.DataFrameSchema)
+    assert isinstance(load_schema, pa.DataFrameSchema)
+
+    # Check if the schemas have the expected columns
+    assert isinstance(extract_schema(raw_synth_data), pd.DataFrame)
+    assert isinstance(load_schema(tf_synth_data), pd.DataFrame)
+
+    with pytest.raises(SchemaError):
+        ex_cols = raw_synth_data.columns
+        extract_schema(
+            raw_synth_data.rename(
+                columns={col: col + "_renamed" for col in ex_cols}
+            )
+        )
+
+    with pytest.raises(SchemaError):
+        tf_cols = tf_synth_data.columns
+        load_schema(
+            tf_synth_data.rename(
+                columns={col: col + "_renamed" for col in tf_cols}
+            )
+        )
+
+
+def test_covid19_vax_trends_transform():
+    extract_df = raw_synth_data
+    tf_df = transform(extract_df)
+
+    # Check if the transformed DataFrame matches the expected schema
+    assert isinstance(load_schema(tf_df), pd.DataFrame)