Skip to content

Commit 667b1fb

Browse files
cdc-ap66cdc-ap66
authored
Ap66-issue41-Synthetic_Test_Data_Generation_PART_1 (#48)
* update to fix main merge workflow * covid vax trends update * placeholder file * version bump * updated to include timestamp * Update coverage badge
1 parent 78910ee commit 667b1fb

6 files changed

Lines changed: 126 additions & 26 deletions

File tree

‎.github/workflows/run_qaqc.yaml‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,18 @@ jobs:
3030
poetry install --no-root --with dev
3131
poetry run pytest
3232
poetry run coverage-badge -f -o docs/assets/badges/coverage.svg
33+
echo '<!-- timestamp: '"$(date '+%Y-%m-%d %H:%M:%S')"' -->' >> docs/assets/badges/coverage.svg
3334
3435
- name: Check-in Coverage Badge
3536
run: |
36-
git config user.name "${{ github.actor }}"
37-
git config user.email "${{ github.actor }}@users.noreply.github.com"
38-
git add docs/assets/badges/coverage.svg
39-
git commit -m "Update coverage badge"
40-
git push origin HEAD:${{ github.head_ref }}
37+
if [ -z "${{ github.head_ref }}" ];
38+
then
39+
echo "Not a pull request, skipping badge update."
40+
exit 0
41+
else
42+
git config user.name "${{ github.actor }}"
43+
git config user.email "${{ github.actor }}@users.noreply.github.com"
44+
git add docs/assets/badges/coverage.svg
45+
git commit -m "Update coverage badge"
46+
git push origin HEAD:${{ github.head_ref }}
47+
fi

‎cfa/scenarios/dataops/datasets/schemas/covid19vax_trends.py‎

Lines changed: 93 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,104 @@
1+
from typing import Iterable, Optional, Union
2+
3+
import pandas as pd
14
import pandera.pandas as pa
5+
from pandera import dtypes
6+
from pandera.engines import pandas_engine
7+
8+
9+
@pandas_engine.Engine.register_dtype
10+
@dtypes.immutable
11+
class FixedLenArray(pandas_engine.NpString):
12+
def check(
13+
self,
14+
pandera_dtype: dtypes.DataType,
15+
data_container: Optional[pd.Series] = None,
16+
) -> Union[bool, Iterable[bool]]:
17+
# ensure that the data container's data type is a string,
18+
# using the parent class's check implementation
19+
correct_type = super().check(pandera_dtype)
20+
if not correct_type:
21+
return correct_type
22+
23+
# ensure the filepaths actually exist locally
24+
if data_container is None:
25+
return True
26+
else:
27+
length = len(data_container[0])
28+
return data_container.map(lambda x: len(x) == length)
29+
30+
def __str__(self) -> str:
31+
return str(self.__class__.__name__)
32+
33+
def __repr__(self) -> str:
34+
return f"DataType({self})"
35+
236

337
extract_schema = pa.DataFrameSchema(
438
{
5-
"date": pa.Column(str),
6-
"location": pa.Column(str),
7-
"demographic_category": pa.Column(str),
8-
"census": pa.Column(float, nullable=True),
9-
"administered_dose1": pa.Column(float, nullable=True),
10-
"series_complete_yes": pa.Column(float, nullable=True),
11-
"booster_doses": pa.Column(float, nullable=True),
12-
"second_booster": pa.Column(float, nullable=True),
13-
"administered_dose1_pct_agegroup": pa.Column(float, nullable=True),
14-
"series_complete_pop_pct_agegroup": pa.Column(float, nullable=True),
15-
"booster_doses_vax_pct_agegroup": pa.Column(float, nullable=True),
16-
"second_booster_vax_pct_agegroup": pa.Column(float, nullable=True),
39+
"date": pa.Column(
40+
str,
41+
checks=[
42+
pa.Check.str_matches(
43+
r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}$"
44+
)
45+
],
46+
),
47+
"location": pa.Column(
48+
str, checks=[pa.Check.str_matches(r"^[A-Z]{2}$")]
49+
),
50+
"demographic_category": pa.Column(
51+
str, checks=[pa.Check.str_matches(r"^[A-Za-z0-9\_\-]+$")]
52+
),
53+
"census": pa.Column(
54+
int, checks=[pa.Check.in_range(0, 10_000_000_000)], nullable=True
55+
),
56+
"administered_dose1": pa.Column(
57+
int, checks=[pa.Check.in_range(0, 10_000_000_000)], nullable=True
58+
),
59+
"series_complete_yes": pa.Column(
60+
int, checks=[pa.Check.in_range(0, 10_000_000_000)], nullable=True
61+
),
62+
"booster_doses": pa.Column(
63+
int, checks=[pa.Check.in_range(0, 10_000_000_000)], nullable=True
64+
),
65+
"second_booster": pa.Column(
66+
int, checks=[pa.Check.in_range(0, 10_000_000_000)], nullable=True
67+
),
68+
"administered_dose1_pct_agegroup": pa.Column(
69+
float, checks=[pa.Check.in_range(0, 100)], nullable=True
70+
),
71+
"series_complete_pop_pct_agegroup": pa.Column(
72+
float, checks=[pa.Check.in_range(0, 100)], nullable=True
73+
),
74+
"booster_doses_vax_pct_agegroup": pa.Column(
75+
float, checks=[pa.Check.in_range(0, 100)], nullable=True
76+
),
77+
"second_booster_vax_pct_agegroup": pa.Column(
78+
float, checks=[pa.Check.in_range(0, 100)], nullable=True
79+
),
1780
}
1881
)
1982

2083
load_schema = pa.DataFrameSchema(
2184
{
22-
"date": pa.Column(str),
23-
"state": pa.Column(str),
24-
"age": pa.Column(object),
25-
"census": pa.Column(int, coerce=True),
26-
"total": pa.Column(object),
27-
"percentage": pa.Column(object),
28-
"dose": pa.Column(object),
85+
"date": pa.Column(
86+
str,
87+
checks=[
88+
pa.Check.str_matches(
89+
r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}$"
90+
)
91+
],
92+
),
93+
"state": pa.Column(str, checks=[pa.Check.str_matches(r"^[A-Z]{2}$")]),
94+
"age": pa.Column(
95+
str, checks=[pa.Check.isin(["65+", "18-49", "0-17", "50-64"])]
96+
),
97+
"census": pa.Column(
98+
int, checks=[pa.Check.in_range(0, 10_000_000_000)], coerce=True
99+
),
100+
"total": pa.Column(FixedLenArray),
101+
"percentage": pa.Column(FixedLenArray),
102+
"dose": pa.Column(FixedLenArray),
29103
}
30104
)

‎changelog.md‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,20 @@ The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release})
88

99
---
1010

11+
## [2025.06.06.0a]
12+
13+
### Added
14+
- `conftest.py` placeholder
15+
16+
### Updated
17+
- `cfa/scenarios/dataops/datasets/schemas/covid19vax_trends.py` to include checks and custom dtype for array columns
18+
19+
### Fixed
20+
- `.github/workflows/run_qaqc.yaml` to fix error related to coverage badge creation when merging to main
21+
1122
## [2025.06.03.0a]
1223

13-
## Added
24+
### Added
1425
- dataset schemas and validation logic
1526

1627
## [2025.06.02.0a]

‎docs/assets/badges/coverage.svg‎

Lines changed: 1 addition & 0 deletions
Loading

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "cfa.scenarios.dataops"
3-
version = "2025.06.03.0a"
3+
version = "2025.06.06.0a"
44
description = "Data cataloging, ETL, modeling, verification, and validation for the CFA Scenarios Team"
55
authors = [
66
{name = "Phil Rogers", email = "ap66@cdc.gov"}

‎tests/conftest.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
from pytest import fixture
2+
3+
4+
@fixture(scope="session")
5+
def data_dir(tmpdir_factory):
6+
"""Fixture to create a temporary data directory for tests."""
7+
return tmpdir_factory.mktemp("data")

0 commit comments

Comments
 (0)