Skip to content

Commit 19e8b60

Browse files
ryanraaschCDCryanraaschCDC
authored
Merge pull request #51 from cdcent/44-test-utils-and-workflows
44 test utils and workflows
2 parents b753744 + 6facd22 commit 19e8b60

26 files changed

Lines changed: 3500 additions & 2446 deletions

‎.github/workflows/run_qaqc.yaml‎

Lines changed: 47 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -1,47 +1,47 @@
1-
name: qaqc
2-
3-
on:
4-
pull_request:
5-
push:
6-
branches: [main]
7-
8-
jobs:
9-
TestAndCoverageBadgeUpdate:
10-
name: Test and Coverage Badge Update
11-
runs-on: ubuntu-latest
12-
13-
steps:
14-
- name: Checkout Code
15-
uses: actions/checkout@v4
16-
with:
17-
fetch-depth: 0
18-
19-
- name: Setup Python
20-
uses: actions/setup-python@v5
21-
with:
22-
python-version: '3.10'
23-
24-
- name: Install poetry
25-
run: |
26-
pip install poetry>=2.0
27-
28-
- name: Test
29-
run: |
30-
poetry install --no-root --with dev
31-
poetry run pytest
32-
poetry run coverage-badge -f -o docs/assets/badges/coverage.svg
33-
echo '<!-- timestamp: '"$(date '+%Y-%m-%d %H:%M:%S')"' -->' >> docs/assets/badges/coverage.svg
34-
35-
- name: Check-in Coverage Badge
36-
run: |
37-
if [ -z "${{ github.head_ref }}" ];
38-
then
39-
echo "Not a pull request, skipping badge update."
40-
exit 0
41-
else
42-
git config user.name "${{ github.actor }}"
43-
git config user.email "${{ github.actor }}@users.noreply.github.com"
44-
git add docs/assets/badges/coverage.svg
45-
git commit -m "Update coverage badge"
46-
git push origin HEAD:${{ github.head_ref }}
47-
fi
1+
name: qaqc
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches: [main]
7+
8+
jobs:
9+
TestAndCoverageBadgeUpdate:
10+
name: Test and Coverage Badge Update
11+
runs-on: ubuntu-latest
12+
13+
steps:
14+
- name: Checkout Code
15+
uses: actions/checkout@v4
16+
with:
17+
fetch-depth: 0
18+
19+
- name: Setup Python
20+
uses: actions/setup-python@v5
21+
with:
22+
python-version: '3.10'
23+
24+
- name: Install poetry
25+
run: |
26+
pip install poetry>=2.0
27+
28+
- name: Test
29+
run: |
30+
poetry install --no-root --with dev
31+
poetry run pytest
32+
poetry run coverage-badge -f -o docs/assets/badges/coverage.svg
33+
echo '<!-- timestamp: '"$(date '+%Y-%m-%d %H:%M:%S')"' -->' >> docs/assets/badges/coverage.svg
34+
35+
- name: Check-in Coverage Badge
36+
run: |
37+
if [ -z "${{ github.head_ref }}" ];
38+
then
39+
echo "Not a pull request, skipping badge update."
40+
exit 0
41+
else
42+
git config user.name "${{ github.actor }}"
43+
git config user.email "${{ github.actor }}@users.noreply.github.com"
44+
git add docs/assets/badges/coverage.svg
45+
git commit -m "Update coverage badge"
46+
git push origin HEAD:${{ github.head_ref }}
47+
fi

‎cfa/scenarios/dataops/datasets/hospitalization.toml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# This dataset is static, and thus only needs to be extracted in it's raw form
2+
# once and saved to blob storage. For a human viewable we format, see:
3+
# https://data.cdc.gov/resource/aemt-mg7g.csv
4+
# Raw (extracted) and transformed (loaded) data will be stored in azure blob storage
5+
# which requires the account, container and path for access
6+
7+
[properties]
8+
9+
name = "hospitalization"
10+
automate = false
11+
transform_template = "hospitalization.sql"
12+
schema = "hospitalization.py"
13+
14+
[source]
15+
16+
url = "https://data.cdc.gov/resource/aemt-mg7g.csv"
17+
pagination = {limit = 1000}
18+
19+
[extract]
20+
21+
account = "cfadatalakeprd"
22+
container = "cfapredict"
23+
prefix = "dataops/scenarios/raw/hospitalization"
24+
25+
[load]
26+
27+
account = "cfadatalakeprd"
28+
container = "cfapredict"
29+
prefix = "dataops/scenarios/transformed/hospitalization"
30+
31+
# TODO: add some data schema validation

‎cfa/scenarios/dataops/datasets/schemas/hospitalization.py‎

Lines changed: 257 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,257 @@
1+
import random
2+
3+
import pandas as pd
4+
import pandera.pandas as pa
5+
from faker import Faker
6+
7+
fake = Faker()
8+
df_len = 1000
9+
10+
extract_schema = pa.DataFrameSchema(
11+
{
12+
"week_end_date": pa.Column(str),
13+
"jurisdiction": pa.Column(str),
14+
"weekly_actual_days_reporting_any_data": pa.Column(
15+
float, nullable=True, coerce=True
16+
),
17+
"weekly_percent_days_reporting_any_data": pa.Column(
18+
float, nullable=True, coerce=True
19+
),
20+
"num_hospitals_previous_day_admission_adult_covid_confirmed": pa.Column(
21+
"int", nullable=True
22+
),
23+
"num_hospitals_previous_day_admission_pediatric_covid_confirmed": pa.Column(
24+
"int", nullable=True
25+
),
26+
"num_hospitals_previous_day_admission_influenza_confirmed": pa.Column(
27+
"int", nullable=True
28+
),
29+
"num_hospitals_total_patients_hospitalized_confirmed_influenza": pa.Column(
30+
"int", nullable=True
31+
),
32+
"num_hospitals_icu_patients_confirmed_influenza": pa.Column(
33+
"int", nullable=True
34+
),
35+
"num_hospitals_inpatient_beds": pa.Column("int", nullable=True),
36+
"num_hospitals_total_icu_beds": pa.Column("int", nullable=True),
37+
"num_hospitals_inpatient_beds_used": pa.Column("int", nullable=True),
38+
"num_hospitals_icu_beds_used": pa.Column("int", nullable=True),
39+
"num_hospitals_percent_inpatient_beds_occupied": pa.Column(
40+
"int", nullable=True
41+
),
42+
"num_hospitals_percent_staff_icu_beds_occupied": pa.Column(
43+
"int", nullable=True
44+
),
45+
"num_hospitals_percent_inpatient_beds_covid": pa.Column(
46+
"int", nullable=True
47+
),
48+
"num_hospitals_percent_inpatient_beds_influenza": pa.Column(
49+
"int", nullable=True
50+
),
51+
"num_hospitals_percent_staff_icu_beds_covid": pa.Column(
52+
"int", nullable=True
53+
),
54+
"num_hospitals_percent_icu_beds_influenza": pa.Column(
55+
"int", nullable=True
56+
),
57+
"num_hospitals_admissions_all_covid_confirmed": pa.Column(
58+
"int", nullable=True
59+
),
60+
"num_hospitals_total_patients_hospitalized_covid_confirmed": pa.Column(
61+
"int", nullable=True
62+
),
63+
"num_hospitals_staff_icu_patients_covid_confirmed": pa.Column(
64+
"int", nullable=True
65+
),
66+
"avg_admissions_adult_covid_confirmed": pa.Column(
67+
"float", nullable=True
68+
),
69+
"total_admissions_adult_covid_confirmed": pa.Column(
70+
"float", nullable=True
71+
),
72+
"avg_admissions_pediatric_covid_confirmed": pa.Column(
73+
"float", nullable=True
74+
),
75+
"total_admissions_pediatric_covid_confirmed": pa.Column(
76+
"float", nullable=True
77+
),
78+
"avg_admissions_all_covid_confirmed": pa.Column(
79+
"float", nullable=True
80+
),
81+
"total_admissions_all_covid_confirmed": pa.Column(
82+
"float", nullable=True
83+
),
84+
"avg_admissions_all_influenza_confirmed": pa.Column(
85+
"float", nullable=True
86+
),
87+
"total_admissions_all_influenza_confirmed": pa.Column(
88+
"float", nullable=True
89+
),
90+
"avg_total_patients_hospitalized_covid_confirmed": pa.Column(
91+
"float", nullable=True
92+
),
93+
"avg_total_patients_hospitalized_influenza_confirmed": pa.Column(
94+
"float", nullable=True
95+
),
96+
"avg_staff_icu_patients_covid_confirmed": pa.Column(
97+
"float", nullable=True
98+
),
99+
"avg_icu_patients_influenza_confirmed": pa.Column(
100+
"float", nullable=True
101+
),
102+
"avg_inpatient_beds": pa.Column("float", nullable=True),
103+
"avg_total_icu_beds": pa.Column("float", nullable=True),
104+
"avg_inpatient_beds_used": pa.Column("float", nullable=True),
105+
"avg_icu_beds_used": pa.Column("float", nullable=True),
106+
"avg_percent_inpatient_beds_occupied": pa.Column(
107+
"float", nullable=True
108+
),
109+
"avg_percent_staff_icu_beds_occupied": pa.Column(
110+
"float", nullable=True
111+
),
112+
"avg_percent_inpatient_beds_covid": pa.Column("float", nullable=True),
113+
"avg_percent_inpatient_beds_influenza": pa.Column(
114+
"float", nullable=True
115+
),
116+
"avg_percent_staff_icu_beds_covid": pa.Column("float", nullable=True),
117+
"avg_percent_icu_beds_influenza": pa.Column("float", nullable=True),
118+
"percent_adult_covid_admissions": pa.Column("float", nullable=True),
119+
"percent_pediatric_covid_admissions": pa.Column(
120+
"float", nullable=True
121+
),
122+
"percent_hospitals_previous_day_admission_adult_covid_confirmed": pa.Column(
123+
"float", nullable=True
124+
),
125+
"percent_hospitals_previous_day_admission_pediatric_covid_confirmed": pa.Column(
126+
"float", nullable=True
127+
),
128+
"percent_hospitals_previous_day_admission_influenza_confirmed": pa.Column(
129+
"float", nullable=True
130+
),
131+
"percent_hospitals_total_patients_hospitalized_confirmed_influenza": pa.Column(
132+
"float", nullable=True
133+
),
134+
"percent_hospitals_icu_patients_confirmed_influenza": pa.Column(
135+
"float", nullable=True
136+
),
137+
"percent_hospitals_inpatient_beds": pa.Column("float", nullable=True),
138+
"percent_hospitals_total_icu_beds": pa.Column("float", nullable=True),
139+
"percent_hospitals_inpatient_beds_used": pa.Column(
140+
"float", nullable=True
141+
),
142+
"percent_hospitals_icu_beds_used": pa.Column("float", nullable=True),
143+
"percent_hospitals_percent_inpatient_beds_occupied": pa.Column(
144+
"float", nullable=True
145+
),
146+
"percent_hospitals_percent_staff_icu_beds_occupied": pa.Column(
147+
"float", nullable=True
148+
),
149+
"percent_hospitals_percent_inpatient_beds_covid": pa.Column(
150+
"float", nullable=True
151+
),
152+
"percent_hospitals_percent_inpatient_beds_influenza": pa.Column(
153+
"float", nullable=True
154+
),
155+
"percent_hospitals_percent_staff_icu_beds_covid": pa.Column(
156+
"float", nullable=True
157+
),
158+
"percent_hospitals_percent_icu_beds_influenza": pa.Column(
159+
"float", nullable=True
160+
),
161+
"percent_hospitals_admissions_all_covid_confirmed": pa.Column(
162+
"float", nullable=True
163+
),
164+
"percent_hospitals_total_patients_hospitalized_covid_confirmed": pa.Column(
165+
"float", nullable=True
166+
),
167+
"percent_hospitals_staff_icu_patients_covid_confirmed": pa.Column(
168+
"float", nullable=True
169+
),
170+
"abs_chg_percent_hospitals_previous_day_admission_adult_covid_confirmed": pa.Column(
171+
"float", nullable=True
172+
),
173+
"abs_chg_percent_hospitals_previous_day_admission_pediatric_covid_confirmed": pa.Column(
174+
"float", nullable=True
175+
),
176+
"abs_chg_percent_hospitals_previous_day_admission_influenza_confirmed": pa.Column(
177+
"float", nullable=True
178+
),
179+
"abs_chg_percent_hospitals_total_patients_hospitalized_confirmed_influenza": pa.Column(
180+
"float", nullable=True
181+
),
182+
"abs_chg_percent_hospitals_icu_patients_confirmed_influenza": pa.Column(
183+
"float", nullable=True
184+
),
185+
"abs_chg_percent_hospitals_inpatient_beds": pa.Column(
186+
"float", nullable=True
187+
),
188+
"abs_chg_percent_hospitals_total_icu_beds": pa.Column(
189+
"float", nullable=True
190+
),
191+
"abs_chg_percent_hospitals_inpatient_beds_used": pa.Column(
192+
"float", nullable=True
193+
),
194+
"abs_chg_percent_hospitals_icu_beds_used": pa.Column(
195+
"float", nullable=True
196+
),
197+
"abs_chg_percent_hospitals_percent_inpatient_beds_occupied": pa.Column(
198+
"float", nullable=True
199+
),
200+
"abs_chg_percent_hospitals_percent_staff_icu_beds_occupied": pa.Column(
201+
"float", nullable=True
202+
),
203+
"abs_chg_percent_hospitals_percent_inpatient_beds_covid": pa.Column(
204+
"float", nullable=True
205+
),
206+
"abs_chg_percent_hospitals_percent_inpatient_beds_influenza": pa.Column(
207+
"float", nullable=True
208+
),
209+
"abs_chg_percent_hospitals_percent_staff_icu_beds_covid": pa.Column(
210+
"float", nullable=True
211+
),
212+
"abs_chg_percent_hospitals_percent_icu_beds_influenza": pa.Column(
213+
"float", nullable=True
214+
),
215+
"abs_chg_percent_hospitals_admissions_all_covid_confirmed": pa.Column(
216+
"float", nullable=True
217+
),
218+
"abs_chg_percent_hospitals_total_patients_hospitalized_covid_confirmed": pa.Column(
219+
"float", nullable=True
220+
),
221+
"abs_chg_percent_hospitals_staff_icu_patients_covid_confirmed": pa.Column(
222+
"float", nullable=True
223+
),
224+
}
225+
)
226+
227+
load_schema = pa.DataFrameSchema(
228+
{
229+
"date": pa.Column(str, coerce=True),
230+
"state": pa.Column(str, coerce=True),
231+
"total": pa.Column(str, coerce=True, nullable=True),
232+
"stname": pa.Column(str, coerce=True),
233+
}
234+
)
235+
236+
237+
raw_synth_data = pd.DataFrame({})
238+
239+
stname_tf = {
240+
"CA": "california",
241+
"TX": "texas",
242+
"NY": "new_york",
243+
"FL": "florida",
244+
"IL": "illinois",
245+
}
246+
tf_synth_data = pd.DataFrame(
247+
{
248+
"date": [fake.date_this_year() for _ in range(df_len)],
249+
"state": [
250+
random.choice(["CA", "TX", "NY", "FL", "IL"])
251+
for _ in range(df_len)
252+
],
253+
"total": [random.randint(0, 1000) for _ in range(df_len)],
254+
"stname": [fake.state() for _ in range(df_len)],
255+
}
256+
)
257+
tf_synth_data["stname"] = tf_synth_data["state"].map(stname_tf)

0 commit comments

Comments
 (0)