Skip to content

Commit 70aaf73

Browse files
ryanraaschCDCryanraaschCDC
committed
add tests
1 parent 66a363d commit 70aaf73

5 files changed

Lines changed: 311 additions & 110 deletions

File tree

‎cfa/scenarios/dataops/datasets/schemas/hospitalization.py‎

Lines changed: 231 additions & 94 deletions
Original file line numberDiff line numberDiff line change
@@ -11,95 +11,227 @@
1111
{
1212
"week_end_date": pa.Column(str),
1313
"jurisdiction": pa.Column(str),
14-
"weekly_actual_days_reporting_any_data": pa.Column(float, nullable=True, coerce = True),
15-
"weekly_percent_days_reporting_any_data": pa.Column(float, nullable=True, coerce = True),
16-
"num_hospitals_previous_day_admission_adult_covid_confirmed": pa.Column('int', nullable=True),
17-
"num_hospitals_previous_day_admission_pediatric_covid_confirmed": pa.Column('int', nullable=True),
18-
"num_hospitals_previous_day_admission_influenza_confirmed": pa.Column('int', nullable=True),
19-
"num_hospitals_total_patients_hospitalized_confirmed_influenza": pa.Column('int', nullable=True),
20-
"num_hospitals_icu_patients_confirmed_influenza": pa.Column('int', nullable=True),
21-
"num_hospitals_inpatient_beds": pa.Column('int', nullable=True),
22-
"num_hospitals_total_icu_beds": pa.Column('int', nullable=True),
23-
"num_hospitals_inpatient_beds_used": pa.Column('int', nullable=True),
24-
"num_hospitals_icu_beds_used": pa.Column('int', nullable=True),
25-
"num_hospitals_percent_inpatient_beds_occupied": pa.Column('int', nullable=True),
26-
"num_hospitals_percent_staff_icu_beds_occupied": pa.Column('int', nullable=True),
27-
"num_hospitals_percent_inpatient_beds_covid": pa.Column('int', nullable=True),
28-
"num_hospitals_percent_inpatient_beds_influenza": pa.Column('int', nullable=True),
29-
"num_hospitals_percent_staff_icu_beds_covid": pa.Column('int', nullable=True),
30-
"num_hospitals_percent_icu_beds_influenza": pa.Column('int', nullable=True),
31-
"num_hospitals_admissions_all_covid_confirmed": pa.Column('int', nullable=True),
32-
"num_hospitals_total_patients_hospitalized_covid_confirmed": pa.Column('int', nullable=True),
33-
"num_hospitals_staff_icu_patients_covid_confirmed": pa.Column('int', nullable=True),
34-
"avg_admissions_adult_covid_confirmed": pa.Column('float', nullable=True),
35-
"total_admissions_adult_covid_confirmed": pa.Column('float', nullable=True),
36-
"avg_admissions_pediatric_covid_confirmed": pa.Column('float', nullable=True),
37-
"total_admissions_pediatric_covid_confirmed": pa.Column('float', nullable=True),
38-
"avg_admissions_all_covid_confirmed": pa.Column('float', nullable=True),
39-
"total_admissions_all_covid_confirmed": pa.Column('float', nullable=True),
40-
"avg_admissions_all_influenza_confirmed": pa.Column('float', nullable=True),
41-
"total_admissions_all_influenza_confirmed": pa.Column('float', nullable=True),
42-
"avg_total_patients_hospitalized_covid_confirmed": pa.Column('float', nullable=True),
43-
"avg_total_patients_hospitalized_influenza_confirmed": pa.Column('float', nullable=True),
44-
"avg_staff_icu_patients_covid_confirmed": pa.Column('float', nullable=True),
45-
"avg_icu_patients_influenza_confirmed": pa.Column('float', nullable=True),
46-
"avg_inpatient_beds": pa.Column('float', nullable=True),
47-
"avg_total_icu_beds": pa.Column('float', nullable=True),
48-
"avg_inpatient_beds_used": pa.Column('float', nullable=True),
49-
"avg_icu_beds_used": pa.Column('float', nullable=True),
50-
"avg_percent_inpatient_beds_occupied": pa.Column('float', nullable=True),
51-
"avg_percent_staff_icu_beds_occupied": pa.Column('float', nullable=True),
52-
"avg_percent_inpatient_beds_covid": pa.Column('float', nullable=True),
53-
"avg_percent_inpatient_beds_influenza": pa.Column('float', nullable=True),
54-
"avg_percent_staff_icu_beds_covid": pa.Column('float', nullable=True),
55-
"avg_percent_icu_beds_influenza": pa.Column('float', nullable=True),
56-
"percent_adult_covid_admissions": pa.Column('float', nullable=True),
57-
"percent_pediatric_covid_admissions": pa.Column('float', nullable=True),
58-
"percent_hospitals_previous_day_admission_adult_covid_confirmed": pa.Column('float', nullable=True),
59-
"percent_hospitals_previous_day_admission_pediatric_covid_confirmed": pa.Column('float', nullable=True),
60-
"percent_hospitals_previous_day_admission_influenza_confirmed": pa.Column('float', nullable=True),
61-
"percent_hospitals_total_patients_hospitalized_confirmed_influenza": pa.Column('float', nullable=True),
62-
"percent_hospitals_icu_patients_confirmed_influenza": pa.Column('float', nullable=True),
63-
"percent_hospitals_inpatient_beds": pa.Column('float', nullable=True),
64-
"percent_hospitals_total_icu_beds": pa.Column('float', nullable=True),
65-
"percent_hospitals_inpatient_beds_used": pa.Column('float', nullable=True),
66-
"percent_hospitals_icu_beds_used": pa.Column('float', nullable=True),
67-
"percent_hospitals_percent_inpatient_beds_occupied": pa.Column('float', nullable=True),
68-
"percent_hospitals_percent_staff_icu_beds_occupied": pa.Column('float', nullable=True),
69-
"percent_hospitals_percent_inpatient_beds_covid": pa.Column('float', nullable=True),
70-
"percent_hospitals_percent_inpatient_beds_influenza": pa.Column('float', nullable=True),
71-
"percent_hospitals_percent_staff_icu_beds_covid": pa.Column('float', nullable=True),
72-
"percent_hospitals_percent_icu_beds_influenza": pa.Column('float', nullable=True),
73-
"percent_hospitals_admissions_all_covid_confirmed": pa.Column('float', nullable=True),
74-
"percent_hospitals_total_patients_hospitalized_covid_confirmed": pa.Column('float', nullable=True),
75-
"percent_hospitals_staff_icu_patients_covid_confirmed": pa.Column('float', nullable=True),
76-
"abs_chg_percent_hospitals_previous_day_admission_adult_covid_confirmed": pa.Column('float', nullable=True),
77-
"abs_chg_percent_hospitals_previous_day_admission_pediatric_covid_confirmed": pa.Column('float', nullable=True),
78-
"abs_chg_percent_hospitals_previous_day_admission_influenza_confirmed": pa.Column('float', nullable=True),
79-
"abs_chg_percent_hospitals_total_patients_hospitalized_confirmed_influenza": pa.Column('float', nullable=True),
80-
"abs_chg_percent_hospitals_icu_patients_confirmed_influenza": pa.Column('float', nullable=True),
81-
"abs_chg_percent_hospitals_inpatient_beds": pa.Column('float', nullable=True),
82-
"abs_chg_percent_hospitals_total_icu_beds": pa.Column('float', nullable=True),
83-
"abs_chg_percent_hospitals_inpatient_beds_used": pa.Column('float', nullable=True),
84-
"abs_chg_percent_hospitals_icu_beds_used": pa.Column('float', nullable=True),
85-
"abs_chg_percent_hospitals_percent_inpatient_beds_occupied": pa.Column('float', nullable=True),
86-
"abs_chg_percent_hospitals_percent_staff_icu_beds_occupied": pa.Column('float', nullable=True),
87-
"abs_chg_percent_hospitals_percent_inpatient_beds_covid": pa.Column('float', nullable=True),
88-
"abs_chg_percent_hospitals_percent_inpatient_beds_influenza": pa.Column('float', nullable=True),
89-
"abs_chg_percent_hospitals_percent_staff_icu_beds_covid": pa.Column('float', nullable=True),
90-
"abs_chg_percent_hospitals_percent_icu_beds_influenza": pa.Column('float', nullable=True),
91-
"abs_chg_percent_hospitals_admissions_all_covid_confirmed": pa.Column('float', nullable=True),
92-
"abs_chg_percent_hospitals_total_patients_hospitalized_covid_confirmed": pa.Column('float', nullable=True),
93-
"abs_chg_percent_hospitals_staff_icu_patients_covid_confirmed": pa.Column('float', nullable=True)
14+
"weekly_actual_days_reporting_any_data": pa.Column(
15+
float, nullable=True, coerce=True
16+
),
17+
"weekly_percent_days_reporting_any_data": pa.Column(
18+
float, nullable=True, coerce=True
19+
),
20+
"num_hospitals_previous_day_admission_adult_covid_confirmed": pa.Column(
21+
"int", nullable=True
22+
),
23+
"num_hospitals_previous_day_admission_pediatric_covid_confirmed": pa.Column(
24+
"int", nullable=True
25+
),
26+
"num_hospitals_previous_day_admission_influenza_confirmed": pa.Column(
27+
"int", nullable=True
28+
),
29+
"num_hospitals_total_patients_hospitalized_confirmed_influenza": pa.Column(
30+
"int", nullable=True
31+
),
32+
"num_hospitals_icu_patients_confirmed_influenza": pa.Column(
33+
"int", nullable=True
34+
),
35+
"num_hospitals_inpatient_beds": pa.Column("int", nullable=True),
36+
"num_hospitals_total_icu_beds": pa.Column("int", nullable=True),
37+
"num_hospitals_inpatient_beds_used": pa.Column("int", nullable=True),
38+
"num_hospitals_icu_beds_used": pa.Column("int", nullable=True),
39+
"num_hospitals_percent_inpatient_beds_occupied": pa.Column(
40+
"int", nullable=True
41+
),
42+
"num_hospitals_percent_staff_icu_beds_occupied": pa.Column(
43+
"int", nullable=True
44+
),
45+
"num_hospitals_percent_inpatient_beds_covid": pa.Column(
46+
"int", nullable=True
47+
),
48+
"num_hospitals_percent_inpatient_beds_influenza": pa.Column(
49+
"int", nullable=True
50+
),
51+
"num_hospitals_percent_staff_icu_beds_covid": pa.Column(
52+
"int", nullable=True
53+
),
54+
"num_hospitals_percent_icu_beds_influenza": pa.Column(
55+
"int", nullable=True
56+
),
57+
"num_hospitals_admissions_all_covid_confirmed": pa.Column(
58+
"int", nullable=True
59+
),
60+
"num_hospitals_total_patients_hospitalized_covid_confirmed": pa.Column(
61+
"int", nullable=True
62+
),
63+
"num_hospitals_staff_icu_patients_covid_confirmed": pa.Column(
64+
"int", nullable=True
65+
),
66+
"avg_admissions_adult_covid_confirmed": pa.Column(
67+
"float", nullable=True
68+
),
69+
"total_admissions_adult_covid_confirmed": pa.Column(
70+
"float", nullable=True
71+
),
72+
"avg_admissions_pediatric_covid_confirmed": pa.Column(
73+
"float", nullable=True
74+
),
75+
"total_admissions_pediatric_covid_confirmed": pa.Column(
76+
"float", nullable=True
77+
),
78+
"avg_admissions_all_covid_confirmed": pa.Column(
79+
"float", nullable=True
80+
),
81+
"total_admissions_all_covid_confirmed": pa.Column(
82+
"float", nullable=True
83+
),
84+
"avg_admissions_all_influenza_confirmed": pa.Column(
85+
"float", nullable=True
86+
),
87+
"total_admissions_all_influenza_confirmed": pa.Column(
88+
"float", nullable=True
89+
),
90+
"avg_total_patients_hospitalized_covid_confirmed": pa.Column(
91+
"float", nullable=True
92+
),
93+
"avg_total_patients_hospitalized_influenza_confirmed": pa.Column(
94+
"float", nullable=True
95+
),
96+
"avg_staff_icu_patients_covid_confirmed": pa.Column(
97+
"float", nullable=True
98+
),
99+
"avg_icu_patients_influenza_confirmed": pa.Column(
100+
"float", nullable=True
101+
),
102+
"avg_inpatient_beds": pa.Column("float", nullable=True),
103+
"avg_total_icu_beds": pa.Column("float", nullable=True),
104+
"avg_inpatient_beds_used": pa.Column("float", nullable=True),
105+
"avg_icu_beds_used": pa.Column("float", nullable=True),
106+
"avg_percent_inpatient_beds_occupied": pa.Column(
107+
"float", nullable=True
108+
),
109+
"avg_percent_staff_icu_beds_occupied": pa.Column(
110+
"float", nullable=True
111+
),
112+
"avg_percent_inpatient_beds_covid": pa.Column("float", nullable=True),
113+
"avg_percent_inpatient_beds_influenza": pa.Column(
114+
"float", nullable=True
115+
),
116+
"avg_percent_staff_icu_beds_covid": pa.Column("float", nullable=True),
117+
"avg_percent_icu_beds_influenza": pa.Column("float", nullable=True),
118+
"percent_adult_covid_admissions": pa.Column("float", nullable=True),
119+
"percent_pediatric_covid_admissions": pa.Column(
120+
"float", nullable=True
121+
),
122+
"percent_hospitals_previous_day_admission_adult_covid_confirmed": pa.Column(
123+
"float", nullable=True
124+
),
125+
"percent_hospitals_previous_day_admission_pediatric_covid_confirmed": pa.Column(
126+
"float", nullable=True
127+
),
128+
"percent_hospitals_previous_day_admission_influenza_confirmed": pa.Column(
129+
"float", nullable=True
130+
),
131+
"percent_hospitals_total_patients_hospitalized_confirmed_influenza": pa.Column(
132+
"float", nullable=True
133+
),
134+
"percent_hospitals_icu_patients_confirmed_influenza": pa.Column(
135+
"float", nullable=True
136+
),
137+
"percent_hospitals_inpatient_beds": pa.Column("float", nullable=True),
138+
"percent_hospitals_total_icu_beds": pa.Column("float", nullable=True),
139+
"percent_hospitals_inpatient_beds_used": pa.Column(
140+
"float", nullable=True
141+
),
142+
"percent_hospitals_icu_beds_used": pa.Column("float", nullable=True),
143+
"percent_hospitals_percent_inpatient_beds_occupied": pa.Column(
144+
"float", nullable=True
145+
),
146+
"percent_hospitals_percent_staff_icu_beds_occupied": pa.Column(
147+
"float", nullable=True
148+
),
149+
"percent_hospitals_percent_inpatient_beds_covid": pa.Column(
150+
"float", nullable=True
151+
),
152+
"percent_hospitals_percent_inpatient_beds_influenza": pa.Column(
153+
"float", nullable=True
154+
),
155+
"percent_hospitals_percent_staff_icu_beds_covid": pa.Column(
156+
"float", nullable=True
157+
),
158+
"percent_hospitals_percent_icu_beds_influenza": pa.Column(
159+
"float", nullable=True
160+
),
161+
"percent_hospitals_admissions_all_covid_confirmed": pa.Column(
162+
"float", nullable=True
163+
),
164+
"percent_hospitals_total_patients_hospitalized_covid_confirmed": pa.Column(
165+
"float", nullable=True
166+
),
167+
"percent_hospitals_staff_icu_patients_covid_confirmed": pa.Column(
168+
"float", nullable=True
169+
),
170+
"abs_chg_percent_hospitals_previous_day_admission_adult_covid_confirmed": pa.Column(
171+
"float", nullable=True
172+
),
173+
"abs_chg_percent_hospitals_previous_day_admission_pediatric_covid_confirmed": pa.Column(
174+
"float", nullable=True
175+
),
176+
"abs_chg_percent_hospitals_previous_day_admission_influenza_confirmed": pa.Column(
177+
"float", nullable=True
178+
),
179+
"abs_chg_percent_hospitals_total_patients_hospitalized_confirmed_influenza": pa.Column(
180+
"float", nullable=True
181+
),
182+
"abs_chg_percent_hospitals_icu_patients_confirmed_influenza": pa.Column(
183+
"float", nullable=True
184+
),
185+
"abs_chg_percent_hospitals_inpatient_beds": pa.Column(
186+
"float", nullable=True
187+
),
188+
"abs_chg_percent_hospitals_total_icu_beds": pa.Column(
189+
"float", nullable=True
190+
),
191+
"abs_chg_percent_hospitals_inpatient_beds_used": pa.Column(
192+
"float", nullable=True
193+
),
194+
"abs_chg_percent_hospitals_icu_beds_used": pa.Column(
195+
"float", nullable=True
196+
),
197+
"abs_chg_percent_hospitals_percent_inpatient_beds_occupied": pa.Column(
198+
"float", nullable=True
199+
),
200+
"abs_chg_percent_hospitals_percent_staff_icu_beds_occupied": pa.Column(
201+
"float", nullable=True
202+
),
203+
"abs_chg_percent_hospitals_percent_inpatient_beds_covid": pa.Column(
204+
"float", nullable=True
205+
),
206+
"abs_chg_percent_hospitals_percent_inpatient_beds_influenza": pa.Column(
207+
"float", nullable=True
208+
),
209+
"abs_chg_percent_hospitals_percent_staff_icu_beds_covid": pa.Column(
210+
"float", nullable=True
211+
),
212+
"abs_chg_percent_hospitals_percent_icu_beds_influenza": pa.Column(
213+
"float", nullable=True
214+
),
215+
"abs_chg_percent_hospitals_admissions_all_covid_confirmed": pa.Column(
216+
"float", nullable=True
217+
),
218+
"abs_chg_percent_hospitals_total_patients_hospitalized_covid_confirmed": pa.Column(
219+
"float", nullable=True
220+
),
221+
"abs_chg_percent_hospitals_staff_icu_patients_covid_confirmed": pa.Column(
222+
"float", nullable=True
223+
),
94224
}
95225
)
96226

97-
load_schema = pa.DataFrameSchema({
98-
"date": pa.Column(str, coerce=True),
99-
"state": pa.Column(str, coerce=True),
100-
"total": pa.Column(str, coerce = True, nullable = True),
101-
"stname" : pa.Column(str, coerce=True),
102-
})
227+
load_schema = pa.DataFrameSchema(
228+
{
229+
"date": pa.Column(str, coerce=True),
230+
"state": pa.Column(str, coerce=True),
231+
"total": pa.Column(str, coerce=True, nullable=True),
232+
"stname": pa.Column(str, coerce=True),
233+
}
234+
)
103235

104236

105237
raw_synth_data = pd.DataFrame({})
@@ -109,12 +241,17 @@
109241
"TX": "texas",
110242
"NY": "new_york",
111243
"FL": "florida",
112-
"IL": "illinois"
244+
"IL": "illinois",
113245
}
114-
tf_synth_data = pd.DataFrame({
115-
"date": [fake.date_this_year() for _ in range(df_len)],
116-
"state": [random.choice(["CA", "TX", "NY", "FL", "IL"]) for _ in range(df_len)],
117-
"total": [random.randint(0, 1000) for _ in range(df_len)],
118-
"stname": [fake.state() for _ in range(df_len)],
119-
})
120-
tf_synth_data["stname"] = tf_synth_data["state"].map(stname_tf)
246+
tf_synth_data = pd.DataFrame(
247+
{
248+
"date": [fake.date_this_year() for _ in range(df_len)],
249+
"state": [
250+
random.choice(["CA", "TX", "NY", "FL", "IL"])
251+
for _ in range(df_len)
252+
],
253+
"total": [random.randint(0, 1000) for _ in range(df_len)],
254+
"stname": [fake.state() for _ in range(df_len)],
255+
}
256+
)
257+
tf_synth_data["stname"] = tf_synth_data["state"].map(stname_tf)

‎cfa/scenarios/dataops/etl/hospitalization.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import pandas as pd
88
import pandera.pandas as pa
99
from tqdm import tqdm
10+
1011
from cfa.scenarios.dataops.datasets.catalog import get_data
1112

1213
from ..datasets import datasets
@@ -60,11 +61,11 @@ def transform(df: pd.DataFrame) -> pd.DataFrame: # noqa: W0613
6061
Returns:
6162
pd.DataFrame: the transformed data
6263
"""
63-
region_id = get_data("fips_to_name_improved")
64+
region_id = get_data("fips_to_name_improved") # noqa
6465
template = transform_template_lookup.get_template(
6566
config.properties.transform_template
6667
)
67-
query = template.render(data_source="df", region_id = "region_id")
68+
query = template.render(data_source="df", region_id="region_id")
6869
transformed_db = duckdb.sql(query).df()
6970
return transformed_db
7071

‎cfa/scenarios/dataops/etl/transform_templates/hospitalization.sql‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
WITH tmp as (
2-
SELECT LEFT(week_end_date, 10) as date,
3-
REPLACE(jurisdiction, 'USA', 'US') as state,
2+
SELECT LEFT(week_end_date, 10) as date,
3+
REPLACE(jurisdiction, 'USA', 'US') as state,
44
total_admissions_all_covid_confirmed as total
55
FROM ${data_source}
66
)
7-
SELECT tmp.date as "date", tmp.state, CAST(tmp.total as int) as total, LOWER(REPLACE(ri.stname, ' ', '_')) as stname
7+
SELECT tmp.date as "date", tmp.state, CAST(tmp.total as int) as total, LOWER(REPLACE(ri.stname, ' ', '_')) as stname
88
FROM tmp
99
RIGHT JOIN ${region_id} as ri
10-
ON tmp.state = ri.stusps
10+
ON tmp.state = ri.stusps

0 commit comments

Comments
 (0)