Skip to content

Commit 63e56e1

Browse files
ryanraaschCDCryanraaschCDC
authored
Merge 6c563ae into d017461 
2 parents d017461 + 6c563ae commit 63e56e1

24 files changed

Lines changed: 2492 additions & 4 deletions

‎cfa/scenarios/dataops/datasets/covid19_vax_coverage_adult.toml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# This dataset is static, and thus only needs to be extracted in it's raw form
2+
# once and saved to blob storage. For a human viewable we format, see:
3+
# https://data.cdc.gov/Vaccinations/Weekly-Cumulative-COVID-19-Vaccination-Coverage-an/ksfb-ug5d/about_data
4+
# Raw (extracted) and transformed (loaded) data will be stored in azure blob storage
5+
# which requires the account, container and path for access
6+
7+
[properties]
8+
9+
name = "covid19_vax_coverage_adult"
10+
automate = false
11+
transform_template = "covid19_vax_coverage_adult.sql"
12+
schema = "datasets/schemas/covid19_vax_coverage_adult.py"
13+
14+
[source]
15+
16+
url = "https://data.cdc.gov/resource/ksfb-ug5d.csv"
17+
pagination = {limit = 1000}
18+
19+
[extract]
20+
21+
account = "cfadatalakeprd"
22+
container = "cfapredict"
23+
prefix = "dataops/scenarios/raw/covid19_vax_coverage_adult"
24+
25+
[load]
26+
27+
account = "cfadatalakeprd"
28+
container = "cfapredict"
29+
prefix = "dataops/scenarios/transformed/covid19_vax_coverage_adult"
30+
31+
# TODO: add some data schema validation

‎cfa/scenarios/dataops/datasets/covid19_vax_coverage_child.toml‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# This dataset is static, and thus only needs to be extracted in it's raw form
2+
# once and saved to blob storage. For a human viewable we format, see:
3+
# https://data.cdc.gov/Child-Vaccinations/Weekly-Parental-Intent-for-Vaccination-and-Cumulat/ker6-gs6z/about_data
4+
# Raw (extracted) and transformed (loaded) data will be stored in azure blob storage
5+
# which requires the account, container and path for access
6+
7+
[properties]
8+
9+
name = "covid19_vax_coverage_child"
10+
automate = false
11+
transform_template = "covid19_vax_coverage_child.sql"
12+
schema = "datasets/schemas/covid19_vax_coverage_child.py"
13+
14+
[source]
15+
16+
url = "https://data.cdc.gov/resource/ker6-gs6z.csv"
17+
pagination = {limit = 1000}
18+
19+
[extract]
20+
21+
account = "cfadatalakeprd"
22+
container = "cfapredict"
23+
prefix = "dataops/scenarios/raw/covid19_vax_coverage_child"
24+
25+
[load]
26+
27+
account = "cfadatalakeprd"
28+
container = "cfapredict"
29+
prefix = "dataops/scenarios/transformed/covid19_vax_coverage_child"

‎cfa/scenarios/dataops/datasets/covid_rd18_vax_curves.toml‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# This dataset is static, and thus only needs to be extracted in it's raw form
2+
# once and saved to blob storage.
3+
# Raw (extracted) and transformed (loaded) data will be stored in azure blob storage
4+
# which requires the account, container and path for access
5+
6+
[properties]
7+
name = "covid_rd18_vax_curves"
8+
automate = false
9+
transform_template = ""
10+
schema = ""
11+
12+
[source]
13+
url = ""
14+
pagination = 0
15+
16+
[extract]
17+
account = "cfadatalakeprd"
18+
container = "cfapredict"
19+
prefix = "dataops/scenarios/raw/covid_rd18_vax_curves"
20+
21+
[load]
22+
account = "cfadatalakeprd"
23+
container = "cfapredict"
24+
prefix = "dataops/scenarios/transformed/covid_rd18_vax_curves"
25+
26+
# TODO: add some data schema validation

‎cfa/scenarios/dataops/datasets/schemas/covid19_vax_coverage_adult.py‎

Lines changed: 222 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,222 @@
1+
import random
2+
3+
import pandas as pd
4+
import pandera.pandas as pa
5+
from faker import Faker
6+
7+
fake = Faker()
8+
df_len = 100
9+
10+
extract_schema = pa.DataFrameSchema(
11+
{
12+
"vaccine": pa.Column(str),
13+
"geographic_level": pa.Column(str),
14+
"geographic_name": pa.Column(str),
15+
"demographic_level": pa.Column(str),
16+
"demographic_name": pa.Column(str),
17+
"indicator_label": pa.Column(str),
18+
"indicator_category_label": pa.Column(str),
19+
"month_week": pa.Column(str),
20+
"week_ending": pa.Column(str),
21+
"estimate": pa.Column(float, nullable=True),
22+
"ci_half_width_95pct": pa.Column(float, nullable=True),
23+
"unweighted_sample_size": pa.Column(int, nullable=True),
24+
"current_season_week_ending": pa.Column(str, nullable=True),
25+
"covid_season": pa.Column(str),
26+
"suppression_flag": pa.Column(int),
27+
}
28+
)
29+
30+
load_schema = pa.DataFrameSchema(
31+
{
32+
"vaccine": pa.Column(str),
33+
"geographic_level": pa.Column(str),
34+
"geographic_name": pa.Column(str),
35+
"demographic_level": pa.Column(str),
36+
"demographic_name": pa.Column(str),
37+
"indicator_label": pa.Column(str),
38+
"indicator_category_label": pa.Column(str),
39+
"month_week": pa.Column(str),
40+
"week_ending": pa.Column(str),
41+
"estimate": pa.Column(
42+
float, checks=pa.Check.in_range(0, 100), nullable=True
43+
),
44+
"ci_half_width_95pct": pa.Column(
45+
float, checks=pa.Check.in_range(0, 100), nullable=True
46+
),
47+
"unweighted_sample_size": pa.Column(
48+
int, checks=pa.Check.in_range(0, 10_000_000), nullable=True
49+
),
50+
"covid_season": pa.Column(str),
51+
"suppression_flag": pa.Column(int),
52+
"date": pa.Column(str),
53+
"date1": pa.Column(str),
54+
}
55+
)
56+
geo_name_opts = [
57+
"National",
58+
"Region 7",
59+
"Pennsylvania-Rest of State",
60+
"Oregon",
61+
"Tennessee",
62+
"Region 3",
63+
"Iowa",
64+
"New Jersey",
65+
"Pennsylvania",
66+
"New York-New York City",
67+
"New York",
68+
"Michigan",
69+
"Rhode Island",
70+
"Colorado",
71+
"Delaware",
72+
"New York-Rest of State",
73+
"Arkansas",
74+
"Arizona",
75+
"Illinois-City of Chicago",
76+
"Region 9",
77+
"North Carolina",
78+
"Nevada",
79+
"Utah",
80+
"West Virginia",
81+
"Alaska",
82+
"Pennsylvania-Philadelphia County",
83+
"Region 10",
84+
"Region 4",
85+
"Wisconsin",
86+
"Texas-City of Houston",
87+
"Maine",
88+
"Nebraska",
89+
"Kentucky",
90+
"Region 2",
91+
"Florida",
92+
"Texas-Bexar County",
93+
"Virginia",
94+
"New Hampshire",
95+
"Georgia",
96+
"Wyoming",
97+
"Region 5",
98+
"New Mexico",
99+
"Louisiana",
100+
"Mississippi",
101+
"Puerto Rico",
102+
"Missouri",
103+
"Kansas",
104+
"Oklahoma",
105+
"Texas",
106+
"Alabama",
107+
"Indiana",
108+
"Massachusetts",
109+
"South Dakota",
110+
"Minnesota",
111+
"District of Columbia",
112+
"Texas-Rest of State",
113+
"Ohio",
114+
"Illinois-Rest of State",
115+
"Region 1",
116+
"North Dakota",
117+
"Washington",
118+
"South Carolina",
119+
"Connecticut",
120+
"Montana",
121+
"Illinois",
122+
"Idaho",
123+
"Region 6",
124+
"Vermont",
125+
"Hawaii",
126+
"Region 8",
127+
"California",
128+
"Maryland",
129+
"U.S. Virgin Islands",
130+
"Guam",
131+
]
132+
demo_level_opts = [
133+
"Race and Ethnicity",
134+
"Urbanicity",
135+
"Overall",
136+
"Age",
137+
"Sexual Orientation",
138+
"Sex",
139+
"Disability Status",
140+
"Poverty Status",
141+
"Health Insurance",
142+
"Health Insurance Among 18-64 Years",
143+
]
144+
demo_name_opts = [
145+
"Multiple Race/Other (Excludes Asian, AIAN, PI/NH), Non-Hispanic",
146+
"Rural (Non-MSA)",
147+
"Asian, Non-Hispanic",
148+
"White, Non-Hispanic",
149+
"18+ years",
150+
"65+ years",
151+
"Other, Non-Hispanic",
152+
"Gay/Lesbian/Bisexual/Other",
153+
"65-74 years",
154+
"Urban MSA Principal City",
155+
"Female",
156+
"75+ years",
157+
"Yes",
158+
"30-39 years",
159+
"Don't Know/Refused",
160+
"American Indian/Alaska Native, Non-Hispanic",
161+
"Straight",
162+
"Above Poverty, Income < $75k",
163+
"Male",
164+
"Above Poverty, Income >= $75k",
165+
"No",
166+
"Uninsured",
167+
"Below Poverty",
168+
"Pacific Islander/Native Hawaiian, Non-Hispanic",
169+
"50-64 years",
170+
"60+ years",
171+
"Insured",
172+
"Poverty Status Unknown",
173+
"Suburban (MSA Non-Principal City)",
174+
"18-49 years",
175+
"40-49 years",
176+
"Black, Non-Hispanic",
177+
"18-29 years",
178+
"Hispanic",
179+
]
180+
ind_label_opts = ["4-level vaccination and intent", "Up-to-date"]
181+
ind_cat_label_opts = [
182+
"Definitely will get a vaccine",
183+
"Received a vaccination",
184+
"Yes",
185+
"Probably will get a vaccine or are unsure",
186+
"Definitely or probably will not get a vaccine",
187+
]
188+
year_opts = ["2023-2024", "2024-2025"]
189+
190+
raw_synth_data = pd.DataFrame()
191+
raw_synth_data = raw_synth_data.assign(
192+
vaccine=["COVID" for _ in range(df_len)],
193+
geographic_level=[
194+
random.choice(["National", "Region", "Substate", "State"])
195+
for _ in range(df_len)
196+
],
197+
geographic_name=[random.choice(geo_name_opts) for _ in range(df_len)],
198+
demographic_level=[random.choice(demo_level_opts) for _ in range(df_len)],
199+
demographic_name=[random.choice(demo_name_opts) for _ in range(df_len)],
200+
indicator_label=[random.choice(ind_label_opts) for _ in range(df_len)],
201+
indicator_category_label=[
202+
random.choice(ind_cat_label_opts) for _ in range(df_len)
203+
],
204+
month_week=[
205+
fake.month_name() + " " + str(random.randint(1, 52))
206+
for _ in range(df_len)
207+
],
208+
week_ending=[str(fake.date_this_year()) for _ in range(df_len)],
209+
estimate=[random.uniform(0, 100) for _ in range(df_len)],
210+
ci_half_width_95pct=[random.uniform(0, 100) for _ in range(df_len)],
211+
unweighted_sample_size=[
212+
random.randint(1, 10_000_000) for _ in range(df_len)
213+
],
214+
current_season_week_ending=[
215+
str(fake.date_this_year()) for _ in range(df_len)
216+
],
217+
covid_season=[random.choice(year_opts) for _ in range(df_len)],
218+
suppression_flag=[random.randint(0, 99) for _ in range(df_len)],
219+
)
220+
tf_synth_data = raw_synth_data.copy()
221+
tf_synth_data["date"] = tf_synth_data["week_ending"]
222+
tf_synth_data["date1"] = tf_synth_data["week_ending"]

0 commit comments

Comments
 (0)