Merge 76b6e93 into 35b13ad

Author: ryanraaschCDC

README.md

@@ -23,9 +23,9 @@ To use this repository:
    ```
    poetry install
    ```
-3. Example: run the COVID-19 vaccination trends ETL pipeline:
+3. Example: run the COVID-19 vaccination trends ETL pipeline, with extraction, raw data schema validation (-v) and transformed data validation (-t).
    ```
-   python -m cfa.scenarios.dataops.etl.covid19vax_trends --extract
+   python -m cfa.scenarios.dataops.etl.covid19vax_trends --extract -v -t
    ```
 4. [Optional] installing developer dependencies:
    ```

cfa/scenarios/dataops/datasets/covid19_hospitalizations.toml

@@ -8,6 +8,7 @@
 name = "covid19hospitalizations"
 automate = false
 transform_template = "covid19hospitalizations.sql"
+schema = "datasets/schemas/covid19hospitalizations.py"
 
 [source]
 url = "https://healthdata.gov/resource/sgxm-t72h.csv"

cfa/scenarios/dataops/datasets/covid19_vaccination_trends.toml

@@ -9,6 +9,7 @@
 name = "covid19vax_trends"
 automate = false
 transform_template = "covid19vax_trends.sql"
+schema = "datasets/schemas/covid19vax_trends.py"
 
 [source]
 

cfa/scenarios/dataops/datasets/donor_seroprevalence_2020.toml

@@ -8,6 +8,7 @@
 name = "donor_seroprevalence_2020"
 automate = false
 transform_template = "donor_seroprevalence_2020.sql"
+schema = "datasets/schemas/donor_seroprevalence_2020.py"
 
 [source]
 url = "https://data.cdc.gov/resource/mtc3-kq6r.csv"

cfa/scenarios/dataops/datasets/donor_seroprevalence_2022.toml

@@ -8,6 +8,7 @@
 name = "donor_seroprevalence_2022"
 automate = false
 transform_template = "donor_seroprevalence_2022.sql"
+schema = "datasets/schemas/donor_seroprevalence_2022.py"
 
 [source]
 url = "https://data.cdc.gov/resource/ar8q-3jhn.csv"

cfa/scenarios/dataops/datasets/fips_to_name.toml

@@ -8,6 +8,7 @@
 name = "fips_to_name"
 automate = false
 transform_template = "fips_to_name.sql"
+schema = "datasets/schemas/fips_to_name.py"
 
 [source]
 url = "https://data.transportation.gov/resource/eek5-pv8d.csv"

cfa/scenarios/dataops/datasets/fips_to_name_improved.toml

@@ -1,24 +1,25 @@
-# This dataset is static, and thus only needs to be extracted in it's raw form
-# once and saved to blob storage. For a human viewable we format, see:
-# "https://www.fema.gov/api/open/v2/FemaRegions"
-# Raw (extracted) and transformed (loaded) data will be stored in azure blob storage
-# which requires the account, container and path for access
-
-[properties]
-name = "fips_to_name_improved"
-automate = false
-transform_template = "fips_to_name_improved.sql"
-
-[source]
-url = "https://www.fema.gov/api/open/v2/FemaRegions"
-pagination = {limit = 1000}
-
-[extract]
-account = "cfadatalakeprd"
-container = "cfapredict"
-prefix = "dataops/scenarios/raw/fips_to_name_improved"
-
-[load]
-account = "cfadatalakeprd"
-container = "cfapredict"
-prefix = "dataops/scenarios/transformed/fips_to_name_improved"
+# This dataset is static, and thus only needs to be extracted in it's raw form
+# once and saved to blob storage. For a human viewable we format, see:
+# "https://www.fema.gov/api/open/v2/FemaRegions"
+# Raw (extracted) and transformed (loaded) data will be stored in azure blob storage
+# which requires the account, container and path for access
+
+[properties]
+name = "fips_to_name_improved"
+automate = false
+transform_template = "fips_to_name_improved.sql"
+schema = "datasets/schemas/fips_to_name_improved.py"
+
+[source]
+url = "https://www.fema.gov/api/open/v2/FemaRegions"
+pagination = {limit = 1000}
+
+[extract]
+account = "cfadatalakeprd"
+container = "cfapredict"
+prefix = "dataops/scenarios/raw/fips_to_name_improved"
+
+[load]
+account = "cfadatalakeprd"
+container = "cfapredict"
+prefix = "dataops/scenarios/transformed/fips_to_name_improved"

cfa/scenarios/dataops/datasets/omicron_variant_regions.toml

@@ -1,24 +1,25 @@
-# This dataset is static, and thus only needs to be extracted in it's raw form
-# once and saved to blob storage. For a human viewable we format, see:
-# https://data.transportation.gov/resource/eek5-pv8d.csv
-# Raw (extracted) and transformed (loaded) data will be stored in azure blob storage
-# which requires the account, container and path for access
-
-[properties]
-name = "omicron_variant_regions"
-automate = false
-transform_template = ""
-
-[source]
-url = ""
-pagination = {limit = 1000}
-
-[extract]
-account = "cfadatalakeprd"
-container = "cfapredict"
-prefix = "dataops/scenarios/raw/omicron_variant_regions"
-
-[load]
-account = "cfadatalakeprd"
-container = "cfapredict"
-prefix = "dataops/scenarios/transformed/omicron_variant_regions"
+# This dataset is static, and thus only needs to be extracted in it's raw form
+# once and saved to blob storage. For a human viewable we format, see:
+# https://data.transportation.gov/resource/eek5-pv8d.csv
+# Raw (extracted) and transformed (loaded) data will be stored in azure blob storage
+# which requires the account, container and path for access
+
+[properties]
+name = "omicron_variant_regions"
+automate = false
+transform_template = ""
+schema = ""
+
+[source]
+url = ""
+pagination = {limit = 1000}
+
+[extract]
+account = "cfadatalakeprd"
+container = "cfapredict"
+prefix = "dataops/scenarios/raw/omicron_variant_regions"
+
+[load]
+account = "cfadatalakeprd"
+container = "cfapredict"
+prefix = "dataops/scenarios/transformed/omicron_variant_regions"

cfa/scenarios/dataops/datasets/sars_cov2_proportions.toml

@@ -8,6 +8,7 @@
 name = "sars_cov2_proportions"
 automate = false
 transform_template = "sars_cov2_proportions.sql"
+schema = "datasets/schemas/sars_cov2_proportions.py"
 
 [source]
 url = "https://data.cdc.gov/resource/jr58-6ysp.csv"