Skip to content

Commit 0aeea9c

Browse files
ryanraaschCDCryanraaschCDC
committed
final updates from main
1 parent 56ef47f commit 0aeea9c

7 files changed

Lines changed: 794 additions & 800 deletions

File tree

‎changelog.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ The versioning pattern is `YYYY.MM.DD.micro(a/b/{none if release})
88

99
---
1010

11-
## [2025.06.13.0a]
11+
## [2025.06.26.0a]
1212

1313
### Added
1414

‎docs/assets/badges/coverage.svg‎

Lines changed: 22 additions & 22 deletions
Loading

‎tests/datasets/test_covid19_vax_trends.py‎

Lines changed: 45 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,45 +1,45 @@
1-
import pandas as pd
2-
import pandera.pandas as pa
3-
import pytest
4-
from pandera.errors import SchemaError
5-
6-
from cfa.scenarios.dataops.datasets.schemas.covid19vax_trends import (
7-
extract_schema,
8-
load_schema,
9-
raw_synth_data,
10-
tf_synth_data,
11-
)
12-
from cfa.scenarios.dataops.etl.covid19vax_trends import transform
13-
14-
15-
def test_covid19_vax_trends_schemas():
16-
assert isinstance(extract_schema, pa.DataFrameSchema)
17-
assert isinstance(load_schema, pa.DataFrameSchema)
18-
19-
# Check if the schemas have the expected columns
20-
assert isinstance(extract_schema(raw_synth_data), pd.DataFrame)
21-
assert isinstance(load_schema(tf_synth_data), pd.DataFrame)
22-
23-
with pytest.raises(SchemaError):
24-
ex_cols = raw_synth_data.columns
25-
extract_schema(
26-
raw_synth_data.rename(
27-
columns={col: col + "_renamed" for col in ex_cols}
28-
)
29-
)
30-
31-
with pytest.raises(SchemaError):
32-
tf_cols = tf_synth_data.columns
33-
load_schema(
34-
tf_synth_data.rename(
35-
columns={col: col + "_renamed" for col in tf_cols}
36-
)
37-
)
38-
39-
40-
def test_covid19_vax_trends_transform():
41-
extract_df = raw_synth_data
42-
tf_df = transform(extract_df)
43-
44-
# Check if the transformed DataFrame matches the expected schema
45-
assert isinstance(load_schema(tf_df), pd.DataFrame)
1+
import pandas as pd
2+
import pandera.pandas as pa
3+
import pytest
4+
from pandera.errors import SchemaError
5+
6+
from cfa.scenarios.dataops.datasets.schemas.covid19vax_trends import (
7+
extract_schema,
8+
load_schema,
9+
raw_synth_data,
10+
tf_synth_data,
11+
)
12+
from cfa.scenarios.dataops.etl.covid19vax_trends import transform
13+
14+
15+
def test_covid19_vax_trends_schemas():
16+
assert isinstance(extract_schema, pa.DataFrameSchema)
17+
assert isinstance(load_schema, pa.DataFrameSchema)
18+
19+
# Check if the schemas have the expected columns
20+
assert isinstance(extract_schema(raw_synth_data), pd.DataFrame)
21+
assert isinstance(load_schema(tf_synth_data), pd.DataFrame)
22+
23+
with pytest.raises(SchemaError):
24+
ex_cols = raw_synth_data.columns
25+
extract_schema(
26+
raw_synth_data.rename(
27+
columns={col: col + "_renamed" for col in ex_cols}
28+
)
29+
)
30+
31+
with pytest.raises(SchemaError):
32+
tf_cols = tf_synth_data.columns
33+
load_schema(
34+
tf_synth_data.rename(
35+
columns={col: col + "_renamed" for col in tf_cols}
36+
)
37+
)
38+
39+
40+
def test_covid19_vax_trends_transform():
41+
extract_df = raw_synth_data
42+
tf_df = transform(extract_df)
43+
44+
# Check if the transformed DataFrame matches the expected schema
45+
assert isinstance(load_schema(tf_df), pd.DataFrame)

‎tests/test_config_validator.py‎

Lines changed: 87 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1,87 +1,87 @@
1-
import unittest
2-
3-
from cfa.scenarios.dataops.datasets.config_validator import (
4-
validate_dataset_config,
5-
verify_no_repeats,
6-
)
7-
8-
good_config = {
9-
"properties": {
10-
"name": "test_dataset_name",
11-
"automate": False,
12-
"transform_template": "test_tf.sql",
13-
"schema": "datasets/schemas/tests.py",
14-
},
15-
"source": {"url": "https://data.csv", "pagination": {"limit": 1000}},
16-
"extract": {
17-
"account": "test_account_raw",
18-
"container": "test_container_raw",
19-
"prefix": "dataops/scenarios/raw/test",
20-
},
21-
"load": {
22-
"account": "test_account_tf",
23-
"container": "test_container_tf",
24-
"prefix": "dataops/scenarios/transformed/test",
25-
},
26-
"_metadata": {"filename": "test_filename"},
27-
}
28-
bad_config = {
29-
"properties": {
30-
"name": "test_dataset_name",
31-
"automate": False,
32-
"transform_template": "test_tf.sql",
33-
"schema": "datasets/schemas/tests.py",
34-
},
35-
"source": {"url": "https://data.csv", "pagination": {"limit": 1000}},
36-
"extract": {
37-
"account": "test_account_raw",
38-
"container": "test_container_raw",
39-
},
40-
"load": {
41-
"account": "test_account_tf",
42-
"container": "test_container_tf",
43-
},
44-
"_metadata": {"filename": "test_filename"},
45-
}
46-
47-
48-
class TestConfigVal(unittest.TestCase):
49-
def test_validate_dataset_config(self):
50-
self.assertEqual(validate_dataset_config(good_config), None)
51-
52-
def test_bad_validate_dataset_config(self):
53-
with self.assertRaises(KeyError):
54-
validate_dataset_config(bad_config)
55-
56-
57-
class TestVerifyNoRepeats(unittest.TestCase):
58-
def test_verify_no_repeats_no_duplicates(self):
59-
configs = [
60-
{
61-
"properties": {"name": "dataset_one"},
62-
"_metadata": {"filename": "file1.toml"},
63-
},
64-
{
65-
"properties": {"name": "dataset_two"},
66-
"_metadata": {"filename": "file2.toml"},
67-
},
68-
]
69-
# Should not raise
70-
self.assertIsNone(verify_no_repeats(configs))
71-
72-
def test_verify_no_repeats_with_duplicates(self):
73-
configs = [
74-
{
75-
"properties": {"name": "dataset_one"},
76-
"_metadata": {"filename": "file1.toml"},
77-
},
78-
{
79-
"properties": {"name": "dataset_one"},
80-
"_metadata": {"filename": "file2.toml"},
81-
},
82-
]
83-
with self.assertRaises(AttributeError) as exc:
84-
verify_no_repeats(configs)
85-
assert "dataset_one" in str(exc.exception)
86-
assert "file1.toml" in str(exc.exception)
87-
assert "file2.toml" in str(exc.exception)
1+
import unittest
2+
3+
from cfa.scenarios.dataops.datasets.config_validator import (
4+
validate_dataset_config,
5+
verify_no_repeats,
6+
)
7+
8+
good_config = {
9+
"properties": {
10+
"name": "test_dataset_name",
11+
"automate": False,
12+
"transform_template": "test_tf.sql",
13+
"schema": "datasets/schemas/tests.py",
14+
},
15+
"source": {"url": "https://data.csv", "pagination": {"limit": 1000}},
16+
"extract": {
17+
"account": "test_account_raw",
18+
"container": "test_container_raw",
19+
"prefix": "dataops/scenarios/raw/test",
20+
},
21+
"load": {
22+
"account": "test_account_tf",
23+
"container": "test_container_tf",
24+
"prefix": "dataops/scenarios/transformed/test",
25+
},
26+
"_metadata": {"filename": "test_filename"},
27+
}
28+
bad_config = {
29+
"properties": {
30+
"name": "test_dataset_name",
31+
"automate": False,
32+
"transform_template": "test_tf.sql",
33+
"schema": "datasets/schemas/tests.py",
34+
},
35+
"source": {"url": "https://data.csv", "pagination": {"limit": 1000}},
36+
"extract": {
37+
"account": "test_account_raw",
38+
"container": "test_container_raw",
39+
},
40+
"load": {
41+
"account": "test_account_tf",
42+
"container": "test_container_tf",
43+
},
44+
"_metadata": {"filename": "test_filename"},
45+
}
46+
47+
48+
class TestConfigVal(unittest.TestCase):
49+
def test_validate_dataset_config(self):
50+
self.assertEqual(validate_dataset_config(good_config), None)
51+
52+
def test_bad_validate_dataset_config(self):
53+
with self.assertRaises(KeyError):
54+
validate_dataset_config(bad_config)
55+
56+
57+
class TestVerifyNoRepeats(unittest.TestCase):
58+
def test_verify_no_repeats_no_duplicates(self):
59+
configs = [
60+
{
61+
"properties": {"name": "dataset_one"},
62+
"_metadata": {"filename": "file1.toml"},
63+
},
64+
{
65+
"properties": {"name": "dataset_two"},
66+
"_metadata": {"filename": "file2.toml"},
67+
},
68+
]
69+
# Should not raise
70+
self.assertIsNone(verify_no_repeats(configs))
71+
72+
def test_verify_no_repeats_with_duplicates(self):
73+
configs = [
74+
{
75+
"properties": {"name": "dataset_one"},
76+
"_metadata": {"filename": "file1.toml"},
77+
},
78+
{
79+
"properties": {"name": "dataset_one"},
80+
"_metadata": {"filename": "file2.toml"},
81+
},
82+
]
83+
with self.assertRaises(AttributeError) as exc:
84+
verify_no_repeats(configs)
85+
assert "dataset_one" in str(exc.exception)
86+
assert "file1.toml" in str(exc.exception)
87+
assert "file2.toml" in str(exc.exception)

0 commit comments

Comments
 (0)