Skip to content

Commit d408c8b

Browse files
mmcdermottmmcdermott
authored
Merge pull request #38 from Medical-Event-Data-Standard/codex/update-sample-datasets-with-correct-meds-codes
Use MEDS birth code in sample data
2 parents 59d2967 + 7035bc0 commit d408c8b

4 files changed

Lines changed: 20 additions & 21 deletions

File tree

‎src/meds_testing_helpers/dataset.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -999,7 +999,7 @@ def from_yaml(cls, yaml: str | Path, **schema_overrides) -> "MEDSDataset":
999999
----
10001000
subject_id: [[239684,239684,239684,239684,239684,...,1195293,1195293,1195293,1195293,1195293],[1195293]]
10011001
time: [[null,null,1980-12-28 00:00:00.000000,2010-05-11 17:41:51.000000,2010-05-11 17:41:51.000000,...,2010-06-20 20:12:31.000000,2010-06-20 20:24:44.000000,2010-06-20 20:24:44.000000,2010-06-20 20:41:33.000000,2010-06-20 20:41:33.000000],[2010-06-20 20:50:04.000000]]
1002-
code: [["EYE_COLOR//BROWN","HEIGHT","DOB","ADMISSION//CARDIAC","HR",...,"TEMP","HR","TEMP","HR","TEMP"],["DISCHARGE"]]
1002+
code: [["EYE_COLOR//BROWN","HEIGHT","MEDS_BIRTH","ADMISSION//CARDIAC","HR",...,"TEMP","HR","TEMP","HR","TEMP"],["DISCHARGE"]]
10031003
numeric_value: [[null,175.27112,null,null,102.6,...,99.8,107.7,100,107.5,100.4],[null]]
10041004
- train/1:
10051005
pyarrow.Table
@@ -1010,7 +1010,7 @@ def from_yaml(cls, yaml: str | Path, **schema_overrides) -> "MEDSDataset":
10101010
----
10111011
subject_id: [[68729,68729,68729,68729,68729,...,814703,814703,814703,814703,814703],[814703]]
10121012
time: [[null,null,1978-03-09 00:00:00.000000,2010-05-26 02:30:56.000000,2010-05-26 02:30:56.000000,...,null,1976-03-28 00:00:00.000000,2010-02-05 05:55:39.000000,2010-02-05 05:55:39.000000,2010-02-05 05:55:39.000000],[2010-02-05 07:02:30.000000]]
1013-
code: [["EYE_COLOR//HAZEL","HEIGHT","DOB","ADMISSION//PULMONARY","HR",...,"HEIGHT","DOB","ADMISSION//ORTHOPEDIC","HR","TEMP"],["DISCHARGE"]]
1013+
code: [["EYE_COLOR//HAZEL","HEIGHT","MEDS_BIRTH","ADMISSION//PULMONARY","HR",...,"HEIGHT","MEDS_BIRTH","ADMISSION//ORTHOPEDIC","HR","TEMP"],["DISCHARGE"]]
10141014
numeric_value: [[null,160.39531,null,null,86,...,156.4856,null,null,170.2,100.1],[null]]
10151015
- tuning/0:
10161016
pyarrow.Table
@@ -1021,7 +1021,7 @@ def from_yaml(cls, yaml: str | Path, **schema_overrides) -> "MEDSDataset":
10211021
----
10221022
subject_id: [[754281,754281,754281,754281,754281,754281],[754281]]
10231023
time: [[null,null,1988-12-19 00:00:00.000000,2010-01-03 06:27:59.000000,2010-01-03 06:27:59.000000,2010-01-03 06:27:59.000000],[2010-01-03 08:22:13.000000]]
1024-
code: [["EYE_COLOR//BROWN","HEIGHT","DOB","ADMISSION//PULMONARY","HR","TEMP"],["DISCHARGE"]]
1024+
code: [["EYE_COLOR//BROWN","HEIGHT","MEDS_BIRTH","ADMISSION//PULMONARY","HR","TEMP"],["DISCHARGE"]]
10251025
numeric_value: [[null,166.22261,null,null,142,99.8],[null]]
10261026
- held_out/0:
10271027
pyarrow.Table
@@ -1032,7 +1032,7 @@ def from_yaml(cls, yaml: str | Path, **schema_overrides) -> "MEDSDataset":
10321032
----
10331033
subject_id: [[1500733,1500733,1500733,1500733,1500733,1500733,1500733,1500733,1500733,1500733],[1500733]]
10341034
time: [[null,null,1986-07-20 00:00:00.000000,2010-06-03 14:54:38.000000,2010-06-03 14:54:38.000000,2010-06-03 14:54:38.000000,2010-06-03 15:39:49.000000,2010-06-03 15:39:49.000000,2010-06-03 16:20:49.000000,2010-06-03 16:20:49.000000],[2010-06-03 16:44:26.000000]]
1035-
code: [["EYE_COLOR//BROWN","HEIGHT","DOB","ADMISSION//ORTHOPEDIC","HR","TEMP","HR","TEMP","HR","TEMP"],["DISCHARGE"]]
1035+
code: [["EYE_COLOR//BROWN","HEIGHT","MEDS_BIRTH","ADMISSION//ORTHOPEDIC","HR","TEMP","HR","TEMP","HR","TEMP"],["DISCHARGE"]]
10361036
numeric_value: [[null,158.60132,null,null,91.4,100,84.4,100.3,90.1,100.1],[null]]
10371037
code_metadata:
10381038
pyarrow.Table
@@ -1194,7 +1194,7 @@ def from_yaml(cls, yaml: str | Path, **schema_overrides) -> "MEDSDataset":
11941194
----
11951195
subject_id: [[239684,239684,239684,239684,239684,...,1195293,1195293,1195293,1195293,1195293],[1195293]]
11961196
time: [[null,null,1980-12-28 00:00:00.000000,2010-05-11 17:41:51.000000,2010-05-11 17:41:51.000000,...,2010-06-20 20:12:31.000000,2010-06-20 20:24:44.000000,2010-06-20 20:24:44.000000,2010-06-20 20:41:33.000000,2010-06-20 20:41:33.000000],[2010-06-20 20:50:04.000000]]
1197-
code: [["EYE_COLOR//BROWN","HEIGHT","DOB","ADMISSION//CARDIAC","HR",...,"TEMP","HR","TEMP","HR","TEMP"],["DISCHARGE"]]
1197+
code: [["EYE_COLOR//BROWN","HEIGHT","MEDS_BIRTH","ADMISSION//CARDIAC","HR",...,"TEMP","HR","TEMP","HR","TEMP"],["DISCHARGE"]]
11981198
numeric_value: [[null,175.27112,null,null,102.6,...,99.8,107.7,100,107.5,100.4],[null]]
11991199
- train/1:
12001200
pyarrow.Table
@@ -1205,7 +1205,7 @@ def from_yaml(cls, yaml: str | Path, **schema_overrides) -> "MEDSDataset":
12051205
----
12061206
subject_id: [[68729,68729,68729,68729,68729,...,814703,814703,814703,814703,814703],[814703]]
12071207
time: [[null,null,1978-03-09 00:00:00.000000,2010-05-26 02:30:56.000000,2010-05-26 02:30:56.000000,...,null,1976-03-28 00:00:00.000000,2010-02-05 05:55:39.000000,2010-02-05 05:55:39.000000,2010-02-05 05:55:39.000000],[2010-02-05 07:02:30.000000]]
1208-
code: [["EYE_COLOR//HAZEL","HEIGHT","DOB","ADMISSION//PULMONARY","HR",...,"HEIGHT","DOB","ADMISSION//ORTHOPEDIC","HR","TEMP"],["DISCHARGE"]]
1208+
code: [["EYE_COLOR//HAZEL","HEIGHT","MEDS_BIRTH","ADMISSION//PULMONARY","HR",...,"HEIGHT","MEDS_BIRTH","ADMISSION//ORTHOPEDIC","HR","TEMP"],["DISCHARGE"]]
12091209
numeric_value: [[null,160.39531,null,null,86,...,156.4856,null,null,170.2,100.1],[null]]
12101210
- tuning/0:
12111211
pyarrow.Table
@@ -1216,7 +1216,7 @@ def from_yaml(cls, yaml: str | Path, **schema_overrides) -> "MEDSDataset":
12161216
----
12171217
subject_id: [[754281,754281,754281,754281,754281,754281],[754281]]
12181218
time: [[null,null,1988-12-19 00:00:00.000000,2010-01-03 06:27:59.000000,2010-01-03 06:27:59.000000,2010-01-03 06:27:59.000000],[2010-01-03 08:22:13.000000]]
1219-
code: [["EYE_COLOR//BROWN","HEIGHT","DOB","ADMISSION//PULMONARY","HR","TEMP"],["DISCHARGE"]]
1219+
code: [["EYE_COLOR//BROWN","HEIGHT","MEDS_BIRTH","ADMISSION//PULMONARY","HR","TEMP"],["DISCHARGE"]]
12201220
numeric_value: [[null,166.22261,null,null,142,99.8],[null]]
12211221
- held_out/0:
12221222
pyarrow.Table
@@ -1227,7 +1227,7 @@ def from_yaml(cls, yaml: str | Path, **schema_overrides) -> "MEDSDataset":
12271227
----
12281228
subject_id: [[1500733,1500733,1500733,1500733,1500733,1500733,1500733,1500733,1500733,1500733],[1500733]]
12291229
time: [[null,null,1986-07-20 00:00:00.000000,2010-06-03 14:54:38.000000,2010-06-03 14:54:38.000000,2010-06-03 14:54:38.000000,2010-06-03 15:39:49.000000,2010-06-03 15:39:49.000000,2010-06-03 16:20:49.000000,2010-06-03 16:20:49.000000],[2010-06-03 16:44:26.000000]]
1230-
code: [["EYE_COLOR//BROWN","HEIGHT","DOB","ADMISSION//ORTHOPEDIC","HR","TEMP","HR","TEMP","HR","TEMP"],["DISCHARGE"]]
1230+
code: [["EYE_COLOR//BROWN","HEIGHT","MEDS_BIRTH","ADMISSION//ORTHOPEDIC","HR","TEMP","HR","TEMP","HR","TEMP"],["DISCHARGE"]]
12311231
numeric_value: [[null,158.60132,null,null,91.4,100,84.4,100.3,90.1,100.1],[null]]
12321232
code_metadata:
12331233
pyarrow.Table

‎src/meds_testing_helpers/static_sample_data/README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ This is a legacy dataset used in previous testing code. It is currently in use,
1212
it likely to be deprecated soon. These include:
1313

1414
- It is sharded by split, rather than relying on the more general `metadata/subject_splits.parquet` file.
15-
- It does not use the proper birth or death codes.
1615

1716
This dataset can also be generated under the generation config `dataset_spec/data_generator: sample`
1817

‎src/meds_testing_helpers/static_sample_data/simple_static_sharded_by_split.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ data/train/0.parquet: |2-
22
subject_id,time,code,numeric_value
33
239684,,EYE_COLOR//BROWN,
44
239684,,HEIGHT,175.271115221764
5-
239684,"12/28/1980, 00:00:00",DOB,
5+
239684,"12/28/1980, 00:00:00",MEDS_BIRTH,
66
239684,"05/11/2010, 17:41:51",ADMISSION//CARDIAC,
77
239684,"05/11/2010, 17:41:51",HR,102.6
88
239684,"05/11/2010, 17:41:51",TEMP,96.0
@@ -15,7 +15,7 @@ data/train/0.parquet: |2-
1515
239684,"05/11/2010, 19:27:19",DISCHARGE,
1616
1195293,,EYE_COLOR//BLUE,
1717
1195293,,HEIGHT,164.6868838269085
18-
1195293,"06/20/1978, 00:00:00",DOB,
18+
1195293,"06/20/1978, 00:00:00",MEDS_BIRTH,
1919
1195293,"06/20/2010, 19:23:52",ADMISSION//CARDIAC,
2020
1195293,"06/20/2010, 19:23:52",HR,109.0
2121
1195293,"06/20/2010, 19:23:52",TEMP,100.0
@@ -35,14 +35,14 @@ data/train/1.parquet: |2-
3535
subject_id,time,code,numeric_value
3636
68729,,EYE_COLOR//HAZEL,
3737
68729,,HEIGHT,160.3953106166676
38-
68729,"03/09/1978, 00:00:00",DOB,
38+
68729,"03/09/1978, 00:00:00",MEDS_BIRTH,
3939
68729,"05/26/2010, 02:30:56",ADMISSION//PULMONARY,
4040
68729,"05/26/2010, 02:30:56",HR,86.0
4141
68729,"05/26/2010, 02:30:56",TEMP,97.8
4242
68729,"05/26/2010, 04:51:52",DISCHARGE,
4343
814703,,EYE_COLOR//HAZEL,
4444
814703,,HEIGHT,156.48559093209357
45-
814703,"03/28/1976, 00:00:00",DOB,
45+
814703,"03/28/1976, 00:00:00",MEDS_BIRTH,
4646
814703,"02/05/2010, 05:55:39",ADMISSION//ORTHOPEDIC,
4747
814703,"02/05/2010, 05:55:39",HR,170.2
4848
814703,"02/05/2010, 05:55:39",TEMP,100.1
@@ -52,7 +52,7 @@ data/tuning/0.parquet: |2-
5252
subject_id,time,code,numeric_value
5353
754281,,EYE_COLOR//BROWN,
5454
754281,,HEIGHT,166.22261567137025
55-
754281,"12/19/1988, 00:00:00",DOB,
55+
754281,"12/19/1988, 00:00:00",MEDS_BIRTH,
5656
754281,"01/03/2010, 06:27:59",ADMISSION//PULMONARY,
5757
754281,"01/03/2010, 06:27:59",HR,142.0
5858
754281,"01/03/2010, 06:27:59",TEMP,99.8
@@ -62,7 +62,7 @@ data/held_out/0.parquet: |2-
6262
subject_id,time,code,numeric_value
6363
1500733,,EYE_COLOR//BROWN,
6464
1500733,,HEIGHT,158.60131573580904
65-
1500733,"07/20/1986, 00:00:00",DOB,
65+
1500733,"07/20/1986, 00:00:00",MEDS_BIRTH,
6666
1500733,"06/03/2010, 14:54:38",ADMISSION//ORTHOPEDIC,
6767
1500733,"06/03/2010, 14:54:38",HR,91.4
6868
1500733,"06/03/2010, 14:54:38",TEMP,100.0

‎src/meds_testing_helpers/static_sample_data/simple_static_sharded_by_split_with_tasks.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ data/train/0.parquet: |2-
22
subject_id,time,code,numeric_value
33
239684,,EYE_COLOR//BROWN,
44
239684,,HEIGHT,175.271115221764
5-
239684,"12/28/1980, 00:00:00",DOB,
5+
239684,"12/28/1980, 00:00:00",MEDS_BIRTH,
66
239684,"05/11/2010, 17:41:51",ADMISSION//CARDIAC,
77
239684,"05/11/2010, 17:41:51",HR,102.6
88
239684,"05/11/2010, 17:41:51",TEMP,96.0
@@ -15,7 +15,7 @@ data/train/0.parquet: |2-
1515
239684,"05/11/2010, 19:27:19",DISCHARGE,
1616
1195293,,EYE_COLOR//BLUE,
1717
1195293,,HEIGHT,164.6868838269085
18-
1195293,"06/20/1978, 00:00:00",DOB,
18+
1195293,"06/20/1978, 00:00:00",MEDS_BIRTH,
1919
1195293,"06/20/2010, 19:23:52",ADMISSION//CARDIAC,
2020
1195293,"06/20/2010, 19:23:52",HR,109.0
2121
1195293,"06/20/2010, 19:23:52",TEMP,100.0
@@ -35,14 +35,14 @@ data/train/1.parquet: |2-
3535
subject_id,time,code,numeric_value
3636
68729,,EYE_COLOR//HAZEL,
3737
68729,,HEIGHT,160.3953106166676
38-
68729,"03/09/1978, 00:00:00",DOB,
38+
68729,"03/09/1978, 00:00:00",MEDS_BIRTH,
3939
68729,"05/26/2010, 02:30:56",ADMISSION//PULMONARY,
4040
68729,"05/26/2010, 02:30:56",HR,86.0
4141
68729,"05/26/2010, 02:30:56",TEMP,97.8
4242
68729,"05/26/2010, 04:51:52",DISCHARGE,
4343
814703,,EYE_COLOR//HAZEL,
4444
814703,,HEIGHT,156.48559093209357
45-
814703,"03/28/1976, 00:00:00",DOB,
45+
814703,"03/28/1976, 00:00:00",MEDS_BIRTH,
4646
814703,"02/05/2010, 05:55:39",ADMISSION//ORTHOPEDIC,
4747
814703,"02/05/2010, 05:55:39",HR,170.2
4848
814703,"02/05/2010, 05:55:39",TEMP,100.1
@@ -52,7 +52,7 @@ data/tuning/0.parquet: |2-
5252
subject_id,time,code,numeric_value
5353
754281,,EYE_COLOR//BROWN,
5454
754281,,HEIGHT,166.22261567137025
55-
754281,"12/19/1988, 00:00:00",DOB,
55+
754281,"12/19/1988, 00:00:00",MEDS_BIRTH,
5656
754281,"01/03/2010, 06:27:59",ADMISSION//PULMONARY,
5757
754281,"01/03/2010, 06:27:59",HR,142.0
5858
754281,"01/03/2010, 06:27:59",TEMP,99.8
@@ -62,7 +62,7 @@ data/held_out/0.parquet: |2-
6262
subject_id,time,code,numeric_value
6363
1500733,,EYE_COLOR//BROWN,
6464
1500733,,HEIGHT,158.60131573580904
65-
1500733,"07/20/1986, 00:00:00",DOB,
65+
1500733,"07/20/1986, 00:00:00",MEDS_BIRTH,
6666
1500733,"06/03/2010, 14:54:38",ADMISSION//ORTHOPEDIC,
6767
1500733,"06/03/2010, 14:54:38",HR,91.4
6868
1500733,"06/03/2010, 14:54:38",TEMP,100.0

0 commit comments

Comments
 (0)