diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 00000000..17162841 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,19 @@ +name: Unit tests + +on: + - pull_request + +jobs: + sqlite-import: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + - name: Install pytest + run: pip install pytest + - name: Verify MIMIC-III SQLite CSV discovery + run: pytest tests/test_mimic_iii_sqlite_import.py -q diff --git a/mimic-iii/buildmimic/sqlite/README.md b/mimic-iii/buildmimic/sqlite/README.md index 5c8fbdf3..2dcca611 100644 --- a/mimic-iii/buildmimic/sqlite/README.md +++ b/mimic-iii/buildmimic/sqlite/README.md @@ -34,7 +34,9 @@ If you are using the `import.py` script, it may be necessary to make minor edits to the `import.py` script. For example: - If you are loading the demo, you may need to change `ROW_ID` to lowercase. -- If your files are `.csv` rather than `csv.gz`, you will need to change `csv.gz` to `csv`. + +`import.py` discovers both plain `.csv` and `.csv.gz` files automatically and +prefers `.csv.gz` when both exist for the same table (matching `import.sh`). ## Step 3: Generate the SQLite file diff --git a/mimic-iii/buildmimic/sqlite/import.py b/mimic-iii/buildmimic/sqlite/import.py index 801651c7..256c73d9 100644 --- a/mimic-iii/buildmimic/sqlite/import.py +++ b/mimic-iii/buildmimic/sqlite/import.py @@ -2,7 +2,6 @@ import sys from glob import glob -import pandas as pd DATABASE_NAME = "mimic3.db" @@ -21,21 +20,39 @@ def _table_name_from_csv(filename: str) -> str: return name.lower() -if os.path.exists(DATABASE_NAME): - msg = "File {} already exists.".format(DATABASE_NAME) - print(msg) - sys.exit() - -for f in glob("*.csv.gz"): - print("Starting processing {}".format(f)) - table = _table_name_from_csv(f) - if os.path.getsize(f) < THRESHOLD_SIZE: - df = pd.read_csv(f, index_col="ROW_ID") - df.to_sql(table, CONNECTION_STRING) - else: - # If the file is too large, let's do the work in chunks - for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE): - chunk.to_sql(table, CONNECTION_STRING, if_exists="append") - print("Finished processing {}".format(f)) - -print("Should be all done!") +def _data_files_by_table(csv_paths=None, csv_gz_paths=None): + """Map table name → path, preferring .csv.gz when both exist. + + ``import.sh`` already accepts plain ``.csv`` and ``.csv.gz``; this loader + previously only globbed ``*.csv.gz`` and would silently skip uncompressed + extracts. + """ + files_by_table = {} + for path in csv_paths if csv_paths is not None else glob("*.csv"): + files_by_table[_table_name_from_csv(path)] = path + for path in csv_gz_paths if csv_gz_paths is not None else glob("*.csv.gz"): + files_by_table[_table_name_from_csv(path)] = path + return files_by_table + + +if __name__ == "__main__": + import pandas as pd + + if os.path.exists(DATABASE_NAME): + msg = "File {} already exists.".format(DATABASE_NAME) + print(msg) + sys.exit() + + files_by_table = _data_files_by_table() + for table, f in sorted(files_by_table.items()): + print("Starting processing {}".format(f)) + if os.path.getsize(f) < THRESHOLD_SIZE: + df = pd.read_csv(f, index_col="ROW_ID") + df.to_sql(table, CONNECTION_STRING) + else: + # If the file is too large, let's do the work in chunks + for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE): + chunk.to_sql(table, CONNECTION_STRING, if_exists="append") + print("Finished processing {}".format(f)) + + print("Should be all done!") diff --git a/styleguide.md b/styleguide.md index 14eca419..fa415e45 100644 --- a/styleguide.md +++ b/styleguide.md @@ -27,3 +27,7 @@ For more detail, following the guidelines at: http://www.sqlstyle.guide/ ## Python Following PEP8 guidelines is recommended. Read more here: https://www.python.org/dev/peps/pep-0008/ + +## Unit tests + +Lightweight helpers (for example MIMIC-III SQLite CSV discovery) are checked on every pull request by `.github/workflows/unit-tests.yml`. diff --git a/tests/test_mimic_iii_sqlite_import.py b/tests/test_mimic_iii_sqlite_import.py new file mode 100644 index 00000000..b6a11538 --- /dev/null +++ b/tests/test_mimic_iii_sqlite_import.py @@ -0,0 +1,50 @@ +"""Unit tests for MIMIC-III SQLite CSV discovery helpers.""" + +import importlib.util +import sys +from pathlib import Path + + +def _load_import_module(): + path = ( + Path(__file__).resolve().parents[1] + / "mimic-iii" + / "buildmimic" + / "sqlite" + / "import.py" + ) + spec = importlib.util.spec_from_file_location("mimic_iii_sqlite_import", path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + # Avoid executing the __main__ import path; load definitions only. + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_table_name_from_csv_literal_suffix(): + mod = _load_import_module() + assert mod._table_name_from_csv("ADMISSIONS.csv") == "admissions" + assert mod._table_name_from_csv("ADMISSIONS.csv.gz") == "admissions" + # Literal suffix: do not strip trailing characters that happen to match + # letters inside ".csv" (historical str.strip bug class). + assert mod._table_name_from_csv("notacsvfile.csv") == "notacsvfile" + + +def test_data_files_prefer_csv_gz_over_plain_csv(): + mod = _load_import_module() + mapping = mod._data_files_by_table( + csv_paths=["ADMISSIONS.csv", "PATIENTS.csv"], + csv_gz_paths=["ADMISSIONS.csv.gz"], + ) + assert mapping["admissions"] == "ADMISSIONS.csv.gz" + assert mapping["patients"] == "PATIENTS.csv" + + +def test_data_files_include_plain_csv_only_tables(): + mod = _load_import_module() + mapping = mod._data_files_by_table( + csv_paths=["CHARTEVENTS.csv"], + csv_gz_paths=[], + ) + assert mapping == {"chartevents": "CHARTEVENTS.csv"}