fix(mimic-iii): load plain CSV in sqlite import.py (prefer .csv.gz)

.github/workflows/unit-tests.yml

@@ -0,0 +1,19 @@
+name: Unit tests
+
+on:
+  - pull_request
+
+jobs:
+  sqlite-import:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: "3.12"
+      - name: Install pytest
+        run: pip install pytest
+      - name: Verify MIMIC-III SQLite CSV discovery
+        run: pytest tests/test_mimic_iii_sqlite_import.py -q

mimic-iii/buildmimic/sqlite/README.md

@@ -34,7 +34,9 @@ If you are using the `import.py` script,
 it may be necessary to make minor edits to the `import.py` script. For example:
 
 - If you are loading the demo, you may need to change `ROW_ID` to lowercase.
-- If your files are `.csv` rather than `csv.gz`, you will need to change `csv.gz` to `csv`.
+
+`import.py` discovers both plain `.csv` and `.csv.gz` files automatically and
+prefers `.csv.gz` when both exist for the same table (matching `import.sh`).
 
 ## Step 3: Generate the SQLite file
 

mimic-iii/buildmimic/sqlite/import.py

@@ -2,7 +2,6 @@
 import sys
 
 from glob import glob
-import pandas as pd
 
 
 DATABASE_NAME = "mimic3.db"
@@ -21,21 +20,39 @@ def _table_name_from_csv(filename: str) -> str:
     return name.lower()
 
 
-if os.path.exists(DATABASE_NAME):
-    msg = "File {} already exists.".format(DATABASE_NAME)
-    print(msg)
-    sys.exit()
-
-for f in glob("*.csv.gz"):
-    print("Starting processing {}".format(f))
-    table = _table_name_from_csv(f)
-    if os.path.getsize(f) < THRESHOLD_SIZE:
-        df = pd.read_csv(f, index_col="ROW_ID")
-        df.to_sql(table, CONNECTION_STRING)
-    else:
-        # If the file is too large, let's do the work in chunks
-        for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE):
-            chunk.to_sql(table, CONNECTION_STRING, if_exists="append")
-    print("Finished processing {}".format(f))
-
-print("Should be all done!")
+def _data_files_by_table(csv_paths=None, csv_gz_paths=None):
+    """Map table name → path, preferring .csv.gz when both exist.
+
+    ``import.sh`` already accepts plain ``.csv`` and ``.csv.gz``; this loader
+    previously only globbed ``*.csv.gz`` and would silently skip uncompressed
+    extracts.
+    """
+    files_by_table = {}
+    for path in csv_paths if csv_paths is not None else glob("*.csv"):
+        files_by_table[_table_name_from_csv(path)] = path
+    for path in csv_gz_paths if csv_gz_paths is not None else glob("*.csv.gz"):
+        files_by_table[_table_name_from_csv(path)] = path
+    return files_by_table
+
+
+if __name__ == "__main__":
+    import pandas as pd
+
+    if os.path.exists(DATABASE_NAME):
+        msg = "File {} already exists.".format(DATABASE_NAME)
+        print(msg)
+        sys.exit()
+
+    files_by_table = _data_files_by_table()
+    for table, f in sorted(files_by_table.items()):
+        print("Starting processing {}".format(f))
+        if os.path.getsize(f) < THRESHOLD_SIZE:
+            df = pd.read_csv(f, index_col="ROW_ID")
+            df.to_sql(table, CONNECTION_STRING)
+        else:
+            # If the file is too large, let's do the work in chunks
+            for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE):
+                chunk.to_sql(table, CONNECTION_STRING, if_exists="append")
+        print("Finished processing {}".format(f))
+
+    print("Should be all done!")

styleguide.md

@@ -27,3 +27,7 @@ For more detail, following the guidelines at: http://www.sqlstyle.guide/
 ## Python
 
 Following PEP8 guidelines is recommended. Read more here: https://www.python.org/dev/peps/pep-0008/
+
+## Unit tests
+
+Lightweight helpers (for example MIMIC-III SQLite CSV discovery) are checked on every pull request by `.github/workflows/unit-tests.yml`.

tests/test_mimic_iii_sqlite_import.py

@@ -0,0 +1,50 @@
+"""Unit tests for MIMIC-III SQLite CSV discovery helpers."""
+
+import importlib.util
+import sys
+from pathlib import Path
+
+
+def _load_import_module():
+    path = (
+        Path(__file__).resolve().parents[1]
+        / "mimic-iii"
+        / "buildmimic"
+        / "sqlite"
+        / "import.py"
+    )
+    spec = importlib.util.spec_from_file_location("mimic_iii_sqlite_import", path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    # Avoid executing the __main__ import path; load definitions only.
+    sys.modules[spec.name] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_table_name_from_csv_literal_suffix():
+    mod = _load_import_module()
+    assert mod._table_name_from_csv("ADMISSIONS.csv") == "admissions"
+    assert mod._table_name_from_csv("ADMISSIONS.csv.gz") == "admissions"
+    # Literal suffix: do not strip trailing characters that happen to match
+    # letters inside ".csv" (historical str.strip bug class).
+    assert mod._table_name_from_csv("notacsvfile.csv") == "notacsvfile"
+
+
+def test_data_files_prefer_csv_gz_over_plain_csv():
+    mod = _load_import_module()
+    mapping = mod._data_files_by_table(
+        csv_paths=["ADMISSIONS.csv", "PATIENTS.csv"],
+        csv_gz_paths=["ADMISSIONS.csv.gz"],
+    )
+    assert mapping["admissions"] == "ADMISSIONS.csv.gz"
+    assert mapping["patients"] == "PATIENTS.csv"
+
+
+def test_data_files_include_plain_csv_only_tables():
+    mod = _load_import_module()
+    mapping = mod._data_files_by_table(
+        csv_paths=["CHARTEVENTS.csv"],
+        csv_gz_paths=[],
+    )
+    assert mapping == {"chartevents": "CHARTEVENTS.csv"}