Merge pull request #2069 from MIT-LCP/alistair/sqlglot_mimiciii

Add mimic-iii concepts to sqlglot transpilation

Author: alistairewj

.github/actions/check-concepts-up-to-date/action.yml

@@ -56,8 +56,15 @@ runs:
           echo "what the transpiler produces from the BigQuery sources. Regenerate with:"
           echo
           echo '```sh'
+          echo "# mimic-iv"
           echo "mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_postgres --destination_dialect postgres"
           echo "mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_duckdb --destination_dialect duckdb"
+          echo "# mimic-iii (see mimic-iii/concepts/README.md)"
+          echo "for d in postgres duckdb; do"
+          echo "  mimic_utils convert_folder mimic-iii/concepts mimic-iii/concepts_\$d --destination_dialect \$d \\"
+          echo "    --derived_schema mimiciii_derived --schema_map mimiciii_clinical=mimiciii,mimiciii_notes=mimiciii \\"
+          echo "    --exclude cookbook other-languages functions pivot/pivoted_oasis.sql"
+          echo "done"
           echo '```'
           echo
           echo "…and commit the result, or let the regeneration bot open a PR after this merges."

.github/actions/compare-concepts/action.yml

@@ -0,0 +1,41 @@
+name: "Compare derived concepts across engines"
+description: >-
+  Diffs the derived concepts between PostgreSQL and DuckDB with
+  mimic_utils compare_concepts. Requires mimic_utils to be installed and the
+  PG* environment variables to point at the PostgreSQL database. This action is
+  the single source of truth for the ignore list: tables known to be
+  non-equivalent are skipped here for every equivalence workflow.
+
+inputs:
+  schema:
+    description: "Derived schema to compare."
+    required: false
+    default: "mimiciv_derived"
+  db_file:
+    description: "Path to the DuckDB database file. Relative paths resolve against $GITHUB_WORKSPACE. Defaults to mimic4.db."
+    required: false
+    default: "mimic4.db"
+
+runs:
+  using: "composite"
+  steps:
+    - name: Compare derived concepts across engines
+      shell: bash
+      env:
+        # If tables are known to be non-equivalent, list them here (comma
+        # delimited) to ignore them in the diff.
+        IGNORE_TABLES: ""
+      # Resolve paths at shell time so $GITHUB_WORKSPACE points at the
+      # container-mounted path.
+      run: |
+        set -euo pipefail
+        db_file="${{ inputs.db_file }}"
+        case "$db_file" in
+          /*) ;;
+          *) db_file="$GITHUB_WORKSPACE/$db_file" ;;
+        esac
+        mimic_utils compare_concepts \
+          --pg "host=$PGHOST dbname=$PGDATABASE user=$PGUSER" \
+          --duckdb "$db_file" \
+          --schema "${{ inputs.schema }}" \
+          --ignore "$IGNORE_TABLES"

.github/actions/create-derived-schemas/action.yml

@@ -0,0 +1,37 @@
+name: "Create MIMIC-IV derived schemas"
+description: >-
+  Creates the empty schemas that the MIMIC-IV concepts build writes into:
+  mimiciv_derived and mimiciv_ed on PostgreSQL, mimiciv_derived on DuckDB.
+  Requires the psql or duckdb CLI on PATH (PostgreSQL connection is taken from
+  the PG* environment variables).
+
+inputs:
+  engine:
+    description: "Database engine to create the schemas in (postgres or duckdb)."
+    required: true
+  db_file:
+    description: "Path to the DuckDB database file (duckdb engine only). Defaults to $GITHUB_WORKSPACE/mimic4.db."
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create derived schemas (${{ inputs.engine }})
+      shell: bash
+      run: |
+        set -euo pipefail
+        case "${{ inputs.engine }}" in
+          postgres)
+            psql -v ON_ERROR_STOP=1 -c "CREATE SCHEMA IF NOT EXISTS mimiciv_derived; CREATE SCHEMA IF NOT EXISTS mimiciv_ed;"
+            ;;
+          duckdb)
+            db_file="${{ inputs.db_file }}"
+            db_file="${db_file:-$GITHUB_WORKSPACE/mimic4.db}"
+            duckdb "$db_file" -c "CREATE SCHEMA IF NOT EXISTS mimiciv_derived;"
+            ;;
+          *)
+            echo "Unknown engine: ${{ inputs.engine }}" >&2
+            exit 1
+            ;;
+        esac

.github/actions/download-mimic3-demo/action.yml

@@ -0,0 +1,36 @@
+name: "Download MIMIC-III demo"
+description: >-
+  Downloads the MIMIC-III demo dataset (v1.4) from PhysioNet into
+  <dest>/mimiciii-demo, gzipping the CSV files so the standard gz load scripts
+  can be used. Cached across runs.
+
+inputs:
+  dest:
+    description: "Directory to download the demo data into (mimiciii-demo/ is created here)."
+    required: false
+    default: "."
+
+runs:
+  using: "composite"
+  steps:
+    - name: Restore demo data from cache
+      id: cache
+      uses: actions/cache@v4
+      with:
+        path: ${{ inputs.dest }}/mimiciii-demo
+        key: mimic-iii-demo-1.4
+
+    - name: Download demo data from PhysioNet
+      if: steps.cache.outputs.cache-hit != 'true'
+      shell: bash
+      working-directory: ${{ inputs.dest }}
+      run: |
+        set -euo pipefail
+        echo "Downloading the MIMIC-III demo from PhysioNet."
+        # -4 forces IPv4
+        # -c resumes any partially downloaded files on retry.
+        wget_opts="-4 --tries=5 --retry-connrefused --waitretry=10 --timeout=60"
+        wget -r -N -c -q $wget_opts --accept "*.csv" -nH -np --cut-dirs=3 \
+          -P mimiciii-demo https://physionet.org/files/mimiciii-demo/1.4/
+        gzip -f mimiciii-demo/*.csv
+        ls mimiciii-demo

.github/actions/load-mimic3-duckdb/action.yml

@@ -0,0 +1,37 @@
+name: "Load MIMIC-III demo into DuckDB"
+description: >-
+  Builds a DuckDB database with the demo data loaded into a mimiciii schema
+  (plus an empty mimiciii_derived schema). Requires the duckdb CLI on PATH.
+
+inputs:
+  mimic_data_dir:
+    description: "Directory containing the demo .csv.gz files. Defaults to $GITHUB_WORKSPACE/mimiciii-demo."
+    required: false
+    default: ""
+  db_file:
+    description: "Path to the DuckDB database file to create. Defaults to $GITHUB_WORKSPACE/mimic3.db."
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: Build DuckDB database
+      shell: bash
+      # Resolve paths at shell time so $GITHUB_WORKSPACE points at the
+      # container-mounted path.
+      run: |
+        set -euo pipefail
+        data_dir="${{ inputs.mimic_data_dir }}"
+        data_dir="${data_dir:-$GITHUB_WORKSPACE/mimiciii-demo}"
+        db_file="${{ inputs.db_file }}"
+        db_file="${db_file:-$GITHUB_WORKSPACE/mimic3.db}"
+        duckdb "$db_file" -c "CREATE SCHEMA IF NOT EXISTS mimiciii; CREATE SCHEMA IF NOT EXISTS mimiciii_derived;"
+        # create the tables inside the mimiciii schema using the standard DDL
+        { echo "USE mimiciii;"; cat "$GITHUB_WORKSPACE/mimic-iii/buildmimic/duckdb/duckdb_add_tables.sql"; } \
+          | duckdb "$db_file"
+        for f in "$data_dir"/*.csv.gz; do
+          tbl="$(basename "$f" | cut -d. -f1 | tr '[:upper:]' '[:lower:]')"
+          echo "Loading $tbl"
+          duckdb "$db_file" -c "COPY mimiciii.$tbl FROM '$f' (HEADER, DELIM ',', QUOTE '\"', ESCAPE '\"');"
+        done

.github/actions/load-mimic3-psql/action.yml

@@ -0,0 +1,30 @@
+name: "Load MIMIC-III demo into PostgreSQL"
+description: >-
+  Creates the mimiciii and mimiciii_derived schemas and loads the demo data
+  into mimiciii. Requires psql on PATH and PG* connection env vars.
+
+inputs:
+  mimic_data_dir:
+    description: "Directory containing the demo .csv.gz files. Defaults to $GITHUB_WORKSPACE/mimiciii-demo."
+    required: false
+    default: ""
+
+runs:
+  using: "composite"
+  steps:
+    - name: Create schemas and load demo data
+      shell: bash
+      # Resolve the data dir at shell time so $GITHUB_WORKSPACE points at the
+      # container-mounted path. The github.workspace context yields the host
+      # path, which does not exist inside a container job.
+      run: |
+        set -euo pipefail
+        data_dir="${{ inputs.mimic_data_dir }}"
+        data_dir="${data_dir:-$GITHUB_WORKSPACE/mimiciii-demo}"
+        psql -q -v ON_ERROR_STOP=1 \
+          -c "CREATE SCHEMA IF NOT EXISTS mimiciii; CREATE SCHEMA IF NOT EXISTS mimiciii_derived;"
+        export PGOPTIONS="--search_path=mimiciii"
+        psql -q -v ON_ERROR_STOP=1 \
+          -f "$GITHUB_WORKSPACE/mimic-iii/buildmimic/postgres/postgres_create_tables.sql"
+        psql -q -v ON_ERROR_STOP=1 -v mimic_data_dir="$data_dir" \
+          -f "$GITHUB_WORKSPACE/mimic-iii/buildmimic/postgres/postgres_load_data_gz.sql"

.github/actions/transpile-concepts/action.yml

@@ -1,22 +1,46 @@
 name: "Transpile BigQuery concepts"
 description: >-
   Regenerates the concepts for a SQL dialect from the BigQuery sources, writing
-  over the committed mimic-iv/concepts_<dialect> folder. Downstream steps then
-  build/test against the freshly transpiled SQL rather than what is committed.
-  Requires mimic_utils to be installed (pip install -e .). This action does not
-  report drift; use the check-concepts-up-to-date action for that.
+  over the committed concepts_<dialect> folder of the given project. Downstream
+  steps then build/test against the freshly transpiled SQL rather than what is
+  committed. Requires mimic_utils to be installed (pip install -e .). This
+  action does not report drift; use the check-concepts-up-to-date action for
+  that.
 
 inputs:
   dialect:
     description: "Destination SQL dialect (postgres or duckdb)."
     required: true
+  project:
+    description: "Project whose concepts are transpiled (mimic-iii or mimic-iv)."
+    required: false
+    default: "mimic-iv"
 
 runs:
   using: "composite"
   steps:
-    - name: Transpile ${{ inputs.dialect }} concepts
+    - name: Transpile ${{ inputs.project }} ${{ inputs.dialect }} concepts
       shell: bash
       run: |
-        mimic_utils convert_folder mimic-iv/concepts \
-          "mimic-iv/concepts_${{ inputs.dialect }}" \
-          --destination_dialect "${{ inputs.dialect }}"
+        set -euo pipefail
+        case "${{ inputs.project }}" in
+          mimic-iv)
+            mimic_utils convert_folder mimic-iv/concepts \
+              "mimic-iv/concepts_${{ inputs.dialect }}" \
+              --destination_dialect "${{ inputs.dialect }}"
+            ;;
+          mimic-iii)
+            # pivot/pivoted_oasis.sql is excluded: it has never been runnable
+            # on any engine (undefined aliases/columns and a missing CTE).
+            mimic_utils convert_folder mimic-iii/concepts \
+              "mimic-iii/concepts_${{ inputs.dialect }}" \
+              --destination_dialect "${{ inputs.dialect }}" \
+              --derived_schema mimiciii_derived \
+              --schema_map mimiciii_clinical=mimiciii,mimiciii_notes=mimiciii \
+              --exclude cookbook other-languages functions pivot/pivoted_oasis.sql
+            ;;
+          *)
+            echo "Unknown project: ${{ inputs.project }}" >&2
+            exit 1
+            ;;
+        esac

.github/workflows/ci.yml

@@ -1,9 +1,15 @@
-# Post-approval CI for the MIMIC-IV demo build and derived concepts.
+# Post-approval CI for the MIMIC demo builds and derived concepts.
 #
+# MIMIC-IV:
 #   psql   ─▶ psql-concepts   ─┐
 #                              ├─▶ equivalence
 #   duckdb ─▶ duckdb-concepts ─┘
 #
+# MIMIC-III (the concepts workflows load the demo themselves):
+#   psql-concepts-iii   ─┐
+#                        ├─▶ equivalence-iii
+#   duckdb-concepts-iii ─┘
+#
 # Runs on PR approval, or when a maintainer clicks "Run workflow" and picks the
 # PR's head branch (Actions tab -> CI -> Run workflow).
 # Downstream jobs inherit the gate via `needs`
@@ -34,3 +40,21 @@ jobs:
   equivalence:
     needs: [psql-concepts, duckdb-concepts]
     uses: ./.github/workflows/equivalence.yml
+
+  psql-concepts-iii:
+    if: github.event_name == 'workflow_dispatch' || github.event.review.state == 'approved'
+    uses: ./.github/workflows/concepts-iii-psql.yml
+
+  duckdb-concepts-iii:
+    if: github.event_name == 'workflow_dispatch' || github.event.review.state == 'approved'
+    uses: ./.github/workflows/concepts-iii-duckdb.yml
+
+  equivalence-iii:
+    needs: [psql-concepts-iii, duckdb-concepts-iii]
+    uses: ./.github/workflows/equivalence.yml
+    with:
+      schema: mimiciii_derived
+      pg-artifact: psql-derived-iii
+      pg-dump-file: derived-iii.sql
+      duckdb-artifact: duckdb-db-iii
+      db-file: mimic3.db

.github/workflows/concepts-duckdb.yml

@@ -25,9 +25,9 @@ jobs:
 
       - uses: ./.github/actions/load-mimic-duckdb
 
-      - name: Create derived schema
-        run: |
-          duckdb "$GITHUB_WORKSPACE/mimic4.db" -c "CREATE SCHEMA IF NOT EXISTS mimiciv_derived;"
+      - uses: ./.github/actions/create-derived-schemas
+        with:
+          engine: duckdb
 
       - uses: ./.github/actions/transpile-concepts
         with:

.github/workflows/concepts-iii-duckdb.yml

@@ -0,0 +1,41 @@
+# Transpile the MIMIC-III BigQuery concepts to DuckDB and build them against
+# the MIMIC-III demo. Publish the database file for the equivalence check.
+name: concepts duckdb (mimic-iii)
+
+on:
+  workflow_call:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container: python:3.12
+
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Install tooling
+        run: |
+          apt-get update
+          apt-get install --yes --no-install-recommends wget unzip
+          pip install -e .
+
+      - uses: ./.github/actions/setup-duckdb
+
+      - uses: ./.github/actions/download-mimic3-demo
+
+      - uses: ./.github/actions/load-mimic3-duckdb
+
+      - uses: ./.github/actions/transpile-concepts
+        with:
+          dialect: duckdb
+          project: mimic-iii
+
+      - name: Build concepts
+        working-directory: ./mimic-iii/concepts_duckdb
+        run: |
+          duckdb "$GITHUB_WORKSPACE/mimic3.db" -c ".read duckdb.sql"
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: duckdb-db-iii
+          path: mimic3.db