Merge pull request #2035 from MIT-LCP/alistair/more_gh_action_shenanigans

Fix GH actions and make concepts deterministic

Author: alistairewj

.github/actions/download-demo/action.yml

@@ -10,8 +10,6 @@ inputs:
 runs:
   using: "composite"
   steps:
-    # The demo datasets are version-pinned and immutable, so a static key never
-    # goes stale
     - name: Restore demo data from cache
       id: cache
       uses: actions/cache@v4
@@ -27,6 +25,10 @@ runs:
       shell: bash
       working-directory: ${{ inputs.dest }}
       run: |
+        set -euo pipefail
         echo "Downloading MIMIC-IV and MIMIC-IV-ED demo from PhysioNet."
-        wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/
-        wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/
+        # -4 forces IPv4
+        # -c resumes any partially downloaded files on retry.
+        wget_opts="-4 --tries=5 --retry-connrefused --waitretry=10 --timeout=60"
+        wget -r -N -c -q $wget_opts --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/
+        wget -r -N -c -q $wget_opts --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/

.github/actions/setup-duckdb/action.yml

@@ -13,7 +13,11 @@ runs:
     - name: Install DuckDB CLI
       shell: bash
       run: |
-        wget -q "https://github.com/duckdb/duckdb/releases/download/v${{ inputs.version }}/duckdb_cli-linux-amd64.zip"
+        set -euo pipefail
+        url="https://github.com/duckdb/duckdb/releases/download/v${{ inputs.version }}/duckdb_cli-linux-amd64.zip"
+        # -4 forces IPv4, plus retries to survive transient network failures
+        wget -q -4 --tries=5 --retry-connrefused --waitretry=10 --timeout=60 \
+          -O duckdb_cli-linux-amd64.zip "$url"
         unzip -o duckdb_cli-linux-amd64.zip -d /usr/local/bin
         rm duckdb_cli-linux-amd64.zip
         duckdb --version

.github/workflows/build-psql.yml

@@ -14,6 +14,9 @@ jobs:
         image: postgres:17
         env:
           POSTGRES_PASSWORD: postgres
+          # Force byte-order (C) collation so string ordering matches DuckDB's
+          # binary sort.
+          POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C"
         options: >-
           --health-cmd pg_isready
           --health-interval 10s

.github/workflows/concepts-psql.yml

@@ -15,6 +15,9 @@ jobs:
         image: postgres:17
         env:
           POSTGRES_PASSWORD: postgres
+          # Force byte-order (C) collation so string ordering matches DuckDB's
+          # binary sort.
+          POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C"
         options: >-
           --health-cmd pg_isready
           --health-interval 10s

.github/workflows/equivalence.yml

@@ -5,8 +5,8 @@ on:
   workflow_call:
 
 env:
-  # Tables with known cross-engine differences
-  IGNORE_TABLES: "first_day_gcs,suspicion_of_infection,oxygen_delivery,bg,first_day_bg,first_day_bg_art,kdigo_uo,kdigo_stages,urine_output_rate"
+  # If tables are known to be non-equivalent, list them here (comma delimited) to ignore them in the diff.
+  IGNORE_TABLES: ""
 
 jobs:
   diff:
@@ -18,6 +18,9 @@ jobs:
         image: postgres:17
         env:
           POSTGRES_PASSWORD: postgres
+          # Force byte-order (C) collation so string ordering matches DuckDB's
+          # binary sort.
+          POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C"
         options: >-
           --health-cmd pg_isready
           --health-interval 10s

.github/workflows/mysql.yml

@@ -18,9 +18,13 @@ jobs:
 
       - name: Extract demo data to local folder
         run: |
-          mv hosp/*.csv.gz ./
-          mv icu/*.csv.gz ./
-          mv ed/*.csv.gz ./
+          # download-demo provides the gzipped CSVs under hosp/, icu/ and ed/.
+          # Flatten every nested CSV into the working directory, where the mysql
+          # load.sql scripts expect them (e.g. LOAD DATA INFILE 'admissions.csv').
+          # Using find keeps this robust to how deeply the files are nested.
+          # NB: copy (not move) so hosp/, icu/ and ed/ retain their contents
+          # and the cache is preserved.
+          find . -mindepth 2 -name '*.csv.gz' -exec cp -t ./ {} +
           gzip -d *.csv.gz
 
       - name: Start MySQL service

mimic-iv/concepts/demographics/weight_durations.sql

@@ -6,8 +6,7 @@ WITH wt_stg AS (
         , c.charttime
         , CASE WHEN c.itemid = 226512 THEN 'admit'
             ELSE 'daily' END AS weight_type
-        -- TODO: eliminate obvious outliers if there is a reasonable weight
-        , c.valuenum AS weight
+        , ROUND(CAST(c.valuenum AS NUMERIC), 3) AS weight
     FROM `physionet-data.mimiciv_icu.chartevents` c
     WHERE c.valuenum IS NOT NULL
         AND c.itemid IN
@@ -16,6 +15,7 @@ WITH wt_stg AS (
             , 224639 -- Daily Weight
         )
         AND c.valuenum > 0
+        AND c.valuenum < 1500
 )
 
 -- assign ascending row number

mimic-iv/concepts/firstday/first_day_gcs.sql

@@ -21,10 +21,13 @@ WITH gcs_final AS (
         , g.gcs_unable
         -- This sorts the data by GCS
         -- rn = 1 is the the lowest total GCS value
+        -- tie-break on charttime (unique per stay) so the component columns
+        -- are deterministic across SQL engines when multiple measurements
+        -- share the same minimum GCS
         , ROW_NUMBER() OVER
         (
             PARTITION BY g.stay_id
-            ORDER BY g.gcs
+            ORDER BY g.gcs ASC NULLS LAST, g.charttime DESC NULLS LAST
         ) AS gcs_seq
     FROM `physionet-data.mimiciv_icu.icustays` ie
     -- Only get data for the first 24 hours

mimic-iv/concepts/firstday/first_day_rrt.sql

@@ -5,7 +5,8 @@ SELECT
     , ie.stay_id
     , MAX(dialysis_present) AS dialysis_present
     , MAX(dialysis_active) AS dialysis_active
-    , STRING_AGG(DISTINCT dialysis_type, ', ') AS dialysis_type
+    , STRING_AGG(DISTINCT dialysis_type, ', ' ORDER BY dialysis_type)
+        AS dialysis_type
 FROM `physionet-data.mimiciv_icu.icustays` ie
 LEFT JOIN `physionet-data.mimiciv_derived.rrt` rrt
     ON ie.stay_id = rrt.stay_id

mimic-iv/concepts/make_concepts.sh

@@ -63,7 +63,7 @@ do
 
             # skip certain tables where order matters
             skip=0
-            for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_eqivalent_dose
+            for skip_table in meld icustay_times first_day_sofa kdigo_stages vasoactive_agent norepinephrine_equivalent_dose sepsis3
             do
               if [[ "${tbl}" == "${skip_table}" ]]; then
                 skip=1
@@ -83,7 +83,7 @@ done
 
 echo "Now generating tables which were skipped due to depending on other tables."
 # generate tables after the above, and in a specific order to ensure dependencies are met
-for table_path in firstday/first_day_sofa organfailure/kdigo_stages organfailure/meld medication/vasoactive_agent medication/norepinephrine_equivalent_dose;
+for table_path in firstday/first_day_sofa organfailure/kdigo_stages organfailure/meld medication/vasoactive_agent medication/norepinephrine_equivalent_dose sepsis/sepsis3;
 do
   table=`echo $table_path | rev | cut -d/ -f1 | rev`