Skip to content

Commit c0adc1d

Browse files
alistairewjalistairewj
committed
update bq ci
1 parent fa675e7 commit c0adc1d

17 files changed

Lines changed: 744 additions & 151 deletions

File tree

‎.github/actions/download-demo/action.yml‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,32 @@
1-
name: "Download MIMIC-IV and MIMIC-IV-ED Demo"
2-
description: "Downloads MIMIC-IV and MIMIC-IV-ED demo data from PhysioNet"
1+
name: "Download MIMIC-IV and MIMIC-IV-ED demo"
2+
description: "Downloads the MIMIC-IV and MIMIC-IV-ED demo datasets from PhysioNet, caching them across runs."
3+
4+
inputs:
5+
dest:
6+
description: "Directory to download the demo data into (hosp/, icu/, ed/ are created here)."
7+
required: false
8+
default: "."
9+
310
runs:
411
using: "composite"
512
steps:
13+
# The demo datasets are version-pinned and immutable, so a static key never
14+
# goes stale
15+
- name: Restore demo data from cache
16+
id: cache
17+
uses: actions/cache@v4
18+
with:
19+
path: |
20+
${{ inputs.dest }}/hosp
21+
${{ inputs.dest }}/icu
22+
${{ inputs.dest }}/ed
23+
key: mimic-iv-demo-2.2
24+
625
- name: Download demo data from PhysioNet
26+
if: steps.cache.outputs.cache-hit != 'true'
27+
shell: bash
28+
working-directory: ${{ inputs.dest }}
729
run: |
830
echo "Downloading MIMIC-IV and MIMIC-IV-ED demo from PhysioNet."
9-
wget -r -N -c --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/
10-
wget -r -N -c --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/
11-
shell: bash
31+
wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/
32+
wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/

‎.github/actions/load-mimic-duckdb/action.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: "Load MIMIC-IV demo into DuckDB"
2+
description: "Builds a DuckDB database from the demo hosp/icu data using import_duckdb.sh. Requires the duckdb CLI on PATH."
3+
4+
inputs:
5+
mimic_data_dir:
6+
description: "Directory containing the hosp/ and icu/ demo data folders."
7+
required: false
8+
default: ${{ github.workspace }}
9+
db_file:
10+
description: "Path to the DuckDB database file to create."
11+
required: false
12+
default: ${{ github.workspace }}/mimic4.db
13+
14+
runs:
15+
using: "composite"
16+
steps:
17+
- name: Build DuckDB database
18+
shell: bash
19+
# `echo n` answers the overwrite prompt defensively
20+
run: |
21+
cd "$GITHUB_WORKSPACE/mimic-iv/buildmimic/duckdb"
22+
echo n | ./import_duckdb.sh "${{ inputs.mimic_data_dir }}" "${{ inputs.db_file }}"

‎.github/actions/load-mimic-psql/action.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: "Load MIMIC-IV demo into PostgreSQL"
2+
description: "Creates the MIMIC-IV schemas and loads the demo hosp/icu data. Requires psql on PATH and PG* connection env vars."
3+
4+
inputs:
5+
mimic_data_dir:
6+
description: "Directory containing the hosp/ and icu/ demo data folders."
7+
required: false
8+
default: ${{ github.workspace }}
9+
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Create schemas and load demo data
14+
shell: bash
15+
run: |
16+
psql -q -v ON_ERROR_STOP=1 -f "$GITHUB_WORKSPACE/mimic-iv/buildmimic/postgres/create.sql"
17+
psql -q -v ON_ERROR_STOP=1 -v mimic_data_dir="${{ inputs.mimic_data_dir }}" \
18+
-f "$GITHUB_WORKSPACE/mimic-iv/buildmimic/postgres/load_gz.sql"

‎.github/actions/setup-duckdb/action.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
name: "Set up DuckDB CLI"
2+
description: "Downloads a pinned DuckDB CLI and installs it onto PATH. Requires wget and unzip to be present."
3+
4+
inputs:
5+
version:
6+
description: "DuckDB release version to install."
7+
required: false
8+
default: "1.1.3"
9+
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Install DuckDB CLI
14+
shell: bash
15+
run: |
16+
wget -q "https://github.com/duckdb/duckdb/releases/download/v${{ inputs.version }}/duckdb_cli-linux-amd64.zip"
17+
unzip -o duckdb_cli-linux-amd64.zip -d /usr/local/bin
18+
rm duckdb_cli-linux-amd64.zip
19+
duckdb --version

‎.github/actions/transpile-concepts/action.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: "Transpile BigQuery concepts"
2+
description: >-
3+
Regenerates the concepts for a SQL dialect from the BigQuery sources and emits
4+
a visible warning if the committed files differ from the transpiler output, so
5+
it is clear the build ran against freshly transpiled (not committed) SQL.
6+
Requires mimic_utils to be installed (pip install -e .).
7+
8+
inputs:
9+
dialect:
10+
description: "Destination SQL dialect (postgres or duckdb)."
11+
required: true
12+
13+
runs:
14+
using: "composite"
15+
steps:
16+
- name: Transpile and warn if out of date
17+
shell: bash
18+
run: |
19+
target="mimic-iv/concepts_${{ inputs.dialect }}"
20+
mimic_utils convert_folder mimic-iv/concepts "$target" --destination_dialect "${{ inputs.dialect }}"
21+
22+
changes="$(git status --porcelain -- "$target")"
23+
if [ -n "$changes" ]; then
24+
echo "::warning title=Transpiled ${{ inputs.dialect }} concepts out of date::CI regenerated the ${{ inputs.dialect }} concepts from the BigQuery sources and the result differs from the committed $target. The concept build below ran against the freshly transpiled SQL, NOT what is committed. To sync: mimic_utils convert_folder mimic-iv/concepts $target --destination_dialect ${{ inputs.dialect }}"
25+
echo "Files that differ from the committed sources:"
26+
echo "$changes"
27+
git --no-pager diff -- "$target"
28+
else
29+
echo "Committed $target matches the BigQuery sources."
30+
fi

‎.github/workflows/build-duckdb.yml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Build the MIMIC-IV demo in DuckDB and validate row counts.
2+
name: build duckdb
3+
4+
on:
5+
workflow_call:
6+
7+
env:
8+
DUCKDB_FILE: ${{ github.workspace }}/mimic4.db
9+
10+
jobs:
11+
build:
12+
runs-on: ubuntu-latest
13+
container: python:3.12
14+
15+
steps:
16+
- uses: actions/checkout@v4
17+
18+
- name: Install tooling
19+
run: |
20+
apt-get update
21+
apt-get install --yes --no-install-recommends wget unzip
22+
23+
- uses: ./.github/actions/setup-duckdb
24+
25+
- uses: ./.github/actions/download-demo
26+
27+
- uses: ./.github/actions/load-mimic-duckdb
28+
with:
29+
db_file: ${{ env.DUCKDB_FILE }}
30+
31+
- name: Validate row counts
32+
# validate_demo.sql is in postgres folder but also works for duckdb
33+
run: |
34+
duckdb -csv "$DUCKDB_FILE" < mimic-iv/buildmimic/postgres/validate_demo.sql | tee results
35+
if grep -F -q "FAILED" results; then
36+
echo "::error::DuckDB demo row-count validation failed:"
37+
grep "FAILED" results
38+
exit 1
39+
fi
40+
echo "All row counts matched."

‎.github/workflows/build-psql.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Build the MIMIC-IV demo in PostgreSQL and validate row counts.
2+
name: build psql
3+
4+
on:
5+
workflow_call:
6+
7+
jobs:
8+
build:
9+
runs-on: ubuntu-latest
10+
container: python:3.12
11+
12+
services:
13+
postgres:
14+
image: postgres
15+
env:
16+
POSTGRES_PASSWORD: postgres
17+
options: >-
18+
--health-cmd pg_isready
19+
--health-interval 10s
20+
--health-timeout 5s
21+
--health-retries 5
22+
23+
env:
24+
PGHOST: postgres
25+
PGUSER: postgres
26+
PGPASSWORD: postgres
27+
PGDATABASE: postgres
28+
29+
steps:
30+
- uses: actions/checkout@v4
31+
32+
- name: Install tooling
33+
run: |
34+
apt-get update
35+
apt-get install --yes --no-install-recommends postgresql-client wget
36+
37+
- uses: ./.github/actions/download-demo
38+
39+
- uses: ./.github/actions/load-mimic-psql
40+
41+
- name: Validate row counts
42+
run: |
43+
psql -v ON_ERROR_STOP=1 -f mimic-iv/buildmimic/postgres/validate_demo.sql | tee results
44+
if grep -F -q "FAILED" results; then
45+
echo "::error::PostgreSQL demo row-count validation failed:"
46+
grep "FAILED" results
47+
exit 1
48+
fi
49+
echo "All row counts matched."

‎.github/workflows/ci.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Post-approval CI for the MIMIC-IV demo build and derived concepts.
2+
#
3+
# psql ─▶ psql-concepts ─┐
4+
# ├─▶ equivalence
5+
# duckdb ─▶ duckdb-concepts ─┘
6+
#
7+
# Runs on PR approval, or when a maintainer clicks "Run workflow" and picks the
8+
# PR's head branch (Actions tab -> CI -> Run workflow).
9+
# Downstream jobs inherit the gate via `needs`
10+
name: CI
11+
12+
on:
13+
pull_request_review:
14+
types: [submitted]
15+
workflow_dispatch:
16+
17+
jobs:
18+
psql:
19+
if: github.event_name == 'workflow_dispatch' || github.event.review.state == 'approved'
20+
uses: ./.github/workflows/build-psql.yml
21+
22+
duckdb:
23+
if: github.event_name == 'workflow_dispatch' || github.event.review.state == 'approved'
24+
uses: ./.github/workflows/build-duckdb.yml
25+
26+
psql-concepts:
27+
needs: psql
28+
uses: ./.github/workflows/concepts-psql.yml
29+
30+
duckdb-concepts:
31+
needs: duckdb
32+
uses: ./.github/workflows/concepts-duckdb.yml
33+
34+
equivalence:
35+
needs: [psql-concepts, duckdb-concepts]
36+
uses: ./.github/workflows/equivalence.yml

‎.github/workflows/concepts-duckdb.yml‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# Transpile the BigQuery concepts to DuckDB and build them.
2+
# Publish database file for checking equivalence.
3+
name: concepts duckdb
4+
5+
on:
6+
workflow_call:
7+
8+
env:
9+
DUCKDB_FILE: ${{ github.workspace }}/mimic4.db
10+
11+
jobs:
12+
build:
13+
runs-on: ubuntu-latest
14+
container: python:3.12
15+
16+
steps:
17+
- uses: actions/checkout@v4
18+
19+
- name: Install tooling
20+
run: |
21+
apt-get update
22+
apt-get install --yes --no-install-recommends wget unzip
23+
pip install -e .
24+
25+
- uses: ./.github/actions/setup-duckdb
26+
27+
- uses: ./.github/actions/download-demo
28+
29+
- uses: ./.github/actions/load-mimic-duckdb
30+
with:
31+
db_file: ${{ env.DUCKDB_FILE }}
32+
33+
- name: Create derived schema
34+
run: |
35+
duckdb "$DUCKDB_FILE" -c "CREATE SCHEMA IF NOT EXISTS mimiciv_derived;"
36+
37+
- uses: ./.github/actions/transpile-concepts
38+
with:
39+
dialect: duckdb
40+
41+
- name: Build concepts
42+
working-directory: ./mimic-iv/concepts_duckdb
43+
run: |
44+
duckdb "$DUCKDB_FILE" -c ".read duckdb.sql"
45+
46+
- uses: actions/upload-artifact@v4
47+
with:
48+
name: duckdb-db
49+
path: ${{ env.DUCKDB_FILE }}

‎.github/workflows/concepts-psql.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Transpile the BigQuery concepts to PostgreSQL and build them.
2+
# Publish database file for checking equivalence.
3+
name: concepts psql
4+
5+
on:
6+
workflow_call:
7+
8+
jobs:
9+
build:
10+
runs-on: ubuntu-latest
11+
container: python:3.12
12+
13+
services:
14+
postgres:
15+
image: postgres
16+
env:
17+
POSTGRES_PASSWORD: postgres
18+
options: >-
19+
--health-cmd pg_isready
20+
--health-interval 10s
21+
--health-timeout 5s
22+
--health-retries 5
23+
24+
env:
25+
PGHOST: postgres
26+
PGUSER: postgres
27+
PGPASSWORD: postgres
28+
PGDATABASE: postgres
29+
30+
steps:
31+
- uses: actions/checkout@v4
32+
33+
- name: Install tooling
34+
run: |
35+
apt-get update
36+
apt-get install --yes --no-install-recommends postgresql-client wget
37+
pip install -e .
38+
39+
- uses: ./.github/actions/download-demo
40+
41+
- uses: ./.github/actions/load-mimic-psql
42+
43+
- name: Create derived and ed schemas
44+
run: |
45+
psql -v ON_ERROR_STOP=1 -c "CREATE SCHEMA IF NOT EXISTS mimiciv_derived; CREATE SCHEMA IF NOT EXISTS mimiciv_ed;"
46+
47+
- uses: ./.github/actions/transpile-concepts
48+
with:
49+
dialect: postgres
50+
51+
- name: Build concepts
52+
working-directory: ./mimic-iv/concepts_postgres
53+
run: |
54+
psql -q -f postgres-functions.sql
55+
psql -q -v ON_ERROR_STOP=1 -f postgres-make-concepts.sql
56+
57+
- name: Dump derived schema
58+
run: |
59+
pg_dump --no-owner --no-privileges --schema=mimiciv_derived -f derived.sql
60+
61+
- uses: actions/upload-artifact@v4
62+
with:
63+
name: psql-derived
64+
path: derived.sql

0 commit comments

Comments
 (0)