Skip to content

Commit dcfa216

Browse files
alistairewjalistairewj
authored
Merge pull request #2021 from MIT-LCP/alistair/improve_bq_ci
add bq CI
2 parents a0af19c + ec0eb54 commit dcfa216

30 files changed

Lines changed: 1381 additions & 158 deletions

File tree

‎.github/actions/check-concepts-up-to-date/action.yml‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
name: "Check transpiled concepts are up to date"
2+
description: >-
3+
Compares the working tree against what is committed for the transpiled dialect
4+
folders. If they differ, the committed concepts_postgres / concepts_duckdb
5+
are stale relative to the BigQuery sources, and this action reports a NEUTRAL
6+
(yellow) check.
7+
8+
The calling job must grant `checks: write` (and `contents: read`) permissions.
9+
10+
inputs:
11+
github-token:
12+
description: "Token used to create the neutral check run."
13+
required: true
14+
check-name:
15+
description: >-
16+
Name shown for the neutral check run. This check is only created when the
17+
dialects are out of date, so the name reflects that.
18+
required: false
19+
default: "Dialects out of date with BigQuery"
20+
paths:
21+
description: "Space-separated paths to compare against the committed tree."
22+
required: false
23+
default: "mimic-iv/concepts_postgres mimic-iv/concepts_duckdb"
24+
25+
runs:
26+
using: "composite"
27+
steps:
28+
- name: Detect drift from committed concepts
29+
id: drift
30+
shell: bash
31+
run: |
32+
changes="$(git status --porcelain -- ${{ inputs.paths }})"
33+
if [ -z "$changes" ]; then
34+
echo "Committed dialect concepts match the BigQuery sources."
35+
echo "out_of_date=false" >> "$GITHUB_OUTPUT"
36+
exit 0
37+
fi
38+
39+
echo "out_of_date=true" >> "$GITHUB_OUTPUT"
40+
41+
# Split drift into tracked modifications (which have a diff) and
42+
# untracked files (which do not — git diff never shows them). Untracked
43+
# files just mean a target dialect is missing a committed concept.
44+
tracked="$(printf '%s\n' "$changes" | grep -v '^?? ' || true)"
45+
untracked="$(printf '%s\n' "$changes" | grep '^?? ' | sed 's/^?? //' || true)"
46+
47+
# Persist for the check run step (runs in a separate shell).
48+
printf '%s\n' "$tracked" > concept-drift-files.txt
49+
printf '%s\n' "$untracked" > concept-untracked-files.txt
50+
git --no-pager diff -- ${{ inputs.paths }} > concept-drift.diff
51+
52+
{
53+
echo "### :warning: Transpiled concepts are out of date"
54+
echo
55+
echo "The committed \`concepts_postgres\` / \`concepts_duckdb\` differ from"
56+
echo "what the transpiler produces from the BigQuery sources. Regenerate with:"
57+
echo
58+
echo '```sh'
59+
echo "mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_postgres --destination_dialect postgres"
60+
echo "mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_duckdb --destination_dialect duckdb"
61+
echo '```'
62+
echo
63+
echo "…and commit the result, or let the regeneration bot open a PR after this merges."
64+
echo
65+
66+
if [ -n "$tracked" ]; then
67+
echo "<details><summary>Files that differ</summary>"
68+
echo
69+
echo '```'
70+
printf '%s\n' "$tracked"
71+
echo '```'
72+
echo
73+
echo "</details>"
74+
echo
75+
echo "<details><summary>Actual differences</summary>"
76+
echo
77+
echo '```diff'
78+
cat concept-drift.diff
79+
echo '```'
80+
echo
81+
echo "</details>"
82+
echo
83+
fi
84+
85+
if [ -n "$untracked" ]; then
86+
echo "<details><summary>Untracked files (missing from a target dialect)</summary>"
87+
echo
88+
echo "These files are not committed to git, so a target dialect is missing a concept. Regenerate and commit them:"
89+
echo
90+
echo '```'
91+
printf '%s\n' "$untracked"
92+
echo '```'
93+
echo
94+
echo "</details>"
95+
fi
96+
} >> "$GITHUB_STEP_SUMMARY"
97+
98+
- name: Report neutral check run when out of date
99+
if: steps.drift.outputs.out_of_date == 'true'
100+
uses: actions/github-script@v7
101+
with:
102+
github-token: ${{ inputs.github-token }}
103+
script: |
104+
const fs = require('fs');
105+
const read = (p) => fs.existsSync(p) ? fs.readFileSync(p, 'utf8') : '';
106+
const files = read('concept-drift-files.txt').trim();
107+
const untracked = read('concept-untracked-files.txt').trim();
108+
let diff = read('concept-drift.diff');
109+
110+
// Check run output text is capped at 65535 chars; keep well under it.
111+
const LIMIT = 60000;
112+
if (diff.length > LIMIT) {
113+
diff = diff.slice(0, LIMIT) + '\n... (diff truncated) ...\n';
114+
}
115+
116+
const headSha = context.payload.pull_request
117+
? context.payload.pull_request.head.sha
118+
: context.sha;
119+
120+
const parts = [
121+
'The committed `concepts_postgres` / `concepts_duckdb` are stale relative to',
122+
'the BigQuery sources. Regenerate them by running:',
123+
'',
124+
'```sh',
125+
'mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_postgres --destination_dialect postgres',
126+
'mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_duckdb --destination_dialect duckdb',
127+
'```',
128+
'',
129+
'Then commit the result.',
130+
'',
131+
'Alternatively, there is a bot which watches the main branch and',
132+
'automatically opens a PR to regenerate the dialects after the main',
133+
'branch merges.',
134+
];
135+
136+
if (files) {
137+
parts.push(
138+
'',
139+
'### Files that differ',
140+
'```',
141+
files,
142+
'```',
143+
'',
144+
'### Actual differences',
145+
'```diff',
146+
diff,
147+
'```',
148+
);
149+
}
150+
151+
if (untracked) {
152+
parts.push(
153+
'',
154+
'### Untracked files (missing from a target dialect)',
155+
'These files are not committed to git, so a target dialect is missing a',
156+
'concept. Regenerate and commit them:',
157+
'',
158+
'```',
159+
untracked,
160+
'```',
161+
);
162+
}
163+
164+
const summary = parts.join('\n');
165+
166+
try {
167+
await github.rest.checks.create({
168+
owner: context.repo.owner,
169+
repo: context.repo.repo,
170+
name: '${{ inputs.check-name }}',
171+
head_sha: headSha,
172+
status: 'completed',
173+
conclusion: 'neutral',
174+
output: {
175+
title: 'Transpiled concepts are out of date',
176+
summary,
177+
},
178+
});
179+
} catch (error) {
180+
// The default GITHUB_TOKEN is read-only for pull requests from forks,
181+
// so it cannot create a check run. Fall back to a warning annotation
182+
// rather than failing the job red.
183+
core.warning(
184+
'Transpiled concepts are out of date with the BigQuery sources; ' +
185+
'could not post a neutral check run (' + error.message + '). ' +
186+
'Regenerate the concepts_postgres / concepts_duckdb folders and commit them.'
187+
);
188+
}

‎.github/actions/download-demo/action.yml‎

Lines changed: 26 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,32 @@
1-
name: "Download MIMIC-IV and MIMIC-IV-ED Demo"
2-
description: "Downloads MIMIC-IV and MIMIC-IV-ED demo data from PhysioNet"
1+
name: "Download MIMIC-IV and MIMIC-IV-ED demo"
2+
description: "Downloads the MIMIC-IV and MIMIC-IV-ED demo datasets from PhysioNet, caching them across runs."
3+
4+
inputs:
5+
dest:
6+
description: "Directory to download the demo data into (hosp/, icu/, ed/ are created here)."
7+
required: false
8+
default: "."
9+
310
runs:
411
using: "composite"
512
steps:
13+
# The demo datasets are version-pinned and immutable, so a static key never
14+
# goes stale
15+
- name: Restore demo data from cache
16+
id: cache
17+
uses: actions/cache@v4
18+
with:
19+
path: |
20+
${{ inputs.dest }}/hosp
21+
${{ inputs.dest }}/icu
22+
${{ inputs.dest }}/ed
23+
key: mimic-iv-demo-2.2
24+
625
- name: Download demo data from PhysioNet
26+
if: steps.cache.outputs.cache-hit != 'true'
27+
shell: bash
28+
working-directory: ${{ inputs.dest }}
729
run: |
830
echo "Downloading MIMIC-IV and MIMIC-IV-ED demo from PhysioNet."
9-
wget -r -N -c --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/
10-
wget -r -N -c --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/
11-
shell: bash
31+
wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-demo/2.2/
32+
wget -r -N -c -q --reject "index.html*" -nH -np --cut-dirs=3 https://physionet.org/files/mimic-iv-ed-demo/2.2/

‎.github/actions/load-mimic-duckdb/action.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: "Load MIMIC-IV demo into DuckDB"
2+
description: "Builds a DuckDB database from the demo hosp/icu data using import_duckdb.sh. Requires the duckdb CLI on PATH."
3+
4+
inputs:
5+
mimic_data_dir:
6+
description: "Directory containing the hosp/ and icu/ demo data folders."
7+
required: false
8+
default: ${{ github.workspace }}
9+
db_file:
10+
description: "Path to the DuckDB database file to create."
11+
required: false
12+
default: ${{ github.workspace }}/mimic4.db
13+
14+
runs:
15+
using: "composite"
16+
steps:
17+
- name: Build DuckDB database
18+
shell: bash
19+
# `echo n` answers the overwrite prompt defensively
20+
run: |
21+
cd "$GITHUB_WORKSPACE/mimic-iv/buildmimic/duckdb"
22+
echo n | ./import_duckdb.sh "${{ inputs.mimic_data_dir }}" "${{ inputs.db_file }}"

‎.github/actions/load-mimic-psql/action.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
name: "Load MIMIC-IV demo into PostgreSQL"
2+
description: "Creates the MIMIC-IV schemas and loads the demo hosp/icu data. Requires psql on PATH and PG* connection env vars."
3+
4+
inputs:
5+
mimic_data_dir:
6+
description: "Directory containing the hosp/ and icu/ demo data folders."
7+
required: false
8+
default: ${{ github.workspace }}
9+
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Create schemas and load demo data
14+
shell: bash
15+
run: |
16+
psql -q -v ON_ERROR_STOP=1 -f "$GITHUB_WORKSPACE/mimic-iv/buildmimic/postgres/create.sql"
17+
psql -q -v ON_ERROR_STOP=1 -v mimic_data_dir="${{ inputs.mimic_data_dir }}" \
18+
-f "$GITHUB_WORKSPACE/mimic-iv/buildmimic/postgres/load_gz.sql"

‎.github/actions/setup-duckdb/action.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
name: "Set up DuckDB CLI"
2+
description: "Downloads a pinned DuckDB CLI and installs it onto PATH. Requires wget and unzip to be present."
3+
4+
inputs:
5+
version:
6+
description: "DuckDB release version to install."
7+
required: false
8+
default: "1.1.3"
9+
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Install DuckDB CLI
14+
shell: bash
15+
run: |
16+
wget -q "https://github.com/duckdb/duckdb/releases/download/v${{ inputs.version }}/duckdb_cli-linux-amd64.zip"
17+
unzip -o duckdb_cli-linux-amd64.zip -d /usr/local/bin
18+
rm duckdb_cli-linux-amd64.zip
19+
duckdb --version

‎.github/actions/transpile-concepts/action.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
name: "Transpile BigQuery concepts"
2+
description: >-
3+
Regenerates the concepts for a SQL dialect from the BigQuery sources, writing
4+
over the committed mimic-iv/concepts_<dialect> folder. Downstream steps then
5+
build/test against the freshly transpiled SQL rather than what is committed.
6+
Requires mimic_utils to be installed (pip install -e .). This action does not
7+
report drift; use the check-concepts-up-to-date action for that.
8+
9+
inputs:
10+
dialect:
11+
description: "Destination SQL dialect (postgres or duckdb)."
12+
required: true
13+
14+
runs:
15+
using: "composite"
16+
steps:
17+
- name: Transpile ${{ inputs.dialect }} concepts
18+
shell: bash
19+
run: |
20+
mimic_utils convert_folder mimic-iv/concepts \
21+
"mimic-iv/concepts_${{ inputs.dialect }}" \
22+
--destination_dialect "${{ inputs.dialect }}"

‎.github/workflows/bigquery_dry_run.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: BigQuery syntax check (dry run)
2+
3+
# Verify syntax of bq through `bq query --dry_run`
4+
# Note: dry-run resolves inter-concept references against mimiciv_derived dataset,
5+
# so a brand-new concept that depends on another brand-new concept (not yet published)
6+
# will report the dependency as missing until the first is released.
7+
on:
8+
pull_request_review:
9+
types: [submitted]
10+
11+
jobs:
12+
dry-run:
13+
if: github.event.review.state == 'approved'
14+
runs-on: ubuntu-latest
15+
permissions:
16+
contents: 'read'
17+
id-token: 'write'
18+
steps:
19+
- name: Check out repository code
20+
uses: actions/checkout@v3
21+
22+
- id: 'auth'
23+
uses: 'google-github-actions/auth@v2'
24+
with:
25+
service_account: 'mimic-code@physionet-data.iam.gserviceaccount.com'
26+
workload_identity_provider: 'projects/569883598760/locations/global/workloadIdentityPools/github/providers/mimic-code'
27+
28+
- name: 'Set up Cloud SDK'
29+
uses: 'google-github-actions/setup-gcloud@v2'
30+
with:
31+
version: '>= 363.0.0'
32+
33+
- name: Dry-run all concept queries
34+
working-directory: ./mimic-iv/concepts
35+
run: |
36+
fail=0
37+
for d in demographics comorbidity measurement medication organfailure treatment firstday score sepsis; do
38+
for fn in "${d}"/*.sql; do
39+
echo "Dry-running ${fn}"
40+
if ! bq query --quiet --headless --dry_run --use_legacy_sql=false < "${fn}"; then
41+
echo "::error file=mimic-iv/concepts/${fn}::BigQuery dry-run failed for ${fn}"
42+
fail=1
43+
fi
44+
done
45+
done
46+
if [ "${fail}" -ne 0 ]; then
47+
echo "One or more concept queries failed the BigQuery dry-run."
48+
exit 1
49+
fi
50+
echo "All concept queries passed the BigQuery dry-run."

0 commit comments

Comments
 (0)