Skip to content

Commit fa675e7

Browse files
alistairewjalistairewj
committed
add bq workflows
1 parent a0af19c commit fa675e7

4 files changed

Lines changed: 160 additions & 2 deletions

File tree

‎.github/workflows/bigquery_dry_run.yml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
name: BigQuery syntax check (dry run)
2+
3+
# Verify syntax of bq through `bq query --dry_run`
4+
# Note: dry-run resolves inter-concept references against mimiciv_derived dataset,
5+
# so a brand-new concept that depends on another brand-new concept (not yet published)
6+
# will report the dependency as missing until the first is released.
7+
on:
8+
pull_request_review:
9+
types: [submitted]
10+
11+
jobs:
12+
dry-run:
13+
if: github.event.review.state == 'approved'
14+
runs-on: ubuntu-latest
15+
permissions:
16+
contents: 'read'
17+
id-token: 'write'
18+
steps:
19+
- name: Check out repository code
20+
uses: actions/checkout@v3
21+
22+
- id: 'auth'
23+
uses: 'google-github-actions/auth@v2'
24+
with:
25+
service_account: 'mimic-code@physionet-data.iam.gserviceaccount.com'
26+
workload_identity_provider: 'projects/569883598760/locations/global/workloadIdentityPools/github/providers/mimic-code'
27+
28+
- name: 'Set up Cloud SDK'
29+
uses: 'google-github-actions/setup-gcloud@v2'
30+
with:
31+
version: '>= 363.0.0'
32+
33+
- name: Dry-run all concept queries
34+
working-directory: ./mimic-iv/concepts
35+
run: |
36+
fail=0
37+
for d in demographics comorbidity measurement medication organfailure treatment firstday score sepsis; do
38+
for fn in "${d}"/*.sql; do
39+
echo "Dry-running ${fn}"
40+
if ! bq query --quiet --headless --dry_run --use_legacy_sql=false < "${fn}"; then
41+
echo "::error file=mimic-iv/concepts/${fn}::BigQuery dry-run failed for ${fn}"
42+
fail=1
43+
fi
44+
done
45+
done
46+
if [ "${fail}" -ne 0 ]; then
47+
echo "One or more concept queries failed the BigQuery dry-run."
48+
exit 1
49+
fi
50+
echo "All concept queries passed the BigQuery dry-run."

‎.github/workflows/main.yml‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Generate tables on BigQuery
1+
name: Generate derived concepts on BigQuery
22

33
on:
44
release:
@@ -33,7 +33,12 @@ jobs:
3333
echo "Generating tables on BigQuery"
3434
cd mimic-iv/concepts
3535
bash make_concepts.sh
36-
36+
37+
- name: Validate concepts
38+
run: |
39+
echo "Validating built concepts in mimiciv_derived before publishing"
40+
bash mimic-iv/concepts/validate_concepts.sh mimiciv_derived
41+
3742
- name: Copy to release specific schema
3843
run: |
3944
echo "Copying tables to release specific schema: mimiciv_${MIMIC_IV_VERSION}_derived"

‎.github/workflows/prerelease_bigquery.yml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
name: Pre-release concept build on mimiciv_derived
2+
3+
# Full BigQuery build of the derived concepts.
4+
# Rebuilds mimiciv_derived but does NOT copy to the versioned schema.
5+
on:
6+
release:
7+
types: [prereleased]
8+
9+
jobs:
10+
build-and-validate:
11+
runs-on: ubuntu-latest
12+
permissions:
13+
contents: 'read'
14+
id-token: 'write'
15+
steps:
16+
- name: Check out repository code
17+
uses: actions/checkout@v3
18+
19+
- id: 'auth'
20+
uses: 'google-github-actions/auth@v2'
21+
with:
22+
service_account: 'mimic-code@physionet-data.iam.gserviceaccount.com'
23+
workload_identity_provider: 'projects/569883598760/locations/global/workloadIdentityPools/github/providers/mimic-code'
24+
25+
- name: 'Set up Cloud SDK'
26+
uses: 'google-github-actions/setup-gcloud@v2'
27+
with:
28+
version: '>= 363.0.0'
29+
30+
- name: Run make_concepts
31+
run: |
32+
echo "Generating tables on BigQuery mimiciv_derived"
33+
cd mimic-iv/concepts
34+
bash make_concepts.sh
35+
36+
- name: Validate concepts
37+
run: |
38+
echo "Validating built concepts in mimiciv_derived"
39+
bash mimic-iv/concepts/validate_concepts.sh mimiciv_derived

‎mimic-iv/concepts/validate_concepts.sh‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/bin/bash
2+
# Validates the derived concepts built in a BigQuery dataset by checking:
3+
# 1. every concept SQL file has a corresponding table in the dataset
4+
# 2. no concept table is empty
5+
#
6+
# Usage: validate_concepts.sh [dataset]
7+
# dataset defaults to mimiciv_derived
8+
set -euo pipefail
9+
10+
PROJECT_ID=${PROJECT_ID:-physionet-data}
11+
DATASET=${1:-mimiciv_derived}
12+
13+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
14+
15+
EXPECTED="$(
16+
for d in demographics comorbidity measurement medication organfailure treatment firstday score sepsis; do
17+
for fn in "${SCRIPT_DIR}/${d}"/*.sql; do
18+
basename "${fn}" .sql
19+
done
20+
done | sort -u
21+
)"
22+
23+
echo "Fetching table metadata for ${PROJECT_ID}:${DATASET}"
24+
# row_count is read straight from table metadata; this does not scan the tables.
25+
ACTUAL="$(bq query --quiet --headless --use_legacy_sql=false --format=csv \
26+
"SELECT table_id, row_count FROM \`${PROJECT_ID}.${DATASET}.__TABLES__\`" \
27+
| tail -n +2)"
28+
29+
fail=0
30+
31+
# Check 1: every expected table is present.
32+
missing=0
33+
while read -r tbl; do
34+
[ -z "${tbl}" ] && continue
35+
if ! grep -q "^${tbl}," <<< "${ACTUAL}"; then
36+
echo "MISSING: expected table ${DATASET}.${tbl} was not built"
37+
missing=1
38+
fail=1
39+
fi
40+
done <<< "${EXPECTED}"
41+
if [ "${missing}" -eq 0 ]; then
42+
echo "OK: all $(grep -c . <<< "${EXPECTED}") expected tables are present"
43+
fi
44+
45+
# Check 2: no expected table is empty.
46+
empty=0
47+
while IFS=, read -r tbl rows; do
48+
[ -z "${tbl}" ] && continue
49+
# only validate the tables we actually build (ignore _metadata and any strays)
50+
if grep -qx "${tbl}" <<< "${EXPECTED}" && [ "${rows}" -eq 0 ]; then
51+
echo "EMPTY: table ${DATASET}.${tbl} has 0 rows"
52+
empty=1
53+
fail=1
54+
fi
55+
done <<< "${ACTUAL}"
56+
if [ "${empty}" -eq 0 ]; then
57+
echo "OK: no concept table is empty"
58+
fi
59+
60+
if [ "${fail}" -ne 0 ]; then
61+
echo "Validation FAILED for ${PROJECT_ID}:${DATASET}"
62+
exit 1
63+
fi
64+
echo "Validation passed for ${PROJECT_ID}:${DATASET}"

0 commit comments

Comments
 (0)