Merge pull request #2038 from MIT-LCP/alistair/gh_fork_fixes

Make gh actions work on forks

Author: alistairewj

.github/scripts/check_sql_syntax.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""Validate that every BigQuery concept SQL file parses with sqlglot.
+
+Usage: ``python .github/scripts/check_sql_syntax.py [ROOT ...]``
+(defaults to ``mimic-iv/concepts``). Exits non-zero if any file fails to parse.
+"""
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+
+import sqlglot
+from sqlglot.errors import ParseError
+
+
+def main(argv: list[str]) -> int:
+    roots = [Path(p) for p in argv[1:]] or [Path("mimic-iv/concepts")]
+    files = sorted({f for root in roots for f in root.rglob("*.sql")})
+    if not files:
+        print(f"::error::No .sql files found under: {', '.join(map(str, roots))}")
+        return 1
+
+    failures = 0
+    for f in files:
+        try:
+            sqlglot.parse_one(f.read_text(), read="bigquery")
+        except ParseError as exc:
+            failures += 1
+            # Keep the GitHub annotation to a single line so it anchors to the
+            # file; the full multi-line message follows in the log.
+            first_line = str(exc).splitlines()[0]
+            print(f"::error file={f}::sqlglot failed to parse: {first_line}")
+            print(str(exc))
+        else:
+            print(f"OK  {f}")
+
+    print()
+    if failures:
+        print(f"{failures} of {len(files)} concept file(s) failed to parse.")
+        return 1
+    print(f"All {len(files)} concept file(s) parsed successfully.")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv))

.github/workflows/bigquery_dry_run.yml

@@ -1,16 +1,22 @@
-name: BigQuery syntax check (dry run)
+name: BigQuery dry run
 
-# Verify syntax of bq through `bq query --dry_run`
-# Note: dry-run resolves inter-concept references against mimiciv_derived dataset,
-# so a brand-new concept that depends on another brand-new concept (not yet published)
-# will report the dependency as missing until the first is released.
+# Semantic validation of the BigQuery concept sources via `bq query --dry_run`,
+# which resolves table/column references against the live mimiciv_derived
+# dataset. This requires cloud credentials (OIDC), so it is only run on main.
+#
+# Note: dry-run resolves inter-concept references against mimiciv_derived, so a
+# brand-new concept that depends on another brand-new concept (not yet
+# published) will report the dependency as missing until the first is released.
 on:
-  pull_request_review:
-    types: [submitted]
+  push:
+    branches:
+      - main
+    paths:
+      - 'mimic-iv/concepts/**'
+      - '.github/workflows/bigquery_dry_run.yml'
 
 jobs:
   dry-run:
-    if: github.event.review.state == 'approved'
     runs-on: ubuntu-latest
     permissions:
         contents: 'read'

.github/workflows/sql_syntax_check.yml

@@ -0,0 +1,27 @@
+name: BigQuery SQL syntax check via sqlglot
+
+on:
+  pull_request:
+    paths:
+      - 'mimic-iv/concepts/**'
+      - 'src/mimic_utils/**'
+      - '.github/scripts/check_sql_syntax.py'
+      - '.github/workflows/sql_syntax_check.yml'
+
+permissions:
+  contents: read
+
+jobs:
+  syntax-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install locked dependencies
+        run: pip install -r requirements-lock.txt
+      - name: Install package
+        run: pip install -e . --no-deps
+      - name: Parse concept SQL with sqlglot
+        run: python .github/scripts/check_sql_syntax.py mimic-iv/concepts

mimic-iv/concepts_duckdb/duckdb.sql

@@ -62,6 +62,8 @@
 .read medication/acei.sql
 .print 'medication/antibiotic.sql'
 .read medication/antibiotic.sql
+.print 'medication/arb.sql'
+.read medication/arb.sql
 .print 'medication/dobutamine.sql'
 .read medication/dobutamine.sql
 .print 'medication/dopamine.sql'
@@ -82,6 +84,8 @@
 .read medication/vasopressin.sql
 
 -- treatment
+.print 'treatment/code_status.sql'
+.read treatment/code_status.sql
 .print 'treatment/crrt.sql'
 .read treatment/crrt.sql
 .print 'treatment/invasive_line.sql'