diff --git a/.github/workflows/lint_sqlfluff.yml b/.github/workflows/lint_sqlfluff.yml index 0986785b..d6e1c3f0 100644 --- a/.github/workflows/lint_sqlfluff.yml +++ b/.github/workflows/lint_sqlfluff.yml @@ -1,11 +1,17 @@ name: SQLFluff +# Lints changed mimic-iv/concepts/*.sql on pull requests and posts GitHub +# annotations. Fork PRs skip Annotate — GITHUB_TOKEN cannot create check runs +# from forks (Resource not accessible by integration / 403). on: - pull_request jobs: lint-mimic-iv: runs-on: ubuntu-latest + permissions: + contents: read + checks: write steps: - name: checkout uses: actions/checkout@v7 @@ -35,8 +41,14 @@ jobs: shell: bash run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json - name: Annotate + # Same-repo PRs only: fork tokens get 403 creating check runs, and + # ignore-unauthorized-error does not treat that as unauthorized. + if: > + steps.get_files_to_lint.outputs.lintees != '' && + github.event.pull_request.head.repo.full_name == github.repository uses: yuzutech/annotations-action@v0.6.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" title: "SQLFluff Lint" - input: "./annotations.json" \ No newline at end of file + input: "./annotations.json" + ignore-unauthorized-error: true diff --git a/mimic-iv/concepts/README.md b/mimic-iv/concepts/README.md index f2c55041..f5e87730 100644 --- a/mimic-iv/concepts/README.md +++ b/mimic-iv/concepts/README.md @@ -3,6 +3,10 @@ This folder contains scripts to generate useful abstractions of raw MIMIC-IV data ("concepts"). The scripts are written using the **BigQuery Standard SQL Dialect**. Concepts are categorized into folders if possible, otherwise they remain in the top-level directory. The [concepts_postgres](/mimic-iv/concepts_postgres) and [concepts_duckdb](/mimic-iv/concepts_duckdb) folders contain automatically generated PostgreSQL and DuckDB versions of these scripts; see the [top-level readme](/mimic-iv/README.md#generating-the-concepts) for how these were generated. +Lab measurement concepts under `measurement/` generally exclude non-positive `valuenum` rows +(`valuenum IS NOT NULL AND valuenum > 0`) and, where documented in the source SQL, filter +`valueuom` to the expected unit for a given `itemid`. + The concepts are organized into individual SQL scripts, with each script generating a table. The BigQuery `mimiciv_derived` dataset under `physionet-data` contains the concepts pregenerated. Access to this dataset is available to MIMIC-IV approved users: see the [cloud instructions](https://mimic.mit.edu/docs/gettingstarted/cloud/) on how to access MIMIC-IV on BigQuery (which includes the derived concepts). See the [top-level readme](/mimic-iv/README.md) for more information about generating the concepts. diff --git a/mimic-iv/concepts/measurement/cardiac_marker.sql b/mimic-iv/concepts/measurement/cardiac_marker.sql index ed612191..57c08e27 100644 --- a/mimic-iv/concepts/measurement/cardiac_marker.sql +++ b/mimic-iv/concepts/measurement/cardiac_marker.sql @@ -18,5 +18,6 @@ WHERE le.itemid IN , 50963 -- N-terminal (NT)-pro hormone BNP (NT-proBNP) ) AND valuenum IS NOT NULL + AND valuenum > 0 GROUP BY le.specimen_id ; diff --git a/mimic-iv/concepts/measurement/coagulation.sql b/mimic-iv/concepts/measurement/coagulation.sql index 241ac06a..58158b86 100644 --- a/mimic-iv/concepts/measurement/coagulation.sql +++ b/mimic-iv/concepts/measurement/coagulation.sql @@ -27,5 +27,6 @@ WHERE le.itemid IN , 51275 -- PTT ) AND valuenum IS NOT NULL + AND valuenum > 0 GROUP BY le.specimen_id ; diff --git a/mimic-iv/concepts_duckdb/measurement/cardiac_marker.sql b/mimic-iv/concepts_duckdb/measurement/cardiac_marker.sql index 7dc6cf26..370970a6 100644 --- a/mimic-iv/concepts_duckdb/measurement/cardiac_marker.sql +++ b/mimic-iv/concepts_duckdb/measurement/cardiac_marker.sql @@ -10,6 +10,6 @@ SELECT MAX(CASE WHEN itemid = 50963 THEN valuenum ELSE NULL END) AS ntprobnp FROM mimiciv_hosp.labevents AS le WHERE - le.itemid IN (51003, 50911, 50963) AND NOT valuenum IS NULL + le.itemid IN (51003, 50911, 50963) AND NOT valuenum IS NULL AND valuenum > 0 GROUP BY le.specimen_id \ No newline at end of file diff --git a/mimic-iv/concepts_duckdb/measurement/coagulation.sql b/mimic-iv/concepts_duckdb/measurement/coagulation.sql index 69d00552..5eddc60d 100644 --- a/mimic-iv/concepts_duckdb/measurement/coagulation.sql +++ b/mimic-iv/concepts_duckdb/measurement/coagulation.sql @@ -13,6 +13,8 @@ SELECT MAX(CASE WHEN itemid = 51275 THEN valuenum ELSE NULL END) AS ptt FROM mimiciv_hosp.labevents AS le WHERE - le.itemid IN (51196, 51214, 51297, 51237, 51274, 51275) AND NOT valuenum IS NULL + le.itemid IN (51196, 51214, 51297, 51237, 51274, 51275) + AND NOT valuenum IS NULL + AND valuenum > 0 GROUP BY le.specimen_id \ No newline at end of file diff --git a/mimic-iv/concepts_postgres/measurement/cardiac_marker.sql b/mimic-iv/concepts_postgres/measurement/cardiac_marker.sql index 4a783cc3..84a1b346 100644 --- a/mimic-iv/concepts_postgres/measurement/cardiac_marker.sql +++ b/mimic-iv/concepts_postgres/measurement/cardiac_marker.sql @@ -17,5 +17,6 @@ WHERE 50963 /* N-terminal (NT)-pro hormone BNP (NT-proBNP) */ ) AND NOT valuenum IS NULL + AND valuenum > 0 GROUP BY le.specimen_id \ No newline at end of file diff --git a/mimic-iv/concepts_postgres/measurement/coagulation.sql b/mimic-iv/concepts_postgres/measurement/coagulation.sql index 8891338d..be698bb3 100644 --- a/mimic-iv/concepts_postgres/measurement/coagulation.sql +++ b/mimic-iv/concepts_postgres/measurement/coagulation.sql @@ -22,5 +22,6 @@ WHERE 51275 /* PTT */ ) AND NOT valuenum IS NULL + AND valuenum > 0 GROUP BY le.specimen_id \ No newline at end of file