diff --git a/.github/workflows/lint_sqlfluff.yml b/.github/workflows/lint_sqlfluff.yml index 0986785b..d6e1c3f0 100644 --- a/.github/workflows/lint_sqlfluff.yml +++ b/.github/workflows/lint_sqlfluff.yml @@ -1,11 +1,17 @@ name: SQLFluff +# Lints changed mimic-iv/concepts/*.sql on pull requests and posts GitHub +# annotations. Fork PRs skip Annotate — GITHUB_TOKEN cannot create check runs +# from forks (Resource not accessible by integration / 403). on: - pull_request jobs: lint-mimic-iv: runs-on: ubuntu-latest + permissions: + contents: read + checks: write steps: - name: checkout uses: actions/checkout@v7 @@ -35,8 +41,14 @@ jobs: shell: bash run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json - name: Annotate + # Same-repo PRs only: fork tokens get 403 creating check runs, and + # ignore-unauthorized-error does not treat that as unauthorized. + if: > + steps.get_files_to_lint.outputs.lintees != '' && + github.event.pull_request.head.repo.full_name == github.repository uses: yuzutech/annotations-action@v0.6.0 with: repo-token: "${{ secrets.GITHUB_TOKEN }}" title: "SQLFluff Lint" - input: "./annotations.json" \ No newline at end of file + input: "./annotations.json" + ignore-unauthorized-error: true diff --git a/README.md b/README.md index 53fe9f9f..649df76c 100644 --- a/README.md +++ b/README.md @@ -114,3 +114,7 @@ By committing your code to the [MIMIC Code Repository](https://github.com/mit-lc ### Coding style Please refer to the [style guide](https://github.com/MIT-LCP/mimic-code/blob/main/styleguide.md) for guidelines on formatting your code for the repository. + +### MIMIC-IV timing notes + +- [ICU chartevents timing](mimic-iv/docs/CHARTEVENTS_TIMING.md) diff --git a/mimic-iv/README.md b/mimic-iv/README.md index 919b196c..28a6b7a9 100644 --- a/mimic-iv/README.md +++ b/mimic-iv/README.md @@ -105,4 +105,8 @@ mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_duckdb --destinat # To PostgreSQL: mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_postgres --destination_dialect postgres -``` \ No newline at end of file +``` + +## Timing caveats + +`chartevents.charttime` can legitimately fall outside `icustays` windows — see [docs/CHARTEVENTS_TIMING.md](docs/CHARTEVENTS_TIMING.md). diff --git a/mimic-iv/concepts/documentation/README.md b/mimic-iv/concepts/documentation/README.md new file mode 100644 index 00000000..15576507 --- /dev/null +++ b/mimic-iv/concepts/documentation/README.md @@ -0,0 +1,4 @@ +# Documentation helper queries + +Ad-hoc SQL that illustrates known data quirks. These are **not** part of the +official derived concept build — run them against a local or BigQuery copy. diff --git a/mimic-iv/concepts/documentation/chartevents_after_outtime.sql b/mimic-iv/concepts/documentation/chartevents_after_outtime.sql new file mode 100644 index 00000000..59d90a7b --- /dev/null +++ b/mimic-iv/concepts/documentation/chartevents_after_outtime.sql @@ -0,0 +1,13 @@ +-- Count chartevents whose charttime falls after the linked ICU stay outtime. +-- Useful as a sensitivity check when building in-ICU only cohorts (#1961). + +SELECT + COUNT(*) AS chartevents_rows, + COUNT(*) FILTER (WHERE ce.charttime > ie.outtime) AS after_outtime, + COUNT(*) FILTER ( + WHERE ce.charttime > ie.outtime + AND ce.charttime <= ie.outtime + INTERVAL '6 hours' + ) AS after_outtime_within_6h +FROM icu.chartevents AS ce +INNER JOIN icu.icustays AS ie + ON ce.stay_id = ie.stay_id; diff --git a/mimic-iv/concepts/documentation/chartevents_before_intime.sql b/mimic-iv/concepts/documentation/chartevents_before_intime.sql new file mode 100644 index 00000000..728d536f --- /dev/null +++ b/mimic-iv/concepts/documentation/chartevents_before_intime.sql @@ -0,0 +1,12 @@ +-- Count chartevents charted before ICU intime (less common than after outtime). + +SELECT + COUNT(*) AS before_intime, + COUNT(*) FILTER ( + WHERE ce.charttime < ie.intime + AND ce.charttime >= ie.intime - INTERVAL '2 hours' + ) AS before_intime_within_2h +FROM icu.chartevents AS ce +INNER JOIN icu.icustays AS ie + ON ce.stay_id = ie.stay_id +WHERE ce.charttime < ie.intime; diff --git a/mimic-iv/docs/CHARTEVENTS_TIMING.md b/mimic-iv/docs/CHARTEVENTS_TIMING.md new file mode 100644 index 00000000..df993020 --- /dev/null +++ b/mimic-iv/docs/CHARTEVENTS_TIMING.md @@ -0,0 +1,46 @@ +# Chartevents timing vs ICU stay windows + +Researchers often find `icu.chartevents.charttime` values **after** +`icu.icustays.outtime` (or occasionally before `intime`). Large absolute counts +are expected in the raw PhysioNet files and usually do **not** mean a broken +`stay_id` join. + +See [issue #1961](https://github.com/MIT-LCP/mimic-code/issues/1961). + +## Why charttime can fall outside `[intime, outtime]` + +1. **Late charting.** Nurses and devices often document observations after the + patient has left the ICU; the row still carries the ICU `stay_id`. +2. **Device / interface clocks.** Bedside monitors and EHR interfaces may stamp + times that disagree slightly with ADT `intime`/`outtime`. +3. **Derived stay boundaries.** `icustays` windows are constructed from + transfer / careunit logic; charted data is not clipped to those windows in + the ETL that loads the CSV files. + +The public tables do not expose a "manual vs automatic" provenance flag for +each `charttime`. + +## Practical guidance + +- Treat `icustays.intime` / `outtime` as the **administrative** ICU window. +- For "in-ICU only" cohorts, filter explicitly, e.g. + `charttime >= intime AND charttime <= outtime`, and report how many rows you + dropped. +- Prefer a sensitivity analysis that also keeps a short post-`outtime` grace + period (e.g. 1–6 hours) when studying labs or vitals that are often charted + late. +- Do **not** rewrite `outtime` from the latest `charttime` inside the PhysioNet + loaders — that belongs in a documented derived concept if you need it. + +## Related notes + +- ED module: `mimic-iv-ed/docs/TIMING_NOTES.md` (`edstays.intime` vs pyxis) +- Lab / OMR unit caveats: `mimic-iv/buildmimic/TABLE_NOTES.md` when present + +## What this repository will not change + +MIMIC-Code does **not** clip `chartevents` to `icustays` windows in the +PhysioNet loaders, and it will not silently rewrite `outtime` from the latest +charted observation. Clipping belongs in analysis code or an explicitly +documented derived concept. +