Address review: drop defensive extras, fix SQLFluff annotate perms

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: Chessing234

.github/workflows/lint_sqlfluff.yml

@@ -6,6 +6,9 @@ on:
 jobs:
   lint-mimic-iv:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      checks: write
     steps:
       - name: checkout
         uses: actions/checkout@v7
@@ -35,12 +38,10 @@ jobs:
         shell: bash
         run: sqlfluff lint --format github-annotation --annotation-level failure --nofail ${{ steps.get_files_to_lint.outputs.lintees }} > annotations.json
       - name: Annotate
-        # Fork PRs cannot create check runs with GITHUB_TOKEN; lint already ran.
-        continue-on-error: true
         if: steps.get_files_to_lint.outputs.lintees != ''
         uses: yuzutech/annotations-action@v0.6.0
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           title: "SQLFluff Lint"
           input: "./annotations.json"
-          ignore-missing-file: true
+          ignore-unauthorized-error: true

mimic-iii/buildmimic/duckdb/import_duckdb.sh

@@ -29,7 +29,7 @@ usage () {
     die "
 USAGE: ./import_duckdb.sh mimic_data_dir [output_db]
 WHERE:
-    mimic_data_dir        directory that contains csv.tar.gz or csv files
+    mimic_data_dir        directory that contains csv.gz or csv files
     output_db: optional   filename for duckdb file (default: mimic3.db)\
 "
 }

mimic-iii/buildmimic/postgres/Makefile

@@ -281,7 +281,7 @@ concepts:
 	@echo '---------------------'
 	@echo ''
 	@sleep 2
-	cd ../../concepts/ && psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -f make-concepts.sql
+	cd ../../concepts_postgres/ && psql "$(DBSTRING)" -v ON_ERROR_STOP=1 -c "CREATE SCHEMA IF NOT EXISTS mimiciii_derived;" -f postgres-make-concepts.sql
 
 
 .PHONY: help mimic clean

mimic-iii/buildmimic/sqlite/import.py

@@ -10,34 +10,6 @@
 CHUNKSIZE = 10 ** 6
 CONNECTION_STRING = "sqlite:///{}".format(DATABASE_NAME)
 
-# Column dtypes for tables that trigger pandas mixed-type warnings when
-# loaded with the default low_memory chunked inference (see #1237).
-# Types mirror mimic-iii/buildmimic/postgres/postgres_create_tables.sql.
-TABLE_DTYPES = {
-    "chartevents": {
-        "WARNING": "Int64",
-        "ERROR": "Int64",
-        "RESULTSTATUS": "string",
-        "STOPPED": "string",
-    },
-    "datetimeevents": {
-        "WARNING": "Int64",
-        "ERROR": "Int64",
-        "RESULTSTATUS": "string",
-        "STOPPED": "string",
-    },
-    "inputevents_cv": {
-        "ORIGINALRATE": "float64",
-        "ORIGINALRATEUOM": "string",
-        "ORIGINALSITE": "string",
-    },
-    "noteevents": {
-        "CHARTTIME": "string",
-        "STORETIME": "string",
-        "ISERROR": "string",
-    },
-}
-
 
 def _table_name_from_csv(filename: str) -> str:
     """Derive SQL table name from a CSV path (literal suffix, not str.strip)."""
@@ -49,18 +21,6 @@ def _table_name_from_csv(filename: str) -> str:
     return name.lower()
 
 
-def _read_csv(path, table, **kwargs):
-    # low_memory=False avoids dtype re-inference across chunks; table-specific
-    # dtypes pin the columns that otherwise flip between numeric/object.
-    return pd.read_csv(
-        path,
-        index_col="ROW_ID",
-        low_memory=False,
-        dtype=TABLE_DTYPES.get(table),
-        **kwargs,
-    )
-
-
 if os.path.exists(DATABASE_NAME):
     msg = "File {} already exists.".format(DATABASE_NAME)
     print(msg)
@@ -77,11 +37,11 @@ def _read_csv(path, table, **kwargs):
 for table, f in sorted(files_by_table.items()):
     print("Starting processing {}".format(f))
     if os.path.getsize(f) < THRESHOLD_SIZE:
-        df = _read_csv(f, table)
+        df = pd.read_csv(f, index_col="ROW_ID")
         df.to_sql(table, CONNECTION_STRING)
     else:
         # If the file is too large, let's do the work in chunks
-        for chunk in _read_csv(f, table, chunksize=CHUNKSIZE):
+        for chunk in pd.read_csv(f, index_col="ROW_ID", chunksize=CHUNKSIZE):
             chunk.to_sql(table, CONNECTION_STRING, if_exists="append")
     print("Finished processing {}".format(f))
 

mimic-iii/concepts_duckdb/demographics/icustay_hours.sql

@@ -4,7 +4,7 @@ WITH all_hours AS (
   SELECT
     it.icustay_id,
     STRPTIME(STRFTIME(CAST(it.intime_hr + INTERVAL '59' MINUTE AS TIMESTAMP), '%Y-%m-%d %H:00:00'), '%Y-%m-%d %H:00:00') AS endtime,
-    (SELECT list(g) FROM generate_series(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS BIGINT)) AS t(g)) AS hrs
+    GENERATE_SERIES(-24, CAST(CEIL(DATE_DIFF('HOUR', it.intime_hr, it.outtime_hr)) AS BIGINT)) AS hrs
   FROM mimiciii_derived.icustay_times AS it
 )
 SELECT

mimic-iv-cxr/txt/chexpert/run_chexpert_on_files.sh

@@ -22,20 +22,12 @@ fi
 sleep 2
 
 # loop through each .csv file in the section folder
-for fn_path in "$REPORT_PATH"/*.csv; do
-    [ -f "$fn_path" ] || continue
-    fn=$(basename "$fn_path")
+for fn in "$REPORT_PATH"/*; do
+    fn=${fn##*/}
     echo "$(date): $fn"
-    fn_stem=${fn%.csv}
+    fn_stem=$(echo "$fn" | cut -d. -f 1)
     # run chexpert - must be run from chexpert folder
-    python "$CHEXPERT_PATH/label.py" --verbose \
-      --reports_path "$fn_path" \
-      --output_path "${fn_stem}_labeled.csv" \
-      --mention_phrases_dir "$CHEXPERT_PATH/phrases/mention" \
-      --unmention_phrases_dir "$CHEXPERT_PATH/phrases/unmention" \
-      --pre_negation_uncertainty_path "$CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt" \
-      --negation_path "$CHEXPERT_PATH/patterns/negation.txt" \
-      --post_negation_uncertainty_path "$CHEXPERT_PATH/patterns/post_negation_uncertainty.txt"
+    python "$CHEXPERT_PATH/label.py"  --verbose --reports_path "$REPORT_PATH/$fn" --output_path "${fn_stem}_labeled.csv" --mention_phrases_dir "$CHEXPERT_PATH/phrases/mention" --unmention_phrases_dir "$CHEXPERT_PATH/phrases/unmention" --pre_negation_uncertainty_path "$CHEXPERT_PATH/patterns/pre_negation_uncertainty.txt" --negation_path "$CHEXPERT_PATH/patterns/negation.txt" --post_negation_uncertainty_path "$CHEXPERT_PATH/patterns/post_negation_uncertainty.txt"
     echo "$(date): done!"
     echo ''
-done
+done

mimic-iv-cxr/txt/negbio/run_negbio.sh

@@ -32,21 +32,20 @@ sleep 2
 #  mimic_cxr_001.csv
 #  mimic_cxr_002.csv
 # .. etc
-for fn_path in "$BASE_FOLDER"/mimic_cxr_*.csv;
+for fn in "$BASE_FOLDER"/*;
 do
-  [ -f "$fn_path" ] || continue
-  fn=$(basename "$fn_path")
+  fn=${fn##*/}
   echo "Looping through files with mimic_cxr_###.csv pattern."
   # validate it's a mimic_cxr sections file
-  if [[ "$fn" =~ ^mimic_cxr_[0-9]+\.csv$ ]];
+  if [[ $fn =~ ^mimic_cxr_[0-9]+.csv$ ]];
   then
-    export INPUT_FILE="$fn_path"
+    export INPUT_FILE=${BASE_FOLDER}/$fn
     # remove extension from filename
     # sets the folder location to stem of original filename
     # all intermediate files will be saved in this folder
-    export OUTPUT_DIR="${INPUT_FILE%.csv}"
+    export OUTPUT_DIR=${INPUT_FILE::-4}
 
-    echo "$OUTPUT_DIR - running NegBio.."
+    echo "$OUTPUT_DIR" - running NegBio..
     python "$NEGBIO_PATH/negbio/negbio_csv2bioc.py" --output "$OUTPUT_DIR/report" "$INPUT_FILE"
     python "$NEGBIO_PATH/negbio/negbio_pipeline.py" section_split --pattern "$NEGBIO_PATH/patterns/section_titles_cxr8.txt" --output "$OUTPUT_DIR/sections" "$OUTPUT_DIR"/report/* --workers=6
     python "$NEGBIO_PATH/negbio/negbio_pipeline.py" ssplit --output "$OUTPUT_DIR/ssplit" "$OUTPUT_DIR"/sections/* --workers=6
@@ -56,7 +55,7 @@ do
     python "$NEGBIO_PATH/negbio/negbio_pipeline.py" neg2 --output "$OUTPUT_DIR/neg" --pre-negation-uncertainty-patterns "$NEGBIO_PATH/patterns/chexpert_pre_negation_uncertainty.yml" --neg-patterns "$NEGBIO_PATH/patterns/neg_patterns2.yml" --post-negation-uncertainty-patterns "$NEGBIO_PATH/patterns/post_negation_uncertainty.yml" --neg-regex-patterns "$NEGBIO_PATH/patterns/neg_regex_patterns.yml" --uncertainty-regex-patterns "$NEGBIO_PATH/patterns/uncertainty_regex_patterns.yml" "$OUTPUT_DIR"/dner/* --workers=6
 
     # ultimate filename we save the labels to
-    export OUTPUT_LABELS="$OUTPUT_DIR/${fn%.csv}_labels.csv"
+    export OUTPUT_LABELS=$OUTPUT_DIR/${fn::-4}_labels.csv
     python "$NEGBIO_PATH/negbio/ext/chexpert_collect_labels.py" --phrases_file "$NEGBIO_PATH/patterns/chexpert_phrases.yml" --output "$OUTPUT_LABELS" "$OUTPUT_DIR"/neg/*
   fi
 done

mimic-iv/concepts/validate_concepts.sh

@@ -32,8 +32,7 @@ fail=0
 missing=0
 while read -r tbl; do
   [ -z "${tbl}" ] && continue
-  # exact field match (regex grep can false-hit prefixes)
-  if ! awk -F, -v t="${tbl}" '$1 == t { found=1 } END { exit !found }' <<< "${ACTUAL}"; then
+  if ! grep -q "^${tbl}," <<< "${ACTUAL}"; then
     echo "MISSING: expected table ${DATASET}.${tbl} was not built"
     missing=1
     fail=1
@@ -48,7 +47,7 @@ empty=0
 while IFS=, read -r tbl rows; do
   [ -z "${tbl}" ] && continue
   # only validate the tables we actually build (ignore _metadata and any strays)
-  if grep -Fxq "${tbl}" <<< "${EXPECTED}" && [ "${rows:-}" -eq 0 ]; then
+  if grep -qx "${tbl}" <<< "${EXPECTED}" && [ "${rows}" -eq 0 ]; then
     echo "EMPTY: table ${DATASET}.${tbl} has 0 rows"
     empty=1
     fail=1

src/mimic_utils/sqlglot_dialects/duckdb.py

@@ -3,10 +3,6 @@
 - ``NUMERIC`` is ``DECIMAL(38, 9)``, but DuckDB's bare ``DECIMAL`` defaults
 to ``DECIMAL(18, 3)``, which silently rounds values to three decimal places
 (e.g. ``CAST(0.0255 AS NUMERIC)`` becomes ``0.026`` before any explicit ``ROUND``).
-- ``GENERATE_ARRAY`` must remain a LIST (for later ``UNNEST``). Native sqlglot
-emits ``GENERATE_SERIES``, which is a set-returning function; wrapping as a
-list matches Postgres ``ARRAY(GENERATE_SERIES)`` and keeps ``UNNEST(hrs)``
-reliable across DuckDB versions (#1736).
 """
 import re
 
@@ -45,13 +41,6 @@ def _parse_datetime_sql(self: DuckDB.Generator, expression: exp.ParseDatetime) -
     return f"STRPTIME({this}, {fmt})"
 
 
-# GENERATE_ARRAY(a, b) -> list via generate_series (inclusive), for UNNEST.
-def _generate_array_sql(self: DuckDB.Generator, expression: exp.Expression) -> str:
-    start = self.sql(expression, "start")
-    end = self.sql(expression, "end")
-    return f"(SELECT list(g) FROM generate_series({start}, {end}) AS t(g))"
-
-
 class MimicDuckDB(DuckDB):
     class Generator(DuckDB.Generator):
         def datatype_sql(self, expression: exp.DataType) -> str:
@@ -65,5 +54,4 @@ def datatype_sql(self, expression: exp.DataType) -> str:
             **DuckDB.Generator.TRANSFORMS,
             exp.RegexpExtract: _regexp_extract_sql,
             exp.ParseDatetime: _parse_datetime_sql,
-            exp.GenerateSeries: _generate_array_sql,
         }

tests/test_transpile.py

@@ -74,9 +74,6 @@ def t(bq: str, dialect: str) -> str:
     # GENERATE_ARRAY -> ARRAY(SELECT ... GENERATE_SERIES) (postgres)
     ("generate_array_pg", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "postgres",
      "SELECT ARRAY(SELECT * FROM GENERATE_SERIES(-24, 5)) AS hrs FROM t"),
-    # GENERATE_ARRAY -> list via generate_series (duckdb); must stay list for UNNEST
-    ("generate_array_duckdb", "SELECT GENERATE_ARRAY(-24, 5) AS hrs FROM t", "duckdb",
-     "SELECT (SELECT list(g) FROM generate_series(-24, 5) AS t(g)) AS hrs FROM t"),
 
     # handled natively by sqlglot 30.x (regression guards)
     ("datetime_date_pg", "SELECT DATETIME(me.chartdate) FROM t me", "postgres",