Skip to content

Commit 785ee05

Browse files
alistairewjalistairewj
committed
update actions to provide better warning
1 parent 3cd5874 commit 785ee05

4 files changed

Lines changed: 159 additions & 41 deletions

File tree

‎.github/actions/check-concepts-up-to-date/action.yml‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
name: "Check transpiled concepts are up to date"
2+
description: >-
3+
Compares the working tree against what is committed for the transpiled dialect
4+
folders. If they differ, the committed concepts_postgres / concepts_duckdb
5+
are stale relative to the BigQuery sources, and this action reports a NEUTRAL
6+
(yellow) check.
7+
8+
The calling job must grant `checks: write` (and `contents: read`) permissions.
9+
10+
inputs:
11+
github-token:
12+
description: "Token used to create the neutral check run."
13+
required: true
14+
check-name:
15+
description: "Name shown for the neutral check run."
16+
required: false
17+
default: "Dialects up to date with BigQuery"
18+
paths:
19+
description: "Space-separated paths to compare against the committed tree."
20+
required: false
21+
default: "mimic-iv/concepts_postgres mimic-iv/concepts_duckdb"
22+
23+
runs:
24+
using: "composite"
25+
steps:
26+
- name: Detect drift from committed concepts
27+
id: drift
28+
shell: bash
29+
run: |
30+
changes="$(git status --porcelain -- ${{ inputs.paths }})"
31+
if [ -z "$changes" ]; then
32+
echo "Committed dialect concepts match the BigQuery sources."
33+
echo "out_of_date=false" >> "$GITHUB_OUTPUT"
34+
exit 0
35+
fi
36+
37+
echo "out_of_date=true" >> "$GITHUB_OUTPUT"
38+
39+
# Persist the file list and diff for the check run + job summary.
40+
printf '%s\n' "$changes" > concept-drift-files.txt
41+
git --no-pager diff -- ${{ inputs.paths }} > concept-drift.diff
42+
43+
{
44+
echo "### :warning: Transpiled concepts are out of date"
45+
echo
46+
echo "The committed \`concepts_postgres\` / \`concepts_duckdb\` differ from"
47+
echo "what the transpiler produces from the BigQuery sources. Regenerate with:"
48+
echo
49+
echo '```sh'
50+
echo "mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_postgres --destination_dialect postgres"
51+
echo "mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_duckdb --destination_dialect duckdb"
52+
echo '```'
53+
echo
54+
echo "…and commit the result, or let the regeneration bot open a PR after this merges."
55+
echo
56+
echo "<details><summary>Files that differ</summary>"
57+
echo
58+
echo '```'
59+
cat concept-drift-files.txt
60+
echo '```'
61+
echo
62+
echo "</details>"
63+
} >> "$GITHUB_STEP_SUMMARY"
64+
65+
- name: Report neutral check run when out of date
66+
if: steps.drift.outputs.out_of_date == 'true'
67+
uses: actions/github-script@v7
68+
with:
69+
github-token: ${{ inputs.github-token }}
70+
script: |
71+
const fs = require('fs');
72+
const read = (p) => fs.existsSync(p) ? fs.readFileSync(p, 'utf8') : '';
73+
const files = read('concept-drift-files.txt').trim();
74+
let diff = read('concept-drift.diff');
75+
76+
// Check run output text is capped at 65535 chars; keep well under it.
77+
const LIMIT = 60000;
78+
if (diff.length > LIMIT) {
79+
diff = diff.slice(0, LIMIT) + '\n... (diff truncated) ...\n';
80+
}
81+
82+
const headSha = context.payload.pull_request
83+
? context.payload.pull_request.head.sha
84+
: context.sha;
85+
86+
const summary = [
87+
'The committed `concepts_postgres` / `concepts_duckdb` are stale relative to',
88+
'the BigQuery sources. Regenerate them with the two `mimic_utils convert_folder`',
89+
'commands and commit the result, or let the regeneration bot open a PR after',
90+
'this branch merges.',
91+
'',
92+
'### Files that differ',
93+
'```',
94+
files,
95+
'```',
96+
].join('\n');
97+
98+
try {
99+
await github.rest.checks.create({
100+
owner: context.repo.owner,
101+
repo: context.repo.repo,
102+
name: '${{ inputs.check-name }}',
103+
head_sha: headSha,
104+
status: 'completed',
105+
conclusion: 'neutral',
106+
output: {
107+
title: 'Transpiled concepts are out of date',
108+
summary,
109+
text: diff ? ['```diff', diff, '```'].join('\n') : undefined,
110+
},
111+
});
112+
} catch (error) {
113+
// The default GITHUB_TOKEN is read-only for pull requests from forks,
114+
// so it cannot create a check run. Fall back to a warning annotation
115+
// rather than failing the job red.
116+
core.warning(
117+
'Transpiled concepts are out of date with the BigQuery sources; ' +
118+
'could not post a neutral check run (' + error.message + '). ' +
119+
'Regenerate the concepts_postgres / concepts_duckdb folders and commit them.'
120+
);
121+
}

‎.github/actions/transpile-concepts/action.yml‎

Lines changed: 9 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
11
name: "Transpile BigQuery concepts"
22
description: >-
3-
Regenerates the concepts for a SQL dialect from the BigQuery sources and emits
4-
a visible warning if the committed files differ from the transpiler output, so
5-
it is clear the build ran against freshly transpiled (not committed) SQL.
6-
Requires mimic_utils to be installed (pip install -e .).
3+
Regenerates the concepts for a SQL dialect from the BigQuery sources, writing
4+
over the committed mimic-iv/concepts_<dialect> folder. Downstream steps then
5+
build/test against the freshly transpiled SQL rather than what is committed.
6+
Requires mimic_utils to be installed (pip install -e .). This action does not
7+
report drift; use the check-concepts-up-to-date action for that.
78
89
inputs:
910
dialect:
@@ -13,18 +14,9 @@ inputs:
1314
runs:
1415
using: "composite"
1516
steps:
16-
- name: Transpile and warn if out of date
17+
- name: Transpile ${{ inputs.dialect }} concepts
1718
shell: bash
1819
run: |
19-
target="mimic-iv/concepts_${{ inputs.dialect }}"
20-
mimic_utils convert_folder mimic-iv/concepts "$target" --destination_dialect "${{ inputs.dialect }}"
21-
22-
changes="$(git status --porcelain -- "$target")"
23-
if [ -n "$changes" ]; then
24-
echo "::warning title=Transpiled ${{ inputs.dialect }} concepts out of date::CI regenerated the ${{ inputs.dialect }} concepts from the BigQuery sources and the result differs from the committed $target. The concept build below ran against the freshly transpiled SQL, NOT what is committed. To sync: mimic_utils convert_folder mimic-iv/concepts $target --destination_dialect ${{ inputs.dialect }}"
25-
echo "Files that differ from the committed sources:"
26-
echo "$changes"
27-
git --no-pager diff -- "$target"
28-
else
29-
echo "Committed $target matches the BigQuery sources."
30-
fi
20+
mimic_utils convert_folder mimic-iv/concepts \
21+
"mimic-iv/concepts_${{ inputs.dialect }}" \
22+
--destination_dialect "${{ inputs.dialect }}"

‎.github/workflows/regenerate_dialects.yml‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,13 +30,19 @@ jobs:
3030
with:
3131
python-version: "3.12"
3232

33+
- name: Install locked dependencies
34+
run: pip install -r requirements-lock.txt
35+
3336
- name: Install package
34-
run: pip install -e .
37+
run: pip install -e . --no-deps
38+
39+
- uses: ./.github/actions/transpile-concepts
40+
with:
41+
dialect: postgres
3542

36-
- name: Regenerate concepts from the BigQuery sources
37-
run: |
38-
mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_postgres --destination_dialect postgres
39-
mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_duckdb --destination_dialect duckdb
43+
- uses: ./.github/actions/transpile-concepts
44+
with:
45+
dialect: duckdb
4046

4147
- name: Open or update the regeneration PR
4248
uses: peter-evans/create-pull-request@v8

‎.github/workflows/transpile.yml‎

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ on:
66
jobs:
77
# Unit tests for the transpiler.
88
unit-tests:
9+
name: Transpile unit tests
910
runs-on: ubuntu-latest
1011
steps:
1112
- uses: actions/checkout@v4
@@ -17,31 +18,29 @@ jobs:
1718
- name: Install package
1819
run: pip install -e . --no-deps
1920
- name: Run transpiler unit tests
20-
run: pytest tests/ -q
21+
run: pytest tests/test_transpile.py -q
2122

22-
# Reports whether the committed concepts_postgres / concepts_duckdb match
23-
# what the transpiler produces from the BigQuery sources.
24-
# Emits warnings to the PR.
2523
generated-up-to-date:
24+
name: Check cross-dialect concepts match BQ
2625
runs-on: ubuntu-latest
26+
permissions:
27+
contents: read
28+
checks: write
2729
steps:
2830
- uses: actions/checkout@v4
2931
- uses: actions/setup-python@v5
3032
with:
3133
python-version: "3.12"
34+
- name: Install locked dependencies
35+
run: pip install -r requirements-lock.txt
3236
- name: Install package
33-
run: pip install -e .
34-
- name: Regenerate concepts from the BigQuery sources
35-
run: |
36-
mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_postgres --destination_dialect postgres
37-
mimic_utils convert_folder mimic-iv/concepts mimic-iv/concepts_duckdb --destination_dialect duckdb
38-
- name: Report whether generated concepts are out of date
39-
run: |
40-
changes="$(git status --porcelain -- mimic-iv/concepts_postgres mimic-iv/concepts_duckdb)"
41-
if [ -n "$changes" ]; then
42-
echo "::warning::SQL concept queries in other dialects are out of date with the BigQuery sources. Run the two 'mimic_utils convert_folder' commands and commit the result, or let the transpilation bot make a new PR after merging this one."
43-
echo "$changes"
44-
git --no-pager diff -- mimic-iv/concepts_postgres mimic-iv/concepts_duckdb
45-
else
46-
echo "SQL concepts are consistent across dialects."
47-
fi
37+
run: pip install -e . --no-deps
38+
- uses: ./.github/actions/transpile-concepts
39+
with:
40+
dialect: postgres
41+
- uses: ./.github/actions/transpile-concepts
42+
with:
43+
dialect: duckdb
44+
- uses: ./.github/actions/check-concepts-up-to-date
45+
with:
46+
github-token: ${{ secrets.GITHUB_TOKEN }}

0 commit comments

Comments
 (0)