Skip to content

Commit 9bbbc7c

Browse files
russell-sternrussell-stern
committed
Added script to test against templates and github action to run that test
1 parent cec8f75 commit 9bbbc7c

4 files changed

Lines changed: 608 additions & 0 deletions

File tree

‎.github/workflows/template-compatibility-comment.yml‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
name: Template Compatibility Comment
2+
3+
# Triggered when the unprivileged "Template Compatibility Check" workflow
4+
# completes. This workflow has pull-requests: write so it can post PR comments,
5+
# but it never checks out or executes any external code — it only reads the
6+
# artifact produced by the check workflow.
7+
#
8+
# SECURITY: workflow_run always runs on the default branch, so this workflow
9+
# definition itself cannot be tampered with by a PR contributor. Artifact
10+
# contents are treated as untrusted strings and sanitized before use.
11+
12+
on:
13+
workflow_run:
14+
workflows: ["Template Compatibility Check"]
15+
types: [completed]
16+
workflow_dispatch: # temporary — remove before merging
17+
inputs:
18+
run_id:
19+
required: true
20+
21+
permissions:
22+
pull-requests: write
23+
actions: read # required to download artifacts from another workflow run
24+
25+
jobs:
26+
post-comment:
27+
runs-on: ubuntu-latest
28+
29+
steps:
30+
- name: Download results artifact
31+
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
32+
with:
33+
name: template-compat-results
34+
path: /tmp/compat-results
35+
run-id: ${{ github.event.workflow_run.id }}
36+
github-token: ${{ secrets.GITHUB_TOKEN }}
37+
38+
- name: Post or remove PR comment
39+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
40+
with:
41+
script: |
42+
const fs = require('fs');
43+
44+
const read = (filename) => {
45+
try { return fs.readFileSync(`/tmp/compat-results/${filename}`, 'utf8').trim(); }
46+
catch { return ''; }
47+
};
48+
49+
// Validate PR number — must be a positive integer.
50+
const prNumber = parseInt(read('pr-number.txt'), 10);
51+
if (!Number.isInteger(prNumber) || prNumber <= 0) {
52+
console.log('Invalid or missing PR number in artifact; skipping comment.');
53+
return;
54+
}
55+
56+
const exitCode = read('exit-code.txt');
57+
const fullOutput = read('output.txt');
58+
// Sanitize values read from the artifact before embedding in markdown
59+
// to prevent injection (e.g. a malicious branch name or script output
60+
// containing markdown syntax that escapes a code fence).
61+
const templatesRef = read('templates-ref.txt').replace(/[^a-zA-Z0-9._\/-]/g, '');
62+
const headRef = read('head-ref.txt').replace(/[^a-zA-Z0-9._\/-]/g, '');
63+
64+
const marker = '<!-- template-compat-comment -->';
65+
66+
const { data: comments } = await github.rest.issues.listComments({
67+
owner: context.repo.owner,
68+
repo: context.repo.repo,
69+
issue_number: prNumber,
70+
});
71+
const existing = comments.find(c => c.body.includes(marker));
72+
73+
if (exitCode === '0') {
74+
// Templates pass — remove any stale failure comment.
75+
if (existing) {
76+
await github.rest.issues.deleteComment({
77+
owner: context.repo.owner,
78+
repo: context.repo.repo,
79+
comment_id: existing.id,
80+
});
81+
}
82+
return;
83+
}
84+
85+
// Extract just the "Results" and "Failure Details" sections from output.
86+
const resultsMatch = fullOutput.match(/={8,}\nResults:.*\n={8,}[\s\S]*/);
87+
const failureSummary = resultsMatch ? resultsMatch[0].trim() : fullOutput.trim();
88+
89+
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.payload.workflow_run.id}`;
90+
const refNote = templatesRef === 'main'
91+
? 'tested against `cre-templates:main`'
92+
: `tested against \`cre-templates:${templatesRef}\` (compat branch)`;
93+
94+
const body = [
95+
'## ⚠️ Template Compatibility Failures',
96+
'',
97+
`This PR breaks one or more templates in [cre-templates](https://github.com/smartcontractkit/cre-templates) (${refNote}).`,
98+
'',
99+
'```',
100+
failureSummary,
101+
'```',
102+
'',
103+
`[View full output →](${runUrl})`,
104+
'',
105+
'<details>',
106+
'<summary>What should I do?</summary>',
107+
'',
108+
'- **Accidental break:** Fix the SDK change so existing templates continue to compile.',
109+
`- **Intentional breaking change:** Create a branch in \`cre-templates\` named \`compat/${headRef}\` with the template fixes applied. This job will automatically retest against that branch.`,
110+
'',
111+
'</details>',
112+
].join('\n');
113+
114+
const commentBody = `${marker}\n${body}`;
115+
116+
if (existing) {
117+
await github.rest.issues.updateComment({
118+
owner: context.repo.owner,
119+
repo: context.repo.repo,
120+
comment_id: existing.id,
121+
body: commentBody,
122+
});
123+
} else {
124+
await github.rest.issues.createComment({
125+
owner: context.repo.owner,
126+
repo: context.repo.repo,
127+
issue_number: prNumber,
128+
body: commentBody,
129+
});
130+
}

‎.github/workflows/template-compatibility.yml‎

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
name: Template Compatibility Check
2+
3+
on:
4+
pull_request:
5+
types: [opened, synchronize, reopened]
6+
7+
# This job is informational — it is intentionally NOT a required status check
8+
# so that breaking SDK changes (major version bumps) can still be merged.
9+
# When templates break, a comment is posted on the PR with details.
10+
# For intentional breaking changes, create a matching branch in cre-templates
11+
# named compat/<your-sdk-branch-name> with the template fixes applied.
12+
# The job will automatically detect and test against that branch.
13+
14+
permissions:
15+
contents: read
16+
17+
jobs:
18+
template-compatibility:
19+
runs-on: ubuntu-latest
20+
21+
defaults:
22+
run:
23+
shell: bash {0}
24+
25+
env:
26+
TEMPLATES_REPO: smartcontractkit/cre-templates
27+
28+
steps:
29+
- name: Checkout SDK
30+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
31+
with:
32+
submodules: recursive
33+
34+
- name: Setup Rust (1.85.0) with wasm target
35+
uses: actions-rust-lang/setup-rust-toolchain@150fca883cd4034361b621bd4e6a9d34e5143606 # v1.15.4
36+
with:
37+
toolchain: 1.85.0
38+
target: wasm32-wasip1
39+
override: true
40+
41+
- name: Setup Bun
42+
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
43+
with:
44+
bun-version: 1.3.12
45+
46+
- name: Cache Bun dependencies
47+
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
48+
with:
49+
path: ~/.bun/install/cache
50+
key: ${{ runner.os }}-bun-${{ hashFiles('**/bun.lock') }}
51+
52+
- name: Cache cargo + Javy
53+
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
54+
with:
55+
path: |
56+
~/.cargo/registry
57+
~/.cargo/git
58+
~/.cache/javy
59+
key: ${{ runner.os }}-cre-plugin-v8.1.0-${{ hashFiles('packages/cre-sdk-javy-plugin/src/javy_chainlink_sdk/Cargo.lock', 'packages/cre-sdk-javy-plugin/src/cre_generated_host.Cargo.lock') }}
60+
61+
- name: Install SDK dependencies
62+
run: bun install --frozen-lockfile
63+
64+
# Detect whether a matching compat branch exists in cre-templates.
65+
# If it does, we test against it (coordinated breaking change).
66+
# If not, we fall back to main.
67+
- name: Detect cre-templates ref to test against
68+
id: detect-ref
69+
env:
70+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
71+
HEAD_REF: ${{ github.head_ref }}
72+
run: |
73+
SAFE_HEAD_REF="${HEAD_REF//[^a-zA-Z0-9._\/-]/}"
74+
COMPAT_BRANCH="compat/$SAFE_HEAD_REF"
75+
76+
if gh api "repos/$TEMPLATES_REPO/branches/$COMPAT_BRANCH" &>/dev/null; then
77+
echo "ref=$COMPAT_BRANCH" >> "$GITHUB_OUTPUT"
78+
echo "Using compat branch: $COMPAT_BRANCH"
79+
else
80+
echo "ref=main" >> "$GITHUB_OUTPUT"
81+
echo "No compat branch found, using: main"
82+
fi
83+
84+
- name: Checkout cre-templates (${{ steps.detect-ref.outputs.ref }})
85+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
86+
with:
87+
repository: ${{ env.TEMPLATES_REPO }}
88+
ref: ${{ steps.detect-ref.outputs.ref }}
89+
path: cre-templates
90+
token: ${{ secrets.GITHUB_TOKEN }}
91+
92+
- name: Setup Node (for npm)
93+
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
94+
with:
95+
node-version: '22'
96+
97+
- name: Run template compatibility check
98+
id: template-check
99+
env:
100+
TEMPLATES_DIR: cre-templates
101+
run: |
102+
set +e
103+
OUTPUT=$(./scripts/test-templates.sh 2>&1)
104+
EXIT_CODE=$?
105+
set -e
106+
107+
echo "$OUTPUT" > /tmp/template-check-output.txt
108+
109+
# Surface it in the action log regardless
110+
echo "$OUTPUT"
111+
112+
echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT"
113+
114+
# Bundle everything the comment workflow needs into an artifact.
115+
# The comment workflow runs with pull-requests: write but must never
116+
# execute external code — it only reads these files.
117+
- name: Save results for comment workflow
118+
if: always()
119+
env:
120+
PR_NUMBER: ${{ github.event.pull_request.number }}
121+
TEMPLATES_REF: ${{ steps.detect-ref.outputs.ref }}
122+
HEAD_REF: ${{ github.head_ref }}
123+
EXIT_CODE: ${{ steps.template-check.outputs.exit_code }}
124+
run: |
125+
mkdir -p /tmp/compat-results
126+
printf '%s' "$PR_NUMBER" > /tmp/compat-results/pr-number.txt
127+
printf '%s' "$TEMPLATES_REF" > /tmp/compat-results/templates-ref.txt
128+
printf '%s' "$HEAD_REF" > /tmp/compat-results/head-ref.txt
129+
printf '%s' "$EXIT_CODE" > /tmp/compat-results/exit-code.txt
130+
if [ -f /tmp/template-check-output.txt ]; then
131+
cp /tmp/template-check-output.txt /tmp/compat-results/output.txt
132+
fi
133+
134+
- name: Upload results artifact
135+
if: always()
136+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
137+
with:
138+
name: template-compat-results
139+
path: /tmp/compat-results/
140+
retention-days: 7
141+
142+
# Always exit 0 — this job is informational, not a merge gate
143+
- name: Report result (non-blocking)
144+
if: always()
145+
run: |
146+
EXIT_CODE="${{ steps.template-check.outputs.exit_code }}"
147+
if [ "$EXIT_CODE" = "0" ]; then
148+
echo "✅ All templates are compatible with this SDK change."
149+
else
150+
echo "⚠️ Some templates failed — see PR comment for details."
151+
echo "This check is informational and does not block merging."
152+
fi
153+
exit 0

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
"check:ci": "turbo run check:ci",
1515
"format": "turbo run format",
1616
"full-checks": "./scripts/full-checks.sh",
17+
"test:templates": "./scripts/test-templates.sh",
1718
"lint": "turbo run lint",
1819
"typecheck": "turbo run typecheck"
1920
},

0 commit comments

Comments
 (0)