Skip to content

Commit c7e928e

Browse files
caio-pizzolcaio-pizzol
committed
feat(agent-docs-audit): delta-only high-confidence CI gate
Fails the workflow when a PR introduces new high-confidence findings vs base, while leaving existing baseline debt and heuristic classes warning- only. High-confidence classes that block CI: - broken @imports - broken symlink targets - linked-inverted pairs - unexpected-duplicate pairs Heuristic/advisory classes (still warning-only via the comment): - broken path refs (backtick regex, known false-positive prone) - budget warnings - unresolved pnpm commands Mechanics: a separate gate script worktree-scans origin/$BASE_REF and diffs high-confidence finding identities against the PR head, scoped to files (or pair-dirs) the PR actually changed. Result is written to /tmp/agent-docs-gate.json; the comment script reads it and prepends a 'Blocking' header so reviewers see why CI is red without opening logs.
1 parent a4449ba commit c7e928e

3 files changed

Lines changed: 227 additions & 12 deletions

File tree

‎.github/scripts/agent-docs-pr-comment.mjs‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
*/
88

99
import { execFileSync } from 'node:child_process';
10-
import { writeFileSync } from 'node:fs';
10+
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
1111
import { tmpdir } from 'node:os';
1212
import { dirname, join, relative, resolve } from 'node:path';
1313
import { computeFlags, pairFlaggedForReview, runL1Scan } from './agent-docs-l1.mjs';
@@ -17,8 +17,18 @@ const PR = process.env.PR_NUMBER;
1717
const REPO = process.env.REPO ?? 'superdoc-dev/superdoc';
1818
const REPO_ROOT = resolve(process.env.REPO_ROOT ?? process.cwd());
1919
const SHA = process.env.GITHUB_SHA ?? 'unknown-sha';
20+
const GATE_RESULT_PATH = process.env.GATE_RESULT_PATH || '/tmp/agent-docs-gate.json';
2021
const DRY_RUN = process.argv.includes('--dry-run');
2122

23+
function readGateResult() {
24+
try {
25+
if (!existsSync(GATE_RESULT_PATH)) return null;
26+
return JSON.parse(readFileSync(GATE_RESULT_PATH, 'utf-8'));
27+
} catch {
28+
return null;
29+
}
30+
}
31+
2232
if (!PR && !DRY_RUN) {
2333
console.log('PR_NUMBER not set; not in a PR context. Skipping.');
2434
process.exit(0);
@@ -124,13 +134,16 @@ function formatPairFinding(finding) {
124134
}
125135

126136
function buildFindingsBody(findings) {
127-
const lines = [
128-
MARKER,
129-
'## Agent docs audit',
130-
'',
131-
`Found deterministic findings on ${findings.length} changed agent-doc item(s).`,
132-
'',
133-
];
137+
const gate = readGateResult();
138+
const lines = [MARKER, '## Agent docs audit', ''];
139+
if (gate?.blocking) {
140+
lines.push(
141+
`**Blocking**: this PR introduces ${gate.newFindings.length} new high-confidence finding(s). CI will fail until resolved.`,
142+
);
143+
lines.push('');
144+
}
145+
lines.push(`Found deterministic findings on ${findings.length} changed agent-doc item(s).`);
146+
lines.push('');
134147

135148
for (const finding of findings) {
136149
lines.push(finding.type === 'pair' ? formatPairFinding(finding) : formatFileFinding(finding));

‎.github/scripts/agent-docs-pr-gate.mjs‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
#!/usr/bin/env node
2+
/**
3+
* Delta-only high-confidence gate. Fails the workflow when the PR introduces
4+
* NEW high-confidence agent-doc findings vs base.
5+
*
6+
* High-confidence classes:
7+
* - broken @imports
8+
* - broken symlink targets
9+
* - linked-inverted pairs
10+
* - unexpected-duplicate pairs
11+
*
12+
* Heuristic / advisory classes are explicitly excluded to keep the false-
13+
* positive rate near zero: brokenPathRefs (backtick regex), budget warnings,
14+
* unresolvedCommands.
15+
*
16+
* Writes the result to GATE_RESULT_PATH so the comment step can surface
17+
* "Blocking" state inline. Exits 1 if blocking, 0 otherwise.
18+
*/
19+
20+
import { execFileSync } from 'node:child_process';
21+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
22+
import { tmpdir } from 'node:os';
23+
import { dirname, join, resolve } from 'node:path';
24+
import { runL1Scan } from './agent-docs-l1.mjs';
25+
26+
const REPO_ROOT = resolve(process.env.REPO_ROOT ?? process.cwd());
27+
const BASE_REF = process.env.BASE_REF || 'main';
28+
const PR = process.env.PR_NUMBER;
29+
const REPO = process.env.REPO ?? 'superdoc-dev/superdoc';
30+
const RESULT_PATH = process.env.GATE_RESULT_PATH || '/tmp/agent-docs-gate.json';
31+
const DRY_RUN = process.argv.includes('--dry-run');
32+
33+
function isAgentDocPath(path) {
34+
if (/(?:^|\/)(?:AGENTS|CLAUDE)(?:\.local)?\.md$/.test(path)) return true;
35+
return /(?:^|\/)\.claude\/rules\/.+\.md$/.test(path);
36+
}
37+
38+
function getChangedAgentDocs() {
39+
if (DRY_RUN) {
40+
const idx = process.argv.indexOf('--files');
41+
if (idx < 0) return [];
42+
return (process.argv[idx + 1] || '').split(',').map((s) => s.trim()).filter(Boolean).filter(isAgentDocPath);
43+
}
44+
if (!PR) return [];
45+
try {
46+
const out = execFileSync('gh', ['pr', 'diff', PR, '--repo', REPO, '--name-only'], { encoding: 'utf-8' });
47+
return out.split('\n').map((s) => s.trim()).filter(Boolean).filter(isAgentDocPath);
48+
} catch (err) {
49+
console.log(`Could not list PR changed files: ${err.message}`);
50+
return [];
51+
}
52+
}
53+
54+
function changedPairDirs(paths) {
55+
const dirs = new Set();
56+
for (const path of paths) {
57+
if (/(?:^|\/)(?:AGENTS|CLAUDE)(?:\.local)?\.md$/.test(path)) {
58+
dirs.add(dirname(path));
59+
}
60+
}
61+
return dirs;
62+
}
63+
64+
function highConfidenceFindings(scan) {
65+
const findings = [];
66+
for (const file of scan.files) {
67+
if (file.brokenSymlinkTarget) {
68+
findings.push({
69+
type: 'broken-symlink',
70+
relPath: file.relPath,
71+
target: file.brokenSymlinkTarget,
72+
id: `symlink:${file.relPath}`,
73+
});
74+
}
75+
if (file.isSymlink) continue;
76+
for (const importPath of file.brokenImports) {
77+
findings.push({
78+
type: 'broken-import',
79+
relPath: file.relPath,
80+
importPath,
81+
id: `import:${file.relPath}:${importPath}`,
82+
});
83+
}
84+
}
85+
for (const pair of scan.pairs) {
86+
if (pair.classification === 'linked-inverted' || pair.classification === 'unexpected-duplicate') {
87+
findings.push({
88+
type: 'pair',
89+
dir: pair.dir,
90+
classification: pair.classification,
91+
detail: pair.detail,
92+
id: `pair:${pair.dir}:${pair.classification}`,
93+
});
94+
}
95+
}
96+
return findings;
97+
}
98+
99+
function prepareBaseSnapshot() {
100+
execFileSync('git', ['fetch', '--no-tags', '--depth=1', 'origin', BASE_REF], { cwd: REPO_ROOT, stdio: 'inherit' });
101+
const baseDir = mkdtempSync(join(tmpdir(), 'agent-docs-base-'));
102+
execFileSync('git', ['worktree', 'add', '--detach', baseDir, `origin/${BASE_REF}`], { cwd: REPO_ROOT, stdio: 'inherit' });
103+
return baseDir;
104+
}
105+
106+
function cleanupBaseSnapshot(baseDir) {
107+
try {
108+
execFileSync('git', ['worktree', 'remove', '--force', baseDir], { cwd: REPO_ROOT, stdio: 'ignore' });
109+
} catch {
110+
rmSync(baseDir, { recursive: true, force: true });
111+
}
112+
}
113+
114+
function writeResult(result) {
115+
writeFileSync(RESULT_PATH, JSON.stringify(result, null, 2));
116+
}
117+
118+
const changed = getChangedAgentDocs();
119+
if (changed.length === 0) {
120+
console.log('No agent-doc files changed; gate is a no-op.');
121+
writeResult({ blocking: false, newFindings: [], changed: [] });
122+
process.exit(0);
123+
}
124+
125+
console.log(`Changed agent-doc files: ${changed.join(', ')}`);
126+
127+
const headScan = runL1Scan(REPO_ROOT);
128+
const headFindings = highConfidenceFindings(headScan);
129+
130+
let baseFindings = [];
131+
let baseDir = null;
132+
try {
133+
if (DRY_RUN) {
134+
const baseFromFlag = process.argv.indexOf('--base-root');
135+
if (baseFromFlag >= 0 && process.argv[baseFromFlag + 1]) {
136+
baseFindings = highConfidenceFindings(runL1Scan(resolve(process.argv[baseFromFlag + 1])));
137+
}
138+
} else {
139+
baseDir = prepareBaseSnapshot();
140+
baseFindings = highConfidenceFindings(runL1Scan(baseDir));
141+
}
142+
} finally {
143+
if (baseDir) cleanupBaseSnapshot(baseDir);
144+
}
145+
146+
const baseIds = new Set(baseFindings.map((f) => f.id));
147+
const newFindings = headFindings.filter((f) => !baseIds.has(f.id));
148+
149+
const changedSet = new Set(changed);
150+
const dirSet = changedPairDirs(changed);
151+
152+
const scoped = newFindings.filter((f) => {
153+
if (f.type === 'pair') return dirSet.has(f.dir);
154+
return changedSet.has(f.relPath);
155+
});
156+
157+
const result = { blocking: scoped.length > 0, newFindings: scoped, changed };
158+
writeResult(result);
159+
160+
if (result.blocking) {
161+
console.log('\nBlocking — new high-confidence findings introduced by this PR:');
162+
for (const f of scoped) {
163+
if (f.type === 'broken-import') console.log(` - broken @import in ${f.relPath}: ${f.importPath}`);
164+
else if (f.type === 'broken-symlink') console.log(` - broken symlink ${f.relPath} → ${f.target}`);
165+
else if (f.type === 'pair') console.log(` - pair ${f.dir} ${f.classification}: ${f.detail}`);
166+
}
167+
console.log(`\nWrote ${RESULT_PATH}`);
168+
process.exit(1);
169+
}
170+
171+
console.log('No new high-confidence findings introduced by this PR. Gate passes.');
172+
process.exit(0);

‎.github/workflows/agent-docs-audit.yml‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,12 @@ name: Agent docs audit
1010
# concrete claims via Read/Glob/Grep and produces
1111
# KEEP/TRIM/MOVE/UPDATE/INVESTIGATE findings.
1212
#
13-
# Warning-only for now. Uploads artifacts and writes a Step Summary; does not
14-
# post PR comments, does not fail the workflow on findings.
13+
# PR runs post a diff-scoped sticky comment with L1 findings, then enforce a
14+
# delta-only high-confidence gate: the workflow fails when the PR introduces
15+
# new broken @imports, broken symlink targets, or unexpected pair drift
16+
# (linked-inverted, unexpected-duplicate). Existing baseline debt on touched
17+
# files does not fail CI. Heuristic classes (broken path refs, budget warnings,
18+
# unresolved commands) remain advisory.
1519
#
1620
# AI layers are skipped automatically if ANTHROPIC_API_KEY is unavailable
1721
# (fork PRs, secret not set). In that case the L1 report still uploads.
@@ -125,14 +129,40 @@ jobs:
125129
if-no-files-found: warn
126130
retention-days: 30
127131

132+
# Delta-only high-confidence gate. Runs before the comment step so the
133+
# sticky can surface "Blocking" state inline. Never fails the comment
134+
# step itself (set as a separate step further down with always()).
135+
- name: Delta gate (PR only, scan-and-write)
136+
if: github.event_name == 'pull_request'
137+
id: gate
138+
continue-on-error: true
139+
env:
140+
PR_NUMBER: ${{ github.event.pull_request.number }}
141+
REPO: ${{ github.repository }}
142+
REPO_ROOT: ${{ github.workspace }}
143+
BASE_REF: ${{ github.base_ref }}
144+
GATE_RESULT_PATH: /tmp/agent-docs-gate.json
145+
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
146+
run: node .github/scripts/agent-docs-pr-gate.mjs
147+
128148
# Diff-scoped sticky PR comment. Pull_request runs are L1-only; this
129149
# surfaces deterministic findings only for agent-doc files touched by the
130-
# PR. No AI, no Bash, no secrets.
150+
# PR. Reads the gate result file if present to add a "Blocking" header.
151+
# always() so the comment posts even when the gate step exits non-zero.
131152
- name: Post sticky PR comment with L1 findings
132-
if: github.event_name == 'pull_request'
153+
if: ${{ always() && github.event_name == 'pull_request' }}
133154
env:
134155
PR_NUMBER: ${{ github.event.pull_request.number }}
135156
REPO: ${{ github.repository }}
136157
REPO_ROOT: ${{ github.workspace }}
158+
GATE_RESULT_PATH: /tmp/agent-docs-gate.json
137159
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
138160
run: node .github/scripts/agent-docs-pr-comment.mjs
161+
162+
# Enforce the gate result. Separate step so the comment posts first; this
163+
# step is the one that turns the job red when blocking findings exist.
164+
- name: Enforce delta gate
165+
if: ${{ always() && github.event_name == 'pull_request' && steps.gate.outcome == 'failure' }}
166+
run: |
167+
echo "::error::Agent docs audit gate failed — PR introduces new high-confidence findings. See sticky comment."
168+
exit 1

0 commit comments

Comments
 (0)