Skip to content

Commit e205f25

Browse files
fix(review): classify bounded scope correctly
1 parent 6dd3c68 commit e205f25

2 files changed

Lines changed: 29 additions & 3 deletions

File tree

src/adapter.ts

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1543,7 +1543,7 @@ export function sessionBrief(
15431543
.filter((item): item is JsonObject => Boolean(item) && typeof item === "object" && !Array.isArray(item))
15441544
.map((item) => String(item.filename || "").trim())
15451545
.filter(Boolean);
1546-
let instruction = "This run is evidence-backed. The trusted worker embedded review_context in the session brief; it is not a separate repository file. Do not search /workspace for a review_context artifact or report the absence of a separate file as a limitation. Review the changed files in the workspace and cite each one you inspected in the result summary. Use Read to inspect and cite at least one relevant supporting repository file when one exists; the repository AGENTS.md is relevant supporting context when present because it defines review and contribution constraints. A review intentionally bounded to all changed files plus relevant supporting context is complete, so do not describe the absence of unrelated-file inspection as a limitation.";
1546+
let instruction = "This run is evidence-backed. The trusted worker embedded review_context in the session brief; it is not a separate repository file. Do not search /workspace for a review_context artifact or report the absence of a separate file as a limitation. Review the changed files in the workspace and cite each one you inspected in the result summary. Use Read to inspect and cite at least one relevant supporting repository file when one exists; the repository AGENTS.md is relevant supporting context when present because it defines review and contribution constraints. A review intentionally bounded to all changed files plus relevant supporting context is complete, so do not describe the absence of unrelated-file inspection as a limitation. In the Confidence/limitations section, write `None` when there is no material limitation; never prefix the required bounded review scope with `Limitation:`.";
15471547
if (changedFiles.length) {
15481548
instruction += ` Changed files supplied by the trusted worker: ${JSON.stringify(changedFiles)}.`;
15491549
}
@@ -2707,6 +2707,13 @@ function executionOnlyLimitation(value: string): boolean {
27072707
|| /\b(?:tests?|checks?|commands?|suite)\b.{0,80}\b(?:not run|not executed|were not run|could not run|unable to run)\b/i.test(value);
27082708
}
27092709

2710+
export function expectedReviewScopeStatement(value: string): boolean {
2711+
const describesExpectedScope = /\b(?:bounded(?: review)? scope|bounded review instructions?|reviewed only the changed files?|limited to (?:the )?(?:supplied )?change set)\b/i.test(value)
2712+
&& /\b(?:changed files?|change set|supporting context|agent guidance|AGENTS\.md)\b/i.test(value);
2713+
const reportsMaterialConstraint = /\b(?:unable|could not|couldn't|missing|truncated|unavailable|uncertain|incomplete|not provided|not supplied|failed to inspect|relevant (?:dependency|file|context).{0,30}(?:not|missing|unavailable))\b/i.test(value);
2714+
return describesExpectedScope && !reportsMaterialConstraint;
2715+
}
2716+
27102717
export function trustedValidationFindings(task: JsonObject, receipts: JsonObject[]): JsonObject[] {
27112718
const evidence = (task.review_evidence as JsonObject | undefined) || {};
27122719
const changedFiles = (Array.isArray(evidence.changed_files) ? evidence.changed_files : [])
@@ -2763,10 +2770,11 @@ export function finalizeReviewResult(resultPath: string, attemptDir: string, rec
27632770
workspace_revision: receipt.workspace_revision,
27642771
}));
27652772
const limitations = Array.isArray(review.limitations) ? review.limitations : [];
2773+
const materialLimitations = limitations.filter((item) => typeof item !== "string" || !expectedReviewScopeStatement(item));
27662774
const trustedExecutionAvailable = receipts.length > 0;
27672775
review.limitations = trustedExecutionAvailable
2768-
? limitations.filter((item) => typeof item !== "string" || !executionOnlyLimitation(item))
2769-
: limitations;
2776+
? materialLimitations.filter((item) => typeof item !== "string" || !executionOnlyLimitation(item))
2777+
: materialLimitations;
27702778
if (findings.length) review.no_findings_reason = null;
27712779
result.review = review;
27722780
result.trusted_validation = {

tests/webhook-adapter.test.ts

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import {
1414
createFreshTaskAttemptDirectory,
1515
createConfig,
1616
createFollowupReviewTask,
17+
expectedReviewScopeStatement,
1718
finalizeReviewResult,
1819
githubRequestAllPages,
1920
handleRequest,
@@ -1142,6 +1143,8 @@ test("hosted review brief directs failed trusted validation into source-backed f
11421143
assert.match(instruction, /Use Read to inspect and cite at least one relevant supporting repository file/);
11431144
assert.match(instruction, /AGENTS\.md is relevant supporting context/);
11441145
assert.match(instruction, /do not describe the absence of unrelated-file inspection as a limitation/);
1146+
assert.match(instruction, /write `None` when there is no material limitation/);
1147+
assert.match(instruction, /never prefix the required bounded review scope with `Limitation:`/);
11451148
assert.match(instruction, /probe\/covencat-live\.mjs/);
11461149
assert.match(instruction, /executed outside the model/);
11471150
assert.match(instruction, /SyntaxError: Unexpected end of input/);
@@ -1151,6 +1154,21 @@ test("hosted review brief directs failed trusted validation into source-backed f
11511154
assert.doesNotMatch(instruction, /trust.*runtime/i);
11521155
});
11531156

1157+
test("distinguishes expected bounded review scope from material limitations", () => {
1158+
assert.equal(expectedReviewScopeStatement(
1159+
"High confidence. Limitation: I reviewed only the changed file plus the repo's agent guidance, per the bounded review instructions.",
1160+
), true);
1161+
assert.equal(expectedReviewScopeStatement(
1162+
"The review is limited to the supplied change set and supporting context and does not assess unrelated repository areas.",
1163+
), true);
1164+
assert.equal(expectedReviewScopeStatement(
1165+
"I reviewed only the changed file because relevant dependency context was unavailable.",
1166+
), false);
1167+
assert.equal(expectedReviewScopeStatement(
1168+
"The supplied patch was truncated, so the review is incomplete.",
1169+
), false);
1170+
});
1171+
11541172
test("runtime result reader rejects symlinks and oversized artifacts", () => {
11551173
const root = mkdtempSync(join(tmpdir(), "coven-runtime-result-"));
11561174
const valid = join(root, "valid.json");

0 commit comments

Comments
 (0)