Skip to content

Commit 5d69202

Browse files
haasonsaashaasonsaas
committed
Harden diffscope review operations
1 parent cda6b65 commit 5d69202

26 files changed

Lines changed: 726 additions & 77 deletions

File tree

‎.github/workflows/diffscope.yml‎

Lines changed: 77 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: DiffScope Review
22
on:
33
pull_request:
4-
types: [opened, synchronize, reopened]
4+
types: [opened, synchronize, reopened, review_requested]
55

66
permissions:
77
contents: read
@@ -20,61 +20,109 @@ jobs:
2020
with:
2121
fetch-depth: 0
2222
ref: ${{ github.event.pull_request.head.sha }}
23-
23+
24+
- uses: actions/setup-node@v5
25+
with:
26+
node-version: 24
27+
cache: npm
28+
cache-dependency-path: web/package-lock.json
29+
30+
- uses: dtolnay/rust-toolchain@1.88.0
31+
- uses: Swatinem/rust-cache@v2
32+
33+
- name: Build frontend
34+
working-directory: web
35+
run: |
36+
npm ci
37+
npm run build
38+
39+
- name: Build current DiffScope binary
40+
run: cargo build --release
41+
2442
- name: Get PR diff
25-
id: diff
2643
run: |
2744
git fetch origin ${{ github.base_ref }} --depth=1
2845
git diff origin/${{ github.base_ref }}...HEAD > pr.diff
29-
30-
- name: Check API key
31-
id: check_key
46+
47+
- name: Select review provider
48+
id: provider
49+
env:
50+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
51+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
52+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
3253
run: |
33-
if [ -z "${{ secrets.OPENAI_API_KEY }}" ]; then
34-
echo "skip=true" >> "$GITHUB_OUTPUT"
35-
echo "::notice::DiffScope review skipped — OPENAI_API_KEY secret not configured"
54+
if [ -n "${ANTHROPIC_API_KEY}" ]; then
55+
{
56+
echo "configured=true"
57+
echo "model=anthropic/claude-opus-4.5"
58+
echo "adapter=anthropic"
59+
} >> "$GITHUB_OUTPUT"
60+
elif [ -n "${OPENROUTER_API_KEY}" ]; then
61+
{
62+
echo "configured=true"
63+
echo "model=anthropic/claude-opus-4.5"
64+
echo "adapter=openrouter"
65+
} >> "$GITHUB_OUTPUT"
66+
elif [ -n "${OPENAI_API_KEY}" ]; then
67+
{
68+
echo "configured=true"
69+
echo "model=gpt-4o"
70+
echo "adapter=openai"
71+
} >> "$GITHUB_OUTPUT"
3672
else
37-
echo "skip=false" >> "$GITHUB_OUTPUT"
73+
echo "configured=false" >> "$GITHUB_OUTPUT"
74+
echo "::notice::DiffScope review skipped; no ANTHROPIC_API_KEY, OPENROUTER_API_KEY, or OPENAI_API_KEY secret is configured"
3875
fi
3976
40-
- name: Check image available
41-
id: check_image
77+
- name: Run DiffScope from this branch
78+
if: steps.provider.outputs.configured == 'true'
79+
env:
80+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
81+
OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
82+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
4283
run: |
43-
docker pull ghcr.io/evalops/diffscope:latest 2>/dev/null && echo "available=true" >> "$GITHUB_OUTPUT" || echo "available=false" >> "$GITHUB_OUTPUT"
44-
45-
- name: Notice image unavailable
46-
if: steps.check_image.outputs.available == 'false'
47-
run: echo "::notice::DiffScope review skipped — image ghcr.io/evalops/diffscope:latest not available (merge to main publishes it)"
84+
./target/release/diffscope \
85+
--model "${{ steps.provider.outputs.model }}" \
86+
--adapter "${{ steps.provider.outputs.adapter }}" \
87+
--output-format json \
88+
review --diff pr.diff --output comments.json
4889
49-
- name: Run DiffScope
50-
if: steps.check_key.outputs.skip != 'true' && steps.check_image.outputs.available == 'true'
51-
run: |
52-
docker run --rm \
53-
-e OPENAI_API_KEY="${{ secrets.OPENAI_API_KEY }}" \
54-
-v "$PWD":/workspace -w /workspace \
55-
ghcr.io/evalops/diffscope:latest \
56-
review --diff pr.diff --output-format json --output comments.json
90+
- name: Upload review artifact
91+
if: always() && steps.provider.outputs.configured == 'true'
92+
uses: actions/upload-artifact@v4
93+
with:
94+
name: diffscope-review-${{ github.event.pull_request.number }}
95+
path: comments.json
96+
if-no-files-found: ignore
5797

5898
- name: Post comments
59-
if: steps.check_key.outputs.skip != 'true' && steps.check_image.outputs.available == 'true'
99+
if: steps.provider.outputs.configured == 'true'
60100
uses: actions/github-script@v7
61101
with:
62102
script: |
63103
const fs = require('fs');
104+
if (!fs.existsSync('comments.json')) {
105+
core.notice('DiffScope did not produce comments.json');
106+
return;
107+
}
64108
const comments = JSON.parse(fs.readFileSync('comments.json', 'utf8'));
65109
const headSha = context.payload.pull_request.head.sha;
66110
const fallback = [];
67-
111+
68112
for (const comment of comments) {
69113
if (!comment.file_path || !comment.line_number || comment.line_number < 1) {
70114
continue;
71115
}
116+
const body = [
117+
`**${comment.severity}**: ${comment.content}`,
118+
comment.suggestion ? `\nSuggestion: ${comment.suggestion}` : ''
119+
].join('');
72120
try {
73121
await github.rest.pulls.createReviewComment({
74122
owner: context.repo.owner,
75123
repo: context.repo.repo,
76124
pull_number: context.issue.number,
77-
body: `**${comment.severity}**: ${comment.content}`,
125+
body,
78126
commit_id: headSha,
79127
path: comment.file_path,
80128
line: comment.line_number,
@@ -90,7 +138,7 @@ jobs:
90138
"Some review comments could not be placed inline and are listed below:",
91139
"",
92140
...fallback.slice(0, 100)
93-
].join("\\n");
141+
].join("\n");
94142
await github.rest.issues.createComment({
95143
owner: context.repo.owner,
96144
repo: context.repo.repo,

‎.github/workflows/eval.yml‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,27 @@ env:
1717
CARGO_TERM_COLOR: always
1818

1919
jobs:
20+
contracts:
21+
runs-on: ubuntu-latest
22+
timeout-minutes: 20
23+
steps:
24+
- uses: actions/checkout@v5
25+
- uses: dtolnay/rust-toolchain@1.88.0
26+
- uses: Swatinem/rust-cache@v2
27+
- name: Run always-on review contract tests
28+
run: |
29+
cargo test fetch_additional_context_rejects_absolute_and_parent_traversal_outside_base
30+
cargo test fetch_context_rejects_parent_traversal
31+
cargo test resolve_pattern_repositories_rejects_parent_traversal_local_paths
32+
cargo test resolve_pattern_repositories_rejects_absolute_local_paths_outside_repo
33+
cargo test build_verification_prompt_sanitizes_adversarial_finding_text
34+
cargo test execute_dag_retries_retryable_nodes
35+
cargo test execute_dag_applies_node_timeout
36+
2037
eval:
2138
runs-on: ubuntu-latest
2239
timeout-minutes: 60
40+
needs: contracts
2341
steps:
2442
- name: Check eval secret
2543
id: secret-check
@@ -59,6 +77,7 @@ jobs:
5977
--model gpt-4o-mini \
6078
--temperature 0 \
6179
--fixtures eval/fixtures \
80+
--artifact-dir /tmp/eval-baseline-artifacts \
6281
--output /tmp/eval-baseline.json
6382
6483
- name: Run eval thresholds on current branch
@@ -71,14 +90,15 @@ jobs:
7190
--temperature 0 \
7291
--fixtures eval/fixtures \
7392
--output eval-current.json \
93+
--artifact-dir eval-artifacts \
7494
--baseline /tmp/eval-baseline.json \
75-
--max-micro-f1-drop 0.20 \
76-
--min-micro-f1 0.20 \
77-
--min-verification-health 0.80 \
95+
--max-micro-f1-drop 0.15 \
96+
--min-micro-f1 0.25 \
97+
--min-verification-health 0.85 \
7898
--min-rule-f1 sec.shell.injection=0.10 \
7999
--min-rule-f1 reliability.unwrap_panic=0.10 \
80-
--max-rule-f1-drop sec.shell.injection=0.25 \
81-
--max-rule-f1-drop reliability.unwrap_panic=0.25
100+
--max-rule-f1-drop sec.shell.injection=0.20 \
101+
--max-rule-f1-drop reliability.unwrap_panic=0.20
82102
83103
- name: Upload eval reports
84104
if: ${{ always() && steps.secret-check.outputs.configured == 'true' }}
@@ -87,8 +107,10 @@ jobs:
87107
name: eval-reports
88108
path: |
89109
eval-current.json
110+
eval-artifacts/**
90111
/tmp/eval-baseline.json
112+
/tmp/eval-baseline-artifacts/**
91113
92114
- name: Skip message
93115
if: ${{ steps.secret-check.outputs.configured != 'true' }}
94-
run: echo "Skipping eval workflow because OPENAI_API_KEY secret is not configured."
116+
run: echo "Skipping live eval workflow because OPENAI_API_KEY secret is not configured; offline contract tests still ran."

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ name = "diffscope"
33
version = "0.5.28"
44
edition = "2021"
55
rust-version = "1.88"
6+
build = "build.rs"
67
authors = ["Jonathan Haas <jonathan@haas.holdings>"]
78
description = "A composable code review engine with smart analysis, confidence scoring, and professional reporting"
89
license = "Apache-2.0"

‎action.yml‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,22 @@ branding:
77

88
inputs:
99
model:
10-
description: 'LLM model to use (e.g., gpt-4o, ollama:codellama)'
10+
description: 'LLM model to use (e.g., anthropic/claude-opus-4.5, openai/gpt-4o, ollama:codellama)'
1111
required: false
12-
default: 'gpt-4o'
12+
default: 'anthropic/claude-opus-4.5'
1313
output-format:
1414
description: 'Output format (json, patch, markdown)'
1515
required: false
1616
default: 'json'
1717
openai-api-key:
1818
description: 'OpenAI API key (can also use OPENAI_API_KEY env var)'
1919
required: false
20+
anthropic-api-key:
21+
description: 'Anthropic API key (can also use ANTHROPIC_API_KEY env var)'
22+
required: false
23+
openrouter-api-key:
24+
description: 'OpenRouter API key (can also use OPENROUTER_API_KEY env var)'
25+
required: false
2026

2127
runs:
2228
using: 'docker'
@@ -28,4 +34,6 @@ runs:
2834
- '--output-format'
2935
- ${{ inputs.output-format }}
3036
env:
31-
OPENAI_API_KEY: ${{ inputs.openai-api-key }}
37+
OPENAI_API_KEY: ${{ inputs.openai-api-key }}
38+
ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
39+
OPENROUTER_API_KEY: ${{ inputs.openrouter-api-key }}

‎build.rs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
use std::fs;
2+
use std::path::Path;
3+
4+
fn main() {
5+
println!("cargo:rerun-if-changed=web/dist");
6+
7+
let dist = Path::new("web/dist");
8+
if dist.exists() {
9+
return;
10+
}
11+
12+
fs::create_dir_all(dist).expect("failed to create web/dist fallback directory");
13+
fs::write(
14+
dist.join("index.html"),
15+
r#"<!doctype html><html><head><meta charset="utf-8"><title>DiffScope</title></head><body>Frontend not built. Run npm run build in web/ for the full UI.</body></html>"#,
16+
)
17+
.expect("failed to write web/dist fallback index.html");
18+
}

‎charts/diffscope/templates/deployment.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,12 @@ spec:
9898
- configMapRef:
9999
name: {{ include "diffscope.fullname" . }}
100100
env:
101+
- name: DIFFSCOPE_SERVER_API_KEY
102+
valueFrom:
103+
secretKeyRef:
104+
name: {{ include "diffscope.secretName" . }}
105+
key: {{ .Values.secrets.existingSecretKeys.serverApiKey }}
106+
optional: true
101107
- name: DIFFSCOPE_API_KEY
102108
valueFrom:
103109
secretKeyRef:
@@ -122,6 +128,36 @@ spec:
122128
name: {{ include "diffscope.secretName" . }}
123129
key: {{ .Values.secrets.existingSecretKeys.anthropicApiKey }}
124130
optional: true
131+
- name: GITHUB_TOKEN
132+
valueFrom:
133+
secretKeyRef:
134+
name: {{ include "diffscope.secretName" . }}
135+
key: {{ .Values.secrets.existingSecretKeys.githubToken }}
136+
optional: true
137+
- name: DIFFSCOPE_GITHUB_APP_ID
138+
valueFrom:
139+
secretKeyRef:
140+
name: {{ include "diffscope.secretName" . }}
141+
key: {{ .Values.secrets.existingSecretKeys.githubAppId }}
142+
optional: true
143+
- name: DIFFSCOPE_GITHUB_PRIVATE_KEY
144+
valueFrom:
145+
secretKeyRef:
146+
name: {{ include "diffscope.secretName" . }}
147+
key: {{ .Values.secrets.existingSecretKeys.githubPrivateKey }}
148+
optional: true
149+
- name: DIFFSCOPE_WEBHOOK_SECRET
150+
valueFrom:
151+
secretKeyRef:
152+
name: {{ include "diffscope.secretName" . }}
153+
key: {{ .Values.secrets.existingSecretKeys.githubWebhookSecret }}
154+
optional: true
155+
- name: DIFFSCOPE_AUTOMATION_WEBHOOK_SECRET
156+
valueFrom:
157+
secretKeyRef:
158+
name: {{ include "diffscope.secretName" . }}
159+
key: {{ .Values.secrets.existingSecretKeys.automationWebhookSecret }}
160+
optional: true
125161
livenessProbe:
126162
httpGet:
127163
path: /api/status

‎charts/diffscope/templates/secret.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ metadata:
77
{{- include "diffscope.labels" . | nindent 4 }}
88
type: Opaque
99
stringData:
10+
{{- if .Values.secrets.serverApiKey }}
11+
DIFFSCOPE_SERVER_API_KEY: {{ .Values.secrets.serverApiKey | quote }}
12+
{{- end }}
1013
{{- if .Values.secrets.diffscopeApiKey }}
1114
DIFFSCOPE_API_KEY: {{ .Values.secrets.diffscopeApiKey | quote }}
1215
{{- end }}
@@ -19,4 +22,19 @@ stringData:
1922
{{- if .Values.secrets.anthropicApiKey }}
2023
ANTHROPIC_API_KEY: {{ .Values.secrets.anthropicApiKey | quote }}
2124
{{- end }}
25+
{{- if .Values.secrets.githubToken }}
26+
GITHUB_TOKEN: {{ .Values.secrets.githubToken | quote }}
27+
{{- end }}
28+
{{- if .Values.secrets.githubAppId }}
29+
DIFFSCOPE_GITHUB_APP_ID: {{ .Values.secrets.githubAppId | quote }}
30+
{{- end }}
31+
{{- if .Values.secrets.githubPrivateKey }}
32+
DIFFSCOPE_GITHUB_PRIVATE_KEY: {{ .Values.secrets.githubPrivateKey | quote }}
33+
{{- end }}
34+
{{- if .Values.secrets.githubWebhookSecret }}
35+
DIFFSCOPE_WEBHOOK_SECRET: {{ .Values.secrets.githubWebhookSecret | quote }}
36+
{{- end }}
37+
{{- if .Values.secrets.automationWebhookSecret }}
38+
DIFFSCOPE_AUTOMATION_WEBHOOK_SECRET: {{ .Values.secrets.automationWebhookSecret | quote }}
39+
{{- end }}
2240
{{- end }}

‎charts/diffscope/values.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,31 @@ diffscope:
3535

3636
# -- API keys (stored in a Secret)
3737
secrets:
38+
# API key for protected DiffScope server mutation routes.
39+
serverApiKey: ""
40+
# Legacy/top-level LLM API key fallback. Prefer provider-specific keys below.
3841
diffscopeApiKey: ""
3942
openaiApiKey: ""
4043
openrouterApiKey: ""
4144
anthropicApiKey: ""
45+
githubToken: ""
46+
githubAppId: ""
47+
githubPrivateKey: ""
48+
githubWebhookSecret: ""
49+
automationWebhookSecret: ""
4250
# Use an existing secret instead of creating one
4351
existingSecret: ""
4452
existingSecretKeys:
53+
serverApiKey: "DIFFSCOPE_SERVER_API_KEY"
4554
diffscopeApiKey: "DIFFSCOPE_API_KEY"
4655
openaiApiKey: "OPENAI_API_KEY"
4756
openrouterApiKey: "OPENROUTER_API_KEY"
4857
anthropicApiKey: "ANTHROPIC_API_KEY"
58+
githubToken: "GITHUB_TOKEN"
59+
githubAppId: "DIFFSCOPE_GITHUB_APP_ID"
60+
githubPrivateKey: "DIFFSCOPE_GITHUB_PRIVATE_KEY"
61+
githubWebhookSecret: "DIFFSCOPE_WEBHOOK_SECRET"
62+
automationWebhookSecret: "DIFFSCOPE_AUTOMATION_WEBHOOK_SECRET"
4963

5064
# -- Non-secret environment config
5165
config:

0 commit comments

Comments
 (0)