[codex] Harden DiffScope review operations #178
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Eval Quality | |
| on: | |
| pull_request: | |
| paths: | |
| - 'src/**' | |
| - 'eval/**' | |
| - '.github/workflows/eval.yml' | |
| - 'Cargo.toml' | |
| - 'Cargo.lock' | |
| workflow_dispatch: | |
| permissions: | |
| contents: read | |
| env: | |
| CARGO_TERM_COLOR: always | |
| jobs: | |
| contracts: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: dtolnay/rust-toolchain@1.88.0 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run always-on review contract tests | |
| run: | | |
| cargo test fetch_additional_context_rejects_absolute_and_parent_traversal_outside_base | |
| cargo test fetch_context_rejects_parent_traversal | |
| cargo test resolve_pattern_repositories_rejects_parent_traversal_local_paths | |
| cargo test resolve_pattern_repositories_rejects_absolute_local_paths_outside_repo | |
| cargo test build_verification_prompt_sanitizes_adversarial_finding_text | |
| cargo test execute_dag_retries_retryable_nodes | |
| cargo test execute_dag_applies_node_timeout | |
| eval: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| needs: contracts | |
| steps: | |
| - name: Check eval secret | |
| id: secret-check | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| run: | | |
| if [ -n "${OPENAI_API_KEY}" ]; then | |
| echo "configured=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "configured=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: actions/checkout@v5 | |
| if: ${{ steps.secret-check.outputs.configured == 'true' }} | |
| with: | |
| fetch-depth: 0 | |
| - uses: dtolnay/rust-toolchain@1.88.0 | |
| if: ${{ steps.secret-check.outputs.configured == 'true' }} | |
| - uses: Swatinem/rust-cache@v2 | |
| if: ${{ steps.secret-check.outputs.configured == 'true' }} | |
| - name: Build current branch binary | |
| if: ${{ steps.secret-check.outputs.configured == 'true' }} | |
| run: cargo build --release | |
| - name: Build baseline report from origin/main | |
| if: ${{ steps.secret-check.outputs.configured == 'true' }} | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| run: | | |
| git fetch origin main --depth=1 | |
| git worktree add /tmp/diffscope-main origin/main | |
| cd /tmp/diffscope-main | |
| cargo build --release | |
| ./target/release/diffscope eval \ | |
| --model gpt-4o-mini \ | |
| --temperature 0 \ | |
| --fixtures eval/fixtures \ | |
| --artifact-dir /tmp/eval-baseline-artifacts \ | |
| --output /tmp/eval-baseline.json | |
| - name: Run eval thresholds on current branch | |
| if: ${{ steps.secret-check.outputs.configured == 'true' }} | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| run: | | |
| ./target/release/diffscope eval \ | |
| --model gpt-4o-mini \ | |
| --temperature 0 \ | |
| --fixtures eval/fixtures \ | |
| --output eval-current.json \ | |
| --artifact-dir eval-artifacts \ | |
| --baseline /tmp/eval-baseline.json \ | |
| --max-micro-f1-drop 0.15 \ | |
| --min-micro-f1 0.25 \ | |
| --min-verification-health 0.85 \ | |
| --min-rule-f1 sec.shell.injection=0.10 \ | |
| --min-rule-f1 reliability.unwrap_panic=0.10 \ | |
| --max-rule-f1-drop sec.shell.injection=0.20 \ | |
| --max-rule-f1-drop reliability.unwrap_panic=0.20 | |
| - name: Upload eval reports | |
| if: ${{ always() && steps.secret-check.outputs.configured == 'true' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: eval-reports | |
| path: | | |
| eval-current.json | |
| eval-artifacts/** | |
| /tmp/eval-baseline.json | |
| /tmp/eval-baseline-artifacts/** | |
| - name: Skip message | |
| if: ${{ steps.secret-check.outputs.configured != 'true' }} | |
| run: echo "Skipping live eval workflow because OPENAI_API_KEY secret is not configured; offline contract tests still ran." |