ci: pin lint tooling, install a dev requirements set, and skip the eval gate without secrets

Author: codebybilal18

.github/workflows/ci.yml

@@ -43,15 +43,15 @@ jobs:
           cache: pip
           cache-dependency-path: |
             requirements.txt
-            requirements-local.txt
+            requirements-dev.txt
 
       - name: Install dependencies
-        # The Vercel-safe set plus test tooling. torch is excluded: it is a
-        # multi-minute install for tests that do not exercise it.
+        # requirements-dev.txt pulls in requirements.txt and adds pinned test
+        # and lint tooling. torch is excluded: it is a multi-minute install for
+        # tests that do not exercise it.
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
-          pip install pytest pytest-asyncio anyio ruff
+          pip install -r requirements-dev.txt
 
       - name: Lint
         run: ruff check app scripts tests

.github/workflows/evals.yml

@@ -50,25 +50,45 @@ jobs:
       TOLERANCE: ${{ github.event.inputs.tolerance || '-0.03' }}
 
     steps:
+      - name: Check that the eval secrets are configured
+        id: secrets
+        # The eval gate needs a database and an API key. Until those secrets
+        # exist the whole job is skipped with an explanatory notice rather than
+        # failing — a workflow that goes red because an optional secret is
+        # missing is a workflow people learn to ignore, and this one is meant
+        # to be trusted enough to block a merge.
+        run: |
+          if [ -z "${{ secrets.EVAL_DATABASE_URL }}" ] || [ -z "${{ secrets.OPENAI_API_KEY }}" ]; then
+            echo "configured=false" >> "$GITHUB_OUTPUT"
+            echo "::notice title=Eval gate skipped::Set the EVAL_DATABASE_URL and OPENAI_API_KEY repository secrets to enable it. See the README."
+          else
+            echo "configured=true" >> "$GITHUB_OUTPUT"
+          fi
+
       - uses: actions/checkout@v4
+        if: steps.secrets.outputs.configured == 'true'
         with:
           # compare.py reads the branch name from git, so it needs real refs.
           fetch-depth: 0
 
       - uses: actions/setup-python@v5
+        if: steps.secrets.outputs.configured == 'true'
         with:
           python-version: "3.12"
           cache: pip
 
       - name: Install dependencies
+        if: steps.secrets.outputs.configured == 'true'
         run: |
           python -m pip install --upgrade pip
           pip install -r requirements.txt
 
       - name: Migrate
+        if: steps.secrets.outputs.configured == 'true'
         run: alembic upgrade head
 
       - name: Run evals
+        if: steps.secrets.outputs.configured == 'true'
         # The corpus is assumed already ingested into the shared eval database.
         # Re-ingesting on every PR would cost more than the eval itself and add
         # embedding drift as a confounder — the corpus is a fixture, not part of
@@ -82,6 +102,7 @@ jobs:
             --notes "PR #${{ github.event.pull_request.number }}"
 
       - name: Compare against main
+        if: steps.secrets.outputs.configured == 'true'
         id: compare
         run: |
           python -m app.evals.compare \
@@ -91,7 +112,7 @@ jobs:
             | tee evals/out/comparison.txt
 
       - name: Comment the results on the PR
-        if: always() && github.event_name == 'pull_request'
+        if: always() && steps.secrets.outputs.configured == 'true' && github.event_name == 'pull_request'
         uses: actions/github-script@v7
         with:
           script: |
@@ -116,7 +137,7 @@ jobs:
             });
 
       - name: Upload artefacts
-        if: always()
+        if: always() && steps.secrets.outputs.configured == 'true'
         uses: actions/upload-artifact@v4
         with:
           name: eval-run-${{ github.run_id }}

requirements-dev.txt

@@ -0,0 +1,14 @@
+# Test and lint tooling. Pinned, and deliberately separate from
+# requirements-local.txt so CI can install it without pulling torch.
+#
+# ruff's version is pinned because `ruff format --check` is a CI gate: an
+# unpinned formatter means a new ruff release reformats something and the build
+# goes red with no code change. A formatter that fails on days you did not touch
+# the repo is a formatter people learn to ignore.
+
+-r requirements.txt
+
+pytest==8.3.4
+pytest-asyncio==0.25.0
+anyio==4.7.0
+ruff==0.16.0

requirements-local.txt

@@ -8,13 +8,7 @@
 #
 # Install with:  pip install -r requirements.txt -r requirements-local.txt
 
--r requirements.txt
+-r requirements-dev.txt
 
 sentence-transformers==3.3.1
 torch==2.5.1
-
-# Test + tooling
-pytest==8.3.4
-pytest-asyncio==0.25.0
-anyio==4.7.0
-ruff==0.8.4

scripts/smoke_test.py

@@ -279,8 +279,7 @@ async def check_query(use_llm: bool) -> None:
     check(
         "claims verified and supported by their citations",
         verification.get("verified", False) and coverage >= 0.99,
-        f"coverage {coverage:.2f}, "
-        f"unsupported {verification.get('unsupported_claims') or 'none'}",
+        f"coverage {coverage:.2f}, unsupported {verification.get('unsupported_claims') or 'none'}",
     )
 
     confidence = response.get("confidence") or {}