Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/codeql.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,14 @@ on:
branches: [ "main", "master" ]
pull_request:
branches: [ "main", "master" ]
permissions:
contents: read

jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
actions: read
contents: read
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/pages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ concurrency:
jobs:
build:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
Expand All @@ -29,6 +30,7 @@ jobs:
name: github-pages
url: ${{ steps.deployment.outputs.page_url }}
runs-on: ubuntu-latest
timeout-minutes: 10
needs: build
steps:
- name: Deploy to GitHub Pages
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/pr-lint.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,15 @@ on:
- edited
- synchronize

permissions:
contents: read
pull-requests: read

jobs:
main:
name: Validate PR title
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: amannn/action-semantic-pull-request@v5
env:
Expand Down
6 changes: 6 additions & 0 deletions .github/workflows/stale.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,15 @@ on:
schedule:
- cron: '0 0 * * *'

permissions:
contents: read
issues: write
pull-requests: write

jobs:
stale:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/stale@v8
with:
Expand Down
6 changes: 3 additions & 3 deletions src/cas_evals/reference_product.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@ def post(envelope: dict[str, Any]) -> dict[str, Any]:
with urlopen(request, timeout=timeout_seconds) as response:
payload = response.read(MAX_RESPONSE_BYTES + 1)
except HTTPError as error:
raise ReferenceProductError(f"reference product returned HTTP {error.code}") from None
except (URLError, TimeoutError, OSError):
raise ReferenceProductError("reference product is unavailable") from None
raise ReferenceProductError(f"reference product returned HTTP {error.code}") from error
except (URLError, TimeoutError, OSError) as error:
raise ReferenceProductError("reference product is unavailable") from error
if len(payload) > MAX_RESPONSE_BYTES:
raise ReferenceProductError("reference product response exceeds the size limit")
try:
Expand Down
20 changes: 19 additions & 1 deletion tests/test_reference_product.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,14 @@
import unittest
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from unittest.mock import patch
from urllib.error import URLError

from cas_evals.reference_product import ReferenceProductError, evaluate_reference_suite
from cas_evals.reference_product import (
ReferenceProductError,
_http_transport,
evaluate_reference_suite,
)

ROOT = Path(__file__).parents[1]
GOLDEN = ROOT / "benchmarks/reference-product/v0.1/golden.json"
Expand Down Expand Up @@ -105,6 +111,18 @@ def test_invalid_endpoint_and_timeout_fail_closed(self):
with self.assertRaises(ReferenceProductError):
evaluate_reference_suite(GOLDEN, timeout_seconds=0)

def test_network_failure_preserves_root_cause(self):
failure = URLError("network unavailable")
transport = _http_transport("http://127.0.0.1:8080/api/v1/workflows", 1.0)

with patch("cas_evals.reference_product.urlopen", side_effect=failure):
with self.assertRaisesRegex(
ReferenceProductError, "reference product is unavailable"
) as caught:
transport({"kind": "PromptEnvelope"})

self.assertIs(caught.exception.__cause__, failure)

def test_http_endpoint_is_executable(self):
server = ThreadingHTTPServer(("127.0.0.1", 0), ReferenceHandler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
Expand Down
Loading