Skip to content

Commit 0a600eb

Browse files
m-misiuram-misiura
committed
ci: add OpenAI API spec conformance tracking
1 parent 0811139 commit 0a600eb

7 files changed

Lines changed: 82193 additions & 0 deletions

File tree

‎.github/workflows/api-conformance.yml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
# API conformance checks:
2+
# PR: breaking changes + coverage regression via the api-conformance
3+
# pre-commit hook (.pre-commit-config.yaml).
4+
# Weekly: this workflow compares our spec (fern/openapi.yml) against the
5+
# vendored and latest upstream OpenAI specs, and reports per-endpoint
6+
# conformance gaps in the GitHub Actions step summary.
7+
8+
name: API Conformance
9+
10+
on:
11+
schedule:
12+
- cron: "23 9 * * 1"
13+
14+
workflow_dispatch:
15+
16+
permissions:
17+
contents: read
18+
19+
env:
20+
OASDIFF_VERSION: "1.23.0"
21+
22+
jobs:
23+
drift:
24+
name: Upstream Drift
25+
runs-on: ubuntu-latest
26+
steps:
27+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
28+
with:
29+
persist-credentials: false
30+
31+
- name: Set up Python
32+
uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
33+
with:
34+
python-version: "3.12"
35+
36+
- uses: ./.github/actions/setup-uv
37+
- run: uv sync --locked --group dev
38+
39+
- name: Install oasdiff
40+
run: |
41+
go install github.com/oasdiff/oasdiff@v${{ env.OASDIFF_VERSION }}
42+
echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
43+
44+
- name: Check for upstream spec drift
45+
env:
46+
SPEC_URL: "https://raw.githubusercontent.com/openai/openai-openapi/master/openapi.yaml"
47+
run: |
48+
curl -fsSL "$SPEC_URL" -o /tmp/openai-latest.yml
49+
LATEST=$(grep -m1 '^\s*version:' /tmp/openai-latest.yml | sed 's/.*version:\s*//' | tr -d ' ')
50+
PINNED=$(grep -m1 '^\s*version:' schemas/openai-spec.yml | sed 's/.*version:\s*//' | tr -d ' ')
51+
52+
VENDORED_REPORT=$(uv run --locked python scripts/openai_coverage.py 2>&1) || true
53+
54+
{
55+
echo "## OpenAI API Conformance — Weekly Drift Check"
56+
echo ""
57+
echo "Compares the vendored OpenAI spec (\`schemas/openai-spec.yml\`) against"
58+
echo "the latest upstream spec from \`github.com/openai/openai-openapi\`, and"
59+
echo "reports how our guardrails spec (\`fern/openapi.yml\`) differs from each."
60+
echo ""
61+
echo "| | Version |"
62+
echo "|---|---|"
63+
echo "| Vendored (\`schemas/openai-spec.yml\`) | ${PINNED} |"
64+
echo "| Upstream (OpenAI latest) | ${LATEST} |"
65+
echo ""
66+
echo "### How does our API differ from vendored OpenAI v${PINNED}?"
67+
echo '```'
68+
echo "$VENDORED_REPORT"
69+
echo '```'
70+
} >> "$GITHUB_STEP_SUMMARY"
71+
72+
echo "$VENDORED_REPORT"
73+
74+
if [ "$LATEST" = "$PINNED" ]; then
75+
echo "Vendored spec is up to date (${PINNED})"
76+
else
77+
echo "::notice::Spec drift detected: vendored=${PINNED}, upstream=${LATEST}"
78+
UPSTREAM_REPORT=$(uv run --locked python scripts/openai_coverage.py --openai-spec /tmp/openai-latest.yml --output /dev/null 2>&1) || true
79+
{
80+
echo ""
81+
echo "### How does our API differ from upstream OpenAI v${LATEST}?"
82+
echo '```'
83+
echo "$UPSTREAM_REPORT"
84+
echo '```'
85+
} >> "$GITHUB_STEP_SUMMARY"
86+
echo "$UPSTREAM_REPORT"
87+
fi

‎.github/workflows/lint.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ jobs:
2222
- uses: actions/checkout@v7.0.0
2323
with:
2424
persist-credentials: false
25+
fetch-depth: 2 # api-conformance hook needs HEAD~1
2526

2627
- name: Set up Python
2728
uses: actions/setup-python@v6

‎.pre-commit-config.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ repos:
55
- id: check-yaml
66
- id: end-of-file-fixer
77
- id: trailing-whitespace
8+
exclude: ^schemas/openai-spec\.yml$
89

910
- repo: https://github.com/astral-sh/ruff-pre-commit
1011
# Ruff version.
@@ -39,3 +40,16 @@ repos:
3940
pass_filenames: false
4041
always_run: true
4142
require_serial: true
43+
- id: api-conformance
44+
name: OpenAI spec conformance
45+
entry: uv run --locked python scripts/openai_coverage.py --check-breaking --check-regression --update --quiet
46+
language: golang
47+
additional_dependencies:
48+
- github.com/oasdiff/oasdiff@v1.23.0
49+
pass_filenames: false
50+
require_serial: true
51+
verbose: true
52+
files: ^(fern/openapi\.yml|schemas/openai-(spec\.yml|conformance-baseline\.json)|scripts/openai_coverage\.py)$
53+
54+
ci:
55+
skip: [api-conformance]

‎schemas/README.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# schemas/
2+
3+
**openai-spec.yml** — Vendored OpenAI API spec (v2.3.0, OpenAPI 3.1.0).
4+
Source: https://github.com/openai/openai-openapi
5+
6+
To update:
7+
8+
```bash
9+
curl -fsSL https://raw.githubusercontent.com/openai/openai-openapi/master/openapi.yaml -o schemas/openai-spec.yml
10+
python scripts/openai_coverage.py --update
11+
```
12+
13+
**openai-conformance-baseline.json** — oasdiff changelog of path-level differences
14+
between the vendored OpenAI spec and our guardrails spec (`fern/openapi.yml`).
15+
Used for regression detection: if the number of differences grows, pre-commit fails.
16+
Auto-generated by `python scripts/openai_coverage.py --update`. Do not edit manually.

0 commit comments

Comments
 (0)