feat(ci): add OpenAI API spec conformance tracking

.github/workflows/api-conformance-docs.yml

@@ -0,0 +1,71 @@
+# Docs conformance: diffs the hand-written Fern spec (fern/openapi.yml)
+# against the upstream OpenAI spec.  Runs on PRs that touch the spec and
+# weekly to surface upstream drift.
+#
+# PR checks use a pinned upstream ref for reproducibility.
+# Weekly checks track master to surface upstream drift.
+
+name: "API Conformance: Docs"
+
+on:
+  pull_request:
+    paths:
+      - "fern/openapi.yml"
+      - "scripts/openai_coverage.py"
+
+  schedule:
+    - cron: "23 9 * * 1"
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  OASDIFF_VERSION: "1.23.0"
+  OPENAI_SPEC_PIN: "f9400172ebe08522ab228b771d885e3bd5456e42"
+  OPENAI_SPEC_URL: "https://raw.githubusercontent.com/openai/openai-openapi"
+
+jobs:
+  docs:
+    name: fern/openapi.yml vs OpenAI spec
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - uses: ./.github/actions/setup-uv
+      - run: uv sync --locked --group dev
+
+      - name: Install oasdiff
+        run: |
+          go install github.com/oasdiff/oasdiff@v${{ env.OASDIFF_VERSION }}
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+
+      - name: Fetch OpenAI spec
+        run: |
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            REF="master"
+          else
+            REF="${{ env.OPENAI_SPEC_PIN }}"
+          fi
+          curl -fsSL "${{ env.OPENAI_SPEC_URL }}/${REF}/openapi.yaml" \
+            -o /tmp/openai-spec.yml
+
+      - name: Check for breaking changes
+        if: github.event_name == 'pull_request'
+        run: |
+          uv run --locked python scripts/openai_coverage.py \
+            --openai-spec /tmp/openai-spec.yml --check-breaking
+
+      - name: Check conformance
+        run: |
+          uv run --locked python scripts/openai_coverage.py \
+            --openai-spec /tmp/openai-spec.yml \
+            --summary "$GITHUB_STEP_SUMMARY"

.github/workflows/api-conformance-impl.yml

@@ -0,0 +1,66 @@
+# Implementation conformance: diffs the actual FastAPI contract (app.openapi())
+# against the upstream OpenAI spec.  Catches schema drift introduced by
+# Pydantic model changes in nemoguardrails/server/.
+#
+# PR checks use a pinned upstream ref for reproducibility.
+# Weekly checks track master to surface upstream drift.
+
+name: "API Conformance: Implementation"
+
+on:
+  pull_request:
+    paths:
+      - "nemoguardrails/server/**"
+      - "scripts/openai_coverage.py"
+
+  schedule:
+    - cron: "37 9 * * 1"
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+env:
+  OASDIFF_VERSION: "1.23.0"
+  OPENAI_SPEC_PIN: "f9400172ebe08522ab228b771d885e3bd5456e42"
+  OPENAI_SPEC_URL: "https://raw.githubusercontent.com/openai/openai-openapi"
+
+jobs:
+  impl:
+    name: app.openapi() vs OpenAI spec
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+
+      - name: Set up Python
+        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0
+        with:
+          python-version: "3.12"
+
+      - uses: ./.github/actions/setup-uv
+      - run: uv sync --locked --group dev
+
+      - name: Install oasdiff
+        run: |
+          go install github.com/oasdiff/oasdiff@v${{ env.OASDIFF_VERSION }}
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+
+      - name: Fetch OpenAI spec
+        run: |
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            REF="master"
+          else
+            REF="${{ env.OPENAI_SPEC_PIN }}"
+          fi
+          curl -fsSL "${{ env.OPENAI_SPEC_URL }}/${REF}/openapi.yaml" \
+            -o /tmp/openai-spec.yml
+          echo "ref=${REF}" >> "$GITHUB_OUTPUT"
+
+      - name: Check conformance
+        run: |
+          uv run --locked python scripts/openai_coverage.py \
+            --openai-spec /tmp/openai-spec.yml --fastapi \
+            --summary "$GITHUB_STEP_SUMMARY"

scripts/openai_coverage.py

@@ -0,0 +1,259 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""OpenAI API conformance analyzer for NeMo Guardrails.
+
+Compares the NeMo Guardrails API spec against the upstream OpenAI spec using
+oasdiff. Supports two spec sources for the guardrails side:
+
+  --guardrails-spec fern/openapi.yml   (default, the hand-written docs spec)
+  --fastapi                            (exports app.openapi() at runtime)
+
+The OpenAI spec can be a local file or fetched from upstream:
+
+  --openai-spec /path/to/spec.yml      (local file)
+  --fetch                              (download from openai/openai-openapi)
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+from typing import Any
+from urllib.request import urlopen
+
+OPENAI_SPEC_URL = "https://raw.githubusercontent.com/openai/openai-openapi/master/openapi.yaml"
+
+MATCH_PATH = "/(chat/completions|models)"
+
+
+def _run_oasdiff(subcommand: str, base: str, revision: str, *, extra_flags: list[str] | None = None) -> Any:
+    cmd = [
+        "oasdiff",
+        subcommand,
+        base,
+        revision,
+        "--format",
+        "json",
+        "--auto-upgrade",
+        "--flatten-allof",
+    ]
+    if extra_flags:
+        cmd.extend(extra_flags)
+    result = subprocess.run(cmd, capture_output=True, text=True)
+    if result.returncode != 0 and not result.stdout:
+        raise RuntimeError(f"oasdiff {subcommand} failed: {result.stderr}")
+    return json.loads(result.stdout) if result.stdout else {}
+
+
+def _check_breaking(spec: str) -> bool:
+    if not Path(spec).exists():
+        return True
+    result = subprocess.run(
+        [
+            "oasdiff",
+            "breaking",
+            f"HEAD:{spec}",
+            spec,
+            "--fail-on",
+            "ERR",
+            "--auto-upgrade",
+            "--flatten-allof",
+            "--match-path",
+            "^/v1/",
+        ],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        print(result.stdout or result.stderr)
+        return False
+    return True
+
+
+def _fetch_openai_spec(url: str = OPENAI_SPEC_URL) -> Path:
+    f = tempfile.NamedTemporaryFile(suffix=".yml", delete=False)
+    try:
+        with urlopen(url) as resp:  # noqa: S310
+            f.write(resp.read())
+    finally:
+        f.close()
+    return Path(f.name)
+
+
+def _export_fastapi_spec() -> Path:
+    import yaml
+
+    from nemoguardrails.server.api import app
+
+    spec = app.openapi()
+    f = tempfile.NamedTemporaryFile(suffix=".yml", mode="w", delete=False)
+    try:
+        yaml.dump(spec, f, default_flow_style=False, sort_keys=False)
+    finally:
+        f.close()
+    return Path(f.name)
+
+
+def analyze(
+    openai_spec: Path,
+    guardrails_spec: Path,
+    match_path: str | None = None,
+    strip_prefix: str = "/v1",
+) -> dict[str, Any]:
+    import yaml
+
+    flags = ["--strip-prefix-revision", strip_prefix]
+    if match_path:
+        flags.extend(["--match-path", match_path])
+    changelog = _run_oasdiff("changelog", str(openai_spec), str(guardrails_spec), extra_flags=flags)
+
+    changes = [
+        {k: v for k, v in entry.items() if k not in ("baseSource", "revisionSource", "fingerprint")}
+        for entry in (changelog if isinstance(changelog, list) else [])
+        if entry.get("section") == "paths"
+    ]
+    changes.sort(key=lambda e: (e.get("path", ""), e.get("operation", ""), e.get("text", "")))
+
+    spec_text = openai_spec.read_text()
+    spec_data = yaml.safe_load(spec_text) if openai_spec.suffix in (".yml", ".yaml") else json.loads(spec_text)
+
+    return {
+        "openai_version": spec_data.get("info", {}).get("version", "unknown"),
+        "changes": changes,
+    }
+
+
+def _print_report(report: dict[str, Any], *, verbose: bool = False) -> None:
+    ver = report["openai_version"]
+    changes = report["changes"]
+    print(f"OpenAI v{ver}: {len(changes)} conformance gap(s)")
+
+    if not verbose:
+        return
+
+    current_endpoint = ""
+    for c in changes:
+        endpoint = f"{c.get('operation', '?')} {c.get('path', '?')}"
+        if endpoint != current_endpoint:
+            current_endpoint = endpoint
+            print(f"  {endpoint}:")
+        print(f"    {c['text']}")
+
+
+def _markdown_report(report: dict[str, Any], source: str) -> str:
+    ver = report["openai_version"]
+    changes = report["changes"]
+    lines = [
+        f"### `{source}` vs OpenAI v{ver}",
+        "",
+        f"**{len(changes)}** conformance gap(s)",
+        "",
+    ]
+    if changes:
+        lines.append("| Endpoint | Change |")
+        lines.append("|---|---|")
+        for c in changes:
+            endpoint = f"`{c.get('operation', '?')} {c.get('path', '?')}`"
+            lines.append(f"| {endpoint} | {c.get('text', '')} |")
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="OpenAI API conformance analyzer for NeMo Guardrails")
+    parser.add_argument(
+        "--openai-spec",
+        type=Path,
+        default=None,
+        help="Path to OpenAI spec file (default: fetched from upstream if --fetch)",
+    )
+    parser.add_argument(
+        "--guardrails-spec",
+        type=Path,
+        default=Path("fern/openapi.yml"),
+        help="Path to guardrails spec (ignored when --fastapi is set)",
+    )
+    parser.add_argument(
+        "--fastapi",
+        action="store_true",
+        help="Use app.openapi() as the guardrails spec instead of a file",
+    )
+    parser.add_argument(
+        "--fetch",
+        action="store_true",
+        help="Fetch the OpenAI spec from upstream instead of using a local file",
+    )
+    parser.add_argument("--match-path", type=str, default=MATCH_PATH)
+    parser.add_argument("--quiet", action="store_true", help="Only print the summary line")
+    parser.add_argument(
+        "--summary",
+        type=Path,
+        default=None,
+        help="Write a markdown summary to this file (append mode, for $GITHUB_STEP_SUMMARY)",
+    )
+    parser.add_argument(
+        "--check-breaking",
+        action="store_true",
+        help="Fail on breaking API changes vs HEAD (fern spec only)",
+    )
+    args = parser.parse_args()
+
+    openai_spec = args.openai_spec
+    guardrails_spec = args.guardrails_spec
+    tmp_files: list[Path] = []
+
+    try:
+        if args.fetch:
+            if openai_spec is not None:
+                parser.error("--fetch and --openai-spec are mutually exclusive")
+            openai_spec = _fetch_openai_spec()
+            tmp_files.append(openai_spec)
+        elif openai_spec is None:
+            parser.error("either --openai-spec or --fetch is required")
+
+        if args.fastapi:
+            guardrails_spec = _export_fastapi_spec()
+            tmp_files.append(guardrails_spec)
+
+        if args.check_breaking and not args.fastapi:
+            print("Breaking changes (HEAD vs working tree):", end=" ")
+            if _check_breaking(str(args.guardrails_spec)):
+                print("none")
+            else:
+                sys.exit(1)
+
+        report = analyze(openai_spec, guardrails_spec, match_path=args.match_path)
+        source = "app.openapi()" if args.fastapi else str(args.guardrails_spec)
+        print(f"[{source}]", end=" ")
+        _print_report(report, verbose=not args.quiet)
+
+        if args.summary is not None:
+            with open(args.summary, "a") as f:
+                f.write(_markdown_report(report, source) + "\n")
+    except (FileNotFoundError, RuntimeError) as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+    finally:
+        for f in tmp_files:
+            f.unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+    main()