diff --git a/.github/workflows/api-conformance-docs.yml b/.github/workflows/api-conformance-docs.yml new file mode 100644 index 0000000000..160946d8f0 --- /dev/null +++ b/.github/workflows/api-conformance-docs.yml @@ -0,0 +1,71 @@ +# Docs conformance: diffs the hand-written Fern spec (fern/openapi.yml) +# against the upstream OpenAI spec. Runs on PRs that touch the spec and +# weekly to surface upstream drift. +# +# PR checks use a pinned upstream ref for reproducibility. +# Weekly checks track master to surface upstream drift. + +name: "API Conformance: Docs" + +on: + pull_request: + paths: + - "fern/openapi.yml" + - "scripts/openai_coverage.py" + + schedule: + - cron: "23 9 * * 1" + + workflow_dispatch: + +permissions: + contents: read + +env: + OASDIFF_VERSION: "1.23.0" + OPENAI_SPEC_PIN: "f9400172ebe08522ab228b771d885e3bd5456e42" + OPENAI_SPEC_URL: "https://raw.githubusercontent.com/openai/openai-openapi" + +jobs: + docs: + name: fern/openapi.yml vs OpenAI spec + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + + - uses: ./.github/actions/setup-uv + - run: uv sync --locked --group dev + + - name: Install oasdiff + run: | + go install github.com/oasdiff/oasdiff@v${{ env.OASDIFF_VERSION }} + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + + - name: Fetch OpenAI spec + run: | + if [ "${{ github.event_name }}" = "schedule" ]; then + REF="master" + else + REF="${{ env.OPENAI_SPEC_PIN }}" + fi + curl -fsSL "${{ env.OPENAI_SPEC_URL }}/${REF}/openapi.yaml" \ + -o /tmp/openai-spec.yml + + - name: Check for breaking changes + if: github.event_name == 'pull_request' + run: | + uv run --locked python scripts/openai_coverage.py \ + --openai-spec /tmp/openai-spec.yml --check-breaking + + - name: Check conformance + run: | + uv run --locked python scripts/openai_coverage.py \ + --openai-spec /tmp/openai-spec.yml \ + --summary "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/api-conformance-impl.yml b/.github/workflows/api-conformance-impl.yml new file mode 100644 index 0000000000..818c1200e5 --- /dev/null +++ b/.github/workflows/api-conformance-impl.yml @@ -0,0 +1,66 @@ +# Implementation conformance: diffs the actual FastAPI contract (app.openapi()) +# against the upstream OpenAI spec. Catches schema drift introduced by +# Pydantic model changes in nemoguardrails/server/. +# +# PR checks use a pinned upstream ref for reproducibility. +# Weekly checks track master to surface upstream drift. + +name: "API Conformance: Implementation" + +on: + pull_request: + paths: + - "nemoguardrails/server/**" + - "scripts/openai_coverage.py" + + schedule: + - cron: "37 9 * * 1" + + workflow_dispatch: + +permissions: + contents: read + +env: + OASDIFF_VERSION: "1.23.0" + OPENAI_SPEC_PIN: "f9400172ebe08522ab228b771d885e3bd5456e42" + OPENAI_SPEC_URL: "https://raw.githubusercontent.com/openai/openai-openapi" + +jobs: + impl: + name: app.openapi() vs OpenAI spec + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + + - uses: ./.github/actions/setup-uv + - run: uv sync --locked --group dev + + - name: Install oasdiff + run: | + go install github.com/oasdiff/oasdiff@v${{ env.OASDIFF_VERSION }} + echo "$(go env GOPATH)/bin" >> $GITHUB_PATH + + - name: Fetch OpenAI spec + run: | + if [ "${{ github.event_name }}" = "schedule" ]; then + REF="master" + else + REF="${{ env.OPENAI_SPEC_PIN }}" + fi + curl -fsSL "${{ env.OPENAI_SPEC_URL }}/${REF}/openapi.yaml" \ + -o /tmp/openai-spec.yml + echo "ref=${REF}" >> "$GITHUB_OUTPUT" + + - name: Check conformance + run: | + uv run --locked python scripts/openai_coverage.py \ + --openai-spec /tmp/openai-spec.yml --fastapi \ + --summary "$GITHUB_STEP_SUMMARY" diff --git a/scripts/openai_coverage.py b/scripts/openai_coverage.py new file mode 100644 index 0000000000..d45dd23201 --- /dev/null +++ b/scripts/openai_coverage.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""OpenAI API conformance analyzer for NeMo Guardrails. + +Compares the NeMo Guardrails API spec against the upstream OpenAI spec using +oasdiff. Supports two spec sources for the guardrails side: + + --guardrails-spec fern/openapi.yml (default, the hand-written docs spec) + --fastapi (exports app.openapi() at runtime) + +The OpenAI spec can be a local file or fetched from upstream: + + --openai-spec /path/to/spec.yml (local file) + --fetch (download from openai/openai-openapi) +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any +from urllib.request import urlopen + +OPENAI_SPEC_URL = "https://raw.githubusercontent.com/openai/openai-openapi/master/openapi.yaml" + +MATCH_PATH = "/(chat/completions|models)" + + +def _run_oasdiff(subcommand: str, base: str, revision: str, *, extra_flags: list[str] | None = None) -> Any: + cmd = [ + "oasdiff", + subcommand, + base, + revision, + "--format", + "json", + "--auto-upgrade", + "--flatten-allof", + ] + if extra_flags: + cmd.extend(extra_flags) + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0 and not result.stdout: + raise RuntimeError(f"oasdiff {subcommand} failed: {result.stderr}") + return json.loads(result.stdout) if result.stdout else {} + + +def _check_breaking(spec: str) -> bool: + if not Path(spec).exists(): + return True + result = subprocess.run( + [ + "oasdiff", + "breaking", + f"HEAD:{spec}", + spec, + "--fail-on", + "ERR", + "--auto-upgrade", + "--flatten-allof", + "--match-path", + "^/v1/", + ], + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(result.stdout or result.stderr) + return False + return True + + +def _fetch_openai_spec(url: str = OPENAI_SPEC_URL) -> Path: + f = tempfile.NamedTemporaryFile(suffix=".yml", delete=False) + try: + with urlopen(url) as resp: # noqa: S310 + f.write(resp.read()) + finally: + f.close() + return Path(f.name) + + +def _export_fastapi_spec() -> Path: + import yaml + + from nemoguardrails.server.api import app + + spec = app.openapi() + f = tempfile.NamedTemporaryFile(suffix=".yml", mode="w", delete=False) + try: + yaml.dump(spec, f, default_flow_style=False, sort_keys=False) + finally: + f.close() + return Path(f.name) + + +def analyze( + openai_spec: Path, + guardrails_spec: Path, + match_path: str | None = None, + strip_prefix: str = "/v1", +) -> dict[str, Any]: + import yaml + + flags = ["--strip-prefix-revision", strip_prefix] + if match_path: + flags.extend(["--match-path", match_path]) + changelog = _run_oasdiff("changelog", str(openai_spec), str(guardrails_spec), extra_flags=flags) + + changes = [ + {k: v for k, v in entry.items() if k not in ("baseSource", "revisionSource", "fingerprint")} + for entry in (changelog if isinstance(changelog, list) else []) + if entry.get("section") == "paths" + ] + changes.sort(key=lambda e: (e.get("path", ""), e.get("operation", ""), e.get("text", ""))) + + spec_text = openai_spec.read_text() + spec_data = yaml.safe_load(spec_text) if openai_spec.suffix in (".yml", ".yaml") else json.loads(spec_text) + + return { + "openai_version": spec_data.get("info", {}).get("version", "unknown"), + "changes": changes, + } + + +def _print_report(report: dict[str, Any], *, verbose: bool = False) -> None: + ver = report["openai_version"] + changes = report["changes"] + print(f"OpenAI v{ver}: {len(changes)} conformance gap(s)") + + if not verbose: + return + + current_endpoint = "" + for c in changes: + endpoint = f"{c.get('operation', '?')} {c.get('path', '?')}" + if endpoint != current_endpoint: + current_endpoint = endpoint + print(f" {endpoint}:") + print(f" {c['text']}") + + +def _markdown_report(report: dict[str, Any], source: str) -> str: + ver = report["openai_version"] + changes = report["changes"] + lines = [ + f"### `{source}` vs OpenAI v{ver}", + "", + f"**{len(changes)}** conformance gap(s)", + "", + ] + if changes: + lines.append("| Endpoint | Change |") + lines.append("|---|---|") + for c in changes: + endpoint = f"`{c.get('operation', '?')} {c.get('path', '?')}`" + lines.append(f"| {endpoint} | {c.get('text', '')} |") + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description="OpenAI API conformance analyzer for NeMo Guardrails") + parser.add_argument( + "--openai-spec", + type=Path, + default=None, + help="Path to OpenAI spec file (default: fetched from upstream if --fetch)", + ) + parser.add_argument( + "--guardrails-spec", + type=Path, + default=Path("fern/openapi.yml"), + help="Path to guardrails spec (ignored when --fastapi is set)", + ) + parser.add_argument( + "--fastapi", + action="store_true", + help="Use app.openapi() as the guardrails spec instead of a file", + ) + parser.add_argument( + "--fetch", + action="store_true", + help="Fetch the OpenAI spec from upstream instead of using a local file", + ) + parser.add_argument("--match-path", type=str, default=MATCH_PATH) + parser.add_argument("--quiet", action="store_true", help="Only print the summary line") + parser.add_argument( + "--summary", + type=Path, + default=None, + help="Write a markdown summary to this file (append mode, for $GITHUB_STEP_SUMMARY)", + ) + parser.add_argument( + "--check-breaking", + action="store_true", + help="Fail on breaking API changes vs HEAD (fern spec only)", + ) + args = parser.parse_args() + + openai_spec = args.openai_spec + guardrails_spec = args.guardrails_spec + tmp_files: list[Path] = [] + + try: + if args.fetch: + if openai_spec is not None: + parser.error("--fetch and --openai-spec are mutually exclusive") + openai_spec = _fetch_openai_spec() + tmp_files.append(openai_spec) + elif openai_spec is None: + parser.error("either --openai-spec or --fetch is required") + + if args.fastapi: + guardrails_spec = _export_fastapi_spec() + tmp_files.append(guardrails_spec) + + if args.check_breaking and not args.fastapi: + print("Breaking changes (HEAD vs working tree):", end=" ") + if _check_breaking(str(args.guardrails_spec)): + print("none") + else: + sys.exit(1) + + report = analyze(openai_spec, guardrails_spec, match_path=args.match_path) + source = "app.openapi()" if args.fastapi else str(args.guardrails_spec) + print(f"[{source}]", end=" ") + _print_report(report, verbose=not args.quiet) + + if args.summary is not None: + with open(args.summary, "a") as f: + f.write(_markdown_report(report, source) + "\n") + except (FileNotFoundError, RuntimeError) as e: + print(f"Error: {e}") + sys.exit(1) + finally: + for f in tmp_files: + f.unlink(missing_ok=True) + + +if __name__ == "__main__": + main()