diff --git a/src/ucode/agents/__init__.py b/src/ucode/agents/__init__.py index 27b0458..065220d 100644 --- a/src/ucode/agents/__init__.py +++ b/src/ucode/agents/__init__.py @@ -363,6 +363,7 @@ def check_gateway_endpoint(state: dict, tool: str) -> bool: bool(state.get("claude_models")) or bool(state.get("codex_models")) or bool(state.get("gemini_models")) + or bool(state.get("oss_models")) ) return False @@ -373,7 +374,7 @@ def check_gateway_endpoint(state: dict, tool: str) -> bool: "codex": ("codex",), "gemini": ("gemini",), "copilot": ("claude", "codex"), - "pi": ("claude", "codex", "gemini"), + "pi": ("claude", "codex", "gemini", "oss"), } diff --git a/src/ucode/agents/claude.py b/src/ucode/agents/claude.py index 4ac91e3..2528bfa 100644 --- a/src/ucode/agents/claude.py +++ b/src/ucode/agents/claude.py @@ -27,6 +27,7 @@ from ucode.databricks import ( build_auth_shell_command, build_tool_base_url, + claude_model_supports_1m, get_databricks_token, ) from ucode.launcher import exec_or_spawn @@ -69,11 +70,6 @@ def _resolve_web_search_model(state: dict) -> str | None: WEB_SEARCH_MCP_NAME = "web_search" -# Matches both the AI Gateway form (`databricks-claude-opus-4-8`) and the UC -# model-services form (`system.ai.claude-opus-4-8`). -_CLAUDE_MODEL_RE = re.compile( - r"^(?:system\.ai\.)?(?:databricks-)?claude-(opus|sonnet)-(\d+)-(\d+)(.*)$" -) # Env keys the MLflow Stop hook reads to route traces. Written into the # settings `env` block alongside the hook itself. @@ -333,19 +329,9 @@ def render_overlay( def _maybe_add_1m_suffix(model: str) -> str: - if model.endswith("[1m]"): - return model - match = _CLAUDE_MODEL_RE.match(model) - if not match: + if model.endswith("[1m]") or not claude_model_supports_1m(model): return model - - family, major_raw, minor_raw, _ = match.groups() - major = int(major_raw) - minor = int(minor_raw) - should_suffix = (family == "opus" and (major, minor) >= (4, 6)) or ( - family == "sonnet" and (major, minor) >= (4, 6) - ) - return f"{model}[1m]" if should_suffix else model + return f"{model}[1m]" def _register_web_search_mcp(workspace: str, search_model: str, profile: str | None = None) -> bool: diff --git a/src/ucode/agents/pi.py b/src/ucode/agents/pi.py index e7c1760..806c26f 100644 --- a/src/ucode/agents/pi.py +++ b/src/ucode/agents/pi.py @@ -1,12 +1,13 @@ """Pi coding agent: writes ~/.pi/agent/models.json with Databricks-backed providers. -Pi (https://pi.dev) is a multi-provider coding agent. We register three +Pi (https://pi.dev) is a multi-provider coding agent. We register four providers in its `models.json`, each speaking the API dialect best suited to that family's gateway path: - `databricks-claude` (api: anthropic-messages) → /ai-gateway/anthropic - `databricks-openai` (api: openai-responses) → /ai-gateway/codex/v1 - `databricks-gemini` (api: google-generative-ai) → /ai-gateway/gemini/v1beta +- `databricks-mlflow` (api: openai-completions) → /ai-gateway/mlflow/v1 Per-provider `compat` flags work around fields the gateway translators reject: @@ -15,11 +16,17 @@ pi uses for every request. With this flag pi omits the per-tool field and sends the legacy `anthropic-beta: fine-grained-tool-streaming-...` header instead, which the gateway accepts. - -OSS / Databricks-foundation models (Llama, Qwen, etc.) are not exposed via -pi today — they live behind /ai-gateway/mlflow/v1 with per-model -`max_tokens` caps that pi has no global way to honor without per-model -config we don't currently maintain. +- mlflow: `supportsStore: false` and `supportsStrictMode: false` — the MLflow + chat-completions gateway rejects OpenAI's `store` field and + `tools[].function.strict`. + +The `databricks-mlflow` provider carries the validated OSS coding models +(GLM and Kimi) discovered upstream. Per model it sets +`contextWindow`/`maxTokens` from `databricks.model_token_limits` and +`reasoning` from `databricks.model_is_reasoning` (so Pi renders the gateway's +streamed reasoning_content as thinking). Inkling is intentionally not offered +until the gateway emits a terminal `finish_reason` on natural completion +(issue #215). The bearer token is baked into the file and refreshed by a background thread while the session runs (same pattern as OpenCode/Copilot). @@ -44,7 +51,12 @@ from ucode.databricks import ( TOKEN_REFRESH_INTERVAL_SECONDS, build_pi_base_urls, + claude_model_capabilities, get_databricks_token, + gpt_model_token_limits, + model_is_reasoning, + model_token_limits, + preferred_gpt_model, ) from ucode.state import mark_tool_managed, save_state from ucode.telemetry import agent_version, ucode_version @@ -68,6 +80,7 @@ "databricks-claude", "databricks-openai", "databricks-gemini", + "databricks-mlflow", ) PROVIDER_KEYS: list[list[str]] = [["providers", name] for name in PROVIDER_NAMES] @@ -86,6 +99,7 @@ def _resolve_model_selector( claude_models: dict[str, str], codex_models: list[str], gemini_models: list[str], + oss_models: list[str], ) -> str: """Return a Pi model selector in `/` form when possible.""" for name in PROVIDER_NAMES: @@ -97,9 +111,65 @@ def _resolve_model_selector( return f"databricks-openai/{model}" if model in gemini_models: return f"databricks-gemini/{model}" + if model in oss_models: + return f"databricks-mlflow/{model}" return model +def _pi_claude_model_entry(model_id: str) -> dict: + """Build a Claude entry with explicit limits. + + Databricks model ids do not match Pi's built-in Anthropic ids, so a bare + custom entry silently gets Pi's 128k context / 4k output defaults. + """ + capabilities = claude_model_capabilities(model_id) + entry: dict = { + "id": model_id, + "reasoning": True, + "input": ["text", "image"], + "contextWindow": capabilities.context, + "maxTokens": capabilities.output, + } + if capabilities.force_adaptive_thinking: + entry["compat"] = {"forceAdaptiveThinking": True} + return entry + + +def _pi_oss_model_entry(model_id: str) -> dict: + """Build a Pi mlflow model entry enriched from the shared limits/reasoning + tables: `reasoning:true` for reasoning models (Pi renders their streamed + reasoning_content as thinking), and `contextWindow`/`maxTokens` from + `model_token_limits`. Fields are omitted when unknown so Pi keeps its + default.""" + entry: dict = {"id": model_id} + if model_is_reasoning(model_id): + entry["reasoning"] = True + limits = model_token_limits(model_id) + if limits: + if limits.get("context"): + entry["contextWindow"] = limits["context"] + if limits.get("output"): + entry["maxTokens"] = limits["output"] + return entry + + +def _pi_gpt_model_entry(model_id: str) -> dict: + """Build a Pi openai (codex) model entry with `contextWindow`/`maxTokens` + from `databricks.gpt_model_token_limits`. GPT ids aren't in Pi's built-in + catalog, so without an explicit window Pi falls back to a small default and + truncates long sessions.""" + limits = gpt_model_token_limits(model_id) + entry: dict = { + "id": model_id, + "contextWindow": limits["context"], + "maxTokens": limits["output"], + } + if "gpt-5" in model_id.lower().replace(".", "-"): + entry["reasoning"] = True + entry["input"] = ["text", "image"] + return entry + + def render_overlay( model: str, token: str, @@ -107,6 +177,7 @@ def render_overlay( claude_models: dict[str, str], codex_models: list[str], gemini_models: list[str], + oss_models: list[str], ) -> tuple[dict, list[list[str]]]: """Return (overlay, managed_key_paths) for ~/.pi/agent/models.json.""" providers: dict = {} @@ -127,7 +198,7 @@ def render_overlay( # the legacy beta header instead when this is false. "compat": {"supportsEagerToolInputStreaming": False}, "headers": ua_headers, - "models": [{"id": m} for m in claude_ids], + "models": [_pi_claude_model_entry(m) for m in claude_ids], } keys.append(["providers", "databricks-claude"]) if codex_models: @@ -137,7 +208,7 @@ def render_overlay( "apiKey": token, "authHeader": True, "headers": ua_headers, - "models": [{"id": m} for m in codex_models], + "models": [_pi_gpt_model_entry(m) for m in codex_models], } keys.append(["providers", "databricks-openai"]) if gemini_models: @@ -150,8 +221,23 @@ def render_overlay( "models": [{"id": m} for m in gemini_models], } keys.append(["providers", "databricks-gemini"]) + if oss_models: + providers["databricks-mlflow"] = { + "baseUrl": pi_base_urls["oss"], + "api": "openai-completions", + "apiKey": token, + "authHeader": True, + # MLflow chat-completions gateway rejects OpenAI's `store` field + # and per-tool `strict`. Pi omits both when these are false. + "compat": {"supportsStore": False, "supportsStrictMode": False}, + "headers": ua_headers, + "models": [_pi_oss_model_entry(m) for m in oss_models], + } + keys.append(["providers", "databricks-mlflow"]) overlay: dict = { - "model": _resolve_model_selector(model, claude_models, codex_models, gemini_models), + "model": _resolve_model_selector( + model, claude_models, codex_models, gemini_models, oss_models + ), } if providers: overlay["providers"] = providers @@ -178,6 +264,7 @@ def write_tool_config( state.get("claude_models") or {}, state.get("codex_models") or [], state.get("gemini_models") or [], + state.get("oss_models") or [], ) existing = read_json_safe(PI_CONFIG_PATH) providers = existing.get("providers") @@ -206,16 +293,19 @@ def _write_settings(model_selector: str) -> None: def default_model(state: dict) -> str | None: - """Prefer Claude opus → sonnet → haiku; fall back to codex, gemini.""" + """Prefer Claude opus → sonnet → haiku; fall back to codex, Gemini, then OSS.""" claude_models = state.get("claude_models") or {} for family in ("opus", "sonnet", "haiku"): if claude_models.get(family): return claude_models[family] - codex_models = state.get("codex_models") or [] - if codex_models: - return codex_models[0] + codex_model = preferred_gpt_model(state.get("codex_models") or []) + if codex_model: + return codex_model gemini_models = state.get("gemini_models") or [] - return gemini_models[0] if gemini_models else None + if gemini_models: + return gemini_models[0] + oss_models = state.get("oss_models") or [] + return oss_models[0] if oss_models else None def _refresh_token_once(state: dict, *, force_refresh: bool = False) -> str: diff --git a/src/ucode/cli.py b/src/ucode/cli.py index 1b42b4b..6037d4c 100644 --- a/src/ucode/cli.py +++ b/src/ucode/cli.py @@ -97,7 +97,7 @@ "claude": ("claude", "opencode", "copilot", "pi"), "codex": ("codex", "copilot", "pi"), "gemini": ("gemini", "opencode", "pi"), - "oss": ("opencode",), + "oss": ("opencode", "pi"), } @@ -371,7 +371,7 @@ def configure_shared_state( ) want_gemini = fetch_all or "gemini" in tools or "opencode" in tools or "pi" in tools want_codex = fetch_all or "codex" in tools or "copilot" in tools or "pi" in tools - want_oss = fetch_all or "opencode" in tools + want_oss = fetch_all or "opencode" in tools or "pi" in tools claude_reason: str | None = None gemini_reason: str | None = None diff --git a/src/ucode/databricks.py b/src/ucode/databricks.py index 1d32f31..4417ff9 100644 --- a/src/ucode/databricks.py +++ b/src/ucode/databricks.py @@ -23,6 +23,7 @@ from concurrent.futures import ( TimeoutError as FutureTimeoutError, ) +from dataclasses import dataclass from pathlib import Path from typing import Literal, cast, overload from urllib import error as urllib_error @@ -1194,10 +1195,24 @@ def build_auth_shell_command( # Databricks-managed foundation models under `system.ai`. _MODEL_SERVICE_REQUIRED_PREFIX = "system.ai." -# Supported OSS chat families, matched by name substring. Add an entry to -# support a new family. +# OSS families validated as coding models in ucode, matched by name substring. +# Keep this as an explicit product allowlist rather than exposing every model on +# the chat-completions route. Inkling remains excluded until gateway issue #215 +# is fixed; other families require coding-harness validation before inclusion. _OSS_MODEL_FAMILIES = ("kimi-", "glm-") +# Non-chat services must never be offered to a chat agent if a future supported +# family also uses one of these substrings. +_OSS_NON_CHAT_SUBSTRINGS = ("embedding", "embed", "rerank") + + +def _is_oss_chat_model(model_id: str) -> bool: + """True if the id matches an OSS chat family and isn't a non-chat service.""" + if any(bad in model_id for bad in _OSS_NON_CHAT_SUBSTRINGS): + return False + return any(family in model_id for family in _OSS_MODEL_FAMILIES) + + # Claude model families ucode buckets, newest tier first. Each maps to a # Claude Code family alias (ANTHROPIC_DEFAULT__MODEL). Add an entry to # support a new family in both discovery paths (`claude--*` via the @@ -1211,24 +1226,179 @@ def build_auth_shell_command( # config dialect. Both fields are provided because agents like OpenCode require # context and output together. Keyed by family substring; add an entry to bound # a new model. +# +# Output caps probed from the gateway 2026-07-16 (it 400s with "max_tokens (N) +# cannot exceed "); context windows from each model's docs/description +# (conservative when unstated). If the gateway raises a cap or ships a new +# model, update this table. _MODEL_TOKEN_LIMITS: dict[str, dict[str, int]] = { - # GLM-4.6: 200k context, but the gateway caps output well below the model's - # native 128k — pin 25k so requests aren't rejected. + # Keep the version-specific entry before the family fallback: GLM 5.2 has + # materially higher probed gateway limits than earlier/unknown variants. + "glm-5-2": {"context": 1_000_000, "output": 65_536}, "glm": {"context": 200_000, "output": 25_000}, + "kimi": {"context": 128_000, "output": 65_536}, } +# Conservative fallback for a future variant that matches a validated family +# but has no specific entry. Pinning a low output ceiling risks truncation, not +# a gateway 400, so it is the safe failure direction. +_OSS_FALLBACK_LIMITS = {"context": 128_000, "output": 8_192} + +# Validated families that emit reasoning. Pi renders their streamed +# reasoning_content as thinking when the model entry sets reasoning:true. +_OSS_REASONING_FAMILIES = ("glm", "kimi") + + +def model_is_reasoning(model_id: str) -> bool: + """True if the OSS model reports reasoning output (family-matched).""" + return any(family in model_id for family in _OSS_REASONING_FAMILIES) + def model_token_limits(model_id: str) -> dict[str, int] | None: """Return ``{"context": ..., "output": ...}`` limits for ``model_id``, or None. - Matches by family substring (e.g. any ``*glm*`` id). None means the model - has no known limits and the agent should not pin any.""" + Prefers a specific `_MODEL_TOKEN_LIMITS` family entry (e.g. any ``*glm*`` + id). Any other OSS chat model falls back to a conservative floor so it is + never offered uncapped (which would 400). None only for non-OSS ids, where + the agent should not pin any limit.""" for family, limits in _MODEL_TOKEN_LIMITS.items(): if family in model_id: return dict(limits) + if _is_oss_chat_model(model_id): + return dict(_OSS_FALLBACK_LIMITS) return None +# Pi treats every custom model without explicit metadata as 128k context / 4k +# output. Gateway ids are custom ids (not Pi's built-ins), so preserve the +# upstream windows explicitly. Entries are ordered most-specific first after +# normalizing dotted OpenAI ids and hyphenated Databricks ids to one form. +# GPT-5.6 Sol/Terra/Luna support the opt-in 1.05M window; 272k is merely Pi's +# built-in short-context pricing default, not the model's hard context limit. +_GPT_TOKEN_LIMITS: tuple[tuple[str, dict[str, int]], ...] = ( + ("gpt-5-6-sol", {"context": 1_050_000, "output": 128_000}), + ("gpt-5-6-terra", {"context": 1_050_000, "output": 128_000}), + ("gpt-5-6-luna", {"context": 1_050_000, "output": 128_000}), + ("gpt-5-5-pro", {"context": 1_050_000, "output": 128_000}), + ("gpt-5-4-pro", {"context": 1_050_000, "output": 128_000}), + ("gpt-5-5", {"context": 272_000, "output": 128_000}), + ("gpt-5-4-mini", {"context": 400_000, "output": 128_000}), + ("gpt-5-4-nano", {"context": 400_000, "output": 128_000}), + ("gpt-5-4", {"context": 272_000, "output": 128_000}), + ("gpt-5", {"context": 400_000, "output": 128_000}), + ("gpt-4-1", {"context": 1_047_576, "output": 32_768}), + ("gpt-4o", {"context": 128_000, "output": 16_384}), + ("gpt-4-turbo", {"context": 128_000, "output": 4_096}), + ("gpt-4", {"context": 8_192, "output": 8_192}), +) +_GPT_FALLBACK_LIMITS = {"context": 128_000, "output": 16_384} + + +def _normalized_foundation_model_id(model_id: str) -> str: + """Strip route prefixes case-insensitively and normalize dotted versions.""" + tail = model_id.split("/")[-1].lower() + if tail.startswith("system.ai."): + tail = tail[len("system.ai.") :] + if tail.startswith("databricks-"): + tail = tail[len("databricks-") :] + return tail.replace(".", "-") + + +def gpt_model_token_limits(model_id: str) -> dict[str, int]: + """Return Pi metadata limits for a GPT (codex/openai) gateway model.""" + tail = _normalized_foundation_model_id(model_id) + for family, limits in _GPT_TOKEN_LIMITS: + if tail == family or tail.startswith(f"{family}-"): + return dict(limits) + return dict(_GPT_FALLBACK_LIMITS) + + +def preferred_gpt_model(model_ids: list[str]) -> str | None: + """Prefer the newest numeric GPT id, then any other Responses endpoint. + + ``gpt-oss`` is chat-completions-only and must never be selected for the + Responses route, even if stale state supplies it here. + """ + eligible = [ + model_id + for model_id in model_ids + if not _normalized_foundation_model_id(model_id).startswith("gpt-oss") + ] + numeric_gpt = [ + model_id + for model_id in eligible + if re.match(r"^gpt-\d(?:-|$)", _normalized_foundation_model_id(model_id)) + ] + if numeric_gpt: + return min( + numeric_gpt, + key=lambda model_id: model_version_sort_key(_normalized_foundation_model_id(model_id)), + ) + return eligible[0] if eligible else None + + +@dataclass(frozen=True) +class ClaudeModelCapabilities: + context: int + output: int + supports_1m: bool = False + force_adaptive_thinking: bool = False + + +_CLAUDE_FALLBACK_CAPABILITIES = ClaudeModelCapabilities(context=200_000, output=64_000) +_CLAUDE_MODEL_RE = re.compile(r"^claude-(fable|opus|sonnet|haiku)-(\d+)(?:-(\d+))?") + + +def claude_model_capabilities(model_id: str) -> ClaudeModelCapabilities: + """Return the shared Claude capability policy for every agent. + + Opus gained the opt-in 1M window in 4.6; Sonnet gained it in 4.5. + Sonnet's verified 1M tiers retain a conservative 64k output cap. Fable 5 + is 1M by default (so it needs no ``[1m]`` suffix) with a 128k output cap. + Opus 4.5, Haiku, and unrecognized ids use the conservative 200k fallback. + """ + tail = _normalized_foundation_model_id(model_id) + match = _CLAUDE_MODEL_RE.match(tail) + if not match: + return _CLAUDE_FALLBACK_CAPABILITIES + family, major_raw, minor_raw = match.groups() + version = (int(major_raw), int(minor_raw or 0)) + if family == "opus" and version >= (4, 6): + return ClaudeModelCapabilities( + context=1_000_000, + output=128_000, + supports_1m=True, + force_adaptive_thinking=True, + ) + if family == "sonnet" and version >= (4, 6): + return ClaudeModelCapabilities( + context=1_000_000, + output=64_000, + supports_1m=True, + force_adaptive_thinking=True, + ) + if family == "sonnet" and version >= (4, 5): + return ClaudeModelCapabilities(context=1_000_000, output=64_000, supports_1m=True) + if family == "fable" and version >= (5, 0): + return ClaudeModelCapabilities( + context=1_000_000, + output=128_000, + force_adaptive_thinking=True, + ) + return _CLAUDE_FALLBACK_CAPABILITIES + + +def claude_model_supports_1m(model_id: str) -> bool: + """Whether Claude Code should request the model's opt-in ``[1m]`` tier.""" + return claude_model_capabilities(model_id).supports_1m + + +def claude_model_token_limits(model_id: str) -> dict[str, int]: + """Return Pi metadata limits from the shared Claude capability policy.""" + capabilities = claude_model_capabilities(model_id) + return {"context": capabilities.context, "output": capabilities.output} + + def _model_service_id(service: dict) -> str | None: """Extract the `system.ai.` id from one model-service entry. @@ -1354,10 +1524,13 @@ def discover_model_services( if candidates: claude_models[family] = candidates[0] - codex_models = [m for m in ids if "gpt-" in m] + # `gpt-oss-*` also contains "gpt-" but is a chat-completions-only OSS model + # (served via /ai-gateway/mlflow/v1), NOT an openai-responses codex model — + # exclude it here so it isn't offered under the codex provider (which 400s). + codex_models = [m for m in ids if "gpt-" in m and "gpt-oss" not in m] gemini_models = sorted([m for m in ids if "gemini-" in m], key=model_version_sort_key) - oss_models = [m for m in ids if any(family in m for family in _OSS_MODEL_FAMILIES)] + oss_models = [m for m in ids if _is_oss_chat_model(m)] if not (claude_models or codex_models or gemini_models or oss_models): sample = ", ".join(ids[:5]) @@ -2391,6 +2564,7 @@ def build_pi_base_urls(workspace: str) -> dict[str, str]: "claude": build_tool_base_url("claude", workspace), "openai": build_tool_base_url("codex", workspace), "gemini": build_tool_base_url("gemini", workspace) + "/v1beta", + "oss": f"{workspace}/ai-gateway/mlflow/v1", } diff --git a/tests/test_agent_claude.py b/tests/test_agent_claude.py index 60b4a6a..ed18ae0 100644 --- a/tests/test_agent_claude.py +++ b/tests/test_agent_claude.py @@ -48,6 +48,30 @@ def test_adds_1m_suffix_for_sonnet_4_6_and_later(self): overlay["env"]["ANTHROPIC_DEFAULT_SONNET_MODEL"] == "databricks-claude-sonnet-4-7[1m]" ) + def test_adds_1m_suffix_for_sonnet_4_5(self): + # Sonnet 4.5 supports the 1M context window (its 1M beta shipped before + # Opus's), so it must get the [1m] suffix even though it predates the + # Opus 4.6 floor. + overlay, _ = claude.render_overlay( + WS, "s4", claude_models={"sonnet": "databricks-claude-sonnet-4-5"} + ) + assert ( + overlay["env"]["ANTHROPIC_DEFAULT_SONNET_MODEL"] == "databricks-claude-sonnet-4-5[1m]" + ) + + def test_does_not_add_1m_suffix_for_sonnet_4_4(self): + overlay, _ = claude.render_overlay( + WS, "s4", claude_models={"sonnet": "databricks-claude-sonnet-4-4"} + ) + assert overlay["env"]["ANTHROPIC_DEFAULT_SONNET_MODEL"] == "databricks-claude-sonnet-4-4" + + def test_does_not_add_1m_suffix_for_opus_4_5(self): + # Opus's 1M window starts at 4.6, so 4.5 stays on the default context. + overlay, _ = claude.render_overlay( + WS, "s4", claude_models={"opus": "databricks-claude-opus-4-5"} + ) + assert overlay["env"]["ANTHROPIC_DEFAULT_OPUS_MODEL"] == "databricks-claude-opus-4-5" + def test_does_not_add_1m_suffix_for_haiku(self): overlay, _ = claude.render_overlay( WS, "s4", claude_models={"haiku": "databricks-claude-haiku-4-6"} @@ -72,6 +96,19 @@ def test_no_1m_suffix_for_model_services_haiku(self): ) assert overlay["env"]["ANTHROPIC_DEFAULT_HAIKU_MODEL"] == "system.ai.claude-haiku-4-6" + @pytest.mark.parametrize( + ("model_id", "expected"), + [ + ("system.ai.claude-opus-5", "system.ai.claude-opus-5[1m]"), + ("databricks-claude-sonnet-5", "databricks-claude-sonnet-5[1m]"), + ("system.ai.claude-opus-4-5", "system.ai.claude-opus-4-5"), + ("system.ai.claude-fable-5", "system.ai.claude-fable-5"), + ("not-a-claude-model", "not-a-claude-model"), + ], + ) + def test_suffix_uses_shared_capability_policy(self, model_id, expected): + assert claude._maybe_add_1m_suffix(model_id) == expected + def test_sets_anthropic_base_url(self): overlay, _ = claude.render_overlay(WS, "s4") assert overlay["env"]["ANTHROPIC_BASE_URL"] == f"{WS}/ai-gateway/anthropic" diff --git a/tests/test_agent_opencode.py b/tests/test_agent_opencode.py index 71ca2bc..f63d078 100644 --- a/tests/test_agent_opencode.py +++ b/tests/test_agent_opencode.py @@ -98,15 +98,24 @@ def test_glm_gets_token_limits(self): overlay, _ = opencode.render_overlay("system.ai.glm-5-2", "tok", _base_urls(), models) glm = overlay["provider"]["databricks-oss"]["models"]["system.ai.glm-5-2"] # OpenCode's schema requires both context and output on `limit`. - assert glm["limit"] == {"context": 200000, "output": 25000} + # Probed 2026-07-16: glm-5-2 is 1M context / 65536 output. + assert glm["limit"] == {"context": 1_000_000, "output": 65_536} - def test_non_glm_oss_model_has_no_output_cap(self): + def test_kimi_gets_token_limits(self): + # kimi is now a capped OSS family (128k context / 65536 output). models = {"oss": ["system.ai.kimi-k2-7-code"]} overlay, _ = opencode.render_overlay( "system.ai.kimi-k2-7-code", "tok", _base_urls(), models ) kimi = overlay["provider"]["databricks-oss"]["models"]["system.ai.kimi-k2-7-code"] - assert "limit" not in kimi + assert kimi["limit"] == {"context": 128_000, "output": 65_536} + + def test_uncapped_oss_model_has_no_limit(self): + # A model outside the limits table gets no `limit` (client default). + models = {"oss": ["system.ai.mystery-7b"]} + overlay, _ = opencode.render_overlay("system.ai.mystery-7b", "tok", _base_urls(), models) + entry = overlay["provider"]["databricks-oss"]["models"]["system.ai.mystery-7b"] + assert "limit" not in entry def test_token_in_api_key(self): models = {"anthropic": ["claude-sonnet"]} diff --git a/tests/test_agent_pi.py b/tests/test_agent_pi.py index 0afc5fb..b0711ee 100644 --- a/tests/test_agent_pi.py +++ b/tests/test_agent_pi.py @@ -17,6 +17,7 @@ def _base_urls() -> dict[str, str]: "claude": f"{WS}/ai-gateway/anthropic", "openai": f"{WS}/ai-gateway/codex/v1", "gemini": f"{WS}/ai-gateway/gemini/v1beta", + "oss": f"{WS}/ai-gateway/mlflow/v1", } @@ -26,6 +27,7 @@ def _empty() -> dict: "claude_models": {}, "codex_models": [], "gemini_models": [], + "oss_models": [], } @@ -39,6 +41,7 @@ def _overlay(model: str, token: str = "tok", **kwargs): bundle["claude_models"], bundle["codex_models"], bundle["gemini_models"], + bundle["oss_models"], ) @@ -75,23 +78,117 @@ def test_openai_provider_uses_openai_responses(self): assert provider["api"] == "openai-responses" assert provider["baseUrl"] == f"{WS}/ai-gateway/codex/v1" + def test_gpt56_sol_model_entry_pins_1m_context(self): + # Gateway ids are custom to Pi, so explicit metadata is required to + # avoid its 128k custom-model default. + overlay, _ = _overlay("gpt-5-6-sol", codex_models=["gpt-5-6-sol"]) + entry = overlay["providers"]["databricks-openai"]["models"][0] + assert entry["id"] == "gpt-5-6-sol" + assert entry["contextWindow"] == 1_050_000 + assert entry["maxTokens"] == 128_000 + assert entry["reasoning"] is True + assert entry["input"] == ["text", "image"] + + def test_gpt_model_entries_use_model_specific_windows(self): + overlay, _ = _overlay( + "system.ai.gpt-5-2", + codex_models=[ + "system.ai.gpt-5-2", + "databricks-gpt-5-4-nano", + "databricks-gpt-5-6-sol", + ], + ) + windows = { + m["id"]: m["contextWindow"] for m in overlay["providers"]["databricks-openai"]["models"] + } + assert windows == { + "system.ai.gpt-5-2": 400_000, + "databricks-gpt-5-4-nano": 400_000, + "databricks-gpt-5-6-sol": 1_050_000, + } + + def test_claude_entries_pin_limits_and_capabilities(self): + overlay, _ = _overlay( + "databricks-claude-opus-4-8", + claude_models={ + "opus": "databricks-claude-opus-4-8", + "sonnet": "system.ai.claude-sonnet-4-5", + "haiku": "databricks-claude-haiku-4-5", + "fable": "system.ai.claude-fable-5", + }, + ) + entries = {m["id"]: m for m in overlay["providers"]["databricks-claude"]["models"]} + opus = entries["databricks-claude-opus-4-8"] + assert opus["contextWindow"] == 1_000_000 + assert opus["maxTokens"] == 128_000 + assert opus["reasoning"] is True + assert opus["input"] == ["text", "image"] + assert opus["compat"] == {"forceAdaptiveThinking": True} + assert entries["system.ai.claude-sonnet-4-5"]["contextWindow"] == 1_000_000 + assert entries["system.ai.claude-sonnet-4-5"]["maxTokens"] == 64_000 + assert entries["databricks-claude-haiku-4-5"]["contextWindow"] == 200_000 + fable = entries["system.ai.claude-fable-5"] + assert fable["contextWindow"] == 1_000_000 + assert fable["maxTokens"] == 128_000 + assert fable["compat"] == {"forceAdaptiveThinking": True} + def test_gemini_provider_uses_google_generative_ai(self): overlay, _ = _overlay("gemini-2", gemini_models=["gemini-2"]) provider = overlay["providers"]["databricks-gemini"] assert provider["api"] == "google-generative-ai" assert provider["baseUrl"] == f"{WS}/ai-gateway/gemini/v1beta" - def test_all_three_providers_when_all_present(self): + def test_mlflow_provider_uses_openai_completions(self): + overlay, _ = _overlay("system.ai.glm-5-2", oss_models=["system.ai.glm-5-2"]) + provider = overlay["providers"]["databricks-mlflow"] + assert provider["api"] == "openai-completions" + assert provider["baseUrl"] == f"{WS}/ai-gateway/mlflow/v1" + assert provider["compat"] == {"supportsStore": False, "supportsStrictMode": False} + + def test_no_mlflow_provider_when_no_oss_models(self): + overlay, _ = _overlay("gpt-5", codex_models=["gpt-5"]) + assert "databricks-mlflow" not in overlay.get("providers", {}) + + def test_all_four_providers_when_all_present(self): overlay, _ = _overlay( "claude-sonnet", claude_models={"sonnet": "claude-sonnet"}, codex_models=["gpt-5"], gemini_models=["gemini-2"], + oss_models=["system.ai.glm-5-2"], ) assert set(overlay["providers"].keys()) == { "databricks-claude", "databricks-openai", "databricks-gemini", + "databricks-mlflow", + } + + +class TestRenderOverlayOssEnrichment: + """OSS mlflow model entries carry reasoning + contextWindow + maxTokens + from the shared databricks.model_token_limits / model_is_reasoning tables.""" + + def test_reasoning_model_enriched(self): + overlay, _ = _overlay("system.ai.glm-5-2", oss_models=["system.ai.glm-5-2"]) + entry = overlay["providers"]["databricks-mlflow"]["models"][0] + assert entry["id"] == "system.ai.glm-5-2" + assert entry["reasoning"] is True + assert entry["contextWindow"] == 1_000_000 + assert entry["maxTokens"] == 65_536 + + def test_unvalidated_model_has_no_inferred_metadata(self): + # Discovery does not offer this model; even if supplied directly, Pi + # must not infer capabilities for an unvalidated coding model. + overlay, _ = _overlay("system.ai.inkling", oss_models=["system.ai.inkling"]) + entry = overlay["providers"]["databricks-mlflow"]["models"][0] + assert entry == {"id": "system.ai.inkling"} + + def test_unknown_oss_model_bare(self): + # No limits/reasoning table entry -> only id, client keeps defaults. + overlay, _ = _overlay("system.ai.mystery-7b", oss_models=["system.ai.mystery-7b"]) + assert overlay["providers"]["databricks-mlflow"]["models"][0] == { + "id": "system.ai.mystery-7b" } @@ -193,6 +290,10 @@ def test_prefixes_gemini_model(self): overlay, _ = _overlay("gemini-2", gemini_models=["gemini-2"]) assert overlay["model"] == "databricks-gemini/gemini-2" + def test_prefixes_oss_model(self): + overlay, _ = _overlay("system.ai.glm-5-2", oss_models=["system.ai.glm-5-2"]) + assert overlay["model"] == "databricks-mlflow/system.ai.glm-5-2" + def test_preserves_already_prefixed_model(self): overlay, _ = _overlay( "databricks-claude/claude-sonnet", @@ -220,14 +321,38 @@ def test_falls_back_to_haiku(self): state = {"claude_models": {"haiku": "h4"}} assert pi.default_model(state) == "h4" - def test_falls_back_to_codex(self): - state = {"claude_models": {}, "codex_models": ["gpt-5"]} - assert pi.default_model(state) == "gpt-5" + def test_falls_back_to_newest_codex_model(self): + state = { + "claude_models": {}, + "codex_models": ["databricks-gpt-5", "system.ai.gpt-5-6-sol", "gpt-5-5"], + } + assert pi.default_model(state) == "system.ai.gpt-5-6-sol" + + def test_falls_back_to_generic_responses_endpoint(self): + state = {"claude_models": {}, "codex_models": ["c1"]} + assert pi.default_model(state) == "c1" + + def test_does_not_route_gpt_oss_to_responses(self): + state = { + "claude_models": {}, + "codex_models": ["gpt-oss-120b"], + "gemini_models": ["gemini-2"], + } + assert pi.default_model(state) == "gemini-2" def test_falls_back_to_gemini(self): state = {"claude_models": {}, "codex_models": [], "gemini_models": ["gemini-2"]} assert pi.default_model(state) == "gemini-2" + def test_falls_back_to_oss_last(self): + state = { + "claude_models": {}, + "codex_models": [], + "gemini_models": [], + "oss_models": ["system.ai.glm-5-2"], + } + assert pi.default_model(state) == "system.ai.glm-5-2" + def test_returns_none_when_empty(self): assert pi.default_model({}) is None assert ( @@ -296,6 +421,7 @@ def test_stale_managed_providers_removed_before_merge(self, tmp_path, monkeypatc "databricks-claude": {"old": True}, "databricks-openai": {"old": True}, "databricks-gemini": {"old": True}, + "databricks-mlflow": {"old": True}, "user-provider": {"keep": True}, } } @@ -311,6 +437,7 @@ def test_stale_managed_providers_removed_before_merge(self, tmp_path, monkeypatc providers = written.get("providers", {}) assert providers.get("databricks-claude") != {"old": True} assert "old" not in providers.get("databricks-claude", {}) + assert "databricks-mlflow" not in providers assert providers.get("user-provider") == {"keep": True} def test_legacy_providers_removed_on_upgrade(self, tmp_path, monkeypatch): diff --git a/tests/test_agents_init.py b/tests/test_agents_init.py index 97acf30..91c3202 100644 --- a/tests/test_agents_init.py +++ b/tests/test_agents_init.py @@ -189,6 +189,14 @@ def test_pi_available_with_codex(self): def test_pi_available_with_gemini(self): assert check_gateway_endpoint({"gemini_models": ["gemini-2"]}, "pi") is True + def test_pi_available_with_oss(self): + assert check_gateway_endpoint({"oss_models": ["system.ai.glm-5-2"]}, "pi") is True + + def test_pi_oss_discovery_reason_is_reported(self): + state = {"_discovery_reasons": {"oss": "no validated OSS models"}} + detail = agents_mod._availability_failure_detail("pi", state) + assert detail == " (oss discovery: no validated OSS models)" + def test_pi_unavailable_when_no_models(self): assert check_gateway_endpoint({}, "pi") is False @@ -243,6 +251,15 @@ def test_pi_falls_back_to_gemini(self): state = {"claude_models": {}, "codex_models": [], "gemini_models": ["gemini-2"]} assert default_model_for_tool("pi", state) == "gemini-2" + def test_pi_falls_back_to_oss(self): + state = { + "claude_models": {}, + "codex_models": [], + "gemini_models": [], + "oss_models": ["system.ai.glm-5-2"], + } + assert default_model_for_tool("pi", state) == "system.ai.glm-5-2" + def test_pi_returns_none_when_no_models(self): assert default_model_for_tool("pi", {}) is None diff --git a/tests/test_cli.py b/tests/test_cli.py index 83ed6df..2c600d0 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -27,6 +27,17 @@ def _strip_ansi(text: str) -> str: TOOLS = ["codex", "claude", "gemini", "opencode"] +def test_oss_discovery_diagnostic_names_all_consumers(monkeypatch): + import ucode.cli as cli_mod + + notes = [] + monkeypatch.setattr(cli_mod, "print_note", notes.append) + + cli_mod._print_discovery_diagnostics({"_discovery_reasons": {"oss": "not found"}}) + + assert notes[0] == "OSS models (needed for: opencode, pi): not found" + + @pytest.fixture(autouse=True) def no_state_writes(): """Prevent any test from writing to the real state file on disk.""" diff --git a/tests/test_databricks.py b/tests/test_databricks.py index 7e1a73a..45d03ad 100644 --- a/tests/test_databricks.py +++ b/tests/test_databricks.py @@ -173,19 +173,154 @@ def _model_service(model_id: str) -> dict: class TestModelTokenLimits: def test_glm_is_capped(self): + # Probed 2026-07-16: glm-5-2 accepts 1M context / 65536 output. assert db_mod.model_token_limits("system.ai.glm-5-2") == { - "context": 200_000, - "output": 25_000, + "context": 1_000_000, + "output": 65_536, } - def test_glm_matches_any_version(self): - assert db_mod.model_token_limits("system.ai.glm-4-6-flash") == { + @pytest.mark.parametrize( + "model_id", + ["system.ai.glm-4-6-flash", "system.ai.glm-future"], + ) + def test_other_glm_versions_keep_conservative_limits(self, model_id): + assert db_mod.model_token_limits(model_id) == { "context": 200_000, "output": 25_000, } - def test_uncapped_model_returns_none(self): - assert db_mod.model_token_limits("system.ai.kimi-k2-7-code") is None + def test_kimi_is_capped(self): + assert db_mod.model_token_limits("system.ai.kimi-k2-7-code") == { + "context": 128_000, + "output": 65_536, + } + + def test_unvalidated_families_return_none(self): + for model_id in ( + "system.ai.inkling", + "system.ai.gpt-oss-120b", + "system.ai.llama-4-maverick", + "system.ai.qwen35-122b-a10b", + "system.ai.gemma-3-12b", + "system.ai.deepseek-v3", + ): + assert db_mod.model_token_limits(model_id) is None + + def test_embedding_model_returns_none_not_fallback(self): + assert db_mod.model_token_limits("system.ai.qwen3-embedding-0-6b") is None + + +class TestGptModelTokenLimits: + def test_gpt56_sol_serves_1m_context_across_id_forms(self): + for model_id in ( + "gpt-5.6-sol", + "system.ai.gpt-5-6-sol", + "databricks-gpt-5-6-sol", + "databricks-openai/gpt-5.6-sol", + ): + assert db_mod.gpt_model_token_limits(model_id) == { + "context": 1_050_000, + "output": 128_000, + } + + def test_gpt5_windows_are_model_specific(self): + assert db_mod.gpt_model_token_limits("system.ai.gpt-5-2")["context"] == 400_000 + assert db_mod.gpt_model_token_limits("databricks-gpt-5-4")["context"] == 272_000 + assert db_mod.gpt_model_token_limits("databricks-gpt-5-4-nano")["context"] == 400_000 + assert db_mod.gpt_model_token_limits("gpt-5.5-pro")["context"] == 1_050_000 + + def test_gpt41_window(self): + assert db_mod.gpt_model_token_limits("databricks-gpt-4-1") == { + "context": 1_047_576, + "output": 32_768, + } + + @pytest.mark.parametrize( + "model_id", + ["gpt-5.40", "gpt-4.10", "gpt-6-turbo"], + ) + def test_unknown_specific_gpt_uses_family_or_conservative_fallback(self, model_id): + expected = ( + {"context": 400_000, "output": 128_000} + if model_id == "gpt-5.40" + else {"context": 8_192, "output": 8_192} + if model_id == "gpt-4.10" + else {"context": 128_000, "output": 16_384} + ) + assert db_mod.gpt_model_token_limits(model_id) == expected + + def test_route_prefixes_are_stripped_case_insensitively(self): + assert db_mod.gpt_model_token_limits("SYSTEM.AI.GPT-5-6-SOL") == { + "context": 1_050_000, + "output": 128_000, + } + assert db_mod.gpt_model_token_limits("DATABRICKS-GPT-4-1") == { + "context": 1_047_576, + "output": 32_768, + } + + def test_preferred_gpt_model_uses_semantic_version_across_prefixes(self): + assert ( + db_mod.preferred_gpt_model( + ["databricks-gpt-5", "system.ai.gpt-5-6-sol", "databricks-gpt-5-5"] + ) + == "system.ai.gpt-5-6-sol" + ) + assert db_mod.preferred_gpt_model(["not-gpt", "claude-opus-4-8"]) == "not-gpt" + + def test_preferred_gpt_model_falls_back_to_generic_responses_endpoint(self): + assert db_mod.preferred_gpt_model(["c1", "custom-responses"]) == "c1" + + def test_preferred_gpt_model_excludes_gpt_oss(self): + assert db_mod.preferred_gpt_model(["gpt-oss-120b", "c1"]) == "c1" + assert db_mod.preferred_gpt_model(["system.ai.gpt-oss-120b"]) is None + + +class TestClaudeModelCapabilities: + @pytest.mark.parametrize( + ("model_id", "context", "output", "supports_1m", "adaptive"), + [ + ("databricks-claude-opus-4-5", 200_000, 64_000, False, False), + ("databricks-claude-opus-4-6", 1_000_000, 128_000, True, True), + ("system.ai.claude-opus-5", 1_000_000, 128_000, True, True), + ("claude-sonnet-4-4", 200_000, 64_000, False, False), + ("system.ai.claude-sonnet-4-5", 1_000_000, 64_000, True, False), + ("claude-sonnet-4-6[1m]", 1_000_000, 64_000, True, True), + ("claude-sonnet-5", 1_000_000, 64_000, True, True), + ("claude-haiku-4-5", 200_000, 64_000, False, False), + ("system.ai.claude-fable-5", 1_000_000, 128_000, False, True), + ("claude-future", 200_000, 64_000, False, False), + ], + ) + def test_shared_capability_policy( + self, + model_id, + context, + output, + supports_1m, + adaptive, + ): + capabilities = db_mod.claude_model_capabilities(model_id) + assert capabilities.context == context + assert capabilities.output == output + assert capabilities.supports_1m is supports_1m + assert capabilities.force_adaptive_thinking is adaptive + assert db_mod.claude_model_supports_1m(model_id) is supports_1m + assert db_mod.claude_model_token_limits(model_id) == { + "context": context, + "output": output, + } + + +class TestModelIsReasoning: + def test_reasoning_families(self): + assert db_mod.model_is_reasoning("system.ai.glm-5-2") is True + assert db_mod.model_is_reasoning("system.ai.kimi-k2-7-code") is True + + def test_unvalidated_families_are_not_marked_reasoning(self): + assert db_mod.model_is_reasoning("system.ai.inkling") is False + assert db_mod.model_is_reasoning("system.ai.qwen35-122b-a10b") is False + assert db_mod.model_is_reasoning("system.ai.gpt-oss-120b") is False class TestDiscoverModelServices: @@ -220,19 +355,20 @@ def test_buckets_families_by_name(self, monkeypatch): assert codex == ["system.ai.gpt-5"] # Gemini ordered newest-first via the shared sort key. assert gemini[0] == "system.ai.gemini-3-5-flash" - # kimi and glm are the allowlisted OSS families; llama is not. + # Only coding-harness-validated OSS families are offered. assert oss == ["system.ai.glm-5-2", "system.ai.kimi-k2-7-code"] - def test_oss_allowlist_drops_unsupported_families(self, monkeypatch): - # Only kimi/glm are allowlisted; other families are dropped. + def test_oss_allowlist_drops_unvalidated_families(self, monkeypatch): payload = { "model_services": [ _model_service("system.ai.glm-5-2"), _model_service("system.ai.kimi-k2-7-code"), - _model_service("system.ai.qwen-3-coder"), + _model_service("system.ai.qwen35-122b-a10b"), + _model_service("system.ai.inkling"), + _model_service("system.ai.llama-4-maverick"), + _model_service("system.ai.gemma-3-12b"), _model_service("system.ai.deepseek-v3"), - _model_service("system.ai.gte-large-embed"), - _model_service("system.ai.bge-reranker-v2"), + _model_service("system.ai.qwen3-embedding-0-6b"), ] } monkeypatch.setattr( @@ -245,6 +381,25 @@ def test_oss_allowlist_drops_unsupported_families(self, monkeypatch): assert (claude, codex, gemini) == ({}, [], []) assert oss == ["system.ai.glm-5-2", "system.ai.kimi-k2-7-code"] + def test_gpt_oss_is_neither_selectable_oss_nor_codex(self, monkeypatch): + # Keep the independent codex exclusion: gpt-oss contains "gpt-" but + # cannot use the Responses API, even though it is not an offered model. + payload = { + "model_services": [ + _model_service("system.ai.gpt-5"), + _model_service("system.ai.gpt-oss-120b"), + ] + } + monkeypatch.setattr( + db_mod, "_http_get_json", lambda url, token, timeout=10: (payload, None) + ) + + _, codex, _, oss, _ = db_mod.discover_model_services(WS, "token") + + assert codex == ["system.ai.gpt-5"] + assert "system.ai.gpt-oss-120b" not in oss + assert "system.ai.gpt-oss-120b" not in codex + def test_paginates_via_next_page_token(self, monkeypatch): pages = { None: { @@ -281,7 +436,8 @@ def test_http_failure_returns_reason(self, monkeypatch): assert reason == "HTTP 500 Server Error" def test_no_matching_families_reports_sample(self, monkeypatch): - payload = {"model_services": [_model_service("system.ai.llama-4-maverick")]} + # deepseek is outside every claude/gpt/gemini/oss family bucket. + payload = {"model_services": [_model_service("system.ai.deepseek-v3")]} monkeypatch.setattr( db_mod, "_http_get_json", lambda url, token, timeout=10: (payload, None) ) @@ -289,7 +445,7 @@ def test_no_matching_families_reports_sample(self, monkeypatch): claude, codex, gemini, oss, reason = db_mod.discover_model_services(WS, "token") assert (claude, codex, gemini, oss) == ({}, [], [], []) - assert reason is not None and "llama-4-maverick" in reason + assert reason is not None and "deepseek-v3" in reason def test_ignores_non_system_ai_schemas(self, monkeypatch): # The metastore listing returns services from every schema; only