From 6a5ab55227ffaf418dcb693a163d39e8362b6ef0 Mon Sep 17 00:00:00 2001 From: r266-tech Date: Tue, 7 Jul 2026 08:58:16 +0800 Subject: [PATCH 1/3] fix(ollama): make native num_ctx opt-in --- hindsight-api-slim/hindsight_api/config.py | 8 + .../hindsight_api/engine/llm_wrapper.py | 25 +++ .../hindsight_api/engine/memory_engine.py | 5 + .../engine/providers/openai_compatible_llm.py | 20 +- .../tests/test_config_validation.py | 35 ++++ hindsight-api-slim/tests/test_llm_wrapper.py | 195 +++++++++++++++++- .../docs/developer/configuration.md | 3 + .../references/developer/configuration.md | 3 + 8 files changed, 292 insertions(+), 2 deletions(-) diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index 44d2e9e81e..407124d609 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -147,6 +147,7 @@ def normalize_config_dict(config: dict[str, Any]) -> dict[str, Any]: ENV_LLM_DEFAULT_HEADERS = "HINDSIGHT_API_LLM_DEFAULT_HEADERS" ENV_LLM_STRICT_SCHEMA = "HINDSIGHT_API_LLM_STRICT_SCHEMA" ENV_LLM_SEND_BANK_AS_USER = "HINDSIGHT_API_LLM_SEND_BANK_AS_USER" +ENV_LLM_OLLAMA_NUM_CTX = "HINDSIGHT_API_LLM_OLLAMA_NUM_CTX" # Per-operation sampling temperature. Each internal LLM call uses a temperature # tuned for its task (deterministic extraction vs. creative reflection). These @@ -1577,6 +1578,9 @@ class HindsightConfig: # LiteLLM, Helicone) key attribution on the OpenAI `user` field. Opt-in; never # overrides a `user` the caller already set. llm_send_bank_as_user: bool + # Optional native Ollama context window override. Unset lets Ollama use the + # model/server default instead of forcing a Hindsight-wide value. + llm_ollama_num_ctx: int | None # Per-operation sampling temperature. None means the temperature parameter is # omitted from the call (for models that reject explicit temperatures). See @@ -2319,6 +2323,10 @@ def from_env(cls) -> "HindsightConfig": llm_strict_schema=os.getenv(ENV_LLM_STRICT_SCHEMA, str(DEFAULT_LLM_STRICT_SCHEMA)).lower() in ("true", "1"), llm_send_bank_as_user=os.getenv(ENV_LLM_SEND_BANK_AS_USER, str(DEFAULT_LLM_SEND_BANK_AS_USER)).lower() in ("true", "1"), + llm_ollama_num_ctx=_parse_optional_positive_int( + ENV_LLM_OLLAMA_NUM_CTX, + os.getenv(ENV_LLM_OLLAMA_NUM_CTX), + ), llm_temperature_verification=_resolve_operation_temperature( ENV_LLM_TEMPERATURE_VERIFICATION, DEFAULT_LLM_TEMPERATURE_VERIFICATION ), diff --git a/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py b/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py index 620158f198..3394155eb9 100644 --- a/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py +++ b/hindsight-api-slim/hindsight_api/engine/llm_wrapper.py @@ -235,6 +235,17 @@ def requires_api_key(provider: str) -> bool: return provider.lower() not in _PROVIDERS_WITHOUT_API_KEY +def _validate_ollama_num_ctx(value: Any) -> int | None: + """Validate a native Ollama context-window override.""" + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"ollama_num_ctx must be a positive integer, got {value!r}") + if value < 1: + raise ValueError(f"ollama_num_ctx must be >= 1, got {value}") + return value + + def create_llm_provider( provider: str, api_key: str, @@ -254,6 +265,7 @@ def create_llm_provider( litellmrouter_config: dict[str, Any] | None = None, gemini_service_tier: str | None = None, timeout: float | None = None, + ollama_num_ctx: int | None = None, ) -> Any: # Returns LLMInterface """ Factory function to create the appropriate LLM provider implementation. @@ -268,6 +280,8 @@ def create_llm_provider( openai_service_tier: OpenAI service tier (for OpenAI provider) - None (default) or "flex" (50% cheaper). bedrock_service_tier: Bedrock service tier (for Bedrock provider) - None (default), "flex", "priority", or "reserved". gemini_service_tier: Gemini service tier (for Gemini provider) - None (default) or "flex" (50% cheaper). + ollama_num_ctx: Native Ollama context window override. None lets Ollama use the + model/server default. extra_body: Extra request-body params merged into the provider's native call. Threaded into OpenAI-compatible, Fireworks, Anthropic, Gemini/ VertexAI and LiteLLM providers (each merges them in its own parameter @@ -291,6 +305,8 @@ def create_llm_provider( Returns: LLMInterface implementation for the specified provider. """ + ollama_num_ctx = _validate_ollama_num_ctx(ollama_num_ctx) + from .providers import ( AnthropicLLM, ClaudeCodeLLM, @@ -494,6 +510,7 @@ def create_llm_provider( groq_service_tier=groq_service_tier, openai_service_tier=openai_service_tier, extra_body=extra_body, + ollama_num_ctx=ollama_num_ctx, timeout=timeout, ) @@ -531,6 +548,7 @@ def __init__( max_retries: int | None = None, initial_backoff: float | None = None, max_backoff: float | None = None, + ollama_num_ctx: int | None = None, ): """ Initialize LLM provider. @@ -545,6 +563,8 @@ def __init__( openai_service_tier: OpenAI service tier (None or "flex") - from config. bedrock_service_tier: Bedrock service tier (None, "flex", "priority", "reserved") - from config. gemini_service_tier: Gemini service tier (None or "flex") - from config. + ollama_num_ctx: Native Ollama context window override. ``None`` lets Ollama + use the model/server default. gemini_safety_settings: Safety settings for Gemini/VertexAI providers. extra_body: Extra request-body params merged into the provider's native call (OpenAI-compatible, Fireworks, Anthropic, Gemini/VertexAI, LiteLLM). @@ -598,6 +618,7 @@ def __init__( self.openai_service_tier = openai_service_tier self.bedrock_service_tier = bedrock_service_tier self.gemini_service_tier = gemini_service_tier + self.ollama_num_ctx = _validate_ollama_num_ctx(ollama_num_ctx) # Gemini safety settings (instance default; can be overridden per-request via context var) self.gemini_safety_settings = gemini_safety_settings # Gemini prompt caching: when True, retain extraction (and any future @@ -742,6 +763,7 @@ def __init__( gemini_safety_settings=self.gemini_safety_settings, prompt_cache_enabled=self.prompt_cache_enabled, litellmrouter_config=router_config, + ollama_num_ctx=self.ollama_num_ctx, timeout=self.timeout, ) @@ -1260,6 +1282,7 @@ def from_env(cls) -> "LLMProvider": ENV_LLM_GROQ_SERVICE_TIER, ENV_LLM_LITELLMROUTER_CONFIG, ENV_LLM_MODEL, + ENV_LLM_OLLAMA_NUM_CTX, ENV_LLM_OPENAI_SERVICE_TIER, ENV_LLM_PROMPT_CACHE_ENABLED, ENV_LLM_PROVIDER, @@ -1270,6 +1293,7 @@ def from_env(cls) -> "LLMProvider": ENV_LLM_VERTEXAI_SERVICE_ACCOUNT_KEY, _get_default_model_for_provider, _parse_llm_router_config, + _parse_optional_positive_int, parse_gemini_service_tier, ) @@ -1314,6 +1338,7 @@ def from_env(cls) -> "LLMProvider": ), gemini_safety_settings=json.loads(os.getenv(ENV_LLM_GEMINI_SAFETY_SETTINGS, "null")), prompt_cache_enabled=prompt_cache_enabled, + ollama_num_ctx=_parse_optional_positive_int(ENV_LLM_OLLAMA_NUM_CTX, os.getenv(ENV_LLM_OLLAMA_NUM_CTX)), litellmrouter_config=_parse_llm_router_config(ENV_LLM_LITELLMROUTER_CONFIG), vertexai_project_id=os.getenv(ENV_LLM_VERTEXAI_PROJECT_ID) or None, vertexai_region=os.getenv(ENV_LLM_VERTEXAI_REGION) or None, diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index 9ad7ab0b54..57155001e0 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -438,6 +438,7 @@ def _member_to_llm(member: "LLMMemberConfig", config: HindsightConfig, defaults: reasoning_effort=member.reasoning_effort or config.llm_reasoning_effort, extra_body=member.extra_body, default_headers=member.default_headers or config.llm_default_headers, + ollama_num_ctx=config.llm_ollama_num_ctx, bedrock_service_tier=member.bedrock_service_tier, gemini_service_tier=member.gemini_service_tier or config.llm_gemini_service_tier, gemini_safety_settings=_get_raw_config().llm_gemini_safety_settings, @@ -1085,6 +1086,7 @@ def pick(field: str) -> Any: reasoning_effort=config.llm_reasoning_effort, extra_body=config.llm_extra_body, default_headers=config.llm_default_headers, + ollama_num_ctx=config.llm_ollama_num_ctx, litellmrouter_config=config.llm_litellmrouter_config, bedrock_service_tier=config.llm_bedrock_service_tier, gemini_service_tier=config.llm_gemini_service_tier, @@ -1129,6 +1131,7 @@ def pick(field: str) -> Any: reasoning_effort=config.llm_reasoning_effort, extra_body=config.llm_extra_body, default_headers=config.llm_default_headers, + ollama_num_ctx=config.llm_ollama_num_ctx, litellmrouter_config=config.retain_llm_litellmrouter_config or config.llm_litellmrouter_config, bedrock_service_tier=config.llm_bedrock_service_tier, gemini_service_tier=config.llm_gemini_service_tier, @@ -1167,6 +1170,7 @@ def pick(field: str) -> Any: reasoning_effort=config.llm_reasoning_effort, extra_body=config.llm_extra_body, default_headers=config.llm_default_headers, + ollama_num_ctx=config.llm_ollama_num_ctx, litellmrouter_config=config.reflect_llm_litellmrouter_config or config.llm_litellmrouter_config, bedrock_service_tier=config.llm_bedrock_service_tier, gemini_service_tier=config.llm_gemini_service_tier, @@ -1205,6 +1209,7 @@ def pick(field: str) -> Any: reasoning_effort=config.llm_reasoning_effort, extra_body=config.llm_extra_body, default_headers=config.llm_default_headers, + ollama_num_ctx=config.llm_ollama_num_ctx, litellmrouter_config=config.consolidation_llm_litellmrouter_config or config.llm_litellmrouter_config, bedrock_service_tier=config.llm_bedrock_service_tier, gemini_service_tier=config.llm_gemini_service_tier, diff --git a/hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py b/hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py index aade75602b..6e51cb99b5 100644 --- a/hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py +++ b/hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py @@ -48,6 +48,18 @@ DEFAULT_LLM_SEED = 4242 JSON_MODE_USER_HINT = "Return valid json only." + +def _validate_ollama_num_ctx(value: Any) -> int | None: + """Validate a native Ollama context-window override.""" + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, int): + raise ValueError(f"ollama_num_ctx must be a positive integer, got {value!r}") + if value < 1: + raise ValueError(f"ollama_num_ctx must be >= 1, got {value}") + return value + + # Self-hosted OpenAI-compatible servers that advertise tool_choice="required" # but silently ignore it: instead of forcing a tool call they return # finish_reason "stop"/"tool_calls" with an EMPTY tool_calls array and no error. @@ -435,6 +447,8 @@ def __init__( timeout: float | None = None, groq_service_tier: str | None = None, extra_body: dict[str, Any] | None = None, + *, + ollama_num_ctx: int | None = None, **kwargs: Any, ): """ @@ -449,6 +463,8 @@ def __init__( timeout: Request timeout in seconds (uses env var or 120s default). groq_service_tier: Groq service tier ("on_demand", "flex", "auto"). extra_body: Extra body params merged into every API call. + ollama_num_ctx: Native Ollama context window override. None lets Ollama use + the model/server default. **kwargs: Additional provider-specific parameters. """ super().__init__(provider, api_key, base_url, model, reasoning_effort, **kwargs) @@ -529,6 +545,7 @@ def __init__( # Service tier configuration (from config, not env vars) self.groq_service_tier = groq_service_tier self.openai_service_tier = kwargs.get("openai_service_tier") + self.ollama_num_ctx = _validate_ollama_num_ctx(ollama_num_ctx) # User-configured extra body params (merged into every API call) self._config_extra_body = extra_body or {} @@ -1337,9 +1354,10 @@ async def _call_ollama_native( # Add optional parameters with optimized defaults for Ollama options: dict[str, Any] = { - "num_ctx": 16384, # 16k context window for larger prompts "num_batch": 512, # Optimal batch size for prompt processing } + if self.ollama_num_ctx is not None: + options["num_ctx"] = self.ollama_num_ctx if max_completion_tokens: options["num_predict"] = max_completion_tokens if temperature is not None: diff --git a/hindsight-api-slim/tests/test_config_validation.py b/hindsight-api-slim/tests/test_config_validation.py index 8b4fd3713a..dcead9aa5e 100644 --- a/hindsight-api-slim/tests/test_config_validation.py +++ b/hindsight-api-slim/tests/test_config_validation.py @@ -122,6 +122,41 @@ def test_retain_structured_chunk_size_reads_from_env(): assert config.retain_structured_chunk_size == 9000 +def test_llm_ollama_num_ctx_defaults_to_none(monkeypatch): + """Unset Ollama num_ctx override lets Ollama use its model/server default.""" + from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, HindsightConfig + + monkeypatch.delenv(ENV_LLM_OLLAMA_NUM_CTX, raising=False) + monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock") + + config = HindsightConfig.from_env() + + assert config.llm_ollama_num_ctx is None + + +def test_llm_ollama_num_ctx_reads_positive_int(monkeypatch): + """The native Ollama context override is parsed as a positive integer.""" + from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, HindsightConfig + + monkeypatch.setenv(ENV_LLM_OLLAMA_NUM_CTX, "65536") + monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock") + + config = HindsightConfig.from_env() + + assert config.llm_ollama_num_ctx == 65536 + + +def test_llm_ollama_num_ctx_rejects_non_positive_values(monkeypatch): + """Zero would be accepted by neither Ollama nor downstream range logic.""" + from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, HindsightConfig + + monkeypatch.setenv(ENV_LLM_OLLAMA_NUM_CTX, "0") + monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "mock") + + with pytest.raises(ValueError, match=ENV_LLM_OLLAMA_NUM_CTX): + HindsightConfig.from_env() + + def test_retain_structured_chunk_size_can_be_less_than_chunk_size(): """Structured-chunk cap can be smaller than the retain chunk target.""" from hindsight_api.config import HindsightConfig diff --git a/hindsight-api-slim/tests/test_llm_wrapper.py b/hindsight-api-slim/tests/test_llm_wrapper.py index 8e523003a2..2cac2075e9 100644 --- a/hindsight-api-slim/tests/test_llm_wrapper.py +++ b/hindsight-api-slim/tests/test_llm_wrapper.py @@ -1,6 +1,199 @@ import pytest -from hindsight_api.engine.llm_wrapper import sanitize_llm_output +from hindsight_api.engine.llm_wrapper import create_llm_provider, sanitize_llm_output + + +def test_create_llm_provider_preserves_positional_timeout_compatibility(): + """The new Ollama knob must not steal the old positional timeout slot.""" + impl = create_llm_provider( + "ollama", + "", + "", + "llama3.2", + "low", + None, + None, + None, + None, + None, + None, + None, + None, + None, + False, + None, + None, + 7.5, + ) + + assert impl.timeout == 7.5 + assert impl.ollama_num_ctx is None + + +def test_llm_provider_preserves_positional_timeout_compatibility(): + """The new Ollama knob must not steal the old positional timeout slot.""" + from hindsight_api.engine.llm_wrapper import LLMProvider + + provider = LLMProvider( + "ollama", + "", + "", + "llama3.2", + "low", + None, + None, + None, + None, + False, + None, + None, + None, + None, + None, + None, + None, + 7.5, + ) + + assert provider.timeout == 7.5 + assert provider.ollama_num_ctx is None + + +def test_llm_provider_threads_ollama_num_ctx_to_provider_impl(): + """LLMProvider carries the native Ollama context override to the implementation.""" + from hindsight_api.engine.llm_wrapper import LLMProvider + + provider = LLMProvider( + provider="ollama", + api_key="", + base_url="", + model="llama3.2", + ollama_num_ctx=65536, + ) + + assert provider.ollama_num_ctx == 65536 + assert provider._provider_impl.ollama_num_ctx == 65536 + + +@pytest.mark.parametrize("bad_value", [0, -1, 1.5, "65536", True]) +def test_llm_provider_rejects_invalid_ollama_num_ctx(bad_value): + """Direct callers should fail before sending invalid Ollama request options.""" + from hindsight_api.engine.llm_wrapper import LLMProvider + + with pytest.raises(ValueError, match="ollama_num_ctx"): + LLMProvider( + provider="ollama", + api_key="", + base_url="", + model="llama3.2", + ollama_num_ctx=bad_value, + ) + + +@pytest.mark.parametrize("bad_value", [0, -1, 1.5, "65536", True]) +def test_openai_compatible_llm_rejects_invalid_ollama_num_ctx(bad_value): + """The provider implementation also validates direct construction.""" + from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM + + with pytest.raises(ValueError, match="ollama_num_ctx"): + OpenAICompatibleLLM( + provider="ollama", + api_key="", + base_url="", + model="llama3.2", + ollama_num_ctx=bad_value, + ) + + +def test_llm_provider_from_env_reads_ollama_num_ctx(monkeypatch): + """Direct env construction uses the same optional positive-int parser.""" + from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, clear_config_cache + from hindsight_api.engine.llm_wrapper import LLMProvider + + monkeypatch.setenv("HINDSIGHT_API_LLM_PROVIDER", "ollama") + monkeypatch.setenv(ENV_LLM_OLLAMA_NUM_CTX, "32768") + clear_config_cache() + + provider = LLMProvider.from_env() + + assert provider.ollama_num_ctx == 32768 + assert provider._provider_impl.ollama_num_ctx == 32768 + clear_config_cache() + + +@pytest.mark.asyncio +async def test_native_ollama_omits_num_ctx_unless_configured(monkeypatch): + """Native Ollama calls should not override the model context window by default.""" + from pydantic import BaseModel + + from hindsight_api.engine.providers.openai_compatible_llm import OpenAICompatibleLLM + + class Answer(BaseModel): + ok: bool + + class FakeResponse: + def raise_for_status(self): + return None + + def json(self): + return {"message": {"content": '{"ok": true}'}} + + calls = [] + + class FakeAsyncClient: + def __init__(self, **kwargs): + self.kwargs = kwargs + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, tb): + return None + + async def post(self, url, json, headers): + calls.append({"url": url, "json": json, "headers": headers}) + return FakeResponse() + + monkeypatch.setattr("hindsight_api.engine.providers.openai_compatible_llm.httpx.AsyncClient", FakeAsyncClient) + + default_provider = OpenAICompatibleLLM( + provider="ollama", + api_key="", + base_url="", + model="llama3.2", + ) + await default_provider._call_ollama_native( + messages=[{"role": "user", "content": "ping"}], + response_format=Answer, + max_completion_tokens=None, + temperature=None, + max_retries=0, + initial_backoff=1, + max_backoff=1, + skip_validation=False, + ) + + configured_provider = OpenAICompatibleLLM( + provider="ollama", + api_key="", + base_url="", + model="llama3.2", + ollama_num_ctx=65536, + ) + await configured_provider._call_ollama_native( + messages=[{"role": "user", "content": "ping"}], + response_format=Answer, + max_completion_tokens=None, + temperature=None, + max_retries=0, + initial_backoff=1, + max_backoff=1, + skip_validation=False, + ) + + assert "num_ctx" not in calls[0]["json"]["options"] + assert calls[0]["json"]["options"]["num_batch"] == 512 + assert calls[1]["json"]["options"]["num_ctx"] == 65536 @pytest.mark.parametrize( diff --git a/hindsight-docs/docs/developer/configuration.md b/hindsight-docs/docs/developer/configuration.md index 52cb8e4bbc..c593b305a5 100644 --- a/hindsight-docs/docs/developer/configuration.md +++ b/hindsight-docs/docs/developer/configuration.md @@ -187,9 +187,12 @@ For non-English banks (especially CJK) and the language/extraction-language trad | `HINDSIGHT_API_LLM_EXTRA_BODY` | JSON dict of extra request-body params (e.g. `temperature`, `top_p`, `max_tokens`) merged into every LLM call. Applied across the OpenAI-compatible, Fireworks, Anthropic, Gemini/VertexAI and LiteLLM (incl. Bedrock/Router) providers. Each provider merges them in its own native parameter space, so use that provider's field names (e.g. `max_tokens` for OpenAI/Anthropic vs `max_output_tokens` for Gemini). Also useful for custom model servers (e.g. vLLM `chat_template_kwargs`). | `null` | | `HINDSIGHT_API_LLM_DEFAULT_HEADERS` | JSON dict passed as `default_headers` to provider SDK clients. Used by operators routing through proxies / request-tracing middleware (e.g. Cloudflare AI Gateway, Helicone, corporate proxies). Currently wired into the Anthropic provider; other providers can opt in. | `null` | | `HINDSIGHT_API_LLM_STRICT_SCHEMA` | Grammar-enforce structured output via `json_schema` `strict: true` instead of the soft "schema-in-prompt + `json_object`" path. Use it with weaker self-hosted models that return prose preambles, markdown ` ```json ` fences, or invalid JSON — which otherwise fail to parse and wedge retain/consolidation. Applies to OpenAI-compatible backends (OpenAI, llama.cpp, vLLM) and LiteLLM; Gemini already enforces its native `response_schema` regardless, and providers without a strict mode ignore it. | `false` | +| `HINDSIGHT_API_LLM_OLLAMA_NUM_CTX` | Optional native Ollama `num_ctx` override for structured-output calls. Leave unset to use the model/server default; set a positive integer only when you need a larger context window. | Unset | | `HINDSIGHT_API_LLM_GEMINI_SAFETY_SETTINGS` | JSON-encoded list of `{category, threshold}` dicts for Gemini/VertexAI content safety filtering | `null` | | `HINDSIGHT_API_LLM_PROMPT_CACHE_ENABLED` | Reuse the fixed system prefix via the provider's explicit prompt cache, billed at the cached-input rate (Gemini/Vertex `CachedContent`). The cached prefix is shared across all banks and soft-fails to an uncached call. Set to `false` to disable. See [Models](./models#provider-capabilities). | `true` | +When `HINDSIGHT_API_LLM_PROVIDER=ollama`, Hindsight no longer sends the previous native API default `num_ctx=16384` unless you set it explicitly. To keep the old request behavior, set `HINDSIGHT_API_LLM_OLLAMA_NUM_CTX=16384`; otherwise Ollama uses the model Modelfile or server default. + **Provider Examples** ```bash diff --git a/skills/hindsight-docs/references/developer/configuration.md b/skills/hindsight-docs/references/developer/configuration.md index 42240315a0..315f61a9fc 100644 --- a/skills/hindsight-docs/references/developer/configuration.md +++ b/skills/hindsight-docs/references/developer/configuration.md @@ -187,9 +187,12 @@ For non-English banks (especially CJK) and the language/extraction-language trad | `HINDSIGHT_API_LLM_EXTRA_BODY` | JSON dict of extra request-body params (e.g. `temperature`, `top_p`, `max_tokens`) merged into every LLM call. Applied across the OpenAI-compatible, Fireworks, Anthropic, Gemini/VertexAI and LiteLLM (incl. Bedrock/Router) providers. Each provider merges them in its own native parameter space, so use that provider's field names (e.g. `max_tokens` for OpenAI/Anthropic vs `max_output_tokens` for Gemini). Also useful for custom model servers (e.g. vLLM `chat_template_kwargs`). | `null` | | `HINDSIGHT_API_LLM_DEFAULT_HEADERS` | JSON dict passed as `default_headers` to provider SDK clients. Used by operators routing through proxies / request-tracing middleware (e.g. Cloudflare AI Gateway, Helicone, corporate proxies). Currently wired into the Anthropic provider; other providers can opt in. | `null` | | `HINDSIGHT_API_LLM_STRICT_SCHEMA` | Grammar-enforce structured output via `json_schema` `strict: true` instead of the soft "schema-in-prompt + `json_object`" path. Use it with weaker self-hosted models that return prose preambles, markdown ` ```json ` fences, or invalid JSON — which otherwise fail to parse and wedge retain/consolidation. Applies to OpenAI-compatible backends (OpenAI, llama.cpp, vLLM) and LiteLLM; Gemini already enforces its native `response_schema` regardless, and providers without a strict mode ignore it. | `false` | +| `HINDSIGHT_API_LLM_OLLAMA_NUM_CTX` | Optional native Ollama `num_ctx` override for structured-output calls. Leave unset to use the model/server default; set a positive integer only when you need a larger context window. | Unset | | `HINDSIGHT_API_LLM_GEMINI_SAFETY_SETTINGS` | JSON-encoded list of `{category, threshold}` dicts for Gemini/VertexAI content safety filtering | `null` | | `HINDSIGHT_API_LLM_PROMPT_CACHE_ENABLED` | Reuse the fixed system prefix via the provider's explicit prompt cache, billed at the cached-input rate (Gemini/Vertex `CachedContent`). The cached prefix is shared across all banks and soft-fails to an uncached call. Set to `false` to disable. See [Models](./models#provider-capabilities). | `true` | +When `HINDSIGHT_API_LLM_PROVIDER=ollama`, Hindsight no longer sends the previous native API default `num_ctx=16384` unless you set it explicitly. To keep the old request behavior, set `HINDSIGHT_API_LLM_OLLAMA_NUM_CTX=16384`; otherwise Ollama uses the model Modelfile or server default. + **Provider Examples** ```bash From da12fd93c084cc9692c3646a09d1b26523223be3 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 7 Jul 2026 10:25:36 -0400 Subject: [PATCH 2/3] docs(ollama): add HINDSIGHT_API_LLM_OLLAMA_NUM_CTX to .env.example --- .env.example | 9 +++++++++ hindsight-embed/hindsight_embed/env.example | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/.env.example b/.env.example index 6a981d63ec..b59c375ca5 100644 --- a/.env.example +++ b/.env.example @@ -59,6 +59,15 @@ HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1 # HINDSIGHT_API_LLM_BASE_URL=http://localhost:1234/v1 # HINDSIGHT_API_LLM_MODEL=qwen2.5-32b-instruct +# Example: Ollama local configuration (native provider) +# HINDSIGHT_API_LLM_PROVIDER=ollama +# HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1 +# HINDSIGHT_API_LLM_MODEL=gemma3:12b +# Native Ollama context-window override (num_ctx). Leave unset to let Ollama use +# the model Modelfile / server default; set a positive integer only to force a +# specific context size (e.g. 16384 to keep the previous request behavior). +# HINDSIGHT_API_LLM_OLLAMA_NUM_CTX=16384 + # Multi-LLM strategies: configure extra LLMs by index alongside the primary above, # then pick a routing strategy. Unset = single primary LLM (default). Members are # numbered from 1; indices must be contiguous. Each operation can override with a diff --git a/hindsight-embed/hindsight_embed/env.example b/hindsight-embed/hindsight_embed/env.example index 6a981d63ec..b59c375ca5 100644 --- a/hindsight-embed/hindsight_embed/env.example +++ b/hindsight-embed/hindsight_embed/env.example @@ -59,6 +59,15 @@ HINDSIGHT_API_LLM_BASE_URL=https://api.openai.com/v1 # HINDSIGHT_API_LLM_BASE_URL=http://localhost:1234/v1 # HINDSIGHT_API_LLM_MODEL=qwen2.5-32b-instruct +# Example: Ollama local configuration (native provider) +# HINDSIGHT_API_LLM_PROVIDER=ollama +# HINDSIGHT_API_LLM_BASE_URL=http://localhost:11434/v1 +# HINDSIGHT_API_LLM_MODEL=gemma3:12b +# Native Ollama context-window override (num_ctx). Leave unset to let Ollama use +# the model Modelfile / server default; set a positive integer only to force a +# specific context size (e.g. 16384 to keep the previous request behavior). +# HINDSIGHT_API_LLM_OLLAMA_NUM_CTX=16384 + # Multi-LLM strategies: configure extra LLMs by index alongside the primary above, # then pick a routing strategy. Unset = single primary LLM (default). Members are # numbered from 1; indices must be contiguous. Each operation can override with a From cc207113cb9699c706f27ff0be0ab56cb0f41dfd Mon Sep 17 00:00:00 2001 From: Evo Date: Wed, 8 Jul 2026 08:39:18 +0800 Subject: [PATCH 3/3] fix(config): keep Ollama num_ctx optional for direct config construction --- hindsight-api-slim/hindsight_api/config.py | 2 +- hindsight-api-slim/tests/test_config_validation.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/hindsight-api-slim/hindsight_api/config.py b/hindsight-api-slim/hindsight_api/config.py index 407124d609..296e5b75dc 100644 --- a/hindsight-api-slim/hindsight_api/config.py +++ b/hindsight-api-slim/hindsight_api/config.py @@ -1580,7 +1580,7 @@ class HindsightConfig: llm_send_bank_as_user: bool # Optional native Ollama context window override. Unset lets Ollama use the # model/server default instead of forcing a Hindsight-wide value. - llm_ollama_num_ctx: int | None + llm_ollama_num_ctx: int | None = field(default=None, kw_only=True) # Per-operation sampling temperature. None means the temperature parameter is # omitted from the call (for models that reject explicit temperatures). See diff --git a/hindsight-api-slim/tests/test_config_validation.py b/hindsight-api-slim/tests/test_config_validation.py index dcead9aa5e..4217db6f60 100644 --- a/hindsight-api-slim/tests/test_config_validation.py +++ b/hindsight-api-slim/tests/test_config_validation.py @@ -134,6 +134,18 @@ def test_llm_ollama_num_ctx_defaults_to_none(monkeypatch): assert config.llm_ollama_num_ctx is None +def test_llm_ollama_num_ctx_keeps_direct_construction_default(): + """Direct HindsightConfig construction should not require the new field.""" + from dataclasses import fields + + from hindsight_api.config import HindsightConfig + + config_field = next(item for item in fields(HindsightConfig) if item.name == "llm_ollama_num_ctx") + + assert config_field.default is None + assert config_field.kw_only + + def test_llm_ollama_num_ctx_reads_positive_int(monkeypatch): """The native Ollama context override is parsed as a positive integer.""" from hindsight_api.config import ENV_LLM_OLLAMA_NUM_CTX, HindsightConfig