fix: bound active memory rerank latency

Author: deeflect

plugins/hermes-dory/provider.py

@@ -1437,7 +1437,7 @@ def _build_tool_schemas() -> list[dict[str, Any]]:
         },
         {
             "name": "dory_active_memory",
-            "description": "Run the bounded active-memory pre-reply pass. Limits: budget_tokens <= 1200, timeout_ms <= 5000. Set include_wake=false if wake was already called.",
+            "description": "Run the bounded active-memory pre-reply pass. Limits: budget_tokens <= 1200, timeout_ms <= 30000. Set include_wake=false if wake was already called.",
             "parameters": {
                 "type": "object",
                 "properties": {
@@ -1458,7 +1458,7 @@ def _build_tool_schemas() -> list[dict[str, Any]]:
                         },
                     },
                     "profile": {"type": "string"},
-                    "timeout_ms": {"type": "integer", "minimum": 100, "maximum": 5000},
+                    "timeout_ms": {"type": "integer", "minimum": 100, "maximum": 30000},
                     "budget_tokens": {"type": "integer", "minimum": 100, "maximum": 1200},
                     "include_wake": {"type": "boolean"},
                     "rerank": {"type": "string", "enum": ["auto", "true", "false"]},

src/dory_core/active_memory.py

@@ -45,6 +45,7 @@
 _PLANNER_MIN_REMAINING_MS = 1800
 _COMPOSER_MIN_REMAINING_MS = 2200
 _COMPOSER_TIMEOUT_HEADROOM_MS = 6000
+_RERANK_TIMEOUT_HEADROOM_MS = 6000
 _TOPIC_TOKEN_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
 _TOPIC_STOPWORDS = {
     "about",
@@ -232,7 +233,7 @@ def _retrieve_evidence(
                 mode="hybrid",
                 corpus="durable",
                 include_content=True,
-                rerank="true" if req.rerank == "auto" else req.rerank,
+                rerank=_active_memory_rerank_mode(req.rerank, deadline),
                 deadline=deadline,
                 source_policy=source_policy,
                 min_remaining_ms=_COMPOSER_MIN_REMAINING_MS,
@@ -516,6 +517,16 @@ def _planning_context_from_helper(helper: WikiHelperContext) -> ActiveMemoryPlan
     )
 
 
+def _active_memory_rerank_mode(
+    requested: Literal["auto", "true", "false"], deadline: "_Deadline"
+) -> Literal["auto", "true", "false"]:
+    if requested == "false":
+        return "false"
+    if deadline.total_ms <= _RERANK_TIMEOUT_HEADROOM_MS:
+        return "false"
+    return "true" if requested == "auto" else requested
+
+
 def _search_candidates(
     search_engine: _SearchEngine,
     *,

src/dory_core/config.py

@@ -84,14 +84,14 @@ class DorySettings(BaseSettings):
     query_expansion_max: int = Field(default=2, ge=0, le=5)
     query_reranker_enabled: bool = False
     query_reranker_provider: Literal["openrouter", "local"] = "openrouter"
-    query_reranker_candidate_limit: int = Field(default=40, ge=2, le=100)
+    query_reranker_candidate_limit: int = Field(default=8, ge=2, le=100)
     local_reranker_api_key: str | None = Field(
         default=None,
         validation_alias=AliasChoices("DORY_LOCAL_RERANKER_API_KEY", "DORY_LOCAL_LLM_API_KEY"),
     )
     local_reranker_base_url: str = "http://127.0.0.1:8000/v1"
     local_reranker_model: str = "qwen3-rerank"
-    local_reranker_timeout_seconds: float = Field(default=30.0, gt=0.0, le=300.0)
+    local_reranker_timeout_seconds: float = Field(default=5.0, gt=0.0, le=300.0)
     eval_judge_enabled: bool = True
     max_write_bytes: int = Field(default=10_240, ge=1)
     default_wake_budget_tokens: int = Field(default=600, ge=1, le=1500)

src/dory_core/tool_registry.py

@@ -82,7 +82,7 @@ class DoryTool:
         http_path="/v1/active-memory",
         description=(
             "Run the bounded active-memory pre-reply pass. Limits: budget_tokens <= 1200, "
-            "timeout_ms <= 5000. Set include_wake=false if wake was already called."
+            "timeout_ms <= 30000. Set include_wake=false if wake was already called."
         ),
         request_model=ActiveMemoryReq,
         handler="active_memory",

src/dory_core/types.py

@@ -133,7 +133,7 @@ class ActiveMemoryReq(BaseModel):
     project: str | None = None
     scope: SearchScope = Field(default_factory=SearchScope)
     profile: ActiveMemoryProfile = "auto"
-    timeout_ms: int = Field(default=3000, ge=100, le=5000)
+    timeout_ms: int = Field(default=3000, ge=100, le=30000)
     budget_tokens: int = Field(default=400, ge=100, le=1200)
     include_wake: bool = True
     rerank: Literal["auto", "true", "false"] = "auto"

tests/unit/test_active_memory.py

@@ -2,7 +2,7 @@
 
 from datetime import UTC, datetime
 from pathlib import Path
-from time import sleep
+from time import monotonic, sleep
 
 from dory_core.active_memory import ActiveMemoryEngine
 from dory_core.retrieval_planner import ActiveMemoryComposition, ActiveMemoryPlanningContext, ActiveMemoryRetrievalPlan
@@ -153,6 +153,7 @@ def test_active_memory_builds_memory_block_for_state_question(tmp_path: Path) ->
             prompt="what are we working on today",
             agent="claude",
             cwd=str(tmp_path),
+            timeout_ms=7000,
         )
     )
 
@@ -358,6 +359,7 @@ def search(self, req: SearchReq):  # pragma: no cover - test stub
             prompt="Before answering a coding question about Dory agent integrations, retrieve only the memory that matters.",
             agent="codex",
             include_wake=True,
+            timeout_ms=7000,
         )
     )
 
@@ -554,8 +556,8 @@ def search(self, req: SearchReq):  # pragma: no cover - test stub
             prompt="debug Dory Docker MCP setup",
             agent="codex",
             include_wake=False,
-            timeout_ms=5000,
-        ).model_copy(update={"timeout_ms": 7000})
+            timeout_ms=7000,
+        )
     )
 
     assert "Docker MCP setup fails when the daemon URL is stale." in result.block
@@ -845,8 +847,8 @@ def test_active_memory_uses_planner_queries_and_llm_composition_when_budget_allo
             prompt="what are we working on today",
             agent="claude",
             cwd=str(tmp_path),
-            timeout_ms=5000,
-        ).model_copy(update={"timeout_ms": 7000})
+            timeout_ms=7000,
+        )
     )
 
     assert result.summary == "Rooster remains the active focus."
@@ -916,8 +918,8 @@ def compose_active_memory(
             prompt="what are we working on today",
             agent="claude",
             include_wake=False,
-            timeout_ms=5000,
-        ).model_copy(update={"timeout_ms": 7000})
+            timeout_ms=7000,
+        )
     )
 
     assert result.summary.startswith("Rooster is the active focus this week.")
@@ -1095,3 +1097,61 @@ def search(self, req: SearchReq):
 
     assert 1 <= len(search_engine.requests) < 4
     assert result.kind == "memory"
+
+
+def test_active_memory_disables_rerank_when_total_timeout_cannot_absorb_it(tmp_path: Path) -> None:
+    class ManyQueryPlanner:
+        def plan_active_memory(
+            self,
+            *,
+            prompt: str,
+            context: ActiveMemoryPlanningContext,
+        ) -> ActiveMemoryRetrievalPlan:
+            del prompt, context
+            return ActiveMemoryRetrievalPlan(
+                durable_queries=("one", "two", "three"),
+                session_queries=(),
+                include_sessions=False,
+                durable_limit=8,
+                session_limit=0,
+            )
+
+    class RerankSensitiveSearchEngine(_StubSearchEngine):
+        def search(self, req: SearchReq):
+            self.requests.append(req)
+            if req.rerank != "false":
+                sleep(0.08)
+            return super().search(req)
+
+    search_engine = RerankSensitiveSearchEngine()
+    engine = ActiveMemoryEngine(
+        wake_builder=WakeBuilder(root=tmp_path),
+        search_engine=search_engine,
+        planner=ManyQueryPlanner(),
+    )
+
+    started = monotonic()
+    result = engine.build(
+        ActiveMemoryReq(
+            prompt="what are we working on today",
+            agent="claude",
+            include_wake=False,
+            timeout_ms=5000,
+        )
+    )
+    elapsed = monotonic() - started
+
+    assert result.kind == "memory"
+    assert search_engine.requests
+    assert {req.rerank for req in search_engine.requests} == {"false"}
+    assert elapsed < 0.08
+
+
+def test_active_memory_request_accepts_larger_timeout_for_slow_local_models() -> None:
+    req = ActiveMemoryReq(
+        prompt="what are we working on today",
+        agent="claude",
+        timeout_ms=12000,
+    )
+
+    assert req.timeout_ms == 12000

tests/unit/test_rerank_orchestrator.py

@@ -112,3 +112,12 @@ def test_rerank_telemetry_logs_safe_metrics_without_content(caplog) -> None:
     assert "snippet_chars_after=" in messages
     assert sensitive_text not in messages
     assert "needle detail" not in messages
+
+
+def test_rerank_orchestrator_default_candidate_limit_matches_local_latency_budget() -> None:
+    from dory_core.config import DorySettings
+
+    settings = DorySettings()
+
+    assert settings.query_reranker_candidate_limit == 8
+    assert settings.local_reranker_timeout_seconds == 5.0