feat: web search fallback chain + loop-guard rework (#467 #544 #571)

- _search_with_fallbacks now tries the configured fallback chain when the
  primary backend reports failure OR when it returns empty results and a
  fallback chain is configured. Empty results with no fallback chain remain
  success=True so real 'no hits' outcomes are not turned into provider-dead
  errors (#467 rework).
- DDGS provider surfaces provider-dead only when empty results occur and a
  fallback chain exists.
- Update fallback-chain tests to match the new semantics and keep
  regression coverage.
- Carry forward prior loop-guard / tool-diagnostics improvements from #544.

Closes #467
Closes #544
Closes #571

Co-Authored-By: Hermes Evolution <evolution@hermes.ai>

Author: Hermes Evolution

agent/loop_guard.py

@@ -34,6 +34,7 @@
 
 from __future__ import annotations
 
+import json
 import re
 from typing import Any, Dict, List, Optional, Tuple
 
@@ -57,6 +58,8 @@
     "exit status",
     "is not recognized",
     "could not be found",
+    "no results",
+    "no results found",
 )
 
 _EXIT_CODE_RE = re.compile(r"exit code[:\s]+([1-9]\d*)", re.IGNORECASE)
@@ -69,49 +72,52 @@
 _NON_RETRYABLE = frozenset({"timeout", "permission", "missing_command", "limit"})
 _NONRETRY_THRESHOLD = 2
 
+# Idempotent tools that are especially prone to content-free repetition and that
+# the issue evidence shows spiraling with no progress even when individual calls
+# return "success". Count these as non-progress after a shorter run so the model
+# is nudged toward a different query / tool / strategy.
+_SHORT_CIRCUIT_IDEMPOTENT = frozenset({"search_files", "web_search", "web_extract"})
+_SHORT_CIRCUIT_REPEAT_THRESHOLD = 4
+
 # Mutating tools get LOWER thresholds than idempotent tools because a fixation
 # on mutating operations (writing files, running commands) is more costly and
 # indicates a deeper strategy problem (#432).
-_IDEMPOTENT_TOOLS = frozenset(
-    {
-        "read_file",
-        "search_files",
-        "web_search",
-        "web_extract",
-        "session_search",
-        "browser_snapshot",
-        "browser_console",
-        "browser_get_images",
-        "mcp_filesystem_read_file",
-        "mcp_filesystem_read_text_file",
-        "mcp_filesystem_read_multiple_files",
-        "mcp_filesystem_list_directory",
-        "mcp_filesystem_list_directory_with_sizes",
-        "mcp_filesystem_directory_tree",
-        "mcp_filesystem_get_file_info",
-        "mcp_filesystem_search_files",
-    }
-)
-_MUTATING_TOOLS = frozenset(
-    {
-        "terminal",
-        "execute_code",
-        "write_file",
-        "patch",
-        "todo",
-        "memory",
-        "skill_manage",
-        "browser_click",
-        "browser_type",
-        "browser_press",
-        "browser_scroll",
-        "browser_navigate",
-        "send_message",
-        "cronjob",
-        "delegate_task",
-        "process",
-    }
-)
+_IDEMPOTENT_TOOLS = frozenset({
+    "read_file",
+    "search_files",
+    "web_search",
+    "web_extract",
+    "session_search",
+    "browser_snapshot",
+    "browser_console",
+    "browser_get_images",
+    "mcp_filesystem_read_file",
+    "mcp_filesystem_read_text_file",
+    "mcp_filesystem_read_multiple_files",
+    "mcp_filesystem_list_directory",
+    "mcp_filesystem_list_directory_with_sizes",
+    "mcp_filesystem_directory_tree",
+    "mcp_filesystem_get_file_info",
+    "mcp_filesystem_search_files",
+})
+_MUTATING_TOOLS = frozenset({
+    "terminal",
+    "execute_code",
+    "write_file",
+    "patch",
+    "todo",
+    "memory",
+    "skill_manage",
+    "browser_click",
+    "browser_type",
+    "browser_press",
+    "browser_scroll",
+    "browser_navigate",
+    "send_message",
+    "cronjob",
+    "delegate_task",
+    "process",
+})
 # Default thresholds: lower for mutating tools, higher for idempotent (#432).
 # Mutating:  repeat at 4, fail at 2, escalate at 8
 # Idempotent: repeat at 8, fail at 4, escalate at 15
@@ -143,27 +149,49 @@ def _looks_like_failure(content: Any) -> bool:
     return bool(_EXIT_CODE_RE.search(content))
 
 
-def _recent_tool_runs(messages: List[Dict[str, Any]]) -> List[Tuple[str, bool, Optional[str]]]:
-    """Most-recent-first list of (single_tool_name, result_failed, failure_class)
+def _tool_result_arg_hash(content: Any) -> Optional[str]:
+    """Return a canonical JSON key of the tool-call arguments embedded in a
+    tool result, or None if no arguments can be parsed. Used to detect
+    identical-query repetition for tools like web_search where the same
+    arguments produce the same empty result and drive a spiral (#467).
+    """
+    if not isinstance(content, str) or not content:
+        return None
+    try:
+        parsed = json.loads(content)
+    except Exception:
+        return None
+    if not isinstance(parsed, dict):
+        return None
+    args = parsed.get("parameters") or parsed.get("args") or parsed.get("arguments")
+    if not isinstance(args, dict):
+        return None
+    return json.dumps(args, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
+
+
+def _recent_tool_runs(
+    messages: List[Dict[str, Any]],
+) -> List[Tuple[str, bool, Optional[str], Optional[str]]]:
+    """Most-recent-first list of
+    (single_tool_name, result_failed, failure_class, arg_hash)
     for the trailing run of assistant turns that each called EXACTLY ONE tool.
     ``failure_class`` is the tool_diagnostics category of the failing result (or
-    None when the turn did not fail).
+    None when the turn did not fail). ``arg_hash`` is a canonical JSON of the
+    parsed arguments when they can be recovered from the tool result.
 
     Stops at the first assistant turn that is not a single-tool call (a text
     reply, or a multi-tool turn) — that breaks the "stuck on one tool" run.
     Multi-tool turns are normal varied work, not a single-tool spiral.
     """
-    runs: List[Tuple[str, bool, Optional[str]]] = []
+    runs: List[Tuple[str, bool, Optional[str], Optional[str]]] = []
     i = len(messages) - 1
     # Collect tool results by id as we walk back so we can mark failures.
     while i >= 0:
         msg = messages[i]
         if msg.get("role") == "assistant" and msg.get("tool_calls"):
             tcs = [tc for tc in msg["tool_calls"] if isinstance(tc, dict)]
             names = [
-                tc.get("function", {}).get("name")
-                for tc in tcs
-                if tc.get("function")
+                tc.get("function", {}).get("name") for tc in tcs if tc.get("function")
             ]
             names = [n for n in names if n]
             if len(set(names)) != 1:
@@ -174,14 +202,17 @@ def _recent_tool_runs(messages: List[Dict[str, Any]]) -> List[Tuple[str, bool, O
             # Results for this turn are the "tool" messages that follow it.
             failed = False
             category: Optional[str] = None
+            arg_hash: Optional[str] = None
             for j in range(i + 1, len(messages)):
                 tm = messages[j]
                 if tm.get("role") != "tool":
                     break
-                if _looks_like_failure(tm.get("content")):
+                content = tm.get("content")
+                if _looks_like_failure(content):
                     failed = True
-                    category = _failure_category(tm.get("content")) or category
-            runs.append((tool, failed, category))
+                    category = _failure_category(content) or category
+                arg_hash = _tool_result_arg_hash(content) or arg_hash
+            runs.append((tool, failed, category, arg_hash))
             i -= 1
         elif msg.get("role") == "tool":
             i -= 1  # skip result messages; handled with their assistant turn
@@ -223,11 +254,13 @@ def maybe_nudge(
 ) -> Optional[str]:
     """Return a nudge string if the trailing single-tool run is stuck, else None.
 
-    Three trigger levels (each is lower for mutating tools than idempotent):
+    Trigger levels (each is lower for mutating tools than idempotent):
       1. Non-retryable failure class repeated twice (highest priority, #231)
       2. Generic failures >= fail_threshold
       3. Same tool called >= repeat_threshold times in a row
       4. Escalated interrupt at higher counts (#432)
+      5. Same *arguments* repeated for short-circuit idempotent tools
+         (search_files / web_search / web_extract) >= 4 times (#467)
 
     Returns None when the agent is making varied progress (not stuck).
     """
@@ -243,18 +276,30 @@ def maybe_nudge(
     is_unknown = cat == "unknown"
     if repeat_threshold is None:
         repeat_threshold = (
-            _MUTATING_REPEAT_THRESHOLD if (is_mutating or is_unknown)
+            _MUTATING_REPEAT_THRESHOLD
+            if (is_mutating or is_unknown)
             else _IDEMPOTENT_REPEAT_THRESHOLD
         )
     if fail_threshold is None:
         fail_threshold = (
-            _MUTATING_FAIL_THRESHOLD if (is_mutating or is_unknown)
+            _MUTATING_FAIL_THRESHOLD
+            if (is_mutating or is_unknown)
             else _IDEMPOTENT_FAIL_THRESHOLD
         )
     escalate_threshold = (
-        _MUTATING_ESCALATE_THRESHOLD if (is_mutating or is_unknown)
+        _MUTATING_ESCALATE_THRESHOLD
+        if (is_mutating or is_unknown)
         else _IDEMPOTENT_ESCALATE_THRESHOLD
     )
+    short_circuit_threshold = (
+        _MUTATING_REPEAT_THRESHOLD
+        if (is_mutating or is_unknown)
+        else (
+            _SHORT_CIRCUIT_REPEAT_THRESHOLD
+            if tool in _SHORT_CIRCUIT_IDEMPOTENT
+            else repeat_threshold
+        )
+    )
 
     # All entries in `runs` share the same tool (run breaks on tool change),
     # but guard anyway:
@@ -264,7 +309,7 @@ def maybe_nudge(
     consec_nonretry = 0
     nonretry_class: Optional[str] = None
     counting_nonretry = True
-    for _t, failed, category in same:
+    for _t, failed, category, _arg_hash in same:
         if failed:
             consec_fail += 1
         else:
@@ -312,6 +357,23 @@ def maybe_nudge(
             f"again the same way."
         )
 
+    # Same-argument repetition for known spiral-prone idempotent tools (#467).
+    # This catches web_search returning "no results" / search_files returning
+    # nothing, where each individual call technically "succeeded" but repeating
+    # the exact same query is still a loop.
+    if tool in _SHORT_CIRCUIT_IDEMPOTENT and count >= short_circuit_threshold:
+        arg_hashes = [r[3] for r in same if r[3] is not None]
+        if arg_hashes and len(set(arg_hashes)) == 1:
+            score = _tool_spiral_score(tool, count, short_circuit_threshold)
+            score_line = f"\n{score}" if score else ""
+            return (
+                f"[loop-guard] You have called `{tool}` {count} times with the "
+                f"SAME arguments and the result is not making progress.{score_line} "
+                f"Do NOT repeat `{tool}` with those identical arguments. Rephrase "
+                f"the query, broaden or narrow it, switch to a different information "
+                f"source, or state the blocker if no relevant results are available."
+            )
+
     if count >= repeat_threshold:
         # Build diversity score for the nudge.
         score = _tool_spiral_score(tool, count, repeat_threshold)

agent/tool_diagnostics.py

@@ -19,30 +19,71 @@
 
 # Ordered most-specific first; first match wins. (regex, category, hint).
 _RULES: tuple[tuple[re.Pattern, str, str], ...] = (
-    (re.compile(r"command not found|not recognized as|: No such file or directory.*\b(sh|bash|exec)\b|executable file not found", re.I),
-     "missing_command",
-     "A required binary/command is missing. Check prerequisites first "
-     "(`which <cmd>` / install it), or use a different tool — do NOT repeat the same command."),
-    (re.compile(r"permission denied|access denied|not permitted|forbidden|refusing to write|operation not permitted|EACCES", re.I),
-     "permission",
-     "Access is denied and the agent can't elevate. Do NOT retry the same path — "
-     "use an allowed path/tool, or report exactly what access is needed."),
-    (re.compile(r"timed out|timeout|deadline exceeded|ClosedResourceError|unreachable|connection refused|ETIMEDOUT", re.I),
-     "timeout",
-     "The operation timed out / the resource is unreachable. Set it aside, route to a "
-     "fallback if one exists, and do NOT loop on health checks or retry blindly."),
-    (re.compile(r"\b(char(acter)?s?|byte)s?\b.*\b(limit|exceed|too (long|large|big)|maximum)\b|exceeds the maximum|max(imum)? (length|size)|too many tokens|context length", re.I),
-     "limit",
-     "A size/length limit was hit. Don't resend as-is — chunk the work, summarize, "
-     "or raise the relevant config limit; a near-identical retry will fail the same way."),
-    (re.compile(r"no such file or directory|not found|does not exist|cannot find|no matches found|0 results|no results", re.I),
-     "not_found",
-     "The target wasn't found. Re-check the path/name (it may be dynamic), broaden the "
-     "search, or create the prerequisite first — don't repeat the same lookup."),
-    (re.compile(r"traceback \(most recent call|exit code[:\s]+[1-9]|exit status [1-9]|non-zero exit|error:|exception|failed", re.I),
-     "runtime_error",
-     "The call errored. Read the message, fix the root cause, and CHANGE the call — "
-     "retrying the same arguments will reproduce it."),
+    (
+        re.compile(
+            r"command not found|not recognized as|: No such file or directory.*\b(sh|bash|exec)\b|executable file not found",
+            re.I,
+        ),
+        "missing_command",
+        "A required binary/command is missing. Check prerequisites first "
+        "(`which <cmd>` / install it), or use a different tool — do NOT repeat the same command.",
+    ),
+    (
+        re.compile(
+            r"permission denied|access denied|not permitted|forbidden|refusing to write|operation not permitted|EACCES",
+            re.I,
+        ),
+        "permission",
+        "Access is denied and the agent can't elevate. Do NOT retry the same path — "
+        "use an allowed path/tool, or report exactly what access is needed.",
+    ),
+    (
+        re.compile(
+            r"timed out|timeout|deadline exceeded|ClosedResourceError|unreachable|connection refused|ETIMEDOUT",
+            re.I,
+        ),
+        "timeout",
+        "The operation timed out / the resource is unreachable. Set it aside, route to a "
+        "fallback if one exists, and do NOT loop on health checks or retry blindly.",
+    ),
+    (
+        re.compile(
+            r"\b(char(acter)?s?|byte)s?\b.*\b(limit|exceed|too (long|large|big)|maximum)\b|exceeds the maximum|max(imum)? (length|size)|too many tokens|context length",
+            re.I,
+        ),
+        "limit",
+        "A size/length limit was hit. Don't resend as-is — chunk the work, summarize, "
+        "or raise the relevant config limit; a near-identical retry will fail the same way.",
+    ),
+    (
+        re.compile(
+            r"\bno results\b|\bno results found\b|duckduckgo search failed|brave search returned http|could not reach .* search|searxng returned http",
+            re.I,
+        ),
+        "provider_dead",
+        "The active search provider is not returning results. Switch to a different "
+        "search_backend in config.yaml or via hermes tools, or report the blocker "
+        "if no alternative provider is configured. Do not retry the same query with "
+        "the same provider.",
+    ),
+    (
+        re.compile(
+            r"no such file or directory|not found|does not exist|cannot find|no matches found|0 results|no results",
+            re.I,
+        ),
+        "not_found",
+        "The target wasn't found. Re-check the path/name (it may be dynamic), broaden the "
+        "search, or create the prerequisite first — don't repeat the same lookup.",
+    ),
+    (
+        re.compile(
+            r"traceback \(most recent call|exit code[:\s]+[1-9]|exit status [1-9]|non-zero exit|error:|exception|failed",
+            re.I,
+        ),
+        "runtime_error",
+        "The call errored. Read the message, fix the root cause, and CHANGE the call — "
+        "retrying the same arguments will reproduce it.",
+    ),
 )