test: harden agents/LLM indexability tests

- Retry transient Weaviate Cloud connection blips in the agents runner (wrap
  execute_py_script_as_module; add connection-error transient markers).
- test_claude_can_fetch_llms_txt: fix stale section assertion (llms.txt no
  longer has 'agents'/'cloud' top-level sections -> check 'quickstart' + 'the
  weaviate stack'), bump max_tokens 2048->4096 to avoid truncated responses.
- test_chatgpt_can_search_code_tabs: web_search paraphrases code, so verify the
  vectorizer is identified (normalized text2vec-weaviate) instead of requiring
  verbatim lines, and drop the vectorizer hint from the prompt so the check
  actually proves the code tabs were read from the site.

Author: g-despot

tests/test_agents.py

@@ -27,7 +27,10 @@
 )
 def test_on_blank_instance_pyv4(script_loc):
     proc_script = utils.load_and_prep_script(script_loc)
-    utils.execute_py_script_as_module(proc_script, Path(script_loc).stem)
+    utils.retry_on_transient(
+        lambda: utils.execute_py_script_as_module(proc_script, Path(script_loc).stem),
+        label=str(script_loc),
+    )
 
 
 @pytest.mark.pyv4
@@ -44,7 +47,10 @@ def test_on_blank_instance_pyv4(script_loc):
 )
 def test_recipes_requiring_openai_pyv4(script_loc):
     proc_script = utils.load_and_prep_script(script_loc)
-    utils.execute_py_script_as_module(proc_script, Path(script_loc).stem)
+    utils.retry_on_transient(
+        lambda: utils.execute_py_script_as_module(proc_script, Path(script_loc).stem),
+        label=str(script_loc),
+    )
 
 
 @pytest.mark.ts

tests/test_docs_indexability.py

@@ -451,16 +451,16 @@ def test_claude_can_fetch_llms_txt():
     url = f"{BASE_URL}/llms.txt"
 
     # The llms.txt file starts with "# Weaviate Documentation" and contains
-    # section headings like "## agents", "## cloud", "## weaviate".
+    # section headings like "## Quickstart", "## The Weaviate stack".
     # Ask Claude to quote specific content to prove it fetched the real file.
     # LLM responses are non-deterministic, so retry a few times: pass as soon
     # as one attempt satisfies ALL conditions; only fail if every attempt does.
-    required_sections = ["agents", "cloud", "weaviate"]
+    required_sections = ["quickstart", "the weaviate stack"]
     last_text = ""
     for attempt in range(3):
         response = client.messages.create(
             model="claude-haiku-4-5-20251001",
-            max_tokens=2048,
+            max_tokens=4096,
             tools=[{
                 "type": "web_fetch_20250910",
                 "name": "web_fetch",
@@ -579,7 +579,7 @@ def test_chatgpt_can_search_code_tabs():
             "Tell me: 1) The exact URL you found "
             "2) What programming languages have code examples "
             "3) For each language, what is the exact vectorizer configuration "
-            "line from the code in the quickstart (e.g. text2vec_weaviate, text2VecWeaviate, etc.)"
+            "line from the code in the quickstart"
         ),
     )
 
@@ -601,16 +601,16 @@ def test_chatgpt_can_search_code_tabs():
         f"Response:\n{text[:1000]}"
     )
 
-    # Must find at least 3 of the 5 exact vectorizer config lines.
-    # web_search_preview may not extract all tabs verbatim, but should
-    # get most of them from the indexed page content.
-    vectorizer_found = sum(
-        1 for line in QUICKSTART_VECTORIZER_LINES.values()
-        if line in text
-    )
-    assert vectorizer_found >= 3, (
-        f"ChatGPT only found {vectorizer_found}/5 vectorizer lines (expected 3+). "
-        f"Response:\n{text[:2000]}"
+    # web_search paraphrases code, so instead of requiring verbatim lines,
+    # confirm ChatGPT identified the text2vec-weaviate vectorizer from the code
+    # tabs (the vectorizer name isn't in the prompt — it can only come from
+    # reading the page). Verbatim per-language extraction is hard-verified
+    # separately by test_claude_can_fetch_code_tabs (direct web_fetch).
+    import re
+    normalized = re.sub(r"[_\-\s]", "", text_lower)
+    assert "text2vecweaviate" in normalized, (
+        f"ChatGPT didn't identify the text2vec-weaviate vectorizer from the "
+        f"quickstart code tabs. Response:\n{text[:2000]}"
     )
 
 

tests/utils.py

@@ -19,6 +19,8 @@
     "connection reset", "econnreset", "connection aborted",
     "502", "503", "504", "429", "too many requests",
     "rate limit", "overloaded", "temporarily unavailable",
+    "error while connecting", "connection error",
+    "weaviate_cloud_connection_error",
 )