test(indexability): follow llms.txt redirect to weaviate.io; skip on network reset

/llms.txt 301-redirects to weaviate.io/llms.txt. Add weaviate.io to the Claude
web_fetch allowed_domains so it can follow the redirect, and make
test_llms_txt_accessible use a browser UA and treat a transient connection
reset/timeout as an inconclusive skip (matching test_llms_txt_code.py) rather
than a hard failure.

Author: g-despot

tests/test_docs_indexability.py

@@ -277,11 +277,21 @@ def test_llm_notice_present(path):
 @pytest.mark.indexability
 def test_llms_txt_accessible():
     """/llms.txt returns 200, has substantial content, and mentions Weaviate."""
-    resp = requests.get(
-        f"{BASE_URL}/llms.txt",
-        timeout=30,
-        headers={"User-Agent": "WeaviateDocsIndexabilityTest/1.0"},
-    )
+    try:
+        resp = requests.get(
+            f"{BASE_URL}/llms.txt",
+            timeout=30,
+            headers={
+                "User-Agent": (
+                    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
+                    "AppleWebKit/537.36 (KHTML, like Gecko) "
+                    "Chrome/124.0.0.0 Safari/537.36"
+                )
+            },
+        )
+    except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc:
+        # network failure must not flake the suite
+        pytest.skip(f"Network issue fetching /llms.txt: {exc}")
     assert resp.status_code == 200, f"/llms.txt returned {resp.status_code}"
     assert len(resp.text) > 500, f"/llms.txt content too short ({len(resp.text)} chars)"
     assert "weaviate" in resp.text.lower(), "/llms.txt doesn't mention Weaviate"
@@ -450,7 +460,7 @@ def test_claude_can_fetch_llms_txt():
             "type": "web_fetch_20250910",
             "name": "web_fetch",
             "max_uses": 1,
-            "allowed_domains": ["docs.weaviate.io"],
+            "allowed_domains": ["docs.weaviate.io", "weaviate.io"],
         }],
         messages=[{
             "role": "user",