Indexer: Fix F95ZONE_UNAVAILABLE for missing threads, improve error

WillyJL · WillyJL · commit 5d6ede0636fe · 2025-12-22T22:30:42.000+01:00
reporting
diff --git a/indexer/cache.py b/indexer/cache.py
@@ -64,7 +64,6 @@ async def lifespan():
     try:
         yield
     finally:
-
         await redis.aclose()
         redis = None
 
@@ -145,6 +144,8 @@ async def _update_thread_cache(id: int, name: str) -> None:
             INDEX_ERROR: result.error_flag,
             EXPIRE_TIME: int(now + result.retry_delay),
         }
+        if result.details:
+            new_fields[INDEX_ERROR] += f": {result.details}"
         # Consider new error as a change
         if old_fields.get(INDEX_ERROR) != new_fields.get(INDEX_ERROR):
             new_fields[LAST_CHANGE] = int(now)
diff --git a/indexer/f95zone.py b/indexer/f95zone.py
@@ -10,7 +10,10 @@
 import aiohttp
 import aiolimiter
 
-from common import meta
+from common import (
+    meta,
+    parser,
+)
 
 RATELIMIT = aiolimiter.AsyncLimiter(max_rate=1, time_period=0.5)
 TIMEOUT = aiohttp.ClientTimeout(total=30, connect=30, sock_read=30, sock_connect=30)
@@ -32,7 +35,6 @@
 TEMP_ERROR_MESSAGES = (
     b'<div id="cf-error-details" class="p-0">',
     b"<b>504 - Gateway Timeout .</b>",
-    b'<body data-template="error">',
     b"<title>502 Bad Gateway</title>",
     b"<title>Error 502</title>",
     b"An unexpected error occurred. Please try again later.",
@@ -44,6 +46,10 @@
     b'<script src="https://static.f95zone.to/assets/SamF95/ErrorPage',
     b'<div class="blockMessage"><p>Please check back in 10 mins</p></div>',
 )
+THREAD_MISSING_MESSAGES = (
+    "The requested thread could not be found.",
+    "You do not have permission to view this page or perform this action.",
+)
 
 logger = logging.getLogger(__name__)
 session: aiohttp.ClientSession = None
@@ -68,6 +74,7 @@
 class IndexerError:
     error_flag: str
     retry_delay: int
+    details: str = ""
 
 
 ERROR_SESSION_LOGGED_OUT = IndexerError(
@@ -79,6 +86,9 @@ class IndexerError:
 ERROR_F95ZONE_UNAVAILABLE = IndexerError(
     "F95ZONE_UNAVAILABLE", dt.timedelta(minutes=15).total_seconds()
 )
+ERROR_F95ZONE_ERROR = IndexerError(
+    "F95ZONE_ERROR", dt.timedelta(minutes=15).total_seconds()
+)
 ERROR_THREAD_MISSING = IndexerError(
     "THREAD_MISSING", dt.timedelta(days=14).total_seconds()
 )
@@ -114,7 +124,6 @@ async def lifespan():
     try:
         yield
     finally:
-
         await session.close()
         session = None
 
@@ -123,10 +132,24 @@ def check_error(
     res: bytes | dict | Exception, logger: logging.Logger
 ) -> IndexerError | None:
     if isinstance(res, bytes):
-        if any((msg in res) for msg in LOGIN_ERROR_MESSAGES):
-            logger.error("Logged out of F95zone")
-            # TODO: maybe auto login, but xf_user cookie should be enough for a long time
-            return ERROR_SESSION_LOGGED_OUT
+        if b'<body data-template="error">' in res:
+            try:
+                html = parser.html(res)
+                message = (
+                    html.select_one(".p-body-pageContent .blockMessage")
+                    .get_text()
+                    .strip()
+                )
+                if message in THREAD_MISSING_MESSAGES:
+                    return ERROR_THREAD_MISSING
+                else:
+                    logger.error(f"F95zone Forum returned an error: {message}")
+                    return dataclasses.replace(ERROR_F95ZONE_ERROR, details=message)
+            except Exception:
+                logger.error(
+                    f"F95zone Forum returned an error that could not be parsed: {res}"
+                )
+                return ERROR_UNKNOWN_RESPONSE
 
         if any((msg in res) for msg in RATELIMIT_FORUM_ERRORS):
             logger.error("Hit F95zone Forum ratelimit")
@@ -136,15 +159,27 @@ def check_error(
             logger.warning("F95zone temporarily unreachable")
             return ERROR_F95ZONE_UNAVAILABLE
 
+        if any((msg in res) for msg in LOGIN_ERROR_MESSAGES):
+            logger.error("Logged out of F95zone")
+            # TODO: maybe auto login, but xf_user cookie should be enough for a long time
+            return ERROR_SESSION_LOGGED_OUT
+
     elif isinstance(res, dict):
         if res.get("status") == "error":
+            message = res.get("msg")
 
-            if any((msg == res.get("msg")) for msg in RATELIMIT_API_ERRORS):
+            if any((msg == message) for msg in RATELIMIT_API_ERRORS):
                 logger.error("Hit F95zone API ratelimit")
                 return ERROR_F95ZONE_RATELIMIT
 
-            logger.error(f"F95zone API returned an error: {res}")
-            return ERROR_UNKNOWN_RESPONSE
+            if isinstance(message, str):
+                logger.error(f"F95zone API returned an error: {message}")
+                return dataclasses.replace(ERROR_F95ZONE_ERROR, details=message)
+            else:
+                logger.error(
+                    f"F95zone API returned an error that could not be parsed: {res}"
+                )
+                return ERROR_UNKNOWN_RESPONSE
 
     elif isinstance(res, Exception):
         if isinstance(res, (asyncio.TimeoutError, aiohttp.ClientConnectionError)):
diff --git a/indexer/scraper.py b/indexer/scraper.py
@@ -38,10 +38,6 @@ async def thread(id: int) -> dict[str, str] | f95zone.IndexerError | None:
     loop = asyncio.get_event_loop()
     ret = await loop.run_in_executor(None, parser.thread, res)
     if isinstance(ret, parser.ParserError):
-
-        if ret.message == "Thread structure missing" and req.status in (403, 404):
-            return f95zone.ERROR_THREAD_MISSING
-
         logger.error(f"Thread {id} parsing failed: {ret.message}\n{ret.dump}")
         return f95zone.ERROR_PARSING_FAILED
 
@@ -80,7 +76,6 @@ async def thread(id: int) -> dict[str, str] | f95zone.IndexerError | None:
     if version:
         query = f95zone.latest_updates_search_sanitize_query(ret.name)
         for category in f95zone.LATEST_UPDATES_CATEGORIES:
-
             try:
                 async with f95zone.session.get(
                     f95zone.LATEST_UPDATES_SEARCH_URL.format(
@@ -163,13 +158,6 @@ async def thread(id: int) -> dict[str, str] | f95zone.IndexerError | None:
     else:
         reviews = await loop.run_in_executor(None, parser.reviews, res)
         if isinstance(reviews, parser.ParserError):
-
-            if reviews.message == "Thread structure missing" and req.status in (
-                403,
-                404,
-            ):
-                return f95zone.ERROR_THREAD_MISSING
-
             logger.error(
                 f"Thread {id} reviews parsing failed: {reviews.message}\n{reviews.dump}"
             )
diff --git a/indexer/watcher.py b/indexer/watcher.py
@@ -30,7 +30,6 @@ async def lifespan():
     try:
         yield
     finally:
-
         updates_task.cancel()
         versions_task.cancel()
 
@@ -179,7 +178,6 @@ async def poll_versions():
         invalidate_cache = cache.redis.pipeline()
 
         for names_chunk in chunks(names, WATCH_VERSIONS_CHUNK_SIZE):
-
             cached_data = cache.redis.pipeline()
             csv = ""
             ids = []
