Even more reliable latest updates search + BUILD

WillyJL · WillyJL · commit 033ef53095e1 · 2025-08-01T00:44:12.000+02:00
diff --git a/indexer/f95zone.py b/indexer/f95zone.py
@@ -4,6 +4,7 @@
 import datetime as dt
 import logging
 import os
+import re
 import sys
 
 import aiohttp
@@ -47,52 +48,17 @@
 
 HOST = "https://f95zone.to"
 THREAD_URL = f"{HOST}/threads/{{thread}}"
-VERCHK_URL = f"{HOST}/sam/checker.php?threads={{threads}}"
-SEARCH_URL = f"{HOST}/sam/latest_alpha/latest_data.php?cmd={{cmd}}&cat={{cat}}&page={{page}}&{{search}}={{query}}&sort={{sort}}&rows={{rows}}&_={{ts}}"
-LATEST_URL = f"{HOST}/sam/latest_alpha/latest_data.php?cmd={{cmd}}&cat={{cat}}&page={{page}}&sort={{sort}}&rows={{rows}}&_={{ts}}"
-LATEST_CATEGORIES = (
+BULK_VERSION_CHECK_URL = f"{HOST}/sam/checker.php?threads={{threads}}"
+LATEST_UPDATES_SEARCH_URL = f"{HOST}/sam/latest_alpha/latest_data.php?cmd={{cmd}}&cat={{cat}}&page={{page}}&{{search}}={{query}}&sort={{sort}}&rows={{rows}}&_={{ts}}"
+LATEST_UPDATES_URL = f"{HOST}/sam/latest_alpha/latest_data.php?cmd={{cmd}}&cat={{cat}}&page={{page}}&sort={{sort}}&rows={{rows}}&_={{ts}}"
+LATEST_UPDATES_CATEGORIES = (
     "games",
     "comics",
     "animations",
     "assets",
     # Doesn't seem to work
     # "mods",
 )
-LATEST_STOPWORDS = (
-    "a",
-    "is",
-    "the",
-    "an",
-    "and",
-    "are",
-    "as",
-    "at",
-    "be",
-    "but",
-    "by",
-    "for",
-    "if",
-    "in",
-    "into",
-    "it",
-    "no",
-    "not",
-    "of",
-    "on",
-    "or",
-    "such",
-    "that",
-    "their",
-    "then",
-    "there",
-    "these",
-    "they",
-    "this",
-    "to",
-    "was",
-    "will",
-    "with",
-)
 
 
 @dataclasses.dataclass(slots=True)
@@ -181,3 +147,61 @@ def check_error(
         if isinstance(res, (asyncio.TimeoutError, aiohttp.ClientConnectionError)):
             logger.warning("F95zone temporarily unreachable")
             return ERROR_F95ZONE_UNAVAILABLE
+
+
+def latest_updates_search_sanitize_query(query: str):
+    redis_stopwords = (
+        "a",
+        "is",
+        "the",
+        "an",
+        "and",
+        "are",
+        "as",
+        "at",
+        "be",
+        "but",
+        "by",
+        "for",
+        "if",
+        "in",
+        "into",
+        "it",
+        "no",
+        "not",
+        "of",
+        "on",
+        "or",
+        "such",
+        "that",
+        "their",
+        "then",
+        "there",
+        "these",
+        "they",
+        "this",
+        "to",
+        "was",
+        "will",
+        "with",
+    )
+    query = query.encode("ascii", errors="replace").decode()
+    query = re.sub(r"\.+ | \.+", " ", query)
+    for char in "?&/':;-.+!~()":
+        query = query.replace(char, " ")
+    query = re.sub(r"\s+", " ", query).strip()
+    words = query.split(" ")
+    for stopword in redis_stopwords:
+        for word in words.copy():
+            if word.lower() == stopword:
+                words.remove(word)
+    query = ""
+    while words:
+        append = f"{' ' if query else ''}{words.pop(0)}"
+        if len(query + append) > 30:
+            append = append[: 30 - len(query)]
+            if len(append) > 3 and append.strip().lower() not in redis_stopwords:
+                query += append
+            break
+        query += append
+    return query
diff --git a/indexer/scraper.py b/indexer/scraper.py
@@ -2,7 +2,6 @@
 import dataclasses
 import json
 import logging
-import re
 import time
 
 from common import parser
@@ -53,7 +52,7 @@ async def thread(id: int) -> dict[str, str] | f95zone.IndexerError | None:
     version = ""
     try:
         async with f95zone.session.get(
-            f95zone.VERCHK_URL.format(threads=id),
+            f95zone.BULK_VERSION_CHECK_URL.format(threads=id),
         ) as req:
             res = await req.read()
     except Exception as exc:
@@ -79,22 +78,12 @@ async def thread(id: int) -> dict[str, str] | f95zone.IndexerError | None:
 
     # If tracked by latest updates, try to search the thread there to get more precise details
     if version:
-        query = ret.name.encode("ascii", errors="replace").decode()
-        query = re.sub(r"\.+ | \.+", " ", query)
-        for char in "?&/':;-":
-            query = query.replace(char, " ")
-        query = re.sub(r"\s+", " ", query).strip()[:30]
-        words = query.split(" ")
-        for stopword in f95zone.LATEST_STOPWORDS:
-            for word in words.copy():
-                if word.lower() == stopword:
-                    words.remove(word)
-        query = " ".join(words)
-        for category in f95zone.LATEST_CATEGORIES:
+        query = f95zone.latest_updates_search_sanitize_query(ret.name)
+        for category in f95zone.LATEST_UPDATES_CATEGORIES:
 
             try:
                 async with f95zone.session.get(
-                    f95zone.SEARCH_URL.format(
+                    f95zone.LATEST_UPDATES_SEARCH_URL.format(
                         cmd="list",
                         cat=category,
                         page=1,
diff --git a/indexer/watcher.py b/indexer/watcher.py
@@ -14,7 +14,7 @@
 )
 
 WATCH_UPDATES_INTERVAL = dt.timedelta(minutes=5).total_seconds()
-WATCH_UPDATES_CATEGORIES = f95zone.LATEST_CATEGORIES
+WATCH_UPDATES_CATEGORIES = f95zone.LATEST_UPDATES_CATEGORIES
 WATCH_UPDATES_PAGES = 4
 WATCH_VERSIONS_INTERVAL = dt.timedelta(hours=12).total_seconds()
 WATCH_VERSIONS_CHUNK_SIZE = 1000
@@ -55,7 +55,7 @@ async def poll_updates():
 
                 try:
                     async with f95zone.session.get(
-                        f95zone.LATEST_URL.format(
+                        f95zone.LATEST_UPDATES_URL.format(
                             cmd="list",
                             cat=category,
                             page=page,
@@ -77,9 +77,7 @@ async def poll_updates():
                 try:
                     updates = json.loads(res)
                 except Exception:
-                    raise Exception(
-                        f"Latest updates returned invalid JSON: {res}"
-                    )
+                    raise Exception(f"Latest updates returned invalid JSON: {res}")
                 if index_error := f95zone.check_error(updates, logger):
                     raise Exception(index_error)
 
@@ -162,9 +160,7 @@ async def poll_updates():
             # )
             # continue
         else:
-            logger.error(
-                f"Error polling updates: {error.text()}\n{error.traceback()}"
-            )
+            logger.error(f"Error polling updates: {error.text()}\n{error.traceback()}")
 
 
 async def watch_updates():
@@ -179,9 +175,7 @@ async def poll_versions():
     try:
         logger.info("Poll versions start")
 
-        names = [
-            n async for n in cache.redis.scan_iter("thread:*", 10000, "hash")
-        ]
+        names = [n async for n in cache.redis.scan_iter("thread:*", 10000, "hash")]
         invalidate_cache = cache.redis.pipeline()
 
         for names_chunk in chunks(names, WATCH_VERSIONS_CHUNK_SIZE):
@@ -198,7 +192,7 @@ async def poll_versions():
 
             try:
                 async with f95zone.session.get(
-                    f95zone.VERCHK_URL.format(threads=csv),
+                    f95zone.BULK_VERSION_CHECK_URL.format(threads=csv),
                 ) as req:
                     # Await together for efficiency
                     res, cached_data = await asyncio.gather(
@@ -246,9 +240,7 @@ async def poll_versions():
         if len(invalidate_cache):
             result = await invalidate_cache.execute()
             invalidated = sum(ret != "0" for ret in result)
-            logger.warning(
-                f"Versions: Invalidated cache for {invalidated} threads"
-            )
+            logger.warning(f"Versions: Invalidated cache for {invalidated} threads")
 
         logger.info("Poll versions done")
 
@@ -269,9 +261,7 @@ async def poll_versions():
             # )
             # continue
         else:
-            logger.error(
-                f"Error polling versions: {error.text()}\n{error.traceback()}"
-            )
+            logger.error(f"Error polling versions: {error.text()}\n{error.traceback()}")
 
 
 async def watch_versions():
diff --git a/modules/api.py b/modules/api.py
@@ -450,53 +450,61 @@ def cleanup_temp_files():
             pass
 
 
-latest_updates_search_redis_stopwords = [
-    "a",
-    "is",
-    "the",
-    "an",
-    "and",
-    "are",
-    "as",
-    "at",
-    "be",
-    "but",
-    "by",
-    "for",
-    "if",
-    "in",
-    "into",
-    "it",
-    "no",
-    "not",
-    "of",
-    "on",
-    "or",
-    "such",
-    "that",
-    "their",
-    "then",
-    "there",
-    "these",
-    "they",
-    "this",
-    "to",
-    "was",
-    "will",
-    "with",
-]
 def latest_updates_search_sanitize_query(query: str):
+    redis_stopwords = (
+        "a",
+        "is",
+        "the",
+        "an",
+        "and",
+        "are",
+        "as",
+        "at",
+        "be",
+        "but",
+        "by",
+        "for",
+        "if",
+        "in",
+        "into",
+        "it",
+        "no",
+        "not",
+        "of",
+        "on",
+        "or",
+        "such",
+        "that",
+        "their",
+        "then",
+        "there",
+        "these",
+        "they",
+        "this",
+        "to",
+        "was",
+        "will",
+        "with",
+    )
     query = query.encode("ascii", errors="replace").decode()
     query = re.sub(r"\.+ | \.+", " ", query)
-    for char in "?&/':;-.":
+    for char in "?&/':;-.+!~()":
         query = query.replace(char, " ")
-    query = re.sub(r"\s+", " ", query).strip()[:30]
+    query = re.sub(r"\s+", " ", query).strip()
     words = query.split(" ")
-    for stopword in latest_updates_search_redis_stopwords:
+    for stopword in redis_stopwords:
         for word in words.copy():
             if word.lower() == stopword:
                 words.remove(word)
-    query = " ".join(words)
+    query = ""
+    while words:
+        append = f"{' ' if query else ''}{words.pop(0)}"
+        if len(query + append) > 30:
+            append = append[: 30 - len(query)]
+            if len(append) > 3 and append.strip().lower() not in redis_stopwords:
+                query += append
+            break
+        query += append
     return query
 
 
