switch to ids.txt and urls.txt

Author: mythz

.gitignore

@@ -5,6 +5,7 @@
 
 # Custom
 .DS_Store
+scripts/*.txt
 all-technologies.json
 chat*.json
 QueryTechnology.json

scripts/process_posts.py

@@ -16,35 +16,35 @@
 import sys
 from pathlib import Path
 
-from utils import MIN_HN_POINTS, MIN_REDDIT_POINTS, SCRIPT_DIR, PYTHON
+from utils import MIN_HN_POINTS, MIN_REDDIT_POINTS, POSTS_DIR, SCRIPT_DIR, FAILED_DIR, PYTHON, append_to_file, file_set
 
-def load_done_urls() -> set:
-    urls = set()
-    for d in ["completed", "failed"]:
-        urls_path = Path(SCRIPT_DIR) / d / "urls.txt"
-        if urls_path.exists():
-            urls.update(line.strip().rstrip('/') for line in urls_path.read_text().splitlines() if line.strip())
-    return urls
+def load_urls() -> set:
+    done = set()
+    done.update(file_set(os.path.join(SCRIPT_DIR, "urls_completed.txt")))
+    done.update(file_set(os.path.join(SCRIPT_DIR, "urls_failed.txt")))
+    return done
 
 
-def load_done() -> set:
+def load_ids() -> set:
     done = set()
-    for d in ["posts", "completed", "failed"]:
-        dir_path = Path(SCRIPT_DIR) / d
-        if dir_path.is_dir():
-            for f in dir_path.glob("*.json"):
-                done.add(f.stem)
+    done.update(file_set(os.path.join(SCRIPT_DIR, "ids_completed.txt")))
+    done.update(file_set(os.path.join(SCRIPT_DIR, "ids_failed.txt")))
     return done
 
 
 def mark_failed(post: dict, error_msg: str):
-    failed_dir = Path(SCRIPT_DIR) / "failed"
-    failed_dir.mkdir(exist_ok=True)
-    failed_path = failed_dir / f"{post['id']}.json"
+    post_id = str(post["id"])
+    append_to_file(os.path.join(SCRIPT_DIR, "ids_failed.txt"), post_id)
+    post_url = post.get("url", "")
+    if post_url:
+        append_to_file(os.path.join(SCRIPT_DIR, "urls_failed.txt"), post_url.rstrip('/'))
+
+    failed_path = os.path.join(FAILED_DIR, f"{post_id}.json")
     failed_data = {**post, "error": error_msg}
-    failed_path.write_text(json.dumps(failed_data, indent=2))
+    with open(failed_path, "w") as f:
+        json.dump(failed_data, f, indent=2)
     # Remove from posts if it exists there
-    posts_path = Path(SCRIPT_DIR) / "posts" / f"{post['id']}.json"
+    posts_path = Path(POSTS_DIR) / f"{post_id}.json"
     if posts_path.exists():
         posts_path.unlink()
 
@@ -105,8 +105,8 @@ def is_blacklisted_url(url: str) -> bool:
     def is_content_url(url: str) -> bool:
         return url.startswith("http") and not is_image_url(url) and not is_video_url(url) and not is_blacklisted_url(url)
 
-    done = load_done()
-    done_urls = load_done_urls()
+    done = load_ids()
+    done_urls = load_urls()
     to_process = [p for p in eligible if str(p["id"]) not in done and p.get("url", "").rstrip('/') not in done_urls and is_content_url(p.get("url", ""))]
     print(f"Already processed: {len(eligible) - len(to_process)}, remaining: {len(to_process)}")
 

scripts/publish_posts.py

@@ -16,11 +16,7 @@
 from pathlib import Path
 
 import requests
-from utils import TECHSTACKS_BASE, SCRIPT_DIR, create_cookie_jar
-
-POSTS_DIR = os.path.join(SCRIPT_DIR, "posts")
-COMPLETED_DIR = os.path.join(SCRIPT_DIR, "completed")
-FAILED_DIR = os.path.join(SCRIPT_DIR, "failed")
+from utils import TECHSTACKS_BASE, SCRIPT_DIR, POSTS_DIR, COMPLETED_DIR, FAILED_DIR, create_cookie_jar, append_to_file
 
 IMPORT_POST_URL = f"{TECHSTACKS_BASE}/api/ImportNewsPost"
 SYNC_POST_URL = f"{TECHSTACKS_BASE}/api/SyncStats"
@@ -41,11 +37,18 @@ def import_post(post_file):
         verify=False,
     )
 
+    post_url = post_data.get("url", "")
+
     if resp.ok:
         print(f"  Success: {resp.status_code}")
         os.makedirs(COMPLETED_DIR, exist_ok=True)
         dest = os.path.join(COMPLETED_DIR, f"{post_id}.json")
         shutil.move(post_file, dest)
+        
+        append_to_file(os.path.join(SCRIPT_DIR, "ids_completed.txt"), str(post_id))
+        if post_url:
+            append_to_file(os.path.join(SCRIPT_DIR, "urls_completed.txt"), post_url.rstrip('/'))
+
         print(f"  Moved to {dest}")
         return True
     else:
@@ -57,6 +60,11 @@ def import_post(post_file):
         with open(failed_dest, "w") as f:
             json.dump(post_data, f, indent=2)
         os.remove(post_file)
+
+        append_to_file(os.path.join(SCRIPT_DIR, "ids_failed.txt"), str(post_id))
+        if post_url:
+            append_to_file(os.path.join(SCRIPT_DIR, "urls_failed.txt"), post_url.rstrip('/'))
+
         print(f"  Moved to {failed_dest}")
         return False
 

scripts/update.sh

@@ -1,16 +1,6 @@
 #!/bin/bash
 
-echo "Updating data and indexes..."
-pushd completed
-./update.sh
-popd
-
-echo "Updating failed urls index..."
-pushd failed
-./update.sh
-popd
-
 echo "Updating technology data..."
 pushd data
 ./update.sh
-popd
+popd

scripts/utils.py

@@ -21,6 +21,10 @@
 
 PYTHON = sys.executable
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+POSTS_DIR = os.path.join(SCRIPT_DIR, "posts")
+COMPLETED_DIR = os.path.join(SCRIPT_DIR, "done", "completed")
+FAILED_DIR = os.path.join(SCRIPT_DIR, "done", "failed")
+
 REPO_ROOT = os.path.dirname(os.path.dirname(SCRIPT_DIR))  # llms repo root
 LLMS_SH = shutil.which("llms")
 LLMS_MODEL = os.getenv("LLMS_MODEL", "MiniMax-M2.1")
@@ -30,6 +34,20 @@
 if not LLMS_SH:
     raise RuntimeError("llms command not found in PATH. Please ensure llms is installed and available.")
 
+def file_set(ids_file):
+    """Return a set of post IDs from an ids_*.txt file."""
+    if os.path.exists(ids_file):
+        with open(ids_file, "r") as f:
+            return set(line.strip() for line in f)
+    return set()
+
+def append_to_file(file_path, symbol: str):
+    """Append a post ID to ids_*.txt if not already present."""
+    existing_symbols = file_set(file_path)
+    if symbol not in existing_symbols:
+        with open(file_path, "a") as f:
+            f.write(f"{symbol}\n")
+
 def create_cookie_jar():
     parsed = urlparse(TECHSTACKS_BASE)
     jar = requests.cookies.RequestsCookieJar()