Merge pull request #220 from kaifcodec/feat/speed-up

feat: speed up username scans by removing category-wise scan waits

Author: kaifcodec

user_scanner/core/orchestrator.py

@@ -48,7 +48,7 @@ def run_user_category(category_path: Path, username: str, show_url: bool = False
     results = []
     modules = load_modules(category_path)
 
-    with ThreadPoolExecutor(max_workers=10) as executor:
+    with ThreadPoolExecutor(max_workers=20) as executor:
         exec_map = executor.map(lambda m: _worker_single(m, username), modules)
         for result in exec_map:
             result.update(category=category_name)
@@ -60,11 +60,32 @@ def run_user_category(category_path: Path, username: str, show_url: bool = False
 
 def run_user_full(username: str, show_url: bool = False) -> List[Result]:
     results = []
+    all_modules = []
+    categories = list(load_categories().items())
+    module_to_cat = {}
+    printed_categories = set()
+
+    for cat_name, cat_path in categories:
+        modules = load_modules(cat_path)
+        display_name = cat_name.capitalize()
+        for m in modules:
+            all_modules.append(m)
+            module_to_cat[get_site_name(m)] = display_name
+
+    with ThreadPoolExecutor(max_workers=60) as executor:
+        exec_map = executor.map(
+            lambda m: _worker_single(m, username), all_modules)
+        for result in exec_map:
+            site_name = result.site_name
+            cat_name = module_to_cat.get(site_name, "Unknown") if site_name else "Unknown"
+
+            if cat_name not in printed_categories:
+                print(f"\n{Fore.MAGENTA}== {cat_name} SITES =={Style.RESET_ALL}")
+                printed_categories.add(cat_name)
 
-    categories = list(load_categories().values())
-    for category_path in categories:
-        temp = run_user_category(category_path, username, show_url=show_url)
-        results.extend(temp)
+            result.update(category=cat_name)
+            results.append(result)
+            result.show(show_url=show_url)
 
     return results
 

user_scanner/user_scan/gaming/battlenet.py

@@ -56,7 +56,7 @@ def process(response):
             return Result.error("Failed to parse response")
 
     return generic_validate(
-        url, process, show_url=show_url, headers=headers, timeout=15.0, follow_redirects=True
+        url, process, show_url=show_url, headers=headers, timeout=6.0, follow_redirects=True
     )
 
 

user_scanner/user_scan/social/tiktok.py

@@ -37,7 +37,7 @@ def process(response) -> Result:
                 return Result.taken()
         return Result.error("Unable to load tiktok")
 
-    return generic_validate(url, process, show_url=show_url, headers=headers)
+    return generic_validate(url, process, show_url=show_url, headers=headers, timeout=4.0)
 
 
 if __name__ == "__main__":
@@ -49,4 +49,4 @@ def process(response) -> Result:
     elif result == 0:
         print("Unavailable!")
     else:
-        print(f"Error occurred! Reason: {result.get_reason()}")
+        print(f"Error occurred! Reason: {result.get_reason()}")

user_scanner/user_scan/social/youtube.py

@@ -35,7 +35,7 @@ def validate_youtube(user) -> Result:
     }
 
 
-    return status_validate(url, 404, 200, show_url=show_url, headers=headers)
+    return status_validate(url, 404, 200, show_url=show_url, headers=headers, follow_redirects=True)
 
 
 if __name__ == "__main__":