Merge pull request sherlock-project#2898 from sherlock-project/improvements

Make Minor Improvements

Author: sdushantha

docs/README.md

@@ -33,7 +33,7 @@
 
 Community-maintained packages are available for Debian (>= 13), Ubuntu (>= 22.10), Homebrew, Kali, and BlackArch. These packages are not directly supported or maintained by the Sherlock Project.
 
-See all alternative installation methods [here](https://sherlockproject.xyz/installation)
+See all alternative installation methods [here](https://sherlockproject.xyz/installation).
 
 ## General usage
 
@@ -51,51 +51,41 @@ Accounts found will be stored in an individual text file with the corresponding
 
 ```console
 $ sherlock --help
-usage: sherlock [-h] [--version] [--verbose] [--folderoutput FOLDEROUTPUT]
-                [--output OUTPUT] [--tor] [--unique-tor] [--csv] [--xlsx]
-                [--site SITE_NAME] [--proxy PROXY_URL] [--json JSON_FILE]
-                [--timeout TIMEOUT] [--print-all] [--print-found] [--no-color]
-                [--browse] [--local] [--nsfw]
+usage: sherlock [-h] [--version] [--verbose] [--folderoutput FOLDEROUTPUT] [--output OUTPUT] [--csv] [--xlsx] [--site SITE_NAME] [--proxy PROXY_URL] [--dump-response]
+                [--json JSON_FILE] [--timeout TIMEOUT] [--print-all] [--print-found] [--no-color] [--browse] [--local] [--nsfw] [--txt] [--ignore-exclusions]
                 USERNAMES [USERNAMES ...]
 
-Sherlock: Find Usernames Across Social Networks (Version 0.14.3)
+Sherlock: Find Usernames Across Social Networks (Version 0.16.0)
 
 positional arguments:
-  USERNAMES             One or more usernames to check with social networks.
-                        Check similar usernames using {?} (replace to '_', '-', '.').
+  USERNAMES             One or more usernames to check with social networks. Check similar usernames using {?} (replace to '_', '-', '.').
 
-optional arguments:
+options:
   -h, --help            show this help message and exit
   --version             Display version information and dependencies.
   --verbose, -v, -d, --debug
                         Display extra debugging information and metrics.
   --folderoutput FOLDEROUTPUT, -fo FOLDEROUTPUT
-                        If using multiple usernames, the output of the results will be
-                        saved to this folder.
+                        If using multiple usernames, the output of the results will be saved to this folder.
   --output OUTPUT, -o OUTPUT
-                        If using single username, the output of the result will be saved
-                        to this file.
-  --tor, -t             Make requests over Tor; increases runtime; requires Tor to be
-                        installed and in system path.
-  --unique-tor, -u      Make requests over Tor with new Tor circuit after each request;
-                        increases runtime; requires Tor to be installed and in system
-                        path.
+                        If using single username, the output of the result will be saved to this file.
   --csv                 Create Comma-Separated Values (CSV) File.
-  --xlsx                Create the standard file for the modern Microsoft Excel
-                        spreadsheet (xlsx).
-  --site SITE_NAME      Limit analysis to just the listed sites. Add multiple options to
-                        specify more than one site.
+  --xlsx                Create the standard file for the modern Microsoft Excel spreadsheet (xlsx).
+  --site SITE_NAME      Limit analysis to just the listed sites. Add multiple options to specify more than one site.
   --proxy PROXY_URL, -p PROXY_URL
                         Make requests over a proxy. e.g. socks5://127.0.0.1:1080
+  --dump-response       Dump the HTTP response to stdout for targeted debugging.
   --json JSON_FILE, -j JSON_FILE
-                        Load data from a JSON file or an online, valid, JSON file.
+                        Load data from a JSON file or an online, valid, JSON file. Upstream PR numbers also accepted.
   --timeout TIMEOUT     Time (in seconds) to wait for response to requests (Default: 60)
   --print-all           Output sites where the username was not found.
-  --print-found         Output sites where the username was found.
+  --print-found         Output sites where the username was found (also if exported as file).
   --no-color            Don't color terminal output
   --browse, -b          Browse to all results on default browser.
   --local, -l           Force the use of the local data.json file.
   --nsfw                Include checking of NSFW sites from default list.
+  --txt                 Enable creation of a txt file
+  --ignore-exclusions   Ignore upstream exclusions (may return more false positives)
 ```
 
 ## Credits

sherlock_project/notify.py

@@ -37,7 +37,6 @@ def __init__(self, result=None):
 
         self.result = result
 
-        # return
 
     def start(self, message=None):
         """Notify Start.
@@ -56,7 +55,6 @@ def start(self, message=None):
         Nothing.
         """
 
-        # return
 
     def update(self, result):
         """Notify Update.
@@ -75,7 +73,6 @@ def update(self, result):
 
         self.result = result
 
-        # return
 
     def finish(self, message=None):
         """Notify Finish.
@@ -94,7 +91,6 @@ def finish(self, message=None):
         Nothing.
         """
 
-        # return
 
     def __str__(self):
         """Convert Object To String.
@@ -137,7 +133,6 @@ def __init__(self, result=None, verbose=False, print_all=False, browse=False):
         self.print_all = print_all
         self.browse = browse
 
-        return
 
     def start(self, message):
         """Notify Start.
@@ -163,7 +158,6 @@ def start(self, message):
         # An empty line between first line and the result(more clear output)
         print('\r')
 
-        return
 
     def countResults(self):
         """This function counts the number of results. Every time the function is called,
@@ -238,7 +232,7 @@ def update(self, result):
                       Fore.WHITE + "]" +
                       Fore.GREEN + f" {self.result.site_name}:" +
                       Fore.YELLOW + f" {msg}")
-                
+
         elif result.status == QueryStatus.WAF:
             if self.print_all:
                 print(Style.BRIGHT + Fore.WHITE + "[" +
@@ -254,10 +248,9 @@ def update(self, result):
                 f"Unknown Query Status '{result.status}' for site '{self.result.site_name}'"
             )
 
-        return
 
     def finish(self, message="The processing has been finished."):
-        """Notify Start.
+        """Notify Finish.
         Will print the last line to the standard output.
         Keyword Arguments:
         self                   -- This object.

sherlock_project/sherlock.py

@@ -675,16 +675,6 @@ def main():
         help="Include checking of NSFW sites from default list.",
     )
 
-    # TODO deprecated in favor of --txt, retained for workflow compatibility, to be removed
-    # in future release
-    parser.add_argument(
-        "--no-txt",
-        action="store_true",
-        dest="no_txt",
-        default=False,
-        help="Disable creation of a txt file - WILL BE DEPRECATED",
-    )
-
     parser.add_argument(
         "--txt",
         action="store_true",

sherlock_project/sites.py

@@ -8,7 +8,7 @@
 import secrets
 
 
-MANIFEST_URL = "https://raw.githubusercontent.com/sherlock-project/sherlock/master/sherlock_project/resources/data.json"
+MANIFEST_URL = "https://data.sherlockproject.xyz"
 EXCLUSIONS_URL = "https://raw.githubusercontent.com/sherlock-project/sherlock/refs/heads/exclusions/false_positive_exclusions.txt"
 
 class SiteInformation:
@@ -121,11 +121,6 @@ def __init__(
             # users from creating issue about false positives which has already been fixed or having outdated data
             data_file_path = MANIFEST_URL
 
-        # Ensure that specified data file has correct extension.
-        if not data_file_path.lower().endswith(".json"):
-            raise FileNotFoundError(f"Incorrect JSON file extension for data file '{data_file_path}'.")
-
-        # if "http://"  == data_file_path[:7].lower() or "https://" == data_file_path[:8].lower():
         if data_file_path.lower().startswith("http"):
             # Reference is to a URL.
             try: