path_crawler.py

import requests
import urllib3
import logging
from concurrent.futures import ThreadPoolExecutor

urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
#logging.getLogger("urllib3").setLevel(logging.CRITICAL)

def getwordlist(input_file="paths.txt"):
    wordlist = []
    with open(input_file, 'r') as f:
        for line in f:
            wordlist.append(line.strip())
    return wordlist


#could use another function and parse http and https
def check_path(base_url, i, path):
    false_pos_lengths = ()
    found_paths = {}
    i= i+1
    
    try:
        url = f"http://{base_url}/{path}"
        print(f"requesting {url}: line {i}")
        r = requests.get(url, timeout=1, verify=False)
        if r.status_code != 404 and len(r.content) not in false_pos_lengths:
            if not false_pos_lengths:
                false_pos_lengths = (0, len(requests.get(f"https://{base_url}/GARBAGEEEMAN", timeout=3, verify=False).content))
                print(f"added {len(r.content)} to false positives")
            print(f"[{r.status_code}] {url} (length: {len(r.content)})")
            found_paths[url] = {"status_code": r.status_code, "content_len": len(r.content)}

    except requests.RequestException:
        pass

    try:
        url = f"https://{base_url}/{path}"
        print(f"requesting {url}")
        r = requests.get(url, timeout=1, verify=False)
        if r.status_code != 404 and len(r.content) not in false_pos_lengths:
            if not false_pos_lengths:
                false_pos_lengths = (0, len(requests.get(f"https://{base_url}/GARBAGEEEMAN", timeout=3, verify=False).content))
                print(f"added {len(r.content)} to false positives")
            print(f"[{r.status_code}] {url} (length: {len(requests.get(f"https://{base_url}/GARBAGEEEMAN", timeout=3).content)})")
            found_paths[url] = {"status_code": r.status_code, "content_len": len(r.content)}

    except requests.RequestException as r:
        print(r)
        
    return found_paths or None

    
def scan(base_url, threads=20):
    i = 0
    wordlist = getwordlist()
    found_paths = {}
    with ThreadPoolExecutor(max_workers=threads) as executor:
        results = executor.map(lambda w: check_path(base_url,i, w), wordlist)
        for path in results:
            if path:
                found_paths.update(path)
        print(f"SCAN FOR {base_url} ENDED--------------------------")
        return found_paths if found_paths else None