Release v5.16.0 (#112)

* Fix stacktrace sniffer bucket naming

* Fix HTML report status tab navigation

* Remove unused six and tabulate dependencies

* - (fix) fixed runtime debug classification compatibility with  metadata so  no longer breaks auto-calibration or scan progress rendering.

* Restore calibrated response progress output

* Fix calibrated progress line rotation

* Improve browser calibration coverage

* Add secret response sniffer

* Add shadow copy sniffer

* Fix filtered progress line clearing before WAF warnings

* Fix filtered progress rotation after WAF warning

* Fix stacktrace false positive on notice-like 404 paths

* Fix stacktrace false positives on notice and warning-like 404 paths

* Document Mobirise fingerprint detection

* Fix graceful proxy failure handling

* Add 403 path bypass normalization variants

* Add verified open redirect sniffer

* extend gitignore

* Refactor sniffer architecture for multi-finding support

* Add staged lint and dead-code quality gates

* Apply safe Ruff cleanup rules

* added  to passively classify suspicious malware, webshell, injected script

* Clean noisy stdout summary fields

* Add QRATOR infrastructure fingerprint

* (feat) Add differential report comparison

* fix(scanner): clean WAF diagnostic logging

* fix(proxy): support authenticated HTTP CONNECT

* fix(proxy): harden rotating proxy handling

* fix(wordlist): harden extension filters

* Harden response filter handling

* Harden response filter handling

* fix(runtime): keep request provider stable for filtered directory scans

* fix(scanner): stabilize filtered scans and report e2e contract

* fix(scanner): harden proxy and transport-loss handling

* docs(update): document safe cross-platform update flow

* noisy debug output in  mode by logging proxy selection only when a new proxy pool is created

* Prepare managed scan temp workspace

* Prepare managed scan temp workspace

* Add socks5h proxy scheme support without changing proxy debug output

* test: cover output and calibration edge branches

* Hardened WordPress fingerprinting by adding static asset probes and preventing weak login/xmlrpc-only evidence from becoming a primary CMS match

* add legacy tls compatibility

* fix stacktrace false positives on localized 404 pages

* Update readme

* Update readme

* Update readme

* Update readme

* Fix security scanner findings

* Disable noisy CodeQL cleartext logging rule

Author: stanislav-web

.dockerignore

@@ -44,6 +44,7 @@ reports
 syslog
 site
 *.log
+debug.log
 
 # Local binaries / keys
 opendoor
@@ -66,11 +67,11 @@ benchmarks
 docs
 tests
 e2e-server.log
-data/test.dat
 
 # Local helper scripts are not needed in runtime image
 scripts
 release.sh
+debug.sh
 
 # Debian/Kali packaging build artifacts
 .pybuild
@@ -91,4 +92,5 @@ debian/*.substvars
 *.tar.xz
 
 # Misc
-TODO
+TODO
+ROADMAP.md

.github/codeql/codeql-config.yml

@@ -5,4 +5,6 @@ paths-ignore:
 
 query-filters:
   - exclude:
-      id: py/incomplete-url-substring-sanitization
+      id: py/incomplete-url-substring-sanitization
+  - exclude:
+      id: py/clear-text-logging-sensitive-data

.github/e2e/server.py

@@ -52,4 +52,4 @@ def log_message(self, fmt: str, *args: object) -> None:
 if __name__ == "__main__":
     server = HTTPServer(("127.0.0.1", 8088), Handler)
     print("E2E server listening on http://127.0.0.1:8088", flush=True)
-    server.serve_forever()
+    server.serve_forever()

.github/e2e/validate.py

@@ -1,5 +1,5 @@
 """
-Validate OpenDoor GitHub Actions E2E reports against v5.15.2 report shape.
+Validate OpenDoor GitHub Actions E2E reports against current filtered report shape.
 """
 
 import json
@@ -18,7 +18,7 @@
     "/login",
 }
 
-EXPECTED_IGNORED_404_PATHS = {
+EXPECTED_FILTERED_404_PATHS = {
     "/nonexistent",
     "/ghost",
     "/random-miss",
@@ -62,22 +62,6 @@ def item_urls(report: dict, bucket: str) -> list[str]:
     ]
 
 
-def item_details(report: dict, bucket: str) -> list[dict]:
-    details = report.get("report_items", {}).get(bucket)
-
-    if isinstance(details, list):
-        return [
-            item
-            for item in details
-            if isinstance(item, dict)
-        ]
-
-    return [
-        {"url": str(item), "code": "-"}
-        for item in report.get("items", {}).get(bucket, [])
-    ]
-
-
 def has_path(urls: list[str], path: str) -> bool:
     return any(url.endswith(path) or path in url for url in urls)
 
@@ -90,7 +74,10 @@ def validate_json_report() -> None:
     assert_true(total.get("forbidden") == 1, "JSON: forbidden bucket has exactly 1 hit")
     assert_true(total.get("auth") == 1, "JSON: auth bucket has exactly 1 hit")
     assert_true(total.get("redirect") == 1, "JSON: redirect bucket has exactly 1 hit")
-    assert_true(total.get("ignored") == 4, "JSON: ignored bucket has exactly 4 filtered misses")
+    assert_true(
+        total.get("ignored") in (None, 0),
+        "JSON: ignored bucket is absent or empty for filtered misses",
+    )
 
     success_urls = item_urls(report, "success")
 
@@ -107,16 +94,12 @@ def validate_json_report() -> None:
         "JSON: /auth-required is in auth bucket",
     )
 
-    ignored_items = item_details(report, "ignored")
+    ignored_urls = item_urls(report, "ignored")
 
-    for path in sorted(EXPECTED_IGNORED_404_PATHS):
+    for path in sorted(EXPECTED_FILTERED_404_PATHS):
         assert_true(
-            any(
-                has_path([str(item.get("url", ""))], path)
-                and str(item.get("code")) == "404"
-                for item in ignored_items
-            ),
-            f"JSON: {path} is preserved as ignored 404",
+            not has_path(ignored_urls, path),
+            f"JSON: {path} is not preserved as an ignored report item",
         )
 
     active_buckets = ("success", "forbidden", "auth", "redirect")
@@ -126,7 +109,7 @@ def validate_json_report() -> None:
         for url in item_urls(report, bucket)
     ]
 
-    for path in sorted(EXPECTED_IGNORED_404_PATHS):
+    for path in sorted(EXPECTED_FILTERED_404_PATHS):
         assert_true(
             not has_path(active_urls, path),
             f"JSON: {path} is not in active finding buckets",
@@ -190,10 +173,15 @@ def result_matches(rule_id: str, path: str | None = None, code: int | None = Non
         "SARIF: redirect bucket has status 301",
     )
 
-    for path in sorted(EXPECTED_IGNORED_404_PATHS):
+    assert_true(
+        not result_matches("opendoor.finding.ignored"),
+        "SARIF: ignored filtered misses are not reported as findings",
+    )
+
+    for path in sorted(EXPECTED_FILTERED_404_PATHS):
         assert_true(
-            result_matches("opendoor.finding.ignored", path, 404),
-            f"SARIF: {path} is ignored/404",
+            not result_matches("opendoor.finding.ignored", path, 404),
+            f"SARIF: {path} is not reported as ignored/404",
         )
 
 
@@ -210,4 +198,4 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    raise SystemExit(main())
+    raise SystemExit(main())

.github/workflows/lint.yml

@@ -0,0 +1,79 @@
+name: Lint
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: lint-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  ruff:
+    name: Ruff
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    env:
+      PYTHONUTF8: "1"
+      PYTHONIOENCODING: utf-8
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: |
+            requirements.txt
+            requirements-dev.txt
+            pyproject.toml
+
+      - name: Install dev dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements-dev.txt
+
+      - name: Run Ruff
+        run: python -m ruff check .
+  vulture:
+    name: Vulture advisory
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    continue-on-error: true
+
+    env:
+      PYTHONUTF8: "1"
+      PYTHONIOENCODING: utf-8
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+          cache: pip
+          cache-dependency-path: |
+            requirements.txt
+            requirements-dev.txt
+            pyproject.toml
+
+      - name: Install dev dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements-dev.txt
+
+      - name: Run Vulture advisory scan
+        run: |
+          vulture src tests --min-confidence 65

.gitignore

@@ -39,8 +39,9 @@ reports/
 scripts/
 syslog/
 site/
-data/test.dat
 release.sh
+debug.sh
+debug.log
 
 # Local binaries / keys
 opendoor
@@ -57,6 +58,8 @@ results.sarif
 
 # Misc
 TODO
+ROADMAP.md
+
 # Local transport profiles may contain secrets.
 data/wireguard-profiles/*.conf
 !data/wireguard-profiles/example.conf
@@ -86,3 +89,4 @@ debian/*.substvars
 *.changes
 *.dsc
 *.tar.xz
+/debug.sh