fix(security): honor robots.txt for all discovery sources; cap HATEOAS request budget

- robots.txt now applies to endpoints sourced from an OpenAPI/Swagger spec too.
  Phase 1 records spec endpoints unchecked, and phases 4 (method testing, incl.
  POST/PUT/DELETE under --test-all-methods) and 5 (schema extraction) previously
  probed them with no robots.txt check, bypassing the tool's ethical contract.
- Response-driven (HATEOAS) discovery now respects the max_requests budget; it
  had no such parameter and followed unbounded links per round, exceeding the
  configured request limit that the other strategies honor.
- tests: TestResponseDrivenBudget + TestRobotsAppliesToAllSources; both verified
  red against the unfixed code (no false pass). Suite 19 -> 21 green.
- chore: gitignore LOCK*.txt.

Reviewed by a fresh reviewer subagent; Fable 5 was flagged on this security tool
so Opus 4.8 took over (module-review loop run 9).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011c2rFLwVHQVshAQJVNzWDS

Author: Lukas Geiger

.gitignore

@@ -44,3 +44,6 @@ config.local.json
 .DS_Store
 Thumbs.db
 desktop.ini
+
+# Multi-Agent Lock-System (lokal, nie committen)
+LOCK*.txt

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## 2026-07-04
+
+- Security/ethics: robots.txt is now honored for endpoints that came from an
+  OpenAPI/Swagger spec, too. Previously such endpoints were probed in the
+  method-testing and schema-extraction phases (including POST/PUT/DELETE with
+  `--test-all-methods`) without any robots.txt check.
+- Rate limiting: the response-driven (HATEOAS) discovery phase now respects the
+  `max_requests` budget. Previously it followed unbounded links per round and
+  could exceed the configured request limit.
+- Added regression tests for both (`test_smoke.py`): request-budget cap in
+  response-driven discovery and robots.txt enforcement across discovery sources.
+
 ## 2026-06-12
 
 - Updated the smoke-test workflow to `actions/checkout@v6` and `actions/setup-python@v6`.

discovery/orchestrator.py

@@ -180,6 +180,11 @@ def on_endpoint_found(path, resp):
             for ep in endpoints:
                 if self._check_limits(max_requests):
                     break
+                # robots.txt gilt auch fuer Endpoints aus einer OpenAPI-Spec
+                # (Phase 1 traegt sie ungeprueft ein) -- sonst wuerden hier
+                # sogar POST/PUT/DELETE an gesperrte Pfade gehen
+                if robots and not robots.is_allowed(ep["path"]):
+                    continue
                 method_info = test_methods(
                     self.client, base_url, ep["path"],
                     skip_destructive=skip_destructive
@@ -207,6 +212,8 @@ def on_endpoint_found(path, resp):
                 methods = json.loads(ep.get("methods_json", "[]"))
                 if "GET" not in methods:
                     continue
+                if robots and not robots.is_allowed(ep["path"]):
+                    continue
                 url = f"{base_url}{ep['path']}"
                 resp = self.client.get(url)
                 if resp.ok and resp.body:
@@ -238,7 +245,7 @@ def on_endpoint_found(path, resp):
                 self.client, base_url, self.db, service_id,
                 robots_checker=robots, known_paths=known_paths,
                 max_depth=self.config.get("max_depth", 2),
-                callback=on_endpoint_found
+                callback=on_endpoint_found, max_requests=max_requests
             )
             self._process_results(service_id, results, "response_driven")
             print(f"  {len(results)} neue Endpoints entdeckt")

discovery/response_driven.py

@@ -9,7 +9,7 @@
 
 def discover_from_responses(client, base_url, db, service_id,
                             robots_checker=None, known_paths=None,
-                            max_depth=2, callback=None):
+                            max_depth=2, callback=None, max_requests=None):
     """Entdeckt neue Endpoints durch Link-Following.
 
     Liest bestehende Responses aus der DB, extrahiert Links,
@@ -24,6 +24,9 @@ def discover_from_responses(client, base_url, db, service_id,
         known_paths: Set bekannter Pfade
         max_depth: Maximale Rekursionstiefe
         callback: Funktion(path, response)
+        max_requests: Maximale Gesamtzahl Requests (Client-Zaehler); die
+            HATEOAS-Runden folgen sonst unbegrenzt vielen Links und
+            sprengen das Budget der anderen Strategien.
 
     Returns:
         list: [(path, response), ...]
@@ -33,6 +36,8 @@ def discover_from_responses(client, base_url, db, service_id,
     all_results = []
 
     for depth in range(max_depth):
+        if max_requests and client.request_count >= max_requests:
+            break
         new_links = set()
 
         # Bekannte Endpoints durchgehen und Links aus deren Responses sammeln
@@ -60,6 +65,8 @@ def discover_from_responses(client, base_url, db, service_id,
         # Neue Links testen
         round_results = []
         for path in sorted(new_links):
+            if max_requests and client.request_count >= max_requests:
+                break
             if robots_checker and not robots_checker.is_allowed(path):
                 continue
 

test_smoke.py

@@ -352,6 +352,146 @@ def test_404_means_everything_allowed(self, monkeypatch):
         assert checker.is_allowed("/api") is True
 
 
+class TestResponseDrivenBudget:
+    """Regressionstest: HATEOAS-Discovery (Phase 6) respektiert max_requests.
+
+    Vorher hatte discover_from_responses keinen max_requests-Parameter und
+    folgte pro Runde beliebig vielen Links -- das Budget der anderen
+    Strategien wurde umgangen.
+    """
+
+    def _ensure_package_importable(self):
+        parent_dir = str(API_PROBER_DIR.parent)
+        if parent_dir not in sys.path:
+            sys.path.insert(0, parent_dir)
+
+    def test_stops_at_max_requests(self, monkeypatch):
+        self._ensure_package_importable()
+        from ApiProber.discovery import response_driven
+
+        # Jede Response liefert viele frische Links
+        monkeypatch.setattr(
+            response_driven, "extract_links_from_json",
+            lambda data, base: [f"/item/{i}" for i in range(50)]
+        )
+
+        class FakeResp:
+            status_code = 200
+
+        class FakeClient:
+            def __init__(self):
+                self.request_count = 0
+
+            def get(self, url):
+                self.request_count += 1
+                return FakeResp()
+
+        class FakeDB:
+            def get_endpoints(self, sid):
+                return [{"id": 1, "path": "/seed"}]
+
+            def get_responses(self, eid):
+                return [{"body_sample": '{"any": "thing"}'}]
+
+        client = FakeClient()
+        response_driven.discover_from_responses(
+            client, "https://example.invalid", FakeDB(), 1,
+            robots_checker=None, known_paths=set(),
+            max_depth=5, callback=None, max_requests=3
+        )
+        assert client.request_count <= 3, \
+            f"HATEOAS-Discovery sprengte das Budget: {client.request_count} > 3"
+
+
+class TestRobotsAppliesToAllSources:
+    """Regressionstest: robots.txt gilt auch fuer Endpoints aus einer
+    OpenAPI-Spec.
+
+    Vorher trug Phase 1 Spec-Endpoints ungeprueft ein; Phase 4 (Method-
+    Testing, inkl. POST/PUT/DELETE) und Phase 5 (Schema-Extraktion) fragten
+    sie dann ohne robots.txt-Pruefung an.
+    """
+
+    def _ensure_package_importable(self):
+        parent_dir = str(API_PROBER_DIR.parent)
+        if parent_dir not in sys.path:
+            sys.path.insert(0, parent_dir)
+
+    def test_phase4_5_skip_robots_disallowed_paths(self, tmp_path, monkeypatch):
+        self._ensure_package_importable()
+        from copy import deepcopy
+        from ApiProber.core.config import DEFAULT_CONFIG
+        from ApiProber.discovery import orchestrator as orch_mod
+
+        class FakeRobots:
+            crawl_delay = None
+            unavailable_status = None
+
+            def __init__(self, base_url, ua=""):
+                pass
+
+            def load(self):
+                return True, "User-agent: *\nDisallow: /private\n"
+
+            def is_allowed(self, path):
+                return not path.startswith("/private")
+
+        monkeypatch.setattr(orch_mod, "RobotsChecker", FakeRobots)
+
+        requested = []
+
+        class FakeResp:
+            def __init__(self):
+                self.status_code = 200
+                self.ok = True
+                self.headers = {}
+                self.content_type = "application/json"
+                self.body = '{"id": 1}'
+                self.error = ""
+                self.elapsed_ms = 1
+                self.method = "GET"
+
+        class FakeClient:
+            def __init__(self):
+                self.request_count = 0
+                self.delay_ms = 0
+
+            def get(self, url):
+                requested.append(url)
+                self.request_count += 1
+                return FakeResp()
+
+            def head(self, url):
+                requested.append(url)
+                self.request_count += 1
+                return FakeResp()
+
+            def request(self, url, method="GET"):
+                requested.append(url)
+                self.request_count += 1
+                return FakeResp()
+
+        config = deepcopy(DEFAULT_CONFIG)
+        config["db_path"] = str(tmp_path / "test.db")
+        config["strategies"] = []  # Phase 1-3 + 6 aus -- nur Phase 4/5 testen
+        config["delay_ms"] = 0
+
+        orch = orch_mod.ProbeOrchestrator(config)
+        orch.client = FakeClient()
+
+        base = "https://example.invalid"
+        sid = orch.db.upsert_service("example", base)
+        orch.db.upsert_endpoint(sid, "/public", methods=["GET"], discovered_by="openapi")
+        orch.db.upsert_endpoint(sid, "/private", methods=["GET"], discovered_by="openapi")
+
+        orch.probe(base)
+
+        assert not any("/private" in u for u in requested), \
+            f"robots-gesperrter Spec-Pfad wurde angefragt: {requested}"
+        assert any("/public" in u for u in requested), \
+            "erlaubter Pfad haette angefragt werden muessen"
+
+
 class TestApiProberExport:
     """Test: Export-Funktionen (konzeptionell)."""