refactor: direct in-process tool execution, security hardening

- Extract execute_recipe_endpoint() as public API for MCP bridge/server
- MCP server calls recipes directly (no more httpx self-call)
- MCP bridge uses execute_recipe_endpoint() instead of faking Request objects
- Sanitize upload filenames with path traversal protection
- Replace regex param validation with Python identifier check
- Configurable host port via WEB2API_HOST_PORT env var
- Add tests: path traversal, MCP special chars, keyword param rejection
- Update fallback version to 0.4.0

Author: Endogen

README.md

@@ -348,7 +348,7 @@ All recipe endpoints follow the pattern: `GET /{slug}/{endpoint}?page=1&q=...`
 - `page` — pagination (default: 1)
 - `q` — query text (required when `requires_query: true`)
 - additional query params are passed to custom scrapers
-- extra query param names must match `[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}` and values are capped at 512 chars
+- extra query param names must be valid Python identifiers (and not keywords); values are capped at 512 chars
 
 ### Error Codes
 

docker-compose.yml

@@ -5,7 +5,7 @@ services:
       dockerfile: Dockerfile
     restart: unless-stopped
     ports:
-      - "127.0.0.1:8010:8000"
+      - "127.0.0.1:${WEB2API_HOST_PORT:-8010}:8000"
     volumes:
       - ./data/recipes:/data/recipes
     environment:

tests/e2e/test_e2e.py

@@ -4,6 +4,7 @@
 
 import os
 import shutil
+import socket
 import subprocess
 import time
 import uuid
@@ -17,7 +18,6 @@
 
 PROJECT_ROOT = Path(__file__).resolve().parents[2]
 COMPOSE_FILE = PROJECT_ROOT / "docker-compose.yml"
-BASE_URL = "http://127.0.0.1:8010"
 HEALTH_TIMEOUT_SECONDS = 180
 HEALTH_POLL_INTERVAL_SECONDS = 2.0
 LIVE_ERROR_MARKERS = (
@@ -62,6 +62,12 @@ def _is_docker_unavailable(stderr_stdout: str) -> bool:
     return any(marker in message for marker in DOCKER_UNAVAILABLE_MARKERS)
 
 
+def _allocate_host_port() -> int:
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+        sock.bind(("127.0.0.1", 0))
+        return int(sock.getsockname()[1])
+
+
 def _wait_for_health(base_url: str) -> None:
     deadline = time.monotonic() + HEALTH_TIMEOUT_SECONDS
     last_error = "no response"
@@ -129,6 +135,9 @@ def dockerized_web2api() -> Iterator[str]:
     compose_project = f"web2api-e2e-{uuid.uuid4().hex[:8]}"
     env = os.environ.copy()
     env["COMPOSE_PROJECT_NAME"] = compose_project
+    host_port = _allocate_host_port()
+    env["WEB2API_HOST_PORT"] = str(host_port)
+    base_url = f"http://127.0.0.1:{host_port}"
 
     started = False
     try:
@@ -140,8 +149,8 @@ def dockerized_web2api() -> Iterator[str]:
             pytest.fail(f"docker compose up failed:\n{combined}")
 
         started = True
-        _wait_for_health(BASE_URL)
-        yield BASE_URL
+        _wait_for_health(base_url)
+        yield base_url
     finally:
         if started:
             _run_compose(

tests/integration/test_api.py

@@ -5,6 +5,7 @@
 import asyncio
 import logging
 import subprocess
+import uuid
 from datetime import UTC, datetime
 from pathlib import Path
 
@@ -259,6 +260,9 @@ async def fake_scrape(
                 invalid_extra_resp = await client.get("/alpha/read?bad!param=1")
                 assert invalid_extra_resp.status_code == 400
                 assert invalid_extra_resp.json()["error"]["code"] == "INVALID_PARAMS"
+                invalid_identifier_resp = await client.get("/alpha/read?model-id=1")
+                assert invalid_identifier_resp.status_code == 400
+                assert invalid_identifier_resp.json()["error"]["code"] == "INVALID_PARAMS"
 
                 # Non-existent endpoint on a recipe (404 from FastAPI)
                 unknown_ep_resp = await client.get("/beta/search")
@@ -527,6 +531,126 @@ async def test_recipe_management_uninstall_force_for_unmanaged_local(
             assert "local-only" not in slugs
 
 
+@pytest.mark.asyncio
+async def test_mcp_bridge_preserves_special_characters_in_params(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    recipes_dir = tmp_path / "recipes"
+    _write_recipe(
+        recipes_dir,
+        "alpha",
+        endpoints={
+            "search": {
+                "url": "https://example.com/search?q={query}&page={page}",
+                "requires_query": True,
+                "params": {
+                    "tools_url": {
+                        "description": "MCP bridge URL",
+                        "required": False,
+                    },
+                },
+                "items": {"container": ".item", "fields": {"title": {"selector": ".title"}}},
+                "pagination": {"type": "page_param", "param": "page"},
+            },
+        },
+    )
+
+    captured: dict[str, object] = {}
+
+    async def fake_scrape(
+        *,
+        pool: FakePool,
+        recipe,
+        endpoint: str,
+        page: int = 1,
+        query: str | None = None,
+        extra_params: dict[str, str] | None = None,
+        scrape_timeout: float = 30.0,
+    ) -> ApiResponse:
+        _ = pool, recipe, endpoint, page, scrape_timeout
+        captured["query"] = query
+        captured["extra_params"] = dict(extra_params or {})
+        return _success_response(slug="alpha", endpoint="search", page=1, query=query)
+
+    monkeypatch.setattr("web2api.main.scrape", fake_scrape)
+
+    fake_pool = FakePool()
+    app = create_app(recipes_dir=recipes_dir, pool=fake_pool)
+
+    async with app.router.lifespan_context(app):
+        transport = ASGITransport(app=app)
+        async with AsyncClient(transport=transport, base_url="http://testserver") as client:
+            response = await client.post(
+                "/mcp/tools/alpha__search",
+                json={
+                    "q": "cats & dogs",
+                    "tools_url": "http://localhost:8100/mcp/tools?x=1&y=2",
+                },
+            )
+
+    assert response.status_code == 200
+    assert captured["query"] == "cats & dogs"
+    assert captured["extra_params"] == {
+        "tools_url": "http://localhost:8100/mcp/tools?x=1&y=2",
+    }
+
+
+@pytest.mark.asyncio
+async def test_post_upload_rejects_path_traversal_filenames(
+    tmp_path: Path,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    recipes_dir = tmp_path / "recipes"
+    _write_recipe(recipes_dir, "alpha")
+
+    captured: dict[str, object] = {}
+    escaped_name = f"web2api_escape_{uuid.uuid4().hex}.txt"
+    escaped_path = Path("/tmp") / escaped_name
+    if escaped_path.exists():
+        escaped_path.unlink()
+
+    async def fake_scrape(
+        *,
+        pool: FakePool,
+        recipe,
+        endpoint: str,
+        page: int = 1,
+        query: str | None = None,
+        extra_params: dict[str, str] | None = None,
+        scrape_timeout: float = 30.0,
+    ) -> ApiResponse:
+        _ = pool, recipe, endpoint, page, query, scrape_timeout
+        captured["extra_params"] = dict(extra_params or {})
+        return _success_response(slug="alpha", endpoint="read", page=1)
+
+    monkeypatch.setattr("web2api.main.scrape", fake_scrape)
+
+    fake_pool = FakePool()
+    app = create_app(recipes_dir=recipes_dir, pool=fake_pool)
+
+    async with app.router.lifespan_context(app):
+        transport = ASGITransport(app=app)
+        async with AsyncClient(transport=transport, base_url="http://testserver") as client:
+            response = await client.post(
+                "/alpha/read",
+                files={
+                    "files": (f"../{escaped_name}", b"uploaded-content", "text/plain"),
+                },
+            )
+
+    assert response.status_code == 200
+    assert escaped_path.exists() is False
+    extra_params = captured.get("extra_params")
+    assert isinstance(extra_params, dict)
+    file_paths = extra_params.get("file_paths", [])
+    assert isinstance(file_paths, list)
+    assert len(file_paths) == 1
+    uploaded_path = Path(str(file_paths[0]))
+    assert uploaded_path.name == escaped_name
+    assert "/../" not in str(uploaded_path)
+
+
 @pytest.mark.asyncio
 async def test_check_updates_endpoint(
     tmp_path: Path,

tests/unit/test_config.py

@@ -217,12 +217,22 @@ def test_reserved_param_name_page_is_rejected() -> None:
 
 
 def test_invalid_param_name_is_rejected() -> None:
-    """Param names that don't match the pattern should fail."""
+    """Param names that are not Python identifiers should fail."""
     data = _valid_recipe_data()
     data["endpoints"]["read"]["params"] = {
         "bad!name": {"description": "invalid chars"},
     }
-    with pytest.raises(ValidationError, match="must match pattern"):
+    with pytest.raises(ValidationError, match="valid Python identifier"):
+        RecipeConfig.model_validate(data)
+
+
+def test_keyword_param_name_is_rejected() -> None:
+    """Python keywords are not valid parameter names."""
+    data = _valid_recipe_data()
+    data["endpoints"]["read"]["params"] = {
+        "class": {"description": "invalid keyword"},
+    }
+    with pytest.raises(ValidationError, match="valid Python identifier"):
         RecipeConfig.model_validate(data)
 
 

web2api/__init__.py

@@ -7,4 +7,4 @@
 try:
     __version__ = version("web2api")
 except PackageNotFoundError:
-    __version__ = "0.2.0"
+    __version__ = "0.4.0"

web2api/config.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import keyword
 import re
 from collections.abc import Mapping
 from typing import Literal
@@ -12,7 +13,6 @@
 PaginationType = Literal["page_param", "next_link", "offset_param"]
 FieldContext = Literal["self", "next_sibling", "parent"]
 RESERVED_RECIPE_SLUGS = {"api", "health", "docs", "openapi", "redoc"}
-_PARAM_NAME_PATTERN = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}$")
 _RESERVED_PARAM_NAMES = {"q", "page"}
 FieldTransform = Literal[
     "regex_int",
@@ -24,6 +24,11 @@
 ]
 
 
+def is_valid_param_name(name: str) -> bool:
+    """Return ``True`` if *name* is a valid Python identifier and not a keyword."""
+    return name.isidentifier() and not keyword.iskeyword(name)
+
+
 class ActionConfig(BaseModel):
     """Playwright action executed before extraction."""
 
@@ -137,10 +142,10 @@ def validate_params(self) -> EndpointConfig:
                 raise ValueError(
                     f"param name '{name}' conflicts with reserved query parameter"
                 )
-            if not _PARAM_NAME_PATTERN.match(name):
+            if not is_valid_param_name(name):
                 raise ValueError(
-                    f"param name '{name}' must match pattern "
-                    "[a-zA-Z0-9][a-zA-Z0-9_-]{0,63}"
+                    f"param name '{name}' must be a valid Python identifier "
+                    "(letters/digits/underscore, not starting with a digit, not a keyword)"
                 )
         return self