feat: add MCP executable path support (#360)

Author: D4Vinci

docs/ai/mcp-server.md

@@ -154,6 +154,35 @@ Here's the main article from Anthropic on [how to add MCP servers to Claude code
 
 Then, after you've added the server, you need to completely quit and restart the app you used above. In Claude Desktop, you should see an MCP server indicator (🔧) in the bottom-right corner of the chat input or see `ScraplingServer` in the `Search and tools` dropdown in the chat input box.
 
+### Custom Browser Executable
+
+Browser-based tools (`fetch`, `bulk_fetch`, `stealthy_fetch`, `bulk_stealthy_fetch`, and `open_session`) can use a custom Chromium-compatible browser executable instead of the bundled Chromium. This is useful for custom browser builds or lightweight browser engines.
+
+To configure it once for the whole MCP server, pass the executable path when starting the server:
+
+```bash
+scrapling mcp --executable-path "/path/to/chromium"
+```
+
+In a Claude Desktop configuration, add the option to the server arguments:
+
+```json
+{
+  "mcpServers": {
+    "ScraplingServer": {
+      "command": "/Users/<MyUsername>/.venv/bin/scrapling",
+      "args": [
+        "mcp",
+        "--executable-path",
+        "/path/to/chromium"
+      ]
+    }
+  }
+}
+```
+
+You can also set the `SCRAPLING_EXECUTABLE_PATH` environment variable before starting the server. Tool calls can still pass `executable_path` directly when a single request or session needs a different browser executable.
+
 ### Streamable HTTP
 As per version 0.3.6, we have added the ability to make the MCP server use the 'Streamable HTTP' transport mode instead of the traditional 'stdio' transport.
 
@@ -356,4 +385,4 @@ This protection runs automatically on all MCP tool responses. Keep `main_content
 
 ---
 
-*Built with ❤️ by the Scrapling team. Happy scraping!*
+*Built with ❤️ by the Scrapling team. Happy scraping!*

docs/api-reference/mcp-server.md

@@ -22,6 +22,12 @@ server = ScraplingMCPServer()
 server.serve(http=False, host="0.0.0.0", port=8000)
 ```
 
+To set a custom Chromium-compatible browser executable for browser-based MCP tools, pass `executable_path`:
+
+```python
+server = ScraplingMCPServer(executable_path="/path/to/chromium")
+```
+
 ## Response Model
 
 The standardized response structure that's returned by all MCP server tools:
@@ -52,4 +58,4 @@ The main MCP server class that provides all web scraping tools:
 
 ## ::: scrapling.core.ai.ScraplingMCPServer
     handler: python
-    :docstring:
+    :docstring:

scrapling/cli.py

@@ -157,10 +157,16 @@ def install(force):  # pragma: no cover
 @option(
     "--port", type=int, default=8000, help="The port to use if streamable-http transport is enabled (Default: 8000)"
 )
-def mcp(http, host, port):
+@option(
+    "--executable-path",
+    type=str,
+    default=None,
+    help="Path to a custom Chromium-compatible browser executable for browser-based MCP tools",
+)
+def mcp(http, host, port, executable_path):
     from scrapling.core.ai import ScraplingMCPServer
 
-    server = ScraplingMCPServer()
+    server = ScraplingMCPServer(executable_path=executable_path)
     server.serve(http, host, port)
 
 

scrapling/core/ai.py

@@ -1,4 +1,5 @@
 from uuid import uuid4
+from os import environ
 from asyncio import gather
 from datetime import datetime, timezone
 from dataclasses import dataclass, field
@@ -33,6 +34,7 @@
 
 SessionType = Literal["dynamic", "stealthy"]
 ScreenshotType = Literal["png", "jpeg"]
+MCP_EXECUTABLE_PATH_ENV = "SCRAPLING_EXECUTABLE_PATH"
 
 
 class ResponseModel(BaseModel):
@@ -105,8 +107,18 @@ def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tu
 
 
 class ScraplingMCPServer:
-    def __init__(self):
+    def __init__(self, executable_path: Optional[str] = None):
+        """Create a Scrapling MCP server.
+
+        :param executable_path: Optional global Chromium-compatible browser executable path for browser tools.
+            If omitted, the SCRAPLING_EXECUTABLE_PATH environment variable is used when set.
+        """
         self._sessions: Dict[str, _SessionEntry] = {}
+        self._executable_path = executable_path or environ.get(MCP_EXECUTABLE_PATH_ENV) or None
+
+    def _resolve_executable_path(self, executable_path: Optional[str]) -> Optional[str]:
+        """Return a per-call executable path or the server-wide default."""
+        return executable_path or self._executable_path
 
     def _get_session(self, session_id: str, expected_type: Optional[SessionType]) -> _SessionEntry:
         """Look up a session by ID, optionally validating its type. Pass `None` to skip the type check."""
@@ -136,6 +148,7 @@ async def open_session(
         extra_headers: Optional[Dict[str, str]] = None,
         useragent: Optional[str] = None,
         cdp_url: Optional[str] = None,
+        executable_path: Optional[str] = None,
         timeout: int | float = 30000,
         disable_resources: bool = False,
         wait_selector: Optional[str] = None,
@@ -166,6 +179,7 @@ async def open_session(
         :param extra_headers: A dictionary of extra headers to add to the request.
         :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
+        :param executable_path: Absolute path to a custom Chromium-compatible browser executable. Overrides the server-wide default for this session.
         :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000.
         :param disable_resources: Drop requests for unnecessary resources for a speed boost.
         :param wait_selector: Wait for a specific CSS selector to be in a specific state.
@@ -202,6 +216,7 @@ async def open_session(
             wait_selector=wait_selector,
             google_search=google_search,
             extra_headers=extra_headers,
+            executable_path=self._resolve_executable_path(executable_path),
             disable_resources=disable_resources,
             wait_selector_state=wait_selector_state,
         )
@@ -494,6 +509,7 @@ async def fetch(
         extra_headers: Optional[Dict[str, str]] = None,
         useragent: Optional[str] = None,
         cdp_url: Optional[str] = None,
+        executable_path: Optional[str] = None,
         timeout: int | float = 30000,
         disable_resources: bool = False,
         wait_selector: Optional[str] = None,
@@ -530,6 +546,7 @@ async def fetch(
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
+        :param executable_path: Absolute path to a custom Chromium-compatible browser executable. Overrides the server-wide default for this request.
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
@@ -550,6 +567,7 @@ async def fetch(
             extra_headers=extra_headers,
             useragent=useragent,
             cdp_url=cdp_url,
+            executable_path=executable_path,
             timeout=timeout,
             disable_resources=disable_resources,
             wait_selector=wait_selector,
@@ -576,6 +594,7 @@ async def bulk_fetch(
         extra_headers: Optional[Dict[str, str]] = None,
         useragent: Optional[str] = None,
         cdp_url: Optional[str] = None,
+        executable_path: Optional[str] = None,
         timeout: int | float = 30000,
         disable_resources: bool = False,
         wait_selector: Optional[str] = None,
@@ -612,6 +631,7 @@ async def bulk_fetch(
         :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.
         :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
+        :param executable_path: Absolute path to a custom Chromium-compatible browser executable. Overrides the server-wide default for this request.
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
@@ -653,6 +673,7 @@ async def bulk_fetch(
                 wait_selector=wait_selector,
                 google_search=google_search,
                 extra_headers=extra_headers,
+                executable_path=self._resolve_executable_path(executable_path),
                 disable_resources=disable_resources,
                 wait_selector_state=wait_selector_state,
             ) as session:
@@ -678,6 +699,7 @@ async def stealthy_fetch(
         useragent: Optional[str] = None,
         hide_canvas: bool = False,
         cdp_url: Optional[str] = None,
+        executable_path: Optional[str] = None,
         timeout: int | float = 30000,
         disable_resources: bool = False,
         wait_selector: Optional[str] = None,
@@ -722,6 +744,7 @@ async def stealthy_fetch(
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak.
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
+        :param executable_path: Absolute path to a custom Chromium-compatible browser executable. Overrides the server-wide default for this request.
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
@@ -744,6 +767,7 @@ async def stealthy_fetch(
             useragent=useragent,
             hide_canvas=hide_canvas,
             cdp_url=cdp_url,
+            executable_path=executable_path,
             timeout=timeout,
             disable_resources=disable_resources,
             wait_selector=wait_selector,
@@ -775,6 +799,7 @@ async def bulk_stealthy_fetch(
         useragent: Optional[str] = None,
         hide_canvas: bool = False,
         cdp_url: Optional[str] = None,
+        executable_path: Optional[str] = None,
         timeout: int | float = 30000,
         disable_resources: bool = False,
         wait_selector: Optional[str] = None,
@@ -819,6 +844,7 @@ async def bulk_stealthy_fetch(
         :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.
         :param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak.
         :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.
+        :param executable_path: Absolute path to a custom Chromium-compatible browser executable. Overrides the server-wide default for this request.
         :param google_search: Enabled by default, Scrapling will set a Google referer header.
         :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._
         :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.
@@ -864,6 +890,7 @@ async def bulk_stealthy_fetch(
                 wait_selector=wait_selector,
                 google_search=google_search,
                 extra_headers=extra_headers,
+                executable_path=self._resolve_executable_path(executable_path),
                 additional_args=additional_args,
                 solve_cloudflare=solve_cloudflare,
                 disable_resources=disable_resources,

tests/ai/test_ai_mcp.py

@@ -1,5 +1,6 @@
 import base64
 import struct
+from typing import Any
 from contextlib import contextmanager
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from threading import Thread
@@ -16,6 +17,48 @@
     SessionClosedModel,
     _normalize_credentials,
 )
+from scrapling.engines.toolbelt.custom import Response
+
+
+class _FakeAsyncBrowserSession:
+    instances: list["_FakeAsyncBrowserSession"] = []
+
+    def __init__(self, **kwargs: Any) -> None:
+        self.kwargs = kwargs
+        self._is_alive = False
+        type(self).instances.append(self)
+
+    async def __aenter__(self):
+        self._is_alive = True
+        return self
+
+    async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:
+        self._is_alive = False
+
+    async def start(self) -> None:
+        self._is_alive = True
+
+    async def close(self) -> None:
+        self._is_alive = False
+
+    async def fetch(self, url: str, **kwargs: Any) -> Response:
+        return Response(
+            url=url,
+            content="<html><body>ok</body></html>",
+            status=200,
+            reason="OK",
+            cookies={},
+            headers={},
+            request_headers={},
+        )
+
+
+class _FakeDynamicSession(_FakeAsyncBrowserSession):
+    instances = []
+
+
+class _FakeStealthySession(_FakeAsyncBrowserSession):
+    instances = []
 
 
 @pytest_httpbin.use_class_based_httpbin
@@ -204,6 +247,60 @@ async def test_open_session_duplicate_id_raises(self, server):
         await server.close_session("dupe")
 
 
+class TestExecutablePath:
+    """Test custom browser executable path plumbing in the MCP browser tools"""
+
+    @pytest.fixture(autouse=True)
+    def reset_fakes(self):
+        _FakeDynamicSession.instances = []
+        _FakeStealthySession.instances = []
+
+    @pytest.mark.asyncio
+    async def test_open_session_passes_executable_path(self, monkeypatch):
+        """open_session forwards per-session executable_path to the dynamic session"""
+        monkeypatch.setattr("scrapling.core.ai.AsyncDynamicSession", _FakeDynamicSession)
+        server = ScraplingMCPServer()
+
+        created = await server.open_session(session_type="dynamic", executable_path="/tmp/chrome")
+
+        assert _FakeDynamicSession.instances[0].kwargs["executable_path"] == "/tmp/chrome"
+        await server.close_session(created.session_id)
+
+    @pytest.mark.asyncio
+    async def test_open_session_uses_environment_default(self, monkeypatch):
+        """open_session uses SCRAPLING_EXECUTABLE_PATH when no per-call value is provided"""
+        monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/custom-chromium")
+        monkeypatch.setattr("scrapling.core.ai.AsyncStealthySession", _FakeStealthySession)
+        server = ScraplingMCPServer()
+
+        created = await server.open_session(session_type="stealthy")
+
+        assert _FakeStealthySession.instances[0].kwargs["executable_path"] == "/opt/custom-chromium"
+        await server.close_session(created.session_id)
+
+    @pytest.mark.asyncio
+    async def test_fetch_overrides_global_executable_path(self, monkeypatch):
+        """fetch forwards a per-call executable_path instead of the server default"""
+        monkeypatch.setattr("scrapling.core.ai.AsyncDynamicSession", _FakeDynamicSession)
+        server = ScraplingMCPServer(executable_path="/opt/default-chromium")
+
+        result = await server.fetch(url="https://example.com", executable_path="/opt/request-chromium")
+
+        assert isinstance(result, ResponseModel)
+        assert _FakeDynamicSession.instances[0].kwargs["executable_path"] == "/opt/request-chromium"
+
+    @pytest.mark.asyncio
+    async def test_stealthy_fetch_uses_global_executable_path(self, monkeypatch):
+        """stealthy_fetch forwards the server executable_path default"""
+        monkeypatch.setattr("scrapling.core.ai.AsyncStealthySession", _FakeStealthySession)
+        server = ScraplingMCPServer(executable_path="/opt/default-chromium")
+
+        result = await server.stealthy_fetch(url="https://example.com")
+
+        assert isinstance(result, ResponseModel)
+        assert _FakeStealthySession.instances[0].kwargs["executable_path"] == "/opt/default-chromium"
+
+
 def _png_height(data: bytes) -> int:
     """Read the height field from a PNG IHDR chunk."""
     return struct.unpack(">I", data[20:24])[0]

tests/cli/test_cli.py

@@ -57,7 +57,19 @@ def test_mcp_command(self, runner):
 
             result = runner.invoke(mcp)
             assert result.exit_code == 0
-            mock_instance.serve.assert_called_once()
+            mock_server.assert_called_once_with(executable_path=None)
+            mock_instance.serve.assert_called_once_with(False, "0.0.0.0", 8000)
+
+    def test_mcp_command_with_executable_path(self, runner):
+        """Test MCP command with a custom browser executable"""
+        with patch('scrapling.core.ai.ScraplingMCPServer') as mock_server:
+            mock_instance = MagicMock()
+            mock_server.return_value = mock_instance
+
+            result = runner.invoke(mcp, ['--executable-path', '/opt/custom-chromium'])
+            assert result.exit_code == 0
+            mock_server.assert_called_once_with(executable_path='/opt/custom-chromium')
+            mock_instance.serve.assert_called_once_with(False, "0.0.0.0", 8000)
 
     def test_extract_get_command(self, runner, tmp_path, html_url):
         """Test extract `get` command"""