feat(studio): add FastAPI backend + React frontend (Part A + B)

Backend (studio/backend/):
- db.py: SQLite init via SQLModel + lifespan context manager
- models.py: Setting, Test, Step, Run, StepResult tables
- deps.py: get_session dependency + artifact/baseline path constants
- routes_settings.py: GET/PUT /settings for LLM config
- routes_tests.py: full CRUD, async record endpoint, pytest export
- routes_runs.py: trigger run, list/get runs, dashboard stats, WebSocket live feed
- run_executor.py: SandboxRunner wrapper + WSConnectionHub for live updates
- main.py: FastAPI app with lifespan, CORS, static mounts

Frontend (studio/frontend/src/):
- App.tsx: react-router-dom v6 routing shell
- main.tsx: React 18 entry with TanStack Query provider
- lib/api.ts: typed fetch client for all backend endpoints
- pages/: Dashboard, Settings, Tests, NewTest, TestDetail, RunDetail, RunHistory

CLI:
- scrapewizard start: launches uvicorn on 127.0.0.1:8000

Tests:
- tests/integration/test_studio_backend.py: 4 HTTP integration tests (all pass)

Fixes:
- Replace datetime.utcnow() with datetime.now(timezone.utc) (py3.12 compat)
- Fix /studio gitignore rule that was blocking studio/ from being tracked
- Add *.db / *.db-shm / *.db-wal to .gitignore (runtime artifacts)
- record_test endpoint is async def (fixes asyncio.create_task in threadpool)

All 43 tests pass.

Author: pras-ops

.gitignore

@@ -36,6 +36,11 @@ cookies.json
 storage_state.json
 .learn.md
 
+# SQLite databases (runtime artifacts — never commit)
+*.db
+*.db-shm
+*.db-wal
+
 # Testing
 .pytest_cache/
 .coverage
@@ -51,6 +56,8 @@ htmlcov/
 # OS
 .DS_Store
 ScrapeWizard Studio
-/studio
 .learn.md
 
+# Frontend build output (generated — not committed)
+studio/frontend/dist/
+studio/frontend/node_modules/

README.md

@@ -152,6 +152,8 @@ The CLI scraper documented above remains the current, working product.
 
 - **[PLATFORM_PLAN.md](PLATFORM_PLAN.md)** — the full roadmap and architecture (source of truth)
 - **[BUILD_GUIDE.md](BUILD_GUIDE.md)** — step-by-step how-to for building each stage
+- **[FRONTEND_PLAN.md](FRONTEND_PLAN.md)** — detailed spec for the application (the GUI/portal)
+- **[APP_BUILD_STEPS.md](APP_BUILD_STEPS.md)** — step-by-step build order: backend API + SQLite, then frontend slices
 
 ## License
 MIT

pyproject.toml

@@ -30,7 +30,8 @@ dependencies = [
     "pandas>=2.0.0",
     "openpyxl>=3.1.0",
     "python-json-logger>=2.0.0",
-    "yaspin>=2.0.0"
+    "yaspin>=2.0.0",
+    "sqlmodel"
 ]
 
 [project.scripts]

scrapewizard/cli/commands/utils.py

@@ -123,5 +123,40 @@ def resume(project_id: str = typer.Argument(..., help="The ID of the project to
         orchestrator = Orchestrator(project_dir)
         orchestrator.run()
     except Exception as e:
-        rprint(f"[red]Resume failed: {e}[/red]")
+        log(f"Resume failed: {e}", level="error")
         raise typer.Exit(code=1)
+
+def start_studio(
+    port: int = typer.Option(8000, "--port", "-p", help="Port to run the studio server on"),
+    open_browser: bool = typer.Option(True, "--open/--no-open", help="Automatically open ScrapeWizard Studio in the default browser")
+) -> None:
+    """Start ScrapeWizard Studio web interface."""
+    import uvicorn
+    import webbrowser
+    import threading
+    import time
+    from studio.backend.db import init_db
+    
+    rprint("[bold cyan]Starting ScrapeWizard Studio...[/bold cyan]")
+    
+    # Initialize DB before booting
+    init_db()
+    
+    url = f"http://127.0.0.1:{port}"
+    rprint(f"• Studio is running at: [bold green]{url}[/bold green]")
+    rprint("• Press [bold yellow]Ctrl+C[/bold yellow] to stop the server.")
+    
+    if open_browser:
+        def open_url():
+            time.sleep(1.5)
+            try:
+                webbrowser.open(url)
+            except Exception:
+                pass
+        threading.Thread(target=open_url, daemon=True).start()
+        
+    try:
+        from studio.backend.main import app as fastapi_app
+        uvicorn.run(fastapi_app, host="127.0.0.1", port=port, log_level="error")
+    except KeyboardInterrupt:
+        rprint("\n[yellow]Studio server stopped.[/yellow]")

scrapewizard/cli/main.py

@@ -27,6 +27,7 @@
 app.command()(utils.clean)
 app.command()(utils.doctor)
 app.command()(utils.resume)
+app.command(name="start")(utils.start_studio)
 app.command(name="record")(engine.record)
 app.command(name="test")(engine.test)
 

studio/backend/db.py

@@ -0,0 +1,34 @@
+import os
+from pathlib import Path
+from sqlmodel import SQLModel, create_engine, Session
+from scrapewizard.core.config import ConfigManager
+
+# SQLite database path inside ~/.scrapewizard/
+DB_DIR = ConfigManager.CONFIG_DIR
+DB_DIR.mkdir(parents=True, exist_ok=True)
+DB_PATH = DB_DIR / "studio.db"
+
+# Create the engine
+database_url = f"sqlite:///{DB_PATH}"
+engine = create_engine(
+    database_url,
+    connect_args={"check_same_thread": False}  # Safe for SQLite with multiple threads
+)
+
+def init_db():
+    """Create all SQLModel tables in SQLite if they do not exist."""
+    SQLModel.metadata.create_all(engine)
+    
+    # Store schema version if not set
+    from studio.backend.models import Setting
+    with Session(engine) as session:
+        version_setting = session.get(Setting, "schema_version")
+        if not version_setting:
+            version_setting = Setting(key="schema_version", value="1.0.0")
+            session.add(version_setting)
+            session.commit()
+
+def get_session():
+    """Dependency for obtaining database sessions in FastAPI route handlers."""
+    with Session(engine) as session:
+        yield session

studio/backend/deps.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+from scrapewizard.core.config import ConfigManager
+
+# Re-export get_session for convenience
+from studio.backend.db import get_session
+
+# Root path for all studio execution screenshots and diff outputs
+STUDIO_ARTIFACTS_DIR = Path.home() / ".scrapewizard" / "artifacts"
+STUDIO_ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True)
+
+# Shared stable path for visual test baselines
+STUDIO_BASELINES_DIR = Path.home() / ".scrapewizard" / "baselines"
+STUDIO_BASELINES_DIR.mkdir(parents=True, exist_ok=True)

studio/backend/main.py

@@ -1,181 +1,52 @@
-from fastapi import FastAPI, WebSocket, WebSocketDisconnect, BackgroundTasks, HTTPException
-from fastapi.middleware.cors import CORSMiddleware
-import uvicorn
-import asyncio
-import sys
 import os
-import json
-from typing import List, Optional
+import sys
+from pathlib import Path
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
 
-# Add root to sys.path to access scrapewizard core
+# Add project root to sys.path to resolve scrapewizard imports correctly
 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")))
 
 from scrapewizard.core.logging import log
-from studio.backend.state import StudioState, StudioProject, FieldDefinition
-from studio.backend.browser_manager import StudioBrowserManager
+from studio.backend.db import init_db
+from studio.backend.deps import STUDIO_ARTIFACTS_DIR
+from studio.backend.routes_settings import router as settings_router
+from studio.backend.routes_tests import router as tests_router
+from studio.backend.routes_runs import router as runs_router
 
-app = FastAPI(title="ScrapeWizard Studio Backend")
-browser_manager = StudioBrowserManager()
+from contextlib import asynccontextmanager
 
-app.add_middleware(
-    CORSMiddleware,
-    # Local-only: portal dev server origins. Do not widen — this API can drive a browser.
-    allow_origins=["http://localhost:5173", "http://127.0.0.1:5173"],
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    log("Initializing SQLite database on startup...")
+    init_db()
+    yield
 
-active_sessions = {}
+app = FastAPI(title="ScrapeWizard Studio Backend", version="1.2.0", lifespan=lifespan)
 
 @app.get("/health")
-async def health():
-    """Health check endpoint for CLI and Electron."""
-    return {"status": "ok", "version": "1.0.0", "engine": "ScrapeWizard-Studio"}
-
-@app.get("/")
-async def root():
-    return {"status": "online", "message": "ScrapeWizard Studio Orchestrator Active"}
-
-@app.post("/session/start")
-async def start_session(url: str, background_tasks: BackgroundTasks):
-    # Use simple validation for now, or import from shared.validators if needed
-    if not url.startswith("http"):
-        raise HTTPException(status_code=400, detail="Invalid URL")
-    
-    project_id = f"proj_{len(active_sessions) + 1}"
-    project = StudioProject(project_id=project_id, url=url, state=StudioState.NAVIGATION)
-    active_sessions[project_id] = project
-    
-    # Start browser in background
-    await browser_manager.start(url)
-    
-    return {"status": "started", "project_id": project_id, "state": project.state}
-
-@app.post("/session/compile")
-async def compile_project(project_id: str):
-    if project_id not in active_sessions:
-        return {"error": "Project not found"}
-    
-    project = active_sessions[project_id]
-    # TODO: Invoke CodeGenerator with AET
-    return {"status": "compiled", "project_id": project_id}
-
-@app.get("/session/dom")
-async def get_dom_project(project_id: str):
-    if project_id not in active_sessions:
-        return {"error": "Project not found"}
-    tree = await browser_manager.get_dom_tree()
-    return {"project_id": project_id, "tree": tree}
-
-@app.websocket("/cdp/ws")
-async def cdp_proxy(ws: WebSocket):
-    """Robust bi-directional CDP proxy for Studio Inspector."""
-    await ws.accept()
-    log("CDP WebSocket connection established", level="info")
-    
-    # Use the existing browser_manager to get a session
-    # or follow the user's "launch as you go" plan for the proxy.
-    # Given the isolation, we'll try to get the existing session first.
-    try:
-        if not browser_manager.page:
-            # If no page is active, start a default one (or wait for session/start)
-            await browser_manager.start("about:blank")
-            
-        cdp_session = await browser_manager.get_cdp_session()
-
-        # Browser -> client forwarding is handled by the cdp_session.on(...) event
-        # subscriptions below (see create_forwarder).
-
-        async def client_to_browser():
-            """Forward messages from Studio Client (WS) -> Browser (CDP)."""
-            try:
-                while True:
-                    msg = await ws.receive_json()
-                    method = msg.get("method")
-                    params = msg.get("params", {})
-                    msg_id = msg.get("id")
-                    
-                    if method == "Input.dispatchMouseEvent":
-                        # Params: type, x, y, button, etc.
-                        # browser_manager expects: event_type='mouse', params={action, x, y, ...}
-                        # We map CDP-like params to our simple manager
-                        etype = params.get("type")
-                        mapping = {
-                            "mousePressed": "down",
-                            "mouseReleased": "up",
-                            "mouseMoved": "move",
-                            "mouseWheel": "wheel"
-                        }
-                        if etype in mapping:
-                            await browser_manager.handle_input_event("mouse", {
-                                "action": mapping[etype],
-                                "x": params.get("x"),
-                                "y": params.get("y"),
-                                "deltaX": params.get("deltaX", 0),
-                                "deltaY": params.get("deltaY", 0),
-                                "button": params.get("button", "left")
-                            })
-                    elif method == "Input.dispatchKeyEvent":
-                        # TODO: Map keys if needed
-                        pass
-                    elif method:
-                        result = await cdp_session.send(method, params)
-                        if msg_id is not None:
-                            await ws.send_json({"id": msg_id, "result": result})
-            except Exception as e:
-                log(f"CDP Client -> Browser error: {e}", level="error")
-
-        # Hook into browser_manager's internal screencast logic
-        async def handle_screencast_frame(data):
-            # Forward the frame to the frontend
-            try:
-                await ws.send_json({"method": "Page.screencastFrame", "params": {"data": data}})
-            except Exception:
-                pass
-
-        browser_manager.on_frame = handle_screencast_frame
-        
-        # Hook into browser_manager's inspector logic
-        async def handle_selection(data_str: str):
-            try:
-                data = json.loads(data_str)
-                if data['type'] == 'hover':
-                    await ws.send_json({"method": "Inspector.highlight", "params": data})
-                elif data['type'] == 'select':
-                    await ws.send_json({"method": "Inspector.selected", "params": data})
-            except Exception as e:
-                log(f"Inspector error: {e}", level="error")
-
-        browser_manager.on_selection = handle_selection
-
-        # In Playwright, .on() can take a sync or async function.
-        # We'll use a wrapper to ensure it forwards to the WS.
-        def create_forwarder(event_name):
-            def handler(params):
-                # Create a task to send the message
-                asyncio.create_task(ws.send_json({"method": event_name, "params": params}))
-            return handler
-
-        # Standard events for Inspector
-        cdp_session.on("Page.screencastFrame", create_forwarder("Page.screencastFrame"))
-        cdp_session.on("Runtime.consoleAPICalled", create_forwarder("Runtime.consoleAPICalled"))
-        cdp_session.on("Network.requestWillBeSent", create_forwarder("Network.requestWillBeSent"))
-        cdp_session.on("DOM.documentUpdated", create_forwarder("DOM.documentUpdated"))
-        cdp_session.on("Page.loadEventFired", create_forwarder("Page.loadEventFired"))
-
-        # Run the command loop
-        await client_to_browser()
-        
-    except WebSocketDisconnect:
-        log("CDP WebSocket disconnected", level="info")
-    except Exception as e:
-        log(f"CDP Proxy failed: {e}", level="error")
-    finally:
-        try:
-            await ws.close()
-        except Exception:
-            pass
+def health():
+    """Health check endpoint containing engine version details."""
+    return {"status": "ok", "version": "1.2.0", "engine": "ScrapeWizard-Studio"}
+
+# Register Settings, Tests and Runs routers
+app.include_router(settings_router)
+app.include_router(tests_router)
+app.include_router(runs_router)
+
+# Mount artifacts folder to serve screenshots and visual diff crops
+STUDIO_ARTIFACTS_DIR.mkdir(parents=True, exist_ok=True)
+app.mount("/artifacts", StaticFiles(directory=str(STUDIO_ARTIFACTS_DIR)), name="artifacts")
+
+# Mount React static frontend dist output (prod bundle)
+frontend_dist = Path(__file__).parent / ".." / "frontend" / "dist"
+if frontend_dist.exists():
+    app.mount("/", StaticFiles(directory=str(frontend_dist), html=True), name="frontend")
+else:
+    log("React frontend dist bundle not found; serving API endpoints only.", level="warning")
 
 if __name__ == "__main__":
-    # Local-only bind: this API can drive a real browser; never expose it on the network.
+    import uvicorn
+    # Bound to local loopback interface only for secure local operation
     uvicorn.run(app, host="127.0.0.1", port=8000)