-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathserver.py
More file actions
1732 lines (1486 loc) · 59.7 KB
/
Copy pathserver.py
File metadata and controls
1732 lines (1486 loc) · 59.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
Seamless browser remote - FastAPI Backend (v2)
Controls Brave Browser via Chrome DevTools Protocol using Playwright.
All inputs are injected via CDP so the active window focus is never stolen.
Key features:
- Live mirror mode: stream screenshots to phone, tap-to-click
- Mouse click/scroll/hover injection via CDP
- Tab management: reuse existing tabs, don't open duplicates
- D-Pad keyboard mode as fallback
- Text input for search
"""
import asyncio
import base64
import ctypes
import ctypes.wintypes
import json
import logging
import os
import socket
import subprocess
import sys
import time
from contextlib import asynccontextmanager
from typing import Optional
from urllib.parse import urlparse
from fastapi import FastAPI, HTTPException, Request
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, StreamingResponse
from fastapi.staticfiles import StaticFiles
from fastapi.websockets import WebSocket, WebSocketDisconnect
from pydantic import BaseModel
from playwright.async_api import async_playwright, Browser, Page, BrowserContext
import config
# ---------------------------------------------------------------------------
# Logging
# ---------------------------------------------------------------------------
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
log = logging.getLogger("remote")
# ---------------------------------------------------------------------------
# Global state
# ---------------------------------------------------------------------------
pw_instance = None
browser: Optional[Browser] = None
page: Optional[Page] = None
browser_process: Optional[subprocess.Popen] = None
pc_mode = False
pc_monitor = 0 # which monitor index to use in PC mode (0 = primary)
def detect_monitors() -> list[dict]:
"""Detect monitors via EnumDisplayMonitors."""
monitors = []
try:
MONITOR_ENUM_PROC = ctypes.WINFUNCTYPE(
ctypes.c_int, ctypes.c_ulong, ctypes.c_ulong,
ctypes.POINTER(ctypes.wintypes.RECT), ctypes.c_double,
)
def callback(hMonitor, hdcMonitor, lprcMonitor, dwData):
r = lprcMonitor.contents
monitors.append({
"left": r.left, "top": r.top,
"width": r.right - r.left, "height": r.bottom - r.top,
})
return 1
ctypes.windll.user32.EnumDisplayMonitors(
None, None, MONITOR_ENUM_PROC(callback), 0
)
except Exception:
pass
if not monitors:
w, h = get_screen_size()
monitors.append({"left": 0, "top": 0, "width": w, "height": h})
return monitors
def get_active_monitor() -> dict:
"""Return the monitor dict for the currently selected pc_monitor index."""
monitors = detect_monitors()
idx = pc_monitor if 0 <= pc_monitor < len(monitors) else 0
return monitors[idx]
# ---------------------------------------------------------------------------
# OS-level mouse control (Windows ctypes — zero external deps)
# ---------------------------------------------------------------------------
class POINT(ctypes.Structure):
_fields_ = [("x", ctypes.c_long), ("y", ctypes.c_long)]
def os_cursor_move_rel(dx: float, dy: float):
pt = POINT()
ctypes.windll.user32.GetCursorPos(ctypes.byref(pt))
ctypes.windll.user32.SetCursorPos(pt.x + int(dx), pt.y + int(dy))
def os_cursor_set(x: float, y: float):
ctypes.windll.user32.SetCursorPos(int(x), int(y))
def os_cursor_click():
MOUSEEVENTF_LEFTDOWN = 0x0002
MOUSEEVENTF_LEFTUP = 0x0004
ctypes.windll.user32.mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0)
ctypes.windll.user32.mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0)
def os_cursor_position():
pt = POINT()
ctypes.windll.user32.GetCursorPos(ctypes.byref(pt))
return pt.x, pt.y
def get_screen_size():
"""Get full virtual screen size."""
w = ctypes.windll.user32.GetSystemMetrics(78) # SM_CXVIRTUALSCREEN
h = ctypes.windll.user32.GetSystemMetrics(79) # SM_CYVIRTUALSCREEN
if w == 0 or h == 0:
w = ctypes.windll.user32.GetSystemMetrics(0) # SM_CXSCREEN
h = ctypes.windll.user32.GetSystemMetrics(1) # SM_CYSCREEN
return w, h
# ---------------------------------------------------------------------------
# Allowed keys
# ---------------------------------------------------------------------------
ALLOWED_KEYS = {
"ArrowUp", "ArrowDown", "ArrowLeft", "ArrowRight",
"Enter", "Escape", "Backspace", "Tab",
"MediaPlayPause", "MediaTrackNext", "MediaTrackPrevious",
"k", "j", "l", "f", "m", " ",
}
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
async def get_all_pages() -> list[Page]:
if browser is None:
return []
pages = []
for ctx in browser.contexts:
pages.extend(ctx.pages)
return pages
async def find_target_page() -> Optional[Page]:
pages = await get_all_pages()
return pages[0] if pages else None
async def get_viewport():
vp = page.viewport_size
if vp and vp.get("width"):
return vp
return await page.evaluate("({width: window.innerWidth, height: window.innerHeight})")
async def ensure_connection():
global pw_instance, browser, page
if page is not None:
try:
await page.title()
return
except Exception:
log.warning("Lost connection, reconnecting...")
page = None
browser = None
if pw_instance is None:
pw_instance = await async_playwright().start()
cdp_url = f"http://localhost:{config.CDP_PORT}"
log.info(f"Connecting to CDP at {cdp_url} ...")
try:
browser = await pw_instance.chromium.connect_over_cdp(cdp_url)
except Exception as e:
log.error(f"CDP connection failed: {e}")
raise HTTPException(status_code=503, detail=f"Cannot connect to Brave CDP: {e}")
page = await find_target_page()
if page is None:
raise HTTPException(status_code=503, detail="No browser page found")
log.info(f"Connected! {await page.title()} | {page.url}")
def launch_browser():
"""Launch browser process from config (server is the boss)."""
global browser_process
cfg = config.load()
browser_path = cfg["brave_path"]
cdp_port = cfg["cdp_port"]
default_url = cfg["default_url"]
user_data_dir = cfg["user_data_dir"]
monitors = cfg.get("monitors", {})
proj_mon = str(cfg.get("projector_monitor", 0))
if not os.path.isfile(browser_path):
log.error(f"Browser not found: {browser_path}")
return
args = [
browser_path,
f"--remote-debugging-port={cdp_port}",
"--start-fullscreen",
"--no-first-run",
"--disable-features=TranslateUI",
"--autoplay-policy=no-user-gesture-required",
]
if user_data_dir and os.path.isdir(user_data_dir):
args.append(f"--user-data-dir={user_data_dir}")
mon = monitors.get(proj_mon, monitors.get(int(proj_mon) if proj_mon.isdigit() else -1))
if mon:
args.append(f"--window-position={mon['left']},{mon['top']}")
args.append(default_url)
try:
browser_process = subprocess.Popen(
args,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
creationflags=subprocess.DETACHED_PROCESS,
)
log.info(f"Browser launched (PID {browser_process.pid})")
except Exception as e:
log.error(f"Failed to launch browser: {e}")
async def smart_navigate(url: str) -> Page:
global page
await ensure_connection()
target_domain = urlparse(url).netloc.replace("www.", "")
pages = await get_all_pages()
for p in pages:
try:
pd = urlparse(p.url).netloc.replace("www.", "")
if target_domain and target_domain in pd:
page = p
await page.bring_to_front()
if p.url.rstrip("/") != url.rstrip("/"):
await page.goto(url, wait_until="domcontentloaded", timeout=15000)
log.info(f"Reused tab for {url}")
return page
except Exception:
continue
await page.goto(url, wait_until="domcontentloaded", timeout=15000)
log.info(f"Navigated to {url}")
return page
def get_local_ip() -> str:
"""Get the real LAN IP by connecting to an external address (no traffic sent)."""
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(("8.8.8.8", 80))
ip = s.getsockname()[0]
s.close()
return ip
except Exception:
return "127.0.0.1"
# ---------------------------------------------------------------------------
# HTTP -> HTTPS redirect server
# ---------------------------------------------------------------------------
def _find_ssl_files() -> tuple[str | None, str | None]:
"""Locate key.pem and cert.pem."""
search_dirs = []
if getattr(sys, 'frozen', False):
search_dirs.append(os.path.dirname(sys.executable))
search_dirs.append(getattr(sys, '_MEIPASS', ''))
else:
search_dirs.append(os.path.dirname(os.path.abspath(__file__)))
for d in search_dirs:
if not d:
continue
key = os.path.join(d, 'key.pem')
cert = os.path.join(d, 'cert.pem')
if os.path.isfile(key) and os.path.isfile(cert):
return key, cert
return None, None
async def _http_redirect_handler(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
"""Handle an HTTP request and redirect it to HTTPS."""
try:
# Read the request line
request_line = await asyncio.wait_for(reader.readline(), timeout=5.0)
if not request_line:
writer.close()
return
# Parse the request to get the path
parts = request_line.decode('utf-8', errors='ignore').split()
path = parts[1] if len(parts) > 1 else '/'
# Read and discard headers
while True:
line = await asyncio.wait_for(reader.readline(), timeout=5.0)
if line in (b'\r\n', b'\n', b''):
break
# Build redirect URL
local_ip = get_local_ip()
https_port = config.PORT
redirect_url = f"https://{local_ip}:{https_port}{path}"
# Send redirect response
response = (
f"HTTP/1.1 301 Moved Permanently\r\n"
f"Location: {redirect_url}\r\n"
f"Content-Length: 0\r\n"
f"Connection: close\r\n"
f"\r\n"
)
writer.write(response.encode('utf-8'))
await writer.drain()
except Exception:
pass
finally:
try:
writer.close()
await writer.wait_closed()
except Exception:
pass
async def start_http_redirect_server(http_port: int = 80) -> asyncio.Server | None:
"""Start a simple HTTP server that redirects all requests to HTTPS."""
try:
server = await asyncio.start_server(
_http_redirect_handler,
host='0.0.0.0',
port=http_port,
)
log.info(f"HTTP redirect server started on port {http_port} -> HTTPS:{config.PORT}")
return server
except PermissionError:
log.warning(f"Cannot bind to port {http_port} (requires admin). HTTP redirect disabled.")
return None
except OSError as e:
if e.errno == 10048: # Port already in use (Windows)
log.warning(f"Port {http_port} already in use. HTTP redirect disabled.")
else:
log.warning(f"Could not start HTTP redirect server: {e}")
return None
except Exception as e:
log.warning(f"HTTP redirect server failed: {e}")
return None
# ---------------------------------------------------------------------------
# Lifespan
# ---------------------------------------------------------------------------
@asynccontextmanager
async def lifespan(app: FastAPI):
log.info("=== browser remote v4 starting ===")
# Check if browser launch should be skipped
no_browser = (
os.environ.get("BR_NO_BROWSER", "") == "1"
or config.load().get("no_browser", False)
)
if no_browser:
log.info("Server-only mode — browser will NOT be launched automatically.")
# Still try connecting to an already-running browser
try:
await ensure_connection()
log.info("Connected to existing browser")
except Exception:
log.info("No browser running — waiting for remote launch via /api/start")
else:
# Try connecting to an existing browser
try:
await ensure_connection()
log.info("Connected to existing browser")
except Exception:
log.info("No browser found — launching from config...")
try:
launch_browser()
await asyncio.sleep(3)
await ensure_connection()
log.info("Browser launched and connected")
await _open_startup_urls()
except Exception as e:
log.warning(f"Browser launch/connect failed: {e} — server in standby")
local_ip = get_local_ip()
# Start HTTP redirect server if SSL is enabled
http_redirect_server = None
key_file, cert_file = _find_ssl_files()
if key_file:
# SSL enabled - start HTTP redirect on port 80
http_redirect_server = await start_http_redirect_server(80)
log.info(f" Local: https://localhost:{config.PORT}")
log.info(f" Network: https://{local_ip}:{config.PORT}")
else:
log.info(f" Local: http://localhost:{config.PORT}")
log.info(f" Network: http://{local_ip}:{config.PORT}")
yield
# Cleanup redirect server
if http_redirect_server:
http_redirect_server.close()
await http_redirect_server.wait_closed()
log.info("HTTP redirect server stopped")
global pw_instance, browser, page
if browser:
try: await browser.close()
except: pass
if pw_instance:
await pw_instance.stop()
log.info("=== Stopped ===")
# ---------------------------------------------------------------------------
# App
# ---------------------------------------------------------------------------
app = FastAPI(title="browser remote", lifespan=lifespan)
static_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "static")
os.makedirs(static_dir, exist_ok=True)
app.mount("/static", StaticFiles(directory=static_dir), name="static")
# ---------------------------------------------------------------------------
# Models
# ---------------------------------------------------------------------------
class KeyPress(BaseModel):
key: str
class TypeText(BaseModel):
text: str
class Navigate(BaseModel):
url: str
class ClickAction(BaseModel):
x: float
y: float
class ScrollAction(BaseModel):
x: float
y: float
deltaX: float = 0
deltaY: float = 0
class CursorMove(BaseModel):
dx: float = 0
dy: float = 0
class CursorSet(BaseModel):
x: float
y: float
class CursorScrollAction(BaseModel):
deltaX: float = 0
deltaY: float = 0
class ModeSwitch(BaseModel):
mode: str # "browser" or "pc"
class MonitorSwitch(BaseModel):
monitor: int
# ---------------------------------------------------------------------------
# Frontend
# ---------------------------------------------------------------------------
@app.get("/", response_class=HTMLResponse)
async def serve_index():
return FileResponse(os.path.join(static_dir, "index.html"))
# ---------------------------------------------------------------------------
# Status
# ---------------------------------------------------------------------------
@app.get("/api/status")
async def status():
global page
connected = False
title = url = ""
vp_w = vp_h = 0
browser_alive = browser_process is not None and browser_process.poll() is None
if page:
try:
title = await page.title()
url = page.url
vp = await get_viewport()
vp_w, vp_h = vp["width"], vp["height"]
connected = True
except: pass
monitors = detect_monitors()
return {"connected": connected, "title": title, "url": url,
"viewport": {"width": vp_w, "height": vp_h},
"browser_alive": browser_alive, "pc_mode": pc_mode,
"pc_monitor": pc_monitor, "monitors": monitors}
@app.post("/api/connect")
async def connect():
await ensure_connection()
return {"status": "connected", "title": await page.title() if page else ""}
@app.post("/api/start")
async def start_browser_endpoint():
"""Launch / re-launch the browser (standby → active)."""
global page, browser
# Reset stale connection
page = None
browser = None
launch_browser()
# Wait for browser to init, then connect
for attempt in range(6):
await asyncio.sleep(1)
try:
await ensure_connection()
return {"status": "ok", "title": await page.title() if page else ""}
except Exception:
pass
return {"status": "starting", "message": "Browser launching — refresh in a few seconds"}
@app.post("/api/mode")
async def set_mode(data: ModeSwitch):
"""Toggle between Browser mode and PC mode."""
global pc_mode
pc_mode = (data.mode == "pc")
log.info(f"Mode switched to: {'PC' if pc_mode else 'Browser'}")
return {"mode": "pc" if pc_mode else "browser"}
@app.get("/api/mode")
async def get_mode():
return {"mode": "pc" if pc_mode else "browser"}
@app.get("/api/monitors")
async def list_monitors():
"""List all detected monitors."""
monitors = detect_monitors()
return {"monitors": monitors, "active": pc_monitor}
@app.post("/api/pc-monitor")
async def set_pc_monitor(data: MonitorSwitch):
"""Set the active monitor for PC mode."""
global pc_monitor
monitors = detect_monitors()
if data.monitor < 0 or data.monitor >= len(monitors):
raise HTTPException(400, f"Monitor {data.monitor} out of range (0-{len(monitors)-1})")
pc_monitor = data.monitor
mon = monitors[pc_monitor]
log.info(f"PC monitor set to {pc_monitor}: {mon['width']}x{mon['height']} at ({mon['left']},{mon['top']})")
return {"status": "ok", "monitor": pc_monitor, "info": mon}
@app.get("/api/pc-monitor")
async def get_pc_monitor():
monitors = detect_monitors()
idx = pc_monitor if 0 <= pc_monitor < len(monitors) else 0
return {"monitor": idx, "info": monitors[idx]}
# ---------------------------------------------------------------------------
# Tabs
# ---------------------------------------------------------------------------
@app.get("/api/tabs")
async def list_tabs():
await ensure_connection()
pages = await get_all_pages()
tabs = []
for i, p in enumerate(pages):
try: tabs.append({"index": i, "title": await p.title(), "url": p.url, "active": p == page})
except: tabs.append({"index": i, "title": "?", "url": "?", "active": False})
return {"tabs": tabs}
@app.post("/api/tabs/{index}")
async def switch_tab(index: int):
global page
await ensure_connection()
pages = await get_all_pages()
if index < 0 or index >= len(pages):
raise HTTPException(400, f"Tab {index} out of range")
page = pages[index]
await page.bring_to_front()
return {"status": "ok", "title": await page.title(), "url": page.url}
# ---------------------------------------------------------------------------
# Mouse (Mirror mode core)
# ---------------------------------------------------------------------------
@app.post("/api/click")
async def click(data: ClickAction):
if pc_mode:
mon = get_active_monitor()
ax = mon["left"] + data.x * mon["width"]
ay = mon["top"] + data.y * mon["height"]
os_cursor_set(ax, ay)
os_cursor_click()
log.info(f"PC Click ({ax:.0f}, {ay:.0f}) on monitor {pc_monitor}")
return {"status": "ok", "x": ax, "y": ay}
await ensure_connection()
vp = await get_viewport()
ax, ay = data.x * vp["width"], data.y * vp["height"]
await page.mouse.click(ax, ay)
log.info(f"Click ({ax:.0f}, {ay:.0f})")
return {"status": "ok", "x": ax, "y": ay}
@app.post("/api/dblclick")
async def dblclick(data: ClickAction):
await ensure_connection()
vp = await get_viewport()
ax, ay = data.x * vp["width"], data.y * vp["height"]
await page.mouse.dblclick(ax, ay)
return {"status": "ok"}
@app.post("/api/hover")
async def hover(data: ClickAction):
await ensure_connection()
vp = await get_viewport()
await page.mouse.move(data.x * vp["width"], data.y * vp["height"])
return {"status": "ok"}
@app.post("/api/scroll")
async def scroll(data: ScrollAction):
if pc_mode:
mon = get_active_monitor()
ax = mon["left"] + data.x * mon["width"]
ay = mon["top"] + data.y * mon["height"]
os_cursor_set(ax, ay)
# Windows mouse_event scroll: MOUSEEVENTF_WHEEL=0x0800, delta in multiples of 120
MOUSEEVENTF_WHEEL = 0x0800
MOUSEEVENTF_HWHEEL = 0x01000
if data.deltaY != 0:
wheel_delta = int(-data.deltaY) # deltaY comes as px-like; negate for natural scroll
ctypes.windll.user32.mouse_event(MOUSEEVENTF_WHEEL, 0, 0, wheel_delta, 0)
if data.deltaX != 0:
wheel_delta = int(data.deltaX)
ctypes.windll.user32.mouse_event(MOUSEEVENTF_HWHEEL, 0, 0, wheel_delta, 0)
log.info(f"PC Scroll dy={data.deltaY:.0f}")
return {"status": "ok"}
await ensure_connection()
vp = await get_viewport()
await page.mouse.move(data.x * vp["width"], data.y * vp["height"])
await page.mouse.wheel(data.deltaX, data.deltaY)
log.info(f"Scroll dy={data.deltaY:.0f}")
return {"status": "ok"}
# ---------------------------------------------------------------------------
# Fake Cursor (Trackpad mode)
# ---------------------------------------------------------------------------
# Server-side cursor position in viewport pixels
cursor_pos = {"x": 0.0, "y": 0.0, "visible": False}
# Cursor auto-hide: hide after configurable delay of no movement, re-show on movement
cursor_last_move_time: float = 0.0
cursor_hide_task: Optional[asyncio.Task] = None
CURSOR_INJECT_JS = """
() => {
if (document.getElementById('__rc_cursor')) return;
// Inject ripple keyframes once
if (!document.getElementById('__rc_styles')) {
const st = document.createElement('style');
st.id = '__rc_styles';
st.textContent = `
@keyframes __rc_ripple {
0% { transform: translate(-50%,-50%) scale(0.3); opacity: 1; }
100% { transform: translate(-50%,-50%) scale(2.5); opacity: 0; }
}
.__rc_click_ring {
position: fixed; pointer-events: none; z-index: 2147483646;
width: 40px; height: 40px; border-radius: 50%;
border: 3px solid rgba(233,69,96,0.9);
transform: translate(-50%,-50%) scale(0.3);
animation: __rc_ripple 0.45s ease-out forwards;
}
`;
document.documentElement.appendChild(st);
}
const c = document.createElement('div');
c.id = '__rc_cursor';
c.style.cssText = `
position: fixed; z-index: 2147483647;
width: 32px; height: 32px; pointer-events: none;
left: 50%; top: 50%;
filter: drop-shadow(0 2px 4px rgba(0,0,0,0.5));
will-change: left, top;
`;
c.innerHTML = `<svg width="32" height="32" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M4 2L4 20L8.5 15.5L12.5 22L15 20.5L11 14L17 14L4 2Z"
fill="white" stroke="black" stroke-width="1.2" stroke-linejoin="round"/>
</svg>`;
document.documentElement.appendChild(c);
}
"""
CURSOR_UPDATE_JS = """
(pos) => {
const c = document.getElementById('__rc_cursor');
if (!c) return false;
c.style.left = pos.x + 'px';
c.style.top = pos.y + 'px';
return true;
}
"""
CURSOR_CLICK_RIPPLE_JS = """
(pos) => {
const ring = document.createElement('div');
ring.className = '__rc_click_ring';
ring.style.left = pos.x + 'px';
ring.style.top = pos.y + 'px';
document.documentElement.appendChild(ring);
setTimeout(() => ring.remove(), 500);
}
"""
CURSOR_REMOVE_JS = """
() => {
const c = document.getElementById('__rc_cursor');
if (c) c.remove();
const st = document.getElementById('__rc_styles');
if (st) st.remove();
document.querySelectorAll('.__rc_click_ring').forEach(e => e.remove());
}
"""
@app.post("/api/cursor/show")
async def cursor_show():
"""Inject the fake cursor into the page DOM and center it."""
global cursor_pos
if pc_mode:
cursor_pos["visible"] = True
x, y = os_cursor_position()
cursor_pos["x"], cursor_pos["y"] = float(x), float(y)
return {"status": "ok", "x": cursor_pos["x"], "y": cursor_pos["y"]}
await ensure_connection()
vp = await get_viewport()
cursor_pos = {"x": vp["width"] / 2, "y": vp["height"] / 2, "visible": True}
await page.evaluate(CURSOR_INJECT_JS)
await page.evaluate(CURSOR_UPDATE_JS, {"x": cursor_pos["x"], "y": cursor_pos["y"]})
log.info("Cursor shown")
return {"status": "ok", "x": cursor_pos["x"], "y": cursor_pos["y"]}
@app.post("/api/cursor/hide")
async def cursor_hide():
"""Remove the fake cursor from the DOM."""
global cursor_pos
cursor_pos["visible"] = False
if not pc_mode:
await ensure_connection()
await page.evaluate(CURSOR_REMOVE_JS)
log.info("Cursor hidden")
return {"status": "ok"}
@app.post("/api/cursor/move")
async def cursor_move(data: CursorMove):
"""Move cursor by relative delta."""
global cursor_pos
if pc_mode:
os_cursor_move_rel(data.dx, data.dy)
x, y = os_cursor_position()
cursor_pos["x"], cursor_pos["y"] = float(x), float(y)
return {"status": "ok", "x": cursor_pos["x"], "y": cursor_pos["y"]}
await ensure_connection()
vp = await get_viewport()
cursor_pos["x"] = max(0, min(vp["width"], cursor_pos["x"] + data.dx))
cursor_pos["y"] = max(0, min(vp["height"], cursor_pos["y"] + data.dy))
await page.evaluate(CURSOR_UPDATE_JS, {"x": cursor_pos["x"], "y": cursor_pos["y"]})
return {"status": "ok", "x": cursor_pos["x"], "y": cursor_pos["y"]}
@app.post("/api/cursor/set")
async def cursor_set(data: CursorSet):
"""Set cursor to absolute position."""
global cursor_pos
if pc_mode:
os_cursor_set(data.x, data.y)
cursor_pos["x"], cursor_pos["y"] = data.x, data.y
return {"status": "ok", "x": cursor_pos["x"], "y": cursor_pos["y"]}
await ensure_connection()
vp = await get_viewport()
cursor_pos["x"] = max(0, min(vp["width"], data.x))
cursor_pos["y"] = max(0, min(vp["height"], data.y))
await page.evaluate(CURSOR_UPDATE_JS, {"x": cursor_pos["x"], "y": cursor_pos["y"]})
return {"status": "ok", "x": cursor_pos["x"], "y": cursor_pos["y"]}
@app.post("/api/cursor/click")
async def cursor_click():
"""Click at the current cursor position with visual ripple."""
global cursor_pos
if pc_mode:
os_cursor_click()
x, y = os_cursor_position()
cursor_pos["x"], cursor_pos["y"] = float(x), float(y)
log.info(f"PC cursor click ({x}, {y})")
return {"status": "ok", "x": cursor_pos["x"], "y": cursor_pos["y"]}
await ensure_connection()
if not cursor_pos["visible"]:
raise HTTPException(400, "Cursor not visible")
x, y = cursor_pos["x"], cursor_pos["y"]
await page.evaluate(CURSOR_CLICK_RIPPLE_JS, {"x": x, "y": y})
await page.mouse.click(x, y)
log.info(f"Cursor click ({x:.0f}, {y:.0f})")
return {"status": "ok", "x": x, "y": y}
@app.post("/api/cursor/scroll")
async def cursor_scroll(data: CursorScrollAction):
"""Scroll at the current cursor position."""
global cursor_pos
if pc_mode:
MOUSEEVENTF_WHEEL = 0x0800
MOUSEEVENTF_HWHEEL = 0x01000
if data.deltaY != 0:
ctypes.windll.user32.mouse_event(
MOUSEEVENTF_WHEEL, 0, 0, int(-data.deltaY), 0)
if data.deltaX != 0:
ctypes.windll.user32.mouse_event(
MOUSEEVENTF_HWHEEL, 0, 0, int(data.deltaX), 0)
log.info(f"PC cursor scroll dy={data.deltaY:.0f}")
return {"status": "ok"}
await ensure_connection()
await page.mouse.move(cursor_pos["x"], cursor_pos["y"])
await page.mouse.wheel(data.deltaX, data.deltaY)
log.info(f"Cursor scroll dy={data.deltaY:.0f}")
return {"status": "ok"}
@app.get("/api/cursor/position")
async def cursor_position():
"""Get current cursor position."""
return cursor_pos
# ---------------------------------------------------------------------------
# Screenshot (fallback)
# ---------------------------------------------------------------------------
@app.get("/api/screenshot.jpg")
async def screenshot_jpg(quality: int = 35):
await ensure_connection()
q = max(10, min(quality, 90))
img = await page.screenshot(type="jpeg", quality=q)
return StreamingResponse(
iter([img]), media_type="image/jpeg",
headers={"Cache-Control": "no-cache, no-store"},
)
# ---------------------------------------------------------------------------
# WebSocket Screencast (CDP Page.screencastFrame)
# ---------------------------------------------------------------------------
@app.websocket("/ws/screencast")
async def ws_screencast(websocket: WebSocket):
"""
Stream live frames via CDP Page.startScreencast.
Much more efficient than polling screenshots — browser only encodes
frames when the page actually changes.
"""
await websocket.accept()
await ensure_connection()
cdp = await page.context.new_cdp_session(page)
frame_queue: asyncio.Queue = asyncio.Queue(maxsize=2)
async def on_frame(params):
data = params.get("data", "")
session_id = params.get("sessionId", 0)
# Ack frame so CDP keeps sending
try:
await cdp.send("Page.screencastFrameAck", {"sessionId": session_id})
except Exception:
pass
# Drop frames if consumer is slow
if not frame_queue.full():
await frame_queue.put(data)
cdp.on("Page.screencastFrame", on_frame)
try:
await cdp.send("Page.startScreencast", {
"format": "jpeg",
"quality": 40,
"maxWidth": 1280,
"maxHeight": 720,
"everyNthFrame": 1,
})
while True:
data = await frame_queue.get()
await websocket.send_text(data)
except (WebSocketDisconnect, Exception) as e:
log.info(f"Screencast WS closed: {e}")
finally:
try:
await cdp.send("Page.stopScreencast")
await cdp.detach()
except Exception:
pass
# ---------------------------------------------------------------------------
# WebSocket Desktop Screencast (PC mode — mss multi-monitor capture)
# ---------------------------------------------------------------------------
@app.websocket("/ws/screencast-desktop")
async def ws_screencast_desktop(websocket: WebSocket):
"""
Stream entire desktop (all monitors stitched) as JPEG frames.
Uses mss for multi-monitor screenshots at ~10-15 FPS.
"""
await websocket.accept()
log.info("Desktop screencast WS connected")
try:
import mss
import mss.tools
from io import BytesIO
from PIL import Image
except ImportError:
log.error("Desktop screencast requires 'mss' and 'Pillow'. Install with: pip install mss Pillow")
await websocket.close(1011, "mss/Pillow not installed")
return
try:
with mss.mss() as sct:
log.info("Desktop capture started")
while True:
# Capture only the selected monitor
# mss.monitors: [0]=all combined, [1]=primary, [2]=second, ...
mon = get_active_monitor()
target = {
"left": mon["left"],
"top": mon["top"],
"width": mon["width"],
"height": mon["height"],
}
shot = sct.grab(target)
img = Image.frombytes("RGB", shot.size, shot.bgra, "raw", "BGRX")
# Scale down for bandwidth (max 1920 wide)
if img.width > 1920:
ratio = 1920 / img.width
img = img.resize(
(1920, int(img.height * ratio)),
Image.LANCZOS,
)
buf = BytesIO()
img.save(buf, format="JPEG", quality=40, optimize=True)
b64 = base64.b64encode(buf.getvalue()).decode("ascii")
await websocket.send_text(b64)
await asyncio.sleep(0.07) # ~14 FPS cap
except (WebSocketDisconnect, Exception) as e:
log.info(f"Desktop screencast WS closed: {e}")
# ---------------------------------------------------------------------------
# WebSocket Gyro Cursor (air-mouse velocity-based)
# ---------------------------------------------------------------------------
@app.websocket("/ws/gyro")
async def ws_gyro(websocket: WebSocket):
"""
Air-mouse gyro: receives {dx, dy} velocity-based deltas from DeviceMotionEvent
rotationRate. Server accumulates position. Also handles {type:"click"}.
Works in both Browser mode (Playwright DOM cursor) and PC mode (OS cursor).
"""
global cursor_pos
await websocket.accept()
log.info("Gyro WS connected")
# Resolve bounds once
vp = None
max_x, max_y = 1920.0, 1080.0