From 4300907befb2e0d978a6ad4fcc7d5f7a43f9d9d2 Mon Sep 17 00:00:00 2001 From: steadying Date: Sat, 2 May 2026 00:26:47 +0100 Subject: [PATCH 01/77] add USB HID touchscreen digitizer backend --- config.go | 1 + internal/usbgadget/config.go | 4 + internal/usbgadget/hid_touchscreen.go | 114 ++++++++++++++++++++++++++ internal/usbgadget/usbgadget.go | 32 ++++++-- jsonrpc.go | 3 + usb.go | 4 + 6 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 internal/usbgadget/hid_touchscreen.go diff --git a/config.go b/config.go index 32b3b659b..8a4256b71 100644 --- a/config.go +++ b/config.go @@ -176,6 +176,7 @@ var ( AbsoluteMouse: true, RelativeMouse: true, Keyboard: true, + Touchscreen: true, MassStorage: true, Audio: true, } diff --git a/internal/usbgadget/config.go b/internal/usbgadget/config.go index 6af97b04e..0d3f5ad44 100644 --- a/internal/usbgadget/config.go +++ b/internal/usbgadget/config.go @@ -60,6 +60,8 @@ var defaultGadgetConfig = map[string]gadgetConfigItem{ "relative_mouse": relativeMouseConfig, // USB audio sink "audio": audioConfig, + // touchscreen/digitizer HID + "touchscreen": touchscreenConfig, // mass storage "mass_storage_base": massStorageBaseConfig, "mass_storage_lun0": massStorageLun0Config, @@ -75,6 +77,8 @@ func (u *UsbGadget) isGadgetConfigItemEnabled(itemKey string) bool { return u.enabledDevices.RelativeMouse case "keyboard": return u.enabledDevices.Keyboard + case "touchscreen": + return u.enabledDevices.Touchscreen case "mass_storage_base": return u.enabledDevices.MassStorage case "mass_storage_lun0": diff --git a/internal/usbgadget/hid_touchscreen.go b/internal/usbgadget/hid_touchscreen.go new file mode 100644 index 000000000..d3bb04d10 --- /dev/null +++ b/internal/usbgadget/hid_touchscreen.go @@ -0,0 +1,114 @@ +package usbgadget + +import ( + "fmt" + "os" +) + +var touchscreenConfig = gadgetConfigItem{ + order: 1003, + device: "hid.usb3", + path: []string{"functions", "hid.usb3"}, + configPath: []string{"hid.usb3"}, + attrs: gadgetAttributes{ + "protocol": "0", + "subclass": "0", + "report_length": "5", + "no_out_endpoint": "1", + "wakeup_on_write": "1", + }, + reportDesc: touchscreenReportDesc, +} + +// Single-touch digitizer (Android-compatible baseline) +var touchscreenReportDesc = []byte{ + 0x05, 0x0D, + 0x09, 0x04, + 0xA1, 0x01, + + 0x09, 0x22, + 0xA1, 0x02, + + 0x09, 0x42, + 0x15, 0x00, + 0x25, 0x01, + 0x75, 0x01, + 0x95, 0x01, + 0x81, 0x02, + + 0x09, 0x32, + 0x75, 0x01, + 0x95, 0x01, + 0x81, 0x02, + + 0x75, 0x01, + 0x95, 0x06, + 0x81, 0x03, + + 0x05, 0x01, + 0x09, 0x30, + 0x09, 0x31, + 0x16, 0x00, 0x00, + 0x26, 0xFF, 0x7F, + 0x36, 0x00, 0x00, + 0x46, 0xFF, 0x7F, + 0x75, 0x10, + 0x95, 0x02, + 0x81, 0x02, + + 0xC0, + 0xC0, +} + +func (u *UsbGadget) touchscreenWriteHidFile(data []byte) error { + if u.touchscreenHidFile == nil { + var err error + u.touchscreenHidFile, err = os.OpenFile("/dev/hidg3", os.O_RDWR, 0666) + if err != nil { + return fmt.Errorf("failed to open hidg3: %w", err) + } + } + + _, err := u.writeWithTimeout(u.touchscreenHidFile, data) + if err != nil { + u.touchscreenHidFile.Close() + u.touchscreenHidFile = nil + return err + } + return nil +} + +func (u *UsbGadget) HasTouchscreen() bool { + return u.enabledDevices.Touchscreen +} + +func (u *UsbGadget) TouchscreenReport(x int, y int, touching bool) error { + if !u.enabledDevices.Touchscreen { + return nil + } + + if x < 0 { + x = 0 + } else if x > 32767 { + x = 32767 + } + + if y < 0 { + y = 0 + } else if y > 32767 { + y = 32767 + } + + flags := byte(0) + if touching { + flags = 0x03 + } + + return u.touchscreenWriteHidFile([]byte{ + flags, + byte(x), + byte(x >> 8), + byte(y), + byte(y >> 8), + }) +} diff --git a/internal/usbgadget/usbgadget.go b/internal/usbgadget/usbgadget.go index bb4a9f98a..e96d04d53 100644 --- a/internal/usbgadget/usbgadget.go +++ b/internal/usbgadget/usbgadget.go @@ -19,6 +19,7 @@ type Devices struct { AbsoluteMouse bool `json:"absolute_mouse"` RelativeMouse bool `json:"relative_mouse"` Keyboard bool `json:"keyboard"` + Touchscreen bool `json:"touchscreen"` MassStorage bool `json:"mass_storage"` SerialConsole bool `json:"serial_console"` Audio bool `json:"audio"` @@ -41,6 +42,7 @@ var defaultUsbGadgetDevices = Devices{ AbsoluteMouse: true, RelativeMouse: true, Keyboard: true, + Touchscreen: true, MassStorage: true, Audio: false, } @@ -62,14 +64,16 @@ type UsbGadget struct { configLock sync.Mutex - keyboardHidFile *os.File - keyboardLock sync.Mutex - wakeHidFile *os.File - wakeHidLock sync.Mutex - absMouseHidFile *os.File - absMouseLock sync.Mutex - relMouseHidFile *os.File - relMouseLock sync.Mutex + keyboardHidFile *os.File + keyboardLock sync.Mutex + wakeHidFile *os.File + wakeHidLock sync.Mutex + absMouseHidFile *os.File + absMouseLock sync.Mutex + relMouseHidFile *os.File + relMouseLock sync.Mutex + touchscreenHidFile *os.File + touchscreenHidLock sync.Mutex keyboardState byte // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana) keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys) @@ -137,6 +141,7 @@ func newUsbGadget(name string, configMap map[string]gadgetConfigItem, enabledDev wakeHidLock: sync.Mutex{}, absMouseLock: sync.Mutex{}, relMouseLock: sync.Mutex{}, + touchscreenHidLock: sync.Mutex{}, txLock: sync.Mutex{}, keyboardStateCtx: keyboardCtx, keyboardStateCancel: keyboardCancel, @@ -195,6 +200,10 @@ func (u *UsbGadget) Close() error { u.relMouseHidFile.Close() u.relMouseHidFile = nil } + if u.touchscreenHidFile != nil { + u.touchscreenHidFile.Close() + u.touchscreenHidFile = nil + } return nil } @@ -224,4 +233,11 @@ func (u *UsbGadget) ResetHIDFiles() { u.relMouseHidFile = nil } unlockWithLog(&u.relMouseLock, u.log, "relMouseHidFile reset") + + u.touchscreenHidLock.Lock() + if u.touchscreenHidFile != nil { + u.touchscreenHidFile.Close() + u.touchscreenHidFile = nil + } + unlockWithLog(&u.touchscreenHidLock, u.log, "touchscreenHidFile reset") } diff --git a/jsonrpc.go b/jsonrpc.go index 7a60656e1..c98f132cc 100644 --- a/jsonrpc.go +++ b/jsonrpc.go @@ -1012,6 +1012,8 @@ func rpcSetUsbDeviceState(device string, enabled bool) error { config.UsbDevices.RelativeMouse = enabled case "keyboard": config.UsbDevices.Keyboard = enabled + case "touchscreen": + config.UsbDevices.Touchscreen = enabled case "massStorage": config.UsbDevices.MassStorage = enabled case "serialConsole": @@ -1344,6 +1346,7 @@ var rpcHandlers = map[string]RPCHandler{ "keypressReport": {Func: rpcKeypressReport, Params: []string{"key", "press"}}, "absMouseReport": {Func: rpcAbsMouseReport, Params: []string{"x", "y", "buttons"}}, "relMouseReport": {Func: rpcRelMouseReport, Params: []string{"dx", "dy", "buttons"}}, + "touchscreenReport": {Func: rpcTouchscreenReport, Params: []string{"x", "y", "touching"}}, "wheelReport": {Func: rpcWheelReport, Params: []string{"wheelY", "wheelX"}}, "wakeHost": {Func: rpcWakeHost}, "getVideoState": {Func: rpcGetVideoState}, diff --git a/usb.go b/usb.go index 843abcabc..7cb8157a6 100644 --- a/usb.go +++ b/usb.go @@ -93,6 +93,10 @@ func rpcRelMouseReport(dx int8, dy int8, buttons uint8) error { return rpcHidReport(func() error { return gadget.RelMouseReport(dx, dy, buttons) }) } +func rpcTouchscreenReport(x int, y int, touching bool) error { + return rpcHidReport(func() error { return gadget.TouchscreenReport(x, y, touching) }) +} + func rpcWheelReport(wheelY int8, wheelX int8) error { return rpcHidReport(func() error { if gadget.HasAbsoluteMouse() { From 7823c93a0c10e3d70e3e02d52d704ce3e4b84691 Mon Sep 17 00:00:00 2001 From: steadying Date: Sat, 2 May 2026 00:33:15 +0100 Subject: [PATCH 02/77] add temporary PicPhone touchscreen UI routing --- ui/src/components/WebRTCVideo.tsx | 45 +++++++++++++++++++++++++------ ui/src/hooks/useMouse.ts | 43 +++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 36a4be61f..1e3a1bfd9 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -4,7 +4,7 @@ import { useResizeObserver } from "usehooks-ts"; import { cx } from "@/cva.config"; import { isWindows } from "@/utils"; import useKeyboard from "@hooks/useKeyboard"; -import useMouse from "@hooks/useMouse"; +import useMouse, { isPicphoneTouchscreenMode } from "@hooks/useMouse"; import { useRTCStore, useSettingsStore, useUiStore, useVideoStore } from "@hooks/stores"; import { JsonRpcResponse, useJsonRpc } from "@hooks/useJsonRpc"; import VirtualKeyboard from "@components/VirtualKeyboard"; @@ -53,6 +53,7 @@ export default function WebRTCVideo({ const { getRelMouseMoveHandler, getAbsMouseMoveHandler, + getTouchscreenMoveHandler, getMouseWheelHandler, resetMousePosition, } = useMouse(); @@ -249,7 +250,9 @@ export default function WebRTCVideo({ const abortController = new AbortController(); const signal = abortController.signal; - document.addEventListener("pointerlockchange", handlePointerLockChange, { signal }); + document.addEventListener("pointerlockchange", handlePointerLockChange, { + signal, + }); return () => { abortController.abort(); @@ -297,6 +300,17 @@ export default function WebRTCVideo({ const relMouseMoveHandler = useMemo(() => getRelMouseMoveHandler(), [getRelMouseMoveHandler]); + const touchscreenMoveHandler = useMemo( + () => + getTouchscreenMoveHandler({ + videoClientWidth, + videoClientHeight, + videoWidth, + videoHeight, + }), + [getTouchscreenMoveHandler, videoClientWidth, videoClientHeight, videoWidth, videoHeight], + ); + const mouseWheelHandler = useMemo(() => getMouseWheelHandler(), [getMouseWheelHandler]); function getAdjustedKeyCode(e: KeyboardEvent) { @@ -537,7 +551,9 @@ export default function WebRTCVideo({ document.addEventListener("keyup", keyUpHandler, { signal }); window.addEventListener("blur", resetKeyboardState, { signal }); - document.addEventListener("visibilitychange", resetKeyboardState, { signal }); + document.addEventListener("visibilitychange", resetKeyboardState, { + signal, + }); return () => { abortController.abort(); @@ -556,7 +572,9 @@ export default function WebRTCVideo({ const signal = abortController.signal; // To prevent the video from being paused when the user presses a space in fullscreen mode - videoElmRefValue.addEventListener("keydown", videoKeyDownHandler, { signal }); + videoElmRefValue.addEventListener("keydown", videoKeyDownHandler, { + signal, + }); videoElmRefValue.addEventListener("keyup", videoKeyUpHandler, { signal }); // We need to know when the video is playing to update state and video size @@ -576,13 +594,19 @@ export default function WebRTCVideo({ if (!videoElmRefValue) return; const isRelativeMouseMode = settings.mouseMode === "relative"; - const mouseHandler = isRelativeMouseMode ? relMouseMoveHandler : absMouseMoveHandler; + const mouseHandler = isPicphoneTouchscreenMode() + ? touchscreenMoveHandler + : isRelativeMouseMode + ? relMouseMoveHandler + : absMouseMoveHandler; const abortController = new AbortController(); const signal = abortController.signal; videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal }); - videoElmRefValue.addEventListener("pointerdown", mouseHandler, { signal }); + videoElmRefValue.addEventListener("pointerdown", mouseHandler, { + signal, + }); videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal }); videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { signal, @@ -602,11 +626,15 @@ export default function WebRTCVideo({ } else { // Reset the mouse position when the window is blurred or the document is hidden window.addEventListener("blur", resetMousePosition, { signal }); - document.addEventListener("visibilitychange", resetMousePosition, { signal }); + document.addEventListener("visibilitychange", resetMousePosition, { + signal, + }); } const preventContextMenu = (e: MouseEvent) => e.preventDefault(); - videoElmRefValue.addEventListener("contextmenu", preventContextMenu, { signal }); + videoElmRefValue.addEventListener("contextmenu", preventContextMenu, { + signal, + }); // Suppress browser Back/Forward navigation on X1/X2 mouse buttons so // those presses are forwarded to the remote target instead. @@ -627,6 +655,7 @@ export default function WebRTCVideo({ requestPointerLock, absMouseMoveHandler, relMouseMoveHandler, + touchscreenMoveHandler, mouseWheelHandler, resetMousePosition, settings.mouseMode, diff --git a/ui/src/hooks/useMouse.ts b/ui/src/hooks/useMouse.ts index e69d2a3d8..afb47d090 100644 --- a/ui/src/hooks/useMouse.ts +++ b/ui/src/hooks/useMouse.ts @@ -6,6 +6,9 @@ import { useMouseStore, useSettingsStore } from "./stores"; const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos); +export const isPicphoneTouchscreenMode = () => + typeof window !== "undefined" && window.localStorage.getItem("picphoneTouchscreen") === "1"; + export interface AbsMouseMoveHandlerProps { videoClientWidth: number; videoClientHeight: number; @@ -122,6 +125,45 @@ export default function useMouse() { [mouseMode, sendAbsMouseMovement], ); + const getTouchscreenMoveHandler = useCallback( + ({ videoClientWidth, videoClientHeight, videoWidth, videoHeight }: AbsMouseMoveHandlerProps) => + (e: MouseEvent) => { + if (!videoClientWidth || !videoClientHeight) return; + if (!isPicphoneTouchscreenMode()) return; + + const videoElementAspectRatio = videoClientWidth / videoClientHeight; + const videoStreamAspectRatio = videoWidth / videoHeight; + + let effectiveWidth = videoClientWidth; + let effectiveHeight = videoClientHeight; + let offsetX = 0; + let offsetY = 0; + + if (videoElementAspectRatio > videoStreamAspectRatio) { + effectiveWidth = videoClientHeight * videoStreamAspectRatio; + offsetX = (videoClientWidth - effectiveWidth) / 2; + } else if (videoElementAspectRatio < videoStreamAspectRatio) { + effectiveHeight = videoClientWidth / videoStreamAspectRatio; + offsetY = (videoClientHeight - effectiveHeight) / 2; + } + + const clampedX = Math.min(Math.max(offsetX, e.offsetX), offsetX + effectiveWidth); + const clampedY = Math.min(Math.max(offsetY, e.offsetY), offsetY + effectiveHeight); + + const relativeX = (clampedX - offsetX) / effectiveWidth; + const relativeY = (clampedY - offsetY) / effectiveHeight; + + const x = Math.round(relativeX * 32767); + const y = Math.round(relativeY * 32767); + const touching = e.buttons !== 0; + + send("touchscreenReport", { x, y, touching }); + setMousePosition(x, y); + lastAbsPos.current = { x, y }; + }, + [send, setMousePosition], + ); + const getMouseWheelHandler = useCallback( () => (e: WheelEvent) => { if (scrollThrottling && blockWheelEvent) { @@ -160,6 +202,7 @@ export default function useMouse() { return { getRelMouseMoveHandler, getAbsMouseMoveHandler, + getTouchscreenMoveHandler, getMouseWheelHandler, resetMousePosition, }; From b0cf662ae129f28e7438255ef52d35001dc0718e Mon Sep 17 00:00:00 2001 From: steadying Date: Sat, 2 May 2026 01:51:34 +0100 Subject: [PATCH 03/77] Expose touchscreen HID as Android direct touch digitizer --- internal/usbgadget/hid_touchscreen.go | 100 ++++++++++++++++---------- 1 file changed, 62 insertions(+), 38 deletions(-) diff --git a/internal/usbgadget/hid_touchscreen.go b/internal/usbgadget/hid_touchscreen.go index d3bb04d10..daa5bdbeb 100644 --- a/internal/usbgadget/hid_touchscreen.go +++ b/internal/usbgadget/hid_touchscreen.go @@ -13,51 +13,71 @@ var touchscreenConfig = gadgetConfigItem{ attrs: gadgetAttributes{ "protocol": "0", "subclass": "0", - "report_length": "5", + "report_length": "7", "no_out_endpoint": "1", "wakeup_on_write": "1", }, reportDesc: touchscreenReportDesc, } -// Single-touch digitizer (Android-compatible baseline) +// One-contact HID multitouch digitizer. +// +// Report layout, 7 bytes: +// +// byte 0: bit0 Tip Switch, bit1 In Range, bits2-7 padding +// byte 1: Contact Identifier +// byte 2-3: X, little-endian, 0..32767 +// byte 4-5: Y, little-endian, 0..32767 +// byte 6: Contact Count var touchscreenReportDesc = []byte{ - 0x05, 0x0D, - 0x09, 0x04, - 0xA1, 0x01, - - 0x09, 0x22, - 0xA1, 0x02, - - 0x09, 0x42, - 0x15, 0x00, - 0x25, 0x01, - 0x75, 0x01, - 0x95, 0x01, - 0x81, 0x02, - - 0x09, 0x32, - 0x75, 0x01, - 0x95, 0x01, - 0x81, 0x02, - - 0x75, 0x01, - 0x95, 0x06, - 0x81, 0x03, - - 0x05, 0x01, - 0x09, 0x30, - 0x09, 0x31, - 0x16, 0x00, 0x00, - 0x26, 0xFF, 0x7F, - 0x36, 0x00, 0x00, - 0x46, 0xFF, 0x7F, - 0x75, 0x10, - 0x95, 0x02, - 0x81, 0x02, - - 0xC0, - 0xC0, + 0x05, 0x0D, // Usage Page (Digitizers) + 0x09, 0x04, // Usage (Touch Screen) + 0xA1, 0x01, // Collection (Application) + + 0x09, 0x22, // Usage (Finger) + 0xA1, 0x02, // Collection (Logical) + + 0x09, 0x42, // Usage (Tip Switch) + 0x09, 0x32, // Usage (In Range) + 0x15, 0x00, // Logical Minimum (0) + 0x25, 0x01, // Logical Maximum (1) + 0x75, 0x01, // Report Size (1) + 0x95, 0x02, // Report Count (2) + 0x81, 0x02, // Input (Data,Var,Abs) + + 0x75, 0x01, // Report Size (1) + 0x95, 0x06, // Report Count (6) + 0x81, 0x03, // Input (Const,Var,Abs) padding + + 0x09, 0x51, // Usage (Contact Identifier) + 0x15, 0x00, // Logical Minimum (0) + 0x25, 0x0F, // Logical Maximum (15) + 0x75, 0x08, // Report Size (8) + 0x95, 0x01, // Report Count (1) + 0x81, 0x02, // Input (Data,Var,Abs) + + 0x05, 0x01, // Usage Page (Generic Desktop) + 0x09, 0x30, // Usage (X) + 0x09, 0x31, // Usage (Y) + 0x16, 0x00, 0x00, // Logical Minimum (0) + 0x26, 0xFF, 0x7F, // Logical Maximum (32767) + 0x36, 0x00, 0x00, // Physical Minimum (0) + 0x46, 0xFF, 0x7F, // Physical Maximum (32767) + 0x75, 0x10, // Report Size (16) + 0x95, 0x02, // Report Count (2) + 0x81, 0x02, // Input (Data,Var,Abs) + + 0xC0, // End Collection + + 0x05, 0x0D, // Usage Page (Digitizers) + 0x09, 0x54, // Usage (Contact Count) + 0x15, 0x00, // Logical Minimum (0) + 0x25, 0x01, // Logical Maximum (1) + 0x75, 0x08, // Report Size (8) + 0x95, 0x01, // Report Count (1) + 0x81, 0x02, // Input (Data,Var,Abs) + + 0xC0, // End Collection } func (u *UsbGadget) touchscreenWriteHidFile(data []byte) error { @@ -100,15 +120,19 @@ func (u *UsbGadget) TouchscreenReport(x int, y int, touching bool) error { } flags := byte(0) + contactCount := byte(0) if touching { flags = 0x03 + contactCount = 0x01 } return u.touchscreenWriteHidFile([]byte{ flags, + 0x00, byte(x), byte(x >> 8), byte(y), byte(y >> 8), + contactCount, }) } From 74da49aaa147be2d521c303c85fb577f272012db Mon Sep 17 00:00:00 2001 From: steadying Date: Sun, 3 May 2026 12:47:54 +0100 Subject: [PATCH 04/77] PicPhone: default to touchscreen HID routing --- ui/src/hooks/useMouse.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/hooks/useMouse.ts b/ui/src/hooks/useMouse.ts index afb47d090..a4c209e42 100644 --- a/ui/src/hooks/useMouse.ts +++ b/ui/src/hooks/useMouse.ts @@ -7,7 +7,7 @@ import { useMouseStore, useSettingsStore } from "./stores"; const calcDelta = (pos: number) => (Math.abs(pos) < 10 ? pos * 2 : pos); export const isPicphoneTouchscreenMode = () => - typeof window !== "undefined" && window.localStorage.getItem("picphoneTouchscreen") === "1"; + typeof window !== "undefined" && window.localStorage.getItem("picphoneTouchscreen") !== "0"; export interface AbsMouseMoveHandlerProps { videoClientWidth: number; From 0b52fca14cef1c74446ef403867a80f2ce391954 Mon Sep 17 00:00:00 2001 From: steadying Date: Sun, 3 May 2026 14:57:56 +0100 Subject: [PATCH 05/77] PicPhone: fix phone controller touchscreen dragging --- ui/src/components/WebRTCVideo.tsx | 48 +++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index 1e3a1bfd9..a2330c7fd 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -594,7 +594,8 @@ export default function WebRTCVideo({ if (!videoElmRefValue) return; const isRelativeMouseMode = settings.mouseMode === "relative"; - const mouseHandler = isPicphoneTouchscreenMode() + const isTouchscreenMode = isPicphoneTouchscreenMode(); + const mouseHandler = isTouchscreenMode ? touchscreenMoveHandler : isRelativeMouseMode ? relMouseMoveHandler @@ -603,11 +604,46 @@ export default function WebRTCVideo({ const abortController = new AbortController(); const signal = abortController.signal; - videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal }); - videoElmRefValue.addEventListener("pointerdown", mouseHandler, { - signal, - }); - videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal }); + if (isTouchscreenMode) { + videoElmRefValue.style.touchAction = "none"; + videoElmRefValue.style.userSelect = "none"; + videoElmRefValue.draggable = false; + + const pointerHandler = (e: PointerEvent) => { + e.preventDefault(); + + if (e.type === "pointerdown") { + try { + videoElmRefValue.setPointerCapture(e.pointerId); + } catch (err) { + console.debug("Unable to capture pointer", err); + } + } + + mouseHandler(e); + + if (e.type === "pointerup" || e.type === "pointercancel") { + try { + if (videoElmRefValue.hasPointerCapture(e.pointerId)) { + videoElmRefValue.releasePointerCapture(e.pointerId); + } + } catch (err) { + console.debug("Unable to release pointer capture", err); + } + } + }; + + videoElmRefValue.addEventListener("pointerdown", pointerHandler, { signal }); + videoElmRefValue.addEventListener("pointermove", pointerHandler, { signal }); + videoElmRefValue.addEventListener("pointerup", pointerHandler, { signal }); + videoElmRefValue.addEventListener("pointercancel", pointerHandler, { signal }); + } else { + videoElmRefValue.addEventListener("mousemove", mouseHandler, { signal }); + videoElmRefValue.addEventListener("pointerdown", mouseHandler, { + signal, + }); + videoElmRefValue.addEventListener("pointerup", mouseHandler, { signal }); + } videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { signal, passive: true, From fb0f1a8f552594f2e6f9da8bc6c7c24148896c23 Mon Sep 17 00:00:00 2001 From: steadying Date: Sun, 3 May 2026 19:54:29 +0100 Subject: [PATCH 06/77] PicPhone: preserve no-bars aligned controller LKG --- ui/src/components/WebRTCVideo.tsx | 133 ++++++++++++++++++++++++++---- 1 file changed, 119 insertions(+), 14 deletions(-) diff --git a/ui/src/components/WebRTCVideo.tsx b/ui/src/components/WebRTCVideo.tsx index a2330c7fd..89be37504 100644 --- a/ui/src/components/WebRTCVideo.tsx +++ b/ui/src/components/WebRTCVideo.tsx @@ -9,6 +9,10 @@ import { useRTCStore, useSettingsStore, useUiStore, useVideoStore } from "@hooks import { JsonRpcResponse, useJsonRpc } from "@hooks/useJsonRpc"; import VirtualKeyboard from "@components/VirtualKeyboard"; import Actionbar from "@components/ActionBar"; + +const isPicphoneDisplayCropMode = () => + typeof window !== "undefined" && window.localStorage.getItem("picphoneDisplayCrop") !== "0"; + import MacroBar from "@components/MacroBar"; import InfoBar from "@components/InfoBar"; import { @@ -49,7 +53,28 @@ export default function WebRTCVideo({ // Store hooks const settings = useSettingsStore(); - const { handleKeyPress, resetKeyboardState } = useKeyboard(); + const { executeMacro, handleKeyPress, resetKeyboardState } = useKeyboard(); + + const sendPicphoneKeyTap = useCallback( + (key: number) => { + void handleKeyPress(key, true); + window.setTimeout(() => void handleKeyPress(key, false), 80); + }, + [handleKeyPress], + ); + + const sendPicphoneAndroidBack = useCallback(() => { + sendPicphoneKeyTap(keys.Escape); + }, [sendPicphoneKeyTap]); + + const sendPicphoneAndroidHome = useCallback(() => { + sendPicphoneKeyTap(keys.Home); + }, [sendPicphoneKeyTap]); + + const sendPicphoneAndroidRecents = useCallback(() => { + void executeMacro([{ keys: ["Tab"], modifiers: ["AltLeft"], delay: 100 }]); + }, [executeMacro]); + const { getRelMouseMoveHandler, getAbsMouseMoveHandler, @@ -612,6 +637,12 @@ export default function WebRTCVideo({ const pointerHandler = (e: PointerEvent) => { e.preventDefault(); + if (e.type === "pointerdown" && e.button === 2) { + e.stopPropagation(); + sendPicphoneAndroidBack(); + return; + } + if (e.type === "pointerdown") { try { videoElmRefValue.setPointerCapture(e.pointerId); @@ -646,7 +677,7 @@ export default function WebRTCVideo({ } videoElmRefValue.addEventListener("wheel", mouseWheelHandler, { signal, - passive: true, + passive: !isTouchscreenMode, }); if (isRelativeMouseMode) { @@ -695,6 +726,7 @@ export default function WebRTCVideo({ mouseWheelHandler, resetMousePosition, settings.mouseMode, + sendPicphoneAndroidBack, ], ); @@ -743,6 +775,17 @@ export default function WebRTCVideo({ }; }, [videoSaturation, videoBrightness, videoContrast]); + const showPicphoneNavOverlay = useMemo(() => { + if (!isPicphoneTouchscreenMode()) return false; + if (typeof window === "undefined") return false; + + const setting = window.localStorage.getItem("picphoneNavOverlay"); + if (setting === "0") return false; + if (setting === "1") return true; + + return window.matchMedia("(pointer: coarse)").matches; + }, []); + return (
@@ -773,7 +816,12 @@ export default function WebRTCVideo({