feat(geisterhand): add /key, /scroll endpoints, widget filtering, iOS parity, accessibility bridge

Baked-in API improvements:
- POST /key: fire menu callbacks by shortcut string (case-insensitive)
- POST /scroll/:handle: programmatic scrollview scrolling with x,y offsets
- GET /widgets?label=&type=: query parameter filtering on widget list
- Registry now includes shortcut field in JSON output

iOS widget registration parity (previously only buttons):
- TextField, Slider, Toggle, Picker, Menu items now registered with geisterhand
- ScrollView registered as new type 8

Accessibility improvements:
- iOS: setAccessibilityLabel on Button, Text, TextField, Toggle
- Both platforms: accessibilityIdentifier set to "gh-{handle}" on all widgets
  (bridges baked-in handles to external CLI accessibility tree)

macOS menu shortcuts now registered via register_with_shortcut for /key lookup

Author: proggeramlug

crates/perry-runtime/src/geisterhand_registry.rs

@@ -14,6 +14,7 @@ pub const WIDGET_PICKER: u8 = 4;
 pub const WIDGET_MENU: u8 = 5;
 pub const WIDGET_SHORTCUT: u8 = 6;
 pub const WIDGET_TABLE: u8 = 7;
+pub const WIDGET_SCROLLVIEW: u8 = 8;
 
 /// Callback kind identifiers
 pub const CB_ON_CLICK: u8 = 0;
@@ -30,6 +31,7 @@ pub struct RegisteredWidget {
     pub callback_kind: u8,
     pub closure_f64: f64,
     pub label: String,
+    pub shortcut: String,
 }
 
 /// An action queued for main-thread execution
@@ -38,6 +40,7 @@ pub enum PendingAction {
     SetState { handle: i64, value: f64 },
     CaptureScreenshot,
     SetText { handle: i64, text: String },
+    ScrollTo { handle: i64, x: f64, y: f64 },
 }
 
 static REGISTRY: Mutex<Vec<RegisteredWidget>> = Mutex::new(Vec::new());
@@ -64,6 +67,7 @@ use std::sync::atomic::{AtomicPtr, Ordering};
 static UI_STATE_SET_FN: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut());
 static UI_SCREENSHOT_CAPTURE_FN: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut());
 static UI_TEXTFIELD_SET_STRING_FN: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut());
+static UI_SCROLL_SET_FN: AtomicPtr<()> = AtomicPtr::new(std::ptr::null_mut());
 
 /// Register the platform UI crate's state_set function.
 #[no_mangle]
@@ -85,6 +89,12 @@ pub extern "C" fn perry_geisterhand_register_textfield_set_string(f: extern "C"
     UI_TEXTFIELD_SET_STRING_FN.store(f as *mut (), Ordering::Release);
 }
 
+/// Register the platform UI crate's scroll_set function.
+#[no_mangle]
+pub extern "C" fn perry_geisterhand_register_scroll_set(f: extern "C" fn(i64, f64, f64)) {
+    UI_SCROLL_SET_FN.store(f as *mut (), Ordering::Release);
+}
+
 /// Register a widget callback in the global registry.
 /// Called from platform UI crates when widgets are created or callbacks attached.
 ///
@@ -123,10 +133,89 @@ pub extern "C" fn perry_geisterhand_register(
             callback_kind,
             closure_f64,
             label,
+            shortcut: String::new(),
+        });
+    }
+}
+
+/// Register a widget callback with an associated keyboard shortcut string.
+/// Used by menu items that have shortcuts (e.g., "s" for Cmd+S).
+#[no_mangle]
+pub extern "C" fn perry_geisterhand_register_with_shortcut(
+    handle: i64,
+    widget_type: u8,
+    callback_kind: u8,
+    closure_f64: f64,
+    label_ptr: *const u8,
+    shortcut_ptr: *const u8,
+    shortcut_len: usize,
+) {
+    let label = if label_ptr.is_null() {
+        String::new()
+    } else {
+        unsafe {
+            let len = *(label_ptr as *const u32) as usize;
+            let data = label_ptr.add(std::mem::size_of::<[u64; 1]>());
+            if len > 0 && len < 10000 {
+                String::from_utf8_lossy(std::slice::from_raw_parts(data, len)).into_owned()
+            } else {
+                String::new()
+            }
+        }
+    };
+    let shortcut = if shortcut_ptr.is_null() || shortcut_len == 0 {
+        String::new()
+    } else {
+        unsafe {
+            String::from_utf8_lossy(std::slice::from_raw_parts(shortcut_ptr, shortcut_len)).into_owned()
+        }
+    };
+    if let Ok(mut reg) = REGISTRY.lock() {
+        reg.push(RegisteredWidget {
+            handle,
+            widget_type,
+            callback_kind,
+            closure_f64,
+            label,
+            shortcut,
         });
     }
 }
 
+/// Find a registered callback by shortcut string. Case-insensitive match.
+/// Returns the closure_f64 or 0.0 if not found.
+#[no_mangle]
+pub extern "C" fn perry_geisterhand_find_by_shortcut(
+    shortcut_ptr: *const u8,
+    shortcut_len: usize,
+) -> f64 {
+    if shortcut_ptr.is_null() || shortcut_len == 0 {
+        return 0.0;
+    }
+    let query = unsafe {
+        String::from_utf8_lossy(std::slice::from_raw_parts(shortcut_ptr, shortcut_len))
+    }.to_lowercase();
+    match REGISTRY.lock() {
+        Ok(reg) => {
+            for w in reg.iter() {
+                if !w.shortcut.is_empty() && w.shortcut.to_lowercase() == query {
+                    return w.closure_f64;
+                }
+            }
+            0.0
+        }
+        Err(_) => 0.0,
+    }
+}
+
+/// Queue a scroll action for main-thread dispatch.
+#[no_mangle]
+pub extern "C" fn perry_geisterhand_queue_scroll(handle: i64, x: f64, y: f64) {
+    if let Ok(mut q) = PENDING_ACTIONS.lock() {
+        q.push(PendingAction::ScrollTo { handle, x, y });
+    }
+}
+
 /// Queue a callback invocation for main-thread dispatch.
 #[no_mangle]
 pub extern "C" fn perry_geisterhand_queue_action(closure_f64: f64) {
@@ -225,6 +314,15 @@ pub extern "C" fn perry_geisterhand_pump() {
                     }
                 }
             }
+            PendingAction::ScrollTo { handle, x, y } => {
+                let f = UI_SCROLL_SET_FN.load(Ordering::Acquire);
+                if !f.is_null() {
+                    unsafe {
+                        let func: extern "C" fn(i64, f64, f64) = std::mem::transmute(f);
+                        func(handle, x, y);
+                    }
+                }
+            }
             PendingAction::CaptureScreenshot => {
                 let f = UI_SCREENSHOT_CAPTURE_FN.load(Ordering::Acquire);
                 let (ptr, len) = if !f.is_null() {
@@ -261,10 +359,12 @@ pub extern "C" fn perry_geisterhand_get_registry_json(out_len: *mut usize) -> *m
             let mut s = String::from("[");
             for (i, w) in reg.iter().enumerate() {
                 if i > 0 { s.push(','); }
+                let escaped_label = w.label.replace('\\', "\\\\").replace('"', "\\\"");
+                let escaped_shortcut = w.shortcut.replace('\\', "\\\\").replace('"', "\\\"");
                 s.push_str(&format!(
-                    r#"{{"handle":{},"widget_type":{},"callback_kind":{},"label":"{}"}}"#,
+                    r#"{{"handle":{},"widget_type":{},"callback_kind":{},"label":"{}","shortcut":"{}"}}"#,
                     w.handle, w.widget_type, w.callback_kind,
-                    w.label.replace('\\', "\\\\").replace('"', "\\\"")
+                    escaped_label, escaped_shortcut
                 ));
             }
             s.push(']');

crates/perry-ui-geisterhand/src/server.rs

@@ -1,6 +1,6 @@
 //! HTTP server for geisterhand.
 //! Routes: /widgets, /click/:handle, /type/:handle, /slide/:handle,
-//! /toggle/:handle, /state/:handle, /key, /chaos/start, /chaos/stop, /chaos/status
+//! /toggle/:handle, /state/:handle, /key, /scroll/:handle, /chaos/start, /chaos/stop, /chaos/status
 
 use tiny_http::{Server, Response, Header, Method};
 
@@ -12,6 +12,8 @@ extern "C" {
     fn perry_geisterhand_queue_action1(closure_f64: f64, arg: f64);
     fn perry_geisterhand_queue_state_set(handle: i64, value: f64);
     fn perry_geisterhand_request_screenshot(out_len: *mut usize) -> *mut u8;
+    fn perry_geisterhand_find_by_shortcut(shortcut_ptr: *const u8, shortcut_len: usize) -> f64;
+    fn perry_geisterhand_queue_scroll(handle: i64, x: f64, y: f64);
 }
 
 // Callback kind constants (must match perry-runtime/src/geisterhand_registry.rs)
@@ -49,6 +51,34 @@ fn parse_handle(path: &str, prefix: &str) -> Option<i64> {
     rest.parse::<i64>().ok()
 }
 
+/// Parse a query parameter value from a URL (e.g., "/widgets?label=Save" → Some("Save"))
+fn query_param<'a>(url: &'a str, key: &str) -> Option<&'a str> {
+    let query = url.split('?').nth(1)?;
+    let needle = format!("{}=", key);
+    for pair in query.split('&') {
+        if let Some(val) = pair.strip_prefix(&needle) {
+            return Some(val);
+        }
+    }
+    None
+}
+
+/// Map widget type name to code
+fn widget_type_from_name(name: &str) -> Option<u8> {
+    match name {
+        "button" => Some(0),
+        "textfield" | "text_field" => Some(1),
+        "slider" => Some(2),
+        "toggle" => Some(3),
+        "picker" => Some(4),
+        "menu" => Some(5),
+        "shortcut" => Some(6),
+        "table" => Some(7),
+        "scrollview" | "scroll_view" => Some(8),
+        _ => name.parse::<u8>().ok(),
+    }
+}
+
 /// Read request body as string
 fn read_body(request: &mut tiny_http::Request) -> String {
     let mut body = String::new();
@@ -67,7 +97,8 @@ pub fn run_server(port: u16) {
     };
 
     for mut request in server.incoming_requests() {
-        let path = request.url().to_string();
+        let full_url = request.url().to_string();
+        let path = full_url.split('?').next().unwrap_or(&full_url);
         let method = request.method().clone();
 
         // Handle CORS preflight
@@ -80,8 +111,8 @@ pub fn run_server(port: u16) {
             continue;
         }
 
-        let response = match (method, path.as_str()) {
-            // GET /widgets — list all registered widgets
+        let response = match (method, path) {
+            // GET /widgets — list all registered widgets (supports ?label= and ?type= filters)
             (Method::Get, "/widgets") => {
                 let mut len: usize = 0;
                 let ptr = unsafe { perry_geisterhand_get_registry_json(&mut len) };
@@ -92,7 +123,42 @@ pub fn run_server(port: u16) {
                 } else {
                     "[]".to_string()
                 };
-                ok_json(&json)
+
+                // Apply query param filters
+                let label_filter = query_param(&full_url, "label");
+                let type_filter = query_param(&full_url, "type")
+                    .and_then(|t| widget_type_from_name(t));
+
+                if label_filter.is_some() || type_filter.is_some() {
+                    if let Ok(arr) = serde_json::from_str::<Vec<serde_json::Value>>(&json) {
+                        let filtered: Vec<&serde_json::Value> = arr.iter().filter(|w| {
+                            if let Some(label) = label_filter {
+                                if let Some(wl) = w.get("label").and_then(|l| l.as_str()) {
+                                    if !wl.to_lowercase().contains(&label.to_lowercase()) {
+                                        return false;
+                                    }
+                                } else {
+                                    return false;
+                                }
+                            }
+                            if let Some(wt) = type_filter {
+                                if let Some(wt_val) = w.get("widget_type").and_then(|t| t.as_u64()) {
+                                    if wt_val != wt as u64 {
+                                        return false;
+                                    }
+                                } else {
+                                    return false;
+                                }
+                            }
+                            true
+                        }).collect();
+                        ok_json(&serde_json::to_string(&filtered).unwrap_or_else(|_| "[]".to_string()))
+                    } else {
+                        ok_json(&json)
+                    }
+                } else {
+                    ok_json(&json)
+                }
             }
 
             // POST /click/:handle — fire onClick
@@ -293,6 +359,47 @@ pub fn run_server(port: u16) {
                 }
             }
 
+            // POST /key — fire a keyboard shortcut by matching registered menu shortcuts
+            (Method::Post, "/key") => {
+                let body = read_body(&mut request);
+                let shortcut = match serde_json::from_str::<serde_json::Value>(&body) {
+                    Ok(v) => v.get("shortcut").and_then(|s| s.as_str()).unwrap_or("").to_string(),
+                    Err(_) => body.trim().to_string(),
+                };
+                if shortcut.is_empty() {
+                    error_json(400, "missing shortcut field")
+                } else {
+                    let closure = unsafe {
+                        perry_geisterhand_find_by_shortcut(shortcut.as_ptr(), shortcut.len())
+                    };
+                    if closure != 0.0 {
+                        unsafe { perry_geisterhand_queue_action(closure); }
+                        ok_json(r#"{"ok":true}"#)
+                    } else {
+                        error_json(404, "no registered shortcut matches")
+                    }
+                }
+            }
+
+            // POST /scroll/:handle — scroll a scrollview
+            (Method::Post, p) if p.starts_with("/scroll/") => {
+                match parse_handle(p, "/scroll/") {
+                    Some(handle) => {
+                        let body = read_body(&mut request);
+                        let (x, y) = match serde_json::from_str::<serde_json::Value>(&body) {
+                            Ok(v) => (
+                                v.get("x").and_then(|v| v.as_f64()).unwrap_or(0.0),
+                                v.get("y").and_then(|v| v.as_f64()).unwrap_or(0.0),
+                            ),
+                            Err(_) => (0.0, 0.0),
+                        };
+                        unsafe { perry_geisterhand_queue_scroll(handle, x, y); }
+                        ok_json(r#"{"ok":true}"#)
+                    }
+                    None => error_json(400, "invalid handle"),
+                }
+            }
+
             _ => error_json(404, "not found"),
         };
 

crates/perry-ui-ios/src/app.rs

@@ -336,11 +336,13 @@ pub fn app_run(_app_handle: i64) {
                 f: extern "C" fn(*mut usize) -> *mut u8,
             );
             fn perry_geisterhand_register_textfield_set_string(f: extern "C" fn(i64, i64));
+            fn perry_geisterhand_register_scroll_set(f: extern "C" fn(i64, f64, f64));
         }
         unsafe {
             perry_geisterhand_register_state_set(crate::perry_ui_state_set);
             perry_geisterhand_register_screenshot_capture(crate::screenshot::perry_ui_screenshot_capture);
             perry_geisterhand_register_textfield_set_string(crate::perry_ui_textfield_set_string);
+            perry_geisterhand_register_scroll_set(crate::widgets::scrollview::perry_ui_scroll_set_offset);
         }
     }
 

crates/perry-ui-ios/src/menu.rs

@@ -83,6 +83,11 @@ pub fn add_item(menu_handle: i64, title_ptr: *const u8, callback: f64) {
             });
         }
     });
+    #[cfg(feature = "geisterhand")]
+    {
+        extern "C" { fn perry_geisterhand_register(h: i64, wt: u8, ck: u8, cb: f64, lbl: *const u8); }
+        unsafe { perry_geisterhand_register(menu_handle, 5, 0, callback, title_ptr); }
+    }
 }
 
 /// Add an item with a keyboard shortcut (e.g. "Cmd+N").
@@ -101,10 +106,26 @@ pub fn add_item_with_shortcut(
             menus[idx].push(MenuItemEntry::Item {
                 title,
                 callback,
-                shortcut: Some(shortcut),
+                shortcut: Some(shortcut.clone()),
             });
         }
     });
+    #[cfg(feature = "geisterhand")]
+    {
+        extern "C" {
+            fn perry_geisterhand_register_with_shortcut(
+                h: i64, wt: u8, ck: u8, cb: f64, lbl: *const u8,
+                shortcut_ptr: *const u8, shortcut_len: usize,
+            );
+        }
+        let shortcut_bytes = shortcut.as_bytes();
+        unsafe {
+            perry_geisterhand_register_with_shortcut(
+                menu_handle, 5, 0, callback, title_ptr,
+                shortcut_bytes.as_ptr(), shortcut_bytes.len(),
+            );
+        }
+    }
 }
 
 /// Remove all items from a menu.

crates/perry-ui-ios/src/widgets/button.rs

@@ -95,6 +95,7 @@ pub fn create(label_ptr: *const u8, on_press: f64) -> i64 {
 
         let ns_string = NSString::from_str(label);
         let _: () = msg_send![&*button, setTitle: &*ns_string, forState: 0u64]; // UIControlStateNormal = 0
+        let _: () = msg_send![&*button, setAccessibilityLabel: &*ns_string];
 
         let _: () = msg_send![&*button, setTranslatesAutoresizingMaskIntoConstraints: false];