Merge pull request #37 from NeptuneKit/feat/20260608-ios-media-playback-helpers

Add iOS media playback snapshot helpers

Author: linhay

CLI/Tests/TritonKitCLITests/SchemaFactSourceTests.swift

@@ -275,6 +275,15 @@ struct SchemaFactSourceTests {
         #expect(connectedObserveIOS.nextAction?.args == ["current", "--platform", "ios", "--json"])
         #expect(connectedObserveIOS.evidence == ["surface-tree", "runtime-ax", "host-layout"])
 
+        let mediaPlayback = try #require(connected["media-playback"])
+        #expect(mediaPlayback.supported)
+        #expect(mediaPlayback.group == "observe")
+        #expect(mediaPlayback.requiredBy.contains("assert"))
+        #expect(mediaPlayback.requiredBy.contains("evidence"))
+        #expect(mediaPlayback.nextAction?.command == "snapshot")
+        #expect(mediaPlayback.nextAction?.args == ["--include", "media,ax,screenshot-metadata", "--json"])
+        #expect(mediaPlayback.evidence == ["runtime-media", "runtime-ax", "screenshot-metadata"])
+
         let webViewList = try #require(disconnected["webview-list"])
         #expect(webViewList.supported)
         #expect(webViewList.group == "webview")
@@ -536,7 +545,7 @@ struct SchemaFactSourceTests {
         let schemas = commandSchemaMap()
         let observeSchema = try #require(schemas["observe"])
         let nodeSchema = try #require(schemas["node"])
-        #expect(observeSchema.providedCapabilities == ["observe", "observe-ios", "observe-android", "observe-harmony"])
+        #expect(observeSchema.providedCapabilities == ["observe", "observe-ios", "media-playback", "observe-android", "observe-harmony"])
         #expect(nodeSchema.providedCapabilities == ["node", "node-resolve"])
 
         let connected = connectedCapabilityMap()
@@ -556,6 +565,7 @@ struct SchemaFactSourceTests {
         )] = [
             ("observe", "observe", ["action", "assert", "evidence"], ["surface-tree", "runtime-ax", "host-layout"], true, true, "observe", ["current", "--json"], "observe", ["current", "--json"]),
             ("observe-ios", "observe", ["action", "assert", "evidence"], ["surface-tree", "runtime-ax", "host-layout"], true, false, "observe", ["current", "--platform", "ios", "--json"], "observe", ["current", "--platform", "ios", "--json"]),
+            ("media-playback", "observe", ["assert", "evidence", "observe"], ["runtime-media", "runtime-ax", "screenshot-metadata"], true, false, "snapshot", ["--include", "media,ax,screenshot-metadata", "--json"], "status", ["--json"]),
             ("observe-android", "observe", ["action", "assert", "evidence"], ["surface-tree", "runtime-ax", "host-layout"], true, true, "observe", ["tree", "--platform", "android", "--device", "<selector>", "--json"], "observe", ["tree", "--platform", "android", "--device", "<selector>", "--json"]),
             ("observe-harmony", "observe", ["action", "assert", "evidence"], ["surface-tree", "runtime-ax", "host-layout"], true, true, "observe", ["tree", "--platform", "harmony", "--device", "<selector>", "--json"], "observe", ["tree", "--platform", "harmony", "--device", "<selector>", "--json"]),
             ("node", "observe", ["action", "assert", "evidence"], ["hierarchy-node", "surface-tree"], true, false, "node", ["--oid", "<oid>", "--json"], "status", ["--json"]),
@@ -1273,7 +1283,7 @@ struct SchemaFactSourceTests {
 
         let runtimeReasonCapabilities = Set([
             "runtime-manifest", "state-app", "state-scene", "state-route", "state-responder",
-            "snapshot", "focus", "set-text", "select-segment", "set-switch", "semantic-action", "ledger",
+            "snapshot", "media-playback", "focus", "set-text", "select-segment", "set-switch", "semantic-action", "ledger",
             "observe-ios",
             "inspect", "hierarchy", "nodes", "node", "attrs", "object",
             "export-json", "export-archive", "geometry", "ax", "hit", "screenshot",
@@ -1529,7 +1539,7 @@ struct SchemaFactSourceTests {
             "target": ["target"],
             "runtime": ["runtime", "state", "snapshot", "focus", "set-text", "select-segment", "set-switch", "ledger", "schema", "status", "serve"],
             "host": ["device", "sim", "app", "ax"],
-            "observe": ["observe", "list", "inspect", "hierarchy", "nodes", "node", "attrs", "object", "export", "geometry", "ax", "screenshot", "hit", "wait", "status", "serve"],
+            "observe": ["observe", "snapshot", "list", "inspect", "hierarchy", "nodes", "node", "attrs", "object", "export", "geometry", "ax", "screenshot", "hit", "wait", "status", "serve"],
             "webview": ["webview"],
             "route": ["route"],
             "evidence": ["evidence", "status", "serve"],
@@ -3713,7 +3723,7 @@ struct SchemaFactSourceTests {
         #expect(snapshot.nextCommands.contains("triton assert text-exists <text> --json"))
         expectContract(snapshot, selector: "runtime.snapshot", fields: [
             "ok", "capturedAt", "runtime", "targetConnectionState", "include", "app", "scene",
-            "route", "responder", "geometry", "ax", "screenshot", "artifacts", "skipped", "truncation",
+            "route", "responder", "media", "geometry", "ax", "screenshot", "artifacts", "skipped", "truncation",
         ])
 
         #expect(observe.failureCodes.contains("target_not_found"))
@@ -4023,7 +4033,7 @@ private func capabilityEvidenceTaxonomy() -> Set<String> {
         "host-layout", "host-targets.json", "hierarchy-node", "input.result",
         "page-events", "provider-url", "route-assertion", "runtime-ax",
         "runtime-ledger", "runtime-manifest", "runtime-provider",
-        "runtime-samples", "runtime-snapshot", "screenshot", "screenshot-metadata",
+        "runtime-media", "runtime-samples", "runtime-snapshot", "screenshot", "screenshot-metadata",
         "smoke-summary", "snapshot-json", "status-json", "stdout-json",
         "surface-tree", "target.resolution", "trace", "tritonplan",
         "unsupported-envelope", "wait.result", "wait-samples", "webview-candidates",

README.md

@@ -563,6 +563,16 @@ triton ledger --limit 50 --jsonl
 
 `set-text --secure` redacts the text value in command output and runtime ledger while preserving inserted length. `ledger --jsonl` is the recent embedded request/action replay stream for debugging selector resolution, runtime errors, elapsed time, and redaction state.
 
+For iOS video regression with `AVPlayer` or `AVPlayerViewController`, include the media snapshot section before asserting playback behavior:
+
+```bash
+triton snapshot --include media,ax,screenshot-metadata --json
+triton screenshot --json
+triton evidence --output /tmp/video-regression.tritonevidence --json
+```
+
+The `media` section reports visible AVPlayer-backed surfaces, player status/rate/time metadata when public APIs expose it, AX playback-control candidates, automation confidence, fallback advice, and evidence commands. System `AVPlayerViewController` controls are not guaranteed to expose stable actionable AX nodes in every route; when the snapshot is `surface-only`, add app-owned DEBUG overlay controls with stable accessibility identifiers for play, pause, seek, progress, elapsed time, and duration, then assert those controls with `wait`, `find`, `tap`, and `assert`.
+
 When a pass/fail decision needs attachable evidence, export a bundle with a machine-readable manifest:
 
 ```bash

Sources/TritonKit/TKRuntimeMediaSnapshot.swift

@@ -0,0 +1,236 @@
+import Foundation
+import TritonKitShared
+#if canImport(UIKit)
+import UIKit
+#endif
+#if canImport(AVFoundation)
+import AVFoundation
+#endif
+#if canImport(AVKit)
+import AVKit
+#endif
+
+#if canImport(UIKit)
+@MainActor
+func currentMediaState(axNodes: [TKAXNode]?) -> TKRuntimeMediaStateResponse {
+    var surfaces: [TKRuntimeMediaSurface] = []
+    let windows = keyWindows()
+
+    for (windowIndex, window) in windows.enumerated() {
+        surfaces.append(contentsOf: mediaLayerSurfaces(in: window, windowIndex: windowIndex))
+    }
+
+    #if canImport(AVKit)
+    let controllers = windows.compactMap(\.rootViewController).flatMap(mediaControllerTree)
+    for controller in controllers {
+        guard let playerController = controller as? AVPlayerViewController else { continue }
+        let frame = playerController.view.window.map { tkRect(playerController.view.convert(playerController.view.bounds, to: $0)) }
+        surfaces.append(mediaSurface(
+            id: "media-controller-\(surfaces.count + 1)",
+            kind: "avplayer-view-controller",
+            className: NSStringFromClass(type(of: playerController)),
+            frame: frame,
+            visible: isAXVisible(playerController.view),
+            player: playerController.player,
+            controllerClassName: NSStringFromClass(type(of: playerController))
+        ))
+    }
+    #endif
+
+    let controls = TKRuntimeMediaControlCandidates(from: axNodes ?? [])
+    let warnings = surfaces.isEmpty
+        ? ["No AVPlayer-backed media surface was discovered from public runtime APIs"]
+        : []
+
+    return TKRuntimeMediaStateResponse(
+        capturedAt: currentStateTimestamp(),
+        surfaces: deduplicateMediaSurfaces(surfaces),
+        controls: controls,
+        warnings: warnings
+    )
+}
+
+@MainActor
+private func mediaLayerSurfaces(in window: UIWindow, windowIndex: Int) -> [TKRuntimeMediaSurface] {
+    var surfaces: [TKRuntimeMediaSurface] = []
+
+    func walk(view: UIView) {
+        surfaces.append(contentsOf: mediaLayerSurfaces(
+            layer: view.layer,
+            window: window,
+            prefix: "window-\(windowIndex)-view-\(oid(for: view) ?? 0)",
+            controllerClassName: nearestViewControllerClassName(from: view)
+        ))
+        for subview in view.subviews {
+            walk(view: subview)
+        }
+    }
+
+    walk(view: window)
+    return surfaces
+}
+
+@MainActor
+private func mediaLayerSurfaces(
+    layer: CALayer,
+    window: UIWindow,
+    prefix: String,
+    controllerClassName: String?
+) -> [TKRuntimeMediaSurface] {
+    var surfaces: [TKRuntimeMediaSurface] = []
+
+    #if canImport(AVFoundation)
+    if let playerLayer = layer as? AVPlayerLayer {
+        let frame = tkRect(window.layer.convert(playerLayer.bounds, from: playerLayer))
+        surfaces.append(mediaSurface(
+            id: "\(prefix)-layer-\(ObjectIdentifier(playerLayer).hashValue)",
+            kind: "avplayer-layer",
+            className: NSStringFromClass(type(of: playerLayer)),
+            frame: frame,
+            visible: !playerLayer.isHidden && playerLayer.opacity > 0 && playerLayer.bounds.width > 0 && playerLayer.bounds.height > 0,
+            player: playerLayer.player,
+            controllerClassName: controllerClassName
+        ))
+    }
+    #endif
+
+    for sublayer in layer.sublayers ?? [] {
+        surfaces.append(contentsOf: mediaLayerSurfaces(
+            layer: sublayer,
+            window: window,
+            prefix: prefix,
+            controllerClassName: controllerClassName
+        ))
+    }
+
+    return surfaces
+}
+
+#if canImport(AVFoundation)
+private func mediaSurface(
+    id: String,
+    kind: String,
+    className: String,
+    frame: TKRect?,
+    visible: Bool,
+    player: AVPlayer?,
+    controllerClassName: String?
+) -> TKRuntimeMediaSurface {
+    let elapsed = player.map { seconds(from: $0.currentTime()) }
+    let duration = player?.currentItem.map { seconds(from: $0.duration) }
+    let boundedDuration = duration.flatMap { $0.isFinite && $0 > 0 ? $0 : nil }
+    let progress = if let elapsed, let boundedDuration {
+        min(max(elapsed / boundedDuration, 0), 1)
+    } else {
+        Optional<Double>.none
+    }
+
+    return TKRuntimeMediaSurface(
+        id: id,
+        kind: kind,
+        className: className,
+        frame: frame,
+        visible: visible,
+        playerStatus: player?.currentItem.map { playerItemStatusName($0.status) },
+        playbackState: playerPlaybackState(player),
+        rate: player.map { Double($0.rate) },
+        elapsedTimeSeconds: elapsed?.isFinite == true ? elapsed : nil,
+        durationSeconds: boundedDuration,
+        progress: progress,
+        controllerClassName: controllerClassName
+    )
+}
+
+private func seconds(from time: CMTime) -> Double {
+    guard time.isValid, !time.isIndefinite else { return .nan }
+    return CMTimeGetSeconds(time)
+}
+
+private func playerItemStatusName(_ status: AVPlayerItem.Status) -> String {
+    switch status {
+    case .unknown: return "unknown"
+    case .readyToPlay: return "readyToPlay"
+    case .failed: return "failed"
+    @unknown default: return "unknown"
+    }
+}
+
+private func playerPlaybackState(_ player: AVPlayer?) -> String? {
+    guard let player else { return nil }
+    if player.rate > 0 {
+        return "playing"
+    }
+    switch player.timeControlStatus {
+    case .paused: return "paused"
+    case .waitingToPlayAtSpecifiedRate: return "waiting"
+    case .playing: return "playing"
+    @unknown default: return "unknown"
+    }
+}
+#else
+private func mediaSurface(
+    id: String,
+    kind: String,
+    className: String,
+    frame: TKRect?,
+    visible: Bool,
+    player: Any?,
+    controllerClassName: String?
+) -> TKRuntimeMediaSurface {
+    TKRuntimeMediaSurface(
+        id: id,
+        kind: kind,
+        className: className,
+        frame: frame,
+        visible: visible,
+        controllerClassName: controllerClassName
+    )
+}
+#endif
+
+@MainActor
+private func mediaControllerTree(_ root: UIViewController) -> [UIViewController] {
+    var controllers = [root]
+    controllers.append(contentsOf: root.children.flatMap(mediaControllerTree))
+    if let presented = root.presentedViewController {
+        controllers.append(contentsOf: mediaControllerTree(presented))
+    }
+    return controllers
+}
+
+@MainActor
+private func nearestViewControllerClassName(from view: UIView) -> String? {
+    var responder: UIResponder? = view
+    while let current = responder {
+        if let controller = current as? UIViewController {
+            return NSStringFromClass(type(of: controller))
+        }
+        responder = current.next
+    }
+    return nil
+}
+
+private func deduplicateMediaSurfaces(_ surfaces: [TKRuntimeMediaSurface]) -> [TKRuntimeMediaSurface] {
+    var seen = Set<String>()
+    return surfaces.filter { surface in
+        let key = [
+            surface.kind,
+            surface.className,
+            surface.controllerClassName ?? "",
+            surface.frame.map { "\($0.x),\($0.y),\($0.width),\($0.height)" } ?? "",
+        ].joined(separator: "|")
+        return seen.insert(key).inserted
+    }
+}
+#else
+func currentMediaState(axNodes: [TKAXNode]?) -> TKRuntimeMediaStateResponse {
+    TKRuntimeMediaStateResponse(
+        capturedAt: currentStateTimestamp(),
+        surfaces: [],
+        controls: TKRuntimeMediaControlCandidates(from: axNodes ?? []),
+        unsupported: [
+            TKRuntimeUnsupportedState(field: "media", reason: "UIKit is not available on this platform"),
+        ]
+    )
+}
+#endif

Sources/TritonKit/TKRuntimeStateSnapshot.swift

@@ -209,13 +209,16 @@ func currentRuntimeSnapshot(_ request: TKRuntimeSnapshotRequest) -> TKRuntimeSna
     }
 
     let ax: [TKAXNode]?
-    if includes("ax") || includes("accessibility") {
+    let axForMedia: [TKAXNode]?
+    let shouldBuildAX = includes("ax") || includes("accessibility") || includes("media")
+    if shouldBuildAX {
         let maxNodes = max(1, request.maxAXNodes ?? 800)
         var context = AXBuildContext(maxNodes: maxNodes)
         let nodes = keyWindows().map { window in
             buildAXWindowNode(for: window, context: &context)
         }
-        ax = nodes
+        axForMedia = nodes
+        ax = (includes("ax") || includes("accessibility")) ? nodes : nil
         if context.remaining == 0 {
             truncation = TKRuntimeSnapshotTruncation(
                 truncated: true,
@@ -224,12 +227,24 @@ func currentRuntimeSnapshot(_ request: TKRuntimeSnapshotRequest) -> TKRuntimeSna
                 returnedCount: maxNodes
             )
         }
-        artifact("ax")
+        if includes("ax") || includes("accessibility") {
+            artifact("ax")
+        }
     } else {
         ax = nil
+        axForMedia = nil
         skipped.append(TKRuntimeSnapshotSkipped(name: "ax", reason: "not requested"))
     }
 
+    let media: TKRuntimeMediaStateResponse?
+    if includes("media") {
+        media = currentMediaState(axNodes: axForMedia)
+        artifact("media")
+    } else {
+        media = nil
+        skipped.append(TKRuntimeSnapshotSkipped(name: "media", reason: "not requested"))
+    }
+
     let screenshot: TKRuntimeScreenshotMetadata?
     if includes("screenshot-metadata") || includes("screenshot") {
         if let window = keyWindows().first {
@@ -255,6 +270,7 @@ func currentRuntimeSnapshot(_ request: TKRuntimeSnapshotRequest) -> TKRuntimeSna
         scene: scene,
         route: route,
         responder: responder,
+        media: media,
         geometry: geometry,
         ax: ax,
         screenshot: screenshot,