Skip to content

Commit 3a260a5

Browse files
author
Vladislav Prusakov
committed
New tools
1 parent 5ca71f1 commit 3a260a5

14 files changed

Lines changed: 1032 additions & 569 deletions

README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ The important distinction is:
1818

1919
- 😳 World, entity, component, resource, and asset inspection.
2020
- 📸 Screenshot capture for live render output.
21-
- 💉 AdaUI inspection tools such as `ui.list_windows`, `ui.get_tree`, `ui.get_node`, `ui.find_nodes`, and `ui.hit_test`.
21+
- 💉 AdaUI inspection tools such as `ui.list_windows`, `ui.get_tree`, `ui.get_node`, `ui.find_nodes`, `ui.capture_node_screenshot`, and `ui.hit_test`.
2222
- 🧑‍🏫 AdaUI diagnostics and safe actions such as focus traversal, deterministic tap, and scroll-to-node.
2323
- 📦 MCP resources under `ada://...`, including `ada://ui/windows`, `ada://ui/window/{id}`, `ada://ui/tree/{id}`, and `ada://ui/node/{windowId}/{nodeRef}`.
2424

@@ -69,4 +69,4 @@ AdaUI support is intentionally inspection-first. The main flow is:
6969
3. Apply a limited, deterministic action.
7070
4. Re-read the tree or diagnostics to verify the result.
7171

72-
External callers should target nodes by `accessibilityIdentifier`. `runtimeId` is returned in payloads as a session-local helper, but it is not intended to be a durable contract.
72+
External callers should target nodes by `accessibilityIdentifier`. `ui.find_nodes` can also search by `nodeType` or `viewType`, for example to locate all scroll views. `runtimeId` is returned in payloads as a session-local helper, but it is not intended to be a durable contract.

Sources/AdaMCPCore/AdaMCPCore.docc/AdaMCPCore.md

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,18 @@ Use `AdaMCPCore` when you need to expose live engine state to tools, agents, or
88

99
- Inspect worlds, entities, components, resources, and assets.
1010
- Capture render output for visual verification.
11-
- Inspect AdaUI windows and view trees.
11+
- Inspect AdaUI windows and view trees, including node lookup by accessibility identifier, runtime ID, node type, or view type.
12+
- Capture a screenshot cropped to the visible bounds of a selected AdaUI node.
1213
- Run a small, deterministic set of AdaUI actions such as focus traversal, scrolling, and tapping a resolved node.
1314

1415
`AdaMCP` is transport-agnostic at the runtime layer. `AdaMCPServer` handles MCP server wiring, while `AdaMCPPlugin` embeds the runtime inside an AdaEngine app.
1516

17+
## AdaUI inspection highlights
18+
19+
Use `ui.find_nodes` to locate live AdaUI nodes. It accepts `accessibilityIdentifier`, `runtimeId`, `nodeType`, or `viewType`; type queries use case-insensitive substring matching, so `nodeType: "ScrollView"` can find scroll view nodes.
20+
21+
Use `ui.capture_node_screenshot` when an agent needs visual context for one UI element instead of the whole render target. The tool resolves a unique node, captures the current render output, crops the PNG to the node's visible frame, and returns both the source screenshot metadata and the cropped image path.
22+
1623
## Topics
1724

1825
### Essentials
Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import AdaEngine
2+
import MCP
3+
4+
extension AdaMCPRuntime {
5+
func assetFindPayload(arguments: [String: Value]) async throws -> Value {
6+
let assets = await AssetsManager.cachedAssets().filter { asset in
7+
if let path = arguments["path"]?.stringValue, !asset.assetPath.contains(path) {
8+
return false
9+
}
10+
if let name = arguments["name"]?.stringValue, !asset.assetName.localizedCaseInsensitiveContains(name) {
11+
return false
12+
}
13+
if let type = arguments["type"]?.stringValue, asset.typeName != type {
14+
return false
15+
}
16+
if let assetID = arguments["assetId"]?.stringValue, asset.assetID != assetID {
17+
return false
18+
}
19+
return true
20+
}
21+
return ["assets": .array(assets.map(self.makeAssetPayload))]
22+
}
23+
24+
func assetGetPayload(arguments: [String: Value]) async throws -> Value {
25+
let assetsValue = try await self.assetFindPayload(arguments: arguments)
26+
guard let first = assetsValue.objectValue?["assets"]?.arrayValue?.first else {
27+
let query = arguments.map { "\($0.key)=\($0.value)" }.sorted().joined(separator: ", ")
28+
throw AdaMCPError.assetNotFound(query)
29+
}
30+
return first
31+
}
32+
}
33+
34+
private extension AdaMCPRuntime {
35+
func makeAssetPayload(_ asset: AssetsManager.CachedAssetInfo) -> Value {
36+
let diagnostics: [Value]
37+
if registry.descriptor(named: asset.typeName) == nil {
38+
diagnostics = [
39+
self.diagnosticValue(
40+
code: "unregistered_asset_descriptor",
41+
message: "Asset type \(asset.typeName) has no explicit MCP descriptor."
42+
)
43+
]
44+
} else {
45+
diagnostics = []
46+
}
47+
48+
return .object([
49+
"type": .string(asset.typeName),
50+
"kind": .string(MCPTypeKind.asset.rawValue),
51+
"id": asset.assetID.map(Value.string) ?? .null,
52+
"name": .string(asset.assetName),
53+
"world": .null,
54+
"summary": .object([
55+
"isLoaded": .bool(asset.isLoaded),
56+
"handleCount": .int(asset.handleCount)
57+
]),
58+
"fields": .object([
59+
"assetPath": .string(asset.assetPath),
60+
"assetName": .string(asset.assetName),
61+
"assetID": asset.assetID.map(Value.string) ?? .null,
62+
"isLoaded": .bool(asset.isLoaded),
63+
"handleCount": .int(asset.handleCount)
64+
]),
65+
"diagnostics": .array(diagnostics)
66+
])
67+
}
68+
}
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
import AdaEngine
2+
import MCP
3+
4+
extension AdaMCPRuntime {
5+
func entityByIDPayload(arguments: [String: Value]) throws -> Value {
6+
guard let entityID = arguments["id"]?.intValue else {
7+
throw AdaMCPError.invalidArguments("Argument 'id' is required.")
8+
}
9+
let worldName = arguments["world"]?.stringValue ?? AppWorldName.main.rawValue
10+
return try self.entityPayload(worldName: worldName, entityID: entityID)
11+
}
12+
13+
func entityByNamePayload(arguments: [String: Value]) throws -> Value {
14+
guard let entityName = arguments["name"]?.stringValue, !entityName.isEmpty else {
15+
throw AdaMCPError.invalidArguments("Argument 'name' is required.")
16+
}
17+
let resolved = try self.resolveWorld(named: arguments["world"]?.stringValue)
18+
guard let entity = resolved.world.main.getEntityByName(entityName) else {
19+
throw AdaMCPError.entityNamedNotFound(world: resolved.name, entityName: entityName)
20+
}
21+
return try self.makeEntityPayload(worldName: resolved.name, world: resolved.world.main, entity: entity)
22+
}
23+
24+
func findEntitiesPayload(arguments: [String: Value]) throws -> Value {
25+
let resolved = try self.resolveWorld(named: arguments["world"]?.stringValue)
26+
let nameQuery = arguments["name"]?.stringValue?.lowercased()
27+
let active = arguments["active"]?.boolValue
28+
let componentType = arguments["componentType"]?.stringValue
29+
30+
let entities = try resolved.world.main.getEntities().filter { entity in
31+
if let nameQuery, !entity.name.lowercased().contains(nameQuery) {
32+
return false
33+
}
34+
if let active, entity.isActive != active {
35+
return false
36+
}
37+
if let componentType, !resolved.world.main.hasComponent(named: componentType, in: entity.id) {
38+
return false
39+
}
40+
return true
41+
}
42+
.map { entity in
43+
try self.makeEntityPayload(worldName: resolved.name, world: resolved.world.main, entity: entity)
44+
}
45+
46+
return ["entities": .array(entities)]
47+
}
48+
49+
func entityComponentsPayload(arguments: [String: Value]) throws -> Value {
50+
guard let entityID = arguments["entityId"]?.intValue else {
51+
throw AdaMCPError.invalidArguments("Argument 'entityId' is required.")
52+
}
53+
let resolved = try self.resolveWorld(named: arguments["world"]?.stringValue)
54+
guard let entity = resolved.world.main.getEntityByID(entityID) else {
55+
throw AdaMCPError.entityNotFound(world: resolved.name, entityID: entityID)
56+
}
57+
let components = try self.inspectComponents(world: resolved.world.main, entity: entity)
58+
return ["components": .array(components.payloads), "diagnostics": .array(components.diagnostics)]
59+
}
60+
61+
func componentPayload(arguments: [String: Value]) throws -> Value {
62+
guard let entityID = arguments["entityId"]?.intValue else {
63+
throw AdaMCPError.invalidArguments("Argument 'entityId' is required.")
64+
}
65+
guard let componentType = arguments["componentType"]?.stringValue, !componentType.isEmpty else {
66+
throw AdaMCPError.invalidArguments("Argument 'componentType' is required.")
67+
}
68+
let resolved = try self.resolveWorld(named: arguments["world"]?.stringValue)
69+
guard resolved.world.main.getEntityByID(entityID) != nil else {
70+
throw AdaMCPError.entityNotFound(world: resolved.name, entityID: entityID)
71+
}
72+
guard let component = resolved.world.main.getComponent(named: componentType, from: entityID) else {
73+
throw AdaMCPError.componentNotFound(
74+
world: resolved.name,
75+
entityID: entityID,
76+
componentType: componentType
77+
)
78+
}
79+
return try self.inspectValue(component)
80+
}
81+
82+
func entityPayload(worldName: String, entityID: Int) throws -> Value {
83+
let resolved = try self.resolveWorld(named: worldName)
84+
guard let entity = resolved.world.main.getEntityByID(entityID) else {
85+
throw AdaMCPError.entityNotFound(world: resolved.name, entityID: entityID)
86+
}
87+
return try self.makeEntityPayload(worldName: resolved.name, world: resolved.world.main, entity: entity)
88+
}
89+
}
90+
91+
private extension AdaMCPRuntime {
92+
func makeEntityPayload(worldName: String, world: World, entity: Entity) throws -> Value {
93+
let components = try self.inspectComponents(world: world, entity: entity)
94+
return .object([
95+
"type": "entity",
96+
"id": .int(entity.id),
97+
"name": .string(entity.name),
98+
"world": .string(worldName),
99+
"summary": .object([
100+
"isActive": .bool(entity.isActive),
101+
"componentCount": .int(entity.components.count),
102+
"inspectableComponentCount": .int(components.payloads.count)
103+
]),
104+
"fields": .object([
105+
"isActive": .bool(entity.isActive)
106+
]),
107+
"components": .array(components.payloads),
108+
"diagnostics": .array(components.diagnostics)
109+
])
110+
}
111+
112+
func inspectComponents(
113+
world: World,
114+
entity: Entity
115+
) throws -> (payloads: [Value], diagnostics: [Value]) {
116+
var payloads: [Value] = []
117+
var diagnostics: [Value] = []
118+
119+
for (typeName, component) in world.getComponents(for: entity.id) {
120+
do {
121+
payloads.append(try self.inspectValue(component))
122+
} catch {
123+
diagnostics.append(self.diagnosticValue(
124+
code: "not_inspectable",
125+
message: "Component \(typeName) on entity \(entity.id) is not inspectable."
126+
))
127+
}
128+
}
129+
130+
return (payloads, diagnostics)
131+
}
132+
}
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import MCP
2+
3+
extension AdaMCPRuntime {
4+
func captureScreenshotPayload(arguments: [String: Value]) async throws -> Value {
5+
let result = try await renderCaptureService.capture(
6+
cameraEntityID: arguments["cameraEntityId"]?.intValue,
7+
cameraName: arguments["cameraName"]?.stringValue,
8+
pauseBeforeCapture: arguments["pauseBeforeCapture"]?.boolValue ?? true,
9+
refreshFrame: true
10+
)
11+
return try Value(result)
12+
}
13+
}
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
import MCP
2+
3+
extension AdaMCPRuntime {
4+
func listResourcesPayload(arguments: [String: Value]) throws -> Value {
5+
let resolved = try self.resolveWorld(named: arguments["world"]?.stringValue)
6+
var payloads: [Value] = []
7+
var diagnostics: [Value] = []
8+
9+
for resource in resolved.world.main.getResources() {
10+
do {
11+
payloads.append(try self.inspectValue(resource))
12+
} catch {
13+
diagnostics.append(self.diagnosticValue(
14+
code: "not_inspectable",
15+
message: "Resource \(String(reflecting: type(of: resource))) is not inspectable."
16+
))
17+
}
18+
}
19+
20+
return ["resources": .array(payloads), "diagnostics": .array(diagnostics)]
21+
}
22+
23+
func resourcePayload(arguments: [String: Value]) throws -> Value {
24+
guard let resourceType = arguments["resourceType"]?.stringValue, !resourceType.isEmpty else {
25+
throw AdaMCPError.invalidArguments("Argument 'resourceType' is required.")
26+
}
27+
let resolved = try self.resolveWorld(named: arguments["world"]?.stringValue)
28+
guard let resource = resolved.world.main.getResource(named: resourceType) else {
29+
throw AdaMCPError.resourceNotFound(resourceType)
30+
}
31+
return try self.inspectValue(resource)
32+
}
33+
34+
func inspectValue(_ value: Any) throws -> Value {
35+
guard let descriptor = registry.descriptor(for: value) else {
36+
throw AdaMCPError.notInspectable(String(reflecting: type(of: value)))
37+
}
38+
guard let serialized = try registry.serialize(value) else {
39+
throw AdaMCPError.notInspectable(descriptor.name)
40+
}
41+
let fields = serialized.objectValue ?? ["value": serialized]
42+
return .object([
43+
"type": .string(descriptor.name),
44+
"kind": .string(descriptor.kind.rawValue),
45+
"summary": .object([
46+
"fieldCount": .int(fields.count)
47+
]),
48+
"fields": .object(fields),
49+
"diagnostics": .array([])
50+
])
51+
}
52+
53+
func diagnosticValue(code: String, message: String) -> Value {
54+
.object([
55+
"code": .string(code),
56+
"message": .string(message)
57+
])
58+
}
59+
}
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import AdaEngine
2+
import MCP
3+
4+
extension AdaMCPRuntime {
5+
func pausePayload(reason: String?) throws -> Value {
6+
let control = appWorlds.main.getOrInitRefResource(SimulationControl.self) {
7+
SimulationControl()
8+
}
9+
control.wrappedValue.mode = .paused
10+
control.wrappedValue.reason = reason ?? "runtime.pause"
11+
control.wrappedValue.pendingStepCount = 0
12+
return try Value(control.wrappedValue)
13+
}
14+
15+
func resumePayload() throws -> Value {
16+
let control = appWorlds.main.getOrInitRefResource(SimulationControl.self) {
17+
SimulationControl()
18+
}
19+
control.wrappedValue.mode = .running
20+
control.wrappedValue.reason = nil
21+
control.wrappedValue.pendingStepCount = 0
22+
return try Value(control.wrappedValue)
23+
}
24+
25+
func stepFramePayload(frames: Int) async throws -> Value {
26+
let frameCount = max(frames, 1)
27+
let control = appWorlds.main.getOrInitRefResource(SimulationControl.self) {
28+
SimulationControl()
29+
}
30+
control.wrappedValue.mode = .paused
31+
control.wrappedValue.reason = "runtime.step_frame"
32+
control.wrappedValue.pendingStepCount += frameCount
33+
34+
for _ in 0..<frameCount {
35+
try await appWorlds.update()
36+
}
37+
38+
return try Value(control.wrappedValue)
39+
}
40+
}
Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import Foundation
2+
import MCP
3+
4+
extension AdaMCPRuntime {
5+
func traceSpansPayload(arguments: [String: Value]) throws -> Value {
6+
guard let traceRecorder else {
7+
throw AdaMCPError.tracingUnavailable
8+
}
9+
return traceRecorder.spansPayload(arguments: arguments)
10+
}
11+
12+
func traceOTLPPayload(arguments: [String: Value]) throws -> Value {
13+
guard let traceRecorder else {
14+
throw AdaMCPError.tracingUnavailable
15+
}
16+
return traceRecorder.otlpPayload(arguments: arguments)
17+
}
18+
19+
func traceClearPayload() throws -> Value {
20+
guard let traceRecorder else {
21+
throw AdaMCPError.tracingUnavailable
22+
}
23+
let clearedCount = traceRecorder.finishedSpanCount
24+
traceRecorder.clearFinishedSpans()
25+
return .object([
26+
"clearedSpanCount": .int(clearedCount),
27+
"activeSpanCount": .int(traceRecorder.activeSpanCount)
28+
])
29+
}
30+
31+
func traceResourcePayload(url: URL, uri: String) throws -> Value {
32+
let parts = url.path.split(separator: "/", omittingEmptySubsequences: true).map(String.init)
33+
guard let format = parts.first else {
34+
throw AdaMCPError.invalidResourceURI(uri)
35+
}
36+
37+
switch format {
38+
case "spans":
39+
return try self.traceSpansPayload(arguments: [:])
40+
case "otlp":
41+
return try self.traceOTLPPayload(arguments: [:])
42+
default:
43+
throw AdaMCPError.invalidResourceURI(uri)
44+
}
45+
}
46+
}

0 commit comments

Comments
 (0)