Skip to content

Commit 08884ac

Browse files
ScriptwonderScriptwonder
committed
Agent-based update
1 parent d303324 commit 08884ac

10 files changed

Lines changed: 1446 additions & 4 deletions

File tree

‎MCPForUnity/Editor/Tools/ManageCode.cs‎

Lines changed: 391 additions & 0 deletions
Large diffs are not rendered by default.

‎MCPForUnity/Editor/Tools/ManageCode.cs.meta‎

Lines changed: 11 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎MCPForUnity/Editor/Tools/ManageInput.cs‎

Lines changed: 270 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,270 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using MCPForUnity.Editor.Helpers;
5+
using Newtonsoft.Json.Linq;
6+
using UnityEditor;
7+
using UnityEngine;
8+
9+
namespace MCPForUnity.Editor.Tools
10+
{
11+
/// <summary>
12+
/// Simulates player input during play mode for AI game testing.
13+
/// Supports key presses, mouse clicks, axis simulation, and state queries.
14+
/// </summary>
15+
[McpForUnityTool("simulate_input", AutoRegister = false)]
16+
public static class ManageInput
17+
{
18+
public static object HandleCommand(JObject @params)
19+
{
20+
if (@params == null)
21+
return new ErrorResponse("Parameters cannot be null.");
22+
23+
var p = new ToolParams(@params);
24+
var actionResult = p.GetRequired("action");
25+
if (!actionResult.IsSuccess)
26+
return new ErrorResponse(actionResult.ErrorMessage);
27+
28+
string action = actionResult.Value.ToLowerInvariant();
29+
30+
switch (action)
31+
{
32+
case "send_key":
33+
return HandleSendKey(p);
34+
case "send_mouse_click":
35+
return HandleSendMouseClick(p);
36+
case "send_mouse_move":
37+
return HandleSendMouseMove(p);
38+
case "send_sequence":
39+
return HandleSendSequence(p, @params);
40+
case "get_state":
41+
return HandleGetState(p);
42+
default:
43+
return new ErrorResponse(
44+
$"Unknown action: '{action}'. Valid actions: send_key, send_mouse_click, send_mouse_move, send_sequence, get_state.");
45+
}
46+
}
47+
48+
private static object RequirePlayMode()
49+
{
50+
if (!EditorApplication.isPlaying)
51+
return new ErrorResponse("Input simulation requires Play Mode. Use manage_editor action='play' first.");
52+
return null;
53+
}
54+
55+
private static object HandleSendKey(ToolParams p)
56+
{
57+
var guard = RequirePlayMode();
58+
if (guard != null) return guard;
59+
60+
string key = p.Get("key");
61+
if (string.IsNullOrEmpty(key))
62+
return new ErrorResponse("'key' parameter is required (e.g., 'W', 'Space', 'Mouse0').");
63+
64+
int holdDurationMs = p.GetInt("holdDuration") ?? 100;
65+
bool press = p.GetBool("press", true);
66+
bool release = p.GetBool("release", true);
67+
68+
var simulator = GetOrCreateSimulator();
69+
if (simulator == null)
70+
return new ErrorResponse("Failed to create InputSimulator in the scene.");
71+
72+
simulator.QueueKeyAction(key, holdDurationMs, press, release);
73+
74+
return new SuccessResponse(
75+
$"Queued key '{key}' (hold {holdDurationMs}ms, press={press}, release={release}).",
76+
new { key, holdDurationMs, press, release });
77+
}
78+
79+
private static object HandleSendMouseClick(ToolParams p)
80+
{
81+
var guard = RequirePlayMode();
82+
if (guard != null) return guard;
83+
84+
float x = (float)(p.GetInt("x") ?? 0);
85+
float y = (float)(p.GetInt("y") ?? 0);
86+
int button = p.GetInt("button") ?? 0; // 0=left, 1=right, 2=middle
87+
88+
var simulator = GetOrCreateSimulator();
89+
if (simulator == null)
90+
return new ErrorResponse("Failed to create InputSimulator in the scene.");
91+
92+
simulator.QueueMouseClick(x, y, button);
93+
94+
return new SuccessResponse(
95+
$"Queued mouse click at ({x}, {y}) button={button}.",
96+
new { x, y, button });
97+
}
98+
99+
private static object HandleSendMouseMove(ToolParams p)
100+
{
101+
var guard = RequirePlayMode();
102+
if (guard != null) return guard;
103+
104+
float deltaX = (float)(p.GetInt("deltaX") ?? 0);
105+
float deltaY = (float)(p.GetInt("deltaY") ?? 0);
106+
107+
var simulator = GetOrCreateSimulator();
108+
if (simulator == null)
109+
return new ErrorResponse("Failed to create InputSimulator in the scene.");
110+
111+
simulator.QueueMouseMove(deltaX, deltaY);
112+
113+
return new SuccessResponse(
114+
$"Queued mouse move delta ({deltaX}, {deltaY}).",
115+
new { deltaX, deltaY });
116+
}
117+
118+
private static object HandleSendSequence(ToolParams p, JObject raw)
119+
{
120+
var guard = RequirePlayMode();
121+
if (guard != null) return guard;
122+
123+
var stepsToken = raw["steps"];
124+
if (stepsToken == null || stepsToken.Type != JTokenType.Array)
125+
return new ErrorResponse("'steps' parameter is required and must be an array of {action, params, duration_ms} objects.");
126+
127+
var simulator = GetOrCreateSimulator();
128+
if (simulator == null)
129+
return new ErrorResponse("Failed to create InputSimulator in the scene.");
130+
131+
var steps = (JArray)stepsToken;
132+
int queued = 0;
133+
134+
foreach (var step in steps)
135+
{
136+
if (step.Type != JTokenType.Object) continue;
137+
var stepObj = (JObject)step;
138+
string stepAction = stepObj["action"]?.ToString()?.ToLowerInvariant() ?? "";
139+
int durationMs = ParamCoercion.CoerceIntNullable(stepObj["duration_ms"] ?? stepObj["durationMs"]) ?? 100;
140+
141+
switch (stepAction)
142+
{
143+
case "key":
144+
string key = stepObj["key"]?.ToString() ?? "";
145+
if (!string.IsNullOrEmpty(key))
146+
{
147+
simulator.QueueKeyAction(key, durationMs, true, true);
148+
queued++;
149+
}
150+
break;
151+
case "mouse_click":
152+
float cx = (float)(ParamCoercion.CoerceIntNullable(stepObj["x"]) ?? 0);
153+
float cy = (float)(ParamCoercion.CoerceIntNullable(stepObj["y"]) ?? 0);
154+
int btn = ParamCoercion.CoerceIntNullable(stepObj["button"]) ?? 0;
155+
simulator.QueueMouseClick(cx, cy, btn);
156+
queued++;
157+
break;
158+
case "mouse_move":
159+
float dx = (float)(ParamCoercion.CoerceIntNullable(stepObj["deltaX"] ?? stepObj["delta_x"]) ?? 0);
160+
float dy = (float)(ParamCoercion.CoerceIntNullable(stepObj["deltaY"] ?? stepObj["delta_y"]) ?? 0);
161+
simulator.QueueMouseMove(dx, dy);
162+
queued++;
163+
break;
164+
case "wait":
165+
simulator.QueueWait(durationMs);
166+
queued++;
167+
break;
168+
}
169+
}
170+
171+
return new SuccessResponse(
172+
$"Queued {queued} input steps.",
173+
new { queued, totalSteps = steps.Count });
174+
}
175+
176+
private static object HandleGetState(ToolParams p)
177+
{
178+
var guard = RequirePlayMode();
179+
if (guard != null) return guard;
180+
181+
string query = p.Get("query") ?? "default";
182+
var state = new Dictionary<string, object>();
183+
184+
// Always include basic state
185+
state["isPlaying"] = EditorApplication.isPlaying;
186+
state["isPaused"] = EditorApplication.isPaused;
187+
state["time"] = Time.time;
188+
state["frameCount"] = Time.frameCount;
189+
190+
// Find player-like objects
191+
var cameras = UnityEngine.Object.FindObjectsOfType<Camera>();
192+
if (cameras.Length > 0)
193+
{
194+
var mainCam = Camera.main ?? cameras[0];
195+
state["cameraPosition"] = new[] { mainCam.transform.position.x, mainCam.transform.position.y, mainCam.transform.position.z };
196+
state["cameraRotation"] = new[] { mainCam.transform.eulerAngles.x, mainCam.transform.eulerAngles.y, mainCam.transform.eulerAngles.z };
197+
}
198+
199+
// Look for common player patterns
200+
var player = GameObject.FindWithTag("Player");
201+
if (player != null)
202+
{
203+
state["playerPosition"] = new[] { player.transform.position.x, player.transform.position.y, player.transform.position.z };
204+
state["playerRotation"] = new[] { player.transform.eulerAngles.x, player.transform.eulerAngles.y, player.transform.eulerAngles.z };
205+
206+
// Check for Rigidbody velocity
207+
var rb = player.GetComponent<Rigidbody>();
208+
if (rb != null)
209+
{
210+
#if UNITY_2023_3_OR_NEWER
211+
state["playerVelocity"] = new[] { rb.linearVelocity.x, rb.linearVelocity.y, rb.linearVelocity.z };
212+
state["playerSpeed"] = rb.linearVelocity.magnitude;
213+
#else
214+
state["playerVelocity"] = new[] { rb.velocity.x, rb.velocity.y, rb.velocity.z };
215+
state["playerSpeed"] = rb.velocity.magnitude;
216+
#endif
217+
}
218+
219+
var rb2d = player.GetComponent<Rigidbody2D>();
220+
if (rb2d != null)
221+
{
222+
#if UNITY_2023_3_OR_NEWER
223+
state["playerVelocity"] = new[] { rb2d.linearVelocity.x, rb2d.linearVelocity.y };
224+
state["playerSpeed"] = rb2d.linearVelocity.magnitude;
225+
#else
226+
state["playerVelocity"] = new[] { rb2d.velocity.x, rb2d.velocity.y };
227+
state["playerSpeed"] = rb2d.velocity.magnitude;
228+
#endif
229+
}
230+
}
231+
232+
// Active UI elements (for click-based games)
233+
if (query == "ui" || query == "default")
234+
{
235+
var canvases = UnityEngine.Object.FindObjectsOfType<Canvas>();
236+
var activeUiElements = new List<object>();
237+
foreach (var canvas in canvases)
238+
{
239+
if (!canvas.gameObject.activeInHierarchy) continue;
240+
var buttons = canvas.GetComponentsInChildren<UnityEngine.UI.Button>(false);
241+
foreach (var btn in buttons)
242+
{
243+
if (!btn.gameObject.activeInHierarchy || !btn.interactable) continue;
244+
var rt = btn.GetComponent<RectTransform>();
245+
activeUiElements.Add(new Dictionary<string, object>
246+
{
247+
{ "name", btn.gameObject.name },
248+
{ "type", "Button" },
249+
{ "interactable", btn.interactable },
250+
{ "position", rt != null ? new[] { rt.position.x, rt.position.y } : null },
251+
});
252+
}
253+
}
254+
if (activeUiElements.Count > 0)
255+
state["activeUiElements"] = activeUiElements;
256+
}
257+
258+
return new SuccessResponse("Game state snapshot.", state);
259+
}
260+
261+
private static MCPInputSimulator GetOrCreateSimulator()
262+
{
263+
var existing = UnityEngine.Object.FindObjectsOfType<MCPInputSimulator>();
264+
if (existing.Length > 0) return existing[0];
265+
var go = new GameObject("__MCP_InputSimulator__");
266+
go.hideFlags = HideFlags.HideAndDontSave;
267+
return go.AddComponent<MCPInputSimulator>();
268+
}
269+
}
270+
}

‎MCPForUnity/Editor/Tools/ManageInput.cs.meta‎

Lines changed: 11 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)