test(replay): expand fixtures for deny, failure, and MCP

rexleimo · rexleimo · commit 023f5d23d902 · 2026-04-05T20:33:30.000+08:00
diff --git a/crates/rexos/tests/fixtures/replay/session_mcp_echo.json b/crates/rexos/tests/fixtures/replay/session_mcp_echo.json
@@ -0,0 +1,37 @@
+[
+  {
+    "choices": [
+      {
+        "index": 0,
+        "message": {
+          "role": "assistant",
+          "content": null,
+          "tool_calls": [
+            {
+              "id": "call_1",
+              "type": "function",
+              "function": {
+                "name": "mcp_stub__echo",
+                "arguments": "{\"text\":\"yo\"}"
+              }
+            }
+          ]
+        },
+        "finish_reason": "tool_calls"
+      }
+    ]
+  },
+  {
+    "choices": [
+      {
+        "index": 0,
+        "message": {
+          "role": "assistant",
+          "content": "done"
+        },
+        "finish_reason": "stop"
+      }
+    ]
+  }
+]
+
diff --git a/crates/rexos/tests/fixtures/replay/session_tool_failed_invalid_path.json b/crates/rexos/tests/fixtures/replay/session_tool_failed_invalid_path.json
@@ -0,0 +1,25 @@
+[
+  {
+    "choices": [
+      {
+        "index": 0,
+        "message": {
+          "role": "assistant",
+          "content": null,
+          "tool_calls": [
+            {
+              "id": "call_1",
+              "type": "function",
+              "function": {
+                "name": "fs_write",
+                "arguments": "{\"path\":\"../hello.txt\",\"content\":\"hello\"}"
+              }
+            }
+          ]
+        },
+        "finish_reason": "tool_calls"
+      }
+    ]
+  }
+]
+
diff --git a/crates/rexos/tests/fixtures/replay/session_tool_not_allowed.json b/crates/rexos/tests/fixtures/replay/session_tool_not_allowed.json
@@ -0,0 +1,25 @@
+[
+  {
+    "choices": [
+      {
+        "index": 0,
+        "message": {
+          "role": "assistant",
+          "content": null,
+          "tool_calls": [
+            {
+              "id": "call_1",
+              "type": "function",
+              "function": {
+                "name": "fs_write",
+                "arguments": "{\"path\":\"hello.txt\",\"content\":\"hello\"}"
+              }
+            }
+          ]
+        },
+        "finish_reason": "tool_calls"
+      }
+    ]
+  }
+]
+
diff --git a/crates/rexos/tests/session_replay.rs b/crates/rexos/tests/session_replay.rs
@@ -7,14 +7,11 @@ use rexos::security::SecurityConfig;
 
 mod support;
 
-#[tokio::test]
-async fn replay_fixture_drives_session_and_tool_calls() {
-    let fixture = support::openai_compat_fixture::load_json_array(include_str!(
-        "fixtures/replay/session_write_file.json"
-    ));
-    let server = support::openai_compat_fixture::FixtureServer::spawn(fixture).await;
-
-    let tmp = tempfile::tempdir().unwrap();
+fn fixture_agent(
+    tmp: &tempfile::TempDir,
+    fixture_base_url: String,
+    security: SecurityConfig,
+) -> (rexos::agent::AgentRuntime, RexosPaths, std::path::PathBuf) {
     let paths = RexosPaths {
         base_dir: tmp.path().join(".loopforge"),
     };
@@ -28,14 +25,13 @@ async fn replay_fixture_drives_session_and_tool_calls() {
         "fixture".to_string(),
         ProviderConfig {
             kind: ProviderKind::OpenAiCompatible,
-            base_url: server.base_url.clone(),
+            base_url: fixture_base_url,
             api_key_env: String::new(),
             default_model: "fixture-model".to_string(),
             aws_bedrock: None,
         },
     );
 
-    let security = SecurityConfig::default();
     let cfg = RexosConfig {
         llm: Default::default(),
         providers,
@@ -62,6 +58,23 @@ async fn replay_fixture_drives_session_and_tool_calls() {
     let agent =
         rexos::agent::AgentRuntime::new_with_security_config(memory, llms, router, security);
 
+    (agent, paths, workspace_root)
+}
+
+#[tokio::test]
+async fn replay_fixture_drives_session_and_tool_calls() {
+    let fixture = support::openai_compat_fixture::load_json_array(include_str!(
+        "fixtures/replay/session_write_file.json"
+    ));
+    let server = support::openai_compat_fixture::FixtureServer::spawn(fixture).await;
+
+    let tmp = tempfile::tempdir().unwrap();
+    let (agent, _paths, workspace_root) = fixture_agent(
+        &tmp,
+        server.base_url.clone(),
+        rexos::security::SecurityConfig::default(),
+    );
+
     let session_id = "s-replay";
     agent
         .set_session_allowed_tools(session_id, vec!["fs_write".to_string()])
@@ -94,3 +107,136 @@ async fn replay_fixture_drives_session_and_tool_calls() {
 
     server.abort();
 }
+
+#[tokio::test]
+async fn replay_fixture_blocks_tool_not_in_allowed_tools() {
+    let fixture = support::openai_compat_fixture::load_json_array(include_str!(
+        "fixtures/replay/session_tool_not_allowed.json"
+    ));
+    let server = support::openai_compat_fixture::FixtureServer::spawn(fixture).await;
+
+    let tmp = tempfile::tempdir().unwrap();
+    let (agent, _paths, workspace_root) = fixture_agent(
+        &tmp,
+        server.base_url.clone(),
+        rexos::security::SecurityConfig::default(),
+    );
+
+    let session_id = "s-replay-deny";
+    agent
+        .set_session_allowed_tools(session_id, vec!["fs_read".to_string()])
+        .unwrap();
+
+    let err = agent
+        .run_session(
+            workspace_root,
+            session_id,
+            None,
+            "try write",
+            TaskKind::Coding,
+        )
+        .await
+        .unwrap_err();
+    let err_text = err.to_string();
+    assert!(
+        err_text.contains("tool not allowed"),
+        "expected deny error, got: {err_text}"
+    );
+
+    let requests = server.requests.lock().unwrap().clone();
+    assert_eq!(requests.len(), 1, "expected one chat completions call");
+
+    server.abort();
+}
+
+#[tokio::test]
+async fn replay_fixture_surfaces_tool_failure_errors() {
+    let fixture = support::openai_compat_fixture::load_json_array(include_str!(
+        "fixtures/replay/session_tool_failed_invalid_path.json"
+    ));
+    let server = support::openai_compat_fixture::FixtureServer::spawn(fixture).await;
+
+    let tmp = tempfile::tempdir().unwrap();
+    let (agent, _paths, workspace_root) = fixture_agent(
+        &tmp,
+        server.base_url.clone(),
+        rexos::security::SecurityConfig::default(),
+    );
+
+    let session_id = "s-replay-tool-failed";
+    agent
+        .set_session_allowed_tools(session_id, vec!["fs_write".to_string()])
+        .unwrap();
+
+    let err = agent
+        .run_session(
+            workspace_root,
+            session_id,
+            None,
+            "write outside workspace",
+            TaskKind::Coding,
+        )
+        .await
+        .unwrap_err();
+    let err_text = err.to_string();
+    assert!(
+        err_text.contains("parent traversal"),
+        "expected relative-path validation error, got: {err_text}"
+    );
+
+    let requests = server.requests.lock().unwrap().clone();
+    assert_eq!(requests.len(), 1, "expected one chat completions call");
+
+    server.abort();
+}
+
+#[tokio::test]
+async fn replay_fixture_executes_mcp_tool_calls() {
+    let fixture = support::openai_compat_fixture::load_json_array(include_str!(
+        "fixtures/replay/session_mcp_echo.json"
+    ));
+    let server = support::openai_compat_fixture::FixtureServer::spawn(fixture).await;
+
+    let tmp = tempfile::tempdir().unwrap();
+    let (agent, _paths, workspace_root) = fixture_agent(
+        &tmp,
+        server.base_url.clone(),
+        rexos::security::SecurityConfig::default(),
+    );
+
+    let mcp_stub = support::mcp_stub::write_mcp_stub(&workspace_root);
+
+    let session_id = "s-replay-mcp";
+    agent
+        .set_session_mcp_config(session_id, support::mcp_stub::mcp_config_json(&mcp_stub))
+        .unwrap();
+    agent
+        .set_session_allowed_tools(session_id, vec!["mcp_stub__echo".to_string()])
+        .unwrap();
+
+    let out = agent
+        .run_session(
+            workspace_root,
+            session_id,
+            None,
+            "call mcp",
+            TaskKind::Coding,
+        )
+        .await
+        .unwrap();
+    assert_eq!(out, "done");
+
+    let requests = server.requests.lock().unwrap().clone();
+    assert_eq!(requests.len(), 2, "expected two chat completions calls");
+    assert_eq!(requests[1]["messages"][2]["role"], "tool");
+    assert_eq!(requests[1]["messages"][2]["name"], "mcp_stub__echo");
+    assert_eq!(requests[1]["messages"][2]["tool_call_id"], "call_1");
+
+    let tool_content = requests[1]["messages"][2]["content"]
+        .as_str()
+        .expect("tool message content");
+    let tool_json: serde_json::Value = serde_json::from_str(tool_content).expect("tool JSON");
+    assert_eq!(tool_json["content"][0]["text"], "yo");
+
+    server.abort();
+}
diff --git a/crates/rexos/tests/support/mcp_stub.rs b/crates/rexos/tests/support/mcp_stub.rs
@@ -0,0 +1,92 @@
+use std::path::{Path, PathBuf};
+
+const MCP_STUB_PY: &str = r#"
+import json
+import sys
+
+def send(obj):
+    sys.stdout.write(json.dumps(obj) + "\n")
+    sys.stdout.flush()
+
+for line in sys.stdin:
+    line = line.strip()
+    if not line:
+        continue
+    try:
+        msg = json.loads(line)
+    except Exception:
+        continue
+
+    method = msg.get("method")
+    if not method:
+        continue
+
+    # Notifications have no id; ignore them.
+    if "id" not in msg:
+        continue
+
+    msg_id = msg.get("id")
+    params = msg.get("params") or {}
+
+    if method == "initialize":
+        send({"jsonrpc": "2.0", "id": msg_id, "result": {}})
+    elif method == "tools/list":
+        send({
+            "jsonrpc": "2.0",
+            "id": msg_id,
+            "result": {
+                "tools": [
+                    {
+                        "name": "echo",
+                        "description": "Echo input text",
+                        "inputSchema": {
+                            "type": "object",
+                            "properties": {"text": {"type": "string"}},
+                            "required": ["text"],
+                            "additionalProperties": False
+                        }
+                    }
+                ]
+            }
+        })
+    elif method == "tools/call":
+        name = params.get("name")
+        arguments = params.get("arguments") or {}
+        if name == "echo":
+            send({
+                "jsonrpc": "2.0",
+                "id": msg_id,
+                "result": {"content": [{"type": "text", "text": arguments.get("text", "")}]}
+            })
+        else:
+            send({"jsonrpc": "2.0", "id": msg_id, "error": {"code": -32601, "message": "unknown tool"}})
+    else:
+        send({"jsonrpc": "2.0", "id": msg_id, "error": {"code": -32601, "message": "unknown method"}})
+"#;
+
+fn mcp_python_exe() -> &'static str {
+    if cfg!(windows) {
+        "python"
+    } else {
+        "python3"
+    }
+}
+
+pub fn write_mcp_stub(workspace: &Path) -> PathBuf {
+    let path = workspace.join("mcp_stub.py");
+    std::fs::write(&path, MCP_STUB_PY).expect("write mcp stub script");
+    path
+}
+
+pub fn mcp_config_json(script: &Path) -> String {
+    serde_json::json!({
+        "servers": {
+            "stub": {
+                "command": mcp_python_exe(),
+                "args": ["-u", script.to_string_lossy()],
+                "cwd": ".",
+            }
+        }
+    })
+    .to_string()
+}
diff --git a/crates/rexos/tests/support/mod.rs b/crates/rexos/tests/support/mod.rs
@@ -1 +1,2 @@
+pub mod mcp_stub;
 pub mod openai_compat_fixture;

Original file line number	Diff line number	Diff line change
`@@ -1 +1,2 @@`
	`1`	`+pub mod mcp_stub;`
`1`	`2`	`pub mod openai_compat_fixture;`