Migrate integration tests to exact MCP responses

src/sandbox/seatbelt_base_policy.sbpl

@@ -22,6 +22,8 @@
   (require-all
     (path "/dev/null")
     (vnode-type CHARACTER-DEVICE)))
+; Do not allow /dev/stdout, /dev/stderr, or /dev/fd aliases without a real
+; package or user workflow that requires reopening inherited stdio by path.
 ; Allow uv's SystemConfiguration probe to touch /dev/dtracehelper on macOS.
 (allow file-write-data
   (require-all

tests/codex_integration.rs

@@ -40,9 +40,6 @@ mod unix_impl {
     const MOCK_TOOL_INPUT: &str = "cat(\"CODEX_MOCK_MCP_OK\\n\")\n";
     const LIVE_FINAL_MARKER: &str = "CODEX_LIVE_DONE";
     const INSTALL_SCRIPTED_TOOL_CALL_MARKER: &str = "INSTALL_SCRIPTED_TOOL_CALL";
-    #[cfg(any(target_os = "macos", target_os = "linux"))]
-    const FULL_ACCESS_TEST_ENV: &str = "MCP_REPL_ENABLE_FULL_ACCESS_TUI_TEST";
-
     fn codex_exec_test_mutex() -> &'static tokio::sync::Mutex<()> {
         static TEST_MUTEX: OnceLock<tokio::sync::Mutex<()>> = OnceLock::new();
         TEST_MUTEX.get_or_init(|| tokio::sync::Mutex::new(()))
@@ -426,17 +423,10 @@ mod unix_impl {
     #[cfg(any(target_os = "macos", target_os = "linux"))]
     pub(super) async fn run_codex_tui_full_access_sandbox_update() -> TestResult<()> {
         const TEST_NAME: &str = "codex_tui_full_access_sandbox_update";
-        if !full_access_test_enabled() {
-            print_skip_banner(TEST_NAME, &format!("{FULL_ACCESS_TEST_ENV} is not set"));
-            return Ok(());
-        }
         if !codex_client_ready(TEST_NAME) {
             return Ok(());
         }
-        if !common::sandbox_exec_available() {
-            print_skip_banner(TEST_NAME, "sandbox-exec unavailable");
-            return Ok(());
-        }
+        assert!(common::sandbox_exec_available(), "sandbox-exec unavailable");
         if !loopback_bind_available().await {
             print_skip_banner(TEST_NAME, "loopback TCP bind unavailable");
             return Ok(());
@@ -459,34 +449,33 @@ mod unix_impl {
         driver.drain(Duration::from_millis(800));
         driver.ensure_running("after startup")?;
         driver.wait_for_warmup(Duration::from_secs(10))?;
-        wait_for_log_contains(&env.debug_dir, "workspace-write", Duration::from_secs(10))?;
 
         driver.send_line(&format!(
             "{FULL_ACCESS_MARKER}: probe write before full access"
         ))?;
         driver.wait_for_contains("WRITE_ERROR:", Duration::from_secs(20))?;
         driver.wait_for_contains("Tool call 1 completed", Duration::from_secs(20))?;
+        wait_for_log_contains(&env.debug_dir, "workspace-write", Duration::from_secs(10))?;
 
-        driver.send_line("/approvals")?;
+        driver.send_line("/permissions")?;
         driver.wait_for_contains("Update Model Permissions", Duration::from_secs(15))?;
         driver.send("3")?;
         driver.wait_for_contains(
             "Permissions updated to Full Access",
             Duration::from_secs(15),
         )?;
 
+        driver.send_line(&format!(
+            "{FULL_ACCESS_MARKER}: probe write after full access"
+        ))?;
+        driver.wait_for_contains("WRITE_OK", Duration::from_secs(20))?;
+        driver.wait_for_contains("Tool call 2 completed", Duration::from_secs(20))?;
         wait_for_log_contains(
             &env.debug_dir,
             "danger-full-access",
             Duration::from_secs(20),
         )?;
         wait_for_log_contains(&env.debug_dir, "tool-call-meta", Duration::from_secs(20))?;
-
-        driver.send_line(&format!(
-            "{FULL_ACCESS_MARKER}: probe write after full access"
-        ))?;
-        driver.wait_for_contains("WRITE_OK", Duration::from_secs(20))?;
-        driver.wait_for_contains("Tool call 2 completed", Duration::from_secs(20))?;
         driver.kill()?;
 
         let outputs = mock_server.function_call_outputs().await;
@@ -538,11 +527,6 @@ mod unix_impl {
         Ok(())
     }
 
-    #[cfg(any(target_os = "macos", target_os = "linux"))]
-    fn full_access_test_enabled() -> bool {
-        std::env::var_os(FULL_ACCESS_TEST_ENV).is_some()
-    }
-
     async fn loopback_bind_available() -> bool {
         TcpListener::bind("127.0.0.1:0").await.is_ok()
     }
@@ -1513,7 +1497,11 @@ trust_level = "trusted"
         let nanos = std::time::SystemTime::now()
             .duration_since(std::time::UNIX_EPOCH)?
             .as_nanos();
-        let target = std::env::temp_dir().join(format!("mcp-repl-codex-probe-{nanos}.txt"));
+        let home = std::env::var_os("HOME")
+            .or_else(|| std::env::var_os("USERPROFILE"))
+            .map(PathBuf::from)
+            .ok_or_else(|| "missing HOME/USERPROFILE for outside-workspace probe".to_string())?;
+        let target = home.join(format!(".mcp-repl-codex-probe-{nanos}.txt"));
         let target_literal = serde_json::to_string(&target.to_string_lossy().to_string())
             .map_err(|err| format!("failed to encode target path: {err}"))?;
         Ok(r#"target <- __TARGET__

tests/common/mod.rs

@@ -212,11 +212,13 @@ impl Drop for SuiteServerLockToken {
 pub fn sandbox_exec_available() -> bool {
     static AVAILABLE: OnceLock<bool> = OnceLock::new();
     *AVAILABLE.get_or_init(|| {
-        if std::env::var_os("CODEX_SANDBOX").is_some() {
-            return false;
-        }
         std::process::Command::new("/usr/bin/sandbox-exec")
-            .args(["-p", "(version 1)", "--", "/usr/bin/true"])
+            .args([
+                "-p",
+                "(version 1)(allow process-exec)(allow file-read*)",
+                "--",
+                "/usr/bin/true",
+            ])
             .stdin(std::process::Stdio::null())
             .stdout(std::process::Stdio::null())
             .stderr(std::process::Stdio::null())

tests/debug_repl_prompt.rs

@@ -12,11 +12,13 @@ type TestResult<T> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
 #[cfg(target_os = "macos")]
 fn sandbox_exec_available() -> bool {
     // Mirror tests/common/mod.rs: sandbox-exec may exist but be unusable (status 71).
-    if std::env::var_os("CODEX_SANDBOX").is_some() {
-        return false;
-    }
     Command::new("/usr/bin/sandbox-exec")
-        .args(["-p", "(version 1)", "--", "/usr/bin/true"])
+        .args([
+            "-p",
+            "(version 1)(allow process-exec)(allow file-read*)",
+            "--",
+            "/usr/bin/true",
+        ])
         .stdin(Stdio::null())
         .stdout(Stdio::null())
         .stderr(Stdio::null())

tests/python_backend.rs

@@ -234,19 +234,26 @@ fn real_python_executable() -> TestResult<String> {
 async fn python_discovery_keeps_venv_probe_inside_sandbox() -> TestResult<()> {
     use std::os::unix::fs::PermissionsExt;
 
-    if !common::sandbox_exec_available() {
-        eprintln!("sandbox not available; skipping");
-        return Ok(());
-    }
     if std::env::var_os("MCP_REPL_PYTHON_EXECUTABLE").is_some() {
-        eprintln!("explicit Python executable set; skipping discovery test");
+        eprintln!("explicit Python executable set; skipping discovery sandbox coverage test");
         return Ok(());
     }
+    assert!(common::sandbox_exec_available(), "sandbox unavailable");
 
     let _guard = lock_test_mutex();
+    let real_python = real_python_executable()?;
     let workspace = tempdir()?;
-    let outside = tempdir()?;
-    let marker = outside.path().join("python-discovery-marker");
+    let empty_bin = workspace.path().join("empty-bin");
+    fs::create_dir_all(&empty_bin)?;
+    let home = std::env::var_os("HOME")
+        .or_else(|| std::env::var_os("USERPROFILE"))
+        .map(PathBuf::from)
+        .ok_or("missing HOME/USERPROFILE for Python discovery sandbox marker")?;
+    let marker = home.join(format!(
+        ".mcp-repl-python-discovery-marker-{}",
+        std::process::id()
+    ));
+    let _ = fs::remove_file(&marker);
     let marker_text = marker
         .to_str()
         .ok_or("marker path must be valid utf-8")?
@@ -256,7 +263,22 @@ async fn python_discovery_keeps_venv_probe_inside_sandbox() -> TestResult<()> {
     let shim = venv_bin.join("python");
     fs::write(
         &shim,
-        "#!/bin/sh\nprintf probe > \"$MCP_REPL_TEST_PYTHON_PROBE_MARKER\"\nexit 1\n",
+        concat!(
+            "#!/bin/sh\n",
+            "exec \"$MCP_REPL_REAL_PYTHON\" - <<'PY'\n",
+            "import os\n",
+            "import sys\n",
+            "from pathlib import Path\n",
+            "\n",
+            "try:\n",
+            "    Path(os.environ['MCP_REPL_TEST_PYTHON_PROBE_MARKER']).write_text('probe')\n",
+            "except Exception as err:\n",
+            "    print(f'MCP_REPL_TEST_PYTHON_PROBE_WRITE_ERROR:{type(err).__name__}', file=sys.stderr)\n",
+            "else:\n",
+            "    print('MCP_REPL_TEST_PYTHON_PROBE_WRITE_OK', file=sys.stderr)\n",
+            "raise SystemExit(1)\n",
+            "PY\n",
+        ),
     )?;
     let mut permissions = fs::metadata(&shim)?.permissions();
     permissions.set_mode(0o755);
@@ -269,16 +291,26 @@ async fn python_discovery_keeps_venv_probe_inside_sandbox() -> TestResult<()> {
             "--sandbox".to_string(),
             "read-only".to_string(),
         ],
-        vec![("MCP_REPL_TEST_PYTHON_PROBE_MARKER".to_string(), marker_text)],
+        vec![
+            ("PATH".to_string(), empty_bin.display().to_string()),
+            ("MCP_REPL_REAL_PYTHON".to_string(), real_python),
+            ("MCP_REPL_TEST_PYTHON_PROBE_MARKER".to_string(), marker_text),
+        ],
         Some(workspace.path().to_path_buf()),
     )
     .await?;
     let result = session.write_stdin_raw_with("1+1", Some(5.0)).await?;
     let text = result_text(&result);
     session.cancel().await?;
+    let marker_exists = marker.exists();
+    let _ = fs::remove_file(&marker);
 
     assert!(
-        !marker.exists(),
+        text.contains("MCP_REPL_TEST_PYTHON_PROBE_WRITE_ERROR:"),
+        "expected Python discovery probe write failure in reply, got: {text:?}"
+    );
+    assert!(
+        !marker_exists,
         "Python discovery probe wrote outside the sandbox; reply was: {text:?}"
     );
     Ok(())

tests/repl_surface.rs

@@ -336,10 +336,14 @@ async fn direct_stdout_fd_write_falls_back_to_raw_stream() -> TestResult<()> {
         "if (!file.exists('/dev/stdout')) { ",
         "cat('SKIP_DIRECT_FD\\n') ",
         "} else { ",
-        "con <- suppressWarnings(file('/dev/stdout', open = 'wb')); ",
+        "con <- tryCatch(suppressWarnings(file('/dev/stdout', open = 'wb')), error = function(e) NULL); ",
+        "if (is.null(con)) { ",
+        "cat('SKIP_DIRECT_FD\\n') ",
+        "} else { ",
         "writeBin(charToRaw('direct-fd\\n'), con); ",
         "flush(con); close(con); ",
         "cat('r-owned\\n') ",
+        "} ",
         "}",
     );
     let result = session.write_stdin_raw_with(input, Some(30.0)).await?;