feat: add benchmarking infrastructure for Moving AI datasets

The project now includes:
- mapf-bench crate: A tool for running multi-agent negotiation benchmarks on Moving AI grid maps.
- scripts/benchmark.py: An automated script that downloads maps/scenarios, runs benchmarks for 2-50 agents, and generates reports with strict timeouts.
- .gitignore update: The cache/ directory used for benchmarking data is now ignored.

Generated-by: Gemini-CLI
Signed-off-by: Arjo Chakravarty <arjoc@intrinsic.ai>

Author: arjo129

.gitignore

@@ -1,2 +1,4 @@
 /target
 Cargo.lock
+cache/
+benchmark_report.json

Cargo.toml

@@ -2,6 +2,7 @@
 members = [
     "mapf",
     "mapf-viz",
+    "mapf-bench",
 ]
 
 resolver = "2"

mapf-bench/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "mapf-bench"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+mapf = { path = "../mapf" }
+movingai = "0.2"
+anyhow = "1.0"
+clap = { version = "4.0", features = ["derive"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+indicatif = "0.17"

mapf-bench/src/main.rs

@@ -0,0 +1,65 @@
+mod movingai;
+
+use anyhow::Result;
+use clap::Parser;
+use std::path::PathBuf;
+use std::time::Instant;
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+    #[arg(short, long)]
+    map: PathBuf,
+
+    #[arg(short, long)]
+    scen: PathBuf,
+
+    #[arg(short, long, default_value_t = 0.45)]
+    radius: f64,
+
+    #[arg(short, long, default_value_t = 1.0)]
+    speed: f64,
+
+    #[arg(short, long, default_value_t = 10)]
+    num_agents: usize,
+
+    #[arg(short, long, default_value_t = 30)]
+    timeout: u64,
+}
+
+fn main() -> Result<()> {
+    let args = Args::parse();
+
+    println!("Loading map: {:?}", args.map);
+    let map = movingai::Map::from_file(&args.map)?;
+
+    println!("Loading scenario: {:?}", args.scen);
+    let scenario = movingai::MovingAIScenario::from_file(&args.scen)?;
+
+    let negotiation_scenario = scenario.to_negotiation_scenario(
+        &map,
+        args.num_agents,
+        args.radius,
+        args.speed,
+        60_f64.to_radians(),
+    );
+
+    println!("Running negotiation with {} agents", args.num_agents);
+    let start_time = Instant::now();
+    // We don't have a clean way to pass wall-clock timeout into negotiate yet,
+    // so we'll rely on the parent script to enforce strict timeouts for now,
+    // or we could add a QueueLengthLimit as a proxy.
+    let result = mapf::negotiation::negotiate(&negotiation_scenario, None);
+    let duration = start_time.elapsed();
+
+    match result {
+        Ok((_solution, _arena, _name_map)) => {
+            println!("Negotiation successful in {:?}", duration);
+        }
+        Err(e) => {
+            println!("Negotiation failed: {:?}", e);
+        }
+    }
+
+    Ok(())
+}

mapf-bench/src/movingai.rs

@@ -0,0 +1,149 @@
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+use std::path::Path;
+use anyhow::Result;
+use std::collections::HashMap;
+
+use mapf::negotiation::{Agent, Scenario};
+use std::collections::BTreeMap;
+
+pub struct Map {
+    pub width: usize,
+    pub height: usize,
+    pub grid: Vec<Vec<char>>,
+}
+
+impl Map {
+    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
+        let file = File::open(path)?;
+        let reader = BufReader::new(file);
+        let mut lines = reader.lines();
+
+        let mut width = 0;
+        let mut height = 0;
+
+        // Parse header
+        while let Some(line) = lines.next() {
+            let line = line?;
+            if line.starts_with("type") {
+                continue;
+            } else if line.starts_with("height") {
+                height = line.split_whitespace().nth(1).unwrap().parse()?;
+            } else if line.starts_with("width") {
+                width = line.split_whitespace().nth(1).unwrap().parse()?;
+            } else if line.starts_with("map") {
+                break;
+            }
+        }
+
+        let mut grid = Vec::with_capacity(height);
+        for _ in 0..height {
+            if let Some(line) = lines.next() {
+                let line = line?;
+                grid.push(line.chars().collect());
+            }
+        }
+
+        Ok(Map { width, height, grid })
+    }
+
+    pub fn to_occupancy_map(&self) -> HashMap<i64, Vec<i64>> {
+        let mut occupancy = HashMap::new();
+        for y in 0..self.height {
+            let mut row = Vec::new();
+            for x in 0..self.width {
+                let c = self.grid[y][x];
+                if c != '.' && c != 'G' && c != 'S' {
+                    row.push(x as i64);
+                }
+            }
+            if !row.is_empty() {
+                occupancy.insert(y as i64, row);
+            }
+        }
+        occupancy
+    }
+}
+
+pub struct ScenarioEntry {
+    pub bucket: usize,
+    pub map_file: String,
+    pub map_width: usize,
+    pub map_height: usize,
+    pub start_x: usize,
+    pub start_y: usize,
+    pub goal_x: usize,
+    pub goal_y: usize,
+    pub optimal_length: f64,
+}
+
+pub struct MovingAIScenario {
+    pub entries: Vec<ScenarioEntry>,
+}
+
+impl MovingAIScenario {
+    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
+        let file = File::open(path)?;
+        let reader = BufReader::new(file);
+        let mut lines = reader.lines();
+
+        // Skip version line
+        lines.next();
+
+        let mut entries = Vec::new();
+        for line in lines {
+            let line = line?;
+            let parts: Vec<&str> = line.split_whitespace().collect();
+            if parts.len() < 9 {
+                continue;
+            }
+
+            entries.push(ScenarioEntry {
+                bucket: parts[0].parse()?,
+                map_file: parts[1].to_string(),
+                map_width: parts[2].parse()?,
+                map_height: parts[3].parse()?,
+                start_x: parts[4].parse()?,
+                start_y: parts[5].parse()?,
+                goal_x: parts[6].parse()?,
+                goal_y: parts[7].parse()?,
+                optimal_length: parts[8].parse()?,
+            });
+        }
+
+        Ok(MovingAIScenario { entries })
+    }
+
+    pub fn to_negotiation_scenario(
+        &self,
+        map: &Map,
+        num_agents: usize,
+        radius: f64,
+        speed: f64,
+        spin: f64,
+    ) -> Scenario {
+        let mut agents = BTreeMap::new();
+        for i in 0..num_agents.min(self.entries.len()) {
+            let entry = &self.entries[i];
+            agents.insert(
+                format!("agent_{}", i),
+                Agent {
+                    start: [entry.start_x as i64, entry.start_y as i64],
+                    yaw: 0.0,
+                    goal: [entry.goal_x as i64, entry.goal_y as i64],
+                    radius,
+                    speed,
+                    spin,
+                },
+            );
+        }
+
+        Scenario {
+            agents,
+            obstacles: Vec::new(),
+            occupancy: map.to_occupancy_map(),
+            cell_size: 1.0,
+            camera_bounds: None,
+        }
+    }
+}

scripts/benchmark.py

@@ -0,0 +1,147 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+import zipfile
+import urllib.request
+import json
+import time
+import argparse
+from pathlib import Path
+
+# Configuration
+CACHE_DIR = Path("cache")
+MAPS_ZIP_URL = "https://www.movingai.com/benchmarks/mapf/mapf-map.zip"
+SCEN_ZIP_URL = "https://www.movingai.com/benchmarks/mapf/mapf-scen-random.zip"
+DEFAULT_AGENT_COUNTS = [2, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50]
+
+def download_file(url, dest):
+    if dest.exists():
+        return
+    print(f"Downloading {url} to {dest}...")
+    urllib.request.urlretrieve(url, dest)
+
+def unzip_file(zip_path, extract_to):
+    print(f"Unzipping {zip_path} to {extract_to}...")
+    with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+        zip_ref.extractall(extract_to)
+
+def run_benchmark(map_path, scen_path, num_agents, timeout):
+    cmd = [
+        "cargo", "run", "-p", "mapf-bench", "--release", "--",
+        "--map", str(map_path),
+        "--scen", str(scen_path),
+        "--num-agents", str(num_agents),
+        "--timeout", str(timeout)
+    ]
+    
+    start_time = time.time()
+    try:
+        # Strict timeout enforcement via subprocess
+        result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout + 5)
+        duration = time.time() - start_time
+        
+        success = "Negotiation successful" in result.stdout
+        return {
+            "success": success,
+            "duration": duration,
+            "stdout": result.stdout,
+            "stderr": result.stderr
+        }
+    except subprocess.TimeoutExpired:
+        return {
+            "success": False,
+            "duration": timeout,
+            "error": "Timeout"
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "error": str(e)
+        }
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--timeout", type=int, default=30, help="Timeout in seconds per run")
+    parser.add_argument("--max-scenarios", type=int, default=1, help="Max random scenarios per map")
+    parser.add_argument("--maps", nargs="+", default=["empty-32-32.map", "room-32-32-4.map", "maze-32-32-2.map"])
+    args = parser.parse_args()
+
+    CACHE_DIR.mkdir(exist_ok=True)
+    
+    maps_zip = CACHE_DIR / "mapf-map.zip"
+    scen_zip = CACHE_DIR / "mapf-scen-random.zip"
+    
+    download_file(MAPS_ZIP_URL, maps_zip)
+    download_file(SCEN_ZIP_URL, scen_zip)
+    
+    maps_dir = CACHE_DIR / "maps"
+    scen_dir = CACHE_DIR / "scenarios"
+    
+    if not maps_dir.exists():
+        unzip_file(maps_zip, maps_dir)
+    if not scen_dir.exists():
+        unzip_file(scen_zip, scen_dir)
+    
+    report = {}
+
+    print("Building mapf-bench in release mode...")
+    subprocess.run(["cargo", "build", "-p", "mapf-bench", "--release"], check=True)
+
+    for map_name in args.maps:
+        map_path = maps_dir / map_name
+        if not map_path.exists():
+            print(f"Map {map_name} not found.")
+            continue
+            
+        base_name = map_name.replace(".map", "")
+        
+        # Find all matching scenario files
+        scen_pattern = f"{base_name}-random-*.scen"
+        scen_files = list((scen_dir / "scen-random").glob(scen_pattern))
+        scen_files.sort()
+        
+        if not scen_files:
+            print(f"No scenarios found for {map_name} with pattern {scen_pattern}")
+            continue
+
+        selected_scens = scen_files[:args.max_scenarios]
+        
+        for scen_path in selected_scens:
+            scen_name = scen_path.name
+            print(f"\nBenchmarking {map_name} with scenario {scen_name}...")
+            
+            key = f"{map_name}:{scen_name}"
+            report[key] = []
+            
+            for count in DEFAULT_AGENT_COUNTS:
+                print(f"  Agents: {count}", end=" ", flush=True)
+                res = run_benchmark(map_path, scen_path, count, args.timeout)
+                if res["success"]:
+                    print(f"✅ ({res['duration']:.2f}s)")
+                else:
+                    error_msg = res.get("error", "FAILED")
+                    print(f"❌ ({error_msg})")
+                
+                report[key].append({
+                    "agents": count,
+                    "success": res["success"],
+                    "duration": res.get("duration", 0),
+                    "error": res.get("error")
+                })
+
+    # Save report
+    with open("benchmark_report.json", "w") as f:
+        json.dump(report, f, indent=2)
+    
+    # Print summary table
+    print("\nBenchmark Summary:")
+    print(f"{'Scenario':<40} | {'Agents':<6} | {'Status':<8} | {'Time':<8}")
+    print("-" * 75)
+    for key, results in report.items():
+        for res in results:
+            status = "SUCCESS" if res["success"] else (res.get("error") if res.get("error") else "FAILED")
+            print(f"{key[:40]:<40} | {res['agents']:<6} | {status:<8} | {res['duration']:>7.2f}s")
+
+if __name__ == "__main__":
+    main()