Understanding & Replicating mini-swe-agent scores on swebench.com

[yishangupenn]

Copied from upstream: SWE-agent/mini-swe-agent#756
Original author: @wise-east
Originally created: 2026-02-23

---

Describe the issue

Hi, I'm taking a stab at replicating SWE-Bench Verified scores on swebench.com to establish a baseline to improve on. I'm specifically trying to replicate 🆕 mini-SWE-agent + MiniMax M2.5 (high reasoning), which gets 75.80% and ✅ mini-SWE-agent with gpt-5 mini (medium reasoning), which gets 59.80% (or 56.20%? I'm referring to row with 🆕 mini-SWE-agent + GPT-5 Mini, which doesn't specify the reasoning effort. If reasoning is set to None, GPT-5 mini's reasoning defaults to medium, so I'm wondering why these two rows have different results. Is this gap within variance?).

My attempt with MiniMax M2.5 (high reasoning - is this configurable? I think there is no reasoning effort knob for this model) only got 66%:

Total instances: 500
Instances submitted: 500
Instances completed: 488
Instances incomplete: 0
Instances resolved: 330
Instances unresolved: 157
Instances with empty patches: 2
Instances with errors: 11

My attempt with gpt-5 mini (high reasoning) only got me 57%:

Total instances: 500
Instances submitted: 499
Instances completed: 479
Instances incomplete: 1
Instances resolved: 285
Instances unresolved: 180
Instances with empty patches: 10
Instances with errors: 24

This is the eval script that I set up to use Modal:

#!/usr/bin/env python3
"""Run mini-SWE-agent on SWE-bench Verified using Modal."""

import concurrent.futures
import json
import subprocess
import sys
import time
from enum import Enum
from pathlib import Path
from typing import Any

import argparse

from rich.live import Live

from minisweagent.config import get_config_from_spec
from minisweagent.run.benchmarks.swebench import (
    DATASET_MAPPING,
    filter_instances,
    process_instance,
    remove_from_preds_file,
)
from minisweagent.run.benchmarks.utils.batch_progress import RunBatchProgressManager
from minisweagent.utils.log import add_file_handler, logger
from minisweagent.utils.serialize import recursive_merge

INFRA_MAX_RETRIES = 3
NO_RETRY_STATUSES = {
    "Submitted",
    "ContextWindowExceededError",
    "AuthenticationError",
    "NotFoundError",
}

SUBSET = "verified"
SPLIT = "test"
OUTPUT_DIR = Path("trajectories")
WORKERS = 20


class ModelPreset(str, Enum):
    GPT5_MINI = "gpt-5-mini"
    MINIMAX_M25 = "minimax-m2.5"


MODEL_CONFIGS: dict[ModelPreset, dict[str, Any]] = {
    ModelPreset.GPT5_MINI: {
        "base_config": "swebench",
        "model_name": "openai/gpt-5-mini",
        "model_class": "litellm",
        "model_kwargs": {"reasoning_effort": "high"},
    },
    ModelPreset.MINIMAX_M25: {
        "base_config": "swebench_xml",
        "model_name": "together_ai/MiniMaxAI/MiniMax-M2.5",
        "model_class": "litellm_textbased",
        "model_kwargs": {"drop_params": True},
        "litellm_model_registry": str(Path(__file__).parent / "minimax_m25_registry.json"), # only for cost tracking, set to same as minimax-m2
    },
}


def _should_retry(output_dir: Path, instance_id: str) -> str | None:
    """Return the exit status if the instance should be retried, None otherwise."""
    traj_path = output_dir / instance_id / f"{instance_id}.traj.json"
    if traj_path.exists():
        status = json.loads(traj_path.read_text()).get("info", {}).get("exit_status", "")
        if status in NO_RETRY_STATUSES:
            return None
        return status
    preds_path = output_dir / "preds.json"
    if preds_path.exists():
        preds = json.loads(preds_path.read_text())
        if instance_id in preds and not preds[instance_id].get("model_patch"):
            return "no_trajectory_empty_patch"
    return None


def process_instance_with_retries(instance, output_dir, config, progress_manager):
    import random
    import shutil

    iid = instance["instance_id"]
    for attempt in range(1, INFRA_MAX_RETRIES + 1):
        process_instance(instance, output_dir, config, progress_manager)
        status = _should_retry(output_dir, iid)
        if status is None:
            return
        if attempt < INFRA_MAX_RETRIES:
            backoff = 2**attempt + random.uniform(0, 5)
            logger.warning(f"Retryable error '{status}' on {iid} (attempt {attempt}/{INFRA_MAX_RETRIES}). Retrying in {backoff:.0f}s...")
            shutil.rmtree(output_dir / iid, ignore_errors=True)
            remove_from_preds_file(output_dir / "preds.json", iid)
            time.sleep(backoff)
        else:
            logger.error(f"Retryable error '{status}' on {iid} after {INFRA_MAX_RETRIES} attempts. Giving up.")


def generate(output_dir: Path, instances: list[dict], workers: int, model_config: dict[str, Any]):
    model_config = dict(model_config)
    base_config = model_config.pop("base_config", "swebench")
    config = recursive_merge(
        get_config_from_spec(base_config),
        get_config_from_spec("swebench_modal"),
        {
            "model": model_config,
            "agent": {"step_limit": 250, "cost_limit": 3.0},
            "environment": {
                "timeout": 300,
                "startup_timeout": 600.0,
                "runtime_timeout": 86400.0,
                "deployment_timeout": 86400.0,
            },
        },
    )
    config["model"].pop("provider", None)

    progress_manager = RunBatchProgressManager(len(instances), output_dir / f"exit_statuses_{time.time()}.yaml")

    def process_futures(futures: dict[concurrent.futures.Future, str]):
        for future in concurrent.futures.as_completed(futures):
            try:
                future.result()
            except concurrent.futures.CancelledError:
                pass
            except Exception as e:
                logger.error(f"Error in future for {futures[future]}: {e}", exc_info=True)
                progress_manager.on_uncaught_exception(futures[future], e)

    with Live(progress_manager.render_group, refresh_per_second=4):
        with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor:
            futures = {}
            for instance in instances:
                futures[executor.submit(process_instance_with_retries, instance, output_dir, config, progress_manager)] = instance["instance_id"]
                time.sleep(0.25)
            try:
                process_futures(futures)
            except KeyboardInterrupt:
                logger.info("Cancelling pending jobs. Press ^C again to exit immediately.")
                for future in futures:
                    if not future.running() and not future.done():
                        future.cancel()
                process_futures(futures)


def evaluate(output_dir: Path, model_name: str):
    preds_path = output_dir / "preds.json"
    preds_eval_path = output_dir / "preds_eval.json"
    preds_eval_path.write_text(json.dumps(list(json.loads(preds_path.read_text()).values()), indent=2))

    run_id = f"mini-swe-agent-{int(time.time())}"
    logger.info(f"Running evaluation with run_id={run_id}...")
    subprocess.run(
        [
            sys.executable, "-m", "swebench.harness.run_evaluation",
            "--dataset_name", DATASET_MAPPING[SUBSET],
            "--split", SPLIT,
            "--predictions_path", str(preds_eval_path),
            "--max_workers", str(max(len(json.loads(preds_path.read_text())), 5)),
            "--run_id", run_id,
            "--modal", "true",
            "--report_dir", str(output_dir / "eval_reports"),
        ],
        check=True,
    )

    report_dir = output_dir / "eval_reports" / run_id
    results = {}
    for report_path in sorted(report_dir.rglob("report.json")):
        report = json.loads(report_path.read_text())
        for iid, data in report.items():
            results[iid] = data.get("resolved", False)

    total = len(results)
    resolved = sum(results.values())
    summary = {
        "run_id": run_id,
        "model": model_name,
        "subset": SUBSET,
        "split": SPLIT,
        "total_instances": total,
        "resolved": resolved,
        "resolve_rate": resolved / total if total else 0,
        "instances": {iid: "PASS" if r else "FAIL" for iid, r in sorted(results.items())},
    }
    summary_path = output_dir / "eval_summary.json"
    summary_path.write_text(json.dumps(summary, indent=2))
    rate = 100 * resolved / total if total else 0
    logger.info(f"Evaluation complete: {resolved}/{total} resolved ({rate:.0f}%)")
    logger.info(f"Summary written to {summary_path}")


def main():
    parser = argparse.ArgumentParser(description="Run mini-SWE-agent on SWE-bench Verified using Modal.")
    subparsers = parser.add_subparsers(dest="command", required=True)

    run_parser = subparsers.add_parser("run", help="Generate patches and evaluate")
    run_parser.add_argument("--output-dir", type=Path, default=OUTPUT_DIR)
    run_parser.add_argument("--workers", type=int, default=WORKERS)
    run_parser.add_argument("--model", type=str, default=ModelPreset.GPT5_MINI.value, choices=[m.value for m in ModelPreset])
    run_parser.add_argument("--slice-spec", type=str, default="")
    run_parser.add_argument("--filter-spec", type=str, default="")

    eval_parser = subparsers.add_parser("eval", help="Evaluate existing predictions")
    eval_parser.add_argument("output_dir", type=Path)
    eval_parser.add_argument("--model-name", type=str, default="")

    args = parser.parse_args()

    if args.command == "run":
        model_preset = ModelPreset(args.model)
        model_config = MODEL_CONFIGS[model_preset]
        args.output_dir.mkdir(parents=True, exist_ok=True)
        add_file_handler(args.output_dir / "minisweagent.log")

        from datasets import load_dataset

        dataset_path = DATASET_MAPPING[SUBSET]
        logger.info(f"Loading dataset {dataset_path}, split {SPLIT}, model {model_preset.value}...")
        instances = list(load_dataset(dataset_path, split=SPLIT))
        instances = filter_instances(instances, filter_spec=args.filter_spec, slice_spec=args.slice_spec)

        if (args.output_dir / "preds.json").exists():
            existing = set(json.loads((args.output_dir / "preds.json").read_text()).keys())
            logger.info(f"Skipping {len(existing)} existing instances")
            instances = [i for i in instances if i["instance_id"] not in existing]

        logger.info(f"Running on {len(instances)} instances...")
        generate(args.output_dir, instances, args.workers, model_config)
        evaluate(args.output_dir, model_config["model_name"])

    elif args.command == "eval":
        preds_path = args.output_dir / "preds.json"
        if not preds_path.exists():
            logger.error(f"No preds.json found in {args.output_dir}")
            raise SystemExit(1)
        model_name = args.model_name
        if not model_name:
            first = next(iter(json.loads(preds_path.read_text()).values()), {})
            model_name = first.get("model_name_or_path", "unknown")
        evaluate(args.output_dir, model_name)


if __name__ == "__main__":
    main()

I'm new to swebench , so I'd appreciate if I can get any guidance on what I might be missing to getting the scores on the leaderboard!

One thing that I'm suspecting for Minimax M2.5 results is that my results are with the model provided by Together AI, which is quantized to FP4, compared to the FP8 model on HuggingFace. Still a >9% drop seems pretty sharp. I'm retrying with the model on HuggingFace to verify myself so I'll report that when it's ready. Maybe the score on the leaderboard is with Minimax M2.5 model provided directly through MiniMax, which could possibly be the full precision model, but I couldn't find that information.

Even if it quantization was the reason for Minimax M2.5, it doesn't explain the lower performance with gpt-5-mini (high reasoning) compared to gpt-5 mini with medium reasoning, so I'd love to know if there's a glaring difference in my setup that explains the performance difference. FYI the number I'm computing for resolved rate is Instances resolved / Total instances