inference-with-tools.py

#!/usr/bin/env python3
import ast
import base64
from datetime import datetime
import faulthandler
import json
import math
import os
import posixpath
import signal
import sys
import threading
import time
from argparse import ArgumentParser
from concurrent.futures import ThreadPoolExecutor, as_completed
from contextlib import redirect_stdout
from dataclasses import dataclass, field
from io import BytesIO
from io import StringIO
from typing import Dict, List, Tuple

import requests
from PIL import Image

from benchmark import current_timestamp_utc, read_benchmark, write_benchmark, sort_benchmark
from execute import execute_solution
from llm_client import (
    Endpoint,
    ensure_model_available,
    load_endpoint_file,
    ollama_pull,
    openai_api_check_exist,
    openai_api_list,
)
from llm_model_test import complete_model_capabilities
from execute_clojure import syntax_check_clojure
from execute_java import syntax_check_java
from execute_python import syntax_check_python
from execute_rust import syntax_check_rust


faulthandler.enable(file=sys.stderr, all_threads=True)
if hasattr(signal, "SIGUSR1"):
    faulthandler.register(signal.SIGUSR1, file=sys.stderr, all_threads=True)
if hasattr(signal, "SIGQUIT"):
    faulthandler.register(signal.SIGQUIT, file=sys.stderr, all_threads=True)

LOG_CONTEXT = threading.local()


SYSTEM_PROMPT = (
    "You are a coding agent operating in a virtual workspace. "
    "You can only act by calling the provided tools. "
    "Use this workflow: inspect the workspace, read files when needed, write or "
    "rewrite complete files, run syntax checks, and continue until the program is ready. "
    "When the task has multiple steps or you need to keep track of progress, consider "
    "using update_plan to record a short plan and keep it current. "
    "If it helps, you may use workspace files as temporary notes or a scratchpad via "
    "write_file and read_file. "
    "The final program can only be checked for syntax, not executed inside the tool loop. "
    "Delivery is a blind one-shot submission of the final workspace file. "
    "When the final program is ready, call deliver_code with the final workspace file path. "
    "Do not end with plain text. Every assistant turn must contain a tool call. "
    "If you need to explain progress, put that explanation in the assistant message that accompanies the tool call. "
    "Return runnable source code with no markdown fences in delivered content."
)

TOOLS = [
    {
        "type": "function",
        "function": {
            "name": "list_files",
            "description": (
                "List the current virtual workspace files. Use this first when you need "
                "to discover what files exist."
            ),
            "parameters": {
                "type": "object",
                "properties": {},
                "additionalProperties": False,
            },
            "strict": True,
        },
    },
    {
        "type": "function",
        "function": {
            "name": "read_file",
            "description": (
                "Read one virtual workspace file by relative path. Use this to inspect "
                "existing code before editing."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "path": {
                        "type": "string",
                        "description": "Virtual workspace-relative file path.",
                        "minLength": 1,
                    }
                },
                "required": ["path"],
                "additionalProperties": False,
            },
            "strict": True,
        },
    },
    {
        "type": "function",
        "function": {
            "name": "write_file",
            "description": (
                "Create or fully replace one virtual workspace file. The content must be "
                "the complete file contents, not a diff."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "path": {
                        "type": "string",
                        "description": "Virtual workspace-relative file path.",
                        "minLength": 1,
                    },
                    "content": {
                        "type": "string",
                        "description": "Exact full file content.",
                    },
                },
                "required": ["path", "content"],
                "additionalProperties": False,
            },
            "strict": True,
        },
    },
    {
        "type": "function",
        "function": {
            "name": "delete_file",
            "description": "Delete one virtual workspace file by relative path.",
            "parameters": {
                "type": "object",
                "properties": {
                    "path": {
                        "type": "string",
                        "description": "Virtual workspace-relative file path.",
                        "minLength": 1,
                    }
                },
                "required": ["path"],
                "additionalProperties": False,
            },
            "strict": True,
        },
    },
    {
        "type": "function",
        "function": {
            "name": "rename_file",
            "description": (
                "Rename or move one virtual workspace file from `source_path` to "
                "`destination_path`."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "source_path": {
                        "type": "string",
                        "description": "Existing virtual workspace-relative file path.",
                        "minLength": 1,
                    },
                    "destination_path": {
                        "type": "string",
                        "description": "New virtual workspace-relative file path.",
                        "minLength": 1,
                    },
                },
                "required": ["source_path", "destination_path"],
                "additionalProperties": False,
            },
            "strict": True,
        },
    },
    {
        "type": "function",
        "function": {
            "name": "syntax_check",
            "description": (
                "Check syntax for python, java, rust, or clojure using one virtual "
                "workspace file path."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "language": {
                        "type": "string",
                        "description": "Language to check.",
                        "enum": ["python", "java", "rust", "clojure"],
                    },
                    "path": {
                        "type": "string",
                        "description": "Virtual workspace-relative file path.",
                        "minLength": 1,
                    },
                },
                "required": ["language", "path"],
                "additionalProperties": False,
            },
            "strict": True,
        },
    },
    {
        "type": "function",
        "function": {
            "name": "calculator",
            "description": (
                "Evaluate a mathematical expression for planning or verification. "
                "Use only for arithmetic, not for general code execution."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "expression": {
                        "type": "string",
                        "description": "Mathematical expression to evaluate.",
                        "minLength": 1,
                    }
                },
                "required": ["expression"],
                "additionalProperties": False,
            },
            "strict": True,
        },
    },
    {
        "type": "function",
        "function": {
            "name": "update_plan",
            "description": (
                "Update the current task plan with a short explanation and a list of "
                "steps. Keep the explanation and steps concise. Use this to track "
                "progress across multiple tool calls."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "explanation": {
                        "type": "string",
                        "description": "Optional short explanation of the current phase.",
                    },
                    "plan": {
                        "type": "array",
                        "description": "Ordered list of plan steps and their statuses.",
                        "minItems": 1,
                        "maxItems": 8,
                        "items": {
                            "type": "object",
                            "properties": {
                                "step": {
                                    "type": "string",
                                    "description": "Human-readable description of the step.",
                                    "minLength": 1,
                                },
                                "status": {
                                    "type": "string",
                                    "description": "Current state of the step.",
                                    "enum": ["pending", "in_progress", "completed"],
                                },
                            },
                            "required": ["step", "status"],
                            "additionalProperties": False,
                        },
                    },
                },
                "required": ["plan"],
                "additionalProperties": False,
            },
            "strict": True,
        },
    },
    {
        "type": "function",
        "function": {
            "name": "deliver_code",
            "description": (
                "Finish the task and deliver the final runnable program using one virtual "
                "workspace file path. Call this only when the code is complete."
            ),
            "parameters": {
                "type": "object",
                "properties": {
                    "path": {
                        "type": "string",
                        "description": "Virtual workspace-relative file path to deliver.",
                        "minLength": 1,
                    },
                },
                "required": ["path"],
                "additionalProperties": False,
            },
            "strict": True,
        },
    },
]

MAX_TOOL_CALLS = 12
MAX_PRE_TOOL_STREAM_BYTES = 8192


def _sanitize_tool_schema(value):
    if isinstance(value, dict):
        sanitized = {}
        for key, item in value.items():
            # Some OpenAI-compatible servers accept basic tool schemas but hang or
            # reject requests once strict JSON schema features are added.
            if key in {"strict", "oneOf", "anyOf", "allOf"}:
                continue
            sanitized[key] = _sanitize_tool_schema(item)
        return sanitized
    if isinstance(value, list):
        return [_sanitize_tool_schema(item) for item in value]
    return value


def get_api_tools(tools: List[dict]) -> List[dict]:
    return [_sanitize_tool_schema(tool) for tool in tools]


@dataclass
class State:
    max_tool_calls: int = MAX_TOOL_CALLS
    tool_call_counts: Dict[str, int] = field(default_factory=dict)
    plan: List[Dict[str, str]] = field(default_factory=list)
    plan_explanation: str = ""


@dataclass
class ToolResult:
    exit_code: int
    stdout: str
    stderr: str


@dataclass
class DeliveryResult:
    delivered: bool
    content: str = ""
    path: str = ""


@dataclass
class VirtualFileSystem:
    files: Dict[str, str] = field(default_factory=dict)

    def list_files(self) -> List[str]:
        return sorted(self.files)

    def read_file(self, path: str) -> str:
        return self.files[path]

    def write_file(self, path: str, content: str) -> None:
        self.files[path] = content

    def delete_file(self, path: str) -> None:
        del self.files[path]

    def rename_file(self, source_path: str, destination_path: str) -> None:
        self.files[destination_path] = self.files.pop(source_path)


def read_template(template_path):
    with open(template_path, "r", encoding="utf-8") as file:
        return file.read()


def get_extension(language):
    if language == "java":
        return "java"
    if language == "rust":
        return "rs"
    if language == "python":
        return "py"
    if language == "clojure":
        return "clj"
    raise Exception(f"Unsupported language: {language}")


def get_tooling_series_name(language: str, max_problem_number: int) -> str:
    return f"{language}-{max_problem_number}-tool-test"


def get_tooling_score_name(language: str, max_problem_number: int) -> str:
    return f"{language}-{max_problem_number}-tool"


def get_tooling_batch_bounds(args) -> Tuple[int, int, int]:
    if args.n100:
        return 100, 1, 100
    if args.n200:
        return 200, 101, 200
    if args.n300:
        return 300, 201, 300
    if args.n400:
        return 400, 301, 400
    if args.nall:
        return 9999, 1, 9999
    return 200, 101, 200


def get_recorded_tooling_vector(store_name: str, language: str, max_problem_number: int) -> str:
    benchmark = read_benchmark()
    entry = benchmark.get(store_name, {})
    return entry.get(get_tooling_series_name(language, max_problem_number), "")


def tooling_result_recorded(test_vector: str, problem_number: str, problem_start: int, problem_end: int) -> bool:
    index = int(problem_number) - problem_start
    vector_length = problem_end - problem_start + 1
    if not (problem_start <= int(problem_number) <= problem_end):
        return False
    if len(test_vector) < vector_length:
        return False
    return 0 <= index < len(test_vector) and test_vector[index] in {"0", "1"}


def update_tooling_score(
    store_name: str,
    language: str,
    max_problem_number: int,
    problem_start: int,
    problem_end: int,
    benchmark_lock: threading.Lock,
) -> None:
    with benchmark_lock:
        benchmark = read_benchmark()
        entry = benchmark.get(store_name, {})
        tooling_series_name = get_tooling_series_name(language, max_problem_number)
        vector_length = problem_end - problem_start + 1
        test_vector = entry.get(tooling_series_name, "")
        if len(test_vector) < vector_length:
            test_vector = test_vector + ("?" * (vector_length - len(test_vector)))
        current_vector = test_vector[:vector_length]
        if not os.path.exists("solutions.json"):
            return

        with open("solutions.json", "r", encoding="utf-8") as json_file:
            expected_solutions = json.load(json_file)

        candidate_points = 0.0
        total_count = 0
        for offset, problem_number_int in enumerate(range(problem_start, problem_end + 1)):
            marker = current_vector[offset]
            if marker not in {"0", "1"}:
                continue
            problem_number = f"{problem_number_int:04d}"
            expected = expected_solutions.get(problem_number)
            if not expected:
                continue
            total_count += 1
            if marker == "1":
                candidate_points += expected.get("points", 0.0)
        if total_count == 0:
            return

        entry[get_tooling_score_name(language, max_problem_number)] = round(candidate_points / total_count, 2)
        entry["timestamp"] = current_timestamp_utc()
        benchmark[store_name] = entry
        write_benchmark(sort_benchmark(benchmark))


def record_tooling_result(
    store_name: str,
    language: str,
    problem_number: str,
    output: str,
    correct: bool,
    max_problem_number: int,
    problem_start: int,
    problem_end: int,
    solutions_json_path: str,
    benchmark_lock: threading.Lock,
) -> None:
    t0 = time.monotonic()
    with benchmark_lock:
        solutions = {}
        if os.path.exists(solutions_json_path):
            with open(solutions_json_path, "r", encoding="utf-8") as json_file:
                try:
                    solutions = json.load(json_file)
                except json.JSONDecodeError:
                    solutions = {}
        solutions[problem_number] = output
        with open(solutions_json_path, "w", encoding="utf-8") as json_file:
            json.dump(solutions, json_file, indent=4)

        benchmark = read_benchmark()
        entry = benchmark.get(store_name, {})
        tooling_series_name = get_tooling_series_name(language, max_problem_number)
        vector_length = problem_end - problem_start + 1
        test_vector = list(entry.get(tooling_series_name, "?" * vector_length))
        if len(test_vector) < vector_length:
            test_vector.extend(["?"] * (vector_length - len(test_vector)))
        index = int(problem_number) - problem_start
        if 0 <= index < vector_length:
            test_vector[index] = "1" if correct else "0"
        entry[tooling_series_name] = "".join(test_vector)
        entry["timestamp"] = current_timestamp_utc()
        benchmark[store_name] = entry
        write_benchmark(sort_benchmark(benchmark))
    update_tooling_score(
        store_name,
        language,
        max_problem_number,
        problem_start,
        problem_end,
        benchmark_lock,
    )
    log(
        f"[{problem_number}] Recorded benchmark result in {time.monotonic() - t0:.2f}s: "
        f"key={get_tooling_series_name(language, max_problem_number)} value={'1' if correct else '0'}"
    )


def log(message: str) -> None:
    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    endpoint = getattr(LOG_CONTEXT, "endpoint", "")
    endpoint_text = f" endpoint={endpoint}" if endpoint else ""
    print(f"[tool-inference {timestamp}{endpoint_text}] {message}", flush=True)


def set_log_endpoint(endpoint: str) -> None:
    LOG_CONTEXT.endpoint = endpoint


def preview_text(text: str, limit: int = 160) -> str:
    compact = " ".join(text.split())
    if len(compact) <= limit:
        return compact
    return compact[: limit - 3] + "..."


def byte_size(text: str) -> int:
    if not text:
        return 0
    return len(text.encode("utf-8"))


def summarize_tool_args(tool_name: str, parsed_args: dict) -> str:
    if tool_name in {"read_file", "write_file", "delete_file", "deliver_code"}:
        path = parsed_args.get("path")
        if path:
            return f"path={path}"
    if tool_name == "rename_file":
        source_path = parsed_args.get("source_path")
        destination_path = parsed_args.get("destination_path")
        if source_path and destination_path:
            return f"source_path={source_path} destination_path={destination_path}"
    if tool_name == "syntax_check":
        language = parsed_args.get("language", "?")
        path = parsed_args.get("path")
        return f"language={language} path={path}" if path else f"language={language}"
    if tool_name == "calculator":
        expr = parsed_args.get("expression", "")
        return f"expression={preview_text(expr, 80)}"
    return ""


def summarize_response(response_json: dict) -> str:
    message = response_json.get("choices", [{}])[0].get("message", {})
    chunk = extract_text_content(message.get("content"))
    tool_calls = message.get("tool_calls") or []
    usage = response_json.get("usage", {}) if isinstance(response_json.get("usage", {}), dict) else {}
    prompt_tokens = usage.get("prompt_tokens")
    completion_tokens = usage.get("completion_tokens", usage.get("output_tokens"))
    usage_summary = []
    if isinstance(prompt_tokens, int):
        usage_summary.append(f"prompt_tokens={prompt_tokens}")
    if isinstance(completion_tokens, int):
        usage_summary.append(f"completion_tokens={completion_tokens}")
    usage_text = f"{', '.join(usage_summary)}; " if usage_summary else ""
    if tool_calls:
        first_call = tool_calls[0]
        function = first_call.get("function", {})
        tool_name = function.get("name") or first_call.get("name") or "unknown_tool"
        raw_args = function.get("arguments", "")
        try:
            parsed_args = json.loads(raw_args) if raw_args else {}
        except json.JSONDecodeError:
            parsed_args = {}
        detail = summarize_tool_args(tool_name, parsed_args)
        arg_bytes = byte_size(raw_args)
        content_bytes = byte_size(chunk)
        if chunk and detail:
            return (
                f"{usage_text}model plans to {tool_name} ({detail}); "
                f"arg_bytes={arg_bytes}; content_bytes={content_bytes}; "
                f"note={preview_text(chunk, 120)}"
            )
        if detail:
            return f"{usage_text}model plans to {tool_name} ({detail}); arg_bytes={arg_bytes}"
        if chunk:
            return f"{usage_text}model plans to {tool_name}; arg_bytes={arg_bytes}; content_bytes={content_bytes}; note={preview_text(chunk, 120)}"
        return f"{usage_text}model plans to {tool_name}; arg_bytes={arg_bytes}"
    if chunk:
        return f"{usage_text}model replied without a tool call; content_bytes={byte_size(chunk)}; text={preview_text(chunk, 120)}"
    return f"{usage_text}model returned neither tool call nor text"


def normalize_virtual_path(path: str) -> str:
    if not path:
        raise ValueError("Path must not be empty.")
    normalized = posixpath.normpath(path.replace("\\", "/"))
    if normalized in (".", ""):
        raise ValueError("Path must not be empty.")
    if normalized.startswith("../") or normalized == ".." or normalized.startswith("/"):
        raise ValueError("Path must be workspace-relative.")
    return normalized


def run_calculator(expression: str) -> ToolResult:
    if not expression or not expression.strip():
        return ToolResult(exit_code=1, stdout="", stderr="expression must not be empty.")

    try:
        parsed = ast.parse(expression, mode="eval")
    except SyntaxError as exc:
        return ToolResult(exit_code=1, stdout="", stderr=f"Invalid expression: {exc.msg}")

    allowed_names = {name: getattr(math, name) for name in dir(math) if not name.startswith("_")}
    allowed_names.update(
        {
            "abs": abs,
            "round": round,
            "min": min,
            "max": max,
            "sum": sum,
            "pow": pow,
        }
    )

    allowed_nodes = (
        ast.Expression,
        ast.BinOp,
        ast.UnaryOp,
        ast.Call,
        ast.Name,
        ast.Load,
        ast.Constant,
        ast.Tuple,
        ast.List,
        ast.Add,
        ast.Sub,
        ast.Mult,
        ast.Div,
        ast.FloorDiv,
        ast.Mod,
        ast.Pow,
        ast.USub,
        ast.UAdd,
    )

    for node in ast.walk(parsed):
        if not isinstance(node, allowed_nodes):
            return ToolResult(
                exit_code=1,
                stdout="",
                stderr=f"Unsupported expression element: {type(node).__name__}",
            )
        if isinstance(node, ast.Call):
            if not isinstance(node.func, ast.Name) or node.func.id not in allowed_names:
                return ToolResult(exit_code=1, stdout="", stderr="Unsupported function call.")
        if isinstance(node, ast.Name) and node.id not in allowed_names:
            return ToolResult(exit_code=1, stdout="", stderr=f"Unknown name: {node.id}")

    try:
        value = eval(compile(parsed, "<calculator>", "eval"), {"__builtins__": {}}, allowed_names)
    except Exception as exc:
        return ToolResult(exit_code=1, stdout="", stderr=f"Calculation error: {exc}")

    return ToolResult(exit_code=0, stdout=str(value), stderr="")


def run_syntax_check(vfs: VirtualFileSystem, parsed_args: dict) -> ToolResult:
    language = (parsed_args.get("language") or "").strip().lower()
    if language not in {"python", "java", "rust", "clojure"}:
        return ToolResult(
            exit_code=1,
            stdout="",
            stderr="language must be one of: python, java, rust, clojure.",
        )

    try:
        filename = normalize_virtual_path(parsed_args.get("path", ""))
    except ValueError as exc:
        return ToolResult(exit_code=1, stdout="", stderr=str(exc))
    if filename not in vfs.files:
        return ToolResult(exit_code=1, stdout="", stderr=f"File not found: {filename}")
    code = vfs.read_file(filename)

    if language == "python":
        exit_code, stdout, stderr = syntax_check_python(code, filename)
        return ToolResult(exit_code=exit_code, stdout=stdout, stderr=stderr)
    if language == "java":
        exit_code, stdout, stderr = syntax_check_java(code)
        return ToolResult(exit_code=exit_code, stdout=stdout, stderr=stderr)
    if language == "rust":
        exit_code, stdout, stderr = syntax_check_rust(code)
        return ToolResult(exit_code=exit_code, stdout=stdout, stderr=stderr)
    exit_code, stdout, stderr = syntax_check_clojure(code)
    return ToolResult(exit_code=exit_code, stdout=stdout, stderr=stderr)


def extract_text_content(content) -> str:
    if isinstance(content, str):
        return content
    if isinstance(content, list):
        parts = []
        for item in content:
            if isinstance(item, dict) and item.get("type") == "text":
                parts.append(item.get("text", ""))
        return "".join(parts)
    return ""


def extract_response_fields(response_json: dict) -> Tuple[str, str, str, str]:
    message = response_json.get("choices", [{}])[0].get("message", {})
    chunk = extract_text_content(message.get("content"))
    tool_calls = message.get("tool_calls") or []
    if not tool_calls:
        return chunk, "", "", ""

    first_call = tool_calls[0]
    function = first_call.get("function", {})
    tool_name = function.get("name") or first_call.get("name") or ""
    tool_id = first_call.get("id", "")
    raw_args = function.get("arguments", "")
    return chunk, tool_name, tool_id, raw_args


def raise_for_status_with_body(response, context: str) -> None:
    try:
        response.raise_for_status()
    except requests.HTTPError as exc:
        body = ""
        try:
            body = response.text or ""
        except Exception:
            body = ""
        body_preview = preview_text(body, 800) if body else "<empty body>"
        log(
            f"{context}: HTTP {response.status_code} from backend; "
            f"response_body={body_preview}"
        )
        raise exc


def _extract_delta_text(delta_value) -> str:
    if isinstance(delta_value, str):
        return delta_value
    if isinstance(delta_value, list):
        parts = []
        for item in delta_value:
            if isinstance(item, str):
                parts.append(item)
            elif isinstance(item, dict):
                text = item.get("text")
                if isinstance(text, str):
                    parts.append(text)
        return "".join(parts)
    if isinstance(delta_value, dict):
        text = delta_value.get("text")
        if isinstance(text, str):
            return text
    return ""


def consume_streaming_tool_response(response, problem_number: str, turn: int) -> dict:
    usage = {}
    content_parts: List[str] = []
    reasoning_parts: List[str] = []
    tool_calls_by_index: Dict[int, dict] = {}
    args_bytes_logged: Dict[int, int] = {}
    current_stream_kind = ""
    pre_tool_text_bytes = 0
    pre_tool_reasoning_bytes = 0

    def ensure_stream_prefix(stream_kind: str) -> None:
        nonlocal current_stream_kind
        if current_stream_kind == stream_kind:
            return
        if current_stream_kind:
            sys.stdout.write("\n")
        current_stream_kind = stream_kind
        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
        endpoint = getattr(LOG_CONTEXT, "endpoint", "")
        endpoint_text = f" endpoint={endpoint}" if endpoint else ""
        sys.stdout.write(
            f"[tool-inference {timestamp}{endpoint_text}] "
            f"[{problem_number}] Turn {turn}: stream {stream_kind}: "
        )
        sys.stdout.flush()

    def close_stream_prefix() -> None:
        nonlocal current_stream_kind
        if current_stream_kind:
            sys.stdout.write("\n")
            sys.stdout.flush()
            current_stream_kind = ""

    for raw_line in response.iter_lines(decode_unicode=True):
        if not raw_line:
            continue
        if not raw_line.startswith("data: "):
            continue
        payload_line = raw_line[len("data: "):].strip()
        if payload_line == "[DONE]":
            close_stream_prefix()
            break
        try:
            event = json.loads(payload_line)
        except json.JSONDecodeError:
            close_stream_prefix()
            log(f"[{problem_number}] Turn {turn}: stream event was not valid JSON.")
            continue

        event_usage = event.get("usage")
        if isinstance(event_usage, dict):
            usage = event_usage

        choices = event.get("choices", [])
        if not choices:
            continue

        delta = choices[0].get("delta", {})
        text_delta = _extract_delta_text(delta.get("content"))
        if text_delta:
            content_parts.append(text_delta)
            ensure_stream_prefix("text")
            sys.stdout.write(text_delta)
            sys.stdout.flush()
            if not tool_calls_by_index:
                pre_tool_text_bytes += len(text_delta.encode("utf-8"))
                if pre_tool_text_bytes > MAX_PRE_TOOL_STREAM_BYTES:
                    close_stream_prefix()
                    raise RuntimeError(
                        f"Model exceeded pre-tool text budget of {MAX_PRE_TOOL_STREAM_BYTES} bytes "
                        f"without calling a tool on turn {turn}."
                    )

        reasoning_delta = _extract_delta_text(delta.get("reasoning"))
        if reasoning_delta:
            reasoning_parts.append(reasoning_delta)
            ensure_stream_prefix("reasoning")
            sys.stdout.write(reasoning_delta)
            sys.stdout.flush()
            if not tool_calls_by_index:
                pre_tool_reasoning_bytes += len(reasoning_delta.encode("utf-8"))
                if pre_tool_reasoning_bytes > MAX_PRE_TOOL_STREAM_BYTES:
                    close_stream_prefix()
                    raise RuntimeError(
                        f"Model exceeded pre-tool reasoning budget of {MAX_PRE_TOOL_STREAM_BYTES} bytes "
                        f"without calling a tool on turn {turn}."
                    )

        for tool_call in delta.get("tool_calls") or []:
            close_stream_prefix()
            index = tool_call.get("index", 0)
            current = tool_calls_by_index.setdefault(
                index,
                {
                    "id": "",
                    "type": "function",
                    "function": {"name": "", "arguments": ""},
                },
            )
            if tool_call.get("id"):
                current["id"] = tool_call["id"]
            if tool_call.get("type"):
                current["type"] = tool_call["type"]

            function = tool_call.get("function", {})
            if function.get("name"):
                current["function"]["name"] += function["name"]
                log(
                    f"[{problem_number}] Turn {turn}: stream tool name[{index}]="
                    f"{current['function']['name']}"
                )
            if function.get("arguments"):
                current["function"]["arguments"] += function["arguments"]
                args_text = current["function"]["arguments"]
                args_bytes = len(args_text.encode("utf-8"))
                previous_bytes = args_bytes_logged.get(index, 0)
                if args_bytes > previous_bytes:
                    args_bytes_logged[index] = args_bytes
                    log(
                        f"[{problem_number}] Turn {turn}: stream tool args[{index}] "
                        f"total {args_bytes}B: {preview_text(args_text, 120)}"
                    )

    close_stream_prefix()

    tool_calls = [tool_calls_by_index[index] for index in sorted(tool_calls_by_index)]
    message = {}
    if content_parts:
        message["content"] = "".join(content_parts)
    else:
        message["content"] = ""
    if reasoning_parts:
        message["reasoning"] = "".join(reasoning_parts)
    if tool_calls:
        message["tool_calls"] = tool_calls

    return {
        "choices": [{"message": message}],
        "usage": usage,
    }


def tool_result_json(tool_name: str, result: ToolResult) -> str:
    return json.dumps(
        {
            "tool": tool_name,
            "exit_code": result.exit_code,
            "stdout": result.stdout,
            "stderr": result.stderr,
        }
    )


def log_plan_update(problem_number: str, parsed_args: dict, label: str = "Plan updated") -> None:
    plan = parsed_args.get("plan") or []
    explanation = parsed_args.get("explanation", "")
    lines = [f"[{problem_number}] {label}:"]
    if explanation:
        lines.append(f"[{problem_number}]   explanation: {preview_text(explanation, 200)}")
    for index, item in enumerate(plan, start=1):
        if not isinstance(item, dict):
            continue
        step = item.get("step", "")
        status = item.get("status", "?")
        lines.append(f"[{problem_number}]   {index}. [{status}] {preview_text(step, 200)}")
    if len(lines) == 1:
        lines.append(f"[{problem_number}]   empty plan")
    for line in lines:
        log(line)


def get_visible_tools(state: State) -> List[dict]:
    visible_tools = []
    for tool in TOOLS:
        tool_name = tool["function"]["name"]
        if state.tool_call_counts.get(tool_name, 0) < state.max_tool_calls:
            visible_tools.append(tool)
    return visible_tools


def run_update_plan(state: State, parsed_args: dict) -> ToolResult:
    plan = parsed_args.get("plan")
    if not isinstance(plan, list):
        return ToolResult(exit_code=1, stdout="", stderr="plan must be an array.")
    if not plan:
        return ToolResult(exit_code=1, stdout="", stderr="plan must contain at least one step.")
    if len(plan) > 8:
        return ToolResult(exit_code=1, stdout="", stderr="plan must contain at most 8 steps.")

    normalized_plan: List[Dict[str, str]] = []
    in_progress_count = 0
    completed_count = 0
    warnings: List[str] = []
    seen_steps = set()
    for index, item in enumerate(plan):
        if not isinstance(item, dict):
            return ToolResult(
                exit_code=1,
                stdout="",
                stderr=f"plan[{index}] must be an object.",
            )
        step = item.get("step")
        status = item.get("status")
        if not isinstance(step, str) or not step.strip():
            return ToolResult(
                exit_code=1,
                stdout="",
                stderr=f"plan[{index}].step must be a non-empty string.",
            )
        if status not in {"pending", "in_progress", "completed"}:
            return ToolResult(
                exit_code=1,
                stdout="",
                stderr=(
                    f"plan[{index}].status must be one of: pending, in_progress, completed."
                ),
            )
        if status == "in_progress":
            in_progress_count += 1
        if status == "completed":
            completed_count += 1
        normalized_step = step.strip()
        if normalized_step.lower() in seen_steps:
            warnings.append(f"Duplicate step text: {normalized_step}")
        seen_steps.add(normalized_step.lower())
        normalized_plan.append({"step": normalized_step, "status": status})

    if in_progress_count > 1:
        return ToolResult(
            exit_code=1,
            stdout="",
            stderr="At most one plan step can be in_progress.",
        )

    explanation = parsed_args.get("explanation", "")
    if explanation is None:
        explanation = ""
    if not isinstance(explanation, str):
        return ToolResult(exit_code=1, stdout="", stderr="explanation must be a string.")

    state.plan = normalized_plan
    state.plan_explanation = explanation
    in_progress_step = next(
        (item["step"] for item in normalized_plan if item["status"] == "in_progress"),
        "",
    )
    if not in_progress_step and completed_count < len(normalized_plan):
        warnings.append("No step is currently marked in_progress.")
    return ToolResult(
        exit_code=0,
        stdout=json.dumps(
            {
                "ok": True,
                "explanation": explanation,
                "plan": normalized_plan,
                "summary": {
                    "step_count": len(normalized_plan),
                    "completed_count": completed_count,
                    "in_progress_step": in_progress_step,
                },
                "warnings": warnings,
            }
        ),
        stderr="",
    )


def run_virtual_tool(state: State, vfs: VirtualFileSystem, tool_name: str, parsed_args: dict) -> ToolResult:
    if tool_name == "list_files":
        return ToolResult(exit_code=0, stdout="\n".join(vfs.list_files()), stderr="")

    if tool_name == "read_file":
        try:
            path = normalize_virtual_path(parsed_args.get("path", ""))
        except ValueError as exc:
            return ToolResult(exit_code=1, stdout="", stderr=str(exc))
        if path not in vfs.files:
            return ToolResult(exit_code=1, stdout="", stderr=f"File not found: {path}")
        return ToolResult(exit_code=0, stdout=vfs.read_file(path), stderr="")

    if tool_name == "write_file":
        try:
            path = normalize_virtual_path(parsed_args.get("path", ""))
        except ValueError as exc:
            return ToolResult(exit_code=1, stdout="", stderr=str(exc))
        vfs.write_file(path, parsed_args.get("content", ""))
        return ToolResult(exit_code=0, stdout=f"Wrote {path}", stderr="")

    if tool_name == "delete_file":
        try:
            path = normalize_virtual_path(parsed_args.get("path", ""))
        except ValueError as exc:
            return ToolResult(exit_code=1, stdout="", stderr=str(exc))
        if path not in vfs.files:
            return ToolResult(exit_code=1, stdout="", stderr=f"File not found: {path}")
        vfs.delete_file(path)
        return ToolResult(exit_code=0, stdout=f"Deleted {path}", stderr="")

    if tool_name == "rename_file":
        try:
            source_path = normalize_virtual_path(parsed_args.get("source_path", ""))
            destination_path = normalize_virtual_path(parsed_args.get("destination_path", ""))
        except ValueError as exc:
            return ToolResult(exit_code=1, stdout="", stderr=str(exc))
        if source_path not in vfs.files:
            return ToolResult(exit_code=1, stdout="", stderr=f"File not found: {source_path}")
        if destination_path in vfs.files:
            return ToolResult(
                exit_code=1,
                stdout="",
                stderr=f"Destination already exists: {destination_path}",
            )
        vfs.rename_file(source_path, destination_path)
        return ToolResult(
            exit_code=0,
            stdout=f"Renamed {source_path} to {destination_path}",
            stderr="",
        )

    if tool_name == "syntax_check":
        return run_syntax_check(vfs, parsed_args)

    if tool_name == "calculator":
        return run_calculator(parsed_args.get("expression", ""))

    if tool_name == "update_plan":
        return run_update_plan(state, parsed_args)

    return ToolResult(exit_code=1, stdout="", stderr=f"Unsupported tool: {tool_name}")


def handle_deliver_code(vfs: VirtualFileSystem, parsed_args: dict) -> Tuple[ToolResult, DeliveryResult]:
    raw_path = parsed_args.get("path", "")
    try:
        path = normalize_virtual_path(raw_path)
    except ValueError as exc:
        return ToolResult(exit_code=1, stdout="", stderr=str(exc)), DeliveryResult(delivered=False)
    if path not in vfs.files:
        return (
            ToolResult(exit_code=1, stdout="", stderr=f"File not found: {path}"),
            DeliveryResult(delivered=False),
        )
    return (
        ToolResult(exit_code=0, stdout=f"Delivered {path}", stderr=""),
        DeliveryResult(delivered=True, content=vfs.read_file(path), path=path),
    )


def handle_tool_call(
    state: State,
    vfs: VirtualFileSystem,
    messages: List[dict],
    assistant_content: str,
    tool_name: str,
    tool_id: str,
    tool_args_unescaped: str,
    problem_number: str,
) -> DeliveryResult:
    if not tool_name or not tool_args_unescaped:
        log(f"[{problem_number}] Model returned no tool call.")
        return DeliveryResult(delivered=False)
    t0 = time.monotonic()

    try:
        parsed_args = json.loads(tool_args_unescaped)
    except json.JSONDecodeError:
        result = ToolResult(exit_code=1, stdout="", stderr="Tool arguments are not valid JSON.")
        delivery = DeliveryResult(delivered=False)
        log(f"[{problem_number}] Executing {tool_name}: invalid JSON arguments.")
    else:
        summary = summarize_tool_args(tool_name, parsed_args)
        if tool_name == "update_plan":
            log_plan_update(problem_number, parsed_args, label="Executing update_plan")
        elif summary:
            log(f"[{problem_number}] Executing {tool_name}: {summary}")
        else:
            log(f"[{problem_number}] Executing {tool_name}.")
        if state.tool_call_counts.get(tool_name, 0) >= state.max_tool_calls:
            result = ToolResult(
                exit_code=1,
                stdout="",
                stderr=f"Tool {tool_name} is no longer available because it reached the call limit.",
            )
            delivery = DeliveryResult(delivered=False)
        else:
            state.tool_call_counts[tool_name] = state.tool_call_counts.get(tool_name, 0) + 1
            if tool_name == "deliver_code":
                result, delivery = handle_deliver_code(vfs, parsed_args)
            else:
                result = run_virtual_tool(state, vfs, tool_name, parsed_args)
                delivery = DeliveryResult(delivered=False)

    call_id = tool_id or "call_1"
    messages.append(
        {
            "role": "assistant",
            "content": assistant_content or "",
            "tool_calls": [
                {
                    "id": call_id,
                    "type": "function",
                    "function": {
                        "name": tool_name,
                        "arguments": tool_args_unescaped,
                    },
                }
            ],
        }
    )
    messages.append(
        {
            "role": "tool",
            "tool_call_id": call_id,
            "content": tool_result_json(tool_name, result),
        }
    )
    if tool_name == "update_plan":
        log(
            f"[{problem_number}] Tool finished: {tool_name} exit={result.exit_code} "
            f"result_bytes={byte_size(result.stdout) + byte_size(result.stderr)} "
            f"stderr={preview_text(result.stderr, 120) if result.stderr else ''} "
            f"duration={time.monotonic() - t0:.2f}s"
        )
    else:
        log(
            f"[{problem_number}] Tool finished: {tool_name} exit={result.exit_code} "
            f"result_bytes={byte_size(result.stdout) + byte_size(result.stderr)} "
            f"stdout={preview_text(result.stdout, 120) if result.stdout else ''} "
            f"stderr={preview_text(result.stderr, 120) if result.stderr else ''} "
            f"duration={time.monotonic() - t0:.2f}s"
        )
    return delivery


def build_user_message(prompt: str, base64_image: str = None) -> dict:
    if not base64_image:
        return {"role": "user", "content": prompt}

    image_type = "jpeg"
    magic_types = {"/9j/": "jpeg", "iVBO": "png", "R0lG": "gif"}
    for magic, candidate in magic_types.items():
        if base64_image.startswith(magic):
            image_type = candidate
            break
    if image_type == "gif":
        image = Image.open(BytesIO(base64.b64decode(base64_image)))
        png_image = BytesIO()
        image.save(png_image, format="PNG")
        base64_image = base64.b64encode(png_image.getvalue()).decode("utf-8")
        image_type = "png"

    return {
        "role": "user",
        "content": [
            {"type": "text", "text": prompt},
            {"type": "image_url", "image_url": {"url": f"data:image/{image_type};base64,{base64_image}"}},
        ],
    }


def validate_tool_agent_contract(
    endpoint: Endpoint,
    think: bool = False,
    no_think: bool = False,
) -> None:
    headers = {"Content-Type": "application/json", "Accept": "application/json"}
    if endpoint.key:
        headers["Authorization"] = f"Bearer {endpoint.key}"

    probe_messages = [
        {"role": "system", "content": SYSTEM_PROMPT},
        {"role": "user", "content": "List the workspace files, then stop."},
    ]
    payload = {
        "model": endpoint.model_name,
        "messages": probe_messages,
        "tools": get_api_tools(get_visible_tools(State())),
        "stream": False,
        "temperature": 0.0,
        "max_tokens": 512,
    }
    if no_think:
        payload["enable_thinking"] = False
    elif think:
        payload["enable_thinking"] = True

    log(
        f"Preflight: validating tool-agent contract on {endpoint.model_name} "
        f"with {len(payload['tools'])} tools."
    )
    t0 = time.monotonic()
    response = requests.post(
        endpoint.url,
        headers=headers,
        json=payload,
        verify=False,
        timeout=(60, 600),
    )
    raise_for_status_with_body(response, "Preflight request failed")
    response_json = response.json()
    chunk, tool_name, _, _ = extract_response_fields(response_json)
    if not tool_name:
        raise RuntimeError(
            "Tool-agent preflight failed: backend returned no tool call for the "
            f"actual agent payload. Assistant text was: {preview_text(chunk, 240)}"
        )
    log(
        f"Preflight succeeded in {time.monotonic() - t0:.2f}s: "
        f"first tool call was {tool_name}."
    )


def run_tool_agent(
    endpoint: Endpoint,
    prompt: str,
    problem_number: str,
    base64_image: str = None,
    think: bool = False,
    no_think: bool = False,
    stream: bool = False,
) -> str:
    state = State()
    messages = [
        {"role": "system", "content": SYSTEM_PROMPT},
        build_user_message(prompt, base64_image=base64_image),
    ]
    vfs = VirtualFileSystem()
    url = endpoint.url
    headers = {"Content-Type": "application/json", "Accept": "application/json"}
    if endpoint.key:
        headers["Authorization"] = f"Bearer {endpoint.key}"
    delivered = DeliveryResult(delivered=False)

    for turn in range(MAX_TOOL_CALLS * 4):
        payload = {
            "model": endpoint.model_name,
            "messages": messages,
            "tools": get_api_tools(get_visible_tools(state)),
            "stream": stream,
            "temperature": 0.0,
        }
        if no_think:
            payload["enable_thinking"] = False
        elif think:
            payload["enable_thinking"] = True

        request_t0 = time.monotonic()
        log(
            f"[{problem_number}] Turn {turn + 1}: waiting for model response from "
            f"{endpoint.model_name} with {len(payload['tools'])} tools."
        )
        response = requests.post(
            url,
            headers=headers,
            json=payload,
            verify=False,
            stream=stream,
            timeout=(60, 600),
        )
        raise_for_status_with_body(
            response,
            f"[{problem_number}] Turn {turn + 1} request failed",
        )
        if stream:
            response_json = consume_streaming_tool_response(response, problem_number, turn + 1)
        else:
            response_json = response.json()
        log(
            f"[{problem_number}] Turn {turn + 1}: model response received in "
            f"{time.monotonic() - request_t0:.2f}s; {summarize_response(response_json)}"
        )

        chunk, tool_name, tool_id, tool_args_unescaped = extract_response_fields(response_json)
        if chunk:
            log(f"[{problem_number}] Assistant text: {preview_text(chunk, 220)}")
        else:
            log(f"[{problem_number}] Assistant returned no text chunk.")

        if not tool_name:
            if chunk:
                messages.append({"role": "assistant", "content": chunk})
            raise RuntimeError(
                f"Model violated tool-calling contract on turn {turn + 1}: "
                "assistant response did not include a tool call."
            )

        delivery = handle_tool_call(
            state,
            vfs,
            messages,
            chunk,
            tool_name,
            tool_id,
            tool_args_unescaped,
            problem_number,
        )
        if delivery.delivered:
            log(f"[{problem_number}] deliver_code succeeded.")
            delivered = delivery
            break

    if not delivered.delivered:
        raise RuntimeError("Task is not complete until deliver_code is called.")
    return delivered.content


def process_single_problem(
    endpoint: Endpoint,
    store_name: str,
    language: str,
    problem_number: str,
    prompt: str,
    output_path: str,
    expected: dict,
    max_problem_number: int,
    problem_start: int,
    problem_end: int,
    solutions_json_path: str,
    benchmark_lock: threading.Lock,
    base64_image: str = None,
    think: bool = False,
    no_think: bool = False,
    stream: bool = False,
) -> Tuple[str, str, bool]:
    t0 = time.monotonic()
    previous_endpoint = getattr(LOG_CONTEXT, "endpoint", "")
    set_log_endpoint(endpoint.url)
    try:
        log(
            f"[{problem_number}] Starting generation on {endpoint.model_name}; "
            f"output_path={output_path}"
        )
        code = run_tool_agent(
            endpoint,
            prompt,
            problem_number,
            base64_image=base64_image,
            think=think,
            no_think=no_think,
            stream=stream,
        )
        with open(output_path, "w", encoding="utf-8") as file:
            file.write(code)
        log(
            f"[{problem_number}] Saved {language} solution to {output_path} "
            f"after generation time {time.monotonic() - t0:.2f}s"
        )
        return evaluate_existing_solution(
            store_name,
            language,
            problem_number,
            output_path,
            expected,
            max_problem_number,
            problem_start,
            problem_end,
            solutions_json_path,
            benchmark_lock,
            reused_existing=False,
        )
    finally:
        set_log_endpoint(previous_endpoint)


def evaluate_existing_solution(
    store_name: str,
    language: str,
    problem_number: str,
    output_path: str,
    expected: dict,
    max_problem_number: int,
    problem_start: int,
    problem_end: int,
    solutions_json_path: str,
    benchmark_lock: threading.Lock,
    reused_existing: bool = True,
) -> Tuple[str, str, bool]:
    previous_endpoint = getattr(LOG_CONTEXT, "endpoint", "")
    if not previous_endpoint:
        set_log_endpoint("local")
    if reused_existing:
        log(f"[{problem_number}] Evaluating existing solution from {output_path}")
    else:
        log(f"[{problem_number}] Evaluating generated solution from {output_path}")
    exec_t0 = time.monotonic()
    output = execute_solution(output_path, expected)
    expected_solution = expected.get("solution", "") if expected else ""
    correct = output == expected_solution
    log(
        f"[{problem_number}] Execution result on {store_name}: output={preview_text(output, 120)} "
        f"expected={expected_solution} correct={correct} "
        f"duration={time.monotonic() - exec_t0:.2f}s"
    )
    try:
        record_tooling_result(
            store_name,
            language,
            problem_number,
            output,
            correct,
            max_problem_number,
            problem_start,
            problem_end,
            solutions_json_path,
            benchmark_lock,
        )
        return problem_number, output, correct
    finally:
        if not previous_endpoint:
            set_log_endpoint(previous_endpoint)


def prepare_endpoints(endpoints: List[Endpoint]) -> List[Endpoint]:
    available_endpoints = []
    for endpoint in endpoints:
        try:
            if ensure_model_available(endpoint, attempts=3, fail_if_unavailable=False):
                available_endpoints.append(endpoint)
        except Exception as e:
            log(f"Error loading endpoint {endpoint}: {e}")
        if endpoint not in available_endpoints:
            log(f"Failed to load endpoint {endpoint} after 3 attempts.")
    return available_endpoints


def build_tool_prompt(template_content: str, problem_content: str, language: str) -> str:
    prompt = template_content.replace("$$$PROBLEM$$$", problem_content)
    prompt = remove_code_block_instructions(prompt, language)
    return (
        f"{prompt}\n\n"
        "Additional instructions:\n"
        f"- Produce only runnable {language} source code.\n"
        "- Use the tools to draft files, inspect them, and check syntax before delivery.\n"
        "- Finish by calling deliver_code with the final workspace file path.\n"
        "- Do not return markdown fences or explanations in the delivered code.\n"
    )


def remove_code_block_instructions(prompt: str, language: str) -> str:
    lines = prompt.splitlines()
    filtered_lines: List[str] = []
    skip_example_block = False

    for line in lines:
        stripped = line.strip()
        if stripped == f"- Wrap your code in a {language} code block using triple backticks":
            continue
        if stripped == "EXAMPLE FORMAT:":
            skip_example_block = True
            continue
        if skip_example_block:
            if stripped.startswith("This would output ") or stripped == "":
                if stripped.startswith("This would output "):
                    skip_example_block = False
                continue
            if stripped.startswith("```") or stripped.startswith("print("):
                continue
        filtered_lines.append(line)

    cleaned = "\n".join(filtered_lines)
    cleaned = cleaned.replace("\n\n\n", "\n\n")
    return cleaned


def process_problem_files(
    problems_dir,
    template_content,
    endpoints: List[Endpoint],
    language,
    max_problem_number=9999,
    problem_start=1,
    problem_end=9999,
    overwrite_existing=False,
    overwrite_failed=False,
    expected_solutions={},
    think=False,
    no_think=False,
    stream=False,
):
    log(f"Processing problems in {problems_dir} with language {language} and endpoint: {endpoints[0]}")
    store_name = endpoints[0].store_name
    if think:
        store_name += "-think"
    if no_think:
        store_name += "-no_think"
    solutions_dir = os.path.join("solutions", store_name, language)
    os.makedirs(solutions_dir, exist_ok=True)
    solutions_json_path = os.path.join(solutions_dir, "solutions.json")
    extension = get_extension(language)

    ensure_model_available(endpoints[0], attempts=3, fail_if_unavailable=True)

    available_endpoints = prepare_endpoints(endpoints)
    if not available_endpoints:
        raise Exception("No usable endpoints are available.")
    validate_tool_agent_contract(
        available_endpoints[0],
        think=think,
        no_think=no_think,
    )

    benchmark = read_benchmark()
    entry = benchmark.get(store_name, {})
    entry, entry_changed = complete_model_capabilities(entry, available_endpoints[0], think=think, no_think=no_think)
    if not entry.get("has_tooling", False):
        entry["has_tooling"] = True
        entry_changed = True
        log(
            f"Overriding cached has_tooling=False for {store_name} because "
            f"tool-agent preflight succeeded on {available_endpoints[0].url}."
        )
    if entry_changed:
        benchmark[store_name] = entry
        write_benchmark(benchmark)
    has_tool_calling = bool(entry.get("has_tooling", False))

    if not has_tool_calling:
        raise Exception(f"Model {store_name} does not support tool calling on {available_endpoints[0].url}")
    is_vision = bool(entry.get("has_vision", False))
    tooling_series_name = get_tooling_series_name(language, max_problem_number)
    benchmark_lock = threading.Lock()
    recorded_vector = get_recorded_tooling_vector(store_name, language, max_problem_number)

    problem_jobs = []
    execution_only_jobs = []
    for problem_file in sorted(os.listdir(problems_dir)):
        if problem_file.startswith(".") or not problem_file.endswith(".txt"):
            continue
        problem_number = problem_file[:-4]
        if int(problem_number) < problem_start:
            continue
        if int(problem_number) > problem_end:
            break

        output_path = os.path.join(solutions_dir, f"tool-{problem_number}.{extension}")
        expected = expected_solutions.get(problem_number, {})
        already_recorded = tooling_result_recorded(recorded_vector, problem_number, problem_start, problem_end)
        if already_recorded and not overwrite_existing:
            log(f"[{problem_number}] Skipping because tooling result is already recorded.")
            continue
        if os.path.exists(output_path):
            if overwrite_existing:
                pass
            else:
                execution_only_jobs.append((problem_number, output_path, expected))
                continue

        problem_path = os.path.join(problems_dir, problem_file)
        with open(problem_path, "r", encoding="utf-8") as file:
            problem_content = file.read()

        base64_image = None
        for ext in ["-0.png", "-0.jpg", "-0.gif"]:
            image_path = os.path.join(problems_dir, problem_number + ext)
            if os.path.exists(image_path):
                with open(image_path, "rb") as image_file:
                    base64_image = base64.b64encode(image_file.read()).decode("utf-8")
                break

        if base64_image and not is_vision:
            log(
                f"Problem {problem_number} requires a vision-capable model for image processing but the model lacks vision support."
            )
            base64_image = None

        prompt = build_tool_prompt(template_content, problem_content, language)
        problem_jobs.append((problem_number, prompt, output_path, base64_image, expected))

    if not problem_jobs and not execution_only_jobs:
        log("No problems queued.")
        return

    max_workers = len(available_endpoints)
    if max_workers == 1:
        log("Using serial execution because only one endpoint is available.")

    with ThreadPoolExecutor(max_workers=max_workers) as executor:
        future_to_problem = {}
        if max_workers == 1:
            for problem_number, output_path, expected in execution_only_jobs:
                future = executor.submit(
                    evaluate_existing_solution,
                    store_name,
                    language,
                    problem_number,
                    output_path,
                    expected,
                    max_problem_number,
                    problem_start,
                    problem_end,
                    solutions_json_path,
                    benchmark_lock,
                    True,
                )
                future_to_problem[future] = (problem_number, store_name, expected)
                log(f"[{problem_number}] Queued existing tool solution for evaluation on {store_name}")
                problem_number_done, endpoint_name, expected_done = future_to_problem[future]
                try:
                    future.result()
                except Exception as e:
                    failure_output = f"Error: {e}"
                    record_tooling_result(
                        store_name,
                        language,
                        problem_number_done,
                        failure_output,
                        False,
                        max_problem_number,
                        problem_start,
                        problem_end,
                        solutions_json_path,
                        benchmark_lock,
                    )
                    log(f"[{problem_number_done}] Failed on {endpoint_name}: {e}")
            for problem_number, prompt, output_path, base64_image, expected in problem_jobs:
                endpoint = available_endpoints[0]
                future = executor.submit(
                    process_single_problem,
                    endpoint,
                    store_name,
                    language,
                    problem_number,
                    prompt,
                    output_path,
                    expected,
                    max_problem_number,
                    problem_start,
                    problem_end,
                    solutions_json_path,
                    benchmark_lock,
                    base64_image,
                    think,
                    no_think,
                    stream,
                )
                future_to_problem[future] = (problem_number, endpoint.store_name, expected)
                log(f"[{problem_number}] Queued generation on {endpoint.store_name}")
                problem_number_done, endpoint_name, expected_done = future_to_problem[future]
                try:
                    future.result()
                except Exception as e:
                    failure_output = f"Error: {e}"
                    record_tooling_result(
                        store_name,
                        language,
                        problem_number_done,
                        failure_output,
                        False,
                        max_problem_number,
                        problem_start,
                        problem_end,
                        solutions_json_path,
                        benchmark_lock,
                    )
                    log(f"[{problem_number_done}] Failed on {endpoint_name}: {e}")
        else:
            for problem_number, output_path, expected in execution_only_jobs:
                future = executor.submit(
                    evaluate_existing_solution,
                    store_name,
                    language,
                    problem_number,
                    output_path,
                    expected,
                    max_problem_number,
                    problem_start,
                    problem_end,
                    solutions_json_path,
                    benchmark_lock,
                    True,
                )
                future_to_problem[future] = (problem_number, store_name, expected)
                log(f"[{problem_number}] Queued existing tool solution for evaluation on {store_name}")
            for index, (problem_number, prompt, output_path, base64_image, expected) in enumerate(problem_jobs):
                endpoint = available_endpoints[index % len(available_endpoints)]
                future = executor.submit(
                    process_single_problem,
                    endpoint,
                    store_name,
                    language,
                    problem_number,
                    prompt,
                    output_path,
                    expected,
                    max_problem_number,
                    problem_start,
                    problem_end,
                    solutions_json_path,
                    benchmark_lock,
                    base64_image,
                    think,
                    no_think,
                    stream,
                )
                future_to_problem[future] = (problem_number, endpoint.store_name, expected)
                log(f"[{problem_number}] Queued generation on {endpoint.store_name}")

            for future in as_completed(future_to_problem):
                problem_number, endpoint_name, expected = future_to_problem[future]
                try:
                    future.result()
                except Exception as e:
                    failure_output = f"Error: {e}"
                    record_tooling_result(
                        store_name,
                        language,
                        problem_number,
                        failure_output,
                        False,
                        max_problem_number,
                        problem_start,
                        problem_end,
                        solutions_json_path,
                        benchmark_lock,
                    )
                    log(f"[{problem_number}] Failed on {endpoint_name}: {e}")

    log("All problems processed!")


def main():
    parser = ArgumentParser(description="Process Euler problems and send them to an LLM.")
    parser.add_argument("--api", action="append", help="Specify (multiple) backend OpenAI API endpoints (i.e. ollama); can be used multiple times")
    parser.add_argument("--api_base", required=False, default="http://localhost:11434", help="API base URL for the LLM or a list of such urls (comma-separated), default is http://localhost:11434")
    parser.add_argument("--endpoint", required=False, default="", help="Name of an <endpoint>.json file in the endpoints directory")
    parser.add_argument("--store_name", help="Storage name when the endpoint file omits store_name")
    parser.add_argument("--model_name", help="API model name when the endpoint file omits model_name")
    parser.add_argument("--allmodels", action="store_true", help="loop over all models provided by ollama and run those which are missing in benchmark.json")
    parser.add_argument("--model", required=False, default="llama3.2:latest", help="Name of the model to use, default is llama3.2:latest")
    parser.add_argument("--think", action="store_true", help="enable thinking mode via backend request parameters (when supported)")
    parser.add_argument("--no_think", action="store_true", help="disable thinking mode via backend request parameters (when supported)")
    parser.add_argument("--stream", action="store_true", help="stream tool-agent model responses for transparency")
    parser.add_argument("--language", required=False, default="python,java,rust,clojure", help="Name of the languages to test, default is python,java,rust,clojure")
    parser.add_argument("--overwrite_existing", action="store_true", help="if set, re-calculate all problems that already have an answer")
    parser.add_argument("--overwrite_failed", action="store_true", help="if set, re-calculate those problems with wrong answers")
    parser.add_argument("--n100", action="store_true", help="problems 1 to 100")
    parser.add_argument("--n200", action="store_true", help="problems 1 to 200")
    parser.add_argument("--n300", action="store_true", help="problems 201 to 300")
    parser.add_argument("--n400", action="store_true", help="problems 301 to 400")
    parser.add_argument("--nall", action="store_true", help="all problems")

    args = parser.parse_args()
    api_base = args.api if args.api else args.api_base.split(",") if "," in args.api_base else [args.api_base]
    store_name = args.model
    max_problem_number, problem_start, problem_end = get_tooling_batch_bounds(args)

    os.chdir(os.path.dirname(os.path.realpath(__file__)))

    with open("solutions.json", "r", encoding="utf-8") as json_file:
        expected_solutions = json.load(json_file)

    languages = args.language.split(",")
    for language in languages:
        endpoint_name = args.endpoint
        problems_dir = "problems"
        template_path = os.path.join("templates", "template_" + language + ".md")

        if not os.path.exists(problems_dir):
            raise Exception(f"Problems directory {problems_dir} does not exist. You must create it using the problems_scraper.py script.")
        if not os.path.exists(template_path):
            raise Exception(f"Template file {template_path} does not exist.")

        template_content = read_template(template_path)

        if args.allmodels:
            if endpoint_name:
                raise Exception("The --allmodels option cannot be used in combination with --endpoint.")

            local_endpoint = Endpoint(store_name=store_name, model_name=store_name, key="", url=f"{api_base[0]}/v1/chat/completions")
            models = openai_api_list(local_endpoint)
            log(f"Found {len(models)} models in ollama.")
            for model in models:
                benchmark = read_benchmark()
                tooling_bench_name = get_tooling_series_name(language, max_problem_number)
                if model not in benchmark or tooling_bench_name not in benchmark[model]:
                    log(f"Inference with tools: Using model {model} and language {language}")
                    endpoints = [
                        Endpoint(store_name=model, model_name=model, key="", url=f"{api_stub}/v1/chat/completions")
                        for api_stub in api_base
                    ]
                    process_problem_files(
                        problems_dir,
                        template_content,
                        endpoints,
                        language,
                        max_problem_number=max_problem_number,
                        problem_start=problem_start,
                        problem_end=problem_end,
                        overwrite_existing=args.overwrite_existing,
                        overwrite_failed=args.overwrite_failed,
                        expected_solutions=expected_solutions,
                        think=args.think,
                        no_think=args.no_think,
                        stream=args.stream,
                    )
        else:
            endpoints = []
            if endpoint_name:
                log(f"Inference with tools: Using endpoint {endpoint_name} and language {language}")
                endpoint_path = os.path.join("endpoints", f"{endpoint_name}.json")
                log(f"Using endpoint file {endpoint_path}")
                if not os.path.exists(endpoint_path):
                    raise Exception(f"Endpoint file {endpoint_path} does not exist.")
                endpoints = [
                    load_endpoint_file(
                        endpoint_path,
                        store_name=args.store_name,
                        model_name=args.model_name,
                    )
                ]
            else:
                log(f"Inference with tools: Using model {store_name} and language {language}")
                endpoints = [
                    Endpoint(store_name=store_name, model_name=store_name, key="", url=f"{api_stub}/v1/chat/completions")
                    for api_stub in api_base
                ]

            process_problem_files(
                problems_dir,
                template_content,
                endpoints,
                language,
                max_problem_number=max_problem_number,
                problem_start=problem_start,
                problem_end=problem_end,
                overwrite_existing=args.overwrite_existing,
                overwrite_failed=args.overwrite_failed,
                expected_solutions=expected_solutions,
                think=args.think,
                no_think=args.no_think,
                stream=args.stream,
            )


if __name__ == "__main__":
    main()