Merge pull request #147 from ImogenBits/logs

Improved match logs

Author: Benezivas

algobattle/battle.py

@@ -4,6 +4,7 @@
 some basic battle types, and related classed.
 """
 from dataclasses import dataclass
+from enum import StrEnum
 from functools import wraps
 from importlib.metadata import entry_points
 from abc import abstractmethod
@@ -30,6 +31,7 @@
     ConfigDict,
     Field,
     GetCoreSchemaHandler,
+    SerializeAsAny,
     ValidationError,
     ValidationInfo,
     ValidatorFunctionWrapHandler,
@@ -39,12 +41,20 @@
 
 from algobattle.program import (
     Generator,
-    ProgramRunInfo,
+    GeneratorResult,
+    ProgramResult,
     ProgramUi,
+    RunConfigOverride,
     Solver,
+    SolverResult,
+)
+from algobattle.problem import InstanceModel, Problem, SolutionModel
+from algobattle.util import (
+    Encodable,
+    EncodableModel,
+    ExceptionInfo,
+    BaseModel,
 )
-from algobattle.problem import Problem
-from algobattle.util import Encodable, ExceptionInfo, BaseModel
 
 
 _BattleConfig: TypeAlias = Any
@@ -61,6 +71,54 @@
 Type = type
 
 
+class ProgramLogConfigTime(StrEnum):
+    """When to log a programs i/o."""
+
+    never = "never"
+    error = "error"
+    always = "always"
+
+
+class ProgramLogConfigLocation(StrEnum):
+    """Where to log a programs i/o."""
+
+    disabled = "disabled"
+    inline = "inline"
+
+
+class ProgramLogConfigView(Protocol):  # noqa: D101
+    when: ProgramLogConfigTime = ProgramLogConfigTime.error
+    output: ProgramLogConfigLocation = ProgramLogConfigLocation.inline
+
+
+class ProgramRunInfo(BaseModel):
+    """Data about a program's execution."""
+
+    runtime: float = 0
+    overriden: RunConfigOverride = Field(default_factory=dict)
+    error: ExceptionInfo | None = None
+    battle_data: SerializeAsAny[EncodableModel] | None = None
+    instance: SerializeAsAny[InstanceModel] | None = None
+    solution: SerializeAsAny[SolutionModel[InstanceModel]] | None = None
+
+    @classmethod
+    def from_result(cls, result: ProgramResult, *, inline_output: bool) -> Self:
+        """Converts the program run info into a jsonable model."""
+        info = cls(
+            runtime=result.runtime,
+            overriden=result.overriden,
+            error=result.error,
+        )
+        if inline_output:
+            if isinstance(result.battle_data, EncodableModel):
+                info.battle_data = result.battle_data
+            if isinstance(result.solution, SolutionModel):
+                info.solution = result.solution
+            if isinstance(result, GeneratorResult) and isinstance(result.instance, InstanceModel):
+                info.instance = result.instance
+        return info
+
+
 class Fight(BaseModel):
     """The result of one fight between the participating teams.
 
@@ -79,6 +137,28 @@ class Fight(BaseModel):
     solver: ProgramRunInfo | None
     """Data about the solver's execution."""
 
+    @classmethod
+    def from_results(
+        cls,
+        max_size: int,
+        score: float,
+        generator: GeneratorResult,
+        solver: SolverResult | None,
+        *,
+        config: ProgramLogConfigView,
+    ) -> Self:
+        """Turns the involved result objects into a jsonable model."""
+        inline_output = config.when == "always" or (
+            config.when == "error"
+            and (generator.error is not None or (solver is not None and solver.error is not None))
+        )
+        return cls(
+            max_size=max_size,
+            score=score,
+            generator=ProgramRunInfo.from_result(generator, inline_output=inline_output),
+            solver=ProgramRunInfo.from_result(solver, inline_output=inline_output) if solver is not None else None,
+        )
+
 
 class FightUi(ProgramUi, Protocol):
     """Provides an interface for :class:`Fight` to update the ui."""
@@ -113,6 +193,7 @@ class FightHandler:
     battle: "Battle"
     ui: FightUi
     set_cpus: str | None
+    log_config: ProgramLogConfigView
 
     @_save_result
     async def run(
@@ -175,8 +256,14 @@ async def run(
             set_cpus=self.set_cpus,
             ui=ui,
         )
-        if gen_result.info.error is not None:
-            return Fight(score=1, max_size=max_size, generator=gen_result.info, solver=None)
+        if gen_result.error is not None:
+            return Fight.from_results(
+                score=1,
+                max_size=max_size,
+                generator=gen_result,
+                solver=None,
+                config=self.log_config,
+            )
         assert gen_result.instance is not None
 
         sol_result = await self.solver.run(
@@ -190,8 +277,10 @@ async def run(
             set_cpus=self.set_cpus,
             ui=ui,
         )
-        if sol_result.info.error is not None:
-            return Fight(score=0, max_size=max_size, generator=gen_result.info, solver=sol_result.info)
+        if sol_result.error is not None:
+            return Fight.from_results(
+                score=0, max_size=max_size, generator=gen_result, solver=sol_result, config=self.log_config
+            )
         assert sol_result.solution is not None
 
         if self.problem.with_solution:
@@ -202,7 +291,13 @@ async def run(
         else:
             score = self.problem.score(gen_result.instance, solution=sol_result.solution)
         score = max(0, min(1, float(score)))
-        return Fight(score=score, max_size=max_size, generator=gen_result.info, solver=sol_result.info)
+        return Fight.from_results(
+            score=score,
+            max_size=max_size,
+            generator=gen_result,
+            solver=sol_result,
+            config=self.log_config,
+        )
 
 
 # We need this to be here to prevent an import cycle between match.py and battle.py

algobattle/cli.py

@@ -183,9 +183,8 @@ def run_match(
                 console.print(Padding(leaderboard, (1, 0, 0, 0)))
 
             if save:
-                res_string = result.model_dump_json(exclude_defaults=True)
                 out_path = config.project.results.joinpath(f"match-{timestamp()}.json")
-                out_path.write_text(res_string)
+                out_path.write_text(result.format(error_detail=config.project.error_detail))
                 console.print("Saved match result to ", out_path)
             return result
         except KeyboardInterrupt:

algobattle/match.py

@@ -5,7 +5,7 @@
 from itertools import combinations
 from pathlib import Path
 import tomllib
-from typing import Annotated, Any, Iterable, Protocol, ClassVar, Self, TypeAlias, TypeVar, cast
+from typing import Annotated, Any, Iterable, Literal, Protocol, ClassVar, Self, TypeAlias, TypeVar, cast
 from typing_extensions import override
 from typing_extensions import TypedDict
 
@@ -28,7 +28,15 @@
 from anyio.to_thread import current_default_thread_limiter
 from docker.types import LogConfig, Ulimit
 
-from algobattle.battle import Battle, FightHandler, FightUi, BattleUi, Iterated
+from algobattle.battle import (
+    Battle,
+    FightHandler,
+    FightUi,
+    BattleUi,
+    Iterated,
+    ProgramLogConfigLocation,
+    ProgramLogConfigTime,
+)
 from algobattle.program import ProgramConfigView, ProgramUi, Matchup, TeamHandler, BuildUi
 from algobattle.problem import Problem
 from algobattle.util import (
@@ -91,6 +99,7 @@ async def _run_battle(
                 battle=battle,
                 ui=battle_ui,
                 set_cpus=set_cpus,
+                log_config=config.project.log_program_io,
             )
             try:
                 await battle.run_battle(
@@ -184,6 +193,30 @@ def calculate_points(self, total_points_per_team: int) -> dict[str, float]:
 
         return points
 
+    def format(self, *, indent: int | None = 2, error_detail: Literal["high", "low"] = "low") -> str:
+        """Nicely formats the match result into a json string."""
+        match error_detail:
+            case "high":
+                exclude = None
+            case "low":
+                detail = {"detail"}
+                program = {"error": detail}
+                exclude = {
+                    "excluded_teams": {"__all__": detail},
+                    "battles": {
+                        "__all__": {
+                            "runtime_error": detail,
+                            "fights": {
+                                "__all__": {
+                                    "generator": program,
+                                    "solver": program,
+                                }
+                            },
+                        }
+                    },
+                }
+        return self.model_dump_json(exclude_defaults=True, indent=indent, exclude=exclude)
+
 
 class Ui(BuildUi, Protocol):
     """Base class for a UI that observes a Match and displays its data.
@@ -581,6 +614,13 @@ class DynamicProblemConfig(BaseModel):
 class ProjectConfig(BaseModel):
     """Various project settings."""
 
+    class ProgramOutputConfig(BaseModel):
+        """How to log program output."""
+
+        # a bit janky atm, allows for future expansion
+        when: ProgramLogConfigTime = ProgramLogConfigTime.error
+        output: ProgramLogConfigLocation = ProgramLogConfigLocation.inline
+
     parallel_battles: int = 1
     """Number of battles exectuted in parallel."""
     name_images: bool = True
@@ -589,6 +629,12 @@ class ProjectConfig(BaseModel):
     """Whether to clean up the images after we use them."""
     set_cpus: str | list[str] | None = None
     """Wich cpus to run programs on, if it is a list each battle will use a different cpu specification for it."""
+    error_detail: Literal["low", "high"] = "high"
+    """How detailed error messages should be.
+    Higher settings help in debugging, but may leak information from other teams.
+    """
+    log_program_io: ProgramOutputConfig = ProgramOutputConfig()
+    """How to log program output."""
     points: int = 100
     """Highest number of points each team can achieve."""
     results: RelativePath = Field(default=Path("./results"), validate_default=True)

algobattle/program.py

@@ -21,7 +21,6 @@
 from docker.models.containers import Container as DockerContainer
 from docker.types import Mount
 from requests import Timeout, ConnectionError
-from pydantic import Field
 from anyio import run as run_async
 from anyio.to_thread import run_sync
 from urllib3.exceptions import ReadTimeoutError
@@ -37,7 +36,6 @@
     TempDir,
     ValidationError,
     Role,
-    BaseModel,
 )
 from algobattle.problem import Problem, Instance, Solution
 
@@ -162,35 +160,25 @@ def __exit__(self, exc: Any, val: Any, tb: Any):
         self._output.__exit__(exc, val, tb)
 
 
-class ProgramRunInfo(BaseModel):
-    """Data about a program's execution."""
+@dataclass(frozen=True)
+class SolverResult:
+    """The result of a solver execution."""
 
     runtime: float = 0
-    overriden: RunConfigOverride = Field(default_factory=dict)
+    overriden: RunConfigOverride = field(default_factory=RunConfigOverride)
     error: ExceptionInfo | None = None
-
-
-@dataclass
-class ProgramResult:
-    """The result of a program execution."""
-
-    info: ProgramRunInfo
     battle_data: Encodable | None = None
+    solution: Solution[Instance] | None = None
 
 
-@dataclass
-class GeneratorResult(ProgramResult):
-    """Result of a single generator execution."""
+@dataclass(frozen=True)
+class GeneratorResult(SolverResult):
+    """The result of a generator execution."""
 
     instance: Instance | None = None
-    solution: Solution[Instance] | None = None
-
 
-@dataclass
-class SolverResult(ProgramResult):
-    """Result of a single solver execution."""
 
-    solution: Solution[Instance] | None = None
+ProgramResult = GeneratorResult | SolverResult
 
 
 @dataclass
@@ -573,7 +561,9 @@ async def run(
             except Exception as e:
                 exception_info = ExceptionInfo.from_exception(e)
             return GeneratorResult(
-                info=ProgramRunInfo(runtime=runtime, overriden=specs.overriden, error=exception_info),
+                runtime=runtime,
+                overriden=specs.overriden,
+                error=exception_info,
                 battle_data=battle_data,
                 instance=instance,
                 solution=solution,
@@ -582,8 +572,8 @@ async def run(
     def test(self, max_size: int | None = None) -> Instance | ExceptionInfo:
         """Tests whether the generator runs without issues and creates a syntactically valid instance."""
         res = run_async(self.run, max_size or self.problem.min_size)
-        if res.info.error:
-            return res.info.error
+        if res.error:
+            return res.error
         else:
             assert res.instance is not None
             return res.instance
@@ -658,16 +648,18 @@ async def run(
             except Exception as e:
                 exception_info = ExceptionInfo.from_exception(e)
             return SolverResult(
-                info=ProgramRunInfo(runtime=runtime, overriden=specs.overriden, error=exception_info),
+                runtime=runtime,
+                overriden=specs.overriden,
+                error=exception_info,
                 battle_data=battle_data,
                 solution=solution,
             )
 
     def test(self, instance: Instance) -> ExceptionInfo | None:
         """Tests whether the solver runs without issues and creates a syntactically valid solution."""
         res = run_async(self.run, instance, instance.size)
-        if res.info.error:
-            return res.info.error
+        if res.error:
+            return res.error
         else:
             return None
 
@@ -828,8 +820,6 @@ async def build(
             except Exception as e:
                 handler.excluded[name] = ExceptionInfo.from_exception(e)
                 ui.finish_build(name, False)
-            except BaseException:
-                raise
             else:
                 ui.finish_build(name, True)
         return handler