secret seeds in database

Author: ngc92

src/discord-cluster-manager/cogs/leaderboard_cog.py

@@ -1,4 +1,5 @@
 import asyncio
+import copy
 from datetime import datetime, timedelta
 from io import StringIO
 from typing import TYPE_CHECKING, Callable, List, Optional
@@ -46,12 +47,20 @@ async def async_submit_cog_job(
         script: discord.Attachment,
         command: Callable,
         task: LeaderboardTask,
+        seed: Optional[int],
         reporter: RunProgressReporter,
         gpu: app_commands.Choice[str],
         runner_name: str,
         mode: SubmissionMode,
         submission_id: int,
     ):
+        if seed is not None:
+            # careful, we've got a reference here
+            # that is shared with the other run
+            # invocations.
+            task = copy.copy(task)
+            task.seed = seed
+
         result: FullResult = await command(
             interaction,
             script,
@@ -152,8 +161,8 @@ async def before_submit_hook(
                 return None, None
 
             gpus = db.get_leaderboard_gpu_types(leaderboard_name)
-            task = leaderboard_item["task"]
-        return task, gpus
+            task: LeaderboardTask = leaderboard_item["task"]
+        return task, gpus, leaderboard_item['secret_seed']
 
     def _get_run_command(self, gpu) -> Optional[Callable]:
         runner_cog = self.bot.get_cog(f"{gpu.runner}Cog")
@@ -230,7 +239,7 @@ async def on_submit_hook(  # noqa: C901
             )
             return -1
 
-        task, task_gpus = await self.before_submit_hook(
+        task, task_gpus, secret_seed = await self.before_submit_hook(
             interaction,
             leaderboard_name,
         )
@@ -297,6 +306,7 @@ async def on_submit_hook(  # noqa: C901
                     script,
                     command,
                     task,
+                    None,
                     reporter.add_run(f"{gpu.name} on {gpu.runner}"),
                     app_commands.Choice(name=gpu.name, value=gpu.value),
                     gpu.runner,
@@ -315,6 +325,7 @@ async def on_submit_hook(  # noqa: C901
                         script,
                         command,
                         task,
+                        secret_seed,
                         reporter.add_run(f"{gpu.name} on {gpu.runner} (secret)"),
                         app_commands.Choice(name=gpu.name, value=gpu.value),
                         gpu.runner,

src/discord-cluster-manager/leaderboard_db.py

@@ -418,7 +418,7 @@ def get_leaderboard_gpu_types(self, leaderboard_name: str) -> List[str] | None:
     def get_leaderboard(self, leaderboard_name: str) -> LeaderboardItem | None:
         self.cursor.execute(
             """
-            SELECT id, name, deadline, task, creator_id, forum_id
+            SELECT id, name, deadline, task, creator_id, forum_id, secret_seed
             FROM leaderboard.leaderboard
             WHERE name = %s
             """,
@@ -436,6 +436,7 @@ def get_leaderboard(self, leaderboard_name: str) -> LeaderboardItem | None:
                 task=task,
                 creator_id=res[4],
                 forum_id=res[5],
+                secret_seed=res[6],
             )
         else:
             return None

src/discord-cluster-manager/migrations/20250329_01_7VjJJ-add-a-secret-seed-column.py

@@ -0,0 +1,16 @@
+"""
+Add a secret seed column.
+Note: double epsilon is eps=2.22e-16, so 1/eps = 4504504504504504 >> 2^31 = 2147483648,
+so we should get reasonably uniform integers here
+"""
+
+from yoyo import step
+
+__depends__ = {"20250316_01_5oMi3-remember-forum-id"}
+
+steps = [
+    step(
+        "ALTER TABLE leaderboard.leaderboard ADD COLUMN secret_seed BIGINT NOT NULL "
+        "DEFAULT FLOOR(RANDOM() * 2147483648) ",
+    ),
+]

src/discord-cluster-manager/run_eval.py

@@ -217,13 +217,14 @@ def compile_cuda_script(  # # noqa: C901
     )
 
 
-def run_program(args: list[str], seed: int, timeout: int) -> RunResult:
+def run_program(args: list[str], seed: Optional[int], timeout: int) -> RunResult:
     print("[Running]")
     # set up a pipe so the tester can communicate its verdict with us
     env = os.environ.copy()
     pipe_read, pipe_write = os.pipe()
     env["POPCORN_FD"] = str(pipe_write)
-    env["POPCORN_SEED"] = str(seed)
+    if seed is not None:
+        env["POPCORN_SEED"] = str(seed)
 
     execution_start_time = time.perf_counter()
     try:
@@ -283,7 +284,7 @@ def run_single_evaluation(
     test_timeout: int = Timeout.TEST,
     benchmark_timeout: int = Timeout.BENCHMARK,
     ranked_timeout: int = Timeout.RANKED,
-    seed: Optional[int] = 42,
+    seed: Optional[int] = None,
 ) -> RunResult:
     """
     A single runner run, either in the context of test files, or in the
@@ -542,7 +543,7 @@ def run_config(config: dict):
     common_args = {
         "tests": build_test_string(config.get("tests", [])),
         "benchmarks": build_test_string(config.get("benchmarks", [])),
-        "seed": 42,
+        "seed": config.get("seed", None),
     }
     if config["lang"] == "py":
         runner = functools.partial(

src/discord-cluster-manager/task.py

@@ -2,7 +2,7 @@
 import dataclasses
 import json
 from pathlib import Path
-from typing import Dict, Union
+from typing import Dict, Optional, Union
 
 from consts import Language
 
@@ -60,6 +60,7 @@ class LeaderboardTask:
     benchmark_timeout: int = 60
     ranked_timeout: int = 90
     templates: dict[str, str] = dataclasses.field(default_factory=dict)
+    seed: Optional[int] = None
 
     @staticmethod
     def from_dict(data: dict):

src/discord-cluster-manager/utils.py

@@ -2,7 +2,7 @@
 import functools
 import logging
 import subprocess
-from typing import Any, List, Optional, TypedDict
+from typing import Any, List, NotRequired, Optional, TypedDict
 
 import discord
 from consts import Language, SubmissionMode
@@ -191,6 +191,7 @@ class LeaderboardItem(TypedDict):
     task: LeaderboardTask
     gpu_types: List[str]
     forum_id: int
+    secret_seed: NotRequired[int]
 
 
 class LeaderboardRankedEntry(TypedDict):
@@ -266,18 +267,23 @@ def build_task_config(
             else:
                 all_files[n] = c
 
+        common = {
+            "lang": task.lang.value,
+            "arch": arch,
+            "benchmarks": task.benchmarks,
+            "tests": task.tests,
+            "mode": mode.value,
+            "test_timeout": task.test_timeout,
+            "benchmark_timeout": task.benchmark_timeout,
+            "ranked_timeout": task.ranked_timeout,
+            "seed": task.seed,
+        }
+
         if task.lang == Language.Python:
             return {
-                "lang": task.lang.value,
-                "arch": arch,
                 "main": task.config.main,
                 "sources": all_files,
-                "benchmarks": task.benchmarks,
-                "tests": task.tests,
-                "mode": mode.value,
-                "test_timeout": task.test_timeout,
-                "benchmark_timeout": task.benchmark_timeout,
-                "ranked_timeout": task.ranked_timeout,
+                **common
             }
         else:
             sources = {}
@@ -289,17 +295,9 @@ def build_task_config(
                     headers[f] = all_files[f]
 
             return {
-                "lang": task.lang.value,
-                "arch": arch,
                 "sources": sources,
                 "headers": headers,
-                "tests": task.tests,
-                "benchmarks": task.benchmarks,
                 "include_dirs": task.config.include_dirs,
-                "mode": mode.value,
-                "test_timeout": task.test_timeout,
-                "benchmark_timeout": task.benchmark_timeout,
-                "ranked_timeout": task.ranked_timeout,
             }