Add per-user submission rate limit (1 per hour) (#436)

Enforce a per-user rate limit of 1 submission per hour, scoped to Modal
B200 GPU submissions on leaderboard ID 730 only. The 429 response suggests
using the NVIDIA runner instead of Modal for faster iteration.

Author: Mark Saroufim

src/kernelbot/api/api_utils.py

@@ -225,6 +225,21 @@ async def to_submit_info(
 
     try:
         with db_context as db:
+            # Per-user rate limit: max 1 submission per hour on Modal B200 for leaderboard 730
+            if gpu_type == "B200":
+                lb_id = db.get_leaderboard_id(leaderboard_name)
+                if lb_id == 730:
+                    last_submission_time = db.check_user_rate_limit(user_id)
+                    if last_submission_time:
+                        raise HTTPException(
+                            status_code=429,
+                            detail=(
+                                f"Rate limit exceeded. You can submit once per hour. "
+                                f"Last submission: {last_submission_time.isoformat()}. "
+                                f"Consider using the NVIDIA runner instead of Modal for faster iteration."
+                            ),
+                        )
+
             leaderboard_item = db.get_leaderboard(leaderboard_name)
             gpus = leaderboard_item.get("gpu_types", [])
             if gpu_type not in gpus:
@@ -239,7 +254,7 @@ async def to_submit_info(
     except Exception as e:
         raise HTTPException(
             status_code=500,
-            detail=f"Internal server error while validating leaderboard/GPU: {e}",
+            detail=f"Internal server error while validating submission: {e}",
         ) from e
 
     try:

src/libkernelbot/leaderboard_db.py

@@ -1172,6 +1172,23 @@ def cleanup_temp_users(self):
             logger.exception("Could not cleanup temp users", exc_info=e)
             raise KernelBotError("Database error while cleaning up temp users") from e
 
+    def check_user_rate_limit(self, user_id: str, hours: int = 1) -> Optional[datetime.datetime]:
+        """Check if user has submitted within the last `hours` hours.
+        Returns the most recent submission_time if rate-limited, None if allowed."""
+        self.cursor.execute(
+            """
+            SELECT submission_time
+            FROM leaderboard.submission
+            WHERE user_id = %s
+              AND submission_time > NOW() - INTERVAL '%s hours'
+            ORDER BY submission_time DESC
+            LIMIT 1
+            """,
+            (str(user_id), hours),
+        )
+        row = self.cursor.fetchone()
+        return row[0] if row else None
+
     def validate_cli_id(self, cli_id: str) -> Optional[dict[str, str]]:
         """
         Validates a CLI ID and returns the associated user ID if valid.

tests/test_admin_api.py

@@ -1,5 +1,6 @@
 """Tests for admin API endpoints."""
 
+import datetime
 from unittest.mock import MagicMock, patch
 
 import pytest
@@ -405,3 +406,101 @@ def test_update_problems_with_errors(self, test_client, mock_backend):
             assert data["status"] == "ok"
             assert len(data["errors"]) == 1
             assert data["errors"][0]["name"] == "bad-problem"
+
+
+class TestSubmissionRateLimit:
+    """Test per-user submission rate limiting on Modal B200 for leaderboard 730."""
+
+    def test_rate_limit_blocks_b200_leaderboard_730(self, test_client, mock_backend):
+        """Second B200 submission to leaderboard 730 within 1 hour is rejected with 429."""
+        mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db)
+        mock_backend.db.__exit__ = MagicMock(return_value=None)
+
+        recent_time = datetime.datetime.now(tz=datetime.timezone.utc)
+        mock_backend.db.check_user_rate_limit = MagicMock(return_value=recent_time)
+        mock_backend.db.get_leaderboard_id = MagicMock(return_value=730)
+        mock_backend.db.validate_cli_id = MagicMock(
+            return_value={"user_id": "123", "user_name": "testuser"}
+        )
+
+        response = test_client.post(
+            "/test-lb/B200/test",
+            headers={"X-Popcorn-Cli-Id": "test-cli-id"},
+            files={"file": ("solution.py", b"print('hello')", "text/plain")},
+        )
+        assert response.status_code == 429
+        assert "Rate limit exceeded" in response.json()["detail"]
+        assert "NVIDIA runner" in response.json()["detail"]
+
+    def test_rate_limit_skipped_for_non_b200(self, test_client, mock_backend):
+        """Rate limit is not enforced for non-B200 GPUs even on leaderboard 730."""
+        mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db)
+        mock_backend.db.__exit__ = MagicMock(return_value=None)
+        mock_backend.accepts_jobs = True
+
+        mock_backend.db.validate_cli_id = MagicMock(
+            return_value={"user_id": "123", "user_name": "testuser"}
+        )
+
+        mock_lb = MagicMock()
+        mock_lb.__getitem__ = lambda self, key: {"gpu_types": ["H100"]}[key]
+        mock_lb.get = lambda key, default=None: {"gpu_types": ["H100"]}.get(key, default)
+        mock_backend.db.get_leaderboard = MagicMock(return_value=mock_lb)
+
+        response = test_client.post(
+            "/test-lb/H100/test",
+            headers={"X-Popcorn-Cli-Id": "test-cli-id"},
+            files={"file": ("solution.py", b"print('hello')", "text/plain")},
+        )
+        # Should not hit rate limit at all — check_user_rate_limit should not be called
+        assert response.status_code != 429
+
+    def test_rate_limit_skipped_for_other_leaderboard(self, test_client, mock_backend):
+        """Rate limit is not enforced for B200 on a leaderboard other than 730."""
+        mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db)
+        mock_backend.db.__exit__ = MagicMock(return_value=None)
+        mock_backend.accepts_jobs = True
+
+        recent_time = datetime.datetime.now(tz=datetime.timezone.utc)
+        mock_backend.db.check_user_rate_limit = MagicMock(return_value=recent_time)
+        mock_backend.db.get_leaderboard_id = MagicMock(return_value=999)
+        mock_backend.db.validate_cli_id = MagicMock(
+            return_value={"user_id": "123", "user_name": "testuser"}
+        )
+
+        mock_lb = MagicMock()
+        mock_lb.__getitem__ = lambda self, key: {"gpu_types": ["B200"]}[key]
+        mock_lb.get = lambda key, default=None: {"gpu_types": ["B200"]}.get(key, default)
+        mock_backend.db.get_leaderboard = MagicMock(return_value=mock_lb)
+
+        response = test_client.post(
+            "/other-lb/B200/test",
+            headers={"X-Popcorn-Cli-Id": "test-cli-id"},
+            files={"file": ("solution.py", b"print('hello')", "text/plain")},
+        )
+        # Should not be rate limited since leaderboard ID is not 730
+        assert response.status_code != 429
+
+    def test_rate_limit_allows_first_b200_submission(self, test_client, mock_backend):
+        """First B200 submission to leaderboard 730 passes the rate limit check."""
+        mock_backend.db.__enter__ = MagicMock(return_value=mock_backend.db)
+        mock_backend.db.__exit__ = MagicMock(return_value=None)
+        mock_backend.accepts_jobs = True
+
+        mock_backend.db.check_user_rate_limit = MagicMock(return_value=None)
+        mock_backend.db.get_leaderboard_id = MagicMock(return_value=730)
+        mock_backend.db.validate_cli_id = MagicMock(
+            return_value={"user_id": "123", "user_name": "testuser"}
+        )
+
+        mock_lb = MagicMock()
+        mock_lb.__getitem__ = lambda self, key: {"gpu_types": ["B200"]}[key]
+        mock_lb.get = lambda key, default=None: {"gpu_types": ["B200"]}.get(key, default)
+        mock_backend.db.get_leaderboard = MagicMock(return_value=mock_lb)
+
+        response = test_client.post(
+            "/test-lb/B200/test",
+            headers={"X-Popcorn-Cli-Id": "test-cli-id"},
+            files={"file": ("solution.py", b"print('hello')", "text/plain")},
+        )
+        assert response.status_code != 429

tests/test_leaderboard_db.py

@@ -605,6 +605,51 @@ def test_generate_stats(database, submit_leaderboard):
         }
 
 
+def test_check_user_rate_limit_no_submissions(database, submit_leaderboard):
+    """Test rate limit returns None when user has no submissions"""
+    with database as db:
+        result = db.check_user_rate_limit("999")
+        assert result is None
+
+
+def test_check_user_rate_limit_recent_submission(database, submit_leaderboard):
+    """Test rate limit returns submission_time when user submitted recently"""
+    submit_time = datetime.datetime.now(tz=datetime.timezone.utc)
+    with database as db:
+        db.create_submission(
+            "submit-leaderboard", "file.py", 5, "code", submit_time, user_name="user"
+        )
+        result = db.check_user_rate_limit("5")
+        assert result is not None
+        assert abs((result - submit_time).total_seconds()) < 2
+
+
+def test_check_user_rate_limit_old_submission(database, submit_leaderboard):
+    """Test rate limit returns None when submission is older than the window"""
+    old_time = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(hours=2)
+    with database as db:
+        db.create_submission(
+            "submit-leaderboard", "file.py", 5, "code", old_time, user_name="user"
+        )
+        result = db.check_user_rate_limit("5")
+        assert result is None
+
+
+def test_check_user_rate_limit_different_user(database, submit_leaderboard):
+    """Test rate limit only applies to the specific user"""
+    submit_time = datetime.datetime.now(tz=datetime.timezone.utc)
+    with database as db:
+        db.create_submission(
+            "submit-leaderboard", "file.py", 5, "code", submit_time, user_name="user5"
+        )
+        # User 6 should not be rate limited
+        result = db.check_user_rate_limit("6")
+        assert result is None
+        # User 5 should be rate limited
+        result = db.check_user_rate_limit("5")
+        assert result is not None
+
+
 def test_get_user_submissions_empty(database, submit_leaderboard):
     """Test get_user_submissions returns empty list for user with no submissions"""
     with database as db: