Tweaks for test submission flow in MM setup

Author: jmgasper

ECS_RUNNER_LIFETIME.md

@@ -41,9 +41,9 @@ This is the normal path for `EXAMPLE` and `PROVISIONAL` scoring during the Submi
 
 ### Validation submission scoring
 
-`POST /v6/marathon-match/challenge/:challengeId/test-submission` uploads a validation submission through the configured Submission API and then launches one ECS runner for the requested `configType` (default `PROVISIONAL`). The response includes `submissionId`, `taskArn`, `taskId`, and usually `cloudWatchLogsConsoleUrl`.
+`POST /v6/marathon-match/challenge/:challengeId/test-submission` stores the uploaded ZIP as an isolated Marathon Match validation run, then launches one ECS runner for the requested `configType` (default `PROVISIONAL`). The runner downloads the ZIP from Marathon Match API using `VALIDATION_SUBMISSION_DOWNLOAD_URL`, includes `VALIDATION_RUN_ID` in progress/final callbacks, skips Submission API artifact uploads, and `ScoringResultService` stores progress/final score details on the validation run instead of creating Review API summations. The response includes `testSubmissionId`, `submissionId` (same validation-run id for runner compatibility), `status`, `taskArn`, `taskId`, and usually `cloudWatchLogsConsoleUrl`.
 
-We do this to support admins / copilots having the ability to test the scorer before the challenge launches.
+Use `GET /v6/marathon-match/challenge/:challengeId/test-submission/:testSubmissionId` to poll for completion and display the score/metadata. We do this to support admins / copilots having the ability to test the scorer before the challenge launches without adding a real submission or review to the challenge.
 
 ### Manual latest-submission rerun
 

ecs-runner/boilerplate/src/main/java/com/topcoder/scorer/services/SubmissionService.java

@@ -85,6 +85,21 @@ public void updateSubmissionStatus(String submissionId, String status) throws Ex
      */
     public void downloadSubmission(String submissionId, String targetDir) throws Exception {
         String url = submissionApiUrl + "/submissions/" + submissionId + "/download";
+        downloadSubmissionFromUrl(url, targetDir, submissionId);
+    }
+
+    /**
+     * Downloads a submission zip from an explicit URL and extracts it to the target directory.
+     * @param url Absolute URL returning the submission ZIP.
+     * @param targetDir Directory to extract the submission to.
+     * @param submissionId Submission or validation-run ID for contextual logs.
+     * @throws Exception if download or extraction fails.
+     */
+    public void downloadSubmissionFromUrl(
+        String url,
+        String targetDir,
+        String submissionId
+    ) throws Exception {
         HttpGet get = new HttpGet(url);
         get.setHeader("Authorization", "Bearer " + accessToken);
         logInfo(submissionId, "GET " + url + " (submission zip download)");

ecs-runner/src/main/java/com/topcoder/runner/EcsRunnerMain.java

@@ -194,6 +194,8 @@ public static void main(String[] args) {
         String reviewTypeId = null;
         String scorecardId = null;
         String submissionApiUrl = null;
+        String validationRunId = null;
+        String validationSubmissionDownloadUrl = null;
 
         try {
             challengeId = getRequiredEnv("TESTER_CONFIG_ID");
@@ -209,9 +211,24 @@ public static void main(String[] args) {
             if (reviewId != null && reviewId.isEmpty()) {
                 reviewId = null;
             }
+            validationRunId = getOptionalEnv("VALIDATION_RUN_ID", "");
+            if (validationRunId != null && validationRunId.isEmpty()) {
+                validationRunId = null;
+            }
+            validationSubmissionDownloadUrl = getOptionalEnv(
+                "VALIDATION_SUBMISSION_DOWNLOAD_URL",
+                ""
+            );
+            if (
+                validationSubmissionDownloadUrl != null
+                    && validationSubmissionDownloadUrl.isEmpty()
+            ) {
+                validationSubmissionDownloadUrl = null;
+            }
             testPhase = normalizeTestPhase(getOptionalEnv("TEST_PHASE", "provisional"));
             long phaseStartSeed = getOptionalLongEnv("PHASE_START_SEED", 0L);
             int phaseNumberOfTests = getOptionalIntEnv("PHASE_NUMBER_OF_TESTS", 0);
+            boolean validationMode = !isBlank(validationRunId);
             setLogContext(challengeId, submissionId, testPhase);
 
             logInfo(
@@ -230,6 +247,8 @@ public static void main(String[] args) {
                     + phaseStartSeed
                     + ", phaseNumberOfTests="
                     + phaseNumberOfTests
+                    + ", validationRunId="
+                    + (validationMode ? validationRunId : "<none>")
             );
             requireTrustedRunnerProcess();
 
@@ -318,9 +337,24 @@ public static void main(String[] args) {
                         + " to "
                         + submissionDir
                         + " via "
-                        + config.getSubmissionApiUrl()
+                        + (validationMode
+                            ? validationSubmissionDownloadUrl
+                            : config.getSubmissionApiUrl())
                 );
-                submissionService.downloadSubmission(submissionId, submissionDir.toString());
+                if (validationMode) {
+                    if (isBlank(validationSubmissionDownloadUrl)) {
+                        throw new RuntimeException(
+                            "VALIDATION_SUBMISSION_DOWNLOAD_URL is required when VALIDATION_RUN_ID is set."
+                        );
+                    }
+                    submissionService.downloadSubmissionFromUrl(
+                        validationSubmissionDownloadUrl,
+                        submissionDir.toString(),
+                        submissionId
+                    );
+                } else {
+                    submissionService.downloadSubmission(submissionId, submissionDir.toString());
+                }
                 logInfo("api.download-submission", "Submission download and extraction complete");
                 logDirectorySnapshot(submissionDir, 100);
 
@@ -345,6 +379,7 @@ public static void main(String[] args) {
                             testPhase,
                             reviewTypeId,
                             reviewId,
+                            validationRunId,
                             scorecardId,
                             0.0,
                             TEST_STATUS_IN_PROGRESS,
@@ -380,6 +415,7 @@ public static void main(String[] args) {
                         accessTokenProvider,
                         reviewTypeId,
                         reviewId,
+                        validationRunId,
                         scorecardId
                     );
                 } finally {
@@ -427,16 +463,24 @@ public static void main(String[] args) {
                     callbackMetadata
                 );
 
-                logInfo("artifacts.upload", "Uploading submission artifacts");
-                uploadArtifacts(
-                    httpClient,
-                    config.getSubmissionApiUrl(),
-                    accessTokenProvider,
-                    submissionId,
-                    testPhase,
-                    submissionDir
-                );
-                logInfo("artifacts.upload", "Artifact upload completed");
+                if (validationMode) {
+                    logInfo(
+                        "artifacts.upload",
+                        "Skipping submission artifact upload for validation run "
+                            + validationRunId
+                    );
+                } else {
+                    logInfo("artifacts.upload", "Uploading submission artifacts");
+                    uploadArtifacts(
+                        httpClient,
+                        config.getSubmissionApiUrl(),
+                        accessTokenProvider,
+                        submissionId,
+                        testPhase,
+                        submissionDir
+                    );
+                    logInfo("artifacts.upload", "Artifact upload completed");
+                }
 
                 ScoringCallbackRequest callbackRequest = new ScoringCallbackRequest(
                     challengeId,
@@ -445,6 +489,7 @@ public static void main(String[] args) {
                     testPhase,
                     reviewTypeId,
                     reviewId,
+                    validationRunId,
                     scorerConfig.getScoreCardId(),
                     callbackMetadata,
                     callbackCurrentReview,
@@ -486,18 +531,27 @@ public static void main(String[] args) {
                 error
             );
             writeFailureArtifactLog(submissionDir, submissionId, error);
-            uploadFailureArtifactsSafely(
-                submissionApiUrl,
-                accessTokenProvider,
-                submissionId,
-                testPhase,
-                submissionDir
-            );
+            if (isBlank(validationRunId)) {
+                uploadFailureArtifactsSafely(
+                    submissionApiUrl,
+                    accessTokenProvider,
+                    submissionId,
+                    testPhase,
+                    submissionDir
+                );
+            } else {
+                logInfo(
+                    "artifacts.upload",
+                    "Skipping failure artifact upload for validation run "
+                        + validationRunId
+                );
+            }
             postFailureProgressSafely(
                 challengeId,
                 submissionId,
                 testPhase,
                 reviewId,
+                validationRunId,
                 accessTokenProvider,
                 marathonMatchBaseUrl,
                 reviewTypeId,
@@ -646,6 +700,7 @@ private static TesterExecutionResult runTesterInIsolation(
         AccessTokenProvider accessTokenProvider,
         String reviewTypeId,
         String reviewId,
+        String validationRunId,
         String scorecardId
     ) throws Exception {
         Path scorerConfigPath = null;
@@ -712,6 +767,7 @@ private static TesterExecutionResult runTesterInIsolation(
                                     testPhase,
                                     reviewTypeId,
                                     reviewId,
+                                    validationRunId,
                                     scorecardId,
                                     progressUpdate.getProgress(),
                                     progressUpdate.getStatus(),
@@ -2520,6 +2576,7 @@ private static void postScoringProgressSafely(
      * @param submissionId Submission ID.
      * @param testPhase Scoring phase.
      * @param reviewId Optional review ID for system scoring.
+     * @param validationRunId Optional isolated validation run ID for Score Operations tests.
      * @param accessTokenProvider Refresh-capable bearer token provider.
      * @param marathonMatchBaseUrl Marathon Match API base URL, when available.
      * @param reviewTypeId Review type ID, when available.
@@ -2531,6 +2588,7 @@ private static void postFailureProgressSafely(
         String submissionId,
         String testPhase,
         String reviewId,
+        String validationRunId,
         AccessTokenProvider accessTokenProvider,
         String marathonMatchBaseUrl,
         String reviewTypeId,
@@ -2561,6 +2619,7 @@ private static void postFailureProgressSafely(
                     testPhase,
                     reviewTypeId,
                     reviewId,
+                    validationRunId,
                     scorecardId,
                     lastReportedProgress,
                     TEST_STATUS_FAILED,
@@ -6166,6 +6225,9 @@ private static class ScoringCallbackRequest {
         @JsonProperty("reviewId")
         private final String reviewId;
 
+        @JsonProperty("validationRunId")
+        private final String validationRunId;
+
         @JsonProperty("scorecardId")
         private final String scorecardId;
 
@@ -6185,6 +6247,7 @@ private static class ScoringCallbackRequest {
             String testPhase,
             String reviewTypeId,
             String reviewId,
+            String validationRunId,
             String scorecardId,
             Map<String, Object> metadata,
             Map<String, Object> currentReview,
@@ -6196,6 +6259,7 @@ private static class ScoringCallbackRequest {
             this.testPhase = testPhase;
             this.reviewTypeId = reviewTypeId;
             this.reviewId = reviewId;
+            this.validationRunId = validationRunId;
             this.scorecardId = scorecardId;
             this.metadata = metadata;
             this.currentReview = currentReview;
@@ -6222,6 +6286,9 @@ private static class ScoringProgressRequest {
         @JsonProperty("reviewId")
         private final String reviewId;
 
+        @JsonProperty("validationRunId")
+        private final String validationRunId;
+
         @JsonProperty("scorecardId")
         private final String scorecardId;
 
@@ -6252,6 +6319,7 @@ private static class ScoringProgressRequest {
             String testPhase,
             String reviewTypeId,
             String reviewId,
+            String validationRunId,
             String scorecardId,
             double progress,
             String status,
@@ -6266,6 +6334,7 @@ private static class ScoringProgressRequest {
             this.testPhase = testPhase;
             this.reviewTypeId = reviewTypeId;
             this.reviewId = reviewId;
+            this.validationRunId = validationRunId;
             this.scorecardId = scorecardId;
             this.progress = Math.min(1.0, Math.max(0.0, progress));
             this.status = status;

prisma/migrations/20260617120000_add_test_submission_run/migration.sql

@@ -0,0 +1,33 @@
+CREATE TABLE "marathon_match"."testSubmissionRun" (
+    "id" VARCHAR(14) NOT NULL DEFAULT nanoid(),
+    "challengeId" VARCHAR(36) NOT NULL,
+    "configType" "marathon_match"."PhaseConfigType" NOT NULL,
+    "memberId" TEXT,
+    "fileName" TEXT NOT NULL,
+    "mimeType" TEXT,
+    "fileSize" INTEGER NOT NULL,
+    "fileContent" BYTEA NOT NULL,
+    "status" TEXT NOT NULL DEFAULT 'QUEUED',
+    "score" DOUBLE PRECISION,
+    "message" TEXT,
+    "metadata" JSONB,
+    "currentReview" JSONB,
+    "impactedReviews" JSONB,
+    "progress" DOUBLE PRECISION,
+    "completedTests" INTEGER,
+    "totalTests" INTEGER,
+    "failedTests" INTEGER,
+    "taskArn" TEXT,
+    "taskId" TEXT,
+    "cloudWatchLogsConsoleUrl" TEXT,
+    "completedAt" TIMESTAMP(3),
+    "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updatedAt" TIMESTAMP(3) NOT NULL,
+    "createdBy" TEXT,
+
+    CONSTRAINT "testSubmissionRun_pkey" PRIMARY KEY ("id")
+);
+
+CREATE INDEX "testSubmissionRun_challengeId_idx" ON "marathon_match"."testSubmissionRun"("challengeId");
+CREATE INDEX "testSubmissionRun_status_idx" ON "marathon_match"."testSubmissionRun"("status");
+CREATE INDEX "testSubmissionRun_createdAt_idx" ON "marathon_match"."testSubmissionRun"("createdAt");

prisma/schema.prisma

@@ -121,6 +121,39 @@ model submissionRunnerLog {
   @@schema("marathon_match")
 }
 
+model testSubmissionRun {
+  id                       String          @id @default(dbgenerated("nanoid()")) @db.VarChar(14)
+  challengeId              String          @db.VarChar(36)
+  configType               PhaseConfigType
+  memberId                 String?
+  fileName                 String
+  mimeType                 String?
+  fileSize                 Int
+  fileContent              Bytes
+  status                   String          @default("QUEUED")
+  score                    Float?
+  message                  String?         @db.Text
+  metadata                 Json?
+  currentReview            Json?
+  impactedReviews          Json?
+  progress                 Float?
+  completedTests           Int?
+  totalTests               Int?
+  failedTests              Int?
+  taskArn                  String?
+  taskId                   String?
+  cloudWatchLogsConsoleUrl String?
+  completedAt              DateTime?
+  createdAt                DateTime        @default(now())
+  updatedAt                DateTime        @updatedAt
+  createdBy                String?
+
+  @@index([challengeId])
+  @@index([status])
+  @@index([createdAt])
+  @@schema("marathon_match")
+}
+
 model scoringCompletionEmailNotification {
   id               String    @id @default(dbgenerated("nanoid()")) @db.VarChar(14)
   challengeId      String    @db.VarChar(36)