PM-4299: allow rerun dispatch when ECS listing is denied

jmgasper · jmgasper · commit 38080d8a06b8 · 2026-06-10T12:36:54.000+10:00
What was broken
Changing the tester on an active Marathon Match config triggered the latest-submission rerun path, but ECS scorer launches could all fail before RunTask when the service role lacked permission for ListTasks. Work Manager then showed the saved scorer config while previous scores remained because no new runner tasks were launched.

Root cause (if identifiable)
A later ECS active-task de-duplication and concurrency check made ListTasks/DescribeTasks mandatory for every scorer launch. In dev, that optional inspection path is not authorized, so the PM-4299 rerun trigger selected submissions but could not dispatch them.

What was changed
Made active ECS task inspection best-effort. When ECS denies the inspection call, the service logs a warning and continues with direct RunTask dispatch while preserving duplicate-task reuse and concurrency checks whenever inspection is available.

Any added/updated tests
Added an EcsService unit test covering the AccessDenied ListTasks fallback and verifying the scorer task still launches and persists the runner log mapping.
diff --git a/src/shared/modules/global/ecs.service.spec.ts b/src/shared/modules/global/ecs.service.spec.ts
@@ -310,6 +310,85 @@ describe('EcsService', () => {
     ).toBe(false);
   });
 
+  it('launches scorer tasks when active task listing is not permitted', async () => {
+    const { service, m2mService, prisma, send } = createService();
+    m2mService.getM2MToken.mockResolvedValue('launch-token');
+    send.mockImplementation((command) => {
+      if (command instanceof ListTasksCommand) {
+        return Promise.reject(
+          Object.assign(
+            new Error('User is not authorized to perform: ecs:ListTasks'),
+            {
+              name: 'AccessDeniedException',
+            },
+          ),
+        );
+      }
+      if (command instanceof RunTaskCommand) {
+        return Promise.resolve({
+          tasks: [
+            {
+              taskArn:
+                'arn:aws:ecs:us-east-1:123456789012:task/cluster/task-123',
+            },
+          ],
+        });
+      }
+      if (command instanceof DescribeTaskDefinitionCommand) {
+        return Promise.resolve({
+          taskDefinition: {
+            containerDefinitions: [
+              {
+                name: 'tc-mm-runner',
+                logConfiguration: {
+                  options: {
+                    'awslogs-group': '/ecs/mm-runner',
+                    'awslogs-stream-prefix': 'mm',
+                  },
+                },
+              },
+            ],
+          },
+        });
+      }
+
+      throw new Error(`Unexpected command ${command.constructor.name}`);
+    });
+
+    const result = await service.launchScorerTask(
+      'challenge-1',
+      'submission-1',
+      baseTaskConfig,
+      basePhaseConfig,
+      undefined,
+      { memberId: 'member-1' },
+    );
+
+    expect(result.taskId).toBe('task-123');
+    expect(result.logStreamName).toBe('mm/tc-mm-runner/task-123');
+    expect(m2mService.getM2MToken).toHaveBeenCalledTimes(1);
+    expect(
+      send.mock.calls.some(([command]) => command instanceof RunTaskCommand),
+    ).toBe(true);
+    expect(
+      send.mock.calls.some(([command]) => command instanceof StopTaskCommand),
+    ).toBe(false);
+    expect(mockLogger.warn).toHaveBeenCalledWith(
+      expect.objectContaining({
+        message:
+          'Unable to inspect active ECS scorer tasks; launching without duplicate or concurrency checks.',
+        taskDefinitionName: 'mm-ecs-runner',
+      }),
+    );
+    expect(prisma.submissionRunnerLog.upsert).toHaveBeenCalledWith(
+      expect.objectContaining({
+        where: {
+          taskArn: 'arn:aws:ecs:us-east-1:123456789012:task/cluster/task-123',
+        },
+      }),
+    );
+  });
+
   it('passes Auth0 refresh settings to the ECS runner for long system scoring callbacks', async () => {
     const { service, m2mService, send } = createService();
     m2mService.getM2MToken.mockResolvedValue('launch-token');
diff --git a/src/shared/modules/global/ecs.service.ts b/src/shared/modules/global/ecs.service.ts
@@ -95,9 +95,9 @@ export class EcsService {
    * AWS_REGION, MARATHON_MATCH_API_URL, REVIEW_TYPE_ID, and Auth0 M2M settings
    * used by the runner to refresh tokens during long scoring tasks. Optional
    * ECS_SCORER_MAX_CONCURRENT_TASKS controls the global pending/running scorer
-   * task cap and defaults to 20.
-   * @throws Error when required ENV vars are missing, token fetch fails, the scorer task cap is reached,
-   * or ECS launch/cancellation fails.
+   * task cap when the role can list active ECS tasks and defaults to 20.
+   * @throws Error when required ENV vars are missing, token fetch fails, the scorer task cap is reached
+   * after active task lookup succeeds, or ECS launch/cancellation fails.
    */
   async launchScorerTask(
     challengeId: string,
@@ -130,53 +130,58 @@ export class EcsService {
 
     try {
       return await this.runWithScorerLaunchLock(async () => {
-        const activeTasks = await this.listActiveScorerTasks(
+        const activeTasks = await this.listActiveScorerTasksIfPermitted(
           cluster,
           taskDefinitionName,
           containerName,
         );
-        const launchableActiveTasks =
-          await this.stopSupersededMemberScorerTasks(cluster, activeTasks, {
+        const launchableActiveTasks = activeTasks
+          ? await this.stopSupersededMemberScorerTasks(cluster, activeTasks, {
+              challengeId,
+              submissionId,
+              memberId,
+            })
+          : null;
+        let maxConcurrentScorerTasks: number | null = null;
+
+        if (launchableActiveTasks) {
+          const duplicateTask = this.findDuplicateActiveScorerTask(
+            launchableActiveTasks,
             challengeId,
             submissionId,
-            memberId,
-          });
-        const duplicateTask = this.findDuplicateActiveScorerTask(
-          launchableActiveTasks,
-          challengeId,
-          submissionId,
-          scoringPhase.configType,
-        );
-
-        if (duplicateTask) {
-          const launchResult = await this.buildLaunchResultFromActiveTask(
-            duplicateTask,
-            taskDefinition,
-            containerName,
+            scoringPhase.configType,
           );
-          await this.persistSubmissionRunnerLogMapping({
-            challengeId,
-            submissionId,
-            scoringPhase,
-            launchResult,
-          });
-          this.logger.log({
-            message: 'Skipped duplicate ECS scorer task launch',
-            challengeId,
-            submissionId,
-            memberId: memberId ?? null,
-            phaseConfigType: scoringPhase.configType,
-            taskArn: launchResult.taskArn,
-            taskId: launchResult.taskId,
-          });
-          return launchResult;
-        }
 
-        const maxConcurrentScorerTasks = this.getMaxConcurrentScorerTasks();
-        if (launchableActiveTasks.length >= maxConcurrentScorerTasks) {
-          throw new Error(
-            `ECS scorer task concurrency limit reached (${launchableActiveTasks.length}/${maxConcurrentScorerTasks}). Deferring submission ${submissionId} to Kafka retry/back-pressure instead of launching another task.`,
-          );
+          if (duplicateTask) {
+            const launchResult = await this.buildLaunchResultFromActiveTask(
+              duplicateTask,
+              taskDefinition,
+              containerName,
+            );
+            await this.persistSubmissionRunnerLogMapping({
+              challengeId,
+              submissionId,
+              scoringPhase,
+              launchResult,
+            });
+            this.logger.log({
+              message: 'Skipped duplicate ECS scorer task launch',
+              challengeId,
+              submissionId,
+              memberId: memberId ?? null,
+              phaseConfigType: scoringPhase.configType,
+              taskArn: launchResult.taskArn,
+              taskId: launchResult.taskId,
+            });
+            return launchResult;
+          }
+
+          maxConcurrentScorerTasks = this.getMaxConcurrentScorerTasks();
+          if (launchableActiveTasks.length >= maxConcurrentScorerTasks) {
+            throw new Error(
+              `ECS scorer task concurrency limit reached (${launchableActiveTasks.length}/${maxConcurrentScorerTasks}). Deferring submission ${submissionId} to Kafka retry/back-pressure instead of launching another task.`,
+            );
+          }
         }
 
         const token = await this.getM2MTokenForScorerLaunch();
@@ -300,7 +305,8 @@ export class EcsService {
           logStreamPrefix: logStreamPrefix ?? null,
           logStreamName: logStreamName ?? null,
           cloudWatchLogsConsoleUrl: cloudWatchLogsConsoleUrl ?? null,
-          activeScorerTaskCountBeforeLaunch: launchableActiveTasks.length,
+          activeScorerTaskCountBeforeLaunch:
+            launchableActiveTasks?.length ?? null,
           maxConcurrentScorerTasks,
         });
 
@@ -522,6 +528,43 @@ export class EcsService {
     return parsedLimit;
   }
 
+  /**
+   * Lists active scorer tasks when ECS permissions allow it; otherwise logs and
+   * returns null so scorer dispatch can continue through RunTask.
+   * @param cluster ECS cluster name or ARN.
+   * @param taskDefinitionName Task definition family configured for scoring.
+   * @param containerName Runner container name whose overrides identify scorer tasks.
+   * @returns Active scorer tasks, or null when listing/describing active tasks is not authorized.
+   * @throws Error for non-permission ECS lookup failures.
+   */
+  private async listActiveScorerTasksIfPermitted(
+    cluster: string,
+    taskDefinitionName: string,
+    containerName: string,
+  ): Promise<ActiveScorerTask[] | null> {
+    try {
+      return await this.listActiveScorerTasks(
+        cluster,
+        taskDefinitionName,
+        containerName,
+      );
+    } catch (error) {
+      if (!this.isEcsPermissionError(error)) {
+        throw error;
+      }
+
+      this.logger.warn({
+        message:
+          'Unable to inspect active ECS scorer tasks; launching without duplicate or concurrency checks.',
+        cluster,
+        taskDefinitionName,
+        containerName,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return null;
+    }
+  }
+
   /**
    * Lists active scorer tasks for the configured task family and container.
    * @param cluster ECS cluster name or ARN.
@@ -586,6 +629,40 @@ export class EcsService {
     return activeTasks;
   }
 
+  /**
+   * Detects AWS authorization failures for optional ECS inspection calls.
+   * @param error Error thrown by the AWS SDK.
+   * @returns True when the error represents a denied ECS action.
+   * Used by scorer launches to fall back to direct RunTask dispatch when active
+   * task inspection is not permitted in an environment.
+   */
+  private isEcsPermissionError(error: unknown): boolean {
+    const typedError = error as {
+      name?: unknown;
+      Code?: unknown;
+      code?: unknown;
+      message?: unknown;
+    };
+    const values = [
+      typedError.name,
+      typedError.Code,
+      typedError.code,
+      typedError.message,
+    ]
+      .map((value) =>
+        typeof value === 'string' ? value.trim().toLowerCase() : '',
+      )
+      .filter(Boolean);
+
+    return values.some(
+      (value) =>
+        value.includes('accessdenied') ||
+        value.includes('access denied') ||
+        value.includes('not authorized') ||
+        value.includes('unauthorized'),
+    );
+  }
+
   /**
    * Converts an ECS task description into active scorer task metadata.
    * @param cluster ECS cluster used for the task.
