Retry failed firestore transaction if re-run requested while check-run is not yet completed. (#5003)

Retry failed firestore transaction if re-run requested while check-run
is not yet completed.

Decreased `delayFactor` to default 200 milliseconds and increased
`maxAttempts` from 8 by default to 10 to lover first retry delay and
potentially decrease response while survive flood of change requested
from scheduled to in progress.

fix: flutter/flutter#184215

Author: ievdokdm

app_dart/lib/src/request_handlers/rerun_all_failed_jobs.dart

@@ -5,6 +5,7 @@
 import 'dart:async';
 
 import 'package:github/github.dart';
+import 'package:retry/retry.dart';
 
 import '../../cocoon_service.dart';
 import '../model/ci_yaml/target.dart';
@@ -72,15 +73,20 @@ final class RerunAllFailedJobs extends ApiRequestHandler {
       );
     }
 
-    final failedChecks = await UnifiedCheckRun.reInitializeFailedJobs(
-      firestoreService: _firestore,
-      slug: slug,
-      prNum: prNumber,
-      guardCheckRunId: guard.checkRunId,
+    // We're doing a transactional update, which could fail if multiple tasks
+    // are running at the same time so retry a sane amount of times before
+    // giving up.
+    final failedChecks = await const RetryOptions().retry(
+      () => UnifiedCheckRun.reInitializeFailedJobs(
+        firestoreService: _firestore,
+        slug: slug,
+        prNum: prNumber,
+        guardCheckRunId: guard.checkRunId,
+      ),
     );
 
     if (failedChecks == null) {
-      return Response.json({'message': 'No failed jobs found to re-run'});
+      throw const BadRequestException('No failed jobs found to re-run');
     }
 
     final (targets, artifacts) = await _scheduler.getAllTargetsForPullRequest(

app_dart/lib/src/request_handlers/rerun_failed_job.dart

@@ -5,6 +5,7 @@
 import 'dart:async';
 
 import 'package:github/github.dart';
+import 'package:retry/retry.dart';
 
 import '../../cocoon_service.dart';
 import '../request_handling/api_request_handler.dart';
@@ -74,12 +75,17 @@ final class RerunFailedJob extends ApiRequestHandler {
       );
     }
 
-    final rerunInfo = await UnifiedCheckRun.reInitializeFailedJob(
-      firestoreService: _firestore,
-      slug: slug,
-      prNum: prNumber,
-      guardCheckRunId: guard.checkRunId,
-      jobName: jobName,
+    // We're doing a transactional update, which could fail if multiple tasks
+    // are running at the same time so retry a sane amount of times before
+    // giving up.
+    final rerunInfo = await const RetryOptions().retry(
+      () => UnifiedCheckRun.reInitializeFailedJob(
+        firestoreService: _firestore,
+        slug: slug,
+        prNum: prNumber,
+        guardCheckRunId: guard.checkRunId,
+        jobName: jobName,
+      ),
     );
 
     if (rerunInfo == null) {

app_dart/lib/src/service/scheduler.dart

@@ -1469,10 +1469,7 @@ $stacktrace
     // We're doing a transactional update, which could fail if multiple tasks
     // are running at the same time so retry a sane amount of times before
     // giving up.
-    const r = RetryOptions(
-      delayFactor: Duration(seconds: 2),
-      maxDelay: Duration(minutes: 2),
-    );
+    const r = RetryOptions(maxAttempts: 10, maxDelay: Duration(minutes: 2));
 
     try {
       return await r.retry(() {

app_dart/test/request_handlers/rerun_all_failed_jobs_test.dart

@@ -2,14 +2,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:convert';
-
 import 'package:cocoon_common/task_status.dart';
 import 'package:cocoon_integration_test/testing.dart';
 import 'package:cocoon_server_test/test_logging.dart';
 import 'package:cocoon_service/cocoon_service.dart';
 import 'package:cocoon_service/src/model/ci_yaml/target.dart';
 import 'package:cocoon_service/src/request_handlers/rerun_all_failed_jobs.dart';
+import 'package:cocoon_service/src/request_handling/exceptions.dart';
 import 'package:cocoon_service/src/service/luci_build_service/engine_artifacts.dart';
 import 'package:github/github.dart';
 import 'package:mockito/mockito.dart';
@@ -177,7 +176,7 @@ void main() {
     expect(response.statusCode, HttpStatus.ok);
   });
 
-  test('Re-run all failed jobs - no failures', () async {
+  test('Re-run all failed jobs - bad request', () async {
     final checkRun = generateCheckRun(1, name: 'Guard');
     final guard = generatePresubmitGuard(
       checkRun: checkRun,
@@ -201,11 +200,10 @@ void main() {
       'pr': guard.prNum,
     };
 
-    final response = await tester.post(handler);
-    final bodyString = await utf8.decoder.bind(response.body).join();
-    final body = jsonDecode(bodyString) as Map<String, dynamic>;
-
-    expect(body['message'], contains('No failed jobs found'));
+    await expectLater(
+      tester.post(handler),
+      throwsA(isA<BadRequestException>()),
+    );
   });
 }