Support post-submit builds for experimental branches. (#4679)

Towards flutter/flutter#168738.

I believe this is the last PR, we can do some validation testing after it (and it's friends) merge.

/cc @vashworth

Author: matanlurey

app_dart/lib/src/service/luci_build_service.dart

@@ -188,6 +188,37 @@ class LuciBuildService {
     return builds;
   }
 
+  /// Checks if [proposedVersion] exists as a recipe branch.
+  ///
+  /// If it does not, logs and falls back to [CipdVersion.defaultRecipe].
+  Future<CipdVersion> _getAndCheckRecipeVersion({
+    required RepositorySlug slug,
+    required String branch,
+  }) async {
+    if (slug != Config.flutterSlug) {
+      log.debug('Using default recipe: $slug is not flutter/flutter');
+      return _config.defaultRecipeBundleRef;
+    }
+    if (branch == Config.defaultBranch(Config.flutterSlug)) {
+      log.debug('Using default recipe: $branch is the default branch');
+      return _config.defaultRecipeBundleRef;
+    }
+    final proposed = CipdVersion(branch: branch);
+    final branches = await _gerritService.branches(
+      'flutter-review.googlesource.com',
+      'recipes',
+      filterRegex: 'flutter-.*|fuchsia.*',
+    );
+    if (branches.contains(proposed.version)) {
+      return proposed;
+    }
+    log.warn(
+      'Falling back to default recipe, could not find "${proposed.version}" '
+      'in $branches.',
+    );
+    return _config.defaultRecipeBundleRef;
+  }
+
   /// Schedules presubmit [targets] on BuildBucket for [pullRequest].
   ///
   /// [engineArtifacts] determines how framework tests download and use the Flutter engine by
@@ -205,33 +236,10 @@ class LuciBuildService {
     final batchRequestList = <bbv2.BatchRequest_Request>[];
     final commitSha = pullRequest.head!.sha!;
     final isFusion = pullRequest.base!.repo!.slug() == Config.flutterSlug;
-    final CipdVersion cipdVersion;
-    {
-      final baseRef = pullRequest.base!.ref!;
-
-      // If this isn't flutter/flutter *OR* it's flutter/flutter master, use the default CIPD recipe.
-      // We don't create CIPD recipes for other repositories (see https://github.com/flutter/flutter/issues/164592).
-      if (!isFusion ||
-          Config.defaultBranch(pullRequest.base!.repo!.slug()) == baseRef) {
-        cipdVersion = CipdVersion.defaultRecipe;
-      } else {
-        final proposedVersion = CipdVersion(branch: pullRequest.base!.ref!);
-        final branches = await _gerritService.branches(
-          'flutter-review.googlesource.com',
-          'recipes',
-          filterRegex: 'flutter-.*|fuchsia.*',
-        );
-        if (branches.contains(proposedVersion.version)) {
-          cipdVersion = proposedVersion;
-        } else {
-          log.warn(
-            'Falling back to default recipe, could not find '
-            '"${proposedVersion.version}" in $branches.',
-          );
-          cipdVersion = _config.defaultRecipeBundleRef;
-        }
-      }
-    }
+    final cipdVersion = await _getAndCheckRecipeVersion(
+      slug: pullRequest.base!.repo!.slug(),
+      branch: pullRequest.base!.ref!,
+    );
 
     final checkRuns = <github.CheckRun>[];
     for (var target in targets) {
@@ -844,8 +852,11 @@ class LuciBuildService {
     processedProperties['git_branch'] = commit.branch;
     processedProperties['git_repo'] = commit.slug.name;
 
-    final cipdExe = 'refs/heads/${commit.branch}';
-    processedProperties['exe_cipd_version'] = cipdExe;
+    final cipdVersion = await _getAndCheckRecipeVersion(
+      slug: commit.slug,
+      branch: commit.branch,
+    );
+    processedProperties['exe_cipd_version'] = cipdVersion.version;
 
     final isFusion = commit.slug == Config.flutterSlug;
     if (isFusion) {
@@ -862,15 +873,20 @@ class LuciBuildService {
           // Prod build bucket, built during the merge queue.
           'flutter_realm': '',
         });
+      } else if (commit.branch != Config.defaultBranch(Config.flutterSlug)) {
+        // Experimental branches do not have:
+        // - A merge queue that prebuilds binaries for the current SHA
+        // - A flutter_release_builder pre-step
+        //
+        // ... so, just re-use the binaries that were built in presubmit.
+        processedProperties['flutter_realm'] = 'flutter_archives_v2';
       }
     }
     final propertiesStruct = bbv2.Struct.create();
     propertiesStruct.mergeFromProto3Json(processedProperties);
 
     final requestedDimensions = target.getDimensions();
-
-    final executable = bbv2.Executable(cipdVersion: cipdExe);
-
+    final executable = bbv2.Executable(cipdVersion: cipdVersion.version);
     log.info(
       'Constructing the postsubmit schedule build request for ${target.name} on commit ${commit.sha}.',
     );

app_dart/lib/src/service/luci_build_service/cipd_version.dart

@@ -14,7 +14,7 @@ final class CipdVersion {
   /// The default recipe to use
   static const defaultRecipe = CipdVersion(branch: 'main');
 
-  /// The version string, in the format, `refs/head/{{branch}}`.
+  /// The version string, in the format, `refs/heads/{{branch}}`.
   final String version;
 
   @override

app_dart/test/service/luci_build_service/schedule_prod_builds_test.dart

@@ -43,17 +43,19 @@ void main() {
   late MockGithubChecksUtil mockGithubChecksUtil;
   late FakeFirestoreService firestore;
   late FakePubSub pubSub;
+  late FakeGerritService gerrit;
 
   setUp(() {
     mockBuildBucketClient = MockBuildBucketClient();
     mockGithubChecksUtil = MockGithubChecksUtil();
     firestore = FakeFirestoreService();
     pubSub = FakePubSub();
+    gerrit = FakeGerritService();
 
     luci = LuciBuildService(
       cache: CacheService(inMemory: true),
       config: FakeConfig(),
-      gerritService: FakeGerritService(),
+      gerritService: gerrit,
       buildBucketClient: mockBuildBucketClient,
       githubChecksUtil: mockGithubChecksUtil,
       pubsub: pubSub,
@@ -248,7 +250,11 @@ void main() {
   });
 
   test('schedules a post-submit build inside of flutter/flutter', () async {
-    final commit = generateFirestoreCommit(1, branch: 'main', repo: 'flutter');
+    final commit = generateFirestoreCommit(
+      1,
+      branch: 'master',
+      repo: 'flutter',
+    );
 
     when(mockBuildBucketClient.listBuilders(any)).thenAnswer((_) async {
       return bbv2.ListBuildersResponse(
@@ -318,7 +324,7 @@ void main() {
     expect(scheduleBuild.properties.fields, {
       'dependencies': bbv2.Value(listValue: bbv2.ListValue()),
       'bringup': bbv2.Value(boolValue: false),
-      'git_branch': bbv2.Value(stringValue: 'main'),
+      'git_branch': bbv2.Value(stringValue: 'master'),
       'git_repo': bbv2.Value(stringValue: 'flutter'),
       'exe_cipd_version': bbv2.Value(stringValue: 'refs/heads/main'),
       'os': bbv2.Value(stringValue: 'debian-10.12'),
@@ -343,7 +349,11 @@ void main() {
   });
 
   test('schedules a post-submit build that is a > attempt #1', () async {
-    final commit = generateFirestoreCommit(1, branch: 'main', repo: 'flutter');
+    final commit = generateFirestoreCommit(
+      1,
+      branch: 'master',
+      repo: 'flutter',
+    );
 
     when(mockBuildBucketClient.listBuilders(any)).thenAnswer((_) async {
       return bbv2.ListBuildersResponse(
@@ -413,7 +423,7 @@ void main() {
     expect(scheduleBuild.properties.fields, {
       'dependencies': bbv2.Value(listValue: bbv2.ListValue()),
       'bringup': bbv2.Value(boolValue: false),
-      'git_branch': bbv2.Value(stringValue: 'main'),
+      'git_branch': bbv2.Value(stringValue: 'master'),
       'git_repo': bbv2.Value(stringValue: 'flutter'),
       'exe_cipd_version': bbv2.Value(stringValue: 'refs/heads/main'),
       'os': bbv2.Value(stringValue: 'debian-10.12'),
@@ -459,6 +469,8 @@ void main() {
       );
     });
 
+    gerrit.branchesValue = ['refs/heads/flutter-0.42-candidate.0'];
+
     await expectLater(
       luci.schedulePostsubmitBuilds(
         commit: commit.toRef(),
@@ -541,6 +553,119 @@ void main() {
     );
   });
 
+  // Regression test for https://github.com/flutter/flutter/issues/168738.
+  test('schedules a post-submit experimental branch build', () async {
+    final commit = generateFirestoreCommit(
+      1,
+      // Notably not a release-candidate branch and not "master".
+      branch: 'ios-experimental',
+      repo: 'flutter',
+    );
+
+    when(mockBuildBucketClient.listBuilders(any)).thenAnswer((_) async {
+      return bbv2.ListBuildersResponse(
+        builders: [
+          bbv2.BuilderItem(
+            id: bbv2.BuilderID(
+              bucket: 'prod',
+              project: 'flutter',
+              builder: 'Linux 1',
+            ),
+          ),
+        ],
+      );
+    });
+
+    // Notably we don't require a recipe branch, and fallback to main.
+    gerrit.branchesValue = [];
+
+    await expectLater(
+      luci.schedulePostsubmitBuilds(
+        commit: commit.toRef(),
+        toBeScheduled: [
+          PendingTask(
+            target: generateTarget(
+              1,
+              properties: {
+                'recipe': 'devicelab/devicelab',
+                'os': 'debian-10.12',
+              },
+              slug: Config.flutterSlug,
+            ),
+            taskName: generateFirestoreTask(1, commitSha: commit.sha).taskName,
+            priority: LuciBuildService.kDefaultPriority,
+            currentAttempt: 1,
+          ),
+        ],
+      ),
+      completion(isEmpty),
+    );
+
+    final bbv2.ScheduleBuildRequest scheduleBuild;
+    {
+      final batchRequest = bbv2.BatchRequest().createEmptyInstance();
+      batchRequest.mergeFromProto3Json(pubSub.messages.single);
+
+      expect(batchRequest.requests, hasLength(1));
+      scheduleBuild = batchRequest.requests.single.scheduleBuild;
+    }
+
+    expect(
+      scheduleBuild.builder,
+      isA<bbv2.BuilderID>()
+          .having((b) => b.bucket, 'bucket', 'prod')
+          .having((b) => b.builder, 'builder', 'Linux 1'),
+    );
+
+    expect(
+      scheduleBuild.notify.pubsubTopic,
+      'projects/flutter-dashboard/topics/build-bucket-postsubmit',
+    );
+
+    expect(
+      PostsubmitUserData.fromBytes(scheduleBuild.notify.userData),
+      PostsubmitUserData(
+        taskId: fs.TaskId.parse('1_task1_1'),
+        checkRunId: null /* Uses batch backfiller */,
+      ),
+    );
+
+    expect(
+      scheduleBuild.properties.fields,
+      isNot(contains('flutter_prebuilt_engine_version')),
+      reason: 'Experimental branches use default engine SHA resolution',
+    );
+
+    expect(scheduleBuild.properties.fields, {
+      'dependencies': bbv2.Value(listValue: bbv2.ListValue()),
+      'bringup': bbv2.Value(boolValue: false),
+      'git_branch': bbv2.Value(stringValue: 'ios-experimental'),
+      'git_repo': bbv2.Value(stringValue: 'flutter'),
+      'exe_cipd_version': bbv2.Value(stringValue: 'refs/heads/main'),
+      'os': bbv2.Value(stringValue: 'debian-10.12'),
+      'recipe': bbv2.Value(stringValue: 'devicelab/devicelab'),
+      'is_fusion': bbv2.Value(stringValue: 'true'),
+
+      // Experimental branches use the presubmit bucket for engine builds.
+      'flutter_realm': bbv2.Value(stringValue: 'flutter_archives_v2'),
+    });
+
+    expect(scheduleBuild.dimensions, [
+      isA<bbv2.RequestedDimension>()
+          .having((d) => d.key, 'key', 'os')
+          .having((d) => d.value, 'value', 'debian-10.12'),
+    ]);
+
+    verifyNever(
+      mockGithubChecksUtil.createCheckRun(
+        any,
+        Config.packagesSlug,
+        any,
+        'Linux 1',
+      ),
+    );
+  });
+
   test('does not run a non-existent builder', () async {
     final commit = generateFirestoreCommit(1, branch: 'main', repo: 'flutter');