diff --git a/app_dart/lib/server.dart b/app_dart/lib/server.dart index a4cb60e7b4..839d7aa169 100644 --- a/app_dart/lib/server.dart +++ b/app_dart/lib/server.dart @@ -131,14 +131,6 @@ Server createServer({ ciYamlFetcher: ciYamlFetcher, luciBuildService: luciBuildService, firestore: firestore, - branchService: branchService, - ), - '/api/v2/scheduler/batch-backfiller': BatchBackfiller( - config: config, - ciYamlFetcher: ciYamlFetcher, - luciBuildService: luciBuildService, - firestore: firestore, - branchService: branchService, ), '/api/v2/scheduler/batch-request-subscription': SchedulerRequestSubscription( diff --git a/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart b/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart index ac190523f7..65dbcceefc 100644 --- a/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart +++ b/app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart @@ -4,7 +4,7 @@ import 'dart:math'; -import 'package:cocoon_common/is_release_branch.dart'; +import 'package:cocoon_common/core_extensions.dart'; import 'package:cocoon_common/task_status.dart'; import 'package:cocoon_server/logging.dart'; import 'package:github/github.dart'; @@ -32,69 +32,60 @@ final class BatchBackfiller extends RequestHandler { required CiYamlFetcher ciYamlFetcher, required LuciBuildService luciBuildService, required FirestoreService firestore, - required BranchService branchService, BackfillStrategy backfillerStrategy = const DefaultBackfillStrategy(), + @visibleForTesting DateTime Function() now = DateTime.now, }) : _ciYamlFetcher = ciYamlFetcher, _luciBuildService = luciBuildService, _backfillerStrategy = backfillerStrategy, _firestore = firestore, - _branchService = branchService; + _now = now; final LuciBuildService _luciBuildService; final CiYamlFetcher _ciYamlFetcher; final BackfillStrategy _backfillerStrategy; final FirestoreService _firestore; - final BranchService _branchService; + final DateTime Function() _now; @override Future get(Request request) async { - await _backfillReleaseBranch(Config.flutterSlug); - await Future.forEach(config.supportedRepos, _backfillDefaultBranch); - await _backfillExperimentalBranch(Config.flutterSlug); - return Response.emptyOk; - } - - // TODO(matanlurey): Remove or make a formal supported feature. - // - // Hardcodes running a branch named `ios-experimental`, which is being tried - // by the iOS team to do MacOS 15.5 validation. It will appear similar to - // any other branch, but use lower-priority scheduling. - // - // See https://github.com/flutter/flutter/issues/168738. - Future _backfillExperimentalBranch(RepositorySlug slug) async { - log.debug('Running experimental branch backfiller for "$slug"'); - final fsGrid = await _firestore.queryRecentCommitsAndTasks( - slug, - commitLimit: config.flags.backfillerCommitLimit, - branch: 'ios-experimental', - ); - await _doBackfillFrom(slug, fsGrid, forceLowPriority: true); - } + log.debug('Finding all branches eligible for backfilling'); + final branches = <(RepositorySlug, String)>{}; + for (final repo in config.supportedRepos) { + // Always include the default branch. + final defaultBranch = Config.defaultBranch(repo); + branches.add((repo, defaultBranch)); - Future _backfillReleaseBranch(RepositorySlug slug) async { - log.debug('Running release branch backfiller for "$slug"'); - final branches = await _branchService.getReleaseBranches(slug: slug); - for (final branch in branches) { - if (!isReleaseCandidateBranch(branchName: branch.reference)) { - continue; - } - final fsGrid = await _firestore.queryRecentCommitsAndTasks( - slug, - commitLimit: config.flags.backfillerCommitLimit, - branch: branch.reference, + // Look for any branch that has received a commit in the last 7 days. + final commits = await _firestore.queryRecentCommits( + slug: repo, + limit: null, + created: TimeRange.after(_now().subtract(const Duration(days: 7))), ); - await _doBackfillFrom(slug, fsGrid); + for (final commit in commits) { + branches.add((repo, commit.branch)); + } } - } - Future _backfillDefaultBranch(RepositorySlug slug) async { - log.debug('Running default branch backfiller for "$slug"'); - final fsGrid = await _firestore.queryRecentCommitsAndTasks( - slug, - commitLimit: config.flags.backfillerCommitLimit, - branch: Config.defaultBranch(slug), + log.debug( + 'Found ${branches.length} branches eligible for backfilling:\n' + '${branches.join('\n')}', ); - return await _doBackfillFrom(slug, fsGrid); + + await Future.forEach(branches, (branch) async { + final (slug, branchName) = branch; + log.debug('Backfilling ${slug.fullName} -> $branchName'); + await _doBackfillFrom( + slug, + await _firestore.queryRecentCommitsAndTasks( + slug, + commitLimit: config.flags.backfillerCommitLimit, + branch: branchName, + ), + ); + }); + return Response.json({ + 'branches': [...branches.map((e) => '$e')], + }); } Future _doBackfillFrom( @@ -103,7 +94,7 @@ final class BatchBackfiller extends RequestHandler { bool forceLowPriority = false, }) async { if (fsGrid.isEmpty) { - log.warn('No commits to backfill'); + log.info('No commits to backfill'); return; } diff --git a/app_dart/lib/src/service/firestore.dart b/app_dart/lib/src/service/firestore.dart index b2b11ec902..38e605e5d6 100644 --- a/app_dart/lib/src/service/firestore.dart +++ b/app_dart/lib/src/service/firestore.dart @@ -68,7 +68,7 @@ mixin FirestoreQueries { Transaction? transaction, }); - static Map _filterByTimeRAnge( + static Map _filterByTimeRAnge( String fieldName, TimeRange range, ) { @@ -76,31 +76,29 @@ mixin FirestoreQueries { IndefiniteTimeRange() => const {}, SpecificTimeRange(:final start, :final end, :final exclusive) => { if (start != null) - '$fieldName ${exclusive ? '>' : '>='}': - '${start.millisecondsSinceEpoch}', + '$fieldName ${exclusive ? '>' : '>='}': start.millisecondsSinceEpoch, if (end != null) - '$fieldName ${exclusive ? '<' : '<='}': - '${end.millisecondsSinceEpoch}', + '$fieldName ${exclusive ? '<' : '<='}': end.millisecondsSinceEpoch, }, }; } /// Queries for recent commits. /// - /// The [limit] argument specifies the maximum number of commits to retrieve. + /// If [limit] is `null`, all commits are returned. /// - /// The returned commits will be ordered by most recent [Commit.timestamp]. + /// The returned commits will be ordered by most recent + /// [Commit.createTimestamp]. Future> queryRecentCommits({ - int limit = 100, + required int? limit, + required RepositorySlug slug, TimeRange? created, String? branch, - required RepositorySlug slug, }) async { - branch ??= Config.defaultBranch(slug); created ??= TimeRange.indefinite; final filterMap = { - '${Commit.fieldBranch} =': branch, '${Commit.fieldRepositoryPath} =': slug.fullName, + if (branch != null) '${Commit.fieldBranch} =': branch, ..._filterByTimeRAnge(Commit.fieldCreateTimestamp, created), }; final orderMap = { diff --git a/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart b/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart index 81f24ba505..2db02c1f0c 100644 --- a/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart +++ b/app_dart/test/request_handlers/scheduler/batch_backfiller_test.dart @@ -2,7 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'package:cocoon_common/rpc_model.dart' as rpc; import 'package:cocoon_common/task_status.dart'; import 'package:cocoon_server_test/test_logging.dart'; import 'package:cocoon_service/src/model/commit_ref.dart'; @@ -13,7 +12,6 @@ import 'package:cocoon_service/src/service/config.dart'; import 'package:cocoon_service/src/service/luci_build_service.dart'; import 'package:cocoon_service/src/service/luci_build_service/pending_task.dart'; import 'package:collection/collection.dart'; -import 'package:github/github.dart'; import 'package:mockito/mockito.dart'; import 'package:test/test.dart'; @@ -22,7 +20,6 @@ import '../../src/request_handling/request_handler_tester.dart'; import '../../src/service/fake_ci_yaml_fetcher.dart'; import '../../src/service/fake_firestore_service.dart'; import '../../src/utilities/entity_generators.dart'; -import '../../src/utilities/mocks.mocks.dart'; void main() { useTestLoggerPerTest(); @@ -31,14 +28,12 @@ void main() { late BatchBackfiller handler; // Dependencies. + final fakeNow = DateTime(2025, 1, 1); late FakeConfig config; late FakeFirestoreService firestore; late FakeCiYamlFetcher ciYamlFetcher; late _FakeLuciBuildService fakeLuciBuildService; - // Used to implement BranchService.getBranches. - late List? branchesForRepository; - // Fixture. late RequestHandlerTester tester; @@ -52,28 +47,13 @@ void main() { ciYamlFetcher = FakeCiYamlFetcher(); fakeLuciBuildService = _FakeLuciBuildService(); - final branchService = MockBranchService(); - branchesForRepository = []; - when(branchService.getReleaseBranches(slug: anyNamed('slug'))).thenAnswer(( - i, - ) async { - final slug = i.namedArguments[#slug] as RepositorySlug; - return branchesForRepository ?? - [ - rpc.Branch( - channel: Config.defaultBranch(slug), - reference: Config.defaultBranch(slug), - ), - ]; - }); - handler = BatchBackfiller( config: config, ciYamlFetcher: ciYamlFetcher, luciBuildService: fakeLuciBuildService, backfillerStrategy: const _NaiveBackfillStrategy(), firestore: firestore, - branchService: branchService, + now: () => fakeNow, ); tester = RequestHandlerTester(); @@ -119,6 +99,7 @@ void main() { ''' enabled_branches: - master + - $branch targets: - name: Linux 0 @@ -133,13 +114,14 @@ void main() { engine: ''' enabled_branches: - master + - $branch targets: - name: Engine 0 ''', ); - var date = DateTime(2025, 1, 1); + var date = fakeNow; for (final (i, row) in statuses.indexed) { final fsCommit = generateFirestoreCommit( i, @@ -286,11 +268,6 @@ void main() { }); test('backfills release candidate branches', () async { - branchesForRepository = [ - rpc.Branch(channel: 'master', reference: 'master'), - rpc.Branch(channel: 'beta', reference: 'flutter-3.32-candidate.0'), - ]; - // dart format off await fillStorageAndSetCiYaml([ [$N, $I, $F, $S], @@ -356,10 +333,6 @@ void main() { // https://github.com/flutter/flutter/issues/168738 test('schedules low-priority targets for "ios-experimental"', () async { - branchesForRepository = [ - // Intentionally left blank. - ]; - // dart format off await fillStorageAndSetCiYaml([ [$N, $I, $F, $S], diff --git a/cron.yaml b/cron.yaml index 6ef4fb5148..90636a24eb 100644 --- a/cron.yaml +++ b/cron.yaml @@ -12,7 +12,7 @@ cron: schedule: every 1 hours - description: backfills builds - url: /api/v2/scheduler/batch-backfiller + url: /api/scheduler/batch-backfiller schedule: every 5 minutes - description: sends build status to GitHub to annotate flutter PRs and commits