Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 0 additions & 8 deletions app_dart/lib/server.dart
Original file line number Diff line number Diff line change
Expand Up @@ -131,14 +131,6 @@ Server createServer({
ciYamlFetcher: ciYamlFetcher,
luciBuildService: luciBuildService,
firestore: firestore,
branchService: branchService,
),
'/api/v2/scheduler/batch-backfiller': BatchBackfiller(
config: config,
ciYamlFetcher: ciYamlFetcher,
luciBuildService: luciBuildService,
firestore: firestore,
branchService: branchService,
),
'/api/v2/scheduler/batch-request-subscription':
SchedulerRequestSubscription(
Expand Down
85 changes: 38 additions & 47 deletions app_dart/lib/src/request_handlers/scheduler/batch_backfiller.dart
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import 'dart:math';

import 'package:cocoon_common/is_release_branch.dart';
import 'package:cocoon_common/core_extensions.dart';
import 'package:cocoon_common/task_status.dart';
import 'package:cocoon_server/logging.dart';
import 'package:github/github.dart';
Expand Down Expand Up @@ -32,69 +32,60 @@ final class BatchBackfiller extends RequestHandler {
required CiYamlFetcher ciYamlFetcher,
required LuciBuildService luciBuildService,
required FirestoreService firestore,
required BranchService branchService,
BackfillStrategy backfillerStrategy = const DefaultBackfillStrategy(),
@visibleForTesting DateTime Function() now = DateTime.now,
}) : _ciYamlFetcher = ciYamlFetcher,
_luciBuildService = luciBuildService,
_backfillerStrategy = backfillerStrategy,
_firestore = firestore,
_branchService = branchService;
_now = now;

final LuciBuildService _luciBuildService;
final CiYamlFetcher _ciYamlFetcher;
final BackfillStrategy _backfillerStrategy;
final FirestoreService _firestore;
final BranchService _branchService;
final DateTime Function() _now;

@override
Future<Response> get(Request request) async {
await _backfillReleaseBranch(Config.flutterSlug);
await Future.forEach(config.supportedRepos, _backfillDefaultBranch);
await _backfillExperimentalBranch(Config.flutterSlug);
return Response.emptyOk;
}

// TODO(matanlurey): Remove or make a formal supported feature.
//
// Hardcodes running a branch named `ios-experimental`, which is being tried
// by the iOS team to do MacOS 15.5 validation. It will appear similar to
// any other branch, but use lower-priority scheduling.
//
// See https://github.com/flutter/flutter/issues/168738.
Future<void> _backfillExperimentalBranch(RepositorySlug slug) async {
log.debug('Running experimental branch backfiller for "$slug"');
final fsGrid = await _firestore.queryRecentCommitsAndTasks(
slug,
commitLimit: config.flags.backfillerCommitLimit,
branch: 'ios-experimental',
);
await _doBackfillFrom(slug, fsGrid, forceLowPriority: true);
}
log.debug('Finding all branches eligible for backfilling');
final branches = <(RepositorySlug, String)>{};
for (final repo in config.supportedRepos) {
// Always include the default branch.
final defaultBranch = Config.defaultBranch(repo);
branches.add((repo, defaultBranch));

Future<void> _backfillReleaseBranch(RepositorySlug slug) async {
log.debug('Running release branch backfiller for "$slug"');
final branches = await _branchService.getReleaseBranches(slug: slug);
for (final branch in branches) {
if (!isReleaseCandidateBranch(branchName: branch.reference)) {
continue;
}
final fsGrid = await _firestore.queryRecentCommitsAndTasks(
slug,
commitLimit: config.flags.backfillerCommitLimit,
branch: branch.reference,
// Look for any branch that has received a commit in the last 7 days.
final commits = await _firestore.queryRecentCommits(
slug: repo,
limit: null,
created: TimeRange.after(_now().subtract(const Duration(days: 7))),
);
await _doBackfillFrom(slug, fsGrid);
for (final commit in commits) {
branches.add((repo, commit.branch));
}
}
}

Future<void> _backfillDefaultBranch(RepositorySlug slug) async {
log.debug('Running default branch backfiller for "$slug"');
final fsGrid = await _firestore.queryRecentCommitsAndTasks(
slug,
commitLimit: config.flags.backfillerCommitLimit,
branch: Config.defaultBranch(slug),
log.debug(
'Found ${branches.length} branches eligible for backfilling:\n'
'${branches.join('\n')}',
);
return await _doBackfillFrom(slug, fsGrid);

await Future.forEach(branches, (branch) async {
final (slug, branchName) = branch;
log.debug('Backfilling ${slug.fullName} -> $branchName');
await _doBackfillFrom(
slug,
await _firestore.queryRecentCommitsAndTasks(
slug,
commitLimit: config.flags.backfillerCommitLimit,
branch: branchName,
),
);
});
return Response.json({
'branches': [...branches.map((e) => '$e')],
});
}

Future<void> _doBackfillFrom(
Expand All @@ -103,7 +94,7 @@ final class BatchBackfiller extends RequestHandler {
bool forceLowPriority = false,
}) async {
if (fsGrid.isEmpty) {
log.warn('No commits to backfill');
log.info('No commits to backfill');
return;
}

Expand Down
20 changes: 9 additions & 11 deletions app_dart/lib/src/service/firestore.dart
Original file line number Diff line number Diff line change
Expand Up @@ -68,39 +68,37 @@ mixin FirestoreQueries {
Transaction? transaction,
});

static Map<String, String> _filterByTimeRAnge(
static Map<String, Object> _filterByTimeRAnge(
String fieldName,
TimeRange range,
) {
return switch (range) {
IndefiniteTimeRange() => const {},
SpecificTimeRange(:final start, :final end, :final exclusive) => {
if (start != null)
'$fieldName ${exclusive ? '>' : '>='}':
'${start.millisecondsSinceEpoch}',
'$fieldName ${exclusive ? '>' : '>='}': start.millisecondsSinceEpoch,
if (end != null)
'$fieldName ${exclusive ? '<' : '<='}':
'${end.millisecondsSinceEpoch}',
'$fieldName ${exclusive ? '<' : '<='}': end.millisecondsSinceEpoch,
},
};
}

/// Queries for recent commits.
///
/// The [limit] argument specifies the maximum number of commits to retrieve.
/// If [limit] is `null`, all commits are returned.
///
/// The returned commits will be ordered by most recent [Commit.timestamp].
/// The returned commits will be ordered by most recent
/// [Commit.createTimestamp].
Future<List<Commit>> queryRecentCommits({
int limit = 100,
required int? limit,
required RepositorySlug slug,
TimeRange? created,
String? branch,
required RepositorySlug slug,
}) async {
branch ??= Config.defaultBranch(slug);
created ??= TimeRange.indefinite;
final filterMap = <String, Object>{
'${Commit.fieldBranch} =': branch,
'${Commit.fieldRepositoryPath} =': slug.fullName,
if (branch != null) '${Commit.fieldBranch} =': branch,
..._filterByTimeRAnge(Commit.fieldCreateTimestamp, created),
};
final orderMap = <String, String>{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'package:cocoon_common/rpc_model.dart' as rpc;
import 'package:cocoon_common/task_status.dart';
import 'package:cocoon_server_test/test_logging.dart';
import 'package:cocoon_service/src/model/commit_ref.dart';
Expand All @@ -13,7 +12,6 @@ import 'package:cocoon_service/src/service/config.dart';
import 'package:cocoon_service/src/service/luci_build_service.dart';
import 'package:cocoon_service/src/service/luci_build_service/pending_task.dart';
import 'package:collection/collection.dart';
import 'package:github/github.dart';
import 'package:mockito/mockito.dart';
import 'package:test/test.dart';

Expand All @@ -22,7 +20,6 @@ import '../../src/request_handling/request_handler_tester.dart';
import '../../src/service/fake_ci_yaml_fetcher.dart';
import '../../src/service/fake_firestore_service.dart';
import '../../src/utilities/entity_generators.dart';
import '../../src/utilities/mocks.mocks.dart';

void main() {
useTestLoggerPerTest();
Expand All @@ -31,14 +28,12 @@ void main() {
late BatchBackfiller handler;

// Dependencies.
final fakeNow = DateTime(2025, 1, 1);
late FakeConfig config;
late FakeFirestoreService firestore;
late FakeCiYamlFetcher ciYamlFetcher;
late _FakeLuciBuildService fakeLuciBuildService;

// Used to implement BranchService.getBranches.
late List<rpc.Branch>? branchesForRepository;

// Fixture.
late RequestHandlerTester tester;

Expand All @@ -52,28 +47,13 @@ void main() {
ciYamlFetcher = FakeCiYamlFetcher();
fakeLuciBuildService = _FakeLuciBuildService();

final branchService = MockBranchService();
branchesForRepository = [];
when(branchService.getReleaseBranches(slug: anyNamed('slug'))).thenAnswer((
i,
) async {
final slug = i.namedArguments[#slug] as RepositorySlug;
return branchesForRepository ??
[
rpc.Branch(
channel: Config.defaultBranch(slug),
reference: Config.defaultBranch(slug),
),
];
});

handler = BatchBackfiller(
config: config,
ciYamlFetcher: ciYamlFetcher,
luciBuildService: fakeLuciBuildService,
backfillerStrategy: const _NaiveBackfillStrategy(),
firestore: firestore,
branchService: branchService,
now: () => fakeNow,
);

tester = RequestHandlerTester();
Expand Down Expand Up @@ -119,6 +99,7 @@ void main() {
'''
enabled_branches:
- master
- $branch

targets:
- name: Linux 0
Expand All @@ -133,13 +114,14 @@ void main() {
engine: '''
enabled_branches:
- master
- $branch

targets:
- name: Engine 0
''',
);

var date = DateTime(2025, 1, 1);
var date = fakeNow;
for (final (i, row) in statuses.indexed) {
final fsCommit = generateFirestoreCommit(
i,
Expand Down Expand Up @@ -286,11 +268,6 @@ void main() {
});

test('backfills release candidate branches', () async {
branchesForRepository = [
rpc.Branch(channel: 'master', reference: 'master'),
rpc.Branch(channel: 'beta', reference: 'flutter-3.32-candidate.0'),
];

// dart format off
await fillStorageAndSetCiYaml([
[$N, $I, $F, $S],
Expand Down Expand Up @@ -356,10 +333,6 @@ void main() {

// https://github.com/flutter/flutter/issues/168738
test('schedules low-priority targets for "ios-experimental"', () async {
branchesForRepository = [
// Intentionally left blank.
];

// dart format off
await fillStorageAndSetCiYaml([
[$N, $I, $F, $S],
Expand Down
2 changes: 1 addition & 1 deletion cron.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ cron:
schedule: every 1 hours

- description: backfills builds
url: /api/v2/scheduler/batch-backfiller
url: /api/scheduler/batch-backfiller
schedule: every 5 minutes

- description: sends build status to GitHub to annotate flutter PRs and commits
Expand Down