Skip to content

Swarming: Adds CountTasks endpoint & Refactors swarming Prpc request #5277

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
IvanBM18 wants to merge 10 commits into
base: master
Choose a base branch
from
Draft

Swarming: Adds CountTasks endpoint & Refactors swarming Prpc request #5277

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
fc94459
Add TasksCountRequest and TasksCount to swarming.proto
IvanBM18 May 14, 2026
56be2f4
Implement CountTasks and refactor Swarming API
IvanBM18 May 14, 2026
d78a8eb
Removes push_swarming_task method & reuses get_swarming_config at the…
IvanBM18 May 15, 2026
0ede2ed
Adds api as a SwarmingService property
IvanBM18 May 15, 2026
57886fc
Fixes Broken UT
IvanBM18 May 15, 2026
5a430c3
Adds endpoint constants
IvanBM18 May 18, 2026
6b5715d
Fixes linter issues
IvanBM18 May 18, 2026
5299ee9
Merge branch 'master' into feature/swarming/count_task_requests
IvanBM18 May 18, 2026
e55457d
Apply suggestion from @letitz
IvanBM18 May 19, 2026
0bf3497
Propagates errors from api to service
IvanBM18 May 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 75 additions & 3 deletions src/clusterfuzz/_internal/protos/swarming.proto
View file
Open in desktop
Comment thread
IvanBM18 marked this conversation as resolved.
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2024 Google LLC
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -12,12 +12,71 @@
// See the License for the specific language governing permissions and
// limitations under the License.

// This file is based on https://source.chromium.org/chromium/infra/infra/+/main:luci/appengine/swarming/proto/api_v2/swarming.proto
// This file is based on https://source.chromium.org/chromium/infra/infra_superproject/+/main:infra/luci/appengine/swarming/proto/api_v2/swarming.proto
// This includes necessary messages to construct a NewTaskRequest
syntax = "proto3";

package swarming.v2;

import "google/protobuf/timestamp.proto";

// Enums

// Use one of the values in this enum to query for tasks in one of the
// specified state.
//
// Use 'ALL' to not use any filtering based on task state.
//
// As an example, this enum enables querying for all tasks with state COMPLETED
// but non-zero exit code via COMPLETED_FAILURE.
//
// Do not confuse StateQuery and TaskState. StateQuery is to query tasks
// via the API. TaskState is the current task state.
enum StateQuery {
// Query for all tasks currently TaskState.PENDING.
QUERY_PENDING = 0;
// Query for all tasks currently TaskState.RUNNING. This includes tasks
// currently in the overhead phase; mapping input files or archiving outputs
// back to the server.
QUERY_RUNNING = 1;
// Query for all tasks currently TaskState.PENDING or TaskState.RUNNING. This
// is the query for the 'active' tasks.
QUERY_PENDING_RUNNING = 2;
// Query for all tasks that completed normally as TaskState.COMPLETED,
// independent of the process exit code.
QUERY_COMPLETED = 3;
// Query for all tasks that completed normally as TaskState.COMPLETED and that
// had exit code 0.
QUERY_COMPLETED_SUCCESS = 4;
// Query for all tasks that completed normally as TaskState.COMPLETED and that
// had exit code not 0.
QUERY_COMPLETED_FAILURE = 5;
// Query for all tasks that are TaskState.EXPIRED.
QUERY_EXPIRED = 6;
// Query for all tasks that are TaskState.TIMED_OUT.
QUERY_TIMED_OUT = 7;
// Query for all tasks that are TaskState.BOT_DIED.
QUERY_BOT_DIED = 8;
// Query for all tasks that are TaskState.CANCELED.
QUERY_CANCELED = 9;
// Query for all tasks, independent of the task state.
//
// In hindsight, this constant should have been the value 0. Sorry, the
// original author was young and foolish.
QUERY_ALL = 10;
// Query for all tasks that are TaskState.COMPLETED but that actually didn't
// run due to TaskProperties.idempotent being True *and* that a previous task
// with the exact same TaskProperties had successfully run before, aka
// COMPLETED_SUCCESS.
QUERY_DEDUPED = 11;
// Query for all tasks that are TaskState.KILLED.
QUERY_KILLED = 12;
// Query for all tasks that are TaskState.NO_RESOURCE.
QUERY_NO_RESOURCE = 13;
// Query for all tasks that are TaskState.CLIENT_ERROR.
QUERY_CLIENT_ERROR = 14;
}

// Messages

// Represents a mapping of string to a string.
Expand Down Expand Up @@ -345,4 +404,17 @@ message NewTaskRequest {
// Task realm.
// See api/swarming.proto for more details.
string realm = 18;
}
}

message TasksCountRequest {
google.protobuf.Timestamp start = 1;
google.protobuf.Timestamp end = 2;
StateQuery state = 3;
repeated string tags = 4;
}

// Returns the count, as requested.
message TasksCount {
int32 count = 1;
google.protobuf.Timestamp now = 2;
}
65 changes: 36 additions & 29 deletions src/clusterfuzz/_internal/protos/swarming_pb2.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,41 +26,48 @@
_sym_db = _symbol_database.Default()


from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2


DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n+clusterfuzz/_internal/protos/swarming.proto\x12\x0bswarming.v2\"(\n\nStringPair\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\",\n\x0eStringListPair\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x03(\t\"*\n\x06\x44igest\x12\x0c\n\x04hash\x18\x01 \x01(\t\x12\x12\n\nsize_bytes\x18\x02 \x01(\x03\"I\n\x0c\x43\x41SReference\x12\x14\n\x0c\x63\x61s_instance\x18\x01 \x01(\t\x12#\n\x06\x64igest\x18\x02 \x01(\x0b\x32\x13.swarming.v2.Digest\"B\n\x0b\x43ipdPackage\x12\x14\n\x0cpackage_name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\"y\n\tCipdInput\x12\x0e\n\x06server\x18\x01 \x01(\t\x12\x30\n\x0e\x63lient_package\x18\x02 \x01(\x0b\x32\x18.swarming.v2.CipdPackage\x12*\n\x08packages\x18\x03 \x03(\x0b\x32\x18.swarming.v2.CipdPackage\"(\n\nCacheEntry\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\"\xf2\x01\n\x0b\x43ontainment\x12\x16\n\x0elower_priority\x18\x01 \x01(\x08\x12\x42\n\x10\x63ontainment_type\x18\x02 \x01(\x0e\x32(.swarming.v2.Containment.ContainmentType\x12\x17\n\x0flimit_processes\x18\x03 \x01(\x03\x12$\n\x1climit_total_committed_memory\x18\x04 \x01(\x03\"H\n\x0f\x43ontainmentType\x12\x11\n\rNOT_SPECIFIED\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\x08\n\x04\x41UTO\x10\x02\x12\x0e\n\nJOB_OBJECT\x10\x03\"\x83\x04\n\x0eTaskProperties\x12\'\n\x06\x63\x61\x63hes\x18\x01 \x03(\x0b\x32\x17.swarming.v2.CacheEntry\x12*\n\ncipd_input\x18\x02 \x01(\x0b\x32\x16.swarming.v2.CipdInput\x12\x0f\n\x07\x63ommand\x18\x03 \x03(\t\x12\x14\n\x0crelative_cwd\x18\x04 \x01(\t\x12+\n\ndimensions\x18\x05 \x03(\x0b\x32\x17.swarming.v2.StringPair\x12$\n\x03\x65nv\x18\x06 \x03(\x0b\x32\x17.swarming.v2.StringPair\x12\x31\n\x0c\x65nv_prefixes\x18\x07 \x03(\x0b\x32\x1b.swarming.v2.StringListPair\x12\x1e\n\x16\x65xecution_timeout_secs\x18\x08 \x01(\x05\x12\x19\n\x11grace_period_secs\x18\t \x01(\x05\x12\x12\n\nidempotent\x18\n \x01(\x08\x12\x31\n\x0e\x63\x61s_input_root\x18\x0b \x01(\x0b\x32\x19.swarming.v2.CASReference\x12\x17\n\x0fio_timeout_secs\x18\x0c \x01(\x05\x12\x0f\n\x07outputs\x18\r \x03(\t\x12\x14\n\x0csecret_bytes\x18\x0e \x01(\x0c\x12-\n\x0b\x63ontainment\x18\x0f \x01(\x0b\x32\x18.swarming.v2.Containment\"p\n\tTaskSlice\x12/\n\nproperties\x18\x01 \x01(\x0b\x32\x1b.swarming.v2.TaskProperties\x12\x17\n\x0f\x65xpiration_secs\x18\x02 \x01(\x05\x12\x19\n\x11wait_for_capacity\x18\x03 \x01(\x08\"\x1d\n\x0bResultDBCfg\x12\x0e\n\x06\x65nable\x18\x01 \x01(\x08\"\xe8\x04\n\x0eNewTaskRequest\x12\x17\n\x0f\x65xpiration_secs\x18\x01 \x01(\x05\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x16\n\x0eparent_task_id\x18\x03 \x01(\t\x12\x10\n\x08priority\x18\x04 \x01(\x05\x12/\n\nproperties\x18\x05 \x01(\x0b\x32\x1b.swarming.v2.TaskProperties\x12+\n\x0btask_slices\x18\x06 \x03(\x0b\x32\x16.swarming.v2.TaskSlice\x12\x0c\n\x04tags\x18\x07 \x03(\t\x12\x0c\n\x04user\x18\x08 \x01(\t\x12\x17\n\x0fservice_account\x18\t \x01(\t\x12\x14\n\x0cpubsub_topic\x18\n \x01(\t\x12\x19\n\x11pubsub_auth_token\x18\x0b \x01(\t\x12\x17\n\x0fpubsub_userdata\x18\x0c \x01(\t\x12\x15\n\revaluate_only\x18\r \x01(\x08\x12M\n\x12pool_task_template\x18\x0e \x01(\x0e\x32\x31.swarming.v2.NewTaskRequest.PoolTaskTemplateField\x12\x1f\n\x17\x62ot_ping_tolerance_secs\x18\x0f \x01(\x05\x12\x14\n\x0crequest_uuid\x18\x10 \x01(\t\x12*\n\x08resultdb\x18\x11 \x01(\x0b\x32\x18.swarming.v2.ResultDBCfg\x12\r\n\x05realm\x18\x12 \x01(\t\"P\n\x15PoolTaskTemplateField\x12\x08\n\x04\x41UTO\x10\x00\x12\x11\n\rCANARY_PREFER\x10\x01\x12\x10\n\x0c\x43\x41NARY_NEVER\x10\x02\x12\x08\n\x04SKIP\x10\x03\x62\x06proto3')
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n+clusterfuzz/_internal/protos/swarming.proto\x12\x0bswarming.v2\x1a\x1fgoogle/protobuf/timestamp.proto\"(\n\nStringPair\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t\",\n\x0eStringListPair\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x03(\t\"*\n\x06\x44igest\x12\x0c\n\x04hash\x18\x01 \x01(\t\x12\x12\n\nsize_bytes\x18\x02 \x01(\x03\"I\n\x0c\x43\x41SReference\x12\x14\n\x0c\x63\x61s_instance\x18\x01 \x01(\t\x12#\n\x06\x64igest\x18\x02 \x01(\x0b\x32\x13.swarming.v2.Digest\"B\n\x0b\x43ipdPackage\x12\x14\n\x0cpackage_name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\"y\n\tCipdInput\x12\x0e\n\x06server\x18\x01 \x01(\t\x12\x30\n\x0e\x63lient_package\x18\x02 \x01(\x0b\x32\x18.swarming.v2.CipdPackage\x12*\n\x08packages\x18\x03 \x03(\x0b\x32\x18.swarming.v2.CipdPackage\"(\n\nCacheEntry\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04path\x18\x02 \x01(\t\"\xf2\x01\n\x0b\x43ontainment\x12\x16\n\x0elower_priority\x18\x01 \x01(\x08\x12\x42\n\x10\x63ontainment_type\x18\x02 \x01(\x0e\x32(.swarming.v2.Containment.ContainmentType\x12\x17\n\x0flimit_processes\x18\x03 \x01(\x03\x12$\n\x1climit_total_committed_memory\x18\x04 \x01(\x03\"H\n\x0f\x43ontainmentType\x12\x11\n\rNOT_SPECIFIED\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\x08\n\x04\x41UTO\x10\x02\x12\x0e\n\nJOB_OBJECT\x10\x03\"\x83\x04\n\x0eTaskProperties\x12\'\n\x06\x63\x61\x63hes\x18\x01 \x03(\x0b\x32\x17.swarming.v2.CacheEntry\x12*\n\ncipd_input\x18\x02 \x01(\x0b\x32\x16.swarming.v2.CipdInput\x12\x0f\n\x07\x63ommand\x18\x03 \x03(\t\x12\x14\n\x0crelative_cwd\x18\x04 \x01(\t\x12+\n\ndimensions\x18\x05 \x03(\x0b\x32\x17.swarming.v2.StringPair\x12$\n\x03\x65nv\x18\x06 \x03(\x0b\x32\x17.swarming.v2.StringPair\x12\x31\n\x0c\x65nv_prefixes\x18\x07 \x03(\x0b\x32\x1b.swarming.v2.StringListPair\x12\x1e\n\x16\x65xecution_timeout_secs\x18\x08 \x01(\x05\x12\x19\n\x11grace_period_secs\x18\t \x01(\x05\x12\x12\n\nidempotent\x18\n \x01(\x08\x12\x31\n\x0e\x63\x61s_input_root\x18\x0b \x01(\x0b\x32\x19.swarming.v2.CASReference\x12\x17\n\x0fio_timeout_secs\x18\x0c \x01(\x05\x12\x0f\n\x07outputs\x18\r \x03(\t\x12\x14\n\x0csecret_bytes\x18\x0e \x01(\x0c\x12-\n\x0b\x63ontainment\x18\x0f \x01(\x0b\x32\x18.swarming.v2.Containment\"p\n\tTaskSlice\x12/\n\nproperties\x18\x01 \x01(\x0b\x32\x1b.swarming.v2.TaskProperties\x12\x17\n\x0f\x65xpiration_secs\x18\x02 \x01(\x05\x12\x19\n\x11wait_for_capacity\x18\x03 \x01(\x08\"\x1d\n\x0bResultDBCfg\x12\x0e\n\x06\x65nable\x18\x01 \x01(\x08\"\xe8\x04\n\x0eNewTaskRequest\x12\x17\n\x0f\x65xpiration_secs\x18\x01 \x01(\x05\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x16\n\x0eparent_task_id\x18\x03 \x01(\t\x12\x10\n\x08priority\x18\x04 \x01(\x05\x12/\n\nproperties\x18\x05 \x01(\x0b\x32\x1b.swarming.v2.TaskProperties\x12+\n\x0btask_slices\x18\x06 \x03(\x0b\x32\x16.swarming.v2.TaskSlice\x12\x0c\n\x04tags\x18\x07 \x03(\t\x12\x0c\n\x04user\x18\x08 \x01(\t\x12\x17\n\x0fservice_account\x18\t \x01(\t\x12\x14\n\x0cpubsub_topic\x18\n \x01(\t\x12\x19\n\x11pubsub_auth_token\x18\x0b \x01(\t\x12\x17\n\x0fpubsub_userdata\x18\x0c \x01(\t\x12\x15\n\revaluate_only\x18\r \x01(\x08\x12M\n\x12pool_task_template\x18\x0e \x01(\x0e\x32\x31.swarming.v2.NewTaskRequest.PoolTaskTemplateField\x12\x1f\n\x17\x62ot_ping_tolerance_secs\x18\x0f \x01(\x05\x12\x14\n\x0crequest_uuid\x18\x10 \x01(\t\x12*\n\x08resultdb\x18\x11 \x01(\x0b\x32\x18.swarming.v2.ResultDBCfg\x12\r\n\x05realm\x18\x12 \x01(\t\"P\n\x15PoolTaskTemplateField\x12\x08\n\x04\x41UTO\x10\x00\x12\x11\n\rCANARY_PREFER\x10\x01\x12\x10\n\x0c\x43\x41NARY_NEVER\x10\x02\x12\x08\n\x04SKIP\x10\x03\"\x9d\x01\n\x11TasksCountRequest\x12)\n\x05start\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\x03\x65nd\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12&\n\x05state\x18\x03 \x01(\x0e\x32\x17.swarming.v2.StateQuery\x12\x0c\n\x04tags\x18\x04 \x03(\t\"D\n\nTasksCount\x12\r\n\x05\x63ount\x18\x01 \x01(\x05\x12\'\n\x03now\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp*\xcf\x02\n\nStateQuery\x12\x11\n\rQUERY_PENDING\x10\x00\x12\x11\n\rQUERY_RUNNING\x10\x01\x12\x19\n\x15QUERY_PENDING_RUNNING\x10\x02\x12\x13\n\x0fQUERY_COMPLETED\x10\x03\x12\x1b\n\x17QUERY_COMPLETED_SUCCESS\x10\x04\x12\x1b\n\x17QUERY_COMPLETED_FAILURE\x10\x05\x12\x11\n\rQUERY_EXPIRED\x10\x06\x12\x13\n\x0fQUERY_TIMED_OUT\x10\x07\x12\x12\n\x0eQUERY_BOT_DIED\x10\x08\x12\x12\n\x0eQUERY_CANCELED\x10\t\x12\r\n\tQUERY_ALL\x10\n\x12\x11\n\rQUERY_DEDUPED\x10\x0b\x12\x10\n\x0cQUERY_KILLED\x10\x0c\x12\x15\n\x11QUERY_NO_RESOURCE\x10\r\x12\x16\n\x12QUERY_CLIENT_ERROR\x10\x0e\x62\x06proto3')

_globals = globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'clusterfuzz._internal.protos.swarming_pb2', _globals)
if _descriptor._USE_C_DESCRIPTORS == False:
DESCRIPTOR._options = None
_globals['_STRINGPAIR']._serialized_start=60
_globals['_STRINGPAIR']._serialized_end=100
_globals['_STRINGLISTPAIR']._serialized_start=102
_globals['_STRINGLISTPAIR']._serialized_end=146
_globals['_DIGEST']._serialized_start=148
_globals['_DIGEST']._serialized_end=190
_globals['_CASREFERENCE']._serialized_start=192
_globals['_CASREFERENCE']._serialized_end=265
_globals['_CIPDPACKAGE']._serialized_start=267
_globals['_CIPDPACKAGE']._serialized_end=333
_globals['_CIPDINPUT']._serialized_start=335
_globals['_CIPDINPUT']._serialized_end=456
_globals['_CACHEENTRY']._serialized_start=458
_globals['_CACHEENTRY']._serialized_end=498
_globals['_CONTAINMENT']._serialized_start=501
_globals['_CONTAINMENT']._serialized_end=743
_globals['_CONTAINMENT_CONTAINMENTTYPE']._serialized_start=671
_globals['_CONTAINMENT_CONTAINMENTTYPE']._serialized_end=743
_globals['_TASKPROPERTIES']._serialized_start=746
_globals['_TASKPROPERTIES']._serialized_end=1261
_globals['_TASKSLICE']._serialized_start=1263
_globals['_TASKSLICE']._serialized_end=1375
_globals['_RESULTDBCFG']._serialized_start=1377
_globals['_RESULTDBCFG']._serialized_end=1406
_globals['_NEWTASKREQUEST']._serialized_start=1409
_globals['_NEWTASKREQUEST']._serialized_end=2025
_globals['_NEWTASKREQUEST_POOLTASKTEMPLATEFIELD']._serialized_start=1945
_globals['_NEWTASKREQUEST_POOLTASKTEMPLATEFIELD']._serialized_end=2025
_globals['_STATEQUERY']._serialized_start=2291
_globals['_STATEQUERY']._serialized_end=2626
_globals['_STRINGPAIR']._serialized_start=93
_globals['_STRINGPAIR']._serialized_end=133
_globals['_STRINGLISTPAIR']._serialized_start=135
_globals['_STRINGLISTPAIR']._serialized_end=179
_globals['_DIGEST']._serialized_start=181
_globals['_DIGEST']._serialized_end=223
_globals['_CASREFERENCE']._serialized_start=225
_globals['_CASREFERENCE']._serialized_end=298
_globals['_CIPDPACKAGE']._serialized_start=300
_globals['_CIPDPACKAGE']._serialized_end=366
_globals['_CIPDINPUT']._serialized_start=368
_globals['_CIPDINPUT']._serialized_end=489
_globals['_CACHEENTRY']._serialized_start=491
_globals['_CACHEENTRY']._serialized_end=531
_globals['_CONTAINMENT']._serialized_start=534
_globals['_CONTAINMENT']._serialized_end=776
_globals['_CONTAINMENT_CONTAINMENTTYPE']._serialized_start=704
_globals['_CONTAINMENT_CONTAINMENTTYPE']._serialized_end=776
_globals['_TASKPROPERTIES']._serialized_start=779
_globals['_TASKPROPERTIES']._serialized_end=1294
_globals['_TASKSLICE']._serialized_start=1296
_globals['_TASKSLICE']._serialized_end=1408
_globals['_RESULTDBCFG']._serialized_start=1410
_globals['_RESULTDBCFG']._serialized_end=1439
_globals['_NEWTASKREQUEST']._serialized_start=1442
_globals['_NEWTASKREQUEST']._serialized_end=2058
_globals['_NEWTASKREQUEST_POOLTASKTEMPLATEFIELD']._serialized_start=1978
_globals['_NEWTASKREQUEST_POOLTASKTEMPLATEFIELD']._serialized_end=2058
_globals['_TASKSCOUNTREQUEST']._serialized_start=2061
_globals['_TASKSCOUNTREQUEST']._serialized_end=2218
_globals['_TASKSCOUNT']._serialized_start=2220
_globals['_TASKSCOUNT']._serialized_end=2288
# @@protoc_insertion_point(module_scope)
Loading
Loading