Skip to content

Commit cbe7492

Browse files
IvanBM18IvanBM18
authored
Swarming: Fetch credential with scope & Capitalize's OS in request (#5222)
After testing the remote task gate in `dev` we were able to discover more things that were missing in the swarming implementation. ## Credentials request to retrieve required scopes Credentials retrieved using `credentials.get_default(_SWARMING_SCOPES)` didn't have all the requested scopes, even tough we explicitly requested it, this happened because the GCP metadata server ignored the requested scopes and instead returned the VM scopes <img width="952" height="679" alt="image" src="https://github.com/user-attachments/assets/7f4d9405-edf1-4b2f-b672-01bdb75be8db" /> Note: In red previous workflow, in green new workflow, in gray workflow used for local tests ## Minor Fixes - Swarming requires the target OS to be capitalized, hence it wont schedule the task to any bot. - We added the IS_K8S_ENV as a workaround for this bug: b/495819289
1 parent 35af2d7 commit cbe7492

2 files changed

Lines changed: 38 additions & 16 deletions

File tree

‎src/clusterfuzz/_internal/swarming/__init__.py‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
from clusterfuzz._internal.config import local_config
2626
from clusterfuzz._internal.datastore import data_types
2727
from clusterfuzz._internal.google_cloud_utils import credentials
28+
from clusterfuzz._internal.metrics import logs
2829
from clusterfuzz._internal.protos import swarming_pb2
2930
from clusterfuzz._internal.system import environment
3031

@@ -67,7 +68,7 @@ def _get_task_dimensions(job: data_types.Job, platform_specific_dimensions: list
6768
""" Gets all swarming dimensions for a task.
6869
Job dimensions have more precedence than static dimensions"""
6970
unique_dimensions = {}
70-
unique_dimensions['os'] = job.platform
71+
unique_dimensions['os'] = str(job.platform).capitalize()
7172
unique_dimensions['pool'] = _get_swarming_config().get('swarming_pool')
7273

7374
for dimension in platform_specific_dimensions:
@@ -133,6 +134,7 @@ def _get_new_task_spec(command: str, job_name: str,
133134
swarming_pb2.StringPair(key='UWORKER', value='True'), # pylint: disable=no-member
134135
swarming_pb2.StringPair(key='SWARMING_BOT', value='True'), # pylint: disable=no-member
135136
swarming_pb2.StringPair(key='LOG_TO_GCP', value='True'), # pylint: disable=no-member
137+
swarming_pb2.StringPair(key='IS_K8S_ENV', value='True'), # pylint: disable=no-member
136138
swarming_pb2.StringPair( # pylint: disable=no-member
137139
key='LOGGING_CLOUD_PROJECT_ID',
138140
value=logs_project_id),
@@ -181,7 +183,11 @@ def push_swarming_task(command, download_url, job_type):
181183
raise ValueError('invalid job_name')
182184

183185
task_spec = _get_new_task_spec(command, job_type, download_url)
184-
creds, _ = credentials.get_default(_SWARMING_SCOPES)
186+
creds = credentials.get_scoped_service_account_credentials(_SWARMING_SCOPES)
187+
if not creds:
188+
logs.error(
189+
'[Swarming] Failed to push task into swarming. Reason: No credentials.')
190+
return
185191

186192
if not creds.token:
187193
creds.refresh(requests.Request())
@@ -193,5 +199,11 @@ def push_swarming_task(command, download_url, job_type):
193199
}
194200
swarming_server = _get_swarming_config().get('swarming_server')
195201
url = f'https://{swarming_server}/prpc/swarming.v2.Tasks/NewTask'
196-
utils.post_url(
197-
url=url, data=json_format.MessageToJson(task_spec), headers=headers)
202+
message_body = json_format.MessageToJson(task_spec)
203+
logs.info(
204+
f"""[Swarming] Pushing task for {job_type}
205+
as {creds.service_account_email}""",
206+
url=url,
207+
body=message_body)
208+
response = utils.post_url(url=url, data=message_body, headers=headers)
209+
logs.info(f'[Swarming] Response from {job_type}', response=response)

‎src/clusterfuzz/_internal/tests/core/swarming/swarming_test.py‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ def setUp(self):
3535
'clusterfuzz._internal.base.utils.post_url',
3636
'clusterfuzz._internal.swarming._get_task_name',
3737
'clusterfuzz._internal.google_cloud_utils.credentials.get_default',
38+
'clusterfuzz._internal.google_cloud_utils.credentials.get_scoped_service_account_credentials',
3839
'google.auth.transport.requests.Request',
3940
'clusterfuzz._internal.swarming.FeatureFlags',
4041
])
@@ -62,7 +63,8 @@ def test_get_spec_from_config_with_docker_image(self):
6263
'luci-auth', 'context', '--', './linux_entry_point.sh'
6364
],
6465
dimensions=[
65-
swarming_pb2.StringPair(key='os', value=job.platform),
66+
swarming_pb2.StringPair(
67+
key='os', value=str(job.platform).capitalize()),
6668
swarming_pb2.StringPair(key='pool', value='pool-name')
6769
],
6870
cipd_input=swarming_pb2.CipdInput(), # pylint: disable=no-member
@@ -82,12 +84,13 @@ def test_get_spec_from_config_with_docker_image(self):
8284
swarming_pb2.StringPair(
8385
key='DOCKER_ENV_VARS',
8486
value=
85-
'{"UWORKER": "True", "SWARMING_BOT": "True", "LOG_TO_GCP": "True", "LOGGING_CLOUD_PROJECT_ID": "project_id"}'
87+
'{"UWORKER": "True", "SWARMING_BOT": "True", "LOG_TO_GCP": "True", "IS_K8S_ENV": "True", "LOGGING_CLOUD_PROJECT_ID": "project_id"}'
8688
),
8789
swarming_pb2.StringPair(key='UWORKER', value='True'),
8890
swarming_pb2.StringPair(
8991
key='SWARMING_BOT', value='True'),
9092
swarming_pb2.StringPair(key='LOG_TO_GCP', value='True'),
93+
swarming_pb2.StringPair(key='IS_K8S_ENV', value='True'),
9194
swarming_pb2.StringPair(
9295
key='LOGGING_CLOUD_PROJECT_ID', value='project_id'),
9396
],
@@ -126,7 +129,8 @@ def test_get_spec_from_config_without_docker_image(self):
126129
'luci-auth', 'context', '--', './mac_entry_point.sh'
127130
],
128131
dimensions=[
129-
swarming_pb2.StringPair(key='os', value=job.platform),
132+
swarming_pb2.StringPair(
133+
key='os', value=str(job.platform).capitalize()),
130134
swarming_pb2.StringPair(key='pool', value='pool-name'),
131135
swarming_pb2.StringPair(key='key1', value='value1'),
132136
swarming_pb2.StringPair(key='key2', value='value2'),
@@ -155,12 +159,13 @@ def test_get_spec_from_config_without_docker_image(self):
155159
swarming_pb2.StringPair(
156160
key='DOCKER_ENV_VARS',
157161
value=
158-
'{"UWORKER": "True", "SWARMING_BOT": "True", "LOG_TO_GCP": "True", "LOGGING_CLOUD_PROJECT_ID": "project_id"}'
162+
'{"UWORKER": "True", "SWARMING_BOT": "True", "LOG_TO_GCP": "True", "IS_K8S_ENV": "True", "LOGGING_CLOUD_PROJECT_ID": "project_id"}'
159163
),
160164
swarming_pb2.StringPair(key='UWORKER', value='True'),
161165
swarming_pb2.StringPair(
162166
key='SWARMING_BOT', value='True'),
163167
swarming_pb2.StringPair(key='LOG_TO_GCP', value='True'),
168+
swarming_pb2.StringPair(key='IS_K8S_ENV', value='True'),
164169
swarming_pb2.StringPair(
165170
key='LOGGING_CLOUD_PROJECT_ID', value='project_id'),
166171
],
@@ -196,7 +201,8 @@ def test_get_spec_from_config_for_fuzz_task(self):
196201
'luci-auth', 'context', '--', './linux_entry_point.sh'
197202
],
198203
dimensions=[
199-
swarming_pb2.StringPair(key='os', value=job.platform),
204+
swarming_pb2.StringPair(
205+
key='os', value=str(job.platform).capitalize()),
200206
swarming_pb2.StringPair(key='pool', value='pool-name')
201207
],
202208
cipd_input=swarming_pb2.CipdInput(), # pylint: disable=no-member
@@ -216,12 +222,13 @@ def test_get_spec_from_config_for_fuzz_task(self):
216222
swarming_pb2.StringPair(
217223
key='DOCKER_ENV_VARS',
218224
value=
219-
'{"UWORKER": "True", "SWARMING_BOT": "True", "LOG_TO_GCP": "True", "LOGGING_CLOUD_PROJECT_ID": "project_id"}'
225+
'{"UWORKER": "True", "SWARMING_BOT": "True", "LOG_TO_GCP": "True", "IS_K8S_ENV": "True", "LOGGING_CLOUD_PROJECT_ID": "project_id"}'
220226
),
221227
swarming_pb2.StringPair(key='UWORKER', value='True'),
222228
swarming_pb2.StringPair(
223229
key='SWARMING_BOT', value='True'),
224230
swarming_pb2.StringPair(key='LOG_TO_GCP', value='True'),
231+
swarming_pb2.StringPair(key='IS_K8S_ENV', value='True'),
225232
swarming_pb2.StringPair(
226233
key='LOGGING_CLOUD_PROJECT_ID', value='project_id'),
227234
],
@@ -234,7 +241,7 @@ def test_push_swarming_task(self):
234241
"""Tests that push_swarming_task works as expected."""
235242
mock_creds = mock.MagicMock()
236243
mock_creds.token = 'fake_token'
237-
self.mock.get_default.return_value = (mock_creds, None)
244+
self.mock.get_scoped_service_account_credentials.return_value = mock_creds
238245

239246
job = data_types.Job(name='libfuzzer_chrome_asan', platform='LINUX')
240247
job.put()
@@ -253,7 +260,8 @@ def test_push_swarming_task(self):
253260
'luci-auth', 'context', '--', './linux_entry_point.sh'
254261
],
255262
dimensions=[
256-
swarming_pb2.StringPair(key='os', value=job.platform),
263+
swarming_pb2.StringPair(
264+
key='os', value=str(job.platform).capitalize()),
257265
swarming_pb2.StringPair(key='pool', value='pool-name')
258266
],
259267
cipd_input=swarming_pb2.CipdInput(), # pylint: disable=no-member
@@ -273,20 +281,22 @@ def test_push_swarming_task(self):
273281
swarming_pb2.StringPair(
274282
key='DOCKER_ENV_VARS',
275283
value=
276-
'{"UWORKER": "True", "SWARMING_BOT": "True", "LOG_TO_GCP": "True", "LOGGING_CLOUD_PROJECT_ID": "project_id"}'
284+
'{"UWORKER": "True", "SWARMING_BOT": "True", "LOG_TO_GCP": "True", "IS_K8S_ENV": "True", "LOGGING_CLOUD_PROJECT_ID": "project_id"}'
277285
),
278286
swarming_pb2.StringPair(key='UWORKER', value='True'),
279287
swarming_pb2.StringPair(
280288
key='SWARMING_BOT', value='True'),
281289
swarming_pb2.StringPair(key='LOG_TO_GCP', value='True'),
290+
swarming_pb2.StringPair(key='IS_K8S_ENV', value='True'),
282291
swarming_pb2.StringPair(
283292
key='LOGGING_CLOUD_PROJECT_ID', value='project_id'),
284293
],
285294
secret_bytes=base64.b64encode(
286295
'https://download_url'.encode('utf-8'))))
287296
])
288297

289-
self.mock.get_default.assert_called_with(swarming._SWARMING_SCOPES) # pylint: disable=protected-access
298+
self.mock.get_scoped_service_account_credentials.assert_called_with(
299+
swarming._SWARMING_SCOPES) # pylint: disable=protected-access
290300
expected_headers = {
291301
'Accept': 'application/json',
292302
'Content-Type': 'application/json',
@@ -302,7 +312,7 @@ def test_push_swarming_task_with_refresh(self):
302312
"""Tests that push_swarming_task refreshes credentials if token is missing."""
303313
mock_creds = mock.MagicMock()
304314
mock_creds.token = None
305-
self.mock.get_default.return_value = (mock_creds, None)
315+
self.mock.get_scoped_service_account_credentials.return_value = mock_creds
306316

307317
def refresh_side_effect(_):
308318
mock_creds.token = 'refreshed_token'
@@ -370,7 +380,7 @@ def test_get_task_dimensions_job_precedence(self):
370380
dimensions = spec.task_slices[0].properties.dimensions
371381

372382
expected_dimensions = [
373-
swarming_pb2.StringPair(key='os', value='MAC'),
383+
swarming_pb2.StringPair(key='os', value='Mac'),
374384
swarming_pb2.StringPair(key='pool', value='pool-name'),
375385
swarming_pb2.StringPair(key='key1', value='job_value1'),
376386
swarming_pb2.StringPair(key='key2', value='value2'),

0 commit comments

Comments
 (0)