Skip to content

Commit 0bf63a2

Browse files
authored
Fix Dataflow cost benchmark after Dataflow client migration (#38788)
* Fix Dataflow cost benchmark after Dataflow client migration * Fix formatter
1 parent d521df8 commit 0bf63a2

1 file changed

Lines changed: 9 additions & 11 deletions

File tree

sdks/python/apache_beam/testing/load_tests/dataflow_cost_benchmark.py

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ def _process_metrics_list(self,
141141
return system_metrics
142142

143143
def _get_worker_time_interval(
144-
self, job_id: str) -> tuple[Optional[str], Optional[str]]:
144+
self, job_id: str) -> tuple[Optional[datetime], Optional[datetime]]:
145145
"""Extracts worker start and stop times from job messages."""
146146
start_time, end_time = None, None
147147
page_token = None
@@ -155,7 +155,7 @@ def _get_worker_time_interval(
155155
page_token=page_token,
156156
minimum_importance='JOB_MESSAGE_DEBUG')
157157
for message in messages:
158-
text = message.messageText
158+
text = message.message_text
159159
if getattr(message, 'time', None):
160160
last_message_time = message.time
161161
if text:
@@ -186,8 +186,8 @@ def _get_throughput_metrics(
186186
self,
187187
project: str,
188188
job_id: str,
189-
start_time: str,
190-
end_time: str,
189+
start_time: datetime,
190+
end_time: datetime,
191191
pcollection_name: Optional[str] = None,
192192
) -> dict[str, float]:
193193
"""Query Cloud Monitoring for per-PCollection throughput."""
@@ -256,7 +256,8 @@ def _point_numeric_value(point) -> float:
256256
return metrics
257257

258258
def _get_streaming_throughput_metrics(
259-
self, project: str, start_time: str, end_time: str) -> dict[str, float]:
259+
self, project: str, start_time: datetime,
260+
end_time: datetime) -> dict[str, float]:
260261
if not self.subscription:
261262
return {'AvgThroughputBytes': 0.0, 'AvgThroughputElements': 0.0}
262263

@@ -297,17 +298,14 @@ def _get_streaming_throughput_metrics(
297298
metrics[f"AvgThroughput{key}"] = avg_rate
298299
return metrics
299300

300-
def _get_job_runtime(self, start_time: str, end_time: str) -> float:
301+
def _get_job_runtime(self, start_time: datetime, end_time: datetime) -> float:
301302
"""Calculates the job runtime duration in seconds."""
302-
start_dt = datetime.fromisoformat(start_time[:-1])
303-
end_dt = datetime.fromisoformat(end_time[:-1])
304-
return (end_dt - start_dt).total_seconds()
303+
return (end_time - start_time).total_seconds()
305304

306305
def _get_additional_metrics(self,
307306
result: DataflowPipelineResult) -> dict[str, Any]:
308307
job_id = result.job_id()
309-
job = self.dataflow_client.get_job(job_id)
310-
project = job.projectId
308+
project = self.project_id
311309
start_time, end_time = self._get_worker_time_interval(job_id)
312310
if not start_time or not end_time:
313311
logging.warning('Could not find valid worker start/end times.')

0 commit comments

Comments
 (0)