diff --git a/apps/worker/services/test_analytics/ta_process_flakes.py b/apps/worker/services/test_analytics/ta_process_flakes.py index 61cf26df09..700a2639a9 100644 --- a/apps/worker/services/test_analytics/ta_process_flakes.py +++ b/apps/worker/services/test_analytics/ta_process_flakes.py @@ -1,4 +1,5 @@ import logging +from collections import defaultdict from datetime import timedelta import sentry_sdk @@ -33,13 +34,12 @@ def fetch_current_flakes(repo_id: int) -> dict[bytes, Flake]: } -def get_testruns(upload: ReportSession) -> QuerySet[Testrun]: - upload_filter = Q(upload_id=upload.id) - +def get_testruns_for_uploads(upload_ids: list[int]) -> QuerySet[Testrun]: # we won't process flakes for testruns older than 1 day return Testrun.objects.filter( - Q(timestamp__gte=timezone.now() - timedelta(days=1)) & upload_filter - ).order_by("timestamp") + Q(timestamp__gte=timezone.now() - timedelta(days=1)) + & Q(upload_id__in=upload_ids) + ).order_by("upload_id", "timestamp") def handle_pass(curr_flakes: dict[bytes, Flake], test_id: bytes): @@ -81,10 +81,11 @@ def handle_failure( @sentry_sdk.trace def process_single_upload( - upload: ReportSession, curr_flakes: dict[bytes, Flake], repo_id: int + upload_id: int, + testruns: list[Testrun], + curr_flakes: dict[bytes, Flake], + repo_id: int, ): - testruns = get_testruns(upload) - for testrun in testruns: test_id = bytes(testrun.test_id) match testrun.outcome: @@ -106,7 +107,7 @@ def process_flakes_for_commit(repo_id: int, commit_id: str): log.info( "process_flakes_for_commit: starting processing", ) - uploads = get_relevant_uploads(repo_id, commit_id) + uploads = list(get_relevant_uploads(repo_id, commit_id)) log.info( "process_flakes_for_commit: fetched uploads", @@ -120,8 +121,17 @@ def process_flakes_for_commit(repo_id: int, commit_id: str): extra={"flakes": [flake.test_id.hex() for flake in curr_flakes.values()]}, ) + upload_ids = [upload.id for upload in uploads] + all_testruns = get_testruns_for_uploads(upload_ids) + + testruns_by_upload: dict[int, list[Testrun]] = defaultdict(list) + for testrun in all_testruns: + testruns_by_upload[testrun.upload_id].append(testrun) + for upload in uploads: - process_single_upload(upload, curr_flakes, repo_id) + process_single_upload( + upload.id, testruns_by_upload[upload.id], curr_flakes, repo_id + ) log.info( "process_flakes_for_commit: processed upload", extra={"upload": upload.id},