|
| 1 | +import logging |
| 2 | +from typing import Literal |
| 3 | + |
| 4 | +import sentry_sdk |
| 5 | +from asgiref.sync import async_to_sync |
| 6 | +from shared.celery_config import cache_test_rollups_task_name, process_flakes_task_name |
| 7 | +from shared.django_apps.reports.models import ReportSession, UploadError |
| 8 | +from shared.reports.types import UploadType |
| 9 | +from shared.typings.torngit import AdditionalData |
| 10 | +from shared.yaml import UserYaml |
| 11 | +from sqlalchemy.orm import Session |
| 12 | + |
| 13 | +from app import celery_app |
| 14 | +from database.models import Commit, Repository |
| 15 | +from helpers.notifier import NotifierResult |
| 16 | +from helpers.string import shorten_file_paths |
| 17 | +from services.activation import activate_user, schedule_new_user_activated_task |
| 18 | +from services.redis import get_redis_connection |
| 19 | +from services.repository import ( |
| 20 | + EnrichedPull, |
| 21 | + fetch_and_update_pull_request_information_from_commit, |
| 22 | + get_repo_provider_service, |
| 23 | +) |
| 24 | +from services.seats import ShouldActivateSeat, determine_seat_activation |
| 25 | +from services.test_analytics.ta_metrics import ( |
| 26 | + read_failures_summary, |
| 27 | + read_tests_totals_summary, |
| 28 | +) |
| 29 | +from services.test_analytics.ta_process_flakes import KEY_NAME |
| 30 | +from services.test_analytics.ta_timeseries import ( |
| 31 | + TestInstance, |
| 32 | + get_flaky_tests_dict, |
| 33 | + get_pr_comment_agg, |
| 34 | + get_pr_comment_failures, |
| 35 | +) |
| 36 | +from services.test_results import ( |
| 37 | + ErrorPayload, |
| 38 | + FinisherResult, |
| 39 | + TACommentInDepthInfo, |
| 40 | + TestResultsNotificationFailure, |
| 41 | + TestResultsNotificationPayload, |
| 42 | + TestResultsNotifier, |
| 43 | + should_do_flaky_detection, |
| 44 | +) |
| 45 | + |
| 46 | +log = logging.getLogger(__name__) |
| 47 | + |
| 48 | + |
| 49 | +def get_upload_ids(commitid: str) -> dict[int, ReportSession]: |
| 50 | + return { |
| 51 | + upload.id: upload |
| 52 | + for upload in ReportSession.objects.filter( |
| 53 | + report__commit__commitid=commitid, |
| 54 | + state__in=["processed", "finished", "flake_processed"], |
| 55 | + ) |
| 56 | + } |
| 57 | + |
| 58 | + |
| 59 | +def get_upload_error(upload_ids: list[int]) -> ErrorPayload | None: |
| 60 | + error = UploadError.objects.filter(report_session_id__in=upload_ids).first() |
| 61 | + if error: |
| 62 | + return ErrorPayload( |
| 63 | + error_code=error.error_code, |
| 64 | + error_message=error.error_params.get("error_message"), |
| 65 | + ) |
| 66 | + return None |
| 67 | + |
| 68 | + |
| 69 | +def transform_failures( |
| 70 | + uploads: dict[int, ReportSession], failures: list[TestInstance] |
| 71 | +) -> list[TestResultsNotificationFailure[bytes]]: |
| 72 | + notif_failures = [] |
| 73 | + for failure in failures: |
| 74 | + if failure["failure_message"] is not None: |
| 75 | + failure["failure_message"] = shorten_file_paths( |
| 76 | + failure["failure_message"] |
| 77 | + ).replace("\r", "") |
| 78 | + |
| 79 | + notif_failures.append( |
| 80 | + TestResultsNotificationFailure( |
| 81 | + display_name=failure["computed_name"], |
| 82 | + failure_message=failure["failure_message"], |
| 83 | + test_id=failure["test_id"], |
| 84 | + envs=uploads[failure["upload_id"]].flag_names, |
| 85 | + duration_seconds=failure["duration_seconds"] or 0, |
| 86 | + build_url=uploads[failure["upload_id"]].build_url, |
| 87 | + ) |
| 88 | + ) |
| 89 | + return notif_failures |
| 90 | + |
| 91 | + |
| 92 | +def queue_followup_tasks( |
| 93 | + repo: Repository, |
| 94 | + commit: Commit, |
| 95 | + commit_yaml: UserYaml, |
| 96 | + impl_type: Literal["new", "both"] = "both", |
| 97 | +): |
| 98 | + if ( |
| 99 | + should_do_flaky_detection(repo, commit_yaml) |
| 100 | + and commit.merged is True |
| 101 | + and commit.branch == repo.branch |
| 102 | + ): |
| 103 | + redis_client = get_redis_connection() |
| 104 | + redis_client.set(f"flake_uploads:{repo.repoid}", 0) |
| 105 | + redis_client.lpush(KEY_NAME.format(repo.repoid), commit.commitid) |
| 106 | + |
| 107 | + celery_app.send_task( |
| 108 | + process_flakes_task_name, |
| 109 | + kwargs={ |
| 110 | + "repo_id": repo.repoid, |
| 111 | + "commit_id": commit.commitid, |
| 112 | + "impl_type": impl_type, |
| 113 | + }, |
| 114 | + ) |
| 115 | + |
| 116 | + if commit.branch is not None: |
| 117 | + celery_app.send_task( |
| 118 | + cache_test_rollups_task_name, |
| 119 | + kwargs={ |
| 120 | + "repoid": repo.repoid, |
| 121 | + "branch": commit.branch, |
| 122 | + "impl_type": impl_type, |
| 123 | + }, |
| 124 | + ) |
| 125 | + |
| 126 | + |
| 127 | +def check_seat_activation(db_session: Session, pull: EnrichedPull) -> bool: |
| 128 | + activate_seat_info = determine_seat_activation(pull) |
| 129 | + |
| 130 | + should_show_upgrade_message = True |
| 131 | + |
| 132 | + match activate_seat_info.should_activate_seat: |
| 133 | + case ShouldActivateSeat.AUTO_ACTIVATE: |
| 134 | + assert activate_seat_info.owner_id |
| 135 | + assert activate_seat_info.author_id |
| 136 | + successful_activation = activate_user( |
| 137 | + db_session=db_session, |
| 138 | + org_ownerid=activate_seat_info.owner_id, |
| 139 | + user_ownerid=activate_seat_info.author_id, |
| 140 | + ) |
| 141 | + if successful_activation: |
| 142 | + schedule_new_user_activated_task( |
| 143 | + activate_seat_info.owner_id, |
| 144 | + activate_seat_info.author_id, |
| 145 | + ) |
| 146 | + should_show_upgrade_message = False |
| 147 | + case ShouldActivateSeat.MANUAL_ACTIVATE: |
| 148 | + pass |
| 149 | + case ShouldActivateSeat.NO_ACTIVATE: |
| 150 | + should_show_upgrade_message = False |
| 151 | + |
| 152 | + return should_show_upgrade_message |
| 153 | + |
| 154 | + |
| 155 | +@sentry_sdk.trace |
| 156 | +def new_impl( |
| 157 | + db_session: Session, # only used for seat activation, for now |
| 158 | + repo: Repository, # using sqlalchemy models for now |
| 159 | + commit: Commit, |
| 160 | + commit_yaml: UserYaml, |
| 161 | + impl_type: Literal["new", "both"] = "both", |
| 162 | +) -> FinisherResult: |
| 163 | + repoid = repo.repoid |
| 164 | + commitid = commit.commitid |
| 165 | + |
| 166 | + extra = { |
| 167 | + "repo_id": repoid, |
| 168 | + "commit_id": commitid, |
| 169 | + "impl_type": impl_type, |
| 170 | + } |
| 171 | + |
| 172 | + log.info("Starting new_impl of TA finisher", extra=extra) |
| 173 | + |
| 174 | + queue_followup_tasks(repo, commit, commit_yaml, impl_type) |
| 175 | + |
| 176 | + if not commit_yaml.read_yaml_field("comment", _else=True): |
| 177 | + log.info("Comment is disabled, not posting comment", extra=extra) |
| 178 | + return { |
| 179 | + "notify_attempted": False, |
| 180 | + "notify_succeeded": False, |
| 181 | + "queue_notify": False, |
| 182 | + } |
| 183 | + |
| 184 | + upload_ids = get_upload_ids(commitid) |
| 185 | + error = get_upload_error(list(upload_ids.keys())) |
| 186 | + |
| 187 | + with read_tests_totals_summary.labels(impl="new").time(): |
| 188 | + summary = get_pr_comment_agg(repoid, commitid) |
| 189 | + |
| 190 | + if not summary["failed"] and error is None: |
| 191 | + log.info( |
| 192 | + "No failures and no error so not posting comment but still queueing notify", |
| 193 | + extra=extra, |
| 194 | + ) |
| 195 | + return { |
| 196 | + "notify_attempted": False, |
| 197 | + "notify_succeeded": True, |
| 198 | + "queue_notify": True, |
| 199 | + } |
| 200 | + |
| 201 | + additional_data: AdditionalData = {"upload_type": UploadType.TEST_RESULTS} |
| 202 | + repo_service = get_repo_provider_service(repo, additional_data=additional_data) |
| 203 | + pull = async_to_sync(fetch_and_update_pull_request_information_from_commit)( |
| 204 | + repo_service, commit, commit_yaml |
| 205 | + ) |
| 206 | + |
| 207 | + if not pull: |
| 208 | + log.info("No pull so not posting comment", extra=extra) |
| 209 | + return { |
| 210 | + "notify_attempted": False, |
| 211 | + "notify_succeeded": False, |
| 212 | + "queue_notify": False, |
| 213 | + } |
| 214 | + |
| 215 | + notifier = TestResultsNotifier( |
| 216 | + commit, |
| 217 | + commit_yaml, |
| 218 | + _pull=pull, |
| 219 | + _repo_service=repo_service, |
| 220 | + error=error, |
| 221 | + ) |
| 222 | + |
| 223 | + seat_needs_activation = check_seat_activation(db_session, pull) |
| 224 | + |
| 225 | + if seat_needs_activation: |
| 226 | + success, _ = notifier.upgrade_comment() |
| 227 | + log.info( |
| 228 | + "Seat needs activation, posted upgrade comment", |
| 229 | + extra={**extra, "success": success}, |
| 230 | + ) |
| 231 | + return { |
| 232 | + "notify_attempted": False, |
| 233 | + "notify_succeeded": success, |
| 234 | + "queue_notify": False, |
| 235 | + } |
| 236 | + |
| 237 | + if summary["failed"] == 0: |
| 238 | + # no failures, only error |
| 239 | + log.info("No failures, posting error comment", extra=extra) |
| 240 | + notifier.error_comment() |
| 241 | + |
| 242 | + return { |
| 243 | + "notify_attempted": True, |
| 244 | + "notify_succeeded": False, |
| 245 | + "queue_notify": True, |
| 246 | + } |
| 247 | + |
| 248 | + with read_failures_summary.labels(impl="new").time(): |
| 249 | + failures = get_pr_comment_failures(repoid, commitid) |
| 250 | + |
| 251 | + notif_failures = transform_failures(upload_ids, failures) |
| 252 | + |
| 253 | + flaky_tests = dict() |
| 254 | + |
| 255 | + # flake detection if appropriate |
| 256 | + if should_do_flaky_detection(repo, commit_yaml): |
| 257 | + flaky_tests = get_flaky_tests_dict(repoid) |
| 258 | + |
| 259 | + payload = TestResultsNotificationPayload( |
| 260 | + failed=summary["failed"], |
| 261 | + passed=summary["passed"], |
| 262 | + skipped=summary["skipped"], |
| 263 | + info=TACommentInDepthInfo(notif_failures, flaky_tests), |
| 264 | + ) |
| 265 | + |
| 266 | + notifier.payload = payload |
| 267 | + |
| 268 | + notifier_result = notifier.notify() |
| 269 | + success = True if notifier_result is NotifierResult.COMMENT_POSTED else False |
| 270 | + log.info("Posted TA comment", extra={**extra, "success": success}) |
| 271 | + return { |
| 272 | + "notify_attempted": True, |
| 273 | + "notify_succeeded": success, |
| 274 | + "queue_notify": False, |
| 275 | + } |
0 commit comments