Skip to content

Commit 8dcfe9d

Browse files
felipebergaminfelipebergamin
committed
feat(tree-compare): add backend endpoint to compare tree commits
Part of #1951 Signed-off-by: Felipe Bergamin <felipebergamin6@gmail.com>
1 parent c8fc570 commit 8dcfe9d

10 files changed

Lines changed: 874 additions & 2 deletions

File tree

‎backend/kernelCI_app/helpers/treeCompare.py‎

Lines changed: 220 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,220 @@
1+
from collections import defaultdict
2+
from dataclasses import dataclass, field
3+
from typing import Callable, Literal, Optional
4+
5+
from kernelCI_app.constants.general import UNKNOWN_STRING
6+
from kernelCI_app.constants.process_pending import ROLLUP_STATUS_FIELDS
7+
from kernelCI_app.typeModels.treeCompare import (
8+
CompareDelta,
9+
CompareEntitySummary,
10+
CompareGroupRow,
11+
CompareGroups,
12+
CompareStatusCounts,
13+
CompareSummary,
14+
TreeCompareResponse,
15+
)
16+
17+
BucketKey = Literal["pass", "fail", "inconclusive"]
18+
19+
20+
@dataclass
21+
class _HashAccumulator:
22+
builds: CompareStatusCounts = field(default_factory=CompareStatusCounts)
23+
boots: CompareStatusCounts = field(default_factory=CompareStatusCounts)
24+
tests: CompareStatusCounts = field(default_factory=CompareStatusCounts)
25+
build_groups: dict[str, CompareStatusCounts] = field(
26+
default_factory=lambda: defaultdict(CompareStatusCounts)
27+
)
28+
boot_groups: dict[str, CompareStatusCounts] = field(
29+
default_factory=lambda: defaultdict(CompareStatusCounts)
30+
)
31+
test_groups: dict[str, CompareStatusCounts] = field(
32+
default_factory=lambda: defaultdict(CompareStatusCounts)
33+
)
34+
35+
36+
def _empty_counts() -> CompareStatusCounts:
37+
return CompareStatusCounts()
38+
39+
40+
def _status_to_bucket(status: Optional[str]) -> BucketKey:
41+
if status is None:
42+
return "inconclusive"
43+
normalized = status.upper()
44+
if normalized == "PASS":
45+
return "pass"
46+
if normalized == "FAIL":
47+
return "fail"
48+
return "inconclusive"
49+
50+
51+
def _increment_bucket(
52+
counts: CompareStatusCounts, bucket: BucketKey, amount: int
53+
) -> None:
54+
if amount <= 0:
55+
return
56+
if bucket == "pass":
57+
counts.pass_count += amount
58+
elif bucket == "fail":
59+
counts.fail_count += amount
60+
else:
61+
counts.inconclusive += amount
62+
63+
64+
def _rollup_status_to_bucket(status_name: str) -> BucketKey:
65+
if status_name == "PASS":
66+
return "pass"
67+
if status_name == "FAIL":
68+
return "fail"
69+
return "inconclusive"
70+
71+
72+
def _boot_group_key(row_dict: dict) -> str:
73+
return (
74+
row_dict.get("test_platform")
75+
or row_dict.get("hardware_key")
76+
or UNKNOWN_STRING
77+
)
78+
79+
80+
def _test_group_key(row_dict: dict) -> str:
81+
path_group = row_dict.get("path_group") or UNKNOWN_STRING
82+
arch = row_dict.get("build_architecture") or UNKNOWN_STRING
83+
if arch == UNKNOWN_STRING:
84+
return path_group
85+
return f"{path_group}/{arch}"
86+
87+
88+
def process_rollup_rows(
89+
*,
90+
rows: list[dict],
91+
commit_hashes: list[str],
92+
) -> dict[str, _HashAccumulator]:
93+
accumulators = {commit_hash: _HashAccumulator() for commit_hash in commit_hashes}
94+
95+
for row_dict in rows:
96+
commit_hash = row_dict["git_commit_hash"]
97+
is_boot_row = row_dict["is_boot"]
98+
acc = accumulators.setdefault(commit_hash, _HashAccumulator())
99+
group_id = _boot_group_key(row_dict) if is_boot_row else _test_group_key(row_dict)
100+
target = acc.boots if is_boot_row else acc.tests
101+
group_map = acc.boot_groups if is_boot_row else acc.test_groups
102+
103+
for status_name, field_name in ROLLUP_STATUS_FIELDS.items():
104+
count = row_dict.get(field_name, 0) or 0
105+
if count <= 0:
106+
continue
107+
bucket = _rollup_status_to_bucket(status_name)
108+
_increment_bucket(target, bucket, count)
109+
_increment_bucket(group_map[group_id], bucket, count)
110+
111+
return accumulators
112+
113+
114+
def process_build_rows(
115+
*,
116+
rows: list[dict],
117+
commit_hashes: list[str],
118+
) -> dict[str, _HashAccumulator]:
119+
accumulators = {commit_hash: _HashAccumulator() for commit_hash in commit_hashes}
120+
121+
for row in rows:
122+
commit_hash = row["git_commit_hash"]
123+
config = row.get("config_name") or UNKNOWN_STRING
124+
count = row.get("count") or 0
125+
acc = accumulators.setdefault(commit_hash, _HashAccumulator())
126+
bucket = _status_to_bucket(row.get("status"))
127+
_increment_bucket(acc.builds, bucket, count)
128+
_increment_bucket(acc.build_groups[config], bucket, count)
129+
130+
return accumulators
131+
132+
133+
def _make_delta(side_a: CompareStatusCounts, side_b: CompareStatusCounts) -> CompareDelta:
134+
return CompareDelta(
135+
**{
136+
"pass": side_b.pass_count - side_a.pass_count,
137+
"fail": side_b.fail_count - side_a.fail_count,
138+
}
139+
)
140+
141+
142+
def _make_entity_summary(
143+
*,
144+
side_a: CompareStatusCounts,
145+
side_b: CompareStatusCounts,
146+
) -> CompareEntitySummary:
147+
return CompareEntitySummary(
148+
sideA=side_a,
149+
sideB=side_b,
150+
delta=_make_delta(side_a, side_b),
151+
)
152+
153+
154+
def _make_group_rows(
155+
*,
156+
group_maps: tuple[dict[str, CompareStatusCounts], dict[str, CompareStatusCounts]],
157+
label_fn: Optional[Callable[[str], str]] = None,
158+
) -> list[CompareGroupRow]:
159+
side_a_groups, side_b_groups = group_maps
160+
all_ids = set(side_a_groups) | set(side_b_groups)
161+
rows: list[CompareGroupRow] = []
162+
163+
for group_id in all_ids:
164+
side_a = side_a_groups.get(group_id, _empty_counts())
165+
side_b = side_b_groups.get(group_id, _empty_counts())
166+
label = label_fn(group_id) if label_fn else group_id
167+
rows.append(
168+
CompareGroupRow(
169+
id=group_id,
170+
label=label,
171+
sideA=side_a,
172+
sideB=side_b,
173+
delta=_make_delta(side_a, side_b),
174+
)
175+
)
176+
177+
return rows
178+
179+
180+
def _test_group_label(group_id: str) -> str:
181+
if "/" in group_id:
182+
path_group, arch = group_id.split("/", 1)
183+
return f"{path_group} · {arch}"
184+
return group_id
185+
186+
187+
def build_compare_response(
188+
*,
189+
hash_a: str,
190+
hash_b: str,
191+
tree_name: str,
192+
branch: str,
193+
git_url: str,
194+
accumulators: dict[str, _HashAccumulator],
195+
) -> TreeCompareResponse:
196+
acc_a = accumulators.get(hash_a, _HashAccumulator())
197+
acc_b = accumulators.get(hash_b, _HashAccumulator())
198+
199+
return TreeCompareResponse(
200+
treeName=tree_name,
201+
branch=branch,
202+
gitUrl=git_url,
203+
summary=CompareSummary(
204+
builds=_make_entity_summary(side_a=acc_a.builds, side_b=acc_b.builds),
205+
boots=_make_entity_summary(side_a=acc_a.boots, side_b=acc_b.boots),
206+
tests=_make_entity_summary(side_a=acc_a.tests, side_b=acc_b.tests),
207+
),
208+
groups=CompareGroups(
209+
builds=_make_group_rows(
210+
group_maps=(acc_a.build_groups, acc_b.build_groups),
211+
),
212+
boots=_make_group_rows(
213+
group_maps=(acc_a.boot_groups, acc_b.boot_groups),
214+
),
215+
tests=_make_group_rows(
216+
group_maps=(acc_a.test_groups, acc_b.test_groups),
217+
label_fn=_test_group_label,
218+
),
219+
),
220+
)

‎backend/kernelCI_app/queries/tree.py‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1204,3 +1204,175 @@ def get_latest_tree(
12041204
query = query.order_by("-start_time").first()
12051205

12061206
return query
1207+
1208+
1209+
def _get_compare_checkout_clauses(
1210+
*,
1211+
git_branch_param: Optional[str],
1212+
tree_name: Optional[str],
1213+
) -> tuple[str, str]:
1214+
checkout_clauses = create_checkouts_where_clauses(
1215+
git_url=None,
1216+
git_branch=git_branch_param,
1217+
tree_name=tree_name,
1218+
)
1219+
1220+
git_branch_clause = checkout_clauses.get("git_branch_clause")
1221+
tree_name_clause = checkout_clauses.get("tree_name_clause")
1222+
tree_name_full_clause = "AND " + tree_name_clause if tree_name_clause else ""
1223+
1224+
return git_branch_clause, tree_name_full_clause
1225+
1226+
1227+
def get_tree_compare_rollup(
1228+
*,
1229+
commit_hashes: list[str],
1230+
origin_param: str,
1231+
git_branch_param: Optional[str],
1232+
tree_name: Optional[str] = None,
1233+
) -> list[dict]:
1234+
if not commit_hashes:
1235+
return []
1236+
1237+
cache_key = "treeCompareRollup"
1238+
params = {
1239+
"commit_hashes": commit_hashes,
1240+
"tree_name": tree_name,
1241+
"origin_param": origin_param,
1242+
"git_branch_param": git_branch_param,
1243+
}
1244+
1245+
rows = get_query_cache(cache_key, params)
1246+
if rows is not None:
1247+
return rows
1248+
1249+
git_branch_clause, tree_name_full_clause = _get_compare_checkout_clauses(
1250+
git_branch_param=git_branch_param,
1251+
tree_name=tree_name,
1252+
)
1253+
1254+
query = f"""
1255+
WITH RELEVANT_CHECKOUTS AS (
1256+
SELECT DISTINCT ON (c.git_commit_hash)
1257+
c.git_commit_hash,
1258+
c.tree_name,
1259+
c.git_repository_branch,
1260+
c.git_repository_url,
1261+
c.origin
1262+
FROM
1263+
checkouts c
1264+
WHERE
1265+
c.git_commit_hash = ANY(%(commit_hashes)s)
1266+
{tree_name_full_clause}
1267+
AND {git_branch_clause}
1268+
AND c.origin = %(origin_param)s
1269+
ORDER BY
1270+
c.git_commit_hash,
1271+
c._timestamp DESC
1272+
)
1273+
SELECT
1274+
tr.git_commit_hash,
1275+
tr.path_group,
1276+
tr.build_architecture,
1277+
tr.hardware_key,
1278+
tr.test_platform,
1279+
tr.is_boot,
1280+
tr.pass_tests,
1281+
tr.fail_tests,
1282+
tr.skip_tests,
1283+
tr.error_tests,
1284+
tr.miss_tests,
1285+
tr.done_tests,
1286+
tr.null_tests,
1287+
tr.total_tests
1288+
FROM
1289+
tree_tests_rollup tr
1290+
INNER JOIN RELEVANT_CHECKOUTS rc ON (
1291+
tr.git_commit_hash = rc.git_commit_hash
1292+
AND tr.origin = rc.origin
1293+
AND tr.tree_name IS NOT DISTINCT FROM rc.tree_name
1294+
AND tr.git_repository_branch IS NOT DISTINCT FROM rc.git_repository_branch
1295+
AND tr.git_repository_url IS NOT DISTINCT FROM rc.git_repository_url
1296+
)
1297+
ORDER BY
1298+
tr.total_tests DESC
1299+
"""
1300+
1301+
with connection.cursor() as cursor:
1302+
cursor.execute(query, params)
1303+
rows = dict_fetchall(cursor=cursor)
1304+
set_query_cache(key=cache_key, params=params, rows=rows)
1305+
1306+
return rows
1307+
1308+
1309+
def get_tree_compare_builds(
1310+
*,
1311+
commit_hashes: list[str],
1312+
origin_param: str,
1313+
git_branch_param: Optional[str],
1314+
tree_name: Optional[str] = None,
1315+
) -> list[dict]:
1316+
if not commit_hashes:
1317+
return []
1318+
1319+
cache_key = "treeCompareBuilds"
1320+
params = {
1321+
"commit_hashes": commit_hashes,
1322+
"tree_name": tree_name,
1323+
"origin_param": origin_param,
1324+
"git_branch_param": git_branch_param,
1325+
}
1326+
1327+
rows = get_query_cache(cache_key, params)
1328+
if rows is not None:
1329+
return rows
1330+
1331+
git_branch_clause, tree_name_full_clause = _get_compare_checkout_clauses(
1332+
git_branch_param=git_branch_param,
1333+
tree_name=tree_name,
1334+
)
1335+
1336+
query = f"""
1337+
WITH RELEVANT_CHECKOUTS AS (
1338+
SELECT DISTINCT ON (c.git_commit_hash)
1339+
c.id AS checkout_id,
1340+
c.git_commit_hash,
1341+
c.git_repository_url
1342+
FROM
1343+
checkouts c
1344+
WHERE
1345+
c.git_commit_hash = ANY(%(commit_hashes)s)
1346+
{tree_name_full_clause}
1347+
AND {git_branch_clause}
1348+
AND c.origin = %(origin_param)s
1349+
ORDER BY
1350+
c.git_commit_hash,
1351+
c._timestamp DESC
1352+
)
1353+
SELECT
1354+
rc.git_commit_hash,
1355+
rc.git_repository_url,
1356+
b.config_name,
1357+
b.status,
1358+
COUNT(DISTINCT b.id) AS count
1359+
FROM
1360+
RELEVANT_CHECKOUTS rc
1361+
INNER JOIN builds b ON b.checkout_id = rc.checkout_id
1362+
WHERE
1363+
b.config_name IS NOT NULL
1364+
AND b.id NOT LIKE 'maestro:dummy_%%'
1365+
GROUP BY
1366+
rc.git_commit_hash,
1367+
rc.git_repository_url,
1368+
b.config_name,
1369+
b.status
1370+
"""
1371+
1372+
with connection.cursor() as cursor:
1373+
cursor.execute(query, params)
1374+
rows = dict_fetchall(cursor=cursor)
1375+
set_query_cache(key=cache_key, params=params, rows=rows)
1376+
1377+
return rows
1378+

0 commit comments

Comments
 (0)