Skip to content

Commit 9609ed8

Browse files
committed
Add standard deviation to grants review recap for vote divergence
Add `std_dev` annotation to the grants review recap view to help identify controversial grants that need discussion during review calls. Standard deviation measures how much reviewers disagree: - High std dev = genuine disagreement across reviewers - Low std dev = consensus, grant can be quickly accepted/rejected This helps prioritize which grants to discuss in review meetings by surfacing cases where reviewers have diverging opinions.
1 parent d026a87 commit 9609ed8

3 files changed

Lines changed: 130 additions & 0 deletions

File tree

backend/reviews/admin.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
OuterRef,
1414
Prefetch,
1515
Q,
16+
StdDev,
1617
Subquery,
1718
Sum,
1819
)
@@ -426,6 +427,10 @@ def _review_grants_recap_view(self, request, review_session):
426427
F("total_score") / F("vote_count"),
427428
output_field=FloatField(),
428429
),
430+
std_dev=StdDev(
431+
"userreview__score__numeric_value",
432+
filter=Q(userreview__review_session_id=review_session_id),
433+
),
429434
has_sent_a_proposal=Exists(
430435
Submission.objects.non_cancelled().filter(
431436
speaker_id=OuterRef("user_id"),

backend/reviews/templates/grants-recap.html

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,6 +432,12 @@ <h3>
432432
</div>
433433
<div class="clear"></div>
434434
</th>
435+
<th scope="col">
436+
<div class="text">
437+
<span>Std Dev</span>
438+
</div>
439+
<div class="clear"></div>
440+
</th>
435441
<th scope="col">
436442
<div class="text">
437443
<span>Votes</span>
@@ -573,6 +579,7 @@ <h3>
573579
</ul>
574580
</td>
575581
<td>{{ item.score }}</td>
582+
<td>{{ item.std_dev|floatformat:2 }}</td>
576583
<td class="votes-list">
577584
<ul>
578585
{% for reviewer in item.userreview_set.all %}

backend/reviews/tests/test_admin.py

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,124 @@ def test_grants_review_scores(rf, scores, avg):
191191
assert grant_to_check.score == avg
192192

193193

194+
@pytest.mark.parametrize(
195+
"scores, expected_score_range",
196+
[
197+
# Multiple different scores: range = max - min = 2 - (-2) = 4
198+
(
199+
[
200+
{"user": 0, "score": 2},
201+
{"user": 1, "score": 2},
202+
{"user": 2, "score": 2},
203+
{"user": 3, "score": -1},
204+
{"user": 4, "score": -2},
205+
],
206+
4.0,
207+
),
208+
# All same scores: range = -2 - (-2) = 0
209+
(
210+
[
211+
{"user": 0, "score": -2},
212+
{"user": 1, "score": -2},
213+
{"user": 2, "score": -2},
214+
{"user": 3, "score": -2},
215+
{"user": 4, "score": -2},
216+
],
217+
0.0,
218+
),
219+
# Single score: std_dev = 0
220+
(
221+
[
222+
{"user": 0, "score": 1},
223+
],
224+
0.0,
225+
),
226+
# No scores: std_dev = None
227+
([], None),
228+
# Two different scores (1 and -1): mean=0, std_dev = sqrt(((1-0)^2 + (-1-0)^2) / 2) = 1.0
229+
(
230+
[
231+
{"user": 0, "score": 1},
232+
{"user": 1, "score": -1},
233+
],
234+
1.0,
235+
),
236+
# Three scores with same value: std_dev = 0
237+
(
238+
[
239+
{"user": 0, "score": 1},
240+
{"user": 1, "score": 1},
241+
{"user": 2, "score": 1},
242+
],
243+
0.0,
244+
),
245+
# Mixed scores showing consensus with outlier: 3x score=1, 1x score=-2
246+
# mean = (1+1+1-2)/4 = 0.25
247+
# std_dev = sqrt(((1-0.25)^2 + (1-0.25)^2 + (1-0.25)^2 + (-2-0.25)^2) / 4)
248+
# = sqrt((0.5625 + 0.5625 + 0.5625 + 5.0625) / 4) = sqrt(1.6875) ≈ 1.299
249+
(
250+
[
251+
{"user": 0, "score": 1},
252+
{"user": 1, "score": 1},
253+
{"user": 2, "score": 1},
254+
{"user": 3, "score": -2},
255+
],
256+
1.299,
257+
),
258+
],
259+
)
260+
def test_grants_review_std_dev(rf, scores, expected_std_dev):
261+
conference = ConferenceFactory()
262+
review_session = ReviewSessionFactory(
263+
conference=conference,
264+
session_type=ReviewSession.SessionType.GRANTS,
265+
status=ReviewSession.Status.COMPLETED,
266+
)
267+
268+
users = UserFactory.create_batch(10, is_staff=True, is_superuser=True)
269+
all_scores = {
270+
-2: AvailableScoreOptionFactory(
271+
review_session=review_session, numeric_value=-2, label="Rejected"
272+
),
273+
-1: AvailableScoreOptionFactory(
274+
review_session=review_session, numeric_value=-1, label="Not convinced"
275+
),
276+
0: AvailableScoreOptionFactory(
277+
review_session=review_session, numeric_value=0, label="Maybe"
278+
),
279+
1: AvailableScoreOptionFactory(
280+
review_session=review_session, numeric_value=1, label="Yes"
281+
),
282+
2: AvailableScoreOptionFactory(
283+
review_session=review_session, numeric_value=2, label="Absolutely"
284+
),
285+
}
286+
287+
grant = GrantFactory(conference=conference)
288+
for score in scores:
289+
UserReviewFactory(
290+
review_session=review_session,
291+
grant=grant,
292+
user=users[score["user"]],
293+
score=all_scores[score["score"]],
294+
)
295+
296+
request = rf.get("/")
297+
request.user = users[5]
298+
299+
admin = ReviewSessionAdmin(ReviewSession, AdminSite())
300+
response = admin._review_grants_recap_view(request, review_session)
301+
context_data = response.context_data
302+
items = context_data["items"]
303+
grant_to_check = next(item for item in items if item.id == grant.id)
304+
305+
assert grant_to_check.id == grant.id
306+
if expected_std_dev is None:
307+
assert grant_to_check.std_dev is None
308+
else:
309+
assert grant_to_check.std_dev == pytest.approx(expected_std_dev, abs=0.01)
310+
311+
194312
def test_review_start_view_when_no_items_are_left(rf, mocker):
195313
mock_messages = mocker.patch("reviews.admin.messages")
196314

0 commit comments

Comments
 (0)