Skip to content

Commit 3d43025

Browse files
jaredlockhartjaredlockhart
authored
feat(nimbus): add usage stats CSV report for owners and reviewers (#15131)
Because * We have a per-experiment CSV report but no report tracking platform user engagement over time * Product needs monthly and quarterly metrics on unique owners, reviewers, and experience levels This commit * Adds a new `/api/v5/csv/usage/` endpoint that generates a usage stats CSV with columns: Month, Owners, Reviewers, Novice (0-3), Intermediate (4-9), Advanced (10+) * Tracks cumulative unique experiment owners and reviewers per month * Buckets owners by experience level based on running experiment count * Includes quarterly rollup rows at the end of the CSV * Adds a "Usage Stats" link in the header dropdown * Adds tests for the new view Fixes #15128
1 parent 9c152c0 commit 3d43025

6 files changed

Lines changed: 295 additions & 1 deletion

File tree

‎docs/experimenter/openapi-schema.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,29 @@
6060
]
6161
}
6262
},
63+
"/api/v5/csv/usage/": {
64+
"get": {
65+
"operationId": "listNimbusExperimentUsageStats",
66+
"description": "",
67+
"parameters": [],
68+
"responses": {
69+
"200": {
70+
"content": {
71+
"application/json": {
72+
"schema": {
73+
"type": "array",
74+
"items": {}
75+
}
76+
}
77+
},
78+
"description": ""
79+
}
80+
},
81+
"tags": [
82+
"api"
83+
]
84+
}
85+
},
6386
"/api/v5/yaml/": {
6487
"get": {
6588
"operationId": "listNimbusExperiments",

‎docs/experimenter/swagger-ui.html‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,29 @@
7272
]
7373
}
7474
},
75+
"/api/v5/csv/usage/": {
76+
"get": {
77+
"operationId": "listNimbusExperimentUsageStats",
78+
"description": "",
79+
"parameters": [],
80+
"responses": {
81+
"200": {
82+
"content": {
83+
"application/json": {
84+
"schema": {
85+
"type": "array",
86+
"items": {}
87+
}
88+
}
89+
},
90+
"description": ""
91+
}
92+
},
93+
"tags": [
94+
"api"
95+
]
96+
}
97+
},
7598
"/api/v5/yaml/": {
7699
"get": {
77100
"operationId": "listNimbusExperiments",

‎experimenter/experimenter/experiments/api/v5/urls.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
from experimenter.experiments.api.v5.views import (
44
FmlErrorsView,
55
NimbusExperimentCsvListView,
6+
NimbusExperimentUsageStatsView,
67
NimbusExperimentYamlListView,
78
)
89

@@ -12,6 +13,11 @@
1213
NimbusExperimentCsvListView.as_view(),
1314
name="nimbus-experiments-csv",
1415
),
16+
re_path(
17+
r"^csv/usage/$",
18+
NimbusExperimentUsageStatsView.as_view(),
19+
name="nimbus-experiments-csv-usage",
20+
),
1521
re_path(
1622
r"^yaml/$",
1723
NimbusExperimentYamlListView.as_view(),

‎experimenter/experimenter/experiments/api/v5/views.py‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,16 @@
1+
import csv
2+
import datetime
3+
import io
4+
from collections import defaultdict
5+
16
import yaml
7+
from dateutil.relativedelta import relativedelta
28
from django.db.models import F
9+
from django.http import HttpResponse
310
from rest_framework.generics import ListAPIView, UpdateAPIView
411
from rest_framework.pagination import PageNumberPagination
512
from rest_framework.renderers import BaseRenderer
13+
from rest_framework.views import APIView
614
from rest_framework_csv.renderers import CSVRenderer
715

816
from experimenter.experiments.api.cache import CachedListMixin
@@ -11,7 +19,7 @@
1119
NimbusExperimentCsvSerializer,
1220
NimbusExperimentYamlSerializer,
1321
)
14-
from experimenter.experiments.models import NimbusExperiment
22+
from experimenter.experiments.models import NimbusChangeLog, NimbusExperiment
1523

1624

1725
class NimbusExperimentCsvRenderer(CSVRenderer):
@@ -41,6 +49,98 @@ def get_queryset(self):
4149
)
4250

4351

52+
class NimbusExperimentUsageStatsView(APIView):
53+
NOVICE_MAX = 3
54+
INTERMEDIATE_MAX = 9
55+
56+
@staticmethod
57+
def _generate_usage_csv():
58+
experiments = (
59+
NimbusExperiment.objects.filter(
60+
_start_date__isnull=False,
61+
status=NimbusExperiment.Status.COMPLETE,
62+
)
63+
.values_list("owner__email", "_start_date")
64+
.order_by("_start_date")
65+
)
66+
approvals = (
67+
NimbusChangeLog.objects.filter(
68+
old_publish_status=NimbusExperiment.PublishStatus.REVIEW,
69+
new_publish_status=NimbusExperiment.PublishStatus.APPROVED,
70+
experiment___start_date__isnull=False,
71+
)
72+
.values_list("changed_by__email", "experiment___start_date")
73+
.order_by("experiment___start_date")
74+
)
75+
76+
owner_events = defaultdict(list)
77+
for email, start_date in experiments:
78+
owner_events[start_date.strftime("%Y-%m")].append(email)
79+
80+
reviewer_events = defaultdict(set)
81+
for email, start_date in approvals:
82+
reviewer_events[start_date.strftime("%Y-%m")].add(email)
83+
84+
if not owner_events and not reviewer_events:
85+
return ""
86+
87+
last_month = max(set(owner_events) | set(reviewer_events))
88+
end_date = datetime.datetime.strptime(last_month, "%Y-%m").date()
89+
90+
seen_owners = set()
91+
seen_reviewers = set()
92+
counts = defaultdict(int)
93+
rows = []
94+
current = datetime.date(2020, 1, 1)
95+
novice_max = NimbusExperimentUsageStatsView.NOVICE_MAX
96+
intermediate_max = NimbusExperimentUsageStatsView.INTERMEDIATE_MAX
97+
while current <= end_date:
98+
month = current.strftime("%Y-%m")
99+
for email in owner_events.get(month, []):
100+
seen_owners.add(email)
101+
counts[email] += 1
102+
seen_reviewers |= reviewer_events.get(month, set())
103+
104+
experience_levels = [0, 0, 0]
105+
for count in counts.values():
106+
experience_levels[
107+
0 if count <= novice_max else 1 if count <= intermediate_max else 2
108+
] += 1
109+
110+
rows.append(
111+
[month, len(seen_owners), len(seen_reviewers), *experience_levels]
112+
)
113+
current += relativedelta(months=1)
114+
115+
output = io.StringIO()
116+
header = [
117+
"Month",
118+
"Owners",
119+
"Reviewers",
120+
f"Novice (0-{novice_max})",
121+
f"Intermediate ({novice_max + 1}-{intermediate_max})",
122+
f"Advanced ({intermediate_max + 1}+)",
123+
]
124+
writer = csv.writer(output)
125+
writer.writerow(header)
126+
quarter_ends = {}
127+
for row in rows:
128+
writer.writerow(row)
129+
row_date = datetime.datetime.strptime(row[0], "%Y-%m").date()
130+
quarter = f"{row_date.year} Q{(row_date.month - 1) // 3 + 1}"
131+
quarter_ends[quarter] = row
132+
133+
writer.writerow([])
134+
for quarter in sorted(quarter_ends):
135+
writer.writerow([quarter, *quarter_ends[quarter][1:]])
136+
137+
return output.getvalue()
138+
139+
def get(self, request):
140+
csv_content = self._generate_usage_csv()
141+
return HttpResponse(csv_content, content_type="text/csv; charset=utf-8")
142+
143+
44144
class NimbusExperimentYamlRenderer(BaseRenderer):
45145
media_type = "text/yaml"
46146
format = "yaml"

‎experimenter/experimenter/experiments/tests/api/v5/test_usage_stats_view.py‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
import csv
2+
import datetime
3+
import io
4+
5+
from django.conf import settings
6+
from django.test import TestCase
7+
from django.urls import reverse
8+
9+
from experimenter.experiments.models import NimbusChangeLog, NimbusExperiment
10+
from experimenter.experiments.tests.factories import NimbusExperimentFactory
11+
from experimenter.openidc.tests.factories import UserFactory
12+
13+
LIFECYCLE = NimbusExperimentFactory.Lifecycles.ENDING_APPROVE_APPROVE
14+
15+
16+
def _create_experiment(owner, start_date):
17+
return NimbusExperimentFactory.create_with_lifecycle(
18+
LIFECYCLE,
19+
start_date=start_date,
20+
end_date=start_date + datetime.timedelta(days=28),
21+
owner=owner,
22+
)
23+
24+
25+
class TestNimbusExperimentUsageStatsView(TestCase):
26+
def _get_csv_rows(self):
27+
response = self.client.get(
28+
reverse("nimbus-experiments-csv-usage"),
29+
**{settings.OPENIDC_EMAIL_HEADER: "user@example.com"},
30+
)
31+
self.assertEqual(response.status_code, 200)
32+
self.assertEqual(response["Content-Type"], "text/csv; charset=utf-8")
33+
content = response.content.decode("utf-8")
34+
reader = csv.DictReader(io.StringIO(content))
35+
return list(reader)
36+
37+
@staticmethod
38+
def _month_index(year, month):
39+
return (year - 2020) * 12 + (month - 1)
40+
41+
def test_empty_database_returns_empty_csv(self):
42+
response = self.client.get(
43+
reverse("nimbus-experiments-csv-usage"),
44+
**{settings.OPENIDC_EMAIL_HEADER: "user@example.com"},
45+
)
46+
self.assertEqual(response.status_code, 200)
47+
self.assertEqual(response.content.decode("utf-8"), "")
48+
49+
def test_single_experiment_counts_owner(self):
50+
owner = UserFactory.create(email="owner1@example.com")
51+
_create_experiment(owner, datetime.date(2023, 3, 15))
52+
53+
rows = self._get_csv_rows()
54+
55+
self.assertEqual(len(rows), 52)
56+
self.assertEqual(rows[0]["Month"], "2020-01")
57+
58+
march = rows[self._month_index(2023, 3)]
59+
self.assertEqual(march["Month"], "2023-03")
60+
self.assertEqual(march["Owners"], "1")
61+
self.assertEqual(march["Novice (0-3)"], "1")
62+
self.assertEqual(march["Intermediate (4-9)"], "0")
63+
self.assertEqual(march["Advanced (10+)"], "0")
64+
65+
def test_cumulative_owners_across_months(self):
66+
owner1 = UserFactory.create(email="owner1@example.com")
67+
owner2 = UserFactory.create(email="owner2@example.com")
68+
69+
_create_experiment(owner1, datetime.date(2023, 1, 10))
70+
_create_experiment(owner2, datetime.date(2023, 3, 10))
71+
72+
rows = self._get_csv_rows()
73+
74+
self.assertEqual(rows[self._month_index(2023, 1)]["Owners"], "1")
75+
self.assertEqual(rows[self._month_index(2023, 2)]["Owners"], "1")
76+
self.assertEqual(rows[self._month_index(2023, 3)]["Owners"], "2")
77+
78+
def test_reviewer_counted_from_approval_changelog(self):
79+
owner = UserFactory.create(email="owner@example.com")
80+
reviewer = UserFactory.create(email="reviewer@example.com")
81+
82+
experiment = _create_experiment(owner, datetime.date(2023, 6, 1))
83+
84+
rows = self._get_csv_rows()
85+
self.assertEqual(rows[self._month_index(2023, 6)]["Reviewers"], "1")
86+
87+
NimbusChangeLog.objects.create(
88+
experiment=experiment,
89+
changed_by=reviewer,
90+
old_status=NimbusExperiment.Status.DRAFT,
91+
new_status=NimbusExperiment.Status.DRAFT,
92+
old_publish_status=NimbusExperiment.PublishStatus.REVIEW,
93+
new_publish_status=NimbusExperiment.PublishStatus.APPROVED,
94+
)
95+
96+
rows = self._get_csv_rows()
97+
self.assertEqual(rows[self._month_index(2023, 6)]["Reviewers"], "2")
98+
99+
def test_experience_levels_advance_with_experiment_count(self):
100+
owner = UserFactory.create(email="prolific@example.com")
101+
102+
for i in range(12):
103+
_create_experiment(
104+
owner,
105+
datetime.date(2023, 1, 1) + datetime.timedelta(days=30 * i),
106+
)
107+
108+
rows = self._get_csv_rows()
109+
110+
jan = rows[self._month_index(2023, 1)]
111+
self.assertEqual(jan["Novice (0-3)"], "1")
112+
self.assertEqual(jan["Intermediate (4-9)"], "0")
113+
114+
apr = rows[self._month_index(2023, 4)]
115+
self.assertEqual(apr["Novice (0-3)"], "0")
116+
self.assertEqual(apr["Intermediate (4-9)"], "1")
117+
118+
oct_row = rows[self._month_index(2023, 10)]
119+
self.assertEqual(oct_row["Intermediate (4-9)"], "0")
120+
self.assertEqual(oct_row["Advanced (10+)"], "1")
121+
122+
def test_quarterly_rollups(self):
123+
owner = UserFactory.create(email="owner@example.com")
124+
_create_experiment(owner, datetime.date(2023, 6, 1))
125+
126+
rows = self._get_csv_rows()
127+
128+
self.assertEqual(len(rows), 56)
129+
self.assertEqual(rows[42]["Month"], "2020 Q1")
130+
self.assertEqual(rows[42]["Owners"], "0")
131+
132+
self.assertEqual(rows[-1]["Month"], "2023 Q2")
133+
self.assertEqual(rows[-1]["Owners"], "1")

‎experimenter/experimenter/nimbus_ui/templates/common/header.html‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,15 @@
9090
Reports
9191
</a>
9292
</li>
93+
<li>
94+
<a href="{% url "nimbus-experiments-csv-usage" %}"
95+
class="dropdown-item link-primary"
96+
target="_blank"
97+
rel="noreferrer">
98+
<i class="fa-solid fa-chart-line me-2"></i>
99+
Usage Stats
100+
</a>
101+
</li>
93102
<li>
94103
<a href="{% url "nimbus-experiments-yaml" %}"
95104
class="dropdown-item link-primary"

0 commit comments

Comments
 (0)