Skip to content

Commit ea9b1ed

Browse files
robot-ci-heartexfern-api[bot]matt-bernstein
authored
chore: ROOT-180: Stats-related endpoints in SDK (#588)
Co-authored-by: fern-api <115122769+fern-api[bot]@users.noreply.github.com> Co-authored-by: Matt Bernstein <matt@humansignal.com>
1 parent 0c3c1c7 commit ea9b1ed

24 files changed

Lines changed: 2742 additions & 416 deletions

‎.mock/definition/projects/stats.yml‎

Lines changed: 355 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,77 @@ types:
6161
docs: List of users in the matrix
6262
source:
6363
openapi: openapi/openapi.yaml
64+
StatsAgreementAnnotatorResponse:
65+
properties:
66+
Agreement_per_annotator:
67+
type: optional<double>
68+
docs: Agreement score for the annotator (0-1)
69+
source:
70+
openapi: openapi/openapi.yaml
71+
StatsDataFiltersResponseUserFiltersStatsItem:
72+
properties:
73+
id:
74+
type: optional<string>
75+
docs: User ID or model version identifier (e.g., "model:1.0")
76+
source:
77+
openapi: openapi/openapi.yaml
78+
inline: true
79+
StatsDataFiltersResponseUserFilters:
80+
docs: Data filter statistics by user and model
81+
properties:
82+
stats:
83+
type: optional<list<StatsDataFiltersResponseUserFiltersStatsItem>>
84+
docs: List of filter configurations for users and models
85+
tasks_with_annotations:
86+
type: optional<map<string, unknown>>
87+
docs: Default filter tab for tasks with annotations
88+
source:
89+
openapi: openapi/openapi.yaml
90+
inline: true
91+
StatsDataFiltersResponse:
92+
properties:
93+
user_filters:
94+
type: optional<StatsDataFiltersResponseUserFilters>
95+
docs: Data filter statistics by user and model
96+
source:
97+
openapi: openapi/openapi.yaml
98+
StatsFinishedTasksResponse:
99+
properties:
100+
finished:
101+
type: optional<integer>
102+
docs: Number of finished tasks
103+
id:
104+
type: optional<integer>
105+
docs: User ID
106+
progress:
107+
type: optional<integer>
108+
docs: Progress percentage (0-100)
109+
source:
110+
openapi: openapi/openapi.yaml
111+
StatsLeadTimeResponseLeadTimeStatsItem:
112+
properties:
113+
mean_time:
114+
type: optional<double>
115+
docs: Average lead time for the user
116+
median_time:
117+
type: optional<double>
118+
docs: Median lead time for the user
119+
sum_lead_time:
120+
type: optional<double>
121+
docs: Total lead time for the user
122+
user_id:
123+
type: optional<integer>
124+
docs: User ID
125+
source:
126+
openapi: openapi/openapi.yaml
127+
inline: true
128+
StatsLeadTimeResponse:
129+
properties:
130+
lead_time_stats:
131+
type: optional<list<StatsLeadTimeResponseLeadTimeStatsItem>>
132+
docs: Lead time statistics including mean, median, and distribution
133+
source:
134+
openapi: openapi/openapi.yaml
64135
StatsTotalAgreementResponseZero:
65136
properties:
66137
total_agreement: optional<double>
@@ -80,6 +151,67 @@ types:
80151
- StatsTotalAgreementResponseOne
81152
source:
82153
openapi: openapi/openapi.yaml
154+
StatsUserPredictionAgreementResponseAveragePredictionAgreementPerUser:
155+
discriminated: false
156+
union:
157+
- type: double
158+
docs: >-
159+
Average prediction agreement score for the user (0-1) when
160+
per_label=False
161+
- type: map<string, double>
162+
docs: >-
163+
Average prediction agreement score per label for the user (0-1) when
164+
per_label=True
165+
source:
166+
openapi: openapi/openapi.yaml
167+
inline: true
168+
StatsUserPredictionAgreementResponse:
169+
properties:
170+
average_prediction_agreement_per_user: >-
171+
optional<StatsUserPredictionAgreementResponseAveragePredictionAgreementPerUser>
172+
source:
173+
openapi: openapi/openapi.yaml
174+
StatsUserReviewScoreResponsePerformanceScore:
175+
discriminated: false
176+
union:
177+
- type: double
178+
docs: Performance score for the user when per_label=False
179+
- type: map<string, double>
180+
docs: Performance score per label for the user when per_label=True
181+
source:
182+
openapi: openapi/openapi.yaml
183+
inline: true
184+
StatsUserReviewScoreResponseReviewScore:
185+
discriminated: false
186+
union:
187+
- type: double
188+
docs: Average review score for the user when per_label=False
189+
- type: map<string, double>
190+
docs: Average review score per label for the user when per_label=True
191+
source:
192+
openapi: openapi/openapi.yaml
193+
inline: true
194+
StatsUserReviewScoreResponse:
195+
properties:
196+
performance_score: optional<StatsUserReviewScoreResponsePerformanceScore>
197+
review_score: optional<StatsUserReviewScoreResponseReviewScore>
198+
source:
199+
openapi: openapi/openapi.yaml
200+
StatsUserGroundTruthAgreementResponseAgreement:
201+
discriminated: false
202+
union:
203+
- type: double
204+
docs: Ground truth agreement score for the user (0-1) when per_label=False
205+
- type: map<string, double>
206+
docs: Ground truth agreement scores per label when per_label=True
207+
source:
208+
openapi: openapi/openapi.yaml
209+
inline: true
210+
StatsUserGroundTruthAgreementResponse:
211+
properties:
212+
agreement: optional<StatsUserGroundTruthAgreementResponseAgreement>
213+
source:
214+
openapi: openapi/openapi.yaml
83215
service:
84216
auth: false
85217
base-path: ''
@@ -175,6 +307,110 @@ service:
175307
id: 2
176308
audiences:
177309
- public
310+
agreement_annotator:
311+
path: /api/projects/{id}/stats/agreement_annotator/{user_id}
312+
method: GET
313+
auth: true
314+
docs: Get agreement statistics for a specific annotator within a project.
315+
source:
316+
openapi: openapi/openapi.yaml
317+
path-parameters:
318+
id: integer
319+
user_id: integer
320+
display-name: Get individual annotator agreement stats
321+
response:
322+
docs: Individual annotator agreement statistics
323+
type: StatsAgreementAnnotatorResponse
324+
examples:
325+
- path-parameters:
326+
id: 1
327+
user_id: 1
328+
response:
329+
body:
330+
Agreement_per_annotator: 1.1
331+
audiences:
332+
- public
333+
data_filters:
334+
path: /api/projects/{id}/stats/data_filter
335+
method: GET
336+
auth: true
337+
docs: Get statistics about user data filters and their usage within a project.
338+
source:
339+
openapi: openapi/openapi.yaml
340+
path-parameters:
341+
id: integer
342+
display-name: Get user data filter statistics
343+
response:
344+
docs: User data filter statistics
345+
type: StatsDataFiltersResponse
346+
examples:
347+
- path-parameters:
348+
id: 1
349+
response:
350+
body:
351+
user_filters:
352+
stats:
353+
- {}
354+
tasks_with_annotations:
355+
key: value
356+
audiences:
357+
- public
358+
finished_tasks:
359+
path: /api/projects/{id}/stats/finished
360+
method: GET
361+
auth: true
362+
docs: Get statistics about finished tasks for a project.
363+
source:
364+
openapi: openapi/openapi.yaml
365+
path-parameters:
366+
id: integer
367+
display-name: Get finished tasks statistics
368+
request:
369+
name: StatsFinishedTasksRequest
370+
query-parameters:
371+
user_pk:
372+
type: optional<integer>
373+
docs: User ID to filter statistics by (optional)
374+
response:
375+
docs: Finished tasks statistics
376+
type: StatsFinishedTasksResponse
377+
examples:
378+
- path-parameters:
379+
id: 1
380+
response:
381+
body:
382+
finished: 1
383+
id: 1
384+
progress: 1
385+
audiences:
386+
- public
387+
lead_time:
388+
path: /api/projects/{id}/stats/lead_time
389+
method: GET
390+
auth: true
391+
docs: >-
392+
Get lead time statistics across the project, including average
393+
annotation time.
394+
source:
395+
openapi: openapi/openapi.yaml
396+
path-parameters:
397+
id: integer
398+
display-name: Get lead time statistics
399+
response:
400+
docs: Lead time statistics
401+
type: StatsLeadTimeResponse
402+
examples:
403+
- path-parameters:
404+
id: 1
405+
response:
406+
body:
407+
lead_time_stats:
408+
- mean_time: 1.1
409+
median_time: 1.1
410+
sum_lead_time: 1.1
411+
user_id: 1
412+
audiences:
413+
- public
178414
total_agreement:
179415
path: /api/projects/{id}/stats/total_agreement
180416
method: GET
@@ -208,5 +444,124 @@ service:
208444
total_agreement: 1.1
209445
audiences:
210446
- public
447+
update_stats:
448+
path: /api/projects/{id}/update-stats
449+
method: GET
450+
auth: true
451+
docs: Start stats recalculation for given project
452+
source:
453+
openapi: openapi/openapi.yaml
454+
path-parameters:
455+
id: integer
456+
display-name: Start stats recalculation
457+
request:
458+
name: StatsUpdateStatsRequest
459+
query-parameters:
460+
stat_type:
461+
type: optional<string>
462+
docs: 'Stat type to recalculate. Possible values: label, stats'
463+
response:
464+
docs: Successful response returns job id
465+
type: map<string, unknown>
466+
examples:
467+
- path-parameters:
468+
id: 1
469+
response:
470+
body:
471+
key: value
472+
audiences:
473+
- public
474+
user_prediction_agreement:
475+
path: /api/projects/{id}/user-stats/{user_pk}/prediction
476+
method: GET
477+
auth: true
478+
docs: >-
479+
Get prediction agreement statistics for a specific user within a
480+
project.
481+
source:
482+
openapi: openapi/openapi.yaml
483+
path-parameters:
484+
id: integer
485+
user_pk: integer
486+
display-name: Get individual user prediction agreement
487+
request:
488+
name: StatsUserPredictionAgreementRequest
489+
query-parameters:
490+
per_label:
491+
type: optional<boolean>
492+
docs: Calculate agreement per label
493+
response:
494+
docs: Individual user prediction agreement statistics
495+
type: StatsUserPredictionAgreementResponse
496+
examples:
497+
- path-parameters:
498+
id: 1
499+
user_pk: 1
500+
response:
501+
body:
502+
average_prediction_agreement_per_user: 1.1
503+
audiences:
504+
- public
505+
user_review_score:
506+
path: /api/projects/{id}/user-stats/{user_pk}/review_score
507+
method: GET
508+
auth: true
509+
docs: Get review score statistics for a specific user within a project.
510+
source:
511+
openapi: openapi/openapi.yaml
512+
path-parameters:
513+
id: integer
514+
user_pk: integer
515+
display-name: Get individual user review scores
516+
request:
517+
name: StatsUserReviewScoreRequest
518+
query-parameters:
519+
per_label:
520+
type: optional<boolean>
521+
docs: Calculate agreement per label
522+
response:
523+
docs: Individual user review score statistics
524+
type: StatsUserReviewScoreResponse
525+
examples:
526+
- path-parameters:
527+
id: 1
528+
user_pk: 1
529+
response:
530+
body:
531+
performance_score: 1.1
532+
review_score: 1.1
533+
audiences:
534+
- public
535+
user_ground_truth_agreement:
536+
path: /api/projects/{id}/users/{user_pk}/stats/agreement-groundtruth
537+
method: GET
538+
auth: true
539+
docs: >-
540+
Get ground truth agreement statistics for a specific user within a
541+
project.
542+
source:
543+
openapi: openapi/openapi.yaml
544+
path-parameters:
545+
id: integer
546+
user_pk: integer
547+
display-name: Get individual user ground truth agreement
548+
request:
549+
name: StatsUserGroundTruthAgreementRequest
550+
query-parameters:
551+
per_label:
552+
type: optional<boolean>
553+
docs: Calculate agreement per label
554+
response:
555+
docs: Individual user ground truth agreement statistics
556+
type: StatsUserGroundTruthAgreementResponse
557+
examples:
558+
- path-parameters:
559+
id: 1
560+
user_pk: 1
561+
response:
562+
body:
563+
agreement: 1.1
564+
audiences:
565+
- public
211566
source:
212567
openapi: openapi/openapi.yaml

0 commit comments

Comments
 (0)