Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions server_api/workflow/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
"""Workflow proposal helpers."""

from .correction_preview import (
CORRECTION_RATE_PROCEED_THRESHOLD,
RECENT_CORRECTION_EVENT_TYPES,
RECENT_EXPORT_EVENT_TYPES,
build_preview_correction_impact_proposal,
)

__all__ = [
"CORRECTION_RATE_PROCEED_THRESHOLD",
"RECENT_CORRECTION_EVENT_TYPES",
"RECENT_EXPORT_EVENT_TYPES",
"build_preview_correction_impact_proposal",
]
71 changes: 71 additions & 0 deletions server_api/workflow/correction_preview.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
"""Rule-based proposal for evaluating correction impact before retraining."""

from __future__ import annotations

from typing import Any

PREVIEW_CORRECTION_IMPACT_PROPOSAL_TYPE = "preview_correction_impact"
RECENT_CORRECTION_EVENT_TYPES = frozenset(
{
"correction_applied",
"instance_corrected",
"mask_corrected",
"proofreading_saved",
}
)
RECENT_EXPORT_EVENT_TYPES = frozenset({"export_created", "masks_exported"})
CORRECTION_RATE_PROCEED_THRESHOLD = 0.10


def _count_events(events: list[dict[str, Any]], event_types: frozenset[str]) -> int:
return sum(1 for event in events if event.get("event_type") in event_types)


def build_preview_correction_impact_proposal(
events: list[dict[str, Any]],
) -> dict[str, Any]:
"""Build a transparent recommendation payload from recent workflow events.

This helper is intentionally pure: it reads events and returns a proposal payload
without mutating any external workflow state.
"""

total_events = len(events)
correction_events = _count_events(events, RECENT_CORRECTION_EVENT_TYPES)
export_events = _count_events(events, RECENT_EXPORT_EVENT_TYPES)

correction_rate = (
correction_events / total_events if total_events else 0.0
)
should_proceed = correction_rate <= CORRECTION_RATE_PROCEED_THRESHOLD
recommendation = "proceed" if should_proceed else "continue_proofreading"

rationale = (
"Proceed because correction activity is at or below the explicit threshold"
if should_proceed
else "Continue proofreading because correction activity exceeds the explicit threshold"
)

return {
"type": PREVIEW_CORRECTION_IMPACT_PROPOSAL_TYPE,
"summary": {
"total_recent_events": total_events,
"correction_related_events": correction_events,
"recent_exports": export_events,
"correction_rate": round(correction_rate, 4),
},
"recommendation": recommendation,
"rationale": {
"rule": "correction_rate <= CORRECTION_RATE_PROCEED_THRESHOLD",
"threshold": CORRECTION_RATE_PROCEED_THRESHOLD,
"computed": {
"correction_rate": round(correction_rate, 4),
"correction_events": correction_events,
"total_events": total_events,
},
"explanation": (
f"{rationale}. Corrections={correction_events}, total={total_events}, "
f"rate={round(correction_rate, 4)}, threshold={CORRECTION_RATE_PROCEED_THRESHOLD}."
),
},
}
58 changes: 58 additions & 0 deletions tests/test_workflow_correction_preview.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
from server_api.workflow.correction_preview import (
CORRECTION_RATE_PROCEED_THRESHOLD,
PREVIEW_CORRECTION_IMPACT_PROPOSAL_TYPE,
build_preview_correction_impact_proposal,
)


def test_preview_correction_impact_low_correction_recommends_proceed():
events = [
{"event_type": "masks_exported"},
{"event_type": "model_trained"},
{"event_type": "evaluation_completed"},
{"event_type": "correction_applied"},
{"event_type": "dataset_refreshed"},
{"event_type": "export_created"},
{"event_type": "model_trained"},
{"event_type": "metrics_uploaded"},
{"event_type": "run_archived"},
{"event_type": "checkpoint_saved"},
]

original_events = [dict(item) for item in events]
proposal = build_preview_correction_impact_proposal(events)

assert proposal["type"] == PREVIEW_CORRECTION_IMPACT_PROPOSAL_TYPE
assert proposal["summary"]["correction_related_events"] == 1
assert proposal["summary"]["recent_exports"] == 2
assert proposal["recommendation"] == "proceed"
assert (
proposal["rationale"]["rule"]
== "correction_rate <= CORRECTION_RATE_PROCEED_THRESHOLD"
)
assert proposal["rationale"]["threshold"] == CORRECTION_RATE_PROCEED_THRESHOLD
assert "threshold" in proposal["rationale"]["explanation"]
assert events == original_events


def test_preview_correction_impact_high_correction_recommends_continue_proofreading():
events = [
{"event_type": "correction_applied"},
{"event_type": "instance_corrected"},
{"event_type": "mask_corrected"},
{"event_type": "proofreading_saved"},
{"event_type": "masks_exported"},
{"event_type": "model_trained"},
{"event_type": "dataset_refreshed"},
{"event_type": "correction_applied"},
{"event_type": "instance_corrected"},
{"event_type": "evaluation_completed"},
]

proposal = build_preview_correction_impact_proposal(events)

assert proposal["summary"]["correction_related_events"] == 6
assert proposal["summary"]["recent_exports"] == 1
assert proposal["recommendation"] == "continue_proofreading"
assert proposal["rationale"]["computed"]["correction_rate"] > CORRECTION_RATE_PROCEED_THRESHOLD
assert "Continue proofreading" in proposal["rationale"]["explanation"]
Loading