diff --git a/server_api/workflow/__init__.py b/server_api/workflow/__init__.py new file mode 100644 index 0000000..04537a9 --- /dev/null +++ b/server_api/workflow/__init__.py @@ -0,0 +1,15 @@ +"""Workflow proposal helpers.""" + +from .correction_preview import ( + CORRECTION_RATE_PROCEED_THRESHOLD, + RECENT_CORRECTION_EVENT_TYPES, + RECENT_EXPORT_EVENT_TYPES, + build_preview_correction_impact_proposal, +) + +__all__ = [ + "CORRECTION_RATE_PROCEED_THRESHOLD", + "RECENT_CORRECTION_EVENT_TYPES", + "RECENT_EXPORT_EVENT_TYPES", + "build_preview_correction_impact_proposal", +] diff --git a/server_api/workflow/correction_preview.py b/server_api/workflow/correction_preview.py new file mode 100644 index 0000000..ac4565b --- /dev/null +++ b/server_api/workflow/correction_preview.py @@ -0,0 +1,71 @@ +"""Rule-based proposal for evaluating correction impact before retraining.""" + +from __future__ import annotations + +from typing import Any + +PREVIEW_CORRECTION_IMPACT_PROPOSAL_TYPE = "preview_correction_impact" +RECENT_CORRECTION_EVENT_TYPES = frozenset( + { + "correction_applied", + "instance_corrected", + "mask_corrected", + "proofreading_saved", + } +) +RECENT_EXPORT_EVENT_TYPES = frozenset({"export_created", "masks_exported"}) +CORRECTION_RATE_PROCEED_THRESHOLD = 0.10 + + +def _count_events(events: list[dict[str, Any]], event_types: frozenset[str]) -> int: + return sum(1 for event in events if event.get("event_type") in event_types) + + +def build_preview_correction_impact_proposal( + events: list[dict[str, Any]], +) -> dict[str, Any]: + """Build a transparent recommendation payload from recent workflow events. + + This helper is intentionally pure: it reads events and returns a proposal payload + without mutating any external workflow state. + """ + + total_events = len(events) + correction_events = _count_events(events, RECENT_CORRECTION_EVENT_TYPES) + export_events = _count_events(events, RECENT_EXPORT_EVENT_TYPES) + + correction_rate = ( + correction_events / total_events if total_events else 0.0 + ) + should_proceed = correction_rate <= CORRECTION_RATE_PROCEED_THRESHOLD + recommendation = "proceed" if should_proceed else "continue_proofreading" + + rationale = ( + "Proceed because correction activity is at or below the explicit threshold" + if should_proceed + else "Continue proofreading because correction activity exceeds the explicit threshold" + ) + + return { + "type": PREVIEW_CORRECTION_IMPACT_PROPOSAL_TYPE, + "summary": { + "total_recent_events": total_events, + "correction_related_events": correction_events, + "recent_exports": export_events, + "correction_rate": round(correction_rate, 4), + }, + "recommendation": recommendation, + "rationale": { + "rule": "correction_rate <= CORRECTION_RATE_PROCEED_THRESHOLD", + "threshold": CORRECTION_RATE_PROCEED_THRESHOLD, + "computed": { + "correction_rate": round(correction_rate, 4), + "correction_events": correction_events, + "total_events": total_events, + }, + "explanation": ( + f"{rationale}. Corrections={correction_events}, total={total_events}, " + f"rate={round(correction_rate, 4)}, threshold={CORRECTION_RATE_PROCEED_THRESHOLD}." + ), + }, + } diff --git a/tests/test_workflow_correction_preview.py b/tests/test_workflow_correction_preview.py new file mode 100644 index 0000000..48d4c87 --- /dev/null +++ b/tests/test_workflow_correction_preview.py @@ -0,0 +1,58 @@ +from server_api.workflow.correction_preview import ( + CORRECTION_RATE_PROCEED_THRESHOLD, + PREVIEW_CORRECTION_IMPACT_PROPOSAL_TYPE, + build_preview_correction_impact_proposal, +) + + +def test_preview_correction_impact_low_correction_recommends_proceed(): + events = [ + {"event_type": "masks_exported"}, + {"event_type": "model_trained"}, + {"event_type": "evaluation_completed"}, + {"event_type": "correction_applied"}, + {"event_type": "dataset_refreshed"}, + {"event_type": "export_created"}, + {"event_type": "model_trained"}, + {"event_type": "metrics_uploaded"}, + {"event_type": "run_archived"}, + {"event_type": "checkpoint_saved"}, + ] + + original_events = [dict(item) for item in events] + proposal = build_preview_correction_impact_proposal(events) + + assert proposal["type"] == PREVIEW_CORRECTION_IMPACT_PROPOSAL_TYPE + assert proposal["summary"]["correction_related_events"] == 1 + assert proposal["summary"]["recent_exports"] == 2 + assert proposal["recommendation"] == "proceed" + assert ( + proposal["rationale"]["rule"] + == "correction_rate <= CORRECTION_RATE_PROCEED_THRESHOLD" + ) + assert proposal["rationale"]["threshold"] == CORRECTION_RATE_PROCEED_THRESHOLD + assert "threshold" in proposal["rationale"]["explanation"] + assert events == original_events + + +def test_preview_correction_impact_high_correction_recommends_continue_proofreading(): + events = [ + {"event_type": "correction_applied"}, + {"event_type": "instance_corrected"}, + {"event_type": "mask_corrected"}, + {"event_type": "proofreading_saved"}, + {"event_type": "masks_exported"}, + {"event_type": "model_trained"}, + {"event_type": "dataset_refreshed"}, + {"event_type": "correction_applied"}, + {"event_type": "instance_corrected"}, + {"event_type": "evaluation_completed"}, + ] + + proposal = build_preview_correction_impact_proposal(events) + + assert proposal["summary"]["correction_related_events"] == 6 + assert proposal["summary"]["recent_exports"] == 1 + assert proposal["recommendation"] == "continue_proofreading" + assert proposal["rationale"]["computed"]["correction_rate"] > CORRECTION_RATE_PROCEED_THRESHOLD + assert "Continue proofreading" in proposal["rationale"]["explanation"]