publish_learner_certification_service.py

from __future__ import annotations

import tempfile
from datetime import UTC, datetime
from pathlib import Path
from app.domain.publish import (
    PublishCertificationCheck,
    PublishCertificationCheckStatus,
    PublishCertificationFailureOrigin,
    PublishLearnerCertificationReport,
    PublishSnapshot,
)
from app.services.bundle_validation import validate_seeded_learner_workspace
from app.services.learner_package_runtime import (
    project_brief_markdown,
    remap_assignment_report_to_deliverables,
    seed_workspace_from_snapshot,
)
from app.services.learner_studio_service import LearnerStudioError, LearnerStudioService
from app.storage.workflow_store import WorkflowStore


def default_publish_certification_workspace_dir() -> Path:
    return Path(__file__).resolve().parents[2] / "tmp" / "publish_certification"


class PublishLearnerCertificationService:
    def __init__(
        self,
        *,
        learner_studio_service: LearnerStudioService | None = None,
        base_dir: str | Path | None = None,
        enabled: bool = False,
        store: WorkflowStore | None = None,
    ) -> None:
        self.learner_studio_service = learner_studio_service or LearnerStudioService()
        self.base_dir = Path(base_dir or default_publish_certification_workspace_dir())
        self.base_dir.mkdir(parents=True, exist_ok=True)
        self.enabled = enabled
        self.store = store

    def certify_snapshot(self, snapshot: PublishSnapshot) -> PublishLearnerCertificationReport:
        if not self.enabled:
            return PublishLearnerCertificationReport(
                certified_at=datetime.now(UTC),
                passed=True,
                checks=[
                    PublishCertificationCheck(
                        key="learner_certification",
                        status=PublishCertificationCheckStatus.skipped,
                        summary="Learner-path certification is disabled for this app instance.",
                        detail="Publishing will not run the exact learner-path certification unless the service is enabled.",
                        blocking=False,
                    )
                ],
                notes=["Learner-path certification was skipped because the service is disabled."],
            )

        # Partial starters intentionally ship with handlers that raise
        # `NotImplementedError`-equivalents. The visible/hidden test suites
        # are designed to FAIL against the unimplemented starter (the
        # reviewer_tests baseline matrix verifies that test strength).
        # Running learner-path certification — which expects the seeded
        # workspace's grade run to pass cleanly — is structurally
        # contradictory for partial starters: it cannot pass on a starter
        # where the tests are *supposed* to fail until the learner
        # implements the handlers. Until a reference-solution solvability
        # gate exists upstream (see todo follow-up), skip certification
        # for partial starters and trust the five reviewer gates.
        spec = snapshot.task_agent_spec
        if spec is not None and (
            spec.runtime_dependencies.starter_type.value == "partial"
        ):
            return PublishLearnerCertificationReport(
                certified_at=datetime.now(UTC),
                passed=True,
                checks=[
                    PublishCertificationCheck(
                        key="partial_starter_certification_skipped",
                        status=PublishCertificationCheckStatus.skipped,
                        summary=(
                            "Learner-path certification skipped for partial starter."
                        ),
                        detail=(
                            "Partial starters ship with unimplemented handlers by design; "
                            "visible/hidden tests are expected to fail until the learner "
                            "implements the deliverables. The reviewer_tests baseline "
                            "matrix already verified that test strength upstream."
                        ),
                        blocking=False,
                    )
                ],
                notes=[
                    "Skipped learner-path certification: starter_type=partial. "
                    "A reference-solution solvability gate is the architecturally correct "
                    "replacement; tracked as a follow-up."
                ],
            )

        learner_package = snapshot.learner_package
        if learner_package is None or not learner_package.deliverables:
            return self._failed_report(
                origin=PublishCertificationFailureOrigin.repairable_generation,
                key="learner_package_missing",
                summary="The publish snapshot is missing learner deliverables.",
                detail="Generate the learner package before publishing so the learner workspace can be seeded.",
            )
        if snapshot.task_agent_spec is None:
            return self._failed_report(
                origin=PublishCertificationFailureOrigin.repairable_generation,
                key="grading_spec_missing",
                summary="The publish snapshot is missing the hidden grading spec.",
                detail="Publishing requires the final task-agent spec so the learner submission path can run.",
            )

        checks: list[PublishCertificationCheck] = []
        workspace_session = None
        with tempfile.TemporaryDirectory(
            prefix=f"{snapshot.id}_",
            dir=self.base_dir,
        ) as temp_dir:
            workspace_root = Path(temp_dir) / "workspace"
            try:
                seed_workspace_from_snapshot(workspace_root, snapshot)
            except Exception as exc:  # noqa: BLE001
                return self._failed_report(
                    origin=PublishCertificationFailureOrigin.repairable_generation,
                    key="workspace_seed_failed",
                    summary="The learner workspace could not be seeded from the publish snapshot.",
                    detail=str(exc),
                )

            checks.append(
                PublishCertificationCheck(
                    key="workspace_seeded",
                    status=PublishCertificationCheckStatus.passed,
                    summary="The publish snapshot seeded a learner workspace successfully.",
                )
            )

            required_paths = [
                workspace_root / "README.md",
                workspace_root / "project_brief.md",
                workspace_root / "deliverables.md",
            ]
            missing_paths = [path.relative_to(workspace_root).as_posix() for path in required_paths if not path.exists()]
            if missing_paths:
                return self._failed_report(
                    origin=PublishCertificationFailureOrigin.repairable_generation,
                    key="project_brief_missing",
                    summary="The learner workspace is missing required project brief files.",
                    detail="Missing files: " + ", ".join(missing_paths),
                    extra_checks=checks,
                )

            checks.append(
                PublishCertificationCheck(
                    key="project_brief_present",
                    status=PublishCertificationCheckStatus.passed,
                    summary="The seeded workspace includes the project brief and deliverables overview.",
                    detail=project_brief_markdown(snapshot)[:500] or None,
                )
            )

            workspace_validation = validate_seeded_learner_workspace(
                snapshot.task_agent_spec,
                workspace_root,
                deliverable_ids=[deliverable.deliverable_id for deliverable in learner_package.deliverables],
            )
            if not workspace_validation.valid:
                return self._failed_report(
                    origin=PublishCertificationFailureOrigin.repairable_generation,
                    key="seeded_workspace_invalid",
                    summary="The seeded learner workspace is not coherent enough for a learner to start from directly.",
                    detail="; ".join(f"{issue.code}: {issue.message}" for issue in workspace_validation.errors[:3]),
                    extra_checks=checks,
                )

            checks.append(
                PublishCertificationCheck(
                    key="seeded_workspace_validated",
                    status=PublishCertificationCheckStatus.passed,
                    summary="The seeded learner workspace passed deterministic README and starter-surface validation.",
                )
            )

            primary_deliverable_id = learner_package.deliverables[0].deliverable_id
            course_run = None
            if self.store is not None and snapshot.course_run_id:
                course_run = self.store.get_course_run(snapshot.course_run_id)
            lab_tutor_enabled = bool(course_run.lab_tutor_enabled) if course_run is not None else False
            assignment_title = course_run.title if course_run is not None else None
            try:
                workspace_session = self.learner_studio_service.launch_editor(
                    enrollment_id=f"publish_cert_{snapshot.id}",
                    deliverable_id=primary_deliverable_id,
                    workspace_root=workspace_root,
                    scope=learner_package.workspace_scope,
                    start_support_services=False,
                    lab_tutor_enabled=lab_tutor_enabled,
                    assignment_title=assignment_title,
                )
            except LearnerStudioError as exc:
                return self._failed_report(
                    origin=PublishCertificationFailureOrigin.platform_runtime,
                    key="editor_launch_failed",
                    summary="The learner workspace editor could not launch.",
                    detail=str(exc),
                    extra_checks=checks,
                )

            checks.append(
                PublishCertificationCheck(
                    key="editor_launch",
                    status=PublishCertificationCheckStatus.passed,
                    summary="The learner editor launched against the seeded workspace.",
                    detail=workspace_session.editor_url,
                )
            )

            try:
                live_grade = self.learner_studio_service.grade_assignment(
                    workspace_root=workspace_root,
                    spec=snapshot.task_agent_spec,
                )
            except LearnerStudioError as exc:
                return self._failed_report(
                    origin=self._grade_failure_origin(str(exc)),
                    key="learner_runtime_failed",
                    summary="The learner app did not boot or grade cleanly from the seeded workspace.",
                    detail=str(exc),
                    extra_checks=checks,
                )
            finally:
                self.learner_studio_service.stop_editor(workspace_session)

            checks.append(
                PublishCertificationCheck(
                    key="grading_completed",
                    status=PublishCertificationCheckStatus.passed,
                    summary="The learner submission path completed against the seeded workspace.",
                    detail=(
                        f"Starter status: {live_grade.assignment_report.status.value}; "
                        f"{live_grade.assignment_report.passed_tests}/{live_grade.assignment_report.total_tests} tests passed."
                    ),
                )
            )

            remapped_report = remap_assignment_report_to_deliverables(snapshot, live_grade.assignment_report)
            expected_deliverable_ids = {
                deliverable.deliverable_id
                for deliverable in learner_package.deliverables
            }
            mapped_deliverable_ids = {
                review_area.deliverable_id
                for review_area in remapped_report.review_areas
            }
            if mapped_deliverable_ids != expected_deliverable_ids:
                return self._failed_report(
                    origin=PublishCertificationFailureOrigin.repairable_generation,
                    key="deliverable_mapping_failed",
                    summary="The learner grade report did not map cleanly back to the published deliverables.",
                    detail=(
                        f"Expected deliverables: {sorted(expected_deliverable_ids)}; "
                        f"got: {sorted(mapped_deliverable_ids)}."
                    ),
                    extra_checks=checks,
                )

            checks.append(
                PublishCertificationCheck(
                    key="deliverable_mapping",
                    status=PublishCertificationCheckStatus.passed,
                    summary="The learner review report mapped back to every published deliverable.",
                )
            )

            notes = []
            if remapped_report.status.value == "failed":
                notes.append(
                    "Starter submission failed cleanly, which is acceptable for certification because the learner path still booted and returned deliverable-scoped feedback."
                )
            else:
                notes.append("Starter submission passed cleanly through the learner runtime path.")
            return PublishLearnerCertificationReport(
                certified_at=datetime.now(UTC),
                passed=True,
                checks=checks,
                assignment_status=remapped_report.status,
                passed_tests=remapped_report.passed_tests,
                total_tests=remapped_report.total_tests,
                notes=notes,
            )

    def _failed_report(
        self,
        *,
        origin: PublishCertificationFailureOrigin,
        key: str,
        summary: str,
        detail: str,
        extra_checks: list[PublishCertificationCheck] | None = None,
    ) -> PublishLearnerCertificationReport:
        checks = list(extra_checks or [])
        checks.append(
            PublishCertificationCheck(
                key=key,
                status=PublishCertificationCheckStatus.failed,
                summary=summary,
                detail=detail,
            )
        )
        return PublishLearnerCertificationReport(
            certified_at=datetime.now(UTC),
            passed=False,
            failure_origin=origin,
            checks=checks,
            notes=[summary],
        )

    def _grade_failure_origin(self, error_message: str) -> PublishCertificationFailureOrigin:
        lower = error_message.lower()
        if "could not build learner studio image" in lower:
            return PublishCertificationFailureOrigin.platform_runtime
        if "could not start learner editor container" in lower:
            return PublishCertificationFailureOrigin.platform_runtime
        if "could not build learner runtime image" in lower:
            return PublishCertificationFailureOrigin.repairable_generation
        if "could not start grading container" in lower:
            return PublishCertificationFailureOrigin.ambiguous
        if "timed out waiting for" in lower:
            return PublishCertificationFailureOrigin.repairable_generation
        if "workspace seed" in lower or "starter_manifest" in lower or "project_brief" in lower:
            return PublishCertificationFailureOrigin.repairable_generation
        return PublishCertificationFailureOrigin.ambiguous