fix: decouple CLI from evals part 1

Author: cristipufu

src/uipath/_cli/_evals/_context.py

@@ -0,0 +1,27 @@
+from typing import Any
+
+from uipath.runtime.schema import UiPathRuntimeSchema
+
+from uipath.eval.evaluators.base_evaluator import GenericBaseEvaluator
+from uipath.eval.models.evaluation_set import EvaluationSet
+
+
+class UiPathEvalContext:
+    """Context used for evaluation runs."""
+
+    # Required Fields
+    runtime_schema: UiPathRuntimeSchema
+    evaluation_set: EvaluationSet
+    evaluators: list[GenericBaseEvaluator[Any, Any, Any]]
+    execution_id: str
+
+    # Optional Fields
+    entrypoint: str | None = None
+    workers: int | None = 1
+    eval_set_run_id: str | None = None
+    verbose: bool = False
+    enable_mocker_cache: bool = False
+    report_coverage: bool = False
+    input_overrides: dict[str, Any] | None = None
+    resume: bool = False
+    job_id: str | None = None

src/uipath/_cli/_evals/_evaluate.py

@@ -4,7 +4,8 @@
     UiPathRuntimeResult,
 )
 
-from uipath._cli._evals._runtime import UiPathEvalContext, UiPathEvalRuntime
+from uipath._cli._evals._context import UiPathEvalContext
+from uipath._cli._evals._runtime import UiPathEvalRuntime
 from uipath._events._event_bus import EventBus
 
 

src/uipath/_cli/_evals/_exporters.py

@@ -0,0 +1,98 @@
+import logging
+from collections import defaultdict
+from typing import Sequence
+
+from opentelemetry import context as context_api
+from opentelemetry.sdk.trace import ReadableSpan, Span
+from opentelemetry.sdk.trace.export import (
+    SpanExporter,
+    SpanExportResult,
+)
+from uipath.core.tracing.processors import UiPathExecutionBatchTraceProcessor
+from uipath.runtime.logging import UiPathRuntimeExecutionLogHandler
+
+from uipath._cli._evals._span_collection import ExecutionSpanCollector
+from uipath._cli._evals.mocks.mocks import execution_id_context
+
+
+class ExecutionSpanExporter(SpanExporter):
+    """Custom exporter that stores spans grouped by execution ids."""
+
+    def __init__(self):
+        # { execution_id -> list of spans }
+        self._spans: dict[str, list[ReadableSpan]] = defaultdict(list)
+
+    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
+        for span in spans:
+            if span.attributes is not None:
+                exec_id = span.attributes.get("execution.id")
+                if exec_id is not None and isinstance(exec_id, str):
+                    self._spans[exec_id].append(span)
+
+        return SpanExportResult.SUCCESS
+
+    def get_spans(self, execution_id: str) -> list[ReadableSpan]:
+        """Retrieve spans for a given execution id."""
+        return self._spans.get(execution_id, [])
+
+    def clear(self, execution_id: str | None = None) -> None:
+        """Clear stored spans for one or all executions."""
+        if execution_id:
+            self._spans.pop(execution_id, None)
+        else:
+            self._spans.clear()
+
+    def shutdown(self) -> None:
+        self.clear()
+
+
+class ExecutionSpanProcessor(UiPathExecutionBatchTraceProcessor):
+    """Span processor that adds spans to ExecutionSpanCollector when they start."""
+
+    def __init__(self, span_exporter: SpanExporter, collector: ExecutionSpanCollector):
+        super().__init__(span_exporter)
+        self.collector = collector
+
+    def on_start(
+        self, span: Span, parent_context: context_api.Context | None = None
+    ) -> None:
+        super().on_start(span, parent_context)
+
+        exec_id = span.attributes.get("execution.id") if span.attributes else None
+
+        # Fallback: if execution.id wasn't propagated (e.g., NonRecordingSpan
+        # parent on resume), get it from the execution context variable.
+        if exec_id is None:
+            ctx_exec_id = execution_id_context.get()
+            if ctx_exec_id:
+                span.set_attribute("execution.id", ctx_exec_id)
+                exec_id = ctx_exec_id
+
+        if span.attributes and "execution.id" in span.attributes:
+            exec_id = span.attributes["execution.id"]
+            if isinstance(exec_id, str):
+                self.collector.add_span(span, exec_id)
+
+
+class ExecutionLogsExporter:
+    """Custom exporter that stores multiple execution log handlers."""
+
+    def __init__(self):
+        self._log_handlers: dict[str, UiPathRuntimeExecutionLogHandler] = {}
+
+    def register(
+        self, execution_id: str, handler: UiPathRuntimeExecutionLogHandler
+    ) -> None:
+        self._log_handlers[execution_id] = handler
+
+    def get_logs(self, execution_id: str) -> list[logging.LogRecord]:
+        """Clear stored spans for one or all executions."""
+        log_handler = self._log_handlers.get(execution_id)
+        return log_handler.buffer if log_handler else []
+
+    def clear(self, execution_id: str | None = None) -> None:
+        """Clear stored spans for one or all executions."""
+        if execution_id:
+            self._log_handlers.pop(execution_id, None)
+        else:
+            self._log_handlers.clear()

src/uipath/_cli/_evals/_models/_mocks.py

@@ -5,17 +5,14 @@
 import logging
 import os
 import uuid
+from enum import IntEnum
 from typing import Any
 from urllib.parse import urlparse
 
 from opentelemetry import trace
 from pydantic import BaseModel
 from rich.console import Console
 
-from uipath._cli._evals._models._evaluation_set import (
-    EvaluationItem,
-    EvaluationStatus,
-)
 from uipath._cli._evals._models._sw_reporting import (
     StudioWebAgentSnapshot,
     StudioWebProgressItem,
@@ -41,12 +38,20 @@
 )
 from uipath.eval.evaluators.base_evaluator import GenericBaseEvaluator
 from uipath.eval.models import EvalItemResult, ScoreType
+from uipath.eval.models.evaluation_set import EvaluationItem
 from uipath.platform import UiPath
 from uipath.platform.common import UiPathConfig
 
 logger = logging.getLogger(__name__)
 
 
+class EvaluationStatus(IntEnum):
+    PENDING = 0
+    IN_PROGRESS = 1
+    COMPLETED = 2
+    FAILED = 3
+
+
 def gracefully_handle_errors(func):
     """Decorator to catch and log errors without stopping execution."""
 

src/uipath/_cli/_evals/_progress_reporter.py

@@ -8,17 +8,11 @@
     Awaitable,
     Iterable,
     Iterator,
-    Sequence,
     Tuple,
 )
 
 import coverage
-from opentelemetry import context as context_api
-from opentelemetry.sdk.trace import ReadableSpan, Span
-from opentelemetry.sdk.trace.export import (
-    SpanExporter,
-    SpanExportResult,
-)
+from opentelemetry.sdk.trace import ReadableSpan
 from opentelemetry.trace import (
     NonRecordingSpan,
     SpanContext,
@@ -29,7 +23,6 @@
 )
 from pydantic import BaseModel
 from uipath.core.tracing import UiPathTraceManager
-from uipath.core.tracing.processors import UiPathExecutionBatchTraceProcessor
 from uipath.runtime import (
     UiPathExecuteOptions,
     UiPathExecutionRuntime,
@@ -45,6 +38,12 @@
 from uipath.runtime.logging import UiPathRuntimeExecutionLogHandler
 from uipath.runtime.schema import UiPathRuntimeSchema
 
+from uipath._cli._evals._context import UiPathEvalContext
+from uipath._cli._evals._exporters import (
+    ExecutionLogsExporter,
+    ExecutionSpanExporter,
+    ExecutionSpanProcessor,
+)
 from uipath._cli._evals._span_utils import (
     configure_eval_set_run_span,
     configure_evaluation_span,
@@ -54,6 +53,11 @@
 from uipath._cli._evals.mocks.input_mocker import (
     generate_llm_input,
 )
+from uipath.eval.mocks.types import MockingContext
+from uipath.eval.models.evaluation_set import (
+    EvaluationItem,
+    EvaluationSet,
+)
 
 from ..._events._event_bus import EventBus
 from ..._events._events import (
@@ -69,10 +73,6 @@
 from ...eval.models.models import AgentExecution, EvalItemResult
 from .._utils._parallelization import execute_parallel
 from ._eval_util import apply_input_overrides
-from ._models._evaluation_set import (
-    EvaluationItem,
-    EvaluationSet,
-)
 from ._models._exceptions import EvaluationRuntimeException
 from ._models._output import (
     EvaluationResultDto,
@@ -90,118 +90,12 @@
 from .mocks.mocks import (
     cache_manager_context,
     clear_execution_context,
-    execution_id_context,
     set_execution_context,
 )
-from .mocks.types import MockingContext
 
 logger = logging.getLogger(__name__)
 
 
-class ExecutionSpanExporter(SpanExporter):
-    """Custom exporter that stores spans grouped by execution ids."""
-
-    def __init__(self):
-        # { execution_id -> list of spans }
-        self._spans: dict[str, list[ReadableSpan]] = defaultdict(list)
-
-    def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
-        for span in spans:
-            if span.attributes is not None:
-                exec_id = span.attributes.get("execution.id")
-                if exec_id is not None and isinstance(exec_id, str):
-                    self._spans[exec_id].append(span)
-
-        return SpanExportResult.SUCCESS
-
-    def get_spans(self, execution_id: str) -> list[ReadableSpan]:
-        """Retrieve spans for a given execution id."""
-        return self._spans.get(execution_id, [])
-
-    def clear(self, execution_id: str | None = None) -> None:
-        """Clear stored spans for one or all executions."""
-        if execution_id:
-            self._spans.pop(execution_id, None)
-        else:
-            self._spans.clear()
-
-    def shutdown(self) -> None:
-        self.clear()
-
-
-class ExecutionSpanProcessor(UiPathExecutionBatchTraceProcessor):
-    """Span processor that adds spans to ExecutionSpanCollector when they start."""
-
-    def __init__(self, span_exporter: SpanExporter, collector: ExecutionSpanCollector):
-        super().__init__(span_exporter)
-        self.collector = collector
-
-    def on_start(
-        self, span: Span, parent_context: context_api.Context | None = None
-    ) -> None:
-        super().on_start(span, parent_context)
-
-        exec_id = span.attributes.get("execution.id") if span.attributes else None
-
-        # Fallback: if execution.id wasn't propagated (e.g., NonRecordingSpan
-        # parent on resume), get it from the execution context variable.
-        if exec_id is None:
-            ctx_exec_id = execution_id_context.get()
-            if ctx_exec_id:
-                span.set_attribute("execution.id", ctx_exec_id)
-                exec_id = ctx_exec_id
-
-        if span.attributes and "execution.id" in span.attributes:
-            exec_id = span.attributes["execution.id"]
-            if isinstance(exec_id, str):
-                self.collector.add_span(span, exec_id)
-
-
-class ExecutionLogsExporter:
-    """Custom exporter that stores multiple execution log handlers."""
-
-    def __init__(self):
-        self._log_handlers: dict[str, UiPathRuntimeExecutionLogHandler] = {}
-
-    def register(
-        self, execution_id: str, handler: UiPathRuntimeExecutionLogHandler
-    ) -> None:
-        self._log_handlers[execution_id] = handler
-
-    def get_logs(self, execution_id: str) -> list[logging.LogRecord]:
-        """Clear stored spans for one or all executions."""
-        log_handler = self._log_handlers.get(execution_id)
-        return log_handler.buffer if log_handler else []
-
-    def clear(self, execution_id: str | None = None) -> None:
-        """Clear stored spans for one or all executions."""
-        if execution_id:
-            self._log_handlers.pop(execution_id, None)
-        else:
-            self._log_handlers.clear()
-
-
-class UiPathEvalContext:
-    """Context used for evaluation runs."""
-
-    # Required Fields
-    runtime_schema: UiPathRuntimeSchema
-    evaluation_set: EvaluationSet
-    evaluators: list[GenericBaseEvaluator[Any, Any, Any]]
-    execution_id: str
-
-    # Optional Fields
-    entrypoint: str | None = None
-    workers: int | None = 1
-    eval_set_run_id: str | None = None
-    verbose: bool = False
-    enable_mocker_cache: bool = False
-    report_coverage: bool = False
-    input_overrides: dict[str, Any] | None = None
-    resume: bool = False
-    job_id: str | None = None
-
-
 class UiPathEvalRuntime:
     """Specialized runtime for evaluation runs, with access to the factory."""
 

src/uipath/_cli/_evals/_runtime.py

@@ -5,12 +5,7 @@
 
 from opentelemetry.trace import Span, Status, StatusCode
 from pydantic import BaseModel, ConfigDict, Field
-
-# Type hint for runtime protocol (avoids circular imports)
-try:
-    from uipath.runtime import UiPathRuntimeProtocol, UiPathRuntimeSchema
-except ImportError:
-    UiPathRuntimeProtocol = Any  # type: ignore
+from uipath.runtime.schema import UiPathRuntimeSchema
 
 
 class EvalSetRunOutput(BaseModel):

src/uipath/_cli/_evals/_span_utils.py

@@ -4,7 +4,7 @@
 from datetime import datetime
 from typing import Any
 
-from uipath._cli._evals.mocks.types import (
+from uipath.eval.mocks.types import (
     InputMockingStrategy,
 )
 from uipath.platform import UiPath