|
| 1 | +"""Shared pytest fixtures for openadapt-viewer tests.""" |
| 2 | + |
| 3 | +import pytest |
| 4 | +from datetime import datetime, timedelta |
| 5 | +from pathlib import Path |
| 6 | +import tempfile |
| 7 | +import json |
| 8 | + |
| 9 | +from openadapt_viewer.core.types import ( |
| 10 | + BenchmarkRun, |
| 11 | + BenchmarkTask, |
| 12 | + TaskExecution, |
| 13 | + ExecutionStep, |
| 14 | +) |
| 15 | + |
| 16 | + |
| 17 | +@pytest.fixture |
| 18 | +def sample_execution_step(): |
| 19 | + """Create a sample ExecutionStep for testing.""" |
| 20 | + return ExecutionStep( |
| 21 | + step_number=0, |
| 22 | + timestamp=datetime.now(), |
| 23 | + screenshot_path="tasks/task_001/screenshots/step_000.png", |
| 24 | + action_type="click", |
| 25 | + action_details={"x": 500, "y": 300}, |
| 26 | + reasoning="Clicking on the target button", |
| 27 | + raw_output="Action: CLICK(500, 300)", |
| 28 | + ) |
| 29 | + |
| 30 | + |
| 31 | +@pytest.fixture |
| 32 | +def sample_task_execution(sample_execution_step): |
| 33 | + """Create a sample TaskExecution for testing.""" |
| 34 | + steps = [ |
| 35 | + sample_execution_step, |
| 36 | + ExecutionStep( |
| 37 | + step_number=1, |
| 38 | + timestamp=datetime.now() + timedelta(seconds=2), |
| 39 | + action_type="type", |
| 40 | + action_details={"text": "Hello World"}, |
| 41 | + reasoning="Typing the required text", |
| 42 | + ), |
| 43 | + ExecutionStep( |
| 44 | + step_number=2, |
| 45 | + timestamp=datetime.now() + timedelta(seconds=4), |
| 46 | + action_type="click", |
| 47 | + action_details={"x": 800, "y": 600}, |
| 48 | + reasoning="Clicking the submit button", |
| 49 | + ), |
| 50 | + ] |
| 51 | + return TaskExecution( |
| 52 | + task_id="task_001", |
| 53 | + start_time=datetime.now(), |
| 54 | + end_time=datetime.now() + timedelta(seconds=10), |
| 55 | + steps=steps, |
| 56 | + success=True, |
| 57 | + error=None, |
| 58 | + ) |
| 59 | + |
| 60 | + |
| 61 | +@pytest.fixture |
| 62 | +def failed_task_execution(): |
| 63 | + """Create a failed TaskExecution for testing.""" |
| 64 | + return TaskExecution( |
| 65 | + task_id="task_002", |
| 66 | + start_time=datetime.now(), |
| 67 | + end_time=datetime.now() + timedelta(seconds=5), |
| 68 | + steps=[ |
| 69 | + ExecutionStep( |
| 70 | + step_number=0, |
| 71 | + action_type="click", |
| 72 | + action_details={"x": 100, "y": 100}, |
| 73 | + reasoning="Attempting to click target", |
| 74 | + ) |
| 75 | + ], |
| 76 | + success=False, |
| 77 | + error="Task failed: Could not locate target element", |
| 78 | + ) |
| 79 | + |
| 80 | + |
| 81 | +@pytest.fixture |
| 82 | +def sample_benchmark_task(): |
| 83 | + """Create a sample BenchmarkTask for testing.""" |
| 84 | + return BenchmarkTask( |
| 85 | + task_id="task_001", |
| 86 | + instruction="Open Notepad and type 'Hello World'", |
| 87 | + domain="office", |
| 88 | + difficulty="easy", |
| 89 | + time_limit=300, |
| 90 | + metadata={"source": "test"}, |
| 91 | + ) |
| 92 | + |
| 93 | + |
| 94 | +@pytest.fixture |
| 95 | +def sample_benchmark_run(sample_benchmark_task, sample_task_execution, failed_task_execution): |
| 96 | + """Create a sample BenchmarkRun with multiple tasks for testing.""" |
| 97 | + tasks = [ |
| 98 | + sample_benchmark_task, |
| 99 | + BenchmarkTask( |
| 100 | + task_id="task_002", |
| 101 | + instruction="Navigate to google.com in Chrome", |
| 102 | + domain="browser", |
| 103 | + difficulty="easy", |
| 104 | + time_limit=300, |
| 105 | + ), |
| 106 | + BenchmarkTask( |
| 107 | + task_id="task_003", |
| 108 | + instruction="Create a new folder on Desktop", |
| 109 | + domain="file_management", |
| 110 | + difficulty="medium", |
| 111 | + time_limit=300, |
| 112 | + ), |
| 113 | + ] |
| 114 | + |
| 115 | + # Add a third successful execution |
| 116 | + third_execution = TaskExecution( |
| 117 | + task_id="task_003", |
| 118 | + start_time=datetime.now(), |
| 119 | + end_time=datetime.now() + timedelta(seconds=8), |
| 120 | + steps=[ |
| 121 | + ExecutionStep( |
| 122 | + step_number=0, |
| 123 | + action_type="click", |
| 124 | + action_details={"x": 200, "y": 400}, |
| 125 | + ), |
| 126 | + ExecutionStep( |
| 127 | + step_number=1, |
| 128 | + action_type="type", |
| 129 | + action_details={"text": "New Folder"}, |
| 130 | + ), |
| 131 | + ], |
| 132 | + success=True, |
| 133 | + ) |
| 134 | + |
| 135 | + executions = [sample_task_execution, failed_task_execution, third_execution] |
| 136 | + |
| 137 | + return BenchmarkRun( |
| 138 | + run_id="test_run_001", |
| 139 | + benchmark_name="Test Benchmark", |
| 140 | + model_id="test-model-v1", |
| 141 | + start_time=datetime.now() - timedelta(hours=1), |
| 142 | + end_time=datetime.now(), |
| 143 | + tasks=tasks, |
| 144 | + executions=executions, |
| 145 | + config={"max_steps": 10, "timeout": 300}, |
| 146 | + ) |
| 147 | + |
| 148 | + |
| 149 | +@pytest.fixture |
| 150 | +def temp_dir(): |
| 151 | + """Create a temporary directory for test files.""" |
| 152 | + with tempfile.TemporaryDirectory() as tmpdir: |
| 153 | + yield Path(tmpdir) |
| 154 | + |
| 155 | + |
| 156 | +@pytest.fixture |
| 157 | +def benchmark_data_dir(temp_dir): |
| 158 | + """Create a mock benchmark data directory with test data.""" |
| 159 | + # Create directory structure |
| 160 | + tasks_dir = temp_dir / "tasks" |
| 161 | + task_001_dir = tasks_dir / "task_001" |
| 162 | + screenshots_dir = task_001_dir / "screenshots" |
| 163 | + screenshots_dir.mkdir(parents=True) |
| 164 | + |
| 165 | + # Create metadata.json |
| 166 | + metadata = { |
| 167 | + "run_id": "test_run", |
| 168 | + "benchmark_name": "Test Benchmark", |
| 169 | + "model_id": "test-model", |
| 170 | + "start_time": datetime.now().isoformat(), |
| 171 | + "config": {"max_steps": 10}, |
| 172 | + } |
| 173 | + with open(temp_dir / "metadata.json", "w") as f: |
| 174 | + json.dump(metadata, f) |
| 175 | + |
| 176 | + # Create task.json |
| 177 | + task_data = { |
| 178 | + "task_id": "task_001", |
| 179 | + "instruction": "Test task instruction", |
| 180 | + "domain": "test", |
| 181 | + "difficulty": "easy", |
| 182 | + } |
| 183 | + with open(task_001_dir / "task.json", "w") as f: |
| 184 | + json.dump(task_data, f) |
| 185 | + |
| 186 | + # Create execution.json |
| 187 | + execution_data = { |
| 188 | + "task_id": "task_001", |
| 189 | + "start_time": datetime.now().isoformat(), |
| 190 | + "steps": [ |
| 191 | + { |
| 192 | + "action_type": "click", |
| 193 | + "action_details": {"x": 100, "y": 200}, |
| 194 | + "reasoning": "Test reasoning", |
| 195 | + } |
| 196 | + ], |
| 197 | + "success": True, |
| 198 | + } |
| 199 | + with open(task_001_dir / "execution.json", "w") as f: |
| 200 | + json.dump(execution_data, f) |
| 201 | + |
| 202 | + return temp_dir |
0 commit comments