test

Author: xzrderek

.github/workflows/rollout.yml

@@ -1,27 +1,34 @@
 name: Eval Protocol Rollout
 
-run-name: rollout:${{ inputs.rollout_id }}
+run-name: rollout:${{ fromJSON(inputs.metadata).rollout_id }}
 
 on:
   workflow_dispatch:
     inputs:
       model:
-        description: 'Model to use for the rollout'
+        description: 'Model to use'
         required: true
         type: string
-      rollout_id:
-        description: 'Rollout ID for tracking'
+      metadata:
+        description: 'JSON serialized metadata object'
         required: true
         type: string
-      prompt:
-        description: 'User prompt for the rollout'
+      messages:
+        description: 'JSON serialized messages array'
+        required: true
+        type: string
+      tools:
+        description: 'JSON serialized tools array'
+        required: false
+        type: string
+      model_base_url:
+        description: 'Base URL for the model API'
         required: true
         type: string
 
 jobs:
   rollout:
     runs-on: ubuntu-latest
-    name: rollout-${{ inputs.rollout_id }}
 
     steps:
       - name: Checkout code
@@ -42,14 +49,8 @@ jobs:
           FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
         run: |
           python tests/github_actions/rollout_worker.py \
-            --rollout-id "${{ inputs.rollout_id }}" \
             --model "${{ inputs.model }}" \
-            --prompt "${{ inputs.prompt }}"
-
-      - name: Upload rollout trace
-        uses: actions/upload-artifact@v4
-        if: always()  # Upload even if the rollout failed
-        with:
-          name: rollout-trace-${{ inputs.rollout_id }}
-          path: rollout_trace_${{ inputs.rollout_id }}.json
-          retention-days: 7
+            --metadata '${{ inputs.metadata }}' \
+            --messages '${{ inputs.messages }}' \
+            --tools '${{ inputs.tools }}' \
+            --model-base-url "${{ inputs.model_base_url }}"

eval_protocol/pytest/evaluation_test.py

@@ -47,7 +47,7 @@
 )
 
 
-from eval_protocol.pytest.utils import (
+from eval_protocol.pytest.evaluation_test_utils import (
     AggregationMethod,
     add_cost_metrics,
     log_eval_status_and_rows,

eval_protocol/pytest/evaluation_test_postprocess.py

@@ -10,7 +10,12 @@
 from eval_protocol.models import CompletionParams, EvaluationRow, EvaluationThreshold, Status
 from eval_protocol.pytest.handle_persist_flow import handle_persist_flow
 from eval_protocol.pytest.types import EvaluationTestMode
-from eval_protocol.pytest.utils import AggregationMethod, aggregate, extract_effort_tag, sanitize_filename
+from eval_protocol.pytest.evaluation_test_utils import (
+    AggregationMethod,
+    aggregate,
+    extract_effort_tag,
+    sanitize_filename,
+)
 from eval_protocol.stats.confidence_intervals import compute_fixed_set_mu_ci
 
 

eval_protocol/pytest/utils.py …protocol/pytest/evaluation_test_utils.pyeval_protocol/pytest/utils.py renamed to eval_protocol/pytest/evaluation_test_utils.py eval_protocol/pytest/utils.py renamed to eval_protocol/pytest/evaluation_test_utils.py

@@ -2,7 +2,7 @@
 from eval_protocol.data_loader.models import EvaluationDataLoader
 from eval_protocol.models import CompletionParams, EvaluationRow
 from eval_protocol.pytest.types import Dataset, DatasetPathParam, EvaluationInputParam, InputMessagesParam
-from eval_protocol.pytest.utils import parse_ep_max_rows
+from eval_protocol.pytest.evaluation_test_utils import parse_ep_max_rows
 from collections.abc import Sequence