letta-code/.github/workflows/terminal-bench-regression.yml at main · CompleteTech-LLC-AI-Research/letta-code · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
name: Terminal-Bench Regression

on:
  schedule:
    - cron: "0 12 * * 1" # Monday 5am PT (12pm UTC)
  workflow_dispatch:
    inputs:
      model:
        description: "Override model (blank = run both defaults)"
        default: ""
      concurrency:
        description: "Max concurrent tasks"
        default: "10"

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  regression:
    runs-on: ubuntu-latest
    timeout-minutes: 180
    strategy:
      fail-fast: false
      matrix:
        model: [sonnet-4.6-xhigh, gpt-5.3-codex-xhigh]
    steps:
      - name: Checkout
        uses: actions/checkout@v6

      - name: Setup Python + uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: false

      - name: Create venv and install deps
        run: |
          uv venv .venv
          source .venv/bin/activate
          uv pip install "harbor==0.1.45" "litellm>=1.0.0" "modal>=1.3.5"

      - name: Configure Modal
        env:
          MODAL_TOKEN_ID: ${{ secrets.MODAL_TOKEN_ID }}
          MODAL_TOKEN_SECRET: ${{ secrets.MODAL_TOKEN_SECRET }}
        run: |
          printf '[letta]\ntoken_id = "%s"\ntoken_secret = "%s"\nactive = true\nenvironment = "terminal-bench"\nimage_builder_version = "2025.06"\n' \
            "$MODAL_TOKEN_ID" "$MODAL_TOKEN_SECRET" > ~/.modal.toml

      - name: Run regression tasks
        env:
          LETTA_API_KEY: ${{ secrets.LETTA_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          source .venv/bin/activate

          harbor run \
            --dataset terminal-bench@2.0 \
            --agent-import-path benchmarks.terminal_bench.letta_code_agent:LettaCode \
            --model "${{ matrix.model }}" \
            --env modal \
            --n-concurrent ${{ inputs.concurrency || '10' }} \
            --job-name "regression-${{ matrix.model }}-$(date +%Y%m%d)"

      - name: Upload results artifact
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: tb-results-${{ matrix.model }}
          path: jobs/

  report:
    needs: regression
    if: always()
    runs-on: ubuntu-latest
    permissions:
      issues: write
      contents: read
    steps:
      - name: Checkout
        uses: actions/checkout@v6

      - name: Download all result artifacts
        uses: actions/download-artifact@v4
        with:
          path: results/

      - name: Setup Python + uv
        uses: astral-sh/setup-uv@v6
        with:
          enable-cache: false

      - name: Generate report and update GitHub Issue
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GITHUB_REPOSITORY: ${{ github.repository }}
          GITHUB_RUN_ID: ${{ github.run_id }}
          GITHUB_SERVER_URL: ${{ github.server_url }}
        run: |
          uv run python benchmarks/terminal_bench/report.py \
            --results-dir results/ \
            --baseline benchmarks/terminal_bench/baseline.json \
            --repo "${{ github.repository }}"