Skip to content

Commit 2408f76

Browse files
probably-nebprobably-neb
authored
gh-workflow unit evals (zed-industries#41637)
Closes #ISSUE Release Notes: - N/A *or* Added/Fixed/Improved ...
1 parent df15d2d commit 2408f76

7 files changed

Lines changed: 246 additions & 157 deletions

File tree

‎.github/workflows/eval.yml‎

Lines changed: 0 additions & 71 deletions
This file was deleted.

‎.github/workflows/run_agent_evals.yml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
# Generated from xtask::workflows::run_agent_evals
2+
# Rebuild with `cargo xtask workflows`.
3+
name: run_agent_evals
4+
env:
5+
CARGO_TERM_COLOR: always
6+
CARGO_INCREMENTAL: '0'
7+
RUST_BACKTRACE: '1'
8+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
9+
ZED_CLIENT_CHECKSUM_SEED: ${{ secrets.ZED_CLIENT_CHECKSUM_SEED }}
10+
ZED_EVAL_TELEMETRY: '1'
11+
on:
12+
pull_request:
13+
types:
14+
- synchronize
15+
- reopened
16+
- labeled
17+
branches:
18+
- '**'
19+
schedule:
20+
- cron: 0 0 * * *
21+
workflow_dispatch: {}
22+
jobs:
23+
agent_evals:
24+
if: |
25+
github.repository_owner == 'zed-industries' &&
26+
(github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'run-eval'))
27+
runs-on: namespace-profile-16x32-ubuntu-2204
28+
steps:
29+
- name: steps::checkout_repo
30+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
31+
with:
32+
clean: false
33+
- name: steps::cache_rust_dependencies
34+
uses: swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6
35+
with:
36+
save-if: ${{ github.ref == 'refs/heads/main' }}
37+
- name: steps::setup_linux
38+
run: ./script/linux
39+
shell: bash -euxo pipefail {0}
40+
- name: steps::install_mold
41+
run: ./script/install-mold
42+
shell: bash -euxo pipefail {0}
43+
- name: steps::setup_cargo_config
44+
run: |
45+
mkdir -p ./../.cargo
46+
cp ./.cargo/ci-config.toml ./../.cargo/config.toml
47+
shell: bash -euxo pipefail {0}
48+
- name: cargo build --package=eval
49+
run: cargo build --package=eval
50+
shell: bash -euxo pipefail {0}
51+
- name: run_agent_evals::agent_evals::run_eval
52+
run: cargo run --package=eval -- --repetitions=8 --concurrency=1
53+
shell: bash -euxo pipefail {0}
54+
- name: steps::cleanup_cargo_config
55+
if: always()
56+
run: |
57+
rm -rf ./../.cargo
58+
shell: bash -euxo pipefail {0}
59+
timeout-minutes: 60
60+
concurrency:
61+
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}
62+
cancel-in-progress: true

‎.github/workflows/run_unit_evals.yml‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
# Generated from xtask::workflows::run_agent_evals
2+
# Rebuild with `cargo xtask workflows`.
3+
name: run_agent_evals
4+
env:
5+
CARGO_TERM_COLOR: always
6+
CARGO_INCREMENTAL: '0'
7+
RUST_BACKTRACE: '1'
8+
ZED_CLIENT_CHECKSUM_SEED: ${{ secrets.ZED_CLIENT_CHECKSUM_SEED }}
9+
on:
10+
schedule:
11+
- cron: 47 1 * * 2
12+
workflow_dispatch: {}
13+
jobs:
14+
unit_evals:
15+
runs-on: namespace-profile-16x32-ubuntu-2204
16+
steps:
17+
- name: steps::checkout_repo
18+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
19+
with:
20+
clean: false
21+
- name: steps::setup_cargo_config
22+
run: |
23+
mkdir -p ./../.cargo
24+
cp ./.cargo/ci-config.toml ./../.cargo/config.toml
25+
shell: bash -euxo pipefail {0}
26+
- name: steps::cache_rust_dependencies
27+
uses: swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6
28+
with:
29+
save-if: ${{ github.ref == 'refs/heads/main' }}
30+
- name: steps::setup_linux
31+
run: ./script/linux
32+
shell: bash -euxo pipefail {0}
33+
- name: steps::install_mold
34+
run: ./script/install-mold
35+
shell: bash -euxo pipefail {0}
36+
- name: steps::cargo_install_nextest
37+
run: cargo install cargo-nextest --locked
38+
shell: bash -euxo pipefail {0}
39+
- name: steps::clear_target_dir_if_large
40+
run: ./script/clear-target-dir-if-larger-than 100
41+
shell: bash -euxo pipefail {0}
42+
- name: ./script/run-unit-evals
43+
run: ./script/run-unit-evals
44+
shell: bash -euxo pipefail {0}
45+
env:
46+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
47+
- name: run_agent_evals::unit_evals::send_failure_to_slack
48+
if: ${{ failure() }}
49+
uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52
50+
with:
51+
method: chat.postMessage
52+
token: ${{ secrets.SLACK_APP_ZED_UNIT_EVALS_BOT_TOKEN }}
53+
payload: |
54+
channel: C04UDRNNJFQ
55+
text: "Unit Evals Failed: https://github.com/zed-industries/zed/actions/runs/${{ github.run_id }}"
56+
- name: steps::cleanup_cargo_config
57+
if: always()
58+
run: |
59+
rm -rf ./../.cargo
60+
shell: bash -euxo pipefail {0}
61+
concurrency:
62+
group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }}
63+
cancel-in-progress: true

‎.github/workflows/unit_evals.yml‎

Lines changed: 0 additions & 86 deletions
This file was deleted.

‎script/run-unit-evals‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env bash
2+
3+
set -euxo pipefail
4+
5+
cargo nextest run --workspace --no-fail-fast --features unit-eval --no-capture -E 'test(::eval_)'

‎tooling/xtask/src/tasks/workflows.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ mod release_nightly;
1010
mod run_bundling;
1111

1212
mod release;
13+
mod run_agent_evals;
1314
mod run_tests;
1415
mod runners;
1516
mod steps;
@@ -28,6 +29,8 @@ pub fn run_workflows(_: GenerateWorkflowArgs) -> Result<()> {
2829
("run_tests.yml", run_tests::run_tests()),
2930
("release.yml", release::release()),
3031
("compare_perf.yml", compare_perf::compare_perf()),
32+
("run_unit_evals.yml", run_agent_evals::run_unit_evals()),
33+
("run_agent_evals.yml", run_agent_evals::run_agent_evals()),
3134
];
3235
fs::create_dir_all(dir)
3336
.with_context(|| format!("Failed to create directory: {}", dir.display()))?;

0 commit comments

Comments
 (0)