pyine/scripts/run_prompted_llm_eval_openai_sweep.sh at main · saifh-github/pyine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
#!/usr/bin/env bash
# Sweep script for the prompted LLM guardrail evaluation across model/effort combinations.
#
# Runs the prompted_llm_eval app sequentially for each target configuration, using
# the 'guardrail/prompted_llm_eval_openai' experiment config as a base.
#
# Usage:
#   bash scripts/run_prompted_llm_eval_openai_sweep.sh

set -euo pipefail

BASE_CMD=(
    uv run python -m pyine.apps.guardrail_eval.prompted_llm_eval
    +experiment=guardrail/prompted_llm_eval_openai
    ++config.guardrail_config.prompt_version=score_only
)

run_eval() {
    local model="$1"
    local effort="${2:-default}"
    local timestamp
    timestamp="$(date '+%Y%m%d_%H%M%S')"
    local run_name="${timestamp}_${model}_${effort}"
    local label="${model} (${effort} effort)"

    local cmd=("${BASE_CMD[@]}"
        "config.guardrail_config.llm_provider.model_kwargs.model=${model}"
        "runtime.run_name='${run_name}'"
    )
    if [[ "${effort}" != "default" ]]; then
        cmd+=("++config.guardrail_config.llm_provider.model_kwargs.reasoning_effort=${effort}")
    fi

    echo ""
    echo "================================================================================"
    echo "  ${label}"
    echo "  cmd: ${cmd[*]}"
    echo "================================================================================"
    echo ""
    "${cmd[@]}"
}

run_eval gpt-5-nano low
run_eval gpt-5-nano
run_eval gpt-5.4-nano low
run_eval gpt-5.4-nano
run_eval gpt-5-mini low
run_eval gpt-5-mini
run_eval gpt-5.4-mini low
run_eval gpt-5.4-mini
run_eval gpt-5 low
run_eval gpt-4o
run_eval gpt-5
run_eval gpt-5.4 low
run_eval gpt-5.4