run_airline_model.sh

#!/usr/bin/env bash
# =============================================================================
# run_airline_model.sh — pick a policy model by number, run the AIRLINE local-test.
# Airline twin of run_model.sh. Thin wrapper around training_airline.sh --localtest:
#     POLICY_MODEL         = the model you pick (with the fireworks_ai/ prefix)
#     USE_LEANMCP_GATEWAY  = 1 by default  (route inference through LeanMCP, logged)
#
# Usage:
#     bash FIREWORKS_TRAINING/run_airline_model.sh            # interactive: asks model + sim-user
#     bash FIREWORKS_TRAINING/run_airline_model.sh 1          # pick #1; still asks about sim-user
#     bash FIREWORKS_TRAINING/run_airline_model.sh 1 y        # pick #1, simulated user ON  (no prompts)
#     bash FIREWORKS_TRAINING/run_airline_model.sh 1 n        # pick #1, simulated user OFF (no prompts)
#     USE_LEANMCP_GATEWAY=0 bash FIREWORKS_TRAINING/run_airline_model.sh 1   # gateway OFF
#
# Per repo convention, Claude does NOT run this — you do.
# =============================================================================
set -euo pipefail
HERE="$(cd "$(dirname "$0")" && pwd)"

# -----------------------------------------------------------------------------
# EDIT THIS LIST to add/remove models (slug under accounts/fireworks/models/).
# -----------------------------------------------------------------------------
MODELS=(
  "minimax-m3"
  "minimax-m2p7"
  "glm-5p2"
  "glm-5p1"
  "kimi-k2p7-code"
  "kimi-k2p6"
  "qwen3p7-plus"
  "deepseek-v4-pro"
  "deepseek-v4-flash"
  "gpt-oss-120b"
  "gpt-oss-20b"
  "nemotron-3-ultra-nvfp4"
)

# Gateway ON by default; override with USE_LEANMCP_GATEWAY=0 in the environment.
GATEWAY="${USE_LEANMCP_GATEWAY:-1}"
# Simulated user: honor ENABLE_USER_SIM if preset; else ask (or take 2nd arg).
USERSIM="${ENABLE_USER_SIM:-}"

# Default to a quick 5-Easy subset. Airline_easy has 17 Easy tasks; set MAX_EASY big
# (e.g. 17) for all. Override per bucket via env.
MAX_EASY="${MAX_EASY:-5}"

# -----------------------------------------------------------------------------
# Pick a model: from $1 if given, else show the numbered menu and read a number.
# -----------------------------------------------------------------------------
echo "== pick a policy model  (gateway: $([ "$GATEWAY" = 1 ] && echo ON || echo OFF)) =="
for i in "${!MODELS[@]}"; do
  printf "  %2d) %s\n" "$((i + 1))" "${MODELS[$i]}"
done
echo

choice="${1:-}"
if [ -z "$choice" ]; then
  read -rp "Select a model [1-${#MODELS[@]}]: " choice
fi

case "$choice" in
  ''|*[!0-9]*) echo "ERROR: '$choice' is not a number."; exit 1;;
esac
if [ "$choice" -lt 1 ] || [ "$choice" -gt "${#MODELS[@]}" ]; then
  echo "ERROR: pick a number between 1 and ${#MODELS[@]}."; exit 1
fi

SLUG="${MODELS[$((choice - 1))]}"
POLICY_MODEL="fireworks_ai/accounts/fireworks/models/${SLUG}"

# Resolve simulated-user choice: env preset > 2nd arg (y/n) > interactive prompt.
sim_arg="${2:-}"
if [ -n "$USERSIM" ]; then
  :
elif [ -n "$sim_arg" ]; then
  case "$sim_arg" in
    [Yy]*|1) USERSIM=1;;
    [Nn]*|0) USERSIM=0;;
    *) echo "ERROR: 2nd arg must be y/n (include simulated user?)."; exit 1;;
  esac
else
  read -rp "Include simulated user? [Y/n]: " ans
  case "$ans" in [Nn]*|0) USERSIM=0;; *) USERSIM=1;; esac
fi

echo
echo "  selected      : $SLUG"
echo "  POLICY_MODEL  : $POLICY_MODEL"
echo "  gateway       : $([ "$GATEWAY" = 1 ] && echo "LeanMCP (needs LEANMCP_API_KEY)" || echo "direct Fireworks")"
echo "  simulated user: $([ "$USERSIM" = 1 ] && echo "ON (faithful multi-turn task)" || echo "OFF (one-shot smoke test)")"
echo "  task subset    : MAX_EASY=$MAX_EASY"
echo

# Hand off to the real runner. These env vars are read by test_airline.py:
#   POLICY_MODEL, USE_LEANMCP_GATEWAY, ENABLE_USER_SIM, MAX_EASY (+ USER_SIM_MODEL).
POLICY_MODEL="$POLICY_MODEL" USE_LEANMCP_GATEWAY="$GATEWAY" ENABLE_USER_SIM="$USERSIM" \
  MAX_EASY="$MAX_EASY" \
  bash "$HERE/training_airline.sh" --localtest