run_sweep.sh

#!/bin/bash
# run_sweep.sh
#
# Warmup-Sweep Experiment (response to Check #2 critique).
#
# Varies the C2-C4 warmup (inter-burst interval) across 6 points spanning
# 5s to 120s, and runs both `ewma` (fixed CUSUM) and `ewma_adaptive`
# (Standardized CUSUM) at each point.
#
#   Single-trial:   6 W × 2 modes = 12 runs        (default)
#   Multi-trial:    6 W × 2 modes × N trials       (use --trials N)
#
# Output:
#   logs/sweep_<timestamp>/[trial<N>_]warmup_<W>s_<mode>/
#     server.log, test.log, resource.csv, load_tester_output.txt
#
# Rationale: the professor's Check #2 comment was "you are designing your
# workload to make your system work correctly." This experiment inverts that
# design — we sweep the one parameter (inter-burst interval) that the
# fixed-threshold CUSUM was implicitly tuned for, and show where each
# algorithm breaks.
#
# Usage:
#   ./run_sweep.sh                # default: 6 W × 2 modes × 1 trial
#   ./run_sweep.sh --trials 5     # paper-quality: 6 × 2 × 5 = 60 runs
#
# Then analyze with:
#   python3 analyze_trials.py logs/sweep_<timestamp>/
#
# CSCI 599: Network Systems for Cloud Computing
# University of Southern California

set -e
cd "$(dirname "$0")"

# ── Colour helpers ────────────────────────────────────────────────────────────
RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'; NC='\033[0m'
info()    { echo -e "${CYAN}[INFO]${NC} $*"; }
success() { echo -e "${GREEN}[OK]${NC}   $*"; }
warn()    { echo -e "${YELLOW}[WARN]${NC} $*"; }
error()   { echo -e "${RED}[ERR]${NC}  $*"; }

# ── Args ──────────────────────────────────────────────────────────────────────
N_TRIALS=1
while [ $# -gt 0 ]; do
    case "$1" in
        --trials) N_TRIALS="$2"; shift 2 ;;
        *)        echo "Unknown arg: $1" >&2; exit 1 ;;
    esac
done

# ── Sweep grid ────────────────────────────────────────────────────────────────
# Full grid for paper-quality data. 5s and 120s are the known boundary regimes
# (adaptive τ_σ cliff at left, fixed aliasing miss at right). 10–60s is the
# central comparison band where each algorithm is supposed to work.
WARMUPS=(5 10 20 35 60 120)
MODES=(ewma ewma_adaptive)

# ── Build once ────────────────────────────────────────────────────────────────
info "Building server..."
make -s
success "Build OK"

SWEEP_TS=$(date +%Y%m%d_%H%M%S)
SWEEP_DIR="logs/sweep_${SWEEP_TS}"
mkdir -p "$SWEEP_DIR"
info "Sweep root: ${SWEEP_DIR}"

# run_one <warmup_s> <mode> <trial_num>
run_one() {
    local warmup_s="$1"
    local mode="$2"
    local trial="$3"

    # Single-trial: warmup_<W>s_<mode>/ (preserved for legacy compatibility)
    # Multi-trial:  trial<N>_warmup_<W>s_<mode>/
    local label
    if [ "$N_TRIALS" -eq 1 ]; then
        label="warmup_${warmup_s}s_${mode}"
    else
        label="trial${trial}_warmup_${warmup_s}s_${mode}"
    fi
    local exp_dir="${SWEEP_DIR}/${label}"
    mkdir -p "$exp_dir"

    echo ""
    echo "----------------------------------------------------------------"
    info "Sweep point: warmup=${warmup_s}s  mode=${mode}"
    echo "----------------------------------------------------------------"

    # Kill lingering server by EXACT name (safe: won't hit VSCode server).
    pkill -x server 2>/dev/null || true; sleep 1
    rm -f /tmp/faas_worker_*.sock /tmp/faas_template_ctrl.sock /tmp/faas_arima.sock
    rm -f "logs/server_${mode}.log"

    info "Starting ./server ${mode}"
    ./server "$mode" &
    local SERVER_PID=$!

    # Wait for server to come up
    local ready=0
    for i in $(seq 1 20); do
        sleep 0.5
        if ! kill -0 "$SERVER_PID" 2>/dev/null; then
            error "Server exited unexpectedly (warmup=${warmup_s}s mode=${mode})"
            return 1
        fi
        local log_file="logs/server_${mode}.log"
        if [ -f "$log_file" ] && grep -q "READY\|starting\|Pre-warming" "$log_file" 2>/dev/null; then
            ready=1; break
        fi
    done
    if [ "$ready" -eq 0 ]; then
        warn "Server did not signal ready in 10s — proceeding anyway."
    fi
    sleep 1
    success "Server up (pid=${SERVER_PID})"

    # Resource monitor
    local resource_csv="${exp_dir}/resource.csv"
    python3 resource_monitor.py "$resource_csv" > "${exp_dir}/resource_monitor_output.txt" 2>&1 &
    local MONITOR_PID=$!
    sleep 0.5

    # Sentinel to locate the test log created by this run
    local sentinel; sentinel=$(mktemp logs/.sentinel.XXXXXX)
    info "Running load_tester.py --warmup-c234 ${warmup_s}"
    python3 load_tester.py --warmup-c234 "$warmup_s" \
        2>&1 | tee "${exp_dir}/load_tester_output.txt"

    # Stop monitor + server
    kill "$MONITOR_PID" 2>/dev/null; wait "$MONITOR_PID" 2>/dev/null || true
    kill "$SERVER_PID"  2>/dev/null; wait "$SERVER_PID"  2>/dev/null || true

    # Archive server + test logs
    local server_log="logs/server_${mode}.log"
    if [ -f "$server_log" ]; then
        mv "$server_log" "${exp_dir}/server.log"
    else
        warn "Server log not found: ${server_log}"
    fi
    local test_log
    test_log=$(find logs -maxdepth 1 -name "test_*.log" -newer "$sentinel" 2>/dev/null \
               | sort | tail -1)
    rm -f "$sentinel"
    if [ -n "$test_log" ] && [ -f "$test_log" ]; then
        mv "$test_log" "${exp_dir}/test.log"
    else
        warn "Test log not found for ${label}"
    fi

    success "Archived -> ${exp_dir}/"
}

# ── Main sweep ────────────────────────────────────────────────────────────────
info "Sweep grid: ${#WARMUPS[@]} W × ${#MODES[@]} modes × ${N_TRIALS} trials = $(( ${#WARMUPS[@]} * ${#MODES[@]} * N_TRIALS )) runs"

for trial in $(seq 1 "$N_TRIALS"); do
    if [ "$N_TRIALS" -gt 1 ]; then
        echo ""
        echo "════════════════════════════════════════════════════════════════"
        info "Trial ${trial} / ${N_TRIALS}"
        echo "════════════════════════════════════════════════════════════════"
    fi
    for w in "${WARMUPS[@]}"; do
        for m in "${MODES[@]}"; do
            run_one "$w" "$m" "$trial"
        done
    done
done

echo ""
echo "================================================================"
success "Sweep complete. Results:"
ls -d "${SWEEP_DIR}"/*/ 2>/dev/null | sort | while read d; do
    echo "    ${d}"
done
echo "================================================================"
echo "Next: python3 figures/plot_sweep.py ${SWEEP_DIR}"