diff --git a/.gitmodules b/.gitmodules index b026356de5..1f1fade726 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "utils/aiperf"] path = utils/aiperf url = https://github.com/SemiAnalysisAI/aiperf.git - branch = cquil11/aiperf-agentx-v1.0 + branch = cquil11/deterministic-agentic-warmup-count-main diff --git a/benchmarks/benchmark_lib.sh b/benchmarks/benchmark_lib.sh index c5e6fb2c86..413ac4e162 100644 --- a/benchmarks/benchmark_lib.sh +++ b/benchmarks/benchmark_lib.sh @@ -1744,14 +1744,13 @@ build_replay_cmd() { # utils/aiperf/docs/tutorials/agentx-mvp.md. local result_dir="$1" local duration="$DURATION" - local cache_warmup_duration="${AIPERF_AGENTIC_CACHE_WARMUP_DURATION:-600}" + local warmup_requests_per_lane="${AIPERF_WARMUP_REQUESTS_PER_LANE:-10}" # Fast mode is an e2e-only feedback preset used before canonical one-hour # sweeps. AIPerf already exposes both controls, so no AIPerf patch is # required. if [[ "${AIPERF_EXPERIMENTAL_FAST:-0}" == "1" ]]; then duration=1200 - cache_warmup_duration=300 fi export AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES="${AIPERF_DATASET_WEKA_LIVE_ASSISTANT_RESPONSES:-0}" @@ -1765,6 +1764,9 @@ build_replay_cmd() { # aiperf validates that SERVICE_PROFILE_CONFIGURE_TIMEOUT >= # DATASET_CONFIGURATION_TIMEOUT at startup. Bump it in lockstep. export AIPERF_SERVICE_PROFILE_CONFIGURE_TIMEOUT=1800 + # Headless realtime metrics are opt-in on current AIPerf main. Enable the + # rolling TTFT/ITL/throughput block and emit it every 30 seconds. + export AIPERF_UI_REALTIME_METRICS_ENABLED=true REPLAY_CMD="$AIPERF_CLI profile --scenario inferencex-agentx-mvp" REPLAY_CMD+=" --url http://localhost:$PORT" REPLAY_CMD+=" --endpoint /v1/chat/completions" @@ -1773,6 +1775,7 @@ build_replay_cmd() { REPLAY_CMD+=" --model $MODEL" REPLAY_CMD+=" --concurrency $CONC" REPLAY_CMD+=" --benchmark-duration $duration" + REPLAY_CMD+=" --stats-interval 30" REPLAY_CMD+=" --random-seed 42" # Fail runs once more than 10% of requests error. This keeps known # transient low-rate failures from killing long sweeps while still @@ -1784,10 +1787,11 @@ build_replay_cmd() { # least one profile turn after warmup. REPLAY_CMD+=" --trajectory-start-min-ratio 0.25" REPLAY_CMD+=" --trajectory-start-max-ratio 0.75" - # After the normal t* snapshot warmup, continue those exact trajectories - # with one-token outputs and no idle delays. Profiling begins only after - # those requests drain and resumes from the resulting live state. - REPLAY_CMD+=" --agentic-cache-warmup-duration $cache_warmup_duration" + # After the normal t* snapshot warmup, advance every trajectory lane by + # the same deterministic number of one-token requests with no idle delay. + # Profiling begins after those requests drain and resumes from the + # resulting live state. + REPLAY_CMD+=" --warmup-requests-per-lane $warmup_requests_per_lane" # Give long-context warmup requests up to 30 minutes to drain before # declaring warmup failed. Recipes whose saturation arms carry a larger # in-flight working set may override via AGENTIC_WARMUP_GRACE_PERIOD diff --git a/perf-changelog.yaml b/perf-changelog.yaml index ac1de53c7d..106073ac1c 100644 --- a/perf-changelog.yaml +++ b/perf-changelog.yaml @@ -5191,3 +5191,9 @@ - "Agentic experiment Variant K (probe, of the #2374 Variant G): REMOVES the kimi-k3-container-deps.sh in-container patch (setup_script, script file, and launcher copy) to verify whether the mamba_hybrid index_fill_ dtype patch is still required by the current vllm/vllm-openai:kimi-k3 image — expected to fail at the first decode step if it is" - "Recipe: benchmarks/multi_node/srt-slurm-recipes/vllm/kimi-k3/agentic/agg-b200-tp8pp2-agentic.yaml on the cluster:b200-dgxc pool" pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2391 + +- config-keys: + - kimik3-fp4-b300-vllm-agentic + description: + - "Test deterministic AIPerf warmup on Kimi K3 B300 with 10 requests per lane." + pr-link: https://github.com/SemiAnalysisAI/InferenceX/pull/2408 diff --git a/utils/aiperf b/utils/aiperf index 381758a88f..29749372a4 160000 --- a/utils/aiperf +++ b/utils/aiperf @@ -1 +1 @@ -Subproject commit 381758a88f5fa6335455ef5b0ded9557c1162814 +Subproject commit 29749372a46386280f9d049f8cbd8762f894b9a3