Skip to content

Commit 4fe2170

Browse files
author
miranov25
committed
Updated becnhamrk
1 parent ddf6c3c commit 4fe2170

1 file changed

Lines changed: 105 additions & 7 deletions

File tree

‎UTILS/dfextensions/AliasDataFrame/benchmarks/run_benchmark.sh‎

Lines changed: 105 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,13 @@
1010
# ./run_benchmark.sh --quick # Quick mode (smaller data)
1111
# ./run_benchmark.sh --synthetic-only # Only synthetic benchmarks (no ROOT files)
1212
# ./run_benchmark.sh --generate-data # Generate synthetic ROOT file first
13+
# ./run_benchmark.sh --save-baseline # Save results as new baseline
14+
# ./run_benchmark.sh --compare-baseline # Compare against baseline (detect regressions)
15+
# ./run_benchmark.sh --threshold 15 # Set regression threshold (default: 20%)
1316
#
1417
# Exit Codes:
1518
# 0 - All benchmarks passed
16-
# 1 - Some benchmarks failed (when --strict)
19+
# 1 - Some benchmarks failed or regression detected (when --strict)
1720
# 0 - Always (default, failures reported but not fatal)
1821
#
1922

@@ -39,6 +42,12 @@ SYNTHETIC_ONLY=false
3942
GENERATE_DATA=false
4043
VERBOSE=false
4144

45+
# Baseline comparison configuration
46+
SAVE_BASELINE=false
47+
COMPARE_BASELINE=false
48+
THRESHOLD=20
49+
BASELINE_FILE="${SCRIPT_DIR}/baseline.json"
50+
4251
# Results tracking
4352
declare -a BENCHMARK_NAMES
4453
declare -a BENCHMARK_STATUS
@@ -141,6 +150,22 @@ while [[ $# -gt 0 ]]; do
141150
VERBOSE=true
142151
shift
143152
;;
153+
--save-baseline)
154+
SAVE_BASELINE=true
155+
shift
156+
;;
157+
--compare-baseline)
158+
COMPARE_BASELINE=true
159+
shift
160+
;;
161+
--threshold)
162+
THRESHOLD="$2"
163+
shift 2
164+
;;
165+
--baseline)
166+
BASELINE_FILE="$2"
167+
shift 2
168+
;;
144169
--output)
145170
OUTPUT_DIR="$2"
146171
shift 2
@@ -152,16 +177,27 @@ while [[ $# -gt 0 ]]; do
152177
echo " --quick Quick mode (smaller data, faster)"
153178
echo " --synthetic-only Only run synthetic benchmarks (no ROOT file needed)"
154179
echo " --generate-data Generate synthetic ROOT file before running"
155-
echo " --strict Exit with code 1 if any benchmark fails"
180+
echo " --strict Exit with code 1 if any benchmark fails or regression detected"
156181
echo " --verbose, -v Show detailed output"
157182
echo " --output DIR Output directory (default: benchmarks/results)"
158-
echo " --help, -h Show this help"
183+
echo ""
184+
echo "Regression Detection:"
185+
echo " --save-baseline Save current results as new baseline"
186+
echo " --compare-baseline Compare results against baseline (detect regressions)"
187+
echo " --threshold PCT Regression threshold percentage (default: 20)"
188+
echo " --baseline FILE Baseline file path (default: benchmarks/baseline.json)"
159189
echo ""
160190
echo "Benchmarks:"
161191
echo " benchmark_performance.py Synthetic data tests (always runs)"
162192
echo " benchmark_read_tree.py ROOT file read tests (needs data)"
163193
echo " benchmark_subframe.py Subframe tests (needs data)"
164194
echo " benchmark_parallel.py Parallel scaling tests (needs data)"
195+
echo ""
196+
echo "Examples:"
197+
echo " $0 # Run all benchmarks"
198+
echo " $0 --save-baseline # Run and save as baseline"
199+
echo " $0 --compare-baseline --strict # CI mode: fail on regression"
200+
echo " $0 --compare-baseline --threshold 15 # Custom threshold"
165201
exit 0
166202
;;
167203
*)
@@ -352,14 +388,17 @@ if [[ "$SYNTHETIC_ONLY" = false ]] && [[ -f "$SYNTHETIC_DATA" ]]; then
352388
PARALLEL_ARGS="--repeats 1 --timeout 15 --max-workers 4"
353389
fi
354390

391+
# Save JSON output for baseline comparison
392+
PARALLEL_JSON="${OUTPUT_DIR}/benchmark_parallel_${TIMESTAMP}.json"
393+
355394
START_TIME=$(get_time)
356395

357396
if [[ "$VERBOSE" = true ]]; then
358-
OUTPUT=$(python3 "${SCRIPT_DIR}/benchmark_parallel.py" "$SYNTHETIC_DATA" --treename tree $PARALLEL_ARGS 2>&1)
397+
OUTPUT=$(python3 "${SCRIPT_DIR}/benchmark_parallel.py" "$SYNTHETIC_DATA" --treename tree $PARALLEL_ARGS --json "$PARALLEL_JSON" 2>&1)
359398
PAR_STATUS=$?
360399
echo "$OUTPUT"
361400
else
362-
OUTPUT=$(python3 "${SCRIPT_DIR}/benchmark_parallel.py" "$SYNTHETIC_DATA" --treename tree $PARALLEL_ARGS --quiet 2>&1)
401+
OUTPUT=$(python3 "${SCRIPT_DIR}/benchmark_parallel.py" "$SYNTHETIC_DATA" --treename tree $PARALLEL_ARGS --json "$PARALLEL_JSON" --quiet 2>&1)
363402
PAR_STATUS=$?
364403
fi
365404

@@ -381,6 +420,60 @@ else
381420
echo ""
382421
fi
383422

423+
# =============================================================================
424+
# Baseline Operations
425+
# =============================================================================
426+
427+
# Save baseline if requested
428+
if [[ "$SAVE_BASELINE" = true ]]; then
429+
echo "--- Saving Baseline ---"
430+
431+
if python3 "${SCRIPT_DIR}/baseline_utils.py" merge "${OUTPUT_DIR}" "${BASELINE_FILE}" 2>&1; then
432+
echo ""
433+
echo -e "\033[32m✓ Baseline saved to: ${BASELINE_FILE}\033[0m"
434+
echo " Commit this file to track performance over time."
435+
else
436+
echo -e "\033[31m✗ Failed to save baseline\033[0m"
437+
((TOTAL_FAILED++))
438+
fi
439+
echo ""
440+
fi
441+
442+
# Compare against baseline if requested
443+
REGRESSION_DETECTED=false
444+
if [[ "$COMPARE_BASELINE" = true ]]; then
445+
echo "--- Comparing Against Baseline ---"
446+
447+
if [[ -f "$BASELINE_FILE" ]]; then
448+
COMPARE_ARGS="--threshold $THRESHOLD --latest"
449+
if [[ "$STRICT_MODE" = true ]]; then
450+
COMPARE_ARGS="$COMPARE_ARGS --strict"
451+
fi
452+
453+
# Export comparison results
454+
COMPARISON_JSON="${OUTPUT_DIR}/comparison_${TIMESTAMP}.json"
455+
456+
python3 "${SCRIPT_DIR}/baseline_utils.py" compare "${OUTPUT_DIR}" "$BASELINE_FILE" $COMPARE_ARGS --json "$COMPARISON_JSON"
457+
COMPARE_STATUS=$?
458+
459+
if [[ $COMPARE_STATUS -eq 1 ]]; then
460+
REGRESSION_DETECTED=true
461+
((TOTAL_FAILED++))
462+
log_result "regression_check" "FAILED" "" "Regression detected"
463+
elif [[ $COMPARE_STATUS -eq 0 ]]; then
464+
log_result "regression_check" "PASSED" "" "Within threshold"
465+
else
466+
echo -e "\033[33m⚠️ Comparison error (exit code $COMPARE_STATUS)\033[0m"
467+
fi
468+
else
469+
echo -e "\033[33m⚠️ No baseline.json found at: ${BASELINE_FILE}\033[0m"
470+
echo " Run with --save-baseline to create one."
471+
log_result "regression_check" "SKIPPED" "" "No baseline"
472+
((TOTAL_SKIPPED++))
473+
fi
474+
echo ""
475+
fi
476+
384477
# =============================================================================
385478
# Summary (pytest-style)
386479
# =============================================================================
@@ -438,8 +531,13 @@ echo "Results saved to: ${OUTPUT_DIR}/"
438531
# Exit Code
439532
# =============================================================================
440533

441-
if [[ "$STRICT_MODE" = true ]] && [[ $TOTAL_FAILED -gt 0 ]]; then
442-
exit 1
534+
if [[ "$STRICT_MODE" = true ]]; then
535+
if [[ $TOTAL_FAILED -gt 0 ]]; then
536+
exit 1
537+
fi
538+
if [[ "$REGRESSION_DETECTED" = true ]]; then
539+
exit 1
540+
fi
443541
fi
444542

445543
exit 0

0 commit comments

Comments
 (0)