update regression config

nirmitparikh8 · nirmitparikh8 · commit 2d17fc394a57 · 2025-12-19T12:30:04.000-05:00
diff --git a/experiments/regression_config.rb b/experiments/regression_config.rb
@@ -3,13 +3,13 @@
 # Configuration for regression detection - easily tunable thresholds
 module RegressionConfig
   # Percentile bounds for control limits
-  LOWER_PERCENTILE = 1
-  UPPER_PERCENTILE = 99
+  LOWER_PERCENTILE = 5
+  UPPER_PERCENTILE = 95
 
   # Violation thresholds
-  DEVIATION_VIOLATION_THRESHOLD = 0.3
-  ERROR_RATE_VIOLATION_THRESHOLD = 0.3
-  REJECTION_RATE_VIOLATION_THRESHOLD = 0.3
+  DEVIATION_VIOLATION_THRESHOLD = 0.5
+  ERROR_RATE_VIOLATION_THRESHOLD = 0.8
+  REJECTION_RATE_VIOLATION_THRESHOLD = 0.8
 
   MIN_BASELINE_RUNS = 10
 
diff --git a/experiments/results/baseline/gradual_increase/computed_baseline.txt b/experiments/results/baseline/gradual_increase/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 150
 
-deviation_p3=0.49999999999999456
+deviation_p5=0.7266666666666672
 deviation_p50=7.33333333333334
-deviation_p97=1408.9325000000006
+deviation_p95=1152.9399999999994
 
-error_rate_p3=0.7894
+error_rate_p5=0.868
 error_rate_p50=2.38
-error_rate_p97=4.7618
+error_rate_p95=4.585999999999998
 
-rejection_rate_p3=0.0
+rejection_rate_p5=0.0
 rejection_rate_p50=0.0
-rejection_rate_p97=51.4737
+rejection_rate_p95=49.843499999999985
diff --git a/experiments/results/baseline/gradual_increase_adaptive/computed_baseline.txt b/experiments/results/baseline/gradual_increase_adaptive/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 150
 
-deviation_p3=33.470000000000006
+deviation_p5=46.25
 deviation_p50=82.66666666666667
-deviation_p97=174.05200000000002
+deviation_p95=136.45999999999998
 
-error_rate_p3=0.96
+error_rate_p5=0.99
 error_rate_p50=2.585
-error_rate_p97=4.9006
+error_rate_p95=4.7764999999999995
 
-rejection_rate_p3=0.33
+rejection_rate_p5=0.4035
 rejection_rate_p50=2.5149999999999997
-rejection_rate_p97=5.4819
+rejection_rate_p95=4.765999999999991
diff --git a/experiments/results/baseline/near_target_error_rate/computed_baseline.txt b/experiments/results/baseline/near_target_error_rate/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 90
 
-deviation_p3=0.5583333333333338
+deviation_p5=0.8333333333333341
 deviation_p50=6.666666666666654
-deviation_p97=26.92499999999999
+deviation_p95=23.324999999999996
 
-error_rate_p3=0.8367
+error_rate_p5=0.919
 error_rate_p50=1.14
-error_rate_p97=1.4465999999999999
+error_rate_p95=1.3984999999999999
 
-rejection_rate_p3=0.0
+rejection_rate_p5=0.0
 rejection_rate_p50=0.0
-rejection_rate_p97=0.0
+rejection_rate_p95=0.0
diff --git a/experiments/results/baseline/near_target_error_rate_adaptive/computed_baseline.txt b/experiments/results/baseline/near_target_error_rate_adaptive/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 90
 
-deviation_p3=15.039999999999996
+deviation_p5=21.449999999999996
 deviation_p50=77.5
-deviation_p97=152.20833333333331
+deviation_p95=144.70833333333334
 
-error_rate_p3=0.8801
+error_rate_p5=0.91
 error_rate_p50=1.14
-error_rate_p97=1.3466
+error_rate_p95=1.331
 
-rejection_rate_p3=0.14
+rejection_rate_p5=0.15
 rejection_rate_p50=0.955
-rejection_rate_p97=1.6029999999999998
+rejection_rate_p95=1.523
diff --git a/experiments/results/baseline/one_of_many_services_latency_degradation/computed_baseline.txt b/experiments/results/baseline/one_of_many_services_latency_degradation/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 210
 
-deviation_p3=13.539999999999988
+deviation_p5=22.45
 deviation_p50=993.0
-deviation_p97=1109.73
+deviation_p95=1103.55
 
-error_rate_p3=0.87
+error_rate_p5=0.9145000000000001
 error_rate_p50=1.82
-error_rate_p97=4.373299999999998
+error_rate_p95=3.754999999999998
 
-rejection_rate_p3=0.1727
+rejection_rate_p5=0.24900000000000003
 rejection_rate_p50=8.86
-rejection_rate_p97=10.2646
+rejection_rate_p95=10.161
diff --git a/experiments/results/baseline/one_of_many_services_latency_degradation_adaptive/computed_baseline.txt b/experiments/results/baseline/one_of_many_services_latency_degradation_adaptive/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 210
 
-deviation_p3=94.27
+deviation_p5=99.45
 deviation_p50=1054.0
-deviation_p97=1185.46
+deviation_p95=1169.1999999999998
 
-error_rate_p3=0.91
+error_rate_p5=0.94
 error_rate_p50=2.825
-error_rate_p97=3.8964999999999996
+error_rate_p95=3.7594999999999983
 
-rejection_rate_p3=0.9634999999999999
+rejection_rate_p5=1.0535
 rejection_rate_p50=8.52
-rejection_rate_p97=10.011399999999998
+rejection_rate_p95=9.82
diff --git a/experiments/results/baseline/oscillating_errors/computed_baseline.txt b/experiments/results/baseline/oscillating_errors/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 330
 
-deviation_p3=2.0000000000000018
+deviation_p5=2.8166666666666655
 deviation_p50=622.6666666666666
-deviation_p97=3540.379999999999
+deviation_p95=2923.5000000000014
 
-error_rate_p3=0.6887
+error_rate_p5=0.749
 error_rate_p50=1.895
-error_rate_p97=6.1426
+error_rate_p95=5.8485000000000005
 
-rejection_rate_p3=0.0
+rejection_rate_p5=0.0
 rejection_rate_p50=37.29
-rejection_rate_p97=56.55489999999999
+rejection_rate_p95=54.514500000000005
diff --git a/experiments/results/baseline/oscillating_errors_adaptive/computed_baseline.txt b/experiments/results/baseline/oscillating_errors_adaptive/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 330
 
-deviation_p3=10.000000000000009
+deviation_p5=15.31666666666667
 deviation_p50=56.500000000000014
-deviation_p97=560.7599999999976
+deviation_p95=339.3833333333334
 
-error_rate_p3=0.8587
+error_rate_p5=0.9245
 error_rate_p50=2.275
-error_rate_p97=6.1526000000000005
+error_rate_p95=6.091
 
-rejection_rate_p3=0.11
+rejection_rate_p5=0.2045
 rejection_rate_p50=3.005
-rejection_rate_p97=20.31789999999999
+rejection_rate_p95=5.088500000000007
diff --git a/experiments/results/baseline/slow_query/computed_baseline.txt b/experiments/results/baseline/slow_query/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 210
 
-deviation_p3=4.0000000000000036
+deviation_p5=6.450000000000002
 deviation_p50=828.9999999999999
-deviation_p97=1046.2499999999998
+deviation_p95=1008.0499999999998
 
-error_rate_p3=0.93
+error_rate_p5=0.94
 error_rate_p50=2.07
-error_rate_p97=3.5064999999999995
+error_rate_p95=3.3609999999999998
 
-rejection_rate_p3=0.0
+rejection_rate_p5=0.0
 rejection_rate_p50=6.925000000000001
-rejection_rate_p97=8.9238
+rejection_rate_p95=8.641
diff --git a/experiments/results/baseline/slow_query_adaptive/computed_baseline.txt b/experiments/results/baseline/slow_query_adaptive/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 210
 
-deviation_p3=88.89
+deviation_p5=126.9
 deviation_p50=891.4999999999999
-deviation_p97=1065.0
+deviation_p95=1054.55
 
-error_rate_p3=0.9427
+error_rate_p5=0.97
 error_rate_p50=2.77
-error_rate_p97=3.5273
+error_rate_p95=3.4519999999999995
 
-rejection_rate_p3=0.8916
+rejection_rate_p5=1.107
 rejection_rate_p50=6.86
-rejection_rate_p97=8.804899999999998
+rejection_rate_p95=8.6655
diff --git a/experiments/results/baseline/sudden_error_spike_100/computed_baseline.txt b/experiments/results/baseline/sudden_error_spike_100/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 105
 
-deviation_p3=0.019199999999999606
+deviation_p5=0.22599999999999912
 deviation_p50=10.999999999999998
-deviation_p97=7284.56
+deviation_p95=7044.6
 
-error_rate_p3=0.2536
+error_rate_p5=0.28
 error_rate_p50=1.0
-error_rate_p97=4.250399999999999
+error_rate_p95=4.168
 
-rejection_rate_p3=0.0
+rejection_rate_p5=0.0
 rejection_rate_p50=0.0
-rejection_rate_p97=95.42
+rejection_rate_p95=94.91
diff --git a/experiments/results/baseline/sudden_error_spike_100_adaptive/computed_baseline.txt b/experiments/results/baseline/sudden_error_spike_100_adaptive/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 105
 
-deviation_p3=0.22080000000000727
+deviation_p5=0.6120000000000061
 deviation_p50=63.99999999999999
-deviation_p97=3018.24
+deviation_p95=2792.1999999999994
 
-error_rate_p3=0.77
+error_rate_p5=0.772
 error_rate_p50=1.02
-error_rate_p97=29.89
+error_rate_p95=29.776
 
-rejection_rate_p3=0.12
+rejection_rate_p5=0.132
 rejection_rate_p50=1.69
-rejection_rate_p97=70.904
+rejection_rate_p95=70.548
diff --git a/experiments/results/baseline/sudden_error_spike_20/computed_baseline.txt b/experiments/results/baseline/sudden_error_spike_20/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 105
 
-deviation_p3=0.0
+deviation_p5=1.0000000000000009
 deviation_p50=10.999999999999998
-deviation_p97=7888.439999999996
+deviation_p95=6211.599999999999
 
-error_rate_p3=0.2536
+error_rate_p5=0.366
 error_rate_p50=0.98
-error_rate_p97=4.579199999999999
+error_rate_p95=4.4479999999999995
 
-rejection_rate_p3=0.0
+rejection_rate_p5=0.0
 rejection_rate_p50=0.0
-rejection_rate_p97=80.46119999999999
+rejection_rate_p95=79.19
diff --git a/experiments/results/baseline/sudden_error_spike_20_adaptive/computed_baseline.txt b/experiments/results/baseline/sudden_error_spike_20_adaptive/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 105
 
-deviation_p3=5.239999999999995
+deviation_p5=7.600000000000008
 deviation_p50=64.99999999999999
-deviation_p97=883.4399999999999
+deviation_p95=865.8
 
-error_rate_p3=0.81
+error_rate_p5=0.824
 error_rate_p50=1.05
-error_rate_p97=17.0772
+error_rate_p95=16.794
 
-rejection_rate_p3=0.13
+rejection_rate_p5=0.132
 rejection_rate_p50=0.78
-rejection_rate_p97=15.493999999999998
+rejection_rate_p95=14.862
diff --git a/experiments/results/baseline/sudden_error_spikes/computed_baseline.txt b/experiments/results/baseline/sudden_error_spikes/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 225
 
-deviation_p3=0.6811999999999966
+deviation_p5=1.0000000000000009
 deviation_p50=16.000000000000004
-deviation_p97=8410.08
+deviation_p95=8114.999999999998
 
-error_rate_p3=0.18
+error_rate_p5=0.202
 error_rate_p50=0.97
-error_rate_p97=4.7256
+error_rate_p95=4.678
 
-rejection_rate_p3=0.0
+rejection_rate_p5=0.0
 rejection_rate_p50=13.81
-rejection_rate_p97=94.03999999999999
+rejection_rate_p95=92.828
diff --git a/experiments/results/baseline/sudden_error_spikes_adaptive/computed_baseline.txt b/experiments/results/baseline/sudden_error_spikes_adaptive/computed_baseline.txt
@@ -1,15 +1,15 @@
-# Baseline computed at 2025-12-18 14:16:25 -0500
+# Baseline computed at 2025-12-19 12:09:33 -0500
 # Source files: run_001_time_analysis.csv, run_002_time_analysis.csv, run_003_time_analysis.csv, run_004_time_analysis.csv, run_005_time_analysis.csv, run_006_time_analysis.csv, run_007_time_analysis.csv, run_008_time_analysis.csv, run_009_time_analysis.csv, run_010_time_analysis.csv, run_011_time_analysis.csv, run_012_time_analysis.csv, run_013_time_analysis.csv, run_014_time_analysis.csv, run_015_time_analysis.csv
 # Total data points: 225
 
-deviation_p3=1.1519999999999941
+deviation_p5=1.9400000000000037
 deviation_p50=83.0
-deviation_p97=3555.2799999999997
+deviation_p95=2747.7999999999997
 
-error_rate_p3=0.77
+error_rate_p5=0.79
 error_rate_p50=1.05
-error_rate_p97=32.9856
+error_rate_p95=30.561999999999998
 
-rejection_rate_p3=0.1844
+rejection_rate_p5=0.21
 rejection_rate_p50=1.77
-rejection_rate_p97=70.078
+rejection_rate_p95=68.768
diff --git a/experiments/results/baseline/sustained_load/computed_baseline.txt b/experiments/results/baseline/sustained_load/computed_baseline.txt
diff --git a/experiments/results/baseline/sustained_load_adaptive/computed_baseline.txt b/experiments/results/baseline/sustained_load_adaptive/computed_baseline.txt
