3131FORMAL_RUN_COUNT = 12
3232FORMAL_SAMPLING_MODE = "paired-adjacent"
3333FORMAL_PAIR_COUNT = 6
34+ MINIMUM_TARGET_IMPROVEMENT = 0.05
35+ MAXIMUM_TARGET_MEDIAN_RATIO = 1.0 - MINIMUM_TARGET_IMPROVEMENT
36+ MAXIMUM_CANDIDATE_WORK_RATIO = 0.10
3437
3538
3639def read_json (path : Path ) -> dict [str , Any ]:
@@ -468,8 +471,10 @@ def evaluate(manifest_path: Path, evidence_root: Path, expected_items: int,
468471 candidate_viewer_ratio = candidate_viewer_checks / baseline_full_scan
469472
470473 checks = {
471- "droppedItemMedianRatioAtMost0_50" : dropped ["medianBRatioToA" ] <= 0.50 ,
472- "droppedItemCiExcludesRegression" : dropped ["ratioBootstrap95Ci" ][1 ] < 1.0 ,
474+ "droppedItemMedianImprovementAtLeast5Percent" :
475+ dropped ["medianBRatioToA" ] <= MAXIMUM_TARGET_MEDIAN_RATIO ,
476+ "droppedItemBootstrap95CiUpperBelow1_00" :
477+ dropped ["ratioBootstrap95Ci" ][1 ] < 1.0 ,
473478 "msptP95CiUpperAtMost1_02" : p95 ["ratioBootstrap95Ci" ][1 ] <= 1.02 ,
474479 "msptP99CiUpperAtMost1_05" : p99 ["ratioBootstrap95Ci" ][1 ] <= 1.05 ,
475480 "allRunsRetainedGlobalPopulation" : all (
@@ -482,18 +487,29 @@ def evaluate(manifest_path: Path, evidence_root: Path, expected_items: int,
482487 run ["fullScanCandidates" ] > 0 for run in runs if run ["variant" ] == "A" ),
483488 "candidateTreatmentIsolated" : all (
484489 run ["sourceOwned" ] is (run ["variant" ] == "B" ) for run in runs ),
485- "candidateSpatialCandidatesAtMost10PercentOfBaseline" : candidate_spatial_ratio <= 0.10 ,
486- "candidateViewerChecksAtMost10PercentOfBaseline" : candidate_viewer_ratio <= 0.10 ,
490+ "candidateSpatialCandidatesAtMost10PercentOfBaseline" :
491+ candidate_spatial_ratio <= MAXIMUM_CANDIDATE_WORK_RATIO ,
492+ "candidateViewerChecksAtMost10PercentOfBaseline" :
493+ candidate_viewer_ratio <= MAXIMUM_CANDIDATE_WORK_RATIO ,
487494 }
488495 formal_complete = all (
489496 result .get ("formalComplete" ) is True for result in analyses .values ())
490497 passed = formal_complete and all (checks .values ())
491498 gate = {
492- "schemaVersion" : 2 ,
499+ "schemaVersion" : 3 ,
493500 "scenario" : SCENARIO ,
494501 "abFactor" : AB_FACTOR ,
495502 "formalComplete" : formal_complete ,
496503 "serverRuntimeGatePassed" : passed ,
504+ "acceptancePolicy" : {
505+ "targetMetric" : "droppedItemMs" ,
506+ "minimumMedianImprovementPercent" : MINIMUM_TARGET_IMPROVEMENT * 100.0 ,
507+ "bootstrap95CiUpperMustBeBelow" : 1.0 ,
508+ "msptP95CiUpperAtMost" : 1.02 ,
509+ "msptP99CiUpperAtMost" : 1.05 ,
510+ "candidateWorkRatioAtMost" : MAXIMUM_CANDIDATE_WORK_RATIO ,
511+ "basis" : "docs/phase2-performance-validation.md section 7" ,
512+ },
497513 "workload" : {
498514 "trackedItems" : expected_items ,
499515 "nearbyLabels" : expected_nearby_items ,
@@ -532,9 +548,11 @@ def evaluate(manifest_path: Path, evidence_root: Path, expected_items: int,
532548 f"Workload: `{ expected_items } ` tracked / `{ expected_nearby_items } ` nearby labels.\n \n " )
533549 stream .write (
534550 f"droppedItemMs B/A: `{ dropped ['medianBRatioToA' ]} ` "
535- f"(95% CI `{ dropped ['ratioBootstrap95Ci' ]} `).\n \n " )
551+ f"(95% CI `{ dropped ['ratioBootstrap95Ci' ]} `; required median "
552+ f"`<= { MAXIMUM_TARGET_MEDIAN_RATIO :.2f} ` and CI upper `< 1.0`).\n \n " )
536553 stream .write (
537- f"Candidate/full-scan ratio: `{ candidate_spatial_ratio :.6f} `.\n \n " )
554+ f"Candidate/full-scan ratio: `{ candidate_spatial_ratio :.6f} ` "
555+ f"(required `<= { MAXIMUM_CANDIDATE_WORK_RATIO :.2f} `).\n \n " )
538556 stream .write (
539557 f"MSPT p95 CI upper: `{ p95 ['ratioBootstrap95Ci' ][1 ]} `; "
540558 f"p99 CI upper: `{ p99 ['ratioBootstrap95Ci' ][1 ]} `.\n " )
@@ -700,7 +718,7 @@ def write_manifest() -> None:
700718 write_manifest ()
701719
702720 analysis_parameters = {
703- "droppedItemMs" : (0.40 , [0.35 , 0.45 ]),
721+ "droppedItemMs" : (0.90 , [0.85 , 0.94 ]),
704722 "msptP95" : (0.99 , [0.98 , 1.01 ]),
705723 "msptP99" : (1.00 , [0.99 , 1.04 ]),
706724 }
@@ -773,9 +791,35 @@ def write_analyses() -> None:
773791 summary = os .environ .pop ("GITHUB_STEP_SUMMARY" , None )
774792 try :
775793 gate = evaluate (manifest_path , root , 2_048 , 128 , root / "gate.json" )
794+ assert gate ["schemaVersion" ] == 3
776795 assert gate ["serverRuntimeGatePassed" ] is True
796+ assert gate ["checks" ]["droppedItemMedianImprovementAtLeast5Percent" ] is True
797+ assert gate ["checks" ]["droppedItemBootstrap95CiUpperBelow1_00" ] is True
777798 assert gate ["ratios" ]["candidateSpatialToBaselineFullScan" ] == 0.0625
778799
800+ analysis_parameters ["droppedItemMs" ] = (0.96 , [0.92 , 0.99 ])
801+ write_analyses ()
802+ small_effect_gate = evaluate (
803+ manifest_path , root , 2_048 , 128 , root / "small-effect-gate.json" )
804+ assert small_effect_gate ["serverRuntimeGatePassed" ] is False
805+ assert small_effect_gate ["checks" ][
806+ "droppedItemMedianImprovementAtLeast5Percent" ] is False
807+ assert small_effect_gate ["checks" ][
808+ "droppedItemBootstrap95CiUpperBelow1_00" ] is True
809+
810+ analysis_parameters ["droppedItemMs" ] = (0.90 , [0.85 , 1.01 ])
811+ write_analyses ()
812+ uncertain_effect_gate = evaluate (
813+ manifest_path , root , 2_048 , 128 , root / "uncertain-effect-gate.json" )
814+ assert uncertain_effect_gate ["serverRuntimeGatePassed" ] is False
815+ assert uncertain_effect_gate ["checks" ][
816+ "droppedItemMedianImprovementAtLeast5Percent" ] is True
817+ assert uncertain_effect_gate ["checks" ][
818+ "droppedItemBootstrap95CiUpperBelow1_00" ] is False
819+
820+ analysis_parameters ["droppedItemMs" ] = (0.90 , [0.85 , 0.94 ])
821+ write_analyses ()
822+
779823 declining_metrics_path = run_roots [0 ] / "iv-perf.json"
780824 declining_metrics = read_json (declining_metrics_path )
781825 declining_metrics ["droppedTrackedItemsMin" ] = 2_047
@@ -864,6 +908,9 @@ def write_analyses() -> None:
864908 print (json .dumps ({
865909 "passed" : True ,
866910 "analysisSchemaVersion" : 2 ,
911+ "gateSchemaVersion" : 3 ,
912+ "documentedEffectSizeGate" : True ,
913+ "confidenceIntervalImprovementGate" : True ,
867914 "canonicalWindowGate" : True ,
868915 "analysisEvidenceClosure" : True ,
869916 "mixedStackPaperClientRejected" : True ,
0 commit comments