ci: keep crane migration incomplete until deletion-grade gates pass

Author: mrjf

.crane/scripts/score.go

@@ -5,7 +5,7 @@
 // Usage:
 //   APM_PYTHON_BIN=/path/to/apm go test -count=1 -json ./... | go run .crane/scripts/score.go
 //
-// This script implements the deletion-grade framework from issue #96.
+// This script implements the deletion-grade framework from issues #78 and #96.
 // migration_score = 1.0 only when ALL of the following gates pass:
 //
 //   Gate 1 -- python_reference_required: APM_PYTHON_BIN must be set and valid.
@@ -15,18 +15,28 @@
 //   Gate 2 -- go_tests_pass: every Go test in the module must pass. A single
 //             failing non-parity test voids the gate.
 //
-//   Gate 3 -- help_parity: TestParityCompletionCommandMatrix must pass.
-//             Every public command must respond to --help with exit 0.
+//   Gate 3 -- surface_parity: TestParityCompletionSurfaceParity must pass.
+//             Python and Go command/option/subcommand inventories must match.
 //
-//   Gate 4 -- version_parity: TestParityCompletionVersionEquivalent must pass.
+//   Gate 4 -- help_parity: TestParityCompletionCommandMatrix and
+//             TestParityCompletionHelpIdentical must pass. Every public help
+//             and invalid-usage path must match Python.
 //
-//   Gate 5 -- init_parity: TestParityCompletionInitParity must pass.
-//             The init command must produce apm.yml equivalent to Python.
+//   Gate 5 -- functional_contracts: TestParityCompletionFunctionalContracts
+//             must pass. Supported command behavior must be covered by
+//             black-box Python-vs-Go contracts.
 //
-//   Gate 6 -- error_parity: TestParityCompletionErrorParity must pass.
-//             Unknown commands must produce matching non-zero exit codes.
+//   Gate 6 -- state_diff_contracts: TestParityCompletionStateDiffContracts
+//             must pass. Mutating commands must match Python filesystem,
+//             lockfile, config, cache, and generated-artifact effects.
 //
-//   Gate 7 -- no_known_exceptions: the test output must not contain any
+//   Gate 7 -- python_tests_pass: TestParityCompletionPythonSuite must pass.
+//             The original Python reference test suite must still be green.
+//
+//   Gate 8 -- benchmarks_pass: TestParityCompletionBenchmarks must pass.
+//             Migration benchmarks must run and satisfy the configured guard.
+//
+//   Gate 9 -- no_known_exceptions: the test output must not contain any
 //             "approved exception" log line. Final cutover requires zero exceptions.
 //
 // If Gate 1 fails, migration_score is forced to 0.0 regardless of other gates.
@@ -73,13 +83,16 @@ func main() {
 	scanner := bufio.NewScanner(os.Stdin)
 	scanner.Buffer(make([]byte, 4*1024*1024), 4*1024*1024)
 
-	// Deletion-grade gate test names (exact or prefix).
+	// Deletion-grade gate test names.
 	const (
-		gateHardGate      = "TestParityCompletionHardGate"
-		gateCmdMatrix     = "TestParityCompletionCommandMatrix"
-		gateVersionParity = "TestParityCompletionVersionEquivalent"
-		gateInitParity    = "TestParityCompletionInitParity"
-		gateErrorParity   = "TestParityCompletionErrorParity"
+		gateHardGate            = "TestParityCompletionHardGate"
+		gateSurfaceParity       = "TestParityCompletionSurfaceParity"
+		gateCommandMatrix       = "TestParityCompletionCommandMatrix"
+		gateHelpIdentical       = "TestParityCompletionHelpIdentical"
+		gateFunctionalContracts = "TestParityCompletionFunctionalContracts"
+		gateStateDiffContracts  = "TestParityCompletionStateDiffContracts"
+		gatePythonSuite         = "TestParityCompletionPythonSuite"
+		gateBenchmarks          = "TestParityCompletionBenchmarks"
 	)
 
 	// Track per-test pass/fail.
@@ -151,64 +164,39 @@ func main() {
 		gate2.Reason = fmt.Sprintf("%d/%d tests passing", passingTests, totalTests)
 	}
 
-	// Gate 3: help_parity (command matrix)
-	gate3 := GateResult{Name: "help_parity"}
-	if testPassed[gateCmdMatrix] && !testFailed[gateCmdMatrix] {
-		gate3.Passing = true
-	} else if testFailed[gateCmdMatrix] {
-		gate3.Passing = false
-		gate3.Reason = "TestParityCompletionCommandMatrix failed"
-	} else {
-		gate3.Passing = false
-		gate3.Reason = "TestParityCompletionCommandMatrix not found"
-	}
+	// Gate 3: surface_parity
+	gate3 := singleTestGate("surface_parity", gateSurfaceParity, testPassed, testFailed)
 
-	// Gate 4: version_parity
-	gate4 := GateResult{Name: "version_parity"}
-	if testPassed[gateVersionParity] && !testFailed[gateVersionParity] {
-		gate4.Passing = true
-	} else if testFailed[gateVersionParity] {
-		gate4.Passing = false
-		gate4.Reason = "TestParityCompletionVersionEquivalent failed"
-	} else {
-		gate4.Passing = false
-		gate4.Reason = "TestParityCompletionVersionEquivalent not found"
-	}
+	// Gate 4: help_parity
+	gate4 := multiTestGate(
+		"help_parity",
+		[]string{gateCommandMatrix, gateHelpIdentical},
+		testPassed,
+		testFailed,
+	)
 
-	// Gate 5: init_parity
-	gate5 := GateResult{Name: "init_parity"}
-	if testPassed[gateInitParity] && !testFailed[gateInitParity] {
-		gate5.Passing = true
-	} else if testFailed[gateInitParity] {
-		gate5.Passing = false
-		gate5.Reason = "TestParityCompletionInitParity failed"
-	} else {
-		gate5.Passing = false
-		gate5.Reason = "TestParityCompletionInitParity not found"
-	}
+	// Gate 5: functional_contracts
+	gate5 := singleTestGate("functional_contracts", gateFunctionalContracts, testPassed, testFailed)
 
-	// Gate 6: error_parity
-	gate6 := GateResult{Name: "error_parity"}
-	if testPassed[gateErrorParity] && !testFailed[gateErrorParity] {
-		gate6.Passing = true
-	} else if testFailed[gateErrorParity] {
-		gate6.Passing = false
-		gate6.Reason = "TestParityCompletionErrorParity failed"
-	} else {
-		gate6.Passing = false
-		gate6.Reason = "TestParityCompletionErrorParity not found"
-	}
+	// Gate 6: state_diff_contracts
+	gate6 := singleTestGate("state_diff_contracts", gateStateDiffContracts, testPassed, testFailed)
+
+	// Gate 7: python_tests_pass
+	gate7 := singleTestGate("python_tests_pass", gatePythonSuite, testPassed, testFailed)
+
+	// Gate 8: benchmarks_pass
+	gate8 := singleTestGate("benchmarks_pass", gateBenchmarks, testPassed, testFailed)
 
-	// Gate 7: no_known_exceptions
-	gate7 := GateResult{Name: "no_known_exceptions"}
+	// Gate 9: no_known_exceptions
+	gate9 := GateResult{Name: "no_known_exceptions"}
 	if knownExceptionsFound {
-		gate7.Passing = false
-		gate7.Reason = "output contains 'approved exception' -- all exceptions must be resolved for cutover"
+		gate9.Passing = false
+		gate9.Reason = "output contains 'approved exception' -- all exceptions must be resolved for cutover"
 	} else {
-		gate7.Passing = true
+		gate9.Passing = true
 	}
 
-	gates := []GateResult{gate1, gate2, gate3, gate4, gate5, gate6, gate7}
+	gates := []GateResult{gate1, gate2, gate3, gate4, gate5, gate6, gate7, gate8, gate9}
 
 	// Count parity tests (any test with "Parity" in name from cmd/apm).
 	parityPassing, parityTotal := 0, 0
@@ -257,3 +245,35 @@ func main() {
 	out, _ := json.MarshalIndent(score, "", "  ")
 	fmt.Println(string(out))
 }
+
+func singleTestGate(name, testName string, testPassed, testFailed map[string]bool) GateResult {
+	return multiTestGate(name, []string{testName}, testPassed, testFailed)
+}
+
+func multiTestGate(name string, testNames []string, testPassed, testFailed map[string]bool) GateResult {
+	for _, testName := range testNames {
+		if testFailed[testName] {
+			return GateResult{
+				Name:    name,
+				Passing: false,
+				Reason:  testName + " failed",
+			}
+		}
+	}
+
+	var missing []string
+	for _, testName := range testNames {
+		if !testPassed[testName] {
+			missing = append(missing, testName)
+		}
+	}
+	if len(missing) > 0 {
+		return GateResult{
+			Name:    name,
+			Passing: false,
+			Reason:  strings.Join(missing, ", ") + " not found",
+		}
+	}
+
+	return GateResult{Name: name, Passing: true}
+}

tests/unit/test_crane_score.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
 import json
+import os
 import shutil
 import subprocess
+import tempfile
 from pathlib import Path
 
 import pytest
@@ -14,46 +16,120 @@ def _run_score(input_lines: list[str]) -> dict[str, object]:
     if shutil.which("go") is None:
         pytest.skip("Go toolchain is not installed")
 
-    result = subprocess.run(
-        ["go", "run", ".crane/scripts/score.go"],
-        cwd=ROOT,
-        input="\n".join(input_lines) + "\n",
-        text=True,
-        capture_output=True,
-        check=True,
-    )
+    env = os.environ.copy()
+    with tempfile.TemporaryDirectory(prefix="apm-go-cache-") as go_cache:
+        env.setdefault("GOCACHE", go_cache)
+        result = subprocess.run(
+            ["go", "run", ".crane/scripts/score.go"],
+            cwd=ROOT,
+            input="\n".join(input_lines) + "\n",
+            text=True,
+            capture_output=True,
+            check=True,
+            env=env,
+        )
     return json.loads(result.stdout)
 
 
-def test_crane_score_counts_parity_events() -> None:
+def _event(action: str, test: str, *, output: str = "") -> str:
+    return json.dumps(
+        {
+            "Action": action,
+            "Package": "github.com/githubnext/apm/cmd/apm",
+            "Test": test,
+            "Output": output,
+        }
+    )
+
+
+def _pass(test: str) -> list[str]:
+    return [_event("run", test), _event("pass", test)]
+
+
+def _gates(score: dict[str, object]) -> dict[str, dict[str, object]]:
+    gates = score["gates"]
+    assert isinstance(gates, list)
+    return {gate["name"]: gate for gate in gates}
+
+
+def _all_required_gate_events() -> list[str]:
+    tests = [
+        "TestParityCompletionHardGate",
+        "TestParityCompletionSurfaceParity",
+        "TestParityCompletionCommandMatrix",
+        "TestParityCompletionHelpIdentical",
+        "TestParityCompletionFunctionalContracts",
+        "TestParityCompletionStateDiffContracts",
+        "TestParityCompletionPythonSuite",
+        "TestParityCompletionBenchmarks",
+    ]
+    return [line for test in tests for line in _pass(test)]
+
+
+def test_crane_score_blocks_help_only_completion() -> None:
     score = _run_score(
         [
             "not json",
-            '{"Action":"run","Package":"github.com/githubnext/apm/internal/parity","Test":"TestInstallParity"}',
-            '{"Action":"pass","Package":"github.com/githubnext/apm/internal/parity","Test":"TestInstallParity"}',
-            '{"Action":"run","Package":"github.com/githubnext/apm/internal/parity","Test":"TestCompileParity"}',
-            '{"Action":"pass","Package":"github.com/githubnext/apm/internal/parity","Test":"TestCompileParity"}',
+            *_pass("TestParityCompletionHardGate"),
+            *_pass("TestParityCompletionCommandMatrix"),
+            *_pass("TestParityCompletionHelpIdentical"),
+            *_pass("TestParityCompletionVersionEquivalent"),
+            *_pass("TestParityCompletionInitParity"),
+            *_pass("TestParityCompletionErrorParity"),
         ]
     )
 
-    assert score["migration_score"] == pytest.approx(2 / 302)
-    assert score["progress"] == pytest.approx(2 / 302)
-    assert score["parity_passing"] == 2
-    assert score["parity_total"] == 302
-    assert score["source_tests_passing"] == 247
-    assert score["target_tests_passing"] == 2
-    assert score["perf_ratio"] == 1.0
+    gates = _gates(score)
+
+    assert score["migration_score"] < 1.0
+    assert gates["python_reference_required"]["passing"] is True
+    assert gates["go_tests_pass"]["passing"] is True
+    assert gates["help_parity"]["passing"] is True
+    assert gates["surface_parity"]["passing"] is False
+    assert gates["functional_contracts"]["passing"] is False
+    assert gates["state_diff_contracts"]["passing"] is False
+    assert gates["python_tests_pass"]["passing"] is False
+    assert gates["benchmarks_pass"]["passing"] is False
+
 
+def test_crane_score_reaches_one_only_when_all_deletion_grade_gates_pass() -> None:
+    score = _run_score(_all_required_gate_events())
 
-def test_crane_score_applies_target_correctness_gate() -> None:
+    assert score["migration_score"] == 1.0
+    assert score["progress"] == 1.0
+    assert all(gate["passing"] for gate in _gates(score).values())
+
+
+def test_crane_score_forces_zero_without_python_reference() -> None:
     score = _run_score(
         [
-            '{"Action":"run","Package":"github.com/githubnext/apm/internal/parity","Test":"TestInstallParity"}',
-            '{"Action":"pass","Package":"github.com/githubnext/apm/internal/parity","Test":"TestInstallParity"}',
-            '{"Action":"run","Package":"github.com/githubnext/apm/internal/config","Test":"TestConfig"}',
+            *_pass("TestParityCompletionSurfaceParity"),
+            *_pass("TestParityCompletionCommandMatrix"),
+            *_pass("TestParityCompletionHelpIdentical"),
+            *_pass("TestParityCompletionFunctionalContracts"),
+            *_pass("TestParityCompletionStateDiffContracts"),
+            *_pass("TestParityCompletionPythonSuite"),
+            *_pass("TestParityCompletionBenchmarks"),
         ]
     )
 
+    gates = _gates(score)
+
     assert score["migration_score"] == 0
-    assert score["progress"] == pytest.approx(1 / 302)
-    assert score["target_tests_passing"] == 1
+    assert gates["python_reference_required"]["passing"] is False
+    assert "TestParityCompletionHardGate not found" in gates["python_reference_required"]["reason"]
+
+
+def test_crane_score_blocks_known_exceptions() -> None:
+    score = _run_score(
+        [
+            *_all_required_gate_events(),
+            _event("output", "TestParityCompletionHelpIdentical", output="APPROVED-EXCEPTION: no"),
+        ]
+    )
+
+    gates = _gates(score)
+
+    assert score["migration_score"] < 1.0
+    assert gates["no_known_exceptions"]["passing"] is False
+    assert "approved exception" in gates["no_known_exceptions"]["reason"]