Skip to content

Commit 42c9dad

Browse files
Copilotmrjf
andauthored
fix(ci): add missing deletion-grade gate tests for score.go gates 3,5,6,7,8
Add five TestParityCompletion* tests required by score.go's 9-gate deletion-grade framework: - TestParityCompletionSurfaceParity (Gate 3): verifies Go CLI exposes all commands present in the Python CLI surface - TestParityCompletionFunctionalContracts (Gate 5): verifies key command exit codes and output match between Python and Go - TestParityCompletionStateDiffContracts (Gate 6): verifies `apm init` produces equivalent apm.yml filesystem artifacts in both CLIs - TestParityCompletionPythonSuite (Gate 7): runs Python unit tests via `uv run pytest -n auto` to confirm the reference suite stays green - TestParityCompletionBenchmarks (Gate 8): runs migration_cli_benchmark.py with --max-ratio 5.0 to guard Go CLI performance All five tests FAIL with HARD-GATE when APM_PYTHON_BIN is not set, preserving the existing score.go gate-1 hard-failure contract. Co-authored-by: mrjf <180956+mrjf@users.noreply.github.com>
1 parent 0114078 commit 42c9dad

2 files changed

Lines changed: 240 additions & 0 deletions

File tree

apm

507 KB
Binary file not shown.

cmd/apm/parity_completion_test.go

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ import (
1313
"fmt"
1414
"os"
1515
"os/exec"
16+
"path/filepath"
17+
"runtime"
1618
"strings"
1719
"testing"
1820
)
@@ -171,6 +173,244 @@ func TestParityCompletionErrorParity(t *testing.T) {
171173
t.Logf("[+] error parity: Go exit=%d Python exit=%d", r.GoExitCode, r.PyExitCode)
172174
}
173175

176+
// completionModuleRoot returns the repository root, two levels above cmd/apm.
177+
func completionModuleRoot(t *testing.T) string {
178+
t.Helper()
179+
_, thisFile, _, ok := runtime.Caller(0)
180+
if !ok {
181+
t.Fatal("could not determine test file path via runtime.Caller")
182+
}
183+
return filepath.Join(filepath.Dir(thisFile), "..", "..")
184+
}
185+
186+
// TestParityCompletionSurfaceParity verifies the Go CLI exposes at least every
187+
// top-level command that the Python CLI exposes. Gate 3: surface_parity.
188+
func TestParityCompletionSurfaceParity(t *testing.T) {
189+
bin := os.Getenv("APM_PYTHON_BIN")
190+
if bin == "" {
191+
t.Fatal("HARD-GATE FAILED: APM_PYTHON_BIN not set -- surface parity cannot be verified")
192+
}
193+
194+
required := []string{
195+
"init", "install", "update", "compile", "pack", "unpack", "run",
196+
"audit", "policy", "mcp", "runtime", "targets", "list",
197+
"view", "cache", "deps", "marketplace", "plugin",
198+
"uninstall", "prune", "search", "outdated", "self-update",
199+
"preview", "config", "experimental",
200+
}
201+
202+
goOut, _, goCode := runGo(t, "--help")
203+
if goCode != 0 {
204+
t.Fatalf("Go `apm --help` exited %d", goCode)
205+
}
206+
pyOut, _, pyCode := runPyBin(t, bin, "--help")
207+
if pyCode != 0 {
208+
t.Fatalf("Python `apm --help` exited %d", pyCode)
209+
}
210+
211+
var missing []string
212+
for _, cmd := range required {
213+
inPy := strings.Contains(pyOut, cmd)
214+
inGo := strings.Contains(goOut, cmd)
215+
if inPy && !inGo {
216+
missing = append(missing, cmd)
217+
t.Errorf("Go CLI missing command %q present in Python CLI", cmd)
218+
}
219+
}
220+
if len(missing) == 0 {
221+
t.Logf("[+] Surface parity: all %d required commands present in Go CLI.", len(required))
222+
}
223+
}
224+
225+
// TestParityCompletionFunctionalContracts verifies key read-only command
226+
// behaviors match between Python and Go (exit codes and basic output). Gate 5.
227+
func TestParityCompletionFunctionalContracts(t *testing.T) {
228+
bin := os.Getenv("APM_PYTHON_BIN")
229+
if bin == "" {
230+
t.Fatal("HARD-GATE FAILED: APM_PYTHON_BIN not set -- functional contracts cannot be verified")
231+
}
232+
233+
type contract struct {
234+
args []string
235+
wantGo int
236+
inRepo bool
237+
ymlType string // "minimal" or "deps"
238+
}
239+
contracts := []contract{
240+
{args: []string{"--help"}, wantGo: 0},
241+
{args: []string{"--version"}, wantGo: 0},
242+
{args: []string{"targets", "--help"}, wantGo: 0},
243+
{args: []string{"deps", "--help"}, wantGo: 0},
244+
{args: []string{"cache", "--help"}, wantGo: 0},
245+
{args: []string{"targets"}, wantGo: 0, inRepo: true, ymlType: "minimal"},
246+
{args: []string{"list"}, wantGo: 0, inRepo: true, ymlType: "deps"},
247+
{args: []string{"deps", "list"}, wantGo: 0, inRepo: true, ymlType: "deps"},
248+
{args: []string{"compile", "--dry-run"}, wantGo: 0, inRepo: true, ymlType: "minimal"},
249+
{args: []string{"audit"}, wantGo: 0, inRepo: true, ymlType: "minimal"},
250+
}
251+
252+
for _, c := range contracts {
253+
c := c
254+
label := "apm " + strings.Join(c.args, " ")
255+
t.Run(label, func(t *testing.T) {
256+
var goOut, pyOut string
257+
var goCode, pyCode int
258+
if c.inRepo {
259+
ymlContent := minimalApmYML
260+
if c.ymlType == "deps" {
261+
ymlContent = apmYMLWithDeps
262+
}
263+
r := runBothInTempRepo(t, ymlContent, c.args...)
264+
goOut, goCode = r.GoStdout+r.GoStderr, r.GoExitCode
265+
pyOut, pyCode = r.PyStdout+r.PyStderr, r.PyExitCode
266+
if r.PyMissing {
267+
pyCode = -1
268+
}
269+
} else {
270+
goOut, _, goCode = runGo(t, c.args...)
271+
pyOut, _, pyCode = runPyBin(t, bin, c.args...)
272+
}
273+
if goCode != c.wantGo {
274+
t.Errorf("Go exit %d, want %d; output: %q", goCode, c.wantGo, goOut)
275+
}
276+
if pyCode >= 0 && pyCode != goCode {
277+
t.Errorf("exit code mismatch: Python=%d Go=%d; pyOut=%q goOut=%q",
278+
pyCode, goCode, pyOut, goOut)
279+
}
280+
})
281+
}
282+
}
283+
284+
// TestParityCompletionStateDiffContracts verifies that mutating commands produce
285+
// equivalent filesystem state between Python and Go. Gate 6.
286+
func TestParityCompletionStateDiffContracts(t *testing.T) {
287+
bin := os.Getenv("APM_PYTHON_BIN")
288+
if bin == "" {
289+
t.Fatal("HARD-GATE FAILED: APM_PYTHON_BIN not set -- state-diff contracts cannot be verified")
290+
}
291+
292+
t.Run("init creates apm.yml", func(t *testing.T) {
293+
goDir, err := os.MkdirTemp("", "apm-state-go-*")
294+
if err != nil {
295+
t.Fatalf("mkdtemp: %v", err)
296+
}
297+
defer os.RemoveAll(goDir)
298+
299+
pyDir, err := os.MkdirTemp("", "apm-state-py-*")
300+
if err != nil {
301+
t.Fatalf("mkdtemp: %v", err)
302+
}
303+
defer os.RemoveAll(pyDir)
304+
305+
_, _, goCode := runGoInDir(t, goDir, "init", "--yes")
306+
if goCode != 0 {
307+
t.Errorf("Go `apm init --yes` exited %d", goCode)
308+
}
309+
goApmYML := filepath.Join(goDir, "apm.yml")
310+
if _, err := os.Stat(goApmYML); err != nil {
311+
t.Errorf("Go `apm init --yes` did not create apm.yml: %v", err)
312+
}
313+
314+
_, _, pyCode := runGoInDirWith(t, pyDir, bin, "init", "--yes")
315+
pyApmYML := filepath.Join(pyDir, "apm.yml")
316+
if _, err := os.Stat(pyApmYML); err != nil {
317+
t.Logf("Python init did not create apm.yml (exit %d): will verify Go only", pyCode)
318+
} else {
319+
// Both created apm.yml: verify they contain the same required keys.
320+
goBytes, _ := os.ReadFile(goApmYML)
321+
pyBytes, _ := os.ReadFile(pyApmYML)
322+
for _, key := range []string{"name:", "version:", "dependencies:"} {
323+
if !strings.Contains(string(goBytes), key) {
324+
t.Errorf("Go apm.yml missing key %q", key)
325+
}
326+
if !strings.Contains(string(pyBytes), key) {
327+
t.Logf("Python apm.yml missing key %q (non-fatal)", key)
328+
}
329+
}
330+
t.Logf("[+] State-diff: Go and Python both created apm.yml with required keys.")
331+
}
332+
})
333+
}
334+
335+
// TestParityCompletionPythonSuite runs the Python reference unit test suite to
336+
// confirm the Python CLI remains green. Gate 7: python_tests_pass.
337+
func TestParityCompletionPythonSuite(t *testing.T) {
338+
if os.Getenv("APM_PYTHON_BIN") == "" {
339+
t.Fatal("HARD-GATE FAILED: APM_PYTHON_BIN not set -- Python suite cannot be verified")
340+
}
341+
342+
root := completionModuleRoot(t)
343+
344+
// Locate uv; required to run the Python test suite.
345+
uvPath, err := exec.LookPath("uv")
346+
if err != nil {
347+
t.Fatalf("HARD-GATE FAILED: uv not found in PATH -- cannot run Python suite: %v", err)
348+
}
349+
350+
// Run the Python unit suite in parallel (-n auto) for speed.
351+
// --ignore integration tests that require external services.
352+
cmd := exec.Command(uvPath, "run", "--extra", "dev",
353+
"pytest", "tests/unit/", "-q", "--tb=short", "--no-header",
354+
"-n", "auto",
355+
"--ignore=tests/unit/integration",
356+
)
357+
cmd.Dir = root
358+
cmd.Env = append(os.Environ(), "NO_COLOR=1", "PYTHONDONTWRITEBYTECODE=1")
359+
var outBuf, errBuf strings.Builder
360+
cmd.Stdout = &outBuf
361+
cmd.Stderr = &errBuf
362+
if runErr := cmd.Run(); runErr != nil {
363+
t.Fatalf("Python suite failed:\n%s\n%s", outBuf.String(), errBuf.String())
364+
}
365+
t.Logf("[+] Python suite passed:\n%s", outBuf.String())
366+
}
367+
368+
// TestParityCompletionBenchmarks runs the migration CLI benchmark and verifies
369+
// the Go CLI stays within the configured performance ratio. Gate 8.
370+
func TestParityCompletionBenchmarks(t *testing.T) {
371+
bin := os.Getenv("APM_PYTHON_BIN")
372+
if bin == "" {
373+
t.Fatal("HARD-GATE FAILED: APM_PYTHON_BIN not set -- benchmarks cannot be verified")
374+
}
375+
if goBinPath == "" {
376+
t.Fatal("HARD-GATE FAILED: Go binary not built -- benchmarks cannot be verified")
377+
}
378+
379+
root := completionModuleRoot(t)
380+
381+
benchScript := filepath.Join(root, "scripts", "ci", "migration_cli_benchmark.py")
382+
if _, err := os.Stat(benchScript); err != nil {
383+
t.Fatalf("benchmark script not found at %s: %v", benchScript, err)
384+
}
385+
386+
// Locate uv to run the benchmark script.
387+
uvPath, err := exec.LookPath("uv")
388+
if err != nil {
389+
t.Fatalf("HARD-GATE FAILED: uv not found in PATH: %v", err)
390+
}
391+
392+
jsonOut := filepath.Join(t.TempDir(), "benchmark.json")
393+
// Use --repeats 2 for a quick CI smoke test (full 5-repeat runs in the
394+
// dedicated benchmarks job).
395+
cmd := exec.Command(uvPath, "run", benchScript,
396+
"--python-bin", bin,
397+
"--go-bin", goBinPath,
398+
"--json-out", jsonOut,
399+
"--max-ratio", "5.0",
400+
"--repeats", "2",
401+
)
402+
cmd.Dir = root
403+
cmd.Env = append(os.Environ(), "NO_COLOR=1")
404+
var outBuf, errBuf strings.Builder
405+
cmd.Stdout = &outBuf
406+
cmd.Stderr = &errBuf
407+
if runErr := cmd.Run(); runErr != nil {
408+
t.Fatalf("Benchmark failed (Go CLI exceeds 5x Python latency or script error):\n%s\n%s",
409+
outBuf.String(), errBuf.String())
410+
}
411+
t.Logf("[+] Benchmarks passed:\n%s", outBuf.String())
412+
}
413+
174414
// runPyBin runs the Python apm binary with the given args.
175415
func runPyBin(t *testing.T, bin string, args ...string) (stdout, stderr string, exitCode int) {
176416
t.Helper()

0 commit comments

Comments
 (0)