chore(dev): tag locally-built dev images :dev-<sha> instead of :latest

Under OBOL_DEVELOPMENT, both the manifest rewrite (internal/defaults) and the
image build (internal/stack) used a shared, mutable `<image>:latest` tag. On a
host running more than one obol-stack worktree against the same Docker daemon,
one worktree's build silently overwrites :latest and a different worktree's
`obol stack up` then deploys the wrong binary — observed during rc9 QA, where a
sibling branch's serviceoffer-controller (reading a per-purchase Secret this
branch's RBAC doesn't grant) poisoned an unrelated stack and presented as a
hang, not an obvious image mismatch.

Tag dev images with `dev-<short-git-sha>` instead:
- defaults.DevImageTag() = dev-<sha> of the working tree (`latest` fallback when
  not a git checkout, preserving prior behaviour for tarball builds).
- CopyInfrastructure stamps the rendered manifests with the dev tag AND persists
  it to $CONFIG_DIR/.dev-image-tag, so internal/stack builds/imports the exact
  tag the cluster pins — even if HEAD moves between `stack init` and `stack up`.
- Different branches/worktrees get distinct tags (no cross-worktree poisoning);
  committing changes the SHA and triggers a fresh build; uncommitted changes
  reuse the committed tag unless OBOL_FORCE_REBUILD_LOCAL_DEV_IMAGES is set.

Validated: go test ./... (33 pkgs); `obol stack init` (dev) rewrites both the
controller and buyer pins to :dev-<sha> with no leftover @sha256 and persists the
tag; `docker build -t …:dev-<sha>` + `k3d image import` succeed (the exact
build/import path buildAndImportLocalImages runs).

Author: bussyjd

internal/defaults/defaults.go

@@ -25,8 +25,50 @@ const (
 	stackIDFile      = ".stack-id"
 	stackBackendFile = ".stack-backend"
 	stampFile        = ".obol-defaults-stamp"
+	// devImageTagFile records the tag the dev-mode manifest rewrite stamped
+	// the locally-built images with. internal/stack reads it at build time so
+	// the image it builds/imports matches what the rendered manifests pin —
+	// even if HEAD moved between `stack init` and `stack up`.
+	devImageTagFile = ".dev-image-tag"
 )
 
+// DevImageTag returns the tag used for locally-built dev images under
+// OBOL_DEVELOPMENT. It is `dev-<short-git-sha>` of the working tree, so each
+// branch/worktree builds a distinct tag and parallel dev stacks sharing one
+// Docker daemon never clobber each other's images (the `:latest` collision that
+// let a sibling worktree's build poison an unrelated stack). Committing changes
+// the SHA and triggers a fresh build; uncommitted changes reuse the committed
+// tag unless OBOL_FORCE_REBUILD_LOCAL_DEV_IMAGES is set. Falls back to `latest`
+// when the source is not a git checkout (e.g. a tarball build), preserving the
+// previous behaviour there.
+func DevImageTag() string {
+	cmd := exec.Command("git", "rev-parse", "--short=12", "HEAD")
+	out, err := cmd.Output()
+	if err != nil {
+		return "latest"
+	}
+	sha := strings.TrimSpace(string(out))
+	if !regexp.MustCompile(`^[0-9a-f]{7,40}$`).MatchString(sha) {
+		return "latest"
+	}
+	return "dev-" + sha
+}
+
+// ReadDevImageTag returns the dev image tag persisted at CopyInfrastructure
+// time, or "latest" if none was recorded (non-dev install, or pre-dates this
+// mechanism). internal/stack uses it to tag the images it builds.
+func ReadDevImageTag(cfg *config.Config) string {
+	data, err := os.ReadFile(filepath.Join(cfg.ConfigDir, devImageTagFile))
+	if err != nil {
+		return "latest"
+	}
+	tag := strings.TrimSpace(string(data))
+	if tag == "" {
+		return "latest"
+	}
+	return tag
+}
+
 // RefreshInfrastructureIfChanged refreshes the generated defaults tree when
 // the embedded infrastructure assets, backend, or stack ID changed.
 func RefreshInfrastructureIfChanged(cfg *config.Config, backendName, stackID string) (bool, error) {
@@ -63,16 +105,21 @@ func CopyInfrastructure(cfg *config.Config, backendName, stackID string) error {
 	}
 
 	// Under OBOL_DEVELOPMENT we build images from the working tree and
-	// import them into k3d as `<image>:latest`. The embedded templates
-	// pin published digests for production safety, which means the
-	// cluster ignores our locally-built images and silently uses stale
-	// ghcr.io binaries. Rewrite digest pins to :latest after copy so the
-	// dev cycle Just Works without operators having to kubectl-set-image
-	// every loop.
+	// import them into k3d. The embedded templates pin published digests for
+	// production safety, which means the cluster ignores our locally-built
+	// images and silently uses stale ghcr.io binaries. Rewrite the digest
+	// pins to a per-commit `dev-<sha>` tag after copy so the dev cycle Just
+	// Works without operators having to kubectl-set-image every loop, and so
+	// parallel worktree stacks don't collide on a shared `:latest`. Persist
+	// the tag so internal/stack builds/imports the exact tag we pinned here.
 	if os.Getenv("OBOL_DEVELOPMENT") == "true" {
-		if err := rewriteDevDigestPins(defaultsDir); err != nil {
+		devTag := DevImageTag()
+		if err := rewriteDevDigestPins(defaultsDir, devTag); err != nil {
 			return fmt.Errorf("rewrite dev digest pins: %w", err)
 		}
+		if err := os.WriteFile(filepath.Join(cfg.ConfigDir, devImageTagFile), []byte(devTag), 0o600); err != nil {
+			return fmt.Errorf("persist dev image tag: %w", err)
+		}
 	}
 
 	stamp, err := infrastructureStamp(backendName, stackID)
@@ -106,7 +153,7 @@ var devLocallyBuiltImageBases = []string{
 // — in either case the local dev build is tagged `:latest`, so the
 // rewrite needs to catch both forms or `obol stack up` would pull from
 // the registry instead of using the freshly-built local image.
-func rewriteDevDigestPins(defaultsDir string) error {
+func rewriteDevDigestPins(defaultsDir, devTag string) error {
 	patterns := make([]*regexp.Regexp, 0, len(devLocallyBuiltImageBases))
 	replaceWith := make([]string, 0, len(devLocallyBuiltImageBases))
 	for _, base := range devLocallyBuiltImageBases {
@@ -122,7 +169,7 @@ func rewriteDevDigestPins(defaultsDir string) error {
 		// silently bypasses the local build (root cause of the no-debug-logs
 		// regression in flow-11 step 43 chase, May 2026).
 		patterns = append(patterns, regexp.MustCompile(regexp.QuoteMeta(base)+"(:[a-f0-9]{7,40}@sha256:[a-f0-9]{64}|@sha256:[a-f0-9]{64}|:[a-f0-9]{7,40})"))
-		replaceWith = append(replaceWith, base+":latest")
+		replaceWith = append(replaceWith, base+":"+devTag)
 	}
 
 	return filepath.WalkDir(defaultsDir, func(path string, d fs.DirEntry, walkErr error) error {

internal/defaults/defaults_test.go

@@ -25,25 +25,37 @@ func TestCopyInfrastructure_DevModeRewritesDigestPins(t *testing.T) {
 	}
 	out := string(data)
 
-	// Every locally-built image must have lost its @sha256: pin and gained
-	// :latest, otherwise the cluster pulls a stale ghcr.io binary even
-	// when OBOL_DEVELOPMENT=true rebuilt the image locally.
+	// The dev rewrite swaps the published digest/SHA pins for the per-commit
+	// dev tag (dev-<sha>, or :latest when not a git checkout). In CI this is a
+	// git checkout, so expect dev-<sha>.
+	devTag := DevImageTag()
+
+	// Every locally-built image must have lost its @sha256: pin and gained the
+	// dev tag, otherwise the cluster pulls a stale ghcr.io binary even when
+	// OBOL_DEVELOPMENT=true rebuilt the image locally.
 	for _, base := range devLocallyBuiltImageBases {
 		if strings.Contains(out, base+"@sha256:") {
 			t.Errorf("dev mode left digest pin on %s in %s", base, x402Path)
 		}
 	}
-	for _, want := range []string{
-		"ghcr.io/obolnetwork/x402-verifier:latest",
-		"ghcr.io/obolnetwork/serviceoffer-controller:latest",
+	for _, base := range []string{
+		"ghcr.io/obolnetwork/x402-verifier",
+		"ghcr.io/obolnetwork/serviceoffer-controller",
 	} {
+		want := base + ":" + devTag
 		if !strings.Contains(out, want) {
 			t.Errorf("dev mode did not rewrite to %q in %s", want, x402Path)
 		}
 	}
 
+	// The persisted dev tag MUST equal what was stamped into the manifests, or
+	// internal/stack would build/import a tag the cluster doesn't pin.
+	if got := ReadDevImageTag(cfg); got != devTag {
+		t.Errorf("persisted dev image tag = %q, want %q", got, devTag)
+	}
+
 	// Combo tag+digest form (used by x402-buyer in llm.yaml) must be
-	// rewritten to a clean `:latest` with no stale `@sha256:` suffix.
+	// rewritten to a clean `:<devTag>` with no stale `@sha256:` suffix.
 	// Regression guard for the bug where the old regex matched only
 	// the `:b13254e` part and left `@sha256:...` behind, causing Docker
 	// to silently pull the registry-pinned image instead of the local
@@ -55,17 +67,37 @@ func TestCopyInfrastructure_DevModeRewritesDigestPins(t *testing.T) {
 		t.Fatalf("read llm.yaml: %v", err)
 	}
 	llmOut := string(llmData)
-	if !strings.Contains(llmOut, "ghcr.io/obolnetwork/x402-buyer:latest") {
-		t.Errorf("dev mode did not rewrite x402-buyer to :latest in %s", llmPath)
+	buyer := "ghcr.io/obolnetwork/x402-buyer"
+	if !strings.Contains(llmOut, buyer+":"+devTag) {
+		t.Errorf("dev mode did not rewrite x402-buyer to :%s in %s", devTag, llmPath)
 	}
-	if strings.Contains(llmOut, "ghcr.io/obolnetwork/x402-buyer:latest@sha256:") {
-		t.Errorf("dev mode left orphan @sha256: suffix on x402-buyer:latest in %s — regex missed the combo form", llmPath)
+	if strings.Contains(llmOut, buyer+":"+devTag+"@sha256:") {
+		t.Errorf("dev mode left orphan @sha256: suffix on x402-buyer:%s in %s — regex missed the combo form", devTag, llmPath)
 	}
-	if strings.Contains(llmOut, "ghcr.io/obolnetwork/x402-buyer@sha256:") {
+	if strings.Contains(llmOut, buyer+"@sha256:") {
 		t.Errorf("dev mode left @sha256: digest pin on x402-buyer in %s — regex missed it", llmPath)
 	}
 }
 
+func TestDevImageTag_Format(t *testing.T) {
+	// Tests run inside the git checkout, so expect dev-<sha>; tolerate the
+	// :latest fallback for non-git build environments.
+	tag := DevImageTag()
+	if tag == "latest" {
+		t.Skip("not a git checkout (DevImageTag fell back to latest) — nothing to assert")
+	}
+	if !regexp.MustCompile(`^dev-[0-9a-f]{7,40}$`).MatchString(tag) {
+		t.Errorf("DevImageTag() = %q, want dev-<short-sha> or latest", tag)
+	}
+}
+
+func TestReadDevImageTag_FallbackWhenAbsent(t *testing.T) {
+	cfg := &config.Config{ConfigDir: t.TempDir()}
+	if got := ReadDevImageTag(cfg); got != "latest" {
+		t.Errorf("ReadDevImageTag with no file = %q, want latest", got)
+	}
+}
+
 func TestCopyInfrastructure_ProductionPreservesImagePins(t *testing.T) {
 	// Without OBOL_DEVELOPMENT=true, the immutable image pins must
 	// survive untouched. A regression here would silently downgrade prod

internal/stack/stack.go

@@ -1141,9 +1141,20 @@ func buildAndImportLocalImages(cfg *config.Config, u *ui.UI) {
 	serverCID := k3dServerContainerID(clusterName)
 	cache := loadImportedImageCache(cfg)
 
+	// Tag locally-built images with the per-commit dev tag the manifest
+	// rewrite stamped (defaults.CopyInfrastructure persisted it). This keeps a
+	// branch/worktree's images on their own tag instead of a shared :latest,
+	// so parallel dev stacks on one Docker daemon can't clobber each other's
+	// images (the cross-worktree poisoning that masquerades as a stale build).
+	devTag := stackdefaults.ReadDevImageTag(cfg)
+
 	var built, pulled, imported, total int
 
 	for _, img := range baseLocalImages {
+		// img.tag carries the published `:latest` placeholder; swap it for the
+		// dev tag the rendered manifests actually pin.
+		imgTag := strings.TrimSuffix(img.tag, ":latest") + ":" + devTag
+
 		contextDir := projectRoot
 		if img.contextDir != "" {
 			if filepath.IsAbs(img.contextDir) {
@@ -1163,27 +1174,27 @@ func buildAndImportLocalImages(cfg *config.Config, u *ui.UI) {
 
 		total++
 
-		if shouldForceRebuild(img.tag) || !dockerImageAvailableLocally(img.tag) {
+		if shouldForceRebuild(imgTag) || !dockerImageAvailableLocally(imgTag) {
 			if u != nil {
-				u.Infof("Building %s from %s", img.tag, img.dockerfile)
+				u.Infof("Building %s from %s", imgTag, img.dockerfile)
 			}
 			buildCmd := exec.Command("docker", "build",
 				"-f", dockerfilePath,
-				"-t", img.tag,
+				"-t", imgTag,
 				contextDir,
 			)
 			buildCmd.Stdout = os.Stdout
 			buildCmd.Stderr = os.Stderr
 			if err := buildCmd.Run(); err != nil {
 				if u != nil {
-					u.Warnf("Failed to build %s: %v", img.tag, err)
+					u.Warnf("Failed to build %s: %v", imgTag, err)
 				}
 				continue
 			}
 			built++
 		}
 
-		if importImageWithCache(k3dBinary, clusterName, img.tag, serverCID, &cache, u) {
+		if importImageWithCache(k3dBinary, clusterName, imgTag, serverCID, &cache, u) {
 			imported++
 		}
 	}