feat(orch): make resume-build -gdb work on real nodes + add copy-build -gdb

packages/orchestrator/cmd/copy-build/main.go

@@ -28,20 +28,24 @@ import (
 type Destination struct {
 	Path    string
 	CRC     uint32
+	exists  bool
 	isLocal bool
 }
 
 func NewDestinationFromObject(ctx context.Context, o *googleStorage.ObjectHandle) (*Destination, error) {
 	var crc uint32
+	exists := false
 	if attrs, err := o.Attrs(ctx); err == nil {
 		crc = attrs.CRC32C
+		exists = true
 	} else if !errors.Is(err, googleStorage.ErrObjectNotExist) {
 		return nil, fmt.Errorf("failed to get object attributes: %w", err)
 	}
 
 	return &Destination{
 		Path:    fmt.Sprintf("gs://%s/%s", o.BucketName(), o.ObjectName()),
 		CRC:     crc,
+		exists:  exists,
 		isLocal: false,
 	}, nil
 }
@@ -67,6 +71,7 @@ func NewDestinationFromPath(prefix, file string) (*Destination, error) {
 		return &Destination{
 			Path:    p,
 			CRC:     crc,
+			exists:  true,
 			isLocal: true,
 		}, nil
 	}
@@ -194,6 +199,116 @@ func gcloudCopy(ctx context.Context, from, to *Destination) error {
 	return nil
 }
 
+// readTemplateVersions reads the build's metadata.json (from the destination, where it
+// has just been copied) and returns its kernel and firecracker version strings.
+func readTemplateVersions(ctx context.Context, client *googleStorage.Client, to, metadataPath string) (kernelVer, fcVer string, err error) {
+	var r io.ReadCloser
+	if strings.HasPrefix(to, "gs://") {
+		bucketName, _ := strings.CutPrefix(to, "gs://")
+		r, err = client.Bucket(bucketName).Object(metadataPath).NewReader(ctx)
+	} else {
+		r, err = os.Open(path.Join(to, "templates", metadataPath))
+	}
+	if err != nil {
+		return "", "", err
+	}
+	defer r.Close()
+
+	var meta struct {
+		Template struct {
+			KernelVersion      string `json:"kernel_version"`
+			FirecrackerVersion string `json:"firecracker_version"`
+		} `json:"template"`
+	}
+	if err := json.NewDecoder(r).Decode(&meta); err != nil {
+		return "", "", err
+	}
+
+	return meta.Template.KernelVersion, meta.Template.FirecrackerVersion, nil
+}
+
+// deriveArtifactBuckets maps a template bucket (gs://<prefix>-fc-templates) to the
+// sibling firecracker-versions and kernels buckets in the same project/env. The build
+// records no environment, so the env is taken from the template bucket location.
+func deriveArtifactBuckets(templateLoc string) (fcBucket, kernelBucket string, err error) {
+	if !strings.HasPrefix(templateLoc, "gs://") {
+		return "", "", fmt.Errorf("-gdb requires a gs:// location, got %q", templateLoc)
+	}
+	bucket := strings.TrimSuffix(strings.TrimPrefix(templateLoc, "gs://"), "/")
+	prefix, ok := strings.CutSuffix(bucket, "-fc-templates")
+	if !ok {
+		return "", "", fmt.Errorf("cannot derive versions/kernels buckets from %q (expected a *-fc-templates bucket)", templateLoc)
+	}
+
+	return "gs://" + prefix + "-fc-versions", "gs://" + prefix + "-fc-kernels", nil
+}
+
+// copyGdbArtifacts ensures the build's FC + kernel runtime and debug artifacts exist at
+// the destination, copying each from the source env's versions/kernels buckets only if
+// it is not already present at the destination (so a large vmlinux.debug is not recopied
+// for a version already there). Required artifacts must exist at the source; the optional
+// firecracker-debug.debug (FC's own symbols, not needed for guest-kernel gdb) is skipped
+// if absent.
+func copyGdbArtifacts(ctx context.Context, client *googleStorage.Client, from, to, arch, fcVer, kernelVer string) error {
+	fcFrom, kFrom, err := deriveArtifactBuckets(from)
+	if err != nil {
+		return err
+	}
+	fcTo, kTo, err := deriveArtifactBuckets(to)
+	if err != nil {
+		return err
+	}
+
+	artifacts := []struct {
+		name                      string
+		srcBucket, dstBucket, obj string
+		required                  bool
+	}{
+		{"firecracker", fcFrom, fcTo, path.Join(fcVer, arch, "firecracker"), true},
+		{"firecracker-debug", fcFrom, fcTo, path.Join(fcVer, arch, "firecracker-debug"), true},
+		{"firecracker-debug.debug", fcFrom, fcTo, path.Join(fcVer, arch, "firecracker-debug.debug"), false},
+		{"vmlinux.bin", kFrom, kTo, path.Join(kernelVer, arch, "vmlinux.bin"), true},
+		{"vmlinux.debug", kFrom, kTo, path.Join(kernelVer, arch, "vmlinux.debug"), true},
+	}
+
+	for _, a := range artifacts {
+		srcBucket := strings.TrimPrefix(a.srcBucket, "gs://")
+		dstBucket := strings.TrimPrefix(a.dstBucket, "gs://")
+		src, err := NewDestinationFromObject(ctx, client.Bucket(srcBucket).Object(a.obj))
+		if err != nil {
+			return fmt.Errorf("stat source %s: %w", a.name, err)
+		}
+		dst, err := NewDestinationFromObject(ctx, client.Bucket(dstBucket).Object(a.obj))
+		if err != nil {
+			return fmt.Errorf("stat destination %s: %w", a.name, err)
+		}
+
+		if !src.exists { // not present at source
+			if a.required {
+				return fmt.Errorf("required gdb artifact %s not found at %s (is this version published with gdb support?)", a.name, src.Path)
+			}
+			fmt.Fprintf(os.Stderr, "-> optional gdb artifact %s not at source, skipping\n", a.name)
+
+			continue
+		}
+		// Skip only when the destination already holds identical content (CRC32C match);
+		// otherwise copy, replacing a divergent/stale or absent artifact rather than
+		// trusting it. The dst.exists guard avoids a false 0 == 0 match when the
+		// destination is missing and the source's CRC32C is genuinely zero.
+		if dst.exists && src.CRC == dst.CRC {
+			fmt.Fprintf(os.Stderr, "-> gdb artifact %s already current at destination, skipping\n", a.name)
+
+			continue
+		}
+		fmt.Fprintf(os.Stderr, "+ copying gdb artifact '%s' to '%s'\n", src.Path, dst.Path)
+		if err := gcloudCopy(ctx, src, dst); err != nil {
+			return fmt.Errorf("copy %s: %w", a.name, err)
+		}
+	}
+
+	return nil
+}
+
 func main() {
 	buildId := flag.String("build", "", "build id")
 	from := flag.String("from", "", "from destination")
@@ -204,12 +319,19 @@ func main() {
 	memory := flag.Int("memory", 1024, "memory MB")
 	disk := flag.Int("disk", 1024, "disk MB")
 	tag := flag.String("tag", "default", "build assignment tag")
+	gdb := flag.Bool("gdb", false, "also copy the build's FC + kernel runtime and debug artifacts (firecracker, firecracker-debug, vmlinux.bin, vmlinux.debug) into the matching versions/kernels buckets so the snapshot is gdb-ready at the destination; requires gs:// -from/-to")
+	arch := flag.String("arch", "amd64", "artifact arch for -gdb (amd64 or arm64)")
 
 	flag.Parse()
 
 	if *teamID != "" && *envdVersion == "" {
 		log.Fatal("-envd-version is required when -team is set")
 	}
+	// Validate -gdb's gs:// precondition up front (deriveArtifactBuckets needs it), so a
+	// wrong invocation fails instantly rather than after the multi-GB snapshot copy.
+	if *gdb && (!strings.HasPrefix(*from, "gs://") || !strings.HasPrefix(*to, "gs://")) {
+		log.Fatal("-gdb requires gs:// -from and -to (it stages debug artifacts between bucket environments)")
+	}
 
 	fmt.Fprintf(os.Stderr, "Copying build '%s' from '%s' to '%s'\n", *buildId, *from, *to)
 
@@ -378,48 +500,34 @@ func main() {
 
 	fmt.Fprintf(os.Stderr, "Build '%s' copied to '%s'\n", *buildId, *to)
 
-	if *teamID != "" {
-		// Read metadata.json from destination to get kernel and firecracker versions.
-		var metadataReader io.ReadCloser
-		if strings.HasPrefix(*to, "gs://") {
-			bucketName, _ := strings.CutPrefix(*to, "gs://")
-			obj := googleStorageClient.Bucket(bucketName).Object(metadataPath)
-			r, err := obj.NewReader(ctx)
-			if err != nil {
-				log.Fatalf("failed to read metadata from GCS: %s", err)
-			}
-			metadataReader = r
-		} else {
-			f, err := os.Open(path.Join(*to, "templates", metadataPath))
-			if err != nil {
-				log.Fatalf("failed to read metadata from local path: %s", err)
-			}
-			metadataReader = f
+	if *teamID != "" || *gdb {
+		// metadata.json (just copied to the destination) carries the kernel + FC
+		// versions; both the -gdb artifact copy and the -team SQL seed need them.
+		kernelVer, fcVer, err := readTemplateVersions(ctx, googleStorageClient, *to, metadataPath)
+		if err != nil {
+			log.Fatalf("failed to read template versions from metadata: %s", err)
 		}
 
-		var meta struct {
-			Template struct {
-				KernelVersion      string `json:"kernel_version"`
-				FirecrackerVersion string `json:"firecracker_version"`
-			} `json:"template"`
+		if *gdb {
+			if err := copyGdbArtifacts(ctx, googleStorageClient, *from, *to, *arch, fcVer, kernelVer); err != nil {
+				log.Fatalf("failed to copy gdb artifacts: %s", err)
+			}
+			fmt.Fprintf(os.Stderr, "gdb artifacts ensured at destination (arch %s)\n", *arch)
 		}
-		if err := json.NewDecoder(metadataReader).Decode(&meta); err != nil {
-			metadataReader.Close()
-			log.Fatalf("failed to decode metadata.json: %s", err)
+
+		if *teamID != "" {
+			envID := id.Generate()
+			fmt.Fprintf(os.Stderr, "\n\nGenerated env ID: %s\n\n", envID)
+
+			fmt.Printf("BEGIN;\n")
+			fmt.Printf("INSERT INTO public.envs (id, team_id, updated_at, public, source)\n")
+			fmt.Printf("VALUES ('%s', '%s', NOW(), FALSE, 'template');\n\n", envID, *teamID)
+			fmt.Printf("INSERT INTO public.env_builds (id, env_id, updated_at, finished_at, status, ram_mb, vcpu, kernel_version, firecracker_version, envd_version, free_disk_size_mb, total_disk_size_mb)\n")
+			fmt.Printf("VALUES ('%s', '%s', NOW(), NOW(), 'uploaded', %d, %d, '%s', '%s', '%s', %d, %d);\n\n",
+				*buildId, envID, *memory, *vcpu, kernelVer, fcVer, *envdVersion, *disk, *disk)
+			fmt.Printf("INSERT INTO public.env_build_assignments (env_id, build_id, tag)\n")
+			fmt.Printf("VALUES ('%s', '%s', '%s');\n", envID, *buildId, *tag)
+			fmt.Printf("COMMIT;\n")
 		}
-		metadataReader.Close()
-
-		envID := id.Generate()
-		fmt.Fprintf(os.Stderr, "\n\nGenerated env ID: %s\n\n", envID)
-
-		fmt.Printf("BEGIN;\n")
-		fmt.Printf("INSERT INTO public.envs (id, team_id, updated_at, public, source)\n")
-		fmt.Printf("VALUES ('%s', '%s', NOW(), FALSE, 'template');\n\n", envID, *teamID)
-		fmt.Printf("INSERT INTO public.env_builds (id, env_id, updated_at, finished_at, status, ram_mb, vcpu, kernel_version, firecracker_version, envd_version, free_disk_size_mb, total_disk_size_mb)\n")
-		fmt.Printf("VALUES ('%s', '%s', NOW(), NOW(), 'uploaded', %d, %d, '%s', '%s', '%s', %d, %d);\n\n",
-			*buildId, envID, *memory, *vcpu, meta.Template.KernelVersion, meta.Template.FirecrackerVersion, *envdVersion, *disk, *disk)
-		fmt.Printf("INSERT INTO public.env_build_assignments (env_id, build_id, tag)\n")
-		fmt.Printf("VALUES ('%s', '%s', '%s');\n", envID, *buildId, *tag)
-		fmt.Printf("COMMIT;\n")
 	}
 }

packages/orchestrator/cmd/resume-build/fc-debug.gdb

@@ -23,7 +23,7 @@ define fc-faults
   else
     set $_fc_n = $arg0
   end
-  break *handle_mm_fault
+  hbreak *handle_mm_fault
   set $_fc_i = 0
   while $_fc_i < $_fc_n
     continue

packages/orchestrator/cmd/resume-build/gdb-debugging.md

@@ -12,20 +12,33 @@ kernel state) beyond what host/UFFD telemetry exposes.
 
 `gdb` on PATH. The two debug artifacts — `firecracker-debug` (Firecracker built
 `--features gdb`) and `vmlinux.debug` (the guest kernel's split DWARF symbols) — are
-**fetched automatically by version**, matched to the snapshot's `FirecrackerVersion`
-/ `KernelVersion` (which `resume-build` prints when it loads the build), from
-`https://storage.googleapis.com/e2b-prod-public-builds`. In the common case you pass
-nothing.
-
-Supplying them yourself is only needed when the fetch can't find them — before the
-fc-versions / fc-kernels pipelines publish them, or for a locally-built kernel/FC.
-Then point `E2B_GDB_ARTIFACTS_URL` at a base that serves them, or pass explicit paths
+resolved **locally**, by version (matched to the snapshot's `FirecrackerVersion` /
+`KernelVersion`, which `resume-build` prints when it loads the build): `firecracker-debug`
+next to the snapshot's `firecracker`, and `vmlinux.debug` next to its `vmlinux.bin`. The
+fc-versions / fc-kernels releases publish them into those version dirs (and `copy-build
+-gdb` stages them when bridging a snapshot — see step 1), so in the common case they are
+already present and you pass nothing.
+
+They are **not** fetched over the network. If they aren't present, pass explicit paths
 with `-gdb-fc` / `-gdb-symbols` (see *Preparing the artifacts*).
 
 ## Steps
 
 1. **Copy the build chain** to the dev node's local storage (`copy-build`), as for
-   `resume-prod-snapshot`.
+   `resume-prod-snapshot` — or resume straight from a bucket with `-storage gs://…`.
+
+   To make the debug artifacts resolvable when the snapshot comes from another
+   environment, stage them with a **separate `gs://`→`gs://` bridge** (not part of the
+   local copy above):
+
+   ```bash
+   copy-build -gdb -from gs://<src>-fc-templates -to gs://<dst>-fc-templates
+   ```
+
+   `-gdb` copies `firecracker-debug` / `vmlinux.debug` (plus the runtime FC/kernel) into
+   the destination's `-fc-versions` / `-fc-kernels` buckets, so they sit next to the
+   FC/kernel on the node's mounts and resolve locally. It requires `gs://` for both
+   `-from` and `-to`; it does not write to local paths.
 
 2. **Resume under gdb** (interactive) — the common case needs no extra flags:
 
@@ -51,10 +64,10 @@ connected.)
 
 ## Preparing the artifacts
 
-Normally you don't — `resume-build` fetches `firecracker-debug` and `vmlinux.debug`
-automatically (see *Prerequisites*). Build them by hand only when the release
-pipelines haven't published them for your version, or to debug a locally-built
-kernel/FC:
+Normally you don't — the fc-versions / fc-kernels releases publish `firecracker-debug`
+and `vmlinux.debug` into the FC-version / kernel-version dirs, and `copy-build -gdb`
+stages them next to a bridged snapshot (see *Prerequisites*). Build them by hand only to
+debug a locally-built kernel/FC, or a version the releases predate:
 
 - **`firecracker-debug`** — Firecracker built `--features gdb` (release profile):
   `cargo build --release --features gdb -p firecracker`. Pass with `-gdb-fc`.
@@ -63,10 +76,9 @@ kernel/FC:
   toolchain as the deployed kernel (gcc 13.x / Ubuntu 24.04) so the symbol addresses
   match the snapshot. Pass with `-gdb-symbols`.
 
-Then either pass `-gdb-fc` / `-gdb-symbols` explicitly, stage them at the conventional
-local paths (`firecracker-debug` in the FC-version dir, `vmlinux.debug` in the
-kernel-version dir), or set `E2B_GDB_ARTIFACTS_URL` to a base that serves them at
-`firecrackers/<fcver>/<arch>/firecracker-debug` and `kernels/<kver>/<arch>/vmlinux.debug`.
+Then pass `-gdb-fc` / `-gdb-symbols` explicitly, or stage them at the conventional local
+paths (`firecracker-debug` in the FC-version dir, `vmlinux.debug` in the kernel-version
+dir).
 
 ## Macros (`fc-debug.gdb`)