ci(snapshot): restore per-source depends_on instead of global wait

Previously every restore step waited for the entire snapshot-create
group to finish via a pipeline-wide wait step. Each restore only needs
its own source snapshot, so key each create step by instance/kv and
have each restore depends_on the specific source it consumes. Restores
now start as soon as their source snapshot is ready.

Signed-off-by: Jack Thomson <jackabt@amazon.com>

Author: JackThomson2

.buildkite/pipeline_cross.py

@@ -38,7 +38,18 @@
         "mkdir -pv snapshots",
         "tar cSvf snapshots/{instance}_{kv}.tar snapshot_artifacts",
     ]
-    pipeline.build_group(
+
+    # Buildkite step keys may only contain [A-Za-z0-9_\-:], so dots in
+    # instance names (m5n.metal) and kernel versions (linux_5.10) must be
+    # sanitized. Tarball paths stay unchanged.
+    def create_step_key(instance, kv):
+        return f"snap-create-{instance}-{kv}".replace(".", "_")
+
+    # Key each snapshot-create step so restore steps can depend on the
+    # specific source snapshot they need, rather than waiting for every
+    # snapshot-create step to finish. `build_group` doesn't sanitize
+    # substituted key values, so we set the final key after it fans out.
+    x86_create = pipeline.build_group(
         "snapshot-create",
         commands,
         timeout=30,
@@ -49,15 +60,17 @@
 
     # https://github.com/firecracker-microvm/firecracker/blob/main/docs/kernel-policy.md#experimental-snapshot-compatibility-across-kernel-versions
     aarch64_platforms = [("al2023", "linux_6.1")]
-    pipeline.build_group(
+    aarch64_create = pipeline.build_group(
         "snapshot-create-aarch64",
         commands,
         timeout=30,
         artifact_paths="snapshots/**/*",
         instances=instances_aarch64,
         platforms=aarch64_platforms,
     )
-    pipeline.add_step("wait")
+    for grp in (x86_create, aarch64_create):
+        for s in grp["steps"]:
+            s["key"] = create_step_key(s["agents"]["instance"], s["agents"]["kv"])
 
     # allow-list of what instances can be restored on what other instances (in
     # addition to itself). aarch64 is restricted to same-instance restores.
@@ -111,6 +124,7 @@
             "label": f"snapshot-restore-src-{src_instance}-{src_kv}-dst-{dst_instance}-{dst_kv}",
             "timeout": 30,
             "agents": {"instance": dst_instance, "kv": dst_kv, "os": dst_os},
+            "depends_on": [create_step_key(src_instance, src_kv)],
             **per_instance,
         }
         steps.append(step)