Merge pull request opensandbox-group#837 from fengcone/feature/public-k8s-snapshot

feat(snapshot): support public snapshots api on Kubernetes runtime

Author: ninan-nn

docs/pause-resume.md

@@ -276,7 +276,7 @@ The OpenSandbox controller requires the following RBAC permissions for pause/res
 
 ### Snapshot image naming
 
-Snapshot images are named:
+Internal pause/resume snapshot images are named:
 ```
 <snapshot-registry>/<sandboxName>-<containerName>:snap-gen<N>
 ```
@@ -286,6 +286,17 @@ For example, with `--snapshot-registry=registry.example.com/sandboxes`, sandbox
 registry.example.com/sandboxes/my-sandbox-sandbox:snap-gen1
 ```
 
+Server-managed public snapshots use the same repository layout but a stable
+snapshot-id-derived tag:
+```
+<snapshot-registry>/<sandboxName>-<containerName>:snap-<snapshotIdHex>
+```
+
+The controller distinguishes the two modes by owner reference. Pause/resume
+snapshots are created by the `BatchSandbox` controller and have a controller
+ownerReference to the owning `BatchSandbox`; public snapshots are created by the
+Lifecycle server and do not use that ownerReference.
+
 ### Commit Job
 
 The controller creates a short-lived Kubernetes `Job` for each pause:

docs/public/api/spec-inline.js

@@ -91,10 +91,10 @@ configToml: |
   informer_enabled = true
   informer_resync_seconds = 300
   informer_watch_timeout_seconds = 60
+  snapshot_create_timeout_seconds = 900
   workload_provider = "batchsandbox"
   batchsandbox_template_file = "/etc/opensandbox/example.batchsandbox-template.yaml"
   
   [egress]
   image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.0.10"
   mode = "dns+nft"
-

kubernetes/charts/opensandbox-server/values.yaml

@@ -317,6 +317,7 @@ func TestSandboxSnapshotHandlePending_UsesSourcePodContainersWhenTemplateMissing
 			Name:       "test-bs",
 			Namespace:  "default",
 			Generation: 2,
+			UID:        types.UID("test-bs-uid"),
 		},
 		Spec: sandboxv1alpha1.BatchSandboxSpec{
 			PoolRef: "test-pool",
@@ -342,6 +343,16 @@ func TestSandboxSnapshotHandlePending_UsesSourcePodContainersWhenTemplateMissing
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-snapshot",
 			Namespace: "default",
+			OwnerReferences: []metav1.OwnerReference{
+				{
+					APIVersion:         "sandbox.opensandbox.io/v1alpha1",
+					Kind:               "BatchSandbox",
+					Name:               "test-bs",
+					UID:                types.UID("test-bs-uid"),
+					Controller:         ptrToBool(true),
+					BlockOwnerDeletion: ptrToBool(true),
+				},
+			},
 		},
 		Spec: sandboxv1alpha1.SandboxSnapshotSpec{
 			SandboxName: "test-bs",
@@ -371,6 +382,71 @@ func TestSandboxSnapshotHandlePending_UsesSourcePodContainersWhenTemplateMissing
 	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Name: "test-snapshot-commit", Namespace: "default"}, job))
 }
 
+func TestSandboxSnapshotHandlePending_PublicSnapshotUsesSnapshotIDTag(t *testing.T) {
+	bs := &sandboxv1alpha1.BatchSandbox{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:       "test-bs",
+			Namespace:  "default",
+			Generation: 7,
+		},
+		Spec: sandboxv1alpha1.BatchSandboxSpec{
+			Template: &corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{Name: "sandbox", Image: "python:3.11"},
+					},
+				},
+			},
+		},
+	}
+	pod := &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-bs-0",
+			Namespace: "default",
+			Labels: map[string]string{
+				LabelBatchSandboxNameKey: "test-bs",
+			},
+		},
+		Spec: corev1.PodSpec{
+			NodeName: "node-a",
+			Containers: []corev1.Container{
+				{Name: "sandbox", Image: "python:3.11"},
+			},
+		},
+		Status: corev1.PodStatus{
+			Phase: corev1.PodRunning,
+		},
+	}
+	snapshot := &sandboxv1alpha1.SandboxSnapshot{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "osb-snap-11111111222243338444555555555555",
+			Namespace: "default",
+		},
+		Spec: sandboxv1alpha1.SandboxSnapshotSpec{
+			SandboxName: "test-bs",
+		},
+		Status: sandboxv1alpha1.SandboxSnapshotStatus{
+			Phase: sandboxv1alpha1.SandboxSnapshotPhasePending,
+		},
+	}
+
+	r := newTestSnapshotReconciler(bs, pod, snapshot)
+	r.SnapshotRegistry = "registry.default.svc.cluster.local:5000"
+
+	result, err := r.handlePending(context.Background(), snapshot)
+	require.NoError(t, err)
+	assert.Equal(t, time.Second, result.RequeueAfter)
+
+	updated := &sandboxv1alpha1.SandboxSnapshot{}
+	require.NoError(t, r.Get(context.Background(), types.NamespacedName{Name: snapshot.Name, Namespace: "default"}, updated))
+	require.Len(t, updated.Status.Containers, 1)
+	assert.Equal(
+		t,
+		"registry.default.svc.cluster.local:5000/test-bs-sandbox:snap-11111111222243338444555555555555",
+		updated.Status.Containers[0].ImageURI,
+	)
+}
+
 func TestBuildCommitJob_SetsBoundedBackoffLimit(t *testing.T) {
 	snapshot := &sandboxv1alpha1.SandboxSnapshot{
 		ObjectMeta: metav1.ObjectMeta{

kubernetes/internal/controller/sandboxsnapshot_controller_test.go

@@ -16,6 +16,7 @@ package controller
 
 import (
 	"context"
+	"crypto/sha256"
 	"encoding/json"
 	"fmt"
 	"strings"
@@ -75,7 +76,7 @@ func (r *SandboxSnapshotReconciler) handlePending(ctx context.Context, snapshot
 
 	var containers []sandboxv1alpha1.ContainerSnapshot
 	for _, c := range sourceContainers {
-		imageURI := fmt.Sprintf("%s/%s-%s:snap-gen%d", r.SnapshotRegistry, bs.Name, c.Name, bs.Generation)
+		imageURI := r.snapshotImageURI(snapshot, bs, c.Name)
 		containers = append(containers, sandboxv1alpha1.ContainerSnapshot{
 			ContainerName: c.Name,
 			ImageURI:      imageURI,
@@ -241,6 +242,62 @@ func (r *SandboxSnapshotReconciler) findPodForSandbox(ctx context.Context, bs *s
 	return nil, fmt.Errorf("no running pod found for BatchSandbox %s", bs.Name)
 }
 
+func (r *SandboxSnapshotReconciler) snapshotImageURI(
+	snapshot *sandboxv1alpha1.SandboxSnapshot,
+	bs *sandboxv1alpha1.BatchSandbox,
+	containerName string,
+) string {
+	return fmt.Sprintf(
+		"%s/%s-%s:%s",
+		r.SnapshotRegistry,
+		bs.Name,
+		containerName,
+		snapshotImageTag(snapshot, bs),
+	)
+}
+
+func snapshotImageTag(snapshot *sandboxv1alpha1.SandboxSnapshot, bs *sandboxv1alpha1.BatchSandbox) string {
+	if hasBatchSandboxControllerOwner(snapshot) {
+		return fmt.Sprintf("snap-gen%d", bs.Generation)
+	}
+	return publicSnapshotImageTag(snapshot.Name)
+}
+
+func hasBatchSandboxControllerOwner(snapshot *sandboxv1alpha1.SandboxSnapshot) bool {
+	for _, owner := range snapshot.OwnerReferences {
+		if owner.Kind != "BatchSandbox" {
+			continue
+		}
+		if owner.Controller != nil && *owner.Controller {
+			return true
+		}
+	}
+	return false
+}
+
+func publicSnapshotImageTag(snapshotName string) string {
+	const publicSnapshotNamePrefix = "osb-snap-"
+	if strings.HasPrefix(snapshotName, publicSnapshotNamePrefix) {
+		suffix := strings.TrimPrefix(snapshotName, publicSnapshotNamePrefix)
+		if isLowerHex(suffix) && len(suffix) == 32 {
+			return "snap-" + suffix
+		}
+	}
+
+	sum := sha256.Sum256([]byte(snapshotName))
+	return fmt.Sprintf("snap-%x", sum)[:37]
+}
+
+func isLowerHex(value string) bool {
+	for _, ch := range value {
+		if (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
 func (r *SandboxSnapshotReconciler) imageCommitterImage() string {
 	if r.ImageCommitterImage != "" {
 		return r.ImageCommitterImage

kubernetes/internal/controller/sandboxsnapshot_lifecycle.go

@@ -120,6 +120,7 @@ If `runtime.type = "kubernetes"` and the `[kubernetes]` table is absent, the ser
 | `image_pull_policy` | string \| omitted | `"IfNotPresent"` | Image pull policy for the BatchSandbox main container. Values: **`Always`**, **`IfNotPresent`**, **`Never`**. |
 | `sandbox_create_timeout_seconds` | integer | `60` | Max time to wait for a new sandbox to become ready (e.g. IP assigned), in seconds. |
 | `sandbox_create_poll_interval_seconds` | float | `1.0` | Poll interval while waiting for readiness. |
+| `snapshot_create_timeout_seconds` | integer | `900` | Max time to wait for a Kubernetes public snapshot to become ready, in seconds. Set this greater than the controller snapshot `commitJobTimeout` / `--commit-job-timeout`. |
 | `informer_enabled` | boolean | `true` | **[Beta]** Use informer/watch cache for reads to reduce API load. |
 | `informer_resync_seconds` | integer | `300` | **[Beta]** Full resync period for the informer cache. |
 | `informer_watch_timeout_seconds` | integer | `60` | **[Beta]** Watch stream restart interval. |

server/configuration.md

@@ -558,6 +558,14 @@ class KubernetesRuntimeConfig(BaseModel):
         gt=0,
         description="Polling interval in seconds when waiting for a sandbox to become ready after creation.",
     )
+    snapshot_create_timeout_seconds: int = Field(
+        default=15 * 60,
+        ge=1,
+        description=(
+            "Timeout in seconds to wait for a Kubernetes public snapshot to become ready. "
+            "Set this greater than the controller snapshot commit-job-timeout."
+        ),
+    )
     execd_init_resources: Optional["ExecdInitResources"] = Field(
         default=None,
         description=(

server/opensandbox_server/config.py

@@ -64,6 +64,10 @@ workload_provider = "batchsandbox"
 # Values: "Always", "IfNotPresent", "Never".
 image_pull_policy = "IfNotPresent"
 
+# Public snapshot wait timeout. Keep this greater than the Kubernetes
+# controller snapshot commit-job-timeout.
+snapshot_create_timeout_seconds = 900
+
 # Path to the BatchSandbox template file
 batchsandbox_template_file = "~/batchsandbox-template.yaml"
 

server/opensandbox_server/examples/example.config.k8s.toml

@@ -64,6 +64,10 @@ workload_provider = "batchsandbox"
 # 可选值："Always"、"IfNotPresent"、"Never"。
 image_pull_policy = "IfNotPresent"
 
+# public snapshot 等待超时时间。应大于 Kubernetes controller 的
+# snapshot commit-job-timeout。
+snapshot_create_timeout_seconds = 900
+
 # Path to the BatchSandbox template file
 # Replace with your path
 batchsandbox_template_file = "~/batchsandbox-template.yaml"

server/opensandbox_server/examples/example.config.k8s.zh.toml

@@ -114,6 +114,12 @@ class SandboxErrorCodes:
     INVALID_STATE = "KUBERNETES::INVALID_STATE"
 
 
+class SnapshotErrorCodes:
+    """Canonical error codes for snapshot service operations."""
+
+    INVALID_SOURCE_STATE = "SNAPSHOT::INVALID_SOURCE_STATE"
+
+
 __all__ = [
     "RESERVED_LABEL_PREFIX",
     "SANDBOX_ID_LABEL",
@@ -135,4 +141,5 @@ class SandboxErrorCodes:
     "EGRESS_MODE_ENV",
     "OPENSANDBOX_EGRESS_TOKEN",
     "SandboxErrorCodes",
+    "SnapshotErrorCodes",
 ]