From cc1eee78a22f555b6e1bf900bae01781ca61184e Mon Sep 17 00:00:00 2001 From: yoogo Date: Wed, 13 May 2026 14:56:26 +0800 Subject: [PATCH] feat(server): support static provisioning PV for k8s PVC volumes --- .../opensandbox-server/templates/server.yaml | 3 + .../src/OpenSandbox/Models/Sandboxes.cs | 8 +++ sdks/sandbox/go/types.go | 13 ++-- sdks/sandbox/javascript/src/api/lifecycle.ts | 6 ++ .../javascript/src/models/sandboxes.ts | 6 ++ .../domain/models/sandboxes/SandboxModels.kt | 8 +++ .../converter/SandboxModelConverter.kt | 1 + .../converter/sandbox_model_converter.py | 1 + .../opensandbox/api/lifecycle/models/pvc.py | 21 +++++- .../src/opensandbox/models/sandboxes.py | 9 ++- server/opensandbox_server/api/schema.py | 9 ++- .../opensandbox_server/services/k8s/client.py | 27 ++++++++ .../services/k8s/kubernetes_service.py | 67 +++++++++++++++++-- specs/sandbox-lifecycle.yml | 9 +++ 14 files changed, 175 insertions(+), 13 deletions(-) diff --git a/kubernetes/charts/opensandbox-server/templates/server.yaml b/kubernetes/charts/opensandbox-server/templates/server.yaml index 2dcf1d7fa..fced9be09 100644 --- a/kubernetes/charts/opensandbox-server/templates/server.yaml +++ b/kubernetes/charts/opensandbox-server/templates/server.yaml @@ -29,6 +29,9 @@ rules: - apiGroups: [""] resources: ["persistentvolumeclaims"] verbs: ["create", "get"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["create", "get"] - apiGroups: ["node.k8s.io"] resources: ["runtimeclasses"] verbs: ["get", "list"] diff --git a/sdks/sandbox/csharp/src/OpenSandbox/Models/Sandboxes.cs b/sdks/sandbox/csharp/src/OpenSandbox/Models/Sandboxes.cs index 40a027a32..402d6fe4f 100644 --- a/sdks/sandbox/csharp/src/OpenSandbox/Models/Sandboxes.cs +++ b/sdks/sandbox/csharp/src/OpenSandbox/Models/Sandboxes.cs @@ -185,6 +185,14 @@ public class PVC /// [JsonPropertyName("accessModes")] public IReadOnlyList? AccessModes { get; set; } + + /// + /// Gets or sets the static PersistentVolume spec for Kubernetes. + /// When provided, the server creates a PV with this spec bound to the auto-created PVC. + /// Defaults to dynamic provisioning when omitted. Ignored for Docker volumes. + /// + [JsonPropertyName("pv")] + public IReadOnlyDictionary? Pv { get; set; } } /// diff --git a/sdks/sandbox/go/types.go b/sdks/sandbox/go/types.go index a7f317d10..04e79efdb 100644 --- a/sdks/sandbox/go/types.go +++ b/sdks/sandbox/go/types.go @@ -77,12 +77,13 @@ type Host struct { // PVC represents a platform-managed named volume backend. type PVC struct { - ClaimName string `json:"claimName"` - CreateIfNotExists *bool `json:"createIfNotExists,omitempty"` - DeleteOnSandboxTermination *bool `json:"deleteOnSandboxTermination,omitempty"` - StorageClass *string `json:"storageClass,omitempty"` - Storage *string `json:"storage,omitempty"` - AccessModes []string `json:"accessModes,omitempty"` + ClaimName string `json:"claimName"` + CreateIfNotExists *bool `json:"createIfNotExists,omitempty"` + DeleteOnSandboxTermination *bool `json:"deleteOnSandboxTermination,omitempty"` + StorageClass *string `json:"storageClass,omitempty"` + Storage *string `json:"storage,omitempty"` + AccessModes []string `json:"accessModes,omitempty"` + PV map[string]any `json:"pv,omitempty"` } // OSSFS represents an Alibaba Cloud OSS mount backend via ossfs. diff --git a/sdks/sandbox/javascript/src/api/lifecycle.ts b/sdks/sandbox/javascript/src/api/lifecycle.ts index 2da9809ca..0f2f07e31 100644 --- a/sdks/sandbox/javascript/src/api/lifecycle.ts +++ b/sdks/sandbox/javascript/src/api/lifecycle.ts @@ -1325,6 +1325,12 @@ export interface components { * volumes. */ accessModes?: string[] | null; + /** + * @description Static PersistentVolume spec for Kubernetes. When provided, + * the server creates a PV with this spec bound to the auto-created PVC. + * Defaults to dynamic provisioning when omitted. Ignored for Docker volumes. + */ + pv?: Record | null; }; /** * @description Alibaba Cloud OSS mount backend via ossfs. diff --git a/sdks/sandbox/javascript/src/models/sandboxes.ts b/sdks/sandbox/javascript/src/models/sandboxes.ts index cdfb10212..99f0f8bf8 100644 --- a/sdks/sandbox/javascript/src/models/sandboxes.ts +++ b/sdks/sandbox/javascript/src/models/sandboxes.ts @@ -128,6 +128,12 @@ export interface PVC extends Record { * Ignored for Docker. */ accessModes?: string[] | null; + /** + * Static PersistentVolume spec for Kubernetes. When provided, the server + * creates a PV with this spec bound to the auto-created PVC. + * Defaults to dynamic provisioning when omitted. Ignored for Docker volumes. + */ + pv?: Record | null; } /** diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/models/sandboxes/SandboxModels.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/models/sandboxes/SandboxModels.kt index 69a0ca419..8bbfb5d86 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/models/sandboxes/SandboxModels.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/domain/models/sandboxes/SandboxModels.kt @@ -418,6 +418,7 @@ class PVC private constructor( val storageClass: String?, val storage: String?, val accessModes: List?, + val pv: Map?, ) { companion object { @JvmStatic @@ -434,6 +435,7 @@ class PVC private constructor( private var storageClass: String? = null private var storage: String? = null private var accessModes: List? = null + private var pv: Map? = null fun claimName(claimName: String): Builder { require(claimName.isNotBlank()) { "Claim name cannot be blank" } @@ -471,6 +473,11 @@ class PVC private constructor( return this } + fun pv(pv: Map?): Builder { + this.pv = pv + return this + } + fun build(): PVC { val claimNameValue = claimName ?: throw IllegalArgumentException("Claim name must be specified") return PVC( @@ -480,6 +487,7 @@ class PVC private constructor( storageClass = storageClass, storage = storage, accessModes = accessModes, + pv = pv, ) } } diff --git a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/converter/SandboxModelConverter.kt b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/converter/SandboxModelConverter.kt index d6b99a3e5..a69f0f952 100644 --- a/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/converter/SandboxModelConverter.kt +++ b/sdks/sandbox/kotlin/sandbox/src/main/kotlin/com/alibaba/opensandbox/sandbox/infrastructure/adapters/converter/SandboxModelConverter.kt @@ -205,6 +205,7 @@ internal object SandboxModelConverter { storageClass = this.storageClass, storage = this.storage, accessModes = this.accessModes, + pv = this.pv, ) } diff --git a/sdks/sandbox/python/src/opensandbox/adapters/converter/sandbox_model_converter.py b/sdks/sandbox/python/src/opensandbox/adapters/converter/sandbox_model_converter.py index 304bcb3a8..76e3839b4 100644 --- a/sdks/sandbox/python/src/opensandbox/adapters/converter/sandbox_model_converter.py +++ b/sdks/sandbox/python/src/opensandbox/adapters/converter/sandbox_model_converter.py @@ -125,6 +125,7 @@ def to_api_volume(volume: Volume): storage_class=volume.pvc.storage_class, storage=volume.pvc.storage, access_modes=volume.pvc.access_modes, + pv=volume.pvc.pv, ) api_ossfs = UNSET diff --git a/sdks/sandbox/python/src/opensandbox/api/lifecycle/models/pvc.py b/sdks/sandbox/python/src/opensandbox/api/lifecycle/models/pvc.py index 138e2877f..315f8167c 100644 --- a/sdks/sandbox/python/src/opensandbox/api/lifecycle/models/pvc.py +++ b/sdks/sandbox/python/src/opensandbox/api/lifecycle/models/pvc.py @@ -17,7 +17,7 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Any, TypeVar, cast +from typing import Any, TypeVar, cast, Dict from attrs import define as _attrs_define @@ -66,6 +66,7 @@ class PVC: storage_class: None | str | Unset = UNSET storage: None | str | Unset = UNSET access_modes: list[str] | None | Unset = UNSET + pv: Dict[str, Any] | None | Unset = UNSET def to_dict(self) -> dict[str, Any]: claim_name = self.claim_name @@ -113,6 +114,14 @@ def to_dict(self) -> dict[str, Any]: if access_modes is not UNSET: field_dict["accessModes"] = access_modes + pv: Dict[str, Any] | None | Unset + if isinstance(self.pv, Unset): + pv = UNSET + else: + pv = self.pv + if pv is not UNSET: + field_dict["pv"] = pv + return field_dict @classmethod @@ -159,6 +168,15 @@ def _parse_access_modes(data: object) -> list[str] | None | Unset: access_modes = _parse_access_modes(d.pop("accessModes", UNSET)) + def _parse_pv(data: object) -> Dict[str, Any] | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(Dict[str, Any], data) + + pv = _parse_pv(d.pop("pv", UNSET)) + pvc = cls( claim_name=claim_name, create_if_not_exists=create_if_not_exists, @@ -166,6 +184,7 @@ def _parse_access_modes(data: object) -> list[str] | None | Unset: storage_class=storage_class, storage=storage, access_modes=access_modes, + pv=pv, ) return pvc diff --git a/sdks/sandbox/python/src/opensandbox/models/sandboxes.py b/sdks/sandbox/python/src/opensandbox/models/sandboxes.py index 829af5ee6..a4ce32ed9 100644 --- a/sdks/sandbox/python/src/opensandbox/models/sandboxes.py +++ b/sdks/sandbox/python/src/opensandbox/models/sandboxes.py @@ -21,7 +21,7 @@ import re from datetime import datetime -from typing import Literal +from typing import Any, Dict, Literal, Optional from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -227,6 +227,13 @@ class PVC(BaseModel): "Ignored for Docker." ), ) + pv: Optional[Dict[str, Any]] = Field( + None, + description=( + "static provisioning pv for auto-created PVCs. " + "Defaults dynamic provisioning when omitted. Ignored for Docker volumes." + ), + ) model_config = ConfigDict(populate_by_name=True) diff --git a/server/opensandbox_server/api/schema.py b/server/opensandbox_server/api/schema.py index f3be288e0..c15e3593f 100644 --- a/server/opensandbox_server/api/schema.py +++ b/server/opensandbox_server/api/schema.py @@ -20,7 +20,7 @@ """ from datetime import datetime -from typing import Dict, List, Literal, Optional +from typing import Dict, List, Literal, Optional, Any from pydantic import BaseModel, Field, RootModel, model_validator @@ -213,6 +213,13 @@ class PVC(BaseModel): "Defaults to ['ReadWriteOnce'] when omitted. Ignored for Docker volumes." ), ) + pv: Optional[Dict[str, Any]] = Field( + None, + description=( + "static provisioning pv for auto-created PVCs. " + "Defaults dynamic provisioning when omitted. Ignored for Docker volumes." + ), + ) class Config: populate_by_name = True diff --git a/server/opensandbox_server/services/k8s/client.py b/server/opensandbox_server/services/k8s/client.py index 72f39a686..09539efdd 100644 --- a/server/opensandbox_server/services/k8s/client.py +++ b/server/opensandbox_server/services/k8s/client.py @@ -317,6 +317,33 @@ def create_pvc( body=body, ) + # ------------------------------------------------------------------ + # PersistentVolume operations + # ------------------------------------------------------------------ + + def get_pv( + self, + name: str, + ) -> Optional[Any]: + """Read a PersistentVolume by name. Returns None on 404.""" + if self._read_limiter: + self._read_limiter.acquire() + try: + return self.get_core_v1_api().read_persistent_volume(name) + except ApiException as e: + if e.status == 404: + return None + raise + + def create_pv( + self, + body: Any, + ) -> Any: + """Create a PersistentVolume.""" + if self._write_limiter: + self._write_limiter.acquire() + return self.get_core_v1_api().create_persistent_volume(body) + # ------------------------------------------------------------------ # Secret operations # ------------------------------------------------------------------ diff --git a/server/opensandbox_server/services/k8s/kubernetes_service.py b/server/opensandbox_server/services/k8s/kubernetes_service.py index 4c674efc9..e9a75ca0e 100644 --- a/server/opensandbox_server/services/k8s/kubernetes_service.py +++ b/server/opensandbox_server/services/k8s/kubernetes_service.py @@ -82,6 +82,7 @@ ensure_metadata_labels, ensure_platform_valid, ensure_timeout_within_limit, + ensure_valid_host_path, ensure_volumes_valid, ) from opensandbox_server.services.k8s.client import K8sClient @@ -309,7 +310,7 @@ def _ensure_pvc_volumes(self, volumes: list) -> None: for PVC operations (403), the check is skipped and volume resolution is left to the kubelet at pod scheduling time. """ - from kubernetes.client import V1PersistentVolumeClaim, V1ObjectMeta + from kubernetes.client import V1PersistentVolume, V1PersistentVolumeClaim, V1ObjectMeta from kubernetes.client import ApiException default_size = self.app_config.storage.volume_default_size @@ -341,10 +342,52 @@ def _ensure_pvc_volumes(self, volumes: list) -> None: access_modes = vol.pvc.access_modes or ["ReadWriteOnce"] storage_class = vol.pvc.storage_class # None = cluster default + pv_body = None + if vol.pvc.pv is not None: + allowed_paths = self.app_config.storage.allowed_host_paths + pv_spec_raw = vol.pvc.pv + for source_key, path_key in (("hostPath", "path"), ("local", "path")): + source = pv_spec_raw.get(source_key) + if isinstance(source, dict) and path_key in source: + ensure_valid_host_path(source[path_key], allowed_paths) + + pv_name = claim_name + if self.namespace is not None: + pv_name = f"{claim_name}-{self.namespace}" + if len(pv_name) > 253: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": SandboxErrorCodes.INVALID_PARAMETER, + "message": ( + f"Generated PV name '{pv_name}' exceeds Kubernetes' " + f"253-character limit ({len(pv_name)} chars). " + f"Use a shorter claimName." + ), + }, + ) + spec = pv_spec_raw + spec["claimRef"] = { + "name": claim_name, + "namespace": self.namespace, + } + spec["accessModes"] = access_modes + spec["capacity"] = {"storage": storage} + if storage_class is not None: + spec["storageClassName"] = storage_class + else: + spec["storageClassName"] = "" + pv_body = V1PersistentVolume( + metadata=V1ObjectMeta(name=pv_name), + spec=spec, + ) + if storage_class is None: + storage_class = "" + pvc_body = V1PersistentVolumeClaim( metadata=V1ObjectMeta( name=claim_name, - namespace=self.namespace, + namespace=self.namespace ), spec={ "accessModes": access_modes, @@ -354,6 +397,24 @@ def _ensure_pvc_volumes(self, volumes: list) -> None: if storage_class is not None: pvc_body.spec["storageClassName"] = storage_class + if pv_body is not None: + try: + self.k8s_client.create_pv(pv_body) + logger.info(f"Auto-created PV '{pv_body.metadata.name}'") + except ApiException as e: + if e.status == 409: + logger.info(f"PV '{pv_body.metadata.name}' already exists, proceeding with PVC creation") + elif e.status in (400, 422): + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={ + "code": SandboxErrorCodes.INVALID_PARAMETER, + "message": f"Invalid PV spec for '{claim_name}': {e.reason}", + }, + ) from e + else: + raise + try: self.k8s_client.create_pvc(self.namespace, pvc_body) logger.info( @@ -362,7 +423,6 @@ def _ensure_pvc_volumes(self, volumes: list) -> None: ) except ApiException as e: if e.status == 409: - # Race condition: another request created it between our check and create logger.info(f"PVC '{claim_name}' was created concurrently, proceeding") elif e.status == 403: logger.warning( @@ -440,7 +500,6 @@ async def create_sandbox(self, request: CreateSandboxRequest) -> CreateSandboxRe request.volumes, self.app_config.storage.allowed_host_paths, ) - # Auto-create PVCs that don't exist yet if request.volumes: diff --git a/specs/sandbox-lifecycle.yml b/specs/sandbox-lifecycle.yml index 9809d7d99..ba457625d 100644 --- a/specs/sandbox-lifecycle.yml +++ b/specs/sandbox-lifecycle.yml @@ -1540,6 +1540,15 @@ components: Access modes for auto-created PVCs (e.g. ["ReadWriteOnce"]). Defaults to ["ReadWriteOnce"] when omitted. Ignored for Docker volumes. + pv: + type: object + nullable: true + description: | + Static PersistentVolume spec for Kubernetes. When provided, the + server creates a PV with this spec bound to the auto-created PVC. + Defaults to dynamic provisioning when omitted. Ignored for Docker + volumes. + additionalProperties: true additionalProperties: false OSSFS: