fix(server): k8s patch_sandbox_metadata correctly deletes keys and returns post-patch state (#899)

* fix(server): k8s patch_sandbox_metadata correctly deletes keys and returns post-patch state

Two bugs in KubernetesSandboxService.patch_sandbox_metadata caused the
nightly k8s mini E2E test_02_metadata_filter_and_logic to fail:

1. JSON merge patch (RFC 7396) on metadata.labels merges keys recursively
   — keys absent from the patch body are kept. The previous code computed
   the desired final labels dict (with deleted keys removed) and sent it,
   so deleted keys were never actually removed on the API server.

2. After the PATCH, the code re-fetched the workload via
   _get_workload_or_404, which goes through K8sClient.get_custom_object
   that prefers the informer cache. The informer is eventually consistent,
   so the read could land before the watch event arrived and return the
   pre-patch labels.

Fix both by:
- Building the merge-patch body with explicit None for deleted keys.
- Using the API server's PATCH response (returned from patch_labels)
  directly, instead of re-reading via the cache.

WorkloadProvider.patch_labels now accepts Dict[str, Optional[str]] and
returns the patched workload dict.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

* chore(ci): touch trigger comment to run k8s mini E2E on this PR

The k8s-nightly-build workflow only runs on PRs that touch one of its
trigger paths. This PR fixes server-side k8s logic but does not modify
those paths, so add a date stamp to the existing trigger comment to
include this PR in the matrix.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com>

Author: Pangjiping

scripts/python-k8s-e2e.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# trigger k8s e2e
+# trigger k8s e2e (2026-05-17)
 # Copyright 2026 Alibaba Group Holding Ltd.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

server/opensandbox_server/services/k8s/kubernetes_service.py

@@ -791,21 +791,25 @@ def patch_sandbox_metadata(self, sandbox_id: str, patch: PatchSandboxMetadataReq
 
         new_labels = self._apply_metadata_patch(labels, patch)
 
+        # JSON merge patch (RFC 7396) on metadata.labels treats keys absent
+        # from the body as kept. To delete a label we must send the key with
+        # an explicit null. Build the merge body from the desired final labels
+        # plus null markers for keys removed by this patch.
+        label_patch: Dict[str, Optional[str]] = dict(new_labels)
+        for key, value in patch.items():
+            if value is None:
+                label_patch[key] = None
+
         try:
-            self.workload_provider.patch_labels(
+            updated = self.workload_provider.patch_labels(
                 name=name,
                 namespace=self.namespace,
-                labels=new_labels,
+                labels=label_patch,
             )
         except Exception as e:
             logger.error("Error patching labels for sandbox %s: %s", sandbox_id, e)
             raise _build_k8s_api_error("patch sandbox labels", e) from e
 
-        updated = _get_workload_or_404(
-            self.workload_provider,
-            self.namespace,
-            sandbox_id,
-        )
         return _build_sandbox_from_workload(updated, self.workload_provider)
 
     def get_endpoint(

server/opensandbox_server/services/k8s/workload_provider.py

@@ -202,10 +202,16 @@ def resume_sandbox(self, sandbox_id: str, namespace: str) -> None:
         """
         raise NotImplementedError("Resume is not supported by this provider")
 
-    def patch_labels(self, name: str, namespace: str, labels: Dict[str, str]) -> None:
-        """Patch workload metadata.labels via JSON merge patch."""
+    def patch_labels(
+        self, name: str, namespace: str, labels: Dict[str, Optional[str]]
+    ) -> Dict[str, Any]:
+        """Patch workload metadata.labels via JSON merge patch.
+
+        A None value for a label key deletes that label per RFC 7396.
+        Returns the API server response (the patched workload).
+        """
         body = {"metadata": {"labels": labels}}
-        self.k8s_client.patch_custom_object(
+        return self.k8s_client.patch_custom_object(
             group=self.group,
             version=self.version,
             namespace=namespace,

server/tests/k8s/test_kubernetes_service.py

@@ -1297,3 +1297,62 @@ def test_signed_endpoint_different_expires_produces_different_endpoints(self, k8
         ep2 = k8s_service.get_endpoint("sbx-001", 8080, expires=2000000500)
 
         assert ep1.endpoint != ep2.endpoint
+
+
+class TestPatchSandboxMetadata:
+    """Verify patch_sandbox_metadata builds the JSON merge-patch body correctly
+    and uses the API server's PATCH response (not a cache-prone re-fetch)."""
+
+    @staticmethod
+    def _workload(labels: dict) -> dict:
+        return {
+            "metadata": {
+                "name": "sandbox-sbx-001",
+                "labels": dict(labels),
+                "creationTimestamp": datetime(2026, 1, 1, tzinfo=timezone.utc),
+            },
+            "spec": {},
+            "status": {"conditions": []},
+        }
+
+    @staticmethod
+    def _stub_provider_status(k8s_service) -> None:
+        k8s_service.workload_provider.get_status.return_value = {
+            "state": "Running",
+            "reason": None,
+            "message": None,
+            "last_transition_at": None,
+        }
+        k8s_service.workload_provider.get_expiration.return_value = None
+
+    def test_patch_body_sends_null_for_deleted_keys(self, k8s_service):
+        initial = {"opensandbox.io/id": "sbx-001", "team": "infra", "env": "dev"}
+        patched = {"opensandbox.io/id": "sbx-001", "env": "stage"}
+
+        k8s_service.workload_provider.get_workload.return_value = self._workload(initial)
+        k8s_service.workload_provider.patch_labels.return_value = self._workload(patched)
+        self._stub_provider_status(k8s_service)
+
+        k8s_service.patch_sandbox_metadata("sbx-001", {"env": "stage", "team": None})
+
+        k8s_service.workload_provider.patch_labels.assert_called_once()
+        body_labels = k8s_service.workload_provider.patch_labels.call_args.kwargs["labels"]
+        assert body_labels["env"] == "stage"
+        assert body_labels["team"] is None
+        assert body_labels["opensandbox.io/id"] == "sbx-001"
+
+    def test_returns_sandbox_from_patch_response(self, k8s_service):
+        """The PATCH response is authoritative; re-reading via get_workload
+        could hit a stale informer cache."""
+        initial = {"opensandbox.io/id": "sbx-001", "env": "dev"}
+        patched = {"opensandbox.io/id": "sbx-001", "env": "stage"}
+
+        k8s_service.workload_provider.get_workload.return_value = self._workload(initial)
+        k8s_service.workload_provider.patch_labels.return_value = self._workload(patched)
+        self._stub_provider_status(k8s_service)
+
+        sandbox = k8s_service.patch_sandbox_metadata("sbx-001", {"env": "stage"})
+
+        assert sandbox.metadata == {"env": "stage"}
+        # Pre-patch read only; no second get_workload after patch_labels.
+        assert k8s_service.workload_provider.get_workload.call_count == 1