fix(server): enforce windows profile arch scheduling and merge USER_PORTS defaults

Pangjiping · Pangjiping · commit 6b0efbd9b9bf · 2026-04-19T14:05:46.000+08:00
diff --git a/server/opensandbox_server/services/k8s/batchsandbox_provider.py b/server/opensandbox_server/services/k8s/batchsandbox_provider.py
@@ -47,6 +47,7 @@
     _workload_platform_constraint_scope,
 )
 from opensandbox_server.services.k8s.windows_profile import (
+    apply_windows_profile_arch_selector,
     apply_windows_profile_overrides,
     is_windows_profile,
     validate_windows_profile_resource_limits,
@@ -188,6 +189,17 @@ def create_workload(
                 resource_limits=resource_limits,
                 disable_ipv6_for_egress=disable_ipv6_for_egress,
             )
+            template = self.template_manager.get_base_template()
+            template_spec = (
+                template.get("spec", {})
+                .get("template", {})
+                .get("spec", {})
+            )
+            apply_windows_profile_arch_selector(
+                pod_spec=pod_spec,
+                template_spec=template_spec if isinstance(template_spec, dict) else {},
+                platform=platform,
+            )
         else:
             self._apply_platform_node_selector(pod_spec, platform)
 
diff --git a/server/opensandbox_server/services/k8s/windows_profile.py b/server/opensandbox_server/services/k8s/windows_profile.py
@@ -26,6 +26,7 @@
 WINDOWS_OEM_VOLUME_NAME = "opensandbox-win-oem"
 WINDOWS_KVM_VOLUME_NAME = "opensandbox-win-kvm"
 WINDOWS_TUN_VOLUME_NAME = "opensandbox-win-tun"
+WINDOWS_PROFILE_DEFAULT_USER_PORTS = ["44772", "8080", "3389/tcp", "3389/udp", "8006/tcp"]
 
 
 def is_windows_profile(platform: Optional[PlatformSpec]) -> bool:
@@ -39,12 +40,10 @@ def validate_windows_profile_resource_limits(resource_limits: dict[str, str]) ->
 def build_windows_profile_env(
     env: dict[str, str],
     resource_limits: dict[str, str],
-    exposed_ports: Optional[list[str]] = None,
 ) -> list[dict[str, str]]:
     env_items = [f"{key}={value}" for key, value in env.items()]
     env_items = inject_windows_resource_limits_env(env_items, resource_limits or {})
-    if exposed_ports is not None:
-        env_items = inject_windows_user_ports(env_items, exposed_ports)
+    env_items = inject_windows_user_ports(env_items, WINDOWS_PROFILE_DEFAULT_USER_PORTS)
 
     result: list[dict[str, str]] = []
     for item in env_items:
@@ -130,6 +129,46 @@ def apply_windows_profile_overrides(
     )
 
 
+def apply_windows_profile_arch_selector(
+    pod_spec: Dict[str, Any],
+    template_spec: Dict[str, Any],
+    platform: Optional[PlatformSpec],
+) -> None:
+    """
+    Apply platform.arch constraint for windows profile pods.
+
+    We intentionally avoid forcing kubernetes.io/os=windows for this profile,
+    but still honor arch constraints from API requests and fail early on
+    template conflicts.
+    """
+    if platform is None:
+        return
+
+    requested_arch = platform.arch
+    template_selector = template_spec.get("nodeSelector", {})
+    if not isinstance(template_selector, dict):
+        template_selector = {}
+
+    existing_arch = template_selector.get("kubernetes.io/arch")
+    if existing_arch is not None and existing_arch != requested_arch:
+        raise ValueError(
+            "platform conflict with template nodeSelector: 'kubernetes.io/arch' "
+            f"is '{existing_arch}', request expects '{requested_arch}'."
+        )
+
+    if not _template_allows_arch(template_spec, requested_arch):
+        raise ValueError(
+            "platform conflict with template nodeAffinity: required node affinity "
+            f"does not allow requested architecture '{requested_arch}'."
+        )
+
+    node_selector = pod_spec.setdefault("nodeSelector", {})
+    if not isinstance(node_selector, dict):
+        node_selector = {}
+        pod_spec["nodeSelector"] = node_selector
+    node_selector["kubernetes.io/arch"] = requested_arch
+
+
 def _merge_volume_mounts(container: Dict[str, Any], mounts_to_add: List[Dict[str, str]]) -> None:
     mounts = container.setdefault("volumeMounts", [])
     if not isinstance(mounts, list):
@@ -156,3 +195,48 @@ def _merge_volumes(pod_spec: Dict[str, Any], volumes_to_add: List[Dict[str, Any]
             continue
         volumes.append(volume)
         existing_names.add(name)
+
+
+def _template_allows_arch(template_spec: Dict[str, Any], requested_arch: str) -> bool:
+    affinity = template_spec.get("affinity", {})
+    if not isinstance(affinity, dict):
+        return True
+
+    node_affinity = affinity.get("nodeAffinity", {})
+    if not isinstance(node_affinity, dict):
+        return True
+
+    required = node_affinity.get("requiredDuringSchedulingIgnoredDuringExecution", {})
+    if not isinstance(required, dict):
+        return True
+
+    terms = required.get("nodeSelectorTerms", [])
+    if not isinstance(terms, list) or not terms:
+        return True
+
+    return any(_arch_term_satisfiable(term, requested_arch) for term in terms if isinstance(term, dict))
+
+
+def _arch_term_satisfiable(term: Dict[str, Any], requested_arch: str) -> bool:
+    expressions = term.get("matchExpressions", [])
+    if not isinstance(expressions, list):
+        return True
+
+    for expr in expressions:
+        if not isinstance(expr, dict):
+            continue
+        if expr.get("key") != "kubernetes.io/arch":
+            continue
+        operator = expr.get("operator")
+        values = expr.get("values", [])
+        if not isinstance(values, list):
+            values = []
+
+        if operator == "In" and requested_arch not in values:
+            return False
+        if operator == "NotIn" and requested_arch in values:
+            return False
+        if operator == "DoesNotExist":
+            return False
+
+    return True
diff --git a/server/tests/k8s/test_batchsandbox_provider.py b/server/tests/k8s/test_batchsandbox_provider.py
@@ -175,8 +175,10 @@ def test_create_workload_windows_profile_uses_windows_runtime_shape(self, mock_k
         body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
         pod_spec = body["spec"]["template"]["spec"]
 
-        # windows profile should not force k8s windows nodeSelector.
-        assert "nodeSelector" not in pod_spec
+        # windows profile should enforce requested arch, but not force os=windows.
+        node_selector = pod_spec.get("nodeSelector", {})
+        assert node_selector["kubernetes.io/arch"] == "amd64"
+        assert "kubernetes.io/os" not in node_selector
 
         init_container = pod_spec["initContainers"][0]
         assert init_container["command"] == ["/bin/sh", "-c"]
@@ -192,13 +194,67 @@ def test_create_workload_windows_profile_uses_windows_runtime_shape(self, mock_k
         assert env_dict["CPU_CORES"] == "4"
         assert env_dict["RAM_SIZE"] == "8G"
         assert env_dict["DISK_SIZE"] == "64G"
-        assert "USER_PORTS" not in env_dict
+        assert env_dict["USER_PORTS"] == "44772,8080,3389,8006"
 
         volume_names = {volume["name"] for volume in pod_spec.get("volumes", [])}
         assert "opensandbox-win-oem" in volume_names
         assert "opensandbox-win-kvm" in volume_names
         assert "opensandbox-win-tun" in volume_names
 
+    def test_create_workload_windows_profile_merges_user_ports(self, mock_k8s_client):
+        provider = BatchSandboxProvider(mock_k8s_client)
+        mock_k8s_client.create_custom_object.return_value = {
+            "metadata": {"name": "test-id", "uid": "test-uid"}
+        }
+
+        provider.create_workload(
+            sandbox_id="test-id",
+            namespace="test-ns",
+            image_spec=ImageSpec(uri="dockurr/windows:latest"),
+            entrypoint=["cmd", "/c", "echo hello"],
+            env={"VERSION": "11", "USER_PORTS": "3000,44772"},
+            resource_limits={"cpu": "4", "memory": "8G", "disk": "64G"},
+            labels={"opensandbox.io/id": "test-id"},
+            expires_at=None,
+            execd_image="execd:latest",
+            platform=PlatformSpec(os="windows", arch="amd64"),
+        )
+
+        body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
+        pod_spec = body["spec"]["template"]["spec"]
+        main_container = pod_spec["containers"][0]
+        env_dict = {item["name"]: item["value"] for item in main_container.get("env", [])}
+        assert env_dict["USER_PORTS"] == "3000,44772,8080,3389,8006"
+
+    def test_create_workload_windows_profile_rejects_arch_conflict_with_template_selector(
+        self, mock_k8s_client, tmp_path
+    ):
+        template_file = tmp_path / "template.yaml"
+        template_file.write_text(
+            """
+spec:
+  template:
+    spec:
+      nodeSelector:
+        kubernetes.io/arch: arm64
+"""
+        )
+        provider = BatchSandboxProvider(mock_k8s_client, _app_config_with_template(str(template_file)))
+
+        with pytest.raises(ValueError, match="platform conflict with template nodeSelector"):
+            provider.create_workload(
+                sandbox_id="test-id",
+                namespace="test-ns",
+                image_spec=ImageSpec(uri="dockurr/windows:latest"),
+                entrypoint=["cmd", "/c", "echo hello"],
+                env={"VERSION": "11"},
+                resource_limits={"cpu": "4", "memory": "8G", "disk": "64G"},
+                labels={"opensandbox.io/id": "test-id"},
+                expires_at=None,
+                execd_image="execd:latest",
+                platform=PlatformSpec(os="windows", arch="amd64"),
+            )
+
     def test_create_workload_rejects_platform_conflict_with_template_selector(self, mock_k8s_client, tmp_path):
         template_file = tmp_path / "template.yaml"
         template_file.write_text(
