Exclude backward incompatible fields from rest plugin calls

src/dstack/_internal/core/compatibility/fleets.py

@@ -0,0 +1,72 @@
+from typing import Any, Dict, Optional
+
+from dstack._internal.core.models.fleets import ApplyFleetPlanInput, FleetSpec
+from dstack._internal.core.models.instances import Instance
+
+
+def get_get_plan_excludes(fleet_spec: FleetSpec) -> Dict:
+    get_plan_excludes = {}
+    spec_excludes = get_fleet_spec_excludes(fleet_spec)
+    if spec_excludes:
+        get_plan_excludes["spec"] = spec_excludes
+    return get_plan_excludes
+
+
+def get_apply_plan_excludes(plan_input: ApplyFleetPlanInput) -> Dict:
+    apply_plan_excludes = {}
+    spec_excludes = get_fleet_spec_excludes(plan_input.spec)
+    if spec_excludes:
+        apply_plan_excludes["spec"] = spec_excludes
+    current_resource = plan_input.current_resource
+    if current_resource is not None:
+        current_resource_excludes = {}
+        apply_plan_excludes["current_resource"] = current_resource_excludes
+        if all(map(_should_exclude_instance_cpu_arch, current_resource.instances)):
+            current_resource_excludes["instances"] = {
+                "__all__": {"instance_type": {"resources": {"cpu_arch"}}}
+            }
+    return {"plan": apply_plan_excludes}
+
+
+def get_create_fleet_excludes(fleet_spec: FleetSpec) -> Dict:
+    create_fleet_excludes = {}
+    spec_excludes = get_fleet_spec_excludes(fleet_spec)
+    if spec_excludes:
+        create_fleet_excludes["spec"] = spec_excludes
+    return create_fleet_excludes
+
+
+def get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[Dict]:
+    """
+    Returns `fleet_spec` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    spec_excludes: Dict[str, Any] = {}
+    configuration_excludes: Dict[str, Any] = {}
+    profile_excludes: set[str] = set()
+    profile = fleet_spec.profile
+    if profile.fleets is None:
+        profile_excludes.add("fleets")
+    if fleet_spec.configuration.tags is None:
+        configuration_excludes["tags"] = True
+    if profile.tags is None:
+        profile_excludes.add("tags")
+    if profile.startup_order is None:
+        profile_excludes.add("startup_order")
+    if profile.stop_criteria is None:
+        profile_excludes.add("stop_criteria")
+    if configuration_excludes:
+        spec_excludes["configuration"] = configuration_excludes
+    if profile_excludes:
+        spec_excludes["profile"] = profile_excludes
+    if spec_excludes:
+        return spec_excludes
+    return None
+
+
+def _should_exclude_instance_cpu_arch(instance: Instance) -> bool:
+    try:
+        return instance.instance_type.resources.cpu_arch is None
+    except AttributeError:
+        return True

src/dstack/_internal/core/compatibility/gateways.py

@@ -0,0 +1,34 @@
+from typing import Dict
+
+from dstack._internal.core.models.gateways import GatewayConfiguration, GatewaySpec
+
+
+def get_gateway_spec_excludes(gateway_spec: GatewaySpec) -> Dict:
+    """
+    Returns `gateway_spec` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    spec_excludes = {}
+    spec_excludes["configuration"] = _get_gateway_configuration_excludes(
+        gateway_spec.configuration
+    )
+    return spec_excludes
+
+
+def get_create_gateway_excludes(configuration: GatewayConfiguration) -> Dict:
+    """
+    Returns an exclude mapping to exclude certain fields from the create gateway request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    create_gateway_excludes = {}
+    create_gateway_excludes["configuration"] = _get_gateway_configuration_excludes(configuration)
+    return create_gateway_excludes
+
+
+def _get_gateway_configuration_excludes(configuration: GatewayConfiguration) -> Dict:
+    configuration_excludes = {}
+    if configuration.tags is None:
+        configuration_excludes["tags"] = True
+    return configuration_excludes

src/dstack/_internal/core/compatibility/runs.py

@@ -0,0 +1,125 @@
+from typing import Any, Dict, Optional
+
+from dstack._internal.core.models.configurations import ServiceConfiguration
+from dstack._internal.core.models.runs import ApplyRunPlanInput, JobSubmission, RunSpec
+from dstack._internal.server.schemas.runs import GetRunPlanRequest
+
+
+def get_apply_plan_excludes(plan: ApplyRunPlanInput) -> Optional[Dict]:
+    """
+    Returns `plan` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    apply_plan_excludes = {}
+    run_spec_excludes = get_run_spec_excludes(plan.run_spec)
+    if run_spec_excludes is not None:
+        apply_plan_excludes["run_spec"] = run_spec_excludes
+    current_resource = plan.current_resource
+    if current_resource is not None:
+        current_resource_excludes = {}
+        current_resource_excludes["status_message"] = True
+        apply_plan_excludes["current_resource"] = current_resource_excludes
+        current_resource_excludes["run_spec"] = get_run_spec_excludes(current_resource.run_spec)
+        job_submissions_excludes = {}
+        current_resource_excludes["jobs"] = {
+            "__all__": {"job_submissions": {"__all__": job_submissions_excludes}}
+        }
+        job_submissions = [js for j in current_resource.jobs for js in j.job_submissions]
+        if all(map(_should_exclude_job_submission_jpd_cpu_arch, job_submissions)):
+            job_submissions_excludes["job_provisioning_data"] = {
+                "instance_type": {"resources": {"cpu_arch"}}
+            }
+        if all(map(_should_exclude_job_submission_jrd_cpu_arch, job_submissions)):
+            job_submissions_excludes["job_runtime_data"] = {
+                "offer": {"instance": {"resources": {"cpu_arch"}}}
+            }
+        if all(js.exit_status is None for js in job_submissions):
+            job_submissions_excludes["exit_status"] = True
+        latest_job_submission = current_resource.latest_job_submission
+        if latest_job_submission is not None:
+            latest_job_submission_excludes = {}
+            current_resource_excludes["latest_job_submission"] = latest_job_submission_excludes
+            if _should_exclude_job_submission_jpd_cpu_arch(latest_job_submission):
+                latest_job_submission_excludes["job_provisioning_data"] = {
+                    "instance_type": {"resources": {"cpu_arch"}}
+                }
+            if _should_exclude_job_submission_jrd_cpu_arch(latest_job_submission):
+                latest_job_submission_excludes["job_runtime_data"] = {
+                    "offer": {"instance": {"resources": {"cpu_arch"}}}
+                }
+            if latest_job_submission.exit_status is None:
+                latest_job_submission_excludes["exit_status"] = True
+    return {"plan": apply_plan_excludes}
+
+
+def get_get_plan_excludes(request: GetRunPlanRequest) -> Optional[Dict]:
+    """
+    Excludes new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    get_plan_excludes = {}
+    run_spec_excludes = get_run_spec_excludes(request.run_spec)
+    if run_spec_excludes is not None:
+        get_plan_excludes["run_spec"] = run_spec_excludes
+    if request.max_offers is None:
+        get_plan_excludes["max_offers"] = True
+    return get_plan_excludes
+
+
+def get_run_spec_excludes(run_spec: RunSpec) -> Optional[Dict]:
+    """
+    Returns `run_spec` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    spec_excludes: dict[str, Any] = {}
+    configuration_excludes: dict[str, Any] = {}
+    profile_excludes: set[str] = set()
+    configuration = run_spec.configuration
+    profile = run_spec.profile
+
+    if configuration.fleets is None:
+        configuration_excludes["fleets"] = True
+    if profile is not None and profile.fleets is None:
+        profile_excludes.add("fleets")
+    if configuration.tags is None:
+        configuration_excludes["tags"] = True
+    if profile is not None and profile.tags is None:
+        profile_excludes.add("tags")
+    if isinstance(configuration, ServiceConfiguration) and not configuration.rate_limits:
+        configuration_excludes["rate_limits"] = True
+    if configuration.shell is None:
+        configuration_excludes["shell"] = True
+    if configuration.priority is None:
+        configuration_excludes["priority"] = True
+    if configuration.startup_order is None:
+        configuration_excludes["startup_order"] = True
+    if profile is not None and profile.startup_order is None:
+        profile_excludes.add("startup_order")
+    if configuration.stop_criteria is None:
+        configuration_excludes["stop_criteria"] = True
+    if profile is not None and profile.stop_criteria is None:
+        profile_excludes.add("stop_criteria")
+
+    if configuration_excludes:
+        spec_excludes["configuration"] = configuration_excludes
+    if profile_excludes:
+        spec_excludes["profile"] = profile_excludes
+    if spec_excludes:
+        return spec_excludes
+    return None
+
+
+def _should_exclude_job_submission_jpd_cpu_arch(job_submission: JobSubmission) -> bool:
+    try:
+        return job_submission.job_provisioning_data.instance_type.resources.cpu_arch is None
+    except AttributeError:
+        return True
+
+
+def _should_exclude_job_submission_jrd_cpu_arch(job_submission: JobSubmission) -> bool:
+    try:
+        return job_submission.job_runtime_data.offer.instance.resources.cpu_arch is None
+    except AttributeError:
+        return True

src/dstack/_internal/core/compatibility/volumes.py

@@ -0,0 +1,32 @@
+from typing import Dict
+
+from dstack._internal.core.models.volumes import VolumeConfiguration, VolumeSpec
+
+
+def get_volume_spec_excludes(volume_spec: VolumeSpec) -> Dict:
+    """
+    Returns `volume_spec` exclude mapping to exclude certain fields from the request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    spec_excludes = {}
+    spec_excludes["configuration"] = _get_volume_configuration_excludes(volume_spec.configuration)
+    return spec_excludes
+
+
+def get_create_volume_excludes(configuration: VolumeConfiguration) -> Dict:
+    """
+    Returns an exclude mapping to exclude certain fields from the create volume request.
+    Use this method to exclude new fields when they are not set to keep
+    clients backward-compatibility with older servers.
+    """
+    create_volume_excludes = {}
+    create_volume_excludes["configuration"] = _get_volume_configuration_excludes(configuration)
+    return create_volume_excludes
+
+
+def _get_volume_configuration_excludes(configuration: VolumeConfiguration) -> Dict:
+    configuration_excludes = {}
+    if configuration.tags is None:
+        configuration_excludes["tags"] = True
+    return configuration_excludes

src/dstack/api/server/_fleets.py

@@ -1,9 +1,13 @@
-from typing import Any, Dict, List, Optional, Union
+from typing import List, Union
 
 from pydantic import parse_obj_as
 
+from dstack._internal.core.compatibility.fleets import (
+    get_apply_plan_excludes,
+    get_create_fleet_excludes,
+    get_get_plan_excludes,
+)
 from dstack._internal.core.models.fleets import ApplyFleetPlanInput, Fleet, FleetPlan, FleetSpec
-from dstack._internal.core.models.instances import Instance
 from dstack._internal.server.schemas.fleets import (
     ApplyFleetPlanRequest,
     CreateFleetRequest,
@@ -34,7 +38,7 @@ def get_plan(
         spec: FleetSpec,
     ) -> FleetPlan:
         body = GetFleetPlanRequest(spec=spec)
-        body_json = body.json(exclude=_get_get_plan_excludes(spec))
+        body_json = body.json(exclude=get_get_plan_excludes(spec))
         resp = self._request(f"/api/project/{project_name}/fleets/get_plan", body=body_json)
         return parse_obj_as(FleetPlan.__response__, resp.json())
 
@@ -46,7 +50,7 @@ def apply_plan(
     ) -> Fleet:
         plan_input = ApplyFleetPlanInput.__response__.parse_obj(plan)
         body = ApplyFleetPlanRequest(plan=plan_input, force=force)
-        body_json = body.json(exclude=_get_apply_plan_excludes(plan_input))
+        body_json = body.json(exclude=get_apply_plan_excludes(plan_input))
         resp = self._request(f"/api/project/{project_name}/fleets/apply", body=body_json)
         return parse_obj_as(Fleet.__response__, resp.json())
 
@@ -66,74 +70,6 @@ def create(
         spec: FleetSpec,
     ) -> Fleet:
         body = CreateFleetRequest(spec=spec)
-        body_json = body.json(exclude=_get_create_fleet_excludes(spec))
+        body_json = body.json(exclude=get_create_fleet_excludes(spec))
         resp = self._request(f"/api/project/{project_name}/fleets/create", body=body_json)
         return parse_obj_as(Fleet.__response__, resp.json())
-
-
-def _get_get_plan_excludes(fleet_spec: FleetSpec) -> Dict:
-    get_plan_excludes = {}
-    spec_excludes = _get_fleet_spec_excludes(fleet_spec)
-    if spec_excludes:
-        get_plan_excludes["spec"] = spec_excludes
-    return get_plan_excludes
-
-
-def _get_apply_plan_excludes(plan_input: ApplyFleetPlanInput) -> Dict:
-    apply_plan_excludes = {}
-    spec_excludes = _get_fleet_spec_excludes(plan_input.spec)
-    if spec_excludes:
-        apply_plan_excludes["spec"] = spec_excludes
-    current_resource = plan_input.current_resource
-    if current_resource is not None:
-        current_resource_excludes = {}
-        apply_plan_excludes["current_resource"] = current_resource_excludes
-        if all(map(_should_exclude_instance_cpu_arch, current_resource.instances)):
-            current_resource_excludes["instances"] = {
-                "__all__": {"instance_type": {"resources": {"cpu_arch"}}}
-            }
-    return {"plan": apply_plan_excludes}
-
-
-def _should_exclude_instance_cpu_arch(instance: Instance) -> bool:
-    try:
-        return instance.instance_type.resources.cpu_arch is None
-    except AttributeError:
-        return True
-
-
-def _get_create_fleet_excludes(fleet_spec: FleetSpec) -> Dict:
-    create_fleet_excludes = {}
-    spec_excludes = _get_fleet_spec_excludes(fleet_spec)
-    if spec_excludes:
-        create_fleet_excludes["spec"] = spec_excludes
-    return create_fleet_excludes
-
-
-def _get_fleet_spec_excludes(fleet_spec: FleetSpec) -> Optional[Dict]:
-    """
-    Returns `fleet_spec` exclude mapping to exclude certain fields from the request.
-    Use this method to exclude new fields when they are not set to keep
-    clients backward-compatibility with older servers.
-    """
-    spec_excludes: Dict[str, Any] = {}
-    configuration_excludes: Dict[str, Any] = {}
-    profile_excludes: set[str] = set()
-    profile = fleet_spec.profile
-    if profile.fleets is None:
-        profile_excludes.add("fleets")
-    if fleet_spec.configuration.tags is None:
-        configuration_excludes["tags"] = True
-    if profile.tags is None:
-        profile_excludes.add("tags")
-    if profile.startup_order is None:
-        profile_excludes.add("startup_order")
-    if profile.stop_criteria is None:
-        profile_excludes.add("stop_criteria")
-    if configuration_excludes:
-        spec_excludes["configuration"] = configuration_excludes
-    if profile_excludes:
-        spec_excludes["profile"] = profile_excludes
-    if spec_excludes:
-        return spec_excludes
-    return None