Default Docker registry and credentials

Add the following server environment variables for
configuring the default Docker registry and the
default Docker registry credentials on the
`dstack` server level:

- `DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY` – A
  default Docker registry to use for job images
  that do not specify an explicit registry. E.g.,
  if set to `registry.example`, then
  `image: ubuntu` becomes equivalent to
  `image: registry.example/ubuntu`. **Note**: This
  setting should only be used for configuring
  registries that act as a pull-through cache for
  Docker Hub. The default `dstack` images are also
  pulled from the configured registry.
- `DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_USERNAME`{
  #DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_USERNAME
  } – Username for authenticating with the default
  Docker registry. See
  `DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_PASSWORD`.
- `DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_PASSWORD`{
  #DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_PASSWORD
  } – Password for authenticating with the default
  Docker registry. Applied only when the image has
  no explicit registry and the run configuration
  does not specify `registry_auth`. **Note**: The
  value may be visible to anyone who can SSH into
  instances managed by `dstack`, which usually
  includes all users of that `dstack` server.

Author: jvstme

docs/docs/reference/environment-variables.md

@@ -139,6 +139,9 @@ For more details on the options below, refer to the [server deployment](../guide
 - `DSTACK_SERVER_INSTANCE_HEALTH_TTL_SECONDS`{ #DSTACK_SERVER_INSTANCE_HEALTH_TTL_SECONDS } – Maximum age of instance health checks.
 - `DSTACK_SERVER_INSTANCE_HEALTH_MIN_COLLECT_INTERVAL_SECONDS`{ #DSTACK_SERVER_INSTANCE_HEALTH_MIN_COLLECT_INTERVAL_SECONDS } – Minimum time interval between consecutive health checks of the same instance.
 - `DSTACK_SERVER_EVENTS_TTL_SECONDS`{ #DSTACK_SERVER_EVENTS_TTL_SECONDS } - Maximum age of event records. Set to `0` to disable event storage. Defaults to 30 days.
+- `DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY`{ #DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY } – A default Docker registry to use for job images that do not specify an explicit registry. E.g., if set to `registry.example`, then `image: ubuntu` becomes equivalent to `image: registry.example/ubuntu`. **Note**: This setting should only be used for configuring registries that act as a pull-through cache for Docker Hub. The default `dstack` images are also pulled from the configured registry.
+- `DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_USERNAME`{ #DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_USERNAME } – Username for authenticating with the default Docker registry. See `DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_PASSWORD`.
+- `DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_PASSWORD`{ #DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_PASSWORD } – Password for authenticating with the default Docker registry. Applied only when the image has no explicit registry and the run configuration does not specify `registry_auth`. **Note**: The value may be visible to anyone who can SSH into instances managed by `dstack`, which usually includes all users of that `dstack` server.
 
 ??? info "Internal environment variables"
      The following environment variables are intended for development purposes:

src/dstack/_internal/server/background/pipeline_tasks/jobs_running.py

@@ -67,7 +67,7 @@
 from dstack._internal.server.services.backends.provisioning import (
     get_instance_specific_gpu_devices,
     get_instance_specific_mounts,
-    resolve_provisioning_image_name,
+    resolve_provisioning_image,
 )
 from dstack._internal.server.services.gateways import get_or_add_gateway_connection
 from dstack._internal.server.services.instances import (
@@ -1126,14 +1126,16 @@ def _process_provisioning_with_shim(
     ssh_user: Optional[str],
     ssh_key: Optional[str],
 ) -> bool:
-    job_spec = JobSpec.__response__.parse_raw(job_model.job_spec_data)
+    job_spec = get_job_spec(job_model)
     shim_client = client.ShimClient(port=ports[DSTACK_SHIM_HTTP_PORT])
 
     resp = shim_client.healthcheck()
     if resp is None:
         logger.debug("%s: shim is not available yet", fmt(job_model))
         return False
 
+    image_name, registry_auth = resolve_provisioning_image(job_spec.image_name, registry_auth, jpd)
+
     registry_username = ""
     registry_password = ""
     if registry_auth is not None:
@@ -1167,7 +1169,6 @@ def _process_provisioning_with_shim(
         cpu = None
         memory = None
         network_mode = NetworkMode.HOST
-    image_name = resolve_provisioning_image_name(job_spec, jpd)
     if shim_client.is_api_v2_supported():
         shim_client.submit_task(
             task_id=job_model.id,

src/dstack/_internal/server/background/pipeline_tasks/jobs_submitted.py

@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import uuid
 from contextlib import AsyncExitStack
 from dataclasses import dataclass
@@ -85,6 +86,7 @@
 )
 from dstack._internal.server.services import events
 from dstack._internal.server.services.backends import get_project_backend_by_type_or_error
+from dstack._internal.server.services.docker import apply_server_docker_defaults
 from dstack._internal.server.services.fleets import (
     check_can_create_new_cloud_instance_in_fleet,
     generate_fleet_name,
@@ -1863,6 +1865,11 @@ async def _provision_new_capacity(
     volumes: Optional[list[list[Volume]]] = None,
     fleet_model: Optional[FleetModel] = None,
 ) -> Union[_FailedNewCapacityProvisioning, _ProvisionNewCapacityResult]:
+    jobs = copy.deepcopy(jobs)
+    for job in jobs:
+        job.job_spec.image_name, job.job_spec.registry_auth = apply_server_docker_defaults(
+            job.job_spec.image_name, job.job_spec.registry_auth
+        )
     job = jobs[0]
     if volumes is None:
         volumes = []

src/dstack/_internal/server/background/scheduled_tasks/running_jobs.py

@@ -57,7 +57,7 @@
 from dstack._internal.server.services.backends.provisioning import (
     get_instance_specific_gpu_devices,
     get_instance_specific_mounts,
-    resolve_provisioning_image_name,
+    resolve_provisioning_image,
 )
 from dstack._internal.server.services.instances import (
     get_instance_remote_connection_info,
@@ -750,6 +750,8 @@ def _process_provisioning_with_shim(
         logger.debug("%s: shim is not available yet", fmt(job_model))
         return False  # shim is not available yet
 
+    image_name, registry_auth = resolve_provisioning_image(job_spec.image_name, registry_auth, jpd)
+
     registry_username = ""
     registry_password = ""
     if registry_auth is not None:
@@ -790,7 +792,6 @@ def _process_provisioning_with_shim(
         cpu = None
         memory = None
         network_mode = NetworkMode.HOST
-    image_name = resolve_provisioning_image_name(job_spec, jpd)
     if shim_client.is_api_v2_supported():
         shim_client.submit_task(
             task_id=job_model.id,

src/dstack/_internal/server/background/scheduled_tasks/submitted_jobs.py

@@ -1,4 +1,5 @@
 import asyncio
+import copy
 import itertools
 import uuid
 from contextlib import AsyncExitStack
@@ -81,6 +82,7 @@
 )
 from dstack._internal.server.services import events
 from dstack._internal.server.services.backends import get_project_backend_by_type_or_error
+from dstack._internal.server.services.docker import apply_server_docker_defaults
 from dstack._internal.server.services.fleets import (
     check_can_create_new_cloud_instance_in_fleet,
     generate_fleet_name,
@@ -1169,6 +1171,11 @@ async def _provision_new_capacity(
     and run only the master job in case there are no offers supporting cluster groups.
     Other jobs should be provisioned one-by-one later.
     """
+    jobs = copy.deepcopy(jobs)
+    for job in jobs:
+        job.job_spec.image_name, job.job_spec.registry_auth = apply_server_docker_defaults(
+            job.job_spec.image_name, job.job_spec.registry_auth
+        )
     job = jobs[0]
     if volumes is None:
         volumes = []

src/dstack/_internal/server/services/backends/provisioning.py

@@ -1,10 +1,13 @@
 import re
+from typing import Optional
 
 from dstack._internal import settings
 from dstack._internal.core.models.backends.base import BackendType
-from dstack._internal.core.models.runs import JobProvisioningData, JobSpec
+from dstack._internal.core.models.common import RegistryAuth
+from dstack._internal.core.models.runs import JobProvisioningData
 from dstack._internal.core.models.volumes import InstanceMountPoint
 from dstack._internal.server.schemas.runner import GPUDevice
+from dstack._internal.server.services.docker import apply_server_docker_defaults, parse_image_name
 
 _AWS_EFA_ENABLED_INSTANCE_TYPE_PATTERNS = [
     # TODO: p6-b200 isn't supported yet in gpuhunt
@@ -87,17 +90,18 @@ def get_instance_specific_gpu_devices(
     return gpu_devices
 
 
-def resolve_provisioning_image_name(
-    job_spec: JobSpec,
+def resolve_provisioning_image(
+    image_name: str,
+    registry_auth: Optional[RegistryAuth],
     job_provisioning_data: JobProvisioningData,
-) -> str:
-    image_name = job_spec.image_name
+) -> tuple[str, Optional[RegistryAuth]]:
+    image_name, registry_auth = apply_server_docker_defaults(image_name, registry_auth)
     if job_provisioning_data.backend == BackendType.AWS:
-        return _patch_base_image_for_aws_efa(
+        image_name = _patch_base_image_for_aws_efa(
             image_name,
             job_provisioning_data.instance_type.name,
         )
-    return image_name
+    return image_name, registry_auth
 
 
 def _patch_base_image_for_aws_efa(
@@ -111,7 +115,7 @@ def _patch_base_image_for_aws_efa(
     if not is_efa_enabled:
         return image_name
 
-    if not image_name.startswith(f"{settings.DSTACK_BASE_IMAGE}:"):
+    if parse_image_name(image_name).repo != settings.DSTACK_BASE_IMAGE:
         return image_name
 
     if image_name.endswith(f"-base-ubuntu{settings.DSTACK_BASE_IMAGE_UBUNTU_VERSION}"):

src/dstack/_internal/server/services/docker.py

@@ -14,6 +14,7 @@
     FrozenCoreModel,
     RegistryAuth,
 )
+from dstack._internal.server import settings as server_settings
 from dstack._internal.server.utils.common import join_byte_stream_checked
 
 DEFAULT_PLATFORM = "linux/amd64"
@@ -151,6 +152,26 @@ def is_host(s: str) -> bool:
     return s == "localhost" or ":" in s or "." in s
 
 
+def apply_server_docker_defaults(
+    image_name: str,
+    registry_auth: Optional[RegistryAuth],
+) -> tuple[str, Optional[RegistryAuth]]:
+    if parse_image_name(image_name).registry is not None:
+        return image_name, registry_auth
+    if server_settings.SERVER_DEFAULT_DOCKER_REGISTRY is not None:
+        image_name = f"{server_settings.SERVER_DEFAULT_DOCKER_REGISTRY}/{image_name}"
+    if (
+        registry_auth is None
+        and server_settings.SERVER_DEFAULT_DOCKER_REGISTRY_USERNAME is not None
+        and server_settings.SERVER_DEFAULT_DOCKER_REGISTRY_PASSWORD is not None
+    ):
+        registry_auth = RegistryAuth(
+            username=server_settings.SERVER_DEFAULT_DOCKER_REGISTRY_USERNAME,
+            password=server_settings.SERVER_DEFAULT_DOCKER_REGISTRY_PASSWORD,
+        )
+    return image_name, registry_auth
+
+
 DOCKER_TARGET_PATH_PATTERN = re.compile(r"^(/[^/\0]*)+/?$")
 
 

src/dstack/_internal/server/services/jobs/configurators/base.py

@@ -48,7 +48,11 @@
 from dstack._internal.core.models.volumes import MountPoint, VolumeMountPoint
 from dstack._internal.core.services.profiles import get_retry
 from dstack._internal.core.services.ssh.ports import filter_reserved_ports
-from dstack._internal.server.services.docker import ImageConfig, get_image_config
+from dstack._internal.server.services.docker import (
+    ImageConfig,
+    apply_server_docker_defaults,
+    get_image_config,
+)
 from dstack._internal.utils import crypto
 from dstack._internal.utils.common import run_async
 from dstack._internal.utils.interpolator import InterpolatorError, VariablesInterpolator
@@ -77,7 +81,7 @@ def get_default_python_verison() -> str:
 def get_default_image(nvcc: bool = False) -> str:
     """
     Note: May be overridden by dstack (e.g., EFA-enabled version for AWS EFA-capable instances).
-    See `dstack._internal.server.services.backends.provisioning.resolve_provisioning_image_name`
+    See `dstack._internal.server.services.backends.provisioning.resolve_provisioning_image`
     for details.
 
     Args:
@@ -140,9 +144,10 @@ async def _get_image_config(self) -> ImageConfig:
                 )
             except InterpolatorError as e:
                 raise ServerClientError(e.args[0])
+        image_name, registry_auth = apply_server_docker_defaults(self._image_name(), registry_auth)
         image_config = await run_async(
             _get_image_config,
-            self._image_name(),
+            image_name,
             registry_auth,
         )
         self._image_config = image_config

src/dstack/_internal/server/settings.py

@@ -137,6 +137,14 @@
     os.getenv("DSTACK_DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE", 64 * 1024 * 1024)
 )
 
+SERVER_DEFAULT_DOCKER_REGISTRY = os.getenv("DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY") or None
+SERVER_DEFAULT_DOCKER_REGISTRY_USERNAME = (
+    os.getenv("DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_USERNAME") or None
+)
+SERVER_DEFAULT_DOCKER_REGISTRY_PASSWORD = (
+    os.getenv("DSTACK_SERVER_DEFAULT_DOCKER_REGISTRY_PASSWORD") or None
+)
+
 USER_PROJECT_DEFAULT_QUOTA = int(os.getenv("DSTACK_USER_PROJECT_DEFAULT_QUOTA", 10))
 FORBID_SERVICES_WITHOUT_GATEWAY = os.getenv("DSTACK_FORBID_SERVICES_WITHOUT_GATEWAY") is not None
 

src/tests/_internal/server/background/pipeline_tasks/test_running_jobs.py

@@ -20,6 +20,7 @@
     DevEnvironmentConfiguration,
     ProbeConfig,
     ServiceConfiguration,
+    TaskConfiguration,
 )
 from dstack._internal.core.models.gateways import GatewayStatus
 from dstack._internal.core.models.instances import InstanceStatus
@@ -2023,3 +2024,53 @@ async def invalidate_lock(*args, **kwargs):
             .all()
         )
         assert probes == []
+
+    async def test_provisioning_shim_uses_server_default_registry(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        test_db,
+        session: AsyncSession,
+        worker: JobRunningWorker,
+        ssh_tunnel_mock: Mock,
+        shim_client_mock: Mock,
+    ):
+        monkeypatch.setattr(server_settings, "SERVER_DEFAULT_DOCKER_REGISTRY", "registry.example")
+        monkeypatch.setattr(
+            server_settings, "SERVER_DEFAULT_DOCKER_REGISTRY_USERNAME", "server-user"
+        )
+        monkeypatch.setattr(
+            server_settings, "SERVER_DEFAULT_DOCKER_REGISTRY_PASSWORD", "server-pass"
+        )
+        project = await create_project(session=session)
+        user = await create_user(session=session)
+        repo = await create_repo(session=session, project_id=project.id)
+        run_spec = get_run_spec(
+            repo_id=repo.name,
+            configuration=TaskConfiguration(image="ubuntu"),
+        )
+        run = await create_run(
+            session=session,
+            project=project,
+            repo=repo,
+            user=user,
+            run_spec=run_spec,
+        )
+        instance = await create_instance(
+            session=session, project=project, status=InstanceStatus.BUSY
+        )
+        job = await create_job(
+            session=session,
+            run=run,
+            status=JobStatus.PROVISIONING,
+            job_provisioning_data=get_job_provisioning_data(dockerized=True),
+            instance=instance,
+            instance_assigned=True,
+        )
+
+        await _process_job(session, worker, job)
+
+        shim_client_mock.submit_task.assert_called_once()
+        call_kwargs = shim_client_mock.submit_task.call_args[1]
+        assert call_kwargs["image_name"] == "registry.example/ubuntu"
+        assert call_kwargs["registry_username"] == "server-user"
+        assert call_kwargs["registry_password"] == "server-pass"