[Feature]: Nebius switch to using nebius iam auth-public-key generate #2934

Author: peterschmidt85

docs/docs/concepts/backends.md

@@ -452,14 +452,43 @@ There are two ways to configure GCP: using a service account or using the defaul
     - name: main
       backends:
         - type: gcp
-          project_id: gcp-project-id
+          project_id: my-gcp-project
           creds:
             type: service_account
             filename: ~/.dstack/server/gcp-024ed630eab5.json
     ```
 
     </div>
 
+    ??? info "User interface"
+        If you are configuring the `gcp` backend on the [project settigns page](projects.md#backends), 
+        specify the contents of the JSON file in `data`:
+
+        <div editor-title="~/.dstack/server/config.yml">
+
+        ```yaml
+        type: gcp
+        project_id: my-gcp-project
+        creds:
+          type: service_account
+          data: |
+            {
+              "type": "service_account",
+              "project_id": "my-gcp-project",
+              "private_key_id": "abcd1234efgh5678ijkl9012mnop3456qrst7890",
+              "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEv...rest_of_key...IDAQAB\n-----END PRIVATE KEY-----\n",
+              "client_email": "my-service-account@my-gcp-project.iam.gserviceaccount.com",
+              "client_id": "123456789012345678901",
+              "auth_uri": "https://accounts.google.com/o/oauth2/auth",
+              "token_uri": "https://oauth2.googleapis.com/token",
+              "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
+              "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/my-service-account%40my-gcp-project.iam.gserviceaccount.com",
+              "universe_domain": "googleapis.com"
+            }
+        ```
+
+        </div>
+
 If you don't know your GCP project ID, use [Google Cloud CLI :material-arrow-top-right-thin:{ .external }](https://cloud.google.com/sdk/docs/install-sdk):
 
 ```shell
@@ -638,8 +667,33 @@ projects:
 
 </div>
 
-??? info "Configuring in the UI"
-    If you are configuring the backend in the `dstack` UI, specify the contents of the private key file in `private_key_content`.
+??? info "Credentials file"
+    It's also possible to configure the `nebius` backend using a credentials file [generated :material-arrow-top-right-thin:{ .external }](https://docs.nebius.com/iam/service-accounts/authorized-keys#create){:target="_blank"} by the `nebius` CLI:
+
+    <div class="termy">
+
+    ```shell
+    $ nebius iam auth-public-key generate \
+        --service-account-id <service account ID> \
+        --output ~/.nebius/sa-credentials.json
+    ```
+    
+    </div>
+
+  
+    ```yaml
+    projects:
+    - name: main
+      backends:
+      - type: nebius
+        creds:
+          type: service_account
+          filename: ~/.nebius/sa-credentials.json
+    ```
+
+??? info "User interface"
+    If you are configuring the `nebius` backend on the [project settigns page](projects.md#backends), 
+    specify the contents of the private key file in `private_key_content`:
 
     <div editor-title="~/.dstack/server/config.yml">
 

src/dstack/_internal/core/backends/nebius/models.py

@@ -1,5 +1,10 @@
+import json
+from pathlib import Path
 from typing import Annotated, Literal, Optional, Union
 
+from nebius.base.service_account.credentials_file import (
+    ServiceAccountCredentials,
+)
 from pydantic import Field, root_validator
 
 from dstack._internal.core.backends.base.models import fill_data
@@ -27,30 +32,70 @@ class NebiusServiceAccountCreds(CoreModel):
             )
         ),
     ]
+    filename: Annotated[
+        Optional[str], Field(description="The path to the service account credentials file")
+    ] = None
 
 
 class NebiusServiceAccountFileCreds(CoreModel):
     type: Annotated[Literal["service_account"], Field(description="The type of credentials")] = (
         "service_account"
     )
-    service_account_id: Annotated[str, Field(description="Service account ID")]
-    public_key_id: Annotated[str, Field(description="ID of the service account public key")]
+    service_account_id: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "Service account ID. Set automatically if `filename` is specified. When configuring via the UI, it must be specified explicitly"
+            )
+        ),
+    ] = None
+    public_key_id: Annotated[
+        Optional[str],
+        Field(
+            description=(
+                "ID of the service account public key. Set automatically if `filename` is specified. When configuring via the UI, it must be specified explicitly"
+            )
+        ),
+    ] = None
     private_key_file: Annotated[
-        Optional[str], Field(description=("Path to the service account private key"))
+        Optional[str],
+        Field(
+            description=(
+                "Path to the service account private key. Set automatically if `filename` or `private_key_content` is specified. When configuring via the UI, it must be specified explicitly"
+            )
+        ),
     ] = None
     private_key_content: Annotated[
         Optional[str],
         Field(
             description=(
                 "Content of the service account private key. When configuring via"
                 " `server/config.yml`, it's automatically filled from `private_key_file`."
-                " When configuring via UI, it has to be specified explicitly."
+                " When configuring via UI, it has to be specified explicitly"
             )
         ),
     ] = None
+    filename: Annotated[
+        Optional[str], Field(description="The path to the service account credentials file")
+    ] = None
 
     @root_validator
     def fill_data(cls, values):
+        if filename := values.get("filename"):
+            try:
+                with open(Path(filename).expanduser()) as f:
+                    data = json.load(f)
+                credentials = ServiceAccountCredentials.from_json(data)
+                subject = credentials.subject_credentials
+                values["service_account_id"] = subject.sub
+                values["public_key_id"] = subject.kid
+                values["private_key_content"] = subject.private_key
+            except OSError:
+                raise ValueError(f"No such file {filename}")
+            except Exception as e:
+                raise ValueError(f"Failed to parse credentials file {filename}: {e}")
+            return values
+
         return fill_data(
             values, filename_field="private_key_file", data_field="private_key_content"
         )

src/tests/_internal/server/services/test_backend_configs.py

@@ -0,0 +1,105 @@
+import json
+import sys
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+import yaml
+
+from dstack._internal.server import settings
+from dstack._internal.server.services.config import (
+    ServerConfigManager,
+    file_config_to_config,
+)
+
+
+@pytest.mark.skipif(sys.version_info < (3, 10), reason="Nebius requires Python 3.10")
+class TestNebiusBackendConfig:
+    def test_with_filename(self, tmp_path: Path):
+        creds_json = {
+            "subject-credentials": {
+                "type": "JWT",
+                "alg": "RS256",
+                "private-key": "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----\n",
+                "kid": "publickey-e00test",
+                "iss": "serviceaccount-e00test",
+                "sub": "serviceaccount-e00test",
+            }
+        }
+        creds_file = tmp_path / "nebius_creds.json"
+        creds_file.write_text(json.dumps(creds_json))
+
+        config_yaml_path = tmp_path / "config.yml"
+        config_dict = {
+            "projects": [
+                {
+                    "name": "main",
+                    "backends": [
+                        {
+                            "type": "nebius",
+                            "creds": {"type": "service_account", "filename": str(creds_file)},
+                        }
+                    ],
+                }
+            ]
+        }
+        config_yaml_path.write_text(yaml.dump(config_dict))
+
+        with patch.object(settings, "SERVER_CONFIG_FILE_PATH", config_yaml_path):
+            m = ServerConfigManager()
+            assert m.load_config()
+            assert m.config is not None
+            assert m.config.projects is not None
+            assert len(m.config.projects) > 0
+            assert m.config.projects[0].backends is not None
+            backend_file_cfg = m.config.projects[0].backends[0]
+            backend_cfg = file_config_to_config(backend_file_cfg)
+
+        assert backend_cfg.type == "nebius"
+        assert backend_cfg.creds.service_account_id == "serviceaccount-e00test"
+        assert backend_cfg.creds.public_key_id == "publickey-e00test"
+        assert (
+            backend_cfg.creds.private_key_content
+            == "-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----\n"
+        )
+
+    def test_with_private_key_file(self, tmp_path: Path):
+        pk_file = tmp_path / "private.key"
+        pk_file.write_text("TEST_PRIVATE_KEY")
+
+        config_yaml_path = tmp_path / "config.yml"
+        config_dict = {
+            "projects": [
+                {
+                    "name": "main",
+                    "backends": [
+                        {
+                            "type": "nebius",
+                            "projects": ["project-e00test"],
+                            "creds": {
+                                "type": "service_account",
+                                "service_account_id": "serviceaccount-e00test",
+                                "public_key_id": "publickey-e00test",
+                                "private_key_file": str(pk_file),
+                            },
+                        }
+                    ],
+                }
+            ]
+        }
+        config_yaml_path.write_text(yaml.dump(config_dict))
+
+        with patch.object(settings, "SERVER_CONFIG_FILE_PATH", config_yaml_path):
+            m = ServerConfigManager()
+            assert m.load_config()
+            assert m.config is not None
+            assert m.config.projects is not None
+            assert len(m.config.projects) > 0
+            assert m.config.projects[0].backends is not None
+            backend_file_cfg = m.config.projects[0].backends[0]
+            backend_cfg = file_config_to_config(backend_file_cfg)
+
+        assert backend_cfg.type == "nebius"
+        assert backend_cfg.creds.service_account_id == "serviceaccount-e00test"
+        assert backend_cfg.creds.public_key_id == "publickey-e00test"
+        assert backend_cfg.creds.private_key_content == "TEST_PRIVATE_KEY"