feat: Support explicit model control mode in OpenAI frontend

python/openai/README.md

@@ -542,6 +542,125 @@ available arguments and default values.
 For more information on the `tritonfrontend` python bindings, see the docs
 [here](https://github.com/triton-inference-server/server/blob/main/docs/customization_guide/tritonfrontend.md).
 
+## Model Management
+
+The OpenAI-compatible frontend supports explicit model control, allowing you to
+dynamically load and unload models at runtime without restarting the server.
+This is particularly useful when hosting multiple large models on a shared GPU
+cluster and needing to swap models on demand.
+
+### Model Control Mode
+
+Use `--model-control-mode` to specify how models are managed at startup and
+runtime. The default is `none`.
+
+| Mode | Behavior |
+|------|----------|
+| `none` (default) | All models in the repository are loaded at startup. Load/unload APIs are not available. |
+| `explicit` | No models are loaded at startup unless specified with `--load-model`. Load and unload are controlled via the management API. |
+
+> [!NOTE]
+> This matches the native `tritonserver --model-control-mode` behavior.
+> See [Triton Model Management](../../docs/user_guide/model_management.md) for
+> more details.
+
+### Load Models at Startup (Explicit Mode)
+
+When using `--model-control-mode=explicit`, use `--load-model` to specify which
+models should be loaded at startup. It may be specified multiple times to load
+multiple models.
+
+<details>
+<summary>Example</summary>
+
+```bash
+# Start in explicit mode with no models loaded
+python3 openai_frontend/main.py \
+  --model-repository /path/to/models \
+  --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \
+  --model-control-mode explicit
+
+# Start in explicit mode and load a specific model at startup
+python3 openai_frontend/main.py \
+  --model-repository /path/to/models \
+  --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \
+  --model-control-mode explicit \
+  --load-model llama-3.1-8b-instruct
+
+# Load multiple models at startup
+python3 openai_frontend/main.py \
+  --model-repository /path/to/models \
+  --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \
+  --model-control-mode explicit \
+  --load-model model-a \
+  --load-model model-b
+
+# Load ALL models in the repository at startup
+python3 openai_frontend/main.py \
+  --model-repository /path/to/models \
+  --tokenizer meta-llama/Meta-Llama-3.1-8B-Instruct \
+  --model-control-mode explicit \
+  --load-model '*'
+```
+
+</details>
+
+> [!IMPORTANT]
+> - `--load-model` requires `--model-control-mode=explicit`.
+> - `--load-model=*` can not be used together with loading specific models.
+
+### Dynamic Load / Unload API
+
+Once the server is running in `explicit` mode, models can be loaded and unloaded
+at runtime via the following endpoints:
+
+| Method | Endpoint | Description |
+|--------|----------|-------------|
+| `POST` | `/v1/models/{model_name}/load` | Load a model. Blocks until model is fully loaded and ready. |
+| `POST` | `/v1/models/{model_name}/unload` | Unload a model. Blocks until fully unloaded. In-flight requests complete before removal. |
+
+Both endpoints return an error if `--model-control-mode` is not `explicit`.
+
+#### Load a model
+
+```bash
+MODEL="llama-3.1-8b-instruct"
+curl -s -X POST http://localhost:9000/v1/models/${MODEL}/load | jq
+```
+
+<details>
+<summary>Example output</summary>
+
+```json
+{
+  "id": "llama-3.1-8b-instruct",
+  "object": "model",
+  "created": 1750000000,
+  "owned_by": "Triton Inference Server"
+}
+```
+
+</details>
+
+#### Unload a model
+
+```bash
+MODEL="llama-3.1-8b-instruct"
+curl -s -X POST http://localhost:9000/v1/models/${MODEL}/unload | jq
+```
+
+<details>
+<summary>Example output</summary>
+
+```json
+{
+  "status": "success",
+  "model": "llama-3.1-8b-instruct"
+}
+```
+
+</details>
+
 ## Model Parallelism Support
 
 - [x] vLLM ([EngineArgs](https://github.com/triton-inference-server/vllm_backend/blob/main/README.md#using-the-vllm-backend))
@@ -794,9 +913,11 @@ Use the `--openai-restricted-api` command-line argument to configure endpoint re
     - `POST /v1/completions`
   - **embedding**: Embedding endpoint
     - `POST /v1/embeddings`
-  - **model-repository**: Model listing and information endpoints
+  - **model-repository**: Model listing, information, and dynamic load/unload endpoints
     - `GET /v1/models`
     - `GET /v1/models/{model_name}`
+    - `POST /v1/models/{model_name}/load`
+    - `POST /v1/models/{model_name}/unload`
   - **metrics**: Server metrics endpoint
     - `GET /metrics`
   - **health**: Health check endpoint

python/openai/openai_frontend/engine/engine.py

@@ -1,4 +1,4 @@
-# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -100,3 +100,19 @@ def embedding(self, request: CreateEmbeddingRequest) -> CreateEmbeddingResponse:
         Returns a CreateEmbeddingResponse.
         """
         pass
+
+    async def load_model(self, model_name: str) -> Model:
+        """
+        Loads a model by name. Only available in EXPLICIT model control mode.
+        Blocks until the model is fully loaded and ready, matching standard
+        Triton server load behavior.
+        """
+        pass
+
+    async def unload_model(self, model_name: str) -> None:
+        """
+        Unloads a model by name. Only available in EXPLICIT model control mode.
+        Blocks until the model is fully unloaded, matching standard Triton
+        server unload behavior. In-flight requests complete before unload.
+        """
+        pass

python/openai/openai_frontend/engine/triton_engine.py

@@ -27,6 +27,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import base64
 import json
 import time
@@ -137,10 +138,8 @@ def __init__(
         self.lora_separator = lora_separator
         self.default_max_tokens = default_max_tokens
 
-        # NOTE: Creation time and model metadata will be static at startup for
-        # now, and won't account for dynamically loading/unloading models.
-        self.create_time = int(time.time())
         self.model_metadata = self._get_model_metadata()
+        self._metadata_lock = asyncio.Lock()
         self.tool_call_parser = (
             ToolParserManager.get_tool_parser_cls(tool_call_parser)
             if tool_call_parser
@@ -493,53 +492,114 @@ def _get_tokenizer(self, tokenizer_name: str):
 
         return tokenizer
 
+    def _build_model_metadata(self, name: str) -> TritonModelMetadata:
+        model = self.server.model(name)
+        backend = model.config()["backend"]
+        if not backend and model.config()["platform"] == "ensemble":
+            backend = "ensemble"
+        print(f"Found model: {name=}, {backend=}")
+
+        lora_configs = _parse_lora_configs(
+            self.server.options.model_repository,
+            name,
+            model.version,
+            backend if self.backend is None else self.backend,
+        )
+
+        echo_tensor_name = None
+        for input in model.config()["input"]:
+            if input["name"] in [
+                "exclude_input_in_output",
+                "sampling_param_exclude_input_from_output",
+            ]:
+                echo_tensor_name = input["name"]
+                break
+
+        return TritonModelMetadata(
+            name=name,
+            backend=backend,
+            model=model,
+            tokenizer=self.tokenizer,
+            lora_configs=lora_configs,
+            echo_tensor_name=echo_tensor_name,
+            create_time=int(time.time()),
+            inference_request_converter=self._determine_request_converter(
+                backend, RequestKind.GENERATION
+            ),
+            embedding_request_converter=self._determine_request_converter(
+                backend, RequestKind.EMBEDDING
+            ),
+        )
+
     def _get_model_metadata(self) -> Dict[str, TritonModelMetadata]:
-        # One tokenizer and creation time shared for all loaded models for now.
+        # One tokenizer is shared for all loaded models; creation time is per model.
         model_metadata = {}
+        for name, _ in self.server.models(exclude_not_ready=True).keys():
+            model_metadata[name] = self._build_model_metadata(name)
+        return model_metadata
 
-        # Read all triton models and store the necessary metadata for each
-        for name, _ in self.server.models().keys():
-            model = self.server.model(name)
-            backend = model.config()["backend"]
-            # Explicitly handle ensembles to avoid any runtime validation errors
-            if not backend and model.config()["platform"] == "ensemble":
-                backend = "ensemble"
-            print(f"Found model: {name=}, {backend=}")
-
-            lora_configs = _parse_lora_configs(
-                self.server.options.model_repository,
-                name,
-                model.version,
-                backend if self.backend is None else self.backend,
+    async def load_model(self, model_name: str) -> Model:
+        if (
+            self.server.options.model_control_mode
+            != tritonserver.ModelControlMode.EXPLICIT
+        ):
+            raise ClientError(
+                "Model load/unload requires --model-control-mode=explicit"
             )
 
-            echo_tensor_name = None
-            for input in model.config()["input"]:
-                if input["name"] in [
-                    "exclude_input_in_output",
-                    "sampling_param_exclude_input_from_output",
-                ]:
-                    echo_tensor_name = input["name"]
-                    break
-
-            metadata = TritonModelMetadata(
-                name=name,
-                backend=backend,
-                model=model,
-                tokenizer=self.tokenizer,
-                lora_configs=lora_configs,
-                echo_tensor_name=echo_tensor_name,
-                create_time=self.create_time,
-                inference_request_converter=self._determine_request_converter(
-                    backend, RequestKind.GENERATION
-                ),
-                embedding_request_converter=self._determine_request_converter(
-                    backend, RequestKind.EMBEDDING
-                ),
+        async with self._metadata_lock:
+            if model_name in self.model_metadata:
+                raise ClientError(f"Model '{model_name}' is already loaded")
+
+            # Blocking C API call dispatched to thread pool to avoid blocking
+            # the event loop. The C API blocks until model is fully loaded and
+            # ready, matching standard Triton server behavior.
+            try:
+                metadata = await asyncio.to_thread(self._load_model_sync, model_name)
+            except tritonserver.InvalidArgumentError as e:
+                raise ClientError(f"Failed to load model '{model_name}': {e}")
+            except tritonserver.TritonError as e:
+                raise ServerError(f"Failed to load model '{model_name}': {e}")
+
+            self.model_metadata[model_name] = metadata
+
+        return Model(
+            id=model_name,
+            created=metadata.create_time,
+            object=ObjectType.model,
+            owned_by="Triton Inference Server",
+        )
+
+    def _load_model_sync(self, model_name: str) -> TritonModelMetadata:
+        self.server.load(model_name)
+        return self._build_model_metadata(model_name)
+
+    async def unload_model(self, model_name: str) -> None:
+        if (
+            self.server.options.model_control_mode
+            != tritonserver.ModelControlMode.EXPLICIT
+        ):
+            raise ClientError(
+                "Model load/unload requires --model-control-mode=explicit"
             )
-            model_metadata[name] = metadata
 
-        return model_metadata
+        async with self._metadata_lock:
+            if model_name not in self.model_metadata:
+                raise ClientError(f"Unknown model: {model_name}")
+
+            # Blocking C API call dispatched to thread pool. The C API handles
+            # in-flight request draining and conflict resolution internally.
+            try:
+                await asyncio.to_thread(self._unload_model_sync, model_name)
+            except tritonserver.InvalidArgumentError as e:
+                raise ClientError(f"Failed to unload model '{model_name}': {e}")
+            except tritonserver.TritonError as e:
+                raise ServerError(f"Failed to unload model '{model_name}': {e}")
+
+            del self.model_metadata[model_name]
+
+    def _unload_model_sync(self, model_name: str) -> None:
+        self.server.unload(model_name)
 
     def _get_streaming_chat_response_chunk(
         self,

python/openai/openai_frontend/frontend/fastapi/middleware/api_restriction.py

@@ -1,4 +1,4 @@
-# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions
@@ -36,7 +36,10 @@
         "POST /v1/completions",
         "POST /v1/embeddings",
     ],
-    "model-repository": ["GET /v1/models"],
+    "model-repository": [
+        "GET /v1/models",
+        "POST /v1/models/",
+    ],
     "metrics": ["GET /metrics"],
     "health": ["GET /health/ready"],
 }