feat: support in-place LoRA updates through sidecar

Author: biswapanda

lib/backend-common/src/engine.rs

@@ -260,6 +260,13 @@ pub trait LLMEngine: Send + Sync + 'static {
         Err(unsupported_lora())
     }
 
+    /// Replace one LoRA while preserving its public name and engine ID.
+    /// The default preserves compatibility for engines that only implement
+    /// the original load operation.
+    async fn load_lora_inplace(&self, adapter: LoraAdapter) -> Result<LoraAdapter, DynamoError> {
+        self.load_lora(adapter).await
+    }
+
     /// Unload one logical LoRA adapter by name.
     async fn unload_lora(&self, _name: &str) -> Result<LoraAdapter, DynamoError> {
         Err(unsupported_lora())

lib/backend-common/src/lora.rs

@@ -188,6 +188,12 @@ impl LoraController {
             .and_then(Value::as_str)
             .filter(|uri| !uri.trim().is_empty())
             .ok_or_else(|| "'source.uri' is required in request".to_string())?;
+        let load_inplace = match request.get("load_inplace") {
+            None => false,
+            Some(value) => value
+                .as_bool()
+                .ok_or_else(|| "'load_inplace' must be a boolean".to_string())?,
+        };
         validate_adapter_name(&self.reserved_names, &name)?;
 
         let operation = self.operation_lock(&name);
@@ -214,7 +220,8 @@ impl LoraController {
             name: name.clone(),
             path: path.to_string_lossy().into_owned(),
         };
-        if let Some(existing) = self.loaded.lock().await.get(&name).cloned() {
+        let existing = { self.loaded.lock().await.get(&name).cloned() };
+        if let Some(existing) = existing {
             if existing.adapter == adapter {
                 if !existing.published {
                     self.discovery
@@ -230,9 +237,41 @@ impl LoraController {
                 }
                 return Ok(success(&existing.adapter, "already loaded"));
             }
-            return Err(format!(
-                "LoRA adapter '{name}' conflicts with the loaded source path"
-            ));
+            if !load_inplace {
+                return Err(format!(
+                    "LoRA adapter '{name}' conflicts with the loaded source path"
+                ));
+            }
+            let adapter = self
+                .engine
+                .load_lora_inplace(adapter)
+                .await
+                .map_err(|error| error.to_string())?;
+            let published = if existing.published {
+                true
+            } else {
+                match self.discovery.attach(&adapter).await {
+                    Ok(()) => true,
+                    Err(error) => {
+                        self.loaded.lock().await.insert(
+                            name,
+                            ManagedLora {
+                                adapter,
+                                published: false,
+                            },
+                        );
+                        return Err(format!("failed to register replaced LoRA model: {error}"));
+                    }
+                }
+            };
+            self.loaded.lock().await.insert(
+                name,
+                ManagedLora {
+                    adapter: adapter.clone(),
+                    published,
+                },
+            );
+            return Ok(success(&adapter, "replaced successfully"));
         }
 
         let adapter = self
@@ -488,6 +527,13 @@ mod tests {
             Ok(adapter)
         }
 
+        async fn load_lora_inplace(
+            &self,
+            adapter: LoraAdapter,
+        ) -> Result<LoraAdapter, DynamoError> {
+            self.load_lora(adapter).await
+        }
+
         async fn unload_lora(&self, name: &str) -> Result<LoraAdapter, DynamoError> {
             self.events.lock().await.push(format!("unload:{name}"));
             if self.fail_unload.load(Ordering::SeqCst) {
@@ -664,6 +710,42 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn load_inplace_replaces_a_stable_name_with_a_new_source() {
+        let root = tempfile::tempdir().unwrap();
+        let first = root.path().join("step-1");
+        let second = root.path().join("step-2");
+        std::fs::create_dir(&first).unwrap();
+        std::fs::create_dir(&second).unwrap();
+        let engine = MockEngine::new(Vec::new());
+        let discovery = Arc::new(MockDiscovery::default());
+        let controller = LoraController::new_with(
+            engine.clone(),
+            discovery.clone(),
+            Some(downloader(&root.path().join("cache"))),
+            HashSet::new(),
+        )
+        .await
+        .unwrap();
+
+        for (path, load_inplace) in [(&first, false), (&second, true)] {
+            controller
+                .load(json!({
+                    "lora_name": "adapter-a",
+                    "source": { "uri": format!("file://{}", path.display()) },
+                    "load_inplace": load_inplace,
+                }))
+                .await
+                .unwrap();
+        }
+
+        assert_eq!(
+            engine.loaded.lock().await["adapter-a"].path,
+            std::fs::canonicalize(second).unwrap().to_string_lossy()
+        );
+        assert_eq!(discovery.attached.lock().await.as_slice(), ["adapter-a"]);
+    }
+
     #[tokio::test]
     async fn load_rejects_base_model_name_before_engine_mutation() {
         let dir = tempfile::tempdir().unwrap();

lib/backend-common/src/rl.rs

@@ -44,9 +44,11 @@ pub(crate) fn serve_endpoint(
     }
     let endpoint_name = resolve_endpoint_name(&primary.id().name)?;
     let endpoint = primary.component().endpoint(endpoint_name);
+    let system_url = self_host_base_url(primary.drt())?;
     let handler = Arc::new(RlRouteHandler {
         routes: primary.drt().engine_routes().clone(),
         metadata,
+        system_url,
     });
     let ingress = Ingress::for_engine(handler.clone())?;
     let future = endpoint
@@ -61,6 +63,20 @@ pub(crate) fn serve_endpoint(
     })
 }
 
+fn self_host_base_url(drt: &dynamo_runtime::DistributedRuntime) -> anyhow::Result<Option<String>> {
+    let Some(info) = drt.system_status_server_info() else {
+        return Ok(None);
+    };
+    let configured = dynamo_runtime::RuntimeConfig::from_settings()
+        .unwrap_or_default()
+        .system_host;
+    let host = match configured.as_str() {
+        "0.0.0.0" | "::" | "[::]" => dynamo_runtime::utils::local_ip_for_advertise(),
+        _ => configured,
+    };
+    Ok(Some(format!("http://{host}:{}", info.port())))
+}
+
 fn resolve_endpoint_name(primary_name: &str) -> anyhow::Result<String> {
     let endpoint_name =
         std::env::var("DYN_RL_ENDPOINT").unwrap_or_else(|_| DEFAULT_RL_ENDPOINT.into());
@@ -82,6 +98,7 @@ fn validate_endpoint_name(endpoint_name: &str, primary_name: &str) -> anyhow::Re
 struct RlRouteHandler {
     routes: EngineRouteRegistry,
     metadata: RlWorkerMetadata,
+    system_url: Option<String>,
 }
 
 impl RlRouteHandler {
@@ -104,17 +121,20 @@ impl RlRouteHandler {
             });
         }
 
-        let mut routes = self
-            .routes
-            .routes()
-            .into_iter()
-            .filter(|route| !route.contains('/'))
-            .collect::<Vec<_>>();
+        let mut routes = self.routes.routes().into_iter().collect::<Vec<_>>();
+        routes.retain(|route| {
+            !route.contains('/')
+                || matches!(
+                    route.as_str(),
+                    "update/load_lora" | "update/unload_lora" | "update/list_loras"
+                )
+        });
         routes.sort();
         routes.dedup();
         json!({
             "status": "ok",
             "routes": routes,
+            "system_url": self.system_url,
             "admin_base_url": self.metadata.admin_base_url,
             "world_size": self.metadata.world_size,
         })
@@ -138,12 +158,18 @@ mod tests {
     use super::*;
 
     fn handler() -> RlRouteHandler {
+        let routes = EngineRouteRegistry::new();
+        routes.register(
+            "update/load_lora",
+            Arc::new(|_| Box::pin(async { Ok(json!({"status": "ok"})) })),
+        );
         RlRouteHandler {
-            routes: EngineRouteRegistry::new(),
+            routes,
             metadata: RlWorkerMetadata {
                 admin_base_url: "http://worker:8120".to_string(),
                 world_size: 2,
             },
+            system_url: Some("http://worker:8181".to_string()),
         }
     }
 
@@ -153,7 +179,8 @@ mod tests {
             handler().dispatch(&json!({"method": "routes"})),
             json!({
                 "status": "ok",
-                "routes": [],
+                "routes": ["update/load_lora"],
+                "system_url": "http://worker:8181",
                 "admin_base_url": "http://worker:8120",
                 "world_size": 2,
             })

lib/runtime/src/system_status_server.rs

@@ -104,6 +104,8 @@ impl SystemStatusState {
 pub struct LoadLoraRequest {
     pub lora_name: String,
     pub source: LoraSource,
+    #[serde(default)]
+    pub load_inplace: bool,
 }
 
 /// Source information for loading a LoRA
@@ -373,6 +375,7 @@ async fn load_lora_handler(
             "source": {
                 "uri": request.source.uri
             },
+            "load_inplace": request.load_inplace,
         }),
     )
     .await

lib/vllm-sidecar/proto/vllm_grpc.proto

@@ -308,7 +308,10 @@ message LoraAdapter {
   string source_path = 3;
 }
 
-message LoadLoraRequest { LoraAdapter adapter = 1; }
+message LoadLoraRequest {
+  LoraAdapter adapter = 1;
+  bool load_inplace = 2;
+}
 message LoadLoraResponse {
   LoraAdapter adapter = 1;
   bool already_loaded = 2;

lib/vllm-sidecar/src/engine.rs

@@ -120,6 +120,40 @@ impl VllmSidecarEngine {
         }
     }
 
+    async fn load_lora_with_mode(
+        &self,
+        adapter: LoraAdapter,
+        load_inplace: bool,
+    ) -> Result<LoraAdapter, DynamoError> {
+        let mut client = self
+            .pool
+            .get()
+            .map(Pool::control_client)
+            .ok_or_else(|| client::engine_shutdown("load_lora called before start"))?;
+        let requested = adapter.clone();
+        let response = client
+            .load_lora(pb::LoadLoraRequest {
+                adapter: Some(pb::LoraAdapter {
+                    lora_id: adapter.id,
+                    lora_name: adapter.name,
+                    source_path: adapter.path,
+                }),
+                load_inplace,
+            })
+            .await
+            .map_err(|status| client::status_to_dynamo("LoadLora", status))?
+            .into_inner()
+            .adapter
+            .ok_or_else(|| client::engine_shutdown("LoadLora returned no adapter"))?;
+        let loaded = lora_from_proto(response)?;
+        if loaded != requested {
+            return Err(client::engine_shutdown(
+                "LoadLora returned an adapter identity different from the request",
+            ));
+        }
+        Ok(loaded)
+    }
+
     /// Parse CLI args, bootstrap-discover the engine role + model, and build the
     /// pair `run()` consumes.
     pub fn from_args(argv: Option<Vec<String>>) -> Result<(Self, WorkerConfig), DynamoError> {
@@ -448,32 +482,11 @@ impl LLMEngine for VllmSidecarEngine {
     }
 
     async fn load_lora(&self, adapter: LoraAdapter) -> Result<LoraAdapter, DynamoError> {
-        let mut client = self
-            .pool
-            .get()
-            .map(Pool::control_client)
-            .ok_or_else(|| client::engine_shutdown("load_lora called before start"))?;
-        let requested = adapter.clone();
-        let response = client
-            .load_lora(pb::LoadLoraRequest {
-                adapter: Some(pb::LoraAdapter {
-                    lora_id: adapter.id,
-                    lora_name: adapter.name,
-                    source_path: adapter.path,
-                }),
-            })
-            .await
-            .map_err(|status| client::status_to_dynamo("LoadLora", status))?
-            .into_inner()
-            .adapter
-            .ok_or_else(|| client::engine_shutdown("LoadLora returned no adapter"))?;
-        let loaded = lora_from_proto(response)?;
-        if loaded != requested {
-            return Err(client::engine_shutdown(
-                "LoadLora returned an adapter identity different from the request",
-            ));
-        }
-        Ok(loaded)
+        self.load_lora_with_mode(adapter, false).await
+    }
+
+    async fn load_lora_inplace(&self, adapter: LoraAdapter) -> Result<LoraAdapter, DynamoError> {
+        self.load_lora_with_mode(adapter, true).await
     }
 
     async fn unload_lora(&self, name: &str) -> Result<LoraAdapter, DynamoError> {