fix: fix old plugin not exit normally

internal/core/control_panel/launcher_local.go

@@ -149,17 +149,17 @@ func (c *ControlPanel) LaunchLocalPlugin(
 			})
 		},
 		OnRuntimeCloseImpl: func() {
+			// delete the runtime from the map
+			// this must happen AFTER all instances are confirmed shutdown
+			// to prevent process leaks where the runtime disappears from the map
+			// but subprocesses are still alive and no safety net can find them
+			c.localPluginRuntimes.Delete(pluginUniqueIdentifier)
 			// notify the plugin totally stopped
 			c.WalkNotifiers(func(notifier ControlPanelNotifier) {
 				notifier.OnLocalRuntimeStopped(runtime)
 			})
 		},
 		OnRuntimeStopScheduleImpl: func() {
-			// delete the runtime from the map
-			// Even if the runtime is not ready, deleting it still makes sense
-			// once a plugin is stopping schedule, all new requests to it need to be rejected
-			// so just remove it from map
-			c.localPluginRuntimes.Delete(pluginUniqueIdentifier)
 			// notify the plugin is stopping
 			c.WalkNotifiers(func(notifier ControlPanelNotifier) {
 				notifier.OnLocalRuntimeStop(runtime)

internal/core/plugin_manager/manager.go

@@ -53,6 +53,12 @@ type PluginManager struct {
 	controlPanel *controlpanel.ControlPanel
 }
 
+type PluginShutdownManager interface {
+	RemoveLocalPlugin(pluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier) error
+	ShutdownLocalPluginGracefully(pluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier) (<-chan error, error)
+	ShutdownLocalPluginForcefully(pluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier) (<-chan error, error)
+}
+
 var (
 	manager *PluginManager
 )

internal/core/plugin_manager/runtime.go

@@ -49,3 +49,9 @@ func (p *PluginManager) ShutdownLocalPluginGracefully(
 ) (<-chan error, error) {
 	return p.controlPanel.ShutdownLocalPluginGracefully(pluginUniqueIdentifier)
 }
+
+func (p *PluginManager) ShutdownLocalPluginForcefully(
+	pluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier,
+) (<-chan error, error) {
+	return p.controlPanel.ShutdownLocalPluginForcefully(pluginUniqueIdentifier)
+}

internal/tasks/install_plugin_utils.go

@@ -155,23 +155,65 @@ func DeleteTask(taskId string) error {
 	})
 }
 
+// gracefulShutdownTimeout controls how long RemovePluginIfNeeded waits for
+// graceful and forceful shutdown to complete. Package-level variable for test override.
+var gracefulShutdownTimeout = 30 * time.Second
+
 func RemovePluginIfNeeded(
-	manager *plugin_manager.PluginManager,
+	manager plugin_manager.PluginShutdownManager,
 	originalPluginUniqueIdentifier plugin_entities.PluginUniqueIdentifier,
 	response *curd.UpgradePluginResponse,
 ) error {
 	shouldCleanup := response.IsOriginalPluginDeleted
 
 	if shouldCleanup && response.DeletedPlugin != nil && response.DeletedPlugin.InstallType == plugin_entities.PLUGIN_RUNTIME_TYPE_LOCAL {
-		// uninstall plugin from local install bucket
+		// uninstall plugin from local install bucket first
+		// this must happen before shutdown so that the WatchDog safety net
+		// (removeUnusedLocalPlugins) can retry cleanup if shutdown fails
 		if err := manager.RemoveLocalPlugin(originalPluginUniqueIdentifier); err != nil {
 			return errors.Join(err, errors.New("failed to remove plugin from local install bucket"))
 		}
 
-		// shutdown it gracefully
-		_, err := manager.ShutdownLocalPluginGracefully(originalPluginUniqueIdentifier)
-		if err != nil {
-			return errors.Join(err, errors.New("failed to shutdown plugin gracefully"))
+		shutdownGracefully := func() bool {
+			ch, err := manager.ShutdownLocalPluginGracefully(originalPluginUniqueIdentifier)
+			if err != nil {
+				// runtime not found in map, it may have been already cleaned up
+				return false
+			}
+			if ch == nil {
+				return true
+			}
+
+			// wait for graceful shutdown with a timeout
+			select {
+			case <-ch:
+				return true
+			case <-time.After(gracefulShutdownTimeout):
+				log.Warn("graceful shutdown timed out, trying forceful shutdown",
+					"plugin", originalPluginUniqueIdentifier.String())
+				return false
+			}
+		}
+
+		if shutdownGracefully() {
+			return nil
+		}
+
+		// graceful shutdown failed or timed out, try forceful as fallback
+		forceCh, forceErr := manager.ShutdownLocalPluginForcefully(originalPluginUniqueIdentifier)
+		if forceErr != nil {
+			return errors.Join(
+				forceErr,
+				errors.New("failed to shutdown plugin forcefully after graceful shutdown failed"),
+			)
+		}
+		if forceCh != nil {
+			select {
+			case <-forceCh:
+				// forceful shutdown completed
+			case <-time.After(gracefulShutdownTimeout):
+				return errors.New("forceful shutdown timed out")
+			}
 		}
 	}
 	return nil