feat: add upgrade model support in fileManager remove operation

- Add upgrade property to modelConfig for handling upgrade models
- Implement _removeUpgradeSingleFile method to handle upgrade file removal
- Refactor remove operation to support both regular and upgrade files
- Improve error logging order in retryFileManagerRm function
- Consolidate remove promises into a single execution list

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>

Author: 高魏洪

src/subCommands/model/fileManager.ts

@@ -190,11 +190,29 @@ export class ArtModelService {
     const { nasMountPoints, ossMountPoints, role, vpcConfig, modelConfig, region } = params;
     try {
       const devClient = await initClient(this.inputs, this.region, 'fun-art');
-      const { files } = modelConfig;
+      const { files, upgrade } = modelConfig;
       if (_.isEmpty(files)) {
         logger.info('[Remove-model] No files specified for removal.');
         return;
       }
+      let allRemovePromises = [];
+      if (!_.isEmpty(upgrade)) {
+        logger.info('[Remove-model] Upgrade model, only support once model.');
+        const removeUpgradePromises = Object.keys(upgrade).map((key) =>
+          this._removeUpgradeSingleFile(devClient, upgrade[key], key, {
+            name,
+            nasMountPoints,
+            ossMountPoints: processedOssMountPoints,
+            role,
+            vpcConfig,
+            modelConfig,
+            region,
+            timeout: modelConfig?.timeout,
+          }),
+        );
+        // 将升级文件删除任务添加到总的任务列表中
+        allRemovePromises = allRemovePromises.concat(removeUpgradePromises);
+      }
       const processedOssMountPoints = extractOssMountDir(ossMountPoints);
 
       // 将异步操作重构为并行处理
@@ -210,8 +228,10 @@ export class ArtModelService {
           timeout: modelConfig?.timeout,
         }),
       );
+      // 将删除任务也添加到总的任务列表中
+      allRemovePromises = allRemovePromises.concat(removePromises);
 
-      const removeResults = await Promise.all(removePromises);
+      const removeResults = await Promise.all(allRemovePromises);
 
       // 统计成功和失败的数量
       const successfulRemovals = removeResults.filter((result) => result.success);
@@ -261,27 +281,58 @@ export class ArtModelService {
       timeout: number;
     },
   ) {
-    const { name, nasMountPoints, ossMountPoints, role, vpcConfig, modelConfig, region, timeout } =
-      config;
+    const { nasMountPoints, ossMountPoints, modelConfig } = config;
+
+    let filepath: string;
+    const uri = file.target?.uri || modelConfig.target.uri;
+    const path = file.target?.path || '';
+    const fileName = file.source?.path || 'unknown';
+
+    // 判断uri是否为nas://auto或oss://auto
+    if (uri.startsWith('nas://auto') && nasMountPoints?.length > 0) {
+      const { mountDir } = nasMountPoints[0];
+      filepath = `${mountDir}/${path}`;
+    } else if (uri.startsWith('oss://auto') && ossMountPoints?.length > 0) {
+      const { mountDir } = ossMountPoints[0];
+      filepath = `${mountDir}/${path}`;
+    } else {
+      // 直接拼接uri和path
+      let normalizedUri = uri.endsWith('/') ? uri.slice(0, -1) : uri;
+      normalizedUri = normalizedUri.replace(/^(nas|oss|file):\/\//, '/');
+      filepath = `${normalizedUri}/${path}`;
+    }
+
+    return this._removeFileWithRetry(devClient, filepath, fileName, config);
+  }
+
+  private async _removeUpgradeSingleFile(
+    devClient: DevClient,
+    path: string,
+    version: string,
+    config: {
+      name: string;
+      nasMountPoints: any[];
+      ossMountPoints: any[];
+      role: string;
+      vpcConfig: any;
+      modelConfig: any;
+      region: string;
+      timeout: number;
+    },
+  ) {
+    const fileName = version || 'unknown';
 
+    return this._removeFileWithRetry(devClient, path, fileName, config);
+  }
+
+  private async _removeFileWithRetry(
+    devClient: DevClient,
+    filepath: string,
+    fileName: string,
+    config: any,
+  ): Promise<any> {
     try {
-      let filepath;
-      const uri = file.target?.uri || modelConfig.target.uri;
-      const path = file.target?.path || '';
-
-      // 判断uri是否为nas://auto或oss://auto
-      if (uri.startsWith('nas://auto') && nasMountPoints?.length > 0) {
-        const { mountDir } = nasMountPoints[0];
-        filepath = `${mountDir}/${path}`;
-      } else if (uri.startsWith('oss://auto') && ossMountPoints?.length > 0) {
-        const { mountDir } = ossMountPoints[0];
-        filepath = `${mountDir}/${path}`;
-      } else {
-        // 直接拼接uri和path
-        let normalizedUri = uri.endsWith('/') ? uri.slice(0, -1) : uri;
-        normalizedUri = normalizedUri.replace(/^(nas|oss|file):\/\//, '/');
-        filepath = `${normalizedUri}/${path}`;
-      }
+      const { name, nasMountPoints, ossMountPoints, role, vpcConfig, region, timeout } = config;
 
       const fileManagerRmRequest = new $Dev20230714.FileManagerRmRequest({
         filepath,
@@ -296,17 +347,10 @@ export class ArtModelService {
         }),
       });
 
-      const result = await retryFileManagerRm(
-        devClient,
-        fileManagerRmRequest,
-        file.source.path,
-        3,
-        30,
-      );
+      const result = await retryFileManagerRm(devClient, fileManagerRmRequest, fileName, 3, 30);
 
       return result;
     } catch (error) {
-      const fileName = file.source?.path || 'unknown';
       logger.error(`[Remove-model] Error removing file ${fileName}: ${error.message}`);
       logger.error(`[Remove-model] Error details:`, error.stack || error);
       return {

src/subCommands/model/index.ts

@@ -297,6 +297,7 @@ mountPoints:
         files: modelConfig.files,
         conflictResolution: modelConfig?.downloadStrategy?.conflictResolution || 'overwrite',
         mode: process.env.MODEL_DOWNLOAD_STRATEGY || modelConfig?.downloadStrategy?.mode || 'once',
+        upgrade: modelConfig?.upgrade || {},
         timeout:
           (modelConfig?.downloadStrategy?.timeout &&
             modelConfig?.downloadStrategy?.timeout * 1000) ||

src/subCommands/model/utils/index.ts

@@ -318,14 +318,14 @@ export async function retryFileManagerRm(
 
           if (isInitializeError(modelStatus.errorMessage) && attempts < maxRetries) {
             const delay = baseDelay * Math.pow(2, attempts - 1);
+            logger.error(
+              `[Remove-model] model: ${modelStatus.errorMessage}, requestId: ${getFileManagerTask.body.requestId}`,
+            );
             logger.warn(
               `[Remove-model] Detected initialization error for ${fileName}, retrying... (${
                 attempts + 1
               }/${maxRetries}). Waiting ${delay}s`,
             );
-            logger.error(
-              `[Remove-model] model: ${modelStatus.errorMessage}, requestId: ${getFileManagerTask.body.requestId}`,
-            );
 
             // eslint-disable-next-line no-await-in-loop
             await sleep(delay);