refactor(model):重构模型下载逻辑，支持多种解决方案并优化资源自动部署

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>

Author: 高魏洪

src/subCommands/model/fileManager.ts

@@ -0,0 +1,290 @@
+// 文生图服务下载，删除逻辑
+import logger from '../../logger';
+import DevClient, * as $Dev20230714 from '@alicloud/devs20230714';
+import * as $OpenApi from '@alicloud/openapi-client';
+import { MODEL_DOWNLOAD_TIMEOUT, NEW_MODEL_SERVICE_CLIENT_CONNECT_TIMEOUT, NEW_MODEL_SERVICE_CLIENT_READ_TIMEOUT } from '.';
+import { IInputs } from '../../interface';
+import { sleep } from '../../utils';
+import _ from 'lodash';
+
+export class ArtModelService {
+  logger = logger;
+  client: DevClient;
+  constructor(private inputs: IInputs) {
+  }
+
+  async initClient() {
+    const {
+      AccessKeyID: accessKeyId,
+      AccessKeySecret: accessKeySecret,
+      SecurityToken: securityToken,
+    } = await this.inputs.getCredential();
+
+    let endpoint: string;
+
+    endpoint = 'devs-pre.cn-hangzhou.aliyuncs.com';
+    if (process.env.ARTIFACT_ENDPOINT) {
+      endpoint = process.env.ARTIFACT_ENDPOINT;
+    }
+    if (process.env.artifact_endpoint) {
+      endpoint = process.env.artifact_endpoint;
+    }
+
+    const protocol = 'https';
+
+    const config = new $OpenApi.Config({
+      accessKeyId,
+      accessKeySecret,
+      securityToken,
+      protocol,
+      endpoint,
+      readTimeout: NEW_MODEL_SERVICE_CLIENT_READ_TIMEOUT,
+      connectTimeout: NEW_MODEL_SERVICE_CLIENT_CONNECT_TIMEOUT,
+      userAgent: `${this.inputs.userAgent ||
+        `Component:cap-model;Nodejs:${process.version};OS:${process.platform}-${process.arch}`
+        }`,
+    });
+
+    logger.info(`new models service init, ARTIFACT_ENDPOINT endpoint: ${config.endpoint}`);
+
+    return new DevClient(config);
+  }
+
+  async downloadModel(name, params) {
+    const devClient = await this.initClient();
+    const {
+      nasMountPoints,
+      ossMountPoints,
+      role,
+      modelConfig,
+      vpcConfig,
+      storage,
+      region
+    } = params;
+    const files = modelConfig.files;
+    if (!_.isEmpty(files)) {
+      // 先统一获取已有的任务列表
+      const ListFileManagerTasksRequest = new $Dev20230714.ListFileManagerTasksRequest({ name });
+      const res = await devClient.listFileManagerTasks(ListFileManagerTasksRequest);
+      logger.debug('listFileManagerTasks', JSON.stringify(res, null, 2));
+      const existingTasks = res.body.data.tasks;
+
+      // 第一步：筛选真正需要下载的文件
+      const filesNeedPromises = files.map(async file => {
+        const { source, destination } = this.getSource(storage, modelConfig.source.uri, file, nasMountPoints, ossMountPoints, region);
+
+        const needDownload = !existingTasks.some(
+          task =>
+            task.finished &&
+            task.success &&
+            task.progress.currentBytes === task.progress.totalBytes &&
+            task.parameters.destination === destination &&
+            task.parameters.source === source
+        );
+
+        if (!needDownload) {
+          logger.info(`[Download-model] ${file.source.path} Download model finished.`);
+          return null;
+        }
+
+        return {
+          ...file,
+          source,
+          fileName: file.target.path,
+          destination
+        };
+      });
+
+      const filesNeedResults = await Promise.all(filesNeedPromises);
+      const filesNeed = filesNeedResults.filter(Boolean);
+
+      // 添加调试日志
+      logger.info(`[Download-model] Total files to check: ${files.length}`);
+      logger.info(`[Download-model] Files need to download: ${filesNeed.length}`);
+
+      const filesApi = filesNeed.map(async (file) => {
+        const { source, destination } = file;
+        const fileManagerRsyncRequest = new $Dev20230714.FileManagerRsyncRequest({
+          mountConfig: new $Dev20230714.FileManagerMountConfig({
+            name,
+            nasMountPoints,
+            ossMountPoints,
+            role,
+            region,
+            vpcConfig
+          }),
+          source,
+          destination,
+          conflictHandling: process.env.MODEL_CONFLIC_HANDLING || modelConfig.conflictResolution
+        })
+        logger.debug('FileManagerRsyncRequest', JSON.stringify(fileManagerRsyncRequest, null, 2))
+        const req = await devClient.fileManagerRsync(fileManagerRsyncRequest)
+        logger.debug(`[Download-model] fileManagerRsync response for ${file.fileName}: ${JSON.stringify(req.body, null, 2)}`)
+        if (!req.body.success) {
+          throw new Error(`fileManagerRsync error: ${JSON.stringify(req.body, null, 2)}`)
+        }
+
+        const taskID = req.body.data.taskID;
+        logger.info(`[Download-model] download model requestId for ${file.fileName}: ${req.body.requestId}, taskID: ${taskID}`);
+        const shouldContinue = true;
+        while (shouldContinue) {
+          // eslint-disable-next-line no-await-in-loop
+          const getFileManager = await devClient.getFileManagerTask(taskID)
+          logger.debug('getFileManagerTask', JSON.stringify(getFileManager, null, 2));
+          const modelStatus = getFileManager.body.data;
+          const totalBytes = modelStatus.progress.totalBytes as any - 0;
+          const currentBytes = modelStatus.progress.currentBytes as any - 0;
+
+          if (modelStatus.finished) {
+            // 如果存在错误信息，则抛出异常
+            if (modelStatus.errorMessage) {
+              logger.error(`[Download-model] ${file.fileName}: ${modelStatus.errorMessage}`);
+              throw new Error(`[Download-model] ${file.fileName}: ${modelStatus.errorMessage}`);
+            }
+            // 下载成功完成
+            this._displayProgressComplete(file.fileName, currentBytes, totalBytes);
+            if (modelStatus.progress.total) {
+              const durationMs = modelStatus.finishedTime - modelStatus.startTime;
+              const durationSeconds = Math.floor(durationMs / 1000);
+              logger.info(`Time taken for model download: ${durationSeconds}s.`);
+            }
+            logger.info(`[Download-model] ${file.fileName} Download model finished.`);
+            return true;
+          }
+          // 显示下载进度
+          this._displayProgress(file.fileName, currentBytes, totalBytes);
+
+          if (Date.now() - modelStatus.startTime > MODEL_DOWNLOAD_TIMEOUT) {
+            // 清除进度条并换行
+            process.stdout.write('\n');
+            const errorMessage = `[Model-download] ${file.fileName} Download timeout after ${MODEL_DOWNLOAD_TIMEOUT / 1000 / 60
+              } minutes`;
+            throw new Error(errorMessage);
+          }
+
+          // 根据文件大小调整轮询间隔
+          let sleepTime = 2; // 默认2秒
+          if (totalBytes !== undefined && totalBytes > 1024 * 1024 * 1024) {
+            // 文件大于1GB时，轮询间隔为10秒
+            sleepTime = 10;
+          }
+
+          await sleep(sleepTime);
+        }
+      })
+      await Promise.all(filesApi)
+    }
+  }
+
+  async removeModel(name, params) {
+    const {
+      nasMountPoints,
+      ossMountPoints,
+      role,
+      vpcConfig,
+      modelConfig,
+    } = params;
+    const devClient = await this.initClient();
+    const files = modelConfig.files;
+
+    const filesRm = files.map(async file => {
+      let filepath = this._getDestinationPath(file.target.path, nasMountPoints, ossMountPoints)
+
+      const fileManagerRmRequest = new $Dev20230714.FileManagerRmRequest({
+        filepath,
+        mountConfignew: new $Dev20230714.FileManagerMountConfig({
+          name,
+          nasMountPoints,
+          ossMountPoints,
+          role,
+          vpcConfig
+        }),
+        skipParentDirectory: true
+      })
+      await devClient.fileManagerRm(fileManagerRmRequest);
+    })
+
+    await Promise.all(filesRm);
+  }
+
+  getSource(storage, uri, file, nasMountPoints, ossMountPoints, region) {
+    let source, destination;
+    const validSourcePattern = /^(modelscope|oss|nas):\/\//;
+    // 处理源路径
+    source = this._getSourcePath(file.source.path, uri, validSourcePattern);
+    // 处理目标路径
+    destination = this._getDestinationPath(file.target.path, nasMountPoints, ossMountPoints);
+
+    return {
+      source,
+      destination
+    }
+  }
+
+  private _displayProgressComplete(filePath: string, currentBytes: number, totalBytes: number) {
+    if (totalBytes && currentBytes !== undefined) {
+      const currentMB = (currentBytes / 1024 / 1024).toFixed(1);
+      const totalMB = (totalBytes / 1024 / 1024).toFixed(1);
+
+      const totalBars = 50;
+      const progressBar = '='.repeat(totalBars);
+
+      process.stdout.write(
+        `\r[Download-model] ${filePath} [${progressBar}] 100.00% (${currentMB}MB/${totalMB}MB)\n`,
+      );
+    } else {
+      process.stdout.write('\n');
+    }
+    // 清除进度条并换行
+    process.stdout.write('\n');
+  }
+
+  private _displayProgress(filePath: string, currentBytes: number, totalBytes: number) {
+    if (currentBytes && totalBytes) {
+      const percentage = (currentBytes / totalBytes) * 100;
+      const currentMB = (currentBytes / 1024 / 1024).toFixed(1);
+      const totalMB = (totalBytes / 1024 / 1024).toFixed(1);
+
+      // 每个等号代表2%，向下取整计算等号数量
+      const totalBars = 50; // 总共50个字符位置
+      const filledBars = Math.min(totalBars, Math.floor(percentage / 2)); // 每个等号代表2%
+      const emptyBars = totalBars - filledBars;
+
+      const progressBar = '='.repeat(filledBars) + '.'.repeat(emptyBars);
+
+      process.stdout.write(
+        `\r[Download-model] ${filePath} [${progressBar}] ${percentage.toFixed(
+          2,
+        )}% (${currentMB}MB/${totalMB}MB)`,
+      );
+    }
+  }
+
+  private _getSourcePath(filePath: string, uri: string, validSourcePattern: RegExp): string {
+    if (validSourcePattern.test(filePath)) {
+      return filePath;
+    } else if (validSourcePattern.test(uri)) {
+      const downloadUri = uri.endsWith('/') && uri.length > 1 ? uri.slice(0, -1) : uri;
+      return `${downloadUri}/${filePath}`;
+    } else {
+      throw new Error(`Invalid source path. Expected a valid URI starting with 'modelscope://', 'oss://', or 'nas://', but got: ${filePath}`);
+    }
+  }
+
+  private _getDestinationPath(
+    targetPath: string,
+    nasMountPoints: any[],
+    ossMountPoints: any[],
+  ): string {
+    const nasProtocolPattern = /^(nas):\/\//;
+    const ossProtocolPattern = /^(oss):\/\//;
+    // 通过 targetPath 是nas://或者oss://前缀来判断使用哪个路径
+    if (nasProtocolPattern.test(targetPath)) {
+      const pathWithoutProtocol = targetPath.replace(nasProtocolPattern, '');
+      return pathWithoutProtocol.startsWith('/') ? `file://${nasMountPoints[0].mountDir}${pathWithoutProtocol}` : `file://${pathWithoutProtocol}`;
+    } else if (ossProtocolPattern.test(targetPath)) {
+      const pathWithoutProtocol = targetPath.replace(ossProtocolPattern, '');
+      return pathWithoutProtocol.startsWith('/') ? `file://${ossMountPoints[0].mountDir}${pathWithoutProtocol}` : `file://${pathWithoutProtocol}`;
+    }
+  }
+}