Merge pull request #112 from devsapp/fix-remove

fix: model remove path

Author: rsonghuster

__tests__/e2e/model/s_file.yaml

@@ -20,6 +20,8 @@ resources:
         gpuMemorySize: 16384
         gpuType: fc.gpu.tesla.1
       nasConfig: auto
+      ossMountConfig: auto
+      role: acs:ram::${config('AccountID')}:role/aliyunfcdefaultrole
       runtime: custom-container
       description: test model download files
       cpu: 8
@@ -72,4 +74,5 @@ resources:
                 path: >-
                   function-art/comfyui/models/checkpoints/v1-5-pruned-emaonly-fp16.safetensors
               target:
+                uri: oss://auto
                 path: models/checkpoints/v1-5-pruned-emaonly-fp16.safetensors

__tests__/ut/commands/modelService_test.ts

@@ -33,6 +33,7 @@ jest.mock('../../../src/subCommands/model/utils', () => {
     checkModelStatus: mockCheckModelStatus,
     _displayProgress: jest.fn(),
     _displayProgressComplete: jest.fn(),
+    extractOssMountDir: jest.fn(),
   };
 });
 

src/subCommands/model/fileManager.ts

@@ -3,7 +3,7 @@ import logger from '../../logger';
 import DevClient, * as $Dev20230714 from '@alicloud/devs20230714';
 import { IInputs } from '../../interface';
 import _ from 'lodash';
-import { checkModelStatus, initClient } from './utils';
+import { checkModelStatus, extractOssMountDir, initClient } from './utils';
 
 export class ArtModelService {
   logger = logger;
@@ -42,6 +42,8 @@ export class ArtModelService {
       return;
     }
 
+    const processedOssMountPoints = extractOssMountDir(ossMountPoints);
+
     let existingTasks = null;
     if (modelConfig.mode === 'once') {
       // 先统一获取已有的任务列表
@@ -59,7 +61,7 @@ export class ArtModelService {
         modelConfig.source.uri,
         file,
         nasMountPoints,
-        ossMountPoints,
+        processedOssMountPoints,
         modelConfig.target.uri,
       );
 
@@ -112,7 +114,7 @@ export class ArtModelService {
       this._downloadSingleFile.bind(this, devClient, file, {
         name,
         nasMountPoints,
-        ossMountPoints,
+        ossMountPoints: processedOssMountPoints,
         role,
         region,
         vpcConfig,
@@ -188,13 +190,14 @@ export class ArtModelService {
         logger.info('[Remove-model] No files specified for removal.');
         return;
       }
+      const processedOssMountPoints = extractOssMountDir(ossMountPoints);
 
       // 将异步操作重构为并行处理
       const removePromises = files.map((file) =>
         this._removeSingleFile(devClient, file, {
           name,
           nasMountPoints,
-          ossMountPoints,
+          ossMountPoints: processedOssMountPoints,
           role,
           vpcConfig,
           modelConfig,

src/subCommands/model/model.ts

@@ -2,8 +2,7 @@
 import logger from '../../logger';
 import * as $Dev20230714 from '@alicloud/devs20230714';
 import { IInputs } from '../../interface';
-import { checkModelStatus, initClient } from './utils';
-import { isEmpty } from 'lodash';
+import { checkModelStatus, extractOssMountDir, initClient } from './utils';
 
 export class ModelService {
   logger = logger;
@@ -41,15 +40,14 @@ export class ModelService {
       );
       return;
     }
+
+    const processedOssMountPoints = extractOssMountDir(ossMountPoints);
+
     // mode 是 once 时候，判断是否已经下载过
     const destination =
       storage === 'nas'
         ? `file:/${nasMountPoints[0].mountDir}`
-        : `file:/${
-            ossMountPoints[0].mountDir.length > 48
-              ? ossMountPoints[0].mountDir.substring(0, 48)
-              : ossMountPoints[0].mountDir
-          }`;
+        : `file:/${processedOssMountPoints[0].mountDir}`;
     if (modelConfig.mode === 'once') {
       const ListFileManagerTasksRequest = new $Dev20230714.ListFileManagerTasksRequest({
         name,
@@ -71,17 +69,6 @@ export class ModelService {
       }
     }
 
-    let processedOssMountPoints;
-    if (!isEmpty(ossMountPoints)) {
-      processedOssMountPoints = ossMountPoints.map((ossMountPoint) => ({
-        ...ossMountPoint,
-        mountDir:
-          ossMountPoint.mountDir.length > 48
-            ? ossMountPoint.mountDir.substring(0, 48)
-            : ossMountPoint.mountDir,
-      }));
-    }
-
     const fileManagerRsyncRequest = new $Dev20230714.FileManagerRsyncRequest({
       mountConfig: new $Dev20230714.FileManagerMountConfig({
         name,
@@ -112,19 +99,11 @@ export class ModelService {
     const { nasMountPoints, ossMountPoints, role, region, vpcConfig, storage } = params;
     const devClient = await initClient(this.inputs, this.region, logger, 'fun-model');
 
-    let processedOssMountPoints;
-    if (!isEmpty(ossMountPoints)) {
-      processedOssMountPoints = ossMountPoints.map((ossMountPoint) => ({
-        ...ossMountPoint,
-        mountDir:
-          ossMountPoint.mountDir.length > 48
-            ? ossMountPoint.mountDir.substring(0, 48)
-            : ossMountPoint.mountDir,
-      }));
-    }
+    const processedOssMountPoints = extractOssMountDir(ossMountPoints);
 
     const fileManagerRmRequest = new $Dev20230714.FileManagerRmRequest({
-      filepath: storage === 'nas' ? nasMountPoints[0]?.mountDir : ossMountPoints[0]?.mountDir,
+      filepath:
+        storage === 'nas' ? nasMountPoints[0]?.mountDir : processedOssMountPoints[0]?.mountDir,
       mountConfig: new $Dev20230714.FileManagerMountConfig({
         name,
         nasMountPoints,

src/subCommands/model/utils/index.ts

@@ -6,6 +6,7 @@ import {
 import * as $OpenApi from '@alicloud/openapi-client';
 import DevClient from '@alicloud/devs20230714';
 import { sleep } from '../../../utils';
+import { isEmpty } from 'lodash';
 
 export const _getEndpoint = (region): string => {
   if (process.env.ARTIFACT_ENDPOINT) {
@@ -147,3 +148,17 @@ export async function checkModelStatus(
     await sleep(sleepTime);
   }
 }
+
+export function extractOssMountDir(ossMountPoints) {
+  let processedOssMountPoints;
+  if (!isEmpty(ossMountPoints)) {
+    processedOssMountPoints = ossMountPoints.map((ossMountPoint) => ({
+      ...ossMountPoint,
+      mountDir:
+        ossMountPoint.mountDir.length > 48
+          ? ossMountPoint.mountDir.substring(0, 48)
+          : ossMountPoint.mountDir,
+    }));
+  }
+  return processedOssMountPoints;
+}