Skip to content

Commit 5612c7c

Browse files
chenghh-9609chenghh-9609
authored
add operator create page (#38)
* feat: Update site name to DataMate and refine text for AI data processing * feat: Refactor settings page and implement model access functionality - Created a new ModelAccess component for managing model configurations. - Removed the old Settings component and replaced it with a new SettingsPage component that integrates ModelAccess, SystemConfig, and WebhookConfig. - Added SystemConfig component for managing system settings. - Implemented WebhookConfig component for managing webhook configurations. - Updated API functions for model management in settings.apis.ts. - Adjusted routing to point to the new SettingsPage component. * feat: Implement Data Collection Page with Task Management and Execution Log - Created DataCollectionPage component to manage data collection tasks. - Added TaskManagement and ExecutionLog components for task handling and logging. - Integrated task operations including start, stop, edit, and delete functionalities. - Implemented filtering and searching capabilities in task management. - Introduced SimpleCronScheduler for scheduling tasks with cron expressions. - Updated CreateTask component to utilize new scheduling and template features. - Enhanced BasicInformation component to conditionally render fields based on visibility settings. - Refactored ImportConfiguration component to remove NAS import section. * feat: Update task creation API endpoint and enhance task creation form with new fields and validation * Refactor file upload and operator management components - Removed unnecessary console logs from file download and export functions. - Added size property to TaskItem interface for better task management. - Simplified TaskUpload component by utilizing useFileSliceUpload hook for file upload logic. - Enhanced OperatorPluginCreate component to handle file uploads and parsing more efficiently. - Updated ConfigureStep component to use Ant Design Form for better data handling and validation. - Improved PreviewStep component to navigate back to the operator market. - Added support for additional file types in UploadStep component. - Implemented delete operator functionality in OperatorMarketPage with confirmation prompts. - Cleaned up unused API functions in operator.api.ts to streamline the codebase. - Fixed number formatting utility to handle zero values correctly.
1 parent e0884ab commit 5612c7c

22 files changed

Lines changed: 636 additions & 975 deletions

‎frontend/src/components/CardView.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -246,10 +246,10 @@ function CardView<T extends BaseCardDataType>(props: CardViewProps<T>) {
246246
<div className="grid grid-cols-2 gap-4 py-3">
247247
{item?.statistics?.map((stat, idx) => (
248248
<div key={idx}>
249-
<div className="text-sm text-gray-500">
249+
<div className="text-sm text-gray-500 overflow-hidden whitespace-nowrap text-ellipsis w-full">
250250
{stat?.label}:
251251
</div>
252-
<div className="text-base font-semibold text-gray-900">
252+
<div className="text-base font-semibold text-gray-900 overflow-hidden whitespace-nowrap text-ellipsis w-full">
253253
{stat?.value}
254254
</div>
255255
</div>

‎frontend/src/hooks/useFetchData.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,10 @@ import { useState, useRef, useEffect, useCallback } from "react";
1616
import { useDebouncedEffect } from "./useDebouncedEffect";
1717
import Loading from "@/utils/loading";
1818
import { App } from "antd";
19-
import { AnyObject } from "antd/es/_util/type";
2019

2120
export default function useFetchData<T>(
2221
fetchFunc: (params?: any) => Promise<any>,
23-
mapDataFunc: (data: AnyObject) => T = (data) => data as T,
22+
mapDataFunc: (data: Partial<T>) => T = (data) => data as T,
2423
pollingInterval: number = 30000, // 默认30秒轮询一次
2524
autoRefresh: boolean = true,
2625
additionalPollingFuncs: (() => Promise<any>)[] = [], // 额外的轮询函数

‎frontend/src/hooks/useSliceUpload.tsx‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
import { TaskItem } from "@/pages/DataManagement/dataset.model";
2+
import { calculateSHA256, checkIsFilesExist } from "@/utils/file.util";
3+
import { App } from "antd";
4+
import { useRef, useState } from "react";
5+
6+
export function useFileSliceUpload(
7+
{
8+
preUpload,
9+
uploadChunk,
10+
cancelUpload,
11+
}: {
12+
preUpload: (id: string, params: any) => Promise<{ data: number }>;
13+
uploadChunk: (id: string, formData: FormData, config: any) => Promise<any>;
14+
cancelUpload: ((reqId: number) => Promise<any>) | null;
15+
},
16+
showTaskCenter = true // 上传时是否显示任务中心
17+
) {
18+
const { message } = App.useApp();
19+
const [taskList, setTaskList] = useState<TaskItem[]>([]);
20+
const taskListRef = useRef<TaskItem[]>([]); // 用于固定任务顺序
21+
22+
const createTask = (detail: any = {}) => {
23+
const { dataset } = detail;
24+
const title = `上传数据集: ${dataset.name} `;
25+
const controller = new AbortController();
26+
const task: TaskItem = {
27+
key: dataset.id,
28+
title,
29+
percent: 0,
30+
reqId: -1,
31+
controller,
32+
size: 0,
33+
updateEvent: detail.updateEvent,
34+
};
35+
taskListRef.current = [task, ...taskListRef.current];
36+
37+
setTaskList(taskListRef.current);
38+
return task;
39+
};
40+
41+
const updateTaskList = (task: TaskItem) => {
42+
taskListRef.current = taskListRef.current.map((item) =>
43+
item.key === task.key ? task : item
44+
);
45+
setTaskList(taskListRef.current);
46+
};
47+
48+
const removeTask = (task: TaskItem) => {
49+
const { key } = task;
50+
taskListRef.current = taskListRef.current.filter(
51+
(item) => item.key !== key
52+
);
53+
setTaskList(taskListRef.current);
54+
if (task.isCancel && task.cancelFn) {
55+
task.cancelFn();
56+
}
57+
if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent));
58+
if (showTaskCenter) {
59+
window.dispatchEvent(
60+
new CustomEvent("show:task-popover", { detail: { show: false } })
61+
);
62+
}
63+
};
64+
65+
async function buildFormData({ file, reqId, i, j }) {
66+
const formData = new FormData();
67+
const { slices, name, size } = file;
68+
const checkSum = await calculateSHA256(slices[j]);
69+
formData.append("file", slices[j]);
70+
formData.append("reqId", reqId.toString());
71+
formData.append("fileNo", (i + 1).toString());
72+
formData.append("chunkNo", (j + 1).toString());
73+
formData.append("fileName", name);
74+
formData.append("fileSize", size.toString());
75+
formData.append("totalChunkNum", slices.length.toString());
76+
formData.append("checkSumHex", checkSum);
77+
return formData;
78+
}
79+
80+
async function uploadSlice(task: TaskItem, fileInfo) {
81+
if (!task) {
82+
return;
83+
}
84+
const { reqId, key } = task;
85+
const { loaded, i, j, files, totalSize } = fileInfo;
86+
const formData = await buildFormData({
87+
file: files[i],
88+
i,
89+
j,
90+
reqId,
91+
});
92+
93+
let newTask = { ...task };
94+
await uploadChunk(key, formData, {
95+
onUploadProgress: (e) => {
96+
const loadedSize = loaded + e.loaded;
97+
const curPercent = Number((loadedSize / totalSize) * 100).toFixed(2);
98+
99+
newTask = {
100+
...newTask,
101+
...taskListRef.current.find((item) => item.key === key),
102+
size: loadedSize,
103+
percent: curPercent >= 100 ? 99.99 : curPercent,
104+
};
105+
updateTaskList(newTask);
106+
},
107+
});
108+
}
109+
110+
async function uploadFile({ task, files, totalSize }) {
111+
const { data: reqId } = await preUpload(task.key, {
112+
totalFileNum: files.length,
113+
totalSize,
114+
datasetId: task.key,
115+
});
116+
117+
const newTask: TaskItem = {
118+
...task,
119+
reqId,
120+
isCancel: false,
121+
cancelFn: () => {
122+
task.controller.abort();
123+
cancelUpload?.(reqId);
124+
if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent));
125+
},
126+
};
127+
updateTaskList(newTask);
128+
if (showTaskCenter) {
129+
window.dispatchEvent(
130+
new CustomEvent("show:task-popover", { detail: { show: true } })
131+
);
132+
}
133+
// // 更新数据状态
134+
if (task.updateEvent) window.dispatchEvent(new Event(task.updateEvent));
135+
136+
let loaded = 0;
137+
for (let i = 0; i < files.length; i++) {
138+
const { slices } = files[i];
139+
for (let j = 0; j < slices.length; j++) {
140+
await uploadSlice(newTask, {
141+
loaded,
142+
i,
143+
j,
144+
files,
145+
totalSize,
146+
});
147+
loaded += slices[j].size;
148+
}
149+
}
150+
removeTask(newTask);
151+
}
152+
153+
const handleUpload = async ({ task, files }) => {
154+
const isErrorFile = await checkIsFilesExist(files);
155+
if (isErrorFile) {
156+
message.error("文件被修改或删除，请重新选择文件上传");
157+
removeTask({
158+
...task,
159+
isCancel: false,
160+
...taskListRef.current.find((item) => item.key === task.key),
161+
});
162+
return;
163+
}
164+
165+
try {
166+
const totalSize = files.reduce((acc, file) => acc + file.size, 0);
167+
await uploadFile({ task, files, totalSize });
168+
} catch (err) {
169+
console.error(err);
170+
message.error("文件上传失败，请稍后重试");
171+
removeTask({
172+
...task,
173+
isCancel: true,
174+
...taskListRef.current.find((item) => item.key === task.key),
175+
});
176+
}
177+
};
178+
179+
return {
180+
taskList,
181+
createTask,
182+
removeTask,
183+
handleUpload,
184+
};
185+
}

‎frontend/src/mock/mock-apis.cjs‎

Lines changed: 13 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -115,20 +115,15 @@ const MockAPI = {
115115
batchEvaluationUsingPost: "/evaluation/batch-evaluate", // 批量评测
116116

117117
// 知识生成接口
118-
queryKnowledgeBasesUsingPost: "/knowledge/bases", // 获取知识库列表
119-
createKnowledgeBaseUsingPost: "/knowledge/bases/create", // 创建知识库
120-
queryKnowledgeBaseByIdUsingGet: "/knowledge/bases/:baseId", // 根据ID获取知识库详情
121-
updateKnowledgeBaseByIdUsingPut: "/knowledge/bases/:baseId", // 更新知识库
122-
deleteKnowledgeBaseByIdUsingDelete: "/knowledge/bases/:baseId", // 删除知识库
123-
queryKnowledgeGenerationTasksUsingPost: "/knowledge/tasks", // 获取知识生成任务列表
124-
createKnowledgeGenerationTaskUsingPost: "/knowledge/tasks/create", // 创建知识生成任务
125-
queryKnowledgeGenerationTaskByIdUsingGet: "/knowledge/tasks/:taskId", // 根据ID获取知识生成任务详情
126-
updateKnowledgeGenerationTaskByIdUsingPut: "/knowledge/tasks/:taskId", // 更新知识生成任务
127-
deleteKnowledgeGenerationTaskByIdUsingDelete: "/knowledge/tasks/:taskId", // 删除知识生成任务
128-
executeKnowledgeGenerationTaskByIdUsingPost:
129-
"/knowledge/tasks/:taskId/execute", // 执行知识生成任务
130-
stopKnowledgeGenerationTaskByIdUsingPost: "/knowledge/tasks/:taskId/stop", // 停止知识生成任务
131-
queryKnowledgeStatisticsUsingGet: "/knowledge/statistics", // 获取知识生成
118+
queryKnowledgeBasesUsingPost: "/knowledge-base/list", // 获取知识库列表
119+
createKnowledgeBaseUsingPost: "/knowledge-base/create", // 创建知识库
120+
queryKnowledgeBaseByIdUsingGet: "/knowledge-base/:baseId", // 根据ID获取知识库详情
121+
updateKnowledgeBaseByIdUsingPut: "/knowledge-base/:baseId", // 更新知识库
122+
deleteKnowledgeBaseByIdUsingDelete: "/knowledge-base/:baseId", // 删除知识库
123+
queryKnowledgeGenerationTasksUsingPost: "/knowledge-base/tasks", // 获取知识生成任务列表
124+
addKnowledgeGenerationFilesUsingPost: "/knowledge-base/:baseId/files", // 添加文件到知识库
125+
queryKnowledgeGenerationFilesByIdUsingGet: "/knowledge-base/:baseId/files/:fileId", // 根据ID获取知识生成文件详情
126+
deleteKnowledgeGenerationTaskByIdUsingDelete: "/knowledge-base/:baseId/files", // 删除知识生成文件
132127

133128
// 算子市场
134129
queryOperatorsUsingPost: "/operators/list", // 获取算子列表
@@ -137,6 +132,10 @@ const MockAPI = {
137132
createOperatorUsingPost: "/operators/create", // 创建算子
138133
updateOperatorByIdUsingPut: "/operators/:operatorId", // 更新算子
139134
uploadOperatorUsingPost: "/operators/upload", // 上传算子
135+
uploadFileChunkUsingPost: "/operators/upload/chunk", // 上传切片
136+
preUploadOperatorUsingPost: "/operators/upload/pre-upload", // 预上传文件
137+
cancelUploadOperatorUsingPut: "/operators/upload/cancel-upload", // 取消上传
138+
140139
createLabelUsingPost: "/operators/labels", // 创建算子标签
141140
queryLabelsUsingGet: "/labels", // 获取算子标签列表
142141
deleteLabelsUsingDelete: "/labels", // 删除算子标签
@@ -151,7 +150,6 @@ const MockAPI = {
151150
createModelUsingPost: "/models/create", // 创建模型
152151
updateModelUsingPut: "/models/:id", // 更新模型
153152
deleteModelUsingDelete: "/models/:id", // 删除模型
154-
155153
};
156154

157155
module.exports = addMockPrefix("/api", MockAPI);

0 commit comments

Comments
 (0)