feature: 支持算子上传创建

Author: hhhhsc701

Makefile

@@ -12,7 +12,7 @@ build: backend-docker-build frontend-docker-build runtime-docker-build
 
 .PHONY: create-namespace
 create-namespace:
-	@kubectl get namespace $(NAMESPACE) > /dev/null 2>&1 || kubectl create namespace $(NAMESPACE)
+	kubectl get namespace $(NAMESPACE) > /dev/null 2>&1 || kubectl create namespace $(NAMESPACE)
 
 .PHONY: install-%
 install-%:
@@ -73,37 +73,61 @@ runtime-docker-build:
 label-studio-adapter-docker-build:
 	docker build -t label-studio-adapter:$(VERSION) . -f scripts/images/label-studio-adapter/Dockerfile
 
+.PHONY: deer-flow-docker-build
+deer-flow-docker-build:
+	@if [ -d "../deer-flow/.git" ]; then \
+		cd ../deer-flow && git pull; \
+	else \
+		git clone git@github.com:bytedance/deer-flow.git ../deer-flow; \
+	fi
+	sed -i "s/dark/light/g" "../deer-flow/web/src/components/deer-flow/theme-provider-wrapper.tsx"
+	cp -n deployment/docker/deer-flow/.env.example ../deer-flow/.env
+	cp -n deployment/docker/deer-flow/conf.yaml.example ../deer-flow/conf.yaml
+	cd ../deer-flow && docker compose build
+
 .PHONY: backend-docker-install
 backend-docker-install:
-	cd deployment/docker/datamate && docker-compose up -d backend
+	cd deployment/docker/datamate && docker compose up -d backend
 
 .PHONY: backend-docker-uninstall
 backend-docker-uninstall:
-	cd deployment/docker/datamate && docker-compose down backend
+	cd deployment/docker/datamate && docker compose down backend
 
 .PHONY: frontend-docker-install
 frontend-docker-install:
-	cd deployment/docker/datamate && docker-compose up -d frontend
+	cd deployment/docker/datamate && docker compose up -d frontend
 
 .PHONY: frontend-docker-uninstall
 frontend-docker-uninstall:
-	cd deployment/docker/datamate && docker-compose down frontend
+	cd deployment/docker/datamate && docker compose down frontend
 
 .PHONY: runtime-docker-install
 runtime-docker-install:
-	cd deployment/docker/datamate && docker-compose up -d runtime
+	cd deployment/docker/datamate && docker compose up -d runtime
 
 .PHONY: runtime-docker-uninstall
 runtime-docker-uninstall:
-	cd deployment/docker/datamate && docker-compose down runtime
+	cd deployment/docker/datamate && docker compose down runtime
 
 .PHONY: datamate-docker-install
 datamate-docker-install:
-	cd deployment/docker/datamate && docker-compose up -d
+	cd deployment/docker/datamate && cp -n .env.example .env && docker compose -f docker-compose.yml up -d
 
 .PHONY: datamate-docker-uninstall
 datamate-docker-uninstall:
-	cd deployment/docker/datamate && docker-compose down
+	cd deployment/docker/datamate && docker compose -f docker-compose.yml down -v
+
+.PHONY: deer-flow-docker-install
+deer-flow-docker-install:
+	cd deployment/docker/datamate && cp -n .env.deer-flow.example .env && docker compose -f docker-compose.yml up -d
+	cd deployment/docker/deer-flow && cp -n .env.example .env && cp -n conf.yaml.example conf.yaml && docker compose -f docker-compose.yml up -d
+
+.PHONY: deer-flow-docker-uninstall
+deer-flow-docker-uninstall:
+	@if docker compose ls --filter name=datamate | grep -q datamate; then \
+		cd deployment/docker/datamate && docker compose -f docker-compose.yml up -d; \
+	fi
+	cd deployment/docker/deer-flow && docker compose -f docker-compose.yml down
 
 .PHONY: datamate-k8s-install
 datamate-k8s-install: create-namespace

backend/api-gateway/pom.xml

@@ -7,7 +7,7 @@
 
     <parent>
         <groupId>com.datamate</groupId>
-        <artifactId>data-mate-platform</artifactId>
+        <artifactId>datamate</artifactId>
         <version>1.0.0-SNAPSHOT</version>
         <relativePath>../pom.xml</relativePath>
     </parent>

backend/openapi/specs/data-management.yaml

@@ -39,7 +39,7 @@ paths:
           schema:
             type: integer
             default: 0
-          description: 页码，从0开始
+          description: 页码，从1开始
         - name: size
           in: query
           schema:
@@ -65,7 +65,7 @@ paths:
           in: query
           schema:
             type: string
-            enum: [ACTIVE, INACTIVE, PROCESSING]
+            enum: [DRAFT, ACTIVE, PROCESSING, ARCHIVED, PUBLISHED, DEPRECATED]
           description: 数据集状态过滤
       responses:
         '200':
@@ -231,40 +231,6 @@ paths:
               schema:
                 $ref: '#/components/schemas/PagedDatasetFileResponse'
 
-    post:
-      tags: [DatasetFile]
-      summary: 上传文件到数据集
-      operationId: uploadDatasetFile
-      description: 向指定数据集上传文件
-      parameters:
-        - name: datasetId
-          in: path
-          required: true
-          schema:
-            type: string
-          description: 数据集ID
-      requestBody:
-        required: true
-        content:
-          multipart/form-data:
-            schema:
-              type: object
-              properties:
-                file:
-                  type: string
-                  format: binary
-                  description: 要上传的文件
-                description:
-                  type: string
-                  description: 文件描述
-      responses:
-        '201':
-          description: 上传成功
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/DatasetFileResponse'
-
   /data-management/datasets/{datasetId}/files/{fileId}:
     get:
       tags: [DatasetFile]
@@ -342,6 +308,78 @@ paths:
                 type: string
                 format: binary
 
+  /data-management/datasets/{datasetId}/files/download:
+    get:
+      tags: [ DatasetFile ]
+      operationId: downloadDatasetFileAsZip
+      summary: 下载文件
+      description: 下载数据集中全部文件
+      parameters:
+        - name: datasetId
+          in: path
+          required: true
+          schema:
+            type: string
+          description: 数据集ID
+      responses:
+        '200':
+          description: 文件内容
+          content:
+            application/octet-stream:
+              schema:
+                type: string
+                format: binary
+
+  /data-management/datasets/{datasetId}/files/upload/pre-upload:
+    post:
+      tags: [ DatasetFile ]
+      operationId: preUpload
+      summary: 切片上传预上传
+      description: 预上传接口，返回后续分片上传所需的请求ID
+      parameters:
+        - name: datasetId
+          in: path
+          required: true
+          schema:
+            type: string
+          description: 数据集ID
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: '#/components/schemas/UploadFilesPreRequest'
+      responses:
+        '200':
+          description: 预上传成功，返回请求ID
+          content:
+            application/json:
+              schema:
+                type: string
+
+  /data-management/datasets/{datasetId}/files/upload/chunk:
+    post:
+      tags: [ DatasetFile ]
+      operationId: chunkUpload
+      summary: 切片上传
+      description: 使用预上传返回的请求ID进行分片上传
+      parameters:
+        - name: datasetId
+          in: path
+          required: true
+          schema:
+            type: string
+          description: 数据集ID
+      requestBody:
+        required: true
+        content:
+          multipart/form-data:
+            schema:
+              $ref: '#/components/schemas/UploadFileRequest'
+      responses:
+        '200':
+          description: 上传成功
+
   /data-management/dataset-types:
     get:
       operationId: getDatasetTypes
@@ -548,9 +586,59 @@ components:
           description: 标签列表
         status:
           type: string
-          enum: [ACTIVE, INACTIVE]
+          enum: [DRAFT, ACTIVE, PROCESSING, ARCHIVED, PUBLISHED, DEPRECATED]
           description: 数据集状态
 
+    UploadFilesPreRequest:
+      type: object
+      description: 切片上传预上传请求
+      properties:
+        hasArchive:
+          type: boolean
+          description: 是否为压缩包上传
+          default: false
+        totalFileNum:
+          type: integer
+          format: int32
+          minimum: 1
+          description: 总文件数量
+        totalSize:
+          type: integer
+          format: int64
+          description: 总文件大小（字节）
+      required: [ totalFileNum ]
+
+    UploadFileRequest:
+      type: object
+      description: 分片上传请求
+      properties:
+        reqId:
+          type: string
+          description: 预上传返回的请求ID
+        fileNo:
+          type: integer
+          format: int32
+          description: 文件编号（批量中的第几个）
+        fileName:
+          type: string
+          description: 文件名称
+        totalChunkNum:
+          type: integer
+          format: int32
+          description: 文件总分片数量
+        chunkNo:
+          type: integer
+          format: int32
+          description: 当前分片编号（从1开始）
+        file:
+          type: string
+          format: binary
+          description: 分片二进制内容
+        checkSumHex:
+          type: string
+          description: 分片校验和（十六进制）
+      required: [ reqId, fileNo, fileName, totalChunkNum, chunkNo, file ]
+
     DatasetTypeResponse:
       type: object
       properties: