From 2867f39be2d8b262321a0c87a566a7d5d3e0d492 Mon Sep 17 00:00:00 2001 From: hhhhsc <1710496817@qq.com> Date: Sat, 25 Oct 2025 16:09:16 +0800 Subject: [PATCH 1/3] refactor: enhance CleaningTaskService and related components with validation and repository updates --- .../services/data-cleaning-service/pom.xml | 5 ++ .../DataCleaningServiceConfiguration.java | 6 +- .../application/CleaningTaskService.java | 6 ++ .../application/CleaningTemplateService.java | 6 +- .../common/exception/CleanErrorCode.java | 2 +- .../domain/model/CreateDatasetRequest.java | 26 ------- .../domain/model/DatasetFileResponse.java | 36 --------- .../domain/model/DatasetResponse.java | 44 ----------- .../domain/model/DatasetTypeResponse.java | 23 ------ .../model/PagedDatasetFileResponse.java | 28 ------- .../repository/CleaningTaskRepository.java | 2 + .../OperatorInstanceRepository.java | 3 - .../converter/OperatorInstanceConverter.java | 2 +- .../Impl/CleaningTaskRepositoryImpl.java | 6 ++ .../Impl/OperatorInstanceRepositoryImpl.java | 6 -- .../mapper/OperatorInstanceMapper.java | 7 -- .../validator/CleanTaskValidator.java | 40 ++++++++++ .../interfaces/dto/CleaningTaskDto.java | 1 + .../interfaces/dto/CleaningTemplateDto.java | 1 + .../cleaning/interfaces/dto/OperatorDto.java | 41 ---------- .../interfaces/dto/OperatorInstanceDto.java | 9 +++ .../rest/CleaningTaskController.java | 26 +++---- .../rest/CleaningTemplateController.java | 26 +++---- .../operator/application/LabelService.java | 21 ----- .../operator/application/OperatorService.java | 61 ++++++++++++++- .../domain/contants/OperatorConstant.java | 7 ++ .../operator/domain/model/Category.java | 2 + .../domain/repository/OperatorRepository.java | 2 +- .../converter/OperatorConverter.java | 6 ++ .../exception/OperatorErrorCode.java | 19 +++++ .../infrastructure/parser/AbstractParser.java | 28 +++++++ .../infrastructure/parser/ParserHolder.java | 61 +++++++++++++++ .../infrastructure/parser/TarParser.java | 76 +++++++++++++++++++ .../infrastructure/parser/ZipParser.java | 75 ++++++++++++++++++ .../Impl/OperatorRepositoryImpl.java | 4 +- .../operator/interfaces/dto/CategoryDto.java | 2 + .../interfaces/dto/UploadOperatorRequest.java | 34 +++++++++ .../interfaces/rest/CategoryController.java | 6 +- .../interfaces/rest/LabelController.java | 40 ---------- .../interfaces/rest/OperatorController.java | 33 +++++--- scripts/db/data-cleaning-init.sql | 4 +- scripts/db/data-operator-init.sql | 35 ++++----- 42 files changed, 522 insertions(+), 346 deletions(-) delete mode 100644 backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/CreateDatasetRequest.java delete mode 100644 backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetFileResponse.java delete mode 100644 backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetResponse.java delete mode 100644 backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetTypeResponse.java delete mode 100644 backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/PagedDatasetFileResponse.java create mode 100644 backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/validator/CleanTaskValidator.java delete mode 100644 backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/OperatorDto.java delete mode 100644 backend/services/operator-market-service/src/main/java/com/datamate/operator/application/LabelService.java create mode 100644 backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/contants/OperatorConstant.java create mode 100644 backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/exception/OperatorErrorCode.java create mode 100644 backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/AbstractParser.java create mode 100644 backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/ParserHolder.java create mode 100644 backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/TarParser.java create mode 100644 backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/ZipParser.java create mode 100644 backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/UploadOperatorRequest.java delete mode 100644 backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/LabelController.java diff --git a/backend/services/data-cleaning-service/pom.xml b/backend/services/data-cleaning-service/pom.xml index 56b70fc56..a4f9de249 100644 --- a/backend/services/data-cleaning-service/pom.xml +++ b/backend/services/data-cleaning-service/pom.xml @@ -27,6 +27,11 @@ data-management-service ${project.version} + + com.datamate + operator-market-service + ${project.version} + org.springframework.boot spring-boot-starter-test diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/DataCleaningServiceConfiguration.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/DataCleaningServiceConfiguration.java index 8f1dcce7a..1ea5ad099 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/DataCleaningServiceConfiguration.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/DataCleaningServiceConfiguration.java @@ -1,6 +1,6 @@ package com.datamate.cleaning; -import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; @@ -8,9 +8,11 @@ * 数据归集服务配置类 * 基于DataX的数据归集和同步服务，支持多种数据源的数据采集和归集 */ -@SpringBootApplication @EnableAsync @EnableScheduling +@ComponentScan(basePackages = { + "com.datamate.cleaning" +}) public class DataCleaningServiceConfiguration { // Configuration class for JAR packaging - no main method needed } diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTaskService.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTaskService.java index 9bb2a3c75..6b1d4d4b3 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTaskService.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTaskService.java @@ -10,6 +10,7 @@ import com.datamate.cleaning.domain.repository.CleaningTaskRepository; import com.datamate.cleaning.domain.repository.OperatorInstanceRepository; +import com.datamate.cleaning.infrastructure.validator.CleanTaskValidator; import com.datamate.cleaning.interfaces.dto.CleaningProcess; import com.datamate.cleaning.interfaces.dto.CleaningTaskDto; import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest; @@ -59,6 +60,8 @@ public class CleaningTaskService { private final DatasetFileApplicationService datasetFileService; + private final CleanTaskValidator cleanTaskValidator; + private final String DATASET_PATH = "/dataset"; private final String FLOW_PATH = "/flow"; @@ -80,6 +83,9 @@ public int countTasks(String status, String keywords) { @Transactional public CleaningTaskDto createTask(CreateCleaningTaskRequest request) { + cleanTaskValidator.checkNameDuplication(request.getName()); + cleanTaskValidator.checkInputAndOutput(request.getInstance()); + CreateDatasetRequest createDatasetRequest = new CreateDatasetRequest(); createDatasetRequest.setName(request.getDestDatasetName()); createDatasetRequest.setDatasetType(DatasetType.valueOf(request.getDestDatasetType())); diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTemplateService.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTemplateService.java index 66e760077..bfd499b2d 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTemplateService.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTemplateService.java @@ -5,6 +5,8 @@ import com.datamate.cleaning.domain.repository.OperatorInstanceRepository; import com.datamate.cleaning.interfaces.dto.*; import com.datamate.cleaning.domain.model.entity.TemplateWithInstance; +import com.datamate.operator.domain.repository.OperatorRepository; +import com.datamate.operator.interfaces.dto.OperatorDto; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; @@ -24,8 +26,10 @@ public class CleaningTemplateService { private final OperatorInstanceRepository operatorInstanceRepo; + private final OperatorRepository operatorRepo; + public List getTemplates(String keywords) { - List allOperators = operatorInstanceRepo.findAllOperators(); + List allOperators = operatorRepo.findAllOperators(); Map operatorsMap = allOperators.stream() .collect(Collectors.toMap(OperatorDto::getId, Function.identity())); List allTemplates = cleaningTemplateRepo.findAllTemplates(keywords); diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/common/exception/CleanErrorCode.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/common/exception/CleanErrorCode.java index 9548d394e..ff7e2d41f 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/common/exception/CleanErrorCode.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/common/exception/CleanErrorCode.java @@ -12,7 +12,7 @@ public enum CleanErrorCode implements ErrorCode { */ DUPLICATE_TASK_NAME("clean.0001", "清洗任务名称重复"), - CREATE_DATASET_FAILED("clean.0002", "创建数据集失败"); + IN_AND_OUT_NOT_MATCH("clean.0002", "算子输入输出不匹配"); private final String code; private final String message; diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/CreateDatasetRequest.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/CreateDatasetRequest.java deleted file mode 100644 index 5c1900631..000000000 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/CreateDatasetRequest.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.datamate.cleaning.domain.model; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -import java.util.List; - - -@Getter -@Setter -@NoArgsConstructor -public class CreateDatasetRequest { - /** 数据集名称 */ - private String name; - /** 数据集描述 */ - private String description; - /** 数据集类型 */ - private String datasetType; - /** 标签列表 */ - private List tags; - /** 数据源 */ - private String dataSource; - /** 目标位置 */ - private String targetLocation; -} diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetFileResponse.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetFileResponse.java deleted file mode 100644 index e31337364..000000000 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetFileResponse.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.datamate.cleaning.domain.model; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -import java.time.LocalDateTime; - - -@Getter -@Setter -@NoArgsConstructor -public class DatasetFileResponse { - /** 文件ID */ - private String id; - /** 文件名 */ - private String fileName; - /** 原始文件名 */ - private String originalName; - /** 文件类型 */ - private String fileType; - /** 文件大小（字节） */ - private Long fileSize; - /** 文件状态 */ - private String status; - /** 文件描述 */ - private String description; - /** 文件路径 */ - private String filePath; - /** 上传时间 */ - private LocalDateTime uploadTime; - /** 最后更新时间 */ - private LocalDateTime lastAccessTime; - /** 上传者 */ - private String uploadedBy; -} diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetResponse.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetResponse.java deleted file mode 100644 index 2c63a0e9c..000000000 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetResponse.java +++ /dev/null @@ -1,44 +0,0 @@ -package com.datamate.cleaning.domain.model; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -import java.time.LocalDateTime; - -/** - * 数据集实体（与数据库表 t_dm_datasets 对齐） - */ -@Getter -@Setter -@NoArgsConstructor -@JsonIgnoreProperties(ignoreUnknown = true) -public class DatasetResponse { - /** 数据集ID */ - private String id; - /** 数据集名称 */ - private String name; - /** 数据集描述 */ - private String description; - /** 数据集类型 */ - private String datasetType; - /** 数据集状态 */ - private String status; - /** 数据源 */ - private String dataSource; - /** 目标位置 */ - private String targetLocation; - /** 文件数量 */ - private Integer fileCount; - /** 总大小（字节） */ - private Long totalSize; - /** 完成率（0-100） */ - private Float completionRate; - /** 创建时间 */ - private LocalDateTime createdAt; - /** 更新时间 */ - private LocalDateTime updatedAt; - /** 创建者 */ - private String createdBy; -} diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetTypeResponse.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetTypeResponse.java deleted file mode 100644 index b085583b4..000000000 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/DatasetTypeResponse.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.datamate.cleaning.domain.model; - -import lombok.Getter; -import lombok.Setter; -import java.util.List; - -/** - * 数据集类型响应DTO - */ -@Getter -@Setter -public class DatasetTypeResponse { - /** 类型编码 */ - private String code; - /** 类型名称 */ - private String name; - /** 类型描述 */ - private String description; - /** 支持的文件格式 */ - private List supportedFormats; - /** 图标 */ - private String icon; -} diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/PagedDatasetFileResponse.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/PagedDatasetFileResponse.java deleted file mode 100644 index da33f1719..000000000 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/model/PagedDatasetFileResponse.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.datamate.cleaning.domain.model; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; - -import java.util.List; - - -@Getter -@Setter -@NoArgsConstructor -public class PagedDatasetFileResponse { - /** 文件内容列表 */ - private List content; - /** 当前页码 */ - private Integer page; - /** 每页大小 */ - private Integer size; - /** 总元素数 */ - private Integer totalElements; - /** 总页数 */ - private Integer totalPages; - /** 是否为第一页 */ - private Boolean first; - /** 是否为最后一页 */ - private Boolean last; -} diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/CleaningTaskRepository.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/CleaningTaskRepository.java index 58a06a0d3..e8aeb1f4c 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/CleaningTaskRepository.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/CleaningTaskRepository.java @@ -16,4 +16,6 @@ public interface CleaningTaskRepository extends IRepository { void updateTask(CleaningTaskDto task); void deleteTaskById(String taskId); + + boolean isNameExist(String name); } diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/OperatorInstanceRepository.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/OperatorInstanceRepository.java index 50b42e25d..4fbcc2a2f 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/OperatorInstanceRepository.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/domain/repository/OperatorInstanceRepository.java @@ -1,15 +1,12 @@ package com.datamate.cleaning.domain.repository; import com.baomidou.mybatisplus.extension.repository.IRepository; -import com.datamate.cleaning.interfaces.dto.OperatorDto; import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto; import com.datamate.cleaning.domain.model.entity.OperatorInstance; import java.util.List; public interface OperatorInstanceRepository extends IRepository { - List findAllOperators(); - void insertInstance(String instanceId, List instances); void deleteByInstanceId(String instanceId); diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/converter/OperatorInstanceConverter.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/converter/OperatorInstanceConverter.java index 6bdda571b..eebeeecbe 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/converter/OperatorInstanceConverter.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/converter/OperatorInstanceConverter.java @@ -3,10 +3,10 @@ import com.datamate.cleaning.domain.model.entity.OperatorInstance; import com.datamate.cleaning.domain.model.entity.Operator; -import com.datamate.cleaning.interfaces.dto.OperatorDto; import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto; import com.datamate.common.infrastructure.exception.BusinessException; import com.datamate.common.infrastructure.exception.SystemErrorCode; +import com.datamate.operator.interfaces.dto.OperatorDto; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.mapstruct.Mapper; diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/CleaningTaskRepositoryImpl.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/CleaningTaskRepositoryImpl.java index 8cfedf996..455f8d592 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/CleaningTaskRepositoryImpl.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/CleaningTaskRepositoryImpl.java @@ -51,4 +51,10 @@ public void updateTask(CleaningTaskDto task) { public void deleteTaskById(String taskId) { mapper.deleteById(taskId); } + + public boolean isNameExist(String name) { + LambdaQueryWrapper queryWrapper = new LambdaQueryWrapper<>(); + queryWrapper.eq(CleaningTask::getName, name); + return mapper.exists(queryWrapper); + } } diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/OperatorInstanceRepositoryImpl.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/OperatorInstanceRepositoryImpl.java index 295d7c016..a3b197f4c 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/OperatorInstanceRepositoryImpl.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/Impl/OperatorInstanceRepositoryImpl.java @@ -3,7 +3,6 @@ import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.extension.repository.CrudRepository; import com.datamate.cleaning.infrastructure.converter.OperatorInstanceConverter; -import com.datamate.cleaning.interfaces.dto.OperatorDto; import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto; import com.datamate.cleaning.domain.model.entity.OperatorInstance; import com.datamate.cleaning.domain.repository.OperatorInstanceRepository; @@ -20,11 +19,6 @@ public class OperatorInstanceRepositoryImpl extends CrudRepository findAllOperators() { - return OperatorInstanceConverter.INSTANCE.fromEntityToDto(mapper.findAllOperators()); - } - @Override public void insertInstance(String instanceId, List instances) { List operatorInstances = new ArrayList<>(); diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/OperatorInstanceMapper.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/OperatorInstanceMapper.java index 2649d867a..0b0699d01 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/OperatorInstanceMapper.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/OperatorInstanceMapper.java @@ -1,17 +1,10 @@ package com.datamate.cleaning.infrastructure.persistence.mapper; import com.baomidou.mybatisplus.core.mapper.BaseMapper; -import com.datamate.cleaning.domain.model.entity.Operator; import com.datamate.cleaning.domain.model.entity.OperatorInstance; import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Select; - -import java.util.List; @Mapper public interface OperatorInstanceMapper extends BaseMapper { - @Select("SELECT id, name, description, version, inputs, outputs, runtime, settings, is_star, created_at, " + - "updated_at FROM t_operator") - List findAllOperators(); } diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/validator/CleanTaskValidator.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/validator/CleanTaskValidator.java new file mode 100644 index 000000000..92f3d29e9 --- /dev/null +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/validator/CleanTaskValidator.java @@ -0,0 +1,40 @@ +package com.datamate.cleaning.infrastructure.validator; + +import com.datamate.cleaning.common.exception.CleanErrorCode; +import com.datamate.cleaning.domain.repository.CleaningTaskRepository; +import com.datamate.cleaning.interfaces.dto.OperatorInstanceDto; +import com.datamate.common.infrastructure.exception.BusinessException; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.springframework.stereotype.Component; + +import java.util.List; +import java.util.Locale; + + +@Component +@RequiredArgsConstructor +public class CleanTaskValidator { + private final CleaningTaskRepository cleaningTaskRepo; + + public void checkNameDuplication (String name) { + if (cleaningTaskRepo.isNameExist(name)) { + throw BusinessException.of(CleanErrorCode.DUPLICATE_TASK_NAME); + } + } + + public void checkInputAndOutput (List operators) { + if (operators == null || operators.size() <= 1) { + return; + } + for (int i = 1; i < operators.size(); i++) { + OperatorInstanceDto front = operators.get(i - 1); + OperatorInstanceDto back = operators.get(i); + if (!StringUtils.equals(front.getOutputs(), back.getInputs())) { + throw BusinessException.of(CleanErrorCode.IN_AND_OUT_NOT_MATCH, + String.format(Locale.ROOT, "ops(name: [%s, %s]) inputs and outputs does not match", + front.getName(), back.getName())); + } + } + } +} diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningTaskDto.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningTaskDto.java index be6fbc726..f2cc96516 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningTaskDto.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningTaskDto.java @@ -5,6 +5,7 @@ import java.time.LocalDateTime; import java.util.List; +import com.datamate.operator.interfaces.dto.OperatorDto; import lombok.Getter; import lombok.Setter; import org.springframework.format.annotation.DateTimeFormat; diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningTemplateDto.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningTemplateDto.java index b86bb3e6b..29c710281 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningTemplateDto.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/CleaningTemplateDto.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; +import com.datamate.operator.interfaces.dto.OperatorDto; import lombok.Getter; import lombok.Setter; import org.springframework.format.annotation.DateTimeFormat; diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/OperatorDto.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/OperatorDto.java deleted file mode 100644 index 7b07fd750..000000000 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/OperatorDto.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.datamate.cleaning.interfaces.dto; - -import java.time.LocalDateTime; - -import lombok.Getter; -import lombok.Setter; -import org.springframework.format.annotation.DateTimeFormat; - -/** - * OperatorDto - */ - -@Getter -@Setter -public class OperatorDto { - - private String id; - - private String name; - - private String description; - - private String version; - - private String inputs; - - private String outputs; - - private String runtime; - - private String settings; - - private Boolean isStar; - - @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) - private LocalDateTime createdAt; - - @DateTimeFormat(iso = DateTimeFormat.ISO.DATE_TIME) - private LocalDateTime updatedAt; -} - diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/OperatorInstanceDto.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/OperatorInstanceDto.java index 64439b5cd..5ea821610 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/OperatorInstanceDto.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/dto/OperatorInstanceDto.java @@ -1,6 +1,7 @@ package com.datamate.cleaning.interfaces.dto; import java.util.HashMap; +import java.util.List; import java.util.Map; @@ -17,6 +18,14 @@ public class OperatorInstanceDto { private String id; + private String name; + + private String inputs; + + private String outputs; + + private List categories; + private Map overrides = new HashMap<>(); } diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTaskController.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTaskController.java index 04674b7c1..811f53e86 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTaskController.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTaskController.java @@ -3,10 +3,8 @@ import com.datamate.cleaning.application.CleaningTaskService; import com.datamate.cleaning.interfaces.dto.CleaningTaskDto; import com.datamate.cleaning.interfaces.dto.CreateCleaningTaskRequest; -import com.datamate.common.infrastructure.common.Response; import com.datamate.common.interfaces.PagedResponse; import lombok.RequiredArgsConstructor; -import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import java.util.List; @@ -19,41 +17,41 @@ public class CleaningTaskController { private final CleaningTaskService cleaningTaskService; @GetMapping - public ResponseEntity>> cleaningTasksGet( + public PagedResponse cleaningTasksGet( @RequestParam("page") Integer page, @RequestParam("size") Integer size, @RequestParam(value = "status", required = false) String status, @RequestParam(value = "keywords", required = false) String keywords) { List tasks = cleaningTaskService.getTasks(status, keywords, page, size); int count = cleaningTaskService.countTasks(status, keywords); int totalPages = (count + size + 1) / size; - return ResponseEntity.ok(Response.ok(PagedResponse.of(tasks, page, count, totalPages))); + return PagedResponse.of(tasks, page, count, totalPages); } @PostMapping - public ResponseEntity> cleaningTasksPost(@RequestBody CreateCleaningTaskRequest request) { - return ResponseEntity.ok(Response.ok(cleaningTaskService.createTask(request))); + public CleaningTaskDto cleaningTasksPost(@RequestBody CreateCleaningTaskRequest request) { + return cleaningTaskService.createTask(request); } @PostMapping("/{taskId}/stop") - public ResponseEntity> cleaningTasksStop(@PathVariable("taskId") String taskId) { + public String cleaningTasksStop(@PathVariable("taskId") String taskId) { cleaningTaskService.stopTask(taskId); - return ResponseEntity.ok(Response.ok(null)); + return taskId; } @PostMapping("/{taskId}/execute") - public ResponseEntity> cleaningTasksStart(@PathVariable("taskId") String taskId) { + public String cleaningTasksStart(@PathVariable("taskId") String taskId) { cleaningTaskService.executeTask(taskId); - return ResponseEntity.ok(Response.ok(null)); + return taskId; } @GetMapping("/{taskId}") - public ResponseEntity> cleaningTasksTaskIdGet(@PathVariable("taskId") String taskId) { - return ResponseEntity.ok(Response.ok(cleaningTaskService.getTask(taskId))); + public CleaningTaskDto cleaningTasksTaskIdGet(@PathVariable("taskId") String taskId) { + return cleaningTaskService.getTask(taskId); } @DeleteMapping("/{taskId}") - public ResponseEntity> cleaningTasksTaskIdDelete(@PathVariable("taskId") String taskId) { + public String cleaningTasksTaskIdDelete(@PathVariable("taskId") String taskId) { cleaningTaskService.deleteTask(taskId); - return ResponseEntity.ok(Response.ok(null)); + return taskId; } } diff --git a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTemplateController.java b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTemplateController.java index ffa74cc3b..5079500c6 100644 --- a/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTemplateController.java +++ b/backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/interfaces/rest/CleaningTemplateController.java @@ -4,10 +4,8 @@ import com.datamate.cleaning.interfaces.dto.CleaningTemplateDto; import com.datamate.cleaning.interfaces.dto.CreateCleaningTemplateRequest; import com.datamate.cleaning.interfaces.dto.UpdateCleaningTemplateRequest; -import com.datamate.common.infrastructure.common.Response; import com.datamate.common.interfaces.PagedResponse; import lombok.RequiredArgsConstructor; -import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -29,14 +27,14 @@ public class CleaningTemplateController { private final CleaningTemplateService cleaningTemplateService; @GetMapping - public ResponseEntity>> cleaningTemplatesGet( + public PagedResponse cleaningTemplatesGet( @RequestParam(value = "page", required = false) Integer page, @RequestParam(value = "size", required = false) Integer size, @RequestParam(value = "keywords", required = false) String keyword) { List templates = cleaningTemplateService.getTemplates(keyword); if (page == null || size == null) { - return ResponseEntity.ok(Response.ok(PagedResponse.of(templates.stream() - .sorted(Comparator.comparing(CleaningTemplateDto::getCreatedAt).reversed()).toList()))); + return PagedResponse.of(templates.stream() + .sorted(Comparator.comparing(CleaningTemplateDto::getCreatedAt).reversed()).toList()); } int count = templates.size(); int totalPages = (count + size + 1) / size; @@ -44,31 +42,31 @@ public ResponseEntity>> cleaningTemp .sorted(Comparator.comparing(CleaningTemplateDto::getCreatedAt).reversed()) .skip((long) page * size) .limit(size).toList(); - return ResponseEntity.ok(Response.ok(PagedResponse.of(limitTemplates, page, count, totalPages))); + return PagedResponse.of(limitTemplates, page, count, totalPages); } @PostMapping - public ResponseEntity> cleaningTemplatesPost( + public CleaningTemplateDto cleaningTemplatesPost( @RequestBody CreateCleaningTemplateRequest request) { - return ResponseEntity.ok(Response.ok(cleaningTemplateService.createTemplate(request))); + return cleaningTemplateService.createTemplate(request); } @GetMapping("/{templateId}") - public ResponseEntity> cleaningTemplatesTemplateIdGet( + public CleaningTemplateDto cleaningTemplatesTemplateIdGet( @PathVariable("templateId") String templateId) { - return ResponseEntity.ok(Response.ok(cleaningTemplateService.getTemplate(templateId))); + return cleaningTemplateService.getTemplate(templateId); } @PutMapping("/{templateId}") - public ResponseEntity> cleaningTemplatesTemplateIdPut( + public CleaningTemplateDto cleaningTemplatesTemplateIdPut( @PathVariable("templateId") String templateId, @RequestBody UpdateCleaningTemplateRequest request) { - return ResponseEntity.ok(Response.ok(cleaningTemplateService.updateTemplate(templateId, request))); + return cleaningTemplateService.updateTemplate(templateId, request); } @DeleteMapping("/{templateId}") - public ResponseEntity> cleaningTemplatesTemplateIdDelete( + public String cleaningTemplatesTemplateIdDelete( @PathVariable("templateId") String templateId) { cleaningTemplateService.deleteTemplate(templateId); - return ResponseEntity.noContent().build(); + return templateId; } } diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/application/LabelService.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/application/LabelService.java deleted file mode 100644 index f9fe8be1b..000000000 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/application/LabelService.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.datamate.operator.application; - -import com.datamate.operator.interfaces.dto.LabelDto; -import org.springframework.stereotype.Service; -import java.util.List; -import java.util.Collections; - -@Service -public class LabelService { - public List getLabels(Integer page, Integer size, String keyword) { - // TODO: 查询标签列表 - return Collections.emptyList(); - } - public void updateLabel(String id, List updateLabelDtoRequest) { - // TODO: 更新标签 - } - public void createLabels(LabelDto labelsPostRequest) { - // TODO: 批量创建标签 - } -} - diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/application/OperatorService.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/application/OperatorService.java index 54ff85823..aaa8bc19b 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/application/OperatorService.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/application/OperatorService.java @@ -1,15 +1,25 @@ package com.datamate.operator.application; +import com.datamate.common.domain.model.ChunkUploadPreRequest; +import com.datamate.common.domain.model.FileUploadResult; +import com.datamate.common.domain.service.FileService; +import com.datamate.operator.domain.contants.OperatorConstant; import com.datamate.operator.infrastructure.converter.OperatorConverter; import com.datamate.operator.domain.model.OperatorView; import com.datamate.operator.domain.repository.CategoryRelationRepository; import com.datamate.operator.domain.repository.OperatorRepository; import com.datamate.operator.domain.repository.OperatorViewRepository; +import com.datamate.operator.infrastructure.parser.ParserHolder; import com.datamate.operator.interfaces.dto.OperatorDto; +import com.datamate.operator.interfaces.dto.UploadOperatorRequest; import lombok.RequiredArgsConstructor; +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; +import java.io.File; +import java.io.IOException; import java.util.List; @Service @@ -21,6 +31,17 @@ public class OperatorService { private final CategoryRelationRepository relationRepo; + private final ParserHolder parserHolder; + + private final FileService fileService; + + @Value("${operator.base.path:/operator}") + private String operatorBasePath; + + private final String uploadPath = operatorBasePath + File.separator + "upload"; + + private final String extractPath = operatorBasePath + File.separator + "extract"; + public List getOperators(Integer page, Integer size, List categories, String operatorName, Boolean isStar) { List filteredOperators = operatorViewRepo.findOperatorsByCriteria(page, size, operatorName, @@ -40,17 +61,55 @@ public OperatorDto getOperatorById(String id) { public OperatorDto createOperator(OperatorDto req) { operatorRepo.insertOperator(req); relationRepo.batchInsert(req.getId(), req.getCategories()); + parserHolder.extractTo(getFileType(req.getFileName()), uploadPath + File.separator + req.getFileName(), + extractPath + File.separator + getFileNameWithoutExtension(req.getFileName())); return getOperatorById(req.getId()); } public OperatorDto updateOperator(String id, OperatorDto req) { operatorRepo.updateOperator(req); relationRepo.batchInsert(id, req.getCategories()); + parserHolder.extractTo(getFileType(req.getFileName()), uploadPath + File.separator + req.getFileName(), + extractPath + File.separator + getFileNameWithoutExtension(req.getFileName())); return getOperatorById(id); } - public OperatorDto uploadOperator(MultipartFile file, String description) { + public OperatorDto uploadOperator(MultipartFile multipartFile) { // TODO: 文件上传与解析 + try { + File file = new File(multipartFile.getName()); + // 从MultipartFile获取输入流并复制到目标文件 + FileUtils.copyInputStreamToFile(multipartFile.getInputStream(), file); + OperatorDto operatorDto = parserHolder.parseYamlFromArchive(getFileType(file.getName()), file, + OperatorConstant.YAML_PATH, OperatorDto.class); + } catch (IOException e) { + e.printStackTrace(); + } return new OperatorDto(); } + + public String preUpload() { + ChunkUploadPreRequest request = ChunkUploadPreRequest.builder().build(); + request.setUploadPath(operatorBasePath + File.separator + "upload"); + request.setTotalFileNum(1); + request.setServiceId(OperatorConstant.SERVICE_ID); + return fileService.preUpload(request); + } + + public void chunkUpload(UploadOperatorRequest request) { + FileUploadResult uploadResult = fileService.chunkUpload(OperatorConverter.INSTANCE.toChunkRequest(request)); + if (uploadResult.isAllFilesUploaded()) { + // TODO: 文件上传与解析 + OperatorDto operatorDto = parserHolder.parseYamlFromArchive(getFileType(uploadResult.getFileName()), uploadResult.getSavedFile(), + OperatorConstant.YAML_PATH, OperatorDto.class); + } + } + + private String getFileType(String fileName) { + return fileName.substring(fileName.lastIndexOf('.') + 1); + } + + private String getFileNameWithoutExtension(String fileName) { + return fileName.substring(0, fileName.lastIndexOf('.')); + } } diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/contants/OperatorConstant.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/contants/OperatorConstant.java new file mode 100644 index 000000000..6abff77ed --- /dev/null +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/contants/OperatorConstant.java @@ -0,0 +1,7 @@ +package com.datamate.operator.domain.contants; + +public class OperatorConstant { + public static String SERVICE_ID = "operator"; + + public static String YAML_PATH = "metadata.yml"; +} diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/model/Category.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/model/Category.java index c8f89c6a5..83a65164b 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/model/Category.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/model/Category.java @@ -12,6 +12,8 @@ public class Category { private String name; + private String value; + private String type; private Integer parentId; diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/repository/OperatorRepository.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/repository/OperatorRepository.java index 546df447d..161db9413 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/repository/OperatorRepository.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/repository/OperatorRepository.java @@ -7,7 +7,7 @@ import java.util.List; public interface OperatorRepository extends IRepository { - List findAllOperators(); + List findAllOperators(); void updateOperator(OperatorDto operator); diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/converter/OperatorConverter.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/converter/OperatorConverter.java index ddccf0f6c..78eac9a46 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/converter/OperatorConverter.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/converter/OperatorConverter.java @@ -1,8 +1,10 @@ package com.datamate.operator.infrastructure.converter; +import com.datamate.common.domain.model.ChunkUploadRequest; import com.datamate.operator.domain.model.Operator; import com.datamate.operator.domain.model.OperatorView; import com.datamate.operator.interfaces.dto.OperatorDto; +import com.datamate.operator.interfaces.dto.UploadOperatorRequest; import org.mapstruct.Mapper; import org.mapstruct.Mapping; import org.mapstruct.Named; @@ -19,6 +21,8 @@ public interface OperatorConverter { @Mapping(target = "categories", source = "categories", qualifiedByName = "stringToList") OperatorDto fromEntityToDto(OperatorView operator); + List fromEntityToDto(List operator); + @Named("stringToList") static List stringToList(String input) { if (input == null || input.isEmpty()) { @@ -28,4 +32,6 @@ static List stringToList(String input) { } Operator fromDtoToEntity(OperatorDto operator); + + ChunkUploadRequest toChunkRequest(UploadOperatorRequest request); } diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/exception/OperatorErrorCode.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/exception/OperatorErrorCode.java new file mode 100644 index 000000000..2fdcac459 --- /dev/null +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/exception/OperatorErrorCode.java @@ -0,0 +1,19 @@ +package com.datamate.operator.infrastructure.exception; + +import com.datamate.common.infrastructure.exception.ErrorCode; +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum OperatorErrorCode implements ErrorCode { + /** + * 不支持的文件类型 + */ + UNSUPPORTED_FILE_TYPE("op.0001", "不支持的文件类型"), + + YAML_NOT_FOUND("op.0002", "算子中缺少元数据文件"); + + private final String code; + private final String message; +} diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/AbstractParser.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/AbstractParser.java new file mode 100644 index 000000000..7cd26f02a --- /dev/null +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/AbstractParser.java @@ -0,0 +1,28 @@ +package com.datamate.operator.infrastructure.parser; + +import org.yaml.snakeyaml.Yaml; + +import java.io.File; +import java.nio.file.Path; + +public abstract class AbstractParser { + // 使用 SnakeYAML 的 Yaml 实例，子类直接使用 + protected final Yaml yaml = new Yaml(); + + /** + * 从压缩包内读取指定路径的 yaml 文件并解析为指定类型 + * @param archive 压缩包路径（zip 或 tar） + * @param entryPath 压缩包内部的文件路径，例如 "config/app.yaml" 或 "./config/app.yaml" + * @param clazz 目标类型 + * @param 类型参数 + * @return 解析后的对象 + */ + public abstract T parseYamlFromArchive(File archive, String entryPath, Class clazz); + + /** + * 将压缩包解压到目标目录（保持相对路径） + * @param archive 压缩包路径 + * @param targetDir 目标目录 + */ + public abstract void extractTo(File archive, String targetDir); +} diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/ParserHolder.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/ParserHolder.java new file mode 100644 index 000000000..4989517d3 --- /dev/null +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/ParserHolder.java @@ -0,0 +1,61 @@ +package com.datamate.operator.infrastructure.parser; + +import com.datamate.common.infrastructure.exception.BusinessException; +import com.datamate.common.infrastructure.exception.SystemErrorCode; +import com.datamate.operator.infrastructure.exception.OperatorErrorCode; +import jakarta.annotation.PostConstruct; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +@Component +public class ParserHolder { + // 存放 parser：key 为 parser 类型标识（例如 "zip" 或 "tar"），value 为 parser 实例 + private final Map parserMap = new ConcurrentHashMap<>(); + + // 注册 parser（可在启动时调用） + public void registerParser(String type, AbstractParser parser) { + if (type == null || parser == null) { + throw BusinessException.of(SystemErrorCode.UNKNOWN_ERROR); + } + parserMap.put(type, parser); + } + + // 根据类型获取 parser（可能为 null） + public AbstractParser getParser(String type) { + return parserMap.get(type); + } + + // 便捷代理：从指定类型的压缩包中读取 entry 并解析为 clazz + public T parseYamlFromArchive(String type, File archive, String entryPath, Class clazz) { + AbstractParser parser = getParser(type); + if (parser == null) { + throw BusinessException.of(OperatorErrorCode.UNSUPPORTED_FILE_TYPE, + "No parser registered for type: " + type); + } + return parser.parseYamlFromArchive(archive, entryPath, clazz); + } + + // 便捷代理：将指定类型的压缩包解压到目标目录 + public void extractTo(String type, File archive, String targetDir) { + AbstractParser parser = getParser(type); + if (parser == null) { + throw BusinessException.of(OperatorErrorCode.UNSUPPORTED_FILE_TYPE, + "No parser registered for type: " + type); + } + parser.extractTo(archive, targetDir); + } + + public void extractTo(String type, String sourceDir, String targetDir) { + extractTo(type, new File(sourceDir), targetDir); + } + + @PostConstruct + public void init() { + // 注册 zip 和 tar parser，key 可根据需要调整（例如 "zip"/"tar"） + registerParser("zip", new ZipParser()); + registerParser("tar", new TarParser()); + } +} diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/TarParser.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/TarParser.java new file mode 100644 index 000000000..1d1fe0ec9 --- /dev/null +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/TarParser.java @@ -0,0 +1,76 @@ +package com.datamate.operator.infrastructure.parser; + +import com.datamate.common.infrastructure.exception.BusinessException; +import com.datamate.common.infrastructure.exception.SystemErrorCode; +import com.datamate.operator.infrastructure.exception.OperatorErrorCode; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; + +public class TarParser extends AbstractParser { + + @Override + public T parseYamlFromArchive(File archive, String entryPath, Class clazz) { + // 允许带或不带前导 "./" + String normalized = entryPath.startsWith("./") ? entryPath.substring(2) : entryPath; + try (InputStream fis = Files.newInputStream(archive.toPath()); + TarArchiveInputStream tis = new TarArchiveInputStream(fis)) { + TarArchiveEntry entry; + while ((entry = tis.getNextEntry()) != null) { + String name = entry.getName(); + if (Objects.equals(name, entryPath) || Objects.equals(name, normalized)) { + // 使用 SnakeYAML 解析当前 entry 的内容到目标类型 + return yaml.loadAs(tis, clazz); + } + } + } catch (IOException e) { + throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR, e.getMessage()); + } + throw BusinessException.of(OperatorErrorCode.YAML_NOT_FOUND, "Entry not found in tar: " + entryPath); + } + + @Override + public void extractTo(File archive, String targetDir) { + Path targetPath = Paths.get(targetDir); + try (InputStream fis = Files.newInputStream(archive.toPath()); + TarArchiveInputStream tis = new TarArchiveInputStream(fis)) { + Files.createDirectories(targetPath); + TarArchiveEntry entry; + while ((entry = tis.getNextEntry()) != null) { + String entryName = entry.getName(); + // 去掉可能的前导 "./" + if (entryName.startsWith("./")) { + entryName = entryName.substring(2); + } + + Path resolved = targetPath.resolve(entryName).normalize(); + if (!resolved.startsWith(targetPath.toAbsolutePath().normalize())) { + throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR, "Bad tar entry: " + entryName); + } + + if (entry.isDirectory()) { + Files.createDirectories(resolved); + } else { + Files.createDirectories(resolved.getParent()); + try (OutputStream os = Files.newOutputStream(resolved)) { + byte[] buffer = new byte[8192]; + int len; + while ((len = tis.read(buffer)) != -1) { + os.write(buffer, 0, len); + } + } + } + } + } catch (IOException e) { + throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR, e.getMessage()); + } + } +} diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/ZipParser.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/ZipParser.java new file mode 100644 index 000000000..186e864d2 --- /dev/null +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/parser/ZipParser.java @@ -0,0 +1,75 @@ +package com.datamate.operator.infrastructure.parser; + +import com.datamate.common.infrastructure.exception.BusinessException; +import com.datamate.common.infrastructure.exception.SystemErrorCode; +import com.datamate.operator.infrastructure.exception.OperatorErrorCode; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Enumeration; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +public class ZipParser extends AbstractParser { + + @Override + public T parseYamlFromArchive(File archive, String entryPath, Class clazz) { + try (ZipFile zipFile = new ZipFile(archive)) { + // 允许带或不带前导 "./" + String normalized = entryPath.startsWith("./") ? entryPath.substring(2) : entryPath; + ZipEntry entry = zipFile.getEntry(entryPath); + if (entry == null) { + entry = zipFile.getEntry(normalized); + } + if (entry == null) { + throw BusinessException.of(OperatorErrorCode.YAML_NOT_FOUND, "Entry not found in zip: " + entryPath); + } + try (InputStream is = zipFile.getInputStream(entry)) { + // 使用 SnakeYAML 解析为目标类型 + return yaml.loadAs(is, clazz); + } + } catch (IOException e) { + throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR, e.getMessage()); + } + } + + @Override + public void extractTo(File archive, String targetDir) { + Path targetPath = Paths.get(targetDir); + try (ZipFile zipFile = new ZipFile(archive)) { + Files.createDirectories(targetPath); + Enumeration entries = zipFile.entries(); + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + String entryName = entry.getName(); + + // 防止 Zip Slip：确保解压路径仍在 targetDir 下 + Path resolved = targetPath.resolve(entryName).normalize(); + if (!resolved.startsWith(targetPath.toAbsolutePath().normalize())) { + throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR, "Bad zip entry: " + entryName); + } + + if (entry.isDirectory()) { + Files.createDirectories(resolved); + } else { + Files.createDirectories(resolved.getParent()); + try (InputStream is = zipFile.getInputStream(entry); + OutputStream os = Files.newOutputStream(resolved)) { + byte[] buffer = new byte[8192]; + int len; + while ((len = is.read(buffer)) != -1) { + os.write(buffer, 0, len); + } + } + } + } + } catch (IOException e) { + throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR, e.getMessage()); + } + } +} diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorRepositoryImpl.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorRepositoryImpl.java index 58a69610a..17a5c5c62 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorRepositoryImpl.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorRepositoryImpl.java @@ -17,8 +17,8 @@ public class OperatorRepositoryImpl extends CrudRepository findAllOperators() { - return mapper.selectList(null); + public List findAllOperators() { + return OperatorConverter.INSTANCE.fromEntityToDto(mapper.selectList(null)); } @Override diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/CategoryDto.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/CategoryDto.java index 9681aa3bd..a9f8fd340 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/CategoryDto.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/CategoryDto.java @@ -10,6 +10,8 @@ public class CategoryDto { private String name; + private String value; + private String type; private Integer parentId; diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/UploadOperatorRequest.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/UploadOperatorRequest.java new file mode 100644 index 000000000..c6ebb6111 --- /dev/null +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/UploadOperatorRequest.java @@ -0,0 +1,34 @@ +package com.datamate.operator.interfaces.dto; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.web.multipart.MultipartFile; + +/** + * 上传文件请求 + * 用于分块上传文件时的请求参数封装，支持大文件分片上传功能 + */ +@Getter +@Setter +public class UploadOperatorRequest { + /** 预上传返回的id，用来确认同一个任务 */ + private String reqId; + + /** 文件编号，用于标识批量上传中的第几个文件 */ + private int fileNo; + + /** 文件名称 */ + private String fileName; + + /** 文件总分块数量 */ + private int totalChunkNum; + + /** 当前分块编号，从1开始 */ + private int chunkNo; + + /** 上传的文件分块内容 */ + private MultipartFile file; + + /** 文件分块的校验和（十六进制字符串），用于验证文件完整性 */ + private String checkSumHex; +} diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/CategoryController.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/CategoryController.java index d9bd4625e..2a7dca83e 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/CategoryController.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/CategoryController.java @@ -1,11 +1,9 @@ package com.datamate.operator.interfaces.rest; -import com.datamate.common.infrastructure.common.Response; import com.datamate.common.interfaces.PagedResponse; import com.datamate.operator.application.CategoryService; import com.datamate.operator.interfaces.dto.CategoryTreeResponse; import lombok.RequiredArgsConstructor; -import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -20,8 +18,8 @@ public class CategoryController { private final CategoryService categoryService; @GetMapping("/tree") - public ResponseEntity>> categoryTreeGet() { + public PagedResponse categoryTreeGet() { List allCategories = categoryService.getAllCategories(); - return ResponseEntity.ok(Response.ok(PagedResponse.of(allCategories))); + return PagedResponse.of(allCategories); } } diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/LabelController.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/LabelController.java deleted file mode 100644 index ec07a8ba1..000000000 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/LabelController.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.datamate.operator.interfaces.rest; - -import com.datamate.common.infrastructure.common.Response; -import com.datamate.common.interfaces.PagedResponse; -import com.datamate.operator.application.LabelService; -import com.datamate.operator.interfaces.dto.LabelDto; -import lombok.RequiredArgsConstructor; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.*; - -import java.util.List; - -@RestController -@RequestMapping("/labels") -@RequiredArgsConstructor -public class LabelController { - private final LabelService labelService; - - @GetMapping - public ResponseEntity>> labelsGet(@RequestParam("page") Integer page, - @RequestParam("size") Integer size, - @RequestParam("keyword") String keyword) { - return ResponseEntity.ok(Response.ok(PagedResponse.of(labelService.getLabels(page, size, keyword)))); - } - - @PutMapping("/{id}") - public ResponseEntity> labelsIdPut(@PathVariable("id") String id, - @RequestBody List updateLabelDtoRequest) { - labelService.updateLabel(id, updateLabelDtoRequest); - return ResponseEntity.ok(Response.ok(null)); - } - - @PostMapping - public ResponseEntity> labelsPost(@RequestBody LabelDto labelsPostRequest) { - labelService.createLabels(labelsPostRequest); - return ResponseEntity.ok(Response.ok(null)); - } -} - - diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/OperatorController.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/OperatorController.java index a201b965a..727a757db 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/OperatorController.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/rest/OperatorController.java @@ -5,6 +5,8 @@ import com.datamate.operator.application.OperatorService; import com.datamate.operator.interfaces.dto.OperatorDto; import com.datamate.operator.interfaces.dto.OperatorsListPostRequest; +import com.datamate.operator.interfaces.dto.UploadOperatorRequest; +import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; @@ -19,34 +21,43 @@ public class OperatorController { private final OperatorService operatorService; @PostMapping("/list") - public ResponseEntity>> operatorsListPost(@RequestBody OperatorsListPostRequest request) { + public PagedResponse operatorsListPost(@RequestBody OperatorsListPostRequest request) { List responses = operatorService.getOperators(request.getPage(), request.getSize(), request.getCategories(), request.getOperatorName(), request.getIsStar()); int count = operatorService.getOperatorsCount(request.getCategories(), request.getOperatorName(), request.getIsStar()); int totalPages = (count + request.getSize() + 1) / request.getSize(); - return ResponseEntity.ok(Response.ok(PagedResponse.of(responses, request.getPage(), count, totalPages))); + return PagedResponse.of(responses, request.getPage(), count, totalPages); } @GetMapping("/{id}") - public ResponseEntity> operatorsIdGet(@PathVariable("id") String id) { - return ResponseEntity.ok(Response.ok(operatorService.getOperatorById(id))); + public OperatorDto operatorsIdGet(@PathVariable("id") String id) { + return operatorService.getOperatorById(id); } @PutMapping("/{id}") - public ResponseEntity> operatorsIdPut(@PathVariable("id") String id, + public OperatorDto operatorsIdPut(@PathVariable("id") String id, @RequestBody OperatorDto updateOperatorRequest) { - return ResponseEntity.ok(Response.ok(operatorService.updateOperator(id, updateOperatorRequest))); + return operatorService.updateOperator(id, updateOperatorRequest); } @PostMapping("/create") - public ResponseEntity> operatorsCreatePost(@RequestBody OperatorDto createOperatorRequest) { - return ResponseEntity.ok(Response.ok(operatorService.createOperator(createOperatorRequest))); + public OperatorDto operatorsCreatePost(@RequestBody OperatorDto createOperatorRequest) { + return operatorService.createOperator(createOperatorRequest); } @PostMapping("/upload") - public ResponseEntity> operatorsUploadPost(@RequestPart(value = "file") MultipartFile file, - @RequestParam(value = "description") String description) { - return ResponseEntity.ok(Response.ok(operatorService.uploadOperator(file, description))); + public OperatorDto operatorsUploadPost(@RequestPart(value = "file") MultipartFile file) { + return operatorService.uploadOperator(file); + } + + @PostMapping("/upload/pre-upload") + public String preUpload() { + return operatorService.preUpload(); + } + + @PostMapping("/upload/chunk") + public void chunkUpload(@RequestBody UploadOperatorRequest request) { + operatorService.chunkUpload(request); } } diff --git a/scripts/db/data-cleaning-init.sql b/scripts/db/data-cleaning-init.sql index 47a42ba5e..3ce98ef29 100644 --- a/scripts/db/data-cleaning-init.sql +++ b/scripts/db/data-cleaning-init.sql @@ -3,7 +3,7 @@ USE datamate; CREATE TABLE IF NOT EXISTS t_clean_template ( id varchar(64) primary key not null unique, - name varchar(64), + name varchar(64) unique, description varchar(256), created_at timestamp default current_timestamp, updated_at timestamp default current_timestamp, @@ -13,7 +13,7 @@ CREATE TABLE IF NOT EXISTS t_clean_template CREATE TABLE IF NOT EXISTS t_clean_task ( id varchar(64) primary key, - name varchar(64), + name varchar(64) unique, description varchar(256), status varchar(256), src_dataset_id varchar(64), diff --git a/scripts/db/data-operator-init.sql b/scripts/db/data-operator-init.sql index 4a1b8a63f..1979f19ca 100644 --- a/scripts/db/data-operator-init.sql +++ b/scripts/db/data-operator-init.sql @@ -3,7 +3,7 @@ USE datamate; CREATE TABLE IF NOT EXISTS t_operator ( id varchar(64) primary key, - name varchar(64), + name varchar(64) unique, description varchar(256), version varchar(256), inputs varchar(256), @@ -19,7 +19,8 @@ CREATE TABLE IF NOT EXISTS t_operator CREATE TABLE IF NOT EXISTS t_operator_category ( id int primary key auto_increment, - name varchar(64), + name varchar(64) unique , + value varchar(64) unique , type varchar(64), parent_id int ); @@ -49,21 +50,21 @@ FROM t_operator_category_relation tocr LEFT JOIN t_operator o ON tocr.operator_id = o.id LEFT JOIN t_operator_category toc ON tocr.category_id = toc.id; -INSERT IGNORE INTO t_operator_category(id, name, type, parent_id) -VALUES (1, '模态', 'predefined', 0), - (2, '语言', 'predefined', 0), - (3, '文本', 'predefined', 1), - (4, '图片', 'predefined', 1), - (5, '音频', 'predefined', 1), - (6, '视频', 'predefined', 1), - (7, '多模态', 'predefined', 1), - (8, 'Python', 'predefined', 2), - (9, 'Java', 'predefined', 2), - (10, '来源', 'predefined', 0), - (11, '系统预置', 'predefined', 10), - (12, '用户上传', 'predefined', 10), - (13, '收藏状态', 'predefined', 0), - (14, '已收藏', 'predefined', 13); +INSERT IGNORE INTO t_operator_category(id, name, value, type, parent_id) +VALUES (1, '模态', 'model', 'predefined', 0), + (2, '语言', 'language', 'predefined', 0), + (3, '文本', 'text', 'predefined', 1), + (4, '图片', 'image', 'predefined', 1), + (5, '音频', 'audio', 'predefined', 1), + (6, '视频', 'video', 'predefined', 1), + (7, '多模态', 'multimodal', 'predefined', 1), + (8, 'Python', 'python', 'predefined', 2), + (9, 'Java', 'java', 'predefined', 2), + (10, '来源', 'origin', 'predefined', 0), + (11, '系统预置', 'predefined', 'predefined', 10), + (12, '用户上传', 'customized', 'predefined', 10), + (13, '收藏状态', 'starStatus', 'predefined', 0), + (14, '已收藏', 'isStar', 'predefined', 13); INSERT IGNORE INTO t_operator (id, name, description, version, inputs, outputs, runtime, settings, file_name, is_star) From 53f2e03384e4345e01f64cdbf00bf482460efa8d Mon Sep 17 00:00:00 2001 From: hhhhsc <1710496817@qq.com> Date: Thu, 30 Oct 2025 16:01:06 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feature:=20=E6=94=AF=E6=8C=81=E7=AE=97?= =?UTF-8?q?=E5=AD=90=E4=B8=8A=E4=BC=A0=E5=88=9B=E5=BB=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datamate/operator/application/OperatorService.java | 4 ++-- .../domain/repository/OperatorViewRepository.java | 4 ++-- .../persistence/Impl/OperatorViewRepositoryImpl.java | 4 ++-- .../operator/interfaces/dto/OperatorsListPostRequest.java | 2 +- deployment/helm/datamate/values.yaml | 8 ++++++-- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/application/OperatorService.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/application/OperatorService.java index 2cbe6a3e3..5cd1041f9 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/application/OperatorService.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/application/OperatorService.java @@ -35,14 +35,14 @@ public class OperatorService { @Value("${operator.base.path:/operator}") private String operatorBasePath; - public List getOperators(Integer page, Integer size, List categories, + public List getOperators(Integer page, Integer size, List categories, String operatorName, Boolean isStar) { List filteredOperators = operatorViewRepo.findOperatorsByCriteria(page, size, operatorName, categories, isStar); return filteredOperators.stream().map(OperatorConverter.INSTANCE::fromEntityToDto).toList(); } - public int getOperatorsCount(List categories, String operatorName, Boolean isStar) { + public int getOperatorsCount(List categories, String operatorName, Boolean isStar) { return operatorViewRepo.countOperatorsByCriteria(operatorName, categories, isStar); } diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/repository/OperatorViewRepository.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/repository/OperatorViewRepository.java index cacba8310..fbbdf6437 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/repository/OperatorViewRepository.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/repository/OperatorViewRepository.java @@ -7,9 +7,9 @@ public interface OperatorViewRepository extends IRepository { List findOperatorsByCriteria(Integer page, Integer size, String operatorName, - List categories, Boolean isStar); + List categories, Boolean isStar); - Integer countOperatorsByCriteria(String operatorName, List categories, Boolean isStar); + Integer countOperatorsByCriteria(String operatorName, List categories, Boolean isStar); OperatorView findOperatorById(String id); } diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorViewRepositoryImpl.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorViewRepositoryImpl.java index 19c11fd62..1a64d5a66 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorViewRepositoryImpl.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorViewRepositoryImpl.java @@ -22,7 +22,7 @@ public class OperatorViewRepositoryImpl extends CrudRepository findOperatorsByCriteria(Integer page, Integer size, String operatorName, - List categories, Boolean isStar) { + List categories, Boolean isStar) { QueryWrapper queryWrapper = Wrappers.query(); queryWrapper.in(CollectionUtils.isNotEmpty(categories), "category_id", categories) .like(StringUtils.isNotBlank(operatorName), "operator_name", operatorName) @@ -37,7 +37,7 @@ public List findOperatorsByCriteria(Integer page, Integer size, St } @Override - public Integer countOperatorsByCriteria(String operatorName, List categories, Boolean isStar) { + public Integer countOperatorsByCriteria(String operatorName, List categories, Boolean isStar) { QueryWrapper queryWrapper = Wrappers.query(); queryWrapper.in(CollectionUtils.isNotEmpty(categories),"category_id", categories) .like(StringUtils.isNotBlank(operatorName), "operator_name", operatorName) diff --git a/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/OperatorsListPostRequest.java b/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/OperatorsListPostRequest.java index 973157992..fde4507ca 100644 --- a/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/OperatorsListPostRequest.java +++ b/backend/services/operator-market-service/src/main/java/com/datamate/operator/interfaces/dto/OperatorsListPostRequest.java @@ -19,7 +19,7 @@ public class OperatorsListPostRequest { private Integer size; - private List categories = new ArrayList<>(); + private List categories = new ArrayList<>(); private String operatorName; diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index d38a70e30..c4258f153 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -43,7 +43,7 @@ dataVolume: &dataVolume operatorVolume: &operatorVolume name: operator-volume hostPath: - path: /opt/datamate/data/ + path: /opt/datamate/data/operator backend: volumes: @@ -73,6 +73,7 @@ frontend: subPath: frontend - mountPath: /etc/nginx/conf.d/backend.conf name: datamate-nginx-conf + subPath: backend.conf database: volumes: @@ -112,11 +113,12 @@ ray-cluster: name: flow-volume - mountPath: /opt/runtime/datamate/ops/user name: operator-volume + subPath: extract sidecarContainers: - name: runtime image: datamate-runtime imagePullPolicy: IfNotPresent - command: + args: - python - /opt/runtime/datamate/operator_runtime.py - --port @@ -146,6 +148,7 @@ ray-cluster: name: flow-volume - mountPath: /opt/runtime/datamate/ops/user name: operator-volume + subPath: extract worker: volumes: - *datasetVolume @@ -162,3 +165,4 @@ ray-cluster: name: flow-volume - mountPath: /opt/runtime/datamate/ops/user name: operator-volume + subPath: extract From 1fdfe35ebe85120571dd025eaf115c278dd1b902 Mon Sep 17 00:00:00 2001 From: hhhhsc <1710496817@qq.com> Date: Thu, 30 Oct 2025 17:01:11 +0800 Subject: [PATCH 3/3] merge main --- deployment/helm/datamate/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deployment/helm/datamate/values.yaml b/deployment/helm/datamate/values.yaml index c4258f153..97aba1fdb 100644 --- a/deployment/helm/datamate/values.yaml +++ b/deployment/helm/datamate/values.yaml @@ -134,6 +134,8 @@ ray-cluster: value: "password" - name: MYSQL_DATABASE value: "datamate" + - name: PDF_FORMATTER_BASE_URL + value: "http://datamate-mineru:9001" ports: - containerPort: 8081 volumeMounts: