Skip to content

Commit 1bced5c

Browse files
hefanlihefanli
committed
Merge branch 'main' into dev_0119
# Conflicts: # backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorViewRepositoryImpl.java # backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/mapper/OperatorViewMapper.java # frontend/src/utils/request.ts # scripts/db/data-operator-init.sql
2 parents d9aff0b + 3963998 commit 1bced5c

61 files changed

Lines changed: 3402 additions & 489 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile‎

Lines changed: 22 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ endef
152152
# ========== Build Targets ==========
153153

154154
# Valid build targets
155-
VALID_BUILD_TARGETS := backend database frontend runtime backend-python deer-flow mineru mineru-npu gateway label-studio
155+
VALID_BUILD_TARGETS := frontend backend gateway database runtime backend-python deer-flow label-studio mineru mineru-910B mineru-910C mineru-310P
156156

157157
# Generic docker build target with service name as parameter
158158
# Automatically prefixes image names with "datamate-" unless it's deer-flow
@@ -169,6 +169,12 @@ VALID_BUILD_TARGETS := backend database frontend runtime backend-python deer-flo
169169
@if [ "$*" = "deer-flow" ]; then \
170170
$(call docker-build,deer-flow-backend,deer-flow-backend); \
171171
$(call docker-build,deer-flow-frontend,deer-flow-frontend); \
172+
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ]; then \
173+
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | docker build --network=host -t datamate-mineru -f - .; \
174+
elif [ "$*" = "mineru-910C" ]; then \
175+
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.11.0-a3/g' | docker build --network=host -t datamate-mineru -f - .; \
176+
elif [ "$*" = "mineru-310P" ]; then \
177+
wget -qO - https://gcore.jsdelivr.net/gh/opendatalab/MinerU@master/docker/china/npu.Dockerfile | sed 's/v0.11.0/v0.10.0rc1-310p/g' | docker build --network=host -t datamate-mineru -f - .; \
172178
else \
173179
$(call docker-build,$*,datamate-$*); \
174180
fi
@@ -235,7 +241,7 @@ endif
235241
# ========== Docker Install/Uninstall Targets ==========
236242

237243
# Valid service targets for docker install/uninstall
238-
VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python database gateway redis mineru deer-flow milvus label-studio data-juicer dj
244+
VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python database gateway redis deer-flow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
239245

240246
# Generic docker service install target
241247
.PHONY: %-docker-install
@@ -252,9 +258,11 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python databa
252258
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile label-studio up -d; \
253259
elif [ "$*" = "datamate" ]; then \
254260
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml up -d; \
255-
elif [ "$*" = "mineru" ]; then \
261+
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
256262
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile mineru up -d datamate-mineru; \
257-
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
263+
elif [ "$*" = "mineru-310P" ]; then \
264+
REGISTRY=$(REGISTRY) EXTRA_ARGS="--enforce-eager --dtype float16" docker compose -f deployment/docker/datamate/docker-compose.yml --profile mineru up -d datamate-mineru; \
265+
elif [ "$*" = "data-juicer" ]; then \
258266
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile data-juicer up -d datamate-data-juicer; \
259267
elif [ "$*" = "redis" ]; then \
260268
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile redis up -d datamate-redis; \
@@ -281,7 +289,7 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python databa
281289
fi
282290
@if [ "$*" = "label-studio" ]; then \
283291
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s label-studio; \
284-
elif [ "$*" = "mineru" ]; then \
292+
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ] || [ "$*" = "mineru-310P" ]; then \
285293
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s datamate-mineru; \
286294
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
287295
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s datamate-data-juicer; \
@@ -304,7 +312,7 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python databa
304312
# ========== Kubernetes Install/Uninstall Targets ==========
305313

306314
# Valid k8s targets
307-
VALID_K8S_TARGETS := mineru datamate deer-flow milvus label-studio data-juicer dj
315+
VALID_K8S_TARGETS := datamate deer-flow milvus label-studio data-juicer mineru mineru-910B mineru-910C mineru-310P
308316

309317
# Generic k8s install target
310318
.PHONY: %-k8s-install
@@ -319,8 +327,10 @@ VALID_K8S_TARGETS := mineru datamate deer-flow milvus label-studio data-juicer d
319327
fi
320328
@if [ "$*" = "label-studio" ]; then \
321329
helm upgrade label-studio deployment/helm/label-studio/ -n $(NAMESPACE) --install; \
322-
elif [ "$*" = "mineru" ]; then \
323-
kubectl apply -f deployment/kubernetes/mineru/deploy.yaml -n $(NAMESPACE); \
330+
elif [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
331+
kubectl apply -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
332+
elif [ "$*" = "mineru-310P" ]; then \
333+
kubectl apply -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
324334
elif [ "$*" = "datamate" ]; then \
325335
helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.image.repository=$(REGISTRY); \
326336
elif [ "$*" = "deer-flow" ]; then \
@@ -346,8 +356,10 @@ VALID_K8S_TARGETS := mineru datamate deer-flow milvus label-studio data-juicer d
346356
done; \
347357
exit 1; \
348358
fi
349-
@if [ "$*" = "mineru" ]; then \
350-
kubectl delete -f deployment/kubernetes/mineru/deploy.yaml -n $(NAMESPACE); \
359+
@if [ "$*" = "mineru" ] || [ "$*" = "mineru-910B" ] || [ "$*" = "mineru-910C" ]; then \
360+
kubectl delete -f deployment/kubernetes/mineru/deploy-910.yaml -n $(NAMESPACE); \
361+
elif [ "$*" = "mineru-310P" ]; then \
362+
kubectl delete -f deployment/kubernetes/mineru/deploy-310.yaml -n $(NAMESPACE); \
351363
elif [ "$*" = "datamate" ]; then \
352364
helm uninstall datamate -n $(NAMESPACE) --ignore-not-found; \
353365
elif [ "$*" = "deer-flow" ]; then \
@@ -360,26 +372,6 @@ VALID_K8S_TARGETS := mineru datamate deer-flow milvus label-studio data-juicer d
360372
kubectl delete -f deployment/kubernetes/data-juicer/deploy.yaml -n $(NAMESPACE); \
361373
fi
362374

363-
# ========== Upgrade Targets ==========
364-
365-
# Valid upgrade targets
366-
VALID_UPGRADE_TARGETS := datamate
367-
368-
# Generic docker upgrade target
369-
.PHONY: %-docker-upgrade
370-
%-docker-upgrade:
371-
@if ! echo " $(VALID_UPGRADE_TARGETS) " | grep -q " $* "; then \
372-
echo "Error: Unknown upgrade target '$*'"; \
373-
echo "Valid upgrade targets are:"; \
374-
for target in $(VALID_UPGRADE_TARGETS); do \
375-
echo " - $$target"; \
376-
done; \
377-
exit 1; \
378-
fi
379-
@if [ "$*" = "datamate" ]; then \
380-
docker compose -f deployment/docker/datamate/docker-compose.yml --profile mineru up -d --force-recreate --remove-orphans; \
381-
fi
382-
383375
# ========== Download Targets ==========
384376

385377
# List of all images to download

‎backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/application/CleaningTaskService.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,9 @@ public CleaningTaskDto createTask(CreateCleaningTaskRequest request) {
136136
cleaningTaskRepo.insertTask(task);
137137

138138
operatorInstanceRepo.insertInstance(taskId, request.getInstance());
139+
operatorRepo.incrementUsageCount(request.getInstance().stream()
140+
.map(OperatorInstanceDto::getId)
141+
.collect(Collectors.toList()));
139142

140143
prepareTask(task, request.getInstance(), executorType);
141144
scanDataset(taskId, request.getSrcDatasetId());

‎backend/services/operator-market-service/src/main/java/com/datamate/operator/application/OperatorService.java‎

Lines changed: 90 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.datamate.common.infrastructure.exception.BusinessException;
66
import com.datamate.common.infrastructure.exception.SystemErrorCode;
77
import com.datamate.operator.domain.contants.OperatorConstant;
8+
import com.datamate.operator.domain.repository.OperatorReleaseRepository;
89
import com.datamate.operator.infrastructure.converter.OperatorConverter;
910
import com.datamate.operator.domain.model.OperatorView;
1011
import com.datamate.operator.domain.repository.CategoryRelationRepository;
@@ -13,13 +14,15 @@
1314
import com.datamate.operator.infrastructure.exception.OperatorErrorCode;
1415
import com.datamate.operator.infrastructure.parser.ParserHolder;
1516
import com.datamate.operator.interfaces.dto.OperatorDto;
17+
import com.datamate.operator.interfaces.dto.OperatorReleaseDto;
1618
import com.datamate.operator.interfaces.dto.UploadOperatorRequest;
1719
import com.fasterxml.jackson.core.JsonProcessingException;
1820
import com.fasterxml.jackson.databind.ObjectMapper;
1921
import lombok.RequiredArgsConstructor;
2022
import lombok.extern.slf4j.Slf4j;
2123
import org.apache.commons.collections4.CollectionUtils;
2224
import org.apache.commons.collections4.MapUtils;
25+
import org.apache.commons.io.FileUtils;
2326
import org.apache.commons.lang3.StringUtils;
2427
import org.springframework.beans.factory.annotation.Value;
2528
import org.springframework.core.io.Resource;
@@ -28,10 +31,16 @@
2831
import org.springframework.transaction.annotation.Transactional;
2932

3033
import java.io.File;
34+
import java.io.IOException;
3135
import java.net.MalformedURLException;
32-
import java.util.ArrayList;
33-
import java.util.List;
34-
import java.util.Map;
36+
import java.nio.charset.StandardCharsets;
37+
import java.nio.file.Files;
38+
import java.nio.file.Path;
39+
import java.nio.file.Paths;
40+
import java.time.LocalDateTime;
41+
import java.util.*;
42+
import java.util.stream.Collectors;
43+
import java.util.stream.Stream;
3544

3645
@Service
3746
@Slf4j
@@ -43,6 +52,8 @@ public class OperatorService {
4352

4453
private final CategoryRelationRepository relationRepo;
4554

55+
private final OperatorReleaseRepository operatorReleaseRepo;
56+
4657
private final ParserHolder parserHolder;
4758

4859
private final FileService fileService;
@@ -63,29 +74,57 @@ public int getOperatorsCount(List<List<String>> categories, String keyword, Bool
6374

6475
public OperatorDto getOperatorById(String id) {
6576
OperatorView operator = operatorViewRepo.findOperatorById(id);
66-
return OperatorConverter.INSTANCE.fromEntityToDto(operator);
77+
OperatorDto operatorDto = OperatorConverter.INSTANCE.fromEntityToDto(operator);
78+
if (StringUtils.isNotBlank(operatorDto.getFileName())) {
79+
String filePath = getExtractPath(getStem(operatorDto.getFileName()));
80+
String requirements = filePath + "/requirements.txt";
81+
operatorDto.setRequirements(readRequirements(requirements));
82+
operatorDto.setReadme(getReadmeContent(filePath));
83+
}
84+
operatorDto.setFileName(null);
85+
operatorDto.setReleases(operatorReleaseRepo.findAllByOperatorId(id));
86+
return operatorDto;
6787
}
6888

6989
@Transactional
7090
public OperatorDto createOperator(OperatorDto req) {
7191
overrideSettings(req);
7292
operatorRepo.insertOperator(req);
7393
relationRepo.batchInsert(req.getId(), req.getCategories());
94+
if (CollectionUtils.isNotEmpty(req.getReleases())) {
95+
OperatorReleaseDto release = req.getReleases().getFirst();
96+
release.setId(req.getId());
97+
release.setVersion(req.getVersion());
98+
release.setReleaseDate(LocalDateTime.now());
99+
operatorReleaseRepo.insertOperatorRelease(release);
100+
}
74101
parserHolder.extractTo(getFileType(req.getFileName()), getUploadPath(req.getFileName()),
75-
getExtractPath(getFileNameWithoutExtension(req.getFileName())));
102+
getExtractPath(getStem(req.getFileName())));
76103
return getOperatorById(req.getId());
77104
}
78105

79106
@Transactional
80107
public OperatorDto updateOperator(String id, OperatorDto req) {
108+
OperatorDto operator = getOperatorById(id);
81109
overrideSettings(req);
82110
operatorRepo.updateOperator(req);
83-
if (CollectionUtils.isNotEmpty(req.getCategories())) {
111+
if (StringUtils.isNotBlank(req.getFileName()) && CollectionUtils.isNotEmpty(req.getCategories())) {
84112
relationRepo.batchUpdate(id, req.getCategories());
85113
}
114+
if (CollectionUtils.isNotEmpty(req.getReleases())) {
115+
OperatorReleaseDto release = req.getReleases().getFirst();
116+
release.setId(req.getId());
117+
release.setVersion(req.getVersion());
118+
release.setReleaseDate(LocalDateTime.now());
119+
if (StringUtils.equals(operator.getVersion(), req.getVersion())) {
120+
operatorReleaseRepo.updateOperatorRelease(release);
121+
} else {
122+
operatorReleaseRepo.insertOperatorRelease(release);
123+
}
124+
}
86125
if (StringUtils.isNotBlank(req.getFileName())) {
87126
parserHolder.extractTo(getFileType(req.getFileName()), getUploadPath(req.getFileName()),
88-
getExtractPath(getFileNameWithoutExtension(req.getFileName())));
127+
getExtractPath(getStem(req.getFileName())));
89128
}
90129
return getOperatorById(id);
91130
}
@@ -98,8 +137,11 @@ public void deleteOperator(String id) {
98137
if (relationRepo.operatorIsPredefined(id)) {
99138
throw BusinessException.of(OperatorErrorCode.CANT_DELETE_PREDEFINED_OPERATOR);
100139
}
140+
OperatorView operator = operatorViewRepo.findOperatorById(id);
101141
operatorRepo.deleteOperator(id);
102142
relationRepo.deleteByOperatorId(id);
143+
operatorReleaseRepo.deleteOperatorRelease(id);
144+
FileUtils.deleteQuietly(new File(getExtractPath(getStem(operator.getFileName()))));
103145
}
104146

105147
public OperatorDto uploadOperator(String fileName) {
@@ -123,7 +165,7 @@ private String getFileType(String fileName) {
123165
return fileName.substring(fileName.lastIndexOf('.') + 1);
124166
}
125167

126-
private String getFileNameWithoutExtension(String fileName) {
168+
private String getStem(String fileName) {
127169
return fileName.substring(0, fileName.lastIndexOf('.'));
128170
}
129171

@@ -220,4 +262,44 @@ private void updateProperties(Map<String, Object> setting, Object value) {
220262
setting.put("properties", result);
221263
}
222264
}
265+
266+
private List<String> readRequirements(String filePath) {
267+
Path path = Paths.get(filePath);
268+
if (!Files.exists(path) || !Files.isRegularFile(path)) {
269+
log.warn("requirements文件不存在或路径错误: {}", filePath);
270+
return Collections.emptyList();
271+
}
272+
273+
List<String> requirements = new ArrayList<>();
274+
try (Stream<String> lines = Files.lines(path)) {
275+
requirements = lines.map(String::trim)
276+
.filter(line -> !line.isEmpty())
277+
.filter(line -> !line.startsWith("#"))
278+
.collect(Collectors.toList());
279+
} catch (IOException e) {
280+
log.warn("读取requirements文件异常: {}", e.getMessage());
281+
}
282+
return requirements;
283+
}
284+
285+
private String getReadmeContent(String directoryPath) {
286+
Path dir = Paths.get(directoryPath);
287+
if (!Files.exists(dir) || !Files.isDirectory(dir)) {
288+
System.err.println("目录不存在: " + directoryPath);
289+
return null;
290+
}
291+
List<String> candidateNames = Arrays.asList("README.md", "readme.md", "Readme.md");
292+
for (String fileName : candidateNames) {
293+
Path filePath = dir.resolve(fileName);
294+
if (Files.exists(filePath) && Files.isRegularFile(filePath)) {
295+
try {
296+
byte[] bytes = Files.readAllBytes(filePath);
297+
return new String(bytes, StandardCharsets.UTF_8);
298+
} catch (IOException e) {
299+
log.warn("找到文件但读取失败: {}, 错误: {}", filePath, e.getMessage());
300+
}
301+
}
302+
}
303+
return "";
304+
}
223305
}

‎backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/model/Operator.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ public class Operator extends BaseEntity<String> {
2727

2828
private String fileName;
2929

30+
private Long fileSize;
31+
32+
private String metrics;
33+
34+
private Integer usageCount;
35+
3036
private Boolean isStar;
3137
}
3238

‎backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/model/OperatorRelease.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
package com.datamate.operator.domain.model;
2+
3+
import com.baomidou.mybatisplus.annotation.TableField;
4+
import com.baomidou.mybatisplus.annotation.TableName;
5+
import com.datamate.common.infrastructure.config.PgJsonTypeHandler;
6+
import lombok.Getter;
7+
import lombok.Setter;
8+
9+
import java.time.LocalDateTime;
10+
import java.util.List;
11+
12+
@Getter
13+
@Setter
14+
@TableName(value = "t_operator_release", autoResultMap = true)
15+
public class OperatorRelease {
16+
private String id;
17+
18+
private String version;
19+
20+
private LocalDateTime releaseDate;
21+
22+
@TableField(typeHandler = PgJsonTypeHandler.class)
23+
private List<String> changelog;
24+
}

‎backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/model/OperatorView.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,14 @@ public class OperatorView {
3131

3232
private String settings;
3333

34+
private String fileName;
35+
36+
private Long fileSize;
37+
38+
private String metrics;
39+
40+
private Integer usageCount;
41+
3442
private Boolean isStar;
3543

3644
private LocalDateTime createdAt;

‎backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/repository/OperatorReleaseRepository.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
package com.datamate.operator.domain.repository;
2+
3+
import com.baomidou.mybatisplus.extension.repository.IRepository;
4+
import com.datamate.operator.domain.model.OperatorRelease;
5+
import com.datamate.operator.interfaces.dto.OperatorReleaseDto;
6+
7+
import java.util.List;
8+
9+
public interface OperatorReleaseRepository extends IRepository<OperatorRelease> {
10+
List<OperatorReleaseDto> findAllByOperatorId(String operatorId);
11+
12+
void insertOperatorRelease(OperatorReleaseDto operatorRelease);
13+
14+
void updateOperatorRelease(OperatorReleaseDto operatorRelease);
15+
16+
void deleteOperatorRelease(String operatorId);
17+
}

‎backend/services/operator-market-service/src/main/java/com/datamate/operator/domain/repository/OperatorRepository.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,6 @@ public interface OperatorRepository extends IRepository<Operator> {
1818
int countOperatorByStar(boolean isStar);
1919

2020
boolean operatorInTemplateOrRunning(String operatorId);
21+
22+
void incrementUsageCount(List<String> operatorIds);
2123
}

0 commit comments

Comments
 (0)