Skip to content

Commit 76f0d26

Browse files
Dallas98Dallas98
authored
feat: enhance Docker configuration with additional services and profiles (#231)
* feat: update error handling in RagEtlService and add commons-io dependency * feat: enhance Docker configuration with additional services and profiles
1 parent 5e6a418 commit 76f0d26

15 files changed

Lines changed: 242 additions & 657 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,4 +189,4 @@ Thumbs.db
189189
*.sublime-workspace
190190

191191
# Milvus
192-
deployment/docker/milvus/volumes/
192+
**/volumes/

‎Makefile‎

Lines changed: 24 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,7 @@ endif
238238
# ========== Docker Install/Uninstall Targets ==========
239239

240240
# Valid service targets for docker install/uninstall
241-
VALID_SERVICE_TARGETS := datamate backend frontend runtime mineru "deer-flow" milvus "label-studio" "data-juicer" dj
241+
VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python database gateway redis mineru deer-flow milvus label-studio data-juicer dj
242242

243243
# Generic docker service install target
244244
.PHONY: %-docker-install
@@ -252,21 +252,23 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime mineru "deer-flow" mi
252252
exit 1; \
253253
fi
254254
@if [ "$*" = "label-studio" ]; then \
255-
$(call docker-compose-service,label-studio,up -d,deployment/docker/label-studio); \
256-
elif [ "$*" = "mineru" ]; then \
257-
REGISTRY=$(REGISTRY) && docker compose -f deployment/docker/datamate/docker-compose.yml up -d datamate-mineru; \
255+
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile label-studio up -d; \
258256
elif [ "$*" = "datamate" ]; then \
259257
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml up -d; \
260-
elif [ "$*" = "deer-flow" ]; then \
261-
cp runtime/deer-flow/.env deployment/docker/deer-flow/.env; \
262-
cp runtime/deer-flow/conf.yaml deployment/docker/deer-flow/conf.yaml; \
263-
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/deer-flow/docker-compose.yml up -d; \
264-
elif [ "$*" = "milvus" ]; then \
265-
docker compose -f deployment/docker/milvus/docker-compose.yml up -d; \
258+
elif [ "$*" = "mineru" ]; then \
259+
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile mineru up -d datamate-mineru; \
266260
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
267-
REGISTRY=$(REGISTRY) && docker compose -f deployment/docker/datamate/docker-compose.yml up -d datamate-data-juicer; \
261+
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile data-juicer up -d datamate-data-juicer; \
262+
elif [ "$*" = "redis" ]; then \
263+
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile redis up -d datamate-redis; \
264+
elif [ "$*" = "milvus" ]; then \
265+
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile milvus up -d; \
266+
elif [ "$*" = "deer-flow" ]; then \
267+
cp runtime/deer-flow/.env deployment/docker/datamate/.env; \
268+
cp runtime/deer-flow/conf.yaml deployment/docker/datamate/conf.yaml; \
269+
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml --profile deer-flow up -d; \
268270
else \
269-
$(call docker-compose-service,$*,up -d,deployment/docker/datamate); \
271+
REGISTRY=$(REGISTRY) docker compose -f deployment/docker/datamate/docker-compose.yml up -d datamate-$*; \
270272
fi
271273

272274
# Generic docker service uninstall target
@@ -281,29 +283,23 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime mineru "deer-flow" mi
281283
exit 1; \
282284
fi
283285
@if [ "$*" = "label-studio" ]; then \
284-
if [ "$(DELETE_VOLUMES_CHOICE)" = "1" ]; then \
285-
cd deployment/docker/label-studio && docker compose down -v && cd - >/dev/null; \
286-
else \
287-
cd deployment/docker/label-studio && docker compose down && cd - >/dev/null; \
288-
fi; \
286+
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s label-studio pg-db; \
289287
elif [ "$*" = "mineru" ]; then \
290-
$(call docker-compose-service,datamate-mineru,down,deployment/docker/datamate); \
288+
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s datamate-mineru; \
289+
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
290+
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s datamate-data-juicer; \
291+
elif [ "$*" = "redis" ]; then \
292+
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s datamate-redis; \
291293
elif [ "$*" = "datamate" ]; then \
292294
if [ "$(DELETE_VOLUMES_CHOICE)" = "1" ]; then \
293-
docker compose -f deployment/docker/datamate/docker-compose.yml --profile mineru down -v; \
295+
docker compose -f deployment/docker/datamate/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deer-flow --profile label-studio --profile milvus down -v; \
294296
else \
295-
docker compose -f deployment/docker/datamate/docker-compose.yml --profile mineru down; \
297+
docker compose -f deployment/docker/datamate/docker-compose.yml --profile mineru --profile redis --profile data-juicer --profile deer-flow --profile label-studio --profile milvus down; \
296298
fi; \
297299
elif [ "$*" = "deer-flow" ]; then \
298-
docker compose -f deployment/docker/deer-flow/docker-compose.yml down; \
300+
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s deer-flow-backend deer-flow-frontend; \
299301
elif [ "$*" = "milvus" ]; then \
300-
if [ "$(DELETE_VOLUMES_CHOICE)" = "1" ]; then \
301-
docker compose -f deployment/docker/milvus/docker-compose.yml down -v; \
302-
else \
303-
docker compose -f deployment/docker/milvus/docker-compose.yml down; \
304-
fi; \
305-
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \
306-
$(call docker-compose-service,datamate-data-juicer,down,deployment/docker/datamate); \
302+
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s milvus etcd minio; \
307303
else \
308304
$(call docker-compose-service,$*,down,deployment/docker/datamate); \
309305
fi

‎backend/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@
6565
<type>pom</type>
6666
<scope>import</scope>
6767
</dependency>
68+
<dependency>
69+
<groupId>commons-io</groupId>
70+
<artifactId>commons-io</artifactId>
71+
<version>2.16.1</version>
72+
<scope>compile</scope>
73+
</dependency>
6874
<dependency>
6975
<groupId>com.google.protobuf</groupId>
7076
<artifactId>protobuf-bom</artifactId>

‎backend/services/rag-indexer-service/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,12 @@
8686
<dependency>
8787
<groupId>dev.langchain4j</groupId>
8888
<artifactId>langchain4j-document-parser-apache-poi</artifactId>
89+
<exclusions>
90+
<exclusion>
91+
<groupId>commons-io</groupId>
92+
<artifactId>commons-io</artifactId>
93+
</exclusion>
94+
</exclusions>
8995
</dependency>
9096
<dependency>
9197
<groupId>dev.langchain4j</groupId>

‎backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/infrastructure/event/RagEtlService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ public void processAfterCommit(DataInsertedEvent event) {
7878
// 更新文件状态为已处理
7979
ragFile.setStatus(FileStatus.PROCESSED);
8080
ragFileRepository.updateById(ragFile);
81-
} catch (Exception e) {
81+
} catch (Throwable e) {
8282
// 处理异常
8383
log.error("Error processing RAG file: {}", ragFile.getFileId(), e);
8484
ragFile.setStatus(FileStatus.PROCESS_FAILED);

0 commit comments

Comments
 (0)