feature: 对接deer-flow (#54)

feature: 对接deer-flow

Author: hhhhsc701

.github/workflows/docker-image-deer-flow.yml

@@ -0,0 +1,64 @@
+name: Deer Flow Docker Image CI
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'runtime/deer-flow/**'
+      - 'scripts/images/deer-flow-backend/**'
+      - 'scripts/images/deer-flow-frontend/**'
+      - '.github/workflows/docker-image-deer-flow.yml'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'runtime/deer-flow/**'
+      - 'scripts/images/deer-flow-backend/**'
+      - 'scripts/images/deer-flow-frontend/**'
+      - '.github/workflows/docker-image-deer-flow.yml'
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Login to GitHub Container Registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set Docker Image Tag
+        id: set-tag
+        run: |
+          if [[ $GITHUB_REF == refs/tags/v* ]]; then
+            TAG=${GITHUB_REF#refs/tags/v}
+            echo "TAGS=$TAG" >> $GITHUB_OUTPUT
+          elif [[ $GITHUB_REF == refs/heads/main ]]; then
+            echo "TAGS=latest" >> $GITHUB_OUTPUT
+          else
+            echo "TAGS=temp" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Build Docker Image
+        run: |
+          make build-deer-flow VERSION=latest
+
+      - name: Tag Docker Image
+        run: |
+          LOWERCASE_REPO=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')
+          docker tag deer-flow-backend:latest ghcr.io/$LOWERCASE_REPO/deer-flow-backend:${{ steps.set-tag.outputs.TAGS }}
+          docker tag deer-flow-frontend:latest ghcr.io/$LOWERCASE_REPO/deer-flow-frontend:${{ steps.set-tag.outputs.TAGS }}
+
+      - name: Push Docker Image
+        if: github.event_name != 'pull_request'
+        run: |
+          docker push ghcr.io/$LOWERCASE_REPO/deer-flow-backend:${{ steps.set-tag.outputs.TAGS }}
+          docker push ghcr.io/$LOWERCASE_REPO/deer-flow-frontend:${{ steps.set-tag.outputs.TAGS }}

Makefile

@@ -4,6 +4,14 @@ WITH_MINERU ?= false  # 默认不构建mineru
 VERSION ?= latest
 NAMESPACE ?= datamate
 
+ifdef COMSPEC
+    # Windows 环境
+    MAKE := "C:/Program Files (x86)/GnuWin32/bin/make"
+else
+    # Linux/Mac 环境
+    MAKE := make
+endif
+
 .PHONY: build-%
 build-%:
 	$(MAKE) $*-docker-build
@@ -76,15 +84,10 @@ label-studio-adapter-docker-build:
 
 .PHONY: deer-flow-docker-build
 deer-flow-docker-build:
-	@if [ -d "../deer-flow/.git" ]; then \
-		cd ../deer-flow && git pull; \
-	else \
-		git clone git@github.com:bytedance/deer-flow.git ../deer-flow; \
-	fi
-	sed -i "s/dark/light/g" "../deer-flow/web/src/components/deer-flow/theme-provider-wrapper.tsx"
-	cp -n deployment/docker/deer-flow/.env.example ../deer-flow/.env
-	cp -n deployment/docker/deer-flow/conf.yaml.example ../deer-flow/conf.yaml
-	cd ../deer-flow && docker compose build
+	cp -n runtime/deer-flow/.env.example runtime/deer-flow/.env
+	cp -n runtime/deer-flow/conf.yaml.example runtime/deer-flow/conf.yaml
+	docker build -t deer-flow-backend:$(VERSION) . -f scripts/images/deer-flow-backend/Dockerfile
+	docker build -t deer-flow-frontend:$(VERSION) . -f scripts/images/deer-flow-frontend/Dockerfile
 
 .PHONY: mineru-docker-build
 mineru-docker-build:
@@ -131,16 +134,19 @@ mineru-k8s-uninstall:
 
 .PHONY: datamate-docker-install
 datamate-docker-install:
-	cd deployment/docker/datamate && cp -n .env.example .env && docker compose -f docker-compose.yml up -d
+	cd deployment/docker/datamate && cp .env.example .env && docker compose -f docker-compose.yml up -d
 
 .PHONY: datamate-docker-uninstall
 datamate-docker-uninstall:
 	cd deployment/docker/datamate && docker compose -f docker-compose.yml down -v
 
 .PHONY: deer-flow-docker-install
 deer-flow-docker-install:
-	cd deployment/docker/datamate && cp -n .env.deer-flow.example .env && docker compose -f docker-compose.yml up -d
-	cd deployment/docker/deer-flow && cp -n .env.example .env && cp -n conf.yaml.example conf.yaml && docker compose -f docker-compose.yml up -d
+	cd deployment/docker/datamate && cp .env.deer-flow.example .env && docker compose -f docker-compose.yml up -d
+	cp -n runtime/deer-flow/.env.example runtime/deer-flow/.env
+	cp -n runtime/deer-flow/conf.yaml.example runtime/deer-flow/conf.yaml
+	cp runtime/deer-flow/.env deployment/docker/deer-flow/.env && cp runtime/deer-flow/conf.yaml deployment/docker/deer-flow/conf.yaml
+	cd deployment/docker/deer-flow && docker compose -f docker-compose.yml up -d
 
 .PHONY: deer-flow-docker-uninstall
 deer-flow-docker-uninstall:
@@ -158,3 +164,22 @@ datamate-k8s-install: create-namespace
 datamate-k8s-uninstall:
 	helm uninstall datamate -n $(NAMESPACE) --ignore-not-found
 	kubectl delete configmap datamate-init-sql -n $(NAMESPACE) --ignore-not-found
+
+.PHONY: deer-flow-k8s-install
+deer-flow-k8s-install:
+	helm upgrade datamate deployment/helm/datamate/ -n $(NAMESPACE) --install --set global.deerFlow.enable=true
+	cp runtime/deer-flow/.env deployment/helm/deer-flow/charts/public/.env
+	cp runtime/deer-flow/conf.yaml deployment/helm/deer-flow/charts/public/conf.yaml
+	helm upgrade deer-flow deployment/helm/deer-flow -n $(NAMESPACE) --install
+
+.PHONY: deer-flow-k8s-uninstall
+deer-flow-k8s-uninstall:
+	helm uninstall deer-flow -n $(NAMESPACE) --ignore-not-found
+
+.PHONY: milvus-k8s-install
+milvus-k8s-install:
+	helm upgrade milvus deployment/helm/milvus -n $(NAMESPACE) --install
+
+.PHONY: milvus-k8s-uninstall
+milvus-k8s-uninstall:
+	helm uninstall milvus -n $(NAMESPACE) --ignore-not-found

backend/services/main-application/src/main/resources/application.yml

@@ -164,8 +164,3 @@ datamate:
     quality-control:
       enabled: ${QC_ENABLED:true}
       threshold: ${QC_THRESHOLD:0.8}
-
-  # RAG配置
-  rag:
-    milvus-host: ${MILVUS_HOST:milvus-standalone}
-    milvus-port: ${MILVUS_PORT:19530}

backend/services/operator-market-service/src/main/java/com/datamate/operator/application/OperatorService.java

@@ -32,7 +32,7 @@ public class OperatorService {
 
     private final FileService fileService;
 
-    @Value("${operator.base.path:/operator}")
+    @Value("${operator.base.path:/operators}")
     private String operatorBasePath;
 
     public List<OperatorDto> getOperators(Integer page, Integer size, List<String> categories,

backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/infrastructure/event/RagEtlService.java

@@ -51,9 +51,9 @@
 public class RagEtlService {
     private static final Semaphore SEMAPHORE = new Semaphore(10);
 
-    @Value("${datamate.rag.milvus-host}")
+    @Value("${datamate.rag.milvus-host:-milvus-standalone}")
     private String milvusHost;
-    @Value("${datamate.rag.milvus-port}")
+    @Value("${datamate.rag.milvus-port:-19530}")
     private int milvusPort;
 
     private final RagFileRepository ragFileRepository;
@@ -122,7 +122,7 @@ private void processRagFile(RagFile ragFile, DataInsertedEvent event) {
         // 调用嵌入模型获取嵌入向量
         List<Embedding> content = embeddingModel.embedAll(split).content();
         // 存储嵌入向量到 Milvus
-        embeddingStore(embeddingModel, ragFile.getKnowledgeBaseId()).addAll(content, split);
+        embeddingStore(embeddingModel, event.knowledgeBase().getName()).addAll(content, split);
     }
 
     /**
@@ -152,11 +152,11 @@ public DocumentSplitter documentSplitter(ProcessType processType) {
         };
     }
 
-    public EmbeddingStore<TextSegment> embeddingStore(EmbeddingModel embeddingModel, String knowledgeBaseId) {
+    public EmbeddingStore<TextSegment> embeddingStore(EmbeddingModel embeddingModel, String knowledgeBaseName) {
         return MilvusEmbeddingStore.builder()
                 .host(milvusHost)
                 .port(milvusPort)
-                .collectionName("datamate_" + knowledgeBaseId)
+                .collectionName(knowledgeBaseName)
                 .dimension(embeddingModel.dimension())
                 .build();
     }

deployment/helm/datamate/charts/backend/values.yaml

@@ -19,13 +19,7 @@ imagePullSecrets: []
 nameOverride: "datamate-backend"
 fullnameOverride: "datamate-backend"
 
-env:
-  - name: namespace
-    valueFrom:
-      fieldRef:
-        fieldPath: metadata.namespace
-  - name: SPRING_CONFIG_LOCATION
-    value: file:/opt/backend/application.yml
+env: []
 
 # This section builds out the service account more information can be found here: https://kubernetes.io/docs/concepts/security/service-accounts/
 serviceAccount:

deployment/helm/datamate/charts/frontend/templates/configmap.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.global.deerFlow.enable }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -13,16 +14,72 @@ data:
 
         client_max_body_size 1024M;
 
+        add_header Set-Cookie "NEXT_LOCALE=zh";
+
         location /api/ {
             proxy_pass http://datamate-backend:8080/api/;
             proxy_set_header Host $host;
             proxy_set_header X-Real-IP $remote_addr;
             proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
         }
 
+        location /chat {
+            proxy_pass http://deer-flow-frontend:3000/chat;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        location /_next {
+            proxy_pass http://deer-flow-frontend:3000/_next;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        location /deer-flow-backend/ {
+            proxy_pass http://deer-flow-backend:8000/api/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
         location / {
+            if ($query_string ~* "_rsc=pmmii") {
+                proxy_pass http://deer-flow-frontend:3000;
+                break;
+            }
+
             root   /opt/frontend;
             try_files $uri $uri/ /index.html;
         }
     }
+{{- else }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: datamate-nginx-conf
+data:
+  backend.conf: |
+    server {
+        listen       80;
+        server_name  0.0.0.0;
 
+        access_log /var/log/datamate/frontend/access.log main;
+        error_log /var/log/datamate/frontend/error.log notice;
+
+        client_max_body_size 1024M;
+
+        location /api/ {
+            proxy_pass http://datamate-backend:8080/api/;
+            proxy_set_header Host $host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        }
+
+        location / {
+            root   /opt/frontend;
+            try_files $uri $uri/ /index.html;
+        }
+    }
+{{- end }}

deployment/helm/datamate/charts/frontend/templates/deployment.yaml

@@ -13,10 +13,11 @@ spec:
       {{- include "frontend.selectorLabels" . | nindent 6 }}
   template:
     metadata:
-      {{- with .Values.podAnnotations }}
       annotations:
+        checksum/nginx: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- with .Values.podAnnotations }}
         {{- toYaml . | nindent 8 }}
-      {{- end }}
+        {{- end }}
       labels:
         {{- include "frontend.labels" . | nindent 8 }}
         {{- with .Values.podLabels }}

deployment/helm/deer-flow/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployment/helm/deer-flow/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: deer-flow
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.0.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.0.1"