feature: Implement the basic knowledge generation function

.editorconfig

@@ -4,7 +4,7 @@ root = true
 charset = utf-8
 end_of_line = lf
 indent_style = space
-indent_size = 2
+indent_size = 4
 insert_final_newline = true
 trim_trailing_whitespace = true
 

backend/services/main-application/pom.xml

@@ -130,6 +130,10 @@
             <artifactId>spring-boot-starter-test</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-autoconfigure</artifactId>
+        </dependency>
     </dependencies>
 
     <build>
@@ -141,6 +145,7 @@
                 <configuration>
                     <source>${maven.compiler.source}</source>
                     <target>${maven.compiler.target}</target>
+                    <parameters>true</parameters>
                     <compilerArgs>
                         <arg>-parameters</arg>
                     </compilerArgs>

backend/services/main-application/src/main/java/com/datamate/main/DataMatePlatformApplication.java

@@ -16,31 +16,12 @@
  * @version 1.0.0
  */
 @SpringBootApplication
-@ComponentScan(basePackages = {
-        "com.datamate.main",
-        "com.datamate.datamanagement",
-        "com.datamate.collection",
-        "com.datamate.operator",
-        "com.datamate.cleaning",
-        "com.datamate.synthesis",
-        "com.datamate.annotation",
-        "com.datamate.evaluation",
-        "com.datamate.pipeline",
-        "com.datamate.execution",
-        "com.datamate.common"
-})
-@MapperScan(basePackages = {
-        "com.datamate.collection.infrastructure.persistence.mapper",
-        "com.datamate.datamanagement.infrastructure.persistence.mapper",
-        "com.datamate.operator.infrastructure.persistence.mapper",
-        "com.datamate.cleaning.infrastructure.persistence.mapper",
-        "com.datamate.**.mapper"
-})
+@ComponentScan(basePackages = {"com.datamate"})
+@MapperScan(basePackages = {"com.datamate.**.mapper"})
 @EnableTransactionManagement
 @EnableAsync
 @EnableScheduling
 public class DataMatePlatformApplication {
-
     public static void main(String[] args) {
         SpringApplication.run(DataMatePlatformApplication.class, args);
     }

backend/services/rag-indexer-service/pom.xml

@@ -16,45 +16,97 @@
     <name>RAG Indexer Service</name>
     <description>RAG文档索引服务</description>
 
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>dev.langchain4j</groupId>
+                <artifactId>langchain4j-bom</artifactId>
+                <version>1.8.0</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
     <dependencies>
         <dependency>
             <groupId>com.datamate</groupId>
             <artifactId>domain-common</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <dependency>
+            <groupId>com.datamate</groupId>
+            <artifactId>data-management-service</artifactId>
+            <version>1.0.0-SNAPSHOT</version>
+        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
         </dependency>
         <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>8.0.33</version>
         </dependency>
         <dependency>
-            <groupId>com.mysql</groupId>
-            <artifactId>mysql-connector-j</artifactId>
-            <version>${mysql.version}</version>
+            <groupId>org.springdoc</groupId>
+            <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.openapitools</groupId>
+            <artifactId>jackson-databind-nullable</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>jakarta.validation</groupId>
+            <artifactId>jakarta.validation-api</artifactId>
         </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>org.springframework.cloud</groupId>
-            <artifactId>spring-cloud-starter-openfeign</artifactId>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-open-ai</artifactId>
+            <version>1.8.0</version>
         </dependency>
         <dependency>
-            <groupId>org.springdoc</groupId>
-            <artifactId>springdoc-openapi-starter-webmvc-ui</artifactId>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j</artifactId>
+            <version>1.8.0</version>
         </dependency>
         <dependency>
-            <groupId>org.openapitools</groupId>
-            <artifactId>jackson-databind-nullable</artifactId>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-document-parser-apache-pdfbox</artifactId>
         </dependency>
         <dependency>
-            <groupId>jakarta.validation</groupId>
-            <artifactId>jakarta.validation-api</artifactId>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-document-parser-apache-tika</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-document-parser-apache-poi</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-document-parser-markdown</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-document-transformer-jsoup</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-milvus</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-embeddings-all-minilm-l6-v2</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>milvus</artifactId>
         </dependency>
     </dependencies>
 
@@ -64,31 +116,6 @@
                 <groupId>org.springframework.boot</groupId>
                 <artifactId>spring-boot-maven-plugin</artifactId>
             </plugin>
-            <!--<plugin>
-                <groupId>org.openapitools</groupId>
-                <artifactId>openapi-generator-maven-plugin</artifactId>
-                <version>6.6.0</version>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>generate</goal>
-                        </goals>
-                        <configuration>
-                            <inputSpec>${project.basedir}/../../openapi/specs/rag-services.yaml</inputSpec>
-                            <generatorName>spring</generatorName>
-                            <output>${project.build.directory}/generated-sources/openapi</output>
-                            <apiPackage>com.datamate.rag.indexer.interfaces.api</apiPackage>
-                            <modelPackage>com.datamate.rag.indexer.interfaces.dto</modelPackage>
-                            <configOptions>
-                                <interfaceOnly>true</interfaceOnly>
-                                <useTags>true</useTags>
-                                <useSpringBoot3>true</useSpringBoot3>
-                                <documentationProvider>springdoc</documentationProvider>
-                            </configOptions>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>-->
         </plugins>
     </build>
 </project>

backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/application/KnowledgeBaseService.java

@@ -0,0 +1,118 @@
+package com.datamate.rag.indexer.application;
+
+import com.baomidou.mybatisplus.core.metadata.IPage;
+import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.datamate.rag.indexer.domain.model.FileStatus;
+import com.datamate.rag.indexer.domain.model.KnowledgeBase;
+import com.datamate.rag.indexer.domain.model.RagChunk;
+import com.datamate.rag.indexer.domain.model.RagFile;
+import com.datamate.rag.indexer.domain.repository.KnowledgeBaseRepository;
+import com.datamate.rag.indexer.domain.repository.RagFileRepository;
+import com.datamate.rag.indexer.infrastructure.event.DataInsertedEvent;
+import com.datamate.common.infrastructure.exception.BusinessException;
+import com.datamate.common.infrastructure.exception.KnowledgeBaseErrorCode;
+import com.datamate.common.interfaces.PagedResponse;
+import com.datamate.common.interfaces.PagingQuery;
+import com.datamate.rag.indexer.interfaces.dto.*;
+import lombok.RequiredArgsConstructor;
+import org.springframework.beans.BeanUtils;
+import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.StringUtils;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * 知识库服务类
+ *
+ * @author dallas
+ * @since 2025-10-24
+ */
+@Service
+@RequiredArgsConstructor
+public class KnowledgeBaseService {
+    private final KnowledgeBaseRepository knowledgeBaseRepository;
+    private final RagFileRepository ragFileRepository;
+    private final ApplicationEventPublisher eventPublisher;
+
+
+    /**
+     * 创建知识库
+     *
+     * @param request 知识库创建请求
+     * @return 知识库 ID
+     */
+    public String create(KnowledgeBaseCreateReq request) {
+        KnowledgeBase knowledgeBase = new KnowledgeBase();
+        BeanUtils.copyProperties(request, knowledgeBase);
+        knowledgeBaseRepository.save(knowledgeBase);
+        return knowledgeBase.getId();
+    }
+
+    /**
+     * 更新知识库
+     *
+     * @param knowledgeBaseId 知识库 ID
+     * @param request         知识库更新请求
+     */
+    public void update(String knowledgeBaseId, KnowledgeBaseUpdateReq request) {
+        KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
+                .orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
+        if (StringUtils.hasText(request.getName())) {
+            knowledgeBase.setName(request.getName());
+        }
+        if (StringUtils.hasText(request.getDescription())) {
+            knowledgeBase.setDescription(request.getDescription());
+        }
+        knowledgeBaseRepository.updateById(knowledgeBase);
+    }
+
+    public void delete(String knowledgeBaseId) {
+        knowledgeBaseRepository.removeById(knowledgeBaseId);
+        ragFileRepository.removeByKnowledgeBaseId(knowledgeBaseId);
+        //  TODO: 删除知识库关联的所有文档
+    }
+
+    public KnowledgeBase getById(String knowledgeBaseId) {
+        return Optional.ofNullable(knowledgeBaseRepository.getById(knowledgeBaseId))
+                .orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
+    }
+
+    public PagedResponse<KnowledgeBase> list(KnowledgeBaseQueryReq request) {
+        IPage<KnowledgeBase> page = new Page<>(request.getPage(), request.getSize());
+        page = knowledgeBaseRepository.page(page, request);
+        return PagedResponse.of(page.getRecords(), page.getCurrent(), page.getTotal(), page.getPages());
+    }
+
+    @Transactional(rollbackFor = Exception.class)
+    public void addFiles(AddFilesReq request) {
+        KnowledgeBase knowledgeBase = Optional.ofNullable(knowledgeBaseRepository.getById(request.getKnowledgeBaseId()))
+                .orElseThrow(() -> BusinessException.of(KnowledgeBaseErrorCode.KNOWLEDGE_BASE_NOT_FOUND));
+        List<RagFile> ragFiles = request.getFiles().stream().map(fileInfo -> {
+            RagFile ragFile = new RagFile();
+            ragFile.setKnowledgeBaseId(knowledgeBase.getId());
+            ragFile.setFileId(fileInfo.fileId());
+            ragFile.setFileName(fileInfo.fileName());
+            ragFile.setStatus(FileStatus.UNPROCESSED);
+            return ragFile;
+        }).toList();
+        ragFileRepository.saveBatch(ragFiles, 100);
+        eventPublisher.publishEvent(new DataInsertedEvent(knowledgeBase.getId(), request.getProcessType()));
+    }
+
+    public PagedResponse<RagFile> listFiles(String knowledgeBaseId, RagFileReq request) {
+        IPage<RagFile> page = new Page<>(request.getPage(), request.getSize());
+        page = ragFileRepository.page(page);
+        return PagedResponse.of(page.getRecords(), page.getCurrent(), page.getTotal(), page.getPages());
+    }
+
+    public void deleteFile(String knowledgeBaseId, DeleteFilesReq request) {
+    }
+
+    public PagedResponse<RagChunk> getChunks(String knowledgeBaseId, String ragFileId, PagingQuery pagingQuery) {
+        IPage<RagChunk> page = new Page<>(pagingQuery.getPage(), pagingQuery.getSize());
+        return PagedResponse.of(page.getRecords(), page.getCurrent(), page.getTotal(), page.getPages());
+    }
+}

backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/domain/model/FileStatus.java

@@ -0,0 +1,26 @@
+package com.datamate.rag.indexer.domain.model;
+
+/**
+ * 文件状态枚举
+ *
+ * @author dallas
+ * @since 2025-10-29
+ */
+public enum FileStatus {
+    /**
+     * 未处理
+     */
+    UNPROCESSED,
+    /**
+     * 处理中
+     */
+    PROCESSING,
+    /**
+     * 已处理
+     */
+    PROCESSED,
+    /**
+     * 处理失败
+     */
+    PROCESS_FAILED
+}

backend/services/rag-indexer-service/src/main/java/com/datamate/rag/indexer/domain/model/KnowledgeBase.java

@@ -0,0 +1,37 @@
+package com.datamate.rag.indexer.domain.model;
+
+import com.baomidou.mybatisplus.annotation.TableName;
+import com.datamate.common.domain.model.base.BaseEntity;
+import lombok.Getter;
+import lombok.Setter;
+
+/**
+ * 知识库实体类
+ *
+ * @author dallas
+ * @since 2025-10-24
+ */
+@Getter
+@Setter
+@TableName("t_rag_knowledge_base")
+public class KnowledgeBase extends BaseEntity<String> {
+    /**
+     * 知识库名称
+     */
+    private String name;
+
+    /**
+     * 知识库描述
+     */
+    private String description;
+
+    /**
+     * 嵌入模型
+     */
+    private String embeddingModel;
+
+    /**
+     * 聊天模型
+     */
+    private String chatModel;
+}