Merge branch 'main' into feat/clean

# Conflicts:
#	frontend/src/pages/DataCleansing/Detail/components/LogsTable.tsx

Author: hhhhsc701

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java

@@ -285,7 +285,7 @@ public void processDataSourceAsync(String datasetId, String dataSourceId) {
             if (CollectionUtils.isEmpty(filePaths)) {
                 return;
             }
-            datasetFileApplicationService.copyFilesToDatasetDir(datasetId, new CopyFilesRequest(filePaths));
+            datasetFileApplicationService.addFilesToDataset(datasetId, new AddFilesRequest(filePaths));
             log.info("Success file scan, total files: {}", filePaths.size());
         } catch (Exception e) {
             log.error("处理数据源文件扫描失败，数据集ID: {}, 数据源ID: {}", datasetId, dataSourceId, e);

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java

@@ -24,7 +24,6 @@
 import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetRepository;
 import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
 import com.datamate.datamanagement.interfaces.dto.AddFilesRequest;
-import com.datamate.datamanagement.interfaces.dto.CopyFilesRequest;
 import com.datamate.datamanagement.interfaces.dto.CreateDirectoryRequest;
 import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
 import com.datamate.datamanagement.interfaces.dto.UploadFilesPreRequest;
@@ -58,7 +57,6 @@
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
 import java.util.*;
-import java.util.concurrent.CompletableFuture;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -664,11 +662,9 @@ public void renameFile(String datasetId, String fileId, RenameFileRequest reques
         }
 
         String originalFileName = file.getFileName();
-        String baseName = originalFileName;
         String extension = "";
         int dotIndex = originalFileName.lastIndexOf('.');
         if (dotIndex > 0 && dotIndex < originalFileName.length() - 1) {
-            baseName = originalFileName.substring(0, dotIndex);
             extension = originalFileName.substring(dotIndex); // 包含点号，如 .jpg
         }
 
@@ -828,114 +824,102 @@ private void setDatasetFileId(DatasetFile datasetFile, Dataset dataset) {
     }
 
     /**
-     * 复制文件到数据集目录
+     * 添加文件到数据集（仅创建数据库记录，不执行文件系统操作）
      *
      * @param datasetId 数据集id
-     * @param req       复制文件请求
-     * @return 复制的文件列表
+     * @param req       添加文件请求
+     * @return 添加的文件列表
      */
     @Transactional
-    public List<DatasetFile> copyFilesToDatasetDir(String datasetId, CopyFilesRequest req) {
+    public List<DatasetFile> addFilesToDataset(String datasetId, AddFilesRequest req) {
+        if (!req.isValidPrefix()) {
+            throw BusinessException.of(DataManagementErrorCode.DIRECTORY_NOT_FOUND);
+        }
         Dataset dataset = datasetRepository.getById(datasetId);
         BusinessAssert.notNull(dataset, SystemErrorCode.RESOURCE_NOT_FOUND);
-        List<DatasetFile> copiedFiles = new ArrayList<>();
+        List<DatasetFile> addedFiles = new ArrayList<>();
         List<DatasetFile> existDatasetFiles = datasetFileRepository.findAllByDatasetId(datasetId);
         dataset.setFiles(existDatasetFiles);
-        for (String sourceFilePath : req.sourcePaths()) {
-            Path sourcePath = Paths.get(sourceFilePath);
-            if (!Files.exists(sourcePath) || !Files.isRegularFile(sourcePath)) {
-                log.warn("Source file does not exist or is not a regular file: {}", sourceFilePath);
-                continue;
+        try {
+            ObjectMapper objectMapper = new ObjectMapper();
+
+            for (AddFilesRequest.FileRequest file : req.getFiles()) {
+                DatasetFile datasetFile = getDatasetFileForAdd(req, file, dataset, objectMapper);
+                setDatasetFileId(datasetFile, dataset);
+                dataset.addFile(datasetFile);
+                addedFiles.add(datasetFile);
+                addFile(file.getFilePath(), datasetFile.getFilePath(), req.isSoftAdd());
             }
-            String fileName = sourcePath.getFileName().toString();
-            File sourceFile = sourcePath.toFile();
-            LocalDateTime currentTime = LocalDateTime.now();
-            DatasetFile datasetFile = DatasetFile.builder()
-                    .id(UUID.randomUUID().toString())
-                    .datasetId(datasetId)
-                    .fileName(fileName)
-                    .fileType(AnalyzerUtils.getExtension(fileName))
-                    .fileSize(sourceFile.length())
-                    .filePath(Paths.get(dataset.getPath(), fileName).toString())
-                    .uploadTime(currentTime)
-                    .lastAccessTime(currentTime)
-                    .build();
-            setDatasetFileId(datasetFile, dataset);
-            dataset.addFile(datasetFile);
-            copiedFiles.add(datasetFile);
+        } catch (BusinessException e) {
+            throw e;
+        } catch (Exception e) {
+            log.error("Failed to add file to dataset {}", dataset.getName(), e);
+            throw BusinessException.of(SystemErrorCode.UNKNOWN_ERROR);
         }
-        datasetFileRepository.saveOrUpdateBatch(copiedFiles, 100);
+
+        datasetFileRepository.saveOrUpdateBatch(addedFiles, 100);
         dataset.active();
         datasetRepository.updateById(dataset);
-        CompletableFuture.runAsync(() -> copyFilesToDatasetDir(req.sourcePaths(), dataset));
-        return copiedFiles;
+        return addedFiles;
     }
 
-    private void copyFilesToDatasetDir(List<String> sourcePaths, Dataset dataset) {
-        for (String sourcePath : sourcePaths) {
-            Path sourceFilePath = Paths.get(sourcePath);
-            Path targetFilePath = Paths.get(dataset.getPath(), sourceFilePath.getFileName().toString());
-            try {
-                Files.createDirectories(Path.of(dataset.getPath()));
-                Files.copy(sourceFilePath, targetFilePath);
-            } catch (IOException e) {
-                log.error("Failed to copy file from {} to {}", sourcePath, targetFilePath, e);
-            }
+    private void addFile(String sourPath, String targetPath, boolean softAdd) {
+        if (StringUtils.isBlank(sourPath) || StringUtils.isBlank(targetPath)) {
+            return;
         }
-    }
+        Path source = Paths.get(sourPath).normalize();
+        Path target = Paths.get(targetPath).normalize();
 
-    /**
-     * 添加文件到数据集（仅创建数据库记录，不执行文件系统操作）
-     *
-     * @param datasetId 数据集id
-     * @param req       添加文件请求
-     * @return 添加的文件列表
-     */
-    @Transactional
-    public List<DatasetFile> addFilesToDataset(String datasetId, AddFilesRequest req) {
-        Dataset dataset = datasetRepository.getById(datasetId);
-        BusinessAssert.notNull(dataset, SystemErrorCode.RESOURCE_NOT_FOUND);
-        List<DatasetFile> addedFiles = new ArrayList<>();
-        List<DatasetFile> existDatasetFiles = datasetFileRepository.findAllByDatasetId(datasetId);
-        dataset.setFiles(existDatasetFiles);
+        // 检查源文件是否存在且为普通文件
+        if (!Files.exists(source) || !Files.isRegularFile(source)) {
+            log.warn("Source file does not exist or is not a regular file: {}", sourPath);
+            throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
+        }
 
-        boolean softAdd = req.softAdd();
-        String metadata;
         try {
-            Map<String, Boolean> metadataMap = Map.of("softAdd", softAdd);
-            ObjectMapper objectMapper = new ObjectMapper();
-            metadata = objectMapper.writeValueAsString(metadataMap);
-        } catch (JsonProcessingException e) {
-            log.error("Failed to serialize metadataMap", e);
-            throw BusinessException.of(SystemErrorCode.UNKNOWN_ERROR);
+            Path parent = target.getParent();
+            // 创建目标目录（如果需要）
+            if (parent != null) {
+                Files.createDirectories(parent);
+            }
+            Files.deleteIfExists(target);
+            if (softAdd) {
+                // 优先尝试创建硬链接，失败后尝试创建符号链接；若均失败抛出异常
+                try {
+                    Files.createLink(target, source);
+                    return;
+                } catch (Throwable hardEx) {
+                    log.warn("create hard link failed from {} to {}: {}", source, target, hardEx.getMessage());
+                }
+                Files.createSymbolicLink(target, source);
+            } else {
+                // 覆盖已存在的目标文件，保持与其他地方行为一致
+                Files.copy(source, target);
+            }
+        } catch (IOException e) {
+            log.error("Failed to add file from {} to {}", source, target, e);
+            throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
         }
+    }
 
-        for (String sourceFilePath : req.sourcePaths()) {
-            Path sourcePath = Paths.get(sourceFilePath);
-            String fileName = sourcePath.getFileName().toString();
-            File sourceFile = sourcePath.toFile();
-            LocalDateTime currentTime = LocalDateTime.now();
+    private static DatasetFile getDatasetFileForAdd(AddFilesRequest req, AddFilesRequest.FileRequest file,
+                                                    Dataset dataset, ObjectMapper objectMapper) throws JsonProcessingException {
+        Path sourcePath = Paths.get(file.getFilePath());
+        File sourceFile = sourcePath.toFile();
+        file.getMetadata().put("softAdd", req.isSoftAdd());
+        LocalDateTime currentTime = LocalDateTime.now();
+        String fileName = sourcePath.getFileName().toString();
 
-            DatasetFile datasetFile = DatasetFile.builder()
+        return DatasetFile.builder()
                 .id(UUID.randomUUID().toString())
-                .datasetId(datasetId)
+                .datasetId(dataset.getId())
                 .fileName(fileName)
                 .fileType(AnalyzerUtils.getExtension(fileName))
                 .fileSize(sourceFile.length())
-                .filePath(sourceFilePath)
+                .filePath(Paths.get(dataset.getPath(), req.getPrefix(), fileName).toString())
                 .uploadTime(currentTime)
                 .lastAccessTime(currentTime)
-                .metadata(metadata)
+                .metadata(objectMapper.writeValueAsString(file.getMetadata()))
                 .build();
-            setDatasetFileId(datasetFile, dataset);
-            dataset.addFile(datasetFile);
-            addedFiles.add(datasetFile);
-        }
-        datasetFileRepository.saveOrUpdateBatch(addedFiles, 100);
-        dataset.active();
-        datasetRepository.updateById(dataset);
-        // Note: addFilesToDataset only creates DB records, no file system operations
-        // If file copy is needed, use copyFilesToDatasetDir endpoint instead
-        return addedFiles;
     }
 }

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/dto/AddFilesRequest.java

@@ -1,18 +1,51 @@
 package com.datamate.datamanagement.interfaces.dto;
 
-import jakarta.validation.constraints.NotEmpty;
-import jakarta.validation.constraints.NotNull;
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.util.List;
+import java.util.Map;
 
 /**
- * 添加文件请求DTO（仅创建DB记录，不执行文件系统操作）
+ * AddFilesRequest1
  *
- * @author datamate
- * @since 2025-11-29
+ * @since 2026/2/9
  */
-public record AddFilesRequest(
-        @NotEmpty List<String> sourcePaths,
-        @NotNull Boolean softAdd
-) {
+@Getter
+@Setter
+@NoArgsConstructor
+@AllArgsConstructor
+public class AddFilesRequest {
+    public AddFilesRequest(List<String> paths) {
+        if (CollectionUtils.isEmpty(paths)) {
+            return;
+        }
+        this.files = paths.stream().map(path -> {
+            AddFilesRequest.FileRequest file = new AddFilesRequest.FileRequest();
+            file.setFilePath(path);
+            return  file;
+        }).toList();
+    }
+
+    @Getter
+    @Setter
+    public static class FileRequest {
+        private String filePath;
+
+        private Map<String, Object> metadata;
+    }
+
+    private boolean softAdd;
+
+    private String prefix = "";
+
+    private List<FileRequest> files;
+
+    public boolean isValidPrefix() {
+        return StringUtils.isEmpty(prefix) || (!prefix.startsWith("."));
+    }
 }

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/DatasetFileController.java

@@ -22,7 +22,6 @@
 import jakarta.validation.Valid;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
-import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.core.io.Resource;
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.HttpStatus;
@@ -157,7 +156,8 @@ public ResponseEntity<Void> chunkUpload(@PathVariable("datasetId") String datase
     @PostMapping("/upload/copy")
     public List<DatasetFileResponse> copyFilesToDatasetDir(@PathVariable("datasetId") String datasetId,
                                                       @RequestBody @Valid CopyFilesRequest req) {
-        List<DatasetFile> datasetFiles = datasetFileApplicationService.copyFilesToDatasetDir(datasetId, req);
+        AddFilesRequest addFilesRequest = new AddFilesRequest(req.sourcePaths());
+        List<DatasetFile> datasetFiles = datasetFileApplicationService.addFilesToDataset(datasetId, addFilesRequest);
         return DatasetConverter.INSTANCE.convertToResponseList(datasetFiles);
     }
 

deployment/docker/datamate/docker-compose.yml

@@ -201,6 +201,26 @@ services:
   # ==============================
   # Optional: Label Studio (profile: label-studio)
   # ==============================
+  label-studio-pgbouncer:
+    container_name: label-studio-pgbouncer
+    image: pgbouncer/pgbouncer:latest
+    restart: on-failure
+    ports:
+      - "6432:6432"
+    environment:
+      - DATABASES_HOST=datamate-database
+      - DATABASES_PORT=5432
+      - DATABASES_NAME=labelstudio
+      - DATABASES_USER=postgres
+      - DATABASES_PASSWORD=${DB_PASSWORD:-password}
+      - POOL_MODE=transaction
+      - MAX_CLIENT_CONN=100
+      - DEFAULT_POOL_SIZE=20
+      - MAX_DB_CONNECTIONS=20
+    networks:
+      - datamate
+    profiles: [ label-studio ]
+
   label-studio:
     container_name: label-studio
     stdin_open: true
@@ -214,14 +234,14 @@ services:
     ports:
       - "30001:8000"
     depends_on:
-      - datamate-database
+      - label-studio-pgbouncer
     environment:
       - DJANGO_DB=default
       - POSTGRE_NAME=labelstudio
       - POSTGRE_USER=postgres
       - POSTGRE_PASSWORD=${DB_PASSWORD:-password}
-      - POSTGRE_PORT=5432
-      - POSTGRE_HOST=datamate-database
+      - POSTGRE_PORT=6432
+      - POSTGRE_HOST=label-studio-pgbouncer
       - LABEL_STUDIO_HOST=${LABEL_STUDIO_HOST:-}
       - LOCAL_FILES_SERVING_ENABLED=true
       - LOCAL_FILES_DOCUMENT_ROOT=/label-studio/local

deployment/helm/label-studio/templates/deployment.yaml

@@ -38,9 +38,9 @@ spec:
             - name: POSTGRE_PASSWORD
               value: {{ .Values.env.POSTGRE_PASSWORD | quote }}
             - name: POSTGRE_PORT
-              value: {{ .Values.env.POSTGRE_PORT | quote }}
+              value: {{ if .Values.pgbouncer.enabled }}{{ "6432" | quote }}{{ else }}{{ .Values.env.POSTGRE_PORT | quote }}{{ end }}
             - name: POSTGRE_HOST
-              value: {{ .Values.env.POSTGRE_HOST | quote }}
+              value: {{ if .Values.pgbouncer.enabled }}{{ printf "%s-pgbouncer" (include "label-studio.fullname" .) | quote }}{{ else }}{{ .Values.env.POSTGRE_HOST | quote }}{{ end }}
             - name: LABEL_STUDIO_HOST
               value: {{ .Values.env.LABEL_STUDIO_HOST | quote }}
             - name: LOCAL_FILES_SERVING_ENABLED