Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetRepository;
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
import com.datamate.datamanagement.interfaces.dto.AddFilesRequest;
import com.datamate.datamanagement.interfaces.dto.BatchDeleteFilesRequest;
import com.datamate.datamanagement.interfaces.dto.CreateDirectoryRequest;
import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
import com.datamate.datamanagement.interfaces.dto.UploadFilesPreRequest;
Expand Down Expand Up @@ -247,6 +248,65 @@
}
}

/**
* 批量删除文件
*/
@Transactional
public void batchDeleteFiles(String datasetId, BatchDeleteFilesRequest request) {
Dataset dataset = datasetRepository.getById(datasetId);
if (dataset == null) {
throw BusinessException.of(DataManagementErrorCode.DATASET_NOT_FOUND);
}

List<String> fileIds = request.getFileIds();
if (fileIds == null || fileIds.isEmpty()) {
throw BusinessException.of(CommonErrorCode.PARAM_ERROR);
}

List<DatasetFile> filesToDelete = new ArrayList<>();
List<String> failedFileIds = new ArrayList<>();

for (String fileId : fileIds) {
try {
DatasetFile file = getDatasetFile(dataset, fileId, request.getPrefix());
filesToDelete.add(file);
datasetFileRepository.removeById(fileId);
} catch (Exception e) {
log.error("Failed to delete file with id: {}", fileId, e);
failedFileIds.add(fileId);
}
}

// 更新数据集(避免 ConcurrentModificationException)
List<DatasetFile> datasetFiles = dataset.getFiles();
if (datasetFiles != null) {
// 创建一个新的列表来存储要保留的文件
List<DatasetFile> remainingFiles = new ArrayList<>(datasetFiles);
// 移除要删除的文件
remainingFiles.removeAll(filesToDelete);
dataset.setFiles(remainingFiles);
}
datasetRepository.updateById(dataset);

// 删除文件系统中的文件
for (DatasetFile file : filesToDelete) {
// 上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件,归集过来的文件仅删除数据库中的记录
if (file.getFilePath().startsWith(dataset.getPath())) {
try {
Path filePath = Paths.get(file.getFilePath());
Files.deleteIfExists(filePath);
Comment thread Fixed
} catch (IOException ex) {
log.error("Failed to delete file from filesystem: {}", file.getFilePath(), ex);
}
}
}

// 如果有失败的文件,记录日志但不抛出异常
if (!failedFileIds.isEmpty()) {
log.warn("Failed to delete {} files out of {}", failedFileIds.size(), fileIds.size());
}
}

/**
* 下载文件
*/
Expand Down Expand Up @@ -637,10 +697,14 @@
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}

// 更新数据集
dataset.setFiles(filesToDelete);
for (DatasetFile file : filesToDelete) {
dataset.removeFile(file);
// 更新数据集(避免 ConcurrentModificationException,先获取文件列表再删除)
List<DatasetFile> datasetFiles = dataset.getFiles();
if (datasetFiles != null) {
// 创建一个新的列表来存储要保留的文件
List<DatasetFile> remainingFiles = new ArrayList<>(datasetFiles);
// 移除要删除的文件
remainingFiles.removeAll(filesToDelete);
dataset.setFiles(remainingFiles);
}
datasetRepository.updateById(dataset);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package com.datamate.datamanagement.interfaces.dto;

import jakarta.validation.constraints.NotEmpty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;

import java.util.List;

/**
* 批量删除文件请求
*/
@Data
@NoArgsConstructor
@AllArgsConstructor
public class BatchDeleteFilesRequest {

/**
* 要删除的文件ID列表
*/
@NotEmpty(message = "文件ID列表不能为空")
private List<String> fileIds;

/**
* 文件路径前缀(用于处理子目录中的文件)
*/
private String prefix = "";
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package com.datamate.datamanagement.interfaces.dto;

import com.datamate.datamanagement.interfaces.validation.ValidFileName;
import com.datamate.datamanagement.interfaces.validation.ValidFilePath;
import jakarta.validation.constraints.Min;
import jakarta.validation.constraints.NotBlank;
import jakarta.validation.constraints.NotNull;
Expand All @@ -25,9 +25,9 @@ public class UploadFileRequest {
@Min(value = 0, message = "文件编号必须为非负整数")
private int fileNo;

/** 文件名称 */
/** 文件名称(支持相对路径,用于文件夹上传) */
@NotBlank(message = "文件名称不能为空")
@ValidFileName
@ValidFilePath
@Size(max = 255, message = "文件名称长度不能超过255个字符")
private String fileName;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
import com.datamate.datamanagement.interfaces.dto.AddFilesRequest;
import com.datamate.datamanagement.interfaces.dto.BatchDeleteFilesRequest;
import com.datamate.datamanagement.interfaces.dto.CopyFilesRequest;
import com.datamate.datamanagement.interfaces.dto.CreateDirectoryRequest;
import com.datamate.datamanagement.interfaces.dto.DatasetFileResponse;
Expand Down Expand Up @@ -88,6 +89,21 @@ public ResponseEntity<Response<Void>> deleteDatasetFile(
}
}

/**
* 批量删除文件
*/
@DeleteMapping("/batch")
public ResponseEntity<Response<Void>> batchDeleteFiles(
@PathVariable("datasetId") String datasetId,
@RequestBody @Valid BatchDeleteFilesRequest request) {
try {
datasetFileApplicationService.batchDeleteFiles(datasetId, request);
return ResponseEntity.ok().build();
} catch (IllegalArgumentException e) {
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
}
}

@IgnoreResponseWrap
@GetMapping(value = "/{fileId}/download", produces = MediaType.APPLICATION_OCTET_STREAM_VALUE + ";charset=UTF-8")
public ResponseEntity<Resource> downloadDatasetFileById(@PathVariable("datasetId") String datasetId,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package com.datamate.datamanagement.interfaces.validation;

import jakarta.validation.Constraint;
import jakarta.validation.Payload;

import java.lang.annotation.*;

/**
* 文件路径校验注解
* 验证文件路径不包含非法字符(允许 / 用于支持文件夹上传)
*
* @author DataMate
* @since 2026/03/12
*/
@Documented
@Constraint(validatedBy = ValidFilePathValidator.class)
@Target({ElementType.FIELD, ElementType.PARAMETER})
@Retention(RetentionPolicy.RUNTIME)
public @interface ValidFilePath {

String message() default "文件路径包含非法字符";

Class<?>[] groups() default {};

Class<? extends Payload>[] payload() default {};
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package com.datamate.datamanagement.interfaces.validation;

import com.datamate.datamanagement.infrastructure.exception.DataManagementErrorCode;
import jakarta.validation.ConstraintValidator;
import jakarta.validation.ConstraintValidatorContext;

import java.util.regex.Pattern;

/**
* 文件路径校验器
* 允许路径分隔符 / 用于支持文件夹上传
*
* @author DataMate
* @since 2026/03/12
*/
public class ValidFilePathValidator implements ConstraintValidator<ValidFilePath, String> {

/**
* 文件路径正则表达式
* 不允许包含特殊字符: \ : * ? " < > | \0
* 允许字母、数字、中文、常见符号(- _ . space /)
* 注意:允许 / 是为了支持文件夹上传的相对路径
*/
private static final Pattern FILE_PATH_PATTERN = Pattern.compile(
"^[^\\\\:*?\"<>|\\x00]+$"
);

@Override
public boolean isValid(String value, ConstraintValidatorContext context) {
if (value == null || value.isEmpty()) {
return true; // 空值由 @NotBlank 等其他注解处理
}

boolean isValid = FILE_PATH_PATTERN.matcher(value).matches();

if (!isValid) {
context.disableDefaultConstraintViolation();
context.buildConstraintViolationWithTemplate(
DataManagementErrorCode.FILE_NAME_INVALID.getMessage()
).addConstraintViolation();
}

return isValid;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,24 @@ public static Optional<File> save(ChunkUploadRequest fileUploadRequest, ChunkUpl
}

File finalFile = new File(preUploadReq.getUploadPath(), fileUploadRequest.getFileName());
// 确保父目录存在(处理嵌套文件夹上传的情况)
File parentDir = finalFile.getParentFile();
if (parentDir != null && !parentDir.exists()) {
try {
boolean created = parentDir.mkdirs();
if (!created && !parentDir.exists()) {
// mkdirs 返回 false 且目录仍不存在,才是真正的失败
log.error("failed to create parent directory for file:{}, req Id:{}", finalFile.getPath(), fileUploadRequest.getReqId());
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}
} catch (Exception e) {
log.error("failed to create parent directory for file:{}, req Id:{}, error:{}", finalFile.getPath(), fileUploadRequest.getReqId(), e.getMessage(), e);
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}
}
if (!targetFile.renameTo(finalFile)) {
log.error("failed to mv file:{}, req Id:{}", targetFile.getName(), fileUploadRequest.getReqId());
throw new IllegalArgumentException("failed to move file to target dir");
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}
log.debug("save chunk {} cost {}", fileUploadRequest.getChunkNo(),
ChronoUnit.MILLIS.between(startTime, LocalDateTime.now()));
Expand All @@ -76,6 +91,21 @@ private static InputStream getFileInputStream(MultipartFile file) {
public static File saveFile(ChunkUploadRequest fileUploadRequest, ChunkUploadPreRequest preUploadReq) {
// 保存文件
File targetFile = new File(preUploadReq.getUploadPath(), fileUploadRequest.getFileName());
// 确保父目录存在(处理嵌套文件夹上传的情况)
File parentDir = targetFile.getParentFile();
if (parentDir != null && !parentDir.exists()) {
try {
boolean created = parentDir.mkdirs();
if (!created && !parentDir.exists()) {
// mkdirs 返回 false 且目录仍不存在,才是真正的失败
log.error("failed to create parent directory for file:{}", targetFile.getPath());
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}
} catch (Exception e) {
log.error("failed to create parent directory for file:{}, error:{}", targetFile.getPath(), e.getMessage(), e);
throw BusinessException.of(SystemErrorCode.FILE_SYSTEM_ERROR);
}
}
try {
log.info("file path {}, file size {}", targetFile.toPath(), targetFile.getTotalSpace());
FileUtils.copyInputStreamToFile(getFileInputStream(fileUploadRequest.getFile()), targetFile);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -197,47 +197,84 @@ export default function ImportConfiguration({
const handleUpload = async (dataset: Dataset) => {
console.log('[ImportConfiguration] Uploading with currentPrefix:', currentPrefix);

// 在上传文件前,确保按文件路径在数据集中创建对应的目录结构(支持多级目录)
// 在上传文件前,确保按文件路径在数据集中创建对应的目录结构(支持多级嵌套目录)
let failedDirs: string[] = [];
try {
const basePrefix = currentPrefix || "";
const dirSet = new Set<string>();
const dirMap = new Map<string, { parentPrefix: string | undefined; directoryName: string }>();

// 收集所有需要创建的目录(去重)
fileSliceList.forEach((file) => {
const path = file.name || "";
const parts = path.split("/").filter(Boolean);
if (parts.length <= 1) return; // 没有目录
if (parts.length <= 1) return; // 没有目录层级

let accumulated = "";
// 为每个目录层级创建记录
let accumulatedPath = "";
for (let i = 0; i < parts.length - 1; i++) {
accumulated += parts[i] + "/";
dirSet.add(basePrefix + accumulated);
accumulatedPath += parts[i] + "/";
const fullDirPath = basePrefix + accumulatedPath;

// 如果这个目录还没有被记录,添加到 map 中
if (!dirMap.has(fullDirPath)) {
// 提取父目录前缀和当前目录名
const pathSegments = accumulatedPath.split("/").filter(Boolean);
const directoryName = pathSegments[pathSegments.length - 1];

let parentPrefix: string | undefined;
if (pathSegments.length > 1) {
// 有父目录,计算父目录前缀
const parentPathSegments = pathSegments.slice(0, -1);
parentPrefix = basePrefix + parentPathSegments.join("/") + "/";
} else {
// 顶级目录,父目录前缀是 basePrefix(如果为空则为 undefined)
parentPrefix = basePrefix || undefined;
}

dirMap.set(fullDirPath, { parentPrefix, directoryName });
}
}
});

const dirList = Array.from(dirSet).sort((a, b) => a.length - b.length);
// 按路径深度排序,确保从父目录到子目录的顺序创建
const sortedDirs = Array.from(dirMap.entries()).sort((a, b) => a[0].length - b[0].length);

for (const fullPrefix of dirList) {
const relative = fullPrefix.slice((basePrefix || "").length);
const segments = relative.split("/").filter(Boolean);
if (!segments.length) continue;
const directoryName = segments[segments.length - 1];
const parentPrefix =
segments.length > 1
? basePrefix + segments.slice(0, -1).join("/") + "/"
: basePrefix || undefined;
console.log('[ImportConfiguration] Creating directories:', sortedDirs.length);

// 逐个创建目录
for (const [fullPath, { parentPrefix, directoryName }] of sortedDirs) {
try {
// 验证参数合法性(确保不包含非法字符)
if (directoryName.includes('/') || directoryName.includes('\\') || directoryName.includes('..')) {
console.error(`[ImportConfiguration] Invalid directory name: "${directoryName}", fullPath: ${fullPath}`);
failedDirs.push(fullPath);
continue;
}
if (parentPrefix && (parentPrefix.includes('..'))) {
console.error(`[ImportConfiguration] Invalid parentPrefix: "${parentPrefix}", fullPath: ${fullPath}`);
failedDirs.push(fullPath);
continue;
}

await createDatasetDirectoryUsingPost(dataset.id, {
parentPrefix,
parentPrefix: parentPrefix || undefined, // 确保空字符串转为 undefined
directoryName,
});
} catch (e) {
// 目录已存在等错误不阻断整个上传流程
console.warn("createDirectory failed for", fullPrefix, e);
console.log('[ImportConfiguration] Created directory:', fullPath, 'parentPrefix:', parentPrefix, 'directoryName:', directoryName);
} catch (e: any) {
// 记录失败的目录
failedDirs.push(fullPath);
const errorMsg = e?.response?.data?.message || e?.message || String(e);
console.warn(`[ImportConfiguration] Failed to create directory ${fullPath}:`, errorMsg);
}
}
} catch (e) {
console.warn("ensure directories before upload failed", e);
console.error('[ImportConfiguration] Directory creation error:', e);
}

// 如果有目录创建失败,给出提示(但不阻断上传,因为后端会自动创建父目录)
if (failedDirs.length > 0) {
message.warning(`部分目录创建失败(${failedDirs.length}个),文件上传时会自动创建`);
}

window.dispatchEvent(
Expand Down
Loading
Loading