Skip to content

Commit 87b8b6d

Browse files
hefanlihefanli
authored
fixed the problem that the metadata of files that have not yet been stored in the database cannot be previewed. (#300)
* fix: fix metadata writing * fix: fixed viewing of collection logs when logged in * fix: fixed the problem that the metadata of files that have not yet been stored in the database cannot be previewed.
1 parent 61b1a9b commit 87b8b6d

6 files changed

Lines changed: 55 additions & 74 deletions

File tree

‎backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetFileApplicationService.java‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import com.datamate.common.domain.service.FileService;
88
import com.datamate.common.domain.utils.AnalyzerUtils;
99
import com.datamate.common.domain.utils.ArchiveAnalyzer;
10+
import com.datamate.common.domain.utils.CommonUtils;
1011
import com.datamate.common.infrastructure.exception.BusinessAssert;
1112
import com.datamate.common.infrastructure.exception.BusinessException;
1213
import com.datamate.common.infrastructure.exception.CommonErrorCode;
@@ -189,7 +190,7 @@ private DatasetFile getDatasetFile(Path path, Map<String, DatasetFile> datasetFi
189190
} else {
190191
DatasetFile exist = datasetFilesMap.get(path.toString());
191192
if (exist == null) {
192-
datasetFile.setId("file-" + datasetFile.getFileName());
193+
datasetFile.setId(datasetFile.getFileName());
193194
datasetFile.setFileSize(path.toFile().length());
194195
} else {
195196
datasetFile = exist;
@@ -202,12 +203,21 @@ private DatasetFile getDatasetFile(Path path, Map<String, DatasetFile> datasetFi
202203
* 获取文件详情
203204
*/
204205
@Transactional(readOnly = true)
205-
public DatasetFile getDatasetFile(String datasetId, String fileId) {
206+
public DatasetFile getDatasetFile(Dataset dataset, String fileId) {
207+
if (dataset != null && !CommonUtils.isUUID(fileId) && !fileId.startsWith(".")) {
208+
DatasetFile file = new DatasetFile();
209+
file.setId(fileId);
210+
file.setFileName(fileId);
211+
file.setDatasetId(dataset.getId());
212+
file.setFileSize(0L);
213+
file.setFilePath(dataset.getPath() + File.separator + fileId);
214+
return file;
215+
}
206216
DatasetFile file = datasetFileRepository.getById(fileId);
207-
if (file == null) {
217+
if (file == null || dataset == null) {
208218
throw new IllegalArgumentException("File not found: " + fileId);
209219
}
210-
if (!file.getDatasetId().equals(datasetId)) {
220+
if (!file.getDatasetId().equals(dataset.getId())) {
211221
throw new IllegalArgumentException("File does not belong to the specified dataset");
212222
}
213223
return file;
@@ -218,11 +228,13 @@ public DatasetFile getDatasetFile(String datasetId, String fileId) {
218228
*/
219229
@Transactional
220230
public void deleteDatasetFile(String datasetId, String fileId) {
221-
DatasetFile file = getDatasetFile(datasetId, fileId);
222231
Dataset dataset = datasetRepository.getById(datasetId);
232+
DatasetFile file = getDatasetFile(dataset, fileId);
223233
dataset.setFiles(new ArrayList<>(Collections.singleton(file)));
224234
datasetFileRepository.removeById(fileId);
225-
dataset.removeFile(file);
235+
if (CommonUtils.isUUID(fileId)) {
236+
dataset.removeFile(file);
237+
}
226238
datasetRepository.updateById(dataset);
227239
// 删除文件时，上传到数据集中的文件会同时删除数据库中的记录和文件系统中的文件，归集过来的文件仅删除数据库中的记录
228240
if (file.getFilePath().startsWith(dataset.getPath())) {
@@ -239,10 +251,10 @@ public void deleteDatasetFile(String datasetId, String fileId) {
239251
* 下载文件
240252
*/
241253
@Transactional(readOnly = true)
242-
public Resource downloadFile(String datasetId, String fileId) {
243-
DatasetFile file = getDatasetFile(datasetId, fileId);
254+
public Resource downloadFile(DatasetFile file) {
244255
try {
245256
Path filePath = Paths.get(file.getFilePath()).normalize();
257+
log.info("start download file {}", file.getFilePath());
246258
Resource resource = new UrlResource(filePath.toUri());
247259
if (resource.exists()) {
248260
return resource;
@@ -638,12 +650,12 @@ public void deleteDirectory(String datasetId, String prefix) {
638650
*/
639651
@Transactional
640652
public void renameFile(String datasetId, String fileId, RenameFileRequest request) {
641-
DatasetFile file = getDatasetFile(datasetId, fileId);
642653
Dataset dataset = datasetRepository.getById(datasetId);
643654
if (dataset == null) {
644655
throw BusinessException.of(DataManagementErrorCode.DATASET_NOT_FOUND);
645656
}
646657

658+
DatasetFile file = getDatasetFile(dataset, fileId);
647659
String newName = Optional.ofNullable(request.getNewName()).orElse("").trim();
648660
if (newName.isEmpty()) {
649661
throw BusinessException.of(CommonErrorCode.PARAM_ERROR);

‎backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/DatasetFile.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import com.baomidou.mybatisplus.annotation.TableField;
44
import com.baomidou.mybatisplus.annotation.TableId;
55
import com.baomidou.mybatisplus.annotation.TableName;
6+
import com.datamate.common.infrastructure.config.PgJsonTypeHandler;
67
import com.fasterxml.jackson.core.type.TypeReference;
78
import com.fasterxml.jackson.databind.ObjectMapper;
89
import lombok.*;
@@ -32,7 +33,7 @@ public class DatasetFile {
3233
private Long fileSize; // bytes
3334
private String checkSum;
3435
private String tags;
35-
private LocalDateTime tagsUpdatedAt;
36+
@TableField(typeHandler = PgJsonTypeHandler.class)
3637
private String metadata;
3738
private String status; // UPLOADED, PROCESSING, COMPLETED, ERROR
3839
private LocalDateTime uploadTime;

‎backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/rest/DatasetFileController.java‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
import com.datamate.common.infrastructure.exception.SystemErrorCode;
66
import com.datamate.common.interfaces.PagedResponse;
77
import com.datamate.common.interfaces.PagingQuery;
8+
import com.datamate.datamanagement.application.DatasetApplicationService;
89
import com.datamate.datamanagement.application.DatasetFileApplicationService;
10+
import com.datamate.datamanagement.domain.model.dataset.Dataset;
911
import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
1012
import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
1113
import com.datamate.datamanagement.interfaces.dto.AddFilesRequest;
@@ -18,6 +20,7 @@
1820
import com.datamate.datamanagement.interfaces.dto.RenameDirectoryRequest;
1921
import jakarta.servlet.http.HttpServletResponse;
2022
import jakarta.validation.Valid;
23+
import lombok.RequiredArgsConstructor;
2124
import lombok.extern.slf4j.Slf4j;
2225
import org.springframework.beans.factory.annotation.Autowired;
2326
import org.springframework.core.io.Resource;
@@ -34,15 +37,13 @@
3437
*/
3538
@Slf4j
3639
@RestController
40+
@RequiredArgsConstructor
3741
@RequestMapping("/data-management/datasets/{datasetId}/files")
3842
public class DatasetFileController {
3943

4044
private final DatasetFileApplicationService datasetFileApplicationService;
4145

42-
@Autowired
43-
public DatasetFileController(DatasetFileApplicationService datasetFileApplicationService) {
44-
this.datasetFileApplicationService = datasetFileApplicationService;
45-
}
46+
private final DatasetApplicationService datasetApplicationService;
4647

4748
@GetMapping
4849
public Response<PagedResponse<DatasetFile>> getDatasetFiles(
@@ -66,7 +67,8 @@ public ResponseEntity<Response<DatasetFileResponse>> getDatasetFileById(
6667
@PathVariable("datasetId") String datasetId,
6768
@PathVariable("fileId") String fileId) {
6869
try {
69-
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(datasetId, fileId);
70+
Dataset dataset = datasetApplicationService.getDataset(datasetId);
71+
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(dataset, fileId);
7072
return ResponseEntity.ok(Response.ok(DatasetConverter.INSTANCE.convertToResponse(datasetFile)));
7173
} catch (IllegalArgumentException e) {
7274
return ResponseEntity.status(HttpStatus.NOT_FOUND).body(Response.error(SystemErrorCode.UNKNOWN_ERROR, null));
@@ -90,17 +92,20 @@ public ResponseEntity<Response<Void>> deleteDatasetFile(
9092
public ResponseEntity<Resource> downloadDatasetFileById(@PathVariable("datasetId") String datasetId,
9193
@PathVariable("fileId") String fileId) {
9294
try {
93-
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(datasetId, fileId);
94-
Resource resource = datasetFileApplicationService.downloadFile(datasetId, fileId);
95+
Dataset dataset = datasetApplicationService.getDataset(datasetId);
96+
DatasetFile datasetFile = datasetFileApplicationService.getDatasetFile(dataset, fileId);
97+
Resource resource = datasetFileApplicationService.downloadFile(datasetFile);
9598

9699
return ResponseEntity.ok()
97100
.contentType(MediaType.APPLICATION_OCTET_STREAM)
98101
.header(HttpHeaders.CONTENT_DISPOSITION,
99102
"attachment; filename=\"" + datasetFile.getFileName() + "\"")
100103
.body(resource);
101104
} catch (IllegalArgumentException e) {
105+
log.error("downloadDatasetFileById error: {}", e.getMessage(), e);
102106
return ResponseEntity.status(HttpStatus.NOT_FOUND).build();
103107
} catch (Exception e) {
108+
log.error("downloadDatasetFileById error: {}", e.getMessage(), e);
104109
return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).build();
105110
}
106111
}

‎backend/shared/domain-common/src/main/java/com/datamate/common/domain/utils/CommonUtils.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.datamate.common.domain.utils;
22

33
import java.io.File;
4+
import java.util.UUID;
45

56
/**
67
* 通用工具类
@@ -21,4 +22,20 @@ public static String trimFilePath(String filePath) {
2122
}
2223
return filename;
2324
}
25+
26+
/**
27+
* 判断字符串是否是uuid
28+
*
29+
* @param str 要判断的字符串
30+
* @return 判断结果
31+
*/
32+
public static boolean isUUID(String str) {
33+
if (str == null) return false;
34+
try {
35+
UUID.fromString(str);
36+
return true;
37+
} catch (IllegalArgumentException e) {
38+
return false;
39+
}
40+
}
2441
}

‎frontend/src/pages/DataCollection/collection.apis.ts‎

Lines changed: 2 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import { get, post, put, del } from "@/utils/request";
1+
import {get, post, put, del, download} from "@/utils/request";
22

33
// 数据源任务相关接口
44
export function queryTasksUsingGet(params?: any) {
@@ -9,21 +9,6 @@ export function createTaskUsingPost(data: any) {
99
return post("/api/data-collection/tasks", data);
1010
}
1111

12-
export function queryTaskByIdUsingGet(id: string | number) {
13-
return get(`/api/data-collection/tasks/${id}`);
14-
}
15-
16-
export function updateTaskByIdUsingPut(
17-
id: string | number,
18-
data: any
19-
) {
20-
return put(`/api/data-collection/tasks/${id}`, data);
21-
}
22-
23-
export function queryTaskDetailsByIdUsingGet(id: string | number) {
24-
return get(`/api/data-collection/tasks/${id}`);
25-
}
26-
2712
export function queryDataXTemplatesUsingGet(params?: any) {
2813
return get("/api/data-collection/templates", params);
2914
}
@@ -50,45 +35,6 @@ export function queryExecutionLogUsingPost(params?: any) {
5035
return get("/api/data-collection/executions", params);
5136
}
5237

53-
export function queryExecutionLogByIdUsingGet(id: string | number) {
54-
return get(`/api/data-collection/executions/${id}`);
55-
}
56-
57-
export function queryExecutionLogContentByIdUsingGet(id: string | number) {
58-
return get(`/api/data-collection/executions/${id}/log`);
59-
}
60-
6138
export async function queryExecutionLogFileByIdUsingGet(id: string | number) {
62-
const token = localStorage.getItem("token") || sessionStorage.getItem("token");
63-
const resp = await fetch(`/api/data-collection/executions/${id}/log`, {
64-
method: "GET",
65-
headers: {
66-
...(token ? { Authorization: `Bearer ${token}` } : {}),
67-
},
68-
credentials: "include",
69-
});
70-
71-
if (!resp.ok) {
72-
let detail = "";
73-
try {
74-
detail = await resp.text();
75-
} catch {
76-
detail = resp.statusText;
77-
}
78-
const err: any = new Error(detail || `HTTP error ${resp.status}`);
79-
err.status = resp.status;
80-
err.data = detail;
81-
throw err;
82-
}
83-
84-
const contentDisposition = resp.headers.get("content-disposition") || "";
85-
const filenameMatch = contentDisposition.match(/filename\*?=(?:UTF-8''|\")?([^;\"\n]+)/i);
86-
const filename = filenameMatch?.[1] ? decodeURIComponent(filenameMatch[1].replace(/\"/g, "").trim()) : `execution_${id}.log`;
87-
const blob = await resp.blob();
88-
return { blob, filename };
89-
}
90-
91-
// 监控统计相关接口
92-
export function queryCollectionStatisticsUsingGet(params?: any) {
93-
return get("/api/data-collection/monitor/statistics", params);
39+
return await download(`/api/data-collection/executions/${id}/log`, null, undefined, "preview");
9440
}

‎runtime/datamate-python/app/module/collection/interface/execution.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ async def get_execution_log(
104104

105105
filename = path.name
106106
headers = {
107-
"Content-Disposition": f'inline; filename="{filename}"'
107+
"Content-Disposition": f'inline; filename={filename}'
108108
}
109109
return FileResponse(
110110
path=str(path),

0 commit comments

Comments
 (0)