diff --git a/app-builder/jane/plugins/aipp-plugin/pom.xml b/app-builder/jane/plugins/aipp-plugin/pom.xml index 02143fe668..ba2fcde831 100644 --- a/app-builder/jane/plugins/aipp-plugin/pom.xml +++ b/app-builder/jane/plugins/aipp-plugin/pom.xml @@ -122,6 +122,10 @@ io.opentelemetry opentelemetry-api + + com.opencsv + opencsv + modelengine.jade.service aipp-prompt-builder-service diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AippChatMapper.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AippChatMapper.java index 047ff103b5..07275a9a2b 100644 --- a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AippChatMapper.java +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AippChatMapper.java @@ -58,7 +58,8 @@ List selectChatList(@Param("requestParam") QueryChatRequest reques * @param chatId 会话ID * @return 会话记录数目 */ - long getChatListCount(@Param("requestParam") QueryChatRequest request, @Param("chatId") String chatId, @Param("createBy") String createBy); + long getChatListCount(@Param("requestParam") QueryChatRequest request, @Param("chatId") String chatId, + @Param("createBy") String createBy); /** * 查询会话 @@ -115,6 +116,7 @@ List selectChat(@Param("chatId") String chatId, @Param("offset") Intege /** * 根据instance id列表批量查询对话消息 + * * @param instanceIds instance id列表 * @return 对应会话信息 */ @@ -178,4 +180,43 @@ List selectChat(@Param("chatId") String chatId, @Param("offset") Intege */ List selectChatByCondition(@Param("condition") Map condition, @Param("requestParam") QueryChatInfoRequest queryChatInfoRequest); + + /** + * 获取超期的对话唯一标识。 + * + * @param expiredDays 表示超期时长的 {@code int}。 + * @param limit 表示查询数量的 {@code int}。 + * @return 表示对话唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + */ + List getExpiredChatIds(int expiredDays, int limit); + + /** + * 根据对话标识列表强制删除对话。 + * + * @param chatIds 表示对话唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + */ + void forceDeleteChat(List chatIds); + + /** + * 根据对话标识列表强制删除对话和任务实例关系。 + * + * @param chatIds 表示对话唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + */ + void deleteWideRelationshipByChatIds(List chatIds); + + /** + * 根据对话唯一标识列表批量查询会话记录实体。 + * + * @param chatIds 表示对话唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + * @return 表示会话记录实体列表的 {@link List}{@code <}{@link ChatInfo}{@code >}。 + */ + List selectByChatIds(@Param("chatIds") List chatIds); + + /** + * 根据对话唯一标识列表批量查询会话记录和任务实例的关系。 + * + * @param chatIds 表示对话唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + * @return 表示会话记录和任务实例的关系的 {@link List}{@code <}{@link ChatAndInstanceMap}{@code >}。 + */ + List selectTaskInstanceRelationsByChatIds(@Param("chatIds") List chatIds); } \ No newline at end of file diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AippLogMapper.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AippLogMapper.java index a309ecd39c..83a20d2417 100644 --- a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AippLogMapper.java +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AippLogMapper.java @@ -195,4 +195,28 @@ List selectRecentInstanceIdByAippIds(List aippIds, String aippTy * @param logIds 表示指定的历史记录 id 的 {@link List}{@code <}{@link Long}{@code >}。 */ void deleteInstanceLogs(@Param("logIds") List logIds); + + /** + * 获取超期的调试对话记录唯一标识列表。 + * + * @param expiredDays 表示超期时间的 {@code int}。 + * @param limit 表示查询条数的 {@code int}。 + * @return 表示历史会话记录的id列表的 {@link List}{@code <}{@link Long}{@code >}。 + */ + List getExpireInstanceLogIds(String aippType, int expiredDays, int limit); + + /** + * 根据实例唯一标识列表强制删除会话记录。 + * + * @param logIds 表示会话实例id列表的 {@link List}{@code <}{@link Long}{@code >}。 + */ + void forceDeleteInstanceLogsByIds(List logIds); + + /** + * 根据日志唯一标识列表查询会话历史记录。 + * + * @param logIds 标识日志唯一标识列表的 {@link List}{@code <}{@link Long}{@code >}。 + * @return 表示实例历史记录列表的 {@link List}{@code <}{@link AippInstLog}{@code >}。 + */ + List selectByLogIds(@Param("logIds") List logIds); } diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AppBuilderRuntimeInfoMapper.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AppBuilderRuntimeInfoMapper.java index 9262a7182a..f1c0ccd569 100644 --- a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AppBuilderRuntimeInfoMapper.java +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/mapper/AppBuilderRuntimeInfoMapper.java @@ -31,4 +31,20 @@ public interface AppBuilderRuntimeInfoMapper { * @param appBuilderRuntimeInfoPO {@link AppBuilderRuntimeInfoPo} 对象. */ void insertOne(AppBuilderRuntimeInfoPo appBuilderRuntimeInfoPO); + + /** + * 获取超期的运行时信息唯一标识列表。 + * + * @param expiredDays 表示超期时间的 {@code int}。 + * @param limit 表示查询条数的 {@code int}。 + * @return 表示运行时信息唯一标识列表的 {@link List}{@code <}{@link Long}{@code >}。 + */ + List getExpiredRuntimeInfos(int expiredDays, int limit); + + /** + * 根据运行时信息唯一标识列表强制删除会话记录。 + * + * @param runtimeInfoIds 表示运行时信息唯一标识列表的 {@link List}{@code <}{@link Long}{@code >}。 + */ + void deleteRuntimeInfos(List runtimeInfoIds); } diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/AippChatRepository.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/AippChatRepository.java new file mode 100644 index 0000000000..5c997cc12d --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/AippChatRepository.java @@ -0,0 +1,52 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.repository; + +import modelengine.fit.jober.aipp.entity.ChatAndInstanceMap; +import modelengine.fit.jober.aipp.entity.ChatInfo; + +import java.util.List; + +/** + * 应用对话的存储仓库。 + * + * @author 杨祥宇 + * @since 2025-04-09 + */ +public interface AippChatRepository { + /** + * 获取超期的对话唯一标识列表。 + * + * @param expiredDays 表示超期时长的 {@code int}。 + * @param limit 表示查询数量的 {@code int}。 + * @return 表示对话唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + */ + List getExpiredChatIds(int expiredDays, int limit); + + /** + * 根据对话标识列表强制删除对话。 + * + * @param chatIds 表示对话唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + */ + void forceDeleteChat(List chatIds); + + /** + * 根据对话唯一标识列表批量查询会话记录实体。 + * + * @param chatIds 表示对话唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + * @return 表示会话记录实体列表的 {@link List}{@code <}{@link ChatInfo}{@code >}。 + */ + List selectByChatIds(List chatIds); + + /** + * 根据对话唯一标识列表批量查询会话记录和任务实例的关系。 + * + * @param chatIds 表示对话唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + * @return 表示会话记录和任务实例的关系的 {@link List}{@code <}{@link ChatAndInstanceMap}{@code >}。 + */ + List selectTaskInstanceRelationsByChatIds(List chatIds); +} diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/AippInstanceLogRepository.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/AippInstanceLogRepository.java new file mode 100644 index 0000000000..2d3d74e602 --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/AippInstanceLogRepository.java @@ -0,0 +1,43 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.repository; + +import modelengine.fit.jober.aipp.entity.AippInstLog; + +import java.util.List; + +/** + * 应用实例历史记录的存储仓库。 + * + * @author 杨祥宇 + * @since 2025-04-09 + */ +public interface AippInstanceLogRepository { + /** + * 获取调试类型的应用过期历史记录。 + * + * @param expiredDays 表示超期天数的 {@code int}。 + * @param limit 表示查询条数的 {@code int}。 + * @return 表示超期历史记录id的 {@link List}{@code <}{@link Long}{@code >}。 + */ + List getExpireInstanceLogIds(String aippType, int expiredDays, int limit); + + /** + * 根据日志唯一标识列表强制删除历史记录。 + * + * @param logIds 表示历史记录的唯一标识列表的 {@link List}{@code <}{@link Long}{@code >}。 + */ + void forceDeleteInstanceLogs(List logIds); + + /** + * 根据日志唯一标识列表查询会话历史记录 + * + * @param logIds 标识日志唯一标识列表的 {@link List}{@code <}{@link Long}{@code >}。 + * @return 表示实例历史记录列表的 {@link List}{@code <}{@link AippInstLog}{@code >}。 + */ + List selectByLogIds(List logIds); +} diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/AppBuilderRuntimeInfoRepository.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/AppBuilderRuntimeInfoRepository.java index 55b043d806..19d756c6dc 100644 --- a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/AppBuilderRuntimeInfoRepository.java +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/AppBuilderRuntimeInfoRepository.java @@ -31,4 +31,20 @@ public interface AppBuilderRuntimeInfoRepository { * @param info {@link AppBuilderRuntimeInfo} 运行时信息. */ void insertOne(AppBuilderRuntimeInfo info); + + /** + * 获取运行时信息过期历史记录。 + * + * @param expiredDays 表示超期天数的 {@code int}。 + * @param limit 表示查询条数的 {@code int}。 + * @return 表示超期运行时信息id的 {@link List}{@code <}{@link Long}{@code >}。 + */ + List getExpiredRuntimeInfos(int expiredDays, int limit); + + /** + * 根据运行时信息id列表强制删除历史记录。 + * + * @param runtimeInfoIds 表示历史记录的id列表的 {@link List}{@code <}{@link Long}{@code >}。 + */ + void deleteRuntimeInfos(List runtimeInfoIds); } diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/impl/AippChatRepositoryImpl.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/impl/AippChatRepositoryImpl.java new file mode 100644 index 0000000000..71a3a0954f --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/impl/AippChatRepositoryImpl.java @@ -0,0 +1,67 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.repository.impl; + +import modelengine.fit.jober.aipp.entity.ChatAndInstanceMap; +import modelengine.fit.jober.aipp.entity.ChatInfo; +import modelengine.fit.jober.aipp.mapper.AippChatMapper; +import modelengine.fit.jober.aipp.repository.AippChatRepository; +import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.transaction.Transactional; +import modelengine.fitframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * {@link AippChatRepository} 对应实现类。 + * + * @author 杨祥宇 + * @since 2025-04-09 + */ +@Component +public class AippChatRepositoryImpl implements AippChatRepository { + private final AippChatMapper aippChatMapper; + + /** + * 表示用对话持久层构造 {@link AippChatRepositoryImpl} 的实例。 + * + * @param aippChatMapper 表示对话持久层实例的 {@link AippChatMapper}。 + */ + public AippChatRepositoryImpl(AippChatMapper aippChatMapper) {this.aippChatMapper = aippChatMapper;} + + @Override + public List getExpiredChatIds(int expiredDays, int limit) { + return this.aippChatMapper.getExpiredChatIds(expiredDays, limit); + } + + @Override + @Transactional + public void forceDeleteChat(List chatIds) { + if (CollectionUtils.isEmpty(chatIds)) { + return; + } + this.aippChatMapper.forceDeleteChat(chatIds); + this.aippChatMapper.deleteWideRelationshipByChatIds(chatIds); + } + + @Override + public List selectByChatIds(List chatIds) { + if (CollectionUtils.isEmpty(chatIds)) { + return new ArrayList<>(); + } + return this.aippChatMapper.selectByChatIds(chatIds); + } + + @Override + public List selectTaskInstanceRelationsByChatIds(List chatIds) { + if (CollectionUtils.isEmpty(chatIds)) { + return new ArrayList<>(); + } + return this.aippChatMapper.selectTaskInstanceRelationsByChatIds(chatIds); + } +} diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/impl/AippInstanceLogRepositoryImpl.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/impl/AippInstanceLogRepositoryImpl.java new file mode 100644 index 0000000000..394a605d45 --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/impl/AippInstanceLogRepositoryImpl.java @@ -0,0 +1,51 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.repository.impl; + +import modelengine.fit.jober.aipp.entity.AippInstLog; +import modelengine.fit.jober.aipp.mapper.AippLogMapper; +import modelengine.fit.jober.aipp.repository.AippInstanceLogRepository; +import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.util.CollectionUtils; + +import java.util.List; + +/** + * {@link AippInstanceLogRepository} 对应实现类。 + * + * @author 杨祥宇 + * @since 2025-04-09 + */ +@Component +public class AippInstanceLogRepositoryImpl implements AippInstanceLogRepository { + private final AippLogMapper aippLogMapper; + + /** + * 表示用日志持久层构造 {@link AippInstanceLogRepositoryImpl} 的实例。 + * + * @param aippLogMapper 表示日志持久层实例的 {@link AippLogMapper}。 + */ + public AippInstanceLogRepositoryImpl(AippLogMapper aippLogMapper) {this.aippLogMapper = aippLogMapper;} + + @Override + public List getExpireInstanceLogIds(String aippType, int expiredDays, int limit) { + return this.aippLogMapper.getExpireInstanceLogIds(aippType, expiredDays, limit); + } + + @Override + public void forceDeleteInstanceLogs(List logIds) { + if (CollectionUtils.isEmpty(logIds)) { + return; + } + this.aippLogMapper.forceDeleteInstanceLogsByIds(logIds); + } + + @Override + public List selectByLogIds(List logIds) { + return this.aippLogMapper.selectByLogIds(logIds); + } +} diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/impl/AppBuilderRuntimeInfoRepositoryImpl.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/impl/AppBuilderRuntimeInfoRepositoryImpl.java index a7b02c19e4..dc45f2978b 100644 --- a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/impl/AppBuilderRuntimeInfoRepositoryImpl.java +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/repository/impl/AppBuilderRuntimeInfoRepositoryImpl.java @@ -11,6 +11,7 @@ import modelengine.fit.jober.aipp.repository.AppBuilderRuntimeInfoRepository; import modelengine.fit.jober.aipp.serializer.impl.AppBuilderRuntimeInfoSerializer; import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.util.CollectionUtils; import java.util.List; import java.util.stream.Collectors; @@ -44,4 +45,17 @@ public List selectByTraceId(String traceId) { public void insertOne(AppBuilderRuntimeInfo info) { this.mapper.insertOne(this.serializer.serialize(info)); } + + @Override + public List getExpiredRuntimeInfos(int expiredDays, int limit) { + return this.mapper.getExpiredRuntimeInfos(expiredDays, limit); + } + + @Override + public void deleteRuntimeInfos(List runtimeInfoIds) { + if (CollectionUtils.isEmpty(runtimeInfoIds)) { + return; + } + this.mapper.deleteRuntimeInfos(runtimeInfoIds); + } } diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/AippInstanceLogCleaner.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/AippInstanceLogCleaner.java new file mode 100644 index 0000000000..f514edb09e --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/AippInstanceLogCleaner.java @@ -0,0 +1,140 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.service.scheduletask; + +import com.opencsv.CSVWriter; + +import modelengine.fit.jober.aipp.entity.AippInstLog; +import modelengine.fit.jober.aipp.enums.AippTypeEnum; +import modelengine.fit.jober.aipp.repository.AippInstanceLogRepository; +import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.annotation.Value; +import modelengine.fitframework.log.Logger; +import modelengine.fitframework.util.CollectionUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +import static modelengine.fit.jober.aipp.service.scheduletask.AppBuilderDbCleanScheduler.FILE_MAX_NUM; + +/** + * 应用实例日志清理器。 + * + * @author 杨祥宇 + * @since 2025-04-15 + */ +@Component +public class AippInstanceLogCleaner { + private static final Logger log = Logger.get(AippInstanceLogCleaner.class); + private static final String FILE_NAME = "aipp-instance-log"; + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd"); + private static final String CONNECTOR = "-"; + + private final AippInstanceLogRepository instanceLogRepo; + private final CsvWriterHelper csvWriterHelper; + private final String aippInstanceLogFilePath; + + /** + * 表示用应用日志仓库实例构造 {@link AippInstanceLogCleaner} 的实例。 + * + * @param instanceLogRepo 表示应用日志仓库实例的 {@link AippInstanceLogRepository}。 + * @param csvWriterHelper 表示文件写入助手实例的 {@link CsvWriterHelper}。 + * @param aippInstanceLogFilePath 表示日志文件备份路径的 {@link String}。 + */ + public AippInstanceLogCleaner(AippInstanceLogRepository instanceLogRepo, CsvWriterHelper csvWriterHelper, + @Value("${aipp.instance.log.file.path}") String aippInstanceLogFilePath) { + this.instanceLogRepo = instanceLogRepo; + this.csvWriterHelper = csvWriterHelper; + this.aippInstanceLogFilePath = aippInstanceLogFilePath; + } + + /** + * 清理已发布的应用对话历史记录表数据，并备份。 + * + * @param expiredDays 表示数据最大保留时长的 {@code int}。 + * @param limit 表示批量处理数量的 {@code int}。 + */ + public void cleanAippInstanceNormalLog(int expiredDays, int limit) { + try { + while (true) { + List instanceLogIds = + this.instanceLogRepo.getExpireInstanceLogIds(AippTypeEnum.NORMAL.type(), expiredDays, limit); + if (instanceLogIds.isEmpty()) { + break; + } + backupData(instanceLogIds); + this.instanceLogRepo.forceDeleteInstanceLogs(instanceLogIds); + } + cleanupOldBackups(FILE_MAX_NUM); + } catch (Exception e) { + log.error("Error occurred while business data cleaner, exception:.", e); + } + } + + private void backupData(List logIds) { + List aippInstLogs = this.instanceLogRepo.selectByLogIds(logIds); + if (CollectionUtils.isEmpty(aippInstLogs)) { + return; + } + String currentDate = LocalDate.now().format(DATE_FORMATTER); + Path backupPath = Paths.get(this.aippInstanceLogFilePath, FILE_NAME + CONNECTOR + currentDate + ".csv"); + try (CSVWriter csvWriter = this.csvWriterHelper.createCsvWriter(backupPath, true)) { + List backupData = aippInstLogs.stream().map(aippInstLog -> new String[] { + String.valueOf(aippInstLog.getLogId()), aippInstLog.getAippId(), aippInstLog.getVersion(), + aippInstLog.getInstanceId(), aippInstLog.getLogData(), aippInstLog.getLogType(), + String.valueOf(aippInstLog.getCreateAt()), aippInstLog.getCreateUserAccount(), aippInstLog.getPath() + }).toList(); + csvWriter.writeAll(backupData); + } catch (IOException e) { + log.error("Error occurred while writing aipp-instance-log.", e); + throw new IllegalStateException(e); + } + } + + private void cleanupOldBackups(int fileMaxNum) { + File backupFolder = this.csvWriterHelper.getFile(this.aippInstanceLogFilePath); + File[] backupFiles = backupFolder.listFiles((dir, name) -> name.startsWith(FILE_NAME) && name.endsWith(".csv")); + if (backupFiles == null) { + return; + } + List sortedFiles = + Arrays.stream(backupFiles).sorted(Comparator.comparing(File::getName).reversed()).toList(); + for (int i = fileMaxNum; i < sortedFiles.size(); i++) { + sortedFiles.get(i).delete(); + } + } + + /** + * 清理调试应用对话历史记录表数据。 + * + * @param expiredDays 表示数据最大保留时长的 {@code int}。 + * @param limit 表示批量处理数量的 {@code int}。 + */ + public void cleanAippInstancePreviewLog(int expiredDays, int limit) { + log.info("Start cleaning aipp preview instance logs"); + try { + while (true) { + List instanceLogIds = + this.instanceLogRepo.getExpireInstanceLogIds(AippTypeEnum.PREVIEW.type(), expiredDays, limit); + if (instanceLogIds.isEmpty()) { + break; + } + this.instanceLogRepo.forceDeleteInstanceLogs(instanceLogIds); + } + } catch (Exception e) { + log.error("clean instance logs failed, exception:", e); + } + log.info("Finish cleaning aipp instance logs"); + } +} diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/AppBuilderDbCleanScheduler.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/AppBuilderDbCleanScheduler.java new file mode 100644 index 0000000000..08729ddf36 --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/AppBuilderDbCleanScheduler.java @@ -0,0 +1,76 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.service.scheduletask; + +import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.annotation.Value; +import modelengine.fitframework.log.Logger; +import modelengine.fitframework.schedule.annotation.Scheduled; + +/** + * 数据库清理定时任务执行器。 + * + * @author 杨祥宇 + * @since 2025-04-09 + */ +@Component +public class AppBuilderDbCleanScheduler { + private static final Logger log = Logger.get(AppBuilderDbCleanScheduler.class); + + /** + * 表示待清理的数据行数上限。 + */ + private static final int LIMIT = 1000; + + /** + * 表示备份文件的最大数量。 + */ + public static final int FILE_MAX_NUM = 15; + + private final int nonBusinessDataTtl; + private final int businessDataTtl; + private final AippInstanceLogCleaner aippInstanceLogCleaner; + private final ChatSessionCleaner chatSessionCleaner; + private final AppBuilderRuntimeInfoCleaner appBuilderRuntimeInfoCleaner; + + /** + * 表示用对话清理器和运行时日志清理器构造 {@link AppBuilderDbCleanScheduler} 的实例。 + * + * @param nonBusinessDataTtl 表示非业务数据的过期时间的 {@link String}。 + * @param businessDataTtl 表示业务数据的过期时间的 {@link String}。 + * @param aippInstanceLogCleaner 表示日志清理器的 {@link AippInstanceLogCleaner}。 + * @param chatSessionCleaner 表示对话清理器的 {@link ChatSessionCleaner}。 + * @param appBuilderRuntimeInfoCleaner 表示运行时信息清理器的 {@link AppBuilderRuntimeInfoCleaner}。 + */ + public AppBuilderDbCleanScheduler(@Value("${app-engine.ttl.nonBusinessData}") int nonBusinessDataTtl, + @Value("${app-engine.ttl.businessData}") int businessDataTtl, AippInstanceLogCleaner aippInstanceLogCleaner, + ChatSessionCleaner chatSessionCleaner, AppBuilderRuntimeInfoCleaner appBuilderRuntimeInfoCleaner) { + this.nonBusinessDataTtl = nonBusinessDataTtl; + this.businessDataTtl = businessDataTtl; + this.aippInstanceLogCleaner = aippInstanceLogCleaner; + this.chatSessionCleaner = chatSessionCleaner; + this.appBuilderRuntimeInfoCleaner = appBuilderRuntimeInfoCleaner; + } + + /** + * 每天凌晨 3 点定时清理超期指定天数的应用相关数据。 + */ + @Scheduled(strategy = Scheduled.Strategy.CRON, value = "0 0 3 * * ?") + public void appBuilderDbCleanSchedule() { + try { + // 清理非业务数据 + this.aippInstanceLogCleaner.cleanAippInstancePreviewLog(this.nonBusinessDataTtl, LIMIT); + this.appBuilderRuntimeInfoCleaner.clean(this.nonBusinessDataTtl, LIMIT); + + // 清理业务数据 + this.aippInstanceLogCleaner.cleanAippInstanceNormalLog(this.businessDataTtl, LIMIT); + this.chatSessionCleaner.clean(this.businessDataTtl, LIMIT); + } catch (Exception e) { + log.error("App builder Db Clean Error, exception:", e); + } + } +} diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/AppBuilderRuntimeInfoCleaner.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/AppBuilderRuntimeInfoCleaner.java new file mode 100644 index 0000000000..1e3edbba45 --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/AppBuilderRuntimeInfoCleaner.java @@ -0,0 +1,52 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.service.scheduletask; + +import modelengine.fit.jober.aipp.repository.AppBuilderRuntimeInfoRepository; +import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.log.Logger; + +import java.util.List; + +/** + * 应用编排运行时信息清理器。 + * + * @author 杨祥宇 + * @since 2025-04-15 + */ +@Component +public class AppBuilderRuntimeInfoCleaner { + private static final Logger log = Logger.get(AppBuilderRuntimeInfoCleaner.class); + + private final AppBuilderRuntimeInfoRepository runtimeInfoRepo; + + public AppBuilderRuntimeInfoCleaner(AppBuilderRuntimeInfoRepository runtimeInfoRepo) { + this.runtimeInfoRepo = runtimeInfoRepo; + } + + /** + * 清理对话运行时表数据，并备份。 + * + * @param expiredDays 表示数据最大保留时长的 {@code int}。 + * @param limit 表示批量处理数量的 {@code int}。 + */ + public void clean(int expiredDays, int limit) { + log.info("Start cleaning app builder runtime infos"); + try { + while (true) { + List expiredRuntimeInfoIds = this.runtimeInfoRepo.getExpiredRuntimeInfos(expiredDays, limit); + if (expiredRuntimeInfoIds.isEmpty()) { + break; + } + this.runtimeInfoRepo.deleteRuntimeInfos(expiredRuntimeInfoIds); + } + } catch (Exception e) { + log.error("cleaning app builder runtime infos failed, exception:", e); + } + log.info("Finish cleaning app builder runtime infos"); + } +} diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/ChatSessionCleaner.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/ChatSessionCleaner.java new file mode 100644 index 0000000000..036d2f8301 --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/ChatSessionCleaner.java @@ -0,0 +1,144 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.service.scheduletask; + +import static modelengine.fit.jober.aipp.service.scheduletask.AppBuilderDbCleanScheduler.FILE_MAX_NUM; + +import com.opencsv.CSVWriter; + +import modelengine.fit.jober.aipp.entity.ChatAndInstanceMap; +import modelengine.fit.jober.aipp.entity.ChatInfo; +import modelengine.fit.jober.aipp.repository.AippChatRepository; +import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.annotation.Value; +import modelengine.fitframework.log.Logger; +import modelengine.fitframework.util.CollectionUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +/** + * 聊天会话数据库表清理器。 + * + * @author 杨祥宇 + * @since 2025-04-15 + */ +@Component +public class ChatSessionCleaner { + private static final Logger log = Logger.get(AippInstanceLogCleaner.class); + private static final String CHAT_SESSION_FILE_NAME = "chat-session"; + private static final String INSTANCE_RELATIONS_FILE_NAME = "instance-relations"; + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyyMMdd"); + private static final String CONNECTOR = "-"; + + private final AippChatRepository chatRepo; + private final CsvWriterHelper csvWriterHelper; + private final String chatSessionFilePath; + + /** + * 表示用对话仓库和文件写入助手来构造 {@link ChatSessionCleaner} 的实例。 + * + * @param chatRepo 表示对话仓库实例的 {@link AippChatRepository}。 + * @param csvWriterHelper 表示文件写入助手实例的 {@link CsvWriterHelper}。 + * @param chatSessionFilePath 表示对话文件路径的 {@link String}。 + */ + public ChatSessionCleaner(AippChatRepository chatRepo, CsvWriterHelper csvWriterHelper, + @Value("${chat.session.file.path}") String chatSessionFilePath) { + this.chatRepo = chatRepo; + this.csvWriterHelper = csvWriterHelper; + this.chatSessionFilePath = chatSessionFilePath; + } + + /** + * 清理对话会话相关数据，并备份。 + * + * @param expiredDays 表示数据最大保留天数的 {@code int}。 + * @param limit 表示批量处理数量的 {@code int}。 + */ + public void clean(int expiredDays, int limit) { + try { + while (true) { + List expiredChatIds = this.chatRepo.getExpiredChatIds(expiredDays, limit); + if (expiredChatIds.isEmpty()) { + break; + } + backupChatSessionData(expiredChatIds); + backupInstanceRelationData(expiredChatIds); + this.chatRepo.forceDeleteChat(expiredChatIds); + } + cleanupOldBackups(CHAT_SESSION_FILE_NAME, FILE_MAX_NUM); + cleanupOldBackups(INSTANCE_RELATIONS_FILE_NAME, FILE_MAX_NUM); + } catch (Exception e) { + log.error("Error occurred while business data cleaner, exception:.", e); + } + } + + private void backupChatSessionData(List chatIds) { + List chatSessionPos = this.chatRepo.selectByChatIds(chatIds); + if (CollectionUtils.isEmpty(chatSessionPos)) { + return; + } + String currentDate = LocalDate.now().format(DATE_FORMATTER); + Path backupPath = Paths.get(this.chatSessionFilePath, CHAT_SESSION_FILE_NAME + CONNECTOR + currentDate + ".csv"); + try (CSVWriter csvWriter = this.csvWriterHelper.createCsvWriter(backupPath, true)) { + List backupData = chatSessionPos.stream().map(session -> new String[]{ + session.getChatId(), session.getAppId(), session.getVersion(), session.getChatName(), + session.getAttributes(), String.valueOf(session.getCreateTime()), session.getCreator(), + String.valueOf(session.getUpdateTime()), session.getUpdater() + }).toList(); + csvWriter.writeAll(backupData); + } catch (IOException e) { + log.error("Error occurred while backup chat session data, exception:", e); + throw new RuntimeException(e); + } + } + + private void backupInstanceRelationData(List chatIds) { + List relationPos = this.chatRepo.selectTaskInstanceRelationsByChatIds(chatIds); + if (CollectionUtils.isEmpty(relationPos)) { + return; + } + String currentDate = LocalDate.now().format(DATE_FORMATTER); + Path backupPath = Paths.get(this.chatSessionFilePath, INSTANCE_RELATIONS_FILE_NAME + CONNECTOR + currentDate + ".csv"); + try (CSVWriter csvWriter = this.csvWriterHelper.createCsvWriter(backupPath, true)) { + List backupData = relationPos.stream().map(relationPo -> new String[]{ + relationPo.getMsgId(), relationPo.getChatId(), relationPo.getInstanceId(), + String.valueOf(relationPo.getCreateTime()), relationPo.getCreator(), + String.valueOf(relationPo.getUpdateTime()), relationPo.getUpdater() + }).toList(); + csvWriter.writeAll(backupData); + } catch (IOException e) { + log.error("Error occurred while backup instance relation data, exception:", e); + throw new RuntimeException(e); + } + } + + private void cleanupOldBackups(String fileName, int fileMaxNum) { + try { + File backupFolder = this.csvWriterHelper.getFile(this.chatSessionFilePath); + File[] backupFiles = + backupFolder.listFiles((dir, name) -> name.startsWith(fileName) && name.endsWith(".csv")); + if (backupFiles == null) { + return; + } + List sortedFiles = + Arrays.stream(backupFiles).sorted(Comparator.comparing(File::getName).reversed()).toList(); + for (int i = fileMaxNum; i < sortedFiles.size(); i++) { + sortedFiles.get(i).delete(); + } + } catch (Exception e) { + log.error("Cleanup old backups failed, filename:{}, exception:", fileName, e); + } + } +} diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/CsvWriterHelper.java b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/CsvWriterHelper.java new file mode 100644 index 0000000000..6d33098801 --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/main/java/modelengine/fit/jober/aipp/service/scheduletask/CsvWriterHelper.java @@ -0,0 +1,47 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.service.scheduletask; + +import com.opencsv.CSVWriter; + +import modelengine.fitframework.annotation.Component; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; + +/** + * csv 文件写入助手。 + * + * @author 杨祥宇 + * @since 2025-04-18 + */ +@Component +public class CsvWriterHelper { + /** + * 生成 CsvWriter 对象。 + * + * @param path 表示文件路径的 {@link Path}。 + * @param isAppend 表示是否追加的 {@link Boolean}。 + * @return 表示生成 CSVWriter类的 {@link CSVWriter}。 + * @throws IOException 表示可能抛出异常的 {@link IOException}。 + */ + public CSVWriter createCsvWriter(Path path, boolean isAppend) throws IOException { + return new CSVWriter(new FileWriter(path.toFile(), isAppend)); + } + + /** + * 生成 File 对象。 + * + * @param path 表示文件路径的 {@link Path}。 + * @return 表示生成 File 对象的 {@link File}。 + */ + public File getFile(String path) { + return new File(path); + } +} diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/resources/application-prod.yml b/app-builder/jane/plugins/aipp-plugin/src/main/resources/application-prod.yml index 54c5d33be4..6bba3a8641 100644 --- a/app-builder/jane/plugins/aipp-plugin/src/main/resources/application-prod.yml +++ b/app-builder/jane/plugins/aipp-plugin/src/main/resources/application-prod.yml @@ -51,6 +51,9 @@ app-engine: file: upload: maxStorageRatio: 0.9 + ttl: + businessData: 15 + nonBusinessData: 1 elsa: endpoint: elsaKey: @@ -79,4 +82,13 @@ export-meta: sensitive: replace: - pattern: "\"timestamp\":\"[^\"]+\"" - to: "\"timestamp\":\"***\"" \ No newline at end of file + to: "\"timestamp\":\"***\"" +aipp: + instance: + log: + file: + path: /var/share/backup/aipp-instance-log/ +chat: + session: + file: + path: /var/share/backup/chat-session/ \ No newline at end of file diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AippChatMapper.xml b/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AippChatMapper.xml index b9b61ead42..d5ae54c057 100644 --- a/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AippChatMapper.xml +++ b/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AippChatMapper.xml @@ -15,6 +15,28 @@ + + + + + + + + + + + + + + + + + + + + + + chat_id, app_id, app_version, name, attributes, create_at, create_by, update_at, update_by, status @@ -220,4 +242,54 @@ order by update_at desc offset #{requestParam.offset} limit #{requestParam.limit} + + + + + DELETE FROM + t_chat_session + where chat_id in + + #{item} + + + + + DELETE FROM + t_chat_session_task_instance_wide_relationship + where chat_id in + + #{item} + + + + + + \ No newline at end of file diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AippLogMapper.xml b/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AippLogMapper.xml index e4ede1eca0..75315fc9cd 100644 --- a/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AippLogMapper.xml +++ b/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AippLogMapper.xml @@ -263,4 +263,37 @@ #{item} + + + + + DELETE FROM + aipp_instance_log + where log_id in + + #{item} + + + + \ No newline at end of file diff --git a/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AppBuilderRuntimeInfoMapper.xml b/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AppBuilderRuntimeInfoMapper.xml index b85dbd4178..5019761064 100644 --- a/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AppBuilderRuntimeInfoMapper.xml +++ b/app-builder/jane/plugins/aipp-plugin/src/main/resources/mapper/AppBuilderRuntimeInfoMapper.xml @@ -88,4 +88,24 @@ where trace_id = #{traceId} order by start_time asc + + + + + DELETE + FROM app_builder_runtime_info + where id in + + #{item} + + \ No newline at end of file diff --git a/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/repository/impl/AippInstanceLogRepositoryImplTest.java b/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/repository/impl/AippInstanceLogRepositoryImplTest.java new file mode 100644 index 0000000000..8b71beafec --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/repository/impl/AippInstanceLogRepositoryImplTest.java @@ -0,0 +1,90 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.repository.impl; + +import modelengine.fit.jober.aipp.dto.aipplog.AippLogCreateDto; +import modelengine.fit.jober.aipp.entity.AippInstLog; +import modelengine.fit.jober.aipp.mapper.AippLogMapper; +import modelengine.fit.jober.aipp.repository.AippInstanceLogRepository; +import modelengine.fit.jober.aipp.service.DatabaseBaseTest; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.List; + +/** + * {@link AippInstanceLogRepositoryImpl} 对应测试类。 + * + * @author 杨祥宇 + * @since 2025-04-10 + */ +public class AippInstanceLogRepositoryImplTest extends DatabaseBaseTest { + private final AippLogMapper mapper = sqlSessionManager.openSession(true).getMapper(AippLogMapper.class); + private AippInstanceLogRepository repo; + + @BeforeEach + void setUp() { + this.repo = new AippInstanceLogRepositoryImpl(this.mapper); + } + + @Test + @DisplayName("测试成功刪除调试信息") + void testForceDeleteInstanceLogsSuccess() { + AippLogCreateDto dto = AippLogCreateDto.builder() + .logId("1") + .aippId("1") + .aippType("PREVIEW") + .instanceId("1") + .logData("{}") + .logType("QUESTION") + .createUserAccount("yyy") + .build(); + this.mapper.insertOne(dto); + this.repo.forceDeleteInstanceLogs(Collections.singletonList(1L)); + List expirePreviewInstanceLogs = this.repo.selectByLogIds(Collections.singletonList(1L)); + Assertions.assertEquals(0, expirePreviewInstanceLogs.size()); + } + + @Test + @DisplayName("测试成功获取过期调试信息") + void testGetExpiredPreviewInstanceLogsSuccess() { + AippLogCreateDto dto = AippLogCreateDto.builder() + .logId("2") + .aippId("2") + .aippType("PREVIEW") + .instanceId("2") + .logData("{}") + .logType("QUESTION") + .createUserAccount("yyy") + .build(); + this.mapper.insertOne(dto); + List expirePreviewInstanceLogs = this.repo.selectByLogIds(Collections.singletonList(2L)); + Assertions.assertEquals(1, expirePreviewInstanceLogs.size()); + } + + @Test + @DisplayName("测试根据对话 id 成功查询对话详细信息") + void testSelectByLogIdSuccess() { + AippLogCreateDto dto = AippLogCreateDto.builder() + .logId("3") + .aippId("3") + .aippType("PREVIEW") + .instanceId("3") + .logData("{}") + .logType("QUESTION") + .createUserAccount("yyy") + .build(); + this.mapper.insertOne(dto); + List aippInstLogs = this.repo.selectByLogIds(Collections.singletonList(3L)); + Assertions.assertEquals(1, aippInstLogs.size()); + Assertions.assertEquals("3", aippInstLogs.get(0).getInstanceId()); + } +} \ No newline at end of file diff --git a/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/repository/impl/AppBuilderRuntimeInfoRepositoryImplTest.java b/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/repository/impl/AppBuilderRuntimeInfoRepositoryImplTest.java new file mode 100644 index 0000000000..2c8bea58bc --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/repository/impl/AppBuilderRuntimeInfoRepositoryImplTest.java @@ -0,0 +1,74 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.repository.impl; + +import modelengine.fit.jober.aipp.domain.AppBuilderRuntimeInfo; +import modelengine.fit.jober.aipp.mapper.AppBuilderRuntimeInfoMapper; +import modelengine.fit.jober.aipp.repository.AppBuilderRuntimeInfoRepository; +import modelengine.fit.jober.aipp.service.DatabaseBaseTest; +import modelengine.fit.runtime.entity.Parameter; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.util.Arrays; +import java.util.List; + +/** + * {@link AppBuilderRuntimeInfoRepositoryImpl} 对应测试类。 + * + * @author 杨祥宇 + * @since 2025-04-10 + */ +class AppBuilderRuntimeInfoRepositoryImplTest extends DatabaseBaseTest { + private final AppBuilderRuntimeInfoMapper mapper = + sqlSessionManager.openSession(true).getMapper(AppBuilderRuntimeInfoMapper.class); + private AppBuilderRuntimeInfoRepository runtimeInfoRepository; + + @BeforeEach + void setUp() { + this.runtimeInfoRepository = new AppBuilderRuntimeInfoRepositoryImpl(mapper); + } + + @Test + @DisplayName("测试成功获取超期的运行时信息") + void testGetExpiredRuntimeInfosSuccess() { + AppBuilderRuntimeInfo info = genRuntimeInfo(); + this.runtimeInfoRepository.insertOne(info); + List expiredRuntimeInfos = this.runtimeInfoRepository.selectByTraceId("0"); + Assertions.assertEquals(0, expiredRuntimeInfos.size()); + } + + private AppBuilderRuntimeInfo genRuntimeInfo() { + Parameter parameter = new Parameter(); + return AppBuilderRuntimeInfo.builder() + .traceId("1") + .flowDefinitionId("1") + .instanceId("1") + .nodeId("1") + .nodeType("PREVIEW") + .startTime(1) + .endTime(2) + .status("ERROR") + .parameters(List.of(parameter)) + .createAt(LocalDateTime.now()) + .updateAt(LocalDateTime.now()) + .build(); + } + + @Test + @DisplayName("测试成功根据 id 列表删除运行信息") + void testDeleteRuntimeInfosSuccess() { + this.runtimeInfoRepository.insertOne(genRuntimeInfo()); + this.mapper.deleteRuntimeInfos(Arrays.asList(1L, 2L)); + List expiredRuntimeInfos = this.runtimeInfoRepository.selectByTraceId("1"); + Assertions.assertEquals(0, expiredRuntimeInfos.size()); + } +} \ No newline at end of file diff --git a/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/service/scheduletask/AippInstanceLogCleanerTest.java b/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/service/scheduletask/AippInstanceLogCleanerTest.java new file mode 100644 index 0000000000..5e026fa5ce --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/service/scheduletask/AippInstanceLogCleanerTest.java @@ -0,0 +1,107 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.service.scheduletask; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.opencsv.CSVWriter; + +import modelengine.fit.jober.aipp.entity.AippInstLog; +import modelengine.fit.jober.aipp.enums.AippTypeEnum; +import modelengine.fit.jober.aipp.repository.AippInstanceLogRepository; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +/** + * {@link AippInstanceLogCleaner} 对应测试类。 + * + * @author 杨祥宇 + * @since 2025-04-18 + */ +@ExtendWith(MockitoExtension.class) +public class AippInstanceLogCleanerTest { + @Mock + private AippInstanceLogRepository instanceLogRepo; + + @Mock + private CsvWriterHelper csvWriterHelper; + + private AippInstanceLogCleaner logCleaner; + + @BeforeEach + void setup() { + String aippInstanceLogFilePath = "/var/share/backup/aipp-instance-log/"; + this.logCleaner = new AippInstanceLogCleaner(this.instanceLogRepo, this.csvWriterHelper, aippInstanceLogFilePath); + } + + @Test + @DisplayName("测试清理 Normal 信息并备份") + void cleanAippInstanceNormalLogShouldBackupAndDelete() throws Exception { + List mockLogIds = List.of(1L, 2L); + when(this.instanceLogRepo.getExpireInstanceLogIds(AippTypeEnum.NORMAL.type(), 30, 100)).thenReturn(mockLogIds) + .thenReturn(Collections.emptyList()); + AippInstLog mockLog = new AippInstLog(); + when(this.instanceLogRepo.selectByLogIds(anyList())).thenReturn(List.of(mockLog)); + CSVWriter csvWriter = mock(CSVWriter.class); + when(this.csvWriterHelper.createCsvWriter(any(), anyBoolean())).thenReturn(csvWriter); + File file = mock(File.class); + File csvFile = mock(File.class); + when(this.csvWriterHelper.getFile(anyString())).thenReturn(file); + when(file.listFiles(any(FilenameFilter.class))).thenReturn(new File[] {csvFile}); + this.logCleaner.cleanAippInstanceNormalLog(30, 100); + verify(this.instanceLogRepo, times(2)).getExpireInstanceLogIds(anyString(), anyInt(), anyInt()); + verify(this.instanceLogRepo, times(1)).forceDeleteInstanceLogs(anyList()); + verify(csvWriter, times(1)).writeAll(anyList()); + verify(csvFile, times(0)).delete(); + } + + @Test + @DisplayName("测试 Normal 信息备份失败时不清理数据") + void cleanAippInstanceNormalLogShouldNotDeleteWhenBackupFailed() throws Exception { + List mockLogIds = List.of(1L, 2L); + when(this.instanceLogRepo.getExpireInstanceLogIds(AippTypeEnum.NORMAL.type(), 30, 100)).thenReturn(mockLogIds) + .thenReturn(Collections.emptyList()); + when(this.instanceLogRepo.selectByLogIds(anyList())).thenReturn(Collections.singletonList(new AippInstLog())); + CSVWriter csvWriter = mock(CSVWriter.class); + when(this.csvWriterHelper.createCsvWriter(any(), anyBoolean())).thenThrow(new IOException("error")); + this.logCleaner.cleanAippInstanceNormalLog(30, 100); + verify(this.instanceLogRepo, times(1)).getExpireInstanceLogIds(anyString(), anyInt(), anyInt()); + verify(this.instanceLogRepo, times(0)).forceDeleteInstanceLogs(anyList()); + verify(csvWriter, times(0)).writeAll(anyList()); + } + + @Test + public void cleanAippInstancePreviewLogMultipleBatches_DeletesAll() { + Mockito.when(this.instanceLogRepo.getExpireInstanceLogIds(Mockito.eq(AippTypeEnum.PREVIEW.type()), + Mockito.anyInt(), + Mockito.anyInt())).thenReturn(Arrays.asList(1L, 2L)).thenReturn(Collections.emptyList()); + this.logCleaner.cleanAippInstancePreviewLog(30, 100); + Mockito.verify(this.instanceLogRepo, Mockito.times(2)).getExpireInstanceLogIds(AippTypeEnum.PREVIEW.type(), 30, 100); + Mockito.verify(this.instanceLogRepo).forceDeleteInstanceLogs(Arrays.asList(1L, 2L)); + } +} \ No newline at end of file diff --git a/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/service/scheduletask/ChatSessionCleanerTest.java b/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/service/scheduletask/ChatSessionCleanerTest.java new file mode 100644 index 0000000000..fe3e473105 --- /dev/null +++ b/app-builder/jane/plugins/aipp-plugin/src/test/java/modelengine/fit/jober/aipp/service/scheduletask/ChatSessionCleanerTest.java @@ -0,0 +1,108 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * This file is a part of the ModelEngine Project. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +package modelengine.fit.jober.aipp.service.scheduletask; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.opencsv.CSVWriter; + +import modelengine.fit.jober.aipp.entity.ChatAndInstanceMap; +import modelengine.fit.jober.aipp.entity.ChatInfo; +import modelengine.fit.jober.aipp.repository.AippChatRepository; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import java.io.File; +import java.io.FilenameFilter; +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +/** + * {@link ChatSessionCleaner} 对应测试类。 + * + * @author 杨祥宇 + * @since 2025-04-18 + */ +@ExtendWith(MockitoExtension.class) +public class ChatSessionCleanerTest { + @Mock + private AippChatRepository chatRepo; + + @Mock + private CsvWriterHelper csvWriterHelper; + + private ChatSessionCleaner chatSessionCleaner; + + @BeforeEach + void setup() { + String chatSessionFilePath = "/var/share/backup/chat-session/"; + this.chatSessionCleaner = new ChatSessionCleaner(this.chatRepo, this.csvWriterHelper, chatSessionFilePath); + } + + @Test + @DisplayName("测试没有过期数据时直接返回，不会做备份和删除操作") + void cleanNoExpiredDataShouldDoNothing() throws IOException { + when(this.chatRepo.getExpiredChatIds(anyInt(), anyInt())).thenReturn(Collections.emptyList()); + this.chatSessionCleaner.clean(30, 100); + verify(this.chatRepo, never()).forceDeleteChat(anyList()); + verify(this.csvWriterHelper, never()).createCsvWriter(any(), anyBoolean()); + } + + @Test + @DisplayName("测试有过期数据时备份并成功删除") + void cleanWithExpiredDataShouldBackupAndDelete() throws Exception { + List expiredChatIds = List.of("chat1", "chat2"); + when(this.chatRepo.getExpiredChatIds(anyInt(), anyInt())).thenReturn(expiredChatIds) + .thenReturn(Collections.emptyList()); + when(this.chatRepo.selectByChatIds(expiredChatIds)).thenReturn(List.of(new ChatInfo(), new ChatInfo())); + when(this.chatRepo.selectTaskInstanceRelationsByChatIds(expiredChatIds)).thenReturn(List.of(new ChatAndInstanceMap(), + new ChatAndInstanceMap())); + CSVWriter csvWriter = mock(CSVWriter.class); + when(this.csvWriterHelper.createCsvWriter(any(), anyBoolean())).thenReturn(csvWriter); + File file = mock(File.class); + File csvFile = mock(File.class); + when(this.csvWriterHelper.getFile(anyString())).thenReturn(file); + when(file.listFiles(any(FilenameFilter.class))).thenReturn(new File[] {csvFile}); + this.chatSessionCleaner.clean(30, 100); + verify(this.chatRepo, times(1)).forceDeleteChat(expiredChatIds); + verify(this.csvWriterHelper, atLeast(2)).createCsvWriter(any(), eq(true)); + verify(csvWriter, times(2)).writeAll(anyList()); + verify(csvFile, times(0)).delete(); + } + + @Test + @DisplayName("测试数据备份失败时不会删除过期数据") + void cleanShouldNotDeleteWhenBackupFailed() throws Exception { + List expiredChatIds = List.of("chat1", "chat2"); + when(this.chatRepo.getExpiredChatIds(anyInt(), anyInt())).thenReturn(expiredChatIds) + .thenReturn(Collections.emptyList()); + when(this.chatRepo.selectByChatIds(anyList())).thenReturn(Collections.singletonList(new ChatInfo())); + CSVWriter csvWriter = mock(CSVWriter.class); + when(this.csvWriterHelper.createCsvWriter(any(), anyBoolean())).thenThrow(new IOException("error")); + this.chatSessionCleaner.clean(30, 100); + verify(this.chatRepo, times(1)).getExpiredChatIds(anyInt(), anyInt()); + verify(this.chatRepo, times(0)).forceDeleteChat(anyList()); + verify(csvWriter, times(0)).writeAll(anyList()); + } +} \ No newline at end of file diff --git a/app-builder/jane/plugins/aipp-plugin/src/test/resources/sql/clear_table.sql b/app-builder/jane/plugins/aipp-plugin/src/test/resources/sql/clear_table.sql index 4dbd79dd8c..b3bf632b7b 100644 --- a/app-builder/jane/plugins/aipp-plugin/src/test/resources/sql/clear_table.sql +++ b/app-builder/jane/plugins/aipp-plugin/src/test/resources/sql/clear_table.sql @@ -16,4 +16,8 @@ truncate table app_builder_form; truncate table app_builder_form_property; -truncate table app_template; \ No newline at end of file +truncate table app_template; + +truncate table app_builder_runtime_info; + +truncate table t_chat_session; \ No newline at end of file diff --git a/app-builder/jane/plugins/aipp-plugin/src/test/resources/sql/init.sql b/app-builder/jane/plugins/aipp-plugin/src/test/resources/sql/init.sql index 6858e0a7cf..7bebe480be 100644 --- a/app-builder/jane/plugins/aipp-plugin/src/test/resources/sql/init.sql +++ b/app-builder/jane/plugins/aipp-plugin/src/test/resources/sql/init.sql @@ -145,4 +145,38 @@ create table if not exists app_template ( update_by varchar(64) not null, update_at timestamp(6) not null default CURRENT_TIMESTAMP, is_deleted int2 not null default 0 +); + +create table if not exists app_builder_runtime_info +( + id BIGSERIAL primary key, + trace_id varchar(64) not null, + flow_definition_id varchar(64) not null, + instance_id varchar(64) not null, + node_id varchar(64) not null, + node_type varchar(32) not null, + start_time bigint not null, + end_time bigint not null, + status varchar(32), + published smallint not null, + error_msg text, + next_position_id varchar(64), + parameters json not null DEFAULT '[]', + create_by varchar(64), + create_at timestamp not null default current_timestamp, + update_by varchar(64), + update_at timestamp not null default current_timestamp +); + +CREATE TABLE IF NOT EXISTS t_chat_session ( + chat_id VARCHAR(32) NOT NULL DEFAULT NULL, + app_id VARCHAR(32) NULL DEFAULT NULL, + app_version VARCHAR(32) NULL DEFAULT NULL, + name VARCHAR(2000) NULL DEFAULT NULL, + attributes JSON NULL DEFAULT NULL, + create_at TIMESTAMP(6) NULL DEFAULT NULL, + create_by VARCHAR(32) NULL DEFAULT NULL, + update_at TIMESTAMP(6) NULL DEFAULT NULL, + update_by VARCHAR(32) NULL DEFAULT NULL, + status INT4 NULL DEFAULT NULL ); \ No newline at end of file diff --git a/app-builder/jane/task-new/pom.xml b/app-builder/jane/task-new/pom.xml index d0c7beb5c9..bdfe476a91 100644 --- a/app-builder/jane/task-new/pom.xml +++ b/app-builder/jane/task-new/pom.xml @@ -63,6 +63,10 @@ modelengine.fit.jane aipp-service + + org.fitframework.extension + fit-schedule + diff --git a/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/MetaInstanceServiceImpl.java b/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/MetaInstanceServiceImpl.java index 873db4fe9d..7726d0d0f3 100644 --- a/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/MetaInstanceServiceImpl.java +++ b/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/MetaInstanceServiceImpl.java @@ -20,6 +20,10 @@ import modelengine.fit.task_new.repository.MetaInstanceRepository; import modelengine.fit.task_new.util.UUIDUtil; import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.annotation.Value; +import modelengine.fitframework.log.Logger; +import modelengine.fitframework.schedule.annotation.Scheduled; +import modelengine.fitframework.util.CollectionUtils; import java.util.Collections; import java.util.List; @@ -33,10 +37,26 @@ */ @Component public class MetaInstanceServiceImpl implements MetaInstanceService { + private static final Logger log = Logger.get(MetaInstanceServiceImpl.class); + + /** + * 表示待清理数据的数量的上限。 + */ + private static final int LIMIT = 1000; + private final MetaInstanceRepository metaInstanceRepository; + private final int expiredDays; - public MetaInstanceServiceImpl(MetaInstanceRepository metaInstanceRepository) { + /** + * 表示用元数据仓库和过期时间构造 {@link MetaInstanceServiceImpl} 的实例。 + * + * @param metaInstanceRepository 表示元数据仓库实例的 {@link MetaInstanceRepository}。 + * @param expiredDays 表示过期时间的 {@link String}。 + */ + public MetaInstanceServiceImpl(MetaInstanceRepository metaInstanceRepository, + @Value("${task.expiredDays}") int expiredDays) { this.metaInstanceRepository = metaInstanceRepository; + this.expiredDays = expiredDays; } @Override @@ -104,4 +124,24 @@ public Instance retrieveById(String instanceId, OperationContext context) { RangedResultSet resultSet = this.list(Collections.singletonList(instanceId), 0, 1, null); return resultSet.getResults().get(0); } + + /** + * 每天凌晨 3 点定时清理超期指定天数的任务实例数据。 + */ + @Scheduled(strategy = Scheduled.Strategy.CRON, value = "0 0 3 * * ?") + public void taskInstanceDbCleanSchedule() { + log.info("Start clean task instance db"); + try { + while (true) { + List expiredInstanceIds = this.metaInstanceRepository.getExpiredInstanceIds(expiredDays, LIMIT); + if (CollectionUtils.isEmpty(expiredInstanceIds)) { + break; + } + this.metaInstanceRepository.forceDelete(expiredInstanceIds); + } + } catch (Exception e) { + log.error("Error clean task instance db, exception:", e); + } + log.info("Finish clean task instance db"); + } } diff --git a/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/mapper/MetaInstanceMapper.java b/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/mapper/MetaInstanceMapper.java index e6cf8d3c51..262fe5786b 100644 --- a/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/mapper/MetaInstanceMapper.java +++ b/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/mapper/MetaInstanceMapper.java @@ -12,7 +12,7 @@ import java.util.List; /** - * Meta 实例数据库 Mapper 类 + * Meta 实例数据库 Mapper 类。 * * @author 邬涨财 * @since 2025-03-31 @@ -54,4 +54,20 @@ public interface MetaInstanceMapper { * @return 表示查询后的结果的 {@code long} */ long count(MetaInstanceCondition cond); + + /** + * 根据元数据实例唯一标识列表强制删除会话记录。 + * + * @param ids 表示元数据实例唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + */ + void forceDelete(List ids); + + /** + * 获取超期的元数据实例唯一标识列表。 + * + * @param expiredDays 表示超期时间的 {@code int}。 + * @param limit 表示查询条数的 {@code int}。 + * @return 表示元数据实例的唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + */ + List getExpiredInstanceIds(int expiredDays, int limit); } diff --git a/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/repository/MetaInstanceRepository.java b/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/repository/MetaInstanceRepository.java index c452aa86cc..6b96e39452 100644 --- a/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/repository/MetaInstanceRepository.java +++ b/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/repository/MetaInstanceRepository.java @@ -9,10 +9,11 @@ import modelengine.fit.task_new.condition.MetaInstanceCondition; import modelengine.fit.task_new.entity.MetaInstance; +import java.time.LocalDateTime; import java.util.List; /** - * Meta 实例数据库 Repo 层接口 + * Meta 实例数据库 Repo 层接口。 * * @author 邬涨财 * @since 2025-03-31 @@ -54,4 +55,20 @@ public interface MetaInstanceRepository { * @return 表示查询后的结果的 {@code long} */ long count(MetaInstanceCondition condition); + + /** + * 获取超期的元数据实例唯一标识列表。 + * + * @param expiredDays 表示超期时间的 {@link LocalDateTime}。 + * @param limit 表示查询条数的 {@code int}。 + * @return 表示元数据实例的唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + */ + List getExpiredInstanceIds(int expiredDays, int limit); + + /** + * 根据元数据实例唯一标识列表强制删除会话记录。 + * + * @param ids 表示元数据实例唯一标识列表的 {@link List}{@code <}{@link String}{@code >}。 + */ + void forceDelete(List ids); } diff --git a/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/repository/impl/MetaInstanceRepositoryImpl.java b/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/repository/impl/MetaInstanceRepositoryImpl.java index 087cf20a03..cc1ed45b91 100644 --- a/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/repository/impl/MetaInstanceRepositoryImpl.java +++ b/app-builder/jane/task-new/src/main/java/modelengine/fit/task_new/repository/impl/MetaInstanceRepositoryImpl.java @@ -12,12 +12,13 @@ import modelengine.fit.task_new.repository.MetaInstanceRepository; import modelengine.fit.task_new.serializer.impl.MetaInstanceSerializer; import modelengine.fitframework.annotation.Component; +import modelengine.fitframework.util.CollectionUtils; import java.util.List; import java.util.stream.Collectors; /** - * Meta 实例数据库 Repo 层实现 + * Meta 实例数据库 Repo 层实现。 * * @author 邬涨财 * @since 2025-03-31 @@ -59,4 +60,17 @@ public List select(MetaInstanceCondition cond) { public long count(MetaInstanceCondition cond) { return this.metaInstanceMapper.count(cond); } + + @Override + public List getExpiredInstanceIds(int expiredDays, int limit) { + return this.metaInstanceMapper.getExpiredInstanceIds(expiredDays, limit); + } + + @Override + public void forceDelete(List ids) { + if (CollectionUtils.isEmpty(ids)) { + return; + } + this.metaInstanceMapper.forceDelete(ids); + } } diff --git a/app-builder/jane/task-new/src/main/resources/application-prod.yml b/app-builder/jane/task-new/src/main/resources/application-prod.yml index 41312e05d6..280f27b086 100644 --- a/app-builder/jane/task-new/src/main/resources/application-prod.yml +++ b/app-builder/jane/task-new/src/main/resources/application-prod.yml @@ -25,4 +25,6 @@ fit: testOnBorrow: false testOnReturn: false mybatis: - mapper-locations: mapper/*Mapper.xml \ No newline at end of file + mapper-locations: mapper/*Mapper.xml +task: + expiredDays: 1 \ No newline at end of file diff --git a/app-builder/jane/task-new/src/main/resources/mapper/MetaInstanceMapper.xml b/app-builder/jane/task-new/src/main/resources/mapper/MetaInstanceMapper.xml index d411d60a8d..3911ecb134 100644 --- a/app-builder/jane/task-new/src/main/resources/mapper/MetaInstanceMapper.xml +++ b/app-builder/jane/task-new/src/main/resources/mapper/MetaInstanceMapper.xml @@ -139,4 +139,25 @@ + + + + + DELETE FROM + task_instance_new + where id in + + #{item} + + \ No newline at end of file diff --git a/app-builder/waterflow/java/waterflow-service/src/main/resources/application.yml b/app-builder/waterflow/java/waterflow-service/src/main/resources/application.yml index e2a62fd7f9..6a861ad0e9 100644 --- a/app-builder/waterflow/java/waterflow-service/src/main/resources/application.yml +++ b/app-builder/waterflow/java/waterflow-service/src/main/resources/application.yml @@ -16,6 +16,6 @@ jane: scheduleRate: 10000 maxCount: 0 isNeedFlowCallbackAdapt: false - contextExpiredDays: 7 + contextExpiredDays: 1 distributed-lock-provider: databaseDistributedLockProvider \ No newline at end of file diff --git a/app-knowledge/plugins/edm-knowledge/src/test/java/modelengine/jade/knowledge/EdmKnowledgeBaseManagerTest.java b/app-knowledge/plugins/edm-knowledge/src/test/java/modelengine/jade/knowledge/EdmKnowledgeBaseManagerTest.java index dc2b49756a..bd72e62822 100644 --- a/app-knowledge/plugins/edm-knowledge/src/test/java/modelengine/jade/knowledge/EdmKnowledgeBaseManagerTest.java +++ b/app-knowledge/plugins/edm-knowledge/src/test/java/modelengine/jade/knowledge/EdmKnowledgeBaseManagerTest.java @@ -76,8 +76,8 @@ public void shouldOkWhenListRepo() { "lxh-2", "description", "VECTOR", - Timestamp.valueOf("2024-09-26 16:16:21.054")), - tuple(1L, "lxh-k", "", "VECTOR", Timestamp.valueOf("2024-09-26 12:12:14.320"))); + Timestamp.valueOf("2024-09-26 08:16:21.054")), + tuple(1L, "lxh-k", "", "VECTOR", Timestamp.valueOf("2024-09-26 04:12:14.320"))); } @Test diff --git a/common/dependency/pom.xml b/common/dependency/pom.xml index 06b01bfd6e..55e1e650f7 100644 --- a/common/dependency/pom.xml +++ b/common/dependency/pom.xml @@ -56,6 +56,7 @@ 1.14.5 2.24 1.12.468 + 5.7.1 3.22.0 @@ -552,6 +553,11 @@ aws-java-sdk-s3 ${aws.version} + + com.opencsv + opencsv + ${opencsv.version} +