refactor: replace the database from mysql to pgsql

Makefile

@@ -283,7 +283,7 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python databa
 		exit 1; \
 	fi
 	@if [ "$*" = "label-studio" ]; then \
-		docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s label-studio pg-db; \
+		docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s label-studio; \
 	elif [ "$*" = "mineru" ]; then \
 		docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s datamate-mineru; \
 	elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \

backend/pom.xml

@@ -22,7 +22,7 @@
         <spring-cloud.version>2025.0.0</spring-cloud.version>
         <spring-ai.version>1.0.0-RC1</spring-ai.version>
         <mysql.version>8.0.33</mysql.version>
-        <postgresql.version>42.6.0</postgresql.version>
+        <postgresql.version>42.7.7</postgresql.version>
         <redis.version>3.2.0</redis.version>
         <elasticsearch.version>8.11.0</elasticsearch.version>
         <junit.version>5.10.0</junit.version>
@@ -108,6 +108,12 @@
                 <artifactId>jakarta.persistence-api</artifactId>
                 <version>${jakarta.persistence.version}</version>
             </dependency>
+            <!-- Database: postgresql -->
+            <dependency>
+                <groupId>org.postgresql</groupId>
+                <artifactId>postgresql</artifactId>
+                <version>${postgresql.version}</version>
+            </dependency>
             <!-- MyBatis version alignment -->
             <dependency>
                 <groupId>com.baomidou</groupId>
@@ -129,12 +135,6 @@
                 <version>${lombok.version}</version>
             </dependency>
 
-            <dependency>
-                <groupId>com.mysql</groupId>
-                <artifactId>mysql-connector-j</artifactId>
-                <version>${mysql.version}</version>
-            </dependency>
-
             <dependency>
                 <groupId>org.apache.poi</groupId>
                 <artifactId>poi</artifactId>

backend/services/data-annotation-service/pom.xml

@@ -30,11 +30,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-websocket</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.mysql</groupId>
-            <artifactId>mysql-connector-j</artifactId>
-            <version>${mysql.version}</version>
-        </dependency>
         <!-- OpenAPI Dependencies -->
         <dependency>
             <groupId>org.springdoc</groupId>

backend/services/data-cleaning-service/pom.xml

@@ -53,10 +53,6 @@
         <groupId>org.openapitools</groupId>
         <artifactId>jackson-databind-nullable</artifactId>
       </dependency>
-      <dependency>
-        <groupId>com.mysql</groupId>
-        <artifactId>mysql-connector-j</artifactId>
-      </dependency>
       <dependency>
         <groupId>org.apache.commons</groupId>
         <artifactId>commons-compress</artifactId>

backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/OperatorInstanceMapper.java

@@ -11,13 +11,14 @@
 
 @Mapper
 public interface OperatorInstanceMapper extends BaseMapper<OperatorInstance> {
-    @Select("SELECT o.operator_id as id, o.operator_name as name, description, version, inputs, outputs, runtime, " +
-            "   settings, created_at, updated_at, " +
-            "GROUP_CONCAT(category_id ORDER BY created_at DESC SEPARATOR ',') AS categories " +
-            "FROM t_operator_instance toi LEFT JOIN v_operator o ON toi.operator_id = o.operator_id " +
+    @Select("SELECT o.operator_id as id, o.operator_name as name, o.description, o.version, o.inputs, o.outputs, " +
+            "o.runtime, o.settings, o.created_at, o.updated_at, " +
+            "STRING_AGG(CAST(category_id AS TEXT), ',' ORDER BY o.created_at DESC) AS categories " +
+            "FROM t_operator_instance toi " +
+            "LEFT JOIN v_operator o ON toi.operator_id = o.operator_id " +
             "WHERE toi.instance_id = #{instanceId} " +
-            "GROUP BY o.operator_id, o.operator_name, description, version, inputs, outputs, runtime," +
-            "    settings, created_at, updated_at, op_index " +
+            "GROUP BY o.operator_id, o.operator_name, o.description, o.version, o.inputs, o.outputs, o.runtime, " +
+            "    o.settings, o.created_at, o.updated_at, toi.op_index " +
             "ORDER BY toi.op_index")
     List<OperatorView> findOperatorByInstanceId(String instanceId);
 }

backend/services/data-evaluation-service/pom.xml

@@ -26,11 +26,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.mysql</groupId>
-            <artifactId>mysql-connector-j</artifactId>
-            <version>${mysql.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>

backend/services/data-management-service/pom.xml

@@ -30,11 +30,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-redis</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.mysql</groupId>
-            <artifactId>mysql-connector-j</artifactId>
-            <version>${mysql.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java

@@ -18,6 +18,10 @@
 import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetRepository;
 import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
 import com.datamate.datamanagement.interfaces.dto.*;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
@@ -64,11 +68,7 @@ public Dataset createDataset(CreateDatasetRequest createDatasetRequest) {
         Dataset dataset = DatasetConverter.INSTANCE.convertToDataset(createDatasetRequest);
         dataset.initCreateParam(datasetBasePath);
         // 处理标签
-        Set<Tag> processedTags = Optional.ofNullable(createDatasetRequest.getTags())
-            .filter(CollectionUtils::isNotEmpty)
-            .map(this::processTagNames)
-            .orElseGet(HashSet::new);
-        dataset.setTags(processedTags);
+        dataset.setTags(processTagNames(createDatasetRequest.getTags()));
         datasetRepository.save(dataset);
 
         //todo 需要解耦这块逻辑
@@ -146,7 +146,7 @@ public PagedResponse<DatasetResponse> getDatasets(DatasetPagingQuery query) {
     /**
      * 处理标签名称，创建或获取标签
      */
-    private Set<Tag> processTagNames(List<String> tagNames) {
+    private String processTagNames(List<String> tagNames) {
         Set<Tag> tags = new HashSet<>();
         for (String tagName : tagNames) {
             Tag tag = tagMapper.findByName(tagName);
@@ -161,7 +161,16 @@ private Set<Tag> processTagNames(List<String> tagNames) {
             tagMapper.updateUsageCount(tag.getId(), tag.getUsageCount());
             tags.add(tag);
         }
-        return tags;
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            mapper.registerModule(new JavaTimeModule());
+            // 可选：配置日期时间格式
+            mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+            return mapper.writeValueAsString(tags);
+        } catch (JsonProcessingException e) {
+            log.warn("Parse tags to json error.");
+            return null;
+        }
     }
 
     /**

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/Dataset.java

@@ -2,7 +2,6 @@
 
 import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableName;
-import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
 import com.datamate.common.domain.model.base.BaseEntity;
 import com.datamate.datamanagement.common.enums.DatasetStatusType;
 import com.datamate.datamanagement.common.enums.DatasetType;
@@ -67,8 +66,7 @@ public class Dataset extends BaseEntity<String> {
     /**
      * 标签列表, JSON格式
      */
-    @TableField(typeHandler = JacksonTypeHandler.class)
-    private Collection<Tag> tags = new HashSet<>();
+    private String tags;
     /**
      * 额外元数据，JSON格式
      */

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/converter/DatasetConverter.java

@@ -4,19 +4,21 @@
 import com.datamate.datamanagement.domain.model.dataset.Dataset;
 import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
 import com.datamate.datamanagement.domain.model.dataset.FileTag;
-import com.datamate.datamanagement.interfaces.dto.CreateDatasetRequest;
-import com.datamate.datamanagement.interfaces.dto.DatasetFileResponse;
-import com.datamate.datamanagement.interfaces.dto.DatasetResponse;
-import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
+import com.datamate.datamanagement.domain.model.dataset.Tag;
+import com.datamate.datamanagement.interfaces.dto.*;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.mapstruct.Mapper;
 import org.mapstruct.Mapping;
 import org.mapstruct.Named;
 import org.mapstruct.factory.Mappers;
 
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 /**
  * 数据集文件转换器
@@ -32,6 +34,7 @@ public interface DatasetConverter {
     @Mapping(source = "sizeBytes", target = "totalSize")
     @Mapping(source = "path", target = "targetLocation")
     @Mapping(source = "files", target = "distribution", qualifiedByName = "getDistribution")
+    @Mapping(source = "tags", target = "tags", qualifiedByName = "getDatasetTags")
     DatasetResponse convertToResponse(Dataset dataset);
 
     /**
@@ -88,4 +91,28 @@ default Map<String, Map<String, Long>> getDistribution(List<DatasetFile> dataset
         }
         return distribution;
     }
+
+    /**
+     * 获取数据集标签
+     *
+     * @param datasetTag 数据集标签
+     * @return 标签
+     */
+    @Named("getDatasetTags")
+    default List<TagResponse> getDatasetTags(String datasetTag) {
+        List<TagResponse> tagResponses = new ArrayList<>();
+        if (StringUtils.isBlank(datasetTag)) {
+            return tagResponses;
+        }
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            mapper.registerModule(new JavaTimeModule());
+            // 可选：配置日期时间格式
+            mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+            return mapper.readValue(datasetTag, new TypeReference<List<Tag>>() {
+            }).stream().map(TagConverter.INSTANCE::convertToResponse).toList();
+        } catch (Exception e) {
+            return Collections.emptyList();
+        }
+    }
 }

backend/services/data-synthesis-service/pom.xml

@@ -26,11 +26,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.mysql</groupId>
-            <artifactId>mysql-connector-j</artifactId>
-            <version>${mysql.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>

backend/services/execution-engine-service/pom.xml

@@ -30,11 +30,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-redis</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.mysql</groupId>
-            <artifactId>mysql-connector-j</artifactId>
-            <version>${mysql.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>

backend/services/main-application/pom.xml

@@ -106,10 +106,8 @@
 
         <!-- Database -->
         <dependency>
-            <groupId>com.mysql</groupId>
-            <artifactId>mysql-connector-j</artifactId>
-            <version>8.0.33</version>
-            <scope>runtime</scope>
+            <groupId>org.postgresql</groupId>
+            <artifactId>postgresql</artifactId>
         </dependency>
 
         <!-- 测试依赖 -->

backend/services/main-application/src/main/resources/application.yml

@@ -11,9 +11,9 @@ spring:
 
   # 数据源配置
   datasource:
-    driver-class-name: com.mysql.cj.jdbc.Driver
-    url: jdbc:mysql://datamate-database:3306/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
-    username: ${DB_USERNAME:root}
+    driver-class-name: org.postgresql.Driver
+    url: jdbc:postgresql://datamate-database:5432/datamate?useUnicode=true&characterEncoding=utf8&useSSL=false&serverTimezone=Asia/Shanghai&allowPublicKeyRetrieval=true
+    username: ${DB_USERNAME:postgres}
     password: ${DB_PASSWORD:password}
     hikari:
       maximum-pool-size: 20

backend/services/operator-market-service/pom.xml

@@ -30,10 +30,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-redis</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.mysql</groupId>
-            <artifactId>mysql-connector-j</artifactId>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>

backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorViewRepositoryImpl.java

@@ -26,34 +26,7 @@ public class OperatorViewRepositoryImpl extends CrudRepository<OperatorViewMappe
     @Override
     public List<OperatorDto> findOperatorsByCriteria(Integer page, Integer size, String keyword,
                                                      List<List<String>> categories, Boolean isStar) {
-        QueryWrapper<OperatorView> queryWrapper = Wrappers.query();
-        queryWrapper.eq(isStar != null, "is_star", isStar);
-        if (StringUtils.isNotEmpty(keyword)) {
-            queryWrapper.and(w ->
-                    w.like("operator_name", keyword)
-                    .or()
-                    .like("description", keyword));
-        }
-        StringBuilder havingSql = new StringBuilder();
-        if (CollectionUtils.isNotEmpty(categories)) {
-            queryWrapper.in("category_id", categories.stream().flatMap(List::stream).toList());
-            int index = 0;
-            for (List<String> category : categories) {
-                if (index > 0) {
-                    havingSql.append(" AND ");
-                }
-                havingSql.append("SUM(CASE WHEN category_id IN (");
-                havingSql.append(category.stream()
-                        .map(id -> "'" + id + "'")
-                        .collect(Collectors.joining(",")));
-                havingSql.append(") THEN 1 ELSE 0 END) > 0");
-                index++;
-            }
-        }
-
-        queryWrapper.groupBy("operator_id")
-            .having(!havingSql.isEmpty(), havingSql.toString())
-            .orderByDesc("created_at");
+        QueryWrapper<OperatorView> queryWrapper = getQueryWrapper(keyword, categories, isStar);
 
         Page<OperatorView> queryPage;
         if (size != null && page != null) {
@@ -68,13 +41,24 @@ public List<OperatorDto> findOperatorsByCriteria(Integer page, Integer size, Str
 
     @Override
     public int countOperatorsByCriteria(String keyword, List<List<String>> categories, Boolean isStar) {
+        QueryWrapper<OperatorView> queryWrapper = getQueryWrapper(keyword, categories, isStar);
+        Integer count = mapper.countOperatorsByCriteria(queryWrapper);
+        return count != null ? count : 0;
+    }
+
+    @Override
+    public OperatorView findOperatorById(String id) {
+        return mapper.findOperatorById(id);
+    }
+
+    private QueryWrapper<OperatorView> getQueryWrapper(String keyword, List<List<String>> categories, Boolean isStar) {
         QueryWrapper<OperatorView> queryWrapper = Wrappers.query();
         queryWrapper.eq(isStar != null, "is_star", isStar);
         if (StringUtils.isNotEmpty(keyword)) {
             queryWrapper.and(w ->
                     w.like("operator_name", keyword)
-                    .or()
-                    .like("description", keyword));
+                            .or()
+                            .like("description", keyword));
         }
         StringBuilder havingSql = new StringBuilder();
         if (CollectionUtils.isNotEmpty(categories)) {
@@ -84,22 +68,19 @@ public int countOperatorsByCriteria(String keyword, List<List<String>> categorie
                 if (index > 0) {
                     havingSql.append(" AND ");
                 }
-                havingSql.append("SUM(CASE WHEN category_id IN (");
+                havingSql.append("SUM(CASE WHEN CAST(category_id AS TEXT) IN (");
                 havingSql.append(category.stream()
                         .map(id -> "'" + id + "'")
                         .collect(Collectors.joining(",")));
                 havingSql.append(") THEN 1 ELSE 0 END) > 0");
                 index++;
             }
         }
-        queryWrapper.groupBy("operator_id")
-                .having(!havingSql.isEmpty(), havingSql.toString());
-        Integer count = mapper.countOperatorsByCriteria(queryWrapper);
-        return count != null ? count : 0;
-    }
 
-    @Override
-    public OperatorView findOperatorById(String id) {
-        return mapper.findOperatorById(id);
+        queryWrapper.groupBy("operator_id", "operator_name", "description", "version", "inputs", "outputs",
+                        "runtime", "settings", "is_star", "created_at", "updated_at")
+                .having(!havingSql.isEmpty(), havingSql.toString())
+                .orderByDesc("created_at");
+        return queryWrapper;
     }
 }