refactor: replace the database from mysql to pgsql (#236)

* refactor: replace the database from mysql to pgsql

* refactor: replace the database from mysql to pgsql

* refactor: merge the databases of DataMate and LabelStudio

* refactor: merge the databases of DataMate and LabelStudio

* fix: resolve the conflict

* feat: 适配pgsql

* fix: resolve the annotation task bug

* fix: fix the system param presetting data

---------

Co-authored-by: hhhhsc <1710496817@qq.com>

Author: hefanli

Makefile

@@ -283,7 +283,7 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python databa
 		exit 1; \
 	fi
 	@if [ "$*" = "label-studio" ]; then \
-		docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s label-studio pg-db; \
+		docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s label-studio; \
 	elif [ "$*" = "mineru" ]; then \
 		docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s datamate-mineru; \
 	elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \

backend/pom.xml

@@ -22,7 +22,7 @@
         <spring-cloud.version>2025.0.0</spring-cloud.version>
         <spring-ai.version>1.0.0-RC1</spring-ai.version>
         <mysql.version>8.0.33</mysql.version>
-        <postgresql.version>42.6.0</postgresql.version>
+        <postgresql.version>42.7.7</postgresql.version>
         <redis.version>3.2.0</redis.version>
         <elasticsearch.version>8.11.0</elasticsearch.version>
         <junit.version>5.10.0</junit.version>
@@ -108,6 +108,12 @@
                 <artifactId>jakarta.persistence-api</artifactId>
                 <version>${jakarta.persistence.version}</version>
             </dependency>
+            <!-- Database: postgresql -->
+            <dependency>
+                <groupId>org.postgresql</groupId>
+                <artifactId>postgresql</artifactId>
+                <version>${postgresql.version}</version>
+            </dependency>
             <!-- MyBatis version alignment -->
             <dependency>
                 <groupId>com.baomidou</groupId>
@@ -129,12 +135,6 @@
                 <version>${lombok.version}</version>
             </dependency>
 
-            <dependency>
-                <groupId>com.mysql</groupId>
-                <artifactId>mysql-connector-j</artifactId>
-                <version>${mysql.version}</version>
-            </dependency>
-
             <dependency>
                 <groupId>org.apache.poi</groupId>
                 <artifactId>poi</artifactId>

backend/services/data-annotation-service/pom.xml

@@ -30,11 +30,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-websocket</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.mysql</groupId>
-            <artifactId>mysql-connector-j</artifactId>
-            <version>${mysql.version}</version>
-        </dependency>
         <!-- OpenAPI Dependencies -->
         <dependency>
             <groupId>org.springdoc</groupId>

backend/services/data-cleaning-service/pom.xml

@@ -53,10 +53,6 @@
         <groupId>org.openapitools</groupId>
         <artifactId>jackson-databind-nullable</artifactId>
       </dependency>
-      <dependency>
-        <groupId>com.mysql</groupId>
-        <artifactId>mysql-connector-j</artifactId>
-      </dependency>
       <dependency>
         <groupId>org.apache.commons</groupId>
         <artifactId>commons-compress</artifactId>

backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/OperatorInstanceMapper.java

@@ -11,13 +11,14 @@
 
 @Mapper
 public interface OperatorInstanceMapper extends BaseMapper<OperatorInstance> {
-    @Select("SELECT o.operator_id as id, o.operator_name as name, description, version, inputs, outputs, runtime, " +
-            "   settings, created_at, updated_at, " +
-            "GROUP_CONCAT(category_id ORDER BY created_at DESC SEPARATOR ',') AS categories " +
-            "FROM t_operator_instance toi LEFT JOIN v_operator o ON toi.operator_id = o.operator_id " +
+    @Select("SELECT o.operator_id as id, o.operator_name as name, o.description, o.version, o.inputs, o.outputs, " +
+            "o.runtime, o.settings, o.created_at, o.updated_at, " +
+            "STRING_AGG(CAST(category_id AS TEXT), ',' ORDER BY o.created_at DESC) AS categories " +
+            "FROM t_operator_instance toi " +
+            "LEFT JOIN v_operator o ON toi.operator_id = o.operator_id " +
             "WHERE toi.instance_id = #{instanceId} " +
-            "GROUP BY o.operator_id, o.operator_name, description, version, inputs, outputs, runtime," +
-            "    settings, created_at, updated_at, op_index " +
+            "GROUP BY o.operator_id, o.operator_name, o.description, o.version, o.inputs, o.outputs, o.runtime, " +
+            "    o.settings, o.created_at, o.updated_at, toi.op_index " +
             "ORDER BY toi.op_index")
     List<OperatorView> findOperatorByInstanceId(String instanceId);
 }

backend/services/data-evaluation-service/pom.xml

@@ -26,11 +26,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-web</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.mysql</groupId>
-            <artifactId>mysql-connector-j</artifactId>
-            <version>${mysql.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>

backend/services/data-management-service/pom.xml

@@ -30,11 +30,6 @@
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-redis</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.mysql</groupId>
-            <artifactId>mysql-connector-j</artifactId>
-            <version>${mysql.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-test</artifactId>

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/application/DatasetApplicationService.java

@@ -18,6 +18,10 @@
 import com.datamate.datamanagement.infrastructure.persistence.repository.DatasetRepository;
 import com.datamate.datamanagement.interfaces.converter.DatasetConverter;
 import com.datamate.datamanagement.interfaces.dto.*;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections4.CollectionUtils;
@@ -64,11 +68,7 @@ public Dataset createDataset(CreateDatasetRequest createDatasetRequest) {
         Dataset dataset = DatasetConverter.INSTANCE.convertToDataset(createDatasetRequest);
         dataset.initCreateParam(datasetBasePath);
         // 处理标签
-        Set<Tag> processedTags = Optional.ofNullable(createDatasetRequest.getTags())
-            .filter(CollectionUtils::isNotEmpty)
-            .map(this::processTagNames)
-            .orElseGet(HashSet::new);
-        dataset.setTags(processedTags);
+        dataset.setTags(processTagNames(createDatasetRequest.getTags()));
         datasetRepository.save(dataset);
 
         //todo 需要解耦这块逻辑
@@ -146,7 +146,7 @@ public PagedResponse<DatasetResponse> getDatasets(DatasetPagingQuery query) {
     /**
      * 处理标签名称，创建或获取标签
      */
-    private Set<Tag> processTagNames(List<String> tagNames) {
+    private String processTagNames(List<String> tagNames) {
         Set<Tag> tags = new HashSet<>();
         for (String tagName : tagNames) {
             Tag tag = tagMapper.findByName(tagName);
@@ -161,7 +161,16 @@ private Set<Tag> processTagNames(List<String> tagNames) {
             tagMapper.updateUsageCount(tag.getId(), tag.getUsageCount());
             tags.add(tag);
         }
-        return tags;
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            mapper.registerModule(new JavaTimeModule());
+            // 可选：配置日期时间格式
+            mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+            return mapper.writeValueAsString(tags);
+        } catch (JsonProcessingException e) {
+            log.warn("Parse tags to json error.");
+            return null;
+        }
     }
 
     /**

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/domain/model/dataset/Dataset.java

@@ -2,7 +2,6 @@
 
 import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableName;
-import com.baomidou.mybatisplus.extension.handlers.JacksonTypeHandler;
 import com.datamate.common.domain.model.base.BaseEntity;
 import com.datamate.datamanagement.common.enums.DatasetStatusType;
 import com.datamate.datamanagement.common.enums.DatasetType;
@@ -67,8 +66,7 @@ public class Dataset extends BaseEntity<String> {
     /**
      * 标签列表, JSON格式
      */
-    @TableField(typeHandler = JacksonTypeHandler.class)
-    private Collection<Tag> tags = new HashSet<>();
+    private String tags;
     /**
      * 额外元数据，JSON格式
      */

backend/services/data-management-service/src/main/java/com/datamate/datamanagement/interfaces/converter/DatasetConverter.java

@@ -4,19 +4,21 @@
 import com.datamate.datamanagement.domain.model.dataset.Dataset;
 import com.datamate.datamanagement.domain.model.dataset.DatasetFile;
 import com.datamate.datamanagement.domain.model.dataset.FileTag;
-import com.datamate.datamanagement.interfaces.dto.CreateDatasetRequest;
-import com.datamate.datamanagement.interfaces.dto.DatasetFileResponse;
-import com.datamate.datamanagement.interfaces.dto.DatasetResponse;
-import com.datamate.datamanagement.interfaces.dto.UploadFileRequest;
+import com.datamate.datamanagement.domain.model.dataset.Tag;
+import com.datamate.datamanagement.interfaces.dto.*;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.mapstruct.Mapper;
 import org.mapstruct.Mapping;
 import org.mapstruct.Named;
 import org.mapstruct.factory.Mappers;
 
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 /**
  * 数据集文件转换器
@@ -32,6 +34,7 @@ public interface DatasetConverter {
     @Mapping(source = "sizeBytes", target = "totalSize")
     @Mapping(source = "path", target = "targetLocation")
     @Mapping(source = "files", target = "distribution", qualifiedByName = "getDistribution")
+    @Mapping(source = "tags", target = "tags", qualifiedByName = "getDatasetTags")
     DatasetResponse convertToResponse(Dataset dataset);
 
     /**
@@ -88,4 +91,28 @@ default Map<String, Map<String, Long>> getDistribution(List<DatasetFile> dataset
         }
         return distribution;
     }
+
+    /**
+     * 获取数据集标签
+     *
+     * @param datasetTag 数据集标签
+     * @return 标签
+     */
+    @Named("getDatasetTags")
+    default List<TagResponse> getDatasetTags(String datasetTag) {
+        List<TagResponse> tagResponses = new ArrayList<>();
+        if (StringUtils.isBlank(datasetTag)) {
+            return tagResponses;
+        }
+        ObjectMapper mapper = new ObjectMapper();
+        try {
+            mapper.registerModule(new JavaTimeModule());
+            // 可选：配置日期时间格式
+            mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS);
+            return mapper.readValue(datasetTag, new TypeReference<List<Tag>>() {
+            }).stream().map(TagConverter.INSTANCE::convertToResponse).toList();
+        } catch (Exception e) {
+            return Collections.emptyList();
+        }
+    }
 }