Skip to content

Commit a748d92

Browse files
hefanlihefanli
committed
Merge remote-tracking branch 'refs/remotes/upstream/develop/dev_pgsql' into dev_pgsql
2 parents de7b896 + 4b2b258 commit a748d92

19 files changed

Lines changed: 65 additions & 103 deletions

File tree

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -283,7 +283,7 @@ VALID_SERVICE_TARGETS := datamate backend frontend runtime backend-python databa
283283
exit 1; \
284284
fi
285285
@if [ "$*" = "label-studio" ]; then \
286-
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s label-studio pg-db; \
286+
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s label-studio; \
287287
elif [ "$*" = "mineru" ]; then \
288288
docker compose -f deployment/docker/datamate/docker-compose.yml rm -f -s datamate-mineru; \
289289
elif [ "$*" = "data-juicer" ] || [ "$*" = "dj" ]; then \

‎backend/services/data-cleaning-service/src/main/java/com/datamate/cleaning/infrastructure/persistence/mapper/OperatorInstanceMapper.java‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,14 @@
1111

1212
@Mapper
1313
public interface OperatorInstanceMapper extends BaseMapper<OperatorInstance> {
14-
@Select("SELECT o.operator_id as id, o.operator_name as name, description, version, inputs, outputs, runtime, " +
15-
" settings, created_at, updated_at, " +
16-
"GROUP_CONCAT(category_id ORDER BY created_at DESC SEPARATOR ',') AS categories " +
17-
"FROM t_operator_instance toi LEFT JOIN v_operator o ON toi.operator_id = o.operator_id " +
14+
@Select("SELECT o.operator_id as id, o.operator_name as name, o.description, o.version, o.inputs, o.outputs, " +
15+
"o.runtime, o.settings, o.created_at, o.updated_at, " +
16+
"STRING_AGG(CAST(category_id AS TEXT), ',' ORDER BY o.created_at DESC) AS categories " +
17+
"FROM t_operator_instance toi " +
18+
"LEFT JOIN v_operator o ON toi.operator_id = o.operator_id " +
1819
"WHERE toi.instance_id = #{instanceId} " +
19-
"GROUP BY o.operator_id, o.operator_name, description, version, inputs, outputs, runtime," +
20-
" settings, created_at, updated_at, op_index " +
20+
"GROUP BY o.operator_id, o.operator_name, o.description, o.version, o.inputs, o.outputs, o.runtime, " +
21+
" o.settings, o.created_at, o.updated_at, toi.op_index " +
2122
"ORDER BY toi.op_index")
2223
List<OperatorView> findOperatorByInstanceId(String instanceId);
2324
}

‎backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/Impl/OperatorViewRepositoryImpl.java‎

Lines changed: 20 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -26,34 +26,7 @@ public class OperatorViewRepositoryImpl extends CrudRepository<OperatorViewMappe
2626
@Override
2727
public List<OperatorDto> findOperatorsByCriteria(Integer page, Integer size, String keyword,
2828
List<List<String>> categories, Boolean isStar) {
29-
QueryWrapper<OperatorView> queryWrapper = Wrappers.query();
30-
queryWrapper.eq(isStar != null, "is_star", isStar);
31-
if (StringUtils.isNotEmpty(keyword)) {
32-
queryWrapper.and(w ->
33-
w.like("operator_name", keyword)
34-
.or()
35-
.like("description", keyword));
36-
}
37-
StringBuilder havingSql = new StringBuilder();
38-
if (CollectionUtils.isNotEmpty(categories)) {
39-
queryWrapper.in("category_id", categories.stream().flatMap(List::stream).toList());
40-
int index = 0;
41-
for (List<String> category : categories) {
42-
if (index > 0) {
43-
havingSql.append(" AND ");
44-
}
45-
havingSql.append("SUM(CASE WHEN category_id IN (");
46-
havingSql.append(category.stream()
47-
.map(id -> "'" + id + "'")
48-
.collect(Collectors.joining(",")));
49-
havingSql.append(") THEN 1 ELSE 0 END) > 0");
50-
index++;
51-
}
52-
}
53-
54-
queryWrapper.groupBy("operator_id")
55-
.having(!havingSql.isEmpty(), havingSql.toString())
56-
.orderByDesc("created_at");
29+
QueryWrapper<OperatorView> queryWrapper = getQueryWrapper(keyword, categories, isStar);
5730

5831
Page<OperatorView> queryPage;
5932
if (size != null && page != null) {
@@ -68,13 +41,24 @@ public List<OperatorDto> findOperatorsByCriteria(Integer page, Integer size, Str
6841

6942
@Override
7043
public int countOperatorsByCriteria(String keyword, List<List<String>> categories, Boolean isStar) {
44+
QueryWrapper<OperatorView> queryWrapper = getQueryWrapper(keyword, categories, isStar);
45+
Integer count = mapper.countOperatorsByCriteria(queryWrapper);
46+
return count != null ? count : 0;
47+
}
48+
49+
@Override
50+
public OperatorView findOperatorById(String id) {
51+
return mapper.findOperatorById(id);
52+
}
53+
54+
private QueryWrapper<OperatorView> getQueryWrapper(String keyword, List<List<String>> categories, Boolean isStar) {
7155
QueryWrapper<OperatorView> queryWrapper = Wrappers.query();
7256
queryWrapper.eq(isStar != null, "is_star", isStar);
7357
if (StringUtils.isNotEmpty(keyword)) {
7458
queryWrapper.and(w ->
7559
w.like("operator_name", keyword)
76-
.or()
77-
.like("description", keyword));
60+
.or()
61+
.like("description", keyword));
7862
}
7963
StringBuilder havingSql = new StringBuilder();
8064
if (CollectionUtils.isNotEmpty(categories)) {
@@ -84,22 +68,19 @@ public int countOperatorsByCriteria(String keyword, List<List<String>> categorie
8468
if (index > 0) {
8569
havingSql.append(" AND ");
8670
}
87-
havingSql.append("SUM(CASE WHEN category_id IN (");
71+
havingSql.append("SUM(CASE WHEN CAST(category_id AS TEXT) IN (");
8872
havingSql.append(category.stream()
8973
.map(id -> "'" + id + "'")
9074
.collect(Collectors.joining(",")));
9175
havingSql.append(") THEN 1 ELSE 0 END) > 0");
9276
index++;
9377
}
9478
}
95-
queryWrapper.groupBy("operator_id")
96-
.having(!havingSql.isEmpty(), havingSql.toString());
97-
Integer count = mapper.countOperatorsByCriteria(queryWrapper);
98-
return count != null ? count : 0;
99-
}
10079

101-
@Override
102-
public OperatorView findOperatorById(String id) {
103-
return mapper.findOperatorById(id);
80+
queryWrapper.groupBy("operator_id", "operator_name", "description", "version", "inputs", "outputs",
81+
"runtime", "settings", "is_star", "created_at", "updated_at")
82+
.having(!havingSql.isEmpty(), havingSql.toString())
83+
.orderByDesc("created_at");
84+
return queryWrapper;
10485
}
10586
}

‎backend/services/operator-market-service/src/main/java/com/datamate/operator/infrastructure/persistence/mapper/OperatorViewMapper.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
public interface OperatorViewMapper extends BaseMapper<OperatorView> {
1515
@Select("SELECT operator_id AS id, operator_name AS name, description, version, inputs, outputs, runtime, " +
1616
"settings, is_star, created_at, updated_at, " +
17-
"GROUP_CONCAT(category_id ORDER BY created_at DESC SEPARATOR ',') AS categories " +
17+
"STRING_AGG(CAST(category_id AS TEXT), ',' ORDER BY created_at DESC) AS categories " +
1818
"FROM v_operator ${ew.customSqlSegment}")
1919
IPage<OperatorView> findOperatorsByCriteria(IPage<OperatorView> page,
2020
@Param(Constants.WRAPPER) Wrapper<OperatorView> queryWrapper);
@@ -24,7 +24,8 @@ IPage<OperatorView> findOperatorsByCriteria(IPage<OperatorView> page,
2424

2525
@Select("SELECT operator_id AS id, operator_name AS name, description, version, inputs, outputs, runtime, " +
2626
"settings, is_star, created_at, updated_at, " +
27-
"GROUP_CONCAT(category_name ORDER BY created_at DESC SEPARATOR ',') AS categories " +
28-
"FROM v_operator WHERE operator_id = #{id}")
27+
"STRING_AGG(category_name, ',' ORDER BY created_at DESC) AS categories " +
28+
"FROM v_operator WHERE operator_id = #{id} " +
29+
"GROUP BY operator_id, operator_name, description, version, inputs, outputs, runtime, settings, is_star, created_at, updated_at")
2930
OperatorView findOperatorById(@Param("id") String id);
3031
}

‎backend/services/rag-indexer-service/pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -99,12 +99,6 @@
9999
<dependency>
100100
<groupId>dev.langchain4j</groupId>
101101
<artifactId>langchain4j-milvus</artifactId>
102-
<exclusions>
103-
<exclusion>
104-
<groupId>com.google.protobuf</groupId>
105-
<artifactId>protobuf-java</artifactId>
106-
</exclusion>
107-
</exclusions>
108102
</dependency>
109103
</dependencies>
110104

‎backend/shared/domain-common/src/main/java/com/datamate/common/infrastructure/config/MybatisPlusConfig.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public JacksonTypeHandler jacksonTypeHandler() {
4141
@Bean
4242
public MybatisPlusInterceptor mybatisPlusInterceptor() {
4343
MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor();
44-
interceptor.addInnerInterceptor(new PaginationInnerInterceptor(DbType.MYSQL));
44+
interceptor.addInnerInterceptor(new PaginationInnerInterceptor(DbType.POSTGRE_SQL));
4545
return interceptor;
4646
}
4747
}

‎deployment/docker/datamate/docker-compose.yml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,11 +74,11 @@ services:
7474
environment:
7575
RAY_DEDUP_LOGS: "0"
7676
RAY_TQDM_PATCH_PRINT: "0"
77-
MYSQL_HOST: "datamate-database"
78-
MYSQL_PORT: "3306"
79-
MYSQL_USER: "root"
80-
MYSQL_PASSWORD: "password"
81-
MYSQL_DATABASE: "datamate"
77+
PG_HOST: "datamate-database"
78+
PG_PORT: "5432"
79+
PG_USER: "postgres"
80+
PG_PASSWORD: "password"
81+
PG_DATABASE: "datamate"
8282
command:
8383
- python
8484
- /opt/runtime/datamate/operator_runtime.py

‎deployment/docker/label-studio/docker-compose.yml‎

Whitespace-only changes.

‎deployment/helm/label-studio/values.yaml‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -12,22 +12,6 @@ image:
1212
tag: "latest"
1313
pullPolicy: IfNotPresent
1414

15-
postgres:
16-
enabled: false
17-
image:
18-
repository: pgautoupgrade/pgautoupgrade
19-
tag: "13-alpine"
20-
pullPolicy: IfNotPresent
21-
username: postgres
22-
# In docker-compose, POSTGRES_HOST_AUTH_METHOD=trust is used (no password).
23-
# For production, you should set a password and disable trust auth.
24-
password: ""
25-
authMethod: trust
26-
persistence:
27-
enabled: true
28-
size: 10Gi
29-
storageClass: ""
30-
3115
service:
3216
type: NodePort
3317
port: 8000

‎runtime/ops/filter/img_duplicated_images_cleaner/sql/sql_config.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
22
"query_sql": "SELECT * FROM operator_duplicate_img_features WHERE task_uuid = :task_uuid AND file_feature = :file_feature",
33
"insert_sql": "INSERT INTO operator_duplicate_img_features (task_uuid, file_feature, file_name, timestamp) VALUES (:task_uuid, :file_feature, :file_name, :timestamp)",
4-
"create_tables_sql": "CREATE TABLE IF NOT EXISTS operator_duplicate_img_features (id INT AUTO_INCREMENT PRIMARY KEY,task_uuid VARCHAR(255),file_feature TEXT,file_name TEXT,timestamp DATETIME);"
4+
"create_tables_sql": "CREATE TABLE IF NOT EXISTS operator_duplicate_img_features (id SERIAL PRIMARY KEY, task_uuid VARCHAR(255), file_feature TEXT, file_name TEXT, timestamp TIMESTAMP);"
55
}

0 commit comments

Comments
 (0)