Skip to content

Commit 17617be

Browse files
deardengdeardeng
authored
[fix] (cloud) Fix local/remote tablet size semantics in schema views (#60887)
In storage-compute separation, data size should be represented consistently as remote size. Previously, show tablets and information_schema.partitions could diverge from information_schema.backend_tablets, which made local/remote semantics confusing for users and operators. This change aligns cloud-mode output mapping for local/remote size columns and adds a regression test to guard the behavior.
1 parent a1f66eb commit 17617be

5 files changed

Lines changed: 354 additions & 6 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/DataSizeDisplayUtil.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.catalog;
19+
20+
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
21+
import org.apache.doris.common.Config;
22+
import org.apache.doris.common.Pair;
23+
24+
public class DataSizeDisplayUtil {
25+
private DataSizeDisplayUtil() {
26+
}
27+
28+
public static Pair<Long, Long> getDisplayDataSize(Replica replica) {
29+
return getDisplayDataSize(replica.getDataSize(), replica.getRemoteDataSize(),
30+
getLocalIndexAndSegmentSize(replica));
31+
}
32+
33+
public static Pair<Long, Long> getDisplayDataSize(Partition partition) {
34+
long localDataSize = partition.getDataSize(false);
35+
long remoteDataSize = partition.getRemoteDataSize();
36+
if (!needCloudSizeMapping(remoteDataSize)) {
37+
return Pair.of(localDataSize, remoteDataSize);
38+
}
39+
return getPartitionDisplayDataSize(partition);
40+
}
41+
42+
private static Pair<Long, Long> getDisplayDataSize(long localDataSize, long remoteDataSize,
43+
long localIndexAndSegmentSize) {
44+
if (!needCloudSizeMapping(remoteDataSize)) {
45+
return Pair.of(localDataSize, remoteDataSize);
46+
}
47+
if (localDataSize > 0) {
48+
remoteDataSize = localDataSize;
49+
localDataSize = 0;
50+
} else if (localIndexAndSegmentSize > 0) {
51+
remoteDataSize = localIndexAndSegmentSize;
52+
}
53+
return Pair.of(localDataSize, remoteDataSize);
54+
}
55+
56+
private static boolean needCloudSizeMapping(long remoteDataSize) {
57+
return Config.isCloudMode() && remoteDataSize == 0;
58+
}
59+
60+
private static Pair<Long, Long> getPartitionDisplayDataSize(Partition partition) {
61+
long localDataSize = 0L;
62+
long remoteDataSize = 0L;
63+
for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
64+
for (Tablet tablet : index.getTablets()) {
65+
for (Replica replica : tablet.getReplicas()) {
66+
Pair<Long, Long> displayDataSize = getDisplayDataSize(replica);
67+
localDataSize += displayDataSize.first;
68+
remoteDataSize += displayDataSize.second;
69+
}
70+
}
71+
}
72+
return Pair.of(localDataSize, remoteDataSize);
73+
}
74+
75+
private static long getLocalIndexAndSegmentSize(Replica replica) {
76+
return replica.getLocalInvertedIndexSize() + replica.getLocalSegmentSize();
77+
}
78+
}

‎fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
package org.apache.doris.common.proc;
1919

2020
import org.apache.doris.catalog.CloudTabletStatMgr;
21+
import org.apache.doris.catalog.DataSizeDisplayUtil;
2122
import org.apache.doris.catalog.DiskInfo;
2223
import org.apache.doris.catalog.Env;
2324
import org.apache.doris.catalog.MaterializedIndex;
@@ -29,6 +30,7 @@
2930
import org.apache.doris.common.AnalysisException;
3031
import org.apache.doris.common.Config;
3132
import org.apache.doris.common.FeConstants;
33+
import org.apache.doris.common.Pair;
3234
import org.apache.doris.common.util.ListComparator;
3335
import org.apache.doris.common.util.NetUtils;
3436
import org.apache.doris.common.util.TimeUtils;
@@ -193,8 +195,11 @@ public List<List<Comparable>> fetchComparableResult(long version, long backendId
193195
tabletInfo.add(displayLastSuccessVersion);
194196
tabletInfo.add(replica.getLastFailedVersion());
195197
tabletInfo.add(TimeUtils.longToTimeString(replica.getLastFailedTimestamp()));
196-
tabletInfo.add(replica.getDataSize());
197-
tabletInfo.add(replica.getRemoteDataSize());
198+
Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(replica);
199+
long localDataSize = displayDataSize.first;
200+
long remoteDataSize = displayDataSize.second;
201+
tabletInfo.add(localDataSize);
202+
tabletInfo.add(remoteDataSize);
198203
tabletInfo.add(replica.getRowCount());
199204
tabletInfo.add(replica.getState());
200205

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.doris.blockrule.SqlBlockRule;
2424
import org.apache.doris.catalog.Column;
2525
import org.apache.doris.catalog.DataProperty;
26+
import org.apache.doris.catalog.DataSizeDisplayUtil;
2627
import org.apache.doris.catalog.Database;
2728
import org.apache.doris.catalog.DatabaseIf;
2829
import org.apache.doris.catalog.DistributionInfo;
@@ -46,6 +47,7 @@
4647
import org.apache.doris.common.AnalysisException;
4748
import org.apache.doris.common.ClientPool;
4849
import org.apache.doris.common.ErrorCode;
50+
import org.apache.doris.common.FeConstants;
4951
import org.apache.doris.common.Pair;
5052
import org.apache.doris.common.proc.FrontendsProcNode;
5153
import org.apache.doris.common.proc.PartitionsProcDir;
@@ -1913,17 +1915,21 @@ private static void partitionsForInternalCatalog(UserIdentity currentUserIdentit
19131915
trow.addToColumnValue(new TCell().setStringVal("")); // NODEGROUP (not available)
19141916
trow.addToColumnValue(new TCell().setStringVal("")); // TABLESPACE_NAME (not available)
19151917

1916-
Pair<Double, String> sizePair = DebugUtil.getByteUint(partition.getDataSize(false));
1918+
Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition);
1919+
long localDataSize = displayDataSize.first;
1920+
long remoteDataSize = displayDataSize.second;
1921+
Pair<Double, String> sizePair = DebugUtil.getByteUint(localDataSize);
19171922
String readableDateSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(sizePair.first) + " "
19181923
+ sizePair.second;
19191924
trow.addToColumnValue(new TCell().setStringVal(readableDateSize)); // LOCAL_DATA_SIZE
1920-
sizePair = DebugUtil.getByteUint(partition.getRemoteDataSize());
1925+
sizePair = DebugUtil.getByteUint(remoteDataSize);
19211926
readableDateSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(sizePair.first) + " "
19221927
+ sizePair.second;
19231928
trow.addToColumnValue(new TCell().setStringVal(readableDateSize)); // REMOTE_DATA_SIZE
19241929
trow.addToColumnValue(new TCell().setStringVal(partition.getState().toString())); // STATE
1925-
String replicaAllocation = PartitionsProcDir.getReplicaAllocationDisplay(
1926-
partitionInfo.getReplicaAllocation(partitionId).toCreateStmt());
1930+
String replicaAllocation = getPartitionsReplicaAllocationDisplay(
1931+
PartitionsProcDir.getReplicaAllocationDisplay(partitionInfo.getReplicaAllocation(
1932+
partitionId).toCreateStmt()));
19271933
trow.addToColumnValue(new TCell().setStringVal(replicaAllocation)); // REPLICA_ALLOCATION
19281934
trow.addToColumnValue(new TCell().setIntVal(partitionInfo.getReplicaAllocation(partitionId)
19291935
.getTotalReplicaNum())); // REPLICA_NUM
@@ -1988,6 +1994,10 @@ private static void partitionsForInternalCatalog(UserIdentity currentUserIdentit
19881994
} // for table
19891995
}
19901996

1997+
private static String getPartitionsReplicaAllocationDisplay(String replicaAllocation) {
1998+
return FeConstants.null_string.equals(replicaAllocation) ? "NULL" : replicaAllocation;
1999+
}
2000+
19912001
private static void partitionsForExternalCatalog(UserIdentity currentUserIdentity,
19922002
CatalogIf catalog, DatabaseIf database, List<TableIf> tables, List<TRow> dataBatch, String timeZone) {
19932003
for (TableIf table : tables) {

‎fe/fe-core/src/test/java/org/apache/doris/catalog/DataSizeDisplayUtilTest.java‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.catalog;
19+
20+
import org.apache.doris.catalog.MaterializedIndex.IndexState;
21+
import org.apache.doris.cloud.catalog.CloudReplica;
22+
import org.apache.doris.cloud.catalog.CloudTablet;
23+
import org.apache.doris.common.Config;
24+
import org.apache.doris.common.Pair;
25+
26+
import org.junit.After;
27+
import org.junit.Assert;
28+
import org.junit.Before;
29+
import org.junit.Test;
30+
31+
public class DataSizeDisplayUtilTest {
32+
private String originDeployMode;
33+
private String originCloudUniqueId;
34+
35+
@Before
36+
public void setUp() {
37+
originDeployMode = Config.deploy_mode;
38+
originCloudUniqueId = Config.cloud_unique_id;
39+
Config.deploy_mode = "cloud";
40+
Config.cloud_unique_id = "";
41+
}
42+
43+
@After
44+
public void tearDown() {
45+
Config.deploy_mode = originDeployMode;
46+
Config.cloud_unique_id = originCloudUniqueId;
47+
}
48+
49+
@Test
50+
public void testPartitionDisplaySizeFallbackToReplicaIndexAndSegmentSize() {
51+
MaterializedIndex baseIndex = new MaterializedIndex(10L, IndexState.NORMAL);
52+
CloudTablet tablet = new CloudTablet(20L);
53+
CloudReplica replica = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0,
54+
100L, 200L, 300L, 10L, 0L);
55+
replica.setDataSize(0L);
56+
replica.setLocalInvertedIndexSize(111L);
57+
replica.setLocalSegmentSize(222L);
58+
tablet.addReplica(replica, true);
59+
baseIndex.addTablet(tablet, null, true);
60+
61+
Partition partition = new Partition(300L, "p1", baseIndex, new RandomDistributionInfo(1));
62+
63+
Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition);
64+
Assert.assertEquals(0L, displayDataSize.first.longValue());
65+
Assert.assertEquals(333L, displayDataSize.second.longValue());
66+
}
67+
68+
@Test
69+
public void testPartitionDisplaySizeMapsCloudDataSizeToRemoteSize() {
70+
MaterializedIndex baseIndex = new MaterializedIndex(10L, IndexState.NORMAL);
71+
CloudTablet tablet = new CloudTablet(20L);
72+
CloudReplica replica = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0,
73+
100L, 200L, 300L, 10L, 0L);
74+
replica.setDataSize(123L);
75+
tablet.addReplica(replica, true);
76+
baseIndex.addTablet(tablet, null, true);
77+
78+
Partition partition = new Partition(300L, "p1", baseIndex, new RandomDistributionInfo(1));
79+
80+
Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition);
81+
Assert.assertEquals(0L, displayDataSize.first.longValue());
82+
Assert.assertEquals(123L, displayDataSize.second.longValue());
83+
}
84+
85+
@Test
86+
public void testPartitionDisplaySizeAggregatesMixedReplicaDisplaySize() {
87+
MaterializedIndex baseIndex = new MaterializedIndex(10L, IndexState.NORMAL);
88+
89+
CloudTablet tabletWithDataSize = new CloudTablet(20L);
90+
CloudReplica replicaWithDataSize = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0,
91+
100L, 200L, 300L, 10L, 0L);
92+
replicaWithDataSize.setDataSize(123L);
93+
tabletWithDataSize.addReplica(replicaWithDataSize, true);
94+
baseIndex.addTablet(tabletWithDataSize, null, true);
95+
96+
CloudTablet tabletWithFallbackSize = new CloudTablet(21L);
97+
CloudReplica replicaWithFallbackSize = new CloudReplica(31L, 1L, Replica.ReplicaState.NORMAL, 2L, 0,
98+
100L, 200L, 300L, 10L, 1L);
99+
replicaWithFallbackSize.setDataSize(0L);
100+
replicaWithFallbackSize.setLocalInvertedIndexSize(111L);
101+
replicaWithFallbackSize.setLocalSegmentSize(222L);
102+
tabletWithFallbackSize.addReplica(replicaWithFallbackSize, true);
103+
baseIndex.addTablet(tabletWithFallbackSize, null, true);
104+
105+
Partition partition = new Partition(300L, "p1", baseIndex, new RandomDistributionInfo(2));
106+
107+
Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition);
108+
Assert.assertEquals(0L, displayDataSize.first.longValue());
109+
Assert.assertEquals(456L, displayDataSize.second.longValue());
110+
}
111+
112+
@Test
113+
public void testReplicaDisplaySizeFallbackToReplicaIndexAndSegmentSize() {
114+
CloudReplica replica = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0,
115+
100L, 200L, 300L, 10L, 0L);
116+
replica.setDataSize(0L);
117+
replica.setLocalInvertedIndexSize(111L);
118+
replica.setLocalSegmentSize(222L);
119+
120+
Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(replica);
121+
Assert.assertEquals(0L, displayDataSize.first.longValue());
122+
Assert.assertEquals(333L, displayDataSize.second.longValue());
123+
}
124+
}

0 commit comments

Comments
 (0)