Skip to content

Commit cb40101

Browse files
morningmanmorningman
authored
Revert "[fix](cache) Follower FE sql cache not invalidated on table metadata replay (#63612)" (#63872)
Reverts #63657
1 parent 916286f commit cb40101

9 files changed

Lines changed: 18 additions & 249 deletions

File tree

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2413,18 +2413,6 @@ public class Config extends ConfigBase {
24132413
)
24142414
public static int sql_cache_manage_num = 100;
24152415

2416-
@ConfField(
2417-
mutable = true,
2418-
description = {
2419-
"是否在 DDL 时写入 OP_TABLE_META_CHANGE edit log 通知 follower FE 清理 sql cache。"
2420-
+ "默认 false，开启后 master DDL 会广播表元数据变更信号到所有 follower",
2421-
"Whether to write OP_TABLE_META_CHANGE edit log on DDL to notify follower FEs "
2422-
+ "to invalidate sql cache. Default false. When enabled, master DDL broadcasts "
2423-
+ "table metadata change signal to all followers"
2424-
}
2425-
)
2426-
public static boolean enable_write_op_table_meta_change = false;
2427-
24282416
@ConfField(
24292417
mutable = true,
24302418
callbackClassString = "org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager$UpdateConfig",

‎fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@
7272
import org.apache.doris.common.DdlException;
7373
import org.apache.doris.common.MetaNotFoundException;
7474
import org.apache.doris.common.UserException;
75+
import org.apache.doris.common.cache.NereidsSqlCacheManager;
7576
import org.apache.doris.common.util.DynamicPartitionUtil;
7677
import org.apache.doris.common.util.MetaLockUtils;
7778
import org.apache.doris.common.util.PropertyAnalyzer;
@@ -362,7 +363,8 @@ private boolean processAlterOlapTableInternal(List<AlterClause> alterClauses, Ol
362363

363364
olapTable.writeLock();
364365
try {
365-
Env.getCurrentEnv().notifyTableMetaChange(olapTable);
366+
NereidsSqlCacheManager sqlCacheManager = Env.getCurrentEnv().getSqlCacheManager();
367+
sqlCacheManager.invalidateAboutTable(olapTable);
366368
} finally {
367369
olapTable.writeUnlock();
368370
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 3 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,6 @@
220220
import org.apache.doris.persist.Storage;
221221
import org.apache.doris.persist.StorageInfo;
222222
import org.apache.doris.persist.TableInfo;
223-
import org.apache.doris.persist.TableMetaChange;
224223
import org.apache.doris.persist.TablePropertyInfo;
225224
import org.apache.doris.persist.TableRenameColumnInfo;
226225
import org.apache.doris.persist.TruncateTableInfo;
@@ -6935,9 +6934,7 @@ public void setTableStatusInternal(String dbName, String tableName, OlapTableSta
69356934
LOG.warn("ignore set same state {} for table {}. is replay: {}.",
69366935
olapTable.getState(), tableName, isReplay);
69376936
}
6938-
if (!isReplay) {
6939-
notifyTableMetaChange(olapTable);
6940-
}
6937+
Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable);
69416938
} finally {
69426939
olapTable.writeUnlock();
69436940
}
@@ -7045,9 +7042,7 @@ private void setReplicaVersionInternal(long tabletId, long backendId, Long versi
70457042
LOG.info("set replica {} of tablet {} on backend {} as version {}, last success version {}, "
70467043
+ "last failed version {}, update time {}. is replay: {}", replica.getId(), tabletId,
70477044
backendId, version, lastSuccessVersion, lastFailedVersion, updateTime, isReplay);
7048-
if (!isReplay) {
7049-
notifyTableMetaChange(table);
7050-
}
7045+
Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(table);
70517046
} finally {
70527047
table.writeUnlock();
70537048
}
@@ -7128,9 +7123,7 @@ public int setPartitionVersionInternal(String database, String table, long parti
71287123
+ " {}.", partitionId, oldVersion, visibleVersion, database, table, isReplay);
71297124
}
71307125

7131-
if (!isReplay) {
7132-
notifyTableMetaChange(olapTable);
7133-
}
7126+
Env.getCurrentEnv().getSqlCacheManager().invalidateAboutTable(olapTable);
71347127
} finally {
71357128
olapTable.writeUnlock();
71367129
}
@@ -7359,35 +7352,6 @@ public NereidsSortedPartitionsCacheManager getSortedPartitionsCacheManager() {
73597352
return sortedPartitionsCacheManager;
73607353
}
73617354

7362-
public void notifyTableMetaChange(TableIf table) {
7363-
if (table == null) {
7364-
return;
7365-
}
7366-
TableMetaChange change =
7367-
TableMetaChange.fromTable(table);
7368-
fanOutTableMetaChange(change);
7369-
if (isMaster() && editLog != null && Config.enable_write_op_table_meta_change) {
7370-
editLog.logTableMetaChange(change);
7371-
}
7372-
}
7373-
7374-
public void replayTableMetaChange(TableMetaChange change) {
7375-
if (change == null) {
7376-
return;
7377-
}
7378-
fanOutTableMetaChange(change);
7379-
}
7380-
7381-
private void fanOutTableMetaChange(TableMetaChange change) {
7382-
if (sqlCacheManager != null) {
7383-
sqlCacheManager.invalidateAboutTable(change);
7384-
}
7385-
if (sortedPartitionsCacheManager != null) {
7386-
sortedPartitionsCacheManager.invalidateTable(
7387-
change.getCatalogName(), change.getDbName(), change.getTableName());
7388-
}
7389-
}
7390-
73917355
public SplitSourceManager getSplitSourceManager() {
73927356
return splitSourceManager;
73937357
}

‎fe/fe-core/src/main/java/org/apache/doris/common/cache/NereidsSqlCacheManager.java‎

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,6 @@
5959
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
6060
import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache;
6161
import org.apache.doris.nereids.util.Utils;
62-
import org.apache.doris.persist.TableMetaChange;
6362
import org.apache.doris.proto.InternalService;
6463
import org.apache.doris.proto.Types.PUniqueId;
6564
import org.apache.doris.qe.ConnectContext;
@@ -113,31 +112,29 @@ public Cache<String, SqlCacheContext> getSqlCaches() {
113112
}
114113

115114
public void invalidateAboutTable(TableIf tableIf) {
116-
invalidateAboutTable(TableMetaChange.fromTable(tableIf));
117-
}
118-
119-
public void invalidateAboutTable(TableMetaChange event) {
115+
Set<String> invalidateKeys = new LinkedHashSet<>();
120116
FullTableName invalidateTableName = null;
121-
if (event.getCatalogName() != null && event.getDbName() != null && event.getTableName() != null) {
122-
invalidateTableName = new FullTableName(
123-
event.getCatalogName(), event.getDbName(), event.getTableName());
117+
DatabaseIf database = tableIf.getDatabase();
118+
if (database != null) {
119+
CatalogIf catalog = database.getCatalog();
120+
if (catalog != null) {
121+
invalidateTableName = new FullTableName(
122+
database.getCatalog().getName(), database.getFullName(), tableIf.getName()
123+
);
124+
}
124125
}
125126

126-
Set<String> invalidateKeys = new LinkedHashSet<>();
127127
for (Entry<String, SqlCacheContext> kv : sqlCaches.asMap().entrySet()) {
128128
String key = kv.getKey();
129129
SqlCacheContext context = kv.getValue();
130-
if (context == null) {
131-
continue;
132-
}
133130
for (Entry<FullTableName, TableVersion> nameToVersion : context.getUsedTables().entrySet()) {
134131
FullTableName tableName = nameToVersion.getKey();
135132
TableVersion tableVersion = nameToVersion.getValue();
136-
if (tableVersion.id == event.getTableId()) {
133+
if (tableVersion.id == tableIf.getId()) {
137134
invalidateKeys.add(key);
138135
break;
139136
}
140-
if (invalidateTableName != null && tableName.equals(invalidateTableName)) {
137+
if (tableName.equals(invalidateTableName)) {
141138
invalidateKeys.add(key);
142139
break;
143140
}
@@ -468,9 +465,6 @@ private IsChanged tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext)
468465
if (currentTableVersion != cacheTableVersion) {
469466
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
470467
}
471-
if (olapTable.getBaseSchemaVersion() != tableVersion.schemaVersion) {
472-
return IsChanged.CHANGED_AND_INVALIDATE_CACHE;
473-
}
474468
if (tableIf instanceof MTMV) {
475469
// mtmv maybe access old data when grace_period > 0, we should disable cache at this case
476470
long gracePeriod = ((MTMV) tableIf).getGracePeriod();

‎fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,6 @@
124124
import org.apache.doris.persist.TableAddOrDropInvertedIndicesInfo;
125125
import org.apache.doris.persist.TableBranchOrTagInfo;
126126
import org.apache.doris.persist.TableInfo;
127-
import org.apache.doris.persist.TableMetaChange;
128127
import org.apache.doris.persist.TablePropertyInfo;
129128
import org.apache.doris.persist.TableRenameColumnInfo;
130129
import org.apache.doris.persist.TableStatsDeletionLog;
@@ -993,11 +992,6 @@ public void readFields(DataInput in) throws IOException {
993992
isRead = true;
994993
break;
995994
}
996-
case OperationType.OP_TABLE_META_CHANGE: {
997-
data = TableMetaChange.read(in);
998-
isRead = true;
999-
break;
1000-
}
1001995
default: {
1002996
IOException e = new IOException();
1003997
LOG.error("UNKNOWN Operation Type {}", opCode, e);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/SqlCacheContext.java‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -203,17 +203,12 @@ public synchronized void addUsedTable(TableIf tableIf) {
203203
LOG.warn("table {}, can not get version", tableIf.getName(), e);
204204
}
205205

206-
int schemaVersion = 0;
207-
if (tableIf instanceof OlapTable) {
208-
schemaVersion = ((OlapTable) tableIf).getBaseSchemaVersion();
209-
}
210206
usedTables.put(
211207
new FullTableName(database.getCatalog().getName(), database.getFullName(), tableIf.getName()),
212208
new TableVersion(
213209
tableIf.getId(),
214210
version,
215-
tableIf.getType(),
216-
schemaVersion
211+
tableIf.getType()
217212
)
218213
);
219214
}
@@ -598,7 +593,6 @@ public static class TableVersion {
598593
public final long id;
599594
public final long version;
600595
public final TableType type;
601-
public final int schemaVersion;
602596
}
603597

604598
/** CacheKeyType */

‎fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1418,11 +1418,6 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) {
14181418
// TODO: implement
14191419
break;
14201420
}
1421-
case OperationType.OP_TABLE_META_CHANGE: {
1422-
TableMetaChange op = (TableMetaChange) journal.getData();
1423-
env.replayTableMetaChange(op);
1424-
break;
1425-
}
14261421
default: {
14271422
IOException e = new IOException();
14281423
LOG.error("UNKNOWN Operation Type {}, log id: {}", opCode, logId, e);
@@ -2509,8 +2504,4 @@ public void logOperateKey(KeyOperationInfo info) {
25092504
public long logBeginSnapshot(SnapshotState snapshotState) {
25102505
return logEdit(OperationType.OP_BEGIN_SNAPSHOT, snapshotState);
25112506
}
2512-
2513-
public void logTableMetaChange(TableMetaChange op) {
2514-
logEdit(OperationType.OP_TABLE_META_CHANGE, op);
2515-
}
25162507
}

‎fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -425,11 +425,6 @@ public class OperationType {
425425

426426
public static final short OP_BEGIN_SNAPSHOT = 1100;
427427

428-
// Generic "an operation modified this table's metadata" signal broadcast from
429-
// master to followers so that every FE-local cache keyed by table can be
430-
// invalidated (NereidsSqlCacheManager, NereidsSortedPartitionsCacheManager, …).
431-
public static final short OP_TABLE_META_CHANGE = 1102;
432-
433428
/**
434429
* Get opcode name by op code.
435430
**/

0 commit comments

Comments
 (0)