use master refreshing interval; use runtimeMaxWait in batch execution

Author: JackShi148

src/main/java/com/alipay/oceanbase/rpc/location/model/TableLocations.java

@@ -95,8 +95,12 @@ public TableEntry refreshMeta(String tableName, ServerRoster serverRoster,
         long runtimeMaxWait = tableClient.getRuntimeMaxWait();
         int tableEntryRefreshContinuousFailureCeiling = tableClient
             .getTableEntryRefreshContinuousFailureCeiling();
+        long tableEntryRefreshIntervalBase = tableClient.getTableEntryRefreshIntervalBase();
+        long tableEntryRefreshIntervalCeiling = tableClient.getTableEntryRefreshIntervalCeiling();
         long tableEntryRefreshLockTimeout = tableClient.getTableEntryRefreshLockTimeout();
-        long refreshMetaInterval = 100L;
+        long refreshMetaInterval = (long) (tableEntryRefreshIntervalBase * Math.pow(2,
+                -serverRoster.getMaxPriority()));
+        refreshMetaInterval = Math.min(refreshMetaInterval, tableEntryRefreshIntervalCeiling);
 
         TableEntry tableEntry = locations.get(tableName);
         // avoid bad contention in high concurrent situation
@@ -288,7 +292,7 @@ public TableEntry refreshPartitionLocation(TableEntry tableEntry, String tableNa
         long runtimeMaxWait = tableClient.getRuntimeMaxWait();
         long tableEntryRefreshLockTimeout = tableClient.getTableEntryRefreshLockTimeout();
         long lastRefreshTime = locationInfo.getLastUpdateTime();
-        long tableEntryRefreshInterval = 100L;
+        long tableEntryRefreshInterval = tableClient.getTableEntryRefreshIntervalCeiling();
         long currentTime = System.currentTimeMillis();
         // do not refresh tablet location if refreshed within 300 milliseconds
         if (currentTime - lastRefreshTime < tableEntryRefreshInterval) {
@@ -421,14 +425,18 @@ public TableEntry refreshTabletLocationBatch(TableEntry tableEntry, String table
         long runtimeMaxWait = tableClient.getRuntimeMaxWait();
         long tableEntryRefreshLockTimeout = tableClient.getTableEntryRefreshLockTimeout();
         long lastRefreshTime = tableEntry.getPartitionEntry().getLastRefreshAllTime();
-        long tableEntryRefreshInterval = 100L;
+        long tableEntryRefreshIntervalBase = tableClient.getTableEntryRefreshIntervalBase();
+        long tableEntryRefreshIntervalCeiling = tableClient.getTableEntryRefreshIntervalCeiling();
+        long refreshBatchTabletInterval = (long) (tableEntryRefreshIntervalBase * Math.pow(2,
+                -serverRoster.getMaxPriority()));
+        refreshBatchTabletInterval = Math.min(refreshBatchTabletInterval, tableEntryRefreshIntervalCeiling);
         long currentTime = System.currentTimeMillis();
         // do not refresh tablet location if refreshed within 300 milliseconds
-        if (currentTime - lastRefreshTime < tableEntryRefreshInterval) {
+        if (currentTime - lastRefreshTime < refreshBatchTabletInterval) {
             logger
                 .info(
                     "punish table entry {}, last batch location refresh time {}, punish interval {}, current time {}.",
-                    tableName, lastRefreshTime, tableEntryRefreshInterval, currentTime);
+                    tableName, lastRefreshTime, refreshBatchTabletInterval, currentTime);
             return tableEntry;
         }
         Lock lock = getLocationBatchRefreshLock(tableName);
@@ -463,11 +471,11 @@ public TableEntry refreshTabletLocationBatch(TableEntry tableEntry, String table
                     logger.warn("[latency monitor] success to acquire refresh tablet locations in batch lock, tableName: {}", tableName);
                     lastRefreshTime = tableEntry.getPartitionEntry().getLastRefreshAllTime();
                     currentTime = System.currentTimeMillis();
-                    if (currentTime - lastRefreshTime < tableEntryRefreshInterval) {
+                    if (currentTime - lastRefreshTime < refreshBatchTabletInterval) {
                         logger
                                 .info(
                                         "punish table entry {}, last batch location refresh time {}, punish interval {}, current time {}.",
-                                        tableName, lastRefreshTime, tableEntryRefreshInterval, currentTime);
+                                        tableName, lastRefreshTime, refreshBatchTabletInterval, currentTime);
                         return tableEntry;
                     }
                     logger.warn("[latency monitor] do refresh tablet locations in batch lock, tableName: {}", tableName);

src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientBatchOpsImpl.java

@@ -541,14 +541,23 @@ private boolean shouldRetry(Throwable throwable) {
 
     private void executeWithRetries(ObTableOperationResult[] results, Map.Entry<Long, ObPair<ObTableParam, List<ObPair<Integer, ObTableOperation>>>> entry) throws Exception {
         int retryCount = 0;
-        boolean success = false;
 
         Map<Long, ObPair<ObTableParam, List<ObPair<Integer, ObTableOperation>>>> currentPartitions = new HashMap<>();
         currentPartitions.put(entry.getKey(), entry.getValue());
         int errCode = ResultCodes.OB_SUCCESS.errorCode;
         String errMsg = null;
-        int maxRetryTimes = obTableClient.getRuntimeRetryTimes();
-        while (retryCount < maxRetryTimes && !success) {
+        long runTimeMaxWait = obTableClient.getRuntimeMaxWait();
+        long startExecute = System.currentTimeMillis();
+        while (true) {
+            long costMillis = System.currentTimeMillis() - startExecute;
+            if (costMillis > runTimeMaxWait) {
+                errMsg = tableName + " failed to execute operation after retrying " + retryCount
+                        + " times and it has waited" +  costMillis + " ms"
+                        + " which exceeds runtime max wait timeout " + runTimeMaxWait
+                        + " ms. Last error Msg:" + "[errCode=" + errCode + "] " + errMsg;
+                logger.error(errMsg);
+                throw new ObTableUnexpectedException(errMsg);
+            }
             boolean allPartitionsSuccess = true;
             for (Map.Entry<Long, ObPair<ObTableParam, List<ObPair<Integer, ObTableOperation>>>> currentEntry : currentPartitions.entrySet()) {
                 try {
@@ -569,15 +578,9 @@ private void executeWithRetries(ObTableOperationResult[] results, Map.Entry<Long
             }
 
             if (allPartitionsSuccess) {
-                success = true;
+                break;
             }
         }
-
-        if (!success) {
-            errMsg = "Failed to execute operation after retrying " + retryCount + " times. Last error Msg:" +
-                    "[errCode="+ errCode +"] " + errMsg;
-            throw new ObTableUnexpectedException(errMsg);
-        }
     }
 
     public ObTableBatchOperationResult executeInternal() throws Exception {

src/main/java/com/alipay/oceanbase/rpc/table/ObTableClientLSBatchOpsImpl.java

@@ -827,17 +827,23 @@ private void executeWithRetries(ObTableSingleOpResult[] results,
                                     Map.Entry<Long, TabletOperationsMap> entry) throws Exception {
 
         int retryCount = 0;
-        boolean success = false;
 
         LsOperationsMap currentPartitions = new LsOperationsMap();
         currentPartitions.put(entry.getKey(), entry.getValue());
         int errCode = ResultCodes.OB_SUCCESS.errorCode;
         String errMsg = null;
-        int maxRetryTimes = obTableClient.getRuntimeRetryTimes();
-        // cannot use runTimeWait to retry to timeout
-        // because in huge-partitioned table situation, if refresh all tablets' locations
-        // it's hard to complete refreshing within time limit
-        while (retryCount < maxRetryTimes && !success) {
+        long runTimeMaxWait = obTableClient.getRuntimeMaxWait();
+        long startExecute = System.currentTimeMillis();
+        while (true) {
+            long costMillis = System.currentTimeMillis() - startExecute;
+            if (costMillis > runTimeMaxWait) {
+                errMsg = tableName + " failed to execute operation after retrying " + retryCount
+                        + " times and it has waited" +  costMillis + " ms"
+                        + " which exceeds runtime max wait timeout " + runTimeMaxWait
+                        + " ms. Last error Msg:" + "[errCode=" + errCode + "] " + errMsg;
+                logger.error(errMsg);
+                throw new ObTableUnexpectedException(errMsg);
+            }
             boolean allPartitionsSuccess = true;
 
             for (Map.Entry<Long, TabletOperationsMap> currentEntry : currentPartitions.entrySet()) {
@@ -861,14 +867,9 @@ private void executeWithRetries(ObTableSingleOpResult[] results,
             }
 
             if (allPartitionsSuccess) {
-                success = true;
+                break;
             }
         }
-        if (!success) {
-            errMsg = "Failed to execute operation after retrying " + retryCount
-                     + " times. Last error Msg:" + "[errCode=" + errCode + "] " + errMsg;
-            throw new ObTableUnexpectedException(errMsg);
-        }
     }
 
     /*