hard-invalidate region on RegionNotFound instead of async reload (tikv#1916)

X-Original-Commit: 33f56d8f139de91d3e75bc0019a3d83a0185dc9f

Author: ClamChowderTiDB

internal/locate/region_cache.go

@@ -1828,18 +1828,6 @@ func (c *RegionCache) BatchLoadRegionsFromKey(bo *retry.Backoffer, startKey []by
 	return regions[len(regions)-1].EndKey(), nil
 }
 
-// AsyncInvalidateCachedRegion marks a cached region for reload on next access
-// without invalidating it, so in-flight retries can still proceed.
-// It sets needDelayedReloadReady directly, skipping the GC promotion from
-// needDelayedReloadPending for faster reload.
-func (c *RegionCache) AsyncInvalidateCachedRegion(id RegionVerID) {
-	cachedRegion := c.GetCachedRegionWithRLock(id)
-	if cachedRegion == nil {
-		return
-	}
-	cachedRegion.setSyncFlags(needDelayedReloadReady)
-}
-
 // InvalidateCachedRegion removes a cached Region.
 func (c *RegionCache) InvalidateCachedRegion(id RegionVerID) {
 	c.InvalidateCachedRegionWithReason(id, Other)

internal/locate/region_request_state_test.go

@@ -288,9 +288,9 @@ func TestRegionCacheStaleRead(t *testing.T) {
 			leaderAsyncReload:     util.Some(false),
 			leaderSuccessReplica:  []string{"z1"},
 			leaderSuccessReadType: SuccessStaleRead,
-			followerRegionValid:   true,
-			followerAsyncReload:   util.Some(true),
-			// may async reload region and access it from leader.
+			followerRegionValid:   false,
+			followerAsyncReload:   util.Some(false),
+			// region is hard-invalidated but the current request succeeds via leader retry.
 			followerSuccessReplica:  []string{"z1"},
 			followerSuccessReadType: SuccessStaleRead,
 		},

internal/locate/replica_selector.go

@@ -33,13 +33,14 @@ import (
 
 type replicaSelector struct {
 	baseReplicaSelector
-	replicaReadType kv.ReplicaReadType
-	isStaleRead     bool
-	isReadOnlyReq   bool
-	option          storeSelectorOp
-	target          *replica
-	proxy           *replica
-	attempts        int
+	replicaReadType           kv.ReplicaReadType
+	isStaleRead               bool
+	isReadOnlyReq             bool
+	option                    storeSelectorOp
+	target                    *replica
+	proxy                     *replica
+	attempts                  int
+	regionInvalidatedForRetry bool // set when region is hard-invalidated but this selector should still retry on leader
 }
 
 func newReplicaSelector(
@@ -100,7 +101,13 @@ func buildTiKVReplicas(region *Region) []*replica {
 }
 
 func (s *replicaSelector) next(bo *retry.Backoffer, req *tikvrpc.Request) (rpcCtx *RPCContext, err error) {
-	if !s.region.isValid() {
+	if s.regionInvalidatedForRetry {
+		// Allow one more attempt on this selector even though the region is
+		// invalidated.  This is set by onRegionNotFound so that the current
+		// request can retry on the leader while concurrent requests are
+		// stopped by the hard invalidation.
+		s.regionInvalidatedForRetry = false
+	} else if !s.region.isValid() {
 		metrics.TiKVReplicaSelectorFailureCounter.WithLabelValues("invalid").Inc()
 		return nil, nil
 	}
@@ -528,12 +535,16 @@ func (s *replicaSelector) onRegionNotFound(
 	leaderIdx := s.region.getStore().workTiKVIdx
 	leader := s.replicas[leaderIdx]
 	if !leader.isExhausted(1, 0) {
-		// if the request is not sent to leader, we can retry it with leader and invalidate the region cache asynchronously. It helps in the scenario
-		// where region is split by the leader but not yet created in replica due to replica down.
+		// if the request is not sent to leader, we can retry it with leader and invalidate the region cache
+		// immediately. It helps in the scenario where region is split by the leader but not yet created
+		// in replica due to replica down.
+		// Hard-invalidate the region so that concurrent requests stop using the stale region,
+		// but allow this selector to retry on the leader via regionInvalidatedForRetry.
 		req.ReplicaRead = false
 		req.ReplicaReadType = kv.ReplicaReadLeader
 		s.replicaReadType = kv.ReplicaReadLeader
-		s.regionCache.AsyncInvalidateCachedRegion(ctx.Region)
+		s.regionInvalidatedForRetry = true
+		s.regionCache.InvalidateCachedRegion(ctx.Region)
 		return true, nil
 	}
 	s.regionCache.InvalidateCachedRegion(ctx.Region)

internal/locate/replica_selector_test.go

@@ -1114,14 +1114,16 @@ func TestReplicaReadAccessPathByBasicCase(t *testing.T) {
 					case kv.ReplicaReadLeader:
 						accessPath = []string{"{addr: store1, replica-read: false, stale-read: false}"}
 					case kv.ReplicaReadFollower:
-						// For RegionNotFoundErr from follower, it will retry on leader
+						// For RegionNotFoundErr from follower, it will retry on leader.
+						// The region is hard-invalidated to stop concurrent requests,
+						// but the current selector bypasses the validity check.
 						if tp == RegionNotFoundErr {
 							accessPath = []string{
 								"{addr: store2, replica-read: true, stale-read: false}",
 								"{addr: store1, replica-read: false, stale-read: false}",
 							}
 							respRegionError = nil
-							regionIsValid = true
+							regionIsValid = false
 						} else {
 							accessPath = []string{"{addr: store2, replica-read: true, stale-read: false}"}
 						}