diff --git a/packages/react-native-gesture-handler/src/v3/hooks/useGesture.ts b/packages/react-native-gesture-handler/src/v3/hooks/useGesture.ts
index d82f8f0edf..139dbb7d0e 100644
--- a/packages/react-native-gesture-handler/src/v3/hooks/useGesture.ts
+++ b/packages/react-native-gesture-handler/src/v3/hooks/useGesture.ts
@@ -90,7 +90,10 @@ export function useGesture<THandlerData, TConfig>(
 
   useEffect(() => {
     return () => {
-      currentGestureRef.current = { type: '', handlerTag: -1 };
+      if (currentGestureRef.current.handlerTag === handlerTag) {
+        currentGestureRef.current = { type: '', handlerTag: -1 };
+      }
+
       NativeProxy.dropGestureHandler(handlerTag);
       scheduleFlushOperations();
     };