fix: allow read-only operations when all masters are down

Read-only operations (get, select, pairs, count, min, max) used to
fail with connection errors if all masters in the cluster were unavailable,
even when healthy replicas were up and failover hadn't processed yet.

To resolve this, the following improvements were made:
- Introduced a `read_only` flag to `utils.get_space[s]` to fetch cluster
  schema from any healthy replica if masters are down.
- Updated `get`, `select`, `pairs`, `count`, `min`, `max` to use this
  new flag.
- Rewrote `call.any` to iterate through all replicasets and utilize
  vshard's `callro` instead of `call` to fetch metadata from replicas.

Author: p0rtale

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Fixed
+* Allow read-only operations (`get`, `select`, `pairs`, `count`, `min`, `max`)
+  to execute successfully using healthy replicas when all cluster masters are down.
+
 ## [1.7.4] - 12-02-26
 
 ### Fixed

crud/borders.lua

@@ -72,7 +72,10 @@ local function call_get_border_on_router(vshard_router, border_name, space_name,
         fetch_latest_metadata = '?boolean',
     })
 
-    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, opts.timeout)
+    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, {
+        timeout = opts.timeout,
+        read_only = opts.mode ~= 'write',
+    })
     if err ~= nil then
         return nil, BorderError:new("An error occurred during the operation: %s", err), const.NEED_SCHEMA_RELOAD
     end

crud/common/call.lua

@@ -270,19 +270,33 @@ function call.any(vshard_router, func_name, func_args, opts)
     if replicasets == nil then
         return nil, CallError:new("Failed to get router replicasets: %s", err.err)
     end
-    local replicaset_id, replicaset = next(replicasets)
 
-    local res, err = call_with_retry_and_recovery(vshard_router, replicaset, 'call',
-        func_name, func_args, {timeout = timeout}, false)
-    if err ~= nil then
-        return nil, wrap_vshard_err(vshard_router, err, func_name, replicaset_id)
-    end
+    local last_replicaset_id = nil
+    local last_err = nil
 
-    if res == box.NULL then
-        return nil
+    local deadline = fiber_clock() + timeout
+
+    for replicaset_id, replicaset in pairs(replicasets) do
+        local wait_timeout = deadline - fiber_clock()
+        if wait_timeout < 0 then
+            wait_timeout = 0
+        end
+
+        local res, err = call_with_retry_and_recovery(vshard_router, replicaset, 'callro',
+            func_name, func_args, {timeout = wait_timeout}, false)
+
+        if err == nil then
+            if res == box.NULL then
+                return nil
+            end
+            return res
+        end
+
+        last_replicaset_id = replicaset_id
+        last_err = err
     end
 
-    return res
+    return nil, wrap_vshard_err(vshard_router, last_err, func_name, last_replicaset_id)
 end
 
 return call

crud/common/utils.lua

@@ -123,10 +123,44 @@ local function get_replicaset_by_replica_id(replicasets, id)
     return nil, nil
 end
 
-function utils.get_spaces(vshard_router, timeout, replica_id)
-    local replicasets, replicaset, replicaset_id, master
+local function find_any_healthy_replica_conn(replicasets)
+    for _, replicaset in pairs(replicasets) do
+        for _, replica in pairs(replicaset.replicas) do
+            if replica:is_connected() then
+                return replica.conn
+            end
+        end
+    end
+    return nil
+end
+
+--- Returns all spaces existing in the schema.
+--
+-- @function get_spaces
+--
+-- @param table vshard_router
+--  A vshard router instance to route requests.
+--
+-- @tparam ?number opts.timeout
+--  Function call timeout. If not specified, `const.DEFAULT_VSHARD_CALL_TIMEOUT` is used.
+--
+-- @tparam ?boolean opts.read_only
+--  If true, the function will try to fetch spaces from any available healthy replica.
+--
+-- @tparam ?string opts.replica_id
+--  The exact replica ID to fetch the replicaset from.
+--
+-- @return[1] table map of spaces
+-- @return[1] nil
+-- @return[1] number schema version
+-- @treturn[2] nil
+-- @treturn[2] table Error description
+function utils.get_spaces(vshard_router, opts)
+    local replicasets, replicaset, replicaset_id, master, ro_replica_conn
+
+    opts = opts or {}
 
-    timeout = timeout or const.DEFAULT_VSHARD_CALL_TIMEOUT
+    local timeout = opts.timeout or const.DEFAULT_VSHARD_CALL_TIMEOUT
     local deadline = fiber.clock() + timeout
     local iter_sleep = math.min(timeout / 100, 0.1)
     while (
@@ -136,12 +170,12 @@ function utils.get_spaces(vshard_router, timeout, replica_id)
     ) do
         -- Try to get master with timeout.
         replicasets = vshard_router:routeall()
-        if replica_id ~= nil then
+        if opts.replica_id ~= nil then
             -- Get the same replica on which the last DML operation was performed.
             -- This approach is temporary and is related to [1], [2].
             -- [1] https://github.com/tarantool/crud/issues/236
             -- [2] https://github.com/tarantool/crud/issues/361
-            replicaset_id, replicaset = get_replicaset_by_replica_id(replicasets, replica_id)
+            replicaset_id, replicaset = get_replicaset_by_replica_id(replicasets, opts.replica_id)
             break
         else
             replicaset_id, replicaset = next(replicasets)
@@ -155,9 +189,22 @@ function utils.get_spaces(vshard_router, timeout, replica_id)
             end
         end
 
+        -- If the master check above didn't succeed (or master is dead),
+        -- and this is a read-only operation, try to find any available healthy replica.
+        if opts.read_only then
+            ro_replica_conn = find_any_healthy_replica_conn(replicasets)
+            if ro_replica_conn ~= nil then
+                break
+            end
+        end
+
         fiber.sleep(iter_sleep)
     end
 
+    if opts.read_only and ro_replica_conn ~= nil then
+        return ro_replica_conn.space, nil, ro_replica_conn.schema_version
+    end
+
     if replicaset == nil then
         return nil, GetSpaceError:new(
             'The router returned empty replicasets: ' ..
@@ -184,8 +231,32 @@ function utils.get_spaces(vshard_router, timeout, replica_id)
     return master.conn.space, nil, master.conn.schema_version
 end
 
-function utils.get_space(space_name, vshard_router, timeout, replica_id)
-    local spaces, err, schema_version = utils.get_spaces(vshard_router, timeout, replica_id)
+--- Returns a specific space by its name.
+--
+-- @function get_space
+--
+-- @param string space_name
+--  A space name to fetch.
+--
+-- @param table vshard_router
+--  A vshard router instance to route requests.
+--
+-- @tparam ?number opts.timeout
+--  Function call timeout.
+--
+-- @tparam ?boolean opts.read_only
+--  If true, the function will try to fetch the space from any available healthy replica.
+--
+-- @tparam ?string opts.replica_id
+--  The exact replica ID to fetch the replicaset from.
+--
+-- @return[1] table space object
+-- @return[1] nil
+-- @return[1] number schema version
+-- @treturn[2] nil
+-- @treturn[2] table Error description
+function utils.get_space(space_name, vshard_router, opts)
+    local spaces, err, schema_version = utils.get_spaces(vshard_router, opts)
 
     if spaces == nil then
         return nil, err

crud/count.lua

@@ -131,7 +131,10 @@ local function call_count_on_router(vshard_router, space_name, user_conditions,
         return nil, CountError:new("Failed to parse conditions: %s", err)
     end
 
-    local space, err = utils.get_space(space_name, vshard_router, opts.timeout)
+    local space, err = utils.get_space(space_name, vshard_router, {
+        timeout = opts.timeout,
+        read_only = opts.mode ~= 'write',
+    })
     if err ~= nil then
         return nil, CountError:new("An error occurred during the operation: %s", err), const.NEED_SCHEMA_RELOAD
     end

crud/delete.lua

@@ -81,7 +81,7 @@ local function call_delete_on_router(vshard_router, space_name, key, opts)
         fetch_latest_metadata = '?boolean',
     })
 
-    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, opts.timeout)
+    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, {timeout = opts.timeout})
     if err ~= nil then
         return nil, DeleteError:new("An error occurred during the operation: %s", err), const.NEED_SCHEMA_RELOAD
     end

crud/get.lua

@@ -82,7 +82,10 @@ local function call_get_on_router(vshard_router, space_name, key, opts)
         fetch_latest_metadata = '?boolean',
     })
 
-    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, opts.timeout)
+    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, {
+        timeout = opts.timeout,
+        read_only = opts.mode ~= 'write',
+    })
     if err ~= nil then
         return nil, GetError:new("An error occurred during the operation: %s", err), const.NEED_SCHEMA_RELOAD
     end

crud/insert.lua

@@ -85,7 +85,7 @@ local function call_insert_on_router(vshard_router, space_name, original_tuple,
         fetch_latest_metadata = '?boolean',
     })
 
-    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, opts.timeout)
+    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, {timeout = opts.timeout})
     if err ~= nil then
         return nil, InsertError:new("An error occurred during the operation: %s", err), const.NEED_SCHEMA_RELOAD
     end

crud/insert_many.lua

@@ -183,7 +183,7 @@ local function call_insert_many_on_router(vshard_router, space_name, original_tu
         fetch_latest_metadata = '?boolean',
     })
 
-    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, opts.timeout)
+    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, {timeout = opts.timeout})
     if err ~= nil then
         return nil, {
             InsertManyError:new("An error occurred during the operation: %s", err)

crud/len.lua

@@ -53,7 +53,7 @@ function len.call(space_name, opts)
         return nil, LenError:new(err)
     end
 
-    local space, err = utils.get_space(space_name, vshard_router, opts.timeout)
+    local space, err = utils.get_space(space_name, vshard_router, {timeout = opts.timeout})
     if err ~= nil then
         return nil, LenError:new("An error occurred during the operation: %s", err)
     end