fix: allow read-only operations when all masters are down

Read-only operations (get, select, count, min, max) used to fail with
connection errors if all masters in the cluster were unavailable, even
when healthy replicas were up and failover hadn't processed yet.

To resolve this, the following improvements were made:
- Introduced a `read_only` flag to `utils.get_space[s]` to fetch cluster
  schema from any healthy replica if masters are down.
- Updated `get`, `select`, `count`, `min`, `max` to use this new flag.
- Rewrote `call.any` to iterate through all replicasets and utilize
  vshard's `callro` instead of `call` to fetch metadata from replicas.

Author: p0rtale

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## [Unreleased]
 
+### Fixed
+* Allow read-only operations (`get`, `select`, `count`, `min`, `max`) to execute 
+  successfully using healthy replicas when all cluster masters are down.
+
 ## [1.7.4] - 12-02-26
 
 ### Fixed

crud/borders.lua

@@ -72,7 +72,7 @@ local function call_get_border_on_router(vshard_router, border_name, space_name,
         fetch_latest_metadata = '?boolean',
     })
 
-    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, opts.timeout)
+    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, opts.timeout, nil, true)
     if err ~= nil then
         return nil, BorderError:new("An error occurred during the operation: %s", err), const.NEED_SCHEMA_RELOAD
     end

crud/common/call.lua

@@ -270,19 +270,26 @@ function call.any(vshard_router, func_name, func_args, opts)
     if replicasets == nil then
         return nil, CallError:new("Failed to get router replicasets: %s", err.err)
     end
-    local replicaset_id, replicaset = next(replicasets)
 
-    local res, err = call_with_retry_and_recovery(vshard_router, replicaset, 'call',
-        func_name, func_args, {timeout = timeout}, false)
-    if err ~= nil then
-        return nil, wrap_vshard_err(vshard_router, err, func_name, replicaset_id)
-    end
+    local last_replicaset_id = nil
+    local last_err = nil
 
-    if res == box.NULL then
-        return nil
+    for replicaset_id, replicaset in pairs(replicasets) do
+        local res, err = call_with_retry_and_recovery(vshard_router, replicaset, 'callro',
+            func_name, func_args, {timeout = timeout}, false)
+
+        if err == nil then
+            if res == box.NULL then
+                return nil
+            end
+            return res
+        end
+
+        last_replicaset_id = replicaset_id
+        last_err = err
     end
 
-    return res
+    return nil, wrap_vshard_err(vshard_router, last_err, func_name, last_replicaset_id)
 end
 
 return call

crud/common/utils.lua

@@ -123,8 +123,19 @@ local function get_replicaset_by_replica_id(replicasets, id)
     return nil, nil
 end
 
-function utils.get_spaces(vshard_router, timeout, replica_id)
-    local replicasets, replicaset, replicaset_id, master
+local function find_any_healthy_replica_conn(replicasets)
+    for _, replicaset in pairs(replicasets) do
+        for _, replica in pairs(replicaset.replicas) do
+            if replica.conn ~= nil and replica.conn.error == nil then
+                return replica.conn
+            end
+        end
+    end
+    return nil
+end
+
+function utils.get_spaces(vshard_router, timeout, replica_id, read_only)
+    local replicasets, replicaset, replicaset_id, master, ro_replica_conn
 
     timeout = timeout or const.DEFAULT_VSHARD_CALL_TIMEOUT
     local deadline = fiber.clock() + timeout
@@ -155,9 +166,22 @@ function utils.get_spaces(vshard_router, timeout, replica_id)
             end
         end
 
+        -- If the master check above didn't succeed (or master is dead),
+        -- and this is a read-only operation, try to find any available healthy replica.
+        if read_only then
+            ro_replica_conn = find_any_healthy_replica_conn(replicasets)
+            if ro_replica_conn ~= nil then
+                break
+            end
+        end
+
         fiber.sleep(iter_sleep)
     end
 
+    if read_only and ro_replica_conn ~= nil then
+        return ro_replica_conn.space, nil, ro_replica_conn.schema_version
+    end
+
     if replicaset == nil then
         return nil, GetSpaceError:new(
             'The router returned empty replicasets: ' ..
@@ -184,8 +208,8 @@ function utils.get_spaces(vshard_router, timeout, replica_id)
     return master.conn.space, nil, master.conn.schema_version
 end
 
-function utils.get_space(space_name, vshard_router, timeout, replica_id)
-    local spaces, err, schema_version = utils.get_spaces(vshard_router, timeout, replica_id)
+function utils.get_space(space_name, vshard_router, timeout, replica_id, read_only)
+    local spaces, err, schema_version = utils.get_spaces(vshard_router, timeout, replica_id, read_only)
 
     if spaces == nil then
         return nil, err

crud/count.lua

@@ -131,7 +131,7 @@ local function call_count_on_router(vshard_router, space_name, user_conditions,
         return nil, CountError:new("Failed to parse conditions: %s", err)
     end
 
-    local space, err = utils.get_space(space_name, vshard_router, opts.timeout)
+    local space, err = utils.get_space(space_name, vshard_router, opts.timeout, nil, true)
     if err ~= nil then
         return nil, CountError:new("An error occurred during the operation: %s", err), const.NEED_SCHEMA_RELOAD
     end

crud/get.lua

@@ -82,7 +82,7 @@ local function call_get_on_router(vshard_router, space_name, key, opts)
         fetch_latest_metadata = '?boolean',
     })
 
-    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, opts.timeout)
+    local space, err, netbox_schema_version = utils.get_space(space_name, vshard_router, opts.timeout, nil, true)
     if err ~= nil then
         return nil, GetError:new("An error occurred during the operation: %s", err), const.NEED_SCHEMA_RELOAD
     end

crud/select/compat/select.lua

@@ -52,7 +52,7 @@ local function build_select_iterator(vshard_router, space_name, user_conditions,
         return nil, SelectError:new("Failed to parse conditions: %s", err)
     end
 
-    local space, err, netbox_schema_version  = utils.get_space(space_name, vshard_router)
+    local space, err, netbox_schema_version  = utils.get_space(space_name, vshard_router, nil, nil, true)
     if err ~= nil then
         return nil, SelectError:new("An error occurred during the operation: %s", err), const.NEED_SCHEMA_RELOAD
     end

test/integration/read_calls_strategies_test.lua

@@ -52,6 +52,11 @@ pgroup.before_all(function(g)
         return vshard_call_strategies
     end
 
+    -- Warm up router sharding metadata cache.
+    -- This ensures that internal callro via call.any won't fire
+    -- unpredictably during actual test execution.
+    g.router:call('crud.get', {'customers', 1})
+
     -- patch vshard.router.call* functions
     local vshard_call_names = {'callro', 'callbro', 'callre', 'callbre', 'callrw'}
     g.router:call('patch_vshard_calls', {vshard_call_names})

test/integration/read_dead_masters_test.lua

@@ -0,0 +1,56 @@
+local t = require('luatest')
+
+local helpers = require('test.helper')
+
+local pgroup = t.group('read_only_with_dead_masters', helpers.backend_matrix({
+    {engine = 'memtx'},
+}))
+
+pgroup.before_all(function(g)
+    helpers.start_default_cluster(g, 'srv_simple_operations')
+end)
+
+pgroup.after_all(function(g)
+    helpers.stop_cluster(g.cluster, g.params.backend)
+end)
+
+pgroup.test_read_operations_when_masters_are_down = function(g)
+    local insert_res, err = g.router:call('crud.insert', {'customers', {1, box.NULL, 'Elizabeth', 12}})
+    t.assert_equals(err, nil)
+    t.assert_not_equals(insert_res, nil)
+
+    insert_res, err = g.router:call('crud.insert', {'customers', {5, box.NULL, 'Jack', 35}})
+    t.assert_equals(err, nil)
+    t.assert_not_equals(insert_res, nil)
+
+    -- Stop all masters in the cluster to simulate total master outage.
+    local s1_master = g.cluster:server('s1-master')
+    local s2_master = g.cluster:server('s2-master')
+
+    s1_master:stop()
+    s2_master:stop()
+
+    local select_res, err = g.router:call('crud.select', {'customers', {{'<=', 'age', 35}}})
+    t.assert_equals(err, nil)
+    t.assert_not_equals(select_res, nil)
+    t.assert_equals(#select_res.rows, 2)
+
+    local get_res, err = g.router:call('crud.get', {'customers', 1})
+    t.assert_equals(err, nil)
+    t.assert_not_equals(get_res, nil)
+    t.assert_equals(get_res.rows[1][3], 'Elizabeth')
+
+    local count_res, err = g.router:call('crud.count', {'customers', {{'==', 'age', 35}}})
+    t.assert_equals(err, nil)
+    t.assert_equals(count_res, 1)
+
+    local min_res, err = g.router:call('crud.min', {'customers'})
+    t.assert_equals(err, nil)
+    t.assert_not_equals(min_res, nil)
+    t.assert_equals(min_res.rows[1][1], 1)
+
+    local max_res, err = g.router:call('crud.max', {'customers'})
+    t.assert_equals(err, nil)
+    t.assert_not_equals(max_res, nil)
+    t.assert_equals(max_res.rows[1][1], 5)
+end

test/unit/call_test.lua

@@ -237,8 +237,11 @@ pgroup.test_any_vshard_call = function(g)
         return call.any(vshard.router.static, 'say_hi_politely', {'dude'}, {})
     ]])
 
-    t.assert_equals(results, 'HI, dude! I am 1')
+    t.assert_str_contains(results, 'HI, dude!')
     t.assert_equals(err, nil)
+
+    local vshard_calls = g.get_vshard_calls()
+    t.assert_equals(vshard_calls, {'callro'})
 end
 
 pgroup.test_any_vshard_call_timeout = function(g)
@@ -264,3 +267,64 @@ pgroup.test_any_vshard_call_timeout = function(g)
     helpers.assert_str_contains_pattern_with_replicaset_id(err.err, "Failed for [replicaset_id]")
     helpers.assert_timeout_error(err.err)
 end
+
+pgroup.before_test('test_any_vshard_call_fallback', function(g)
+    -- Mock callro to fail exactly once per replicaset.
+    -- This guarantees that call.any triggers its fallback loop.
+    g.router:eval([[
+        local vshard = require('vshard')
+        local errors = require('errors')
+
+        local replicasets = vshard.router.static:routeall()
+
+        rawset(_G, '__original_callros', {})
+        local originals = rawget(_G, '__original_callros')
+
+        for replicaset_id, replicaset in pairs(replicasets) do
+            originals[replicaset_id] = replicaset.callro
+
+            local calls_count = 0
+
+            replicaset.callro = function(self, ...)
+                calls_count = calls_count + 1
+                if calls_count == 1 then
+                    return nil, errors.new_class('ClientError'):new('Temporary failure')
+                else
+                    return originals[replicaset_id](self, ...)
+                end
+            end
+        end
+    ]])
+end)
+
+pgroup.after_test('test_any_vshard_call_fallback', function(g)
+    g.router:eval([[
+        local vshard = require('vshard')
+
+        local replicasets = vshard.router.static:routeall()
+        local originals = rawget(_G, '__original_callros')
+
+        for replicaset_id, original_fun in pairs(originals) do
+            if replicasets[replicaset_id] ~= nil then
+                replicasets[replicaset_id].callro = original_fun
+            end
+        end
+        rawset(_G, '__original_callros', nil)
+    ]])
+end)
+
+pgroup.test_any_vshard_call_fallback = function(g)
+    g.clear_vshard_calls()
+
+    local results, err = g.router:eval([[
+        local vshard = require('vshard')
+        local call = require('crud.common.call')
+
+        return call.any(vshard.router.static, 'say_hi_politely', {'survivor'}, {})
+    ]])
+
+    -- Ensure call.any successfully routes around the initial failure
+    -- and uses the available fallback.
+    t.assert_str_contains(results, 'HI, survivor!')
+    t.assert_equals(err, nil)
+end