Skip to content

Commit c9b8e5e

Browse files
committed
Fix on_up() destroying healthy pool after replace-with-same-IP
When a node is replaced with the same IP, the driver receives both TOPOLOGY_CHANGE NEW_NODE and STATUS_CHANGE UP events. The NEW_NODE handler runs first, replacing the old host and establishing a new pool. The STATUS_CHANGE UP handler fires later with a stale reference to the old host object. Because Host.__eq__/__hash__ are endpoint-based, the stale on_up() tears down the new host's pool, causing a brief window where queries fail with NoHostAvailable. Add two guards at the top of on_up(): 1. If the host has been replaced in metadata (different object, same endpoint, new host already up), skip processing. 2. If a healthy (non-shutdown) pool already exists for this host, call set_up() and skip the teardown/rebuild cycle. Both guards reset _currently_handling_node_up under host.lock, consistent with the existing cleanup paths. Refs: SCYLLADB-833
1 parent e6f9e9f commit c9b8e5e

2 files changed

Lines changed: 92 additions & 0 deletions

File tree

cassandra/cluster.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1930,6 +1930,27 @@ def on_up(self, host):
19301930
have_future = False
19311931
futures = set()
19321932
try:
1933+
# Guard against stale on_up destroying a healthy pool.
1934+
# Case 1: Host was replaced in metadata (different object, same endpoint).
1935+
current_host = self.metadata.get_host(host.endpoint)
1936+
if current_host is not None and current_host is not host and current_host.is_up:
1937+
log.debug("Host %s has been replaced by %s which is already up; "
1938+
"skipping stale on_up handling", host, current_host)
1939+
with host.lock:
1940+
host._currently_handling_node_up = False
1941+
return
1942+
1943+
# Case 2: A healthy pool already exists (e.g. on_add already ran).
1944+
for session in tuple(self.sessions):
1945+
pool = session._pools.get(host)
1946+
if pool and not pool.is_shutdown:
1947+
log.debug("Host %s already has a healthy pool; "
1948+
"skipping on_up pool teardown/rebuild", host)
1949+
with host.lock:
1950+
host.set_up()
1951+
host._currently_handling_node_up = False
1952+
return
1953+
19331954
log.info("Host %s may be up; will prepare queries and open connection pool", host)
19341955

19351956
reconnector = host.get_and_set_reconnection_handler(None)

tests/unit/test_cluster.py

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -719,3 +719,74 @@ def test_no_warning_adding_lbp_ep_to_cluster_with_contact_points(self):
719719
)
720720

721721
patched_logger.warning.assert_not_called()
722+
723+
724+
class TestOnUpStaleHost(unittest.TestCase):
725+
"""
726+
Tests for on_up() not destroying a healthy pool when called with a stale
727+
host reference after a replace-with-same-IP.
728+
"""
729+
730+
def _make_cluster(self, sessions=None):
731+
"""Create a minimal Cluster object without connecting."""
732+
from threading import Lock
733+
cluster = object.__new__(Cluster)
734+
cluster.is_shutdown = False
735+
cluster.metadata = Mock()
736+
cluster.sessions = sessions or set()
737+
cluster.profile_manager = Mock()
738+
cluster.control_connection = Mock()
739+
cluster._listeners = set()
740+
cluster._listener_lock = Lock()
741+
return cluster
742+
743+
def test_on_up_skips_when_host_replaced_in_metadata(self):
744+
"""
745+
If a NEW_NODE event already replaced the old host with a new one
746+
(same endpoint, different host_id), on_up(old_host) should bail out
747+
instead of tearing down the new host's pool.
748+
"""
749+
from cassandra.connection import DefaultEndPoint
750+
endpoint = DefaultEndPoint('127.0.0.1')
751+
752+
old_host = Host(endpoint, conviction_policy_factory=Mock(), host_id=uuid.uuid4())
753+
old_host.is_up = False
754+
old_host._currently_handling_node_up = False
755+
756+
new_host = Host(endpoint, conviction_policy_factory=Mock(), host_id=uuid.uuid4())
757+
new_host.is_up = True
758+
759+
cluster = self._make_cluster()
760+
cluster.metadata.get_host = Mock(return_value=new_host)
761+
762+
cluster.on_up(old_host)
763+
764+
self.assertFalse(old_host._currently_handling_node_up,
765+
"on_up should have returned early and reset _currently_handling_node_up")
766+
767+
def test_on_up_skips_when_healthy_pool_exists(self):
768+
"""
769+
If on_add already created a healthy pool for this host, a subsequent
770+
on_up should not tear it down and rebuild it.
771+
"""
772+
from cassandra.connection import DefaultEndPoint
773+
endpoint = DefaultEndPoint('127.0.0.1')
774+
775+
host = Host(endpoint, conviction_policy_factory=Mock(), host_id=uuid.uuid4())
776+
host.is_up = False
777+
host._currently_handling_node_up = False
778+
779+
mock_pool = Mock()
780+
mock_pool.is_shutdown = False
781+
mock_session = Mock()
782+
mock_session._pools = {host: mock_pool}
783+
784+
cluster = self._make_cluster(sessions={mock_session})
785+
cluster.metadata.get_host = Mock(return_value=host)
786+
787+
cluster.on_up(host)
788+
789+
mock_session.remove_pool.assert_not_called()
790+
self.assertTrue(host.is_up, "on_up should have marked host as up")
791+
self.assertFalse(host._currently_handling_node_up,
792+
"on_up should have returned early and reset _currently_handling_node_up")

0 commit comments

Comments
 (0)