Skip to content

Commit 582e86d

Browse files
committed
Fix on_up() destroying healthy pool after replace-with-same-IP (SCYLLADB-833)
When a node is replaced with the same IP, the driver receives both TOPOLOGY_CHANGE NEW_NODE and STATUS_CHANGE UP events. The NEW_NODE handler runs first, replacing the old host and establishing a new pool. The STATUS_CHANGE UP handler fires later with a stale reference to the old host object. Because Host.__eq__/__hash__ are endpoint-based, the stale on_up() tears down the new host's pool, causing a brief window where queries fail with NoHostAvailable. Add two guards at the top of on_up(): 1. If the host has been replaced in metadata (different object, same endpoint, new host already up), skip processing. 2. If a healthy (non-shutdown) pool already exists for this host, mark it up and skip the teardown/rebuild cycle. Refs: SCYLLADB-833
1 parent e6f9e9f commit 582e86d

2 files changed

Lines changed: 90 additions & 0 deletions

File tree

cassandra/cluster.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1930,6 +1930,25 @@ def on_up(self, host):
19301930
have_future = False
19311931
futures = set()
19321932
try:
1933+
# SCYLLADB-833: Guard against stale on_up destroying a healthy pool.
1934+
# Case 1: Host was replaced in metadata (different object, same endpoint).
1935+
current_host = self.metadata.get_host(host.endpoint)
1936+
if current_host is not None and current_host is not host and current_host.is_up:
1937+
log.debug("Host %s has been replaced by %s which is already up; "
1938+
"skipping stale on_up handling", host, current_host)
1939+
host._currently_handling_node_up = False
1940+
return
1941+
1942+
# Case 2: A healthy pool already exists (e.g. on_add already ran).
1943+
for session in tuple(self.sessions):
1944+
pool = session._pools.get(host)
1945+
if pool and not pool.is_shutdown:
1946+
log.debug("Host %s already has a healthy pool; "
1947+
"skipping on_up pool teardown/rebuild", host)
1948+
host.is_up = True
1949+
host._currently_handling_node_up = False
1950+
return
1951+
19331952
log.info("Host %s may be up; will prepare queries and open connection pool", host)
19341953

19351954
reconnector = host.get_and_set_reconnection_handler(None)

tests/unit/test_cluster.py

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -719,3 +719,74 @@ def test_no_warning_adding_lbp_ep_to_cluster_with_contact_points(self):
719719
)
720720

721721
patched_logger.warning.assert_not_called()
722+
723+
724+
class TestOnUpStaleHost(unittest.TestCase):
725+
"""
726+
Tests for SCYLLADB-833: on_up() should not destroy a healthy pool when
727+
called with a stale host reference after a replace-with-same-IP.
728+
"""
729+
730+
def _make_cluster(self, sessions=None):
731+
"""Create a minimal Cluster object without connecting."""
732+
from threading import Lock
733+
cluster = object.__new__(Cluster)
734+
cluster.is_shutdown = False
735+
cluster.metadata = Mock()
736+
cluster.sessions = sessions or set()
737+
cluster.profile_manager = Mock()
738+
cluster.control_connection = Mock()
739+
cluster._listeners = set()
740+
cluster._listener_lock = Lock()
741+
return cluster
742+
743+
def test_on_up_skips_when_host_replaced_in_metadata(self):
744+
"""
745+
If a NEW_NODE event already replaced the old host with a new one
746+
(same endpoint, different host_id), on_up(old_host) should bail out
747+
instead of tearing down the new host's pool.
748+
"""
749+
from cassandra.connection import DefaultEndPoint
750+
endpoint = DefaultEndPoint('127.0.0.1')
751+
752+
old_host = Host(endpoint, conviction_policy_factory=Mock(), host_id=uuid.uuid4())
753+
old_host.is_up = False
754+
old_host._currently_handling_node_up = False
755+
756+
new_host = Host(endpoint, conviction_policy_factory=Mock(), host_id=uuid.uuid4())
757+
new_host.is_up = True
758+
759+
cluster = self._make_cluster()
760+
cluster.metadata.get_host = Mock(return_value=new_host)
761+
762+
cluster.on_up(old_host)
763+
764+
self.assertFalse(old_host._currently_handling_node_up,
765+
"on_up should have returned early and reset _currently_handling_node_up")
766+
767+
def test_on_up_skips_when_healthy_pool_exists(self):
768+
"""
769+
If on_add already created a healthy pool for this host, a subsequent
770+
on_up should not tear it down and rebuild it.
771+
"""
772+
from cassandra.connection import DefaultEndPoint
773+
endpoint = DefaultEndPoint('127.0.0.1')
774+
775+
host = Host(endpoint, conviction_policy_factory=Mock(), host_id=uuid.uuid4())
776+
host.is_up = False
777+
host._currently_handling_node_up = False
778+
779+
mock_pool = Mock()
780+
mock_pool.is_shutdown = False
781+
mock_session = Mock()
782+
mock_session._pools = {host: mock_pool}
783+
784+
cluster = self._make_cluster(sessions={mock_session})
785+
cluster.metadata.get_host = Mock(return_value=host)
786+
787+
cluster.on_up(host)
788+
789+
mock_session.remove_pool.assert_not_called()
790+
self.assertTrue(host.is_up, "on_up should have marked host as up")
791+
self.assertFalse(host._currently_handling_node_up,
792+
"on_up should have returned early and reset _currently_handling_node_up")

0 commit comments

Comments
 (0)