control-connection: reconnect after stale failure

Author: dkropachev

cassandra/cluster.py

@@ -5362,10 +5362,10 @@ def _signal_error(self):
                 # host may be None if it's already been removed, but that indicates
                 # that errors have already been reported, so we're fine
                 if host:
-                    self._cluster.signal_connection_failure(
+                    if self._cluster.signal_connection_failure(
                         host, self._connection.last_error, is_host_addition=False,
-                        expected_endpoint=self._connection.endpoint)
-                    return
+                        expected_endpoint=self._connection.endpoint):
+                        return
 
         # if the connection is not defunct or the host already left, reconnect
         # manually

tests/unit/test_control_connection.py

@@ -455,6 +455,34 @@ def test_stale_control_connection_failure_is_endpoint_fenced(self):
             host, self.connection.last_error, is_host_addition=False,
             expected_endpoint=old_endpoint)
 
+    def test_stale_control_connection_failure_reconnects_when_cluster_ignores_signal(self):
+        host_id = uuid.uuid4()
+        old_endpoint = ClientRoutesEndPoint(
+            host_id, Mock(), "127.0.0.1", original_port=9042)
+        new_endpoint = ClientRoutesEndPoint(
+            host_id, Mock(), "127.0.0.1", original_port=9142)
+
+        host = Host(old_endpoint, SimpleConvictionPolicy, host_id=host_id)
+        host.set_up()
+        self.cluster.metadata = Metadata()
+        self.cluster.metadata.add_or_return_host(host)
+        host.endpoint = new_endpoint
+        self.cluster.metadata.update_host(host, old_endpoint)
+
+        self.connection.endpoint = old_endpoint
+        self.connection.is_defunct = True
+        self.connection.last_error = ConnectionException(
+            "stale control connection failed", endpoint=old_endpoint)
+        self.cluster.signal_connection_failure = Mock(return_value=False)
+        self.control_connection.reconnect = Mock()
+
+        self.control_connection._signal_error()
+
+        self.cluster.signal_connection_failure.assert_called_once_with(
+            host, self.connection.last_error, is_host_addition=False,
+            expected_endpoint=old_endpoint)
+        self.control_connection.reconnect.assert_called_once_with()
+
     def test_refresh_nodes_and_tokens_uses_preloaded_results_if_given(self):
         """
         refresh_nodes_and_tokens uses preloaded results if given for shared table queries