Skip to content

Commit 19f27f0

Browse files
committed
connection: tolerate clean startup close
1 parent bcc2d3d commit 19f27f0

6 files changed

Lines changed: 302 additions & 40 deletions

File tree

cassandra/cluster.py

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -345,6 +345,14 @@ def _execution_profile_to_string(name):
345345
return '"%s"' % (name,)
346346

347347

348+
def _raise_if_connection_closed_during_startup(conn, endpoint):
349+
if conn.is_closed:
350+
endpoint = getattr(conn, 'endpoint', endpoint)
351+
raise ConnectionShutdown(
352+
"Connection to %s was closed during the startup handshake" % (endpoint,),
353+
endpoint)
354+
355+
348356
class ExecutionProfile(object):
349357
load_balancing_policy = None
350358
"""
@@ -1749,11 +1757,22 @@ def connection_factory(self, endpoint, host_conn = None, *args, **kwargs):
17491757
Intended for internal use only.
17501758
"""
17511759
kwargs = self._make_connection_kwargs(endpoint, kwargs)
1752-
return self.connection_class.factory(endpoint, self.connect_timeout, host_conn, *args, **kwargs)
1760+
conn = self.connection_class.factory(endpoint, self.connect_timeout, host_conn, *args, **kwargs)
1761+
_raise_if_connection_closed_during_startup(conn, endpoint)
1762+
return conn
17531763

17541764
def _make_connection_factory(self, host, *args, **kwargs):
1755-
kwargs = self._make_connection_kwargs(host.endpoint, kwargs)
1756-
return partial(self.connection_class.factory, host.endpoint, self.connect_timeout, *args, **kwargs)
1765+
endpoint = host.endpoint
1766+
connection_class = self.connection_class
1767+
connect_timeout = self.connect_timeout
1768+
kwargs = self._make_connection_kwargs(endpoint, kwargs)
1769+
1770+
def connection_factory():
1771+
conn = connection_class.factory(endpoint, connect_timeout, *args, **kwargs)
1772+
_raise_if_connection_closed_during_startup(conn, endpoint)
1773+
return conn
1774+
1775+
return connection_factory
17571776

17581777
def _make_connection_kwargs(self, endpoint, kwargs_dict):
17591778
if self._auth_provider_callable:

cassandra/connection.py

Lines changed: 30 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -965,31 +965,42 @@ def create_timer(cls, timeout, callback):
965965
@classmethod
966966
def factory(cls, endpoint, timeout, host_conn = None, *args, **kwargs):
967967
"""
968-
A factory function which returns connections which have
969-
succeeded in connecting and are ready for service (or
970-
raises an exception otherwise).
968+
A factory function which returns a connection once startup has
969+
completed, returns a closed connection if the server closes during
970+
startup, or raises an exception otherwise.
971+
972+
Pool callers pass ``host_conn`` so the connection can be tracked while
973+
startup is in progress and closed during pool shutdown.
971974
"""
972975
start = time.time()
973976
kwargs['connect_timeout'] = timeout
974977
conn = cls(endpoint, *args, **kwargs)
975-
if host_conn is not None:
976-
host_conn._pending_connections.append(conn)
977-
if host_conn.is_shutdown:
978+
pending_registered = False
979+
try:
980+
if host_conn is not None:
981+
host_conn._pending_connections.append(conn)
982+
pending_registered = True
983+
if host_conn.is_shutdown:
984+
conn.close()
985+
elapsed = time.time() - start
986+
conn.connected_event.wait(timeout - elapsed)
987+
if conn.last_error:
988+
if conn.is_unsupported_proto_version:
989+
raise ProtocolVersionUnsupported(endpoint, conn.protocol_version)
990+
if conn.is_closed and not conn.is_defunct and isinstance(conn.last_error, ConnectionShutdown):
991+
return conn
992+
raise conn.last_error
993+
elif not conn.connected_event.is_set():
978994
conn.close()
979-
elapsed = time.time() - start
980-
conn.connected_event.wait(timeout - elapsed)
981-
if conn.last_error:
982-
if conn.is_unsupported_proto_version:
983-
raise ProtocolVersionUnsupported(endpoint, conn.protocol_version)
984-
raise conn.last_error
985-
elif not conn.connected_event.is_set():
986-
conn.close()
987-
raise OperationTimedOut("Timed out creating connection (%s seconds)" % timeout,
988-
timeout=timeout)
989-
elif conn.is_closed:
990-
raise ConnectionShutdown("Connection to %s was closed by server" % conn.endpoint)
991-
else:
995+
raise OperationTimedOut("Timed out creating connection (%s seconds)" % timeout,
996+
timeout=timeout)
992997
return conn
998+
finally:
999+
if pending_registered:
1000+
try:
1001+
host_conn._pending_connections.remove(conn)
1002+
except ValueError:
1003+
pass
9931004

9941005
def _build_ssl_context_from_options(self):
9951006

cassandra/pool.py

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -426,7 +426,10 @@ def __init__(self, host, host_distance, session):
426426
return
427427

428428
log.debug("Initializing connection for host %s", self.host)
429-
first_connection = session.cluster.connection_factory(self.host.endpoint, on_orphaned_stream_released=self.on_orphaned_stream_released)
429+
first_connection = session.cluster.connection_factory(
430+
self.host.endpoint,
431+
host_conn=self,
432+
on_orphaned_stream_released=self.on_orphaned_stream_released)
430433
log.debug("First connection created to %s for shard_id=%i", self.host, first_connection.features.shard_id)
431434
self._connections[first_connection.features.shard_id] = first_connection
432435
self._keyspace = session.keyspace
@@ -605,8 +608,10 @@ def _replace(self, connection):
605608
self._connecting.add(connection.features.shard_id)
606609
self._session.submit(self._open_connection_to_missing_shard, connection.features.shard_id)
607610
else:
608-
connection = self._session.cluster.connection_factory(self.host.endpoint,
609-
on_orphaned_stream_released=self.on_orphaned_stream_released)
611+
connection = self._session.cluster.connection_factory(
612+
self.host.endpoint,
613+
host_conn=self,
614+
on_orphaned_stream_released=self.on_orphaned_stream_released)
610615
if self._keyspace:
611616
connection.set_keyspace_blocking(self._keyspace)
612617
self._connections[connection.features.shard_id] = connection
@@ -725,15 +730,21 @@ def _open_connection_to_missing_shard(self, shard_id):
725730
log.debug("shard_aware_endpoint=%r", shard_aware_endpoint)
726731
if shard_aware_endpoint:
727732
try:
728-
conn = self._session.cluster.connection_factory(shard_aware_endpoint, host_conn=self, on_orphaned_stream_released=self.on_orphaned_stream_released,
729-
shard_id=shard_id,
730-
total_shards=self.host.sharding_info.shards_count)
733+
conn = self._session.cluster.connection_factory(
734+
shard_aware_endpoint,
735+
host_conn=self,
736+
on_orphaned_stream_released=self.on_orphaned_stream_released,
737+
shard_id=shard_id,
738+
total_shards=self.host.sharding_info.shards_count)
731739
conn.original_endpoint = self.host.endpoint
732740
except Exception as exc:
733741
log.error("Failed to open connection to %s, on shard_id=%i: %s", self.host, shard_id, exc)
734742
raise
735743
else:
736-
conn = self._session.cluster.connection_factory(self.host.endpoint, host_conn=self, on_orphaned_stream_released=self.on_orphaned_stream_released)
744+
conn = self._session.cluster.connection_factory(
745+
self.host.endpoint,
746+
host_conn=self,
747+
on_orphaned_stream_released=self.on_orphaned_stream_released)
737748

738749
log.debug(
739750
"Received a connection %s for shard_id=%i on host %s",

tests/unit/test_cluster.py

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
InvalidRequest, Unauthorized, AuthenticationFailed, OperationTimedOut, UnsupportedOperation, RequestValidationException, ConfigurationException, ProtocolVersion
2626
from cassandra.cluster import _Scheduler, Session, Cluster, ResultSet, SchemaAgreementScope, ControlConnectionQueryFallback, default_lbp_factory, \
2727
ExecutionProfile, _ConfigMode, EXEC_PROFILE_DEFAULT
28-
from cassandra.connection import ConnectionBusy, ConnectionException
28+
from cassandra.connection import ConnectionBusy, ConnectionException, ConnectionShutdown
2929
from cassandra.pool import Host
3030
from cassandra.policies import HostDistance, RetryPolicy, RoundRobinPolicy, DowngradingConsistencyRetryPolicy, SimpleConvictionPolicy
3131
from cassandra.query import SimpleStatement, named_tuple_factory, tuple_factory
@@ -268,16 +268,58 @@ def test_connection_factory_passes_compression_kwarg(self):
268268
]
269269

270270
for supported, configured, expected in scenarios:
271+
connection = Mock(is_closed=False)
271272
with patch.dict('cassandra.cluster.locally_supported_compressions', supported, clear=True):
272-
with patch.object(Cluster.connection_class, 'factory', autospec=True, return_value='connection') as factory:
273+
with patch.object(Cluster.connection_class, 'factory', autospec=True, return_value=connection) as factory:
273274
cluster = Cluster(compression=configured)
274275
conn = cluster.connection_factory(endpoint)
275276

276-
assert conn == 'connection'
277+
assert conn is connection
277278
assert factory.call_count == 1
278279
assert factory.call_args.kwargs['compression'] == expected
279280
assert cluster.compression == expected
280281

282+
def test_reconnection_factory_rejects_startup_close(self):
283+
endpoint = Mock(address='127.0.0.1')
284+
host = Mock(endpoint=endpoint)
285+
connection = Mock(is_closed=False)
286+
287+
with patch.object(Cluster.connection_class, 'factory', autospec=True, return_value=connection) as factory:
288+
cluster = Cluster()
289+
conn_factory = cluster._make_connection_factory(host)
290+
conn = conn_factory()
291+
292+
assert conn is connection
293+
assert factory.call_count == 1
294+
295+
def test_connection_factory_rejects_startup_close(self):
296+
endpoint = Mock(address='127.0.0.1')
297+
connection = Mock(is_closed=True)
298+
connection.endpoint = endpoint
299+
300+
with patch.object(Cluster.connection_class, 'factory', autospec=True, return_value=connection):
301+
cluster = Cluster()
302+
303+
with pytest.raises(ConnectionShutdown) as exc_info:
304+
cluster.connection_factory(endpoint)
305+
306+
assert "closed during the startup handshake" in str(exc_info.value)
307+
308+
def test_reconnection_factory_rejects_startup_close_result(self):
309+
endpoint = Mock(address='127.0.0.1')
310+
host = Mock(endpoint=endpoint)
311+
connection = Mock(is_closed=True)
312+
connection.endpoint = endpoint
313+
314+
with patch.object(Cluster.connection_class, 'factory', autospec=True, return_value=connection):
315+
cluster = Cluster()
316+
conn_factory = cluster._make_connection_factory(host)
317+
318+
with pytest.raises(ConnectionShutdown) as exc_info:
319+
conn_factory()
320+
321+
assert "closed during the startup handshake" in str(exc_info.value)
322+
281323

282324
class SchedulerTest(unittest.TestCase):
283325
# TODO: this suite could be expanded; for now just adding a test covering a ticket

tests/unit/test_connection.py

Lines changed: 133 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,11 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414
import itertools
15+
import socket
1516
import unittest
1617
from io import BytesIO
1718
import time
18-
from threading import Lock
19+
from threading import Event, Lock, Thread
1920
from unittest.mock import Mock, ANY, call, patch
2021

2122
from cassandra import OperationTimedOut
@@ -383,6 +384,137 @@ def test_wait_for_responses_shutdown_includes_last_error(self):
383384
assert "already closed" in error_message
384385
assert "Bad file descriptor" in error_message
385386

387+
def test_factory_returns_maintenance_mode_startup_close(self):
388+
"""
389+
Maintenance mode accepts regular CQL sockets and closes them during
390+
startup. The low-level factory keeps that close observable while
391+
still tracking pool-owned startup connections for shutdown cleanup.
392+
"""
393+
394+
class MaintenanceModeCqlServer(object):
395+
def __init__(self):
396+
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
397+
self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
398+
self._sock.bind(('127.0.0.1', 0))
399+
self._sock.listen(1)
400+
self._sock.settimeout(2)
401+
self.port = self._sock.getsockname()[1]
402+
self.first_frame = b''
403+
self.ready = Event()
404+
self.received_frame = Event()
405+
self.error = None
406+
self.thread = Thread(target=self._run)
407+
self.thread.daemon = True
408+
self.thread.start()
409+
410+
def _run(self):
411+
self.ready.set()
412+
try:
413+
client, _ = self._sock.accept()
414+
with client:
415+
client.settimeout(2)
416+
while len(self.first_frame) < 9:
417+
chunk = client.recv(9 - len(self.first_frame))
418+
if not chunk:
419+
break
420+
self.first_frame += chunk
421+
except Exception as exc:
422+
self.error = exc
423+
finally:
424+
self.received_frame.set()
425+
426+
def close(self):
427+
self._sock.close()
428+
self.thread.join(2)
429+
430+
class MaintenanceModeConnection(Connection):
431+
def __init__(self, *args, **kwargs):
432+
super(MaintenanceModeConnection, self).__init__(*args, **kwargs)
433+
self._reader = None
434+
self._connect_socket()
435+
self._send_options_message()
436+
self._reader = Thread(target=self._read_until_server_closes)
437+
self._reader.daemon = True
438+
self._reader.start()
439+
440+
def push(self, data):
441+
self._socket.sendall(data)
442+
443+
def close(self):
444+
with self.lock:
445+
if self.is_closed:
446+
return
447+
self.is_closed = True
448+
449+
if self._socket:
450+
self._socket.close()
451+
452+
if not self.is_defunct:
453+
shutdown_error = ConnectionShutdown("Connection to %s was closed" % self.endpoint)
454+
self.error_all_requests(shutdown_error)
455+
if not self.connected_event.is_set():
456+
self.last_error = shutdown_error
457+
self.connected_event.set()
458+
459+
def _read_until_server_closes(self):
460+
try:
461+
while True:
462+
data = self._socket.recv(self.in_buffer_size)
463+
if not data:
464+
self.close()
465+
return
466+
self._iobuf.write(data)
467+
self.process_io_buffer()
468+
except socket.error as exc:
469+
if not self.is_closed:
470+
self.defunct(exc)
471+
472+
class PendingConnections(list):
473+
def __init__(self):
474+
super(PendingConnections, self).__init__()
475+
self.appended = []
476+
477+
def append(self, conn):
478+
self.appended.append(conn)
479+
super(PendingConnections, self).append(conn)
480+
481+
server = MaintenanceModeCqlServer()
482+
try:
483+
assert server.ready.wait(2)
484+
conn = MaintenanceModeConnection.factory(
485+
DefaultEndPoint('127.0.0.1', server.port), timeout=2)
486+
487+
assert conn.is_closed
488+
assert server.received_frame.wait(2)
489+
assert server.error is None
490+
assert len(server.first_frame) >= 5
491+
assert server.first_frame[4] == 0x05 # OPTIONS
492+
finally:
493+
server.close()
494+
495+
server = MaintenanceModeCqlServer()
496+
try:
497+
assert server.ready.wait(2)
498+
host_conn = Mock()
499+
host_conn.is_shutdown = False
500+
host_conn._pending_connections = PendingConnections()
501+
502+
conn = MaintenanceModeConnection.factory(
503+
DefaultEndPoint('127.0.0.1', server.port),
504+
timeout=2,
505+
host_conn=host_conn)
506+
507+
assert conn.is_closed
508+
assert server.received_frame.wait(2)
509+
assert server.error is None
510+
assert len(server.first_frame) >= 5
511+
assert server.first_frame[4] == 0x05 # OPTIONS
512+
assert host_conn._pending_connections == []
513+
assert len(host_conn._pending_connections.appended) == 1
514+
assert host_conn._pending_connections.appended[0] is conn
515+
finally:
516+
server.close()
517+
386518

387519
@patch('cassandra.connection.ConnectionHeartbeat._raise_if_stopped')
388520
class ConnectionHeartbeatTest(unittest.TestCase):

0 commit comments

Comments
 (0)