Lazy-import novaclient and keystoneauth1 SDK modules

Move novaclient and keystoneauth1 imports from module level into the
functions that use them (nova_login, _server_evacuate,
_handle_nova_exception, main). Eliminates ~135 lines of mock exception
boilerplate across 4 test files. Add shared patch_pipeline() fixture
in conftest.py replacing 7-deep nested patch blocks in process_service
tests. Bind conftest sys.modules submodule mocks to parent mock
attributes so lazy imports and patch() targets resolve to the same
objects.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: lmiccini

templates/instanceha/bin/instanceha.py

@@ -28,13 +28,6 @@
 from collections import defaultdict
 from abc import ABC, abstractmethod
 
-from novaclient import client
-from novaclient.exceptions import Conflict, NotFound, Forbidden, Unauthorized
-
-from keystoneauth1 import loading
-from keystoneauth1 import session as ksc_session
-from keystoneauth1.exceptions.discovery import DiscoveryFailure
-
 from prometheus_client import Counter, Gauge, Histogram, generate_latest, CONTENT_TYPE_LATEST
 
 
@@ -1761,6 +1754,7 @@ def _host_evacuate(connection, failed_service, service, target_host=None) -> boo
 
 def _server_evacuate(connection, server, target_host=None, server_obj=None) -> EvacuationResult:
     """Evacuate a single server instance, returning EvacuationResult."""
+    from novaclient.exceptions import Conflict, NotFound, Forbidden, Unauthorized
     success = False
     error_message = ""
     resp_status_code = None
@@ -2017,6 +2011,11 @@ def _server_evacuate_future(connection, server, target_host=None,
 def _nova_login(plugin_name: str, auth_kwargs: dict, region_name: str,
                 ca_bundle: Optional[str] = None) -> OpenStackClient:
     """Create and return Nova client connection. Raises NovaConnectionError on failure."""
+    from keystoneauth1 import loading
+    from keystoneauth1 import session as ksc_session
+    from keystoneauth1.exceptions.discovery import DiscoveryFailure
+    from novaclient import client
+    from novaclient.exceptions import Unauthorized
     try:
         loader = loading.get_plugin_loader(plugin_name)
         auth = loader.load_from_options(**auth_kwargs)
@@ -2055,15 +2054,14 @@ def nova_login_ac(credentials: ACLoginCredentials, ca_bundle: Optional[str] = No
     }, credentials.region_name, ca_bundle)
 
 
-NOVA_EXCEPTION_MESSAGES = {
-    NotFound: "Resource not found",
-    Conflict: "Conflicting operation",
-}
-
-
 def _handle_nova_exception(operation: str, service_info: str, e: Exception, is_critical: bool = True) -> bool:
     """Handle Nova API exceptions with consistent logging."""
-    error_msg = NOVA_EXCEPTION_MESSAGES.get(type(e))
+    from novaclient.exceptions import NotFound, Conflict
+    nova_exception_messages = {
+        NotFound: "Resource not found",
+        Conflict: "Conflicting operation",
+    }
+    error_msg = nova_exception_messages.get(type(e))
 
     if error_msg:
         logging.error("Failed to %s for %s. %s: %s", operation, service_info, error_msg, type(e).__name__)
@@ -3558,6 +3556,9 @@ def _sigterm_handler(signum, frame):
 
     consecutive_failures = 0
 
+    from novaclient.exceptions import Unauthorized
+    from keystoneauth1.exceptions.discovery import DiscoveryFailure
+
     while not service.shutdown_event.is_set():
         service.update_health_hash()
         poll_interval = service.config.get_config_value('POLL')

test/instanceha/conftest.py

@@ -6,14 +6,18 @@
 
 import os
 import sys
-from unittest.mock import MagicMock
+from contextlib import contextmanager
+from unittest.mock import MagicMock, patch
 
 
 # --- Mock OpenStack dependencies before any test file imports instanceha ---
 
 if 'novaclient' not in sys.modules:
-    sys.modules['novaclient'] = MagicMock()
-    sys.modules['novaclient.client'] = MagicMock()
+    _novaclient_mock = MagicMock()
+    _novaclient_client_mock = MagicMock()
+    _novaclient_mock.client = _novaclient_client_mock
+    sys.modules['novaclient'] = _novaclient_mock
+    sys.modules['novaclient.client'] = _novaclient_client_mock
 
     class NotFound(Exception):
         pass
@@ -32,12 +36,18 @@ class Unauthorized(Exception):
     novaclient_exceptions.Conflict = Conflict
     novaclient_exceptions.Forbidden = Forbidden
     novaclient_exceptions.Unauthorized = Unauthorized
+    _novaclient_mock.exceptions = novaclient_exceptions
     sys.modules['novaclient.exceptions'] = novaclient_exceptions
 
 if 'keystoneauth1' not in sys.modules:
-    sys.modules['keystoneauth1'] = MagicMock()
-    sys.modules['keystoneauth1.loading'] = MagicMock()
-    sys.modules['keystoneauth1.session'] = MagicMock()
+    _keystoneauth_mock = MagicMock()
+    _loading_mock = MagicMock()
+    _session_mock = MagicMock()
+    _keystoneauth_mock.loading = _loading_mock
+    _keystoneauth_mock.session = _session_mock
+    sys.modules['keystoneauth1'] = _keystoneauth_mock
+    sys.modules['keystoneauth1.loading'] = _loading_mock
+    sys.modules['keystoneauth1.session'] = _session_mock
 
     class DiscoveryFailure(Exception):
         pass
@@ -48,6 +58,7 @@ class DiscoveryFailure(Exception):
     exceptions_module = MagicMock()
     exceptions_module.discovery = discovery_module
 
+    _keystoneauth_mock.exceptions = exceptions_module
     sys.modules['keystoneauth1.exceptions'] = exceptions_module
     sys.modules['keystoneauth1.exceptions.discovery'] = discovery_module
 
@@ -59,3 +70,32 @@ class DiscoveryFailure(Exception):
 _abs_path = os.path.abspath(_instanceha_path)
 if _abs_path not in sys.path:
     sys.path.insert(0, _abs_path)
+
+
+# --- Shared test fixtures ---
+
+import instanceha  # noqa: E402
+
+
+@contextmanager
+def patch_pipeline(conn=None, fence=True, disable=True,
+                   reserved=None, evacuate=True, recovery=True):
+    """Patch all process_service pipeline steps with configurable return values.
+
+    Yields a dict of mock objects keyed by step name, so tests can assert
+    on individual steps (e.g. mocks['disable'].assert_not_called()).
+    """
+    if reserved is None:
+        reserved = instanceha.ReservedHostResult(success=True, hostname=None)
+    with patch('instanceha._get_nova_connection', return_value=conn) as m_conn, \
+         patch('instanceha._host_fence', return_value=fence) as m_fence, \
+         patch('instanceha._host_disable', return_value=disable) as m_disable, \
+         patch('instanceha._manage_reserved_hosts', return_value=reserved) as m_reserved, \
+         patch('instanceha._host_evacuate', return_value=evacuate) as m_evacuate, \
+         patch('instanceha._post_evacuation_recovery', return_value=recovery) as m_recovery, \
+         patch('instanceha.track_host_processing') as m_track:
+        yield {
+            'conn': m_conn, 'fence': m_fence, 'disable': m_disable,
+            'reserved': m_reserved, 'evacuate': m_evacuate,
+            'recovery': m_recovery, 'track': m_track,
+        }

test/instanceha/test_coverage_gaps.py

@@ -761,10 +761,6 @@ def test_timeout(self):
 # Main loop backoff tests
 # ============================================================================
 
-class _TestDiscoveryFailure(Exception):
-    pass
-
-
 class TestMainLoopBackoff(unittest.TestCase):
     """Tests for main() exponential backoff on Nova failures."""
 
@@ -782,12 +778,13 @@ def _make_service_mock(self):
         svc.shutdown_event = threading.Event()
         return svc
 
-    @patch('instanceha.DiscoveryFailure', _TestDiscoveryFailure)
     @patch('instanceha.signal.signal')
     @patch('instanceha._establish_nova_connection')
     @patch('instanceha._initialize_service')
     @patch('instanceha.ConfigManager')
     def test_unauthorized_triggers_reconnection(self, mock_cm, mock_init, mock_conn, mock_signal):
+        from novaclient.exceptions import Unauthorized
+
         mock_cm.return_value.get_config_value = Mock(return_value='INFO')
         service = self._make_service_mock()
         mock_init.return_value = service
@@ -799,14 +796,13 @@ def unauthorized_then_stop(*args, **kwargs):
             call_count[0] += 1
             if call_count[0] >= 2:
                 service.shutdown_event.set()
-            raise instanceha.Unauthorized('expired')
+            raise Unauthorized('expired')
         conn.services.list.side_effect = unauthorized_then_stop
 
         instanceha.main()
 
         mock_conn.assert_called()
 
-    @patch('instanceha.DiscoveryFailure', _TestDiscoveryFailure)
     @patch('instanceha.signal.signal')
     @patch('instanceha._establish_nova_connection')
     @patch('instanceha._initialize_service')
@@ -830,7 +826,6 @@ def error_then_stop(*args, **kwargs):
 
         self.assertGreaterEqual(call_count[0], 2)
 
-    @patch('instanceha.DiscoveryFailure', _TestDiscoveryFailure)
     @patch('instanceha.signal.signal')
     @patch('instanceha._process_reenabling')
     @patch('instanceha._process_stale_services')
@@ -860,7 +855,6 @@ def services_then_stop(*args, **kwargs):
 
         mock_cat.assert_called()
 
-    @patch('instanceha.DiscoveryFailure', _TestDiscoveryFailure)
     @patch('instanceha.signal.signal')
     @patch('instanceha._establish_nova_connection')
     @patch('instanceha._initialize_service')

test/instanceha/test_critical_error_paths.py

@@ -178,12 +178,12 @@ def test_nova_login_discovery_failure(self):
         """Test Nova login with discovery failure."""
         from keystoneauth1.exceptions.discovery import DiscoveryFailure
 
-        with patch('instanceha.loading.get_plugin_loader') as mock_loader:
+        with patch('keystoneauth1.loading.get_plugin_loader') as mock_loader:
             mock_auth = Mock()
             mock_loader.return_value.load_from_options.return_value = mock_auth
 
-            with patch('instanceha.ksc_session.Session') as mock_session:
-                with patch('instanceha.client.Client') as mock_client:
+            with patch('keystoneauth1.session.Session') as mock_session:
+                with patch('novaclient.client.Client') as mock_client:
                     mock_nova = Mock()
                     mock_nova.versions.get_current.side_effect = DiscoveryFailure('Discovery failed')
                     mock_client.return_value = mock_nova
@@ -200,12 +200,12 @@ def test_nova_login_unauthorized(self):
         """Test Nova login with unauthorized exception."""
         from novaclient.exceptions import Unauthorized
 
-        with patch('instanceha.loading.get_plugin_loader') as mock_loader:
+        with patch('keystoneauth1.loading.get_plugin_loader') as mock_loader:
             mock_auth = Mock()
             mock_loader.return_value.load_from_options.return_value = mock_auth
 
-            with patch('instanceha.ksc_session.Session') as mock_session:
-                with patch('instanceha.client.Client') as mock_client:
+            with patch('keystoneauth1.session.Session') as mock_session:
+                with patch('novaclient.client.Client') as mock_client:
                     mock_nova = Mock()
                     mock_nova.versions.get_current.side_effect = Unauthorized('Bad credentials')
                     mock_client.return_value = mock_nova