|
12 | 12 | from enum import Enum |
13 | 13 | import concurrent.futures |
14 | 14 | import requests |
15 | | -import requests.exceptions |
16 | 15 | import yaml |
17 | 16 | import subprocess |
18 | 17 | from yaml.loader import SafeLoader |
@@ -179,18 +178,9 @@ class MatchType(Enum): |
179 | 178 | ZONE = "zone" |
180 | 179 |
|
181 | 180 |
|
182 | | -class MigrationStatus(Enum): |
183 | | - """Migration status values.""" |
184 | | - COMPLETED = "completed" |
185 | | - DONE = "done" |
186 | | - ERROR = "error" |
187 | | - FAILED = "failed" |
188 | | - CANCELLED = "cancelled" |
189 | | - |
190 | | - |
191 | 181 | # Migration status constants for checking migration state |
192 | | -MIGRATION_STATUS_COMPLETED = [MigrationStatus.COMPLETED.value, MigrationStatus.DONE.value] |
193 | | -MIGRATION_STATUS_ERROR = [MigrationStatus.ERROR.value, MigrationStatus.FAILED.value, MigrationStatus.CANCELLED.value] |
| 182 | +MIGRATION_STATUS_COMPLETED = ["completed", "done"] |
| 183 | +MIGRATION_STATUS_ERROR = ["error", "failed", "cancelled"] |
194 | 184 |
|
195 | 185 |
|
196 | 186 | # Dataclasses |
@@ -312,8 +302,6 @@ def _retry_with_backoff(func, max_attempts, label, delay=FENCING_RETRY_DELAY_SEC |
312 | 302 | 'power_action': (['on', 'off', 'status', 'reset', 'cycle', 'soft', 'On', 'ForceOff', |
313 | 303 | 'GracefulShutdown', 'GracefulRestart', 'ForceRestart', 'Nmi', |
314 | 304 | 'ForceOn', 'PushPowerButton'], None), |
315 | | - 'ip_address': ('ip', None), # Special marker for IP address validation |
316 | | - 'port': ('port', None), # Special marker for port validation |
317 | 305 | 'username': (r'^[a-zA-Z0-9_-]{1,64}$', USERNAME_MAX_LENGTH), |
318 | 306 | 'hostname': (r'^[a-zA-Z0-9]([a-zA-Z0-9._-]*[a-zA-Z0-9])?$', USERNAME_MAX_LENGTH) |
319 | 307 | } |
@@ -768,7 +756,6 @@ def __init__(self, config_manager: ConfigManager, cloud_client: Optional[OpenSta |
768 | 756 | self.current_hash = "" |
769 | 757 | self.hash_update_successful = False |
770 | 758 | self._last_hash_time = 0 |
771 | | - self._previous_hash = "" |
772 | 759 | self.ready = False |
773 | 760 | self.k8s_api_reachable = False |
774 | 761 |
|
@@ -838,7 +825,6 @@ def update_health_hash(self, hash_interval: Optional[int] = None) -> None: |
838 | 825 | if current_timestamp - self._last_hash_time > hash_interval: |
839 | 826 | self.current_hash = hashlib.sha256(str(current_timestamp).encode()).hexdigest() |
840 | 827 | self.hash_update_successful = True |
841 | | - self._previous_hash = self.current_hash |
842 | 828 | self._last_hash_time = current_timestamp |
843 | 829 |
|
844 | 830 | def get_connection(self) -> Optional[OpenStackClient]: |
@@ -953,8 +939,7 @@ def _get_cached_evacuable(self, cache_attr: str, config_key: str, |
953 | 939 | try: |
954 | 940 | cache_data = fetch_func(connection) |
955 | 941 | except Exception as e: |
956 | | - logging.error("Failed to get evacuable resources (%s): %s", config_key, e) |
957 | | - logging.debug('Exception traceback:', exc_info=True) |
| 942 | + _safe_log_exception(f"Failed to get evacuable resources ({config_key})", e, include_traceback=True) |
958 | 943 | cache_data = [] |
959 | 944 | with self._cache_lock: |
960 | 945 | setattr(self, cache_attr, cache_data) |
@@ -1115,8 +1100,7 @@ def is_aggregate_evacuable(self, connection: OpenStackClient, host: str, aggrega |
1115 | 1100 | return any(host in agg.hosts for agg in aggregates |
1116 | 1101 | if self._is_resource_evacuable(agg, self.evacuable_tag, ['metadata'])) |
1117 | 1102 | except Exception as e: |
1118 | | - logging.error("Failed to check aggregate evacuability for %s: %s", host, e) |
1119 | | - logging.debug('Exception traceback:', exc_info=True) |
| 1103 | + _safe_log_exception(f"Failed to check aggregate evacuability for {host}", e, include_traceback=True) |
1120 | 1104 | return False |
1121 | 1105 |
|
1122 | 1106 | def get_hosts_with_servers_cached(self, connection, services): |
@@ -1290,15 +1274,7 @@ def do_GET(self): |
1290 | 1274 | except Exception as e: |
1291 | 1275 | logging.error('Health check server failed: %s', e) |
1292 | 1276 |
|
1293 | | - def _cleanup_dict_by_condition(self, dictionary: dict, condition_func: Callable[[Any, Any], bool], |
1294 | | - log_message: Optional[str] = None) -> int: |
1295 | | - """Remove dictionary entries matching condition function.""" |
1296 | | - to_remove = [k for k, v in dictionary.items() if condition_func(k, v)] |
1297 | | - for k in to_remove: |
1298 | | - del dictionary[k] |
1299 | | - if log_message: |
1300 | | - logging.debug(log_message.format(k)) |
1301 | | - return len(to_remove) |
| 1277 | + |
1302 | 1278 |
|
1303 | 1279 | def refresh_evacuable_cache(self, connection=None, force=False, cache_timeout=CACHE_TIMEOUT_SECONDS): |
1304 | 1280 | """Refresh evacuable flavors and images cache with intelligent timing.""" |
@@ -1551,8 +1527,7 @@ def _update_service_disable_reason(connection, host, service_id=None) -> bool: |
1551 | 1527 | logging.debug('Updated disabled reason for host %s after evacuation failure', host) |
1552 | 1528 | return True |
1553 | 1529 | except Exception as e: |
1554 | | - logging.error('Failed to update disable_reason for host %s. Error: %s', host, e) |
1555 | | - logging.debug('Exception traceback:', exc_info=True) |
| 1530 | + _safe_log_exception(f"Failed to update disable_reason for host {host}", e, include_traceback=True) |
1556 | 1531 | return False |
1557 | 1532 |
|
1558 | 1533 |
|
@@ -1899,9 +1874,7 @@ def _monitor_evacuation(connection, server_id, response_uuid, start_time, |
1899 | 1874 |
|
1900 | 1875 | except Exception as e: |
1901 | 1876 | api_error_count += 1 |
1902 | | - logging.error("Error checking evacuation status for %s: %s", |
1903 | | - response_uuid, _sanitize_message(str(e))) |
1904 | | - logging.debug('Exception traceback:', exc_info=True) |
| 1877 | + _safe_log_exception(f"Error checking evacuation status for {response_uuid}", e, include_traceback=True) |
1905 | 1878 | if api_error_count >= MAX_API_RETRIES: |
1906 | 1879 | logging.error("Too many API errors checking evacuation status for %s. Giving up.", |
1907 | 1880 | response_uuid) |
@@ -1993,9 +1966,7 @@ def _server_evacuate_future(connection, server, target_host=None, |
1993 | 1966 | return result |
1994 | 1967 |
|
1995 | 1968 | except Exception as e: |
1996 | | - logging.error("Unexpected error during evacuation of server %s: %s", |
1997 | | - server.id, _sanitize_message(str(e))) |
1998 | | - logging.debug('Exception traceback:', exc_info=True) |
| 1969 | + _safe_log_exception(f"Unexpected error during evacuation of server {server.id}", e, include_traceback=True) |
1999 | 1970 | _emit_k8s_event(source_host, 'InstanceEvacuationFailed', |
2000 | 1971 | f'Instance {server.id} evacuation error: {_sanitize_message(str(e))}', |
2001 | 1972 | event_type='Warning') |
@@ -2520,12 +2491,6 @@ def _host_fence(host, action, service): |
2520 | 2491 | logging.error("Invalid fence parameters: missing host") |
2521 | 2492 | return False |
2522 | 2493 |
|
2523 | | - # Validate action parameter at entry point |
2524 | | - if not validate_input(action, 'power_action', "host fencing"): |
2525 | | - logging.error("Invalid fence action: %s", action) |
2526 | | - return False |
2527 | | - |
2528 | | - # Additional check for fence-specific actions |
2529 | 2494 | if action not in ['on', 'off']: |
2530 | 2495 | logging.error("Unsupported fence action: %s (only 'on' and 'off' supported)", action) |
2531 | 2496 | return False |
@@ -2558,14 +2523,7 @@ def _host_fence(host, action, service): |
2558 | 2523 |
|
2559 | 2524 | def _get_nova_connection(service): |
2560 | 2525 | """Establish a connection to Nova using service configuration.""" |
2561 | | - try: |
2562 | | - return service.create_connection() |
2563 | | - except NovaConnectionError: |
2564 | | - logging.error("Nova connection failed") |
2565 | | - return None |
2566 | | - except Exception as e: |
2567 | | - _safe_log_exception("Failed to establish Nova connection", e) |
2568 | | - return None |
| 2526 | + return _establish_nova_connection(service, fatal=False) |
2569 | 2527 |
|
2570 | 2528 |
|
2571 | 2529 | def _enable_matching_reserved_host(conn, failed_service, reserved_hosts, service, |
@@ -3019,7 +2977,7 @@ def _categorize_services(services: List[Any], target_date: datetime) -> tuple: |
3019 | 2977 |
|
3020 | 2978 | return compute_nodes, resume, reenable |
3021 | 2979 |
|
3022 | | -def _check_critical_services(conn, services, compute_nodes): |
| 2980 | +def _check_critical_services(services): |
3023 | 2981 | """Check if critical Nova services are operational for evacuation.""" |
3024 | 2982 | # Check if at least one scheduler is up |
3025 | 2983 | schedulers = [s for s in services if s.binary == 'nova-scheduler'] |
@@ -3048,10 +3006,11 @@ def _filter_processing_hosts(service, compute_nodes, to_resume): |
3048 | 3006 |
|
3049 | 3007 | with service.processing_lock: |
3050 | 3008 | # Clean up expired processing entries |
3051 | | - service._cleanup_dict_by_condition( |
3052 | | - service.hosts_processing, |
3053 | | - lambda h, t: current_time - t > max_processing_time + MAX_PROCESSING_TIME_PADDING_SECONDS, |
3054 | | - 'Cleaned up expired processing entry for {}') |
| 3009 | + expired = [h for h, t in service.hosts_processing.items() |
| 3010 | + if current_time - t > max_processing_time + MAX_PROCESSING_TIME_PADDING_SECONDS] |
| 3011 | + for h in expired: |
| 3012 | + del service.hosts_processing[h] |
| 3013 | + logging.debug('Cleaned up expired processing entry for %s', h) |
3055 | 3014 |
|
3056 | 3015 | # Filter out hosts currently being processed |
3057 | 3016 | original_count = len(compute_nodes) |
@@ -3278,7 +3237,7 @@ def _process_stale_services(conn, service, services, compute_nodes, to_resume): |
3278 | 3237 | # Process evacuations |
3279 | 3238 | if not service.config.get_config_value('DISABLED'): |
3280 | 3239 | # Check if critical services are operational |
3281 | | - can_evacuate, error_msg = _check_critical_services(conn, services, compute_nodes) |
| 3240 | + can_evacuate, error_msg = _check_critical_services(services) |
3282 | 3241 | if not can_evacuate: |
3283 | 3242 | logging.error('Cannot evacuate: %s. Skipping evacuation.', error_msg) |
3284 | 3243 | _cleanup_filtered_hosts(service, marked_hostnames, set(), current_time) |
|
0 commit comments