diff --git a/contents/pods-resource-model.py b/contents/pods-resource-model.py index 304dd4a..8f99a7c 100644 --- a/contents/pods-resource-model.py +++ b/contents/pods-resource-model.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import datetime import logging import sys import os @@ -16,80 +17,94 @@ log = logging.getLogger('kubernetes-model-source') -def nodeCollectData(pod, container, defaults, taglist, mappingList, boEmoticon): - tags = [] - tags.extend(taglist.split(',')) +def format_started_at(started): + # With _preload_content=False the API's startedAt arrives as an RFC 3339 string + # (e.g. "2024-06-15T10:30:00Z") rather than a datetime. Reformat it to the + # "YYYY-MM-DD HH:MM:SS" shape this attribute has always produced. + if not started: + return None + parsed = datetime.datetime.fromisoformat(started.replace('Z', '+00:00')) + return parsed.strftime("%Y-%m-%d %H:%M:%S") - status = pod.status.phase + +def nodeCollectData(pod, container, config): + # config carries the per-run options parsed once in main() (tags, mappings, + # defaults, emoticon flag, config file) so they are not re-parsed for every node. + tags = config['tags'] + boEmoticon = config['emoticon'] + + metadata = pod['metadata'] + pod_status = pod['status'] + container_name = container['name'] + pod_labels = metadata.get('labels') + + status = pod_status.get('phase') statusMessage = None startedAt = None terminated = False container_id = None - if pod.status.container_statuses: + container_statuses = pod_status.get('containerStatuses') + if container_statuses: log.info("------") - log.info("container-name:" + container.name) + log.info("container-name:" + container_name) - for statuses in pod.status.container_statuses: - log.info("pod-container-name:" + statuses.name) + for statuses in container_statuses: + log.info("pod-container-name:" + statuses['name']) - if container.name == statuses.name: - if statuses.state.running is not None: + if container_name == statuses['name']: + state = statuses.get('state') or {} + if state.get('running') is not None: status = "running" - if statuses.state.running.started_at: - startedAt = statuses.state.running.started_at.strftime( - "%Y-%m-%d %H:%M:%S" - ) + startedAt = format_started_at(state['running'].get('startedAt')) - if statuses.state.waiting is not None: + if state.get('waiting') is not None: status = "waiting" - if statuses.state.terminated is not None: + if state.get('terminated') is not None: terminated = True status = "terminated" - container_id = statuses.container_id + container_id = statuses.get('containerID') - if terminated is False and pod.status.conditions is not None: - for info in pod.status.conditions: - if info.status == 'False': - status = info.reason - statusMessage = info.message + if terminated is False and pod_status.get('conditions') is not None: + for info in pod_status['conditions']: + if info.get('status') == 'False': + status = info.get('reason') + statusMessage = info.get('message') labels = [] - if pod.metadata.labels: - for keys, values in pod.metadata.labels.items(): + if pod_labels: + for keys, values in pod_labels.items(): labels.append(keys + ":" + values) default_settings = { # kubernetes:config_file attribute are kept to avoid breaking existing k8s jobs depend on this configuration-override hack # This is just a temporary workaround solution and should be replaced by a layered configuration-override mechanism. - 'kubernetes:config_file': os.environ.get('RD_CONFIG_CONFIG_FILE'), - 'default:pod_id': pod.status.pod_ip, - 'default:host_id': pod.status.host_ip, + 'kubernetes:config_file': config['config_file'], + 'default:pod_id': pod_status.get('podIP'), + 'default:host_id': pod_status.get('hostIP'), 'default:started_at': startedAt, - 'default:name': pod.metadata.name, + 'default:name': metadata['name'], 'default:labels': ','.join(labels), - 'default:namespace': pod.metadata.namespace, - 'default:image': container.image, + 'default:namespace': metadata['namespace'], + 'default:image': container.get('image'), 'default:status': status, 'default:status_message': statusMessage, 'default:container_id': container_id, - 'default:container_name': container.name + 'default:container_name': container_name } - mappings = [] custom_attributes = {} # custom mapping attributes - if mappingList: - log.debug('Mapping: %s', mappingList) - mappings.extend(mappingList.split(',')) + if config['mappings']: + log.debug('Mapping: %s', config['mappings']) - for mapping in mappings: + for mapping in config['mappings']: mapping_array = dict(s.split('=', 1) for s in mapping.split()) for key, value in mapping_array.items(): @@ -107,13 +122,13 @@ def nodeCollectData(pod, container, defaults, taglist, mappingList, boEmoticon): # rundeck attributes data = default_settings - data['nodename'] = default_settings['default:name']+"-"+container.name + data['nodename'] = default_settings['default:name']+"-"+container_name data['hostname'] = default_settings['default:pod_id'] data['terminated'] = terminated # Add labels as its own map of node attributes. - if pod.metadata.labels is not None: - for key, value in pod.metadata.labels.items(): + if pod_labels is not None: + for key, value in pod_labels.items(): data['labels:' + key] = value emoticon = "" @@ -152,26 +167,39 @@ def nodeCollectData(pod, container, defaults, taglist, mappingList, boEmoticon): if custom_attributes: data = dict(list(data.items()) + list(custom_attributes.items())) - data.update(dict(token.split('=') for token in shlex.split(defaults))) + data.update(config['defaults']) return data -def collect_pods_from_api(namespace_filter, label_selector, field_selector): +def collect_pods_from_api(namespace_filter, label_selector, field_selector, use_cache=False): v1 = client.CoreV1Api() log.debug(label_selector) log.debug(field_selector) - kwargs = {'watch': False} + # _preload_content=False returns the raw HTTP response so the JSON can be parsed + # directly into plain dicts. This skips the client's per-object model + # deserialization, which dominates wall-clock time on large pod lists. + kwargs = {'watch': False, '_preload_content': False} + + # resource_version='0' lets the apiserver serve the list from its in-memory watch + # cache instead of a quorum read from etcd: much faster on large clusters and + # lighter on the control plane, at the cost of possibly-stale data. Opt-in. + if use_cache: + kwargs['resource_version'] = '0' + if label_selector: kwargs['label_selector'] = label_selector if field_selector: kwargs['field_selector'] = field_selector if namespace_filter: - return v1.list_namespaced_pod(namespace=namespace_filter, **kwargs) - return v1.list_pod_for_all_namespaces(**kwargs) + resp = v1.list_namespaced_pod(namespace=namespace_filter, **kwargs) + else: + resp = v1.list_pod_for_all_namespaces(**kwargs) + + return json.loads(resp.data).get('items', []) def main(): @@ -181,7 +209,7 @@ def main(): common.connect() - tags = os.environ.get('RD_CONFIG_TAGS') + tags = os.environ.get('RD_CONFIG_TAGS', '') mappingList = os.environ.get('RD_CONFIG_MAPPING') defaults = os.environ.get('RD_CONFIG_ATTRIBUTES') @@ -193,6 +221,10 @@ def main(): if os.environ.get('RD_CONFIG_EMOTICON') == 'true': boEmoticon = True + use_cache = False + if os.environ.get('RD_CONFIG_USE_CACHE') == 'true': + use_cache = True + field_selector = None if os.environ.get('RD_CONFIG_FIELD_SELECTOR'): field_selector = os.environ.get('RD_CONFIG_FIELD_SELECTOR') @@ -201,30 +233,44 @@ def main(): if os.environ.get('RD_CONFIG_NAMESPACE_FILTER'): namespace_filter = os.environ.get('RD_CONFIG_NAMESPACE_FILTER') + # Opt-in: exclude namespaces server-side via the field selector. Defaults to + # empty (no exclusion, no behavior change). Only applied to all-namespace + # queries; a specific Namespace already scopes the result. + exclude_namespaces = os.environ.get('RD_CONFIG_EXCLUDE_NAMESPACES', '') + if not namespace_filter and exclude_namespaces: + exclusions = ['metadata.namespace!=' + ns.strip() + for ns in exclude_namespaces.split(',') if ns.strip()] + if exclusions: + field_selector = ','.join([field_selector] + exclusions) if field_selector else ','.join(exclusions) + label_selector = None if os.environ.get('RD_CONFIG_LABEL_SELECTOR'): label_selector = os.environ.get('RD_CONFIG_LABEL_SELECTOR') + # Parse the per-node options once here rather than re-parsing the same config + # strings inside nodeCollectData for every container. + config = { + 'tags': tags.split(','), + 'mappings': mappingList.split(',') if mappingList else [], + 'defaults': dict(token.split('=') for token in shlex.split(defaults or '')), + 'emoticon': boEmoticon, + 'config_file': os.environ.get('RD_CONFIG_CONFIG_FILE'), + } + node_set = [] - ret = collect_pods_from_api(namespace_filter, label_selector, field_selector) + ret = collect_pods_from_api(namespace_filter, label_selector, field_selector, use_cache=use_cache) - for i in ret.items: - for container in i.spec.containers: + for i in ret: + for container in i['spec']['containers']: log.debug("%s\t%s\t%s\t%s", - i.status.pod_ip, - i.metadata.namespace, - i.metadata.name, - container.name) - - node_data = nodeCollectData(i, - container, - defaults, - tags, - mappingList, - boEmoticon - ) + i['status'].get('podIP'), + i['metadata']['namespace'], + i['metadata']['name'], + container['name']) + + node_data = nodeCollectData(i, container, config) if running is False: if node_data["terminated"] is False: @@ -234,7 +280,7 @@ def main(): if node_data["status"].lower() == "running": node_set.append(node_data) - print(json.dumps(node_set, indent=4, sort_keys=True)) + print(json.dumps(node_set, sort_keys=True)) if __name__ == '__main__': diff --git a/contents/tests/test_pods_resource_model.py b/contents/tests/test_pods_resource_model.py index 6af6591..2c88912 100644 --- a/contents/tests/test_pods_resource_model.py +++ b/contents/tests/test_pods_resource_model.py @@ -2,9 +2,9 @@ Unit tests for pods-resource-model.py functions. """ -import datetime import importlib import os +import shlex import sys import unittest from unittest.mock import MagicMock, patch @@ -13,55 +13,54 @@ # pods-resource-model.py has a hyphenated name, so use importlib sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..')) pods_resource_model = importlib.import_module('pods-resource-model') -nodeCollectData = pods_resource_model.nodeCollectData collect_pods_from_api = pods_resource_model.collect_pods_from_api main = pods_resource_model.main +# nodeCollectData now takes a config dict (parsed once in main) instead of flat config +# strings. Adapt the flat-argument call shape these tests use to the new signature. +_nodeCollectData = pods_resource_model.nodeCollectData + + +def nodeCollectData(pod, container, defaults, taglist, mappingList, boEmoticon): + config = { + 'tags': taglist.split(',') if taglist else [], + 'mappings': mappingList.split(',') if mappingList else [], + 'defaults': dict(token.split('=') for token in shlex.split(defaults or '')), + 'emoticon': boEmoticon, + 'config_file': os.environ.get('RD_CONFIG_CONFIG_FILE'), + } + return _nodeCollectData(pod, container, config) + + +# The resource model parses raw API JSON into plain dicts (camelCase keys), so +# fixtures build dicts that mirror the Kubernetes pod JSON rather than client objects. def make_container(name='app', image='nginx:latest'): - container = MagicMock() - container.name = name - container.image = image - return container + return {'name': name, 'image': image} def make_pod(name='my-pod', namespace='default', pod_ip='10.0.0.1', host_ip='192.168.1.1', phase='Running', labels=None, container_statuses=None, conditions=None): - pod = MagicMock() - pod.metadata.name = name - pod.metadata.namespace = namespace - pod.metadata.labels = labels - pod.status.pod_ip = pod_ip - pod.status.host_ip = host_ip - pod.status.phase = phase - pod.status.container_statuses = container_statuses - pod.status.conditions = conditions - return pod + status = {'phase': phase, 'podIP': pod_ip, 'hostIP': host_ip} + if container_statuses is not None: + status['containerStatuses'] = container_statuses + if conditions is not None: + status['conditions'] = conditions + return { + 'metadata': {'name': name, 'namespace': namespace, 'labels': labels}, + 'spec': {'containers': []}, + 'status': status, + } def make_container_status(name='app', running=True, started_at=None, waiting=False, terminated=False, container_id='docker://abc123'): - status = MagicMock() - status.name = name - status.container_id = container_id - - if running: - status.state.running = MagicMock() - status.state.running.started_at = started_at - else: - status.state.running = None - - if waiting: - status.state.waiting = MagicMock() - else: - status.state.waiting = None - - if terminated: - status.state.terminated = MagicMock() - else: - status.state.terminated = None - - return status + state = { + 'running': {'startedAt': started_at} if running else None, + 'waiting': {} if waiting else None, + 'terminated': {} if terminated else None, + } + return {'name': name, 'containerID': container_id, 'state': state} class TestNodeCollectData(unittest.TestCase): @@ -70,7 +69,7 @@ def setUp(self): os.environ.clear() def test_basic_running_pod(self): - started = datetime.datetime(2024, 6, 15, 10, 30, 0) + started = '2024-06-15T10:30:00Z' container = make_container() cs = make_container_status(name='app', running=True, started_at=started) pod = make_pod(container_statuses=[cs]) @@ -114,10 +113,11 @@ def test_no_container_statuses(self): def test_conditions_not_ready(self): container = make_container() - condition = MagicMock() - condition.status = 'False' - condition.reason = 'ContainersNotReady' - condition.message = 'containers not ready' + condition = { + 'status': 'False', + 'reason': 'ContainersNotReady', + 'message': 'containers not ready', + } pod = make_pod(container_statuses=None, conditions=[condition]) data = nodeCollectData(pod, container, '', 'kubernetes', None, False) @@ -181,10 +181,11 @@ def test_custom_mapping(self): def test_status_message_in_description(self): container = make_container() - condition = MagicMock() - condition.status = 'False' - condition.reason = 'ContainersNotReady' - condition.message = 'waiting for readiness' + condition = { + 'status': 'False', + 'reason': 'ContainersNotReady', + 'message': 'waiting for readiness', + } pod = make_pod(container_statuses=None, conditions=[condition]) data = nodeCollectData(pod, container, '', 'kubernetes', None, False) @@ -201,14 +202,23 @@ def test_config_file_env(self): class TestCollectPodsFromApi(unittest.TestCase): + @staticmethod + def _resp(payload='{"items": "result"}'): + # collect_pods_from_api requests the raw response (_preload_content=False) + # and parses resp.data as JSON, returning the "items" list. + resp = MagicMock() + resp.data = payload + return resp + @patch.object(pods_resource_model.client, 'CoreV1Api') def test_all_namespaces_both_selectors(self, mock_api_class): mock_api = mock_api_class.return_value - mock_api.list_pod_for_all_namespaces.return_value = 'result' + mock_api.list_pod_for_all_namespaces.return_value = self._resp() ret = collect_pods_from_api(None, 'app=web', 'status.phase=Running') mock_api.list_pod_for_all_namespaces.assert_called_once_with( watch=False, + _preload_content=False, field_selector='status.phase=Running', label_selector='app=web', ) @@ -217,11 +227,12 @@ def test_all_namespaces_both_selectors(self, mock_api_class): @patch.object(pods_resource_model.client, 'CoreV1Api') def test_all_namespaces_field_selector_only(self, mock_api_class): mock_api = mock_api_class.return_value - mock_api.list_pod_for_all_namespaces.return_value = 'result' + mock_api.list_pod_for_all_namespaces.return_value = self._resp() ret = collect_pods_from_api(None, None, 'status.phase=Running') mock_api.list_pod_for_all_namespaces.assert_called_once_with( watch=False, + _preload_content=False, field_selector='status.phase=Running', ) self.assertEqual('result', ret) @@ -229,11 +240,12 @@ def test_all_namespaces_field_selector_only(self, mock_api_class): @patch.object(pods_resource_model.client, 'CoreV1Api') def test_all_namespaces_label_selector_only(self, mock_api_class): mock_api = mock_api_class.return_value - mock_api.list_pod_for_all_namespaces.return_value = 'result' + mock_api.list_pod_for_all_namespaces.return_value = self._resp() ret = collect_pods_from_api(None, 'app=web', None) mock_api.list_pod_for_all_namespaces.assert_called_once_with( watch=False, + _preload_content=False, label_selector='app=web', ) self.assertEqual('result', ret) @@ -241,21 +253,25 @@ def test_all_namespaces_label_selector_only(self, mock_api_class): @patch.object(pods_resource_model.client, 'CoreV1Api') def test_all_namespaces_no_selectors(self, mock_api_class): mock_api = mock_api_class.return_value - mock_api.list_pod_for_all_namespaces.return_value = 'result' + mock_api.list_pod_for_all_namespaces.return_value = self._resp() ret = collect_pods_from_api(None, None, None) - mock_api.list_pod_for_all_namespaces.assert_called_once_with(watch=False) + mock_api.list_pod_for_all_namespaces.assert_called_once_with( + watch=False, + _preload_content=False, + ) self.assertEqual('result', ret) @patch.object(pods_resource_model.client, 'CoreV1Api') def test_namespaced(self, mock_api_class): mock_api = mock_api_class.return_value - mock_api.list_namespaced_pod.return_value = 'result' + mock_api.list_namespaced_pod.return_value = self._resp() ret = collect_pods_from_api('prod', 'app=web', 'status.phase=Running') mock_api.list_namespaced_pod.assert_called_once_with( namespace='prod', watch=False, + _preload_content=False, label_selector='app=web', field_selector='status.phase=Running', ) @@ -264,13 +280,36 @@ def test_namespaced(self, mock_api_class): @patch.object(pods_resource_model.client, 'CoreV1Api') def test_namespaced_no_selectors(self, mock_api_class): mock_api = mock_api_class.return_value - mock_api.list_namespaced_pod.return_value = 'result' + mock_api.list_namespaced_pod.return_value = self._resp() ret = collect_pods_from_api('default', None, None) mock_api.list_namespaced_pod.assert_called_once_with( namespace='default', watch=False, + _preload_content=False, ) + self.assertEqual('result', ret) + + @patch.object(pods_resource_model.client, 'CoreV1Api') + def test_use_cache_sets_resource_version(self, mock_api_class): + mock_api = mock_api_class.return_value + mock_api.list_pod_for_all_namespaces.return_value = self._resp() + + collect_pods_from_api(None, None, None, use_cache=True) + mock_api.list_pod_for_all_namespaces.assert_called_once_with( + watch=False, + _preload_content=False, + resource_version='0', + ) + + @patch.object(pods_resource_model.client, 'CoreV1Api') + def test_no_cache_omits_resource_version(self, mock_api_class): + mock_api = mock_api_class.return_value + mock_api.list_pod_for_all_namespaces.return_value = self._resp() + + collect_pods_from_api(None, None, None) + _, kwargs = mock_api.list_pod_for_all_namespaces.call_args + self.assertNotIn('resource_version', kwargs) class TestMain(unittest.TestCase): @@ -279,13 +318,12 @@ def setUp(self): os.environ.clear() def _make_pod_list(self, pods): - ret = MagicMock() + # collect_pods_from_api returns a plain list of pod dicts. items = [] for pod, containers in pods: - pod.spec.containers = containers + pod['spec']['containers'] = containers items.append(pod) - ret.items = items - return ret + return items @patch.object(pods_resource_model, 'collect_pods_from_api') @patch.object(pods_resource_model.common, 'connect') @@ -353,12 +391,58 @@ def test_main_passes_env_to_collect(self, mock_connect, mock_collect): os.environ['RD_CONFIG_LABEL_SELECTOR'] = 'app=web' os.environ['RD_CONFIG_FIELD_SELECTOR'] = 'status.phase=Running' - mock_collect.return_value = MagicMock(items=[]) + mock_collect.return_value = [] + + with patch('builtins.print'): + main() + + mock_collect.assert_called_once_with('prod', 'app=web', 'status.phase=Running', use_cache=False) + + @patch.object(pods_resource_model, 'collect_pods_from_api') + @patch.object(pods_resource_model.common, 'connect') + def test_main_use_cache_flag(self, mock_connect, mock_collect): + os.environ['RD_CONFIG_TAGS'] = 'kubernetes' + os.environ['RD_CONFIG_ATTRIBUTES'] = '' + os.environ['RD_CONFIG_USE_CACHE'] = 'true' + + mock_collect.return_value = [] + + with patch('builtins.print'): + main() + + _, kwargs = mock_collect.call_args + self.assertTrue(kwargs['use_cache']) + + @patch.object(pods_resource_model, 'collect_pods_from_api') + @patch.object(pods_resource_model.common, 'connect') + def test_main_excludes_namespaces_when_configured(self, mock_connect, mock_collect): + os.environ['RD_CONFIG_TAGS'] = 'kubernetes' + os.environ['RD_CONFIG_ATTRIBUTES'] = '' + os.environ['RD_CONFIG_EXCLUDE_NAMESPACES'] = 'kube-system, kube-public' + + mock_collect.return_value = [] + + with patch('builtins.print'): + main() + + _, _, field_selector = mock_collect.call_args[0] + self.assertIn('metadata.namespace!=kube-system', field_selector) + self.assertIn('metadata.namespace!=kube-public', field_selector) + + @patch.object(pods_resource_model, 'collect_pods_from_api') + @patch.object(pods_resource_model.common, 'connect') + def test_main_no_exclusion_by_default(self, mock_connect, mock_collect): + os.environ['RD_CONFIG_TAGS'] = 'kubernetes' + os.environ['RD_CONFIG_ATTRIBUTES'] = '' + + mock_collect.return_value = [] with patch('builtins.print'): main() - mock_collect.assert_called_once_with('prod', 'app=web', 'status.phase=Running') + # No RD_CONFIG_EXCLUDE_NAMESPACES set -> field_selector stays None (no change). + _, _, field_selector = mock_collect.call_args[0] + self.assertIsNone(field_selector) @patch.object(pods_resource_model, 'collect_pods_from_api') @patch.object(pods_resource_model.common, 'connect') @@ -411,7 +495,7 @@ def test_main_empty_pod_list(self, mock_connect, mock_collect): os.environ['RD_CONFIG_TAGS'] = 'kubernetes' os.environ['RD_CONFIG_ATTRIBUTES'] = '' - mock_collect.return_value = MagicMock(items=[]) + mock_collect.return_value = [] with patch('builtins.print') as mock_print: main() @@ -422,4 +506,4 @@ def test_main_empty_pod_list(self, mock_connect, mock_collect): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/plugin.yaml b/plugin.yaml index 2f7d3c6..fbe1f46 100644 --- a/plugin.yaml +++ b/plugin.yaml @@ -45,6 +45,21 @@ providers: default: '' renderingOptions: groupName: Config + - name: exclude_namespaces + type: String + title: "Exclude Namespaces" + description: "Comma-separated namespaces to exclude server-side, only applied when no Namespace is set. Empty (the default) excludes nothing. Example: kube-system,kube-public,kube-node-lease" + required: false + default: '' + renderingOptions: + groupName: Config + - type: Boolean + name: use_cache + title: "Use API Cache?" + description: "Serve the pod list from the apiserver's in-memory watch cache (resource_version=0) instead of a quorum read from etcd. Faster on large clusters, but data may be slightly stale. Default false." + default: "false" + renderingOptions: + groupName: Config - name: field_selector type: String title: "Field Selector"