Merge pull request #16 from cisco-en-programmability/main

Changes from mainline

Author: madhansansel

plugins/modules/provision_playbook_config_generator.py

@@ -296,6 +296,9 @@ def __init__(self, module):
         self.module_schema = self.provision_workflow_manager_mapping()
         self.log("Initialized ProvisionPlaybookGenerator class instance.", "DEBUG")
         self.site_id_name_dict = self.get_site_id_name_mapping()
+        self._cached_provisioned_devices = None
+        self._cached_inventory_devices = None
+        self._cached_inventory_by_id = None
 
     def get_site_id_name_mapping(self):
         """
@@ -324,6 +327,30 @@ def get_site_id_name_mapping(self):
 
         return site_id_name_mapping
 
+    def get_inventory_devices(self):
+        """
+        Retrieve inventory devices once and reuse them throughout the current run.
+
+        Returns:
+            list: Inventory device records from Catalyst Center.
+        """
+        if self._cached_inventory_devices is not None:
+            return self._cached_inventory_devices
+
+        all_devices_response = self.catalystcenter._exec(
+            family="devices",
+            function="get_device_list",
+            op_modifies=False,
+        )
+        self.log("Received API response: {0}".format(all_devices_response), "DEBUG")
+        self._cached_inventory_devices = all_devices_response.get("response", [])
+        self._cached_inventory_by_id = {
+            device.get("id"): device
+            for device in self._cached_inventory_devices
+            if device.get("id")
+        }
+        return self._cached_inventory_devices
+
     def write_dict_to_yaml(self, data_dict, file_path, dumper=OrderedDumper, file_mode="overwrite"):
         """
         Converts a dictionary to YAML format and writes it to a specified file path.
@@ -916,6 +943,10 @@ def get_all_provisioned_devices_internal(self):
             list: List of all provisioned devices
         """
         self.log("Retrieving all provisioned devices from SDA API", "INFO")
+        if self._cached_provisioned_devices is not None:
+            self.log("Using cached provisioned devices list", "DEBUG")
+            return self._cached_provisioned_devices
+
         try:
             # Get all provisioned devices from SDA API
             response = self.catalystcenter._exec(
@@ -928,63 +959,68 @@ def get_all_provisioned_devices_internal(self):
             self.log("Retrieved {0} devices from SDA provisioned devices API".format(len(sda_devices)), "INFO")
 
             # WORKAROUND: Check for missing wireless controllers
-            all_devices_response = self.catalystcenter._exec(
-                family="devices",
-                function="get_device_list",
-                op_modifies=False,
-            )
-            self.log("Received API response: {0}".format(all_devices_response), "DEBUG")
-            all_devices = all_devices_response.get("response", [])
+            all_devices = self.get_inventory_devices()
+            inventory_by_id = self._cached_inventory_by_id or {}
 
             wireless_controllers_found = []
             sda_device_ids = {device.get("networkDeviceId") for device in sda_devices}
 
+            for sda_device in sda_devices:
+                inventory_device = inventory_by_id.get(sda_device.get("networkDeviceId"))
+                if inventory_device:
+                    sda_device.update(
+                        {
+                            "id": inventory_device.get("id"),
+                            "managementIpAddress": inventory_device.get("managementIpAddress"),
+                            "family": inventory_device.get("family"),
+                            "type": inventory_device.get("type"),
+                            "hostname": inventory_device.get("hostname"),
+                            "location": inventory_device.get("location"),
+                            "siteId": sda_device.get("siteId") or inventory_device.get("siteId"),
+                        }
+                    )
+
             for device in all_devices:
                 device_id = device.get("id")
                 management_ip = device.get("managementIpAddress")
 
                 if device_id in sda_device_ids:
                     continue
 
-                try:
-                    device_detail_response = self.catalystcenter._exec(
-                        family="devices",
-                        function="get_device_detail",
-                        op_modifies=False,
-                        params={"search_by": device_id, "identifier": "uuid"},
-                    )
-                    self.log("Received API response: {0}".format(device_detail_response), "DEBUG")
-                    device_info = device_detail_response.get("response", {})
-                    device_family = device_info.get("nwDeviceFamily")
+                device_family = device.get("family") or device.get("nwDeviceFamily")
 
-                    if device_family == "Wireless Controller":
-                        try:
-                            provision_response = self.catalystcenter._exec(
-                                family="sda",
-                                function="get_provisioned_wired_device",
-                                op_modifies=False,
-                                params={"device_management_ip_address": management_ip}
-                            )
-                            self.log("Received API response: {0}".format(provision_response), "DEBUG")
-                            if provision_response.get("status") == "success":
-                                mock_device = {
-                                    "networkDeviceId": device_id,
-                                    "siteId": device.get("siteId"),
-                                    "deviceType": "WirelessController"
-                                }
-                                sda_devices.append(mock_device)
-                                wireless_controllers_found.append(management_ip)
-                        except Exception:
-                            self.log("Wireless controller with IP {0} not provisioned in SDA".format(management_ip), "WARNING")
-                            pass
-                except Exception:
-                    self.log("Could not retrieve details for device ID {0}".format(device_id), "WARNING")
-                    pass
+                if device_family == "Wireless Controller":
+                    try:
+                        provision_response = self.catalystcenter._exec(
+                            family="sda",
+                            function="get_provisioned_wired_device",
+                            op_modifies=False,
+                            params={"device_management_ip_address": management_ip}
+                        )
+                        self.log("Received API response: {0}".format(provision_response), "DEBUG")
+                        if provision_response.get("status") == "success":
+                            mock_device = {
+                                "networkDeviceId": device_id,
+                                "siteId": device.get("siteId"),
+                                "deviceType": "WirelessController",
+                                "id": device.get("id"),
+                                "managementIpAddress": management_ip,
+                                "family": device_family,
+                                "type": device.get("type"),
+                                "hostname": device.get("hostname"),
+                                "location": device.get("location"),
+                            }
+                            sda_devices.append(mock_device)
+                            wireless_controllers_found.append(management_ip)
+                    except Exception:
+                        self.log("Wireless controller with IP {0} not provisioned in SDA".format(management_ip), "WARNING")
+                        pass
 
             self.log("Found {0} additional provisioned wireless controllers".format(
                 len(wireless_controllers_found)), "INFO")
 
-            return sda_devices
+            self._cached_provisioned_devices = sda_devices
+            return self._cached_provisioned_devices
 
         except Exception as e:
             self.log("Error retrieving provisioned devices: {0}".format(str(e)), "ERROR")
@@ -1209,6 +1245,11 @@ def transform_device_site_hierarchy(self, device_details):
             else:
                 self.log("WARNING: Site ID {0} not found in site mapping".format(site_id), "WARNING")
 
+        cached_location = device_details.get("location")
+        if cached_location and str(cached_location).strip():
+            self.log("Using cached location field for site hierarchy: {0}".format(cached_location), "DEBUG")
+            return cached_location
+
         # Fallback: If no siteId or mapping failed, get it from device detail API
         if not site_id or not site_name_hierarchy:
             self.log("Fallback: Getting site hierarchy from device detail API for device {0}".format(device_id), "DEBUG")
@@ -1285,6 +1326,10 @@ def transform_device_family_info(self, device_details):
         Returns:
             str: Device family type (e.g., 'Switches and Hubs', 'Wireless Controller').
         """
+        cached_family = device_details.get("nwDeviceFamily") or device_details.get("family")
+        if cached_family:
+            return cached_family
+
         device_id = device_details.get("networkDeviceId")
         self.log("Transforming device family info for device ID: {0}".format(device_id), "DEBUG")
 
@@ -1329,6 +1374,10 @@ def transform_device_management_ip(self, device_details):
         Returns:
             str: Device management IP address.
         """
+        cached_ip = device_details.get("managementIpAddress") or device_details.get("managementIpAddr")
+        if cached_ip:
+            return cached_ip
+
         self.log("Transforming device management IP for device: {0}".format(device_details.get("networkDeviceId")), "DEBUG")
 
         device_id = device_details.get("networkDeviceId")

tests/unit/modules/catalyst/test_provision_playbook_config_generator.py

@@ -18,7 +18,7 @@
 
 __metaclass__ = type
 
-from unittest.mock import patch
+from unittest.mock import Mock, patch
 
 from ansible_collections.cisco.catalystcenter.plugins.modules import provision_playbook_config_generator
 from .catalystcenter_module import TestCatalystModule, set_module_args, loadPlaybookData
@@ -190,17 +190,22 @@ def test_provision_playbook_config_generator_playbook_global_filters(self):
                 file_mode="overwrite",
             )
         )
-        result = self.execute_module(changed=True, failed=False)
         expected_file_path = (
             "/Users/syedkahm/ansible/catalystcenter/work/collections/ansible_collections/cisco/catalystcenter/"
             "playbooks/brownfield_provision_workflow_playbook.yml"
         )
+        with patch.object(
+            provision_playbook_config_generator.ProvisionPlaybookGenerator,
+            "write_dict_to_yaml",
+            return_value=True,
+        ):
+            result = self.execute_module(changed=True, failed=False)
         self.assertEqual(
             result.get("response"),
             {
                 "YAML config generation Task succeeded for module 'provision_workflow_manager'.": {
                     "file_path": expected_file_path,
-                    "devices_count": 6
+                    "devices_count": 7
                 }
             }
         )
@@ -264,7 +269,7 @@ def test_provision_playbook_config_generator_playbook_global_filters_default_fil
             {
                 "YAML config generation Task succeeded for module 'provision_workflow_manager'.": {
                     "file_path": default_file_path,
-                    "devices_count": 6,
+                    "devices_count": 7,
                 }
             }
         )
@@ -369,3 +374,115 @@ def test_apply_device_filters_matches_list_values(self):
             [device.get("management_ip_address") for device in filtered_devices],
             ["204.1.2.5", "204.1.2.6"],
         )
+
+    def test_get_all_provisioned_devices_internal_uses_cached_inventory_details(self):
+        generator = self.module.ProvisionPlaybookGenerator.__new__(
+            self.module.ProvisionPlaybookGenerator
+        )
+        generator.log = lambda *args, **kwargs: None
+        generator._cached_provisioned_devices = None
+        generator._cached_inventory_devices = None
+        generator._cached_inventory_by_id = None
+
+        exec_mock = Mock(side_effect=[
+            {
+                "response": [
+                    {
+                        "networkDeviceId": "dev-1",
+                        "siteId": "site-1",
+                    }
+                ]
+            },
+            {
+                "response": [
+                    {
+                        "id": "dev-1",
+                        "managementIpAddress": "10.10.10.1",
+                        "family": "Switches and Hubs",
+                        "type": "Cisco Catalyst 9300 Switch",
+                        "hostname": "edge-1",
+                        "location": "Global/USA/SAN JOSE/BLD23",
+                        "siteId": "site-1",
+                    },
+                    {
+                        "id": "dev-2",
+                        "managementIpAddress": "10.10.10.2",
+                        "family": "Wireless Controller",
+                        "type": "Cisco Catalyst 9800",
+                        "hostname": "wlc-1",
+                        "location": "Global/USA/SAN JOSE/BLD23",
+                        "siteId": "site-2",
+                    },
+                ]
+            },
+            {"status": "success"},
+        ])
+        generator.catalystcenter = Mock(_exec=exec_mock)
+
+        devices = generator.get_all_provisioned_devices_internal()
+        cached_devices = generator.get_all_provisioned_devices_internal()
+
+        self.assertIs(devices, cached_devices)
+        self.assertEqual(len(devices), 2)
+        self.assertEqual(devices[0]["managementIpAddress"], "10.10.10.1")
+        self.assertEqual(devices[0]["family"], "Switches and Hubs")
+        self.assertEqual(devices[1]["deviceType"], "WirelessController")
+        self.assertEqual(devices[1]["managementIpAddress"], "10.10.10.2")
+        self.assertEqual(exec_mock.call_count, 3)
+        called_functions = [call.kwargs["function"] for call in exec_mock.call_args_list]
+        self.assertEqual(
+            called_functions,
+            ["get_provisioned_devices", "get_device_list", "get_provisioned_wired_device"],
+        )
+
+    def test_transform_device_site_hierarchy_uses_cached_location(self):
+        generator = self.module.ProvisionPlaybookGenerator.__new__(
+            self.module.ProvisionPlaybookGenerator
+        )
+        generator.log = lambda *args, **kwargs: None
+        generator.site_id_name_dict = {}
+        generator.catalystcenter = Mock()
+
+        site_hierarchy = generator.transform_device_site_hierarchy(
+            {
+                "networkDeviceId": "dev-1",
+                "location": "Global/USA/SAN JOSE/BLD23",
+            }
+        )
+
+        self.assertEqual(site_hierarchy, "Global/USA/SAN JOSE/BLD23")
+        generator.catalystcenter._exec.assert_not_called()
+
+    def test_transform_device_family_info_prefers_cached_family(self):
+        generator = self.module.ProvisionPlaybookGenerator.__new__(
+            self.module.ProvisionPlaybookGenerator
+        )
+        generator.log = lambda *args, **kwargs: None
+        generator.catalystcenter = Mock()
+
+        device_family = generator.transform_device_family_info(
+            {
+                "networkDeviceId": "dev-1",
+                "family": "Wireless Controller",
+            }
+        )
+
+        self.assertEqual(device_family, "Wireless Controller")
+        generator.catalystcenter._exec.assert_not_called()
+
+    def test_transform_device_management_ip_prefers_cached_management_ip(self):
+        generator = self.module.ProvisionPlaybookGenerator.__new__(
+            self.module.ProvisionPlaybookGenerator
+        )
+        generator.log = lambda *args, **kwargs: None
+        generator.catalystcenter = Mock()
+
+        management_ip = generator.transform_device_management_ip(
+            {
+                "networkDeviceId": "dev-1",
+                "managementIpAddress": "10.10.10.1",
+            }
+        )
+
+        self.assertEqual(management_ip, "10.10.10.1")
+        generator.catalystcenter._exec.assert_not_called()