keep repeating discover_nodes until all registered nodes are online

plugwise_usb/constants.py

@@ -40,6 +40,9 @@
 # In bigger networks a response from a Node could take up a while, so lets use 15 seconds.
 NODE_TIME_OUT: Final = 15
 
+# Retry delay discover nodes
+NODE_DISCOVER_INTERVAL = 60
+
 MAX_RETRIES: Final = 3
 SUPPRESS_INITIALIZATION_WARNINGS: Final = 10  # Minutes to suppress (expected) communication warning messages after initialization
 

plugwise_usb/network/__init__.py

@@ -72,6 +72,9 @@
         self._unsubscribe_node_rejoin: Callable[[], None] | None = None
 
         self._discover_sed_tasks: dict[str, Task[bool]] = {}
+        self._registry_stragglers: dict[int, str] = {}
+        self._discover_stragglers_task: Task[None] | None = None
+        self._load_stragglers_task: Task[None] | None = None
 
     # region - Properties
 
@@ -338,7 +341,7 @@
     # endregion
 
     # region - Nodes
-    def _create_node_object(
+    async def _create_node_object(
         self,
         mac: str,
         address: int,
@@ -363,7 +366,7 @@
             return
         self._nodes[mac] = node
         _LOGGER.debug("%s node %s added", node.__class__.__name__, mac)
-        self._register.update_network_registration(address, mac, node_type)
+        await self._register.update_network_registration(address, mac, node_type)
 
         if self._cache_enabled:
             _LOGGER.debug(
@@ -404,22 +407,24 @@
 
         Return True if discovery succeeded.
         """
-        _LOGGER.debug("Start discovery of node %s ", mac)
+        _LOGGER.debug(
+            "Start discovery of node %s with NodeType %s", mac, str(node_type)
+        )
         if self._nodes.get(mac) is not None:
             _LOGGER.debug("Skip discovery of already known node %s ", mac)
             return True
 
         if node_type is not None:
-            self._create_node_object(mac, address, node_type)
+            await self._create_node_object(mac, address, node_type)
             await self._notify_node_event_subscribers(NodeEvent.DISCOVERED, mac)
             return True
 
         # Node type is unknown, so we need to discover it first
-        _LOGGER.debug("Starting the discovery of node %s", mac)
+        _LOGGER.debug("Starting the discovery of node %s with unknown NodeType", mac)
         node_info, node_ping = await self._controller.get_node_details(mac, ping_first)
         if node_info is None:
             return False
-        self._create_node_object(mac, address, node_info.node_type)
+        await self._create_node_object(mac, address, node_info.node_type)
 
         # Forward received NodeInfoResponse message to node
         await self._nodes[mac].message_for_node(node_info)
@@ -431,15 +436,42 @@
     async def _discover_registered_nodes(self) -> None:
         """Discover nodes."""
         _LOGGER.debug("Start discovery of registered nodes")
-        counter = 0
+        registered_counter = 0
         for address, registration in self._register.registry.items():
             mac, node_type = registration
             if mac != "":
                 if self._nodes.get(mac) is None:
-                    await self._discover_node(address, mac, node_type)
-                counter += 1
+                    if not await self._discover_node(address, mac, node_type):
+                        self._registry_stragglers[address] = mac
+                registered_counter += 1
                 await sleep(0)
-        _LOGGER.debug("Total %s registered node(s)", str(counter))
+        if len(self._registry_stragglers) > 0:
+            if (
+                self._discover_stragglers_task is None
+                or self._discover_stragglers_task.done()
+            ):
+                self._discover_stragglers_task = create_task(
+                    self._discover_stragglers()
+                )
+        _LOGGER.debug(
+            "Total %s online of %s registered node(s)",
+            str(len(self._nodes)),
+            str(registered_counter),
+        )
+
+    async def _discover_stragglers(self) -> None:
+        """Repeat Discovery of Nodes with unknown NodeType."""
+        while len(self._registry_stragglers) > 0:
+            await sleep(60)
+            stragglers: dict[int, str] = {}
+            for address, mac in self._registry_stragglers.items():
+                if not await self._discover_node(address, mac, None):
+                    stragglers[address] = mac
+            self._registry_stragglers = stragglers
+            _LOGGER.debug(
+                "Total %s nodes unreachable having unknown NodeType",
+                str(len(stragglers)),
+            )
 
     async def _load_node(self, mac: str) -> bool:
         """Load node."""
@@ -452,6 +484,12 @@
             return True
         return False
 
+    async def _load_stragglers(self) -> None:
+        """Retry failed load operation."""
+        await sleep(60)
+        while not self._load_discovered_nodes():
+            await sleep(60)
+
     async def _load_discovered_nodes(self) -> bool:
         """Load all nodes currently discovered."""
         _LOGGER.debug("_load_discovered_nodes | START | %s", len(self._nodes))
@@ -499,10 +537,11 @@
         await self.discover_network_coordinator(load=load)
         if not self._is_running:
             await self.start()
-
         await self._discover_registered_nodes()
         if load:
-            return await self._load_discovered_nodes()
+            if not await self._load_discovered_nodes():
+                self._load_stragglers_task = create_task(self._load_stragglers())
+                return False
 
         return True
 

plugwise_usb/network/cache.py

@@ -5,11 +5,10 @@
 import logging
 
 from ..api import NodeType
-from ..constants import CACHE_DATA_SEPARATOR
 from ..helpers.cache import PlugwiseCache
 
 _LOGGER = logging.getLogger(__name__)
-_NETWORK_CACHE_FILE_NAME = "nodes.cache"
+_NETWORK_CACHE_FILE_NAME = "nodetype.cache"
 
 
 class NetworkRegistrationCache(PlugwiseCache):
@@ -18,68 +17,58 @@
     def __init__(self, cache_root_dir: str = "") -> None:
         """Initialize NetworkCache class."""
         super().__init__(_NETWORK_CACHE_FILE_NAME, cache_root_dir)
-        self._registrations: dict[int, tuple[str, NodeType | None]] = {}
+        self._nodetypes: dict[str, NodeType] = {}
 
     @property
-    def registrations(self) -> dict[int, tuple[str, NodeType | None]]:
+    def nodetypes(self) -> dict[str, NodeType]:
         """Cached network information."""
-        return self._registrations
+        return self._nodetypes
 
     async def save_cache(self) -> None:
         """Save the node information to file."""
         cache_data_to_save: dict[str, str] = {}
-        for address in range(-1, 64, 1):
-            mac, node_type = self._registrations.get(address, ("", None))
-            if node_type is None:
-                node_value = ""
-            else:
-                node_value = str(node_type)
-            cache_data_to_save[str(address)] = (
-                f"{mac}{CACHE_DATA_SEPARATOR}{node_value}"
-            )
+        for mac, node_type in self._nodetypes.items():
+            node_value = str(node_type)
+            cache_data_to_save[mac] = node_value
+        _LOGGER.debug("Save NodeTypes %s", str(len(cache_data_to_save)))
         await self.write_cache(cache_data_to_save)
 
     async def clear_cache(self) -> None:
         """Clear current cache."""
-        self._registrations = {}
+        self._nodetypes = {}
         await self.delete_cache()
 
     async def restore_cache(self) -> None:
         """Load the previously stored information."""
         data: dict[str, str] = await self.read_cache()
-        self._registrations = {}
-        for _key, _data in data.items():
-            address = int(_key)
-            try:
-                if CACHE_DATA_SEPARATOR in _data:
-                    values = _data.split(CACHE_DATA_SEPARATOR)
-                else:
-                    # legacy data separator can by remove at next version
-                    values = _data.split(";")
-                mac = values[0]
-                node_type: NodeType | None = None
-                if values[1] != "":
-                    node_type = NodeType[values[1][9:]]
-                self._registrations[address] = (mac, node_type)
-                _LOGGER.debug(
-                    "Restore registry address %s with mac %s with node type %s",
-                    address,
-                    mac if mac != "" else "<empty>",
-                    str(node_type),
-                )
-            except (KeyError, IndexError):
-                _LOGGER.warning(
-                    "Skip invalid data '%s' in cache file '%s'",
-                    _data,
-                    self._cache_file,
-                )
+        self._nodetypes = {}
+        for mac, node_value in data.items():
+            node_type: NodeType | None = None
+            if len(node_value) >= 10:
+                node_type = NodeType[node_value[9:]]
+            if node_type is not None:
+                self._nodetypes[mac] = node_type
+            _LOGGER.debug(
+                "Restore NodeType for mac %s with node type %s",
+                mac,
+                str(node_type),
+            )
 
-    def update_registration(
-        self, address: int, mac: str, node_type: NodeType | None
-    ) -> None:
+    async def update_nodetypes(self, mac: str, node_type: NodeType | None) -> None:
         """Save node information in cache."""
-        if self._registrations.get(address) is not None:
-            _, current_node_type = self._registrations[address]
-            if current_node_type is not None and node_type is None:
+        if node_type is None:
+            return
+        if (current_node_type := self._nodetypes.get(mac)) is not None:
+            if current_node_type == node_type:
                 return
-        self._registrations[address] = (mac, node_type)
+            _LOGGER.warning(
+                "Cache contained mismatched NodeType %s replacing with %s",
+                str(current_node_type),
+                str(node_type),
+            )
+        self._nodetypes[mac] = node_type
+        await self.save_cache()
+
+    def get_nodetype(self, mac: str) -> NodeType | None:
+        """Return NodeType from cache."""
+        return self._nodetypes.get(mac)

plugwise_usb/network/registry.py

@@ -107,7 +107,6 @@
         """Initialize load the network registry."""
         if self._cache_enabled:
             await self.restore_network_cache()
-            await self.load_registry_from_cache()
         await self.update_missing_registrations_quick()
 
     async def restore_network_cache(self) -> None:
@@ -121,22 +120,6 @@
             await self._network_cache.restore_cache()
         self._cache_restored = True
 
-    async def load_registry_from_cache(self) -> None:
-        """Load network registry from cache."""
-        if self._network_cache is None:
-            _LOGGER.error(
-                "Unable to restore network registry because cache is not initialized"
-            )
-            return
-
-        if self._cache_restored:
-            return
-
-        for address, registration in self._network_cache.registrations.items():
-            mac, node_type = registration
-            if self._registry.get(address) is None:
-                self._registry[address] = (mac, node_type)
-
     async def retrieve_network_registration(
         self, address: int, retry: bool = True
     ) -> tuple[int, str] | None:
@@ -167,17 +150,22 @@
             raise NodeError("Unable to return network controller details")
         return self.registry[-1]
 
-    def update_network_registration(
+    async def update_network_registration(
         self, address: int, mac: str, node_type: NodeType | None
     ) -> None:
         """Add a network registration."""
-        if self._registry.get(address) is not None:
-            _, current_type = self._registry[address]
-            if current_type is not None and node_type is None:
-                return
+        if node_type is None:
+            if self._registry.get(address) is not None:
+                _, current_type = self._registry[address]
+                if current_type is not None:
+                    return
+            if self._network_cache is not None:
+                node_type = self._network_cache.get_nodetype(mac)
+
         self._registry[address] = (mac, node_type)
-        if self._network_cache is not None:
-            self._network_cache.update_registration(address, mac, node_type)
+        if node_type is not None:
+            if self._network_cache is not None:
+                await self._network_cache.update_nodetypes(mac, node_type)
 
     async def update_missing_registrations_full(self) -> None:
         """Full retrieval of all unknown network registrations from network controller."""
@@ -199,14 +187,10 @@
                     str(nextaddress),
                     "'empty'" if mac == "" else f"set to {mac}",
                 )
-                self.update_network_registration(nextaddress, mac, None)
+                await self.update_network_registration(nextaddress, mac, None)
             await sleep(10)
         _LOGGER.debug("Full network registration finished")
         self._scan_completed = True
-        if self._cache_enabled:
-            _LOGGER.debug("Full network registration finished, save to cache")
-            await self.save_registry_to_cache()
-            _LOGGER.debug("Full network registration finished, post")
         _LOGGER.info("Full network discovery completed")
         if self._full_scan_finished is not None:
             await self._full_scan_finished()
@@ -228,7 +212,7 @@
                     str(nextaddress),
                     "'empty'" if mac == "" else f"set to {mac}",
                 )
-                self.update_network_registration(nextaddress, mac, None)
+                await self.update_network_registration(nextaddress, mac, None)
             await sleep(0.1)
         if self._registration_task is None or self._registration_task.done():
             self._registration_task = create_task(
@@ -239,9 +223,9 @@
                 self._quick_scan_finished = None
             _LOGGER.info("Quick network registration discovery finished")
 
-    def update_node_registration(self, mac: str) -> int:
+    async def update_node_registration(self, mac: str) -> int:
         """Register (re)joined node to Plugwise network and return network address."""
-        self.update_network_registration(self._first_free_address, mac, None)
+        await self.update_network_registration(self._first_free_address, mac, None)
         self._first_free_address += 1
         return self._first_free_address - 1
 
@@ -251,22 +235,6 @@
             return
         self._registration_task.cancel()
 
-    async def save_registry_to_cache(self) -> None:
-        """Save network registry to cache."""
-        if self._network_cache is None:
-            _LOGGER.error(
-                "Unable to save network registry because cache is not initialized"
-            )
-            return
-        _LOGGER.debug(
-            "save_registry_to_cache starting for %s items", str(len(self._registry))
-        )
-        for address, registration in self._registry.items():
-            mac, node_type = registration
-            self._network_cache.update_registration(address, mac, node_type)
-        await self._network_cache.save_cache()
-        _LOGGER.debug("save_registry_to_cache finished")
-
     async def register_node(self, mac: str) -> None:
         """Register node to Plugwise network and return network address."""
         if not validate_mac(mac):
@@ -303,7 +271,7 @@
                 + f" failed to unregister node '{mac}'"
             )
         if (address := self.network_address(mac)) is not None:
-            self.update_network_registration(address, mac, None)
+            await self.update_network_registration(address, mac, None)
 
     async def clear_register_cache(self) -> None:
         """Clear current cache."""
@@ -314,9 +282,3 @@
     async def stop(self) -> None:
         """Unload the network registry."""
         self._stop_registration_task()
-        if (
-            self._cache_enabled
-            and self._network_cache is not None
-            and self._network_cache.initialized
-        ):
-            await self.save_registry_to_cache()