Replace index-based discovery with ScraperRegistry (#554)

Replace fragile array-index lookups (SCRAPER_IDX/PLATFORM_IDX) with a
type-safe ScraperRegistry using deterministic string IDs derived from
config names. This makes scraper/entity lookup reload-safe since order
no longer matters.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: danieldotnl

custom_components/multiscrape/__init__.py

@@ -14,13 +14,14 @@
 from homeassistant.helpers import discovery
 from homeassistant.helpers.reload import (async_integration_yaml_config,
                                           async_reload_integration_platforms)
+from homeassistant.util import slugify
 
-from .const import (CONF_LOG_RESPONSE, COORDINATOR, DOMAIN, PLATFORM_IDX,
-                    SCRAPER, SCRAPER_DATA, SCRAPER_IDX)
+from .const import CONF_LOG_RESPONSE, DOMAIN, ENTITY_KEY, SCRAPER_ID
 from .coordinator import (create_content_request_manager,
                           create_multiscrape_coordinator)
 from .file import create_file_manager
 from .http_session import create_http_session
+from .registry import ScraperInstance, ScraperRegistry
 from .schema import COMBINED_SCHEMA, CONFIG_SCHEMA  # noqa: F401
 from .scraper import create_scraper
 from .service import setup_config_services, setup_integration_services
@@ -65,8 +66,8 @@ async def reload_service_handler(service):
 
 
 def _async_setup_shared_data(hass: HomeAssistant):
-    """Create shared data for platform config and scraper coordinators."""
-    hass.data[DOMAIN] = {key: [] for key in [SCRAPER_DATA, *PLATFORMS]}
+    """Create a fresh ScraperRegistry for platform config and scraper coordinators."""
+    hass.data[DOMAIN] = ScraperRegistry()
 
 
 async def _async_process_config(hass: HomeAssistant, config) -> bool:
@@ -76,11 +77,13 @@ async def _async_process_config(hass: HomeAssistant, config) -> bool:
 
     refresh_tasks = []
     load_tasks = []
+    registry: ScraperRegistry = hass.data[DOMAIN]
 
-    for scraper_idx, conf in enumerate(config[DOMAIN]):
+    for conf in config[DOMAIN]:
         config_name = conf.get(CONF_NAME)
         if config_name is None:
-            config_name = f"Scraper_noname_{scraper_idx}"
+            resource = conf.get(CONF_RESOURCE) or ""
+            config_name = f"scraper_{slugify(resource)}" if resource else "scraper_unnamed"
             _LOGGER.debug(
                 "# Found no name for scraper, generated a unique name: %s", config_name
             )
@@ -89,6 +92,8 @@ async def _async_process_config(hass: HomeAssistant, config) -> bool:
             "%s # Setting up multiscrape with config:\n %s", config_name, conf
         )
 
+        scraper_id = config_name
+
         file_manager = await create_file_manager(hass, config_name, conf.get(CONF_LOG_RESPONSE))
         session = create_http_session(config_name, conf, hass, file_manager)
         scraper = create_scraper(config_name, conf, hass, file_manager)
@@ -108,9 +113,12 @@ async def _shutdown_session(_event, _session=session):
 
         hass.bus.async_listen_once(EVENT_HOMEASSISTANT_STOP, _shutdown_session)
 
-        hass.data[DOMAIN][SCRAPER_DATA].append(
-            {SCRAPER: scraper, COORDINATOR: coordinator}
+        instance = ScraperInstance(
+            scraper_id=scraper_id,
+            scraper=scraper,
+            coordinator=coordinator,
         )
+        registry.register(instance)
 
         await setup_config_services(hass, coordinator, config_name)
 
@@ -119,14 +127,16 @@ async def _shutdown_session(_event, _session=session):
                 continue
 
             for platform_conf in conf[platform_domain]:
-                hass.data[DOMAIN][platform_domain].append(platform_conf)
-                platform_idx = len(hass.data[DOMAIN][platform_domain]) - 1
+                entity_name = platform_conf.get(CONF_NAME, "")
+                entity_key = slugify(entity_name) if entity_name else f"entity_{id(platform_conf)}"
+
+                instance.platform_configs.setdefault(platform_domain, {})[entity_key] = platform_conf
 
                 load = discovery.async_load_platform(
                     hass,
                     platform_domain,
                     DOMAIN,
-                    {SCRAPER_IDX: scraper_idx, PLATFORM_IDX: platform_idx},
+                    {SCRAPER_ID: scraper_id, ENTITY_KEY: entity_key},
                     config,
                 )
                 load_tasks.append(load)
@@ -143,8 +153,7 @@ async def _shutdown_session(_event, _session=session):
 
 async def async_get_config_and_coordinator(hass, platform_domain, discovery_info):
     """Get the config and coordinator for the platform from discovery."""
-    shared_data = hass.data[DOMAIN][SCRAPER_DATA][discovery_info[SCRAPER_IDX]]
-    conf = hass.data[DOMAIN][platform_domain][discovery_info[PLATFORM_IDX]]
-    coordinator = shared_data[COORDINATOR]
-    scraper = shared_data[SCRAPER]
-    return conf, coordinator, scraper
+    registry: ScraperRegistry = hass.data[DOMAIN]
+    instance = registry.get(discovery_info[SCRAPER_ID])
+    conf = instance.platform_configs[platform_domain][discovery_info[ENTITY_KEY]]
+    return conf, instance.coordinator, instance.scraper

custom_components/multiscrape/const.py

@@ -40,13 +40,8 @@
 
 CONF_FIELDS = "fields"
 
-SCRAPER_IDX = "scraper_idx"
-PLATFORM_IDX = "platform_idx"
-
-COORDINATOR = "coordinator"
-SCRAPER = "scraper"
-
-SCRAPER_DATA = "scraper"
+SCRAPER_ID = "scraper_id"
+ENTITY_KEY = "entity_key"
 
 METHODS = ["POST", "GET", "PUT"]
 DEFAULT_SEPARATOR = ","

custom_components/multiscrape/registry.py

@@ -0,0 +1,48 @@
+"""ScraperRegistry for type-safe, reload-safe scraper lookup."""
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass, field
+
+from homeassistant.const import Platform
+
+_LOGGER = logging.getLogger(__name__)
+
+
+@dataclass
+class ScraperInstance:
+    """Holds all data for a single scraper configuration."""
+
+    scraper_id: str
+    scraper: object
+    coordinator: object
+    platform_configs: dict[Platform, dict[str, dict]] = field(default_factory=dict)
+
+
+class ScraperRegistry:
+    """Registry for scraper instances, replacing index-based lookups."""
+
+    def __init__(self):
+        """Initialize an empty registry."""
+        self._scrapers: dict[str, ScraperInstance] = {}
+
+    def register(self, instance: ScraperInstance) -> None:
+        """Register a scraper instance by its unique ID."""
+        if instance.scraper_id in self._scrapers:
+            raise ValueError(
+                f"Scraper '{instance.scraper_id}' is already registered"
+            )
+        self._scrapers[instance.scraper_id] = instance
+        _LOGGER.debug("Registered scraper: %s", instance.scraper_id)
+
+    def get(self, scraper_id: str) -> ScraperInstance:
+        """Get a scraper instance by its unique ID."""
+        return self._scrapers[scraper_id]
+
+    def get_all(self) -> list[ScraperInstance]:
+        """Get all registered scraper instances."""
+        return list(self._scrapers.values())
+
+    def clear(self) -> None:
+        """Remove all registered scrapers."""
+        self._scrapers.clear()

tests/test_init.py

@@ -12,9 +12,8 @@
                                            _async_setup_shared_data,
                                            async_get_config_and_coordinator,
                                            async_setup)
-from custom_components.multiscrape.const import (COORDINATOR, DOMAIN,
-                                                 PLATFORM_IDX, SCRAPER,
-                                                 SCRAPER_DATA, SCRAPER_IDX)
+from custom_components.multiscrape.const import DOMAIN, ENTITY_KEY, SCRAPER_ID
+from custom_components.multiscrape.registry import ScraperRegistry
 
 
 @pytest.fixture
@@ -47,18 +46,13 @@ def empty_config():
 @pytest.mark.async_test
 @pytest.mark.timeout(10)
 async def test_async_setup_shared_data(hass: HomeAssistant):
-    """Test _async_setup_shared_data creates required data structures."""
+    """Test _async_setup_shared_data creates a ScraperRegistry."""
     # Act
     _async_setup_shared_data(hass)
 
     # Assert
     assert DOMAIN in hass.data
-    assert SCRAPER_DATA in hass.data[DOMAIN]
-    assert Platform.SENSOR in hass.data[DOMAIN]
-    assert Platform.BINARY_SENSOR in hass.data[DOMAIN]
-    assert Platform.BUTTON in hass.data[DOMAIN]
-    assert isinstance(hass.data[DOMAIN][SCRAPER_DATA], list)
-    assert isinstance(hass.data[DOMAIN][Platform.SENSOR], list)
+    assert isinstance(hass.data[DOMAIN], ScraperRegistry)
 
 
 @pytest.mark.integration
@@ -119,10 +113,11 @@ async def test_async_process_config_creates_scraper_data(
 
     # Assert
     assert result is True
-    assert len(hass.data[DOMAIN][SCRAPER_DATA]) == 1
-    scraper_data = hass.data[DOMAIN][SCRAPER_DATA][0]
-    assert SCRAPER in scraper_data
-    assert COORDINATOR in scraper_data
+    registry: ScraperRegistry = hass.data[DOMAIN]
+    assert len(registry.get_all()) == 1
+    instance = registry.get("test_scraper")
+    assert instance.scraper is not None
+    assert instance.coordinator is not None
 
 
 @pytest.mark.integration
@@ -154,8 +149,11 @@ async def test_async_process_config_generates_name_for_unnamed_scraper(
 
     # Assert
     assert result is True
-    # Should have generated a name like "Scraper_noname_0"
-    assert len(hass.data[DOMAIN][SCRAPER_DATA]) == 1
+    registry: ScraperRegistry = hass.data[DOMAIN]
+    assert len(registry.get_all()) == 1
+    # Should have generated an ID from the resource URL
+    instance = registry.get_all()[0]
+    assert "example" in instance.scraper_id
 
 
 @pytest.mark.integration
@@ -191,9 +189,11 @@ async def test_async_process_config_stores_platform_configs(
     await _async_process_config(hass, minimal_config)
 
     # Assert
-    assert len(hass.data[DOMAIN][Platform.SENSOR]) == 1
-    sensor_config = hass.data[DOMAIN][Platform.SENSOR][0]
-    assert sensor_config[CONF_NAME] == "test_sensor"
+    registry: ScraperRegistry = hass.data[DOMAIN]
+    instance = registry.get("test_scraper")
+    sensor_configs = instance.platform_configs[Platform.SENSOR]
+    assert len(sensor_configs) == 1
+    assert list(sensor_configs.values())[0][CONF_NAME] == "test_sensor"
 
 
 @pytest.mark.integration
@@ -234,9 +234,10 @@ async def test_async_process_config_with_multiple_platforms(hass: HomeAssistant)
     await _async_process_config(hass, config)
 
     # Assert
-    assert len(hass.data[DOMAIN][Platform.SENSOR]) == 1
-    assert len(hass.data[DOMAIN][Platform.BINARY_SENSOR]) == 1
-    assert len(hass.data[DOMAIN][Platform.BUTTON]) == 1
+    instance = hass.data[DOMAIN].get("multi_platform_scraper")
+    assert len(instance.platform_configs[Platform.SENSOR]) == 1
+    assert len(instance.platform_configs[Platform.BINARY_SENSOR]) == 1
+    assert len(instance.platform_configs[Platform.BUTTON]) == 1
 
 
 @pytest.mark.integration
@@ -267,8 +268,10 @@ async def test_async_process_config_with_multiple_sensors_per_scraper(
     await _async_process_config(hass, config)
 
     # Assert
-    assert len(hass.data[DOMAIN][Platform.SENSOR]) == 3
-    assert len(hass.data[DOMAIN][SCRAPER_DATA]) == 1  # Only one scraper
+    registry: ScraperRegistry = hass.data[DOMAIN]
+    instance = registry.get("multi_sensor_scraper")
+    assert len(instance.platform_configs[Platform.SENSOR]) == 3
+    assert len(registry.get_all()) == 1  # Only one scraper
 
 
 @pytest.mark.integration
@@ -298,8 +301,10 @@ async def test_async_process_config_with_multiple_scrapers(hass: HomeAssistant):
     await _async_process_config(hass, config)
 
     # Assert
-    assert len(hass.data[DOMAIN][SCRAPER_DATA]) == 2
-    assert len(hass.data[DOMAIN][Platform.SENSOR]) == 2
+    registry: ScraperRegistry = hass.data[DOMAIN]
+    assert len(registry.get_all()) == 2
+    assert registry.get("scraper1").platform_configs[Platform.SENSOR].get("sensor1") is not None
+    assert registry.get("scraper2").platform_configs[Platform.SENSOR].get("sensor2") is not None
     # Each scraper should have its own trigger service
     assert hass.services.has_service(DOMAIN, "trigger_scraper1")
     assert hass.services.has_service(DOMAIN, "trigger_scraper2")
@@ -315,7 +320,7 @@ async def test_async_get_config_and_coordinator(hass: HomeAssistant, minimal_con
     _async_setup_shared_data(hass)
     await _async_process_config(hass, minimal_config)
 
-    discovery_info = {SCRAPER_IDX: 0, PLATFORM_IDX: 0}
+    discovery_info = {SCRAPER_ID: "test_scraper", ENTITY_KEY: "test_sensor"}
 
     # Act
     conf, coordinator, scraper = await async_get_config_and_coordinator(
@@ -362,7 +367,8 @@ async def test_async_process_config_with_form_submit(hass: HomeAssistant):
 
     # Assert
     assert result is True
-    assert len(hass.data[DOMAIN][SCRAPER_DATA]) == 1
+    registry: ScraperRegistry = hass.data[DOMAIN]
+    assert len(registry.get_all()) == 1
 
 
 @pytest.mark.integration
@@ -395,7 +401,8 @@ async def test_async_process_config_with_resource_template(hass: HomeAssistant):
 
     # Assert
     assert result is True
-    assert len(hass.data[DOMAIN][SCRAPER_DATA]) == 1
+    registry: ScraperRegistry = hass.data[DOMAIN]
+    assert len(registry.get_all()) == 1
 
 
 @pytest.mark.integration
@@ -426,6 +433,7 @@ async def test_async_process_config_skips_platforms_not_in_config(hass: HomeAssi
     await _async_process_config(hass, config)
 
     # Assert
-    assert len(hass.data[DOMAIN][Platform.SENSOR]) == 1
-    assert len(hass.data[DOMAIN][Platform.BINARY_SENSOR]) == 0
-    assert len(hass.data[DOMAIN][Platform.BUTTON]) == 0
+    instance = hass.data[DOMAIN].get("sensor_only_scraper")
+    assert len(instance.platform_configs.get(Platform.SENSOR, {})) == 1
+    assert len(instance.platform_configs.get(Platform.BINARY_SENSOR, {})) == 0
+    assert len(instance.platform_configs.get(Platform.BUTTON, {})) == 0