test: e2e coverage for HA Discovery startup race protections

Two new test surfaces for the v1.8.1 + v1.8.4 + v1.8.5 features:

1. tests/discoveryE2E.test.js (in-process, runs in npm test):
   Wires a real CommandResponseProcessor to a real HaDiscovery and a
   real EventPublisher (no mocks of either), then drives the full
   pipeline from raw C-Gate response lines through to MQTT publishes.
   Covers nine scenarios:
   - happy path (343/347/344 sequence advances diagnostic to ok)
   - startup race (401 → retry → success)
   - giveup (9× 401 exhausts retry budget, sensor reaches paused)
   - event-driven 742 Network created triggers TREEXML
   - 742 events for non-configured networks are filtered
   - Network created mid-backoff cancels the pending retry
   - non-Network 742 events are ignored
   - parser hardening against UUIDs containing hyphens
   - HA Discovery config payload shape (entity_category, device, etc.)

2. test-env/integration-test.js (podman managed-mode runner, manual):
   New section "6b. Discovery health diagnostic" asserts that the
   per-network diagnostic sensor is published with correct config
   payload shape (entity_category=diagnostic, device.identifiers
   includes cgateweb_bridge, state_topic format) and that at least
   one network reaches discovery_status=ok against a real C-Gate.

Full suite: 1199/1199 passing (was 1190 before these additions).
No behaviour changes — purely additive test coverage.

Author: dougrathbone

test-env/integration-test.js

@@ -383,6 +383,90 @@ async function runTests() {
         info(`Discovery summary: ${discoveryMessages.size} total, ${lightCount} light(s)`);
     }
 
+    // ------------------------------------------------------------------
+    // 6b. Discovery health diagnostic (v1.8.4) — per-network sensor
+    // ------------------------------------------------------------------
+    section('Discovery health diagnostic');
+    info('Collecting per-network discovery_status messages for 3s...');
+
+    const diagReceived = await collectMqtt(
+        ['homeassistant/sensor/cgateweb_discovery_+/config', 'cbus/read/+///discovery_status'],
+        3000
+    );
+    const diagConfigTopics = Object.keys(diagReceived).filter(t =>
+        /^homeassistant\/sensor\/cgateweb_discovery_\w+\/config$/.test(t)
+    );
+    const diagStateTopics = Object.keys(diagReceived).filter(t =>
+        /^cbus\/read\/\w+\/\/\/discovery_status$/.test(t)
+    );
+
+    if (diagConfigTopics.length === 0 && diagStateTopics.length === 0) {
+        info('No discovery diagnostic messages received — HA Discovery may not have run (no networks configured?). Skipping diagnostic assertions.');
+        pass('Discovery diagnostic: no messages (soft pass)');
+        passed++;
+    } else {
+        assert(
+            `discovery diagnostic config published (${diagConfigTopics.length} sensor config(s))`,
+            diagConfigTopics.length > 0
+        );
+        assert(
+            `discovery diagnostic state published (${diagStateTopics.length} network(s))`,
+            diagStateTopics.length > 0
+        );
+
+        // Validate the config payload shape on at least one diagnostic.
+        if (diagConfigTopics.length > 0) {
+            const cfgTopic = diagConfigTopics[0];
+            try {
+                const cfg = JSON.parse(diagReceived[cfgTopic]);
+                assert(
+                    `${cfgTopic} has entity_category=diagnostic`,
+                    cfg.entity_category === 'diagnostic',
+                    `got: ${cfg.entity_category}`
+                );
+                assert(
+                    `${cfgTopic} has unique_id matching cgateweb_discovery_*`,
+                    typeof cfg.unique_id === 'string' && cfg.unique_id.startsWith('cgateweb_discovery_'),
+                    `got: ${cfg.unique_id}`
+                );
+                assert(
+                    `${cfgTopic} state_topic matches cbus/read/<network>///discovery_status`,
+                    /^cbus\/read\/\w+\/\/\/discovery_status$/.test(cfg.state_topic || ''),
+                    `got: ${cfg.state_topic}`
+                );
+                assert(
+                    `${cfgTopic} grouped under cgateweb_bridge device`,
+                    Array.isArray(cfg.device?.identifiers) && cfg.device.identifiers.includes('cgateweb_bridge'),
+                    `got: ${JSON.stringify(cfg.device?.identifiers)}`
+                );
+            } catch (err) {
+                fail(`${cfgTopic} payload not valid JSON: ${err.message}`);
+                failed++;
+            }
+        }
+
+        // Validate the state payload — should be one of {discovering, ok, paused}.
+        const validStates = new Set(['discovering', 'ok', 'paused']);
+        for (const stateTopic of diagStateTopics) {
+            const value = diagReceived[stateTopic];
+            assert(
+                `${stateTopic} = ${value} (one of discovering/ok/paused)`,
+                validStates.has(value),
+                `got: ${value}`
+            );
+        }
+
+        // For a working stack with at least one network, at least one diagnostic
+        // should have reached "ok" (TreeXML succeeded). If everything is still
+        // "discovering" after readiness, something's wrong.
+        const okCount = diagStateTopics.filter(t => diagReceived[t] === 'ok').length;
+        assert(
+            `at least one network reached discovery_status=ok  (${okCount} of ${diagStateTopics.length})`,
+            okCount > 0 || diagStateTopics.length === 0,
+            'all networks still in discovering/paused after readiness'
+        );
+    }
+
     // ------------------------------------------------------------------
     // 7. Stability check — watch for 10s, no reconnects
     // ------------------------------------------------------------------

tests/discoveryE2E.test.js

@@ -0,0 +1,267 @@
+// In-process end-to-end test for HA Discovery startup race protections
+// (v1.8.1 retry + v1.8.4 diagnostic sensor + v1.8.5 event-driven refresh).
+//
+// Wires a real CommandResponseProcessor to a real HaDiscovery (and a real
+// EventPublisher), feeds raw C-Gate command-port lines through processLine(),
+// and asserts that MQTT publishes and outbound C-Gate commands match what HA
+// users would actually observe.
+
+const CommandResponseProcessor = require('../src/commandResponseProcessor');
+const EventPublisher = require('../src/eventPublisher');
+const HaDiscovery = require('../src/haDiscovery');
+const {
+    CGATE_CMD_TREEXML,
+    NEWLINE,
+    DISCOVERY_STATE_DISCOVERING,
+    DISCOVERY_STATE_OK,
+    DISCOVERY_STATE_PAUSED
+} = require('../src/constants');
+
+// Minimal but realistic TreeXML payload — flat application format, two lighting groups.
+const TREE_XML = `<?xml version="1.0" encoding="UTF-8"?>
+<Network>
+  <NetworkNumber>254</NetworkNumber>
+  <Unit>
+    <UnitAddress>1</UnitAddress>
+    <Application>
+      <ApplicationAddress>56</ApplicationAddress>
+      <Group>
+        <GroupAddress>10</GroupAddress>
+        <Label>Kitchen</Label>
+      </Group>
+      <Group>
+        <GroupAddress>11</GroupAddress>
+        <Label>Lounge</Label>
+      </Group>
+    </Application>
+  </Unit>
+</Network>`;
+
+function buildHarness(overrides = {}) {
+    const sentCommands = [];
+    const publishes = [];
+
+    const settings = {
+        ha_discovery_enabled: true,
+        ha_discovery_prefix: 'homeassistant',
+        ha_discovery_networks: ['254'],
+        cbusname: 'PROJECT',
+        getallnetapp: null,
+        eventPublishDedupWindowMs: 0,
+        log_level: 'warn',
+        ...overrides
+    };
+
+    const publishFn = (topic, payload, options) => {
+        publishes.push({ topic, payload, options });
+    };
+
+    const sendCommandFn = (cmd) => {
+        sentCommands.push(cmd);
+    };
+
+    const haDiscovery = new HaDiscovery(settings, publishFn, sendCommandFn);
+
+    const eventPublisher = new EventPublisher({
+        settings,
+        publishFn,
+        mqttOptions: { retain: true, qos: 0 }
+    });
+
+    const processor = new CommandResponseProcessor({
+        eventPublisher,
+        haDiscovery,
+        onObjectStatus: () => {},
+        // Mirrors the wiring in BridgeInitializationService.handleCommandError:
+        // forward 4xx/5xx responses to HaDiscovery so it can recognise the
+        // 401 "Network not found" that signals a TREEXML startup race.
+        onCommandError: (code, statusData) => haDiscovery.handleCommandError(code, statusData)
+    });
+
+    return { processor, haDiscovery, settings, sentCommands, publishes };
+}
+
+function statePublishes(publishes, network) {
+    return publishes
+        .filter(p => p.topic === `cbus/read/${network}///discovery_status`)
+        .map(p => p.payload);
+}
+
+function configPublishes(publishes, network) {
+    return publishes.filter(p => p.topic === `homeassistant/sensor/cgateweb_discovery_${network}/config`);
+}
+
+describe('HA Discovery e2e (real processor + real haDiscovery)', () => {
+    beforeEach(() => {
+        jest.useFakeTimers();
+    });
+
+    afterEach(() => {
+        jest.useRealTimers();
+    });
+
+    it('happy path: TREEXML 343/347/344 sequence advances diagnostic to ok', async () => {
+        const h = buildHarness();
+        h.haDiscovery.trigger();
+
+        // First TREEXML went out, sensor is "discovering"
+        expect(h.sentCommands).toEqual([`${CGATE_CMD_TREEXML} 254${NEWLINE}`]);
+        expect(statePublishes(h.publishes, '254')).toEqual([DISCOVERY_STATE_DISCOVERING]);
+        expect(configPublishes(h.publishes, '254')).toHaveLength(1);
+
+        // C-Gate responds with the standard tree sequence
+        h.processor.processLine('343-Begin TreeXML');
+        for (const line of TREE_XML.split('\n')) {
+            h.processor.processLine(`347-${line}`);
+        }
+        h.processor.processLine('344-End TreeXML');
+
+        // xml2js.parseString is async — flush microtasks
+        await Promise.resolve();
+        await Promise.resolve();
+
+        const states = statePublishes(h.publishes, '254');
+        expect(states[states.length - 1]).toBe(DISCOVERY_STATE_OK);
+
+        // Discovery payloads for Kitchen + Lounge published under homeassistant/light/...
+        const lightConfigs = h.publishes.filter(
+            p => /^homeassistant\/light\/cgateweb_254_56_(10|11)\/config$/.test(p.topic)
+        );
+        expect(lightConfigs).toHaveLength(2);
+    });
+
+    it('startup race: 401 Network not found triggers retry; eventual success → ok', async () => {
+        const h = buildHarness();
+        h.haDiscovery.trigger();
+        expect(h.sentCommands).toHaveLength(1);
+
+        // C-Gate returns 401 because the network isn't loaded yet
+        h.processor.processLine('401-Bad object or device ID: Network not found');
+
+        // No new command yet — retry is pending
+        expect(h.sentCommands).toHaveLength(1);
+
+        // Diagnostic stays "discovering" through the retry (de-duped)
+        expect(statePublishes(h.publishes, '254')).toEqual([DISCOVERY_STATE_DISCOVERING]);
+
+        // Wait for the 2s backoff
+        jest.advanceTimersByTime(2000);
+        expect(h.sentCommands).toHaveLength(2);
+
+        // This time C-Gate has loaded the network — full tree response
+        h.processor.processLine('343-Begin TreeXML');
+        h.processor.processLine(`347-${TREE_XML}`);
+        h.processor.processLine('344-End TreeXML');
+
+        await Promise.resolve();
+        await Promise.resolve();
+
+        const states = statePublishes(h.publishes, '254');
+        expect(states[states.length - 1]).toBe(DISCOVERY_STATE_OK);
+    });
+
+    it('giveup path: 9 consecutive 401s exhaust retry budget, sensor reaches paused', () => {
+        const h = buildHarness();
+        h.haDiscovery.trigger();
+
+        // 8 retries permitted, 9th failure exhausts the budget
+        for (let i = 1; i <= 8; i++) {
+            h.processor.processLine('401-Bad object or device ID: Network not found');
+            jest.runOnlyPendingTimers();
+        }
+        h.processor.processLine('401-Bad object or device ID: Network not found');
+
+        const states = statePublishes(h.publishes, '254');
+        expect(states[states.length - 1]).toBe(DISCOVERY_STATE_PAUSED);
+    });
+
+    it('event-driven refresh: timestamped 742 Network created triggers TREEXML', () => {
+        const h = buildHarness();
+        // No initial trigger — simulate the bridge before discovery has run.
+
+        h.processor.processLine(
+            '20260504-193110.569 742 //PROJECT/254 c2211b00-28c1-103f-94b5-db702a32859b ' +
+            'Network created type=cni address=192.168.0.100:10001'
+        );
+
+        expect(h.sentCommands).toEqual([`${CGATE_CMD_TREEXML} 254${NEWLINE}`]);
+        expect(statePublishes(h.publishes, '254')).toEqual([DISCOVERY_STATE_DISCOVERING]);
+    });
+
+    it('event-driven refresh fires only for configured networks', () => {
+        const h = buildHarness({ ha_discovery_networks: ['254'] });
+
+        h.processor.processLine(
+            '20260504-193110.569 742 //PROJECT/999 abc Network created type=cni'
+        );
+
+        expect(h.sentCommands).toEqual([]);
+    });
+
+    it('Network created mid-backoff short-circuits the v1.8.1 retry timer', () => {
+        const h = buildHarness();
+        h.haDiscovery.trigger();
+        h.processor.processLine('401-Bad object or device ID: Network not found');
+        // Retry scheduled for 2s.
+
+        const sentBefore = h.sentCommands.length;
+        h.processor.processLine(
+            '20260504-193110.569 742 //PROJECT/254 uuid Network created type=cni'
+        );
+        // Event triggered an immediate fresh TREEXML.
+        expect(h.sentCommands).toHaveLength(sentBefore + 1);
+
+        // The original retry was canceled — no extra command after the 2s window.
+        jest.advanceTimersByTime(2500);
+        expect(h.sentCommands).toHaveLength(sentBefore + 1);
+    });
+
+    it('non-Network 742 events do not trigger discovery', () => {
+        const h = buildHarness();
+
+        h.processor.processLine(
+            '20260504-193110.421 742 //PROJECT - Tag information changed at tag address: //PROJECT/Installation oldtag: null newtag: null'
+        );
+        h.processor.processLine(
+            '20260504-193120.394 836 //PROJECT/254/p/12 c21ed110-... unit configuration changed (5 changes)'
+        );
+
+        expect(h.sentCommands).toEqual([]);
+    });
+
+    it('parser hardening: hyphens inside payload UUIDs do not break parsing', () => {
+        const h = buildHarness();
+
+        // Without the v1.8.5 parser fix, the hyphen inside the UUID would be
+        // mistaken for the code/data separator and the line would be skipped.
+        h.processor.processLine(
+            '20260504-193110.569 742 //PROJECT/254 c2211b00-28c1-103f-94b5-db702a32859b Network created type=cni address=192.168.0.100:10001'
+        );
+
+        expect(h.sentCommands).toEqual([`${CGATE_CMD_TREEXML} 254${NEWLINE}`]);
+    });
+
+    it('config payload retains diagnostic shape across the full lifecycle', () => {
+        const h = buildHarness();
+        h.haDiscovery.trigger();
+
+        const cfg = configPublishes(h.publishes, '254')[0];
+        expect(cfg).toBeDefined();
+        const payload = JSON.parse(cfg.payload);
+
+        // Required HA Discovery fields
+        expect(payload.unique_id).toBe('cgateweb_discovery_254');
+        expect(payload.state_topic).toBe('cbus/read/254///discovery_status');
+        expect(payload.entity_category).toBe('diagnostic');
+        expect(payload.availability_topic).toBe('hello/cgateweb');
+        expect(payload.payload_available).toBe('Online');
+        expect(payload.payload_not_available).toBe('Offline');
+
+        // Grouped under the existing cgateweb Bridge device so it sits next to
+        // the other diagnostics (Bridge Ready, MQTT Connected, etc.).
+        expect(payload.device.identifiers).toContain('cgateweb_bridge');
+
+        // Retained so HA always has the latest state cached.
+        expect(cfg.options).toEqual({ retain: true, qos: 0 });
+    });
+});