SFP boot scripts: wait for carrier before loading SGMII+ module (#602)

* Sync SFP SGMII+ boot scripts: carrier wait before module load

Boot scripts now nohup to background and wait up to 90s for the SFP to
establish a 1G link before loading the kernel module. Fixes a boot-order
race with SFPs like the Zyxel PMG3000 that need ~15s to configure their
SerDes. If no carrier appears within the timeout, the module loads anyway
to handle SFPs hard-locked at 2.5G.

* Show carrier wait note when SFP boot script deployed but module not loaded

Explains the up-to-90s link check period that allows the ONT to complete
its boot sequence before the SGMII+ module loads.

* Suppress Issue badge for 90s after SFP deploy, fix alert margin and wording

After deploying an SFP tweak, the boot script waits up to 90s for carrier
before loading the module. During this grace period, show Active instead of
Issue since the module not being loaded yet is expected. Also add bottom
margin to the carrier wait info alert and clarify "kernel module" vs "module"
to avoid confusion with SFP modules.

* Carrier wait note: mention deploy in addition to reboot

Author: tvancott42

src/NetworkOptimizer.Web/Components/Pages/PerformanceTweaks.razor

@@ -220,7 +220,7 @@
         @foreach (var def in _tweakDefs.Where(d => d.IsCompatibleWith(_status?.GatewayModel)))
         {
             var tweakStatus = _status?.Tweaks.GetValueOrDefault(def.Id);
-            var effectiveStatus = GetEffectiveStatus(tweakStatus);
+            var effectiveStatus = GetEffectiveStatus(tweakStatus, def.Id);
             var exclusiveOtherActive = def.MutuallyExclusiveWith != null
                 && _status?.Tweaks.TryGetValue(def.MutuallyExclusiveWith, out var otherTweak) == true
                 && (otherTweak.IsActive || otherTweak.IsManuallyDeployed);
@@ -258,6 +258,14 @@
                         <p class="pt-tweak-description" style="font-style: italic;">@def.ExtraNote</p>
                     }
 
+                    <!-- SFP carrier wait note: boot script deployed but module not yet loaded -->
+                    @if ((def.Id == "sfp-sgmiiplus" || def.Id == "sfp-sgmiiplus-port6") && tweakStatus?.BootScriptDeployed == true && tweakStatus?.RuntimeDetected != true)
+                    {
+                        <div class="alert alert-info" style="margin-top: 0.75rem; margin-bottom: 0.75rem;">
+                            <strong>Waiting for link:</strong> After a reboot or deploy, the boot script waits up to 90 seconds for the SFP to establish a 1 G link before loading the SGMII+ kernel module. This allows time for the ONT's boot sequence and SerDes configuration. If the kernel module has not loaded yet, it may still be in this link check period.
+                        </div>
+                    }
+
                     <!-- Health Check Results -->
                     @if (tweakStatus != null && tweakStatus.HealthChecks.Any())
                     {
@@ -934,6 +942,7 @@ ls -la /var/config/run-syslog.sh</code></pre>
     private bool _showRemoveConfirm;
     private string? _pendingRemoveTweakId;
     private bool _showZyxelInstructions;
+    private readonly Dictionary<string, DateTime> _sfpDeployTimes = new();
     private bool _allConfirmed => _confirmBackup && _confirmBackupDownloaded && _confirmWarranty && _confirmRisk;
     private bool _canDeploy => _status?.UdmBootInstalled == true && _status?.FirmwareSupported == true;
     private List<string> _deploySteps = new();
@@ -1122,7 +1131,11 @@ ls -la /var/config/run-syslog.sh</code></pre>
             var result = await DeployService.DeployTweakAsync(tweakId, progress);
 
             if (result.success)
+            {
+                if (tweakId is "sfp-sgmiiplus" or "sfp-sgmiiplus-port6")
+                    _sfpDeployTimes[tweakId] = DateTime.UtcNow;
                 await LoadStatusAsync();
+            }
         }
         finally
         {
@@ -1163,13 +1176,22 @@ ls -la /var/config/run-syslog.sh</code></pre>
         await LoadStatusAsync();
     }
 
-    private TweakDisplayStatus GetEffectiveStatus(TweakDeploymentStatus? status)
+    private bool IsSfpInCarrierWait(string tweakId) =>
+        _sfpDeployTimes.TryGetValue(tweakId, out var deployTime)
+        && (DateTime.UtcNow - deployTime).TotalSeconds < 90;
+
+    private TweakDisplayStatus GetEffectiveStatus(TweakDeploymentStatus? status, string? tweakId = null)
     {
         if (status == null) return TweakDisplayStatus.NotDeployed;
         if (status.IsManuallyDeployed) return TweakDisplayStatus.Manual;
         if (status.IsActive && string.IsNullOrEmpty(status.IssueDescription)) return TweakDisplayStatus.Active;
         if (status.IsActive && !string.IsNullOrEmpty(status.IssueDescription)) return TweakDisplayStatus.Issue;
-        if (status.BootScriptDeployed && !status.IsActive) return TweakDisplayStatus.Issue;
+        if (status.BootScriptDeployed && !status.IsActive)
+        {
+            if (tweakId != null && IsSfpInCarrierWait(tweakId))
+                return TweakDisplayStatus.Active;
+            return TweakDisplayStatus.Issue;
+        }
         if (status.RuntimeDetected && !status.BootScriptDeployed) return TweakDisplayStatus.Detected;
         return TweakDisplayStatus.NotDeployed;
     }

src/NetworkOptimizer.Web/Resources/PerfTweaks/19-sfp-sgmiiplus-eth5.sh

@@ -1,18 +1,22 @@
 #!/bin/sh
 # 19-sfp-sgmiiplus-eth5.sh: Force 1st SFP+ port (eth5 / Port 6) to SGMII+ 2.5G
 #
-# Loads a kernel module that switches uniphy2 from SGMII 1G to SGMII+ 2.5G
-# by calling the QCA-SSDK's internal uniphy mode set function directly,
-# bypassing SFP EEPROM validation that blocks the speed change.
+# Waits for the SFP to establish a 1G link, then loads a kernel module that
+# switches uniphy2 from SGMII 1G to SGMII+ 2.5G. The wait avoids a boot-order
+# race with SFPs that need time to configure their SerDes (e.g., Zyxel PMG3000
+# takes ~15s after boot to fire its 2.5G override). If no 1G link appears
+# within the timeout, the module loads anyway — this handles SFPs that are
+# hard-locked at 2.5G and can't establish a 1G link without the host matching.
+#
+# The module bypasses the SSDK's SFP EEPROM validation by calling the uniphy
+# mode set function directly. The SSDK's MAC sync polling loop re-reads the
+# SFP EEPROM every ~12s and would revert the 2.5G change. The module excludes
+# eth5 from the polling loop's port bitmap and restarts it — the loop continues
+# to run for all other ports, so eth6 link recovery is unaffected.
 #
 # WARNING: This targets eth5 / Port 6 (the 1st SFP+ port) ONLY.
 # For eth6 / Port 7, use 20-sfp-sgmiiplus.sh instead.
 #
-# The SSDK's MAC sync polling loop re-reads the SFP EEPROM every ~12s and
-# would revert the 2.5G change. The module excludes eth5 from the
-# polling loop's port bitmap and restarts it — the loop continues to run
-# for all other ports, so eth6 link recovery is unaffected.
-#
 # Target: UCG-Fiber / UXG-Fiber (IPQ9574, kernel 5.4.213-ui-ipq9574)
 # Requires: qca-ssdk.ko loaded, module pre-deployed to /data/sfp-sgmiiplus/
 
@@ -22,11 +26,19 @@ MODULE_DIR="/data/sfp-sgmiiplus"
 MODULE_NAME="force_uniphy2_sgmiiplus"
 MODULE_FILE="${MODULE_DIR}/${MODULE_NAME}.ko"
 CLOCK_PATH="/sys/kernel/debug/clk/uniphy2_gcc_tx_clk/clk_rate"
+IFACE="eth5"
+CARRIER_TIMEOUT=90
 
 log() {
     echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "${LOG_FILE}"
 }
 
+# Re-exec in background so on_boot.d doesn't block waiting for carrier
+if [ "$1" != "--bg" ]; then
+    nohup "$0" --bg >/dev/null 2>&1 &
+    exit 0
+fi
+
 # ─── Sanity checks ───
 
 if [ ! -f "${MODULE_FILE}" ]; then
@@ -44,6 +56,23 @@ if ! lsmod | grep -q "qca_ssdk"; then
     exit 1
 fi
 
+# ─── Wait for 1G carrier or timeout ───
+
+elapsed=0
+while [ $elapsed -lt $CARRIER_TIMEOUT ]; do
+    carrier=$(cat /sys/class/net/${IFACE}/carrier 2>/dev/null)
+    if [ "$carrier" = "1" ]; then
+        log "${IFACE} has carrier after ${elapsed}s — loading module"
+        break
+    fi
+    sleep 2
+    elapsed=$((elapsed + 2))
+done
+
+if [ "$carrier" != "1" ]; then
+    log "${IFACE} no carrier after ${CARRIER_TIMEOUT}s — loading module anyway (SFP may be hard-locked at 2.5G)"
+fi
+
 # ─── Load module ───
 
 log "Loading ${MODULE_NAME}..."

src/NetworkOptimizer.Web/Resources/PerfTweaks/20-sfp-sgmiiplus.sh

@@ -1,16 +1,20 @@
 #!/bin/sh
 # 20-sfp-sgmiiplus.sh: Force 2nd SFP+ port (eth6 / Port 7) to SGMII+ 2.5G
 #
-# Loads a kernel module that switches uniphy1 from SGMII 1G to SGMII+ 2.5G
-# by calling the QCA-SSDK's internal uniphy mode set function directly,
-# bypassing SFP EEPROM validation that blocks the speed change.
+# Waits for the SFP to establish a 1G link, then loads a kernel module that
+# switches uniphy1 from SGMII 1G to SGMII+ 2.5G. The wait avoids a boot-order
+# race with SFPs that need time to configure their SerDes (e.g., Zyxel PMG3000
+# takes ~15s after boot to fire its 2.5G override). If no 1G link appears
+# within the timeout, the module loads anyway — this handles SFPs that are
+# hard-locked at 2.5G and can't establish a 1G link without the host matching.
 #
-# WARNING: This targets eth6 / Port 7 (the 2nd SFP+ port) ONLY.
+# The module bypasses the SSDK's SFP EEPROM validation by calling the uniphy
+# mode set function directly. The SSDK's MAC sync polling loop re-reads the
+# SFP EEPROM every ~12s and would revert the 2.5G change. The module (v3+)
+# excludes eth6 from the polling loop's port bitmap and restarts it — the loop
+# continues to run for all other ports, so eth5 link recovery is unaffected.
 #
-# The SSDK's MAC sync polling loop re-reads the SFP EEPROM every ~12s and
-# would revert the 2.5G change. The module (v3+) excludes eth6 from the
-# polling loop's port bitmap and restarts it — the loop continues to run
-# for all other ports, so eth5 link recovery is unaffected.
+# WARNING: This targets eth6 / Port 7 (the 2nd SFP+ port) ONLY.
 #
 # Target: UCG-Fiber / UXG-Fiber (IPQ9574, kernel 5.4.213-ui-ipq9574)
 # Requires: qca-ssdk.ko loaded, module pre-deployed to /data/sfp-sgmiiplus/
@@ -21,11 +25,19 @@ MODULE_DIR="/data/sfp-sgmiiplus"
 MODULE_NAME="force_uniphy1_sgmiiplus"
 MODULE_FILE="${MODULE_DIR}/${MODULE_NAME}.ko"
 CLOCK_PATH="/sys/kernel/debug/clk/uniphy1_gcc_tx_clk/clk_rate"
+IFACE="eth6"
+CARRIER_TIMEOUT=90
 
 log() {
     echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" >> "${LOG_FILE}"
 }
 
+# Re-exec in background so on_boot.d doesn't block waiting for carrier
+if [ "$1" != "--bg" ]; then
+    nohup "$0" --bg >/dev/null 2>&1 &
+    exit 0
+fi
+
 # ─── Sanity checks ───
 
 if [ ! -f "${MODULE_FILE}" ]; then
@@ -43,6 +55,23 @@ if ! lsmod | grep -q "qca_ssdk"; then
     exit 1
 fi
 
+# ─── Wait for 1G carrier or timeout ───
+
+elapsed=0
+while [ $elapsed -lt $CARRIER_TIMEOUT ]; do
+    carrier=$(cat /sys/class/net/${IFACE}/carrier 2>/dev/null)
+    if [ "$carrier" = "1" ]; then
+        log "${IFACE} has carrier after ${elapsed}s — loading module"
+        break
+    fi
+    sleep 2
+    elapsed=$((elapsed + 2))
+done
+
+if [ "$carrier" != "1" ]; then
+    log "${IFACE} no carrier after ${CARRIER_TIMEOUT}s — loading module anyway (SFP may be hard-locked at 2.5G)"
+fi
+
 # ─── Load module ───
 
 log "Loading ${MODULE_NAME}..."