feat(usbgadget): warn in UI when USB endpoint budget is exceeded

The RV1106 dwc3 controller has a limited number of IN (device->host) and OUT
(host->device) endpoints -- separate pools, ~7 each. Each gadget function
claims some: HID mice/wake and audio capture take 1 IN; the keyboard takes
1 IN + 1 OUT (LED reports); mass storage takes 1 IN + 1 OUT; CDC-ACM and
CDC-NCM take 2 IN + 1 OUT each. When the enabled set exceeds either pool a
function silently fails to allocate its endpoint(s) -- CDC-NCM in particular
comes up looking connected (RX works) while TX is a black hole. That's
painful to diagnose, so warn before the user commits to an over-budget combo.

- internal/usbgadget/endpoints.go: per-function {in,out} endpoint cost map,
  the two budget constants (with the empirical derivation documented; the IN
  ceiling of 7 is confirmed on-device, OUT is assumed symmetric), and
  ExceedsEndpointBudget. The true low-level limiter may be dwc3 TX-FIFO SRAM
  rather than a raw count, but the effective ceiling is the same; FIFO RAM is
  not modeled separately.
- internal/usbgadget/config.go: extract isGadgetConfigItemEnabledForDevices so
  endpoint demand can be computed for any hypothetical Devices selection.
- jsonrpc.go: getUsbEndpointReport RPC returning {exceedsBudget} for a given
  device set.
- UsbDeviceSetting.tsx: query it live as toggles change and show an inline
  amber warning when the set exceeds the budget, suggesting the user free an
  endpoint (e.g. disable Relative Mouse).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: mcuelenaere

internal/usbgadget/config.go

@@ -70,21 +70,28 @@ var defaultGadgetConfig = map[string]gadgetConfigItem{
 }
 
 func (u *UsbGadget) isGadgetConfigItemEnabled(itemKey string) bool {
+	return isGadgetConfigItemEnabledForDevices(itemKey, &u.enabledDevices)
+}
+
+// isGadgetConfigItemEnabledForDevices reports whether a gadget config item is
+// enabled for an arbitrary Devices selection. Items without an explicit case
+// (base, base_info, wake_hid, audio, ...) are always enabled.
+func isGadgetConfigItemEnabledForDevices(itemKey string, devices *Devices) bool {
 	switch itemKey {
 	case "absolute_mouse":
-		return u.enabledDevices.AbsoluteMouse
+		return devices.AbsoluteMouse
 	case "relative_mouse":
-		return u.enabledDevices.RelativeMouse
+		return devices.RelativeMouse
 	case "keyboard":
-		return u.enabledDevices.Keyboard
+		return devices.Keyboard
 	case "mass_storage_base":
-		return u.enabledDevices.MassStorage
+		return devices.MassStorage
 	case "mass_storage_lun0":
-		return u.enabledDevices.MassStorage
+		return devices.MassStorage
 	case "serial_console":
-		return u.enabledDevices.SerialConsole
+		return devices.SerialConsole
 	case "ncm":
-		return u.enabledDevices.Ncm
+		return devices.Ncm
 	default:
 		return true
 	}

internal/usbgadget/endpoints.go

@@ -0,0 +1,68 @@
+package usbgadget
+
+// USB endpoint budget for the JetKVM RV1106 dwc3 controller.
+//
+// dwc3 exposes IN (device->host) and OUT (host->device) endpoints as separate
+// hardware resources, so they are budgeted independently — allocating a
+// bulk-OUT never steals from the IN pool.
+//
+// The IN budget of 7 is empirically confirmed: the full default function set
+// plus CDC-NCM needs 8 IN endpoints and NCM's bulk-IN silently fails to
+// allocate (the link enumerates and usb0 comes up RX-only, TX is a black
+// hole); dropping any single IN-using function (-> 7) makes it work. The OUT
+// pool on this part is symmetric with IN and OUT demand stays far below it in
+// practice, so the same ceiling is used for both.
+//
+// Note: the true low-level limiter for IN endpoints may be dwc3's TX-FIFO
+// SRAM (each IN endpoint reserves FIFO space sized to its max packet) rather
+// than a raw endpoint count, but the effective ceiling we observed is 7, so a
+// simple count captures it. FIFO RAM is not modeled separately.
+const (
+	usbInEndpointBudget  uint = 7
+	usbOutEndpointBudget uint = 7
+)
+
+// endpointCost is the number of USB IN and OUT endpoints a gadget function
+// consumes.
+type endpointCost struct {
+	in  uint
+	out uint
+}
+
+// endpointCosts maps a gadget config item key to its endpoint cost. Keys match
+// entries in defaultGadgetConfig; items absent here cost nothing (base,
+// base_info, mass_storage_lun0, ...). HID OUT cost follows the function's
+// no_out_endpoint attribute (keyboard has an interrupt-OUT for LED reports;
+// the mice and wake HID do not).
+var endpointCosts = map[string]endpointCost{
+	"keyboard":          {in: 1, out: 1}, // interrupt IN + interrupt OUT (LED reports)
+	"wake_hid":          {in: 1, out: 0}, // interrupt IN only
+	"absolute_mouse":    {in: 1, out: 0}, // interrupt IN only
+	"relative_mouse":    {in: 1, out: 0}, // interrupt IN only
+	"audio":             {in: 1, out: 0}, // UAC1 capture: isochronous IN
+	"mass_storage_base": {in: 1, out: 1}, // bulk IN + bulk OUT
+	"serial_console":    {in: 2, out: 1}, // CDC-ACM: bulk IN + notify IN + bulk OUT
+	"ncm":               {in: 2, out: 1}, // CDC-NCM: bulk IN + notify IN + bulk OUT
+}
+
+// endpointUsage returns the IN and OUT endpoint demand of a device selection,
+// including always-on functions (wake_hid, audio).
+func endpointUsage(devices *Devices) (in, out uint) {
+	for key, cost := range endpointCosts {
+		if isGadgetConfigItemEnabledForDevices(key, devices) {
+			in += cost.in
+			out += cost.out
+		}
+	}
+	return in, out
+}
+
+// ExceedsEndpointBudget reports whether the given device selection would exceed
+// the controller's IN or OUT endpoint budget. An over-budget gadget still
+// enumerates but leaves some function's endpoint(s) silently unallocated (e.g.
+// CDC-NCM comes up RX-only with a dead TX path), so the UI uses this to warn
+// before the user commits to the combination.
+func ExceedsEndpointBudget(devices *Devices) bool {
+	in, out := endpointUsage(devices)
+	return in > usbInEndpointBudget || out > usbOutEndpointBudget
+}

jsonrpc.go

@@ -968,6 +968,20 @@ func rpcGetUsbDevices() (usbgadget.Devices, error) {
 	return *config.UsbDevices, nil
 }
 
+// UsbEndpointReport tells the UI whether a given device selection would exceed
+// the controller's USB endpoint budget (IN or OUT). Over-budget combinations
+// leave a function silently non-functional — notably CDC-NCM, which needs two
+// IN endpoints, comes up RX-only with a dead TX path.
+type UsbEndpointReport struct {
+	ExceedsBudget bool `json:"exceedsBudget"`
+}
+
+func rpcGetUsbEndpointReport(devices usbgadget.Devices) (UsbEndpointReport, error) {
+	return UsbEndpointReport{
+		ExceedsBudget: usbgadget.ExceedsEndpointBudget(&devices),
+	}, nil
+}
+
 func updateUsbRelatedConfig() error {
 	if err := gadget.UpdateGadgetConfig(); err != nil {
 		return fmt.Errorf("failed to write gadget config: %w", err)
@@ -1399,6 +1413,7 @@ var rpcHandlers = map[string]RPCHandler{
 	"deleteSerialCommandHistory": {Func: rpcDeleteSerialCommandHistory},
 	"setTerminalPaused":          {Func: rpcSetTerminalPaused, Params: []string{"terminalPaused"}},
 	"getUsbDevices":              {Func: rpcGetUsbDevices},
+	"getUsbEndpointReport":       {Func: rpcGetUsbEndpointReport, Params: []string{"devices"}},
 	"setUsbDevices":              {Func: rpcSetUsbDevices, Params: []string{"devices"}},
 	"setUsbDeviceState":          {Func: rpcSetUsbDeviceState, Params: []string{"device", "enabled"}},
 	"setCloudUrl":                {Func: rpcSetCloudUrl, Params: []string{"apiUrl", "appUrl"}},

ui/src/components/UsbDeviceSetting.tsx

@@ -76,6 +76,7 @@ export function UsbDeviceSetting() {
 
   const [usbDeviceConfig, setUsbDeviceConfig] = useState<UsbDeviceConfig>(defaultUsbDeviceConfig);
   const [selectedPreset, setSelectedPreset] = useState<string>("default");
+  const [overBudget, setOverBudget] = useState(false);
 
   const syncUsbDeviceConfig = useCallback(() => {
     send("getUsbDevices", {}, (resp: JsonRpcResponse) => {
@@ -158,6 +159,17 @@ export function UsbDeviceSetting() {
     syncUsbDeviceConfig();
   }, [syncUsbDeviceConfig]);
 
+  // Check whether the currently-selected function set fits the controller's USB
+  // endpoint budget. The dwc3 controller has a limited number of IN/OUT
+  // endpoints; an over-budget combination can leave a function (notably
+  // CDC-NCM) silently non-functional.
+  useEffect(() => {
+    send("getUsbEndpointReport", { devices: usbDeviceConfig }, (resp: JsonRpcResponse) => {
+      if ("error" in resp) return;
+      setOverBudget((resp.result as { exceedsBudget: boolean }).exceedsBudget);
+    });
+  }, [send, usbDeviceConfig]);
+
   return (
     <Fieldset disabled={loading} className="space-y-4">
       <div className="h-px w-full bg-slate-800/10 dark:bg-slate-300/20" />
@@ -250,6 +262,14 @@ export function UsbDeviceSetting() {
               </SettingsItem>
             </div>
           </div>
+          {overBudget && (
+            <div className="mt-4 rounded-md border border-amber-500/30 bg-amber-50 p-3 text-sm text-amber-800 dark:border-amber-400/20 dark:bg-amber-900/20 dark:text-amber-200">
+              This combination exceeds the device&apos;s available USB endpoints. Functions beyond
+              the limit may silently fail to work &mdash; Ethernet over USB (CDC-NCM) in particular
+              will appear connected but won&apos;t pass traffic. Disable a function (e.g. Relative
+              Mouse) to free an endpoint.
+            </div>
+          )}
           <div className="mt-6 flex gap-x-2">
             <Button
               size="SM"