Fix infinite loops and timeout handling in SPI and CAN

- SPI _wait_for_ack: default timeout=0 to 500ms, clamp to 100-500ms range
- SPI _transfer: default timeout=0 to 500ms, add MAX_TIMEOUT_RETRIES (5) limit
- SPI _transfer: add NACK backoff like C++ implementation
- SPI _transfer: run recovery logic for ALL exception types (fixes onroad test)
- CAN can_recv: max 3 retries instead of infinite loop, return [] on failure
- CAN can_send_many: detect no-progress and drop after 3 retries
- CAN_SEND_TIMEOUT_MS: changed from 10ms to 5ms to match C++

These changes align Python behavior with the C++ pandad implementation.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: adeebshihadeh

python/__init__.py

@@ -692,7 +692,8 @@ def set_uart_callback(self, uart, install):
   # The panda will NAK CAN writes when there is CAN congestion.
   # libusb will try to send it again, with a max timeout.
   # Timeout is in ms. If set to 0, the timeout is infinite.
-  CAN_SEND_TIMEOUT_MS = 10
+  CAN_SEND_TIMEOUT_MS = 5  # 5ms like C++ implementation
+  CAN_MAX_RETRIES = 3
 
   def can_reset_communications(self):
     self._handle.controlWrite(Panda.REQUEST_OUT, 0xc0, 0, 0, b'')
@@ -701,8 +702,16 @@ def can_reset_communications(self):
   def can_send_many(self, arr, *, fd=False, timeout=CAN_SEND_TIMEOUT_MS):
     snds = pack_can_buffer(arr, chunk=(not self.spi), fd=fd)
     for tx in snds:
+      retries = 0
       while len(tx) > 0:
         bs = self._handle.bulkWrite(3, tx, timeout=timeout)
+        if bs == 0:
+          retries += 1
+          if retries > self.CAN_MAX_RETRIES:
+            logger.warning("CAN send: no progress after retries, dropping")
+            break
+        else:
+          retries = 0
         tx = tx[bs:]
 
   def can_send(self, addr, dat, bus, *, fd=False, timeout=CAN_SEND_TIMEOUT_MS):
@@ -711,13 +720,16 @@ def can_send(self, addr, dat, bus, *, fd=False, timeout=CAN_SEND_TIMEOUT_MS):
   @ensure_can_packet_version
   def can_recv(self):
     dat = bytearray()
-    while True:
+    for _ in range(self.CAN_MAX_RETRIES):
       try:
         dat = self._handle.bulkRead(1, 16384) # Max receive batch size + 2 extra reserve frames
         break
       except (usb1.USBErrorIO, usb1.USBErrorOverflow):
         logger.error("CAN: BAD RECV, RETRYING")
-        time.sleep(0.1)
+        time.sleep(0.01)  # 10ms sleep, not 100ms
+    else:
+      logger.error("CAN: recv failed after retries")
+      return []
     msgs, self.can_rx_overflow_buffer = unpack_can_buffer(self.can_rx_overflow_buffer + dat)
     return msgs
 

python/spi.py

@@ -25,7 +25,10 @@
 CHECKSUM_START = 0xAB
 
 MIN_ACK_TIMEOUT_MS = 100
+MAX_ACK_TIMEOUT_MS = 500  # like C++ SPI_ACK_TIMEOUT
+DEFAULT_TIMEOUT_MS = 500  # default when timeout=0
 MAX_XFER_RETRY_COUNT = 5
+MAX_TIMEOUT_RETRIES = 5  # like C++
 
 SPI_BUF_SIZE = 4096  # from panda/board/drivers/spi.h
 XFER_SIZE = SPI_BUF_SIZE - 0x40 # give some room for SPI protocol overhead
@@ -127,10 +130,14 @@ def _calc_checksum(self, data: bytes) -> int:
     return cksum
 
   def _wait_for_ack(self, spi, ack_val: int, timeout: int, tx: int, length: int = 1) -> bytes:
-    timeout_s = max(MIN_ACK_TIMEOUT_MS, timeout) * 1e-3
+    # Like C++: default timeout=0 to MAX_ACK_TIMEOUT_MS, clamp to valid range
+    if timeout == 0:
+      timeout = MAX_ACK_TIMEOUT_MS
+    timeout = max(MIN_ACK_TIMEOUT_MS, min(timeout, MAX_ACK_TIMEOUT_MS))
+    timeout_s = timeout * 1e-3
 
     start = time.monotonic()
-    while (timeout == 0) or ((time.monotonic() - start) < timeout_s):
+    while (time.monotonic() - start) < timeout_s:
       dat = spi.xfer2([tx, ] * length)
       if dat[0] == ack_val:
         return bytes(dat)
@@ -182,10 +189,27 @@ def _transfer(self, endpoint: int, data, timeout: int, max_rx_len: int = 1000, e
     logger.debug("starting transfer: endpoint=%d, max_rx_len=%d", endpoint, max_rx_len)
     logger.debug("==============================================")
 
+    # Like C++: default timeout=0 to DEFAULT_TIMEOUT_MS for individual ACK waits
+    if timeout == 0:
+      timeout = DEFAULT_TIMEOUT_MS
+
     n = 0
+    timeout_count = 0
+    nack_count = 0
     start_time = time.monotonic()
     exc = PandaSpiException()
-    while (timeout == 0) or (time.monotonic() - start_time) < timeout*1e-3:
+
+    # Like C++: retry based on count, not overall time limit
+    # C++ loops while (ret < 0 && connected && !timed_out) where timed_out = timeout_count > 5
+    # C++ only exits on consecutive ACK_TIMEOUT, not on NACK (panda is busy, keep trying)
+    # Safety limit: 30 seconds max to prevent infinite loops (C++ uses 'connected' flag we don't have)
+    max_time = 30.0  # seconds
+    while (time.monotonic() - start_time) < max_time:
+      # Like C++: stop after MAX_TIMEOUT_RETRIES consecutive timeouts
+      if timeout_count > MAX_TIMEOUT_RETRIES:
+        logger.warning("SPI: too many timeouts (%d), giving up", timeout_count)
+        break
+
       n += 1
       logger.debug("\ntry #%d", n)
       with self.dev.acquire() as spi:
@@ -197,6 +221,21 @@ def _transfer(self, endpoint: int, data, timeout: int, max_rx_len: int = 1000, e
           if self.no_retry:
             break
 
+          # Track specific error types like C++
+          if isinstance(e, PandaSpiMissingAck):
+            timeout_count += 1
+          elif isinstance(e, PandaSpiNackResponse):
+            nack_count += 1
+            # Reset timeout count - NACK is not a timeout, panda is responding
+            timeout_count = 0
+            # Like C++: backoff on repeated NACKs (panda TX buffer full)
+            if nack_count > 3:
+              sleep_us = max(200, min(nack_count * 10, 2000))
+              time.sleep(sleep_us * 1e-6)
+          else:
+            # For other errors (checksum, etc), reset timeout count
+            timeout_count = 0
+
           # ensure slave is in a consistent state and ready for the next transfer
           # (e.g. slave TX buffer isn't stuck full)
           nack_cnt = 0
@@ -209,6 +248,7 @@ def _transfer(self, endpoint: int, data, timeout: int, max_rx_len: int = 1000, e
             except PandaSpiException:
               nack_cnt = 0
 
+    logger.error("SPI transfer failed after %d tries, %.2fms", n, (time.monotonic() - start_time) * 1000)
     raise exc
 
   def get_protocol_version(self) -> bytes: