fix: handle fragmented ClientHello in auto-sense server

Firefox 148+ sends X25519MLKEM768 as its primary key_share, making the
ClientHello too large for a single DTLS record. The supported_versions
extension ends up in a later fragment, causing the auto-sense server to
miss it and fall back to DTLS 1.2.

Replace the lightweight packet-sniffing version detection (ServerPending)
with a design that uses Server13 directly in auto mode. The DTLS 1.3
engine handles fragment reassembly natively. After reassembling the
complete ClientHello:

- If supported_versions contains DTLS 1.3: proceed as 1.3 (zero overhead)
- If not: return Error::Dtls12Fallback, create Server12, replay packets

Changes:
- Add Error::Dtls12Fallback variant
- Add auto_mode + auto_packets to Server13
- Remove ServerPending from Inner enum and detect.rs
- new_auto() creates Server13::new_auto(), falls back to Server12 on
  Dtls12Fallback or ParseError
- Server13 buffers raw packets during AwaitClientHello, frees on commit
- 47 new tests: server_fallback (15) + cross_matrix (32) covering all
  version combinations at normal/fragmented/heavy MTU

Author: xnorpx

src/detect.rs

@@ -1,14 +1,20 @@
 /// Version detection and hybrid ClientHello for auto-sensing DTLS endpoints.
 ///
-/// Both `client_hello_version` and `server_hello_version` do lightweight
-/// parsing of the first record in a datagram, looking for the
-/// `supported_versions` extension to decide between DTLS 1.2 and 1.3.
+/// `server_hello_version` does lightweight parsing of the first record in a
+/// datagram. It looks for a `HelloVerifyRequest` or a `ServerHello` with the
+/// `supported_versions` extension to decide between DTLS 1.2 and 1.3 for the
+/// client auto-sense path.
 ///
 /// [`HybridClientHello`] constructs a ClientHello compatible with both
-/// DTLS 1.2 and 1.3 servers: it offers both version's cipher suites and
+/// DTLS 1.2 and 1.3 servers: it offers both versions cipher suites and
 /// includes `supported_versions` with both versions. Once the server
 /// responds, the caller inspects the reply with `server_hello_version`
 /// and forks into the appropriate handshake path.
+///
+/// Server-side auto-sense does not use lightweight detection. Instead,
+/// it starts a DTLS 1.3 server which handles fragment reassembly natively
+/// and falls back to DTLS 1.2 via [`Error::Dtls12Fallback`] if the
+/// reassembled ClientHello does not offer DTLS 1.3.
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
@@ -245,35 +251,6 @@ impl HybridClientHello {
     }
 }
 
-/// Auto-sense server waiting for the first ClientHello to determine the DTLS version.
-pub(crate) struct ServerPending {
-    config: Arc<Config>,
-    certificate: DtlsCertificate,
-    last_now: Instant,
-}
-
-impl ServerPending {
-    pub fn new(config: Arc<Config>, certificate: DtlsCertificate, now: Instant) -> Self {
-        ServerPending {
-            config,
-            certificate,
-            last_now: now,
-        }
-    }
-
-    pub fn handle_timeout(&mut self, now: Instant) {
-        self.last_now = now;
-    }
-
-    pub fn poll_output<'a>(&self, _buf: &'a mut [u8]) -> Output<'a> {
-        Output::Timeout(self.last_now + Duration::from_secs(86400))
-    }
-
-    pub fn into_parts(self) -> (Arc<Config>, DtlsCertificate, Instant) {
-        (self.config, self.certificate, self.last_now)
-    }
-}
-
 /// Auto-sense client that sends a hybrid ClientHello and waits for the server's response
 /// to determine the DTLS version.
 pub(crate) struct ClientPending {
@@ -373,117 +350,6 @@ pub(crate) enum DetectedVersion {
     Unknown,
 }
 
-/// Detect DTLS version from a ClientHello packet.
-///
-/// Returns `Dtls13` if the `supported_versions` extension contains
-/// DTLS 1.3 (0xFEFC), otherwise `Dtls12`.
-pub(crate) fn client_hello_version(packet: &[u8]) -> DetectedVersion {
-    match client_hello_version_inner(packet) {
-        Some(true) => DetectedVersion::Dtls13,
-        _ => DetectedVersion::Dtls12,
-    }
-}
-
-fn client_hello_version_inner(packet: &[u8]) -> Option<bool> {
-    // Record header: content_type(1) + version(2) + epoch(2) + seq(6) + length(2) = 13
-    if packet.len() < 13 {
-        return Some(false);
-    }
-
-    // content_type must be 0x16 (Handshake)
-    if packet[0] != 0x16 {
-        return Some(false);
-    }
-
-    let record_len = u16::from_be_bytes([packet[11], packet[12]]) as usize;
-    let record_body = packet.get(13..13 + record_len)?;
-
-    // Handshake header: msg_type(1) + length(3) + message_seq(2) +
-    //   fragment_offset(3) + fragment_length(3) = 12
-    if record_body.len() < 12 {
-        return Some(false);
-    }
-
-    // msg_type must be 1 (ClientHello)
-    if record_body[0] != 1 {
-        return Some(false);
-    }
-
-    let fragment_len = ((record_body[9] as usize) << 16)
-        | ((record_body[10] as usize) << 8)
-        | (record_body[11] as usize);
-    let body = record_body.get(12..12 + fragment_len)?;
-
-    // ClientHello body:
-    //   client_version(2) + random(32) = 34 minimum before session_id
-    if body.len() < 34 {
-        return Some(false);
-    }
-    let mut pos = 34;
-
-    // session_id: 1-byte length + data
-    let sid_len = *body.get(pos)? as usize;
-    pos += 1 + sid_len;
-
-    // cookie: 1-byte length + data
-    let cookie_len = *body.get(pos)? as usize;
-    pos += 1 + cookie_len;
-
-    // cipher_suites: 2-byte length + data
-    if pos + 2 > body.len() {
-        return Some(false);
-    }
-    let cs_len = u16::from_be_bytes([body[pos], body[pos + 1]]) as usize;
-    pos += 2 + cs_len;
-
-    // compression_methods: 1-byte length + data
-    let cm_len = *body.get(pos)? as usize;
-    pos += 1 + cm_len;
-
-    // extensions: 2-byte total length
-    if pos + 2 > body.len() {
-        return Some(false);
-    }
-    let ext_total_len = u16::from_be_bytes([body[pos], body[pos + 1]]) as usize;
-    pos += 2;
-    let ext_end = pos + ext_total_len;
-    if ext_end > body.len() {
-        return Some(false);
-    }
-
-    // Walk extensions looking for supported_versions (0x002B)
-    while pos + 4 <= ext_end {
-        let ext_type = u16::from_be_bytes([body[pos], body[pos + 1]]);
-        let ext_len = u16::from_be_bytes([body[pos + 2], body[pos + 3]]) as usize;
-        pos += 4;
-
-        if ext_type == 0x002B {
-            // supported_versions client format: 1-byte list length, then 2-byte versions
-            if ext_len < 1 {
-                return Some(false);
-            }
-            let list_len = body[pos] as usize;
-            let list_start = pos + 1;
-            if list_start + list_len > pos + ext_len {
-                return Some(false);
-            }
-            let mut i = list_start;
-            while i + 2 <= list_start + list_len {
-                let version = u16::from_be_bytes([body[i], body[i + 1]]);
-                if version == 0xFEFC {
-                    return Some(true);
-                }
-                i += 2;
-            }
-            return Some(false);
-        }
-
-        pos += ext_len;
-    }
-
-    Some(false)
-}
-
 /// Detect DTLS version from a server response (ServerHello or HelloVerifyRequest).
 ///
 /// - HelloVerifyRequest (msg_type 3) → `Dtls12`

src/dtls13/client.rs

@@ -210,7 +210,7 @@ impl Client {
     }
 
     pub fn into_server(self) -> Server {
-        Server::new_with_engine(self.engine, self.last_now)
+        Server::new_with_engine(self.engine, self.last_now, false)
     }
 
     pub(crate) fn state_name(&self) -> &'static str {

src/dtls13/server.rs

@@ -135,6 +135,15 @@ pub struct Server {
 
     /// Whether we need to respond with our own KeyUpdate.
     pending_key_update_response: bool,
+
+    /// When true, a ClientHello without DTLS 1.3 in `supported_versions`
+    /// returns [`Error::Dtls12Fallback`] instead of a security error.
+    /// Used by the auto-sense server path.
+    auto_mode: bool,
+
+    /// Raw packets buffered during auto-sense so they can be replayed
+    /// to a DTLS 1.2 server on fallback.
+    auto_packets: Vec<Vec<u8>>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -156,10 +165,21 @@ impl Server {
     /// Create a new DTLS 1.3 server.
     pub fn new(config: Arc<Config>, certificate: DtlsCertificate, now: Instant) -> Server {
         let engine = Engine::new(config, certificate);
-        Self::new_with_engine(engine, now)
+        Self::new_with_engine(engine, now, false)
+    }
+
+    /// Create a new DTLS 1.3 server in auto-sense mode.
+    ///
+    /// In auto-sense mode, if the ClientHello does not offer DTLS 1.3
+    /// in `supported_versions`, the server returns [`Error::Dtls12Fallback`]
+    /// instead of a fatal security error, allowing the caller to switch
+    /// to a DTLS 1.2 server.
+    pub fn new_auto(config: Arc<Config>, certificate: DtlsCertificate, now: Instant) -> Server {
+        let engine = Engine::new(config, certificate);
+        Self::new_with_engine(engine, now, true)
     }
 
-    pub(crate) fn new_with_engine(mut engine: Engine, now: Instant) -> Server {
+    pub(crate) fn new_with_engine(mut engine: Engine, now: Instant, auto_mode: bool) -> Server {
         let cookie_secret = engine.random_arr();
 
         Server {
@@ -183,23 +203,64 @@ impl Server {
             hello_retry: false,
             cookie_secret,
             pending_key_update_response: false,
+            auto_mode,
+            auto_packets: Vec::new(),
         }
     }
 
     pub fn into_client(self) -> Client {
         Client::new_with_engine(self.engine, self.last_now)
     }
 
+    /// Whether this server is in auto-sense mode.
+    pub fn is_auto_mode(&self) -> bool {
+        self.auto_mode
+    }
+
+    /// The last time instant seen by this server.
+    pub fn last_now(&self) -> Instant {
+        self.last_now
+    }
+
     pub(crate) fn state_name(&self) -> &'static str {
         self.state.name()
     }
 
     pub fn handle_packet(&mut self, packet: &[u8]) -> Result<(), Error> {
+        // In auto-sense mode, buffer raw packets while still waiting for
+        // the ClientHello so they can be replayed to Server12 on fallback.
+        if self.auto_mode && self.state == State::AwaitClientHello {
+            // Cap buffered fragments to prevent unbounded growth from malicious traffic
+            if self.auto_packets.len() >= 64 {
+                return Err(Error::SecurityError(
+                    "too many fragmented packets during auto-sense".to_string(),
+                ));
+            }
+            self.auto_packets.push(packet.to_vec());
+        }
+
         self.engine.parse_packet(packet)?;
         self.make_progress()?;
+
+        // Once past AwaitClientHello, DTLS 1.3 is committed — free the buffer.
+        if self.auto_mode && self.state != State::AwaitClientHello {
+            self.auto_packets.clear();
+            self.auto_mode = false;
+        }
+
         Ok(())
     }
 
+    /// Take the buffered raw packets for DTLS 1.2 fallback replay.
+    pub fn take_auto_packets(&mut self) -> Vec<Vec<u8>> {
+        std::mem::take(&mut self.auto_packets)
+    }
+
+    /// Number of buffered auto-sense packets.
+    pub fn auto_packet_count(&self) -> usize {
+        self.auto_packets.len()
+    }
+
     pub fn poll_output<'a>(&mut self, buf: &'a mut [u8]) -> Output<'a> {
         if let Some(event) = self.local_events.pop_front() {
             return event.into_output(buf, &self.client_certificates);
@@ -392,6 +453,9 @@ impl State {
         }
 
         if !supported_versions_ok {
+            if server.auto_mode {
+                return Err(Error::Dtls12Fallback);
+            }
             return Err(Error::SecurityError(
                 "ClientHello must include DTLS 1.3 in supported_versions".to_string(),
             ));

src/error.rs

@@ -36,6 +36,11 @@ pub enum Error {
     /// resolved.  Callers should buffer the data and retry once the
     /// handshake advances.
     HandshakePending,
+    /// The DTLS 1.3 server received a ClientHello that does not offer
+    /// DTLS 1.3 in `supported_versions`. In auto-sense mode the caller
+    /// should fall back to a DTLS 1.2 server and replay the buffered
+    /// packets.
+    Dtls12Fallback,
 }
 
 impl<'a> From<nom::Err<nom::error::Error<&'a [u8]>>> for Error {
@@ -69,6 +74,12 @@ impl std::fmt::Display for Error {
             Error::HandshakePending => {
                 write!(f, "handshake pending: cannot send application data yet")
             }
+            Error::Dtls12Fallback => {
+                write!(
+                    f,
+                    "dtls 1.2 fallback required: ClientHello does not offer DTLS 1.3"
+                )
+            }
         }
     }
 }