fix: return HandshakePending for auto-sense pending state

Author: xnorpx

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Unreleased
 
+  * Fix panic in auto DTLS version selection #65
+  * Add `Error::HandshakePending` for auto-sense pending state (breaking) #65
   * DTLS 1.2 ECDSA determine curve from certificate, not hash algorithm #57
   * DTLS 1.3 enforce SignatureScheme curve matches certificate key #60
 

src/dtls12/client.rs

@@ -124,7 +124,7 @@ impl Client {
         config: std::sync::Arc<crate::Config>,
         certificate: crate::DtlsCertificate,
         now: Instant,
-    ) -> Client {
+    ) -> Result<Client, Error> {
         let mut engine = Engine::new(config, certificate);
         engine.set_client(true);
         // The hybrid ClientHello was sent with message_seq=0 outside this
@@ -137,7 +137,7 @@ impl Client {
         // Advance epoch-0 record sequence past the hybrid CH record.
         engine.advance_epoch_0_sequence();
 
-        Client {
+        let mut client = Client {
             state: State::AwaitHelloVerifyRequest,
             engine,
             random: Some(random),
@@ -153,7 +153,9 @@ impl Client {
             last_now: now,
             local_events: VecDeque::new(),
             queued_data: Vec::new(),
-        }
+        };
+        client.handle_timeout(now)?;
+        Ok(client)
     }
 
     pub fn into_server(self) -> Server {

src/dtls12/engine.rs

@@ -446,7 +446,9 @@ impl Engine {
             }
             (Timeout::Armed(c), _) => c,
             (_, Timeout::Armed(f)) => f,
-            _ => unreachable!(),
+            // Both Unarmed or mixed Unarmed/Disabled: return current time
+            // to trigger handle_timeout on the next cycle.
+            _ => now,
         }
     }
 

src/dtls13/client.rs

@@ -174,15 +174,15 @@ impl Client {
         config: std::sync::Arc<crate::Config>,
         certificate: crate::DtlsCertificate,
         now: Instant,
-    ) -> Client {
+    ) -> Result<Client, Error> {
         let mut engine = Engine::new(config, certificate);
         engine.set_client(true);
 
         // Inject transcript + sequence state from the hybrid CH that was
         // already sent on the wire by ClientPending.
         engine.inject_hybrid_client_hello(&hybrid.transcript_bytes);
 
-        Client {
+        let mut client = Client {
             state: State::AwaitServerHello,
             engine,
             random: Some(hybrid.random),
@@ -204,7 +204,9 @@ impl Client {
             handshake_secret: None,
             client_hs_traffic_secret: None,
             server_hs_traffic_secret: None,
-        }
+        };
+        client.handle_timeout(now)?;
+        Ok(client)
     }
 
     pub fn into_server(self) -> Server {

src/error.rs

@@ -29,6 +29,12 @@ pub enum Error {
     TooManyRecords,
     /// Peer attempted renegotiation (not supported)
     RenegotiationAttempt,
+    /// Application data cannot be sent because the handshake is not yet complete.
+    ///
+    /// For auto-sense instances this means the version has not yet been
+    /// resolved.  Callers should buffer the data and retry once the
+    /// handshake advances.
+    HandshakePending,
 }
 
 impl<'a> From<nom::Err<nom::error::Error<&'a [u8]>>> for Error {
@@ -59,6 +65,9 @@ impl std::fmt::Display for Error {
             Error::ConfigError(msg) => write!(f, "config error: {}", msg),
             Error::TooManyRecords => write!(f, "too many records in packet"),
             Error::RenegotiationAttempt => write!(f, "peer attempted renegotiation"),
+            Error::HandshakePending => {
+                write!(f, "handshake pending: cannot send application data yet")
+            }
         }
     }
 }

src/lib.rs

@@ -358,21 +358,30 @@ impl Dtls {
                 detect::client_hello_version(packet),
                 detect::DetectedVersion::Dtls13
             );
-            let resolved = if is_13 {
-                let mut server = Server13::new(config, certificate, now);
-                server.handle_timeout(now)?;
-                Inner::Server13(server)
+            self.inner = if is_13 {
+                Some(Inner::Server13(Server13::new(config, certificate, now)))
             } else {
-                let mut server = Server12::new(config, certificate, now);
-                server.handle_timeout(now)?;
-                Inner::Server12(server)
+                Some(Inner::Server12(Server12::new(config, certificate, now)))
             };
-            self.inner = Some(resolved);
+
+            // Arm the server's random + retransmit timers.
+            // inner is already set, so errors won't leave it as None.
+            self.handle_timeout(now)?;
         }
 
         // Auto-sense client: resolve version on first server response
         if matches!(self.inner, Some(Inner::ClientPending(_))) {
             let version = detect::server_hello_version(packet);
+
+            // Check version before taking inner — returning an error
+            // while inner is None would leave us unable to poll/timeout.
+            if matches!(version, detect::DetectedVersion::Unknown) {
+                return Err(Error::UnexpectedMessage(
+                    "Unrecognized response from server".to_string(),
+                ));
+            }
+
+            // unwrap: guarded by the matches! check above
             let inner = self.inner.take().unwrap();
             let Inner::ClientPending(cp) = inner else {
                 unreachable!()
@@ -386,24 +395,26 @@ impl Dtls {
                         config,
                         certificate,
                         now,
-                    );
+                    )?;
                     // Feed the HVR to Client12 — it enters
                     // AwaitHelloVerifyRequest and processes the cookie.
-                    client12.handle_packet(packet)?;
+                    if let Err(e) = client12.handle_packet(packet) {
+                        self.inner = Some(Inner::Client12(client12));
+                        return Err(e);
+                    }
                     self.inner = Some(Inner::Client12(client12));
                     return Ok(());
                 }
                 detect::DetectedVersion::Dtls13 => {
-                    let mut client13 = Client13::new_from_hybrid(hybrid, config, certificate, now);
-                    client13.handle_packet(packet)?;
+                    let mut client13 = Client13::new_from_hybrid(hybrid, config, certificate, now)?;
+                    if let Err(e) = client13.handle_packet(packet) {
+                        self.inner = Some(Inner::Client13(client13));
+                        return Err(e);
+                    }
                     self.inner = Some(Inner::Client13(client13));
                     return Ok(());
                 }
-                detect::DetectedVersion::Unknown => {
-                    return Err(Error::UnexpectedMessage(
-                        "Unrecognized response from server".to_string(),
-                    ));
-                }
+                detect::DetectedVersion::Unknown => unreachable!(),
             }
         }
 
@@ -444,18 +455,17 @@ impl Dtls {
     }
 
     /// Send application data over the established DTLS session.
+    ///
+    /// Returns [`Error::HandshakePending`] if the DTLS version has not
+    /// yet been resolved (auto-sense pending).  Callers should buffer
+    /// the data externally and retry after the handshake progresses.
     pub fn send_application_data(&mut self, data: &[u8]) -> Result<(), Error> {
         match self.inner.as_mut().unwrap() {
             Inner::Client12(client) => client.send_application_data(data),
             Inner::Server12(server) => server.send_application_data(data),
             Inner::Client13(client) => client.send_application_data(data),
             Inner::Server13(server) => server.send_application_data(data),
-            Inner::ServerPending(_) => Err(Error::UnexpectedMessage(
-                "cannot send application data: handshake not started".to_string(),
-            )),
-            Inner::ClientPending(_) => Err(Error::UnexpectedMessage(
-                "cannot send application data: handshake not complete".to_string(),
-            )),
+            Inner::ServerPending(_) | Inner::ClientPending(_) => Err(Error::HandshakePending),
         }
     }
 }
@@ -583,6 +593,34 @@ mod test {
         assert!(matches!(result, Output::Timeout(_)));
     }
 
+    #[test]
+    fn test_auto_client_unknown_version_no_panic() {
+        // Regression: handle_packet returning UnexpectedMessage for an
+        // unrecognized server response must not leave inner as None,
+        // which would panic on the next poll_output/handle_timeout.
+        let mut dtls = new_instance_auto();
+        dtls.set_active(true);
+        let now = Instant::now();
+        dtls.handle_timeout(now).unwrap();
+
+        // Drain the hybrid ClientHello
+        let mut buf = [0u8; 2048];
+        loop {
+            if matches!(dtls.poll_output(&mut buf), Output::Timeout(_)) {
+                break;
+            }
+        }
+
+        // Feed a garbage packet that won't be recognized as DTLS 1.2 or 1.3
+        let garbage = [0xFF; 64];
+        let err = dtls.handle_packet(&garbage).unwrap_err();
+        assert!(matches!(err, Error::UnexpectedMessage(_)));
+
+        // These must NOT panic — inner should still be intact
+        dtls.handle_timeout(now).unwrap();
+        let _ = dtls.poll_output(&mut buf);
+    }
+
     #[test]
     fn is_send() {
         fn is_send<T: Send>(_t: T) {}