Tweak DTLS13 Server auto sensing

Author: algesten

src/detect.rs src/auto.rssrc/detect.rs renamed to src/auto.rs

@@ -170,7 +170,7 @@ impl Client {
     /// already sent on the wire by `ClientPending`, so no record is
     /// enqueued for output.
     pub(crate) fn new_from_hybrid(
-        hybrid: crate::detect::HybridClientHello,
+        hybrid: crate::auto::HybridClientHello,
         config: std::sync::Arc<crate::Config>,
         certificate: crate::DtlsCertificate,
         now: Instant,

src/dtls13/client.rs

@@ -27,7 +27,7 @@ use crate::dtls13::message::Sequence;
 use crate::timer::ExponentialBackoff;
 use crate::types::{HashAlgorithm, Random};
 use crate::window::ReplayWindow;
-use crate::{Config, Error, Output, SeededRng};
+use crate::{Config, DtlsCertificate, Error, Output, SeededRng};
 
 const MAX_DEFRAGMENT_PACKETS: usize = 50;
 
@@ -39,6 +39,9 @@ pub struct Engine {
     /// Configuration options.
     config: Arc<Config>,
 
+    /// Saved certificate
+    certificate: DtlsCertificate,
+
     /// Seedable random number generator for deterministic testing
     rng: SeededRng,
 
@@ -93,9 +96,6 @@ pub struct Engine {
     /// Whether the remote peer has enabled encryption
     peer_encryption_enabled: bool,
 
-    /// Certificate in DER format
-    certificate_der: Vec<u8>,
-
     /// Signing key for CertificateVerify
     signing_key: Box<dyn SigningKey>,
 
@@ -189,7 +189,7 @@ struct Entry {
 }
 
 impl Engine {
-    pub fn new(config: Arc<Config>, certificate: crate::DtlsCertificate) -> Self {
+    pub fn new(config: Arc<Config>, certificate: DtlsCertificate) -> Self {
         let mut rng = SeededRng::new(config.rng_seed());
 
         let flight_backoff =
@@ -206,6 +206,7 @@ impl Engine {
 
         Self {
             config,
+            certificate,
             rng,
             buffers_free: BufferPool::default(),
             sequence_epoch_0: Sequence::new(0),
@@ -224,7 +225,6 @@ impl Engine {
             prev_app_send_seq: 0,
             app_recv_keys: ArrayVec::new(),
             peer_encryption_enabled: false,
-            certificate_der: certificate.certificate,
             signing_key,
             is_client: false,
             peer_handshake_seq_no: 0,
@@ -246,6 +246,10 @@ impl Engine {
         }
     }
 
+    pub fn into_fallback(self) -> (Arc<Config>, DtlsCertificate) {
+        (self.config, self.certificate)
+    }
+
     pub fn set_client(&mut self, is_client: bool) {
         self.is_client = is_client;
     }
@@ -307,7 +311,7 @@ impl Engine {
     }
 
     pub fn certificate_der(&self) -> &[u8] {
-        &self.certificate_der
+        &self.certificate.certificate
     }
 
     pub fn signing_key(&mut self) -> &mut dyn SigningKey {

src/dtls13/engine.rs

@@ -74,6 +74,8 @@ const HRR_RANDOM: [u8; 32] = [
     0xC2, 0xA2, 0x11, 0x16, 0x7A, 0xBB, 0x8C, 0x5E, 0x07, 0x9E, 0x09, 0xE2, 0xC8, 0xA8, 0x33, 0x9C,
 ];
 
+const MAX_RETAINED_CLIENT_HELLO: usize = 64;
+
 /// DTLS 1.3 server
 pub struct Server {
     /// Current server state.
@@ -143,7 +145,7 @@ pub struct Server {
 
     /// Raw packets buffered during auto-sense so they can be replayed
     /// to a DTLS 1.2 server on fallback.
-    auto_packets: Vec<Vec<u8>>,
+    retained_hello: VecDeque<Buf>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -179,7 +181,7 @@ impl Server {
         Self::new_with_engine(engine, now, true)
     }
 
-    pub(crate) fn new_with_engine(mut engine: Engine, now: Instant, auto_mode: bool) -> Server {
+    pub fn new_with_engine(mut engine: Engine, now: Instant, auto_mode: bool) -> Server {
         let cookie_secret = engine.random_arr();
 
         Server {
@@ -204,7 +206,7 @@ impl Server {
             cookie_secret,
             pending_key_update_response: false,
             auto_mode,
-            auto_packets: Vec::new(),
+            retained_hello: VecDeque::with_capacity(10),
         }
     }
 
@@ -217,9 +219,15 @@ impl Server {
         self.auto_mode
     }
 
-    /// The last time instant seen by this server.
-    pub fn last_now(&self) -> Instant {
-        self.last_now
+    /// Take all relevant config from this server instance.
+    ///
+    /// This is used in two cases:
+    ///
+    /// 1. Switching a server pending (auto-mode) to dtls12 server
+    /// 2. set_active(true), turning a server pending (auto-mode) to a ClientPending
+    pub fn into_parts(self) -> (Arc<Config>, DtlsCertificate, Instant, VecDeque<Buf>) {
+        let (config, cert) = self.engine.into_fallback();
+        (config, cert, self.last_now, self.retained_hello)
     }
 
     pub(crate) fn state_name(&self) -> &'static str {
@@ -231,36 +239,24 @@ impl Server {
         // the ClientHello so they can be replayed to Server12 on fallback.
         if self.auto_mode && self.state == State::AwaitClientHello {
             // Cap buffered fragments to prevent unbounded growth from malicious traffic
-            if self.auto_packets.len() >= 64 {
-                return Err(Error::SecurityError(
-                    "too many fragmented packets during auto-sense".to_string(),
-                ));
+            if self.retained_hello.len() >= MAX_RETAINED_CLIENT_HELLO {
+                return Err(Error::TooManyClientHelloFragments);
             }
-            self.auto_packets.push(packet.to_vec());
+            self.retained_hello.push_back(packet.to_buf());
         }
 
         self.engine.parse_packet(packet)?;
         self.make_progress()?;
 
         // Once past AwaitClientHello, DTLS 1.3 is committed — free the buffer.
         if self.auto_mode && self.state != State::AwaitClientHello {
-            self.auto_packets.clear();
+            self.retained_hello.clear();
             self.auto_mode = false;
         }
 
         Ok(())
     }
 
-    /// Take the buffered raw packets for DTLS 1.2 fallback replay.
-    pub fn take_auto_packets(&mut self) -> Vec<Vec<u8>> {
-        std::mem::take(&mut self.auto_packets)
-    }
-
-    /// Number of buffered auto-sense packets.
-    pub fn auto_packet_count(&self) -> usize {
-        self.auto_packets.len()
-    }
-
     pub fn poll_output<'a>(&mut self, buf: &'a mut [u8]) -> Output<'a> {
         if let Some(event) = self.local_events.pop_front() {
             return event.into_output(buf, &self.client_certificates);

src/dtls13/server.rs

@@ -36,10 +36,17 @@ pub enum Error {
     /// resolved.  Callers should buffer the data and retry once the
     /// handshake advances.
     HandshakePending,
+    /// If we are in auto-sense mode for a server and we received too
+    /// many client hello fragments that haven't made a packet.
+    TooManyClientHelloFragments,
     /// The DTLS 1.3 server received a ClientHello that does not offer
     /// DTLS 1.3 in `supported_versions`. In auto-sense mode the caller
     /// should fall back to a DTLS 1.2 server and replay the buffered
     /// packets.
+    ///
+    /// This value should never be seen outside dimpl. It's an internal
+    /// value to communicate from dtls13/server.rs to lib.rs
+    #[doc(hidden)]
     Dtls12Fallback,
 }
 
@@ -74,11 +81,9 @@ impl std::fmt::Display for Error {
             Error::HandshakePending => {
                 write!(f, "handshake pending: cannot send application data yet")
             }
+            Error::TooManyClientHelloFragments => write!(f, "too many client hello fragments"),
             Error::Dtls12Fallback => {
-                write!(
-                    f,
-                    "dtls 1.2 fallback required: ClientHello does not offer DTLS 1.3"
-                )
+                write!(f, "dtls 1.2 fallback (internal)")
             }
         }
     }