ctap2: take &mut Response in trait + outline call_ctap2 arms to reduce stack by 40kB

Author: 0x0ece

CHANGELOG.md

@@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 [Unreleased]: https://github.com/trussed-dev/ctap-types/compare/0.6.0-rc.3...HEAD
 
--
+- Improve `Authenticator` trait to reduce stack usage.
 
 ## [0.6.0-rc.3] 2026-05-28
 

src/ctap2.rs

@@ -508,14 +508,55 @@ pub enum Error {
 /// [`Response`].
 pub trait Authenticator {
     fn get_info(&mut self) -> get_info::Response;
+
+    /// Build a `MakeCredential` response into the caller-supplied buffer.
+    ///
+    /// Returns by-out-parameter instead of by-value because the response
+    /// is large (~6 KB with `mldsa44` enabled) and the CTAP dispatch chain
+    /// is several layers deep — every layer that carries a
+    /// `Result<Response>` reserves a Response-sized slot in its frame even
+    /// when NRVO would elide the final move. Writing into the caller's
+    /// slot keeps the chain flat (~6 KB total instead of ~6 KB per layer).
+    ///
+    /// Same pattern as `std::io::Read::read_to_end(&mut self, buf: &mut Vec<u8>)`
+    /// / `BufRead::read_line(buf: &mut String)`. For callers that don't
+    /// care about the stack copy, [`Authenticator::make_credential`] is a
+    /// convenience by-value wrapper.
+    fn make_credential_into(
+        &mut self,
+        request: &make_credential::Request,
+        response: &mut make_credential::Response,
+    ) -> Result<()>;
+
+    /// Convenience by-value wrapper around [`make_credential_into`] for
+    /// callers that don't need to avoid the stack copy.
     fn make_credential(
         &mut self,
         request: &make_credential::Request,
-    ) -> Result<make_credential::Response>;
+    ) -> Result<make_credential::Response> {
+        let mut response = make_credential::Response::empty();
+        self.make_credential_into(request, &mut response)?;
+        Ok(response)
+    }
+
+    /// Build a `GetAssertion` response into the caller-supplied buffer.
+    /// See [`make_credential_into`] for the rationale.
+    fn get_assertion_into(
+        &mut self,
+        request: &get_assertion::Request,
+        response: &mut get_assertion::Response,
+    ) -> Result<()>;
+
+    /// Convenience by-value wrapper around [`get_assertion_into`].
     fn get_assertion(
         &mut self,
         request: &get_assertion::Request,
-    ) -> Result<get_assertion::Response>;
+    ) -> Result<get_assertion::Response> {
+        let mut response = get_assertion::Response::empty();
+        self.get_assertion_into(request, &mut response)?;
+        Ok(response)
+    }
+
     fn get_next_assertion(&mut self) -> Result<get_assertion::Response>;
     fn reset(&mut self) -> Result<()>;
     fn client_pin(&mut self, request: &client_pin::Request) -> Result<client_pin::Response>;
@@ -537,120 +578,175 @@ pub trait Authenticator {
         Err(Error::InvalidCommand)
     }
 
-    /// Dispatches the enum of possible requests into the appropriate trait method.
+    /// Dispatches the enum of possible requests into the appropriate trait
+    /// method. Writes the result into `*response` in place, so the caller
+    /// owns the (~6 KB with `mldsa44`) Response storage and intermediate
+    /// frames don't carry duplicate copies. Saves ~6 KB per chain layer
+    /// vs the previous by-value-return shape.
+    ///
+    /// Each match arm that constructs a non-trivial Response variant is
+    /// outlined into its own `#[inline(never)]` helper below. Without that,
+    /// LLVM reserves a separate stack region for *every* variant's temporary
+    /// in `call_ctap2`'s frame (measured ~29 KB before outlining); with it,
+    /// only the currently-active arm's helper frame is live, shrinking the
+    /// shared dispatch frame to a small jump table.
     #[inline(never)]
-    fn call_ctap2(&mut self, request: &Request) -> Result<Response> {
+    fn call_ctap2(&mut self, request: &Request, response: &mut Response) -> Result<()> {
         match request {
             // 0x4
             Request::GetInfo => {
                 debug_now!("CTAP2.GI");
-                Ok(Response::GetInfo(self.get_info()))
+                self.dispatch_get_info(response)
             }
-
             // 0x2
             Request::MakeCredential(request) => {
                 debug_now!("CTAP2.MC");
-                Ok(Response::MakeCredential(
-                    self.make_credential(request).inspect_err(|_e| {
-                        debug!("error: {:?}", _e);
-                    })?,
-                ))
+                self.dispatch_make_credential(request, response)
             }
-
             // 0x1
             Request::GetAssertion(request) => {
                 debug_now!("CTAP2.GA");
-                Ok(Response::GetAssertion(
-                    self.get_assertion(request).inspect_err(|_e| {
-                        debug!("error: {:?}", _e);
-                    })?,
-                ))
+                self.dispatch_get_assertion(request, response)
             }
-
             // 0x8
             Request::GetNextAssertion => {
                 debug_now!("CTAP2.GNA");
-                Ok(Response::GetNextAssertion(
-                    self.get_next_assertion().inspect_err(|_e| {
-                        debug!("error: {:?}", _e);
-                    })?,
-                ))
+                self.dispatch_get_next_assertion(response)
             }
-
             // 0x7
             Request::Reset => {
                 debug_now!("CTAP2.RST");
                 self.reset().inspect_err(|_e| {
                     debug!("error: {:?}", _e);
                 })?;
-                Ok(Response::Reset)
+                *response = Response::Reset;
+                Ok(())
             }
-
             // 0x6
             Request::ClientPin(request) => {
                 debug_now!("CTAP2.PIN");
-                Ok(Response::ClientPin(self.client_pin(request).inspect_err(
-                    |_e| {
-                        debug!("error: {:?}", _e);
-                    },
-                )?))
+                self.dispatch_client_pin(request, response)
             }
-
             // 0xA
             Request::CredentialManagement(request) => {
                 debug_now!("CTAP2.CM");
-                Ok(Response::CredentialManagement(
-                    self.credential_management(request).inspect_err(|_e| {
-                        debug!("error: {:?}", _e);
-                    })?,
-                ))
+                self.dispatch_credential_management(request, response)
             }
-
             // 0xB
             Request::Selection => {
                 debug_now!("CTAP2.SEL");
                 self.selection().inspect_err(|_e| {
                     debug!("error: {:?}", _e);
                 })?;
-                Ok(Response::Selection)
+                *response = Response::Selection;
+                Ok(())
             }
-
             // 0xC
             Request::LargeBlobs(request) => {
                 debug_now!("CTAP2.LB");
-                Ok(Response::LargeBlobs(
-                    self.large_blobs(request).inspect_err(|_e| {
-                        debug!("error: {:?}", _e);
-                    })?,
-                ))
+                self.dispatch_large_blobs(request, response)
             }
-
             // 0xD
             Request::Config(request) => {
                 debug_now!("CTAP2.CFG");
                 self.config(request).inspect_err(|_e| {
                     debug!("error: {:?}", _e);
                 })?;
-                Ok(Response::Config)
+                *response = Response::Config;
+                Ok(())
             }
-
             // Not stable
             Request::Vendor(op) => {
                 debug_now!("CTAP2.V");
                 self.vendor(*op).inspect_err(|_e| {
                     debug!("error: {:?}", _e);
                 })?;
-                Ok(Response::Vendor)
+                *response = Response::Vendor;
+                Ok(())
             }
         }
     }
-}
 
-impl<A: Authenticator> crate::Rpc<Error, Request<'_>, Response> for A {
-    /// Dispatches the enum of possible requests into the appropriate trait method.
     #[inline(never)]
-    fn call(&mut self, request: &Request) -> Result<Response> {
-        self.call_ctap2(request)
+    fn dispatch_get_info(&mut self, response: &mut Response) -> Result<()> {
+        *response = Response::GetInfo(self.get_info());
+        Ok(())
+    }
+
+    #[inline(never)]
+    fn dispatch_make_credential(
+        &mut self,
+        request: &make_credential::Request,
+        response: &mut Response,
+    ) -> Result<()> {
+        *response = Response::MakeCredential(make_credential::Response::empty());
+        let Response::MakeCredential(inner) = response else {
+            unreachable!()
+        };
+        self.make_credential_into(request, inner).inspect_err(|_e| {
+            debug!("error: {:?}", _e);
+        })
+    }
+
+    #[inline(never)]
+    fn dispatch_get_assertion(
+        &mut self,
+        request: &get_assertion::Request,
+        response: &mut Response,
+    ) -> Result<()> {
+        *response = Response::GetAssertion(get_assertion::Response::empty());
+        let Response::GetAssertion(inner) = response else {
+            unreachable!()
+        };
+        self.get_assertion_into(request, inner).inspect_err(|_e| {
+            debug!("error: {:?}", _e);
+        })
+    }
+
+    #[inline(never)]
+    fn dispatch_get_next_assertion(&mut self, response: &mut Response) -> Result<()> {
+        *response = Response::GetNextAssertion(self.get_next_assertion().inspect_err(|_e| {
+            debug!("error: {:?}", _e);
+        })?);
+        Ok(())
+    }
+
+    #[inline(never)]
+    fn dispatch_client_pin(
+        &mut self,
+        request: &client_pin::Request,
+        response: &mut Response,
+    ) -> Result<()> {
+        *response = Response::ClientPin(self.client_pin(request).inspect_err(|_e| {
+            debug!("error: {:?}", _e);
+        })?);
+        Ok(())
+    }
+
+    #[inline(never)]
+    fn dispatch_credential_management(
+        &mut self,
+        request: &credential_management::Request,
+        response: &mut Response,
+    ) -> Result<()> {
+        *response = Response::CredentialManagement(
+            self.credential_management(request).inspect_err(|_e| {
+                debug!("error: {:?}", _e);
+            })?,
+        );
+        Ok(())
+    }
+
+    #[inline(never)]
+    fn dispatch_large_blobs(
+        &mut self,
+        request: &large_blobs::Request,
+        response: &mut Response,
+    ) -> Result<()> {
+        *response = Response::LargeBlobs(self.large_blobs(request).inspect_err(|_e| {
+            debug!("error: {:?}", _e);
+        })?);
+        Ok(())
     }
 }
 

src/ctap2/get_assertion.rs

@@ -1,4 +1,4 @@
-use crate::{Bytes, Vec};
+use crate::{Bytes, String, Vec};
 use cosey::EcdhEsHkdf256PublicKey;
 use serde::{Deserialize, Serialize};
 use serde_bytes::ByteArray;
@@ -179,6 +179,24 @@ impl ResponseBuilder {
     }
 }
 
+impl Response {
+    /// Empty `Response` with default fields. Used by `Authenticator::call_ctap2`
+    /// to preallocate the `Response::GetAssertion` variant slot so the inner
+    /// `get_assertion` impl can write via `&mut` — same shape as
+    /// `make_credential::Response::empty()`.
+    pub fn empty() -> Self {
+        ResponseBuilder {
+            credential: PublicKeyCredentialDescriptor {
+                id: Bytes::new(),
+                key_type: String::new(),
+            },
+            auth_data: Bytes::new(),
+            signature: Bytes::new(),
+        }
+        .build()
+    }
+}
+
 #[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
 #[non_exhaustive]
 pub struct UnsignedExtensionOutputs {}

src/ctap2/make_credential.rs

@@ -207,6 +207,24 @@ impl ResponseBuilder {
     }
 }
 
+impl Response {
+    /// Empty `Response` with default fields. Used by `Authenticator::call_ctap2`
+    /// to preallocate the `Response::MakeCredential` variant slot in the
+    /// outer `ctap2::Response` so the inner `make_credential` impl can write
+    /// via `&mut` — saves the ~6 KB return-by-value copy through the
+    /// dispatch chain. Inner is sized to the type's full capacity
+    /// (`auth_data` is `Bytes<AUTHENTICATOR_DATA_LENGTH>` ≈ 2 KB with
+    /// `mldsa44`); the slot is allocated either way, the win is that it
+    /// lives in ONE place instead of three.
+    pub fn empty() -> Self {
+        ResponseBuilder {
+            fmt: AttestationStatementFormat::None,
+            auth_data: super::SerializedAuthenticatorData::new(),
+        }
+        .build()
+    }
+}
+
 #[derive(Clone, Debug, Eq, PartialEq, Serialize)]
 #[cfg_attr(feature = "platform-serde", derive(Deserialize))]
 #[non_exhaustive]