feat: add RateLimitLayer middleware (#320)

Uses governor for RPM limiting, placed below the retry layer so retries
are also throttled. Reads x-ratelimit-remaining-requests and
x-ratelimit-reset-requests headers to apply backpressure when the server
quota is exhausted. Gated behind the rate-limit feature.

Author: Linuxdazhao

async-openai/Cargo.toml

@@ -52,6 +52,7 @@ assistant = ["assistant-types", "_api", ]
 administration = ["administration-types", "_api"]
 completions = ["completion-types", "_api"]
 middleware = ["dep:tower", "_api"]
+rate-limit = ["dep:governor", "middleware"]
 
 # Type feature flags - these enable only the types
 response-types = ["dep:derive_builder"]
@@ -132,6 +133,7 @@ full = [
     "types",
     "byot",
     "middleware",
+    "rate-limit",
 ]
 
 # Internal feature to enable API dependencies
@@ -180,6 +182,7 @@ secrecy = { version = "0.10", features = ["serde"], optional = true }
 serde_urlencoded = { version = "0.7", optional = true }
 url = { version = "2.5", optional = true }
 tower = { version = "0.5", features = ["limit", "retry", "timeout", "util"], optional = true }
+governor = { version = "0.8", optional = true, default-features = false, features = ["std"] }
 eventsource-stream = { version = "0.2", optional = true }
 ## For Webhook signature verification
 hmac = { version = "0.12", optional = true, default-features = false}
@@ -197,6 +200,7 @@ tokio-tungstenite = { version = "0.28", optional = true, default-features = fals
 ## API WASM dependencies
 [target.wasm32-unknown-unknown.dependencies]
 getrandom = { version = "0.3", features = ["wasm_js"] }
+governor = { version = "0.8", optional = true, default-features = false, features = ["std"] }
 
 [dev-dependencies]
 http = "1"

async-openai/MIDDLEWARE.md

@@ -91,6 +91,22 @@ Custom tower retry policies can call `middleware::retry::should_retry` to reuse
 
 On native targets retries wait using `tokio::time::sleep`. On WASM retries are immediate.
 
+## Rate limit layer
+
+Enable the `rate-limit` feature to use `RateLimitLayer`.
+
+```rust
+use async_openai::middleware::{retry::OpenAIRetryLayer, ReqwestService};
+use async_openai::middleware::rate_limit::RateLimitLayer;
+
+let service = tower::ServiceBuilder::new()
+    .layer(OpenAIRetryLayer::default())
+    .layer(RateLimitLayer::per_minute(60))
+    .service(ReqwestService::new(reqwest::Client::new()));
+```
+
+`RateLimitLayer` should sit below `OpenAIRetryLayer` so retries are also throttled. On native targets, the layer provides local request-per-minute limiting and server-feedback backpressure from `x-ratelimit-remaining-requests` plus `x-ratelimit-reset-requests`. On WASM targets, it records local governor state but does not delay requests.
+
 ## Error Handling
 
 `OpenAIError::Boxed` is available only when the `middleware` feature is enabled.

async-openai/src/lib.rs

@@ -29,6 +29,7 @@
 //! ## Making requests
 //!
 //!```
+//!# #[cfg(feature = "responses")]
 //!# tokio_test::block_on(async {
 //! use async_openai::{Client, types::responses::{CreateResponseArgs}};
 //!
@@ -122,6 +123,7 @@
 //!
 //! For demonstration:
 //! ```
+//! # #[cfg(feature = "chat-completion")]
 //! # tokio_test::block_on(async {
 //! # use async_openai::Client;
 //! # use async_openai::traits::RequestOptionsBuilder;

async-openai/src/middleware/mod.rs

@@ -103,6 +103,35 @@
 //!
 //! On native targets retries wait using `tokio::time::sleep`. On WASM retries are immediate.
 //!
+//! ## Rate limit layer
+//!
+//! Enable the `rate-limit` feature to use [`rate_limit::RateLimitLayer`].
+//! Place it below [`retry::OpenAIRetryLayer`] so retry attempts also pass
+//! through the limiter:
+//!
+//! ```no_run
+//! # #[cfg(feature = "rate-limit")]
+//! # fn main() {
+//! # use async_openai::{Client, config::OpenAIConfig};
+//! # use async_openai::middleware::{ReqwestService, retry::OpenAIRetryLayer};
+//! # use async_openai::middleware::rate_limit::RateLimitLayer;
+//! let service = tower::ServiceBuilder::new()
+//!     .layer(OpenAIRetryLayer::default())
+//!     .layer(RateLimitLayer::per_minute(60))
+//!     .service(ReqwestService::new(reqwest::Client::new()));
+//!
+//! let client = Client::with_config(OpenAIConfig::default())
+//!     .with_http_service(service);
+//! # }
+//! # #[cfg(not(feature = "rate-limit"))]
+//! # fn main() {}
+//! ```
+//!
+//! On native targets, the layer limits request RPM locally and applies
+//! server-feedback backpressure when `x-ratelimit-remaining-requests`
+//! reaches zero and `x-ratelimit-reset-requests` is parseable. On WASM
+//! targets, it records local governor state but does not delay requests.
+//!
 //! ## Error Handling
 //!
 //! `OpenAIError::Boxed` is available only when the `middleware` feature is enabled.
@@ -118,6 +147,10 @@
 //! trait bounds with or without middleware. JSON request bodies are serialized
 //! before they enter the replayable middleware request factory.
 
+/// Request rate limiting middleware.
+#[cfg(feature = "rate-limit")]
+pub mod rate_limit;
+
 /// Retry layers and policies for middleware.
 pub mod retry {
     #[doc(inline)]