resolved aws cli request format

Author: ehsan6sha

crates/fula-cli/src/handlers/multipart.rs

@@ -5,6 +5,7 @@ use crate::pinning::{check_can_upload, pin_for_user};
 use crate::state::UserSession;
 use crate::multipart::UploadPart;
 use crate::xml;
+use super::object::try_decode_chunked;
 use axum::{
     extract::{Extension, Path, Query, State},
     http::{HeaderMap, StatusCode},
@@ -85,6 +86,7 @@ pub async fn upload_part(
     Extension(session): Extension<UserSession>,
     Path((bucket, key)): Path<(String, String)>,
     Query(params): Query<MultipartParams>,
+    headers: HeaderMap,
     body: Bytes,
 ) -> Result<Response, ApiError> {
     if !session.can_write() {
@@ -93,7 +95,7 @@ pub async fn upload_part(
 
     let upload_id = params.upload_id
         .ok_or_else(|| ApiError::s3(S3ErrorCode::InvalidArgument, "Missing uploadId"))?;
-    
+
     let part_number = params.part_number
         .ok_or_else(|| ApiError::s3(S3ErrorCode::InvalidArgument, "Missing partNumber"))?;
 
@@ -113,6 +115,12 @@ pub async fn upload_part(
         return Err(ApiError::s3(S3ErrorCode::InvalidArgument, "Bucket/key mismatch"));
     }
 
+    // Decode chunked encoding if present (same as put_object)
+    let body = match try_decode_chunked(&headers, &body) {
+        Some(decoded) => decoded,
+        None => body,
+    };
+
     // Store part data
     let cid = state.block_store.put_block(&body).await?;
 

crates/fula-cli/src/handlers/object.rs

@@ -43,6 +43,14 @@ pub async fn put_object(
         ));
     }
 
+    // Decode HTTP chunked transfer encoding if present in the body.
+    // AWS CLI and other S3 clients may send chunked-encoded bodies that a
+    // reverse proxy (e.g., nginx) forwards without decoding.
+    let body = match try_decode_chunked(&headers, &body) {
+        Some(decoded) => decoded,
+        None => body,
+    };
+
     // Store the data
     let cid = state.block_store.put_block(&body).await?;
 
@@ -265,6 +273,115 @@ pub async fn get_object(
     Ok(response.body(Body::from(body_data)).unwrap())
 }
 
+/// Attempt to decode HTTP chunked transfer encoding from a request body.
+/// Returns Some(decoded) if the body was chunked-encoded, None otherwise.
+///
+/// This handles two cases:
+/// 1. AWS `Content-Encoding: aws-chunked` with streaming SigV4 signatures
+/// 2. Plain HTTP chunked TE where a reverse proxy stripped the Transfer-Encoding header
+pub(crate) fn try_decode_chunked(headers: &HeaderMap, body: &Bytes) -> Option<Bytes> {
+    let has_decoded_len = headers.get("x-amz-decoded-content-length").is_some();
+    let has_aws_chunked = headers
+        .get(header::CONTENT_ENCODING)
+        .and_then(|v| v.to_str().ok())
+        .map(|v| v.contains("aws-chunked"))
+        .unwrap_or(false);
+
+    if !has_decoded_len && !has_aws_chunked && !looks_like_chunked(body) {
+        return None;
+    }
+
+    decode_chunked_body(body).map(|decoded| {
+        tracing::info!(
+            original_len = body.len(),
+            decoded_len = decoded.len(),
+            has_decoded_len,
+            has_aws_chunked,
+            "Decoded chunked request body"
+        );
+        decoded
+    })
+}
+
+/// Check if a body appears to be HTTP chunked transfer-encoded.
+fn looks_like_chunked(body: &[u8]) -> bool {
+    if body.len() < 4 {
+        return false;
+    }
+
+    // Find first \r\n (chunk-size line delimiter)
+    let crlf_pos = match body.windows(2).position(|w| w == b"\r\n") {
+        Some(pos) if pos > 0 && pos <= 100 => pos,
+        _ => return false,
+    };
+
+    // Extract hex size (before any chunk extensions like ";chunk-signature=...")
+    let size_line = match std::str::from_utf8(&body[..crlf_pos]) {
+        Ok(s) => s,
+        Err(_) => return false,
+    };
+    let size_hex = size_line.split(';').next().unwrap_or("");
+
+    let chunk_size = match usize::from_str_radix(size_hex.trim(), 16) {
+        Ok(s) if s > 0 => s,
+        _ => return false,
+    };
+
+    // Chunk data must fit within the remaining body
+    let data_start = crlf_pos + 2;
+    chunk_size <= body.len().saturating_sub(data_start)
+}
+
+/// Decode HTTP chunked transfer encoding from raw bytes.
+/// Handles both plain chunked and aws-chunked (ignoring chunk extensions).
+fn decode_chunked_body(body: &[u8]) -> Option<Bytes> {
+    let mut decoded = Vec::new();
+    let mut pos = 0;
+
+    while pos < body.len() {
+        let remaining = &body[pos..];
+
+        // Find the \r\n ending the chunk-size line
+        let crlf_pos = remaining.windows(2).position(|w| w == b"\r\n")?;
+
+        if crlf_pos == 0 {
+            // Empty line — skip
+            pos += 2;
+            continue;
+        }
+
+        // Parse chunk size (ignore extensions after ';')
+        let size_line = std::str::from_utf8(&remaining[..crlf_pos]).ok()?;
+        let size_hex = size_line.split(';').next()?;
+        let chunk_size = usize::from_str_radix(size_hex.trim(), 16).ok()?;
+
+        if chunk_size == 0 {
+            break; // Terminal chunk
+        }
+
+        let data_start = pos + crlf_pos + 2;
+        let data_end = data_start + chunk_size;
+
+        if data_end > body.len() {
+            return None; // Truncated — probably not chunked after all
+        }
+
+        decoded.extend_from_slice(&body[data_start..data_end]);
+
+        // Skip the \r\n after chunk data
+        pos = data_end;
+        if pos + 2 <= body.len() && body[pos] == b'\r' && body[pos + 1] == b'\n' {
+            pos += 2;
+        }
+    }
+
+    if decoded.is_empty() {
+        None
+    } else {
+        Some(Bytes::from(decoded))
+    }
+}
+
 /// Parse Range header (e.g., "bytes=0-1023" or "bytes=500-" or "bytes=-500")
 fn parse_range_header(range: &str, total_size: usize) -> Result<(usize, usize), ()> {
     let range = range.strip_prefix("bytes=").ok_or(())?;

crates/fula-cli/src/routes.rs

@@ -170,7 +170,7 @@ async fn object_put_handler(
             part_number: query.part_number,
             uploads: query.uploads.clone(),
         };
-        handlers::upload_part(state, session, path, axum::extract::Query(mp_params), body).await
+        handlers::upload_part(state, session, path, axum::extract::Query(mp_params), headers, body).await
     } else if query.tagging.is_some() {
         handlers::put_object_tagging(state, session, path, body).await
     } else if headers.contains_key("x-amz-copy-source") {

target/.rustc_info.json

@@ -1 +1 @@
-{"rustc_fingerprint":7713099781947737724,"outputs":{"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.94.0-nightly (ba86c0460 2025-12-06)\nbinary: rustc\ncommit-hash: ba86c0460b0233319e01fd789a42a7276eade805\ncommit-date: 2025-12-06\nhost: x86_64-pc-windows-msvc\nrelease: 1.94.0-nightly\nLLVM version: 21.1.5\n","stderr":""},"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___.exe\nlib___.rlib\n___.dll\n___.dll\n___.lib\n___.dll\nC:\\Users\\ehsan\\.rustup\\toolchains\\nightly-x86_64-pc-windows-msvc\npacked\n___\ndebug_assertions\nemscripten_wasm_eh\nfmt_debug=\"full\"\noverflow_checks\npanic=\"unwind\"\nproc_macro\nrelocation_model=\"pic\"\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"msvc\"\ntarget_family=\"windows\"\ntarget_feature=\"cmpxchg16b\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"lahfsahf\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_feature=\"sse3\"\ntarget_feature=\"x87\"\ntarget_has_atomic\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_has_atomic_equal_alignment=\"128\"\ntarget_has_atomic_equal_alignment=\"16\"\ntarget_has_atomic_equal_alignment=\"32\"\ntarget_has_atomic_equal_alignment=\"64\"\ntarget_has_atomic_equal_alignment=\"8\"\ntarget_has_atomic_equal_alignment=\"ptr\"\ntarget_has_atomic_load_store\ntarget_has_atomic_load_store=\"128\"\ntarget_has_atomic_load_store=\"16\"\ntarget_has_atomic_load_store=\"32\"\ntarget_has_atomic_load_store=\"64\"\ntarget_has_atomic_load_store=\"8\"\ntarget_has_atomic_load_store=\"ptr\"\ntarget_has_reliable_f128\ntarget_has_reliable_f16\ntarget_has_reliable_f16_math\ntarget_os=\"windows\"\ntarget_pointer_width=\"64\"\ntarget_thread_local\ntarget_vendor=\"pc\"\nub_checks\nwindows\n","stderr":""}},"successes":{}}
+{"rustc_fingerprint":16197101729860176073,"outputs":{"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.so\nlib___.so\nlib___.a\nlib___.so\n/root/.rustup/toolchains/stable-x86_64-unknown-linux-gnu\noff\npacked\nunpacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"x86_64\"\ntarget_endian=\"little\"\ntarget_env=\"gnu\"\ntarget_family=\"unix\"\ntarget_feature=\"fxsr\"\ntarget_feature=\"sse\"\ntarget_feature=\"sse2\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"linux\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"unknown\"\nunix\n","stderr":""},"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.92.0 (ded5c06cf 2025-12-08)\nbinary: rustc\ncommit-hash: ded5c06cf21d2b93bffd5d884aa6e96934ee4234\ncommit-date: 2025-12-08\nhost: x86_64-unknown-linux-gnu\nrelease: 1.92.0\nLLVM version: 21.1.3\n","stderr":""}},"successes":{}}

tests/api_tests.rs

@@ -898,6 +898,160 @@ async fn test_large_compressible_object_no_compression() {
     assert_eq!(body, content);
 }
 
+/// Regression test: PUT with HTTP chunked transfer-encoded body must be decoded.
+///
+/// AWS CLI sends request bodies with chunked transfer encoding that a reverse
+/// proxy (nginx) may forward without decoding. Without decoding, the chunked
+/// framing (e.g., "100000\r\n") gets stored as part of the object data,
+/// corrupting the file.
+#[tokio::test]
+async fn test_put_object_decodes_chunked_body() {
+    let (base_url, _) = spawn_server(false).await;
+    let client = Client::builder()
+        .no_gzip()
+        .no_brotli()
+        .no_deflate()
+        .build()
+        .unwrap();
+
+    let bucket = "chunked-decode-bucket";
+    let key = "photo.jpg";
+
+    // Simulate a JPEG file (starts with FF D8 FF)
+    let original_data: Vec<u8> = {
+        let mut v = vec![0xFF, 0xD8, 0xFF, 0xE0];
+        v.extend_from_slice(&[0x42; 2044]); // 2048 bytes total
+        v
+    };
+
+    // Wrap the data in HTTP chunked transfer encoding framing
+    // This simulates what happens when nginx strips Transfer-Encoding header
+    // but forwards the raw chunked body
+    let chunk_size_hex = format!("{:x}", original_data.len());
+    let mut chunked_body: Vec<u8> = Vec::new();
+    chunked_body.extend_from_slice(chunk_size_hex.as_bytes()); // "800"
+    chunked_body.extend_from_slice(b"\r\n");
+    chunked_body.extend_from_slice(&original_data);
+    chunked_body.extend_from_slice(b"\r\n");
+    chunked_body.extend_from_slice(b"0\r\n\r\n"); // Terminal chunk
+
+    assert_ne!(chunked_body.len(), original_data.len(), "Chunked body should be larger");
+
+    // Create bucket
+    client.put(&format!("{}/{}", base_url, bucket)).send().await.unwrap();
+
+    // Upload the chunked-encoded body (simulating broken proxy behavior)
+    let res = client.put(&format!("{}/{}/{}", base_url, bucket, key))
+        .body(chunked_body)
+        .header("Content-Type", "image/jpeg")
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(res.status(), StatusCode::OK);
+
+    // Download and verify the gateway decoded the chunked framing
+    let res = client.get(&format!("{}/{}/{}", base_url, bucket, key))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(res.status(), StatusCode::OK);
+
+    let body = res.bytes().await.unwrap();
+    assert_eq!(
+        body.len(), original_data.len(),
+        "Downloaded size ({}) should match original ({}), not chunked body",
+        body.len(), original_data.len()
+    );
+    assert_eq!(
+        &body[..4], &[0xFF, 0xD8, 0xFF, 0xE0],
+        "Body must start with JPEG magic, not chunked framing"
+    );
+    assert_eq!(body.as_ref(), original_data.as_slice(), "Decoded body must match original");
+}
+
+/// Test that chunked decoding handles aws-chunked format (with chunk-signature extensions)
+#[tokio::test]
+async fn test_put_object_decodes_aws_chunked_body() {
+    let (base_url, _) = spawn_server(false).await;
+    let client = Client::builder()
+        .no_gzip()
+        .no_brotli()
+        .no_deflate()
+        .build()
+        .unwrap();
+
+    let bucket = "aws-chunked-bucket";
+    let key = "data.bin";
+    let original_data = b"Hello, this is test data for aws-chunked decoding!";
+
+    // Build aws-chunked encoded body:
+    // <hex-size>;chunk-signature=<fake-sig>\r\n<data>\r\n0;chunk-signature=<fake-sig>\r\n\r\n
+    let fake_sig = "a".repeat(64);
+    let mut chunked_body: Vec<u8> = Vec::new();
+    chunked_body.extend_from_slice(format!("{:x};chunk-signature={}\r\n", original_data.len(), fake_sig).as_bytes());
+    chunked_body.extend_from_slice(original_data);
+    chunked_body.extend_from_slice(b"\r\n");
+    chunked_body.extend_from_slice(format!("0;chunk-signature={}\r\n\r\n", fake_sig).as_bytes());
+
+    // Create bucket
+    client.put(&format!("{}/{}", base_url, bucket)).send().await.unwrap();
+
+    // Upload with x-amz-decoded-content-length header (AWS streaming signature signal)
+    let res = client.put(&format!("{}/{}/{}", base_url, bucket, key))
+        .body(chunked_body)
+        .header("Content-Encoding", "aws-chunked")
+        .header("x-amz-decoded-content-length", original_data.len().to_string())
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(res.status(), StatusCode::OK);
+
+    // Download and verify
+    let res = client.get(&format!("{}/{}/{}", base_url, bucket, key))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(res.status(), StatusCode::OK);
+
+    let body = res.bytes().await.unwrap();
+    assert_eq!(body.as_ref(), original_data, "Body must be decoded aws-chunked content");
+}
+
+/// Test that normal (non-chunked) uploads are NOT affected by chunked decoding
+#[tokio::test]
+async fn test_put_object_normal_body_unaffected() {
+    let (base_url, _) = spawn_server(false).await;
+    let client = Client::new();
+
+    let bucket = "normal-body-bucket";
+    let key = "binary.dat";
+
+    // Binary data that could theoretically look like a chunk header if we're not careful
+    // (starts with bytes that are valid hex chars in ASCII)
+    let original_data: Vec<u8> = vec![0x41, 0x42, 0x43, 0x44, 0x45, 0x46]; // "ABCDEF" in ASCII
+
+    // Create bucket
+    client.put(&format!("{}/{}", base_url, bucket)).send().await.unwrap();
+
+    // Upload normally
+    let res = client.put(&format!("{}/{}/{}", base_url, bucket, key))
+        .body(original_data.clone())
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(res.status(), StatusCode::OK);
+
+    // Download - should be identical
+    let res = client.get(&format!("{}/{}/{}", base_url, bucket, key))
+        .send()
+        .await
+        .unwrap();
+    assert_eq!(res.status(), StatusCode::OK);
+
+    let body = res.bytes().await.unwrap();
+    assert_eq!(body.as_ref(), original_data.as_slice(), "Normal upload must not be modified");
+}
+
 /// Test duplicate bucket creation for same user fails
 #[tokio::test]
 async fn test_multitenant_duplicate_bucket_same_user() {