CLDSRV-863: emit trailer in TrailingChecksumTransform

leif-scality · leif-scality · commit 7fb197c87baf · 2026-03-16T17:32:24.000+01:00
diff --git a/lib/auth/streamingV4/trailingChecksumTransform.js b/lib/auth/streamingV4/trailingChecksumTransform.js
@@ -3,10 +3,11 @@ const { errors } = require('arsenal');
 const { maximumAllowedPartSize } = require('../../../constants');
 
 /**
- * This class is designed to handle the chunks sent in a streaming
- * unsigned playload trailer request. In this iteration, we are not checking
- * the checksums, but we are removing them from the stream.
- * S3C-9732 will deal with checksum verification.
+ * This class handles the chunked-upload body format used by
+ * STREAMING-UNSIGNED-PAYLOAD-TRAILER requests. It strips the chunk-size
+ * headers and trailing checksum trailer from the stream, forwarding only
+ * the raw object data. The trailer name and value are emitted via a
+ * 'trailer' event so that ChecksumTransform can validate the checksum.
  */
 class TrailingChecksumTransform extends Transform {
     /**
@@ -20,6 +21,10 @@ class TrailingChecksumTransform extends Transform {
         this.bytesToDiscard = 0; // when trailing \r\n are present, we discard them but they can be in different chunks
         this.bytesToRead = 0; // when a chunk is advertised, the size is put here and we forward all bytes
         this.streamClosed = false;
+        this.readingTrailer = false;
+        this.trailerBuffer = Buffer.alloc(0);
+        this.trailerName = null;
+        this.trailerValue = null;
     }
 
     /**
@@ -30,9 +35,16 @@ class TrailingChecksumTransform extends Transform {
      * @return {function} executes callback with err if applicable
      */
     _flush(callback) {
-        if (!this.streamClosed) {
+        if (!this.streamClosed && this.readingTrailer && this.trailerBuffer.length === 0) {
+            // Nothing came after "0\r\n", don't fail.
+            // If the x-amz-trailer header was present then the trailer is required and ChecksumTransform will fail.
+            return callback();
+        } else if (!this.streamClosed && this.readingTrailer && this.trailerBuffer.length !== 0) {
+            this.log.error('stream ended without trailer "\r\n"');
+            return callback(errors.IncompleteBody);
+        } else if (!this.streamClosed && !this.readingTrailer) {
             this.log.error('stream ended without closing chunked encoding');
-            return callback(errors.InvalidArgument);
+            return callback(errors.IncompleteBody);
         }
         return callback();
     }
@@ -66,6 +78,49 @@ class TrailingChecksumTransform extends Transform {
                 continue;
             }
 
+            // after the 0-size chunk, read the trailer line (e.g. "x-amz-checksum-crc32:YABb/g==")
+            if (this.readingTrailer) {
+                const combined = Buffer.concat([this.trailerBuffer, chunk]);
+                const lineBreakIndex = combined.indexOf('\r\n');
+                if (lineBreakIndex === -1) {
+                    if (combined.byteLength > 1024) {
+                        this.log.error('trailer line too long');
+                        return callback(errors.MalformedTrailerError);
+                    }
+                    // The trailer is not complete yet, continue.
+                    this.trailerBuffer = combined;
+                    return callback();
+                }
+                this.trailerBuffer = Buffer.alloc(0);
+                const fullTrailer = combined.subarray(0, lineBreakIndex);
+                if (fullTrailer.length === 0) {
+                    // The trailer is empty, stop reading.
+                    this.readingTrailer = false;
+                    this.streamClosed = true;
+                    return callback();
+                }
+                let trailerLine = fullTrailer.toString();
+                // Some clients terminate the trailer with \n\r\n instead of
+                // just \r\n, producing a trailing \n in the parsed line.
+                if (trailerLine.endsWith('\n')) {
+                    trailerLine = trailerLine.slice(0, -1);
+                }
+                const colonIndex = trailerLine.indexOf(':');
+                if (colonIndex > 0) {
+                    this.trailerName = trailerLine.slice(0, colonIndex).trim();
+                    this.trailerValue = trailerLine.slice(colonIndex + 1).trim();
+                    this.emit('trailer', this.trailerName, this.trailerValue);
+                } else {
+                    this.log.error('incomplete trailer missing ":"', { trailerLine });
+                    return callback(errors.IncompleteBody);
+                }
+                this.readingTrailer = false;
+                this.streamClosed = true;
+                // The trailer \r\n is the last bytes of the stream per the AWS
+                // chunked upload format, so any remaining bytes are discarded.
+                return callback();
+            }
+
             // we are now looking for the chunk size field
             // no need to look further than 10 bytes since the field cannot be bigger: the max
             // chunk size is 5GB (see constants.maximumAllowedPartSize)
@@ -100,9 +155,9 @@ class TrailingChecksumTransform extends Transform {
             }
             this.chunkSizeBuffer = Buffer.alloc(0);
             if (dataSize === 0) {
-                // TODO: check if the checksum is correct (S3C-9732)
-                // last chunk, no more data to read, the stream is closed
-                this.streamClosed = true;
+                // last chunk, no more data to read; enter trailer-reading mode
+                // bytesToDiscard = 2 below will consume the \r\n after "0"
+                this.readingTrailer = true;
             }
             if (dataSize > maximumAllowedPartSize) {
                 this.log.error('chunk size too big', { dataSize });
diff --git a/tests/functional/raw-node/test/trailingChecksums.js b/tests/functional/raw-node/test/trailingChecksums.js
@@ -6,10 +6,9 @@ const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4');
 const bucket = 'testunsupportedchecksumsbucket';
 const objectKey = 'key';
 const objData = Buffer.alloc(1024, 'a');
-// note this is not the correct checksum in objDataWithTrailingChecksum
 const objDataWithTrailingChecksum = '10\r\n0123456789abcdef\r\n' +
                                     '10\r\n0123456789abcdef\r\n' +
-                                    '0\r\nx-amz-checksum-crc64nvme:YeIDuLa7tU0=\r\n';
+                                    '0\r\nx-amz-checksum-crc64nvme:skQv82y5rgE=\r\n';
 const objDataWithoutTrailingChecksum = '0123456789abcdef0123456789abcdef';
 
 const config = require('../../config.json');
diff --git a/tests/unit/auth/TrailingChecksumTransform.js b/tests/unit/auth/TrailingChecksumTransform.js
@@ -4,9 +4,39 @@ const async = require('async');
 const { Readable } = require('stream');
 
 const TrailingChecksumTransform = require('../../../lib/auth/streamingV4/trailingChecksumTransform');
-const { stripTrailingChecksumStream } = require('../../../lib/api/apiUtils/object/prepareStream');
 const { DummyRequestLogger } = require('../helpers');
 
+// Helper: pipe input chunks through TrailingChecksumTransform, collect output and trailer events
+function runTransform(inputChunks) {
+    const stream = new TrailingChecksumTransform(new DummyRequestLogger());
+    return new Promise((resolve, reject) => {
+        const output = [];
+        const trailerEvents = [];
+        stream.on('data', chunk => output.push(Buffer.from(chunk)));
+        stream.on('trailer', (name, value) => trailerEvents.push({ name, value }));
+        stream.on('finish', () => resolve({ data: Buffer.concat(output), trailers: trailerEvents, stream }));
+        stream.on('error', reject);
+        for (const chunk of inputChunks) {
+            stream.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+        }
+        stream.end();
+    });
+}
+
+// Helper: expect a stream error from the given input chunks
+function expectError(inputChunks) {
+    const stream = new TrailingChecksumTransform(new DummyRequestLogger());
+    return new Promise((resolve, reject) => {
+        stream.on('error', resolve);
+        stream.on('finish', () => reject(new Error('expected error but stream finished cleanly')));
+        for (const chunk of inputChunks) {
+            stream.write(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
+        }
+        stream.end();
+        stream.resume();
+    });
+}
+
 const log = new DummyRequestLogger();
 
 // note this is not the correct checksum in objDataWithTrailingChecksum
@@ -174,30 +204,33 @@ describe('TrailingChecksumTransform class', () => {
     it('should propagate _flush error via errCb when stream closes without chunked encoding', done => {
         const incompleteData = '10\r\n01234\r6789abcd\r\n\r\n';
         const source = new ChunkedReader([Buffer.from(incompleteData)]);
-        source.headers = { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER' };
-        const stream = stripTrailingChecksumStream(source, log, err => {
-            assert.deepStrictEqual(err, errors.InvalidArgument);
+        const stream = new TrailingChecksumTransform(log);
+        stream.on('error', err => {
+            assert.deepStrictEqual(err, errors.IncompleteBody);
             done();
         });
+        source.pipe(stream);
         stream.resume();
     });
 
     it('should propagate _transform error via errCb for invalid chunk size', done => {
         const badData = '500000000000\r\n';
         const source = new ChunkedReader([Buffer.from(badData)]);
-        source.headers = { 'x-amz-content-sha256': 'STREAMING-UNSIGNED-PAYLOAD-TRAILER' };
-        const stream = stripTrailingChecksumStream(source, log, err => {
+        const stream = new TrailingChecksumTransform(log);
+        stream.on('error', err => {
             assert.deepStrictEqual(err, errors.InvalidArgument);
             done();
         });
+        source.pipe(stream);
         stream.resume();
     });
 
-    it('should return early if supplied with an out of specification chunk size', done => {
+    it('should return early if supplied with an out-of-specification chunk size', done => {
         const trailingChecksumTransform = new TrailingChecksumTransform(log);
         const chunks = [
             Buffer.from('500000'),
             Buffer.from('000000\r\n'),
+
             Buffer.alloc(1000000),
             Buffer.alloc(1000000),
             Buffer.alloc(1000000),
@@ -227,3 +260,188 @@ describe('TrailingChecksumTransform class', () => {
         chunkedReader.pipe(trailingChecksumTransform);
     });
 });
+
+describe('TrailingChecksumTransform trailer parsing and emitting', () => {
+    describe('happy path', () => {
+        it('single chunk with data and trailer: forwards data, emits trailer name and value', async () => {
+            const input = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n';
+            const { data, trailers } = await runTransform([input]);
+            assert.strictEqual(data.toString(), 'hello');
+            assert.strictEqual(trailers.length, 1);
+            assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32');
+            assert.strictEqual(trailers[0].value, 'AAAAAA==');
+        });
+
+        it('multiple data chunks followed by trailer: forwards all data, emits trailer once', async () => {
+            const input = '5\r\nhello\r\n5\r\nworld\r\n0\r\nx-amz-checksum-sha256:AAAAAA==\r\n';
+            const { data, trailers } = await runTransform([input]);
+            assert.strictEqual(data.toString(), 'helloworld');
+            assert.strictEqual(trailers.length, 1);
+            assert.strictEqual(trailers[0].name, 'x-amz-checksum-sha256');
+        });
+
+        it('data chunk containing \\r\\n in payload is forwarded correctly', async () => {
+            // 7 bytes: h e l \r \n l o
+            const input = '7\r\nhel\r\nlo\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n';
+            const { data } = await runTransform([input]);
+            assert.strictEqual(data.toString(), 'hel\r\nlo');
+        });
+
+        it('trailer with whitespace around name and value: name and value are trimmed', async () => {
+            const input = '0\r\n x-amz-checksum-crc32 : AAAAAA== \r\n';
+            const { trailers } = await runTransform([input]);
+            assert.strictEqual(trailers.length, 1);
+            assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32');
+            assert.strictEqual(trailers[0].value, 'AAAAAA==');
+        });
+
+        it('trailer terminated with \\n\\r\\n: trailing \\n stripped from parsed line', async () => {
+            const input = '0\r\nx-amz-checksum-crc32:AAAAAA==\n\r\n';
+            const { trailers } = await runTransform([input]);
+            assert.strictEqual(trailers.length, 1);
+            assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32');
+            assert.strictEqual(trailers[0].value, 'AAAAAA==');
+        });
+
+        it('trailer value containing a colon: only first colon used as separator', async () => {
+            const input = '0\r\nx-amz-checksum-crc32:AA:BB==\r\n';
+            const { trailers } = await runTransform([input]);
+            assert.strictEqual(trailers.length, 1);
+            assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32');
+            assert.strictEqual(trailers[0].value, 'AA:BB==');
+        });
+
+        it('bytes after trailer \\r\\n are silently discarded', async () => {
+            const input = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\nextra bytes ignored';
+            const { data, trailers } = await runTransform([input]);
+            assert.strictEqual(data.toString(), 'hello');
+            assert.strictEqual(trailers.length, 1);
+        });
+    });
+
+    describe('chunk boundary edge cases', () => {
+        it('chunk size field split across two input chunks: parsed correctly', async () => {
+            // size 'a' (hex) = 10 bytes; split the size field across two chunks
+            const c1 = 'a';
+            const c2 = '\r\nAAAAAAAAAA\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n';
+            const { data, trailers } = await runTransform([c1, c2]);
+            assert.strictEqual(data.toString(), 'AAAAAAAAAA');
+            assert.strictEqual(trailers.length, 1);
+            assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32');
+        });
+
+        it('data bytes split across two input chunks: all forwarded', async () => {
+            // 5 bytes 'hello', split after 'hel'
+            const c1 = '5\r\nhel';
+            const c2 = 'lo\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n';
+            const { data, trailers } = await runTransform([c1, c2]);
+            assert.strictEqual(data.toString(), 'hello');
+            assert.strictEqual(trailers.length, 1);
+        });
+
+        it('\\r\\n delimiter after chunk size split across two input chunks: parsed correctly', async () => {
+            // '5\r' in chunk1, '\nhello\r\n0\r\n...' in chunk2
+            const c1 = '5\r';
+            const c2 = '\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n';
+            const { data, trailers } = await runTransform([c1, c2]);
+            assert.strictEqual(data.toString(), 'hello');
+            assert.strictEqual(trailers.length, 1);
+        });
+
+        it('trailer line split across two input chunks: emits trailer correctly', async () => {
+            const c1 = '5\r\nhello\r\n0\r\nx-amz-checksum-';
+            const c2 = 'crc32:AAAAAA==\r\n';
+            const { data, trailers } = await runTransform([c1, c2]);
+            assert.strictEqual(data.toString(), 'hello');
+            assert.strictEqual(trailers.length, 1);
+            assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32');
+            assert.strictEqual(trailers[0].value, 'AAAAAA==');
+        });
+
+        it('trailer \\r\\n split across two input chunks: emits trailer correctly', async () => {
+            const c1 = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r';
+            const c2 = '\n';
+            const { data, trailers } = await runTransform([c1, c2]);
+            assert.strictEqual(data.toString(), 'hello');
+            assert.strictEqual(trailers.length, 1);
+            assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32');
+            assert.strictEqual(trailers[0].value, 'AAAAAA==');
+        });
+    });
+
+    describe('zero-size terminator and trailer', () => {
+        it('empty trailer line (0\\r\\n\\r\\n): no trailer event emitted, stream closes cleanly', async () => {
+            const input = '5\r\nhello\r\n0\r\n\r\n';
+            const { data, trailers } = await runTransform([input]);
+            assert.strictEqual(data.toString(), 'hello');
+            assert.strictEqual(trailers.length, 0);
+        });
+
+        it('zero data chunks (only terminator + trailer): no data forwarded, trailer emitted', async () => {
+            const input = '0\r\nx-amz-checksum-crc32:AAAAAA==\r\n';
+            const { data, trailers } = await runTransform([input]);
+            assert.strictEqual(data.length, 0);
+            assert.strictEqual(trailers.length, 1);
+            assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32');
+            assert.strictEqual(trailers[0].value, 'AAAAAA==');
+        });
+    });
+
+    describe('_flush error cases', () => {
+        it('stream ends mid-data (no zero-chunk): IncompleteBody error', async () => {
+            // 5 bytes declared but stream ends after only 3
+            const err = await expectError(['5\r\nhel']);
+            assert.deepStrictEqual(err, errors.IncompleteBody);
+        });
+
+        it('stream ends after zero-chunk with partial trailer content: IncompleteBody error', async () => {
+            // zero-chunk received, trailer starts but no \r\n terminator
+            const err = await expectError(['0\r\nx-amz-checksum-crc32:AAAAAA==']);
+            assert.deepStrictEqual(err, errors.IncompleteBody);
+        });
+
+        it('stream ends after zero-chunk with no trailer content: no error', async () => {
+            // only '0\r\n' — readingTrailer=true, trailerBuffer empty → no error
+            const { data, trailers } = await runTransform(['0\r\n']);
+            assert.strictEqual(data.length, 0);
+            assert.strictEqual(trailers.length, 0);
+        });
+    });
+
+    describe('_transform error cases', () => {
+        it('chunk size field larger than 10 bytes: InvalidArgument error', async () => {
+            // 11 hex digits — exceeds the 10-byte field size limit
+            const err = await expectError(['12345678901\r\n']);
+            assert.deepStrictEqual(err, errors.InvalidArgument);
+        });
+
+        it('chunk size is not valid hex: InvalidArgument error', async () => {
+            // 2 chars, short enough to pass size check, but not valid hex
+            const err = await expectError(['zz\r\n']);
+            assert.deepStrictEqual(err, errors.InvalidArgument);
+        });
+
+        it('chunk size exceeds maximumAllowedPartSize: EntityTooLarge error', async () => {
+            // 0x200000000 = 8589934592 > maximumAllowedPartSize (5GB = 0x140000000)
+            const err = await expectError(['200000000\r\n']);
+            assert.deepStrictEqual(err, errors.EntityTooLarge);
+        });
+
+        it('trailer line longer than 1024 bytes: MalformedTrailerError', async () => {
+            // send zero-chunk then a trailer line > 1024 bytes with no \r\n
+            const longTrailer = 'x'.repeat(1025);
+            const err = await expectError([`0\r\n${longTrailer}`]);
+            assert.deepStrictEqual(err, errors.MalformedTrailerError);
+        });
+
+        it('trailer line missing colon: IncompleteBody error', async () => {
+            const err = await expectError(['0\r\nnocolon\r\n']);
+            assert.deepStrictEqual(err, errors.IncompleteBody);
+        });
+
+        it('trailer line with colon at position 0 (empty name): IncompleteBody error', async () => {
+            const err = await expectError(['0\r\n:value\r\n']);
+            assert.deepStrictEqual(err, errors.IncompleteBody);
+        });
+    });
+});
