CLDSRV-872: store checksum in metadata for objects uploaded with PutObject

leif-scality · leif-scality · commit 85e28c8332c5 · 2026-03-17T15:59:44.000+01:00
diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js
@@ -217,7 +217,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
             if (size === 0) {
                 if (!dontSkipBackend[locationType]) {
                     metadataStoreParams.contentMD5 = constants.emptyFileMd5;
-                    return next(null, null, null);
+                    return next(null, null, null, null);
                 }
 
                 // Handle mdOnlyHeader as a metadata only operation. If
@@ -243,14 +243,14 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
                         dataStoreVersionId: versionId,
                         dataStoreMD5: _md5,
                     };
-                    return next(null, dataGetInfo, _md5);
+                    return next(null, dataGetInfo, _md5, null);
                 }
             }
 
             return dataStore(objectKeyContext, cipherBundle, request, size,
                     streamingV4Params, backendInfo, log, next);
         },
-        function processDataResult(dataGetInfo, calculatedHash, next) {
+        function processDataResult(dataGetInfo, calculatedHash, checksum, next) {
             if (dataGetInfo === null || dataGetInfo === undefined) {
                 return next(null, null);
             }
@@ -275,6 +275,7 @@ function createAndStoreObject(bucketName, bucketMD, objectKey, objMD, authInfo,
                 dataGetInfoArr[0].size = mdOnlySize;
             }
             metadataStoreParams.contentMD5 = calculatedHash;
+            metadataStoreParams.checksum = checksum;
             return next(null, dataGetInfoArr);
         },
         function getVersioningInfo(infoArr, next) {
diff --git a/lib/api/apiUtils/object/storeObject.js b/lib/api/apiUtils/object/storeObject.js
@@ -17,7 +17,7 @@ const { arsenalErrorFromChecksumError } = require('../../apiUtils/integrity/vali
  * @return {function} - calls callback with arguments:
  * error, dataRetrievalInfo, and completedHash (if any)
  */
-function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) {
+function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, checksumStream, log, cb) {
     const contentMD5 = stream.contentMD5;
     const completedHash = hashedStream.completedHash;
     if (contentMD5 && completedHash && contentMD5 !== completedHash) {
@@ -37,7 +37,10 @@ function checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cb) {
             return cb(errors.BadDigest);
         });
     }
-    return cb(null, dataRetrievalInfo, completedHash);
+    const checksum = checksumStream.digest
+        ? { algorithm: checksumStream.algoName, value: checksumStream.digest, type: 'FULL_OBJECT' }
+        : null;
+    return cb(null, dataRetrievalInfo, completedHash, checksum);
 }
 
 /**
@@ -107,7 +110,8 @@ function dataStore(objectContext, cipherBundle, stream, size,
                         return cbOnce(arsenalErrorFromChecksumError(checksumErr));
                     });
                 }
-                return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo, log, cbOnce);
+                return checkHashMatchMD5(stream, hashedStream, dataRetrievalInfo,
+                    checksumedStream.stream, log, cbOnce);
             };
 
             // ChecksumTransform._flush computes the digest asynchronously for
diff --git a/lib/services.js b/lib/services.js
@@ -6,6 +6,7 @@ const { errors, s3middleware } = require('arsenal');
 const ObjectMD = require('arsenal').models.ObjectMD;
 const BucketInfo = require('arsenal').models.BucketInfo;
 const ObjectMDArchive = require('arsenal').models.ObjectMDArchive;
+const ObjectMDChecksum = require('arsenal').models.ObjectMDChecksum;
 const { versioning } = require('arsenal');
 const acl = require('./metadata/acl');
 const constants = require('../constants');
@@ -102,7 +103,7 @@ const services = {
      * @return {function} executes callback with err or ETag as arguments
      */
     metadataStoreObject(bucketName, dataGetInfo, cipherBundle, params, cb) {
-        const { objectKey, authInfo, size, contentMD5, metaHeaders,
+        const { objectKey, authInfo, size, contentMD5, checksum, metaHeaders,
             contentType, cacheControl, contentDisposition, contentEncoding,
             expires, multipart, headers, overrideMetadata, log,
             lastModifiedDate, versioning, versionId, uploadId,
@@ -138,6 +139,9 @@ const services = {
             // CreationTime needs to be carried over so that it remains static
             .setCreationTime(creationTime)
             .setOriginOp(originOp);
+        if (checksum) {
+            md.setChecksum(new ObjectMDChecksum(checksum.algorithm, checksum.value, checksum.type));
+        }
         // Sending in last modified date in object put copy since need
         // to return the exact date in the response
         if (lastModifiedDate) {
diff --git a/tests/functional/raw-node/test/routes/routeMetadata.js b/tests/functional/raw-node/test/routes/routeMetadata.js
@@ -1,7 +1,10 @@
 const assert = require('assert');
+const crypto = require('crypto');
+const { CrtCrc64Nvme } = require('@aws-sdk/crc64-nvme-crt');
 const http = require('http');
-const { CreateBucketCommand, 
-    PutObjectCommand, 
+const { CreateBucketCommand,
+    PutObjectCommand,
+    DeleteObjectCommand,
     DeleteBucketCommand } = require('@aws-sdk/client-s3');
 
 const { makeRequest } = require('../../utils/makeRequest');
@@ -144,3 +147,78 @@ describe('metadata routes with metadata', () => {
         });
     });
 });
+
+describe('checksum stored in object metadata after PutObject', () => {
+    const bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' });
+    const s3 = bucketUtil.s3;
+
+    const bucket = 'bucket1';
+    const objectBody = 'hello checksum';
+    const sha256Key = 'object-with-sha256-checksum';
+    const defaultKey = 'object-with-default-checksum';
+
+    let expectedCrc64nvme;
+
+    before(async function () {
+        if (!process.env.S3_END_TO_END) {
+            this.skip();
+        }
+        const crc = new CrtCrc64Nvme();
+        crc.update(Buffer.from(objectBody));
+        expectedCrc64nvme = Buffer.from(await crc.digest()).toString('base64');
+
+        const sha256Value = crypto.createHash('sha256').update(objectBody).digest('base64');
+        await s3.send(new PutObjectCommand({
+            Bucket: bucket,
+            Key: sha256Key,
+            Body: objectBody,
+            ChecksumSHA256: sha256Value,
+        }));
+        await s3.send(new PutObjectCommand({
+            Bucket: bucket,
+            Key: defaultKey,
+            Body: objectBody,
+        }));
+    });
+
+    after(async () => {
+        if (!process.env.S3_END_TO_END) {
+            return;
+        }
+        await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: sha256Key }));
+        await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: defaultKey }));
+    });
+
+    it('stores sha256 checksum in metadata when x-amz-checksum-sha256 is provided', done => {
+        const expectedValue = crypto.createHash('sha256').update(objectBody).digest('base64');
+        makeMetadataRequest({
+            method: 'GET',
+            authCredentials: metadataAuthCredentials,
+            path: `/_/metadata/default/bucket/${bucket}/${sha256Key}`,
+        }, (err, res) => {
+            assert.ifError(err);
+            assert.strictEqual(res.statusCode, 200);
+            const md = JSON.parse(res.body);
+            assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256');
+            assert.strictEqual(md.checksum.checksumValue, expectedValue);
+            assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT');
+            return done();
+        });
+    });
+
+    it('stores crc64nvme checksum in metadata when no checksum header is provided', done => {
+        makeMetadataRequest({
+            method: 'GET',
+            authCredentials: metadataAuthCredentials,
+            path: `/_/metadata/default/bucket/${bucket}/${defaultKey}`,
+        }, (err, res) => {
+            assert.ifError(err);
+            assert.strictEqual(res.statusCode, 200);
+            const md = JSON.parse(res.body);
+            assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme');
+            assert.strictEqual(md.checksum.checksumValue, expectedCrc64nvme);
+            assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT');
+            return done();
+        });
+    });
+});
diff --git a/tests/unit/api/objectPut.js b/tests/unit/api/objectPut.js
@@ -1,5 +1,6 @@
 const assert = require('assert');
 const async = require('async');
+const crypto = require('crypto');
 const moment = require('moment');
 const { s3middleware, storage, versioning } = require('arsenal');
 const sinon = require('sinon');
@@ -865,6 +866,52 @@ describe('objectPut API', () => {
                 });
         });
     });
+
+    it('should store sha256 checksum in metadata when x-amz-checksum-sha256 header is provided', done => {
+        const sha256Value = crypto.createHash('sha256').update(postBody).digest('base64');
+        const request = new DummyRequest({
+            bucketName,
+            namespace,
+            objectKey: objectName,
+            headers: {
+                host: `${bucketName}.s3.amazonaws.com`,
+                'x-amz-checksum-sha256': sha256Value,
+            },
+            url: '/',
+        }, postBody);
+
+        bucketPut(authInfo, testPutBucketRequest, log, err => {
+            assert.ifError(err);
+            objectPut(authInfo, request, undefined, log, err => {
+                assert.ifError(err);
+                metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => {
+                    assert.ifError(err);
+                    assert(md.checksum, 'checksum should be set in metadata');
+                    assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256');
+                    assert.strictEqual(md.checksum.checksumValue, sha256Value);
+                    assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT');
+                    done();
+                });
+            });
+        });
+    });
+
+    it('should store crc64nvme checksum in metadata when no checksum header is provided', done => {
+        bucketPut(authInfo, testPutBucketRequest, log, err => {
+            assert.ifError(err);
+            objectPut(authInfo, testPutObjectRequest, undefined, log, err => {
+                assert.ifError(err);
+                metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => {
+                    assert.ifError(err);
+                    assert(md.checksum, 'checksum should be set in metadata');
+                    assert.strictEqual(md.checksum.checksumAlgorithm, 'crc64nvme');
+                    assert(md.checksum.checksumValue, 'checksumValue should be set');
+                    assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT');
+                    done();
+                });
+            });
+        });
+    });
 });
 
 describe('objectPut API with versioning', () => {
