rados: Add support for rados_checksum API

Add support for in-OSD calculations of object checksums using one of the supported algorithms:
* XXHash32
* XXHash64
* CRC32

Signed-off-by: Max Naylor <maxnaylor09@gmail.com>

Author: naylorpmax-joyent

docs/api-status.json

@@ -1266,7 +1266,14 @@
         "became_stable_version": "v0.33.0"
       }
     ],
-    "preview_api": []
+    "preview_api": [
+      {
+        "name": "IOContext.Checksum",
+        "comment": "Checksum calculates the checksum of the given object data, using one of the supported checksum algorithms.\n\nImplements:\n\n\tint rados_checksum(rados_ioctx_t io,\n\t                   const char *oid,\n\t                   rados_checksum_type_t type,\n\t                   const char *init_value,\n\t                   size_t init_value_len,\n\t                   size_t len,\n\t                   uint64_t off,\n\t                   size_t chunk_size,\n\t                   char *pchecksum,\n\t                   size_t checksum_len);\n",
+        "added_in_version": "$NEXT_RELEASE",
+        "expected_stable_version": "$NEXT_RELEASE_STABLE"
+      }
+    ]
   },
   "rbd": {
     "deprecated_api": [

docs/api-status.md

@@ -23,7 +23,11 @@ FSAdmin.SubVolumeGroupInfo | $NEXT_RELEASE | $NEXT_RELEASE_STABLE |
 
 ## Package: rados
 
-No Preview/Deprecated APIs found. All APIs are considered stable.
+### Preview APIs
+
+Name | Added in Version | Expected Stable Version | 
+---- | ---------------- | ----------------------- | 
+IOContext.Checksum | $NEXT_RELEASE | $NEXT_RELEASE_STABLE | 
 
 ## Package: rbd
 

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.98.0
 	github.com/aws/smithy-go v1.24.2
 	github.com/gofrs/uuid/v5 v5.4.0
+	github.com/pierrec/xxHash v0.1.5
 	github.com/stretchr/testify v1.11.1
 	golang.org/x/sys v0.42.0
 )

go.sum

@@ -40,6 +40,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/gofrs/uuid/v5 v5.4.0 h1:EfbpCTjqMuGyq5ZJwxqzn3Cbr2d0rUZU7v5ycAk/e/0=
 github.com/gofrs/uuid/v5 v5.4.0/go.mod h1:CDOjlDMVAtN56jqyRUZh58JT31Tiw7/oQyEXZV+9bD8=
+github.com/pierrec/xxHash v0.1.5 h1:n/jBpwTHiER4xYvK3/CdPVnLDPchj8eTJFFLUb4QHBo=
+github.com/pierrec/xxHash v0.1.5/go.mod h1:w2waW5Zoa/Wc4Yqe0wgrIYAGKqRMf7czn2HNKXmuL+I=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=

rados/ioctx_checksum.go

@@ -0,0 +1,86 @@
+//go:build ceph_preview
+
+package rados
+
+/*
+#cgo LDFLAGS: -lrados
+#include <stdlib.h>
+#include <rados/librados.h>
+*/
+import "C"
+
+import "unsafe"
+
+// Checksum calculates the checksum of the given object data, using one of the supported checksum algorithms.
+//
+// Implements:
+//
+//	int rados_checksum(rados_ioctx_t io,
+//	                   const char *oid,
+//	                   rados_checksum_type_t type,
+//	                   const char *init_value,
+//	                   size_t init_value_len,
+//	                   size_t len,
+//	                   uint64_t off,
+//	                   size_t chunk_size,
+//	                   char *pchecksum,
+//	                   size_t checksum_len);
+func (ioctx *IOContext) Checksum(oid string, checksumType ChecksumType, dst []byte, opts *ChecksumOptions) error {
+	// apply defaults
+	if opts == nil {
+		opts = &ChecksumOptions{}
+	}
+	if opts.InitValue == nil {
+		initLen := 4
+		if checksumType == ChecksumTypeXXHash64 {
+			initLen = 8
+		}
+		opts.InitValue = make([]byte, initLen)
+	}
+
+	// call library
+	coid := C.CString(oid)
+	defer C.free(unsafe.Pointer(coid))
+
+	return getError(C.rados_checksum(
+		ioctx.ioctx,
+		coid,
+		C.rados_checksum_type_t(checksumType),
+		(*C.char)(unsafe.Pointer(&opts.InitValue[0])),
+		C.size_t(len(opts.InitValue)),
+		C.size_t(opts.Len),
+		C.uint64_t(opts.Off),
+		C.size_t(opts.ChunkSize),
+		(*C.char)(unsafe.Pointer(&dst[0])),
+		C.size_t(len(dst)),
+	))
+}
+
+// ChecksumType indicates checksum algorithm types supported by the IOContext.Checksum method.
+// Equivalent to the rados_checksum_type_t enum.
+type ChecksumType uint32
+
+const (
+	// ChecksumTypeXXHash32 produces an encoded le32 checksum of the given object.
+	ChecksumTypeXXHash32 = ChecksumType(C.LIBRADOS_CHECKSUM_TYPE_XXHASH32)
+	// ChecksumTypeXXHash64 produces an encoded le64 checksum of the given object.
+	ChecksumTypeXXHash64 = ChecksumType(C.LIBRADOS_CHECKSUM_TYPE_XXHASH64)
+	// ChecksumTypeCRC32C produces an encoded le32 checksum of the given object.
+	ChecksumTypeCRC32C = ChecksumType(C.LIBRADOS_CHECKSUM_TYPE_CRC32C)
+)
+
+// ChecksumOptions exposes non-required parameters for the Checksum method.
+type ChecksumOptions struct {
+	// Off sets the object offset to start checksumming in the object.
+	// By default, the entire object will be checksummed.
+	Off uint64
+	// Len sets the the number of bytes to checksum in the object.
+	// By default, the entire object will be checksummed.
+	Len uint64
+	// ChunkSize sets the length-aligned chunk size for the checksum calculation.
+	// By default, the entire object will be checksummed as a single chunk.
+	ChunkSize uint64
+	// InitValue sets the initial value for the checksum calculation.
+	// By default, the initial value will be a zeroed-out byte slice.
+	InitValue []byte
+}

rados/ioctx_checksum_test.go

@@ -0,0 +1,192 @@
+package rados
+
+import (
+	"encoding/binary"
+	"hash/crc32"
+
+	xxhash32 "github.com/pierrec/xxHash/xxHash32"
+	xxhash64 "github.com/pierrec/xxHash/xxHash64"
+	"github.com/stretchr/testify/assert"
+)
+
+func (suite *RadosTestSuite) TestChecksum() {
+	suite.SetupConnection()
+
+	var (
+		oid      = suite.GenObjectName()
+		contents = suite.RandomBytes(3 * 1024)
+	)
+
+	// write some random data to the object
+	assert.NoError(suite.T(), suite.ioctx.Write(oid, contents, 0))
+
+	// test each checksum type
+	suite.Run("CRC32", func() {
+		init := []byte{0xff, 0xff, 0xff, 0xff}
+
+		dst := make([]byte, 4+4)
+		assert.NoError(suite.T(), suite.ioctx.Checksum(oid, ChecksumTypeCRC32C, dst, &ChecksumOptions{InitValue: init}))
+
+		chunks := binary.LittleEndian.Uint32(dst[:4])
+		assert.Equal(suite.T(), uint32(1), chunks)
+
+		// Note: the CRC32 Go standard library produces checksum results with the final XOR already applied,
+		// while rados_checksum returns raw results; so here we do additional processing to assert correctness.
+		sum := binary.LittleEndian.Uint32(dst[4:]) ^ 0xffffffff
+		want := crc32.Checksum(contents, crc32.MakeTable(crc32.Castagnoli))
+		assert.Equal(suite.T(), want, sum)
+	})
+
+	suite.Run("XXHash32", func() {
+		init := []byte{0x0, 0x0, 0x0, 0x0}
+		seed := binary.LittleEndian.Uint32(init)
+
+		hash := xxhash32.New(seed)
+		_, err := hash.Write(contents)
+		assert.NoError(suite.T(), err)
+		want := binary.LittleEndian.Uint32(hash.Sum(nil))
+
+		dst := make([]byte, 4+4)
+		assert.NoError(suite.T(), suite.ioctx.Checksum(oid, ChecksumTypeXXHash32, dst, nil))
+
+		chunks := binary.LittleEndian.Uint32(dst[:4])
+		assert.Equal(suite.T(), uint32(1), chunks)
+
+		sum := binary.LittleEndian.Uint32(dst[4:])
+		assert.Equal(suite.T(), want, sum)
+	})
+
+	suite.Run("XXHash64", func() {
+		init := []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}
+		seed := binary.LittleEndian.Uint64(init)
+
+		xxh := xxhash64.New(seed)
+		_, err := xxh.Write(contents)
+		assert.NoError(suite.T(), err)
+		want := binary.LittleEndian.Uint64(xxh.Sum(nil))
+
+		dst := make([]byte, 4+8)
+		assert.NoError(suite.T(), suite.ioctx.Checksum(oid, ChecksumTypeXXHash64, dst, nil))
+
+		chunks := binary.LittleEndian.Uint32(dst[:4])
+		assert.Equal(suite.T(), uint32(1), chunks)
+
+		sum := binary.LittleEndian.Uint64(dst[4:])
+		assert.Equal(suite.T(), want, sum)
+	})
+}
+func (suite *RadosTestSuite) TestChecksumWithOpts() {
+	suite.SetupConnection()
+
+	var (
+		off        uint64 = 256
+		chunkSize  uint64 = 1024
+		chunkCount uint64 = 2
+
+		oid      = suite.GenObjectName()
+		contents = suite.RandomBytes(int(off + chunkSize*(chunkCount+1))) // write one extra chunk, to be ignored
+	)
+
+	// write some random data to the object
+	assert.NoError(suite.T(), suite.ioctx.Write(oid, contents, 0))
+
+	// test each checksum type with optional parameters
+	suite.Run("CRC32", func() {
+		init := []byte{0xff, 0xff, 0xff, 0xff}
+
+		table := crc32.MakeTable(crc32.Castagnoli)
+		want := []uint32{
+			crc32.Checksum(contents[off:off+chunkSize], table),
+			crc32.Checksum(contents[off+chunkSize:off+chunkSize*2], table),
+		}
+
+		dst := make([]byte, 4+4*chunkCount) // allocate space for multiple uint32 checksum values
+		assert.NoError(suite.T(), suite.ioctx.Checksum(oid, ChecksumTypeCRC32C, dst, &ChecksumOptions{
+			InitValue: init,
+			ChunkSize: chunkSize,
+			Off:       off,
+			Len:       chunkSize * chunkCount,
+		}))
+
+		chunks := binary.LittleEndian.Uint32(dst[:4])
+		assert.Equal(suite.T(), uint32(chunkCount), chunks)
+
+		got := []uint32{
+			binary.LittleEndian.Uint32(dst[4:8]) ^ 0xffffffff,
+			binary.LittleEndian.Uint32(dst[8:12]) ^ 0xffffffff,
+		}
+		for i := range want {
+			assert.Equal(suite.T(), want[i], got[i])
+		}
+	})
+
+	suite.Run("XXHash32", func() {
+		init := []byte{0xff, 0xff, 0xff, 0xff}
+		seed := binary.LittleEndian.Uint32(init)
+
+		var want [2]uint32
+		xxh := xxhash32.New(seed)
+
+		_, err := xxh.Write(contents[off : off+chunkSize])
+		assert.NoError(suite.T(), err)
+		want[0] = binary.LittleEndian.Uint32(xxh.Sum(nil))
+		xxh.Reset()
+		_, err = xxh.Write(contents[off+chunkSize : off+chunkSize*2])
+		assert.NoError(suite.T(), err)
+		want[1] = binary.LittleEndian.Uint32(xxh.Sum(nil))
+
+		dst := make([]byte, 4+4*chunkCount) // allocate space for multiple uint32 checksum values
+		assert.NoError(suite.T(), suite.ioctx.Checksum(oid, ChecksumTypeXXHash32, dst, &ChecksumOptions{
+			InitValue: init,
+			ChunkSize: chunkSize,
+			Off:       off,
+			Len:       chunkSize * chunkCount,
+		}))
+
+		chunks := binary.LittleEndian.Uint32(dst[:4])
+		assert.Equal(suite.T(), uint32(chunkCount), chunks)
+
+		got := []uint32{
+			binary.LittleEndian.Uint32(dst[4:8]),
+			binary.LittleEndian.Uint32(dst[8:12]),
+		}
+		for i := range want {
+			assert.Equal(suite.T(), want[i], got[i])
+		}
+	})
+
+	suite.Run("XXHash64", func() {
+		init := []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff}
+		seed := binary.LittleEndian.Uint64(init)
+
+		var want [2]uint64
+		xxh := xxhash64.New(seed)
+
+		_, err := xxh.Write(contents[off : off+chunkSize])
+		assert.NoError(suite.T(), err)
+		want[0] = binary.LittleEndian.Uint64(xxh.Sum(nil))
+		xxh.Reset()
+		_, err = xxh.Write(contents[off+chunkSize : off+chunkSize*2])
+		assert.NoError(suite.T(), err)
+		want[1] = binary.LittleEndian.Uint64(xxh.Sum(nil))
+
+		dst := make([]byte, 4+8*chunkCount) // allocate space for multiple uint64 checksum values
+		assert.NoError(suite.T(), suite.ioctx.Checksum(oid, ChecksumTypeXXHash64, dst, &ChecksumOptions{
+			InitValue: init,
+			ChunkSize: chunkSize,
+			Off:       off,
+			Len:       chunkSize * chunkCount,
+		}))
+
+		chunks := binary.LittleEndian.Uint32(dst[:4])
+		assert.Equal(suite.T(), uint32(chunkCount), chunks)
+
+		got := []uint64{
+			binary.LittleEndian.Uint64(dst[4:12]),
+			binary.LittleEndian.Uint64(dst[12:20]),
+		}
+		for i := range want {
+			assert.Equal(suite.T(), want[i], got[i])
+		}
+	})
+}