[ntuple] Rename kTypeDAOS to kTypeObject64 and kTypeS3 to kTypeMulti

Author: JasMehta08

tree/ntuple/doc/BinaryFormatSpecification.md

@@ -253,7 +253,15 @@ The type can take one of the following values
 | 0x01 | Large locator | 64bit size followed by 64bit offset |
 
 Each locator type follows a given format for the payload (see Section "Well-known payload formats" below).
-The range 0x02 - 0x7f is reserved for future use.
+
+Currently assigned non-standard locator types:
+
+| Type | Meaning          | Payload format                                                  |
+|------|------------------|-----------------------------------------------------------------|
+| 0x02 | Object64 locator | Object64 (64-bit object identifier)                             |
+| 0x03 | Multi locator    | Object64 (bit-spliced 64-bit value encoding object ID and byte offset) |
+
+The range 0x04 - 0x7f is reserved for future use.
 
 _Reserved_ is an 8 bit field that can be used by the storage backend corresponding to the type
 in order to store additional information about the locator.
@@ -285,6 +293,25 @@ _Content size_: the number of bytes to read, i.e. the compressed size of the ref
 
 _Content offset_: the 64bit byte offset of the referenced byte range counted from the start of the file.
 
+- _Object64_: A 64-bit object identifier used by object-store backends (types 0x02 and 0x03)
+```
+ 0                   1                   2                   3
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                   Content size (optional)                     +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+|                                                               |
++                        Object64 value                         +
+|                                                               |
++-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+```
+
+The Object64 value is a 64-bit unsigned integer. Its interpretation depends on the locator type:
+for type 0x02, it is a plain object identifier; for type 0x03, the upper 32 bits encode
+an object identifier and the lower 32 bits encode a byte offset within that object.
+
 
 ## Envelopes
 

tree/ntuple/inc/ROOT/RNTupleTypes.hxx

@@ -234,8 +234,8 @@ public:
       // The kTypeFile locator may translate to an on-disk standard locator (type 0x00) or a large locator (type 0x01),
       // if the size of the referenced data block is >2GB
       kTypeFile = 0x00,
-      kTypeDAOS = 0x02,
-      kTypeS3 = 0x03,
+      kTypeObject64 = 0x02,
+      kTypeMulti = 0x03,
 
       kLastSerializableType = 0x7f,
       kTypePageZero = kLastSerializableType + 1,
@@ -249,7 +249,7 @@ private:
    static constexpr std::uint64_t kMaskType = 0x07ULL << 61;
    static constexpr std::uint64_t kMaskReservedBit = 1ull << 60;
 
-   /// To save memory, we use the most significant bits to store the locator type (file, DAOS, zero page,
+   /// To save memory, we use the most significant bits to store the locator type (file, Object64, zero page,
    /// unkown, kTestLocatorType) as well as one "reserved bit" that can be used in future locators.
    /// Consequently, we can only store sizes up to 60 bits (1 EB), which in practice won't be an issue.
    std::uint64_t fFlagsAndNBytes = 0;
@@ -276,7 +276,7 @@ public:
    void SetType(ELocatorType type);
    void SetReserved(std::uint8_t reserved);
 
-   /// Note that for GetPosition() / SetPosition(), the locator type must correspond (kTypeFile, kTypeDAOS).
+   /// Note that for GetPosition() / SetPosition(), the locator type must correspond (kTypeFile, kTypeObject64).
 
    template <typename T>
    T GetPosition() const

tree/ntuple/src/RNTupleSerialize.cxx

@@ -1086,19 +1086,19 @@ ROOT::Internal::RNTupleSerializer::SerializeLocator(const RNTupleLocator &locato
       size += SerializeLocatorPayloadLarge(locator, payloadp);
       locatorType = 0x01;
       break;
-   case RNTupleLocator::kTypeDAOS:
+   case RNTupleLocator::kTypeObject64:
       size += SerializeLocatorPayloadObject64(locator, payloadp);
       locatorType = 0x02;
       break;
-   case RNTupleLocator::kTypeS3:
+   case RNTupleLocator::kTypeMulti:
       size += SerializeLocatorPayloadObject64(locator, payloadp);
       locatorType = 0x03;
       break;
    default:
       if (locator.GetType() == ROOT::Internal::kTestLocatorType) {
          // For the testing locator, use the same payload format as Object64. We won't read it back anyway.
          RNTupleLocator dummy;
-         dummy.SetType(RNTupleLocator::kTypeDAOS);
+         dummy.SetType(RNTupleLocator::kTypeObject64);
          size += SerializeLocatorPayloadObject64(dummy, payloadp);
          locatorType = 0x7e;
       } else {
@@ -1139,11 +1139,11 @@ ROOT::RResult<std::uint32_t> ROOT::Internal::RNTupleSerializer::DeserializeLocat
          DeserializeLocatorPayloadLarge(bytes, locator);
          break;
       case 0x02:
-         locator.SetType(RNTupleLocator::kTypeDAOS);
+         locator.SetType(RNTupleLocator::kTypeObject64);
          DeserializeLocatorPayloadObject64(bytes, payloadSize, locator);
          break;
       case 0x03:
-         locator.SetType(RNTupleLocator::kTypeS3);
+         locator.SetType(RNTupleLocator::kTypeMulti);
          DeserializeLocatorPayloadObject64(bytes, payloadSize, locator);
          break;
       default: locator.SetType(RNTupleLocator::kTypeUnknown);

tree/ntuple/src/RNTupleTypes.cxx

@@ -41,10 +41,10 @@ ROOT::RNTupleLocator::ELocatorType ROOT::RNTupleLocator::GetType() const
    std::uint64_t compactType = fFlagsAndNBytes >> 61;
    switch (compactType) {
    case 0: return kTypeFile;
-   case 1: return kTypeDAOS;
+   case 1: return kTypeObject64;
    case 2: return kTypePageZero;
    case 3: return kTypeUnknown;
-   case 4: return kTypeS3;
+   case 4: return kTypeMulti;
    case 5: return Internal::kTestLocatorType;
    default: break;
    }
@@ -57,10 +57,10 @@ void ROOT::RNTupleLocator::SetType(ELocatorType type)
    std::uint64_t compactType;
    switch (type) {
    case kTypeFile: compactType = 0; break;
-   case kTypeDAOS: compactType = 1; break;
+   case kTypeObject64: compactType = 1; break;
    case kTypePageZero: compactType = 2; break;
    case kTypeUnknown: compactType = 3; break;
-   case kTypeS3: compactType = 4; break;
+   case kTypeMulti: compactType = 4; break;
    default:
       if (type == Internal::kTestLocatorType)
          compactType = 5;
@@ -80,7 +80,7 @@ void ROOT::RNTupleLocator::SetPosition(std::uint64_t position)
 
 void ROOT::RNTupleLocator::SetPosition(RNTupleLocatorObject64 position)
 {
-   if (GetType() != kTypeDAOS && GetType() != kTypeS3)
+   if (GetType() != kTypeObject64 && GetType() != kTypeMulti)
       throw RException(R__FAIL("cannot set position as 64bit object for type " + std::to_string(GetType())));
    fPosition = position.GetLocation();
 }
@@ -95,7 +95,7 @@ std::uint64_t ROOT::Internal::RNTupleLocatorHelper<std::uint64_t>::Get(const RNT
 ROOT::RNTupleLocatorObject64
 ROOT::Internal::RNTupleLocatorHelper<ROOT::RNTupleLocatorObject64>::Get(const RNTupleLocator &loc)
 {
-   if (loc.GetType() != ROOT::RNTupleLocator::kTypeDAOS && loc.GetType() != ROOT::RNTupleLocator::kTypeS3)
+   if (loc.GetType() != ROOT::RNTupleLocator::kTypeObject64 && loc.GetType() != ROOT::RNTupleLocator::kTypeMulti)
       throw RException(R__FAIL("cannot retrieve position as 64bit object for type " + std::to_string(loc.GetType())));
    return RNTupleLocatorObject64{loc.fPosition};
 }

tree/ntuple/src/RPageStorageDaos.cxx

@@ -305,7 +305,7 @@ ROOT::Experimental::Internal::RPageSinkDaos::CommitSealedPageImpl(ROOT::Descript
    }
 
    RNTupleLocator result;
-   result.SetType(RNTupleLocator::kTypeDAOS);
+   result.SetType(RNTupleLocator::kTypeObject64);
    result.SetNBytesOnStorage(sealedPage.GetDataSize());
    result.SetPosition(ROOT::RNTupleLocatorObject64{pageId});
    fCounters->fNPageCommitted.Inc();
@@ -341,7 +341,7 @@ ROOT::Experimental::Internal::RPageSinkDaos::CommitSealedPageVImpl(std::span<RPa
          it->second.Insert(daosKey.fAkey, pageIov);
 
          RNTupleLocator locator;
-         locator.SetType(RNTupleLocator::kTypeDAOS);
+         locator.SetType(RNTupleLocator::kTypeObject64);
          locator.SetNBytesOnStorage(s.GetDataSize());
          locator.SetPosition(ROOT::RNTupleLocatorObject64{pageId});
          locators.push_back(locator);
@@ -382,7 +382,7 @@ ROOT::Experimental::Internal::RPageSinkDaos::CommitClusterGroupImpl(unsigned cha
       daos_obj_id_t{kOidLowPageList, static_cast<decltype(daos_obj_id_t::hi)>(fNTupleIndex)}, kDistributionKeyDefault,
       offsetData, kCidMetadata);
    RNTupleLocator result;
-   result.SetType(RNTupleLocator::kTypeDAOS);
+   result.SetType(RNTupleLocator::kTypeObject64);
    result.SetNBytesOnStorage(szPageListZip);
    result.SetPosition(RNTupleLocatorObject64{offsetData});
    fCounters->fSzWritePayload.Add(static_cast<int64_t>(szPageListZip));

tree/ntuple/test/ntuple_serialize.cxx

@@ -377,14 +377,14 @@ TEST(RNTuple, SerializeLocator)
    EXPECT_EQ(1u, locator.GetPosition<std::uint64_t>());
    EXPECT_EQ(RNTupleLocator::kTypeFile, locator.GetType());
 
-   locator.SetType(RNTupleLocator::kTypeDAOS);
+   locator.SetType(RNTupleLocator::kTypeObject64);
    locator.SetPosition(RNTupleLocatorObject64{1337U});
    locator.SetNBytesOnStorage(420420U);
    locator.SetReserved(0);
    EXPECT_EQ(16u, RNTupleSerializer::SerializeLocator(locator, buffer).Unwrap());
    locator = RNTupleLocator{};
    EXPECT_EQ(16u, RNTupleSerializer::DeserializeLocator(buffer, 16, locator).Unwrap());
-   EXPECT_EQ(locator.GetType(), RNTupleLocator::kTypeDAOS);
+   EXPECT_EQ(locator.GetType(), RNTupleLocator::kTypeObject64);
    EXPECT_EQ(locator.GetNBytesOnStorage(), 420420U);
    EXPECT_EQ(locator.GetReserved(), 0);
    EXPECT_EQ(1337U, locator.GetPosition<RNTupleLocatorObject64>().GetLocation());
@@ -394,32 +394,32 @@ TEST(RNTuple, SerializeLocator)
    EXPECT_EQ(20u, RNTupleSerializer::SerializeLocator(locator, buffer).Unwrap());
    locator = RNTupleLocator{};
    EXPECT_EQ(20u, RNTupleSerializer::DeserializeLocator(buffer, 20, locator).Unwrap());
-   EXPECT_EQ(locator.GetType(), RNTupleLocator::kTypeDAOS);
+   EXPECT_EQ(locator.GetType(), RNTupleLocator::kTypeObject64);
    EXPECT_EQ(locator.GetNBytesOnStorage(), static_cast<std::uint64_t>(std::numeric_limits<std::uint32_t>::max()) + 1);
    EXPECT_EQ(locator.GetReserved(), 1);
    EXPECT_EQ(1337U, locator.GetPosition<RNTupleLocatorObject64>().GetLocation());
 
-   // S3 locator round-trip with 32-bit nBytesOnStorage
+   // Multi locator round-trip with 32-bit nBytesOnStorage
    locator = RNTupleLocator{};
-   locator.SetType(RNTupleLocator::kTypeS3);
+   locator.SetType(RNTupleLocator::kTypeMulti);
    locator.SetPosition(RNTupleLocatorObject64{42U});
    locator.SetNBytesOnStorage(1024U);
    locator.SetReserved(0);
    EXPECT_EQ(16u, RNTupleSerializer::SerializeLocator(locator, buffer).Unwrap());
    locator = RNTupleLocator{};
    EXPECT_EQ(16u, RNTupleSerializer::DeserializeLocator(buffer, 16, locator).Unwrap());
-   EXPECT_EQ(locator.GetType(), RNTupleLocator::kTypeS3);
+   EXPECT_EQ(locator.GetType(), RNTupleLocator::kTypeMulti);
    EXPECT_EQ(locator.GetNBytesOnStorage(), 1024U);
    EXPECT_EQ(locator.GetReserved(), 0);
    EXPECT_EQ(42U, locator.GetPosition<RNTupleLocatorObject64>().GetLocation());
 
-   // S3 locator round-trip with 64-bit nBytesOnStorage and reserved bit
+   // Multi locator round-trip with 64-bit nBytesOnStorage and reserved bit
    locator.SetNBytesOnStorage(static_cast<std::uint64_t>(std::numeric_limits<std::uint32_t>::max()) + 1);
    locator.SetReserved(1);
    EXPECT_EQ(20u, RNTupleSerializer::SerializeLocator(locator, buffer).Unwrap());
    locator = RNTupleLocator{};
    EXPECT_EQ(20u, RNTupleSerializer::DeserializeLocator(buffer, 20, locator).Unwrap());
-   EXPECT_EQ(locator.GetType(), RNTupleLocator::kTypeS3);
+   EXPECT_EQ(locator.GetType(), RNTupleLocator::kTypeMulti);
    EXPECT_EQ(locator.GetNBytesOnStorage(), static_cast<std::uint64_t>(std::numeric_limits<std::uint32_t>::max()) + 1);
    EXPECT_EQ(locator.GetReserved(), 1);
    EXPECT_EQ(42U, locator.GetPosition<RNTupleLocatorObject64>().GetLocation());