Skip to content

Commit 77b7180

Browse files
JasMehta08JasMehta08
committed
[ntuple] Create anchor and its tests for S3 backend
1 parent a9cc13f commit 77b7180

5 files changed

Lines changed: 535 additions & 0 deletions

File tree

‎tree/ntuple/CMakeLists.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,13 @@ if(daos OR daos_mock)
121121
endif()
122122
endif()
123123

124+
# Enable RNTuple support for S3-compatible object storage
125+
if(curl)
126+
set(ROOTNTuple_EXTRA_HEADERS ${ROOTNTuple_EXTRA_HEADERS} ROOT/RPageStorageS3.hxx)
127+
target_sources(ROOTNTuple PRIVATE src/RPageStorageS3.cxx)
128+
target_link_libraries(ROOTNTuple PRIVATE nlohmann_json::nlohmann_json)
129+
endif()
130+
124131
if(MSVC)
125132
target_compile_definitions(ROOTNTuple PRIVATE _USE_MATH_DEFINES)
126133
endif()

‎tree/ntuple/inc/ROOT/RPageStorageS3.hxx‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/// \file ROOT/RPageStorageS3.hxx
2+
/// \author Jas Mehta <jasmehta805@gmail.com>
3+
/// \date 2026-06-01
4+
5+
/*************************************************************************
6+
* Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. *
7+
* All rights reserved. *
8+
* *
9+
* For the licensing terms see $ROOTSYS/LICENSE. *
10+
* For the list of contributors see $ROOTSYS/README/CREDITS. *
11+
*************************************************************************/
12+
13+
#ifndef ROOT_RPageStorageS3
14+
#define ROOT_RPageStorageS3
15+
16+
#include <ROOT/RError.hxx>
17+
#include <ROOT/RNTuple.hxx>
18+
19+
#include <cstdint>
20+
#include <string>
21+
22+
namespace ROOT {
23+
namespace Experimental {
24+
namespace Internal {
25+
26+
// clang-format off
27+
/**
28+
\class ROOT::Experimental::Internal::RNTupleAnchorS3
29+
\ingroup NTuple
30+
\brief Entry point for an RNTuple stored in S3-compatible object storage.
31+
32+
The anchor is serialized as a JSON object and stored at the base URL of the ntuple.
33+
It contains the information needed to locate and read the header and footer envelopes.
34+
The anchor is always the last object written during CommitDatasetImpl, ensuring atomicity:
35+
if the anchor exists, the entire ntuple is complete.
36+
*/
37+
// clang-format on
38+
struct RNTupleAnchorS3 {
39+
/// Allows evolving the anchor JSON schema in future versions
40+
std::uint32_t fVersionAnchor = 0;
41+
/// Version of the RNTuple binary format supported by the writer
42+
std::uint16_t fVersionEpoch = RNTuple::kVersionEpoch;
43+
std::uint16_t fVersionMajor = RNTuple::kVersionMajor;
44+
std::uint16_t fVersionMinor = RNTuple::kVersionMinor;
45+
std::uint16_t fVersionPatch = RNTuple::kVersionPatch;
46+
/// Pattern for resolving object IDs to full S3 URLs.
47+
/// ${baseurl} is replaced with the anchor URL, ${objid} with the numeric object ID.
48+
std::string fUrlTemplate;
49+
/// Object ID and byte offset of the compressed header within the S3 object
50+
std::uint64_t fHeaderObjId = 0;
51+
std::uint64_t fHeaderOffset = 0;
52+
/// Compressed and uncompressed sizes of the header envelope
53+
std::uint64_t fNBytesHeader = 0;
54+
std::uint64_t fLenHeader = 0;
55+
/// Object ID and byte offset of the compressed footer within the S3 object
56+
std::uint64_t fFooterObjId = 0;
57+
std::uint64_t fFooterOffset = 0;
58+
/// Compressed and uncompressed sizes of the footer envelope
59+
std::uint64_t fNBytesFooter = 0;
60+
std::uint64_t fLenFooter = 0;
61+
62+
bool operator==(const RNTupleAnchorS3 &other) const;
63+
64+
/// Serialize the anchor to a JSON string suitable for storage at the base URL
65+
std::string ToJSON() const;
66+
/// Deserialize the anchor from a JSON string. Returns an error on malformed or incompatible input.
67+
static RResult<RNTupleAnchorS3> FromJSON(const std::string &json);
68+
};
69+
70+
} // namespace Internal
71+
} // namespace Experimental
72+
} // namespace ROOT
73+
74+
#endif

‎tree/ntuple/src/RPageStorageS3.cxx‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/// \file RPageStorageS3.cxx
2+
/// \author Jas Mehta <jasmehta805@gmail.com>
3+
/// \date 2026-06-01
4+
5+
/*************************************************************************
6+
* Copyright (C) 1995-2026, Rene Brun and Fons Rademakers. *
7+
* All rights reserved. *
8+
* *
9+
* For the licensing terms see $ROOTSYS/LICENSE. *
10+
* For the list of contributors see $ROOTSYS/README/CREDITS. *
11+
*************************************************************************/
12+
13+
#include <ROOT/RPageStorageS3.hxx>
14+
15+
#include <nlohmann/json.hpp>
16+
17+
#include <string>
18+
19+
/// Field-by-field equality check across all 14 anchor members.
20+
/// Used to verify round-trip correctness in tests.
21+
bool ROOT::Experimental::Internal::RNTupleAnchorS3::operator==(const RNTupleAnchorS3 &other) const
22+
{
23+
return fVersionAnchor == other.fVersionAnchor && fVersionEpoch == other.fVersionEpoch &&
24+
fVersionMajor == other.fVersionMajor && fVersionMinor == other.fVersionMinor &&
25+
fVersionPatch == other.fVersionPatch && fUrlTemplate == other.fUrlTemplate &&
26+
fHeaderObjId == other.fHeaderObjId && fHeaderOffset == other.fHeaderOffset &&
27+
fNBytesHeader == other.fNBytesHeader && fLenHeader == other.fLenHeader &&
28+
fFooterObjId == other.fFooterObjId && fFooterOffset == other.fFooterOffset &&
29+
fNBytesFooter == other.fNBytesFooter && fLenFooter == other.fLenFooter;
30+
}
31+
32+
/// Serialize the anchor to a pretty-printed JSON string (2-space indent).
33+
/// nlohmann/json handles type conversion, string escaping, and uint64 precision.
34+
/// The output is suitable for direct upload to S3 as the anchor object.
35+
std::string ROOT::Experimental::Internal::RNTupleAnchorS3::ToJSON() const
36+
{
37+
nlohmann::json jsonAnchor;
38+
jsonAnchor["anchor_version"] = fVersionAnchor;
39+
jsonAnchor["format_version_epoch"] = fVersionEpoch;
40+
jsonAnchor["format_version_major"] = fVersionMajor;
41+
jsonAnchor["format_version_minor"] = fVersionMinor;
42+
jsonAnchor["format_version_patch"] = fVersionPatch;
43+
jsonAnchor["url_template"] = fUrlTemplate;
44+
jsonAnchor["header_obj_id"] = fHeaderObjId;
45+
jsonAnchor["header_offset"] = fHeaderOffset;
46+
jsonAnchor["nbytes_header"] = fNBytesHeader;
47+
jsonAnchor["len_header"] = fLenHeader;
48+
jsonAnchor["footer_obj_id"] = fFooterObjId;
49+
jsonAnchor["footer_offset"] = fFooterOffset;
50+
jsonAnchor["nbytes_footer"] = fNBytesFooter;
51+
jsonAnchor["len_footer"] = fLenFooter;
52+
return jsonAnchor.dump(2);
53+
}
54+
55+
/// Construct an anchor from a JSON string.
56+
/// The anchor version is checked first; if it does not match the current version,
57+
/// parsing fails immediately. All remaining fields are extracted with jsonAnchor.at()
58+
/// which throws on missing keys or type mismatches.
59+
ROOT::RResult<ROOT::Experimental::Internal::RNTupleAnchorS3>
60+
ROOT::Experimental::Internal::RNTupleAnchorS3::FromJSON(const std::string &json)
61+
{
62+
nlohmann::json jsonAnchor;
63+
try {
64+
jsonAnchor = nlohmann::json::parse(json);
65+
} catch (const nlohmann::json::parse_error &e) {
66+
return R__FAIL("cannot parse S3 anchor JSON: " + std::string(e.what()));
67+
}
68+
69+
RNTupleAnchorS3 anchor;
70+
71+
try {
72+
anchor.fVersionAnchor = jsonAnchor.at("anchor_version").get<std::uint32_t>();
73+
} catch (const nlohmann::json::exception &e) {
74+
return R__FAIL("missing or invalid 'anchor_version' in S3 anchor: " + std::string(e.what()));
75+
}
76+
77+
if (anchor.fVersionAnchor != RNTupleAnchorS3().fVersionAnchor)
78+
return R__FAIL("unsupported S3 anchor version: " + std::to_string(anchor.fVersionAnchor));
79+
80+
try {
81+
anchor.fVersionEpoch = jsonAnchor.at("format_version_epoch").get<std::uint16_t>();
82+
anchor.fVersionMajor = jsonAnchor.at("format_version_major").get<std::uint16_t>();
83+
anchor.fVersionMinor = jsonAnchor.at("format_version_minor").get<std::uint16_t>();
84+
anchor.fVersionPatch = jsonAnchor.at("format_version_patch").get<std::uint16_t>();
85+
anchor.fUrlTemplate = jsonAnchor.at("url_template").get<std::string>();
86+
anchor.fHeaderObjId = jsonAnchor.at("header_obj_id").get<std::uint64_t>();
87+
anchor.fHeaderOffset = jsonAnchor.at("header_offset").get<std::uint64_t>();
88+
anchor.fNBytesHeader = jsonAnchor.at("nbytes_header").get<std::uint64_t>();
89+
anchor.fLenHeader = jsonAnchor.at("len_header").get<std::uint64_t>();
90+
anchor.fFooterObjId = jsonAnchor.at("footer_obj_id").get<std::uint64_t>();
91+
anchor.fFooterOffset = jsonAnchor.at("footer_offset").get<std::uint64_t>();
92+
anchor.fNBytesFooter = jsonAnchor.at("nbytes_footer").get<std::uint64_t>();
93+
anchor.fLenFooter = jsonAnchor.at("len_footer").get<std::uint64_t>();
94+
} catch (const nlohmann::json::exception &e) {
95+
return R__FAIL("missing or invalid field in S3 anchor: " + std::string(e.what()));
96+
}
97+
98+
return anchor;
99+
}

‎tree/ntuple/test/CMakeLists.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,10 @@ if(daos OR daos_mock)
176176
endif()
177177
endif()
178178

179+
if(curl)
180+
ROOT_ADD_GTEST(ntuple_storage_s3 ntuple_storage_s3.cxx LIBRARIES ROOTNTuple)
181+
endif()
182+
179183

180184
# RNTuple Python interface tests
181185
if(pyroot)

0 commit comments

Comments
 (0)