[ntuple] implement RSoAField::ReadGlobalImpl()

Author: jblomer

tree/ntuple/inc/ROOT/RField/RFieldSequenceContainer.hxx

@@ -31,6 +31,10 @@ namespace Detail {
 class RFieldVisitor;
 } // namespace Detail
 
+namespace Experimental {
+class RSoAField;
+}
+
 namespace Internal {
 std::unique_ptr<RFieldBase> CreateEmulatedVectorField(std::string_view fieldName, std::unique_ptr<RFieldBase> itemField,
                                                       std::string_view emulatedFromType);
@@ -112,6 +116,7 @@ public:
 /// The type-erased field for a RVec<Type>
 class RRVecField : public RFieldBase {
    friend class RArrayAsRVecField; // to use the RRVecDeleter and to call ResizeRVec()
+   friend class ROOT::Experimental::RSoAField; // to call ResizeRVec()
 
    // Ensures that the RVec pointed to by rvec has at least nItems valid elements
    // Returns the possibly new "begin pointer" of the RVec, i.e. the pointer to the data area.

tree/ntuple/inc/ROOT/RField/RFieldSoA.hxx

@@ -66,6 +66,8 @@ class RSoAField : public RFieldBase {
    /// The offset of the RVec members in the SoA type in the order of subfields of the underlying record type.
    /// In particular, the order is not necessarily the same then the order of RVec members in the SoA class.
    std::vector<std::size_t> fSoAMemberOffsets;
+   ///< A deleter returned by each record member's GetDeleter()
+   std::vector<std::unique_ptr<RDeleter>> fRecordMemberDeleters;
    std::size_t fMaxAlignment = 1;
    ROOT::Internal::RColumnIndex fNWritten;
 
@@ -85,8 +87,6 @@ protected:
    std::size_t AppendImpl(const void *from) final;
    void ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void *to) final;
 
-   void ReconcileOnDiskField(const RNTupleDescriptor &) final {}
-
    void CommitClusterImpl() final { fNWritten = 0; }
 
 public:

tree/ntuple/src/RFieldMeta.cxx

@@ -662,6 +662,9 @@ ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, const RSoAF
    fTraits = source.GetTraits();
    Attach(source.fSubfields[0]->Clone(source.fSubfields[0]->GetFieldName()));
    fRecordMemberFields = fSubfields[0]->GetMutableSubfields();
+   fRecordMemberDeleters.reserve(fRecordMemberFields.size());
+   for (const auto f : fRecordMemberFields)
+      fRecordMemberDeleters.emplace_back(GetDeleterOf(*f));
 }
 
 ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, std::string_view className)
@@ -700,13 +703,16 @@ ROOT::Experimental::RSoAField::RSoAField(std::string_view fieldName, TClass *clS
    fRecordMemberFields = fSubfields[0]->GetMutableSubfields();
 
    std::unordered_map<std::string, std::size_t> recordFieldNameToIdx;
+   fRecordMemberDeleters.reserve(fRecordMemberFields.size());
+   recordFieldNameToIdx.reserve(fRecordMemberFields.size());
    for (std::size_t i = 0; i < fRecordMemberFields.size(); ++i) {
       const RFieldBase *f = fRecordMemberFields[i];
       assert(!f->GetFieldName().empty());
       if (f->GetFieldName()[0] == ':') {
          throw RException(R__FAIL("SoA fields with inheritance are currently unsupported"));
       }
       recordFieldNameToIdx[f->GetFieldName()] = i;
+      fRecordMemberDeleters.emplace_back(GetDeleterOf(*f));
    }
 
    const auto *bases = fSoAClass->GetListOfBases();
@@ -840,9 +846,28 @@ std::size_t ROOT::Experimental::RSoAField::AppendImpl(const void *from)
    return nbytes + fPrincipalColumn->GetElement()->GetPackedSize();
 }
 
-void ROOT::Experimental::RSoAField::ReadGlobalImpl(ROOT::NTupleSize_t /* globalIndex */, void * /* to */)
+void ROOT::Experimental::RSoAField::ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void *to)
 {
-   throw RException(R__FAIL("not yet implemented"));
+   // Read collection info for this entry
+   ROOT::NTupleSize_t N;
+   RNTupleLocalIndex collectionStart;
+   fPrincipalColumn->GetCollectionInfo(globalIndex, &collectionStart, &N);
+
+   const auto nSoAMembers = fSoAMemberOffsets.size();
+   for (std::size_t i = 0; i < nSoAMembers; ++i) {
+      RFieldBase *memberField = fRecordMemberFields[i];
+      const auto memberSize = memberField->GetValueSize();
+      void *rvecPtr = static_cast<unsigned char *>(to) + fSoAMemberOffsets[i];
+      auto begin = ROOT::RRVecField::ResizeRVec(rvecPtr, N, memberSize, memberField, fRecordMemberDeleters[i].get());
+
+      if (memberField->IsSimple() && N) {
+         GetPrincipalColumnOf(*memberField)->ReadV(collectionStart, N, begin);
+      } else {
+         for (std::size_t j = 0; j < N; ++j) {
+            CallReadOn(*memberField, collectionStart + j, begin + (j * memberSize));
+         }
+      }
+   }
 }
 
 void ROOT::Experimental::RSoAField::ConstructValue(void *where) const

tree/ntuple/test/SoAField.hxx

@@ -91,4 +91,27 @@ struct SoASimpleWrongMember {
    ClassDefNV(SoASimpleWrongMember, 2);
 };
 
+/// class with non-trivial constructor and destructor
+struct ComplexMember {
+   inline static int gNCallConstructor = 0;
+   inline static int gNCallDestructor = 0;
+
+   ComplexMember() { gNCallConstructor++; }
+   ~ComplexMember() { gNCallDestructor++; }
+
+   ClassDefNV(ComplexMember, 2);
+};
+
+struct RecordComplex {
+   ComplexMember fA;
+
+   ClassDefNV(RecordComplex, 2);
+};
+
+struct SoAComplex {
+   ROOT::RVec<ComplexMember> fA;
+
+   ClassDefNV(SoAComplex, 2);
+};
+
 #endif // ROOT_RNTuple_Test_SoAField

tree/ntuple/test/SoAFieldLinkDef.h

@@ -21,4 +21,8 @@
 #pragma link C++ options=rntupleSoARecord(RecordSimple) class SoASimpleMissingMember+;
 #pragma link C++ options=rntupleSoARecord(RecordSimple) class SoASimpleWrongMember+;
 
+#pragma link C++ class ComplexMember+;
+#pragma link C++ class RecordComplex+;
+#pragma link C++ options=rntupleSoARecord(RecordComplex) class SoAComplex+;
+
 #endif // __CLING__

tree/ntuple/test/ntuple_soa.cxx

@@ -264,3 +264,154 @@ TEST(RNTuple, SoASimpleSwapped)
    EXPECT_FLOAT_EQ(1.0, v(0).at(0).fX);
    EXPECT_FLOAT_EQ(2.0, v(0).at(0).fY);
 }
+
+TEST(RNTuple, SoABasicWriteRead)
+{
+   ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_write_read.root");
+
+   {
+      auto model = ROOT::RNTupleModel::Create();
+      model->AddField(std::make_unique<RSoAField>("simple", "SoASimple"));
+      model->AddField(std::make_unique<RSoAField>("empty", "SoA"));
+
+      auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath());
+      auto simpleSoA = writer->GetModel().GetDefaultEntry().GetPtr<SoASimple>("simple");
+      auto emptySoA = writer->GetModel().GetDefaultEntry().GetPtr<SoA>("empty");
+
+      simpleSoA->fX.push_back(1.0);
+      simpleSoA->fY.push_back(2.0);
+      writer->Fill();
+      writer->CommitCluster();
+
+      simpleSoA->fX.clear();
+      simpleSoA->fY.clear();
+      writer->Fill();
+
+      simpleSoA->fX.push_back(3.0);
+      simpleSoA->fY.push_back(4.0);
+      simpleSoA->fX.push_back(5.0);
+      simpleSoA->fY.push_back(6.0);
+      writer->Fill();
+   }
+
+   auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath());
+   EXPECT_EQ(3u, reader->GetNEntries());
+   auto simpleSoA = reader->GetModel().GetDefaultEntry().GetPtr<SoASimple>("simple");
+
+   reader->LoadEntry(0);
+   EXPECT_EQ(1U, simpleSoA->fX.size());
+   EXPECT_EQ(1U, simpleSoA->fY.size());
+   EXPECT_FLOAT_EQ(1.0, simpleSoA->fX[0]);
+   EXPECT_FLOAT_EQ(2.0, simpleSoA->fY[0]);
+
+   reader->LoadEntry(1);
+   EXPECT_TRUE(simpleSoA->fX.empty());
+   EXPECT_TRUE(simpleSoA->fY.empty());
+
+   reader->LoadEntry(2);
+   EXPECT_EQ(2U, simpleSoA->fX.size());
+   EXPECT_EQ(2U, simpleSoA->fY.size());
+   EXPECT_FLOAT_EQ(3.0, simpleSoA->fX[0]);
+   EXPECT_FLOAT_EQ(4.0, simpleSoA->fY[0]);
+   EXPECT_FLOAT_EQ(5.0, simpleSoA->fX[1]);
+   EXPECT_FLOAT_EQ(6.0, simpleSoA->fY[1]);
+}
+
+TEST(RNTuple, SoAReadAdopted)
+{
+   ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_read_adopted.root");
+
+   {
+      auto model = ROOT::RNTupleModel::Create();
+      model->AddField(std::make_unique<RSoAField>("simple", "SoASimple"));
+
+      auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath());
+      auto simpleSoA = writer->GetModel().GetDefaultEntry().GetPtr<SoASimple>("simple");
+
+      simpleSoA->fX.push_back(1.0);
+      simpleSoA->fY.push_back(2.0);
+      simpleSoA->fX.push_back(3.0);
+      simpleSoA->fY.push_back(4.0);
+      writer->Fill();
+   }
+
+   auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath());
+   EXPECT_EQ(1u, reader->GetNEntries());
+
+   auto viewSize = reader->GetView<ROOT::RNTupleCardinality<std::uint64_t>>("simple");
+   EXPECT_EQ(2u, viewSize(0));
+
+   float x[2] = {.0, .0};
+   float y[2] = {.0, .0};
+   SoASimple soa;
+   soa.fX = ROOT::RVec<float>(x, 2);
+   soa.fY = ROOT::RVec<float>(y, 2);
+   auto viewSoA = reader->GetView("simple", &soa, "SoASimple");
+   viewSoA(0);
+   EXPECT_FLOAT_EQ(1.0, x[0]);
+   EXPECT_FLOAT_EQ(2.0, y[0]);
+   EXPECT_FLOAT_EQ(3.0, x[1]);
+   EXPECT_FLOAT_EQ(4.0, y[1]);
+}
+
+TEST(RNTuple, SoAReadComplex)
+{
+   ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_read_complex.root");
+
+   {
+      auto model = ROOT::RNTupleModel::Create();
+      model->AddField(std::make_unique<RSoAField>("complex", "SoAComplex"));
+
+      auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath());
+      auto complexSoA = writer->GetModel().GetDefaultEntry().GetPtr<SoAComplex>("complex");
+
+      complexSoA->fA.resize(2);
+      writer->Fill();
+      complexSoA->fA.clear();
+      writer->Fill();
+   }
+
+   auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath());
+   EXPECT_EQ(2u, reader->GetNEntries());
+
+   auto complexSoA = reader->GetModel().GetDefaultEntry().GetPtr<SoAComplex>("complex");
+   ComplexMember::gNCallConstructor = 0;
+   ComplexMember::gNCallDestructor = 0;
+
+   reader->LoadEntry(0);
+   EXPECT_EQ(2U, complexSoA->fA.size());
+   EXPECT_EQ(2, ComplexMember::gNCallConstructor);
+   EXPECT_EQ(0, ComplexMember::gNCallDestructor);
+
+   reader->LoadEntry(1);
+   EXPECT_TRUE(complexSoA->fA.empty());
+   EXPECT_EQ(2, ComplexMember::gNCallConstructor);
+   EXPECT_EQ(2, ComplexMember::gNCallDestructor);
+}
+
+TEST(RNTuple, SoAFromVector)
+{
+   ROOT::TestSupport::FileRaii fileGuard("test_rntuple_soa_from_vector.root");
+
+   {
+      auto model = ROOT::RNTupleModel::Create();
+      auto v = model->MakeField<std::vector<RecordSimple>>("simple");
+
+      auto writer = ROOT::RNTupleWriter::Recreate(std::move(model), "ntpl", fileGuard.GetPath());
+      v->emplace_back(RecordSimple{1.0, 2.0});
+
+      writer->Fill();
+   }
+
+   auto reader = ROOT::RNTupleReader::Open("ntpl", fileGuard.GetPath());
+   SoASimple soa;
+
+   // Until SoA schema evolution is implemented, the reading the vector as SoA will
+   try {
+      reader->GetView("simple", &soa, "SoASimple");
+      FAIL() << "reading a vector with a SoA field should fail";
+   } catch (const ROOT::RException &e) {
+      EXPECT_THAT(e.what(), testing::HasSubstr(
+                               "in-memory field simple of type SoASimple is incompatible with on-disk field simple"));
+   }
+}