Add computed metrics (randomness, sparseness) to RNTuple

Author: JasMehta08

tree/ntuple/inc/ROOT/RPageStorageFile.hxx

@@ -152,6 +152,19 @@ private:
    RNTupleDescriptorBuilder fDescriptorBuilder;
    /// Populated by LoadStructureImpl(), reset at the end of Attach()
    RStructureBuffer fStructureBuffer;
+   /// Tracks the last read offset for seek distance calculation
+   std::uint64_t fLastOffset = 0;
+
+   /// File-specific I/O performance counters
+   struct RFileCounters {
+      ROOT::Experimental::Detail::RNTupleAtomicCounter &fSzSkip;
+      ROOT::Experimental::Detail::RNTupleCalcPerf &fSzFile;
+      ROOT::Experimental::Detail::RNTupleCalcPerf &fRandomness;
+      ROOT::Experimental::Detail::RNTupleCalcPerf &fSparseness;
+   };
+   std::unique_ptr<RFileCounters> fFileCounters;
+   /// Total file size, set once in AttachImpl()
+   std::int64_t fFileSize = 0;
 
    RPageSourceFile(std::string_view ntupleName, const ROOT::RNTupleReadOptions &options);
 

tree/ntuple/src/RPageStorageFile.cxx

@@ -42,7 +42,20 @@
 #include <functional>
 #include <mutex>
 
+using ROOT::Experimental::Detail::RNTupleAtomicCounter;
 using ROOT::Experimental::Detail::RNTupleAtomicTimer;
+using ROOT::Experimental::Detail::RNTupleCalcPerf;
+using ROOT::Experimental::Detail::RNTupleMetrics;
+using ROOT::Internal::MakeUninitArray;
+using ROOT::Internal::RCluster;
+using ROOT::Internal::RClusterPool;
+using ROOT::Internal::RNTupleCompressor;
+using ROOT::Internal::RNTupleDecompressor;
+using ROOT::Internal::RNTupleFileWriter;
+using ROOT::Internal::RNTupleSerializer;
+using ROOT::Internal::ROnDiskPage;
+using ROOT::Internal::ROnDiskPageMap;
+using ROOT::Internal::RPagePool;
 
 ROOT::Internal::RPageSinkFile::RPageSinkFile(std::string_view ntupleName, const ROOT::RNTupleWriteOptions &options)
    : RPagePersistentSink(ntupleName, options)
@@ -315,6 +328,46 @@ ROOT::Internal::RPageSourceFile::RPageSourceFile(std::string_view ntupleName, co
    : RPageSource(ntupleName, opts)
 {
    EnableDefaultMetrics("RPageSourceFile");
+   fFileCounters = std::make_unique<RFileCounters>(RFileCounters{
+      *fMetrics.MakeCounter<RNTupleAtomicCounter *>("szSkip", "B",
+                                                    "cumulative seek distance (excluding header/footer reads)"),
+      *fMetrics.MakeCounter<RNTupleCalcPerf *>(
+         "szFile", "B", "total file size", fMetrics,
+         [this](const RNTupleMetrics &) -> std::pair<bool, double> {
+            if (fFileSize > 0)
+               return {true, static_cast<double>(fFileSize)};
+            return {false, -1.};
+         }),
+      *fMetrics.MakeCounter<RNTupleCalcPerf *>(
+         "randomness", "",
+         "ratio of seek distance to bytes read (excluding file structure reads)", fMetrics,
+         [](const RNTupleMetrics &metrics) -> std::pair<bool, double> {
+            if (const auto szSkip = metrics.GetLocalCounter("szSkip")) {
+               if (const auto szReadPayload = metrics.GetLocalCounter("szReadPayload")) {
+                  if (const auto szReadOverhead = metrics.GetLocalCounter("szReadOverhead")) {
+                     auto totalRead = szReadPayload->GetValueAsInt() + szReadOverhead->GetValueAsInt();
+                     if (totalRead > 0) {
+                        return {true, (1. * szSkip->GetValueAsInt()) / totalRead};
+                     }
+                  }
+               }
+            }
+            return {false, -1.};
+         }),
+      *fMetrics.MakeCounter<RNTupleCalcPerf *>(
+         "sparseness", "",
+         "ratio of bytes read to total file size (excluding file structure reads)", fMetrics,
+         [this](const RNTupleMetrics &metrics) -> std::pair<bool, double> {
+            if (fFileSize > 0) {
+               if (const auto szReadPayload = metrics.GetLocalCounter("szReadPayload")) {
+                  if (const auto szReadOverhead = metrics.GetLocalCounter("szReadOverhead")) {
+                     auto totalRead = szReadPayload->GetValueAsInt() + szReadOverhead->GetValueAsInt();
+                     return {true, (1. * totalRead) / fFileSize};
+                  }
+               }
+            }
+            return {false, -1.};
+         })});
 }
 
 ROOT::Internal::RPageSourceFile::RPageSourceFile(std::string_view ntupleName,
@@ -460,6 +513,9 @@ ROOT::RNTupleDescriptor ROOT::Internal::RPageSourceFile::AttachImpl(RNTupleSeria
    // For the page reads, we rely on the I/O scheduler to define the read requests
    fFile->SetBuffering(false);
 
+   // Set file size once after buffering is turned off
+   fFileSize = fFile->GetSize();
+
    return desc;
 }
 
@@ -523,8 +579,16 @@ ROOT::Internal::RPageRef ROOT::Internal::RPageSourceFile::LoadPageImpl(ColumnHan
       directReadBuffer = MakeUninitArray<unsigned char>(sealedPage.GetBufferSize());
       {
          RNTupleAtomicTimer timer(fCounters->fTimeWallRead, fCounters->fTimeCpuRead);
-         fReader.ReadBuffer(directReadBuffer.get(), sealedPage.GetBufferSize(),
-                            pageInfo.GetLocator().GetPosition<std::uint64_t>());
+         const auto offset = pageInfo.GetLocator().GetPosition<std::uint64_t>();
+         // Track seek distance (excluding file structure reads)
+         R__ASSERT(fFileCounters);
+         if (fLastOffset != 0) {
+            const auto distance = static_cast<std::uint64_t>(std::abs(
+               static_cast<std::int64_t>(offset) - static_cast<std::int64_t>(fLastOffset)));
+            fFileCounters->fSzSkip.Add(distance);
+         }
+         fReader.ReadBuffer(directReadBuffer.get(), sealedPage.GetBufferSize(), offset);
+         fLastOffset = offset + sealedPage.GetBufferSize();
       }
       fCounters->fNPageRead.Inc();
       fCounters->fNRead.Inc();
@@ -730,6 +794,18 @@ ROOT::Internal::RPageSourceFile::LoadClusters(std::span<RCluster::RKey> clusterK
          }
       }
 
+      // Track seek distance for each read request (excluding file structure reads)
+      R__ASSERT(fFileCounters);
+      for (std::size_t i = 0; i < nBatch; ++i) {
+         const auto offset = readRequests[iReq + i].fOffset;
+         if (fLastOffset != 0) {
+            const auto distance = static_cast<std::uint64_t>(std::abs(
+               static_cast<std::int64_t>(offset) - static_cast<std::int64_t>(fLastOffset)));
+            fFileCounters->fSzSkip.Add(distance);
+         }
+         fLastOffset = offset + readRequests[iReq + i].fSize;
+      }
+
       if (nBatch <= 1) {
          nBatch = 1;
          RNTupleAtomicTimer timer(fCounters->fTimeWallRead, fCounters->fTimeCpuRead);

tree/ntuple/test/ntuple_metrics.cxx

@@ -112,3 +112,45 @@ TEST(Metrics, RNTupleWriter)
    // one page for the int field, one for the float field
    EXPECT_EQ(2, page_counter->GetValueAsInt());
 }
+
+TEST(Metrics, IOMetrics)
+{
+   FileRaii fileGuard("test_ntuple_io_metrics.root");
+
+   {
+      auto model = RNTupleModel::Create();
+      auto int_field = model->MakeField<int>("ints");
+      auto ntuple = RNTupleWriter::Recreate(std::move(model), "ntuple", fileGuard.GetPath());
+      for (int i = 0; i < 1000; ++i) {
+         *int_field = i;
+         ntuple->Fill();
+      }
+      ntuple->CommitCluster();
+   }
+
+   {
+      auto ntupleReader = RNTupleReader::Open("ntuple", fileGuard.GetPath());
+      EXPECT_FALSE(ntupleReader->GetMetrics().IsEnabled());
+      ntupleReader->EnableMetrics();
+      EXPECT_TRUE(ntupleReader->GetMetrics().IsEnabled());
+
+      auto view = ntupleReader->GetView<int>("ints");
+      for (auto i : *ntupleReader) {
+         (void)view(i);
+      }
+
+      const auto &metrics = ntupleReader->GetMetrics();
+      auto *randomness = metrics.GetCounter("RNTupleReader.RPageSourceFile.randomness");
+      auto *sparseness = metrics.GetCounter("RNTupleReader.RPageSourceFile.sparseness");
+      auto *szSkip = metrics.GetCounter("RNTupleReader.RPageSourceFile.szSkip");
+      auto *szFile = metrics.GetCounter("RNTupleReader.RPageSourceFile.szFile");
+
+      ASSERT_NE(randomness, nullptr);
+      ASSERT_NE(sparseness, nullptr);
+      ASSERT_NE(szSkip, nullptr);
+      ASSERT_NE(szFile, nullptr);
+
+      EXPECT_GE(szSkip->GetValueAsInt(), 0);
+      EXPECT_GT(szFile->GetValueAsInt(), 0);
+   }
+}