diff --git a/cmake/modules/RootTestDriver.cmake b/cmake/modules/RootTestDriver.cmake index 6b65c50c62aae..79ff70ea2e1a4 100644 --- a/cmake/modules/RootTestDriver.cmake +++ b/cmake/modules/RootTestDriver.cmake @@ -134,7 +134,12 @@ endif() #---Execute pre-command----------------------------------------------------------------------------- if(PRE) execute_process(COMMAND ${_pre} ${_cwd} RESULT_VARIABLE _rc) - if(_rc) + if(_rc EQUAL 77) + # Exit code 77 from a pre-command means "skip this test". + # Print a sentinel recognised by SKIP_REGULAR_EXPRESSION on the test. + message(STATUS "ROOTTEST_SKIP: pre-command requested skip -- skipping test") + return() + elseif(_rc) message(FATAL_ERROR "pre-command error code : ${_rc}") endif() endif() diff --git a/core/base/inc/TBuffer.h b/core/base/inc/TBuffer.h index 5490b90f5d58c..1ae6ed901e84f 100644 --- a/core/base/inc/TBuffer.h +++ b/core/base/inc/TBuffer.h @@ -45,10 +45,12 @@ class TBuffer : public TObject { protected: typedef std::vector CacheList_t; + // FIXME: decide if this is signed or unsigned. + using Size_t = std::size_t; Bool_t fMode; //Read or write mode Int_t fVersion; //Buffer format version - Int_t fBufSize; //Size of buffer + Size_t fBufSize; //Size of buffer char *fBuffer; //Buffer used to store objects char *fBufCur; //Current position in buffer char *fBufMax; //End of buffer @@ -88,8 +90,8 @@ class TBuffer : public TObject { enum { kInitialSize = 1024, kMinimalSize = 128 }; TBuffer(EMode mode); - TBuffer(EMode mode, Long64_t bufsize); - TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); + TBuffer(EMode mode, ULong64_t bufsize); + TBuffer(EMode mode, ULong64_t bufsize, void *buf, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); virtual ~TBuffer(); Int_t GetBufferVersion() const { return fVersion; } @@ -97,20 +99,20 @@ class TBuffer : public TObject { Bool_t IsWriting() const { return (fMode & kWrite) != 0; } void SetReadMode(); void SetWriteMode(); - void SetBuffer(void *buf, Long64_t bufsize = 0, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); + void SetBuffer(void *buf, ULong64_t bufsize = 0, Bool_t adopt = kTRUE, ReAllocCharFun_t reallocfunc = nullptr); ReAllocCharFun_t GetReAllocFunc() const; void SetReAllocFunc(ReAllocCharFun_t reallocfunc = nullptr); - void SetBufferOffset(Long64_t offset = 0) { fBufCur = fBuffer+offset; } + void SetBufferOffset(ULong64_t offset = 0) { fBufCur = fBuffer+offset; } void SetParent(TObject *parent); TObject *GetParent() const; char *Buffer() const { return fBuffer; } char *GetCurrent() const { return fBufCur; } - Int_t BufferSize() const { return fBufSize; } + Size_t BufferSize() const { return fBufSize; } void DetachBuffer() { fBuffer = nullptr; } - Int_t Length() const { return (Int_t)(fBufCur - fBuffer); } - void Expand(Long64_t newsize, Bool_t copy = kTRUE); // expand buffer to newsize - void AutoExpand(Long64_t size_needed); // expand buffer to newsize - Bool_t ByteSwapBuffer(Long64_t n, EDataType type); // Byte-swap N primitive-elements in the buffer + Size_t Length() const { return (Size_t)(fBufCur - fBuffer); } + void Expand(ULong64_t newsize, Bool_t copy = kTRUE); // expand buffer to newsize + void AutoExpand(ULong64_t size_needed); // expand buffer to newsize + Bool_t ByteSwapBuffer(ULong64_t n, EDataType type); // Byte-swap N primitive-elements in the buffer virtual Bool_t CheckObject(const TObject *obj) = 0; virtual Bool_t CheckObject(const void *obj, const TClass *ptrClass) = 0; @@ -320,7 +322,7 @@ class TBuffer : public TObject { virtual void ReadFastArray(ULong_t *l, Int_t n) = 0; virtual void ReadFastArray(Long64_t *l, Int_t n) = 0; virtual void ReadFastArray(ULong64_t *l, Int_t n) = 0; - virtual void ReadFastArray(Float_t *f, Int_t n) = 0; + virtual void ReadFastArray(Float_t *f, Long64_t n) = 0; virtual void ReadFastArray(Double_t *d, Int_t n) = 0; virtual void ReadFastArrayFloat16(Float_t *f, Int_t n, TStreamerElement *ele = nullptr) = 0; virtual void ReadFastArrayDouble32(Double_t *d, Int_t n, TStreamerElement *ele = nullptr) = 0; diff --git a/core/base/src/TBuffer.cxx b/core/base/src/TBuffer.cxx index 87a801be6e116..d193c071bd980 100644 --- a/core/base/src/TBuffer.cxx +++ b/core/base/src/TBuffer.cxx @@ -19,8 +19,9 @@ Buffer base class used for serializing objects. #include "TClass.h" #include "TProcessID.h" -constexpr Int_t kExtraSpace = 8; // extra space at end of buffer (used for free block count) -constexpr Int_t kMaxBufferSize = 0x7FFFFFFE; // largest possible size. +constexpr UInt_t kExtraSpace = 8; // extra space at end of buffer (used for free block count) +constexpr UInt_t kMaxBufferSize = 0x7FFFFFFE; // largest possible size before we need to switch to the large buffer format (see TBufferFile::SetByteCount). +constexpr ULong64_t kMaxLargeBufferSize = 0x7FFFFFFFFFFFFFFE; // largest possible size. @@ -69,7 +70,7 @@ TBuffer::TBuffer(EMode mode) /// Create an I/O buffer object. Mode should be either TBuffer::kRead or /// TBuffer::kWrite. -TBuffer::TBuffer(EMode mode, Long64_t bufsize) +TBuffer::TBuffer(EMode mode, ULong64_t bufsize) { if (bufsize > kMaxBufferSize) Fatal("TBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%x.", bufsize, kMaxBufferSize); @@ -100,10 +101,10 @@ TBuffer::TBuffer(EMode mode, Long64_t bufsize) /// is provided, a Fatal error will be issued if the Buffer attempts to /// expand. -TBuffer::TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt, ReAllocCharFun_t reallocfunc) +TBuffer::TBuffer(EMode mode, ULong64_t bufsize, void *buf, Bool_t adopt, ReAllocCharFun_t reallocfunc) { - if (bufsize > kMaxBufferSize) - Fatal("TBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%x.", bufsize, kMaxBufferSize); + if (bufsize > kMaxLargeBufferSize) + Fatal("TBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%llx.", bufsize, kMaxLargeBufferSize); fBufSize = bufsize; fMode = mode; fVersion = 0; @@ -111,12 +112,20 @@ TBuffer::TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt, ReAllocC SetBit(kIsOwner); + if (buf && !adopt) + ResetBit(kIsOwner); + + SetReAllocFunc( reallocfunc ); + if (buf) { fBuffer = (char *)buf; if ( (fMode&kWrite)!=0 ) { - fBufSize -= kExtraSpace; + if (fBufSize < kExtraSpace) { + Expand( kMinimalSize ); + } else { + fBufSize -= kExtraSpace; + } } - if (!adopt) ResetBit(kIsOwner); } else { if (fBufSize < kMinimalSize) { fBufSize = kMinimalSize; @@ -125,12 +134,6 @@ TBuffer::TBuffer(EMode mode, Long64_t bufsize, void *buf, Bool_t adopt, ReAllocC } fBufCur = fBuffer; fBufMax = fBuffer + fBufSize; - - SetReAllocFunc( reallocfunc ); - - if (buf && ( (fMode&kWrite)!=0 ) && fBufSize < 0) { - Expand( kMinimalSize ); - } } //////////////////////////////////////////////////////////////////////////////// @@ -154,15 +157,15 @@ TBuffer::~TBuffer() /// If the size_needed is larger than the current size, the policy /// is to expand to double the current size or the size_needed which ever is largest. -void TBuffer::AutoExpand(Long64_t size_needed) +void TBuffer::AutoExpand(ULong64_t size_needed) { - if (size_needed > kMaxBufferSize) { - Fatal("AutoExpand","Request to expand a too large buffer: 0x%llx for a max of 0x%x.", size_needed, kMaxBufferSize); + if (size_needed > kMaxLargeBufferSize) { + Fatal("AutoExpand","Request to expand a too large buffer: 0x%llx for a max of 0x%llx.", size_needed, kMaxLargeBufferSize); } if (size_needed > fBufSize) { - Long64_t doubling = 2LLU * fBufSize; - if (doubling > kMaxBufferSize) - doubling = kMaxBufferSize; + ULong64_t doubling = 2LLU * fBufSize; + if (doubling > kMaxLargeBufferSize) + doubling = kMaxLargeBufferSize; if (size_needed > doubling) { Expand(size_needed); } else { @@ -183,7 +186,7 @@ void TBuffer::AutoExpand(Long64_t size_needed) /// is provided, a Fatal error will be issued if the Buffer attempts to /// expand. -void TBuffer::SetBuffer(void *buf, Long64_t newsiz, Bool_t adopt, ReAllocCharFun_t reallocfunc) +void TBuffer::SetBuffer(void *buf, ULong64_t newsiz, Bool_t adopt, ReAllocCharFun_t reallocfunc) { if (newsiz > kMaxBufferSize) Fatal("SetBuffer","Request to create a too large buffer: 0x%llx for a max of 0x%x.", newsiz, kMaxBufferSize); @@ -195,22 +198,22 @@ void TBuffer::SetBuffer(void *buf, Long64_t newsiz, Bool_t adopt, ReAllocCharFun else ResetBit(kIsOwner); + SetReAllocFunc( reallocfunc ); + fBuffer = (char *)buf; fBufCur = fBuffer; if (newsiz > 0) { if ( (fMode&kWrite)!=0 ) { - fBufSize = newsiz - kExtraSpace; + if (newsiz >= kExtraSpace) { + fBufSize = newsiz - kExtraSpace; + } else { + Expand( kMinimalSize ); + } } else { fBufSize = newsiz; } } fBufMax = fBuffer + fBufSize; - - SetReAllocFunc( reallocfunc ); - - if (buf && ( (fMode&kWrite)!=0 ) && fBufSize < 0) { - Expand( kMinimalSize ); - } } //////////////////////////////////////////////////////////////////////////////// @@ -221,22 +224,23 @@ void TBuffer::SetBuffer(void *buf, Long64_t newsiz, Bool_t adopt, ReAllocCharFun /// In order to avoid losing data, if the current length is greater than /// the requested size, we only shrink down to the current length. -void TBuffer::Expand(Long64_t newsize, Bool_t copy) +void TBuffer::Expand(ULong64_t newsize, Bool_t copy) { - Int_t l = Length(); - if ( (Long64_t(l) > newsize) && copy ) { + ULong64_t l = Length(); + if ( l > newsize && copy ) { newsize = l; } - const Int_t extraspace = (fMode&kWrite)!=0 ? kExtraSpace : 0; + const ULong64_t extraspace = (fMode&kWrite)!=0 ? kExtraSpace : 0; - if ( newsize > kMaxBufferSize - kExtraSpace) { - if (l < kMaxBufferSize) { - newsize = kMaxBufferSize - extraspace; + if ( newsize > kMaxLargeBufferSize - kExtraSpace) { + if (l < kMaxLargeBufferSize) { + newsize = kMaxLargeBufferSize - extraspace; } else { - Fatal("Expand","Requested size (%lld) is too large (max is %d).", newsize, kMaxBufferSize); + Fatal("Expand","Requested size (%llu) is too large (max is %llu).", newsize, kMaxLargeBufferSize); } } if ( (fMode&kWrite)!=0 ) { + // FIXME-LARGE: The size can overflow and the signature of fReAllocFunc is not large enough to handle the size of the buffer. fBuffer = fReAllocFunc(fBuffer, newsize+kExtraSpace, copy ? fBufSize+kExtraSpace : 0); } else { @@ -392,13 +396,13 @@ TVirtualArray *TBuffer::PopDataCache() /// Byte-swap N primitive-elements in the buffer. /// Bulk API relies on this function. -Bool_t TBuffer::ByteSwapBuffer(Long64_t n, EDataType type) +Bool_t TBuffer::ByteSwapBuffer(ULong64_t n, EDataType type) { char *input_buf = GetCurrent(); if ((type == EDataType::kShort_t) || (type == EDataType::kUShort_t)) { #ifdef R__BYTESWAP Short_t *buf __attribute__((aligned(8))) = reinterpret_cast(input_buf); - for (int idx=0; idx(buf + idx); // Makes a copy of the values; frombuf can't handle aliasing. char *tmp_ptr = reinterpret_cast(&tmp); frombuf(tmp_ptr, buf + idx); @@ -407,7 +411,7 @@ Bool_t TBuffer::ByteSwapBuffer(Long64_t n, EDataType type) } else if ((type == EDataType::kFloat_t) || (type == EDataType::kInt_t) || (type == EDataType::kUInt_t)) { #ifdef R__BYTESWAP Float_t *buf __attribute__((aligned(8))) = reinterpret_cast(input_buf); - for (int idx=0; idx(buf + idx); // Makes a copy of the values; frombuf can't handle aliasing. char *tmp_ptr = reinterpret_cast(&tmp); frombuf(tmp_ptr, buf + idx); @@ -416,7 +420,7 @@ Bool_t TBuffer::ByteSwapBuffer(Long64_t n, EDataType type) } else if ((type == EDataType::kDouble_t) || (type == EDataType::kLong64_t) || (type == EDataType::kULong64_t)) { #ifdef R__BYTESWAP Double_t *buf __attribute__((aligned(8))) = reinterpret_cast(input_buf); - for (int idx=0; idx(buf + idx); // Makes a copy of the values; frombuf can't handle aliasing. char *tmp_ptr = reinterpret_cast(&tmp); frombuf(tmp_ptr, buf + idx); diff --git a/core/cont/inc/TVirtualCollectionProxy.h b/core/cont/inc/TVirtualCollectionProxy.h index 12d6e412285d8..ec3d3e7a76cce 100644 --- a/core/cont/inc/TVirtualCollectionProxy.h +++ b/core/cont/inc/TVirtualCollectionProxy.h @@ -71,6 +71,8 @@ class TVirtualCollectionProxy { kCustomAlloc = BIT(5) ///< The collection has a custom allocator. }; + using size_type = ULong64_t; + /// RAII helper class that ensures that `PushProxy()` / `PopProxy()` are called when entering / leaving a C++ context class TPushPop { public: @@ -123,19 +125,19 @@ class TVirtualCollectionProxy { } /// Construct an array of `nElements` container objects and return the base address of the array - virtual void *NewArray(Int_t nElements) const { return !fClass.GetClass() ? nullptr : fClass->NewArray(nElements); } + virtual void *NewArray(Long64_t nElements) const { return !fClass.GetClass() ? nullptr : fClass->NewArray(nElements); } /// Construct an array of `nElements` container objects at the address given by `arena` - virtual void *NewArray(Int_t nElements, void *arena) const + virtual void *NewArray(Long64_t nElements, void *arena) const { return !fClass.GetClass() ? nullptr : fClass->NewArray(nElements, arena); } /// Construct an array of `nElements` container objects and return the base address of the array - virtual TClass::ObjectPtr NewObjectArray(Int_t nElements) const + virtual TClass::ObjectPtr NewObjectArray(Long64_t nElements) const { return !fClass.GetClass() ? TClass::ObjectPtr{} : fClass->NewObjectArray(nElements); } /// Construct an array of `nElements` container objects at the address given by `arena` - virtual TClass::ObjectPtr NewObjectArray(Int_t nElements, void *arena) const + virtual TClass::ObjectPtr NewObjectArray(Long64_t nElements, void *arena) const { return !fClass.GetClass() ? TClass::ObjectPtr{} : fClass->NewObjectArray(nElements, arena); } @@ -155,7 +157,7 @@ class TVirtualCollectionProxy { } /// Return the `sizeof()` of the collection object - virtual UInt_t Sizeof() const = 0; + virtual size_type Sizeof() const = 0; /// Set the address of the container being proxied and keep track of the previous one virtual void PushProxy(void *objectstart) = 0; @@ -174,19 +176,19 @@ class TVirtualCollectionProxy { virtual EDataType GetType() const = 0; /// Return the address of the value at index `idx` - virtual void *At(UInt_t idx) = 0; + virtual void *At(size_type idx) = 0; /// Clear the container virtual void Clear(const char *opt = "") = 0; /// Return the current number of elements in the container - virtual UInt_t Size() const = 0; + virtual size_type Size() const = 0; /// Allocates space for storing at least `n` elements. This function returns a pointer to the actual object on /// which insertions should take place. For associative collections, this function returns a pointer to a temporary /// buffer known as the staging area. If the insertion happened in a staging area (i.e. the returned pointer != /// proxied object), `Commit()` should be called on the value returned by this function. - virtual void* Allocate(UInt_t n, Bool_t forceDelete) = 0; + virtual void* Allocate(size_type n, Bool_t forceDelete) = 0; /// Commits pending elements in a staging area (see Allocate() for more information). virtual void Commit(void*) = 0; @@ -196,7 +198,7 @@ class TVirtualCollectionProxy { virtual void Insert(const void *data, void *container, size_t size) = 0; /// Return the address of the value at index `idx` - char *operator[](UInt_t idx) const { return (char *)(const_cast(this))->At(idx); } + char *operator[](size_type idx) const { return (char *)(const_cast(this))->At(idx); } // Functions related to member-wise actions virtual TStreamerInfoActions::TActionSequence *GetConversionReadMemberWiseActions(TClass *oldClass, Int_t version) = 0; diff --git a/core/meta/inc/TClass.h b/core/meta/inc/TClass.h index fec709736ea2b..acb46de66edf5 100644 --- a/core/meta/inc/TClass.h +++ b/core/meta/inc/TClass.h @@ -544,12 +544,12 @@ friend class TStreamerInfo; void Move(void *arenaFrom, void *arenaTo) const; void *New(ENewType defConstructor = kClassNew, Bool_t quiet = kFALSE) const; void *New(void *arena, ENewType defConstructor = kClassNew) const; - void *NewArray(Long_t nElements, ENewType defConstructor = kClassNew) const; - void *NewArray(Long_t nElements, void *arena, ENewType defConstructor = kClassNew) const; + void *NewArray(Long64_t nElements, ENewType defConstructor = kClassNew) const; + void *NewArray(Long64_t nElements, void *arena, ENewType defConstructor = kClassNew) const; ObjectPtr NewObject(ENewType defConstructor = kClassNew, Bool_t quiet = kFALSE) const; ObjectPtr NewObject(void *arena, ENewType defConstructor = kClassNew) const; - ObjectPtr NewObjectArray(Long_t nElements, ENewType defConstructor = kClassNew) const; - ObjectPtr NewObjectArray(Long_t nElements, void *arena, ENewType defConstructor = kClassNew) const; + ObjectPtr NewObjectArray(Long64_t nElements, ENewType defConstructor = kClassNew) const; + ObjectPtr NewObjectArray(Long64_t nElements, void *arena, ENewType defConstructor = kClassNew) const; virtual void PostLoadCheck(); Long_t Property() const override; Int_t ReadBuffer(TBuffer &b, void *pointer, Int_t version, UInt_t start, UInt_t count); diff --git a/core/meta/src/TClass.cxx b/core/meta/src/TClass.cxx index 37c2b3824ae2c..e16b856825c49 100644 --- a/core/meta/src/TClass.cxx +++ b/core/meta/src/TClass.cxx @@ -5260,8 +5260,9 @@ TClass::ObjectPtr TClass::NewObject(void *arena, ENewType defConstructor) const /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -void *TClass::NewArray(Long_t nElements, ENewType defConstructor) const +void *TClass::NewArray(Long64_t nElements, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); auto obj = NewObjectArray(nElements, defConstructor); if (obj.GetPtr() && obj.GetAllocator()) { // Register the object for special handling in the destructor. @@ -5276,8 +5277,9 @@ void *TClass::NewArray(Long_t nElements, ENewType defConstructor) const /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -TClass::ObjectPtr TClass::NewObjectArray(Long_t nElements, ENewType defConstructor) const +TClass::ObjectPtr TClass::NewObjectArray(Long64_t nElements, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); ObjectPtr p; if (fNewArray) { @@ -5362,8 +5364,9 @@ TClass::ObjectPtr TClass::NewObjectArray(Long_t nElements, ENewType defConstruct /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -void *TClass::NewArray(Long_t nElements, void *arena, ENewType defConstructor) const +void *TClass::NewArray(Long64_t nElements, void *arena, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); auto obj = NewObjectArray(nElements, arena, defConstructor); if (obj.GetPtr() && obj.GetAllocator()) { // Register the object for special handling in the destructor. @@ -5377,8 +5380,9 @@ void *TClass::NewArray(Long_t nElements, void *arena, ENewType defConstructor) c /// The class must have a default constructor. For meaning of /// defConstructor, see TClass::IsCallingNew(). -TClass::ObjectPtr TClass::NewObjectArray(Long_t nElements, void *arena, ENewType defConstructor) const +TClass::ObjectPtr TClass::NewObjectArray(Long64_t nElements, void *arena, ENewType defConstructor) const { + R__ASSERT(nElements >= 0 && "Possible corruption of the heap: negative number of elements requested in TClass::NewObjectArray"); ObjectPtr p; if (fNewArray) { diff --git a/io/io/inc/TBufferFile.h b/io/io/inc/TBufferFile.h index fe73ecf98e446..348b6d405ee7a 100644 --- a/io/io/inc/TBufferFile.h +++ b/io/io/inc/TBufferFile.h @@ -88,7 +88,7 @@ class TBufferFile : public TBufferIO { UInt_t ReserveByteCount(const TClass *cl) override; void WriteObjectClass(const void *actualObjStart, const TClass *actualClass, Bool_t cacheReuse) override; - bool ShouldNotReadCollection(Int_t lengthInBytes, Int_t nElements=1) const; + bool ShouldNotReadCollection(Long64_t lengthInBytes, Long64_t nElements=1) const; public: enum { kStreamedMemberWise = BIT(14) }; //added to version number to know if a collection has been stored member-wise @@ -193,7 +193,7 @@ class TBufferFile : public TBufferIO { void ReadFastArray(ULong_t *l, Int_t n) override; void ReadFastArray(Long64_t *l, Int_t n) override; void ReadFastArray(ULong64_t *l, Int_t n) override; - void ReadFastArray(Float_t *f, Int_t n) override; + void ReadFastArray(Float_t *f, Long64_t n) override; void ReadFastArray(Double_t *d, Int_t n) override; void ReadFastArrayFloat16(Float_t *f, Int_t n, TStreamerElement *ele = nullptr) override; void ReadFastArrayDouble32(Double_t *d, Int_t n, TStreamerElement *ele = nullptr) override; @@ -300,7 +300,7 @@ class TBufferFile : public TBufferIO { //---------------------- TBufferFile inlines --------------------------------------- //______________________________________________________________________________ -inline bool TBufferFile::ShouldNotReadCollection(Int_t lengthInBytes, Int_t nElements) const +inline bool TBufferFile::ShouldNotReadCollection(Long64_t lengthInBytes, Long64_t nElements) const { // Three cases here in which we should not read the collection: // 1. The collection has zero or a negative number of elements or diff --git a/io/io/inc/TBufferJSON.h b/io/io/inc/TBufferJSON.h index a9b6fbaf9da0a..4fd36806315ec 100644 --- a/io/io/inc/TBufferJSON.h +++ b/io/io/inc/TBufferJSON.h @@ -160,7 +160,7 @@ class TBufferJSON final : public TBufferText { void ReadFastArray(ULong_t *l, Int_t n) final; void ReadFastArray(Long64_t *l, Int_t n) final; void ReadFastArray(ULong64_t *l, Int_t n) final; - void ReadFastArray(Float_t *f, Int_t n) final; + void ReadFastArray(Float_t *f, Long64_t n) final; void ReadFastArray(Double_t *d, Int_t n) final; void ReadFastArray(void *start, const TClass *cl, Int_t n = 1, TMemberStreamer *s = nullptr, const TClass *onFileClass = nullptr) final; diff --git a/io/io/inc/TEmulatedCollectionProxy.h b/io/io/inc/TEmulatedCollectionProxy.h index 1c4e7863fd5e2..9ddaa4bcd9a27 100644 --- a/io/io/inc/TEmulatedCollectionProxy.h +++ b/io/io/inc/TEmulatedCollectionProxy.h @@ -157,29 +157,29 @@ class TEmulatedCollectionProxy : public TGenCollectionProxy { TClass::ObjectPtr NewObject(void *memory) const override { return {New(memory), nullptr}; } // Virtual array constructor - void *NewArray(Int_t nElements) const override + void *NewArray(Long64_t nElements) const override { void *arr = ::operator new(nElements * sizeof(Cont_t)); - for (Int_t i = 0; i < nElements; ++i) + for (Long64_t i = 0; i < nElements; ++i) WithCont(static_cast(arr) + i * sizeof(Cont_t), [](auto *c, std::size_t) { new (c) std::decay_t(); }); return arr; } // Virtual in-place array constructor - void *NewArray(Int_t nElements, void *memory) const override + void *NewArray(Long64_t nElements, void *memory) const override { - for (Int_t i = 0; i < nElements; ++i) + for (Long64_t i = 0; i < nElements; ++i) WithCont(static_cast(memory) + i * sizeof(Cont_t), [](auto *c, std::size_t) { new (c) std::decay_t(); }); return memory; } // Virtual array constructor - TClass::ObjectPtr NewObjectArray(Int_t nElements) const override { return {NewArray(nElements), nullptr}; } + TClass::ObjectPtr NewObjectArray(Long64_t nElements) const override { return {NewArray(nElements), nullptr}; } // Virtual in-place array constructor - TClass::ObjectPtr NewObjectArray(Int_t nElements, void *memory) const override + TClass::ObjectPtr NewObjectArray(Long64_t nElements, void *memory) const override { return {NewArray(nElements, memory), nullptr}; } @@ -191,22 +191,22 @@ class TEmulatedCollectionProxy : public TGenCollectionProxy { void DeleteArray(void* p, Bool_t dtorOnly = kFALSE) const override; // TVirtualCollectionProxy overload: Return the sizeof the collection object. - UInt_t Sizeof() const override { return sizeof(Cont_t); } + size_type Sizeof() const override { return sizeof(Cont_t); } // Return the address of the value at index 'idx' - void *At(UInt_t idx) override; + void *At(size_type idx) override; // Clear the container void Clear(const char *opt = "") override; // Resize the container - void Resize(UInt_t n, Bool_t force_delete) override; + void Resize(size_type n, Bool_t force_delete) override; // Return the current size of the container - UInt_t Size() const override; + size_type Size() const override; // Block allocation of containees - void* Allocate(UInt_t n, Bool_t forceDelete) override; + void* Allocate(size_type n, Bool_t forceDelete) override; // Block commit of containees void Commit(void* env) override; diff --git a/io/io/inc/TEmulatedMapProxy.h b/io/io/inc/TEmulatedMapProxy.h index 48e990d85e86d..232a7b1bbd27f 100644 --- a/io/io/inc/TEmulatedMapProxy.h +++ b/io/io/inc/TEmulatedMapProxy.h @@ -38,10 +38,10 @@ class TEmulatedMapProxy : public TEmulatedCollectionProxy { ~TEmulatedMapProxy() override; // Return the address of the value at index 'idx' - void *At(UInt_t idx) override; + void *At(size_type idx) override; // Return the current size of the container - UInt_t Size() const override; + size_type Size() const override; // Read portion of the streamer void ReadBuffer(TBuffer &buff, void *pObj) override; diff --git a/io/io/inc/TGenCollectionProxy.h b/io/io/inc/TGenCollectionProxy.h index 832d53e88de7f..49993c858d77f 100644 --- a/io/io/inc/TGenCollectionProxy.h +++ b/io/io/inc/TGenCollectionProxy.h @@ -376,7 +376,7 @@ class TGenCollectionProxy ULong_t GetIncrement() const override; // Return the sizeof the collection object. - UInt_t Sizeof() const override; + size_type Sizeof() const override; // Push new proxy environment. void PushProxy(void *objstart) override; @@ -394,19 +394,19 @@ class TGenCollectionProxy EDataType GetType() const override; // Return the address of the value at index 'idx'. - void *At(UInt_t idx) override; + void *At(size_type idx) override; // Clear the container. void Clear(const char *opt = "") override; // Resize the container. - virtual void Resize(UInt_t n, Bool_t force_delete); + virtual void Resize(size_type n, Bool_t force_delete); // Return the current size of the container. - UInt_t Size() const override; + size_type Size() const override; // Block allocation of containees. - void* Allocate(UInt_t n, Bool_t forceDelete) override; + void* Allocate(size_type n, Bool_t forceDelete) override; // Insert data into the container where data is a C-style array of the actual type contained in the collection // of the given size. For associative container (map, etc.), the data type is the pair. diff --git a/io/io/inc/TStreamerInfo.h b/io/io/inc/TStreamerInfo.h index ffe008fd308b0..07850cbe5a795 100644 --- a/io/io/inc/TStreamerInfo.h +++ b/io/io/inc/TStreamerInfo.h @@ -253,7 +253,7 @@ class TStreamerInfo : public TVirtualStreamerInfo { Int_t WriteBufferAux (TBuffer &b, const T &arr, TCompInfo *const*const compinfo, Int_t first, Int_t last, Int_t narr,Int_t eoffset,Int_t mode); //WARNING this class version must be the same as TVirtualStreamerInfo - ClassDefOverride(TStreamerInfo, 10) // Streamer information for one class version + ClassDefOverride(TStreamerInfo, 11) // Streamer information for one class version }; diff --git a/io/io/src/TBufferFile.cxx b/io/io/src/TBufferFile.cxx index dd71715942c1d..9e5c455308afa 100644 --- a/io/io/src/TBufferFile.cxx +++ b/io/io/src/TBufferFile.cxx @@ -361,7 +361,7 @@ void TBufferFile::SetByteCount(ULong64_t cntpos, Bool_t packInVersion) assert( (cntpos == kOverflowPosition || (cntpos < kOverflowPosition && (sizeof(UInt_t) + cntpos) < static_cast(fBufCur - fBuffer))) && (fBufCur >= fBuffer) - && static_cast(fBufCur - fBuffer) <= std::numeric_limits::max() + && static_cast(fBufCur - fBuffer) <= std::numeric_limits::max() && "Byte count position is after the end of the buffer"); // We can either make this unconditional or we could split the routine @@ -440,7 +440,10 @@ Long64_t TBufferFile::CheckByteCount(ULong64_t startpos, ULong64_t bcnt, const T // The position is above 4GB but was cached using a 32 bit variable. startpos = fByteCountStack.back().locator; // See below - R__ASSERT((fByteCountStack.back().cl == nullptr || clss == fByteCountStack.back().cl) + R__ASSERT((fByteCountStack.back().cl == nullptr || + (clss == nullptr && (classname == nullptr || classname[0] == '\0')) || + (clss && clss == fByteCountStack.back().cl) || + (classname && strcmp(classname, fByteCountStack.back().cl->GetName()) == 0)) && "Class on the byte count position stack does not match the passed class"); } else { // This assert allows to reject cases that used to be valid (missing or @@ -1527,9 +1530,9 @@ void TBufferFile::ReadFastArray(Long64_t *ll, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// Read array of n floats from the I/O buffer. -void TBufferFile::ReadFastArray(Float_t *f, Int_t n) +void TBufferFile::ReadFastArray(Float_t *f, Long64_t n) { - Int_t l = sizeof(Float_t)*n; + auto l = sizeof(Float_t)*n; if (ShouldNotReadCollection(l, n)) return; #ifdef R__BYTESWAP @@ -1537,7 +1540,7 @@ void TBufferFile::ReadFastArray(Float_t *f, Int_t n) bswapcpy32(f, fBufCur, n); fBufCur += sizeof(Float_t)*n; # else - for (int i = 0; i < n; i++) + for (Long64_t i = 0; i < n; i++) frombuf(fBufCur, &f[i]); # endif #else @@ -2330,15 +2333,15 @@ void TBufferFile::WriteFastArray(const Float_t *f, Long64_t n) { if (n == 0) return; - constexpr Int_t dataWidth = static_cast(sizeof(Float_t)); - const Int_t maxElements = (std::numeric_limits::max() - Length())/dataWidth; + constexpr Long64_t dataWidth = static_cast(sizeof(Float_t)); + const Long64_t maxElements = (std::numeric_limits::max() - Length())/dataWidth; if (n < 0 || n > maxElements) { - Fatal("WriteFastArray", "Not enough space left in the buffer (1GB limit). %lld elements is greater than the max left of %d", n, maxElements); + Fatal("WriteFastArray", "Not enough space left in the buffer. %lld elements is greater than the max left of %lld", n, maxElements); return; // In case the user re-routes the error handler to not die when Fatal is called } - Int_t l = sizeof(Float_t)*n; + auto l = sizeof(Float_t)*n; if (fBufCur + l > fBufMax) AutoExpand(fBufSize+l); #ifdef R__BYTESWAP @@ -2346,7 +2349,7 @@ void TBufferFile::WriteFastArray(const Float_t *f, Long64_t n) bswapcpy32(fBufCur, f, n); fBufCur += l; # else - for (int i = 0; i < n; i++) + for (Long64_t i = 0; i < n; i++) tobuf(fBufCur, f[i]); # endif #else @@ -2658,6 +2661,8 @@ void TBufferFile::SkipObjectAny(Long64_t start, UInt_t count) /// real beginning of the object in memory. You will need to use a /// dynamic_cast later if you need to retrieve it. + + void *TBufferFile::ReadObjectAny(const TClass *clCast) { R__ASSERT(IsReading()); @@ -2665,6 +2670,20 @@ void *TBufferFile::ReadObjectAny(const TClass *clCast) // make sure fMap is initialized InitMap(); + struct CaptureAndCheck { + using ByteCountStack_t = TBufferFile::ByteCountStack_t; + size_t fCurrent; + ByteCountStack_t *fByteCountStack; + + CaptureAndCheck(ByteCountStack_t *stack) : fCurrent(stack->size()), fByteCountStack(stack) {} + ~CaptureAndCheck() { + if (fCurrent != fByteCountStack->size()) { + ::Fatal("CaptureAndCheck", "Byte count stack was not properly cleaned up. Current size is %zu but should be %zu", fByteCountStack->size(), fCurrent); + } + } + }; + CaptureAndCheck checker(&(this->fByteCountStack)); + // before reading object save start position ULong64_t startpos = static_cast(fBufCur-fBuffer); ULong64_t cntpos = startpos <= kMaxCountPosition ? startpos : kOverflowPosition; @@ -3015,6 +3034,14 @@ TClass *TBufferFile::ReadClass(const TClass *clReq, ULong64_t *objTag) // return bytecount in objTag if (objTag) { *objTag = (bcnt & ~kByteCountMask); + if (*objTag == 0) { + // The byte count was stored but is zero, this means the data + // did not fit and thus we stored it in 'fByteCounts' instead. + // Mark this case by setting startpos to kOverflowCount. + *objTag = kOverflowCount; + // We do not have access to the caller's "cntpos" and can not + // update to kOverflowPosition (unlike the same code in ReadVersion). + } if (cl) fByteCountStack.back().alt = cl; } @@ -3159,7 +3186,8 @@ Version_t TBufferFile::ReadVersion(UInt_t *startpos, UInt_t *bcnt, const TClass // before reading object save start position auto full_startpos = fBufCur - fBuffer; *startpos = full_startpos <= kMaxCountPosition ? UInt_t(full_startpos) : kOverflowPosition; - fByteCountStack.push_back({(size_t)full_startpos, cl, nullptr}); + if (bcnt) + fByteCountStack.push_back({(size_t)full_startpos, cl, nullptr}); } // read byte count (older files don't have byte count) @@ -3196,7 +3224,8 @@ Version_t TBufferFile::ReadVersion(UInt_t *startpos, UInt_t *bcnt, const TClass // did not fit and thus we stored it in 'fByteCounts' instead. // Mark this case by setting startpos to kOverflowCount. *bcnt = kOverflowCount; - *startpos = kOverflowPosition; + if (startpos) + *startpos = kOverflowPosition; } } } @@ -3302,7 +3331,8 @@ Version_t TBufferFile::ReadVersionNoCheckSum(UInt_t *startpos, UInt_t *bcnt) auto full_startpos = fBufCur - fBuffer; *startpos = full_startpos < kMaxCountPosition ? UInt_t(full_startpos) : kOverflowPosition; // TODO: Extend ReadVersionNoCheckSum to take the class pointer. - fByteCountStack.push_back({(size_t)full_startpos, nullptr, nullptr}); + if (bcnt) + fByteCountStack.push_back({(size_t)full_startpos, nullptr, nullptr}); } // read byte count (older files don't have byte count) @@ -3338,7 +3368,8 @@ Version_t TBufferFile::ReadVersionNoCheckSum(UInt_t *startpos, UInt_t *bcnt) // did not fit and thus we stored it in 'fByteCounts' instead. // Mark this case by setting startpos to kOverflowCount. *bcnt = kOverflowCount; - *startpos = kOverflowPosition; + if (startpos) + *startpos = kOverflowPosition; } } } @@ -3754,7 +3785,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio sinfo = (TStreamerInfo*)cl->GetConversionStreamerInfo( onFileClass, version ); if( !sinfo ) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length() ); CheckByteCount(start, count, onFileClass); return 0; @@ -3770,7 +3801,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio auto infos = cl->GetStreamerInfos(); auto ninfos = infos->GetSize(); if (version < -1 || version >= ninfos) { - Error("ReadClassBuffer", "class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + Error("ReadClassBuffer", "class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length() ); CheckByteCount(start, count, cl); return 0; @@ -3801,7 +3832,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio CheckByteCount(start, count, cl); return 0; } else { - Error("ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + Error("ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length() ); CheckByteCount(start, count, cl); return 0; @@ -3847,6 +3878,7 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass else version = ReadVersion(&R__s, &R__c, cl); + size_t current = fByteCountStack.size(); Bool_t v2file = kFALSE; TFile *file = (TFile*)GetParent(); if (file && file->GetVersion() < 30000) { @@ -3863,8 +3895,9 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass sinfo = (TStreamerInfo*)cl->GetConversionStreamerInfo( onFileClass, version ); if( !sinfo ) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length() ); + --current; CheckByteCount(R__s, R__c, onFileClass); return 0; } @@ -3887,8 +3920,9 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass Int_t infocapacity = infos->Capacity(); if (infocapacity) { if (version < -1 || version >= infocapacity) { - Error("ReadClassBuffer","class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + Error("ReadClassBuffer","class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length()); + --current; CheckByteCount(R__s, R__c, cl); return 0; } @@ -3947,11 +3981,13 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass // When the object was written the class was version zero, so // there is no StreamerInfo to be found. // Check that the buffer position corresponds to the byte count. + --current; CheckByteCount(R__s, R__c, cl); return 0; } else { - Error( "ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + Error( "ReadClassBuffer", "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length() ); + --current; CheckByteCount(R__s, R__c, cl); return 0; } @@ -3963,6 +3999,9 @@ Int_t TBufferFile::ReadClassBuffer(const TClass *cl, void *pointer, const TClass ApplySequence(*(sinfo->GetReadObjectWiseActions()), (char*)pointer ); if (sinfo->TStreamerInfo::IsRecovered()) R__c=0; // 'TStreamerInfo::' avoids going via a virtual function. + if (current != fByteCountStack.size()) + Fatal("ReadClassBuffer", "We are out of sync at level %zu vs %zu for %s", current, fByteCountStack.size(), cl->GetName()); + --current; // Check that the buffer position corresponds to the byte count. CheckByteCount(R__s, R__c, cl); @@ -4031,8 +4070,11 @@ Int_t TBufferFile::ApplySequence(const TStreamerInfoActions::TActionSequence &se for(TStreamerInfoActions::ActionContainer_t::const_iterator iter = sequence.fActions.begin(); iter != end; ++iter) { + auto current = fByteCountStack.size(); (*iter).PrintDebug(*this,obj); (*iter)(*this,obj); + if (current != fByteCountStack.size()) + Fatal("ApplySequence", "We are out of sync starting at level %zu vs %zu", current, fByteCountStack.size()); } } else { @@ -4041,7 +4083,10 @@ Int_t TBufferFile::ApplySequence(const TStreamerInfoActions::TActionSequence &se for(TStreamerInfoActions::ActionContainer_t::const_iterator iter = sequence.fActions.begin(); iter != end; ++iter) { + auto current = fByteCountStack.size(); (*iter)(*this,obj); + if (current != fByteCountStack.size()) + Fatal("ApplySequence", "ByteCountStack is out of sync starting at level %zu vs %zu", current, fByteCountStack.size()); } } diff --git a/io/io/src/TBufferIO.cxx b/io/io/src/TBufferIO.cxx index 9589eee8cad1c..bafb7072d62f6 100644 --- a/io/io/src/TBufferIO.cxx +++ b/io/io/src/TBufferIO.cxx @@ -161,7 +161,9 @@ void TBufferIO::InitMap() void TBufferIO::MapObject(const TObject *obj, ULong64_t offset) { - R__ASSERT(offset <= kMaxUInt); + // This is too high, the real limit is kMaxLongRange which is + // currently declared inside TBufferFile.cxx. + R__ASSERT(offset <= kMaxULong64); if (IsWriting()) { if (!fMap) InitMap(); @@ -195,7 +197,7 @@ void TBufferIO::MapObject(const TObject *obj, ULong64_t offset) void TBufferIO::MapObject(const void *obj, const TClass *cl, ULong64_t offset) { - R__ASSERT(offset <= kMaxUInt); + R__ASSERT(offset <= kMaxLong64); if (IsWriting()) { if (!fMap) InitMap(); diff --git a/io/io/src/TBufferJSON.cxx b/io/io/src/TBufferJSON.cxx index 2ffbdb5864b2d..432400aacb2f7 100644 --- a/io/io/src/TBufferJSON.cxx +++ b/io/io/src/TBufferJSON.cxx @@ -3014,7 +3014,7 @@ void TBufferJSON::ReadFastArray(ULong64_t *l, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// read array of Float_t from buffer -void TBufferJSON::ReadFastArray(Float_t *f, Int_t n) +void TBufferJSON::ReadFastArray(Float_t *f, Long64_t n) { JsonReadFastArray(f, n); } diff --git a/io/io/src/TBufferText.cxx b/io/io/src/TBufferText.cxx index d1961bc841320..37a32544b56db 100644 --- a/io/io/src/TBufferText.cxx +++ b/io/io/src/TBufferText.cxx @@ -249,7 +249,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio sinfo = (TStreamerInfo *)cl->GetConversionStreamerInfo(onFileClass, version); if (!sinfo) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length()); CheckByteCount(start, count, onFileClass); return 0; @@ -265,7 +265,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio auto infos = cl->GetStreamerInfos(); auto ninfos = infos->GetSize(); if (version < -1 || version >= ninfos) { - Error("ReadBuffer1", "class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + Error("ReadBuffer1", "class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length()); CheckByteCount(start, count, cl); return 0; @@ -294,7 +294,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, Int_t versio return 0; } else { Error("ReadClassBuffer", - "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length()); CheckByteCount(start, count, cl); return 0; @@ -352,7 +352,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, const TClass sinfo = (TStreamerInfo *)cl->GetConversionStreamerInfo(onFileClass, version); if (!sinfo) { Error("ReadClassBuffer", - "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %d", + "Could not find the right streamer info to convert %s version %d into a %s, object skipped at offset %lu", onFileClass->GetName(), version, cl->GetName(), Length()); CheckByteCount(R__s, R__c, onFileClass); return 0; @@ -377,7 +377,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, const TClass if (infocapacity) { if (version < -1 || version >= infocapacity) { Error("ReadClassBuffer", - "class: %s, attempting to access a wrong version: %d, object skipped at offset %d", + "class: %s, attempting to access a wrong version: %d, object skipped at offset %lu", cl->GetName(), version, Length()); CheckByteCount(R__s, R__c, cl); return 0; @@ -440,7 +440,7 @@ Int_t TBufferText::ReadClassBuffer(const TClass *cl, void *pointer, const TClass return 0; } else { Error("ReadClassBuffer", - "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %d", + "Could not find the StreamerInfo for version %d of the class %s, object skipped at offset %lu", version, cl->GetName(), Length()); CheckByteCount(R__s, R__c, cl); return 0; diff --git a/io/io/src/TContainerConverters.cxx b/io/io/src/TContainerConverters.cxx index 6958ad082e2fc..b74fe8b30f4b6 100644 --- a/io/io/src/TContainerConverters.cxx +++ b/io/io/src/TContainerConverters.cxx @@ -72,9 +72,8 @@ void TConvertClonesArrayToProxy::operator()(TBuffer &b, void *pmember, Int_t siz Int_t nobjects, dummy; char nch; TString s; - char classv[256]; - void *env; - UInt_t start, bytecount; + char classv[256]; + void *env; R__ASSERT(b.IsReading()); @@ -95,6 +94,9 @@ void TConvertClonesArrayToProxy::operator()(TBuffer &b, void *pmember, Int_t siz if (size==0) size=1; for(Int_t k=0; kCommit(env); b.CheckByteCount(start, bytecount,TClonesArray::Class()); + if (needAlloc) { + // Balance/check for the ReadClass byte count. + b.CheckByteCount(cntpos, tag, TClonesArray::Class()); + } } } diff --git a/io/io/src/TEmulatedCollectionProxy.cxx b/io/io/src/TEmulatedCollectionProxy.cxx index b951b314b45f6..2ba859ce385b6 100644 --- a/io/io/src/TEmulatedCollectionProxy.cxx +++ b/io/io/src/TEmulatedCollectionProxy.cxx @@ -247,7 +247,7 @@ Bool_t TEmulatedCollectionProxy::IsValid() const return (0 != fCreateEnv.call); } -UInt_t TEmulatedCollectionProxy::Size() const +TVirtualCollectionProxy::size_type TEmulatedCollectionProxy::Size() const { // Return the current size of the container @@ -452,7 +452,7 @@ void TEmulatedCollectionProxy::Expand(UInt_t nCurr, UInt_t left) } } -void TEmulatedCollectionProxy::Resize(UInt_t left, Bool_t force) +void TEmulatedCollectionProxy::Resize(TVirtualCollectionProxy::size_type left, Bool_t force) { // Resize the container @@ -473,7 +473,7 @@ void TEmulatedCollectionProxy::Resize(UInt_t left, Bool_t force) Fatal("TEmulatedCollectionProxy","Resize> Logic error - no proxy object set."); } -void* TEmulatedCollectionProxy::At(UInt_t idx) +void* TEmulatedCollectionProxy::At(TVirtualCollectionProxy::size_type idx) { // Return the address of the value at index 'idx' if ( fEnv && fEnv->fObject ) { @@ -488,7 +488,7 @@ void* TEmulatedCollectionProxy::At(UInt_t idx) return 0; } -void* TEmulatedCollectionProxy::Allocate(UInt_t n, Bool_t forceDelete) +void* TEmulatedCollectionProxy::Allocate(TVirtualCollectionProxy::size_type n, Bool_t forceDelete) { // Allocate the necessary space. diff --git a/io/io/src/TEmulatedMapProxy.cxx b/io/io/src/TEmulatedMapProxy.cxx index f244a8aba6f05..53ea1b80a9f2f 100644 --- a/io/io/src/TEmulatedMapProxy.cxx +++ b/io/io/src/TEmulatedMapProxy.cxx @@ -66,7 +66,7 @@ TVirtualCollectionProxy* TEmulatedMapProxy::Generate() const return new TEmulatedMapProxy(*this); } -void* TEmulatedMapProxy::At(UInt_t idx) +void* TEmulatedMapProxy::At(TVirtualCollectionProxy::size_type idx) { // Return the address of the value at index 'idx'. if ( fEnv && fEnv->fObject ) { @@ -77,7 +77,7 @@ void* TEmulatedMapProxy::At(UInt_t idx) return 0; } -UInt_t TEmulatedMapProxy::Size() const +TVirtualCollectionProxy::size_type TEmulatedMapProxy::Size() const { // Return the current size of the container. if ( fEnv && fEnv->fObject ) { diff --git a/io/io/src/TGenCollectionProxy.cxx b/io/io/src/TGenCollectionProxy.cxx index 9b240e02ad28d..b812642a96f9c 100644 --- a/io/io/src/TGenCollectionProxy.cxx +++ b/io/io/src/TGenCollectionProxy.cxx @@ -47,7 +47,7 @@ class TGenVectorProxy : public TGenCollectionProxy { { } // Return the address of the value at index 'idx' - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { if ( fEnv && fEnv->fObject ) { fEnv->fIdx = idx; @@ -97,7 +97,7 @@ class TGenVectorBoolProxy : public TGenCollectionProxy { { // Standard Destructor. } - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { // Return the address of the value at index 'idx' @@ -143,7 +143,7 @@ class TGenBitsetProxy : public TGenCollectionProxy { { // Standard Destructor. } - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { // Return the address of the value at index 'idx' @@ -198,7 +198,7 @@ class TGenListProxy : public TGenVectorProxy { { } // Return the address of the value at index 'idx' - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { if ( fEnv && fEnv->fObject ) { switch( idx ) { @@ -240,7 +240,7 @@ class TGenSetProxy : public TGenVectorProxy { { } // Return the address of the value at index 'idx' - void* At(UInt_t idx) override + void* At(TVirtualCollectionProxy::size_type idx) override { if ( fEnv && fEnv->fObject ) { if ( fEnv->fUseTemp ) { @@ -1015,7 +1015,7 @@ ULong_t TGenCollectionProxy::GetIncrement() const { //////////////////////////////////////////////////////////////////////////////// /// Return the sizeof the collection object. -UInt_t TGenCollectionProxy::Sizeof() const +TVirtualCollectionProxy::size_type TGenCollectionProxy::Sizeof() const { return fClass->Size(); } @@ -1065,7 +1065,7 @@ EDataType TGenCollectionProxy::GetType() const //////////////////////////////////////////////////////////////////////////////// /// Return the address of the value at index 'idx' -void* TGenCollectionProxy::At(UInt_t idx) +void* TGenCollectionProxy::At(TVirtualCollectionProxy::size_type idx) { if ( fEnv && fEnv->fObject ) { switch (fSTL_type) { @@ -1154,7 +1154,7 @@ void TGenCollectionProxy::Clear(const char* opt) //////////////////////////////////////////////////////////////////////////////// /// Return the current size of the container -UInt_t TGenCollectionProxy::Size() const +TVirtualCollectionProxy::size_type TGenCollectionProxy::Size() const { if ( fEnv && fEnv->fObject ) { if (fEnv->fUseTemp) { @@ -1170,7 +1170,7 @@ UInt_t TGenCollectionProxy::Size() const //////////////////////////////////////////////////////////////////////////////// /// Resize the container -void TGenCollectionProxy::Resize(UInt_t n, Bool_t force) +void TGenCollectionProxy::Resize(TVirtualCollectionProxy::size_type n, Bool_t force) { if ( fEnv && fEnv->fObject ) { if ( force && fPointers ) { @@ -1193,7 +1193,7 @@ void TGenCollectionProxy::Resize(UInt_t n, Bool_t force) /// For associative collection, this returns a TStaging object that /// need to be deleted manually __or__ returned by calling Commit(TStaging*) -void* TGenCollectionProxy::Allocate(UInt_t n, Bool_t /* forceDelete */ ) +void* TGenCollectionProxy::Allocate(TVirtualCollectionProxy::size_type n, Bool_t /* forceDelete */ ) { if ( fEnv && fEnv->fObject ) { switch ( fSTL_type ) { diff --git a/io/io/src/TStreamerInfoActions.cxx b/io/io/src/TStreamerInfoActions.cxx index bebd784eacd7d..72c3723e99dd6 100644 --- a/io/io/src/TStreamerInfoActions.cxx +++ b/io/io/src/TStreamerInfoActions.cxx @@ -19,6 +19,7 @@ #include "TVirtualArray.h" #include "TBufferFile.h" #include "TBufferText.h" +#include "TStreamerInfoCollectionUtils.h" #include "TMemberStreamer.h" #include "TClassEdit.h" #include "TVirtualCollectionIterators.h" @@ -126,7 +127,7 @@ namespace TStreamerInfoActions aElement->GetSequenceType(sequenceType); printf("StreamerInfoAction, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, offset=%d (%s)\n", + " %s, bufpos=%lu, arr=%p, offset=%d (%s)\n", info->GetClass()->GetName(), aElement->GetName(), fElemId, fCompInfo->fType, aElement->ClassName(), buf.Length(), addr, fOffset, sequenceType.Data()); } @@ -1274,7 +1275,7 @@ namespace TStreamerInfoActions { if (gDebug > 1) { TStreamerInfo *info = (TStreamerInfo*)fInfo; - printf("StreamerInfoAction, class:%s, %sDataCache, bufpos=%d, arr=%p, offset=%d, onfileObject=%p\n", + printf("StreamerInfoAction, class:%s, %sDataCache, bufpos=%lu, arr=%p, offset=%d, onfileObject=%p\n", info->GetClass()->GetName(), fOnfileObject ? "Push" : "Pop", buffer.Length(), object, fOffset, fOnfileObject); } @@ -1350,7 +1351,7 @@ namespace TStreamerInfoActions TStreamerInfo *info = (TStreamerInfo*)fInfo; TStreamerElement *aElement = fCompInfo->fElem; fprintf(stdout,"StreamerInfoAction, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, Redirect=%p\n", + " %s, bufpos=%lu, arr=%p, eoffset=%d, Redirect=%p\n", info->GetClass()->GetName(),aElement->GetName(),fElemId,fCompInfo->fType, aElement->ClassName(),b.Length(),addr, 0,b.PeekDataCache() ? b.PeekDataCache()->GetObjectAt(0) : 0); } @@ -2234,21 +2235,20 @@ namespace TStreamerInfoActions TConfigSTL *config = (TConfigSTL*)conf; UInt_t start, count; - /* Version_t vers = */ buf.ReadVersion(&start, &count, config->fOldClass); + Version_t vers = buf.ReadVersion(&start, &count, config->fOldClass); std::vector *const vec = (std::vector*)(((char*)addr)+config->fOffset); - Int_t nvalues; - buf.ReadInt(nvalues); - vec->resize(nvalues); + ULong64_t nvalues64 = TStreamerInfoUtils::ReadCollectionSize(buf, vers); + vec->resize(nvalues64); #ifdef R__VISUAL_CPLUSPLUS - if (nvalues <= 0) { + if (nvalues64 == 0) { buf.CheckByteCount(start,count,config->fTypeName); return 0; } #endif T *begin = vec->data(); - buf.ReadFastArray(begin, nvalues); + buf.ReadFastArray(begin, nvalues64); buf.CheckByteCount(start,count,config->fTypeName); return 0; @@ -2263,8 +2263,8 @@ namespace TStreamerInfoActions UInt_t start = buf.WriteVersion(config->fInfo->IsA(), kTRUE); std::vector *const vec = (std::vector*)(((char*)addr)+config->fOffset); - Int_t nvalues = vec->size(); - buf.WriteInt(nvalues); + auto nvalues = vec->size(); + TStreamerInfoUtils::WriteCollectionSize(buf, nvalues); T *begin = vec->data(); buf.WriteFastArray(begin, nvalues); diff --git a/io/io/src/TStreamerInfoCollectionUtils.h b/io/io/src/TStreamerInfoCollectionUtils.h new file mode 100644 index 0000000000000..aefae12142a26 --- /dev/null +++ b/io/io/src/TStreamerInfoCollectionUtils.h @@ -0,0 +1,68 @@ +// @(#)root/io:$Id$ +// Author: Philippe Canal 04/2026 + +/************************************************************************* + * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. * + * All rights reserved. * + * * + * For the licensing terms see $ROOTSYS/LICENSE. * + * For the list of contributors see $ROOTSYS/README/CREDITS. * + *************************************************************************/ + +#ifndef ROOT_TStreamerInfoCollectionUtils +#define ROOT_TStreamerInfoCollectionUtils + +#include "TBuffer.h" +#include "RtypesCore.h" +#include // std::numeric_limits + +namespace TStreamerInfoUtils { + +/// Read the number-of-objects count from the buffer using the encoding introduced +/// in TStreamerInfo version 11: when the high bit of the 32-bit word is set the +/// remaining 31 bits form the high part of a 64-bit count, and the following +/// UInt_t carries the low 32 bits. +/// +/// \param b The input buffer (operator>> must be available for Int_t/UInt_t). +/// \param vers The TStreamerInfo version that was used when the data was written. +/// \return The number of objects encoded in the stream. +inline ULong64_t ReadCollectionSize(TBuffer &b, Version_t vers) +{ + Int_t nobjects; + b >> nobjects; + ULong64_t nobjects64; + // std::signbit is not available (yet?) on Windows. + // Instead, we check if nobjects is negative to determine if the high bit is set. + if (nobjects < 0 && vers >= 11) { + nobjects64 = (static_cast(nobjects) & 0x7fffffff) << 32; + UInt_t nobjectsLow; + b >> nobjectsLow; + nobjects64 |= nobjectsLow; + } else { + nobjects64 = nobjects; + } + return nobjects64; +} + +/// Write the number-of-objects count to the buffer using the encoding introduced +/// in TStreamerInfo version 11: counts that fit in a non-negative Int_t are written +/// as a single Int_t; larger counts set the high bit of the first Int_t (carrying +/// bits 32–62 of the count) and follow with a UInt_t carrying the low 32 bits. +/// +/// \param b The output buffer (WriteInt/WriteUInt must be available). +/// \param n The number of objects to encode. +inline void WriteCollectionSize(TBuffer &b, ULong64_t n) +{ + if (n <= static_cast(std::numeric_limits::max())) { + b.WriteInt(static_cast(n)); + } else { + // Encode high 31 bits with the sign bit set, then the low 32 bits. + Int_t high = static_cast((n >> 32) | 0x80000000u); + b.WriteInt(high); + b.WriteUInt(static_cast(n & 0xffffffffu)); + } +} + +} // namespace TStreamerInfoUtils + +#endif // ROOT_TStreamerInfoCollectionUtils diff --git a/io/io/src/TStreamerInfoReadBuffer.cxx b/io/io/src/TStreamerInfoReadBuffer.cxx index 611abf936bacb..65b23432aa4d6 100644 --- a/io/io/src/TStreamerInfoReadBuffer.cxx +++ b/io/io/src/TStreamerInfoReadBuffer.cxx @@ -13,6 +13,7 @@ #include "TFile.h" #include "TClass.h" #include "TBufferFile.h" +#include "TStreamerInfoCollectionUtils.h" #include "TClonesArray.h" #include "TError.h" #include "TRef.h" @@ -89,7 +90,7 @@ TStreamerElement *TStreamerInfo::GetCurrentElement() Char_t isArray; \ b >> isArray; \ Int_t *l = (Int_t*)(arr[index]+imethod); \ - if (*l < 0 || *l > b.BufferSize()) continue; \ + if (*l < 0 || static_cast(*l) > b.BufferSize()) continue; \ name **f = (name**)(arr[index]+ioffset); \ int j; \ if (isArray) for(j=0;jfLength;j++) { \ @@ -486,7 +487,7 @@ Int_t TStreamerInfo::ReadBufferSkip(TBuffer &b, const T &arr, const TCompInfo *c DOLOOP { \ b >> isArray; \ Int_t *l = (Int_t*)(arr[k]+imethod); \ - if (*l>0 && *l < b.BufferSize()) { \ + if (*l>0 && static_cast(*l) < b.BufferSize()) { \ readbuf = new name[*l]; \ switch(newtype) { \ case TStreamerInfo::kBool: ConvCBasicPointerTo(Bool_t,ReadArrayFunc); \ @@ -798,7 +799,7 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, } else { if (gDebug > 1) { printf("ReadBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, Redirect=%p\n", + " %s, bufpos=%zu, arr=%p, eoffset=%d, Redirect=%p\n", fClass->GetName(), aElement->GetName(), i, compinfo[i]->fType, aElement->ClassName(), b.Length(), arr[0], eoffset, b.PeekDataCache()->GetObjectAt(0)); } @@ -811,7 +812,7 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, if (gDebug > 1) { printf("ReadBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, offset=%d\n", + " %s, bufpos=%zu, arr=%p, offset=%d\n", fClass->GetName(),aElement->GetName(),i,compinfo[i]->fType, aElement->ClassName(),b.Length(),arr[0], ioffset); } @@ -1191,10 +1192,10 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, cont = contp[j]; } TVirtualCollectionProxy::TPushPop helper( newProxy, cont ); - Int_t nobjects; - b >> nobjects; - env = newProxy->Allocate(nobjects,true); - if (!nobjects && (vers>=7)) { + ULong64_t nobjects64 = TStreamerInfoUtils::ReadCollectionSize(b, vers); + + env = newProxy->Allocate(nobjects64,true); + if (!nobjects64 && (vers>=7)) { // Do nothing but in version 6 of TStreamerInfo and below, // we were calling ReadBuffer for empty collection. } else { @@ -1301,10 +1302,10 @@ Int_t TStreamerInfo::ReadBuffer(TBuffer &b, const T &arr, for(; obj> nobjects; - void* env = newProxy->Allocate(nobjects,true); - if (!nobjects && (vers>=7)) { + ULong64_t nobjects64 = TStreamerInfoUtils::ReadCollectionSize(b, vers); + + void* env = newProxy->Allocate(nobjects64, true); + if (!nobjects64 && (vers>=7)) { // Do nothing but in version 6 of TStreamerInfo and below, // we were calling ReadBuffer for empty collection. } else { diff --git a/io/io/src/TStreamerInfoWriteBuffer.cxx b/io/io/src/TStreamerInfoWriteBuffer.cxx index 7a62f21ae346b..9c37917f8b1fe 100644 --- a/io/io/src/TStreamerInfoWriteBuffer.cxx +++ b/io/io/src/TStreamerInfoWriteBuffer.cxx @@ -17,6 +17,7 @@ #include "TStreamer.h" #include "TStreamerElement.h" #include "TStreamerInfo.h" +#include "TStreamerInfoCollectionUtils.h" #include "TVirtualCollectionProxy.h" #include "TRefTable.h" #include "TFile.h" @@ -151,7 +152,7 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, } else { if (gDebug > 1) { printf("WriteBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, Redirect=%p\n", + " %s, bufpos=%lu, arr=%p, eoffset=%d, Redirect=%p\n", fClass->GetName(), aElement->GetName(), i, compinfo[i]->fType, aElement->ClassName(), b.Length(), arr[0], eoffset, b.PeekDataCache()->GetObjectAt(0)); } @@ -161,7 +162,7 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, } else { if (gDebug > 1) { printf("WriteBuffer, class:%s, name=%s, fType[%d]=%d," - " %s, bufpos=%d, arr=%p, eoffset=%d, not a write rule, skipping.\n", + " %s, bufpos=%lu, arr=%p, eoffset=%d, not a write rule, skipping.\n", fClass->GetName(),aElement->GetName(),i,compinfo[i]->fType, aElement->ClassName(),b.Length(),arr[0], eoffset); } @@ -175,7 +176,7 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, if (gDebug > 1) { printf("WriteBuffer, class:%s, name=%s, fType[%d]=%d, %s, " - "bufpos=%d, arr=%p, offset=%d\n", + "bufpos=%lu, arr=%p, offset=%d\n", fClass->GetName(),aElement->GetName(),i,compinfo[i]->fType,aElement->ClassName(), b.Length(),arr[0],ioffset); } @@ -523,9 +524,9 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, for(int j=0;jfLength;++j) { char *cont = contp[j]; TVirtualCollectionProxy::TPushPop helper( proxy, cont ); - Int_t nobjects = cont ? proxy->Size() : 0; - b << nobjects; - if (nobjects) { + ULong64_t nobjects64 = cont ? proxy->Size() : 0; + TStreamerInfoUtils::WriteCollectionSize(b, nobjects64); + if (nobjects64) { auto actions = proxy->GetWriteMemberWiseActions(); char startbuf[TVirtualCollectionProxy::fgIteratorArenaSize]; @@ -587,9 +588,9 @@ Int_t TStreamerInfo::WriteBufferAux(TBuffer &b, const T &arr, for(Int_t j=0; jSize(); - b << nobjects; - if (nobjects) { + ULong64_t nobjects64 = proxy->Size(); + TStreamerInfoUtils::WriteCollectionSize(b, nobjects64); + if (nobjects64) { auto actions = proxy->GetWriteMemberWiseActions(); char startbuf[TVirtualCollectionProxy::fgIteratorArenaSize]; diff --git a/io/io/test/CMakeLists.txt b/io/io/test/CMakeLists.txt index 1e31fc3d9f98f..cf782cc0552df 100644 --- a/io/io/test/CMakeLists.txt +++ b/io/io/test/CMakeLists.txt @@ -62,4 +62,56 @@ if(NOT _32BIT) MACRO testByteCount.cxx+ OUTREF testByteCount.ref FIXTURES_REQUIRED io-io-tobj-fixture) + + ROOTTEST_COMPILE_MACRO(testLargeCollection.cxx + FIXTURES_SETUP io-io-testcoll-fixture) + + # Memory-check script: exits 0 when enough RAM+swap is available, 77 (skip) otherwise. + # RootTestDriver.cmake propagates exit code 77 from PRECMD to the test's own exit code, + # and SKIP_RETURN_CODE 77 on the main test turns that into a proper CTest "Skipped". + set(_memcheck_script "${CMAKE_CURRENT_BINARY_DIR}/check_mem_for_large_coll.sh") + file(WRITE ${_memcheck_script} + "#!/bin/sh\n" + "# Exit 77 (CTest SKIP via SKIP_RETURN_CODE) if total RAM+swap < 32 GB.\n" + "# We use TOTAL (not free) because the OS reclaims cached pages on demand.\n" + "THRESHOLD_KB=33554432\n" + "if [ -r /proc/meminfo ]; then\n" + " # Linux: use MemTotal + SwapTotal (kernel will reclaim caches as needed)\n" + " RAM_KB=\$(awk '/^MemTotal:/{print \$2}' /proc/meminfo)\n" + " SWAP_KB=\$(awk '/^SwapTotal:/{print \$2}' /proc/meminfo)\n" + " RAM_KB=\${RAM_KB:-0}\n" + " SWAP_KB=\${SWAP_KB:-0}\n" + " TOTAL_KB=\$(( RAM_KB + SWAP_KB ))\n" + "elif [ \"\$(uname)\" = \"Darwin\" ]; then\n" + " # macOS: total physical RAM via sysctl hw.memsize (bytes -> kB)\n" + " RAM_BYTES=\$(sysctl -n hw.memsize 2>/dev/null)\n" + " RAM_KB=\$(( \${RAM_BYTES:-0} / 1024 ))\n" + " # sysctl output: 'total = NNN.NNM used = ... free = ...'\n" + " SWAP_TOTAL_MB=\$(sysctl -n vm.swapusage 2>/dev/null | sed 's/total = \\([0-9.]*\\)M.*/\\1/')\n" + " if [ -n \"\$SWAP_TOTAL_MB\" ]; then\n" + " SWAP_KB=\$(awk \"BEGIN{printf \\\"%d\\\", \$SWAP_TOTAL_MB * 1024}\")\n" + " else\n" + " SWAP_KB=0\n" + " fi\n" + " TOTAL_KB=\$(( RAM_KB + SWAP_KB ))\n" + "else\n" + " echo \"testLargeCollections memory check: unsupported OS, assuming enough memory\"\n" + " exit 0\n" + "fi\n" + "echo \"testLargeCollections memory check: total \${TOTAL_KB} kB (RAM \${RAM_KB} + swap \${SWAP_KB}), need \${THRESHOLD_KB} kB\"\n" + "if [ \"\$TOTAL_KB\" -lt \"\$THRESHOLD_KB\" ]; then\n" + " echo \"Insufficient memory -- skipping testLargeCollections\"\n" + " exit 77\n" + "fi\n" + "exit 0\n" + ) + execute_process(COMMAND chmod +x ${_memcheck_script}) + + ROOTTEST_ADD_TEST(testLargeCollections + TIMEOUT 1200 + MACRO testLargeCollection.cxx+ + OUTREF testLargeCollection.ref + FIXTURES_REQUIRED io-io-testcoll-fixture + PRECMD sh ${_memcheck_script} + PROPERTIES "SKIP_REGULAR_EXPRESSION;ROOTTEST_SKIP") endif() diff --git a/io/io/test/testByteCount.cxx b/io/io/test/testByteCount.cxx index 07a6966786e56..8c9e71a6c97e0 100644 --- a/io/io/test/testByteCount.cxx +++ b/io/io/test/testByteCount.cxx @@ -5,7 +5,7 @@ #include -int testByteCount() +int unittestByteCount() { int errors = 0; unsigned int expectedByteCounts = 1; @@ -124,3 +124,115 @@ int testByteCount() std::cerr << "The end.\n"; return errors; } + +struct LargeByteCountsFixture { + std::size_t fSize{0}; + void resize(size_t size) { + fSize = size; + } + void Streamer(TBuffer &b) { + // Bare minimum to trigger the large byte count mechanism, + // we don't care about the content of the data. + if (b.IsReading()) { + b >> fSize; + b.SetBufferOffset(b.GetCurrent() - b.Buffer() + fSize); + } else { + b << fSize; + b.SetBufferOffset(b.GetCurrent() - b.Buffer() + fSize); + } + } +}; + +#ifdef __ROOTCLING__ +#pragma link C++ class LargeByteCountsFixture-; +#endif + +int readAndCheck(const std::string &msg, TBufferFile &b, size_t expected_size) +{ + // Same as b >> ptr; + auto obj = b.ReadObjectAny(TClass::GetClass(typeid(LargeByteCountsFixture))); + if (!obj) { + std::cerr << msg << ": Failed to read back the object\n"; + return 1; + } else { + auto* readFixture = static_cast(obj); + if (readFixture->fSize != expected_size ) { + std::cerr << msg << ": The size of the data vectors do not match the original ones\n"; + delete readFixture; + return 1; + } + delete readFixture; + } + return 0; +} + + +char *DoNothingAllocator(char* input, size_t, size_t) +{ + // We 'could' check that the requested memory in under what we + // preallocated. + return input; +} + +int testReadWriteObjectAny() +{ + int errors = 0; + + // TBufferFile currently reject size larger than 2GB. + // SetBufferOffset does not check against the size, + // so we can provide and use a larger buffer. + std::vector databuffer{}; + databuffer.reserve(9 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 9 * 1024 * 1024 * 1024ll - 100, databuffer.data(), false /* don't adopt */, DoNothingAllocator); + + LargeByteCountsFixture fixture; + fixture.resize(100); // Small object, should be written with the regular byte count mechanism. + + auto startPos = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + + errors += readAndCheck("Small object written in regular section", b, 100); + + // Large object, should be written with the large byte count mechanism + b.SetWriteMode(); + startPos = b.GetCurrent() - b.Buffer(); + fixture.resize(1024 * 1024 * 256); // 1GB of data + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + errors += readAndCheck("Large object written in regular section", b, 1024 * 1024 * 256); + + // Large object written in long range section, should be written with + // the large byte count mechanism + b.SetWriteMode(); + b.SetBufferOffset(4 * 1024 * 1024 * 1024ll + 100); + startPos = b.GetCurrent() - b.Buffer(); + fixture.resize(1024 * 1024 * 256); // 1GB of data + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + errors += readAndCheck("Large object written in long range section", b, 1024 * 1024 * 256); + + // Very Large object written in long range section, should be written with + // the large byte count mechanism + b.SetWriteMode(); + b.SetBufferOffset(4 * 1024 * 1024 * 1024ll + 200); + startPos = b.GetCurrent() - b.Buffer(); + fixture.resize(1024 * 1024 * 1024 + 8ll); // a bit more than 4GB of data + b.WriteObject(&fixture, false /* cacheReuse */); + b.SetReadMode(); + b.SetBufferOffset(startPos); + errors += readAndCheck("Very Large object written in long range section", b, 1024 * 1024 * 1024 + 8); + + return errors; +} + + +int testByteCount() +{ + int res = unittestByteCount(); + res += testReadWriteObjectAny(); + return res; +} \ No newline at end of file diff --git a/io/io/test/testLargeCollection.cxx b/io/io/test/testLargeCollection.cxx new file mode 100644 index 0000000000000..6e73dc9e6d839 --- /dev/null +++ b/io/io/test/testLargeCollection.cxx @@ -0,0 +1,727 @@ +/* Test large collections + + Need to test: + - direct store in a TBufferFile + - store as part of an object + - those 2 things both for numerical type and structs. + - at least one nested test. + - (maybe not) as part of a split TTree. +*/ + +#include "TBufferFile.h" +#include "TClass.h" + +#include +#include +#include +#include +#include +#if !defined(_WIN32) +#include +#endif +#if defined(__APPLE__) +#include +#elif defined(__linux__) +#include +#elif defined(_WIN32) +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#endif + +// Timing helper — writes to a dedicated file, not captured by the ctest driver. +static std::ofstream &timingLog() +{ +#if defined(_WIN32) + static std::ofstream f("testLargeCollection_timing.txt"); +#else + static std::ofstream f("/tmp/testLargeCollection_timing.txt"); +#endif + return f; +} +static double now_sec() +{ + using namespace std::chrono; + return duration(steady_clock::now().time_since_epoch()).count(); +} + +// Returns the process peak RSS in MB. +// ru_maxrss is bytes on macOS, kilobytes on Linux. +static double peak_rss_mb() +{ +#if defined(_WIN32) + PROCESS_MEMORY_COUNTERS pmc; + if (GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc))) + return pmc.PeakWorkingSetSize / (1024.0 * 1024.0); + return -1.0; +#else + struct rusage ru; + getrusage(RUSAGE_SELF, &ru); +#if defined(__APPLE__) + return ru.ru_maxrss / (1024.0 * 1024.0); +#else + return ru.ru_maxrss / 1024.0; +#endif +#endif +} +// Returns the available (free) physical memory in MB. +static double available_mem_mb() +{ +#if defined(__APPLE__) + uint64_t memsize = 0; + size_t len = sizeof(memsize); + // vm.pagesize * vm_stat free pages gives available memory + int pagesize = 0; + size_t plen = sizeof(pagesize); + sysctlbyname("hw.pagesize", &pagesize, &plen, nullptr, 0); + // Use hw.memsize for total; approximate available via sysctl vm.page_free_count + uint64_t pagefree = 0; + size_t pflen = sizeof(pagefree); + if (sysctlbyname("vm.page_free_count", &pagefree, &pflen, nullptr, 0) == 0 && pagesize > 0) + return (pagefree * (double)pagesize) / (1024.0 * 1024.0); + // fallback: total physical memory + sysctlbyname("hw.memsize", &memsize, &len, nullptr, 0); + return memsize / (1024.0 * 1024.0); +#elif defined(__linux__) + struct sysinfo si; + if (sysinfo(&si) == 0) + return (si.freeram * (double)si.mem_unit) / (1024.0 * 1024.0); + return -1.0; +#else + return -1.0; +#endif +} + +// Returns the available (free) swap space in MB. +static double available_swap_mb() +{ +#if defined(__APPLE__) + struct xsw_usage swapinfo; + size_t len = sizeof(swapinfo); + if (sysctlbyname("vm.swapusage", &swapinfo, &len, nullptr, 0) == 0) + return swapinfo.xsu_avail / (1024.0 * 1024.0); + return -1.0; +#elif defined(__linux__) + struct sysinfo si; + if (sysinfo(&si) == 0) + return (si.freeswap * (double)si.mem_unit) / (1024.0 * 1024.0); + return -1.0; +#elif defined(_WIN32) + MEMORYSTATUSEX ms; + ms.dwLength = sizeof(ms); + if (GlobalMemoryStatusEx(&ms)) + return ms.ullAvailPageFile / (1024.0 * 1024.0); + return -1.0; +#else + return -1.0; +#endif +} + +#define TIME_SUBTEST(timing, label, call) \ + do { \ + std::cerr << (label) << " ...\n"; \ + double _t0 = (timing) ? now_sec() : 0.0; \ + errors += (call); \ + if (timing) { \ + timingLog() << (label) << " done in " << (now_sec() - _t0) \ + << " s peak RSS: " << peak_rss_mb() << " MB\n"; \ + timingLog().flush(); \ + } \ + } while (0) +#define TLOG(timing, msg) do { if (timing) { timingLog() << msg << "\n"; timingLog().flush(); } } while(0) + +// ----------------------------------------------------------------------- +// Spot-check a large array: verify the first/last kSpot elements and +// kSpot elements centred on the 2 GB boundary, rather than comparing +// the entire array element-by-element. +// ----------------------------------------------------------------------- +template +static bool spotCheck(const std::vector &got, const std::vector &expected, const char *tag) +{ + if (got.size() != expected.size()) { + std::cerr << tag << ": size mismatch " << got.size() << " vs " << expected.size() << '\n'; + return false; + } + constexpr Long64_t kSpot = 1024; + const Long64_t n = (Long64_t)got.size(); + const Long64_t boundaryN = (Long64_t)(2LL * 1024 * 1024 * 1024 / sizeof(T)); + bool ok = true; + + auto check = [&](Long64_t i) { + if (i < 0 || i >= n) + return; + if (!(got[i] == expected[i])) { + if (ok) + std::cerr << tag << ": mismatch at index " << i << '\n'; + ok = false; + } + }; + + for (Long64_t i = 0; i < kSpot; ++i) + check(i); // beginning + for (Long64_t i = boundaryN - kSpot; i < boundaryN + kSpot; ++i) + check(i); // around 2 GB boundary + for (Long64_t i = n - kSpot; i < n; ++i) + check(i); // end + + return ok; +} + +// Variant of spotCheck that computes expected values on-the-fly via a +// generator, so the caller can free the source array before read-back. +template +static bool spotCheckFn(const std::vector &got, Long64_t expectedSize, Gen expectedAt, const char *tag) +{ + if ((Long64_t)got.size() != expectedSize) { + std::cerr << tag << ": size mismatch " << got.size() << " vs " << expectedSize << '\n'; + return false; + } + constexpr Long64_t kSpot = 1024; + const Long64_t n = expectedSize; + const Long64_t boundaryN = (Long64_t)(2LL * 1024 * 1024 * 1024 / sizeof(T)); + bool ok = true; + + auto check = [&](Long64_t i) { + if (i < 0 || i >= n) + return; + if (!(got[i] == expectedAt(i))) { + if (ok) + std::cerr << tag << ": mismatch at index " << i << '\n'; + ok = false; + } + }; + + for (Long64_t i = 0; i < kSpot; ++i) + check(i); + for (Long64_t i = boundaryN - kSpot; i < boundaryN + kSpot; ++i) + check(i); + for (Long64_t i = n - kSpot; i < n; ++i) + check(i); + + return ok; +} + +// ----------------------------------------------------------------------- +// Helper: a non-trivial struct so we exercise the object-array path +// ----------------------------------------------------------------------- +struct DataPoint { + float x{0}, y{0}, z{0}; + bool operator==(const DataPoint &o) const { return x == o.x && y == o.y && z == o.z; } +}; + +// ----------------------------------------------------------------------- +// Helper: an object that owns a large numerical vector and a large struct +// vector, with a minimal hand-written Streamer so we control the layout. +// ----------------------------------------------------------------------- +struct LargeCollectionFixture { + std::vector fFloats; + std::vector fPoints; + +#if 0 + void Streamer(TBuffer &b) + { + if (b.IsReading()) { + Long64_t nf = 0, np = 0; + b >> nf; + fFloats.resize(nf); + b.ReadFastArray(fFloats.data(), nf); + b >> np; + fPoints.resize(np); + b.ReadFastArray(fPoints.data(), TClass::GetClass(typeid(Point)), np); + } else { + Long64_t nf = fFloats.size(), np = fPoints.size(); + b << nf; + b.WriteFastArray(fFloats.data(), nf); + b << np; + b.WriteFastArray(fPoints.data(), TClass::GetClass(typeid(Point)), np); + } + } +#endif +}; + +#ifdef __ROOTCLING__ +#pragma link C++ class DataPoint + ; +#pragma link C++ class LargeCollectionFixture + ; +#endif + +// ----------------------------------------------------------------------- +// A do-nothing reallocator: we pre-allocate the buffer ourselves. +// ----------------------------------------------------------------------- +static char *DoNothingAllocator(char *input, size_t, size_t) +{ + return input; +} + +// ----------------------------------------------------------------------- +// 1. Direct store of a numerical array in a TBufferFile +// Returns the large float vector so testDirectVector can reuse it. +// ----------------------------------------------------------------------- +int testDirectNumerical(std::vector &orig_large_out) +{ + int errors = 0; + + // Use a pre-allocated 6 GB region so TBuffer never has to realloc. + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + // --- small array (stays in the <2 GB region) --- + const Long64_t smallN = 1000; + std::vector orig_small(smallN); + for (Long64_t i = 0; i < smallN; ++i) + orig_small[i] = float(i) * 0.1f; + + auto startSmall = b.GetCurrent() - b.Buffer(); + b << smallN; + b.WriteFastArray(orig_small.data(), smallN); + + // --- large array (crosses the 2 GB boundary) --- + // 512 M floats = 2 GB of payload + const Long64_t largeN = 512 * 1024 * 1024ll; + orig_large_out.resize(largeN); + for (Long64_t i = 0; i < largeN; ++i) + orig_large_out[i] = float(i & 0xFFFF); // 65536 = 2^16; & is faster than % + + auto startLarge = b.GetCurrent() - b.Buffer(); + b << largeN; + b.WriteFastArray(orig_large_out.data(), largeN); + + // --- read back small --- + b.SetReadMode(); + b.SetBufferOffset(startSmall); + { + Long64_t n = 0; + b >> n; + if (n != smallN) { + std::cerr << "testDirectNumerical: small array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), n); + if (got != orig_small) { + std::cerr << "testDirectNumerical: small array content mismatch\n"; + ++errors; + } + } + } + + // --- read back large --- + b.SetBufferOffset(startLarge); + { + Long64_t n = 0; + b >> n; + if (n != largeN) { + std::cerr << "testDirectNumerical: large array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), n); + if (!spotCheck(got, orig_large_out, "testDirectNumerical large")) + ++errors; + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// 2. Direct store of a struct array in a TBufferFile +// ----------------------------------------------------------------------- +int testDirectStruct(bool timing = false) +{ + int errors = 0; + + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + auto *pointClass = TClass::GetClass(typeid(DataPoint)); + + // --- small struct array --- + const Long64_t smallN = 500; + std::vector orig_small(smallN); + for (Long64_t i = 0; i < smallN; ++i) + orig_small[i] = {float(i), float(i * 2), float(i * 3)}; + + auto startSmall = b.GetCurrent() - b.Buffer(); + b << smallN; + b.WriteFastArray(orig_small.data(), pointClass, smallN); + + // --- large struct array (crosses 2 GB) --- + // ~170 M Points * 12 bytes = ~2 GB + const Long64_t largeN = 170 * 1024 * 1024ll; + std::vector orig_large(largeN); + // Use bitwise AND (power-of-2 period) instead of % 1000 to avoid + // 510 M integer divisions in this fill loop. + { double t0 = now_sec(); + for (Long64_t i = 0; i < largeN; ++i) + orig_large[i] = {float(i & 0x3FF), float((i + 1) & 0x3FF), float((i + 2) & 0x3FF)}; + TLOG(timing, " testDirectStruct fill: " << (now_sec()-t0) << " s peak RSS: " << peak_rss_mb() << " MB"); } + + auto startLarge = b.GetCurrent() - b.Buffer(); + b << largeN; + { double t0 = now_sec(); + b.WriteFastArray(orig_large.data(), pointClass, largeN); + TLOG(timing, " testDirectStruct write: " << (now_sec()-t0) << " s peak RSS: " << peak_rss_mb() << " MB"); } + + // --- read back small --- + b.SetReadMode(); + b.SetBufferOffset(startSmall); + { + Long64_t n = 0; + b >> n; + if (n != smallN) { + std::cerr << "testDirectStruct: small array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), pointClass, n); + if (got != orig_small) { + std::cerr << "testDirectStruct: small array content mismatch\n"; + ++errors; + } + } + } + + // --- read back large --- + b.SetBufferOffset(startLarge); + { + Long64_t n = 0; + b >> n; + if (n != largeN) { + std::cerr << "testDirectStruct: large array count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + { double t0 = now_sec(); + b.ReadFastArray(got.data(), pointClass, n); + TLOG(timing, " testDirectStruct read: " << (now_sec()-t0) << " s peak RSS: " << peak_rss_mb() << " MB"); } + if (!spotCheckFn(got, largeN, + [](Long64_t i) { + return DataPoint{float(i & 0x3FF), float((i + 1) & 0x3FF), float((i + 2) & 0x3FF)}; + }, + "testDirectStruct large")) + ++errors; + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// 2b. Direct store of std::vector via StreamObject +// Takes the already-built large float vector from testDirectNumerical +// to avoid re-allocating and re-filling 2 GB of data. +// ----------------------------------------------------------------------- +int testDirectVector(std::vector &orig_large_f) +{ + int errors = 0; + + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + auto *floatVecClass = TClass::GetClass("vector"); + auto *pointVecClass = TClass::GetClass("vector"); + + // --- small std::vector --- + std::vector orig_small_f(1000); + for (int i = 0; i < 1000; ++i) + orig_small_f[i] = float(i) * 0.5f; + + auto startSmallF = b.GetCurrent() - b.Buffer(); + b.StreamObject(&orig_small_f, floatVecClass); + + // --- large std::vector (512 M entries = 2 GB) --- + // orig_large_f is passed in from testDirectNumerical — no re-allocation needed. + + auto startLargeF = b.GetCurrent() - b.Buffer(); + b.StreamObject(&orig_large_f, floatVecClass); + + // --- small std::vector --- + std::vector orig_small_p(500); + for (int i = 0; i < 500; ++i) + orig_small_p[i] = {float(i), float(i * 2), float(i * 3)}; + + auto startSmallP = b.GetCurrent() - b.Buffer(); + b.StreamObject(&orig_small_p, pointVecClass); + + // --- read back small float vector --- + b.SetReadMode(); + b.SetBufferOffset(startSmallF); + { + std::vector got; + b.StreamObject(&got, floatVecClass); + if (got != orig_small_f) { + std::cerr << "testDirectVector: small float vector content mismatch\n"; + ++errors; + } + } + + // --- read back large float vector --- + b.SetBufferOffset(startLargeF); + { + std::vector got; + b.StreamObject(&got, floatVecClass); + if (!spotCheckFn(got, 512 * 1024 * 1024ll, + [](Long64_t i) { return float(i & 0xFFFF); }, + "testDirectVector large float")) + ++errors; + } + + // --- read back small DataPoint vector --- + b.SetBufferOffset(startSmallP); + { + std::vector got; + b.StreamObject(&got, pointVecClass); + if (got != orig_small_p) { + std::cerr << "testDirectVector: small DataPoint vector content mismatch\n"; + ++errors; + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// 3. Store as part of an object (numerical + struct members) +// ----------------------------------------------------------------------- +static int readAndCheckFixture(const std::string &msg, TBufferFile &b, Long64_t startPos, size_t expectedFloats, + size_t expectedPoints) +{ + b.SetReadMode(); + b.SetBufferOffset(startPos); + auto *obj = b.ReadObjectAny(TClass::GetClass(typeid(LargeCollectionFixture))); + if (!obj) { + std::cerr << msg << ": Failed to read back object\n"; + return 1; + } + auto *f = static_cast(obj); + int errors = 0; + if (f->fFloats.size() != expectedFloats) { + std::cerr << msg << ": fFloats size mismatch: got " << f->fFloats.size() << " expected " << expectedFloats + << '\n'; + ++errors; + } + if (f->fPoints.size() != expectedPoints) { + std::cerr << msg << ": fPoints size mismatch: got " << f->fPoints.size() << " expected " << expectedPoints + << '\n'; + ++errors; + } + delete f; + return errors; +} + +int testAsPartOfObject() +{ + int errors = 0; + + // Let TBufferFile own and manage its own buffer so it can auto-expand freely. + // Start small; it will grow as needed (up to ~8+ GB for the 2G-float large object). + TBufferFile b(TBuffer::kWrite, 1024); + + { + LargeCollectionFixture fixture; + + // --- small object (well within 2 GB region) --- + fixture.fFloats.assign(1000, 1.0f); + fixture.fPoints.assign(500, {1.f, 2.f, 3.f}); + auto startSmall = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + errors += readAndCheckFixture("small object", b, startSmall, 1000, 500); + } + LargeCollectionFixture fixture; + + // --- large object (floats cross 2 GB, written in regular section) --- + b.SetWriteMode(); + fixture.fFloats.assign(2 * 1024 * 1024 * 1024ll, 2.0f); // 2 G of floats (8GB) + fixture.fPoints.assign(100, {4.f, 5.f, 6.f}); + auto startLarge = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + errors += readAndCheckFixture("large object in regular section", b, startLarge, 2 * 1024 * 1024 * 1024ll, 100); + + // --- large object written past the 4 GB mark --- + b.SetWriteMode(); + b.SetBufferOffset(4 * 1024 * 1024 * 1024ll + 100); + fixture.fFloats.assign(256 * 1024 * 1024ll, 3.0f); // 1 GB of floats + fixture.fPoints.assign(200, {7.f, 8.f, 9.f}); + auto startFar = b.GetCurrent() - b.Buffer(); + b.WriteObject(&fixture, false /* cacheReuse */); + errors += readAndCheckFixture("large object in long-range section", b, startFar, 256 * 1024 * 1024ll, 200); + + return errors; +} + +// ----------------------------------------------------------------------- +// 4. Nested: a vector of LargeCollectionFixture objects +// ----------------------------------------------------------------------- +int testNested() +{ + int errors = 0; + + std::vector raw; + raw.reserve(6 * 1024 * 1024 * 1024ll); + TBufferFile b(TBuffer::kWrite, 6 * 1024 * 1024 * 1024ll - 100, raw.data(), false /* don't adopt */, + DoNothingAllocator); + + auto *fixtureClass = TClass::GetClass(typeid(LargeCollectionFixture)); + const Long64_t nObjects = 3; + std::vector objs(nObjects); + // Give each fixture a different size so we can verify round-trip. + objs[0].fFloats.assign(100, 1.0f); + objs[0].fPoints.assign(10, {1, 1, 1}); + objs[1].fFloats.assign(200, 2.0f); + objs[1].fPoints.assign(20, {2, 2, 2}); + objs[2].fFloats.assign(300, 3.0f); + objs[2].fPoints.assign(30, {3, 3, 3}); + + auto startPos = b.GetCurrent() - b.Buffer(); + b << nObjects; + b.WriteFastArray(objs.data(), fixtureClass, nObjects); + + b.SetReadMode(); + b.SetBufferOffset(startPos); + { + Long64_t n = 0; + b >> n; + if (n != nObjects) { + std::cerr << "testNested: object count mismatch: got " << n << '\n'; + ++errors; + } else { + std::vector got(n); + b.ReadFastArray(got.data(), fixtureClass, n); + for (Long64_t i = 0; i < n; ++i) { + if (got[i].fFloats != objs[i].fFloats || got[i].fPoints != objs[i].fPoints) { + std::cerr << "testNested: content mismatch at index " << i << '\n'; + ++errors; + } + } + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// 5. Minimal reproducer for two related >4 GB TBufferFile bugs: +// +// Bug A (write-side): WriteObject on an object whose serialised content +// crosses the 4 GB mark in the buffer. SetByteCount fires an assert +// because cntpos < 4 GB but fBufCur - fBuffer > 4 GB after streaming. +// Minimal trigger: start writing just below 4 GB, payload > remaining space. +// +// Bug B (read-side): ReadObjectAny on an object whose buffer start +// position > kMaxUInt. MapObject fires "offset <= kMaxUInt" assert. +// Minimal trigger: seek to > 4 GB, WriteObject tiny fixture, ReadObjectAny. +// ----------------------------------------------------------------------- +int testMapObjectLargeOffset() +{ + int errors = 0; + + // Each sub-test uses its own fresh TBufferFile so the class map is clean, + // ensuring every read-back sees a genuine "new class" entry in ReadClass + // and exercises the MapObject path at the relevant buffer offset. + + // --- Bug A: object START is just below 4 GB but the payload (10 M floats = 40 MB) + // pushes fBufCur past kMaxUInt. SetByteCount / WriteObjectClass must handle + // a byte-count position that was recorded below 4 GB while the current + // position is above it. + { + const Long64_t kBufSize = (4LL * 1024 + 128) * 1024 * 1024; // 4.125 GB + std::vector raw(kBufSize, 0); + TBufferFile b(TBuffer::kWrite, kBufSize, raw.data(), false /* don't adopt */, DoNothingAllocator); + + const Long64_t kStartA = 4LL * 1024 * 1024 * 1024 - 256; + b.SetBufferOffset(kStartA); + LargeCollectionFixture fixture; + fixture.fFloats.assign(10 * 1024 * 1024, 1.0f); // 40 MB — crosses the 4 GB boundary + fixture.fPoints.assign(3, {1.f, 2.f, 3.f}); + b.WriteObject(&fixture, false /* cacheReuse */); + + b.SetReadMode(); + b.SetBufferOffset(kStartA); + auto *objA = b.ReadObjectAny(TClass::GetClass(typeid(LargeCollectionFixture))); + if (!objA) { + std::cerr << "testMapObjectLargeOffset BugA: ReadObjectAny returned null\n"; + ++errors; + } else { + auto *f = static_cast(objA); + if (f->fFloats.size() != 10u * 1024 * 1024 || f->fPoints.size() != 3) { + std::cerr << "testMapObjectLargeOffset BugA: size mismatch\n"; + ++errors; + } + delete f; + } + } + + // --- Bug B: object written and read at a start position entirely above kMaxUInt. + // ReadObjectAny -> ReadClass -> MapObject(cl, startpos+kMapOffset) where + // startpos+kMapOffset > kMaxUInt, hitting R__ASSERT(offset <= kMaxUInt) + // in the TObject* overload of TBufferIO::MapObject. + // A fresh TBufferFile is required so that ReadClass sees a "new class" + // (not already cached) and calls MapObject with the >4 GB offset. + { + const Long64_t kBufSize = (4LL * 1024 + 128) * 1024 * 1024; // 4.125 GB + std::vector raw(kBufSize, 0); + TBufferFile b(TBuffer::kWrite, kBufSize, raw.data(), false /* don't adopt */, DoNothingAllocator); + + const Long64_t kStartB = 4LL * 1024 * 1024 * 1024 + 100; // just above kMaxUInt + b.SetBufferOffset(kStartB); + LargeCollectionFixture fixture; + fixture.fFloats.assign(10, 2.0f); + fixture.fPoints.assign(2, {2.f, 3.f, 4.f}); + b.WriteObject(&fixture, false /* cacheReuse */); + + b.SetReadMode(); + b.SetBufferOffset(kStartB); + auto *objB = b.ReadObjectAny(TClass::GetClass(typeid(LargeCollectionFixture))); + if (!objB) { + std::cerr << "testMapObjectLargeOffset BugB: ReadObjectAny returned null\n"; + ++errors; + } else { + auto *f = static_cast(objB); + if (f->fFloats.size() != 10 || f->fPoints.size() != 2) { + std::cerr << "testMapObjectLargeOffset BugB: size mismatch\n"; + ++errors; + } + delete f; + } + } + + return errors; +} + +// ----------------------------------------------------------------------- +// Entry point +// ----------------------------------------------------------------------- +int testLargeCollection(bool timing = false, bool memoryCheck = false) +{ + int errors = 0; + + if (memoryCheck) { + std::cerr << "Available memory at start: " << available_mem_mb() << " MB\n"; + double swap = available_swap_mb(); + if (swap >= 0.0) + std::cerr << "Available swap at start: " << swap << " MB\n"; + } + + std::vector sharedLargeFloats; + TIME_SUBTEST(timing, "testDirectNumerical", testDirectNumerical(sharedLargeFloats)); + TIME_SUBTEST(timing, "testDirectStruct", testDirectStruct(timing)); + TIME_SUBTEST(timing, "testDirectVector", testDirectVector(sharedLargeFloats)); + { std::vector{}.swap(sharedLargeFloats); } // release 2 GB before the large-object test + TIME_SUBTEST(timing, "testAsPartOfObject", testAsPartOfObject()); + TIME_SUBTEST(timing, "testMapObjectLargeOffset", testMapObjectLargeOffset()); + TIME_SUBTEST(timing, "testNested", testNested()); + + std::cerr << "Done. errors=" << errors << '\n'; + return errors; +} diff --git a/io/io/test/testLargeCollection.ref b/io/io/test/testLargeCollection.ref new file mode 100644 index 0000000000000..f817fb669e889 --- /dev/null +++ b/io/io/test/testLargeCollection.ref @@ -0,0 +1,9 @@ +Processing io/io/test/testLargeCollection.C... +testDirectNumerical ... +testDirectStruct ... +testDirectVector ... +testAsPartOfObject ... +testMapObjectLargeOffset ... +testNested ... +Done. errors=0 +(int) 0 diff --git a/io/sql/inc/TBufferSQL2.h b/io/sql/inc/TBufferSQL2.h index 8db572a85b362..1088ce6f20161 100644 --- a/io/sql/inc/TBufferSQL2.h +++ b/io/sql/inc/TBufferSQL2.h @@ -198,7 +198,7 @@ class TBufferSQL2 final : public TBufferText { void ReadFastArray(ULong_t *l, Int_t n) final; void ReadFastArray(Long64_t *l, Int_t n) final; void ReadFastArray(ULong64_t *l, Int_t n) final; - void ReadFastArray(Float_t *f, Int_t n) final; + void ReadFastArray(Float_t *f, Long64_t n) final; void ReadFastArray(Double_t *d, Int_t n) final; void ReadFastArrayString(Char_t *c, Int_t n) final; void ReadFastArray(void *start, const TClass *cl, Int_t n = 1, TMemberStreamer *s = nullptr, diff --git a/io/sql/src/TBufferSQL2.cxx b/io/sql/src/TBufferSQL2.cxx index cbbc40f755aa7..4ff5a04b6eb79 100644 --- a/io/sql/src/TBufferSQL2.cxx +++ b/io/sql/src/TBufferSQL2.cxx @@ -1252,7 +1252,7 @@ void TBufferSQL2::ReadFastArray(ULong64_t *l, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// Read array of Float_t from buffer -void TBufferSQL2::ReadFastArray(Float_t *f, Int_t n) +void TBufferSQL2::ReadFastArray(Float_t *f, Long64_t n) { SqlReadFastArray(f, n); } diff --git a/io/xml/inc/TBufferXML.h b/io/xml/inc/TBufferXML.h index b25aff793a838..db9202f2f983c 100644 --- a/io/xml/inc/TBufferXML.h +++ b/io/xml/inc/TBufferXML.h @@ -126,7 +126,7 @@ class TBufferXML final : public TBufferText, public TXMLSetup { void ReadFastArray(ULong_t *l, Int_t n) final; void ReadFastArray(Long64_t *l, Int_t n) final; void ReadFastArray(ULong64_t *l, Int_t n) final; - void ReadFastArray(Float_t *f, Int_t n) final; + void ReadFastArray(Float_t *f, Long64_t n) final; void ReadFastArray(Double_t *d, Int_t n) final; void ReadFastArrayString(Char_t *c, Int_t n) final; void ReadFastArray(void *start, const TClass *cl, Int_t n = 1, TMemberStreamer *s = nullptr, diff --git a/io/xml/src/TBufferXML.cxx b/io/xml/src/TBufferXML.cxx index 71ea61657da58..93ea70243cef3 100644 --- a/io/xml/src/TBufferXML.cxx +++ b/io/xml/src/TBufferXML.cxx @@ -1779,7 +1779,7 @@ void TBufferXML::ReadFastArray(ULong64_t *l, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// Read array of Float_t from buffer -void TBufferXML::ReadFastArray(Float_t *f, Int_t n) +void TBufferXML::ReadFastArray(Float_t *f, Long64_t n) { XmlReadFastArray(f, n); } diff --git a/net/net/src/TMessage.cxx b/net/net/src/TMessage.cxx index f36e8b296804c..4105aadcf9f41 100644 --- a/net/net/src/TMessage.cxx +++ b/net/net/src/TMessage.cxx @@ -392,10 +392,16 @@ Int_t TMessage::Compress() fCompPos = fBufCur; bufcur = fBufComp; + if (static_cast(CompLength()) > static_cast(std::numeric_limits::max())) { + Warning("Compress", "Compressed buffer length too large (%d)", CompLength()); + } tobuf(bufcur, (UInt_t)(CompLength() - sizeof(UInt_t))); Int_t what = fWhat | kMESS_ZIP; tobuf(bufcur, what); - tobuf(bufcur, Length()); // original uncompressed buffer length + if (static_cast(Length()) > static_cast(std::numeric_limits::max())) { + Warning("Compress", "Original buffer length too large (%zu)", Length()); + } + tobuf(bufcur, (UInt_t)Length()); // original uncompressed buffer length return 0; } diff --git a/roottest/root/io/buffer/ownership.ref b/roottest/root/io/buffer/ownership.ref index 5c633dbad1d39..e39e5dca5c19e 100644 --- a/roottest/root/io/buffer/ownership.ref +++ b/roottest/root/io/buffer/ownership.ref @@ -1,7 +1,7 @@ Processing runownership.C+... Fatal in : Failed to expand the data buffer because TBuffer does not own it and no custom memory reallocator was provided. The expected error "Failed to expand the data buffer because TBuffer does not own it and no custom memory reallocator was provided." was seen -Running custom allocator (increase from 3 to 136) +Running custom allocator (increase from 128 to 136) Running custom allocator (increase from 136 to 264) The test has completed successfully (int) 0 diff --git a/roottest/root/io/filemerger/CMakeLists.txt b/roottest/root/io/filemerger/CMakeLists.txt index 2c4d05d148051..4ab605b9104b3 100644 --- a/roottest/root/io/filemerger/CMakeLists.txt +++ b/roottest/root/io/filemerger/CMakeLists.txt @@ -346,7 +346,7 @@ if(ZLIB_NG) else() ROOTTEST_ADD_TEST(simple-zlib-compr-level1 MACRO testSimpleFile.C - MACROARG "\"hsimple101.root\",25000,101,415025,25" + MACROARG "\"hsimple101.root\",25000,101,415004,25" FIXTURES_REQUIRED root-io-filemerger-hsimple101-fixture) ROOTTEST_ADD_TEST(simple-zlib-compr-level6 diff --git a/roottest/root/tree/fastcloning/references/execCheckClusterRange.ref32 b/roottest/root/tree/fastcloning/references/execCheckClusterRange.ref32 index d6cb093e57d19..8a1bbe9488ff7 100644 --- a/roottest/root/tree/fastcloning/references/execCheckClusterRange.ref32 +++ b/roottest/root/tree/fastcloning/references/execCheckClusterRange.ref32 @@ -141,7 +141,7 @@ Cluster #47 starts at 1740 and ends at 1769 Cluster #48 starts at 1770 and ends at 1799 ****************************************************************************** *Tree :t1 : * -*Entries : 1800 : Total = 29346 bytes File Size = 17990 * +*Entries : 1800 : Total = 29346 bytes File Size = 17985 * * : : Tree compression factor = 1.59 * ****************************************************************************** Cluster Range # Entry Start Last Entry Size Number of clusters @@ -221,7 +221,7 @@ Cluster #67 starts at 2340 and ends at 2369 Cluster #68 starts at 2370 and ends at 2399 ****************************************************************************** *Tree :t1 : * -*Entries : 2400 : Total = 33566 bytes File Size = 20848 * +*Entries : 2400 : Total = 33566 bytes File Size = 20841 * * : : Tree compression factor = 1.56 * ****************************************************************************** Cluster Range # Entry Start Last Entry Size Number of clusters diff --git a/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib.ref b/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib.ref index 2e65ecc45e778..2ac6f68b65ff9 100644 --- a/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib.ref +++ b/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib.ref @@ -221,7 +221,7 @@ Cluster #67 starts at 2340 and ends at 2369 Cluster #68 starts at 2370 and ends at 2399 ****************************************************************************** *Tree :t1 : * -*Entries : 2400 : Total = 33566 bytes File Size = 20846 * +*Entries : 2400 : Total = 33566 bytes File Size = 20845 * * : : Tree compression factor = 1.56 * ****************************************************************************** Cluster Range # Entry Start Last Entry Size Number of clusters diff --git a/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib_win64.ref b/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib_win64.ref index 00f9d698c83b4..8a08d9b2bca9a 100644 --- a/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib_win64.ref +++ b/roottest/root/tree/fastcloning/references/execCheckClusterRange_zlib_win64.ref @@ -141,7 +141,7 @@ Cluster #47 starts at 1740 and ends at 1769 Cluster #48 starts at 1770 and ends at 1799 ****************************************************************************** *Tree :t1 : * -*Entries : 1800 : Total = 29346 bytes File Size = 17989 * +*Entries : 1800 : Total = 29346 bytes File Size = 17988 * * : : Tree compression factor = 1.59 * ****************************************************************************** Cluster Range # Entry Start Last Entry Size Number of clusters @@ -221,7 +221,7 @@ Cluster #67 starts at 2340 and ends at 2369 Cluster #68 starts at 2370 and ends at 2399 ****************************************************************************** *Tree :t1 : * -*Entries : 2400 : Total = 33566 bytes File Size = 20845 * +*Entries : 2400 : Total = 33566 bytes File Size = 20847 * * : : Tree compression factor = 1.56 * ****************************************************************************** Cluster Range # Entry Start Last Entry Size Number of clusters diff --git a/tree/ntuple/test/CustomStruct.hxx b/tree/ntuple/test/CustomStruct.hxx index 0cde0ccb3e091..e5b53bd1b1db8 100644 --- a/tree/ntuple/test/CustomStruct.hxx +++ b/tree/ntuple/test/CustomStruct.hxx @@ -362,16 +362,16 @@ public: TVirtualCollectionProxy *Generate() const final { return new CyclicCollectionProxy(); } Int_t GetCollectionType() const final { return 0; } ULong_t GetIncrement() const final { return 0; } - UInt_t Sizeof() const final { return 0; } + size_type Sizeof() const final { return 0; } bool HasPointers() const final { return false; } TClass *GetValueClass() const final; EDataType GetType() const final { return EDataType::kOther_t; } void PushProxy(void *) final {} void PopProxy() final {} - void *At(UInt_t) final { return nullptr; } + void *At(size_type) final { return nullptr; } void Clear(const char * = "") final {} - UInt_t Size() const final { return 0; } - void *Allocate(UInt_t, bool) final { return nullptr; } + size_type Size() const final { return 0; } + void *Allocate(size_type, bool) final { return nullptr; } void Commit(void *) final {} void Insert(const void *, void *, size_t) final {} TStreamerInfoActions::TActionSequence *GetConversionReadMemberWiseActions(TClass *, Int_t) final { return nullptr; } diff --git a/tree/ntuple/test/SimpleCollectionProxy.hxx b/tree/ntuple/test/SimpleCollectionProxy.hxx index 6d152f76c8482..bbf2ca6c4cdd1 100644 --- a/tree/ntuple/test/SimpleCollectionProxy.hxx +++ b/tree/ntuple/test/SimpleCollectionProxy.hxx @@ -55,7 +55,7 @@ public: } Int_t GetCollectionType() const override { return CollectionKind; } ULong_t GetIncrement() const override { return sizeof(typename CollectionT::ValueType); } - UInt_t Sizeof() const override { return sizeof(CollectionT); } + size_type Sizeof() const override { return sizeof(CollectionT); } bool HasPointers() const override { return false; } TClass *GetValueClass() const override @@ -80,10 +80,10 @@ public: void PushProxy(void *objectstart) override { fObject = static_cast(objectstart); } void PopProxy() override { fObject = nullptr; } - void *At(UInt_t idx) override { return &fObject->v[idx]; } + void *At(size_type idx) override { return &fObject->v[idx]; } void Clear(const char * /*opt*/ = "") override { fObject->v.clear(); } - UInt_t Size() const override { return fObject->v.size(); } - void *Allocate(UInt_t n, bool /*forceDelete*/) override + size_type Size() const override { return fObject->v.size(); } + void *Allocate(size_type n, bool /*forceDelete*/) override { fObject->v.resize(n); return fObject; diff --git a/tree/ntuple/test/ntuple_emulated.cxx b/tree/ntuple/test/ntuple_emulated.cxx index 013a8f1fd30bf..31d6d4b938399 100644 --- a/tree/ntuple/test/ntuple_emulated.cxx +++ b/tree/ntuple/test/ntuple_emulated.cxx @@ -566,7 +566,7 @@ TEST(RNTupleEmulated, CollectionProxy) } Int_t GetCollectionType() const override { return ROOT::kSTLvector; } ULong_t GetIncrement() const override { return sizeof(typename CollectionT::ValueType); } - UInt_t Sizeof() const override { return sizeof(CollectionT); } + size_type Sizeof() const override { return sizeof(CollectionT); } bool HasPointers() const override { return false; } TClass *GetValueClass() const override @@ -589,10 +589,10 @@ TEST(RNTupleEmulated, CollectionProxy) void PushProxy(void *objectstart) override { fObject = static_cast(objectstart); } void PopProxy() override { fObject = nullptr; } - void *At(UInt_t idx) override { return &fObject->v[idx]; } + void *At(size_type idx) override { return &fObject->v[idx]; } void Clear(const char * /*opt*/ = "") override { fObject->v.clear(); } - UInt_t Size() const override { return fObject->v.size(); } - void *Allocate(UInt_t n, bool /*forceDelete*/) override + size_type Size() const override { return fObject->v.size(); } + void *Allocate(size_type n, bool /*forceDelete*/) override { fObject->v.resize(n); return fObject; diff --git a/tree/tree/inc/TBasket.h b/tree/tree/inc/TBasket.h index 66a737ac2ec2c..e926496513c76 100644 --- a/tree/tree/inc/TBasket.h +++ b/tree/tree/inc/TBasket.h @@ -39,11 +39,11 @@ friend class TBranch; TBasket& operator=(const TBasket&); ///< TBasket objects are not copiable. // Internal corner cases for ReadBasketBuffers - Int_t ReadBasketBuffersUnzip(char*, Int_t, bool, TFile*); + Int_t ReadBasketBuffersUnzip(char*, ULong64_t, bool, TFile*); Int_t ReadBasketBuffersUncompressedCase(); // Helper for managing the compressed buffer. - void InitializeCompressedBuffer(Int_t len, TFile* file); + void InitializeCompressedBuffer(ULong64_t len, TFile* file); // Handles special logic around deleting / reseting the entry offset pointer. void ResetEntryOffset(); @@ -130,9 +130,9 @@ friend class TBranch; Int_t GetNevBufSize() const {return fNevBufSize;} Int_t GetLast() const {return fLast;} virtual void MoveEntries(Int_t dentries); - virtual void PrepareBasket(Long64_t /* entry */) {}; - Int_t ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file); - Int_t ReadBasketBytes(Long64_t pos, TFile *file); + virtual void PrepareBasket(ULong64_t /* entry */) {}; + Int_t ReadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file); + Int_t ReadBasketBytes(ULong64_t pos, TFile *file); virtual void WriteReset(); // Time spent reseting basket sizes (typically, at event cluster boundaries), in microseconds @@ -142,7 +142,7 @@ friend class TBranch; // Count of resets performed of basket size. bool GetResetAllocationCount() const { return fResetAllocation; } - Int_t LoadBasketBuffers(Long64_t pos, Int_t len, TFile *file, TTree *tree = nullptr); + Int_t LoadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file, TTree *tree = nullptr); Long64_t CopyTo(TFile *to); void SetBranch(TBranch *branch) { fBranch = branch; } diff --git a/tree/tree/inc/TBasketSQL.h b/tree/tree/inc/TBasketSQL.h index efe9caab8261c..ba7925e8d33e7 100644 --- a/tree/tree/inc/TBasketSQL.h +++ b/tree/tree/inc/TBasketSQL.h @@ -45,9 +45,9 @@ class TBasketSQL : public TBasket TBranch *branch, TSQLResult **rs, TString *insert_query, std::vector *vc, TSQLRow **row); ~TBasketSQL() override; - void PrepareBasket(Long64_t entry) override; - virtual Int_t ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file); - virtual Int_t ReadBasketBytes(Long64_t pos, TFile *file); + void PrepareBasket(ULong64_t entry) override; + virtual Int_t ReadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file); + virtual Int_t ReadBasketBytes(ULong64_t pos, TFile *file); TSQLResult * GetResultSet() { return *fResultPtr;} void CreateBuffer(const char *name, TString title, std::vector * vc, TBranch *branch, TSQLResult ** rs); diff --git a/tree/tree/inc/TBranch.h b/tree/tree/inc/TBranch.h index f91d1e8380495..64be8447bdc58 100644 --- a/tree/tree/inc/TBranch.h +++ b/tree/tree/inc/TBranch.h @@ -263,7 +263,7 @@ class TBranch : public TNamed, public TAttFill { virtual bool GetMakeClass() const; TBranch *GetMother() const; TBranch *GetSubBranch(const TBranch *br) const; - TBuffer *GetTransientBuffer(Int_t size); + TBuffer *GetTransientBuffer(ULong64_t size); bool IsAutoDelete() const; bool IsFolder() const override; virtual void KeepCircular(Long64_t maxEntries); diff --git a/tree/tree/inc/TBufferSQL.h b/tree/tree/inc/TBufferSQL.h index 72908333e689c..d988488b7bd49 100644 --- a/tree/tree/inc/TBufferSQL.h +++ b/tree/tree/inc/TBufferSQL.h @@ -116,7 +116,7 @@ class TBufferSQL final : public TBufferFile { void ReadFastArray(ULong_t *, Int_t ) final; void ReadFastArray(Long64_t *, Int_t ) final; void ReadFastArray(ULong64_t *, Int_t ) final; - void ReadFastArray(Float_t *, Int_t ) final; + void ReadFastArray(Float_t *, Long64_t ) final; void ReadFastArray(Double_t *, Int_t ) final; void ReadFastArrayFloat16(Float_t *f, Int_t n, TStreamerElement *ele=nullptr) final; void ReadFastArrayDouble32(Double_t *d, Int_t n, TStreamerElement *ele=nullptr) final; diff --git a/tree/tree/inc/TTree.h b/tree/tree/inc/TTree.h index 4250c4789da0b..64988c1ff1a82 100644 --- a/tree/tree/inc/TTree.h +++ b/tree/tree/inc/TTree.h @@ -599,7 +599,7 @@ class TTree : public TNamed, public TAttLine, public TAttFill, public TAttMarker TTreeFormula *GetSelect() { return GetPlayer()->GetSelect(); } virtual Long64_t GetSelectedRows() { return GetPlayer()->GetSelectedRows(); } virtual Int_t GetTimerInterval() const { return fTimerInterval; } - TBuffer* GetTransientBuffer(Int_t size); + TBuffer* GetTransientBuffer(ULong64_t size); virtual Long64_t GetTotBytes() const { return fTotBytes; } virtual TTree *GetTree() const { return const_cast(this); } virtual TVirtualIndex *GetTreeIndex() const { return fTreeIndex; } diff --git a/tree/tree/src/TBasket.cxx b/tree/tree/src/TBasket.cxx index fd17cc22a9c74..68c090d4b080d 100644 --- a/tree/tree/src/TBasket.cxx +++ b/tree/tree/src/TBasket.cxx @@ -243,7 +243,7 @@ Int_t TBasket::GetEntryPointer(Int_t entry) /// This function is called by TTreeCloner. /// The function returns 0 in case of success, 1 in case of error. -Int_t TBasket::LoadBasketBuffers(Long64_t pos, Int_t len, TFile *file, TTree *tree) +Int_t TBasket::LoadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file, TTree *tree) { if (fBufferRef) { // Reuse the buffer if it exist. @@ -378,7 +378,7 @@ Int_t TBasket::ReadBasketBuffersUncompressedCase() //////////////////////////////////////////////////////////////////////////////// /// We always create the TBuffer for the basket but it hold the buffer from the cache. -Int_t TBasket::ReadBasketBuffersUnzip(char* buffer, Int_t size, bool mustFree, TFile* file) +Int_t TBasket::ReadBasketBuffersUnzip(char* buffer, ULong64_t size, bool mustFree, TFile* file) { if (fBufferRef) { fBufferRef->SetBuffer(buffer, size, mustFree); @@ -408,15 +408,19 @@ Int_t TBasket::ReadBasketBuffersUnzip(char* buffer, Int_t size, bool mustFree, T //////////////////////////////////////////////////////////////////////////////// /// Initialize a buffer for reading if it is not already initialized -static inline TBuffer* R__InitializeReadBasketBuffer(TBuffer* bufferRef, Int_t len, TFile* file) +static inline TBuffer* R__InitializeReadBasketBuffer(TBuffer* bufferRef, ULong64_t len, TFile* file) { TBuffer* result; if (R__likely(bufferRef)) { bufferRef->SetReadMode(); - Int_t curBufferSize = bufferRef->BufferSize(); + ULong64_t curBufferSize = bufferRef->BufferSize(); if (curBufferSize < len) { // Experience shows that giving 5% "wiggle-room" decreases churn. - bufferRef->Expand(Int_t(len*1.05)); + double newSizeD = len * 1.05; + auto newSize = newSizeD < (double)std::numeric_limits::max() + ? static_cast(newSizeD) + : std::numeric_limits::max(); + bufferRef->Expand(newSize); } bufferRef->Reset(); result = bufferRef; @@ -430,7 +434,7 @@ static inline TBuffer* R__InitializeReadBasketBuffer(TBuffer* bufferRef, Int_t l //////////////////////////////////////////////////////////////////////////////// /// Initialize the compressed buffer; either from the TTree or create a local one. -void inline TBasket::InitializeCompressedBuffer(Int_t len, TFile* file) +void inline TBasket::InitializeCompressedBuffer(ULong64_t len, TFile* file) { bool compressedBufferExists = fCompressedBufferRef != nullptr; fCompressedBufferRef = R__InitializeReadBasketBuffer(fCompressedBufferRef, len, file); @@ -462,7 +466,7 @@ void TBasket::ResetEntryOffset() /// There is a lot of code duplication but it was necessary to assure /// the expected behavior when there is no cache. -Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) +Int_t TBasket::ReadBasketBuffers(ULong64_t pos, ULong64_t len, TFile *file) { if(!fBranch->GetDirectory()) { return -1; @@ -607,7 +611,7 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) // Note that in previous versions we didn't allocate buffers until we verified // the zip headers; this is no longer beforehand as the buffer lifetime is scoped // to the TBranch. - uncompressedBufferLen = len > fObjlen+fKeylen ? len : fObjlen+fKeylen; + uncompressedBufferLen = len > static_cast(fObjlen+fKeylen) ? len : fObjlen+fKeylen; fBufferRef = R__InitializeReadBasketBuffer(fBufferRef, uncompressedBufferLen, file); rawUncompressedBuffer = fBufferRef->Buffer(); fBuffer = rawUncompressedBuffer; @@ -699,7 +703,10 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) if (R__unlikely(!fEntryOffset)) { fEntryOffset = new Int_t[fNevBuf+1]; fEntryOffset[0] = fKeylen; - Warning("ReadBasketBuffers","basket:%s has fNevBuf=%d but fEntryOffset=0, pos=%lld, len=%d, fNbytes=%d, fObjlen=%d, trying to repair",GetName(),fNevBuf,pos,len,fNbytes,fObjlen); + Warning("ReadBasketBuffers", + "basket:%s has fNevBuf=%d but fEntryOffset=0, pos=%llu, len=%llu, fNbytes=%d, fObjlen=%d, trying to " + "repair", + GetName(), fNevBuf, pos, len, fNbytes, fObjlen); return 0; } if (fIOBits & static_cast(TBasket::EIOBits::kGenerateOffsetMap)) { @@ -731,7 +738,7 @@ Int_t TBasket::ReadBasketBuffers(Long64_t pos, Int_t len, TFile *file) /// Read first bytes of a logical record starting at position pos /// return record length (first 4 bytes of record). -Int_t TBasket::ReadBasketBytes(Long64_t pos, TFile *file) +Int_t TBasket::ReadBasketBytes(ULong64_t pos, TFile *file) { const Int_t len = 128; char buffer[len]; diff --git a/tree/tree/src/TBasketSQL.cxx b/tree/tree/src/TBasketSQL.cxx index 5c0edf3639030..6bb0ed64a33b2 100644 --- a/tree/tree/src/TBasketSQL.cxx +++ b/tree/tree/src/TBasketSQL.cxx @@ -111,7 +111,7 @@ void TBasketSQL::CreateBuffer(const char *name, TString title, //////////////////////////////////////////////////////////////////////////////// /// Prepare the basket for the next entry. -void TBasketSQL::PrepareBasket(Long64_t entry) +void TBasketSQL::PrepareBasket(ULong64_t entry) { ((TBufferSQL*)fBufferRef)->ResetOffset(); ((TTreeSQL*)fBranch->GetTree())->PrepEntry(entry); @@ -121,7 +121,7 @@ void TBasketSQL::PrepareBasket(Long64_t entry) //////////////////////////////////////////////////////////////////////////////// /// See TBasket::ReadBasketBytes. This is not implemented in TBasketSQL. -Int_t TBasketSQL::ReadBasketBytes(Long64_t , TFile *) +Int_t TBasketSQL::ReadBasketBytes(ULong64_t , TFile *) { Error("ReadBasketBytes","This member function should not be called!"); return 0; @@ -130,7 +130,7 @@ Int_t TBasketSQL::ReadBasketBytes(Long64_t , TFile *) //////////////////////////////////////////////////////////////////////////////// /// See TBasket::ReadBasketBuffers. This is not implemented in TBasketSQL. -Int_t TBasketSQL::ReadBasketBuffers(Long64_t , Int_t, TFile *) +Int_t TBasketSQL::ReadBasketBuffers(ULong64_t , ULong64_t, TFile *) { Error("ReadBasketBuffers","This member function should not be called!"); return 0; diff --git a/tree/tree/src/TBranch.cxx b/tree/tree/src/TBranch.cxx index c0e3640e7572f..41d01d30d8cab 100644 --- a/tree/tree/src/TBranch.cxx +++ b/tree/tree/src/TBranch.cxx @@ -523,7 +523,7 @@ TBranch::~TBranch() //////////////////////////////////////////////////////////////////////////////// /// Returns the transient buffer currently used by this TBranch for reading/writing baskets. -TBuffer* TBranch::GetTransientBuffer(Int_t size) +TBuffer* TBranch::GetTransientBuffer(ULong64_t size) { if (fTransientBuffer) { if (fTransientBuffer->BufferSize() < size) { diff --git a/tree/tree/src/TBufferSQL.cxx b/tree/tree/src/TBufferSQL.cxx index ba4d09306ef5c..eae27a605a42b 100644 --- a/tree/tree/src/TBufferSQL.cxx +++ b/tree/tree/src/TBufferSQL.cxx @@ -850,9 +850,9 @@ void TBufferSQL::ReadFastArray(ULong64_t *ull, Int_t n) //////////////////////////////////////////////////////////////////////////////// /// ReadFastArray SQL implementation. -void TBufferSQL::ReadFastArray(Float_t *f, Int_t n) +void TBufferSQL::ReadFastArray(Float_t *f, Long64_t n) { - for(int i=0; iGetField(*fIter)); ++fIter; } diff --git a/tree/tree/src/TTree.cxx b/tree/tree/src/TTree.cxx index dd0e35e3c9494..d2dd147b93025 100644 --- a/tree/tree/src/TTree.cxx +++ b/tree/tree/src/TTree.cxx @@ -1062,7 +1062,7 @@ TTree::~TTree() //////////////////////////////////////////////////////////////////////////////// /// Returns the transient buffer currently used by this TTree for reading/writing baskets. -TBuffer* TTree::GetTransientBuffer(Int_t size) +TBuffer* TTree::GetTransientBuffer(ULong64_t size) { if (fTransientBuffer) { if (fTransientBuffer->BufferSize() < size) {