Large files test

JeWe37 · JeWe37 · commit dd961ae03c5c · 2023-08-26T04:07:03.000+02:00
diff --git a/src/blockhash.cc b/src/blockhash.cc
@@ -14,6 +14,7 @@
 
 #include <config.h>
 #include "blockhash.h"
+#include <cstddef>
 #include <stdint.h>  // uint32_t
 #include <string.h>  // memcpy, memcmp
 #include <algorithm>  // std::min
@@ -27,7 +28,7 @@ typedef unsigned long uword_t;  // a machine word                         NOLINT
 
 BlockHash::BlockHash(const char* source_data,
                      size_t source_size,
-                     int starting_offset)
+                     size_t starting_offset)
     : source_data_(source_data),
       source_size_(source_size),
       hash_table_mask_(0),
@@ -95,7 +96,7 @@ BlockHash* BlockHash::CreateTargetHash(const char* target_data,
                                        size_t dictionary_size) {
   BlockHash* new_target_hash = new BlockHash(target_data,
                                              target_size,
-                                             static_cast<int>(dictionary_size));
+                                             static_cast<ptrdiff_t>(dictionary_size));
   if (!new_target_hash->Init(/* populate_hash_table = */ false)) {
     delete new_target_hash;
     return NULL;
@@ -156,9 +157,9 @@ void BlockHash::AddBlock(uint32_t hash_value) {
     return;
   }
   // The initial value of last_block_added_ is -1.
-  int block_number = last_block_added_ + 1;
-  const int total_blocks =
-      static_cast<int>(source_size_ / kBlockSize);  // round down
+  size_t block_number = last_block_added_ + 1;
+  const size_t total_blocks =
+      static_cast<size_t>(source_size_ / kBlockSize);  // round down
   if (block_number >= total_blocks) {
     VCD_DFATAL << "BlockHash::AddBlock() called"
                   " with block number " << block_number
@@ -174,7 +175,7 @@ void BlockHash::AddBlock(uint32_t hash_value) {
     return;
   }
   const uint32_t hash_table_index = GetHashTableIndex(hash_value);
-  const int first_matching_block = hash_table_[hash_table_index];
+  const size_t first_matching_block = hash_table_[hash_table_index];
   if (first_matching_block < 0) {
     // This is the first entry with this hash value
     hash_table_[hash_table_index] = block_number;
@@ -200,14 +201,14 @@ void BlockHash::AddAllBlocks() {
   AddAllBlocksThroughIndex(static_cast<int>(source_size_));
 }
 
-void BlockHash::AddAllBlocksThroughIndex(int end_index) {
-  if (end_index > static_cast<int>(source_size_)) {
+void BlockHash::AddAllBlocksThroughIndex(size_t end_index) {
+  if (end_index > static_cast<size_t>(source_size_)) {
     VCD_DFATAL << "BlockHash::AddAllBlocksThroughIndex() called"
                   " with index " << end_index
                << " higher than end index  " << source_size_ << VCD_ENDL;
     return;
   }
-  const int last_index_added = last_block_added_ * kBlockSize;
+  const size_t last_index_added = last_block_added_ * kBlockSize;
   if (end_index <= last_index_added) {
     VCD_DFATAL << "BlockHash::AddAllBlocksThroughIndex() called"
                   " with index " << end_index
@@ -221,11 +222,11 @@ void BlockHash::AddAllBlocksThroughIndex(int end_index) {
     // See: https://github.com/google/open-vcdiff/issues/40
     return;
   }
-  int end_limit = end_index;
+  size_t end_limit = end_index;
   // Don't allow reading any indices at or past source_size_.
   // The Hash function extends (kBlockSize - 1) bytes past the index,
   // so leave a margin of that size.
-  int last_legal_hash_index = static_cast<int>(source_size() - kBlockSize);
+  size_t last_legal_hash_index = static_cast<size_t>(source_size() - kBlockSize);
   if (end_limit > last_legal_hash_index) {
     end_limit = last_legal_hash_index + 1;
   }
@@ -300,9 +301,9 @@ bool BlockHash::BlockContentsMatch(const char* block1, const char* block2) {
   return BlockContentsMatchInline(block1, block2);
 }
 
-inline int BlockHash::SkipNonMatchingBlocks(int block_number,
+inline size_t BlockHash::SkipNonMatchingBlocks(size_t block_number,
                                             const char* block_ptr) const {
-  int probes = 0;
+  size_t probes = 0;
   while ((block_number >= 0) &&
          !BlockContentsMatchInline(block_ptr,
                                    &source_data_[block_number * kBlockSize])) {
@@ -317,18 +318,18 @@ inline int BlockHash::SkipNonMatchingBlocks(int block_number,
 // Init() must have been called and returned true before using
 // FirstMatchingBlock or NextMatchingBlock.  No check is performed
 // for this condition; the code will crash if this condition is violated.
-inline int BlockHash::FirstMatchingBlockInline(uint32_t hash_value,
+inline size_t BlockHash::FirstMatchingBlockInline(uint32_t hash_value,
                                                const char* block_ptr) const {
   return SkipNonMatchingBlocks(hash_table_[GetHashTableIndex(hash_value)],
                                block_ptr);
 }
 
-int BlockHash::FirstMatchingBlock(uint32_t hash_value,
+size_t BlockHash::FirstMatchingBlock(uint32_t hash_value,
                                   const char* block_ptr) const {
   return FirstMatchingBlockInline(hash_value, block_ptr);
 }
 
-int BlockHash::NextMatchingBlock(int block_number,
+size_t BlockHash::NextMatchingBlock(size_t block_number,
                                  const char* block_ptr) const {
   if (static_cast<size_t>(block_number) >= GetNumberOfBlocks()) {
     VCD_DFATAL << "NextMatchingBlock called for invalid block number "
@@ -343,7 +344,7 @@ int BlockHash::NextMatchingBlock(int block_number,
 // dictionary is made up of spaces (' ') and the search string is also
 // made up of spaces, there will be one match for each block in the
 // dictionary.
-inline bool BlockHash::TooManyMatches(int* match_counter) {
+inline bool BlockHash::TooManyMatches(size_t* match_counter) {
   ++(*match_counter);
   return (*match_counter) > kMaxMatchesToCheck;
 }
@@ -352,12 +353,12 @@ inline bool BlockHash::TooManyMatches(int* match_counter) {
 // that match the corresponding bytes to the left of target_match_start.
 // Will not examine more than max_bytes bytes, which is to say that
 // the return value will be in the range [0, max_bytes] inclusive.
-int BlockHash::MatchingBytesToLeft(const char* source_match_start,
+size_t BlockHash::MatchingBytesToLeft(const char* source_match_start,
                                    const char* target_match_start,
-                                   int max_bytes) {
+                                   size_t max_bytes) {
   const char* source_ptr = source_match_start;
   const char* target_ptr = target_match_start;
-  int bytes_found = 0;
+  size_t bytes_found = 0;
   while (bytes_found < max_bytes) {
     --source_ptr;
     --target_ptr;
@@ -373,12 +374,12 @@ int BlockHash::MatchingBytesToLeft(const char* source_match_start,
 // that match the corresponding bytes starting at target_match_end.
 // Will not examine more than max_bytes bytes, which is to say that
 // the return value will be in the range [0, max_bytes] inclusive.
-int BlockHash::MatchingBytesToRight(const char* source_match_end,
+size_t BlockHash::MatchingBytesToRight(const char* source_match_end,
                                     const char* target_match_end,
-                                    int max_bytes) {
+                                    size_t max_bytes) {
   const char* source_ptr = source_match_end;
   const char* target_ptr = target_match_end;
-  int bytes_found = 0;
+  size_t bytes_found = 0;
   while ((bytes_found < max_bytes) && (*source_ptr == *target_ptr)) {
     ++bytes_found;
     ++source_ptr;
@@ -399,24 +400,24 @@ void BlockHash::FindBestMatch(uint32_t hash_value,
                               const char* target_start,
                               size_t target_size,
                               Match* best_match) const {
-  int match_counter = 0;
-  for (int block_number = FirstMatchingBlockInline(hash_value,
+  size_t match_counter = 0;
+  for (ptrdiff_t block_number = FirstMatchingBlockInline(hash_value,
                                                    target_candidate_start);
        (block_number >= 0) && !TooManyMatches(&match_counter);
        block_number = NextMatchingBlock(block_number, target_candidate_start)) {
-    int source_match_offset = block_number * kBlockSize;
-    const int source_match_end = source_match_offset + kBlockSize;
+    ptrdiff_t source_match_offset = block_number * kBlockSize;
+    const ptrdiff_t source_match_end = source_match_offset + kBlockSize;
 
-    int target_match_offset =
-        static_cast<int>(target_candidate_start - target_start);
-    const int target_match_end = target_match_offset + kBlockSize;
+    ptrdiff_t target_match_offset =
+        static_cast<ptrdiff_t>(target_candidate_start - target_start);
+    const ptrdiff_t target_match_end = target_match_offset + kBlockSize;
 
     size_t match_size = kBlockSize;
     {
       // Extend match start towards beginning of unencoded data
-      const int limit_bytes_to_left = std::min(source_match_offset,
+      const ptrdiff_t limit_bytes_to_left = std::min(source_match_offset,
                                                target_match_offset);
-      const int matching_bytes_to_left =
+      const ptrdiff_t matching_bytes_to_left =
           MatchingBytesToLeft(source_data_ + source_match_offset,
                               target_start + target_match_offset,
                               limit_bytes_to_left);
@@ -433,7 +434,7 @@ void BlockHash::FindBestMatch(uint32_t hash_value,
       match_size +=
           MatchingBytesToRight(source_data_ + source_match_end,
                                target_start + target_match_end,
-                               static_cast<int>(limit_bytes_to_right));
+                               static_cast<ptrdiff_t>(limit_bytes_to_right));
     }
     // Update in/out parameter if the best match found was better
     // than any match already stored in *best_match.
diff --git a/src/blockhash.h b/src/blockhash.h
@@ -79,8 +79,8 @@ class BlockHash {
     Match() : size_(0), source_offset_(-1), target_offset_(-1) { }
 
     void ReplaceIfBetterMatch(size_t candidate_size,
-                              int candidate_source_offset,
-                              int candidate_target_offset) {
+                              size_t candidate_source_offset,
+                              size_t candidate_target_offset) {
       if (candidate_size > size_) {
         size_ = candidate_size;
         source_offset_ = candidate_source_offset;
@@ -89,20 +89,20 @@ class BlockHash {
     }
 
     size_t size() const { return size_; }
-    int source_offset() const { return source_offset_; }
-    int target_offset() const { return target_offset_; }
+    size_t source_offset() const { return source_offset_; }
+    size_t target_offset() const { return target_offset_; }
 
    private:
      // The size of the best (longest) match passed to ReplaceIfBetterMatch().
     size_t size_;
 
     // The source offset of the match, including the starting_offset_
     // of the BlockHash for which the match was found.
-    int source_offset_;
+    size_t source_offset_;
 
     // The target offset of the match.  An offset of 0 corresponds to the
     // data at target_start, which is an argument of FindBestMatch().
-    int target_offset_;
+    size_t target_offset_;
 
     // Making these private avoids implicit copy constructor
     // & assignment operator
@@ -119,7 +119,7 @@ class BlockHash {
   // starting_offset_ will be zero; for a hash of previously encoded
   // target data, starting_offset_ will be equal to the dictionary size.
   //
-  BlockHash(const char* source_data, size_t source_size, int starting_offset);
+  BlockHash(const char* source_data, size_t source_size, size_t starting_offset);
 
   ~BlockHash();
 
@@ -176,7 +176,7 @@ class BlockHash {
   // of whether the block is aligned evenly on a block boundary.  The
   // BlockHash will only store hash entries for the evenly-aligned blocks.
   //
-  void AddOneIndexHash(int index, uint32_t hash_value) {
+  void AddOneIndexHash(size_t index, uint32_t hash_value) {
     if (index == NextIndexToAdd()) {
       AddBlock(hash_value);
     }
@@ -204,7 +204,7 @@ class BlockHash {
   // VCDiffEngine::Encode (in vcdiffengine.cc) uses this function to
   // add a whole range of data to a target hash when a COPY instruction
   // is generated.
-  void AddAllBlocksThroughIndex(int end_index);
+  void AddAllBlocksThroughIndex(size_t end_index);
 
   // FindBestMatch takes a position within the unencoded target data
   // (target_candidate_start) and the hash value of the kBlockSize bytes
@@ -321,12 +321,12 @@ class BlockHash {
   // The expected number of true matches scales super-linearly
   // with the inverse of kBlockSize, but here a linear scale is used
   // for block sizes smaller than 32.
-  static const int kMaxMatchesToCheck = (kBlockSize >= 32) ? 32 :
+  static const size_t kMaxMatchesToCheck = (kBlockSize >= 32) ? 32 :
                                             (32 * (32 / kBlockSize));
 
   // Do not skip more than this number of non-matching hash collisions
   // to find the next matching entry in the hash chain.
-  static const int kMaxProbes = 16;
+  static const size_t kMaxProbes = 16;
 
   // Internal routine which calculates a hash table size based on kBlockSize and
   // the dictionary_size.  Will return a power of two if successful, or 0 if an
@@ -346,11 +346,11 @@ class BlockHash {
 
   // The index within source_data_ of the next block
   // for which AddBlock() should be called.
-  int NextIndexToAdd() const {
+  size_t NextIndexToAdd() const {
     return (last_block_added_ + 1) * kBlockSize;
   }
 
-  static inline bool TooManyMatches(int* match_counter);
+  static inline bool TooManyMatches(size_t* match_counter);
 
   const char* source_data() { return source_data_; }
   size_t source_size() { return source_size_; }
@@ -400,18 +400,18 @@ class BlockHash {
   // so if this function is called before the hash table has been populated
   // using AddAllBlocks() or AddBlock(), it will simply return -1
   // for any value of hash_value.
-  int FirstMatchingBlock(uint32_t hash_value, const char* block_ptr) const;
+  size_t FirstMatchingBlock(uint32_t hash_value, const char* block_ptr) const;
 
   // Given a block number returned by FirstMatchingBlock()
   // or by a previous call to NextMatchingBlock(), returns
   // the next block number that matches the same hash value.
   // Returns -1 if no match was found.
-  int NextMatchingBlock(int block_number, const char* block_ptr) const;
+  size_t NextMatchingBlock(size_t block_number, const char* block_ptr) const;
 
   // Inline version of FirstMatchingBlock.  This saves the cost of a function
   // call when this routine is called from within the module.  The external
   // (non-inlined) version is called only by unit tests.
-  inline int FirstMatchingBlockInline(uint32_t hash_value,
+  inline size_t FirstMatchingBlockInline(uint32_t hash_value,
                                       const char* block_ptr) const;
 
   // Walk through the hash entry chain, skipping over any false matches
@@ -420,23 +420,23 @@ class BlockHash {
   // the first true match found, or -1 if no true match was found.
   // If block_number is a matching block, the function will return block_number
   // without skipping to the next block.
-  int SkipNonMatchingBlocks(int block_number, const char* block_ptr) const;
+  size_t SkipNonMatchingBlocks(size_t block_number, const char* block_ptr) const;
 
   // Returns the number of bytes to the left of source_match_start
   // that match the corresponding bytes to the left of target_match_start.
   // Will not examine more than max_bytes bytes, which is to say that
   // the return value will be in the range [0, max_bytes] inclusive.
-  static int MatchingBytesToLeft(const char* source_match_start,
+  static size_t MatchingBytesToLeft(const char* source_match_start,
                                  const char* target_match_start,
-                                 int max_bytes);
+                                 size_t max_bytes);
 
   // Returns the number of bytes starting at source_match_end
   // that match the corresponding bytes starting at target_match_end.
   // Will not examine more than max_bytes bytes, which is to say that
   // the return value will be in the range [0, max_bytes] inclusive.
-  static int MatchingBytesToRight(const char* source_match_end,
+  static size_t MatchingBytesToRight(const char* source_match_end,
                                   const char* target_match_end,
-                                  int max_bytes);
+                                  size_t max_bytes);
 
   // The protected functions BlockContentsMatch, FirstMatchingBlock,
   // NextMatchingBlock, MatchingBytesToLeft, and MatchingBytesToRight
@@ -455,12 +455,12 @@ class BlockHash {
   // GetHashTableIndex(), or -1 if there is no matching block.  This value can
   // then be used as an index into next_block_table_ to retrieve the entire set
   // of matching block numbers.
-  std::vector<int> hash_table_;
+  std::vector<ptrdiff_t> hash_table_;
 
   // An array containing one element for each source block.  Each element is
   // either -1 (== not found) or the index of the next block whose hash value
   // would produce a matching result from GetHashTableIndex().
-  std::vector<int> next_block_table_;
+  std::vector<ptrdiff_t> next_block_table_;
 
   // This vector has the same size as next_block_table_.  For every block number
   // B that is referenced in hash_table_, last_block_table_[B] will contain
@@ -473,7 +473,7 @@ class BlockHash {
   // lists, so that the match with the lowest index is returned first.  This
   // should result in a more compact encoding because the VCDIFF format favors
   // smaller index values and repeated index values.
-  std::vector<int> last_block_table_;
+  std::vector<ptrdiff_t> last_block_table_;
 
   // Performing a bitwise AND with hash_table_mask_ will produce a value ranging
   // from 0 to the number of elements in hash_table_.
@@ -489,12 +489,12 @@ class BlockHash {
   // For a hash of source (dictionary) data, starting_offset_ will be zero;
   // for a hash of previously encoded target data, starting_offset_ will be
   // equal to the dictionary size.
-  const int starting_offset_;
+  const size_t starting_offset_;
 
   // The last index added by AddBlock().  This determines the block number
   // for successive calls to AddBlock(), and is also
   // used to determine the starting block for AddAllBlocksThroughIndex().
-  int last_block_added_;
+  size_t last_block_added_;
 
   // Making these private avoids implicit copy constructor & assignment operator
   BlockHash(const BlockHash&);  // NOLINT
