Compute serialized size bytes

Author: geonove

ddsketch/include/collapsing_dense_store.hpp

@@ -36,6 +36,7 @@ class CollapsingDenseStore : public DenseStore<Derived, Allocator> {
 
   void serialize(std::ostream& os) const;
   static Derived deserialize(std::istream& is);
+  int get_serialized_size_bytes() const;
 
   ~CollapsingDenseStore() = default;
 

ddsketch/include/collapsing_dense_store_impl.hpp

@@ -24,7 +24,10 @@
 
 namespace datasketches {
 template<class Derived, typename Allocator>
-CollapsingDenseStore<Derived, Allocator>::CollapsingDenseStore(size_type max_num_bins): DenseStore<Derived, Allocator>(), max_num_bins(max_num_bins), is_collapsed(false) {}
+CollapsingDenseStore<Derived, Allocator>::CollapsingDenseStore(size_type max_num_bins):
+  DenseStore<Derived, Allocator>(),
+  max_num_bins(max_num_bins),
+  is_collapsed(false) {}
 
 template<class Derived, typename Allocator>
 CollapsingDenseStore<Derived, Allocator>& CollapsingDenseStore<Derived, Allocator>::operator=(const CollapsingDenseStore<Derived, Allocator>& other) {
@@ -76,5 +79,26 @@ Derived CollapsingDenseStore<Derived, Allocator>::deserialize(std::istream& is)
   return store;
 }
 
+template<class Derived, typename Allocator>
+int CollapsingDenseStore<Derived, Allocator>::get_serialized_size_bytes() const {
+  int size_bytes = 0;
+  size_bytes += sizeof(max_num_bins);
+  size_bytes += sizeof(is_collapsed);
+  size_bytes += sizeof(this->min_index);
+  size_bytes += sizeof(this->max_index);
+  size_bytes += sizeof(size_type);
+
+  size_type non_empty_bins = 0;
+  for (const double& count : this->bins) {
+    non_empty_bins += (count > 0.0);
+  }
+
+  size_bytes += sizeof(non_empty_bins);
+  size_bytes += non_empty_bins * sizeof(size_type);
+  size_bytes += non_empty_bins * sizeof(double);
+
+  return size_bytes;
+}
+
 }
 #endif //COLLAPSING_DENSE_STORE_IMPL_HPP

ddsketch/include/ddsketch.hpp

@@ -65,7 +65,7 @@ class DDSketch {
 
   void serialize(std::ostream& os) const;
   static DDSketch deserialize(std::istream& is);
-
+  int get_serialized_size_bytes() const;
 protected:
   Store positive_store;
   Store negative_store;

ddsketch/include/sparse_store.hpp

@@ -61,6 +61,7 @@ class SparseStore {
 
   void serialize(std::ostream& os) const;
   static SparseStore deserialize(std::istream& is);
+  int get_serialized_size_bytes() const;
 
   iterator begin() const;
   iterator end() const;

ddsketch/include/sparse_store_impl.hpp

@@ -219,6 +219,17 @@ SparseStore<Allocator> SparseStore<Allocator>::deserialize(std::istream& is) {
   return store;
 }
 
+template<typename Allocator>
+int SparseStore<Allocator>::get_serialized_size_bytes() const {
+  int size_bytes = 0;
+  size_bytes += sizeof(typename SparseStore<Allocator>::bins_type::size_type);
+  size_bytes += bins.size() * sizeof(typename SparseStore<Allocator>::bins_type::key_type);
+  size_bytes += bins.size() * sizeof(typename SparseStore<Allocator>::bins_type::mapped_type);
+
+  return size_bytes;
+}
+
+
 }
 
 #endif //SPARSE_STORE_IMPL_HPP

ddsketch/include/unbounded_size_dense_store.hpp

@@ -43,7 +43,7 @@ class UnboundedSizeDenseStore: public DenseStore<UnboundedSizeDenseStore<Allocat
 
   void serialize(std::ostream& os) const;
   static UnboundedSizeDenseStore deserialize(std::istream& is);
-
+  int get_serialized_size_bytes() const;
 protected:
   size_type normalize(size_type index);
   void adjust(size_type new_min_index, size_type new_max_index);

ddsketch/include/unbounded_size_dense_store_impl.hpp

@@ -95,6 +95,25 @@ UnboundedSizeDenseStore<Allocator> UnboundedSizeDenseStore<Allocator>::deseriali
   return store;
 }
 
+template<typename Allocator>
+int UnboundedSizeDenseStore<Allocator>::get_serialized_size_bytes() const {
+  int size_bytes = 0;
+  size_bytes += sizeof(this->min_index);
+  size_bytes += sizeof(this->max_index);
+  size_bytes += sizeof(size_type);
+
+  size_type non_empty_bins = 0;
+  for (const double& count : this->bins) {
+    non_empty_bins += (count > 0.0);
+  }
+
+  size_bytes += sizeof(non_empty_bins);
+  size_bytes += non_empty_bins * sizeof(size_type);
+  size_bytes += non_empty_bins * sizeof(double);
+
+  return size_bytes;
+}
+
 
 
 

ddsketch/test/StoreTest.cpp

@@ -524,27 +524,6 @@ TEMPLATE_TEST_CASE("test cross merge", "[storetest]",
   }
 }
 
-TEST_CASE("merge test", "[mergetest]") {
-  CollapsingHighestDenseStore<std::allocator<uint8_t>> s1(1024);
-  CollapsingHighestDenseStore<std::allocator<uint8_t>> s2(1024);
-  CollapsingLowestDenseStore<std::allocator<uint8_t>> s3(1024);
-
-  SparseStore<std::allocator<uint8_t>> ss;
-  s1.merge(s2);
-  s1.merge(s3);
-
-  ss.merge(s1);
-  s1.merge(ss);
-
-  LinearlyInterpolatedMapping mapping(0.01);
-  DDSketch sketch(s1, s2, mapping, 1., 1.);
-
-  sketch.update(10.);
-  // DDSketch<SparseStore<std::allocator<uint8_t>>, LinearlyInterpolatedMapping> sketch2(ss, ss, mapping);
-
-  //DDSketch<CollapsingHighestDenseStore<A>, LinearlyInterpolatedMapping> sketch4<1024>(0.99);
-}
-
 TEMPLATE_TEST_CASE("dense store serialization test", "[serialization]",
   CollapsingLowestStoreTestCase<8>,
   CollapsingLowestStoreTestCase<128>,
@@ -582,6 +561,7 @@ TEMPLATE_TEST_CASE("dense store serialization test", "[serialization]",
   REQUIRE_FALSE(deserialized_store.is_empty());
   REQUIRE(stream.peek() == std::istream::traits_type::eof());
   REQUIRE(store == deserialized_store);
+
 }
 }