Skip to content

Commit 16bab2c

Browse files
jmalkinjmalkin
committed
skip updating kxq in HLL merge
1 parent 4778e3a commit 16bab2c

9 files changed

Lines changed: 111 additions & 51 deletions

‎hll/include/Hll4Array-internal.hpp‎

Lines changed: 16 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,22 @@ Hll4Array<A>::Hll4Array(const Hll4Array<A>& that) :
5151
}
5252
}
5353

54+
template<typename A>
55+
Hll4Array<A>::Hll4Array(const HllArray<A>& other) :
56+
HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_4, other.isStartFullSize(), other.getAllocator()),
57+
auxHashMap_(nullptr)
58+
{
59+
const int numBytes = this->hll4ArrBytes(this->lgConfigK_);
60+
this->hllByteArr_.resize(numBytes, 0);
61+
this->oooFlag_ = other.isOutOfOrderFlag();
62+
63+
for (const auto& coupon : other) { // all = false, so skip empty values
64+
internalCouponUpdate(coupon); // updates KxQ registers
65+
}
66+
this->hipAccum_ = other.getHipAccum();
67+
this->rebuild_kxq_curmin_ = false;
68+
}
69+
5470
template<typename A>
5571
Hll4Array<A>::~Hll4Array() {
5672
// hllByteArr deleted in parent
@@ -327,13 +343,6 @@ typename HllArray<A>::const_iterator Hll4Array<A>::end() const {
327343
this->tgtHllType_, auxHashMap_, this->curMin_, false);
328344
}
329345

330-
template<typename A>
331-
void Hll4Array<A>::mergeHll(const HllArray<A>& src) {
332-
for (const auto coupon: src) {
333-
internalCouponUpdate(coupon);
334-
}
335-
}
336-
337346
}
338347

339348
#endif // _HLL4ARRAY_INTERNAL_HPP_

‎hll/include/Hll4Array.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ class Hll4Array final : public HllArray<A> {
3030
public:
3131
explicit Hll4Array(uint8_t lgConfigK, bool startFullSize, const A& allocator);
3232
explicit Hll4Array(const Hll4Array<A>& that);
33+
explicit Hll4Array(const HllArray<A>& that);
3334

3435
virtual ~Hll4Array();
3536
virtual std::function<void(HllSketchImpl<A>*)> get_deleter() const;
@@ -44,7 +45,6 @@ class Hll4Array final : public HllArray<A> {
4445
virtual uint32_t getHllByteArrBytes() const;
4546

4647
virtual HllSketchImpl<A>* couponUpdate(uint32_t coupon) final;
47-
void mergeHll(const HllArray<A>& src);
4848

4949
virtual AuxHashMap<A>* getAuxHashMap() const;
5050
// does *not* delete old map if overwriting

‎hll/include/Hll6Array-internal.hpp‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,25 @@ HllArray<A>(lgConfigK, target_hll_type::HLL_6, startFullSize, allocator)
3434
this->hllByteArr_.resize(numBytes, 0);
3535
}
3636

37+
template<typename A>
38+
Hll6Array<A>::Hll6Array(const HllArray<A>& other) :
39+
HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_6, other.isStartFullSize(), other.getAllocator())
40+
{
41+
const int numBytes = this->hll6ArrBytes(this->lgConfigK_);
42+
this->hllByteArr_.resize(numBytes, 0);
43+
this->oooFlag_ = other.isOutOfOrderFlag();
44+
uint32_t num_zeros = 1 << this->lgConfigK_;
45+
46+
for (const auto& coupon : other) { // all = false, so skip empty values
47+
num_zeros--;
48+
internalCouponUpdate(coupon); // updates KxQ registers
49+
}
50+
51+
this->numAtCurMin_ = num_zeros;
52+
this->hipAccum_ = other.getHipAccum();
53+
this->rebuild_kxq_curmin_ = false;
54+
}
55+
3756
template<typename A>
3857
std::function<void(HllSketchImpl<A>*)> Hll6Array<A>::get_deleter() const {
3958
return [](HllSketchImpl<A>* ptr) {
@@ -101,13 +120,6 @@ void Hll6Array<A>::internalCouponUpdate(uint32_t coupon) {
101120
}
102121
}
103122

104-
template<typename A>
105-
void Hll6Array<A>::mergeHll(const HllArray<A>& src) {
106-
for (const auto coupon: src) {
107-
internalCouponUpdate(coupon);
108-
}
109-
}
110-
111123
}
112124

113125
#endif // _HLL6ARRAY_INTERNAL_HPP_

‎hll/include/Hll6Array.hpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ template<typename A>
3131
class Hll6Array final : public HllArray<A> {
3232
public:
3333
Hll6Array(uint8_t lgConfigK, bool startFullSize, const A& allocator);
34+
explicit Hll6Array(const HllArray<A>& that);
3435

3536
virtual ~Hll6Array() = default;
3637
virtual std::function<void(HllSketchImpl<A>*)> get_deleter() const;
@@ -41,7 +42,6 @@ class Hll6Array final : public HllArray<A> {
4142
inline void putSlot(uint32_t slotNo, uint8_t value);
4243

4344
virtual HllSketchImpl<A>* couponUpdate(uint32_t coupon) final;
44-
void mergeHll(const HllArray<A>& src);
4545

4646
virtual uint32_t getHllByteArrBytes() const;
4747

‎hll/include/Hll8Array-internal.hpp‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,25 @@ HllArray<A>(lgConfigK, target_hll_type::HLL_8, startFullSize, allocator)
3232
this->hllByteArr_.resize(numBytes, 0);
3333
}
3434

35+
template<typename A>
36+
Hll8Array<A>::Hll8Array(const HllArray<A>& other):
37+
HllArray<A>(other.getLgConfigK(), target_hll_type::HLL_8, other.isStartFullSize(), other.getAllocator())
38+
{
39+
const int numBytes = this->hll8ArrBytes(this->lgConfigK_);
40+
this->hllByteArr_.resize(numBytes, 0);
41+
this->oooFlag_ = other.isOutOfOrderFlag();
42+
uint32_t num_zeros = 1 << this->lgConfigK_;
43+
44+
for (const auto& coupon : other) { // all = false, so skip empty values
45+
num_zeros--;
46+
internalCouponUpdate(coupon); // updates KxQ registers
47+
}
48+
49+
this->numAtCurMin_ = num_zeros;
50+
this->hipAccum_ = other.getHipAccum();
51+
this->rebuild_kxq_curmin_ = false;
52+
}
53+
3554
template<typename A>
3655
std::function<void(HllSketchImpl<A>*)> Hll8Array<A>::get_deleter() const {
3756
return [](HllSketchImpl<A>* ptr) {
@@ -128,16 +147,14 @@ void Hll8Array<A>::mergeHll(const HllArray<A>& src) {
128147
++i;
129148
}
130149
}
150+
this->setRebuildKxqCurminFlag(true);
131151
}
132152

153+
133154
template<typename A>
134155
void Hll8Array<A>::processValue(uint32_t slot, uint32_t mask, uint8_t new_val) {
135-
const uint8_t old_val = this->hllByteArr_[slot & mask];
136-
if (new_val > old_val) {
137-
this->hllByteArr_[slot & mask] = new_val;
138-
this->hipAndKxQIncrementalUpdate(old_val, new_val);
139-
this->numAtCurMin_ -= old_val == 0;
140-
}
156+
const size_t index = slot & mask;
157+
this->hllByteArr_[index] = std::max(this->hllByteArr_[index], new_val);
141158
}
142159

143160
}

‎hll/include/Hll8Array.hpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ template<typename A>
3131
class Hll8Array final : public HllArray<A> {
3232
public:
3333
Hll8Array(uint8_t lgConfigK, bool startFullSize, const A& allocator);
34+
explicit Hll8Array(const HllArray<A>& that);
3435

3536
virtual ~Hll8Array() = default;
3637
virtual std::function<void(HllSketchImpl<A>*)> get_deleter() const;

‎hll/include/HllArray-internal.hpp‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -43,20 +43,42 @@ kxq1_(0.0),
4343
hllByteArr_(allocator),
4444
curMin_(0),
4545
numAtCurMin_(1 << lgConfigK),
46-
oooFlag_(false)
46+
oooFlag_(false),
47+
rebuild_kxq_curmin_(false)
48+
{}
49+
50+
template<typename A>
51+
HllArray<A>::HllArray(const HllArray& other, target_hll_type tgtHllType) :
52+
HllSketchImpl<A>(other.getLgConfigK(), tgtHllType, hll_mode::HLL, other.isStartFullSize()),
53+
// remaining fields are initialized to empty sketch defaults
54+
// and left to subclass constructor to populate
55+
hipAccum_(0.0),
56+
kxq0_(1 << other.getLgConfigK()),
57+
kxq1_(0.0),
58+
hllByteArr_(other.getAllocator()),
59+
curMin_(0),
60+
numAtCurMin_(1 << other.getLgConfigK()),
61+
oooFlag_(false),
62+
rebuild_kxq_curmin_(false)
4763
{}
4864

4965
template<typename A>
5066
HllArray<A>* HllArray<A>::copyAs(target_hll_type tgtHllType) const {
51-
if (tgtHllType == this->getTgtHllType()) {
67+
// we may need to recompute KxQ and curMin data for a union gadget,
68+
// so only use a direct copy if we have a valid sketch
69+
if (tgtHllType == this->getTgtHllType() && !this->isRebuildKxqCurminFlag()) {
5270
return static_cast<HllArray*>(copy());
5371
}
54-
if (tgtHllType == target_hll_type::HLL_4) {
55-
return HllSketchImplFactory<A>::convertToHll4(*this);
56-
} else if (tgtHllType == target_hll_type::HLL_6) {
57-
return HllSketchImplFactory<A>::convertToHll6(*this);
58-
} else { // tgtHllType == HLL_8
59-
return HllSketchImplFactory<A>::convertToHll8(*this);
72+
73+
// the factory methods replay the coupons and will always rebuild
74+
// the sketch in a consistent way
75+
switch (tgtHllType) {
76+
case target_hll_type::HLL_4:
77+
return HllSketchImplFactory<A>::convertToHll4(*this);
78+
case target_hll_type::HLL_6:
79+
return HllSketchImplFactory<A>::convertToHll6(*this);
80+
case target_hll_type::HLL_8:
81+
return HllSketchImplFactory<A>::convertToHll8(*this);
6082
}
6183
}
6284

@@ -562,6 +584,16 @@ double HllArray<A>::getHllRawEstimate() const {
562584
return hyperEst;
563585
}
564586

587+
template<typename A>
588+
void HllArray<A>::setRebuildKxqCurminFlag(bool rebuild) {
589+
rebuild_kxq_curmin_ = rebuild;
590+
}
591+
592+
template<typename A>
593+
bool HllArray<A>::isRebuildKxqCurminFlag() const {
594+
return rebuild_kxq_curmin_;
595+
}
596+
565597
template<typename A>
566598
typename HllArray<A>::const_iterator HllArray<A>::begin(bool all) const {
567599
return const_iterator(hllByteArr_.data(), 1 << this->lgConfigK_, 0, this->tgtHllType_, nullptr, 0, all);
@@ -604,6 +636,8 @@ auto HllArray<A>::const_iterator::operator*() const -> reference {
604636

605637
template<typename A>
606638
uint8_t HllArray<A>::const_iterator::get_value(const uint8_t* array, uint32_t index, target_hll_type hll_type, const AuxHashMap<A>* exceptions, uint8_t offset) {
639+
// TODO: we should be able to improve efficiency here by reading multiple bytes at a time
640+
// for HLL4 and HLL6
607641
if (hll_type == target_hll_type::HLL_4) {
608642
uint8_t value = array[index >> 1];
609643
if ((index & 1) > 0) { // odd

‎hll/include/HllArray.hpp‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ template<typename A>
3232
class HllArray : public HllSketchImpl<A> {
3333
public:
3434
HllArray(uint8_t lgConfigK, target_hll_type tgtHllType, bool startFullSize, const A& allocator);
35+
explicit HllArray(const HllArray& other, target_hll_type tgtHllType);
3536

3637
static HllArray* newHll(const void* bytes, size_t len, const A& allocator);
3738
static HllArray* newHll(std::istream& is, const A& allocator);
@@ -86,6 +87,9 @@ class HllArray : public HllSketchImpl<A> {
8687

8788
virtual AuxHashMap<A>* getAuxHashMap() const;
8889

90+
virtual void setRebuildKxqCurminFlag(bool rebuild);
91+
virtual bool isRebuildKxqCurminFlag() const;
92+
8993
class const_iterator;
9094
virtual const_iterator begin(bool all = false) const;
9195
virtual const_iterator end() const;
@@ -106,6 +110,7 @@ class HllArray : public HllSketchImpl<A> {
106110
uint8_t curMin_; //always zero for Hll6 and Hll8, only tracked by Hll4Array
107111
uint32_t numAtCurMin_; //interpreted as num zeros when curMin == 0
108112
bool oooFlag_; //Out-Of-Order Flag
113+
bool rebuild_kxq_curmin_; // flag to recompute
109114

110115
friend class HllSketchImplFactory<A>;
111116
};

‎hll/include/HllSketchImplFactory.hpp‎

Lines changed: 3 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -136,38 +136,20 @@ HllSketchImpl<A>* HllSketchImplFactory<A>::reset(HllSketchImpl<A>* impl, bool st
136136

137137
template<typename A>
138138
Hll4Array<A>* HllSketchImplFactory<A>::convertToHll4(const HllArray<A>& srcHllArr) {
139-
const uint8_t lgConfigK = srcHllArr.getLgConfigK();
140139
using Hll4Alloc = typename std::allocator_traits<A>::template rebind_alloc<Hll4Array<A>>;
141-
Hll4Array<A>* hll4Array = new (Hll4Alloc(srcHllArr.getAllocator()).allocate(1))
142-
Hll4Array<A>(lgConfigK, srcHllArr.isStartFullSize(), srcHllArr.getAllocator());
143-
hll4Array->putOutOfOrderFlag(srcHllArr.isOutOfOrderFlag());
144-
hll4Array->mergeHll(srcHllArr);
145-
hll4Array->putHipAccum(srcHllArr.getHipAccum());
146-
return hll4Array;
140+
return new (Hll4Alloc(srcHllArr.getAllocator()).allocate(1)) Hll4Array<A>(srcHllArr);
147141
}
148142

149143
template<typename A>
150144
Hll6Array<A>* HllSketchImplFactory<A>::convertToHll6(const HllArray<A>& srcHllArr) {
151-
const uint8_t lgConfigK = srcHllArr.getLgConfigK();
152145
using Hll6Alloc = typename std::allocator_traits<A>::template rebind_alloc<Hll6Array<A>>;
153-
Hll6Array<A>* hll6Array = new (Hll6Alloc(srcHllArr.getAllocator()).allocate(1))
154-
Hll6Array<A>(lgConfigK, srcHllArr.isStartFullSize(), srcHllArr.getAllocator());
155-
hll6Array->putOutOfOrderFlag(srcHllArr.isOutOfOrderFlag());
156-
hll6Array->mergeHll(srcHllArr);
157-
hll6Array->putHipAccum(srcHllArr.getHipAccum());
158-
return hll6Array;
146+
return new (Hll6Alloc(srcHllArr.getAllocator()).allocate(1)) Hll6Array<A>(srcHllArr);
159147
}
160148

161149
template<typename A>
162150
Hll8Array<A>* HllSketchImplFactory<A>::convertToHll8(const HllArray<A>& srcHllArr) {
163-
const uint8_t lgConfigK = srcHllArr.getLgConfigK();
164151
using Hll8Alloc = typename std::allocator_traits<A>::template rebind_alloc<Hll8Array<A>>;
165-
Hll8Array<A>* hll8Array = new (Hll8Alloc(srcHllArr.getAllocator()).allocate(1))
166-
Hll8Array<A>(lgConfigK, srcHllArr.isStartFullSize(), srcHllArr.getAllocator());
167-
hll8Array->putOutOfOrderFlag(srcHllArr.isOutOfOrderFlag());
168-
hll8Array->mergeHll(srcHllArr);
169-
hll8Array->putHipAccum(srcHllArr.getHipAccum());
170-
return hll8Array;
152+
return new (Hll8Alloc(srcHllArr.getAllocator()).allocate(1)) Hll8Array<A>(srcHllArr);
171153
}
172154

173155
}

0 commit comments

Comments
 (0)