Fix API contract and compatibility bugs.

Validate window, sync docs, dedup cardinality estimator.

Author: svpcom

README.rst

@@ -9,26 +9,26 @@ Use ``pip install hyperloglog`` to install from PyPI.
 Usage:
 =====
 
-::
+.. code-block:: python
 
     import hyperloglog
     hll = hyperloglog.HyperLogLog(0.01)  # accept 1% counting error
     hll.add("hello")
-    print len(hll)  # 1
+    print(len(hll))  # 1
     hll.add("hello")
-    print len(hll)  # 1 as items aren't added more than once
+    print(len(hll))  # 1 as items aren't added more than once
     hll.add("hello again")
-    print len(hll)  # 2
+    print(len(hll))  # 2
 
 If we add a further 1000 random strings (giving a total of 1002 strings) we'll have a count roughly within 1% of the true value, in this case it counts 1007 (within +/- 10.2 of the true value)
 
-::
+.. code-block:: python
 
     # add 1000 random 30 char strings to hll
     import random
     import string
     [hll.add("".join([string.ascii_letters[random.randint(0, len(string.ascii_letters)-1)] for n in range(30)])) for m in range(1000)]  
-    print len(hll)  # 1007
+    print(len(hll))  # 1007
 
 
 Changes:

hyperloglog/hll.py

@@ -81,12 +81,28 @@ def bit_length_vec(arr):
 
 
 def get_rho_vec(w, max_width):
-    rho = max_width - bit_length_vec(w) + 1
+    # compare before subtracting: bit_length_vec may return an unsigned
+    # dtype (NumPy 2.x), where a negative rho would silently wrap around
+    bits = bit_length_vec(w)
 
-    if np.count_nonzero(rho <= 0):
+    if np.count_nonzero(bits > max_width):
         raise ValueError('w overflow')
 
-    return rho
+    return max_width - bits + 1
+
+
+def get_estimate(M, m, p, alpha):
+    # small-range linear counting below the empirical threshold, otherwise
+    # the HLL raw estimate (bias-corrected while E <= 5m)
+    V = np.count_nonzero(M == 0)
+
+    if V > 0:
+        H = m * math.log(m / V)
+        if H <= get_treshold(p):
+            return H
+
+    E = alpha * (m ** 2) / np.power(2.0, -M, dtype=float).sum()
+    return (E - estimate_bias(E, p)) if E <= 5 * m else E
 
 
 class HyperLogLog(object):
@@ -124,6 +140,10 @@ def __setstate__(self, d):
         for key in d:
             setattr(self, key, d[key])
 
+        # normalize M from pickles created by pre-NumPy versions (plain list)
+        # or under the other NumPy major version (different counter dtype)
+        self.M = np.asarray(self.M, HLL_COUNTER_TYPE)
+
     def add(self, value):
         """
         Adds the item to the HyperLogLog
@@ -152,7 +172,8 @@ def add_bulk(self, values):
         # w = <x_{p}x_{p+1}..>
         # M[j] = max(M[j], rho(w))
 
-        assert not isinstance(values, (bytes, str)) and hasattr(values, '__iter__')
+        if isinstance(values, (bytes, str)) or not hasattr(values, '__iter__'):
+            raise TypeError('values must be a non-string iterable')
 
         x = np.fromiter((int.from_bytes(sha1(packb(value)).digest()[:8], byteorder='big') for value in values), np.uint64)
         j = x & (self.m - 1)
@@ -180,32 +201,17 @@ def update(self, *others):
         self.M = np.maximum.reduce(ml)
 
     def __eq__(self, other):
-        if self.m != other.m:
-            raise ValueError('Counters precisions should be equal')
-
-        return np.array_equal(self.M, other.M)
+        if not isinstance(other, HyperLogLog):
+            return NotImplemented
 
-    def __ne__(self, other):
-        return not self.__eq__(other)
+        return self.m == other.m and np.array_equal(self.M, other.M)
 
     def __len__(self):
         return round(self.card())
 
-    def _Ep(self):
-        E = self.alpha * (self.m ** 2) / np.power(2.0, -self.M, dtype=float).sum()
-        return (E - estimate_bias(E, self.p)) if E <= 5 * self.m else E
-
     def card(self):
         """
         Returns the estimate of the cardinality
         """
-
-        #count number of registers equal to 0
-        V = np.count_nonzero(self.M == 0)
-
-        if V > 0:
-            H = self.m * math.log(self.m / V)
-            return H if H <= get_treshold(self.p) else self._Ep()
-        else:
-            return self._Ep()
+        return get_estimate(self.M, self.m, self.p, self.alpha)
 

hyperloglog/shll.py

@@ -8,7 +8,7 @@
 
 from hashlib import sha1
 from msgpack import packb
-from .hll import get_treshold, estimate_bias, get_alpha, get_rho
+from .hll import get_alpha, get_rho, get_estimate
 
 
 class SlidingHyperLogLog(object):
@@ -27,14 +27,17 @@ def __init__(self, error_rate, window, lpfm=None):
 
         self.window = window
 
+        if not (window > 0):
+            raise ValueError('window must be > 0')
+
         if lpfm is not None:
             m = len(lpfm)
-            p = round(math.log(m, 2))
 
-            if (1 << p) != m:
+            if m == 0 or (m & (m - 1)) != 0:
                 raise ValueError('List length is not power of 2')
 
-            self.LPFM = lpfm
+            p = m.bit_length() - 1
+            self.LPFM = list(lpfm)
 
         else:
             if not (0 < error_rate < 1):
@@ -58,6 +61,10 @@ def __setstate__(self, d):
         for key in d:
             setattr(self, key, d[key])
 
+        # normalize LPFM from pickles created by older versions, which used
+        # None for empty registers and ascending timestamp order
+        self.LPFM = [tuple(sorted(reg, reverse=True)) if reg else tuple() for reg in self.LPFM]
+
     @classmethod
     def from_list(cls, lpfm, window):
         return cls(None, window, lpfm)
@@ -103,6 +110,9 @@ def update(self, *others):
             if self.m != item.m:
                 raise ValueError('Counters precisions should be equal')
 
+            if self.window != item.window:
+                raise ValueError('Counters windows should be equal')
+
         for j, lpfms_j in enumerate(zip(self.LPFM, *list(item.LPFM for item in others))):
             Rmax = None
             tmp = []
@@ -122,20 +132,13 @@ def update(self, *others):
             self.LPFM[j] = tuple(tmp)
 
     def __eq__(self, other):
-        if self.m != other.m:
-            raise ValueError('Counters precisions should be equal')
-
-        return self.LPFM == other.LPFM
+        if not isinstance(other, SlidingHyperLogLog):
+            return NotImplemented
 
-    def __ne__(self, other):
-        return not self.__eq__(other)
+        return self.m == other.m and self.window == other.window and self.LPFM == other.LPFM
 
     def __len__(self):
-        raise NotImplemented
-
-    def _Ep(self, M):
-        E = self.alpha * (self.m ** 2) / np.power(2.0, -M, dtype=float).sum()
-        return (E - estimate_bias(E, self.p)) if E <= 5 * self.m else E
+        raise NotImplementedError('use card(timestamp) to estimate cardinality')
 
     def card(self, timestamp, window=None):
         """
@@ -150,14 +153,7 @@ def card(self, timestamp, window=None):
         _t = timestamp - window
         M = np.fromiter((np.max(np.fromiter((R for ts, R in lpfm if ts >= _t), int), initial=0) if lpfm else 0 for lpfm in self.LPFM), int)
 
-        #count number or registers equal to 0
-        V = np.count_nonzero(M == 0)
-
-        if V > 0:
-            H = self.m * math.log(self.m / V)
-            return H if H <= get_treshold(self.p) else self._Ep(M)
-        else:
-            return self._Ep(M)
+        return get_estimate(M, self.m, self.p, self.alpha)
 
     def card_wlist(self, timestamp, window_list):
         """
@@ -188,13 +184,5 @@ def card_wlist(self, timestamp, window_list):
 
         res = []
         for M in M_list:
-            #count number of registers equal to 0
-            V = M.count(0)
-            M = np.array(M, int)
-
-            if V > 0:
-                H = self.m * math.log(self.m / V)
-                res.append(H if H <= get_treshold(self.p) else self._Ep(M))
-            else:
-                res.append(self._Ep(M))
+            res.append(get_estimate(np.array(M, int), self.m, self.p, self.alpha))
         return res

setup.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python
 
-from distutils.core import setup
+from setuptools import setup
 
-version = '0.1.5'
+version = '0.1.6'
 
 setup(
     name='hyperloglog',