Improve naming in PrototypeDecomposer

Author: IlyaMuravjov

cfpq_decomposer/constants.py

@@ -6,3 +6,7 @@
 HASH_FUNCTIONS_COUNT = 3
 MIN_REDUCTION_RATIO = 0.05
 MAX_SIZE_RATIO = 0.3
+
+PROTOTYPE_MIN_VALUES_PER_ROW = 5
+PROTOTYPE_OUTLIER_THRESHOLD = 0.05
+PROTOTYPE_MIN_LSH_BUCKET_SIZE = 5

cfpq_decomposer/prototype_decomposer.py

@@ -1,109 +1,113 @@
-import random
 from collections import defaultdict
+import random
 
 from graphblas.binary import plus
 from graphblas.core.dtypes import BOOL, INT32
 from graphblas.core.matrix import Matrix
 from graphblas.core.vector import Vector
 
 from cfpq_decomposer.abstract_decomposer import AbstractDecomposer
+from cfpq_decomposer.constants import HASH_PRIME_MODULUS, HASH_FUNCTIONS_COUNT, PROTOTYPE_MIN_LSH_BUCKET_SIZE, \
+    PROTOTYPE_OUTLIER_THRESHOLD, PROTOTYPE_MIN_VALUES_PER_ROW
 
 
 class PrototypeDecomposer(AbstractDecomposer):
-    def row_based_decompose(self, M: Matrix):
-        n_rows, n_cols = M.shape
-
-        I, J, V = M.to_coo()
-
-        rows = defaultdict(set)
-        for i, j in zip(I, J):
-            rows[i].add(j)
-
-        p = 2147483647
-        num_hashes = 3
-        hash_funcs = []
-        for _ in range(num_hashes):
-            a = random.randint(1, p - 1)
-            b = random.randint(0, p - 1)
-            hash_funcs.append((a, b))
-
-        minhashes = dict()
-
-        for i, S_i in rows.items():
-            minhash_values = []
-            if len(S_i) < 5:
+    def row_based_decompose(self, input_matrix: Matrix):
+        number_of_rows, number_of_columns = input_matrix.shape
+        row_indices, column_indices, _ = input_matrix.to_coo()
+
+        row_to_column_sets = defaultdict(set)
+        for row_index, column_index in zip(row_indices, column_indices):
+            row_to_column_sets[row_index].add(column_index)
+
+        hash_coefficients_and_offsets = []
+        for _ in range(HASH_FUNCTIONS_COUNT):
+            coefficient = random.randint(1, HASH_PRIME_MODULUS - 1)
+            offset = random.randint(0, HASH_PRIME_MODULUS - 1)
+            hash_coefficients_and_offsets.append((coefficient, offset))
+
+        row_to_minhash_signature = {}
+        for row_index, column_set in row_to_column_sets.items():
+            if len(column_set) < PROTOTYPE_MIN_VALUES_PER_ROW:
                 continue
-            for a, b in hash_funcs:
-                min_hash = min(((a * x + b) % p) for x in S_i)
-                minhash_values.append(min_hash)
-            minhashes[i] = tuple(minhash_values)
-
-        master_hashes = dict()
-        for i, minhash_values in minhashes.items():
-            master_hash = hash(minhash_values)
-            master_hashes[i] = master_hash
-
-        buckets = defaultdict(list)
-        for i, master_hash in master_hashes.items():
-            buckets[master_hash].append(i)
-
-        buckets = {h: idxs for h, idxs in buckets.items() if len(idxs) >= 5}
-
-        LEFT_columns = []
-        RIGHT_rows = []
-
-        for h, B in buckets.items():
-            N = len(B)
-            M_B: Matrix = M[B, :].new()
-            A1 = M_B.dup(dtype=INT32).reduce_columnwise(plus).new()
-
-            threshold = int(0.95 * N)
-            A2: Vector = A1.select('>=', threshold).new()
-
-            if A2.nvals == 0:
+            signature = []
+            for coefficient, offset in hash_coefficients_and_offsets:
+                min_hash = min((coefficient * col + offset) % HASH_PRIME_MODULUS for col in column_set)
+                signature.append(min_hash)
+            row_to_minhash_signature[row_index] = tuple(signature)
+
+        row_to_master_hash = {
+            row_index: hash(signature)
+            for row_index, signature in row_to_minhash_signature.items()
+        }
+
+        master_hash_to_rows = defaultdict(list)
+        for row_index, master_hash in row_to_master_hash.items():
+            master_hash_to_rows[master_hash].append(row_index)
+
+        buckets_with_enough_rows = {
+            master_hash: rows
+            for master_hash, rows in master_hash_to_rows.items()
+            if len(rows) >= PROTOTYPE_MIN_LSH_BUCKET_SIZE
+        }
+
+        left_factor_column_vectors = []
+        right_factor_row_signatures = []
+
+        for master_hash, bucket_row_indices in buckets_with_enough_rows.items():
+            bucket_size = len(bucket_row_indices)
+            bucket_submatrix = input_matrix[bucket_row_indices, :].new()
+            column_sums = bucket_submatrix.dup(dtype=INT32).reduce_columnwise(plus).new()
+
+            first_threshold = int((1 - PROTOTYPE_OUTLIER_THRESHOLD) * bucket_size)
+            frequent_columns_after_first_filter = column_sums.select('>=', first_threshold).new()
+            if frequent_columns_after_first_filter.nvals == 0:
                 continue
 
-            S_A2 = set(A2.to_coo()[0])
-
-            B_prime = [i for i in B if S_A2 <= rows[i]]
-
-            K = len(B_prime)
-            if K == 0:
+            frequent_column_indices = set(frequent_columns_after_first_filter.to_coo()[0])
+            first_filtered_rows = [
+                row_index
+                for row_index in bucket_row_indices
+                if frequent_column_indices <= row_to_column_sets[row_index]
+            ]
+            if not first_filtered_rows:
                 continue
 
-            M_B_prime = M[B_prime, :].new()
-            A3 = M_B_prime.dup(dtype=INT32).reduce_columnwise(plus)
-
-            threshold = int(0.95 * K)
-            A4 = A3.select('>=', threshold).new()
+            filtered_submatrix = input_matrix[first_filtered_rows, :].new()
+            filtered_column_sums = filtered_submatrix.dup(dtype=INT32).reduce_columnwise(plus)
 
-            if A4.nvals == 0:
+            second_threshold = int((1 - PROTOTYPE_OUTLIER_THRESHOLD) * len(first_filtered_rows))
+            frequent_columns_after_second_filter = filtered_column_sums.select('>=', second_threshold).new()
+            if frequent_columns_after_second_filter.nvals == 0:
                 continue
 
-            S_A4 = set(A4.to_coo()[0])
-
-            B_double_prime = [i for i in B_prime if S_A4 <= rows[i]]
-
-            if len(B_double_prime) < 5:
+            frequent_filtered_column_indices = set(frequent_columns_after_second_filter.to_coo()[0])
+            second_filtered_rows = [
+                row_index
+                for row_index in first_filtered_rows
+                if frequent_filtered_column_indices <= row_to_column_sets[row_index]
+            ]
+            if len(second_filtered_rows) < PROTOTYPE_MIN_LSH_BUCKET_SIZE:
                 continue
 
-            RIGHT_rows.append(A4)
+            right_factor_row_signatures.append(frequent_columns_after_second_filter)
 
-            CORE = Vector(BOOL, size=n_rows)
-            for i in B_double_prime:
-                CORE[i] = True
-            LEFT_columns.append(CORE)
+            core_membership_vector = Vector(BOOL, size=number_of_rows)
+            for core_row in second_filtered_rows:
+                core_membership_vector[core_row] = True
+            left_factor_column_vectors.append(core_membership_vector)
 
-        num_buckets_remaining = len(LEFT_columns)
-        if num_buckets_remaining == 0:
-            return Matrix(M.dtype, M.nrows, 0), Matrix(M.dtype, 0, M.ncols)
+        bucket_count = len(left_factor_column_vectors)
+        if bucket_count == 0:
+            return Matrix(input_matrix.dtype, number_of_rows, 0), \
+                   Matrix(input_matrix.dtype, 0, number_of_columns)
 
-        LEFT = Matrix(bool, n_rows, num_buckets_remaining)
-        for idx, CORE in enumerate(LEFT_columns):
-            LEFT[:, idx] = CORE
+        left_factor = Matrix(bool, number_of_rows, bucket_count)
+        for idx, column_vector in enumerate(left_factor_column_vectors):
+            left_factor[:, idx] = column_vector
 
-        RIGHT = Matrix(bool, num_buckets_remaining, n_cols)
-        for idx, A4 in enumerate(RIGHT_rows):
-            RIGHT[idx, :] = A4
+        right_factor = Matrix(bool, bucket_count, number_of_columns)
+        for idx, row_signature in enumerate(right_factor_row_signatures):
+            right_factor[idx, :] = row_signature
 
-        return LEFT, RIGHT
+        return left_factor, right_factor