Added support for large vectors. Fixes issue #9

Author: tdebatty

src/main/java/info/debatty/java/lsh/LSHMinHash.java

@@ -33,31 +33,31 @@ public class LSHMinHash extends LSH {
 
     /**
      * Instantiates a LSH instance that internally uses MinHash,
-     * with s stages (or bands) and b buckets (per stage), for sets out of a 
+     * with s stages (or bands) and b buckets (per stage), for sets out of a
      * dictionary of n elements.
-     * 
+     *
      * Attention: the number of buckets should be chosen such that we have at
      * least 100 items per bucket.
-     * 
+     *
      * @param s stages
      * @param b buckets (per stage)
      * @param n dictionary size
      */
     public LSHMinHash(int s, int b, int n) {
         super(s, b, n);
-        
+
         /**
          * "Mining of Massive Datasets", p.88.
-         * It can be shown that, using MinHash, the probability that the 
-         * signatures of 2 sets with Jaccard similarity s agree in all the 
-         * rows of at least one stage (band), and therefore become a candidate 
+         * It can be shown that, using MinHash, the probability that the
+         * signatures of 2 sets with Jaccard similarity s agree in all the
+         * rows of at least one stage (band), and therefore become a candidate
          * pair, is 1−(1−s^R)^b
          * where R = signature_size / b (number of rows in a stage/band)
-         * Thus, the curve that shows the probability that 2 items fall in the 
-         * same bucket for at least one of the stages, as a function of their 
+         * Thus, the curve that shows the probability that 2 items fall in the
+         * same bucket for at least one of the stages, as a function of their
          * Jaccard index similarity, has a S shape.
-         * The threshold (the value of similarity at which the probability of 
-         * becoming a candidate is 1/2) is a function of the number of stages 
+         * The threshold (the value of similarity at which the probability of
+         * becoming a candidate is 1/2) is a function of the number of stages
          * (s, or bands b in the book) and the signature size:
          * threshold ≃ (1/s)^(1/R)
          * Hence the signature size can be computed as:
@@ -69,12 +69,12 @@ public LSHMinHash(int s, int b, int n) {
         int signature_size = R * s;
         this.mh = new MinHash(signature_size, n);
     }
-    
+
     public int[] hash(boolean[] vector) {
         return hashSignature(this.mh.signature(vector));
     }
-    
-    public int[][] getCoefficients() {
+
+    public long[][] getCoefficients() {
         return mh.getCoefficients();
     }
 }

src/main/java/info/debatty/java/lsh/MinHash.java

@@ -81,7 +81,7 @@ public static int size(double error) {
     /**
      * Random a and b coefficients for the random hash functions
      */
-    private int[][] hash_coefs;
+    private long[][] hash_coefs;
 
     /**
      * Dictionary size
@@ -154,8 +154,8 @@ public int[] signature(Set<Integer> set) {
 
         for (final int r : list) {
 
-            // However, if c has 1 in row r, then for each i = 1, 2, . . . ,n 
-            // set SIG(i, c) to the smaller of the current value of 
+            // However, if c has 1 in row r, then for each i = 1, 2, . . . ,n
+            // set SIG(i, c) to the smaller of the current value of
             // SIG(i, c) and hi(r)
             for (int i = 0; i < n; i++) {
                 sig[i] = Math.min(
@@ -207,7 +207,7 @@ private void init(int size, int dict_size) {
         // h = (a * x) + b
         // a and b should be randomly generated
         Random r = new Random();
-        hash_coefs = new int[n][2];
+        hash_coefs = new long[n][2];
         for (int i = 0; i < n; i++) {
             hash_coefs[i][0] = r.nextInt(dict_size); // a
             hash_coefs[i][1] = r.nextInt(dict_size); // b
@@ -222,10 +222,11 @@ private void init(int size, int dict_size) {
      * @return the hashed value of x, using ith hash function
      */
     private int h(int i, int x) {
-        return (hash_coefs[i][0] * x + hash_coefs[i][1]) % dict_size;
+        return (int)
+                ((hash_coefs[i][0] * (long) x + hash_coefs[i][1]) % dict_size);
     }
 
-    public int[][] getCoefficients() {
+    public long[][] getCoefficients() {
         return hash_coefs;
     }
 }

src/test/java/info/debatty/java/lsh/LSHMinHashTest.java

@@ -0,0 +1,80 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2016 Thibault Debatty.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+package info.debatty.java.lsh;
+
+import java.util.Random;
+import org.junit.Test;
+
+/**
+ *
+ * @author Thibault Debatty
+ */
+public class LSHMinHashTest {
+
+  /**
+   * Test of hash method, of class LSHMinHash.
+   */
+  @Test
+  public void testHash() {
+    System.out.println("hash");
+
+    // proportion of 0's in the vectors
+    // if the vectors are dense (lots of 1's), the average jaccard similarity
+    // will be very high (especially for large vectors), and LSH
+    // won't be able to distinguish them
+    // as a result, all vectors will be binned in the same bucket...
+    double sparsity = 0.75;
+
+    // Number and size of vectors
+    int count = 10000;
+    int n = 100000;
+
+    int stages = 2;
+    int buckets = 10;
+
+    // Let's generate some random sets
+    boolean[][] vectors = new boolean[count][n];
+    Random rand = new Random();
+
+    for (int i = 0; i < count; i++) {
+      for (int j = 0; j < n; j++) {
+        vectors[i][j] = rand.nextDouble() > sparsity;
+      }
+    }
+
+    LSHMinHash lsh = new LSHMinHash(stages, buckets, n);
+    int[][] counts = new int[stages][buckets];
+
+    // Perform hashing
+    for (boolean[] vector : vectors) {
+      int[] hash = lsh.hash(vector);
+
+      for (int i = 0; i < hash.length; i++) {
+        // this will raise an ArrayIndexOutOfBoundsException
+        // if the bin values are negatives or too large
+        counts[i][hash[i]]++;
+      }
+    }
+  }
+}