Removed neighbor.attributes, which was wasting a lot of space on large graphs

tdebatty · tdebatty · commit d59a0a9505da · 2017-03-17T10:05:03.000+01:00
diff --git a/src/main/java/info/debatty/java/graphs/Edge.java b/src/main/java/info/debatty/java/graphs/Edge.java
@@ -1,31 +1,78 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2016 Thibault Debatty.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
 package info.debatty.java.graphs;
 
 /**
- * Represent a weighted edge (a link from node n1 to node n2)
+ * Represent a weighted edge (a link from node n1 to node n2).
+ * Internally, the edge is a source node plus a neighbor.
  *
  * @author Thibault Debatty
  */
 public class Edge<T> {
 
     public T n1;
-    public T n2;
-    public double weight = 0;
-
-    public static final String SEPARATOR = ";";
+    public Neighbor<T> neighbor;
 
     public Edge() {
 
     }
 
-    public Edge(T n1, T n2, double weight) {
+    public Edge(T n1, Neighbor<T> neighbor) {
         this.n1 = n1;
-        this.n2 = n2;
-        this.weight = weight;
+        this.neighbor = neighbor;
     }
 
     @Override
     public String toString() {
-        return n1.toString() + SEPARATOR + n2.toString()+ SEPARATOR + weight;
+        return n1.toString() + " => " + neighbor.toString();
+    }
 
+    @Override
+    public int hashCode() {
+        int hash = 3;
+        hash = 83 * hash + (this.n1 != null ? this.n1.hashCode() : 0);
+        hash = 83 * hash + (this.neighbor != null ? this.neighbor.hashCode() : 0);
+        return hash;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        final Edge<?> other = (Edge<?>) obj;
+        if (this.n1 != other.n1 && (this.n1 == null || !this.n1.equals(other.n1))) {
+            return false;
+        }
+        if (this.neighbor != other.neighbor && (this.neighbor == null || !this.neighbor.equals(other.neighbor))) {
+            return false;
+        }
+        return true;
     }
+
+
 }
diff --git a/src/main/java/info/debatty/java/graphs/Neighbor.java b/src/main/java/info/debatty/java/graphs/Neighbor.java
@@ -1,8 +1,30 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2016 Thibault Debatty.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
 package info.debatty.java.graphs;
 
 import java.io.Serializable;
 import java.security.InvalidParameterException;
-import java.util.HashMap;
 
 /**
  * Neighbor of an edge (stores the other node, and the similarity)
@@ -15,45 +37,27 @@ public class Neighbor<T>
     public T node;
     public double similarity;
 
-    protected HashMap<String, Object> attributes;
-
-    public Neighbor(T node, double similarity) {
-        this.attributes = new HashMap<String, Object>();
-        this.node = node;
-        this.similarity = similarity;
-    }
-
-    /**
-     *
-     * @param key
-     * @param value
-     */
-    public void setAttribute(String key, Object value) {
-        attributes.put(key, value);
-    }
-
     /**
-     * Returns the value of this attribute, or null if this neighbor has no such
-     * attribute
      *
-     * @param key
-     * @return
+     * @param node
+     * @param similarity
      */
-    public Object getAttribute(String key) {
-        return attributes.get(key);
+    public Neighbor(final T node, final double similarity) {
+        this.node = node;
+        this.similarity = similarity;
     }
 
     /**
      *
-     * @return (node.id,node.value,similarity)
+     * @return (node.toString(),similarity)
      */
     @Override
-    public String toString() {
-        return "(" + node.toString() + "," + similarity + ")"; //node.value + "," + similarity + ")";
+    public final String toString() {
+        return "(" + node.toString() + "," + similarity + ")";
     }
 
     @Override
-    public boolean equals(Object other) {
+    public final boolean equals(final Object other) {
         if (!other.getClass().getName().equals(this.getClass().getName())) {
             return false;
         }
@@ -63,14 +67,19 @@ public boolean equals(Object other) {
     }
 
     @Override
-    public int hashCode() {
+    public final int hashCode() {
         int hash = 7;
         hash = 17 * hash + (this.node != null ? this.node.hashCode() : 0);
         return hash;
     }
 
+    /**
+     * This > other if this.similarity > other.similarity.
+     * @param other
+     * @return
+     */
     @Override
-    public int compareTo(Object other) {
+    public final int compareTo(final Object other) {
         if (other == null) {
             return 1;
         }
@@ -87,6 +96,10 @@ public int compareTo(Object other) {
             return 0;
         }
 
-        return this.similarity > ((Neighbor) other).similarity ? 1 : -1;
+        if (this.similarity > ((Neighbor) other).similarity) {
+            return 1;
+        }
+
+        return -1;
     }
 }
diff --git a/src/main/java/info/debatty/java/graphs/NeighborList.java b/src/main/java/info/debatty/java/graphs/NeighborList.java
@@ -2,9 +2,6 @@
 
 import info.debatty.java.util.SynchronizedBoundedPriorityQueue;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.PriorityQueue;
 
 /**
@@ -15,24 +12,6 @@
 public class NeighborList extends SynchronizedBoundedPriorityQueue<Neighbor>
         implements Serializable {
 
-    public static <T> ArrayList<Edge>
-        Convert2Edges(HashMap<T, NeighborList> graph) {
-        ArrayList<Edge> edges = new ArrayList<Edge>();
-
-        for (Map.Entry<T, NeighborList> pair : graph.entrySet()) {
-            for (Neighbor<T> neighbor : pair.getValue()) {
-                edges.add(
-                        new Edge(
-                                pair.getKey(),
-                                neighbor.node,
-                                neighbor.similarity));
-
-            }
-        }
-
-        return edges;
-    }
-
     /**
      * Copy constructor.
      *
diff --git a/src/main/java/info/debatty/java/graphs/build/NNDescent.java b/src/main/java/info/debatty/java/graphs/build/NNDescent.java
@@ -1,14 +1,40 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2016 Thibault Debatty.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
 package info.debatty.java.graphs.build;
 
+import info.debatty.java.graphs.Edge;
 import info.debatty.java.graphs.Graph;
 import info.debatty.java.graphs.Neighbor;
 import info.debatty.java.graphs.NeighborList;
 import java.security.InvalidParameterException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.Set;
 
 /**
  * Implementation of NN-Descent k-nn graph building algorithm. Based on the
@@ -30,7 +56,12 @@ public class NNDescent<T> extends GraphBuilder<T> {
     protected int iterations = 0;
     protected int c;
 
-    protected static final String IS_PROCESSED = "NNDescent_IS_PROCESSED_KEY";
+    /**
+     * Contains the list of neighbors that have been processed. Has we use a
+     * hashset, we have to use edges (which contain a reference to the source
+     * node) instead of neighbors for the concrete implementation.
+     */
+    protected Set<Edge> processed;
 
     /**
      * Get the number of edges modified at the last iteration
@@ -105,6 +136,7 @@ public void setMaxIterations(int max_iterations) {
     protected Graph<T> _computeGraph(List<T> nodes) {
 
         iterations = 0;
+        processed = new HashSet<Edge>(nodes.size() * k);
 
         if (nodes.size() <= (k + 1)) {
             return MakeFullyLinked(nodes);
@@ -135,8 +167,8 @@ protected Graph<T> _computeGraph(List<T> nodes) {
             // Mark sampled items in B[v] as false;
             for (int i = 0; i < nodes.size(); i++) {
                 T v = nodes.get(i);
-                old_lists.put(v, PickFalses(neighborlists.getNeighbors(v)));
-                new_lists.put(v, PickTruesAndMark(neighborlists.getNeighbors(v)));
+                old_lists.put(v, PickFalses(v, neighborlists.getNeighbors(v)));
+                new_lists.put(v, PickTruesAndMark(v, neighborlists.getNeighbors(v)));
 
             }
 
@@ -242,10 +274,11 @@ protected NeighborList RandomNeighborList(List<T> nodes, T for_node) {
         return nl;
     }
 
-    protected ArrayList<T> PickFalses(NeighborList neighborList) {
+    protected ArrayList<T> PickFalses(T node, NeighborList neighborList) {
         ArrayList<T> falses = new ArrayList<T>();
         for (Neighbor<T> n : neighborList) {
-            if (n.getAttribute(IS_PROCESSED) != null) { // !n.is_new
+            Edge edge = new Edge(node, n);
+            if (processed.contains(edge)) {
                 falses.add(n.node);
             }
         }
@@ -259,11 +292,12 @@ protected ArrayList<T> PickFalses(NeighborList neighborList) {
      * @param neighborList
      * @return
      */
-    protected ArrayList<T> PickTruesAndMark(NeighborList neighborList) {
+    protected ArrayList<T> PickTruesAndMark(T node, NeighborList neighborList) {
         ArrayList<T> r = new ArrayList<T>();
         for (Neighbor<T> n : neighborList) {
-            if (n.getAttribute(IS_PROCESSED) == null && Math.random() < rho) { // n.is_new
-                n.setAttribute(IS_PROCESSED, true); // n.is_new = false;
+            Edge<T> edge = new Edge<T>(node, n);
+            if (!processed.contains(edge) && Math.random() < rho) {
+                processed.add(edge);
                 r.add(n.node);
             }
         }
diff --git a/src/main/java/info/debatty/java/graphs/build/ThreadedNNDescent.java b/src/main/java/info/debatty/java/graphs/build/ThreadedNNDescent.java
@@ -1,8 +1,11 @@
 package info.debatty.java.graphs.build;
 
+import info.debatty.java.graphs.Edge;
 import info.debatty.java.graphs.Graph;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.concurrent.Callable;
 import java.util.concurrent.ConcurrentHashMap;
@@ -34,6 +37,8 @@ protected final Graph<T> _computeGraph(final List<T> nodes) {
         ExecutorService executor = Executors.newFixedThreadPool(thread_count);
 
         iterations = 0;
+        processed = Collections.synchronizedSet(
+                new HashSet<Edge>(nodes.size() * k));
 
         if (nodes.size() <= (k + 1)) {
             return MakeFullyLinked(nodes);
@@ -65,8 +70,8 @@ protected final Graph<T> _computeGraph(final List<T> nodes) {
             // Mark sampled items in B[v] as false;
             for (int i = 0; i < nodes.size(); i++) {
                 T v = nodes.get(i);
-                old_lists.put(v, PickFalses(graph.getNeighbors(v)));
-                new_lists.put(v, PickTruesAndMark(graph.getNeighbors(v)));
+                old_lists.put(v, PickFalses(v, graph.getNeighbors(v)));
+                new_lists.put(v, PickTruesAndMark(v, graph.getNeighbors(v)));
             }
 
             // old′ ←Reverse(old)
