rebase fix

Signed-off-by: Samuel Herman <sherman8915@gmail.com>

Author: sam-herman

jvector-base/src/main/java/io/github/jbellis/jvector/graph/GraphIndexBuilder.java

@@ -957,4 +957,58 @@ private void loadV3(RandomAccessReader in, int size) throws IOException {
         graph.updateEntryNode(new NodeAtLevel(0, entryNode));
         graph.setDegrees(List.of(maxDegree));
     }
+
+    /**
+     * Convenience method to build a new graph from an existing one, with the addition of new nodes.
+     * This is useful when we want to merge a new set of vectors into an existing graph that is already on disk.
+     *
+     * @param onDiskGraphIndex the on-disk representation of the graph index to be processed and converted.
+     * @param perLevelNeighborsScoreCache the cache containing pre-computed neighbor scores,
+     * @param newVectors a super set RAVV containing the new vectors to be added to the graph as well as the old ones that are already in the graph
+     * @param buildScoreProvider the provider responsible for calculating build scores.
+     * @param startingNodeOffset the offset in the newVectors RAVV where the new vectors start
+     * @param graphToRavvOrdMap a mapping from the old graph's node ids to the newVectors RAVV node ids
+     * @param beamWidth the width of the beam used during the graph building process.
+     * @param overflowRatio the ratio of extra neighbors to allow temporarily when inserting a node.
+     * @param alpha the weight factor for balancing score computations.
+     * @param addHierarchy whether to add hierarchical structures while building the graph.
+     *
+     * @return the in-memory representation of the graph index.
+     * @throws IOException if an I/O error occurs during the graph loading or conversion process.
+     */
+    public static ImmutableGraphIndex buildAndMergeNewNodes(OnDiskGraphIndex onDiskGraphIndex,
+                                                         NeighborsScoreCache perLevelNeighborsScoreCache,
+                                                         RandomAccessVectorValues newVectors,
+                                                         BuildScoreProvider buildScoreProvider,
+                                                         int startingNodeOffset,
+                                                         int[] graphToRavvOrdMap,
+                                                         int beamWidth,
+                                                         float overflowRatio,
+                                                         float alpha,
+                                                         boolean addHierarchy) throws IOException {
+
+
+
+        try (GraphIndexBuilder builder = new GraphIndexBuilder(buildScoreProvider,
+                onDiskGraphIndex,
+                perLevelNeighborsScoreCache,
+                beamWidth,
+                overflowRatio,
+                alpha,
+                addHierarchy,
+                true,
+                PhysicalCoreExecutor.pool(),
+                ForkJoinPool.commonPool())) {
+
+            var vv = newVectors.threadLocalSupplier();
+
+            // parallel graph construction from the merge documents Ids
+            PhysicalCoreExecutor.pool().submit(() -> IntStream.range(startingNodeOffset, newVectors.size()).parallel().forEach(ord -> {
+                builder.addGraphNode(ord, vv.get().getVector(graphToRavvOrdMap[ord]));
+            })).join();
+
+            builder.cleanup();
+            return builder.getGraph();
+        }
+    }
 }

jvector-base/src/main/java/io/github/jbellis/jvector/graph/OnHeapGraphIndex.java

@@ -303,6 +303,10 @@ public View getView() {
         }
     }
 
+    public FrozenView getFrozenView() {
+        return new FrozenView();
+    }
+
     public ThreadSafeGrowableBitSet getDeletedNodes() {
         return deletedNodes;
     }
@@ -661,7 +665,7 @@ public static OnHeapGraphIndex convertToHeap(OnDiskGraphIndex diskIndex,
                     final NodeArray neighbors = levelNeighborsScoreCache.get(nodeId).copy();
 
                     // Add the node with its neighbors
-                    heapIndex.addNode(level, nodeId, neighbors);
+                    heapIndex.connectNode(level, nodeId, neighbors);
                     heapIndex.markComplete(new NodeAtLevel(level, nodeId));
                 }
             }

jvector-tests/src/test/java/io/github/jbellis/jvector/graph/OnHeapGraphIndexTest.java

@@ -77,9 +77,9 @@ public class OnHeapGraphIndexTest extends RandomizedTest  {
     private BuildScoreProvider baseBuildScoreProvider;
     private BuildScoreProvider newBuildScoreProvider;
     private BuildScoreProvider allBuildScoreProvider;
-    private OnHeapGraphIndex baseGraphIndex;
-    private OnHeapGraphIndex newGraphIndex;
-    private OnHeapGraphIndex allGraphIndex;
+    private ImmutableGraphIndex baseGraphIndex;
+    private ImmutableGraphIndex newGraphIndex;
+    private ImmutableGraphIndex allGraphIndex;
 
     @Before
     public void setup() throws IOException {
@@ -149,7 +149,7 @@ public void testReconstructionOfOnHeapGraphIndex() throws IOException {
         TestUtil.writeGraph(baseGraphIndex, baseVectorsRavv, graphOutputPath);
 
         log.info("Writing neighbors score cache to {}", neighborsScoreCacheOutputPath);
-        final NeighborsScoreCache neighborsScoreCache = new NeighborsScoreCache(baseGraphIndex);
+        final NeighborsScoreCache neighborsScoreCache = new NeighborsScoreCache((OnHeapGraphIndex) baseGraphIndex);
         try (SimpleWriter writer = new SimpleWriter(neighborsScoreCacheOutputPath.toAbsolutePath())) {
             neighborsScoreCache.write(writer);
         }
@@ -182,14 +182,14 @@ public void testReconstructionOfOnHeapGraphIndex() throws IOException {
     public void testIncrementalInsertionFromOnDiskIndex() throws IOException {
         var outputPath = testDirectory.resolve("testReconstructionOfOnHeapGraphIndex_" + baseGraphIndex.getClass().getSimpleName());
         log.info("Writing graph to {}", outputPath);
-        final NeighborsScoreCache neighborsScoreCache = new NeighborsScoreCache(baseGraphIndex);
+        final NeighborsScoreCache neighborsScoreCache = new NeighborsScoreCache((OnHeapGraphIndex) baseGraphIndex);
         TestUtil.writeGraph(baseGraphIndex, baseVectorsRavv, outputPath);
         try (var readerSupplier = new SimpleMappedReader.Supplier(outputPath.toAbsolutePath());
              var onDiskGraph = OnDiskGraphIndex.load(readerSupplier)) {
             TestUtil.assertGraphEquals(baseGraphIndex, onDiskGraph);
             // We will create a trivial 1:1 mapping between the new graph and the ravv
             final int[] graphToRavvOrdMap = IntStream.range(0, allVectorsRavv.size()).toArray();
-            OnHeapGraphIndex reconstructedAllNodeOnHeapGraphIndex = GraphIndexBuilder.buildAndMergeNewNodes(onDiskGraph, neighborsScoreCache, allVectorsRavv, allBuildScoreProvider, NUM_BASE_VECTORS, graphToRavvOrdMap, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA, ADD_HIERARCHY);
+            ImmutableGraphIndex reconstructedAllNodeOnHeapGraphIndex = GraphIndexBuilder.buildAndMergeNewNodes(onDiskGraph, neighborsScoreCache, allVectorsRavv, allBuildScoreProvider, NUM_BASE_VECTORS, graphToRavvOrdMap, BEAM_WIDTH, NEIGHBOR_OVERFLOW, ALPHA, ADD_HIERARCHY);
 
             // Verify that the recall is similar
             float recallFromReconstructedAllNodeOnHeapGraphIndex = calculateRecall(reconstructedAllNodeOnHeapGraphIndex, allBuildScoreProvider, queryVector, groundTruthAllVectors, TOP_K);
@@ -231,7 +231,7 @@ private static int[] getGroundTruth(RandomAccessVectorValues ravv, VectorFloat<?
         return exactResults.stream().limit(topK).mapToInt(nodeScore -> nodeScore.node).toArray();
     }
 
-    private static float calculateRecall(OnHeapGraphIndex graphIndex, BuildScoreProvider buildScoreProvider, VectorFloat<?> queryVector, int[] groundTruth, int k) throws IOException {
+    private static float calculateRecall(ImmutableGraphIndex graphIndex, BuildScoreProvider buildScoreProvider, VectorFloat<?> queryVector, int[] groundTruth, int k) throws IOException {
         try (GraphSearcher graphSearcher = new GraphSearcher(graphIndex)){
             SearchScoreProvider ssp = buildScoreProvider.searchProviderFor(queryVector);
             var searchResults = graphSearcher.search(ssp, k, Bits.ALL);