iter

Author: kaivalnp

lucene/core/src/java/org/apache/lucene/codecs/dedup/DedupFlatVectorsFormat.java

@@ -47,8 +47,18 @@
  *   <li>-1 sentinel marking end of fields
  * </ul>
  *
- * <p>During merge, vectors are streamed one at a time. Only a hash map of {@code long hash → int
- * dictOrd} is held in memory (~12 bytes per unique vector). Vector data is never buffered.
+ * <p>The writer performs no temporary-file IO. Dictionary bytes stream directly into the main
+ * {@code .dvd} output. Hash-collision verification uses already-resident sources: on-heap typed
+ * arrays during flush, and already-open {@link org.apache.lucene.codecs.KnnVectorsReader}s from
+ * {@link org.apache.lucene.index.MergeState} during merge.
+ *
+ * <p>When a source segment being merged was itself written with this format, the merge path skips
+ * per-doc hashing through a lazy {@code sourceDictOrd → targetDictOrd} cache and a same-sub int-ord
+ * shortcut on hash collision. This also drops source dict entries whose referencing docs have all
+ * been deleted — their dict ords are never touched, so dead vectors do not accumulate across merge
+ * cycles.
+ *
+ * <p>See {@code lucene/dev-docs/dedup-vectors-format-design.md} for the full design.
  *
  * @lucene.experimental
  */

lucene/core/src/java/org/apache/lucene/codecs/dedup/DedupFlatVectorsReader.java

@@ -39,6 +39,10 @@
 import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.VectorScorer;
 import org.apache.lucene.store.ChecksumIndexInput;
+import org.apache.lucene.store.DataAccessHint;
+import org.apache.lucene.store.FileDataHint;
+import org.apache.lucene.store.FileTypeHint;
+import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.hnsw.RandomVectorScorer;
@@ -59,6 +63,13 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {
   private final FieldInfos fieldInfos;
   private final IndexInput data;
 
+  /**
+   * IO context used to open {@link #data}. Captured so that {@link #getMergeInstance()} can switch
+   * the underlying access hint to {@link DataAccessHint#SEQUENTIAL} for the duration of a merge,
+   * and {@link #finishMerge()} can revert it back to the original context for search.
+   */
+  private final IOContext dataContext;
+
   DedupFlatVectorsReader(SegmentReadState state, FlatVectorsScorer scorer) throws IOException {
     super(scorer);
     this.fieldInfos = state.fieldInfos;
@@ -71,7 +82,8 @@ final class DedupFlatVectorsReader extends FlatVectorsReader {
     String dataFileName =
         IndexFileNames.segmentFileName(
             state.segmentInfo.name, state.segmentSuffix, DedupFlatVectorsFormat.DATA_EXTENSION);
-    IndexInput dataIn = state.directory.openInput(dataFileName, state.context);
+    this.dataContext = state.context.withHints(FileTypeHint.DATA, FileDataHint.KNN_VECTORS);
+    IndexInput dataIn = state.directory.openInput(dataFileName, dataContext);
     try {
       int versionData =
           CodecUtil.checkIndexHeader(
@@ -223,16 +235,83 @@ public RandomVectorScorer getRandomVectorScorer(String field, byte[] target) thr
     return vectorScorer.getRandomVectorScorer(entry.simFunc, getByteVectorValues(field), target);
   }
 
+  // ---- Package-private accessors for merge-path optimization ----
+  //
+  // These let a dedup-aware writer reuse this reader's existing dictionary + ordToDict mapping
+  // instead of hashing each per-doc vector individually at merge time. See
+  // DedupFlatVectorsWriter#buildFloatSubs / #buildByteSubs.
+
+  /**
+   * Return the per-doc {@code ordToDict} mapping for the given field, or {@code null} if the field
+   * has no duplicates (identity mapping — no value in trying to skip hashing since there are no
+   * duplicates to collapse).
+   */
+  int[] getOrdToDict(String field) {
+    return getFieldEntry(field).ordToDict;
+  }
+
+  /**
+   * Return a random-access view over the {@code dictSize} unique float vectors in the field's
+   * dictionary, indexed by source dict ord. The merge writer walks this once, hashes each vector
+   * into its own target dict, and builds a {@code sourceDictOrd → targetDictOrd} mapping — which it
+   * then uses to resolve every per-doc ord with a single array lookup (no per-doc hashing or
+   * equality check).
+   *
+   * <p>Returns {@code null} if the field's encoding is not float32.
+   */
+  FloatVectorValues getDictFloatVectorValues(String field) throws IOException {
+    FieldEntry entry = getFieldEntry(field);
+    if (entry.encoding != VectorEncoding.FLOAT32) {
+      return null;
+    }
+    IndexInput dictSlice = data.slice("dedup-dict", entry.dictDataOffset, entry.dictDataLength());
+    return new OffHeapFloatVectorValues.DenseOffHeapVectorValues(
+        entry.dictDimension,
+        entry.dictSize,
+        dictSlice,
+        entry.vectorByteSize,
+        vectorScorer,
+        entry.simFunc);
+  }
+
+  /** Byte-encoded mirror of {@link #getDictFloatVectorValues(String)}. */
+  ByteVectorValues getDictByteVectorValues(String field) throws IOException {
+    FieldEntry entry = getFieldEntry(field);
+    if (entry.encoding != VectorEncoding.BYTE) {
+      return null;
+    }
+    IndexInput dictSlice = data.slice("dedup-dict", entry.dictDataOffset, entry.dictDataLength());
+    return new OffHeapByteVectorValues.DenseOffHeapVectorValues(
+        entry.dictDimension,
+        entry.dictSize,
+        dictSlice,
+        entry.vectorByteSize,
+        vectorScorer,
+        entry.simFunc);
+  }
+
   @Override
   public void checkIntegrity() throws IOException {
     CodecUtil.checksumEntireFile(data);
   }
 
   @Override
-  public FlatVectorsReader getMergeInstance() {
+  public FlatVectorsReader getMergeInstance() throws IOException {
+    // During merge, the dedup writer walks each source segment's vectors mostly in forward order
+    // (via DocIDMerger). Switch the IO hint to SEQUENTIAL so the directory can enable readahead.
+    // Occasional random-access reads for hash-collision verification target recently-read vectors,
+    // which typically remain resident in the page cache, so they don't materially suffer from the
+    // SEQUENTIAL hint.
+    data.updateIOContext(dataContext.withHints(DataAccessHint.SEQUENTIAL));
     return this;
   }
 
+  @Override
+  public void finishMerge() throws IOException {
+    // Revert the access hint to the original context used for search.
+    data.updateIOContext(dataContext);
+  }
+
   @Override
   public void close() throws IOException {
     IOUtils.close(data);