add dense bulk path for multi-dimensional point fields via BinaryColumn

Author: iprithv

lucene/CHANGES.txt

@@ -284,7 +284,7 @@ Other
 
 API Changes
 ---------------------
-(No changes)
+* GITHUB#16209: Add dense bulk path for multi-dimensional point fields via BinaryColumn. (Prithvi S)
 
 New Features
 ---------------------
@@ -346,7 +346,6 @@ API Changes
 
 * GITHUB#16224 Add TokenStreamColumn for experimental columnar batch indexing. (Tim Brooks)
 
-
 New Features
 ---------------------
 

lucene/core/src/java/org/apache/lucene/document/column/BinaryColumn.java

@@ -21,9 +21,9 @@
 import org.apache.lucene.util.BytesRef;
 
 /**
- * A {@link Column} that provides variable-size binary values via a tuple cursor. Used for {@link
- * org.apache.lucene.index.DocValuesType#BINARY BINARY}, {@link
- * org.apache.lucene.index.DocValuesType#SORTED SORTED}, and {@link
+ * A {@link Column} that provides variable-size binary values via a tuple cursor, and dense values
+ * via a {@link BytesRefValuesCursor}. Used for {@link org.apache.lucene.index.DocValuesType#BINARY
+ * BINARY}, {@link org.apache.lucene.index.DocValuesType#SORTED SORTED}, and {@link
  * org.apache.lucene.index.DocValuesType#SORTED_SET SORTED_SET} doc values, and for stored/indexed
  * binary or text fields. Values fed to points are passed through unchanged, so callers are
  * responsible for producing sort-encoded bytes of the correct total length.

lucene/core/src/java/org/apache/lucene/index/PointValuesWriter.java

@@ -58,6 +58,11 @@ class PointValuesWriter {
             + (PointValues.MAX_NUM_BYTES * BKDConfig.MAX_DIMS);
   }
 
+  /** Minimum number of values to process per chunk in the dense N-D bulk path. */
+  private static final int MIN_VALUES_PER_CHUNK = 64;
+
+  private byte[] densePointsBuffer;
+
   PointValuesWriter(Counter bytesUsed, FieldInfo fieldInfo, SharedIndexingScratch sharedScratch) {
     this.fieldInfo = fieldInfo;
     this.iwBytesUsed = bytesUsed;
@@ -137,15 +142,30 @@ void addDense1DLongValues(int firstDocID, LongValuesCursor cursor) throws IOExce
     commitDenseRange(firstDocID, size, ramBefore);
   }
 
+  /**
+   * Bulk-adds dense N-dimensional packed point values from a {@link BytesRefValuesCursor}. Each value
+   * is a pre-encoded packed byte array of {@code packedBytesLength} bytes.
+   */
   void addDenseNDValues(int firstDocID, BytesRefValuesCursor cursor) throws IOException {
     final int size = cursor.size();
     if (size == 0) {
       return;
     }
     final long ramBefore = reserveDenseRange(firstDocID, size);
     final int width = packedBytesLength;
-    final int perChunk = SharedIndexingScratch.BYTES_SCRATCH_SIZE / width;
-    final byte[] buffer = sharedScratch.bytesScratch();
+    final byte[] buffer;
+    final int perChunk;
+    if (width * MIN_VALUES_PER_CHUNK <= SharedIndexingScratch.BYTES_SCRATCH_SIZE) {
+      // Common case (packed <= ~64 bytes): reuse the shared 4 KiB scratch which already gives
+      // at least MIN_VALUES_PER_CHUNK values per chunk. Matches the 1D dense path behavior.
+      buffer = sharedScratch.bytesScratch();
+      perChunk = SharedIndexingScratch.BYTES_SCRATCH_SIZE / width;
+    } else {
+      // Wide points: shared 4 KiB would yield < MIN_VALUES_PER_CHUNK; allocate a dedicated
+      // larger buffer (sized for at least 64 values) and keep it for the lifetime of this writer.
+      buffer = pointsDenseBuffer(width);
+      perChunk = buffer.length / width;
+    }
     int remaining = size;
     while (remaining > 0) {
       int chunk = Math.min(perChunk, remaining);
@@ -156,6 +176,26 @@ void addDenseNDValues(int firstDocID, BytesRefValuesCursor cursor) throws IOExce
     commitDenseRange(firstDocID, size, ramBefore);
   }
 
+  /**
+   * Returns (and caches) a dedicated buffer for wide packed point values, sized to hold at least
+   * {@code MIN_VALUES_PER_CHUNK} values. Only called when the shared scratch would result in fewer
+   * than {@link #MIN_VALUES_PER_CHUNK} values per chunk (i.e. packed length > ~64 bytes).
+   *
+   * <p>The allocated size is charged to {@code iwBytesUsed}.
+   */
+  private byte[] pointsDenseBuffer(int packedLength) {
+    final int minBytes = packedLength * MIN_VALUES_PER_CHUNK;
+    if (densePointsBuffer == null) {
+      densePointsBuffer = new byte[minBytes];
+      iwBytesUsed.addAndGet(minBytes);
+    } else if (densePointsBuffer.length < minBytes) {
+      final int old = densePointsBuffer.length;
+      densePointsBuffer = new byte[minBytes];
+      iwBytesUsed.addAndGet(minBytes - (long) old);
+    }
+    return densePointsBuffer;
+  }
+
   private void validate1DPacked(int byteWidth) {
     if (fieldInfo.getPointDimensionCount() != 1 || fieldInfo.getPointNumBytes() != byteWidth) {
       throw new IllegalArgumentException(