Make doc values skip index extensible and add DocValuesField/SkipStat<T> API

lucene/CHANGES.txt

@@ -306,6 +306,8 @@ New Features
 
 Improvements
 ---------------------
+* GITHUB#16358: Make doc values skip index extensible via type-tagged stats. (Sagar Upadhyaya)
+
 * GITHUB#16170: Alter TopGroups.merge() to not return null if no groups. (Binlong Gao)
 
 Optimizations

lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java

@@ -19,6 +19,7 @@
 import java.io.Closeable;
 import java.io.IOException;
 import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.index.DocValuesField;
 import org.apache.lucene.index.DocValuesSkipIndexType;
 import org.apache.lucene.index.DocValuesSkipper;
 import org.apache.lucene.index.DocValuesType;
@@ -83,6 +84,46 @@ protected DocValuesProducer() {}
    */
   public abstract DocValuesSkipper getSkipper(FieldInfo field) throws IOException;
 
+  /**
+   * Returns a {@link DocValuesField} providing global statistics and skipper access. The default
+   * wraps {@link #getSkipper(FieldInfo)}. Codecs should override to serve global stats from
+   * already-loaded .dvm metadata without IO.
+   *
+   * @lucene.experimental
+   */
+  public DocValuesField getDocValuesField(FieldInfo field) throws IOException {
+    final DocValuesSkipper skipper = getSkipper(field);
+    if (skipper == null) {
+      return null;
+    }
+    return new DocValuesField() {
+      @Override
+      public long minValue() {
+        return skipper.minValue();
+      }
+
+      @Override
+      public long maxValue() {
+        return skipper.maxValue();
+      }
+
+      @Override
+      public int docCount() {
+        return skipper.docCount();
+      }
+
+      @Override
+      public int maxValueCount() {
+        return skipper.maxValueCount();
+      }
+
+      @Override
+      public DocValuesSkipper getSkipper() {
+        return skipper;
+      }
+    };
+  }
+
   /**
    * Checks consistency of this producer
    *

lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java

@@ -315,6 +315,9 @@ private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer)
     meta.writeInt(globalDocCount);
     meta.writeInt(maxDocId);
     meta.writeInt(globalMaxValueCount);
+    // Bitmask of optional SkipStat types written per skip interval (beyond base range stats).
+    // Bit N set means SkipStat with ordinal N is present. Currently 0 (only base range stats).
+    meta.writeInt(0);
   }
 
   private void writeLevels(List<SkipAccumulator> accumulators) throws IOException {
@@ -323,23 +326,48 @@ private void writeLevels(List<SkipAccumulator> accumulators) throws IOException
     for (int i = 0; i < SKIP_INDEX_MAX_LEVEL - 1; i++) {
       accumulatorsLevels.add(buildLevel(accumulatorsLevels.get(i)));
     }
+    // Reusable buffer for writing all level entries of one interval before lengths are known.
+    // IndexOutput is forward-only, so we buffer the entire interval, then write the total
+    // interval length followed by the buffered contents.
+    final ByteBuffersDataOutput intervalBuffer = ByteBuffersDataOutput.newResettableInstance();
+    final ByteBuffersDataOutput entryBuffer = ByteBuffersDataOutput.newResettableInstance();
     int totalAccumulators = accumulators.size();
     for (int index = 0; index < totalAccumulators; index++) {
       // compute how many levels we need to write for the current accumulator
       final int levels = getLevels(index, totalAccumulators);
       // write the number of levels
       skipIndex.writeByte((byte) levels);
-      // write intervals in reverse order. This is done so we don't
-      // need to read all of them in case of slipping
+
+      intervalBuffer.reset();
+
+      // Write all level entries for this interval into intervalBuffer in reverse order.
+      // The outermost entry (highest level) has the widest docID span and is checked first
+      // by advance(), which can skip the entire interval via the interval length if its
+      // maxDocID < target, avoiding reading any lower-level entries.
       for (int level = levels - 1; level >= 0; level--) {
         final SkipAccumulator accumulator =
             accumulatorsLevels.get(level).get(index >> (SKIP_INDEX_LEVEL_SHIFT * level));
-        skipIndex.writeInt(accumulator.maxDocID);
-        skipIndex.writeInt(accumulator.minDocID);
-        skipIndex.writeLong(accumulator.maxValue);
-        skipIndex.writeLong(accumulator.minValue);
-        skipIndex.writeInt(accumulator.docCount);
+
+        entryBuffer.reset();
+
+        // SKIP_STAT_RANGE(default) is always written first in each level entry.
+        entryBuffer.writeByte(Lucene90DocValuesFormat.SKIP_STAT_RANGE);
+        entryBuffer.writeInt(accumulator.minDocID);
+        entryBuffer.writeLong(accumulator.maxValue);
+        entryBuffer.writeLong(accumulator.minValue);
+        entryBuffer.writeInt(accumulator.docCount);
+
+        intervalBuffer.writeInt(accumulator.maxDocID);
+        intervalBuffer.writeInt(Math.toIntExact(entryBuffer.size()));
+        entryBuffer.copyTo(intervalBuffer);
       }
+
+      // Write the total interval length then all level entries.
+      // advance() reads the highest-level maxDocID first. If it rejects, it uses this
+      // length to seek past the entire interval (all levels), without needing a
+      // precomputed jump table.
+      skipIndex.writeInt(Math.toIntExact(intervalBuffer.size()));
+      intervalBuffer.copyTo(skipIndex);
     }
   }
 

lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java

@@ -190,7 +190,8 @@ public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOExcepti
   static final int VERSION_START = 0;
   static final int VERSION_SKIPPER_SEPARATE_FILE = 1;
   static final int VERSION_SKIPPER_MAX_VALUE_COUNT = 2;
-  static final int VERSION_CURRENT = VERSION_SKIPPER_MAX_VALUE_COUNT;
+  static final int VERSION_SKIPPER_STATS_FORMAT = 3;
+  static final int VERSION_CURRENT = VERSION_SKIPPER_STATS_FORMAT;
 
   // indicates docvalues type
   static final byte NUMERIC = 0;
@@ -219,29 +220,38 @@ public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOExcepti
   //   * 16 bytes: min / max value,
   //   * 8 bytes:  min / max docID
   //   * 4 bytes: number of documents
-  private static final long SKIP_INDEX_INTERVAL_BYTES = 29L;
+  private static final long SKIP_INDEX_INTERVAL_BYTES_LEGACY = 29L;
   // number of intervals represented as a shift to create a new level, this is 1 << 3 == 8
   // intervals.
   static final int SKIP_INDEX_LEVEL_SHIFT = 3;
   // max number of levels
   // Increasing this number, it increases how much heap we need at index time.
   // we currently need (1 * 8 * 8 * 8)  = 512 accumulators on heap
   static final int SKIP_INDEX_MAX_LEVEL = 4;
-  // number of bytes to skip when skipping a level. It does not take into account the
-  // current interval that is being read.
-  static final long[] SKIP_INDEX_JUMP_LENGTH_PER_LEVEL = new long[SKIP_INDEX_MAX_LEVEL];
+  // Legacy precomputed jump table for reading segments written before VERSION_SKIPPER_STATS_FORMAT.
+  // Each entry gives the number of bytes to skip past the remaining levels when a level rejects.
+  static final long[] SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY = new long[SKIP_INDEX_MAX_LEVEL];
+
+  // Stat type tags for the stats-based .dvs format (VERSION_SKIPPER_STATS_FORMAT and above).
+  // Each level entry is length-prefixed and contains a sequence of type-tagged stats.
+  // New stats can be appended without a version bump: readers that encounter an unknown
+  // type tag skip to the entry end via the length prefix.
+  // This must be a compile-time constant for use in switch statements.
+  static final byte SKIP_STAT_RANGE = 0x01; // == SkipStat.RANGE.id()
 
   static {
     // Size of the interval minus read bytes (1 byte for level and 4 bytes for maxDocID)
-    SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[0] = SKIP_INDEX_INTERVAL_BYTES - 5L;
+    SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY[0] = SKIP_INDEX_INTERVAL_BYTES_LEGACY - 5L;
     for (int level = 1; level < SKIP_INDEX_MAX_LEVEL; level++) {
       // jump from previous level
-      SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level] = SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level - 1];
+      SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY[level] =
+          SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY[level - 1];
       // nodes added by new level
-      SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level] +=
-          (1 << (level * SKIP_INDEX_LEVEL_SHIFT)) * SKIP_INDEX_INTERVAL_BYTES;
+      SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY[level] +=
+          (1 << (level * SKIP_INDEX_LEVEL_SHIFT)) * SKIP_INDEX_INTERVAL_BYTES_LEGACY;
       // remove the byte levels added in the previous level
-      SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level] -= (1 << ((level - 1) * SKIP_INDEX_LEVEL_SHIFT));
+      SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY[level] -=
+          (1 << ((level - 1) * SKIP_INDEX_LEVEL_SHIFT));
     }
   }
 }