From 7c0dc264a32ae4ba8c5daf977b88afd94a0edcfd Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Sun, 5 Jul 2026 23:52:35 -0700 Subject: [PATCH 1/6] Make .dvs skip index extensible via type-tagged stats --- .../lucene90/Lucene90DocValuesConsumer.java | 45 +++++++-- .../lucene90/Lucene90DocValuesFormat.java | 29 ++++-- .../lucene90/Lucene90DocValuesProducer.java | 95 +++++++++++++++---- 3 files changed, 131 insertions(+), 38 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java index a7dff6b9e2c9..ae1bea15cb73 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java @@ -323,23 +323,54 @@ private void writeLevels(List accumulators) throws IOException for (int i = 0; i < SKIP_INDEX_MAX_LEVEL - 1; i++) { accumulatorsLevels.add(buildLevel(accumulatorsLevels.get(i))); } + // Reusable buffer for writing all level entries of one interval before lengths are known. + // IndexOutput is forward-only, so we buffer the entire interval, then write the total + // interval length followed by the buffered contents. + final ByteBuffersDataOutput intervalBuffer = ByteBuffersDataOutput.newResettableInstance(); + final ByteBuffersDataOutput entryBuffer = ByteBuffersDataOutput.newResettableInstance(); int totalAccumulators = accumulators.size(); for (int index = 0; index < totalAccumulators; index++) { // compute how many levels we need to write for the current accumulator final int levels = getLevels(index, totalAccumulators); // write the number of levels skipIndex.writeByte((byte) levels); - // write intervals in reverse order. This is done so we don't - // need to read all of them in case of slipping + + intervalBuffer.reset(); + + // Write all level entries for this interval into intervalBuffer in reverse order. + // The outermost entry (highest level) has the widest docID span and is checked first + // by advance(), which can skip the entire interval via the interval length if its + // maxDocID < target, avoiding reading any lower-level entries. for (int level = levels - 1; level >= 0; level--) { final SkipAccumulator accumulator = accumulatorsLevels.get(level).get(index >> (SKIP_INDEX_LEVEL_SHIFT * level)); - skipIndex.writeInt(accumulator.maxDocID); - skipIndex.writeInt(accumulator.minDocID); - skipIndex.writeLong(accumulator.maxValue); - skipIndex.writeLong(accumulator.minValue); - skipIndex.writeInt(accumulator.docCount); + + entryBuffer.reset(); + + // SKIP_STAT_RANGE(default) is always written first in each level entry. + entryBuffer.writeByte(Lucene90DocValuesFormat.SKIP_STAT_RANGE); + entryBuffer.writeInt(accumulator.minDocID); + entryBuffer.writeLong(accumulator.maxValue); + entryBuffer.writeLong(accumulator.minValue); + entryBuffer.writeInt(accumulator.docCount); + + // Future optional stats can be appended here. + + // Per-level on-disk layout: + // [4 bytes: maxDocID] ← read first for cheap rejection + // [4 bytes: entry length] ← bytes of stats following (= entryBuffer.size()) + // [N bytes: stats] + intervalBuffer.writeInt(accumulator.maxDocID); + intervalBuffer.writeInt(Math.toIntExact(entryBuffer.size())); + entryBuffer.copyTo(intervalBuffer); } + + // Write the total interval length then all level entries. + // advance() reads the highest-level maxDocID first. If it rejects, it uses this + // length to seek past the entire interval (all levels), without needing a + // precomputed jump table. + skipIndex.writeInt(Math.toIntExact(intervalBuffer.size())); + intervalBuffer.copyTo(skipIndex); } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java index 22cd7581f1db..f2e0f08f1c53 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java @@ -190,7 +190,8 @@ public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOExcepti static final int VERSION_START = 0; static final int VERSION_SKIPPER_SEPARATE_FILE = 1; static final int VERSION_SKIPPER_MAX_VALUE_COUNT = 2; - static final int VERSION_CURRENT = VERSION_SKIPPER_MAX_VALUE_COUNT; + static final int VERSION_SKIPPER_STATS_FORMAT = 3; + static final int VERSION_CURRENT = VERSION_SKIPPER_STATS_FORMAT; // indicates docvalues type static final byte NUMERIC = 0; @@ -219,7 +220,7 @@ public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOExcepti // * 16 bytes: min / max value, // * 8 bytes: min / max docID // * 4 bytes: number of documents - private static final long SKIP_INDEX_INTERVAL_BYTES = 29L; + private static final long SKIP_INDEX_INTERVAL_BYTES_LEGACY = 29L; // number of intervals represented as a shift to create a new level, this is 1 << 3 == 8 // intervals. static final int SKIP_INDEX_LEVEL_SHIFT = 3; @@ -227,21 +228,29 @@ public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOExcepti // Increasing this number, it increases how much heap we need at index time. // we currently need (1 * 8 * 8 * 8) = 512 accumulators on heap static final int SKIP_INDEX_MAX_LEVEL = 4; - // number of bytes to skip when skipping a level. It does not take into account the - // current interval that is being read. - static final long[] SKIP_INDEX_JUMP_LENGTH_PER_LEVEL = new long[SKIP_INDEX_MAX_LEVEL]; + // Legacy precomputed jump table for reading segments written before VERSION_SKIPPER_STATS_FORMAT. + // Each entry gives the number of bytes to skip past the remaining levels when a level rejects. + static final long[] SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY = new long[SKIP_INDEX_MAX_LEVEL]; + + // Stat type tags for the stats-based .dvs format (VERSION_SKIPPER_STATS_FORMAT and above). + // Each level entry is length-prefixed and contains a sequence of type-tagged stats. + // New stats can be appended without a version bump: readers that encounter an unknown + // type tag skip to the entry end via the length prefix. + static final byte SKIP_STAT_RANGE = 0x01; static { // Size of the interval minus read bytes (1 byte for level and 4 bytes for maxDocID) - SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[0] = SKIP_INDEX_INTERVAL_BYTES - 5L; + SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY[0] = SKIP_INDEX_INTERVAL_BYTES_LEGACY - 5L; for (int level = 1; level < SKIP_INDEX_MAX_LEVEL; level++) { // jump from previous level - SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level] = SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level - 1]; + SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY[level] = + SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY[level - 1]; // nodes added by new level - SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level] += - (1 << (level * SKIP_INDEX_LEVEL_SHIFT)) * SKIP_INDEX_INTERVAL_BYTES; + SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY[level] += + (1 << (level * SKIP_INDEX_LEVEL_SHIFT)) * SKIP_INDEX_INTERVAL_BYTES_LEGACY; // remove the byte levels added in the previous level - SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level] -= (1 << ((level - 1) * SKIP_INDEX_LEVEL_SHIFT)); + SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY[level] -= + (1 << ((level - 1) * SKIP_INDEX_LEVEL_SHIFT)); } } } diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java index f3599924c324..7448fc3fa4e4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java @@ -16,7 +16,6 @@ */ package org.apache.lucene.codecs.lucene90; -import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.SKIP_INDEX_JUMP_LENGTH_PER_LEVEL; import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.SKIP_INDEX_MAX_LEVEL; import static org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat.TERMS_DICT_BLOCK_LZ4_SHIFT; @@ -2489,30 +2488,84 @@ public void advance(int target) throws IOException { // find next interval assert target > maxDocID[0] : "target " + target + " must be bigger that current interval " + maxDocID[0]; - while (true) { - levels = input.readByte(); - assert levels <= SKIP_INDEX_MAX_LEVEL && levels > 0 - : "level out of range [" + levels + "]"; - boolean valid = true; - // check if current interval is competitive or we can jump to the next position - for (int level = levels - 1; level >= 0; level--) { - if ((maxDocID[level] = input.readInt()) < target) { - input.skipBytes(SKIP_INDEX_JUMP_LENGTH_PER_LEVEL[level]); // the jump for the level - valid = false; - break; - } - minDocID[level] = input.readInt(); - maxValue[level] = input.readLong(); - minValue[level] = input.readLong(); - docCount[level] = input.readInt(); + if (version >= Lucene90DocValuesFormat.VERSION_SKIPPER_STATS_FORMAT) { + advanceStatsFormat(input, target); + } else { + advanceLegacyFormat(input, target); + } + } + } + + private void advanceStatsFormat(IndexInput input, int target) throws IOException { + while (true) { + levels = input.readByte(); + assert levels <= SKIP_INDEX_MAX_LEVEL && levels > 0 + : "level out of range [" + levels + "]"; + + // Read the total interval length covering all level entries. + // Used to skip the entire interval when the highest level rejects. + final int intervalLength = input.readInt(); + final long intervalEnd = input.getFilePointer() + intervalLength; + + boolean valid = true; + for (int level = levels - 1; level >= 0; level--) { + if ((maxDocID[level] = input.readInt()) < target) { + // This level rejects: skip the rest of this interval entirely. + input.seek(intervalEnd); + valid = false; + break; } - if (valid) { - // adjust levels - while (levels < SKIP_INDEX_MAX_LEVEL && maxDocID[levels] >= target) { - levels++; + final int entryLength = input.readInt(); + final long entryEnd = input.getFilePointer() + entryLength; + while (input.getFilePointer() < entryEnd) { + final byte statType = input.readByte(); + switch (statType) { + case Lucene90DocValuesFormat.SKIP_STAT_RANGE -> { + minDocID[level] = input.readInt(); + maxValue[level] = input.readLong(); + minValue[level] = input.readLong(); + docCount[level] = input.readInt(); + } + // Unknown stat types: skip to entry end. This allows new stats to be + // added without a version bump — older readers just ignore them. + default -> input.seek(entryEnd); } + } + } + if (valid) { + while (levels < SKIP_INDEX_MAX_LEVEL && maxDocID[levels] >= target) { + levels++; + } + break; + } + } + } + + private void advanceLegacyFormat(IndexInput input, int target) throws IOException { + while (true) { + levels = input.readByte(); + assert levels <= SKIP_INDEX_MAX_LEVEL && levels > 0 + : "level out of range [" + levels + "]"; + boolean valid = true; + // check if current interval is competitive or we can jump to the next position + for (int level = levels - 1; level >= 0; level--) { + if ((maxDocID[level] = input.readInt()) < target) { + input.skipBytes( + Lucene90DocValuesFormat.SKIP_INDEX_JUMP_LENGTH_PER_LEVEL_LEGACY[level]); + valid = false; break; } + minDocID[level] = input.readInt(); + maxValue[level] = input.readLong(); + minValue[level] = input.readLong(); + docCount[level] = input.readInt(); + } + if (valid) { + // adjust levels + while (levels < SKIP_INDEX_MAX_LEVEL && maxDocID[levels] >= target) { + levels++; + } + break; } } } From c50ce8d15fc2083b2272c19f93c56b483293aa2a Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Mon, 6 Jul 2026 00:07:02 -0700 Subject: [PATCH 2/6] Remove unncessary comment --- .../lucene/codecs/lucene90/Lucene90DocValuesConsumer.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java index ae1bea15cb73..c984ecbb661b 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java @@ -354,12 +354,6 @@ private void writeLevels(List accumulators) throws IOException entryBuffer.writeLong(accumulator.minValue); entryBuffer.writeInt(accumulator.docCount); - // Future optional stats can be appended here. - - // Per-level on-disk layout: - // [4 bytes: maxDocID] ← read first for cheap rejection - // [4 bytes: entry length] ← bytes of stats following (= entryBuffer.size()) - // [N bytes: stats] intervalBuffer.writeInt(accumulator.maxDocID); intervalBuffer.writeInt(Math.toIntExact(entryBuffer.size())); entryBuffer.copyTo(intervalBuffer); From f6eeef1962277e462ce125ac7b5d5ee5de1f3e70 Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Mon, 6 Jul 2026 00:14:42 -0700 Subject: [PATCH 3/6] Add changelog --- lucene/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 72fcd7427cdb..a8867f024df5 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -306,6 +306,8 @@ New Features Improvements --------------------- +* GITHUB#16358: Make .dvs skip index extensible via type-tagged stats. (Sagar Upadhyaya) + * GITHUB#16170: Alter TopGroups.merge() to not return null if no groups. (Binlong Gao) Optimizations From 60a1e4fc88879662f6361c43e590266217cd1c4a Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Mon, 6 Jul 2026 00:24:29 -0700 Subject: [PATCH 4/6] Update changelog --- lucene/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index a8867f024df5..9d7c5f87c440 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -306,7 +306,7 @@ New Features Improvements --------------------- -* GITHUB#16358: Make .dvs skip index extensible via type-tagged stats. (Sagar Upadhyaya) +* GITHUB#16358: Make doc values skip index extensible via type-tagged stats. (Sagar Upadhyaya) * GITHUB#16170: Alter TopGroups.merge() to not return null if no groups. (Binlong Gao) From cecf3cc3f413e7a307b7bef97bdd5b4762155532 Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Tue, 7 Jul 2026 16:48:14 -0700 Subject: [PATCH 5/6] Add docValuesField and SkipStat API --- .../lucene/codecs/DocValuesProducer.java | 41 +++++++++ .../lucene90/Lucene90DocValuesConsumer.java | 3 + .../lucene90/Lucene90DocValuesFormat.java | 3 +- .../lucene90/Lucene90DocValuesProducer.java | 72 +++++++++++++++- .../org/apache/lucene/index/CodecReader.java | 10 +++ .../apache/lucene/index/DocValuesField.java | 63 ++++++++++++++ .../apache/lucene/index/DocValuesSkipper.java | 11 +++ .../org/apache/lucene/index/LeafReader.java | 11 +++ .../org/apache/lucene/index/SkipStat.java | 84 +++++++++++++++++++ 9 files changed, 294 insertions(+), 4 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/index/DocValuesField.java create mode 100644 lucene/core/src/java/org/apache/lucene/index/SkipStat.java diff --git a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java index ea9cb78b0ed4..f6df029ef772 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/DocValuesProducer.java @@ -19,6 +19,7 @@ import java.io.Closeable; import java.io.IOException; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValuesField; import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesSkipper; import org.apache.lucene.index.DocValuesType; @@ -83,6 +84,46 @@ protected DocValuesProducer() {} */ public abstract DocValuesSkipper getSkipper(FieldInfo field) throws IOException; + /** + * Returns a {@link DocValuesField} providing global statistics and skipper access. The default + * wraps {@link #getSkipper(FieldInfo)}. Codecs should override to serve global stats from + * already-loaded .dvm metadata without IO. + * + * @lucene.experimental + */ + public DocValuesField getDocValuesField(FieldInfo field) throws IOException { + final DocValuesSkipper skipper = getSkipper(field); + if (skipper == null) { + return null; + } + return new DocValuesField() { + @Override + public long minValue() { + return skipper.minValue(); + } + + @Override + public long maxValue() { + return skipper.maxValue(); + } + + @Override + public int docCount() { + return skipper.docCount(); + } + + @Override + public int maxValueCount() { + return skipper.maxValueCount(); + } + + @Override + public DocValuesSkipper getSkipper() { + return skipper; + } + }; + } + /** * Checks consistency of this producer * diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java index c984ecbb661b..9239dd003bab 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumer.java @@ -315,6 +315,9 @@ private void writeSkipIndex(FieldInfo field, DocValuesProducer valuesProducer) meta.writeInt(globalDocCount); meta.writeInt(maxDocId); meta.writeInt(globalMaxValueCount); + // Bitmask of optional SkipStat types written per skip interval (beyond base range stats). + // Bit N set means SkipStat with ordinal N is present. Currently 0 (only base range stats). + meta.writeInt(0); } private void writeLevels(List accumulators) throws IOException { diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java index f2e0f08f1c53..669b60fb9989 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesFormat.java @@ -236,7 +236,8 @@ public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOExcepti // Each level entry is length-prefixed and contains a sequence of type-tagged stats. // New stats can be appended without a version bump: readers that encounter an unknown // type tag skip to the entry end via the length prefix. - static final byte SKIP_STAT_RANGE = 0x01; + // This must be a compile-time constant for use in switch statements. + static final byte SKIP_STAT_RANGE = 0x01; // == SkipStat.RANGE.id() static { // Size of the interval minus read bytes (1 byte for level and 4 bytes for maxDocID) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java index 7448fc3fa4e4..2b4fdd4b75f8 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java @@ -21,14 +21,17 @@ import java.io.IOException; import java.util.Arrays; +import java.util.Set; import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.DocValuesProducer; import org.apache.lucene.index.BaseTermsEnum; import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.DocValuesField; import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesSkipper; +import org.apache.lucene.index.SkipStat; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.ImpactsEnum; @@ -264,7 +267,8 @@ private void inferMaxValueCounts(FieldInfos fieldInfos) { entry.maxValue, entry.docCount, entry.maxDocId, - inferredMaxValueCount)); + inferredMaxValueCount, + entry.availableStatsBitmask)); } } } @@ -315,9 +319,16 @@ private DocValuesSkipperEntry readDocValueSkipperMeta(IndexInput meta) throws IO } else { maxValueCount = docCount == 0 ? 0 : -1; } + final int availableStatsBitmask; + if (version >= Lucene90DocValuesFormat.VERSION_SKIPPER_STATS_FORMAT) { + availableStatsBitmask = meta.readInt(); + } else { + availableStatsBitmask = 0; + } return new DocValuesSkipperEntry( - offset, length, minValue, maxValue, docCount, maxDocID, maxValueCount); + offset, length, minValue, maxValue, docCount, maxDocID, maxValueCount, + availableStatsBitmask); } private void readNumeric(IndexInput meta, NumericEntry entry) throws IOException { @@ -456,7 +467,8 @@ private record DocValuesSkipperEntry( long maxValue, int docCount, int maxDocId, - int maxValueCount) {} + int maxValueCount, + int availableStatsBitmask) {} // Cached VectorizationProvider instance to avoid repeated stack walks in ensureCaller() private static final org.apache.lucene.internal.vectorization.DocValuesRangeSupport @@ -2455,6 +2467,60 @@ long getLongValue(long index) throws IOException { } } + @Override + public DocValuesField getDocValuesField(FieldInfo field) throws IOException { + final DocValuesSkipperEntry entry = skippers.get(field.number); + if (entry == null) { + return null; + } + final Lucene90DocValuesProducer producer = this; + final Set> available = decodeStatsBitmask(entry.availableStatsBitmask()); + return new DocValuesField() { + @Override + public long minValue() { + return entry.minValue(); + } + + @Override + public long maxValue() { + return entry.maxValue(); + } + + @Override + public int docCount() { + return entry.docCount(); + } + + @Override + public int maxValueCount() { + return entry.maxValueCount(); + } + + @Override + public Set> availableBlockStats() { + return available; + } + + @Override + public DocValuesSkipper getSkipper() throws IOException { + return producer.getSkipper(field); + } + }; + } + + private static Set> decodeStatsBitmask(int bitmask) { + if (bitmask == 0) { + return Set.of(); + } + Set> stats = new java.util.HashSet<>(); + for (SkipStat stat : SkipStat.values()) { + if ((bitmask & (1 << stat.id())) != 0) { + stats.add(stat); + } + } + return java.util.Collections.unmodifiableSet(stats); + } + @Override public DocValuesSkipper getSkipper(FieldInfo field) throws IOException { final DocValuesSkipperEntry entry = skippers.get(field.number); diff --git a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java index 9e579a23a338..3f8427e2fc9c 100644 --- a/lucene/core/src/java/org/apache/lucene/index/CodecReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/CodecReader.java @@ -206,6 +206,16 @@ public final DocValuesSkipper getDocValuesSkipper(String field) throws IOExcepti return getDocValuesReader().getSkipper(fi); } + @Override + public final DocValuesField getDocValuesField(String field) throws IOException { + ensureOpen(); + FieldInfo fi = getFieldInfos().fieldInfo(field); + if (fi == null || fi.docValuesSkipIndexType() == DocValuesSkipIndexType.NONE) { + return null; + } + return getDocValuesReader().getDocValuesField(fi); + } + @Override public final NumericDocValues getNormValues(String field) throws IOException { ensureOpen(); diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesField.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesField.java new file mode 100644 index 000000000000..538a0d53ea98 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesField.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.Set; + +/** + * Per-field doc values metadata and skipper access for a single segment. Global statistics are + * available without IO. The {@link DocValuesSkipper} is created lazily via {@link #getSkipper()}. + * + * @lucene.experimental + */ +public abstract class DocValuesField { + + /** Sole constructor. */ + protected DocValuesField() {} + + /** Returns the global minimum value across all documents with this field in this segment. */ + public abstract long minValue(); + + /** Returns the global maximum value across all documents with this field in this segment. */ + public abstract long maxValue(); + + /** Returns the number of documents that have a value for this field in this segment. */ + public abstract int docCount(); + + /** + * Returns the maximum number of values any single document has for this field. Returns -1 if + * unknown. Returns 0 if docCount is 0. A field is single-valued if this returns 1. + */ + public int maxValueCount() { + return docCount() == 0 ? 0 : -1; + } + + /** + * Returns the set of optional per-block statistics available from this field's skipper. This can + * be checked without creating the skipper (no IO beyond what was done at segment open). + */ + public Set> availableBlockStats() { + return Set.of(); + } + + /** + * Returns a {@link DocValuesSkipper} for navigating per-block intervals. This may perform IO on + * first call. The returned instance should be confined to the thread that created it. + */ + public abstract DocValuesSkipper getSkipper() throws IOException; +} diff --git a/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.java b/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.java index 0dd105d6b465..339a26f998e7 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.java +++ b/lucene/core/src/java/org/apache/lucene/index/DocValuesSkipper.java @@ -111,6 +111,17 @@ public int maxValueCount() { return docCount() == 0 ? 0 : -1; } + /** + * Returns the value of the given optional stat for the current interval at the given level, or + * null if the stat is not available. Callers should check {@link + * DocValuesField#availableBlockStats()} before using this. + * + * @lucene.experimental + */ + public T stat(SkipStat type, int level) { + return null; + } + /** * Advance this skipper so that all levels intersects the range given by {@code minValue} and * {@code maxValue}. If there are no intersecting levels, the skipper is exhausted. diff --git a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java index 8e010151274e..c3159dc24799 100644 --- a/lucene/core/src/java/org/apache/lucene/index/LeafReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/LeafReader.java @@ -211,6 +211,17 @@ public final PostingsEnum postings(Term term) throws IOException { */ public abstract DocValuesSkipper getDocValuesSkipper(String field) throws IOException; + /** + * Returns a {@link DocValuesField} providing global statistics and skipper access for the given + * field, or {@code null} if no skip index was indexed. Global statistics are available without + * additional IO. + * + * @lucene.experimental + */ + public DocValuesField getDocValuesField(String field) throws IOException { + return null; + } + /** * Returns {@link FloatVectorValues} for this field, or null if no {@link FloatVectorValues} were * indexed. The returned instance should only be used by a single thread. diff --git a/lucene/core/src/java/org/apache/lucene/index/SkipStat.java b/lucene/core/src/java/org/apache/lucene/index/SkipStat.java new file mode 100644 index 000000000000..13fe7b7ea28a --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/SkipStat.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +/** + * Identifies a per-block statistic that a {@link DocValuesSkipper} may provide. Each instance binds + * an ID (used as both the .dvs type tag and the .dvm bitmask bit) with a Java return type. + * + *

Callers check availability via {@link DocValuesField#availableBlockStats()} and access values + * via {@link DocValuesSkipper#stat(SkipStat, int)}. + * + * @param the Java type of the stat value + * @lucene.experimental + */ +public final class SkipStat { + + private final byte id; + private final String name; + private final Class type; + + private SkipStat(byte id, String name, Class type) { + this.id = id; + this.name = name; + this.type = type; + } + + // ID 0x01 is reserved for RANGE (the mandatory base contract accessed via + // minValue(level)/maxValue(level)/docCount(level) on DocValuesSkipper). + // Optional stats start at 0x02. + + // Example of how optional sum stats would be defined: + // public static final SkipStat SUM = new SkipStat<>((byte) 0x02, "SUM", Long.class); + + private static final SkipStat[] ALL = {}; + + /** Returns the ID used as both the .dvs type tag and the .dvm bitmask bit position. */ + public byte id() { + return id; + } + + /** Returns the human-readable name. */ + public String name() { + return name; + } + + /** Returns the Java type of the stat value. */ + public Class type() { + return type; + } + + /** Returns all registered SkipStat instances. */ + public static SkipStat[] values() { + return ALL.clone(); + } + + /** Resolve a SkipStat by its ID, or null if unknown. */ + public static SkipStat fromId(int id) { + for (SkipStat stat : ALL) { + if (stat.id == id) { + return stat; + } + } + return null; + } + + @Override + public String toString() { + return name; + } +} From 97170b0c7ef43099e51b5e2c1161bed4a13243a2 Mon Sep 17 00:00:00 2001 From: Sagar Upadhyaya Date: Tue, 7 Jul 2026 16:52:57 -0700 Subject: [PATCH 6/6] Fix tidy issue --- .../codecs/lucene90/Lucene90DocValuesProducer.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java index 2b4fdd4b75f8..40a8da09aca3 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducer.java @@ -31,7 +31,6 @@ import org.apache.lucene.index.DocValuesField; import org.apache.lucene.index.DocValuesSkipIndexType; import org.apache.lucene.index.DocValuesSkipper; -import org.apache.lucene.index.SkipStat; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.ImpactsEnum; @@ -40,6 +39,7 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SkipStat; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; @@ -327,7 +327,13 @@ private DocValuesSkipperEntry readDocValueSkipperMeta(IndexInput meta) throws IO } return new DocValuesSkipperEntry( - offset, length, minValue, maxValue, docCount, maxDocID, maxValueCount, + offset, + length, + minValue, + maxValue, + docCount, + maxDocID, + maxValueCount, availableStatsBitmask); }