From ecfcb41d1a7cd967aeba0c736598774cd1c49c15 Mon Sep 17 00:00:00 2001 From: zhouhui Date: Sun, 17 Mar 2024 22:31:13 +0800 Subject: [PATCH 1/7] Binary search term leaf. --- .../blocktree/SegmentTermsEnumFrame.java | 427 +++++++++++++++++- .../org/apache/lucene/store/DataInput.java | 15 + .../lucene99/TestLucene99PostingsFormat.java | 15 +- .../index/BasePostingsFormatTestCase.java | 149 +++++- 4 files changed, 595 insertions(+), 11 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java index 66231313e520..331720a12644 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java @@ -25,6 +25,7 @@ import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.fst.FST; final class SegmentTermsEnumFrame { @@ -51,6 +52,11 @@ final class SegmentTermsEnumFrame { byte[] suffixLengthBytes; final ByteArrayDataInput suffixLengthsReader; + int[] suffixes; + int[] postions; + int[] offsets; + FixedBitSet termExists; + long[] subCodes; byte[] statBytes = new byte[64]; int statsSingletonRunLength = 0; final ByteArrayDataInput statsReader = new ByteArrayDataInput(); @@ -75,6 +81,9 @@ final class SegmentTermsEnumFrame { // True if all entries are terms boolean isLeafBlock; + // True if all entries have the same length. + boolean allEqual; + long lastSubFP; int nextFloorLabel; @@ -183,7 +192,7 @@ void loadBlock() throws IOException { suffixesReader.reset(suffixBytes, 0, numSuffixBytes); int numSuffixLengthBytes = ste.in.readVInt(); - final boolean allEqual = (numSuffixLengthBytes & 0x01) != 0; + allEqual = (numSuffixLengthBytes & 0x01) != 0; numSuffixLengthBytes >>>= 1; if (suffixLengthBytes.length < numSuffixLengthBytes) { suffixLengthBytes = new byte[ArrayUtil.oversize(numSuffixLengthBytes, 1)]; @@ -196,6 +205,38 @@ void loadBlock() throws IOException { suffixLengthsReader.reset(suffixLengthBytes, 0, numSuffixLengthBytes); totalSuffixBytes = ste.in.getFilePointer() - startSuffixFP; + // Prepare suffixes, offsets to binary search. + if (allEqual == false) { + suffixes = new int[entCount]; + // TODO: remove postions if it is necessary. + postions = new int[entCount]; + if (isLeafBlock) { + for (int i = 0; i < suffixes.length; i++) { + suffixes[i] = suffixLengthsReader.readVInt(); + postions[i] = suffixLengthsReader.getPosition(); + } + } else { + // Handle subCode for non leaf block. + termExists = new FixedBitSet(entCount); + subCodes = new long[entCount]; + for (int i = 0; i < suffixes.length; i++) { + code = suffixLengthsReader.readVInt(); + suffixes[i] = code >>> 1; + if ((code & 1) == 0) { + termExists.set(i); + } else { + // read subCode. + subCodes[i] = suffixLengthsReader.readVLong(); + } + postions[i] = suffixLengthsReader.getPosition(); + } + } + // Reset suffixLengthsReader's position. + suffixLengthsReader.setPosition(0); + offsets = getOffsets(suffixes); + assert assertOffset(suffixes, offsets); + } + /*if (DEBUG) { if (arc == null) { System.out.println(" loadBlock (next) fp=" + fp + " entCount=" + entCount + " prefixLen=" + prefix + " isLastInFloor=" + isLastInFloor + " leaf?=" + isLeafBlock); @@ -523,7 +564,9 @@ public void scanToSubBlock(long subFP) { // NOTE: sets startBytePos/suffix as a side effect public SeekStatus scanToTerm(BytesRef target, boolean exactOnly) throws IOException { - return isLeafBlock ? scanToTermLeaf(target, exactOnly) : scanToTermNonLeaf(target, exactOnly); + return isLeafBlock + ? binarySearchTermLeaf(target, exactOnly) + : scanToTermNonLeaf(target, exactOnly); } private int startBytePos; @@ -554,8 +597,6 @@ public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOEx assert prefixMatches(target); - // TODO: binary search when all terms have the same length, which is common for ID fields, - // which are also the most sensitive to lookup performance? // Loop over each entry (term or sub-block) in this block: do { nextEnt++; @@ -592,6 +633,14 @@ public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOEx // return NOT_FOUND: fillTerm(); + // System.out.println("target: " + target.utf8ToString() + // + ", nextEnt: " + nextEnt + // + ", startBytePos: " + startBytePos + // + ", suffix: " + suffix + // + ", lenPos: " + suffixLengthsReader.getPosition() + // + ", sufPos: " + suffixesReader.getPosition() + // ); + // if (DEBUG) System.out.println(" not found"); return SeekStatus.NOT_FOUND; } else { @@ -604,6 +653,13 @@ public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOEx assert ste.termExists; fillTerm(); // if (DEBUG) System.out.println(" found!"); + // System.out.println("target: " + target.utf8ToString() + // + ", nextEnt: " + nextEnt + // + ", startBytePos: " + startBytePos + // + ", suffix: " + suffix + // + ", lenPos: " + suffixLengthsReader.getPosition() + // + ", sufPos: " + suffixesReader.getPosition() + // ); return SeekStatus.FOUND; } } while (nextEnt < entCount); @@ -622,12 +678,256 @@ public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOEx fillTerm(); } + // System.out.println("target: " + target.utf8ToString() + // + ", nextEnt: " + nextEnt + // + ", startBytePos: " + startBytePos + // + ", suffix: " + suffix + // + ", lenPos: " + suffixLengthsReader.getPosition() + // + ", sufPos: " + suffixesReader.getPosition() + // ); + // TODO: not consistent that in the // not-exact case we don't next() into the next // frame here return SeekStatus.END; } + // TODO: take this to scanToTermNonLeaf + // Target's prefix matches this block's prefix; + // But these suffixes' length are not same, + // we binary search the entries check if the suffix matches. + private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnly) + throws IOException { + // if (DEBUG) System.out.println(" binarySearchTermLeaf: block fp=" + fp + " prefix=" + + // prefix + " + // nextEnt=" + nextEnt + " (of " + entCount + ") target=" + brToString(target) + " term=" + + // brToString(term)); + + assert nextEnt != -1; + + ste.termExists = true; + subCode = 0; + + if (nextEnt == entCount) { + if (exactOnly) { + fillTerm(); + } + return SeekStatus.END; + } + + assert prefixMatches(target); + assert assertOffset(suffixes, offsets); + + int start = nextEnt; + int end = entCount - 1; + // Binary search the entries (terms) in this leaf block: + int cmp = 0; + int lastGreaterPos = 0; + while (start <= end) { + int mid = (start + end) / 2; + nextEnt = mid + 1; + startBytePos = offsets[mid]; + suffix = suffixes[mid]; + suffixesReader.setPosition(startBytePos + suffix); + + // TODO: is it necessary to set suffixLengthsReader's position? + suffixLengthsReader.setPosition(postions[mid]); + + // Binary search bytes in the suffix, comparing to the target + cmp = + Arrays.compareUnsigned( + suffixBytes, + startBytePos, + startBytePos + suffix, + target.bytes, + target.offset + prefix, + target.offset + target.length); + if (cmp < 0) { + start = mid + 1; + } else if (cmp > 0) { + end = mid - 1; + lastGreaterPos = mid; + } else { + // Exact match! + + // This cannot be a sub-block because we + // would have followed the index to this + // sub-block from the start: + + assert ste.termExists; + fillTerm(); + // if (DEBUG) System.out.println(" found!"); + // System.out.println("target: " + target.utf8ToString() + // + ", nextEnt: " + nextEnt + // + ", startBytePos: " + startBytePos + // + ", suffix: " + suffix + // + ", lenPos: " + suffixLengthsReader.getPosition() + // + ", sufPos: " + suffixesReader.getPosition() + // ); + return SeekStatus.FOUND; + } + } + + // It is possible (and OK) that terms index pointed us + // at this block, but, we searched the entire block and + // did not find the term to position to. This happens + // when the target is after the last term in the block + // (but, before the next term in the index). EG + // target could be foozzz, and terms index pointed us + // to the foo* block, but the last term in this block + // was fooz (and, eg, first term in the next block will + // bee fop). + // if (DEBUG) System.out.println(" block end"); + SeekStatus seekStatus = end < entCount - 1 ? SeekStatus.NOT_FOUND : SeekStatus.END; + if (exactOnly || seekStatus == SeekStatus.NOT_FOUND) { + // If binary search ended at the less term, and greater term exists. + // We need to advance to the greater term. + if (cmp < 0 && seekStatus == SeekStatus.NOT_FOUND) { + suffix = suffixes[lastGreaterPos]; + startBytePos = offsets[lastGreaterPos]; + suffixesReader.setPosition(startBytePos + suffix); + nextEnt = lastGreaterPos + 1; + + suffixLengthsReader.setPosition(postions[lastGreaterPos]); + } + fillTerm(); + } + + // TODO: not consistent that in the + // not-exact case we don't next() into the next + // frame here + // System.out.println("target: " + target.utf8ToString() + // + ", nextEnt: " + nextEnt + // + ", startBytePos: " + startBytePos + // + ", suffix: " + suffix + // + ", lenPos: " + suffixLengthsReader.getPosition() + // + ", sufPos: " + suffixesReader.getPosition() + // ); + return seekStatus; + } + + private int[] getOffsets(int[] suffixes) { + int[] offsets = new int[suffixes.length]; + offsets[0] = 0; + for (int i = 1; i < suffixes.length; i++) { + offsets[i] = offsets[i - 1] + suffixes[i - 1]; + } + return offsets; + } + + // Used only by assert + private boolean assertOffset(int[] suffixes, int[] offsets) { + if (offsets[0] != 0) { + return false; + } + for (int i = 1; i < offsets.length; i++) { + if (suffixes[i - 1] != offsets[i] - offsets[i - 1]) { + return false; + } + } + return true; + } + + // Target's prefix matches this block's prefix; + // we binary search the entries check if the suffix matches. + public SeekStatus binarySearchTermLeaf(BytesRef target, boolean exactOnly) throws IOException { + return allEqual + ? binarySearchTermLeafAllEqual(target, exactOnly) + : binarySearchTermLeafUnEqual(target, exactOnly); + } + + // Target's prefix matches this block's prefix; + // And all suffixes have the same length in this block, + // we binary search the entries check if the suffix matches. + private SeekStatus binarySearchTermLeafAllEqual(BytesRef target, boolean exactOnly) + throws IOException { + // if (DEBUG) System.out.println(" binarySearchTermLeaf: block fp=" + fp + " prefix=" + + // prefix + " + // nextEnt=" + nextEnt + " (of " + entCount + ") target=" + brToString(target) + " term=" + + // brToString(term)); + + assert nextEnt != -1; + + ste.termExists = true; + subCode = 0; + + if (nextEnt == entCount) { + if (exactOnly) { + fillTerm(); + } + return SeekStatus.END; + } + + assert prefixMatches(target); + + suffix = suffixLengthsReader.readVInt(); + // TODO early terminate when target length unequals suffix + prefix. + // But we need to keep the same status with scanToTermLeaf. + int start = nextEnt; + int end = entCount - 1; + // Binary search the entries (terms) in this leaf block: + int cmp = 0; + while (start <= end) { + int mid = (start + end) / 2; + nextEnt = mid + 1; + startBytePos = mid * suffix; + suffixesReader.setPosition(startBytePos + suffix); + + // Binary search bytes in the suffix, comparing to the target + cmp = + Arrays.compareUnsigned( + suffixBytes, + startBytePos, + startBytePos + suffix, + target.bytes, + target.offset + prefix, + target.offset + target.length); + if (cmp < 0) { + start = mid + 1; + } else if (cmp > 0) { + end = mid - 1; + } else { + // Exact match! + + // This cannot be a sub-block because we + // would have followed the index to this + // sub-block from the start: + + assert ste.termExists; + fillTerm(); + // if (DEBUG) System.out.println(" found!"); + return SeekStatus.FOUND; + } + } + + // It is possible (and OK) that terms index pointed us + // at this block, but, we searched the entire block and + // did not find the term to position to. This happens + // when the target is after the last term in the block + // (but, before the next term in the index). EG + // target could be foozzz, and terms index pointed us + // to the foo* block, but the last term in this block + // was fooz (and, eg, first term in the next block will + // bee fop). + // if (DEBUG) System.out.println(" block end"); + SeekStatus seekStatus = end < entCount - 1 ? SeekStatus.NOT_FOUND : SeekStatus.END; + if (exactOnly || seekStatus == SeekStatus.NOT_FOUND) { + // If binary search ended at the less term, and greater term exists. + // We need to advance to the greater term. + if (cmp < 0 && seekStatus == SeekStatus.NOT_FOUND) { + startBytePos += suffix; + suffixesReader.skipBytes(suffix); + nextEnt++; + } + fillTerm(); + } + + // TODO: not consistent that in the + // not-exact case we don't next() into the next + // frame here + return seekStatus; + } + // Target's prefix matches this block's prefix; we // scan the entries check if the suffix matches. public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws IOException { @@ -750,6 +1050,125 @@ public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws I return SeekStatus.END; } + // Target's prefix matches this block's prefix; we + // binary search the entries check if the suffix matches. + public SeekStatus binarySearchNonLeaf(BytesRef target, boolean exactOnly) throws IOException { + + // if (DEBUG) System.out.println(" scanToTermNonLeaf: block fp=" + fp + " prefix=" + prefix + + // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + + // ToStringUtils.bytesRefToString(target) + + // " term=" + ToStringUtils.bytesRefToString(term)); + + assert nextEnt != -1; + + if (nextEnt == entCount) { + if (exactOnly) { + fillTerm(); + ste.termExists = subCode == 0; + } + return SeekStatus.END; + } + + assert prefixMatches(target); + assert assertOffset(suffixes, offsets); + + int start = nextEnt; + int end = entCount - 1; + // Binary search the entries (terms) in this non leaf block: + int cmp = 0; + int termLen = 0; + int lastMid = 0; + + while (start <= end) { + int mid = (start + end) / 2; + nextEnt = mid + 1; + startBytePos = offsets[mid]; + suffix = suffixes[mid]; + suffixesReader.setPosition(startBytePos + suffix); + + termLen = prefix + suffix; + ste.termExists = termExists.get(mid); + if (ste.termExists) { + state.termBlockOrd++; + subCode = 0; + } else { + // subCode = subCodes[mid]; + // lastSubFP = fp - subCode; + } + + // Binary search bytes in the suffix, comparing to the target + cmp = + Arrays.compareUnsigned( + suffixBytes, + startBytePos, + startBytePos + suffix, + target.bytes, + target.offset + prefix, + target.offset + target.length); + if (cmp < 0) { + start = mid + 1; + } else if (cmp > 0) { + end = mid - 1; + lastMid = mid; + } else { + // Exact match! + + // This cannot be a sub-block because we + // would have followed the index to this + // sub-block from the start: + + assert ste.termExists; + fillTerm(); + // if (DEBUG) System.out.println(" found!"); + return SeekStatus.FOUND; + } + } + + // It is possible (and OK) that terms index pointed us + // at this block, but, we scanned the entire block and + // did not find the term to position to. This happens + // when the target is after the last term in the block + // (but, before the next term in the index). EG + // target could be foozzz, and terms index pointed us + // to the foo* block, but the last term in this block + // was fooz (and, eg, first term in the next block will + // bee fop). + // if (DEBUG) System.out.println(" block end"); + SeekStatus seekStatus = end < entCount - 1 ? SeekStatus.NOT_FOUND : SeekStatus.END; + if (exactOnly || seekStatus == SeekStatus.NOT_FOUND) { + // If binary search ended at the less term, and greater term exists. + // We need to advance to the greater term. + if (cmp < 0 && seekStatus == SeekStatus.NOT_FOUND) { + startBytePos += suffix; + suffixesReader.skipBytes(suffix); + nextEnt++; + } + fillTerm(); + } + + if (seekStatus == SeekStatus.NOT_FOUND && !exactOnly && !termExists.get(lastMid)) { + subCode = subCodes[lastMid]; + lastSubFP = fp - subCode; + // System.out.println(" now pushFrame"); + // TODO this + // We are on a sub-block, and caller wants + // us to position to the next term after + // the target, so we must recurse into the + // sub-frame(s): + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen); + ste.currentFrame.loadBlock(); + while (ste.currentFrame.next()) { + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length()); + ste.currentFrame.loadBlock(); + } + } + + // TODO: not consistent that in the + // not-exact case we don't next() into the next + // frame here + return seekStatus; + } + private void fillTerm() { final int termLength = prefix + suffix; ste.term.setLength(termLength); diff --git a/lucene/core/src/java/org/apache/lucene/store/DataInput.java b/lucene/core/src/java/org/apache/lucene/store/DataInput.java index 781066f02ab5..96aee04b527b 100644 --- a/lucene/core/src/java/org/apache/lucene/store/DataInput.java +++ b/lucene/core/src/java/org/apache/lucene/store/DataInput.java @@ -189,6 +189,21 @@ public void readInts(int[] dst, int offset, int length) throws IOException { } } + // TODO: read VLongs, etc. + /** + * Reads a specified number of vints into an array at the specified offset. + * + * @param dst the array to read bytes into + * @param offset the offset in the array to start storing ints + * @param length the number of vints to read + */ + public void readVInts(int[] dst, int offset, int length) throws IOException { + Objects.checkFromIndexSize(offset, length, dst.length); + for (int i = 0; i < length; ++i) { + dst[offset + i] = readVInt(); + } + } + /** * Reads a specified number of floats into an array at the specified offset. * diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java index 99c0e0a6ae28..07d55452dd4a 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java @@ -29,10 +29,7 @@ import org.apache.lucene.codecs.lucene99.Lucene99ScoreSkipReader.MutableImpactList; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.Impact; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.*; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -41,6 +38,7 @@ import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.index.BasePostingsFormatTestCase; import org.apache.lucene.tests.util.TestUtil; +import org.apache.lucene.util.BytesRef; public class TestLucene99PostingsFormat extends BasePostingsFormatTestCase { private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene99PostingsFormat()); @@ -143,4 +141,13 @@ private void doTestImpactSerialization(List impacts) throws IOException } } } + + @Override + protected void subCheckBinarySearch(TermsEnum termsEnum) throws Exception { + // 10004a matched block's entries: [100001, 100003, ..., 100049]. + // if target greater than the last entry of the matched block, + // termsEnum.term should be the last entry. + assertFalse(termsEnum.seekExact(new BytesRef("10004a"))); + assertEquals(termsEnum.term(), new BytesRef("100049")); + } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java index 45e8bd8e858b..e3684608df46 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java @@ -58,9 +58,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.tests.analysis.CannedTokenStream; @@ -369,6 +367,151 @@ public void testGhosts() throws Exception { dir.close(); } + protected void subCheckBinarySearch(TermsEnum termsEnum) throws Exception {} + + // public void testSubBlock() throws Exception { + // Directory dir = newDirectory(); + // // Set minTermBlockSize to 2, maxTermBlockSize to 3, to generate subBlock. + // PostingsFormat postingsFormat = getDefaultPostingsFormat(2, 3); + // + // IndexWriter iw = + // new IndexWriter(dir, + // newIndexWriterConfig().setCodec(alwaysPostingsFormat(postingsFormat))); + // String[] categories = new String[] {"regular", "request", "rest", "teacher", "team"}; + // + // for (String category : categories) { + // Document doc = new Document(); + // doc.add(newStringField("category", category, Field.Store.YES)); + // iw.addDocument(doc); + // } + // + // IndexReader reader = DirectoryReader.open(iw); + // iw.commit(); + // iw.forceMerge(1); + // + // TermsEnum termsEnum = getOnlyLeafReader(reader).terms("category").iterator(); + // + // // test seekExact. + // for (String category : categories) { + // BytesRef target = newBytesRef(category); + // assertTrue(termsEnum.seekExact(target)); + // assertEquals(termsEnum.term(), target); + // } + // + // // test seekCeil. + // for (String category : categories) { + // BytesRef target = newBytesRef(category); + // assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(target)); + // assertEquals(termsEnum.term(), target); + // } + // + // iw.close(); + // reader.close(); + // dir.close(); + // } + + // public void testDeepSubBlock() throws Exception { + // Directory dir = newDirectory(); + // // Set minTermBlockSize to 2, maxTermBlockSize to 3, to generate deep subBlock. + // PostingsFormat postingsFormat = getDefaultPostingsFormat(2, 3); + // + // IndexWriter iw = + // new IndexWriter(dir, + // newIndexWriterConfig().setCodec(alwaysPostingsFormat(postingsFormat))); + // String[] categories = + // new String[] { + // "regular", "request1", "request2", "request3", "request4", "rest", "teacher", "team" + // }; + // + // for (String category : categories) { + // Document doc = new Document(); + // doc.add(newStringField("category", category, Field.Store.YES)); + // iw.addDocument(doc); + // } + // + // IndexReader reader = DirectoryReader.open(iw); + // iw.commit(); + // iw.forceMerge(1); + // + // TermsEnum termsEnum = getOnlyLeafReader(reader).terms("category").iterator(); + // + // BytesRef target = newBytesRef("reques"); + // assertFalse(termsEnum.seekExact(target)); + // assertEquals(termsEnum.term(), new BytesRef("request")); + // assertEquals(SeekStatus.NOT_FOUND, termsEnum.seekCeil(target)); + // + // // test seekExact. + // for (String category : categories) { + // target = newBytesRef(category); + // assertTrue(termsEnum.seekExact(target)); + // assertEquals(termsEnum.term(), target); + // } + // + // // test seekCeil. + // for (String category : categories) { + // target = newBytesRef(category); + // assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(target)); + // assertEquals(termsEnum.term(), target); + // } + // + // iw.close(); + // reader.close(); + // dir.close(); + // } + + public void testBinarySearchTermLeaf() throws Exception { + Directory dir = newDirectory(); + + IndexWriterConfig iwc = newIndexWriterConfig(null); + iwc.setCodec(getCodec()); + iwc.setMergePolicy(newTieredMergePolicy()); + IndexWriter iw = new IndexWriter(dir, iwc); + + for (int i = 100000; i <= 100400; i++) { + // only add odd number + if (i % 2 == 1) { + Document document = new Document(); + document.add(new StringField("id", i + "", Field.Store.NO)); + iw.addDocument(document); + } + } + iw.commit(); + iw.forceMerge(1); + + DirectoryReader reader = DirectoryReader.open(iw); + TermsEnum termsEnum = getOnlyLeafReader(reader).terms("id").iterator(); + // test seekExact + for (int i = 100000; i <= 100400; i++) { + BytesRef target = new BytesRef(i + ""); + if (i % 2 == 1) { + assertTrue(termsEnum.seekExact(target)); + assertEquals(termsEnum.term(), target); + } else { + assertFalse(termsEnum.seekExact(target)); + } + } + + subCheckBinarySearch(termsEnum); + // test seekCeil + for (int i = 100000; i < 100400; i++) { + BytesRef target = new BytesRef(i + ""); + if (i % 2 == 1) { + assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(target)); + assertEquals(termsEnum.term(), target); + if (i <= 100397) { + assertEquals(new BytesRef(i + 2 + ""), termsEnum.next()); + } + } else { + assertEquals(SeekStatus.NOT_FOUND, termsEnum.seekCeil(target)); + assertEquals(new BytesRef(i + 1 + ""), termsEnum.term()); + } + } + assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef(100400 + ""))); + reader.close(); + iw.close(); + dir.close(); + } + // tests that level 2 ghost fields still work public void testLevel2Ghosts() throws Exception { Directory dir = newDirectory(); From d3dc1c660acbe80cd2687f9c056ded49213c8f81 Mon Sep 17 00:00:00 2001 From: zhouhui Date: Sat, 23 Mar 2024 20:07:56 +0800 Subject: [PATCH 2/7] Binary search term non leaf. --- .../blocktree/SegmentTermsEnumFrame.java | 563 ++++++++---------- 1 file changed, 263 insertions(+), 300 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java index 331720a12644..ce448b487498 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java @@ -55,6 +55,8 @@ final class SegmentTermsEnumFrame { int[] suffixes; int[] postions; int[] offsets; + int[] termBlockOrds; + int[] lastSubIndices; FixedBitSet termExists; long[] subCodes; byte[] statBytes = new byte[64]; @@ -206,9 +208,53 @@ void loadBlock() throws IOException { totalSuffixBytes = ste.in.getFilePointer() - startSuffixFP; // Prepare suffixes, offsets to binary search. - if (allEqual == false) { + if (allEqual) { + if (isLeafBlock) { + suffix = suffixLengthsReader.readVInt(); + } else { + // Handle subCode for non leaf block. + postions = new int[entCount]; + termExists = new FixedBitSet(entCount); + subCodes = new long[entCount]; + termBlockOrds = new int[entCount]; + lastSubIndices = new int[entCount]; + int termBlockOrd = 0; + int lastSubIndex = -1; + // read first vint to set suffix, byt the way, set termExist, subCode. + code = suffixLengthsReader.readVInt(); + suffix = code >>> 1; + if ((code & 1) == 0) { + termExists.set(0); + termBlockOrd++; + } else { + // read subCode. + subCodes[0] = suffixLengthsReader.readVLong(); + lastSubIndex = 0; + } + termBlockOrds[0] = termBlockOrd; + postions[0] = suffixLengthsReader.getPosition(); + lastSubIndices[0] = lastSubIndex; + for (int i = 1; i < suffixes.length; i++) { + code = suffixLengthsReader.readVInt(); + suffixes[i] = code >>> 1; + if ((code & 1) == 0) { + termExists.set(i); + termBlockOrd++; + } else { + // read subCode. + subCodes[i] = suffixLengthsReader.readVLong(); + lastSubIndex = i; + } + termBlockOrds[i] = termBlockOrd; + postions[i] = suffixLengthsReader.getPosition(); + lastSubIndices[i] = lastSubIndex; + } + } + // Reset suffixLengthsReader's position. + suffixLengthsReader.setPosition(0); + } else { suffixes = new int[entCount]; - // TODO: remove postions if it is necessary. + // TODO: remove postions if it is unnecessary. postions = new int[entCount]; if (isLeafBlock) { for (int i = 0; i < suffixes.length; i++) { @@ -219,16 +265,24 @@ void loadBlock() throws IOException { // Handle subCode for non leaf block. termExists = new FixedBitSet(entCount); subCodes = new long[entCount]; + termBlockOrds = new int[entCount]; + lastSubIndices = new int[entCount]; + int termBlockOrd = 0; + int lastSubIndex = -1; for (int i = 0; i < suffixes.length; i++) { code = suffixLengthsReader.readVInt(); suffixes[i] = code >>> 1; if ((code & 1) == 0) { termExists.set(i); + termBlockOrd++; } else { // read subCode. subCodes[i] = suffixLengthsReader.readVLong(); + lastSubIndex = i; } + termBlockOrds[i] = termBlockOrd; postions[i] = suffixLengthsReader.getPosition(); + lastSubIndices[i] = lastSubIndex; } } // Reset suffixLengthsReader's position. @@ -566,7 +620,7 @@ public void scanToSubBlock(long subFP) { public SeekStatus scanToTerm(BytesRef target, boolean exactOnly) throws IOException { return isLeafBlock ? binarySearchTermLeaf(target, exactOnly) - : scanToTermNonLeaf(target, exactOnly); + : binarySearchTermNonLeaf(target, exactOnly); } private int startBytePos; @@ -574,131 +628,43 @@ public SeekStatus scanToTerm(BytesRef target, boolean exactOnly) throws IOExcept private long subCode; CompressionAlgorithm compressionAlg = CompressionAlgorithm.NO_COMPRESSION; - // Target's prefix matches this block's prefix; we - // scan the entries check if the suffix matches. - public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOException { - - // if (DEBUG) System.out.println(" scanToTermLeaf: block fp=" + fp + " prefix=" + prefix + - // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + - // ToStringUtils.bytesRefToString(target) + - // " term=" + ToStringUtils.bytesRefToString(term)); - - assert nextEnt != -1; - - ste.termExists = true; - subCode = 0; - - if (nextEnt == entCount) { - if (exactOnly) { - fillTerm(); - } - return SeekStatus.END; + private int[] getOffsets(int[] suffixes) { + int[] offsets = new int[suffixes.length]; + offsets[0] = 0; + for (int i = 1; i < suffixes.length; i++) { + offsets[i] = offsets[i - 1] + suffixes[i - 1]; } + return offsets; + } - assert prefixMatches(target); - - // Loop over each entry (term or sub-block) in this block: - do { - nextEnt++; - - suffix = suffixLengthsReader.readVInt(); - - // if (DEBUG) { - // BytesRef suffixBytesRef = new BytesRef(); - // suffixBytesRef.bytes = suffixBytes; - // suffixBytesRef.offset = suffixesReader.getPosition(); - // suffixBytesRef.length = suffix; - // System.out.println(" cycle: term " + (nextEnt-1) + " (of " + entCount + ") suffix=" - // + ToStringUtils.bytesRefToString(suffixBytesRef)); - // } - - startBytePos = suffixesReader.getPosition(); - suffixesReader.skipBytes(suffix); - - // Loop over bytes in the suffix, comparing to the target - final int cmp = - Arrays.compareUnsigned( - suffixBytes, - startBytePos, - startBytePos + suffix, - target.bytes, - target.offset + prefix, - target.offset + target.length); - - if (cmp < 0) { - // Current entry is still before the target; - // keep scanning - } else if (cmp > 0) { - // Done! Current entry is after target -- - // return NOT_FOUND: - fillTerm(); - - // System.out.println("target: " + target.utf8ToString() - // + ", nextEnt: " + nextEnt - // + ", startBytePos: " + startBytePos - // + ", suffix: " + suffix - // + ", lenPos: " + suffixLengthsReader.getPosition() - // + ", sufPos: " + suffixesReader.getPosition() - // ); - - // if (DEBUG) System.out.println(" not found"); - return SeekStatus.NOT_FOUND; - } else { - // Exact match! - - // This cannot be a sub-block because we - // would have followed the index to this - // sub-block from the start: - - assert ste.termExists; - fillTerm(); - // if (DEBUG) System.out.println(" found!"); - // System.out.println("target: " + target.utf8ToString() - // + ", nextEnt: " + nextEnt - // + ", startBytePos: " + startBytePos - // + ", suffix: " + suffix - // + ", lenPos: " + suffixLengthsReader.getPosition() - // + ", sufPos: " + suffixesReader.getPosition() - // ); - return SeekStatus.FOUND; + // Used only by assert + private boolean assertOffset(int[] suffixes, int[] offsets) { + if (offsets[0] != 0) { + return false; + } + for (int i = 1; i < offsets.length; i++) { + if (suffixes[i - 1] != offsets[i] - offsets[i - 1]) { + return false; } - } while (nextEnt < entCount); - - // It is possible (and OK) that terms index pointed us - // at this block, but, we scanned the entire block and - // did not find the term to position to. This happens - // when the target is after the last term in the block - // (but, before the next term in the index). EG - // target could be foozzz, and terms index pointed us - // to the foo* block, but the last term in this block - // was fooz (and, eg, first term in the next block will - // bee fop). - // if (DEBUG) System.out.println(" block end"); - if (exactOnly) { - fillTerm(); } + return true; + } - // System.out.println("target: " + target.utf8ToString() - // + ", nextEnt: " + nextEnt - // + ", startBytePos: " + startBytePos - // + ", suffix: " + suffix - // + ", lenPos: " + suffixLengthsReader.getPosition() - // + ", sufPos: " + suffixesReader.getPosition() - // ); - - // TODO: not consistent that in the - // not-exact case we don't next() into the next - // frame here - return SeekStatus.END; + // Target's prefix matches this leaf block's prefix; + // we binary search the entries check if the suffix matches. + public SeekStatus binarySearchTermLeaf(BytesRef target, boolean exactOnly) throws IOException { + return allEqual + ? binarySearchTermLeafAllEqual(target, exactOnly) + : binarySearchTermLeafUnEqual(target, exactOnly); } - // TODO: take this to scanToTermNonLeaf - // Target's prefix matches this block's prefix; - // But these suffixes' length are not same, + // Target's prefix matches this leaf block's prefix; + // And all suffixes have the same length in this block, // we binary search the entries check if the suffix matches. - private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnly) + private SeekStatus binarySearchTermLeafAllEqual(BytesRef target, boolean exactOnly) throws IOException { - // if (DEBUG) System.out.println(" binarySearchTermLeaf: block fp=" + fp + " prefix=" + + // if (DEBUG) System.out.println(" binarySearchTermLeafAllEqual: block fp=" + fp + " prefix=" + // + // prefix + " // nextEnt=" + nextEnt + " (of " + entCount + ") target=" + brToString(target) + " term=" + // brToString(term)); @@ -716,22 +682,17 @@ private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnl } assert prefixMatches(target); - assert assertOffset(suffixes, offsets); + // TODO early terminate when target length unequals suffix + prefix. + // But we need to keep the same status with scanToTermLeaf. int start = nextEnt; int end = entCount - 1; // Binary search the entries (terms) in this leaf block: int cmp = 0; - int lastGreaterPos = 0; while (start <= end) { int mid = (start + end) / 2; nextEnt = mid + 1; - startBytePos = offsets[mid]; - suffix = suffixes[mid]; - suffixesReader.setPosition(startBytePos + suffix); - - // TODO: is it necessary to set suffixLengthsReader's position? - suffixLengthsReader.setPosition(postions[mid]); + startBytePos = mid * suffix; // Binary search bytes in the suffix, comparing to the target cmp = @@ -746,24 +707,15 @@ private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnl start = mid + 1; } else if (cmp > 0) { end = mid - 1; - lastGreaterPos = mid; } else { // Exact match! - + suffixesReader.setPosition(startBytePos + suffix); // This cannot be a sub-block because we // would have followed the index to this // sub-block from the start: - assert ste.termExists; fillTerm(); // if (DEBUG) System.out.println(" found!"); - // System.out.println("target: " + target.utf8ToString() - // + ", nextEnt: " + nextEnt - // + ", startBytePos: " + startBytePos - // + ", suffix: " + suffix - // + ", lenPos: " + suffixLengthsReader.getPosition() - // + ", sufPos: " + suffixesReader.getPosition() - // ); return SeekStatus.FOUND; } } @@ -779,69 +731,34 @@ private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnl // bee fop). // if (DEBUG) System.out.println(" block end"); SeekStatus seekStatus = end < entCount - 1 ? SeekStatus.NOT_FOUND : SeekStatus.END; - if (exactOnly || seekStatus == SeekStatus.NOT_FOUND) { + if (seekStatus == SeekStatus.NOT_FOUND) { // If binary search ended at the less term, and greater term exists. // We need to advance to the greater term. - if (cmp < 0 && seekStatus == SeekStatus.NOT_FOUND) { - suffix = suffixes[lastGreaterPos]; - startBytePos = offsets[lastGreaterPos]; - suffixesReader.setPosition(startBytePos + suffix); - nextEnt = lastGreaterPos + 1; - - suffixLengthsReader.setPosition(postions[lastGreaterPos]); + if (cmp < 0) { + startBytePos += suffix; + nextEnt++; } + suffixesReader.setPosition(startBytePos + suffix); fillTerm(); + } else { + suffixesReader.setPosition(startBytePos + suffix); + if (exactOnly) { + fillTerm(); + } } - // TODO: not consistent that in the // not-exact case we don't next() into the next // frame here - // System.out.println("target: " + target.utf8ToString() - // + ", nextEnt: " + nextEnt - // + ", startBytePos: " + startBytePos - // + ", suffix: " + suffix - // + ", lenPos: " + suffixLengthsReader.getPosition() - // + ", sufPos: " + suffixesReader.getPosition() - // ); return seekStatus; } - private int[] getOffsets(int[] suffixes) { - int[] offsets = new int[suffixes.length]; - offsets[0] = 0; - for (int i = 1; i < suffixes.length; i++) { - offsets[i] = offsets[i - 1] + suffixes[i - 1]; - } - return offsets; - } - - // Used only by assert - private boolean assertOffset(int[] suffixes, int[] offsets) { - if (offsets[0] != 0) { - return false; - } - for (int i = 1; i < offsets.length; i++) { - if (suffixes[i - 1] != offsets[i] - offsets[i - 1]) { - return false; - } - } - return true; - } - - // Target's prefix matches this block's prefix; - // we binary search the entries check if the suffix matches. - public SeekStatus binarySearchTermLeaf(BytesRef target, boolean exactOnly) throws IOException { - return allEqual - ? binarySearchTermLeafAllEqual(target, exactOnly) - : binarySearchTermLeafUnEqual(target, exactOnly); - } - - // Target's prefix matches this block's prefix; - // And all suffixes have the same length in this block, + // Target's prefix matches this leaf block's prefix; + // But these suffixes' length are not same, // we binary search the entries check if the suffix matches. - private SeekStatus binarySearchTermLeafAllEqual(BytesRef target, boolean exactOnly) + private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnly) throws IOException { - // if (DEBUG) System.out.println(" binarySearchTermLeaf: block fp=" + fp + " prefix=" + + // if (DEBUG) System.out.println(" binarySearchTermLeafUnEqual: block fp=" + fp + " prefix=" + // + // prefix + " // nextEnt=" + nextEnt + " (of " + entCount + ") target=" + brToString(target) + " term=" + // brToString(term)); @@ -859,10 +776,8 @@ private SeekStatus binarySearchTermLeafAllEqual(BytesRef target, boolean exactOn } assert prefixMatches(target); + assert assertOffset(suffixes, offsets); - suffix = suffixLengthsReader.readVInt(); - // TODO early terminate when target length unequals suffix + prefix. - // But we need to keep the same status with scanToTermLeaf. int start = nextEnt; int end = entCount - 1; // Binary search the entries (terms) in this leaf block: @@ -870,8 +785,8 @@ private SeekStatus binarySearchTermLeafAllEqual(BytesRef target, boolean exactOn while (start <= end) { int mid = (start + end) / 2; nextEnt = mid + 1; - startBytePos = mid * suffix; - suffixesReader.setPosition(startBytePos + suffix); + startBytePos = offsets[mid]; + suffix = suffixes[mid]; // Binary search bytes in the suffix, comparing to the target cmp = @@ -889,6 +804,7 @@ private SeekStatus binarySearchTermLeafAllEqual(BytesRef target, boolean exactOn } else { // Exact match! + setSuffixesReaderUnEqual(); // This cannot be a sub-block because we // would have followed the index to this // sub-block from the start: @@ -911,28 +827,44 @@ private SeekStatus binarySearchTermLeafAllEqual(BytesRef target, boolean exactOn // bee fop). // if (DEBUG) System.out.println(" block end"); SeekStatus seekStatus = end < entCount - 1 ? SeekStatus.NOT_FOUND : SeekStatus.END; - if (exactOnly || seekStatus == SeekStatus.NOT_FOUND) { + if (seekStatus == SeekStatus.NOT_FOUND) { // If binary search ended at the less term, and greater term exists. // We need to advance to the greater term. - if (cmp < 0 && seekStatus == SeekStatus.NOT_FOUND) { - startBytePos += suffix; - suffixesReader.skipBytes(suffix); + if (cmp < 0) { nextEnt++; } + setSuffixesReaderUnEqual(); fillTerm(); + } else { + setSuffixesReaderUnEqual(); + if (exactOnly) { + fillTerm(); + } } // TODO: not consistent that in the // not-exact case we don't next() into the next // frame here + return seekStatus; } - // Target's prefix matches this block's prefix; we - // scan the entries check if the suffix matches. - public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws IOException { + // Target's prefix matches this non-leaf block's prefix; + // we binary search the entries check if the suffix matches. + public SeekStatus binarySearchTermNonLeaf(BytesRef target, boolean exactOnly) throws IOException { + return allEqual + ? binarySearchTermNonLeafAllEqual(target, exactOnly) + : binarySearchTermNonLeafUnEqual(target, exactOnly); + } + + // Target's prefix matches this non-leaf block's prefix; + // And all suffixes have the same length in this block, + // we binary search the entries check if the suffix matches. + private SeekStatus binarySearchTermNonLeafAllEqual(BytesRef target, boolean exactOnly) + throws IOException { - // if (DEBUG) System.out.println(" scanToTermNonLeaf: block fp=" + fp + " prefix=" + prefix + + // if (DEBUG) System.out.println(" binarySearchTermNonLeafAllEqual: block fp=" + fp + " + // prefix=" + prefix + // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + // ToStringUtils.bytesRefToString(target) + // " term=" + ToStringUtils.bytesRefToString(term)); @@ -948,38 +880,21 @@ public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws I } assert prefixMatches(target); + assert assertOffset(suffixes, offsets); - // Loop over each entry (term or sub-block) in this block: - while (nextEnt < entCount) { - - nextEnt++; - - final int code = suffixLengthsReader.readVInt(); - suffix = code >>> 1; - - // if (DEBUG) { - // BytesRef suffixBytesRef = new BytesRef(); - // suffixBytesRef.bytes = suffixBytes; - // suffixBytesRef.offset = suffixesReader.getPosition(); - // suffixBytesRef.length = suffix; - // System.out.println(" cycle: " + ((code&1)==1 ? "sub-block" : "term") + " " + - // (nextEnt-1) + " (of " + entCount + ") suffix=" + - // ToStringUtils.bytesRefToString(suffixBytesRef)); - // } + int start = nextEnt; + int end = entCount - 1; + // Binary search the entries (terms) in this non leaf block: + int cmp = 0; + int mid = 0; - final int termLen = prefix + suffix; - startBytePos = suffixesReader.getPosition(); - suffixesReader.skipBytes(suffix); - ste.termExists = (code & 1) == 0; - if (ste.termExists) { - state.termBlockOrd++; - subCode = 0; - } else { - subCode = suffixLengthsReader.readVLong(); - lastSubFP = fp - subCode; - } + while (start <= end) { + mid = (start + end) / 2; + nextEnt = mid + 1; + startBytePos = mid * suffix; - final int cmp = + // Binary search bytes in the suffix, comparing to the target + cmp = Arrays.compareUnsigned( suffixBytes, startBytePos, @@ -987,45 +902,23 @@ public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws I target.bytes, target.offset + prefix, target.offset + target.length); - if (cmp < 0) { - // Current entry is still before the target; - // keep scanning + start = mid + 1; } else if (cmp > 0) { - // Done! Current entry is after target -- - // return NOT_FOUND: - fillTerm(); - - // if (DEBUG) System.out.println(" maybe done exactOnly=" + exactOnly + - // " ste.termExists=" + ste.termExists); - - if (!exactOnly && !ste.termExists) { - // System.out.println(" now pushFrame"); - // TODO this - // We are on a sub-block, and caller wants - // us to position to the next term after - // the target, so we must recurse into the - // sub-frame(s): - ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen); - ste.currentFrame.loadBlock(); - while (ste.currentFrame.next()) { - ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length()); - ste.currentFrame.loadBlock(); - } - } - - // if (DEBUG) System.out.println(" not found"); - return SeekStatus.NOT_FOUND; + end = mid - 1; } else { // Exact match! + setSuffixesReaderAllEqual(); + setSubStates(); // This cannot be a sub-block because we // would have followed the index to this // sub-block from the start: - assert ste.termExists; + fillTerm(); // if (DEBUG) System.out.println(" found!"); + return SeekStatus.FOUND; } } @@ -1040,21 +933,52 @@ public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws I // was fooz (and, eg, first term in the next block will // bee fop). // if (DEBUG) System.out.println(" block end"); - if (exactOnly) { + SeekStatus seekStatus = end < entCount - 1 ? SeekStatus.NOT_FOUND : SeekStatus.END; + // If binary search ended at the less term, and greater term exists. + // We need to advance to the greater term. + if (seekStatus == SeekStatus.NOT_FOUND) { + if (cmp < 0) { + nextEnt++; + } + setSuffixesReaderAllEqual(); + setSubStates(); + fillTerm(); + if (!exactOnly && !termExists.get(nextEnt - 1)) { + // System.out.println(" now pushFrame"); + // TODO this + // We are on a sub-block, and caller wants + // us to position to the next term after + // the target, so we must recurse into the + // sub-frame(s): + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, prefix + suffix); + ste.currentFrame.loadBlock(); + while (ste.currentFrame.next()) { + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length()); + ste.currentFrame.loadBlock(); + } + } + } else { + setSuffixesReaderAllEqual(); + setSubStates(); + if (exactOnly) { + fillTerm(); + } } - // TODO: not consistent that in the // not-exact case we don't next() into the next // frame here - return SeekStatus.END; + return seekStatus; } - // Target's prefix matches this block's prefix; we - // binary search the entries check if the suffix matches. - public SeekStatus binarySearchNonLeaf(BytesRef target, boolean exactOnly) throws IOException { + // Target's prefix matches this non-leaf block's prefix; + // But these suffixes' length are not same, + // we binary search the entries check if the suffix matches. + private SeekStatus binarySearchTermNonLeafUnEqual(BytesRef target, boolean exactOnly) + throws IOException { - // if (DEBUG) System.out.println(" scanToTermNonLeaf: block fp=" + fp + " prefix=" + prefix + + // if (DEBUG) System.out.println(" binarySearchTermNonLeafUnEqual: block fp=" + fp + " + // prefix=" + prefix + // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + // ToStringUtils.bytesRefToString(target) + // " term=" + ToStringUtils.bytesRefToString(term)); @@ -1076,25 +1000,13 @@ public SeekStatus binarySearchNonLeaf(BytesRef target, boolean exactOnly) throws int end = entCount - 1; // Binary search the entries (terms) in this non leaf block: int cmp = 0; - int termLen = 0; - int lastMid = 0; + int mid = 0; while (start <= end) { - int mid = (start + end) / 2; + mid = (start + end) / 2; nextEnt = mid + 1; startBytePos = offsets[mid]; suffix = suffixes[mid]; - suffixesReader.setPosition(startBytePos + suffix); - - termLen = prefix + suffix; - ste.termExists = termExists.get(mid); - if (ste.termExists) { - state.termBlockOrd++; - subCode = 0; - } else { - // subCode = subCodes[mid]; - // lastSubFP = fp - subCode; - } // Binary search bytes in the suffix, comparing to the target cmp = @@ -1109,15 +1021,17 @@ public SeekStatus binarySearchNonLeaf(BytesRef target, boolean exactOnly) throws start = mid + 1; } else if (cmp > 0) { end = mid - 1; - lastMid = mid; } else { // Exact match! + setSuffixesReaderUnEqual(); + setSubStates(); // This cannot be a sub-block because we // would have followed the index to this // sub-block from the start: assert ste.termExists; + fillTerm(); // if (DEBUG) System.out.println(" found!"); return SeekStatus.FOUND; @@ -1135,31 +1049,36 @@ public SeekStatus binarySearchNonLeaf(BytesRef target, boolean exactOnly) throws // bee fop). // if (DEBUG) System.out.println(" block end"); SeekStatus seekStatus = end < entCount - 1 ? SeekStatus.NOT_FOUND : SeekStatus.END; - if (exactOnly || seekStatus == SeekStatus.NOT_FOUND) { - // If binary search ended at the less term, and greater term exists. - // We need to advance to the greater term. - if (cmp < 0 && seekStatus == SeekStatus.NOT_FOUND) { - startBytePos += suffix; - suffixesReader.skipBytes(suffix); + // If binary search ended at the less term, and greater term exists. + // We need to advance to the greater term. + if (seekStatus == SeekStatus.NOT_FOUND) { + if (cmp < 0) { nextEnt++; } - fillTerm(); - } - if (seekStatus == SeekStatus.NOT_FOUND && !exactOnly && !termExists.get(lastMid)) { - subCode = subCodes[lastMid]; - lastSubFP = fp - subCode; - // System.out.println(" now pushFrame"); - // TODO this - // We are on a sub-block, and caller wants - // us to position to the next term after - // the target, so we must recurse into the - // sub-frame(s): - ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen); - ste.currentFrame.loadBlock(); - while (ste.currentFrame.next()) { - ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length()); + setSuffixesReaderUnEqual(); + setSubStates(); + + fillTerm(); + if (!exactOnly && !termExists.get(nextEnt - 1)) { + // System.out.println(" now pushFrame"); + // TODO this + // We are on a sub-block, and caller wants + // us to position to the next term after + // the target, so we must recurse into the + // sub-frame(s): + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, prefix + suffix); ste.currentFrame.loadBlock(); + while (ste.currentFrame.next()) { + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length()); + ste.currentFrame.loadBlock(); + } + } + } else { + setSuffixesReaderUnEqual(); + setSubStates(); + if (exactOnly) { + fillTerm(); } } @@ -1169,6 +1088,50 @@ public SeekStatus binarySearchNonLeaf(BytesRef target, boolean exactOnly) throws return seekStatus; } + // Set sub block's states. + private void setSubStates() { + int currentEnt = nextEnt - 1; + + // Set blockOrd, termExists. + state.termBlockOrd = termBlockOrds[currentEnt]; + ste.termExists = termExists.get(currentEnt); + + // Set subCode. + subCode = subCodes[currentEnt]; + + // Set lastSubFP. + if (lastSubIndices[currentEnt] != -1) { + lastSubFP = fp - subCodes[lastSubIndices[currentEnt]]; + } else { + lastSubFP = -1; + } + } + + // Set suffixesReader's position. + private void setSuffixesReaderAllEqual() { + int currentEnt = nextEnt - 1; + + startBytePos = currentEnt * suffix; + suffixesReader.setPosition(startBytePos + suffix); + + // Set suffixLengthsReader's position. + // TODO: is it necessary to set suffixLengthsReader's position? + suffixLengthsReader.setPosition(postions[currentEnt]); + } + + // Set suffixesReader's position. + private void setSuffixesReaderUnEqual() { + int currentEnt = nextEnt - 1; + + startBytePos = offsets[currentEnt]; + suffix = suffixes[currentEnt]; + suffixesReader.setPosition(startBytePos + suffix); + + // Set suffixLengthsReader's position. + // TODO: is it necessary to set suffixLengthsReader's position? + suffixLengthsReader.setPosition(postions[currentEnt]); + } + private void fillTerm() { final int termLength = prefix + suffix; ste.term.setLength(termLength); From a7783bed97b3b75bb42ecf65f37edba2af961c08 Mon Sep 17 00:00:00 2001 From: zhouhui Date: Fri, 26 Apr 2024 10:49:17 +0800 Subject: [PATCH 3/7] Revert conflict files. --- .../blocktree/SegmentTermsEnumFrame.java | 576 +++--------------- .../lucene99/TestLucene99PostingsFormat.java | 15 +- .../index/BasePostingsFormatTestCase.java | 151 +---- 3 files changed, 105 insertions(+), 637 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java index ce448b487498..66231313e520 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java @@ -25,7 +25,6 @@ import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.fst.FST; final class SegmentTermsEnumFrame { @@ -52,13 +51,6 @@ final class SegmentTermsEnumFrame { byte[] suffixLengthBytes; final ByteArrayDataInput suffixLengthsReader; - int[] suffixes; - int[] postions; - int[] offsets; - int[] termBlockOrds; - int[] lastSubIndices; - FixedBitSet termExists; - long[] subCodes; byte[] statBytes = new byte[64]; int statsSingletonRunLength = 0; final ByteArrayDataInput statsReader = new ByteArrayDataInput(); @@ -83,9 +75,6 @@ final class SegmentTermsEnumFrame { // True if all entries are terms boolean isLeafBlock; - // True if all entries have the same length. - boolean allEqual; - long lastSubFP; int nextFloorLabel; @@ -194,7 +183,7 @@ void loadBlock() throws IOException { suffixesReader.reset(suffixBytes, 0, numSuffixBytes); int numSuffixLengthBytes = ste.in.readVInt(); - allEqual = (numSuffixLengthBytes & 0x01) != 0; + final boolean allEqual = (numSuffixLengthBytes & 0x01) != 0; numSuffixLengthBytes >>>= 1; if (suffixLengthBytes.length < numSuffixLengthBytes) { suffixLengthBytes = new byte[ArrayUtil.oversize(numSuffixLengthBytes, 1)]; @@ -207,90 +196,6 @@ void loadBlock() throws IOException { suffixLengthsReader.reset(suffixLengthBytes, 0, numSuffixLengthBytes); totalSuffixBytes = ste.in.getFilePointer() - startSuffixFP; - // Prepare suffixes, offsets to binary search. - if (allEqual) { - if (isLeafBlock) { - suffix = suffixLengthsReader.readVInt(); - } else { - // Handle subCode for non leaf block. - postions = new int[entCount]; - termExists = new FixedBitSet(entCount); - subCodes = new long[entCount]; - termBlockOrds = new int[entCount]; - lastSubIndices = new int[entCount]; - int termBlockOrd = 0; - int lastSubIndex = -1; - // read first vint to set suffix, byt the way, set termExist, subCode. - code = suffixLengthsReader.readVInt(); - suffix = code >>> 1; - if ((code & 1) == 0) { - termExists.set(0); - termBlockOrd++; - } else { - // read subCode. - subCodes[0] = suffixLengthsReader.readVLong(); - lastSubIndex = 0; - } - termBlockOrds[0] = termBlockOrd; - postions[0] = suffixLengthsReader.getPosition(); - lastSubIndices[0] = lastSubIndex; - for (int i = 1; i < suffixes.length; i++) { - code = suffixLengthsReader.readVInt(); - suffixes[i] = code >>> 1; - if ((code & 1) == 0) { - termExists.set(i); - termBlockOrd++; - } else { - // read subCode. - subCodes[i] = suffixLengthsReader.readVLong(); - lastSubIndex = i; - } - termBlockOrds[i] = termBlockOrd; - postions[i] = suffixLengthsReader.getPosition(); - lastSubIndices[i] = lastSubIndex; - } - } - // Reset suffixLengthsReader's position. - suffixLengthsReader.setPosition(0); - } else { - suffixes = new int[entCount]; - // TODO: remove postions if it is unnecessary. - postions = new int[entCount]; - if (isLeafBlock) { - for (int i = 0; i < suffixes.length; i++) { - suffixes[i] = suffixLengthsReader.readVInt(); - postions[i] = suffixLengthsReader.getPosition(); - } - } else { - // Handle subCode for non leaf block. - termExists = new FixedBitSet(entCount); - subCodes = new long[entCount]; - termBlockOrds = new int[entCount]; - lastSubIndices = new int[entCount]; - int termBlockOrd = 0; - int lastSubIndex = -1; - for (int i = 0; i < suffixes.length; i++) { - code = suffixLengthsReader.readVInt(); - suffixes[i] = code >>> 1; - if ((code & 1) == 0) { - termExists.set(i); - termBlockOrd++; - } else { - // read subCode. - subCodes[i] = suffixLengthsReader.readVLong(); - lastSubIndex = i; - } - termBlockOrds[i] = termBlockOrd; - postions[i] = suffixLengthsReader.getPosition(); - lastSubIndices[i] = lastSubIndex; - } - } - // Reset suffixLengthsReader's position. - suffixLengthsReader.setPosition(0); - offsets = getOffsets(suffixes); - assert assertOffset(suffixes, offsets); - } - /*if (DEBUG) { if (arc == null) { System.out.println(" loadBlock (next) fp=" + fp + " entCount=" + entCount + " prefixLen=" + prefix + " isLastInFloor=" + isLastInFloor + " leaf?=" + isLeafBlock); @@ -618,9 +523,7 @@ public void scanToSubBlock(long subFP) { // NOTE: sets startBytePos/suffix as a side effect public SeekStatus scanToTerm(BytesRef target, boolean exactOnly) throws IOException { - return isLeafBlock - ? binarySearchTermLeaf(target, exactOnly) - : binarySearchTermNonLeaf(target, exactOnly); + return isLeafBlock ? scanToTermLeaf(target, exactOnly) : scanToTermNonLeaf(target, exactOnly); } private int startBytePos; @@ -628,46 +531,14 @@ public SeekStatus scanToTerm(BytesRef target, boolean exactOnly) throws IOExcept private long subCode; CompressionAlgorithm compressionAlg = CompressionAlgorithm.NO_COMPRESSION; - private int[] getOffsets(int[] suffixes) { - int[] offsets = new int[suffixes.length]; - offsets[0] = 0; - for (int i = 1; i < suffixes.length; i++) { - offsets[i] = offsets[i - 1] + suffixes[i - 1]; - } - return offsets; - } + // Target's prefix matches this block's prefix; we + // scan the entries check if the suffix matches. + public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOException { - // Used only by assert - private boolean assertOffset(int[] suffixes, int[] offsets) { - if (offsets[0] != 0) { - return false; - } - for (int i = 1; i < offsets.length; i++) { - if (suffixes[i - 1] != offsets[i] - offsets[i - 1]) { - return false; - } - } - return true; - } - - // Target's prefix matches this leaf block's prefix; - // we binary search the entries check if the suffix matches. - public SeekStatus binarySearchTermLeaf(BytesRef target, boolean exactOnly) throws IOException { - return allEqual - ? binarySearchTermLeafAllEqual(target, exactOnly) - : binarySearchTermLeafUnEqual(target, exactOnly); - } - - // Target's prefix matches this leaf block's prefix; - // And all suffixes have the same length in this block, - // we binary search the entries check if the suffix matches. - private SeekStatus binarySearchTermLeafAllEqual(BytesRef target, boolean exactOnly) - throws IOException { - // if (DEBUG) System.out.println(" binarySearchTermLeafAllEqual: block fp=" + fp + " prefix=" - // + - // prefix + " - // nextEnt=" + nextEnt + " (of " + entCount + ") target=" + brToString(target) + " term=" + - // brToString(term)); + // if (DEBUG) System.out.println(" scanToTermLeaf: block fp=" + fp + " prefix=" + prefix + + // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + + // ToStringUtils.bytesRefToString(target) + + // " term=" + ToStringUtils.bytesRefToString(term)); assert nextEnt != -1; @@ -683,113 +554,28 @@ private SeekStatus binarySearchTermLeafAllEqual(BytesRef target, boolean exactOn assert prefixMatches(target); - // TODO early terminate when target length unequals suffix + prefix. - // But we need to keep the same status with scanToTermLeaf. - int start = nextEnt; - int end = entCount - 1; - // Binary search the entries (terms) in this leaf block: - int cmp = 0; - while (start <= end) { - int mid = (start + end) / 2; - nextEnt = mid + 1; - startBytePos = mid * suffix; - - // Binary search bytes in the suffix, comparing to the target - cmp = - Arrays.compareUnsigned( - suffixBytes, - startBytePos, - startBytePos + suffix, - target.bytes, - target.offset + prefix, - target.offset + target.length); - if (cmp < 0) { - start = mid + 1; - } else if (cmp > 0) { - end = mid - 1; - } else { - // Exact match! - suffixesReader.setPosition(startBytePos + suffix); - // This cannot be a sub-block because we - // would have followed the index to this - // sub-block from the start: - assert ste.termExists; - fillTerm(); - // if (DEBUG) System.out.println(" found!"); - return SeekStatus.FOUND; - } - } - - // It is possible (and OK) that terms index pointed us - // at this block, but, we searched the entire block and - // did not find the term to position to. This happens - // when the target is after the last term in the block - // (but, before the next term in the index). EG - // target could be foozzz, and terms index pointed us - // to the foo* block, but the last term in this block - // was fooz (and, eg, first term in the next block will - // bee fop). - // if (DEBUG) System.out.println(" block end"); - SeekStatus seekStatus = end < entCount - 1 ? SeekStatus.NOT_FOUND : SeekStatus.END; - if (seekStatus == SeekStatus.NOT_FOUND) { - // If binary search ended at the less term, and greater term exists. - // We need to advance to the greater term. - if (cmp < 0) { - startBytePos += suffix; - nextEnt++; - } - suffixesReader.setPosition(startBytePos + suffix); - fillTerm(); - } else { - suffixesReader.setPosition(startBytePos + suffix); - if (exactOnly) { - fillTerm(); - } - } - // TODO: not consistent that in the - // not-exact case we don't next() into the next - // frame here - return seekStatus; - } - - // Target's prefix matches this leaf block's prefix; - // But these suffixes' length are not same, - // we binary search the entries check if the suffix matches. - private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnly) - throws IOException { - // if (DEBUG) System.out.println(" binarySearchTermLeafUnEqual: block fp=" + fp + " prefix=" - // + - // prefix + " - // nextEnt=" + nextEnt + " (of " + entCount + ") target=" + brToString(target) + " term=" + - // brToString(term)); + // TODO: binary search when all terms have the same length, which is common for ID fields, + // which are also the most sensitive to lookup performance? + // Loop over each entry (term or sub-block) in this block: + do { + nextEnt++; - assert nextEnt != -1; + suffix = suffixLengthsReader.readVInt(); - ste.termExists = true; - subCode = 0; + // if (DEBUG) { + // BytesRef suffixBytesRef = new BytesRef(); + // suffixBytesRef.bytes = suffixBytes; + // suffixBytesRef.offset = suffixesReader.getPosition(); + // suffixBytesRef.length = suffix; + // System.out.println(" cycle: term " + (nextEnt-1) + " (of " + entCount + ") suffix=" + // + ToStringUtils.bytesRefToString(suffixBytesRef)); + // } - if (nextEnt == entCount) { - if (exactOnly) { - fillTerm(); - } - return SeekStatus.END; - } + startBytePos = suffixesReader.getPosition(); + suffixesReader.skipBytes(suffix); - assert prefixMatches(target); - assert assertOffset(suffixes, offsets); - - int start = nextEnt; - int end = entCount - 1; - // Binary search the entries (terms) in this leaf block: - int cmp = 0; - while (start <= end) { - int mid = (start + end) / 2; - nextEnt = mid + 1; - startBytePos = offsets[mid]; - suffix = suffixes[mid]; - - // Binary search bytes in the suffix, comparing to the target - cmp = + // Loop over bytes in the suffix, comparing to the target + final int cmp = Arrays.compareUnsigned( suffixBytes, startBytePos, @@ -797,14 +583,20 @@ private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnl target.bytes, target.offset + prefix, target.offset + target.length); + if (cmp < 0) { - start = mid + 1; + // Current entry is still before the target; + // keep scanning } else if (cmp > 0) { - end = mid - 1; + // Done! Current entry is after target -- + // return NOT_FOUND: + fillTerm(); + + // if (DEBUG) System.out.println(" not found"); + return SeekStatus.NOT_FOUND; } else { // Exact match! - setSuffixesReaderUnEqual(); // This cannot be a sub-block because we // would have followed the index to this // sub-block from the start: @@ -814,10 +606,10 @@ private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnl // if (DEBUG) System.out.println(" found!"); return SeekStatus.FOUND; } - } + } while (nextEnt < entCount); // It is possible (and OK) that terms index pointed us - // at this block, but, we searched the entire block and + // at this block, but, we scanned the entire block and // did not find the term to position to. This happens // when the target is after the last term in the block // (but, before the next term in the index). EG @@ -826,45 +618,21 @@ private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnl // was fooz (and, eg, first term in the next block will // bee fop). // if (DEBUG) System.out.println(" block end"); - SeekStatus seekStatus = end < entCount - 1 ? SeekStatus.NOT_FOUND : SeekStatus.END; - if (seekStatus == SeekStatus.NOT_FOUND) { - // If binary search ended at the less term, and greater term exists. - // We need to advance to the greater term. - if (cmp < 0) { - nextEnt++; - } - setSuffixesReaderUnEqual(); + if (exactOnly) { fillTerm(); - } else { - setSuffixesReaderUnEqual(); - if (exactOnly) { - fillTerm(); - } } // TODO: not consistent that in the // not-exact case we don't next() into the next // frame here - - return seekStatus; + return SeekStatus.END; } - // Target's prefix matches this non-leaf block's prefix; - // we binary search the entries check if the suffix matches. - public SeekStatus binarySearchTermNonLeaf(BytesRef target, boolean exactOnly) throws IOException { - return allEqual - ? binarySearchTermNonLeafAllEqual(target, exactOnly) - : binarySearchTermNonLeafUnEqual(target, exactOnly); - } - - // Target's prefix matches this non-leaf block's prefix; - // And all suffixes have the same length in this block, - // we binary search the entries check if the suffix matches. - private SeekStatus binarySearchTermNonLeafAllEqual(BytesRef target, boolean exactOnly) - throws IOException { + // Target's prefix matches this block's prefix; we + // scan the entries check if the suffix matches. + public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws IOException { - // if (DEBUG) System.out.println(" binarySearchTermNonLeafAllEqual: block fp=" + fp + " - // prefix=" + prefix + + // if (DEBUG) System.out.println(" scanToTermNonLeaf: block fp=" + fp + " prefix=" + prefix + // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + // ToStringUtils.bytesRefToString(target) + // " term=" + ToStringUtils.bytesRefToString(term)); @@ -880,21 +648,38 @@ private SeekStatus binarySearchTermNonLeafAllEqual(BytesRef target, boolean exac } assert prefixMatches(target); - assert assertOffset(suffixes, offsets); - int start = nextEnt; - int end = entCount - 1; - // Binary search the entries (terms) in this non leaf block: - int cmp = 0; - int mid = 0; + // Loop over each entry (term or sub-block) in this block: + while (nextEnt < entCount) { - while (start <= end) { - mid = (start + end) / 2; - nextEnt = mid + 1; - startBytePos = mid * suffix; + nextEnt++; - // Binary search bytes in the suffix, comparing to the target - cmp = + final int code = suffixLengthsReader.readVInt(); + suffix = code >>> 1; + + // if (DEBUG) { + // BytesRef suffixBytesRef = new BytesRef(); + // suffixBytesRef.bytes = suffixBytes; + // suffixBytesRef.offset = suffixesReader.getPosition(); + // suffixBytesRef.length = suffix; + // System.out.println(" cycle: " + ((code&1)==1 ? "sub-block" : "term") + " " + + // (nextEnt-1) + " (of " + entCount + ") suffix=" + + // ToStringUtils.bytesRefToString(suffixBytesRef)); + // } + + final int termLen = prefix + suffix; + startBytePos = suffixesReader.getPosition(); + suffixesReader.skipBytes(suffix); + ste.termExists = (code & 1) == 0; + if (ste.termExists) { + state.termBlockOrd++; + subCode = 0; + } else { + subCode = suffixLengthsReader.readVLong(); + lastSubFP = fp - subCode; + } + + final int cmp = Arrays.compareUnsigned( suffixBytes, startBytePos, @@ -902,136 +687,43 @@ private SeekStatus binarySearchTermNonLeafAllEqual(BytesRef target, boolean exac target.bytes, target.offset + prefix, target.offset + target.length); + if (cmp < 0) { - start = mid + 1; + // Current entry is still before the target; + // keep scanning } else if (cmp > 0) { - end = mid - 1; - } else { - // Exact match! - - setSuffixesReaderAllEqual(); - setSubStates(); - // This cannot be a sub-block because we - // would have followed the index to this - // sub-block from the start: - assert ste.termExists; - + // Done! Current entry is after target -- + // return NOT_FOUND: fillTerm(); - // if (DEBUG) System.out.println(" found!"); - return SeekStatus.FOUND; - } - } - - // It is possible (and OK) that terms index pointed us - // at this block, but, we scanned the entire block and - // did not find the term to position to. This happens - // when the target is after the last term in the block - // (but, before the next term in the index). EG - // target could be foozzz, and terms index pointed us - // to the foo* block, but the last term in this block - // was fooz (and, eg, first term in the next block will - // bee fop). - // if (DEBUG) System.out.println(" block end"); - SeekStatus seekStatus = end < entCount - 1 ? SeekStatus.NOT_FOUND : SeekStatus.END; - // If binary search ended at the less term, and greater term exists. - // We need to advance to the greater term. - if (seekStatus == SeekStatus.NOT_FOUND) { - if (cmp < 0) { - nextEnt++; - } - setSuffixesReaderAllEqual(); - setSubStates(); - - fillTerm(); - if (!exactOnly && !termExists.get(nextEnt - 1)) { - // System.out.println(" now pushFrame"); - // TODO this - // We are on a sub-block, and caller wants - // us to position to the next term after - // the target, so we must recurse into the - // sub-frame(s): - ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, prefix + suffix); - ste.currentFrame.loadBlock(); - while (ste.currentFrame.next()) { - ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length()); + // if (DEBUG) System.out.println(" maybe done exactOnly=" + exactOnly + + // " ste.termExists=" + ste.termExists); + + if (!exactOnly && !ste.termExists) { + // System.out.println(" now pushFrame"); + // TODO this + // We are on a sub-block, and caller wants + // us to position to the next term after + // the target, so we must recurse into the + // sub-frame(s): + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, termLen); ste.currentFrame.loadBlock(); + while (ste.currentFrame.next()) { + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length()); + ste.currentFrame.loadBlock(); + } } - } - } else { - setSuffixesReaderAllEqual(); - setSubStates(); - if (exactOnly) { - fillTerm(); - } - } - // TODO: not consistent that in the - // not-exact case we don't next() into the next - // frame here - return seekStatus; - } - // Target's prefix matches this non-leaf block's prefix; - // But these suffixes' length are not same, - // we binary search the entries check if the suffix matches. - private SeekStatus binarySearchTermNonLeafUnEqual(BytesRef target, boolean exactOnly) - throws IOException { - - // if (DEBUG) System.out.println(" binarySearchTermNonLeafUnEqual: block fp=" + fp + " - // prefix=" + prefix + - // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + - // ToStringUtils.bytesRefToString(target) + - // " term=" + ToStringUtils.bytesRefToString(term)); - - assert nextEnt != -1; - - if (nextEnt == entCount) { - if (exactOnly) { - fillTerm(); - ste.termExists = subCode == 0; - } - return SeekStatus.END; - } - - assert prefixMatches(target); - assert assertOffset(suffixes, offsets); - - int start = nextEnt; - int end = entCount - 1; - // Binary search the entries (terms) in this non leaf block: - int cmp = 0; - int mid = 0; - - while (start <= end) { - mid = (start + end) / 2; - nextEnt = mid + 1; - startBytePos = offsets[mid]; - suffix = suffixes[mid]; - - // Binary search bytes in the suffix, comparing to the target - cmp = - Arrays.compareUnsigned( - suffixBytes, - startBytePos, - startBytePos + suffix, - target.bytes, - target.offset + prefix, - target.offset + target.length); - if (cmp < 0) { - start = mid + 1; - } else if (cmp > 0) { - end = mid - 1; + // if (DEBUG) System.out.println(" not found"); + return SeekStatus.NOT_FOUND; } else { // Exact match! - setSuffixesReaderUnEqual(); - setSubStates(); // This cannot be a sub-block because we // would have followed the index to this // sub-block from the start: assert ste.termExists; - fillTerm(); // if (DEBUG) System.out.println(" found!"); return SeekStatus.FOUND; @@ -1048,88 +740,14 @@ private SeekStatus binarySearchTermNonLeafUnEqual(BytesRef target, boolean exact // was fooz (and, eg, first term in the next block will // bee fop). // if (DEBUG) System.out.println(" block end"); - SeekStatus seekStatus = end < entCount - 1 ? SeekStatus.NOT_FOUND : SeekStatus.END; - // If binary search ended at the less term, and greater term exists. - // We need to advance to the greater term. - if (seekStatus == SeekStatus.NOT_FOUND) { - if (cmp < 0) { - nextEnt++; - } - - setSuffixesReaderUnEqual(); - setSubStates(); - + if (exactOnly) { fillTerm(); - if (!exactOnly && !termExists.get(nextEnt - 1)) { - // System.out.println(" now pushFrame"); - // TODO this - // We are on a sub-block, and caller wants - // us to position to the next term after - // the target, so we must recurse into the - // sub-frame(s): - ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, prefix + suffix); - ste.currentFrame.loadBlock(); - while (ste.currentFrame.next()) { - ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length()); - ste.currentFrame.loadBlock(); - } - } - } else { - setSuffixesReaderUnEqual(); - setSubStates(); - if (exactOnly) { - fillTerm(); - } } // TODO: not consistent that in the // not-exact case we don't next() into the next // frame here - return seekStatus; - } - - // Set sub block's states. - private void setSubStates() { - int currentEnt = nextEnt - 1; - - // Set blockOrd, termExists. - state.termBlockOrd = termBlockOrds[currentEnt]; - ste.termExists = termExists.get(currentEnt); - - // Set subCode. - subCode = subCodes[currentEnt]; - - // Set lastSubFP. - if (lastSubIndices[currentEnt] != -1) { - lastSubFP = fp - subCodes[lastSubIndices[currentEnt]]; - } else { - lastSubFP = -1; - } - } - - // Set suffixesReader's position. - private void setSuffixesReaderAllEqual() { - int currentEnt = nextEnt - 1; - - startBytePos = currentEnt * suffix; - suffixesReader.setPosition(startBytePos + suffix); - - // Set suffixLengthsReader's position. - // TODO: is it necessary to set suffixLengthsReader's position? - suffixLengthsReader.setPosition(postions[currentEnt]); - } - - // Set suffixesReader's position. - private void setSuffixesReaderUnEqual() { - int currentEnt = nextEnt - 1; - - startBytePos = offsets[currentEnt]; - suffix = suffixes[currentEnt]; - suffixesReader.setPosition(startBytePos + suffix); - - // Set suffixLengthsReader's position. - // TODO: is it necessary to set suffixLengthsReader's position? - suffixLengthsReader.setPosition(postions[currentEnt]); + return SeekStatus.END; } private void fillTerm() { diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java index 07d55452dd4a..99c0e0a6ae28 100644 --- a/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java +++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene99/TestLucene99PostingsFormat.java @@ -29,7 +29,10 @@ import org.apache.lucene.codecs.lucene99.Lucene99ScoreSkipReader.MutableImpactList; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.*; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.Impact; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -38,7 +41,6 @@ import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.index.BasePostingsFormatTestCase; import org.apache.lucene.tests.util.TestUtil; -import org.apache.lucene.util.BytesRef; public class TestLucene99PostingsFormat extends BasePostingsFormatTestCase { private final Codec codec = TestUtil.alwaysPostingsFormat(new Lucene99PostingsFormat()); @@ -141,13 +143,4 @@ private void doTestImpactSerialization(List impacts) throws IOException } } } - - @Override - protected void subCheckBinarySearch(TermsEnum termsEnum) throws Exception { - // 10004a matched block's entries: [100001, 100003, ..., 100049]. - // if target greater than the last entry of the matched block, - // termsEnum.term should be the last entry. - assertFalse(termsEnum.seekExact(new BytesRef("10004a"))); - assertEquals(termsEnum.term(), new BytesRef("100049")); - } } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java index e3684608df46..da6ac9f67dbc 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java @@ -58,7 +58,9 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.search.*; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.tests.analysis.CannedTokenStream; @@ -367,151 +369,6 @@ public void testGhosts() throws Exception { dir.close(); } - protected void subCheckBinarySearch(TermsEnum termsEnum) throws Exception {} - - // public void testSubBlock() throws Exception { - // Directory dir = newDirectory(); - // // Set minTermBlockSize to 2, maxTermBlockSize to 3, to generate subBlock. - // PostingsFormat postingsFormat = getDefaultPostingsFormat(2, 3); - // - // IndexWriter iw = - // new IndexWriter(dir, - // newIndexWriterConfig().setCodec(alwaysPostingsFormat(postingsFormat))); - // String[] categories = new String[] {"regular", "request", "rest", "teacher", "team"}; - // - // for (String category : categories) { - // Document doc = new Document(); - // doc.add(newStringField("category", category, Field.Store.YES)); - // iw.addDocument(doc); - // } - // - // IndexReader reader = DirectoryReader.open(iw); - // iw.commit(); - // iw.forceMerge(1); - // - // TermsEnum termsEnum = getOnlyLeafReader(reader).terms("category").iterator(); - // - // // test seekExact. - // for (String category : categories) { - // BytesRef target = newBytesRef(category); - // assertTrue(termsEnum.seekExact(target)); - // assertEquals(termsEnum.term(), target); - // } - // - // // test seekCeil. - // for (String category : categories) { - // BytesRef target = newBytesRef(category); - // assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(target)); - // assertEquals(termsEnum.term(), target); - // } - // - // iw.close(); - // reader.close(); - // dir.close(); - // } - - // public void testDeepSubBlock() throws Exception { - // Directory dir = newDirectory(); - // // Set minTermBlockSize to 2, maxTermBlockSize to 3, to generate deep subBlock. - // PostingsFormat postingsFormat = getDefaultPostingsFormat(2, 3); - // - // IndexWriter iw = - // new IndexWriter(dir, - // newIndexWriterConfig().setCodec(alwaysPostingsFormat(postingsFormat))); - // String[] categories = - // new String[] { - // "regular", "request1", "request2", "request3", "request4", "rest", "teacher", "team" - // }; - // - // for (String category : categories) { - // Document doc = new Document(); - // doc.add(newStringField("category", category, Field.Store.YES)); - // iw.addDocument(doc); - // } - // - // IndexReader reader = DirectoryReader.open(iw); - // iw.commit(); - // iw.forceMerge(1); - // - // TermsEnum termsEnum = getOnlyLeafReader(reader).terms("category").iterator(); - // - // BytesRef target = newBytesRef("reques"); - // assertFalse(termsEnum.seekExact(target)); - // assertEquals(termsEnum.term(), new BytesRef("request")); - // assertEquals(SeekStatus.NOT_FOUND, termsEnum.seekCeil(target)); - // - // // test seekExact. - // for (String category : categories) { - // target = newBytesRef(category); - // assertTrue(termsEnum.seekExact(target)); - // assertEquals(termsEnum.term(), target); - // } - // - // // test seekCeil. - // for (String category : categories) { - // target = newBytesRef(category); - // assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(target)); - // assertEquals(termsEnum.term(), target); - // } - // - // iw.close(); - // reader.close(); - // dir.close(); - // } - - public void testBinarySearchTermLeaf() throws Exception { - Directory dir = newDirectory(); - - IndexWriterConfig iwc = newIndexWriterConfig(null); - iwc.setCodec(getCodec()); - iwc.setMergePolicy(newTieredMergePolicy()); - IndexWriter iw = new IndexWriter(dir, iwc); - - for (int i = 100000; i <= 100400; i++) { - // only add odd number - if (i % 2 == 1) { - Document document = new Document(); - document.add(new StringField("id", i + "", Field.Store.NO)); - iw.addDocument(document); - } - } - iw.commit(); - iw.forceMerge(1); - - DirectoryReader reader = DirectoryReader.open(iw); - TermsEnum termsEnum = getOnlyLeafReader(reader).terms("id").iterator(); - // test seekExact - for (int i = 100000; i <= 100400; i++) { - BytesRef target = new BytesRef(i + ""); - if (i % 2 == 1) { - assertTrue(termsEnum.seekExact(target)); - assertEquals(termsEnum.term(), target); - } else { - assertFalse(termsEnum.seekExact(target)); - } - } - - subCheckBinarySearch(termsEnum); - // test seekCeil - for (int i = 100000; i < 100400; i++) { - BytesRef target = new BytesRef(i + ""); - if (i % 2 == 1) { - assertEquals(SeekStatus.FOUND, termsEnum.seekCeil(target)); - assertEquals(termsEnum.term(), target); - if (i <= 100397) { - assertEquals(new BytesRef(i + 2 + ""), termsEnum.next()); - } - } else { - assertEquals(SeekStatus.NOT_FOUND, termsEnum.seekCeil(target)); - assertEquals(new BytesRef(i + 1 + ""), termsEnum.term()); - } - } - assertEquals(SeekStatus.END, termsEnum.seekCeil(new BytesRef(100400 + ""))); - reader.close(); - iw.close(); - dir.close(); - } - // tests that level 2 ghost fields still work public void testLevel2Ghosts() throws Exception { Directory dir = newDirectory(); @@ -1762,7 +1619,7 @@ public void testLineFileDocs() throws IOException { // Use a FS dir and a non-randomized IWC to not slow down indexing try (Directory dir = newFSDirectory(createTempDir())) { try (LineFileDocs docs = new LineFileDocs(random()); - IndexWriter w = new IndexWriter(dir, new IndexWriterConfig())) { + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig())) { final int numDocs = atLeast(10_000); for (int i = 0; i < numDocs; ++i) { // Only keep the body field, and don't index term vectors on it, we only care about From c7bae565c185d2ce0de698f86a72e94c07705f83 Mon Sep 17 00:00:00 2001 From: zhouhui Date: Fri, 26 Apr 2024 14:40:06 +0800 Subject: [PATCH 4/7] Binary search all terms. --- .../blocktree/SegmentTermsEnumFrame.java | 528 +++++++++++++----- .../index/BasePostingsFormatTestCase.java | 2 +- 2 files changed, 403 insertions(+), 127 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java index c1552c8ce607..ffd0486efa37 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java @@ -25,6 +25,7 @@ import org.apache.lucene.store.ByteArrayDataInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.fst.FST; final class SegmentTermsEnumFrame { @@ -51,6 +52,13 @@ final class SegmentTermsEnumFrame { byte[] suffixLengthBytes; final ByteArrayDataInput suffixLengthsReader; + int[] suffixes; + int[] postions; + int[] offsets; + int[] termBlockOrds; + int[] lastSubIndices; + FixedBitSet termExists; + long[] subCodes; byte[] statBytes = new byte[64]; int statsSingletonRunLength = 0; final ByteArrayDataInput statsReader = new ByteArrayDataInput(); @@ -199,6 +207,90 @@ void loadBlock() throws IOException { suffixLengthsReader.reset(suffixLengthBytes, 0, numSuffixLengthBytes); totalSuffixBytes = ste.in.getFilePointer() - startSuffixFP; + // Prepare suffixes, offsets to binary search. + if (allEqual) { + if (isLeafBlock) { + suffix = suffixLengthsReader.readVInt(); + } else { + // Handle subCode for non leaf block. + postions = new int[entCount]; + termExists = new FixedBitSet(entCount); + subCodes = new long[entCount]; + termBlockOrds = new int[entCount]; + lastSubIndices = new int[entCount]; + int termBlockOrd = 0; + int lastSubIndex = -1; + // read first vint to set suffix, byt the way, set termExist, subCode. + code = suffixLengthsReader.readVInt(); + suffix = code >>> 1; + if ((code & 1) == 0) { + termExists.set(0); + termBlockOrd++; + } else { + // read subCode. + subCodes[0] = suffixLengthsReader.readVLong(); + lastSubIndex = 0; + } + termBlockOrds[0] = termBlockOrd; + postions[0] = suffixLengthsReader.getPosition(); + lastSubIndices[0] = lastSubIndex; + for (int i = 1; i < suffixes.length; i++) { + code = suffixLengthsReader.readVInt(); + suffixes[i] = code >>> 1; + if ((code & 1) == 0) { + termExists.set(i); + termBlockOrd++; + } else { + // read subCode. + subCodes[i] = suffixLengthsReader.readVLong(); + lastSubIndex = i; + } + termBlockOrds[i] = termBlockOrd; + postions[i] = suffixLengthsReader.getPosition(); + lastSubIndices[i] = lastSubIndex; + } + } + // Reset suffixLengthsReader's position. + suffixLengthsReader.setPosition(0); + } else { + suffixes = new int[entCount]; + // TODO: remove postions if it is unnecessary. + postions = new int[entCount]; + if (isLeafBlock) { + for (int i = 0; i < suffixes.length; i++) { + suffixes[i] = suffixLengthsReader.readVInt(); + postions[i] = suffixLengthsReader.getPosition(); + } + } else { + // Handle subCode for non leaf block. + termExists = new FixedBitSet(entCount); + subCodes = new long[entCount]; + termBlockOrds = new int[entCount]; + lastSubIndices = new int[entCount]; + int termBlockOrd = 0; + int lastSubIndex = -1; + for (int i = 0; i < suffixes.length; i++) { + code = suffixLengthsReader.readVInt(); + suffixes[i] = code >>> 1; + if ((code & 1) == 0) { + termExists.set(i); + termBlockOrd++; + } else { + // read subCode. + subCodes[i] = suffixLengthsReader.readVLong(); + lastSubIndex = i; + } + termBlockOrds[i] = termBlockOrd; + postions[i] = suffixLengthsReader.getPosition(); + lastSubIndices[i] = lastSubIndex; + } + } + // Reset suffixLengthsReader's position. + suffixLengthsReader.setPosition(0); + offsets = getOffsets(suffixes); + assert assertOffset(suffixes, offsets); + } + /*if (DEBUG) { if (arc == null) { System.out.println(" loadBlock (next) fp=" + fp + " entCount=" + entCount + " prefixLen=" + prefix + " isLastInFloor=" + isLastInFloor + " leaf?=" + isLeafBlock); @@ -526,15 +618,9 @@ public void scanToSubBlock(long subFP) { // NOTE: sets startBytePos/suffix as a side effect public SeekStatus scanToTerm(BytesRef target, boolean exactOnly) throws IOException { - if (isLeafBlock) { - if (allEqual) { - return binarySearchTermLeaf(target, exactOnly); - } else { - return scanToTermLeaf(target, exactOnly); - } - } else { - return scanToTermNonLeaf(target, exactOnly); - } + return isLeafBlock + ? binarySearchTermLeaf(target, exactOnly) + : binarySearchTermNonLeaf(target, exactOnly); } private int startBytePos; @@ -542,14 +628,46 @@ public SeekStatus scanToTerm(BytesRef target, boolean exactOnly) throws IOExcept private long subCode; CompressionAlgorithm compressionAlg = CompressionAlgorithm.NO_COMPRESSION; - // Target's prefix matches this block's prefix; we - // scan the entries to check if the suffix matches. - public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOException { + private int[] getOffsets(int[] suffixes) { + int[] offsets = new int[suffixes.length]; + offsets[0] = 0; + for (int i = 1; i < suffixes.length; i++) { + offsets[i] = offsets[i - 1] + suffixes[i - 1]; + } + return offsets; + } - // if (DEBUG) System.out.println(" scanToTermLeaf: block fp=" + fp + " prefix=" + prefix + - // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + - // ToStringUtils.bytesRefToString(target) + - // " term=" + ToStringUtils.bytesRefToString(term)); + // Used only by assert + private boolean assertOffset(int[] suffixes, int[] offsets) { + if (offsets[0] != 0) { + return false; + } + for (int i = 1; i < offsets.length; i++) { + if (suffixes[i - 1] != offsets[i] - offsets[i - 1]) { + return false; + } + } + return true; + } + + // Target's prefix matches this leaf block's prefix; + // we binary search the entries to check if the suffix matches. + public SeekStatus binarySearchTermLeaf(BytesRef target, boolean exactOnly) throws IOException { + return allEqual + ? binarySearchTermLeafAllEqual(target, exactOnly) + : binarySearchTermLeafUnEqual(target, exactOnly); + } + + // Target's prefix matches this leaf block's prefix; + // And all suffixes have the same length in this block, + // we binary search the entries to check if the suffix matches. + private SeekStatus binarySearchTermLeafAllEqual(BytesRef target, boolean exactOnly) + throws IOException { + // if (DEBUG) System.out.println(" binarySearchTermLeafAllEqual: block fp=" + fp + " prefix=" + // + + // prefix + " + // nextEnt=" + nextEnt + " (of " + entCount + ") target=" + brToString(target) + " term=" + + // brToString(term)); assert nextEnt != -1; @@ -565,26 +683,19 @@ public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOEx assert prefixMatches(target); - // Loop over each entry (term or sub-block) in this block: - do { - nextEnt++; - - suffix = suffixLengthsReader.readVInt(); - - // if (DEBUG) { - // BytesRef suffixBytesRef = new BytesRef(); - // suffixBytesRef.bytes = suffixBytes; - // suffixBytesRef.offset = suffixesReader.getPosition(); - // suffixBytesRef.length = suffix; - // System.out.println(" cycle: term " + (nextEnt-1) + " (of " + entCount + ") suffix=" - // + ToStringUtils.bytesRefToString(suffixBytesRef)); - // } - - startBytePos = suffixesReader.getPosition(); - suffixesReader.skipBytes(suffix); + // TODO early terminate when target length unequals suffix + prefix. + // But we need to keep the same status with scanToTermLeaf. + int start = nextEnt; + int end = entCount - 1; + // Binary search the entries (terms) in this leaf block: + int cmp = 0; + while (start <= end) { + int mid = (start + end) >>> 1; + nextEnt = mid + 1; + startBytePos = mid * suffix; - // Loop over bytes in the suffix, comparing to the target - final int cmp = + // Binary search bytes in the suffix, comparing to the target. + cmp = Arrays.compareUnsigned( suffixBytes, startBytePos, @@ -592,32 +703,21 @@ public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOEx target.bytes, target.offset + prefix, target.offset + target.length); - if (cmp < 0) { - // Current entry is still before the target; - // keep scanning + start = mid + 1; } else if (cmp > 0) { - // Done! Current entry is after target -- - // return NOT_FOUND: - fillTerm(); - - // if (DEBUG) System.out.println(" not found"); - return SeekStatus.NOT_FOUND; + end = mid - 1; } else { // Exact match! - - // This cannot be a sub-block because we - // would have followed the index to this - // sub-block from the start: - + suffixesReader.setPosition(startBytePos + suffix); fillTerm(); // if (DEBUG) System.out.println(" found!"); return SeekStatus.FOUND; } - } while (nextEnt < entCount); + } // It is possible (and OK) that terms index pointed us - // at this block, but, we scanned the entire block and + // at this block, but, we searched the entire block and // did not find the term to position to. This happens // when the target is after the last term in the block // (but, before the next term in the index). EG @@ -626,21 +726,36 @@ public SeekStatus scanToTermLeaf(BytesRef target, boolean exactOnly) throws IOEx // was fooz (and, eg, first term in the next block will // bee fop). // if (DEBUG) System.out.println(" block end"); - if (exactOnly) { + SeekStatus seekStatus; + if (end < entCount - 1) { + seekStatus = SeekStatus.NOT_FOUND; + // If binary search ended at the less term, and greater term exists. + // We need to advance to the greater term. + if (cmp < 0) { + startBytePos += suffix; + nextEnt++; + } + suffixesReader.setPosition(startBytePos + suffix); fillTerm(); + } else { + seekStatus = SeekStatus.END; + suffixesReader.setPosition(startBytePos + suffix); + if (exactOnly) { + fillTerm(); + } } - // TODO: not consistent that in the // not-exact case we don't next() into the next // frame here - return SeekStatus.END; + return seekStatus; } - // Target's prefix matches this block's prefix; - // And all suffixes have the same length in this block, + // Target's prefix matches this leaf block's prefix; + // But these suffixes' length are not same, // we binary search the entries to check if the suffix matches. - public SeekStatus binarySearchTermLeaf(BytesRef target, boolean exactOnly) throws IOException { - // if (DEBUG) System.out.println(" binarySearchTermLeaf: block fp=" + fp + " prefix=" + + private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnly) + throws IOException { + // if (DEBUG) System.out.println(" binarySearchTermLeafUnEqual: block fp=" + fp + " prefix=" // prefix + " // nextEnt=" + nextEnt + " (of " + entCount + ") target=" + brToString(target) + " term=" + // brToString(term)); @@ -658,10 +773,8 @@ public SeekStatus binarySearchTermLeaf(BytesRef target, boolean exactOnly) throw } assert prefixMatches(target); + assert assertOffset(suffixes, offsets); - suffix = suffixLengthsReader.readVInt(); - // TODO early terminate when target length unequals suffix + prefix. - // But we need to keep the same status with scanToTermLeaf. int start = nextEnt; int end = entCount - 1; // Binary search the entries (terms) in this leaf block: @@ -669,9 +782,10 @@ public SeekStatus binarySearchTermLeaf(BytesRef target, boolean exactOnly) throw while (start <= end) { int mid = (start + end) >>> 1; nextEnt = mid + 1; - startBytePos = mid * suffix; + startBytePos = offsets[mid]; + suffix = suffixes[mid]; - // Binary search bytes in the suffix, comparing to the target. + // Binary search bytes in the suffix, comparing to the target cmp = Arrays.compareUnsigned( suffixBytes, @@ -686,7 +800,7 @@ public SeekStatus binarySearchTermLeaf(BytesRef target, boolean exactOnly) throw end = mid - 1; } else { // Exact match! - suffixesReader.setPosition(startBytePos + suffix); + setSuffixesReaderUnEqual(); fillTerm(); // if (DEBUG) System.out.println(" found!"); return SeekStatus.FOUND; @@ -709,29 +823,39 @@ public SeekStatus binarySearchTermLeaf(BytesRef target, boolean exactOnly) throw // If binary search ended at the less term, and greater term exists. // We need to advance to the greater term. if (cmp < 0) { - startBytePos += suffix; nextEnt++; } - suffixesReader.setPosition(startBytePos + suffix); + setSuffixesReaderUnEqual(); fillTerm(); } else { seekStatus = SeekStatus.END; - suffixesReader.setPosition(startBytePos + suffix); + setSuffixesReaderUnEqual(); if (exactOnly) { fillTerm(); } } + // TODO: not consistent that in the // not-exact case we don't next() into the next // frame here return seekStatus; } - // Target's prefix matches this block's prefix; we - // scan the entries to check if the suffix matches. - public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws IOException { + // Target's prefix matches this non-leaf block's prefix; + // we binary search the entries to check if the suffix matches. + public SeekStatus binarySearchTermNonLeaf(BytesRef target, boolean exactOnly) throws IOException { + return allEqual + ? binarySearchTermNonLeafAllEqual(target, exactOnly) + : binarySearchTermNonLeafUnEqual(target, exactOnly); + } - // if (DEBUG) System.out.println(" scanToTermNonLeaf: block fp=" + fp + " prefix=" + prefix + + // Target's prefix matches this non-leaf block's prefix; + // And all suffixes have the same length in this block, + // we binary search the entries to check if the suffix matches. + private SeekStatus binarySearchTermNonLeafAllEqual(BytesRef target, boolean exactOnly) + throws IOException { + // if (DEBUG) System.out.println(" binarySearchTermNonLeafAllEqual: block fp=" + fp + " + // prefix=" + prefix + // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + // ToStringUtils.bytesRefToString(target) + // " term=" + ToStringUtils.bytesRefToString(term)); @@ -747,37 +871,21 @@ public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws I } assert prefixMatches(target); + assert assertOffset(suffixes, offsets); - // Loop over each entry (term or sub-block) in this block: - while (nextEnt < entCount) { - - nextEnt++; - - final int code = suffixLengthsReader.readVInt(); - suffix = code >>> 1; - - // if (DEBUG) { - // BytesRef suffixBytesRef = new BytesRef(); - // suffixBytesRef.bytes = suffixBytes; - // suffixBytesRef.offset = suffixesReader.getPosition(); - // suffixBytesRef.length = suffix; - // System.out.println(" cycle: " + ((code&1)==1 ? "sub-block" : "term") + " " + - // (nextEnt-1) + " (of " + entCount + ") suffix=" + - // ToStringUtils.bytesRefToString(suffixBytesRef)); - // } + int start = nextEnt; + int end = entCount - 1; + // Binary search the entries (terms) in this non leaf block: + int cmp = 0; + int mid = 0; - startBytePos = suffixesReader.getPosition(); - suffixesReader.skipBytes(suffix); - ste.termExists = (code & 1) == 0; - if (ste.termExists) { - state.termBlockOrd++; - subCode = 0; - } else { - subCode = suffixLengthsReader.readVLong(); - lastSubFP = fp - subCode; - } + while (start <= end) { + mid = (start + end) >>> 1; + nextEnt = mid + 1; + startBytePos = mid * suffix; - final int cmp = + // Binary search bytes in the suffix, comparing to the target + cmp = Arrays.compareUnsigned( suffixBytes, startBytePos, @@ -785,43 +893,136 @@ public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws I target.bytes, target.offset + prefix, target.offset + target.length); - if (cmp < 0) { - // Current entry is still before the target; - // keep scanning + start = mid + 1; } else if (cmp > 0) { - // Done! Current entry is after target -- - // return NOT_FOUND: + end = mid - 1; + } else { + // Exact match! + + setSuffixesReaderAllEqual(); + setSubStates(); + // This cannot be a sub-block because we + // would have followed the index to this + // sub-block from the start: + assert ste.termExists; + fillTerm(); + // if (DEBUG) System.out.println(" found!"); - // if (DEBUG) System.out.println(" maybe done exactOnly=" + exactOnly + - // " ste.termExists=" + ste.termExists); - - if (!exactOnly && !ste.termExists) { - // System.out.println(" now pushFrame"); - // TODO this - // We are on a sub-block, and caller wants - // us to position to the next term after - // the target, so we must recurse into the - // sub-frame(s): - ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, prefix + suffix); + return SeekStatus.FOUND; + } + } + + // It is possible (and OK) that terms index pointed us + // at this block, but, we scanned the entire block and + // did not find the term to position to. This happens + // when the target is after the last term in the block + // (but, before the next term in the index). EG + // target could be foozzz, and terms index pointed us + // to the foo* block, but the last term in this block + // was fooz (and, eg, first term in the next block will + // bee fop). + // if (DEBUG) System.out.println(" block end"); + SeekStatus seekStatus; + if (end < entCount - 1) { + seekStatus = SeekStatus.NOT_FOUND; + // If binary search ended at the less term, and greater term exists. + // We need to advance to the greater term. + if (cmp < 0) { + nextEnt++; + } + setSuffixesReaderAllEqual(); + setSubStates(); + + fillTerm(); + if (!exactOnly && !termExists.get(nextEnt - 1)) { + // System.out.println(" now pushFrame"); + // TODO this + // We are on a sub-block, and caller wants + // us to position to the next term after + // the target, so we must recurse into the + // sub-frame(s): + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, prefix + suffix); + ste.currentFrame.loadBlock(); + while (ste.currentFrame.next()) { + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length()); ste.currentFrame.loadBlock(); - while (ste.currentFrame.next()) { - ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length()); - ste.currentFrame.loadBlock(); - } } + } + } else { + seekStatus = SeekStatus.END; + setSuffixesReaderAllEqual(); + setSubStates(); + if (exactOnly) { + fillTerm(); + } + } + // TODO: not consistent that in the + // not-exact case we don't next() into the next + // frame here + return seekStatus; + } + + // Target's prefix matches this non-leaf block's prefix; + // But these suffixes' length are not same, + // we binary search the entries to check if the suffix matches. + private SeekStatus binarySearchTermNonLeafUnEqual(BytesRef target, boolean exactOnly) + throws IOException { + + // if (DEBUG) System.out.println(" binarySearchTermNonLeafUnEqual: block fp=" + fp + " + // prefix=" + prefix + + // " nextEnt=" + nextEnt + " (of " + entCount + ") target=" + + // ToStringUtils.bytesRefToString(target) + + // " term=" + ToStringUtils.bytesRefToString(term)); + + assert nextEnt != -1; - // if (DEBUG) System.out.println(" not found"); - return SeekStatus.NOT_FOUND; + if (nextEnt == entCount) { + if (exactOnly) { + fillTerm(); + ste.termExists = subCode == 0; + } + return SeekStatus.END; + } + + assert prefixMatches(target); + assert assertOffset(suffixes, offsets); + + int start = nextEnt; + int end = entCount - 1; + // Binary search the entries (terms) in this non leaf block: + int cmp = 0; + int mid = 0; + + while (start <= end) { + mid = (start + end) >>> 1; + nextEnt = mid + 1; + startBytePos = offsets[mid]; + suffix = suffixes[mid]; + + // Binary search bytes in the suffix, comparing to the target + cmp = + Arrays.compareUnsigned( + suffixBytes, + startBytePos, + startBytePos + suffix, + target.bytes, + target.offset + prefix, + target.offset + target.length); + if (cmp < 0) { + start = mid + 1; + } else if (cmp > 0) { + end = mid - 1; } else { // Exact match! - + setSuffixesReaderUnEqual(); + setSubStates(); // This cannot be a sub-block because we // would have followed the index to this // sub-block from the start: - assert ste.termExists; + fillTerm(); // if (DEBUG) System.out.println(" found!"); return SeekStatus.FOUND; @@ -838,14 +1039,89 @@ public SeekStatus scanToTermNonLeaf(BytesRef target, boolean exactOnly) throws I // was fooz (and, eg, first term in the next block will // bee fop). // if (DEBUG) System.out.println(" block end"); - if (exactOnly) { + SeekStatus seekStatus; + if (end < entCount - 1) { + seekStatus = SeekStatus.NOT_FOUND; + // If binary search ended at the less term, and greater term exists. + // We need to advance to the greater term. + if (cmp < 0) { + nextEnt++; + } + + setSuffixesReaderUnEqual(); + setSubStates(); + fillTerm(); + if (!exactOnly && !termExists.get(nextEnt - 1)) { + // System.out.println(" now pushFrame"); + // TODO this + // We are on a sub-block, and caller wants + // us to position to the next term after + // the target, so we must recurse into the + // sub-frame(s): + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, prefix + suffix); + ste.currentFrame.loadBlock(); + while (ste.currentFrame.next()) { + ste.currentFrame = ste.pushFrame(null, ste.currentFrame.lastSubFP, ste.term.length()); + ste.currentFrame.loadBlock(); + } + } + } else { + seekStatus = SeekStatus.END; + setSuffixesReaderUnEqual(); + setSubStates(); + if (exactOnly) { + fillTerm(); + } } - // TODO: not consistent that in the // not-exact case we don't next() into the next // frame here - return SeekStatus.END; + return seekStatus; + } + + // Set sub block's states. + private void setSubStates() { + int currentEnt = nextEnt - 1; + + // Set blockOrd, termExists. + state.termBlockOrd = termBlockOrds[currentEnt]; + ste.termExists = termExists.get(currentEnt); + + // Set subCode. + subCode = subCodes[currentEnt]; + + // Set lastSubFP. + if (lastSubIndices[currentEnt] != -1) { + lastSubFP = fp - subCodes[lastSubIndices[currentEnt]]; + } else { + lastSubFP = -1; + } + } + + // Set suffixesReader's position. + private void setSuffixesReaderAllEqual() { + int currentEnt = nextEnt - 1; + + startBytePos = currentEnt * suffix; + suffixesReader.setPosition(startBytePos + suffix); + + // Set suffixLengthsReader's position. + // TODO: is it necessary to set suffixLengthsReader's position? + suffixLengthsReader.setPosition(postions[currentEnt]); + } + + // Set suffixesReader's position. + private void setSuffixesReaderUnEqual() { + int currentEnt = nextEnt - 1; + + startBytePos = offsets[currentEnt]; + suffix = suffixes[currentEnt]; + suffixesReader.setPosition(startBytePos + suffix); + + // Set suffixLengthsReader's position. + // TODO: is it necessary to set suffixLengthsReader's position? + suffixLengthsReader.setPosition(postions[currentEnt]); } private void fillTerm() { diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java index 05d9ada635b9..8f8233ee680e 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java @@ -1710,7 +1710,7 @@ public void testLineFileDocs() throws IOException { // Use a FS dir and a non-randomized IWC to not slow down indexing try (Directory dir = newFSDirectory(createTempDir())) { try (LineFileDocs docs = new LineFileDocs(random()); - IndexWriter w = new IndexWriter(dir, new IndexWriterConfig())) { + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig())) { final int numDocs = atLeast(10_000); for (int i = 0; i < numDocs; ++i) { // Only keep the body field, and don't index term vectors on it, we only care about From 4ebd068551958d82747d6fe947e91bbe2f7d4300 Mon Sep 17 00:00:00 2001 From: zhouhui Date: Fri, 26 Apr 2024 15:49:53 +0800 Subject: [PATCH 5/7] Typo: postions -> positions. --- .../blocktree/SegmentTermsEnumFrame.java | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java index ffd0486efa37..720354a0dbe4 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java @@ -53,7 +53,7 @@ final class SegmentTermsEnumFrame { final ByteArrayDataInput suffixLengthsReader; int[] suffixes; - int[] postions; + int[] positions; int[] offsets; int[] termBlockOrds; int[] lastSubIndices; @@ -213,7 +213,7 @@ void loadBlock() throws IOException { suffix = suffixLengthsReader.readVInt(); } else { // Handle subCode for non leaf block. - postions = new int[entCount]; + positions = new int[entCount]; termExists = new FixedBitSet(entCount); subCodes = new long[entCount]; termBlockOrds = new int[entCount]; @@ -232,7 +232,7 @@ void loadBlock() throws IOException { lastSubIndex = 0; } termBlockOrds[0] = termBlockOrd; - postions[0] = suffixLengthsReader.getPosition(); + positions[0] = suffixLengthsReader.getPosition(); lastSubIndices[0] = lastSubIndex; for (int i = 1; i < suffixes.length; i++) { code = suffixLengthsReader.readVInt(); @@ -246,7 +246,7 @@ void loadBlock() throws IOException { lastSubIndex = i; } termBlockOrds[i] = termBlockOrd; - postions[i] = suffixLengthsReader.getPosition(); + positions[i] = suffixLengthsReader.getPosition(); lastSubIndices[i] = lastSubIndex; } } @@ -254,12 +254,12 @@ void loadBlock() throws IOException { suffixLengthsReader.setPosition(0); } else { suffixes = new int[entCount]; - // TODO: remove postions if it is unnecessary. - postions = new int[entCount]; + // TODO: remove positions if it is unnecessary. + positions = new int[entCount]; if (isLeafBlock) { for (int i = 0; i < suffixes.length; i++) { suffixes[i] = suffixLengthsReader.readVInt(); - postions[i] = suffixLengthsReader.getPosition(); + positions[i] = suffixLengthsReader.getPosition(); } } else { // Handle subCode for non leaf block. @@ -281,7 +281,7 @@ void loadBlock() throws IOException { lastSubIndex = i; } termBlockOrds[i] = termBlockOrd; - postions[i] = suffixLengthsReader.getPosition(); + positions[i] = suffixLengthsReader.getPosition(); lastSubIndices[i] = lastSubIndex; } } @@ -1108,7 +1108,7 @@ private void setSuffixesReaderAllEqual() { // Set suffixLengthsReader's position. // TODO: is it necessary to set suffixLengthsReader's position? - suffixLengthsReader.setPosition(postions[currentEnt]); + suffixLengthsReader.setPosition(positions[currentEnt]); } // Set suffixesReader's position. @@ -1121,7 +1121,7 @@ private void setSuffixesReaderUnEqual() { // Set suffixLengthsReader's position. // TODO: is it necessary to set suffixLengthsReader's position? - suffixLengthsReader.setPosition(postions[currentEnt]); + suffixLengthsReader.setPosition(positions[currentEnt]); } private void fillTerm() { From 7c0c6997a104eede60303cfe9868c81667239951 Mon Sep 17 00:00:00 2001 From: zhouhui Date: Fri, 26 Apr 2024 15:57:52 +0800 Subject: [PATCH 6/7] Remove todo on positions. --- .../codecs/lucene90/blocktree/SegmentTermsEnumFrame.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java index 720354a0dbe4..80efc8711fec 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java @@ -254,7 +254,6 @@ void loadBlock() throws IOException { suffixLengthsReader.setPosition(0); } else { suffixes = new int[entCount]; - // TODO: remove positions if it is unnecessary. positions = new int[entCount]; if (isLeafBlock) { for (int i = 0; i < suffixes.length; i++) { @@ -1107,7 +1106,6 @@ private void setSuffixesReaderAllEqual() { suffixesReader.setPosition(startBytePos + suffix); // Set suffixLengthsReader's position. - // TODO: is it necessary to set suffixLengthsReader's position? suffixLengthsReader.setPosition(positions[currentEnt]); } @@ -1120,7 +1118,6 @@ private void setSuffixesReaderUnEqual() { suffixesReader.setPosition(startBytePos + suffix); // Set suffixLengthsReader's position. - // TODO: is it necessary to set suffixLengthsReader's position? suffixLengthsReader.setPosition(positions[currentEnt]); } From 19cdd7e031ecd45fa5fc939a14cba52f8c98a061 Mon Sep 17 00:00:00 2001 From: Zhou Hui Date: Sat, 18 Jul 2026 09:44:28 +0800 Subject: [PATCH 7/7] Rename vars. --- .../blocktree/SegmentTermsEnumFrame.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java index 80efc8711fec..d67cb041421a 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/blocktree/SegmentTermsEnumFrame.java @@ -52,9 +52,9 @@ final class SegmentTermsEnumFrame { byte[] suffixLengthBytes; final ByteArrayDataInput suffixLengthsReader; - int[] suffixes; - int[] positions; - int[] offsets; + int[] suffixLengths; + int[] suffixLengthPositions; + int[] suffixOffsets; int[] termBlockOrds; int[] lastSubIndices; FixedBitSet termExists; @@ -213,7 +213,7 @@ void loadBlock() throws IOException { suffix = suffixLengthsReader.readVInt(); } else { // Handle subCode for non leaf block. - positions = new int[entCount]; + suffixLengthPositions = new int[entCount]; termExists = new FixedBitSet(entCount); subCodes = new long[entCount]; termBlockOrds = new int[entCount]; @@ -232,11 +232,11 @@ void loadBlock() throws IOException { lastSubIndex = 0; } termBlockOrds[0] = termBlockOrd; - positions[0] = suffixLengthsReader.getPosition(); + suffixLengthPositions[0] = suffixLengthsReader.getPosition(); lastSubIndices[0] = lastSubIndex; - for (int i = 1; i < suffixes.length; i++) { + for (int i = 1; i < suffixLengths.length; i++) { code = suffixLengthsReader.readVInt(); - suffixes[i] = code >>> 1; + suffixLengths[i] = code >>> 1; if ((code & 1) == 0) { termExists.set(i); termBlockOrd++; @@ -246,19 +246,19 @@ void loadBlock() throws IOException { lastSubIndex = i; } termBlockOrds[i] = termBlockOrd; - positions[i] = suffixLengthsReader.getPosition(); + suffixLengthPositions[i] = suffixLengthsReader.getPosition(); lastSubIndices[i] = lastSubIndex; } } // Reset suffixLengthsReader's position. suffixLengthsReader.setPosition(0); } else { - suffixes = new int[entCount]; - positions = new int[entCount]; + suffixLengths = new int[entCount]; + suffixLengthPositions = new int[entCount]; if (isLeafBlock) { - for (int i = 0; i < suffixes.length; i++) { - suffixes[i] = suffixLengthsReader.readVInt(); - positions[i] = suffixLengthsReader.getPosition(); + for (int i = 0; i < suffixLengths.length; i++) { + suffixLengths[i] = suffixLengthsReader.readVInt(); + suffixLengthPositions[i] = suffixLengthsReader.getPosition(); } } else { // Handle subCode for non leaf block. @@ -268,9 +268,9 @@ void loadBlock() throws IOException { lastSubIndices = new int[entCount]; int termBlockOrd = 0; int lastSubIndex = -1; - for (int i = 0; i < suffixes.length; i++) { + for (int i = 0; i < suffixLengths.length; i++) { code = suffixLengthsReader.readVInt(); - suffixes[i] = code >>> 1; + suffixLengths[i] = code >>> 1; if ((code & 1) == 0) { termExists.set(i); termBlockOrd++; @@ -280,14 +280,14 @@ void loadBlock() throws IOException { lastSubIndex = i; } termBlockOrds[i] = termBlockOrd; - positions[i] = suffixLengthsReader.getPosition(); + suffixLengthPositions[i] = suffixLengthsReader.getPosition(); lastSubIndices[i] = lastSubIndex; } } // Reset suffixLengthsReader's position. suffixLengthsReader.setPosition(0); - offsets = getOffsets(suffixes); - assert assertOffset(suffixes, offsets); + suffixOffsets = getOffsets(suffixLengths); + assert assertOffset(suffixLengths, suffixOffsets); } /*if (DEBUG) { @@ -772,7 +772,7 @@ private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnl } assert prefixMatches(target); - assert assertOffset(suffixes, offsets); + assert assertOffset(suffixLengths, suffixOffsets); int start = nextEnt; int end = entCount - 1; @@ -781,8 +781,8 @@ private SeekStatus binarySearchTermLeafUnEqual(BytesRef target, boolean exactOnl while (start <= end) { int mid = (start + end) >>> 1; nextEnt = mid + 1; - startBytePos = offsets[mid]; - suffix = suffixes[mid]; + startBytePos = suffixOffsets[mid]; + suffix = suffixLengths[mid]; // Binary search bytes in the suffix, comparing to the target cmp = @@ -870,7 +870,7 @@ private SeekStatus binarySearchTermNonLeafAllEqual(BytesRef target, boolean exac } assert prefixMatches(target); - assert assertOffset(suffixes, offsets); + assert assertOffset(suffixLengths, suffixOffsets); int start = nextEnt; int end = entCount - 1; @@ -986,7 +986,7 @@ private SeekStatus binarySearchTermNonLeafUnEqual(BytesRef target, boolean exact } assert prefixMatches(target); - assert assertOffset(suffixes, offsets); + assert assertOffset(suffixLengths, suffixOffsets); int start = nextEnt; int end = entCount - 1; @@ -997,8 +997,8 @@ private SeekStatus binarySearchTermNonLeafUnEqual(BytesRef target, boolean exact while (start <= end) { mid = (start + end) >>> 1; nextEnt = mid + 1; - startBytePos = offsets[mid]; - suffix = suffixes[mid]; + startBytePos = suffixOffsets[mid]; + suffix = suffixLengths[mid]; // Binary search bytes in the suffix, comparing to the target cmp = @@ -1106,19 +1106,19 @@ private void setSuffixesReaderAllEqual() { suffixesReader.setPosition(startBytePos + suffix); // Set suffixLengthsReader's position. - suffixLengthsReader.setPosition(positions[currentEnt]); + suffixLengthsReader.setPosition(suffixLengthPositions[currentEnt]); } // Set suffixesReader's position. private void setSuffixesReaderUnEqual() { int currentEnt = nextEnt - 1; - startBytePos = offsets[currentEnt]; - suffix = suffixes[currentEnt]; + startBytePos = suffixOffsets[currentEnt]; + suffix = suffixLengths[currentEnt]; suffixesReader.setPosition(startBytePos + suffix); // Set suffixLengthsReader's position. - suffixLengthsReader.setPosition(positions[currentEnt]); + suffixLengthsReader.setPosition(suffixLengthPositions[currentEnt]); } private void fillTerm() {