From 6830ef424ca65dfdcc5075414cd0f17926705508 Mon Sep 17 00:00:00 2001 From: Shubham Sharma Date: Tue, 26 May 2026 12:11:49 +0100 Subject: [PATCH 1/7] Promoted PrimaryKeyLookup and added tests --- lucene/CHANGES.txt | 3 + .../apache/lucene/index/PrimaryKeyLookup.java | 156 ++++++++++++++ .../lucene/index/TestPrimaryKeyLookup.java | 196 ++++++++++++++++++ .../lucene/tests/index/PerThreadPKLookup.java | 115 +--------- 4 files changed, 365 insertions(+), 105 deletions(-) create mode 100644 lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java create mode 100644 lucene/core/src/test/org/apache/lucene/index/TestPrimaryKeyLookup.java diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index ed1fed222a59..c8dbf3952732 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -100,6 +100,9 @@ API Changes New Features --------------------- +* GITHUB#15520: Promote PerThreadPKLookup from the test-framework to core as + PrimaryKeyLookup. The original PerThreadPKLookup is now a deprecated. (Shubham Sharma) + * GITHUB#15505: Upgrade snowball to 2d2e312df56f2ede014a4ffb3e91e6dea43c24be. New stemmer: PolishStemmer (and PolishSnowballAnalyzer in the stempel package) (Justas Sakalauskas, Dawid Weiss) diff --git a/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java b/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java new file mode 100644 index 000000000000..91f62a7c4a58 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.lucene.index.IndexReader.CacheHelper; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; + +/** + * Class to do efficient primary-key lookups. Instances are not thread-safe. Each calling thread must own its own + * instance, and an instance must only ever be used from the thread that constructed it. When assertions are + * enabled this invariant is checked on every lookup and reopen call. Do not use this if a term may appear in more + * than one document! It will only return the first one it finds. + * + * @lucene.experimental + */ +public class PrimaryKeyLookup { + + protected final String idFieldName; + protected final TermsEnum[] termsEnums; + protected final PostingsEnum[] postingsEnums; + protected final Bits[] liveDocs; + protected final int[] docBases; + protected final int numEnums; + protected final boolean hasDeletions; + protected final Map enumIndexes; + + // Thread-stickiness guard. + private final long ownerThreadId; + + public PrimaryKeyLookup(IndexReader reader, String idFieldName) throws IOException { + this(reader, idFieldName, Collections.emptyMap(), null, null); + } + + /** + * Construct a {@code PrimaryKeyLookup} instance. + */ + protected PrimaryKeyLookup( + IndexReader reader, + String idFieldName, + Map prevEnumIndexes, + TermsEnum[] reusableTermsEnums, + PostingsEnum[] reusablePostingsEnums) + throws IOException { + this.idFieldName = idFieldName; + this.ownerThreadId = Thread.currentThread().threadId(); + + List leaves = new ArrayList<>(reader.leaves()); + // Larger segments are more likely to have the id, so we sort largest to smallest by numDocs: + leaves.sort((c1, c2) -> c2.reader().numDocs() - c1.reader().numDocs()); + + termsEnums = new TermsEnum[leaves.size()]; + postingsEnums = new PostingsEnum[leaves.size()]; + liveDocs = new Bits[leaves.size()]; + docBases = new int[leaves.size()]; + enumIndexes = new HashMap<>(); + int numEnums = 0; + boolean hasDeletions = false; + + for (LeafReaderContext context : leaves) { + LeafReader leafReader = context.reader(); + CacheHelper cacheHelper = leafReader.getCoreCacheHelper(); + IndexReader.CacheKey cacheKey = cacheHelper == null ? null : cacheHelper.getKey(); + + if (cacheKey != null && prevEnumIndexes.containsKey(cacheKey)) { + // Reuse termsEnum, postingsEnum. + int seg = prevEnumIndexes.get(cacheKey); + termsEnums[numEnums] = reusableTermsEnums[seg]; + postingsEnums[numEnums] = reusablePostingsEnums[seg]; + } else { + // New or empty segment. + Terms terms = leafReader.terms(idFieldName); + if (terms != null) { + termsEnums[numEnums] = terms.iterator(); + assert termsEnums[numEnums] != null; + } + } + + if (termsEnums[numEnums] != null) { + if (cacheKey != null) { + enumIndexes.put(cacheKey, numEnums); + } + + docBases[numEnums] = context.docBase; + liveDocs[numEnums] = leafReader.getLiveDocs(); + hasDeletions |= leafReader.hasDeletions(); + + numEnums++; + } + } + + this.numEnums = numEnums; + this.hasDeletions = hasDeletions; + } + + /** Returns docID if found, else -1. */ + public int lookup(BytesRef id) throws IOException { + assert assertOwnerThread(); + for (int seg = 0; seg < numEnums; seg++) { + if (termsEnums[seg].seekExact(id)) { + postingsEnums[seg] = termsEnums[seg].postings(postingsEnums[seg], 0); + int docID; + while ((docID = postingsEnums[seg].nextDoc()) != PostingsEnum.NO_MORE_DOCS) { + if (liveDocs[seg] == null || liveDocs[seg].get(docID)) { + return docBases[seg] + docID; + } + } + assert hasDeletions; + } + } + + return -1; + } + + /** + * Returns a new PrimaryKeyLookup instance bound reusing this instance per-segment TermsEnum/PostingsEnum etc. + */ + public PrimaryKeyLookup reopen(IndexReader reader) throws IOException { + assert assertOwnerThread(); + if (reader == null) { + return null; + } + return new PrimaryKeyLookup( + reader, this.idFieldName, this.enumIndexes, this.termsEnums, this.postingsEnums); + } + + private boolean assertOwnerThread() { + long current = Thread.currentThread().threadId(); + assert current == ownerThreadId + : "PrimaryKeyLookup is not thread-safe: it was created by thread " + + ownerThreadId + + " but accessed from thread " + + current; + return true; + } +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestPrimaryKeyLookup.java b/lucene/core/src/test/org/apache/lucene/index/TestPrimaryKeyLookup.java new file mode 100644 index 000000000000..e121ae7cdde6 --- /dev/null +++ b/lucene/core/src/test/org/apache/lucene/index/TestPrimaryKeyLookup.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.KeywordField; +import org.apache.lucene.store.Directory; +import org.apache.lucene.tests.analysis.MockAnalyzer; +import org.apache.lucene.tests.util.LuceneTestCase; + +public class TestPrimaryKeyLookup extends LuceneTestCase { + + public void testReopen() throws Exception { + Directory dir = newDirectory(); + IndexWriter writer = + new IndexWriter( + dir, + new IndexWriterConfig(new MockAnalyzer(random())) + .setMergePolicy(NoMergePolicy.INSTANCE)); + + Document doc; + doc = new Document(); + doc.add(new KeywordField("PK", "1", Field.Store.NO)); + writer.addDocument(doc); + + doc = new Document(); + doc.add(new KeywordField("PK", "2", Field.Store.NO)); + writer.addDocument(doc); + writer.flush(); + + // Segment with no PK terms. + doc = new Document(); + doc.add(new KeywordField("PK2", "3", Field.Store.NO)); + writer.addDocument(doc); + + doc = new Document(); + doc.add(new KeywordField("PK2", "4", Field.Store.NO)); + writer.addDocument(doc); + writer.flush(); + + DirectoryReader reader1 = DirectoryReader.open(writer); + PrimaryKeyLookup pk1 = new PrimaryKeyLookup(reader1, "PK"); + + doc = new Document(); + doc.add(new KeywordField("PK", "5", Field.Store.NO)); + writer.addDocument(doc); + + doc = new Document(); + doc.add(new KeywordField("PK", "6", Field.Store.NO)); + writer.addDocument(doc); + writer.deleteDocuments(new Term("PK", "1")); + writer.flush(); + + doc = new Document(); + doc.add(new KeywordField("PK2", "7", Field.Store.NO)); + writer.addDocument(doc); + + doc = new Document(); + doc.add(new KeywordField("PK2", "8", Field.Store.NO)); + writer.addDocument(doc); + writer.flush(); + + assertEquals(0, pk1.lookup(newBytesRef("1"))); + assertEquals(1, pk1.lookup(newBytesRef("2"))); + assertEquals(-1, pk1.lookup(newBytesRef("5"))); + assertEquals(-1, pk1.lookup(newBytesRef("8"))); + + DirectoryReader reader2 = DirectoryReader.openIfChanged(reader1); + PrimaryKeyLookup pk2 = pk1.reopen(reader2); + + assertEquals(-1, pk2.lookup(newBytesRef("1"))); + assertEquals(1, pk2.lookup(newBytesRef("2"))); + assertEquals(4, pk2.lookup(newBytesRef("5"))); + assertEquals(-1, pk2.lookup(newBytesRef("8"))); + + doc = new Document(); + doc.add(new KeywordField("PK", "9", Field.Store.NO)); + writer.addDocument(doc); + + doc = new Document(); + doc.add(new KeywordField("PK", "10", Field.Store.NO)); + writer.addDocument(doc); + writer.flush(); + + assertEquals(-1, pk2.lookup(newBytesRef("9"))); + DirectoryReader reader3 = DirectoryReader.openIfChanged(reader2); + PrimaryKeyLookup pk3 = pk2.reopen(reader3); + assertEquals(8, pk3.lookup(newBytesRef("9"))); + + DirectoryReader reader4 = DirectoryReader.openIfChanged(reader3); + assertNull(pk3.reopen(reader4)); + + writer.close(); + reader1.close(); + reader2.close(); + reader3.close(); + dir.close(); + } + + public void testPKLookupWithUpdate() throws Exception { + Directory dir = newDirectory(); + IndexWriter writer = + new IndexWriter( + dir, + new IndexWriterConfig(new MockAnalyzer(random())) + .setMergePolicy(NoMergePolicy.INSTANCE)); + + Document doc; + doc = new Document(); + doc.add(new KeywordField("PK", "1", Field.Store.NO)); + doc.add(new KeywordField("version", "1", Field.Store.NO)); + writer.addDocument(doc); + + doc = new Document(); + doc.add(new KeywordField("PK", "1", Field.Store.NO)); + doc.add(new KeywordField("version", "2", Field.Store.NO)); + writer.updateDocument(new Term("PK", "1"), doc); + + doc = new Document(); + doc.add(new KeywordField("PK", "1", Field.Store.NO)); + doc.add(new KeywordField("version", "3", Field.Store.NO)); + writer.updateDocument(new Term("PK", "1"), doc); + writer.flush(); + writer.close(); + + DirectoryReader reader = DirectoryReader.open(dir); + PrimaryKeyLookup pk = new PrimaryKeyLookup(reader, "PK"); + assertEquals(2, pk.lookup(newBytesRef("1"))); + + reader.close(); + dir.close(); + } + + /** + * Constructing on one thread and calling {@link PrimaryKeyLookup#lookup} from another must fire + * the thread-stickiness assertion. + */ + public void testThreadStickinessAssertion() throws Exception { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))); + Document doc = new Document(); + doc.add(new KeywordField("PK", "1", Field.Store.NO)); + writer.addDocument(doc); + writer.commit(); + writer.close(); + + DirectoryReader reader = DirectoryReader.open(dir); + PrimaryKeyLookup pk = new PrimaryKeyLookup(reader, "PK"); + + if (TEST_ASSERTS_ENABLED) { + // pk was constructed on this thread; calling lookup from any other thread must trip + // the thread-stickiness assertion in PrimaryKeyLookup. + class CrossThreadLookup extends Thread { + Throwable thrown; + + @Override + public void run() { + try { + pk.lookup(newBytesRef("1")); + } catch (Throwable t) { + thrown = t; + } + } + } + + CrossThreadLookup other = new CrossThreadLookup(); + other.setName("pk-cross-thread-lookup"); + other.start(); + other.join(); + + assertTrue( + "expected AssertionError but got: " + other.thrown, + other.thrown instanceof AssertionError); + assertTrue( + "unexpected message: " + other.thrown.getMessage(), + other.thrown.getMessage().contains("not thread-safe")); + } + reader.close(); + dir.close(); + } +} diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java index 5db9a2409e8c..0743b9f9bc55 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java @@ -17,124 +17,29 @@ package org.apache.lucene.tests.index; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReader.CacheHelper; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.LeafReaderContext; -import org.apache.lucene.index.PostingsEnum; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; +import org.apache.lucene.index.PrimaryKeyLookup; /** - * Utility class to do efficient primary-key (only 1 doc contains the given term) lookups by - * segment, re-using the enums. This class is not thread safe, so it is the caller's job to create - * and use one instance of this per thread. Do not use this if a term may appear in more than one - * document! It will only return the first one it finds. + * @deprecated Use {@link PrimaryKeyLookup} instead. */ -public class PerThreadPKLookup { - - private final String idFieldName; - protected final TermsEnum[] termsEnums; - protected final PostingsEnum[] postingsEnums; - protected final Bits[] liveDocs; - protected final int[] docBases; - protected final int numEnums; - protected final boolean hasDeletions; - private final Map enumIndexes; +@Deprecated +public class PerThreadPKLookup extends PrimaryKeyLookup { public PerThreadPKLookup(IndexReader reader, String idFieldName) throws IOException { - this(reader, idFieldName, Collections.emptyMap(), null, null); - } - - private PerThreadPKLookup( - IndexReader reader, - String idFieldName, - Map prevEnumIndexes, - TermsEnum[] reusableTermsEnums, - PostingsEnum[] reusablePostingsEnums) - throws IOException { - this.idFieldName = idFieldName; - - List leaves = new ArrayList<>(reader.leaves()); - // Larger segments are more likely to have the id, so we sort largest to smallest by numDocs: - leaves.sort((c1, c2) -> c2.reader().numDocs() - c1.reader().numDocs()); - - termsEnums = new TermsEnum[leaves.size()]; - postingsEnums = new PostingsEnum[leaves.size()]; - liveDocs = new Bits[leaves.size()]; - docBases = new int[leaves.size()]; - enumIndexes = new HashMap<>(); - int numEnums = 0; - boolean hasDeletions = false; - - for (int i = 0; i < leaves.size(); i++) { - LeafReaderContext context = leaves.get(i); - LeafReader leafReader = context.reader(); - CacheHelper cacheHelper = leafReader.getCoreCacheHelper(); - IndexReader.CacheKey cacheKey = cacheHelper == null ? null : cacheHelper.getKey(); - - if (cacheKey != null && prevEnumIndexes.containsKey(cacheKey)) { - // Reuse termsEnum, postingsEnum. - int seg = prevEnumIndexes.get(cacheKey); - termsEnums[numEnums] = reusableTermsEnums[seg]; - postingsEnums[numEnums] = reusablePostingsEnums[seg]; - } else { - // New or empty segment. - Terms terms = leafReader.terms(idFieldName); - if (terms != null) { - termsEnums[numEnums] = terms.iterator(); - assert termsEnums[numEnums] != null; - } - } - - if (termsEnums[numEnums] != null) { - if (cacheKey != null) { - enumIndexes.put(cacheKey, numEnums); - } - - docBases[numEnums] = context.docBase; - liveDocs[numEnums] = leafReader.getLiveDocs(); - hasDeletions |= leafReader.hasDeletions(); - - numEnums++; - } - } - - this.numEnums = numEnums; - this.hasDeletions = hasDeletions; + super(reader, idFieldName); } - /** Returns docID if found, else -1. */ - public int lookup(BytesRef id) throws IOException { - for (int seg = 0; seg < numEnums; seg++) { - if (termsEnums[seg].seekExact(id)) { - postingsEnums[seg] = termsEnums[seg].postings(postingsEnums[seg], 0); - int docID = -1; - while ((docID = postingsEnums[seg].nextDoc()) != PostingsEnum.NO_MORE_DOCS) { - if (liveDocs[seg] == null || liveDocs[seg].get(docID)) { - return docBases[seg] + docID; - } - } - assert hasDeletions; - } - } - - return -1; + private PerThreadPKLookup(IndexReader reader, PerThreadPKLookup prev) throws IOException { + super(reader, prev.idFieldName, prev.enumIndexes, prev.termsEnums, prev.postingsEnums); } - /** Reuse previous PerThreadPKLookup's termsEnum and postingsEnum. */ + /** Narrows {@link PrimaryKeyLookup#reopen} to keep the historical return type. */ + @Override public PerThreadPKLookup reopen(IndexReader reader) throws IOException { if (reader == null) { return null; } - return new PerThreadPKLookup( - reader, this.idFieldName, this.enumIndexes, this.termsEnums, this.postingsEnums); + return new PerThreadPKLookup(reader, this); } } From a04a4a595105cbf7d8839b87c440ca499f286c90 Mon Sep 17 00:00:00 2001 From: Shubham Sharma Date: Tue, 26 May 2026 12:14:48 +0100 Subject: [PATCH 2/7] Tidy --- .../apache/lucene/index/PrimaryKeyLookup.java | 16 ++++++++-------- .../lucene/tests/index/PerThreadPKLookup.java | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java b/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java index 91f62a7c4a58..819ce3feeffd 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java +++ b/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java @@ -27,10 +27,11 @@ import org.apache.lucene.util.BytesRef; /** - * Class to do efficient primary-key lookups. Instances are not thread-safe. Each calling thread must own its own - * instance, and an instance must only ever be used from the thread that constructed it. When assertions are - * enabled this invariant is checked on every lookup and reopen call. Do not use this if a term may appear in more - * than one document! It will only return the first one it finds. + * Class to do efficient primary-key lookups. Instances are not thread-safe. Each calling thread + * must own its own instance, and an instance must only ever be used from the thread that + * constructed it. When assertions are enabled this invariant is checked on every lookup and reopen + * call. Do not use this if a term may appear in more than one document! It will only return the + * first one it finds. * * @lucene.experimental */ @@ -52,9 +53,7 @@ public PrimaryKeyLookup(IndexReader reader, String idFieldName) throws IOExcepti this(reader, idFieldName, Collections.emptyMap(), null, null); } - /** - * Construct a {@code PrimaryKeyLookup} instance. - */ + /** Construct a {@code PrimaryKeyLookup} instance. */ protected PrimaryKeyLookup( IndexReader reader, String idFieldName, @@ -133,7 +132,8 @@ public int lookup(BytesRef id) throws IOException { } /** - * Returns a new PrimaryKeyLookup instance bound reusing this instance per-segment TermsEnum/PostingsEnum etc. + * Returns a new PrimaryKeyLookup instance bound reusing this instance per-segment + * TermsEnum/PostingsEnum etc. */ public PrimaryKeyLookup reopen(IndexReader reader) throws IOException { assert assertOwnerThread(); diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java index 0743b9f9bc55..f546a23887ef 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java @@ -21,7 +21,7 @@ import org.apache.lucene.index.PrimaryKeyLookup; /** - * @deprecated Use {@link PrimaryKeyLookup} instead. + * @deprecated Use {@link PrimaryKeyLookup} instead. */ @Deprecated public class PerThreadPKLookup extends PrimaryKeyLookup { From dee58dea8181bbda0b9c3553d4cb83995089a9ee Mon Sep 17 00:00:00 2001 From: Shubham Sharma Date: Tue, 26 May 2026 12:29:50 +0100 Subject: [PATCH 3/7] Tidy and javadoc --- .../apache/lucene/index/PrimaryKeyLookup.java | 19 +++++++++++++++++++ .../lucene/tests/index/PerThreadPKLookup.java | 3 +++ 2 files changed, 22 insertions(+) diff --git a/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java b/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java index 819ce3feeffd..e22b55a19cf6 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java +++ b/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java @@ -37,18 +37,37 @@ */ public class PrimaryKeyLookup { + /** Name of the primary-key field. */ protected final String idFieldName; + + /** Per-segment {@link TermsEnum}. */ protected final TermsEnum[] termsEnums; + + /** Per-segment {@link PostingsEnum}. */ protected final PostingsEnum[] postingsEnums; + + /** Per-segment live-docs bitset. */ protected final Bits[] liveDocs; + + /** Per-segment {@code docBase} used to translate leaf docIDs to absolute docIDs. */ protected final int[] docBases; + + /** Number of populated entries in termsEnums/postingsEnums. */ protected final int numEnums; + + /** True if any segment in this lookup has deletions. */ protected final boolean hasDeletions; + + /** + * Maps a segment core cache key to its index in termsEnums/postingsEnums, so that reopen can + * transfer per-segment enum state to the next generation. + */ protected final Map enumIndexes; // Thread-stickiness guard. private final long ownerThreadId; + /** Construct a {@code PrimaryKeyLookup} bound to {@code reader}. */ public PrimaryKeyLookup(IndexReader reader, String idFieldName) throws IOException { this(reader, idFieldName, Collections.emptyMap(), null, null); } diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java index f546a23887ef..9ea7d10c45f9 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java @@ -21,11 +21,14 @@ import org.apache.lucene.index.PrimaryKeyLookup; /** + * A {@link PerThreadPKLookup} that can be used for primary key lookups. + * * @deprecated Use {@link PrimaryKeyLookup} instead. */ @Deprecated public class PerThreadPKLookup extends PrimaryKeyLookup { + /** Construct a {@code PerThreadPKLookup} bound to {@code reader}. */ public PerThreadPKLookup(IndexReader reader, String idFieldName) throws IOException { super(reader, idFieldName); } From 31fb1095266bf6a6d577cdb97e434b1e0a44557f Mon Sep 17 00:00:00 2001 From: Shubham Sharma Date: Tue, 26 May 2026 12:33:42 +0100 Subject: [PATCH 4/7] Javadoc --- lucene/CHANGES.txt | 6 +++--- .../org/apache/lucene/tests/index/PerThreadPKLookup.java | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index cc6b748cea09..a764a8574d5b 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -100,9 +100,6 @@ API Changes New Features --------------------- -* GITHUB#15520: Promote PerThreadPKLookup from the test-framework to core as - PrimaryKeyLookup. The original PerThreadPKLookup is now a deprecated. (Shubham Sharma) - * GITHUB#15505: Upgrade snowball to 2d2e312df56f2ede014a4ffb3e91e6dea43c24be. New stemmer: PolishStemmer (and PolishSnowballAnalyzer in the stempel package) (Justas Sakalauskas, Dawid Weiss) @@ -114,6 +111,9 @@ New Features * GITHUB#15818: Add BM25 k3 query-term frequency saturation to BM25Similarity. (Sagar Upadhyaya) +* GITHUB#15520: Promote PerThreadPKLookup from the test-framework to core as + PrimaryKeyLookup. The original PerThreadPKLookup is now deprecated. (Shubham Sharma) + Improvements --------------------- * GITHUB#15704: Replace LinkedList with more efficient data structure. (Renato Haeberli) diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java index 9ea7d10c45f9..3ca2f88c4e9b 100644 --- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java +++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/PerThreadPKLookup.java @@ -21,7 +21,10 @@ import org.apache.lucene.index.PrimaryKeyLookup; /** - * A {@link PerThreadPKLookup} that can be used for primary key lookups. + * Utility class to do efficient primary-key (only 1 doc contains the given term) lookups by + * segment, re-using the enums. This class is not thread safe, so it is the caller's job to create + * and use one instance of this per thread. Do not use this if a term may appear in more than one + * document! It will only return the first one it finds. * * @deprecated Use {@link PrimaryKeyLookup} instead. */ From 005eaae8b71b4f0d53708a5ba7636fd6691a7662 Mon Sep 17 00:00:00 2001 From: Shubham Sharma Date: Thu, 28 May 2026 15:27:56 +0100 Subject: [PATCH 5/7] Added fast and slow paths --- .../apache/lucene/index/PrimaryKeyLookup.java | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java b/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java index e22b55a19cf6..d5c7cc85cbaa 100644 --- a/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java +++ b/lucene/core/src/java/org/apache/lucene/index/PrimaryKeyLookup.java @@ -135,18 +135,28 @@ protected PrimaryKeyLookup( public int lookup(BytesRef id) throws IOException { assert assertOwnerThread(); for (int seg = 0; seg < numEnums; seg++) { - if (termsEnums[seg].seekExact(id)) { - postingsEnums[seg] = termsEnums[seg].postings(postingsEnums[seg], 0); - int docID; - while ((docID = postingsEnums[seg].nextDoc()) != PostingsEnum.NO_MORE_DOCS) { - if (liveDocs[seg] == null || liveDocs[seg].get(docID)) { - return docBases[seg] + docID; - } + if (termsEnums[seg].seekExact(id) == false) { + continue; + } + postingsEnums[seg] = termsEnums[seg].postings(postingsEnums[seg], 0); + + if (liveDocs[seg] == null) { + // Fast path: segment has no deletions. + int docID = postingsEnums[seg].nextDoc(); + assert docID != PostingsEnum.NO_MORE_DOCS; + return docBases[seg] + docID; + } + + // Slow path: segment has deletions; scan until we find a live doc or exhaust. + int docID; + while ((docID = postingsEnums[seg].nextDoc()) != PostingsEnum.NO_MORE_DOCS) { + if (liveDocs[seg].get(docID)) { + return docBases[seg] + docID; } - assert hasDeletions; } - } + assert hasDeletions; + } return -1; } From af7eb9ff863e6c7c116f609e00d9b786e8b8058c Mon Sep 17 00:00:00 2001 From: Shubham Sharma Date: Fri, 29 May 2026 16:03:49 +0100 Subject: [PATCH 6/7] Changed extending class --- .../sandbox/codecs/idversion/TestIDVersionPostingsFormat.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/idversion/TestIDVersionPostingsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/idversion/TestIDVersionPostingsFormat.java index 8520103c541e..f10900af0d24 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/idversion/TestIDVersionPostingsFormat.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/idversion/TestIDVersionPostingsFormat.java @@ -39,6 +39,7 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.PrimaryKeyLookup; import org.apache.lucene.index.Term; import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.sandbox.codecs.idversion.StringAndPayloadField.SingleTokenWithPayloadTokenStream; @@ -334,7 +335,7 @@ public void testRandom() throws Exception { dir.close(); } - private static class PerThreadVersionPKLookup extends PerThreadPKLookup { + private static class PerThreadVersionPKLookup extends PrimaryKeyLookup { public PerThreadVersionPKLookup(IndexReader r, String field) throws IOException { super(r, field); } From bcf82974b0dff47e2d1fa84d25cc30e8abc48bc3 Mon Sep 17 00:00:00 2001 From: Shubham Sharma Date: Fri, 29 May 2026 16:18:28 +0100 Subject: [PATCH 7/7] gradle tidy --- .../sandbox/codecs/idversion/TestIDVersionPostingsFormat.java | 1 - 1 file changed, 1 deletion(-) diff --git a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/idversion/TestIDVersionPostingsFormat.java b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/idversion/TestIDVersionPostingsFormat.java index f10900af0d24..a31dc72009a8 100644 --- a/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/idversion/TestIDVersionPostingsFormat.java +++ b/lucene/sandbox/src/test/org/apache/lucene/sandbox/codecs/idversion/TestIDVersionPostingsFormat.java @@ -52,7 +52,6 @@ import org.apache.lucene.tests.analysis.MockAnalyzer; import org.apache.lucene.tests.analysis.MockTokenFilter; import org.apache.lucene.tests.analysis.MockTokenizer; -import org.apache.lucene.tests.index.PerThreadPKLookup; import org.apache.lucene.tests.index.RandomIndexWriter; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.lucene.tests.util.TestUtil;