Upgrading lucene version to 10.5.0

gradle/libs.versions.toml

@@ -1,6 +1,6 @@
 [versions]
 opensearch        = "3.8.0"
-lucene            = "10.4.0"
+lucene            = "10.5.0"
 
 bundled_jdk_vendor = "adoptium"
 bundled_jdk = "25.0.3+9"

libs/core/licenses/lucene-core-10.5.0.jar.sha1

@@ -0,0 +1 @@
+383deeee9b44ef15daaf7541d57ea5183d52a7b5

libs/core/src/main/java/org/opensearch/Version.java

@@ -173,7 +173,7 @@ private static String legacyFriendlyIdToString(int versionId) {
     public static final Version V_3_6_1 = new Version(3060199, org.apache.lucene.util.Version.LUCENE_10_4_0);
     public static final Version V_3_7_0 = new Version(3070099, org.apache.lucene.util.Version.LUCENE_10_4_0);
     public static final Version V_3_7_1 = new Version(3070199, org.apache.lucene.util.Version.LUCENE_10_4_0);
-    public static final Version V_3_8_0 = new Version(3080099, org.apache.lucene.util.Version.LUCENE_10_4_0);
+    public static final Version V_3_8_0 = new Version(3080099, org.apache.lucene.util.Version.LUCENE_10_5_0);
     public static final Version CURRENT = V_3_8_0;
 
     protected static final Map<Integer, Version> idToVersion;

libs/netty4/licenses/lucene-core-10.5.0.jar.sha1

@@ -0,0 +1 @@
+383deeee9b44ef15daaf7541d57ea5183d52a7b5

modules/lang-expression/licenses/lucene-expressions-10.5.0.jar.sha1

@@ -0,0 +1 @@
+a1820135f8c0f51a6635905d754705472592a008

plugins/analysis-icu/licenses/lucene-analysis-icu-10.5.0.jar.sha1

@@ -0,0 +1 @@
+cb67d073b767ed1d1d24a9b03117ef9bbac5c1aa

plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-10.5.0.jar.sha1

@@ -0,0 +1 @@
+c205946d6a177bf516f590e89e4cda36a537c33f

plugins/analysis-nori/licenses/lucene-analysis-nori-10.5.0.jar.sha1

@@ -0,0 +1 @@
+2a2db150e967984e18d293619142b2cd27f09f63

plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-10.5.0.jar.sha1

@@ -0,0 +1 @@
+2dfd6a62be5ed3822b01c96a0da805fa55e962ef

plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-10.5.0.jar.sha1

@@ -0,0 +1 @@
+c6ef06ff204f4e370584a37674cd676b708d5ceb

plugins/analysis-stempel/licenses/lucene-analysis-stempel-10.5.0.jar.sha1

@@ -0,0 +1 @@
+4bbed0d5695daefcbf6c380d72c1b54b99ddf397

plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-10.5.0.jar.sha1

@@ -0,0 +1 @@
+1ea877703aeefcd8109934c6894387ee6fde7163

server/licenses/lucene-analysis-common-10.5.0.jar.sha1

@@ -0,0 +1 @@
+2b314f57f18f3d9ad855f611f6f391519ac78a8d

server/licenses/lucene-backward-codecs-10.5.0.jar.sha1

@@ -0,0 +1 @@
+0967b73127de6f17bbd83c015d258872359e8f97

server/licenses/lucene-core-10.5.0.jar.sha1

@@ -0,0 +1 @@
+383deeee9b44ef15daaf7541d57ea5183d52a7b5

server/licenses/lucene-grouping-10.5.0.jar.sha1

@@ -0,0 +1 @@
+ed85d8acb132c4aa175c92778132f8e54a00ce0e

server/licenses/lucene-highlighter-10.5.0.jar.sha1

@@ -0,0 +1 @@
+fccffbe10fa6fd84bc884df844eb850ffcc8d51b

server/licenses/lucene-join-10.5.0.jar.sha1

@@ -0,0 +1 @@
+b34a23804c461495fb98ef597d72cfc134466b3b

server/licenses/lucene-memory-10.5.0.jar.sha1

@@ -0,0 +1 @@
+a18c7a57e054836d2dc7ece7dfe28a9cad61f660

server/licenses/lucene-misc-10.5.0.jar.sha1

@@ -0,0 +1 @@
+113d957ae3c80c8c52052f4d38badc9ae5c1b60d

server/licenses/lucene-queries-10.5.0.jar.sha1

@@ -0,0 +1 @@
+c96555ec32a411d860527ea13add388d72a614b2

server/licenses/lucene-queryparser-10.5.0.jar.sha1

@@ -0,0 +1 @@
+dea0c58101825f086127abf5adce13ced0bd0fc2

server/licenses/lucene-sandbox-10.5.0.jar.sha1

@@ -0,0 +1 @@
+928527693dc814ba0125659b07316e260713e38f

server/licenses/lucene-spatial-extras-10.5.0.jar.sha1

@@ -0,0 +1 @@
+900c6d78202cfd96a8e513f2271c972c4af98388

server/licenses/lucene-spatial3d-10.5.0.jar.sha1

@@ -0,0 +1 @@
+e2a6601ca2d16796a068589128787114d15935c3

server/licenses/lucene-suggest-10.5.0.jar.sha1

@@ -0,0 +1 @@
+62ee5896308143da8ee6b8c0f6a1a2f724c1ffb4

server/src/main/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesConsumerWrapper.java

@@ -31,15 +31,19 @@ public Lucene90DocValuesConsumerWrapper(
         String dataCodec,
         String dataExtension,
         String metaCodec,
-        String metaExtension
+        String metaExtension,
+        String skipIndexCodec,
+        String skipIndexExtension
     ) throws IOException {
         lucene90DocValuesConsumer = new Lucene90DocValuesConsumer(
             state,
             skipIndexIntervalSize,
             dataCodec,
             dataExtension,
             metaCodec,
-            metaExtension
+            metaExtension,
+            skipIndexCodec,
+            skipIndexExtension
         );
     }
 

server/src/main/java/org/apache/lucene/codecs/lucene90/Lucene90DocValuesProducerWrapper.java

@@ -30,9 +30,19 @@ public Lucene90DocValuesProducerWrapper(
         String dataCodec,
         String dataExtension,
         String metaCodec,
-        String metaExtension
+        String metaExtension,
+        String skipIndexCodec,
+        String skipIndexExtension
     ) throws IOException {
-        lucene90DocValuesProducer = new Lucene90DocValuesProducer(state, dataCodec, dataExtension, metaCodec, metaExtension);
+        lucene90DocValuesProducer = new Lucene90DocValuesProducer(
+            state,
+            dataCodec,
+            dataExtension,
+            metaCodec,
+            metaExtension,
+            skipIndexCodec,
+            skipIndexExtension
+        );
     }
 
     public DocValuesProducer getLucene90DocValuesProducer() {

server/src/main/java/org/apache/lucene/index/SortedSetDocValuesWriterWrapper.java

@@ -33,7 +33,12 @@ public class SortedSetDocValuesWriterWrapper implements DocValuesWriterWrapper<S
      * @see SortedSetDocValuesWriter
      */
     public SortedSetDocValuesWriterWrapper(FieldInfo fieldInfo, Counter counter, ByteBlockPool byteBlockPool) {
-        sortedSetDocValuesWriterDelegate = new SortedSetDocValuesWriter(fieldInfo, counter, byteBlockPool);
+        sortedSetDocValuesWriterDelegate = new SortedSetDocValuesWriter(
+            fieldInfo,
+            counter,
+            byteBlockPool,
+            new SharedIndexingScratch(counter)
+        );
     }
 
     /**

server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesConsumerFactory.java

@@ -11,6 +11,7 @@
 import org.apache.lucene.codecs.DocValuesConsumer;
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesConsumerWrapper;
 import org.apache.lucene.index.SegmentWriteState;
+import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat;
 
 import java.io.IOException;
 
@@ -58,7 +59,9 @@ public static DocValuesConsumer getDocValuesConsumerForCompositeCodec(
             dataCodec,
             dataExtension,
             metaCodec,
-            metaExtension
+            metaExtension,
+            Composite912DocValuesFormat.SKIP_INDEX_DOC_VALUES_CODEC,
+            Composite912DocValuesFormat.SKIP_INDEX_DOC_VALUES_EXTENSION
         );
         return lucene90DocValuesConsumerWrapper.getLucene90DocValuesConsumer();
 

server/src/main/java/org/opensearch/index/codec/composite/LuceneDocValuesProducerFactory.java

@@ -12,6 +12,7 @@
 import org.apache.lucene.codecs.lucene90.Lucene90DocValuesProducerWrapper;
 import org.apache.lucene.index.SegmentReadState;
 import org.opensearch.index.codec.composite.composite912.Composite912Codec;
+import org.opensearch.index.codec.composite.composite912.Composite912DocValuesFormat;
 
 import java.io.IOException;
 
@@ -45,7 +46,9 @@ public static DocValuesProducer getDocValuesProducerForCompositeCodec(
                     dataCodec,
                     dataExtension,
                     metaCodec,
-                    metaExtension
+                    metaExtension,
+                    Composite912DocValuesFormat.SKIP_INDEX_DOC_VALUES_CODEC,
+                    Composite912DocValuesFormat.SKIP_INDEX_DOC_VALUES_EXTENSION
                 );
                 return lucene90DocValuesProducerWrapper.getLucene90DocValuesProducer();
             default:

server/src/main/java/org/opensearch/index/codec/composite/composite912/Composite912DocValuesFormat.java

@@ -61,6 +61,12 @@ public class Composite912DocValuesFormat extends DocValuesFormat {
     /** Filename extension for the composite index meta doc values */
     public static final String META_DOC_VALUES_EXTENSION = "cidvm";
 
+    /** Skip index doc values codec name for Composite Doc Values Format */
+    public static final String SKIP_INDEX_DOC_VALUES_CODEC = "Composite912DocValuesSkipIndex";
+
+    /** Filename extension for the composite index skip index doc values */
+    public static final String SKIP_INDEX_DOC_VALUES_EXTENSION = "cidvs";
+
     /** Initial version for the Composite90DocValuesFormat */
     public static final int VERSION_START = 0;
 

server/src/main/java/org/opensearch/index/query/MoreLikeThisQueryBuilder.java

@@ -1128,7 +1128,7 @@ private static Fields[] getFieldsFor(MultiTermVectorsResponse responses) throws
             }
             likeFields.add(getResponse.getFields());
         }
-        return likeFields.toArray(Fields.EMPTY_ARRAY);
+        return likeFields.toArray(new Fields[0]);
     }
 
     private static void checkRoutingMissingException(MultiTermVectorsItemResponse response) {

server/src/main/java/org/opensearch/lucene/util/CombinedBitSet.java

@@ -142,4 +142,17 @@ public int nextSetBit(int start, int end) {
         return next;
 
     }
+
+    @Override
+    public int nextClearBit(int start, int upperBound) {
+        assert start >= 0 && start < length() : "start=" + start + " numBits=" + length();
+        assert upperBound >= start && upperBound <= length() : "upperBound=" + upperBound + " numBits=" + length();
+
+        for (int i = start; i < upperBound; i++) {
+            if (get(i) == false) {
+                return i;
+            }
+        }
+        return DocIdSetIterator.NO_MORE_DOCS;
+    }
 }

server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeBuilderTestCase.java

@@ -166,7 +166,7 @@ public void setup() throws IOException {
         }
         writeState = getWriteState(5, UUID.randomUUID().toString().substring(0, 16).getBytes(StandardCharsets.UTF_8));
 
-        mergeState = new MergeState(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, false);
+        mergeState = new MergeState(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, false, null);
 
         dataFileName = IndexFileNames.segmentFileName(
             writeState.segmentInfo.name,

server/src/test/java/org/opensearch/lucene/util/CombinedBitSetTests.java

@@ -79,6 +79,7 @@ private void testCase(int numBits, float percent1, float percent2) {
         for (int i = 0; i < numBits; ++i) {
             assertEquals(expected.nextSetBit(i), actual.nextSetBit(i));
             assertEquals(Integer.toString(i), expected.prevSetBit(i), actual.prevSetBit(i));
+            assertEquals(Integer.toString(i), expected.nextClearBit(i, numBits), actual.nextClearBit(i, numBits));
         }
     }
 

test/framework/licenses/lucene-codecs-10.5.0.jar.sha1

@@ -0,0 +1 @@
+d128c42988203efb7c3a2c7fb8beb788cb578dae

test/framework/licenses/lucene-test-framework-10.5.0.jar.sha1

@@ -0,0 +1 @@
+fbca09b1ae940d26b6c3a90befecda40d5f1c3c2