Defer Sorter.DocMap packing until after flush

lucene/CHANGES.txt

@@ -367,6 +367,8 @@ Optimizations
 
 * GITHUB#16001: IndexSearcher.count() was calling query.rewrite twice, a regression since v9.10 (David Smiley)
 
+* GITHUB#16048: Defer Sorter.DocMap packing until after flush (Tim Brooks)
+
 Bug Fixes
 ---------------------
 * GITHUB#15754: Fix HTMLStripCharFilter to prevent tags from incorrectly consuming subsequent

lucene/core/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java

@@ -407,15 +407,15 @@ FlushedSegment flush(DocumentsWriter.FlushNotifications flushNotifications) thro
           "DWPT",
           "flush postings as segment " + flushState.segmentInfo.name + " numDocs=" + numDocsInRAM);
     }
-    final Sorter.DocMap sortMap;
+    final Sorter.PackableDocMap packableSortMap;
     try {
       DocIdSetIterator softDeletedDocs;
       if (indexWriterConfig.getSoftDeletesField() != null) {
         softDeletedDocs = indexingChain.getHasDocValues(indexWriterConfig.getSoftDeletesField());
       } else {
         softDeletedDocs = null;
       }
-      sortMap = indexingChain.flush(flushState);
+      packableSortMap = indexingChain.flush(flushState);
       if (softDeletedDocs == null) {
         flushState.softDelCountOnFlush = 0;
       } else {
@@ -496,8 +496,10 @@ FlushedSegment flush(DocumentsWriter.FlushNotifications flushNotifications) thro
               segmentDeletes,
               flushState.liveDocs,
               flushState.delCountOnFlush,
-              sortMap);
-      sealFlushedSegment(fs, sortMap, flushNotifications);
+              packableSortMap != null
+                  ? packableSortMap.pack()
+                  : null); // Use a packed version as the lifetime of FlushedSegment is long.
+      sealFlushedSegment(fs, packableSortMap, flushNotifications);
       if (infoStream.isEnabled("DWPT")) {
         infoStream.message(
             "DWPT",

lucene/core/src/java/org/apache/lucene/index/IndexingChain.java

@@ -226,7 +226,7 @@ public FieldInfos getFieldInfos() {
     };
   }
 
-  private Sorter.DocMap maybeSortSegment(SegmentWriteState state) throws IOException {
+  private Sorter.PackableDocMap maybeSortSegment(SegmentWriteState state) throws IOException {
     Sort indexSort = state.segmentInfo.getIndexSort();
     if (indexSort == null) {
       return null;
@@ -272,15 +272,15 @@ private Sorter.DocMap maybeSortSegment(SegmentWriteState state) throws IOExcepti
     }
     Sorter sorter = new Sorter(indexSort);
     // returns null if the documents are already sorted
-    return sorter.sort(
+    return sorter.sortAndLeaveUnpacked(
         state.segmentInfo.maxDoc(), comparators.toArray(IndexSorter.DocComparator[]::new));
   }
 
-  Sorter.DocMap flush(SegmentWriteState state) throws IOException {
+  Sorter.PackableDocMap flush(SegmentWriteState state) throws IOException {
 
     // NOTE: caller (DocumentsWriterPerThread) handles
     // aborting on any exception from this method
-    Sorter.DocMap sortMap = maybeSortSegment(state);
+    Sorter.PackableDocMap sortMap = maybeSortSegment(state);
     int maxDoc = state.segmentInfo.maxDoc();
     long t0 = System.nanoTime();
     writeNorms(state, sortMap);

lucene/core/src/java/org/apache/lucene/index/Sorter.java

@@ -129,7 +129,7 @@ protected int compareSaved(int i, int j) {
   }
 
   /** Computes the old-to-new permutation over the given comparator. */
-  private static Sorter.DocMap sort(final int maxDoc, IndexSorter.DocComparator comparator) {
+  private static PackableDocMap sort(final int maxDoc, IndexSorter.DocComparator comparator) {
     // check if the index is sorted
     boolean sorted = true;
     for (int i = 1; i < maxDoc; ++i) {
@@ -168,30 +168,7 @@ private static Sorter.DocMap sort(final int maxDoc, IndexSorter.DocComparator co
       docs[(int) newToOld.get(i)] = i;
     } // docs is now the oldToNew mapping
 
-    final PackedLongValues.Builder oldToNewBuilder =
-        PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
-    for (int i = 0; i < maxDoc; ++i) {
-      oldToNewBuilder.add(docs[i]);
-    }
-    final PackedLongValues oldToNew = oldToNewBuilder.build();
-
-    return new Sorter.DocMap() {
-
-      @Override
-      public int oldToNew(int docID) {
-        return (int) oldToNew.get(docID);
-      }
-
-      @Override
-      public int newToOld(int docID) {
-        return (int) newToOld.get(docID);
-      }
-
-      @Override
-      public int size() {
-        return maxDoc;
-      }
-    };
+    return new PackableDocMap(docs, newToOld, maxDoc);
   }
 
   /**
@@ -241,6 +218,12 @@ DocMap sort(LeafReader reader) throws IOException {
   }
 
   DocMap sort(int maxDoc, IndexSorter.DocComparator[] comparators) throws IOException {
+    PackableDocMap packableDocMap = sortAndLeaveUnpacked(maxDoc, comparators);
+    return packableDocMap != null ? packableDocMap.pack() : packableDocMap;
+  }
+
+  PackableDocMap sortAndLeaveUnpacked(int maxDoc, IndexSorter.DocComparator[] comparators)
+      throws IOException {
     final IndexSorter.DocComparator comparator =
         (docID1, docID2) -> {
           for (int i = 0; i < comparators.length; i++) {
@@ -270,4 +253,60 @@ public String getID() {
   public String toString() {
     return getID();
   }
+
+  /** A {@link DocMap} that can keep oldToNew in either packed or unpacked form. */
+  static final class PackableDocMap extends DocMap {
+
+    private final PackedLongValues newToOld;
+    private final int maxDoc;
+
+    private final int[] oldToNewUnpacked;
+    private final PackedLongValues oldToNewPacked;
+
+    private PackableDocMap(int[] oldToNewUnpacked, PackedLongValues newToOld, int maxDoc) {
+      this.oldToNewUnpacked = oldToNewUnpacked;
+      this.oldToNewPacked = null;
+      this.newToOld = newToOld;
+      this.maxDoc = maxDoc;
+    }
+
+    private PackableDocMap(PackedLongValues oldToNewPacked, PackedLongValues newToOld, int maxDoc) {
+      this.oldToNewUnpacked = null;
+      this.oldToNewPacked = oldToNewPacked;
+      this.newToOld = newToOld;
+      this.maxDoc = maxDoc;
+    }
+
+    @Override
+    public int oldToNew(int docID) {
+      if (oldToNewUnpacked != null) {
+        return oldToNewUnpacked[docID];
+      } else {
+        return (int) oldToNewPacked.get(docID);
+      }
+    }
+
+    @Override
+    public int newToOld(int docID) {
+      return (int) newToOld.get(docID);
+    }
+
+    /** Pack the internal representations into more compact forms. No-op if already packed. */
+    DocMap pack() {
+      if (oldToNewPacked != null) {
+        return this;
+      }
+      final PackedLongValues.Builder oldToNewBuilder =
+          PackedLongValues.monotonicBuilder(PackedInts.COMPACT);
+      for (int i = 0; i < maxDoc; ++i) {
+        oldToNewBuilder.add(oldToNewUnpacked[i]);
+      }
+      return new PackableDocMap(oldToNewBuilder.build(), newToOld, maxDoc);
+    }
+
+    @Override
+    public int size() {
+      return maxDoc;
+    }
+  }
 }