Skip to content

Commit b6ca26f

Browse files
tlrxtlrx
authored
Check if merge is aborted during file integrity checksums (#16281)
Adds a new `CodecUtil.checksumEntireFile(IndexInput, OneMerge)` that periodically checks whether the merge has been aborted while reading through the file. This avoids spending time checksumming large files when the merge is already cancelled. The check runs every 1 MB. It also adds a `OneMerge` parameter to `checkIntegrity()` to all codec reader/producer base classes. See #16086 for original discussion.
1 parent b614a53 commit b6ca26f

95 files changed

Lines changed: 453 additions & 241 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎lucene/CHANGES.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,9 @@ API Changes
9898
* GITHUB#16052: Remove IOException from LeafReader#getPointValues, LeafReader#terms,
9999
LeafReader#getDocCount signatures. (Ignacio Vera)
100100

101+
* GITHUB#16281: The checkIntegrity() method of codec Producer APIs gains a OneMerge parameter,
102+
which is used to check for aborted merges during checksum validation. (Tanguy Leroux)
103+
101104
New Features
102105
---------------------
103106
* GITHUB#16241: Add HangulCompositionCharFilter to analysis-nori for opt-in composition of
@@ -142,6 +145,8 @@ Improvements
142145

143146
* GITHUB#15558: Refactor QueryCache for performance. (Sagar Upadhyaya)
144147

148+
* GITHUB#16281: File integrity checks periodically check if a merge is aborted. (Tanguy Leroux)
149+
145150
Optimizations
146151
---------------------
147152
* GITHUB#16254: Enable the vectorized findNextGEQ path on aarch64, where it was disabled on cores

‎lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene101/Lucene101PostingsReader.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.apache.lucene.index.ImpactsEnum;
3939
import org.apache.lucene.index.IndexFileNames;
4040
import org.apache.lucene.index.IndexOptions;
41+
import org.apache.lucene.index.MergePolicy;
4142
import org.apache.lucene.index.PostingsEnum;
4243
import org.apache.lucene.index.SegmentReadState;
4344
import org.apache.lucene.store.ByteArrayDataInput;
@@ -1404,15 +1405,15 @@ static FreqAndNormBuffer readImpacts(ByteArrayDataInput in, FreqAndNormBuffer re
14041405
}
14051406

14061407
@Override
1407-
public void checkIntegrity() throws IOException {
1408+
public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {
14081409
if (docIn != null) {
1409-
CodecUtil.checksumEntireFile(docIn);
1410+
CodecUtil.checksumEntireFile(docIn, merge);
14101411
}
14111412
if (posIn != null) {
1412-
CodecUtil.checksumEntireFile(posIn);
1413+
CodecUtil.checksumEntireFile(posIn, merge);
14131414
}
14141415
if (payIn != null) {
1415-
CodecUtil.checksumEntireFile(payIn);
1416+
CodecUtil.checksumEntireFile(payIn, merge);
14161417
}
14171418
}
14181419

‎lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene102/Lucene102BinaryQuantizedVectorsReader.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import org.apache.lucene.index.FloatVectorValues;
4444
import org.apache.lucene.index.IndexFileNames;
4545
import org.apache.lucene.index.KnnVectorValues;
46+
import org.apache.lucene.index.MergePolicy;
4647
import org.apache.lucene.index.SegmentReadState;
4748
import org.apache.lucene.index.SegmentWriteState;
4849
import org.apache.lucene.index.VectorEncoding;
@@ -211,9 +212,9 @@ public RandomVectorScorer getRandomVectorScorer(String field, byte[] target) thr
211212
}
212213

213214
@Override
214-
public void checkIntegrity() throws IOException {
215-
rawVectorsReader.checkIntegrity();
216-
CodecUtil.checksumEntireFile(quantizedVectorData);
215+
public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {
216+
rawVectorsReader.checkIntegrity(merge);
217+
CodecUtil.checksumEntireFile(quantizedVectorData, merge);
217218
}
218219

219220
@Override

‎lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene103/Lucene103PostingsReader.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.apache.lucene.index.ImpactsEnum;
3939
import org.apache.lucene.index.IndexFileNames;
4040
import org.apache.lucene.index.IndexOptions;
41+
import org.apache.lucene.index.MergePolicy;
4142
import org.apache.lucene.index.PostingsEnum;
4243
import org.apache.lucene.index.SegmentReadState;
4344
import org.apache.lucene.search.DocAndFloatFeatureBuffer;
@@ -1449,15 +1450,15 @@ static FreqAndNormBuffer readImpacts(ByteArrayDataInput in, FreqAndNormBuffer re
14491450
}
14501451

14511452
@Override
1452-
public void checkIntegrity() throws IOException {
1453+
public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {
14531454
if (docIn != null) {
1454-
CodecUtil.checksumEntireFile(docIn);
1455+
CodecUtil.checksumEntireFile(docIn, merge);
14551456
}
14561457
if (posIn != null) {
1457-
CodecUtil.checksumEntireFile(posIn);
1458+
CodecUtil.checksumEntireFile(posIn, merge);
14581459
}
14591460
if (payIn != null) {
1460-
CodecUtil.checksumEntireFile(payIn);
1461+
CodecUtil.checksumEntireFile(payIn, merge);
14611462
}
14621463
}
14631464

‎lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene80/Lucene80DocValuesProducer.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.apache.lucene.index.FieldInfos;
3232
import org.apache.lucene.index.ImpactsEnum;
3333
import org.apache.lucene.index.IndexFileNames;
34+
import org.apache.lucene.index.MergePolicy;
3435
import org.apache.lucene.index.NumericDocValues;
3536
import org.apache.lucene.index.PostingsEnum;
3637
import org.apache.lucene.index.SegmentReadState;
@@ -1683,8 +1684,8 @@ public DocValuesSkipper getSkipper(FieldInfo field) {
16831684
}
16841685

16851686
@Override
1686-
public void checkIntegrity() throws IOException {
1687-
CodecUtil.checksumEntireFile(data);
1687+
public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {
1688+
CodecUtil.checksumEntireFile(data, merge);
16881689
}
16891690

16901691
/**

‎lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90HnswVectorsReader.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.apache.lucene.index.FieldInfos;
3333
import org.apache.lucene.index.FloatVectorValues;
3434
import org.apache.lucene.index.IndexFileNames;
35+
import org.apache.lucene.index.MergePolicy;
3536
import org.apache.lucene.index.SegmentReadState;
3637
import org.apache.lucene.index.VectorSimilarityFunction;
3738
import org.apache.lucene.internal.hppc.IntObjectHashMap;
@@ -218,9 +219,9 @@ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOExceptio
218219
}
219220

220221
@Override
221-
public void checkIntegrity() throws IOException {
222-
CodecUtil.checksumEntireFile(vectorData);
223-
CodecUtil.checksumEntireFile(vectorIndex);
222+
public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {
223+
CodecUtil.checksumEntireFile(vectorData, merge);
224+
CodecUtil.checksumEntireFile(vectorIndex, merge);
224225
}
225226

226227
private FieldEntry getFieldEntry(String field) {

‎lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/Lucene90PostingsReader.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import org.apache.lucene.index.ImpactsEnum;
3737
import org.apache.lucene.index.IndexFileNames;
3838
import org.apache.lucene.index.IndexOptions;
39+
import org.apache.lucene.index.MergePolicy;
3940
import org.apache.lucene.index.PostingsEnum;
4041
import org.apache.lucene.index.SegmentReadState;
4142
import org.apache.lucene.index.SlowImpactsEnum;
@@ -2042,15 +2043,15 @@ public long cost() {
20422043
}
20432044

20442045
@Override
2045-
public void checkIntegrity() throws IOException {
2046+
public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {
20462047
if (docIn != null) {
2047-
CodecUtil.checksumEntireFile(docIn);
2048+
CodecUtil.checksumEntireFile(docIn, merge);
20482049
}
20492050
if (posIn != null) {
2050-
CodecUtil.checksumEntireFile(posIn);
2051+
CodecUtil.checksumEntireFile(posIn, merge);
20512052
}
20522053
if (payIn != null) {
2053-
CodecUtil.checksumEntireFile(payIn);
2054+
CodecUtil.checksumEntireFile(payIn, merge);
20542055
}
20552056
}
20562057

‎lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene90/blocktree/Lucene90BlockTreeTermsReader.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.lucene.index.FieldInfos;
3030
import org.apache.lucene.index.IndexFileNames;
3131
import org.apache.lucene.index.IndexOptions;
32+
import org.apache.lucene.index.MergePolicy;
3233
import org.apache.lucene.index.SegmentReadState;
3334
import org.apache.lucene.index.Terms;
3435
import org.apache.lucene.internal.hppc.IntCursor;
@@ -327,15 +328,15 @@ public int size() {
327328
}
328329

329330
@Override
330-
public void checkIntegrity() throws IOException {
331+
public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {
331332
// terms index
332-
CodecUtil.checksumEntireFile(indexIn);
333+
CodecUtil.checksumEntireFile(indexIn, merge);
333334

334335
// term dictionary
335-
CodecUtil.checksumEntireFile(termsIn);
336+
CodecUtil.checksumEntireFile(termsIn, merge);
336337

337338
// postings
338-
postingsReader.checkIntegrity();
339+
postingsReader.checkIntegrity(merge);
339340
}
340341

341342
@Override

‎lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene91/Lucene91HnswVectorsReader.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.lucene.index.FieldInfos;
3535
import org.apache.lucene.index.FloatVectorValues;
3636
import org.apache.lucene.index.IndexFileNames;
37+
import org.apache.lucene.index.MergePolicy;
3738
import org.apache.lucene.index.SegmentReadState;
3839
import org.apache.lucene.index.VectorSimilarityFunction;
3940
import org.apache.lucene.internal.hppc.IntObjectHashMap;
@@ -214,9 +215,9 @@ private FieldEntry readField(IndexInput input, FieldInfo info) throws IOExceptio
214215
}
215216

216217
@Override
217-
public void checkIntegrity() throws IOException {
218-
CodecUtil.checksumEntireFile(vectorData);
219-
CodecUtil.checksumEntireFile(vectorIndex);
218+
public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {
219+
CodecUtil.checksumEntireFile(vectorData, merge);
220+
CodecUtil.checksumEntireFile(vectorIndex, merge);
220221
}
221222

222223
private FieldEntry getFieldEntry(String field) {

‎lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene912/Lucene912PostingsReader.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
import org.apache.lucene.index.ImpactsEnum;
3939
import org.apache.lucene.index.IndexFileNames;
4040
import org.apache.lucene.index.IndexOptions;
41+
import org.apache.lucene.index.MergePolicy;
4142
import org.apache.lucene.index.PostingsEnum;
4243
import org.apache.lucene.index.SegmentReadState;
4344
import org.apache.lucene.index.SlowImpactsEnum;
@@ -1795,15 +1796,15 @@ static FreqAndNormBuffer readImpacts(ByteArrayDataInput in, FreqAndNormBuffer re
17951796
}
17961797

17971798
@Override
1798-
public void checkIntegrity() throws IOException {
1799+
public void checkIntegrity(MergePolicy.OneMerge merge) throws IOException {
17991800
if (docIn != null) {
1800-
CodecUtil.checksumEntireFile(docIn);
1801+
CodecUtil.checksumEntireFile(docIn, merge);
18011802
}
18021803
if (posIn != null) {
1803-
CodecUtil.checksumEntireFile(posIn);
1804+
CodecUtil.checksumEntireFile(posIn, merge);
18041805
}
18051806
if (payIn != null) {
1806-
CodecUtil.checksumEntireFile(payIn);
1807+
CodecUtil.checksumEntireFile(payIn, merge);
18071808
}
18081809
}
18091810

0 commit comments

Comments
 (0)