[#290] Report leftover MVV mark bits in IntegrityCheck (#291)

* Report leftover MVV mark bits in IntegrityCheck

A prune interrupted before the issue #286 fix could leave 0x8000 mark
bits behind in on-disk version length fields. Since PR #288 the read
paths strip the mark bit silently, so such versions read normally and
icheck reported affected volumes as clean.

Add MVV.countMarkedVersions and scan every data record during icheck:
marked versions are counted in a new MarkedVersions counter (report,
CSV and getMarkedVersionCount) and reported as a fault so affected
volumes can be identified. The scan runs only after visitAllVersions
has validated the version structure, so a structurally corrupt MVV
cannot produce a bogus marked-version fault.

icheck -p remains the remediation: pruning rewrites the marked
versions (issue #292 makes prune clear stale marks unconditionally).
With icheck -P the marked-version faults are reported but do not
block clearing the TransactionIndex, since the prune pass has already
rewritten the marks. The icheck -c CSV example in Management.rst is
updated for the new MarkedVersions column.

Fixes #290

* Compare fault counts in the icheck -P gate, not the capped fault list

The pruneAndClear gate excluded marked-version faults by comparing
_faults.size() with the marked-fault count, but both stop moving once
the fault list reaches MAX_FAULTS: on a volume with more than
MAX_FAULTS marked-version records a genuine fault found afterwards
(the garbage chain is checked after the trees) is dropped from the
list and the arithmetic still converges, so -P cleared the
TransactionIndex despite real corruption. Count every fault found -
recorded in the list or not - and gate on the counts instead.

Author: vharseko

persistit/core/src/main/java/com/persistit/IntegrityCheck.java

@@ -60,6 +60,7 @@ public class IntegrityCheck extends Task {
 
     private Volume _currentVolume;
     private Tree _currentTree;
+    private long _currentPage;
     private LongBitSet _usedPageBits = new LongBitSet();
     private long _totalPages = 0;
     private long _pagesVisited = 0;
@@ -79,6 +80,8 @@ public class IntegrityCheck extends Task {
     private boolean _csv;
 
     private final ArrayList<Fault> _faults = new ArrayList<Fault>();
+    private int _faultCount;
+    private int _markedVersionFaultCount;
     private final ArrayList<CleanupIndexHole> _holes = new ArrayList<CleanupIndexHole>();
 
     // Used in checking long values
@@ -98,6 +101,7 @@ private static class Counters {
         private long _mvvCount = 0;
         private long _mvvOverhead = 0;
         private long _mvvAntiValues = 0;
+        private long _markedVersionCount = 0;
         private long _pruningErrorCount = 0;
         private long _prunedPageCount = 0;
         private long _garbagePageCount = 0;
@@ -118,6 +122,7 @@ private static class Counters {
             _mvvCount = counters._mvvCount;
             _mvvOverhead = counters._mvvOverhead;
             _mvvAntiValues = counters._mvvAntiValues;
+            _markedVersionCount = counters._markedVersionCount;
             _pruningErrorCount = counters._pruningErrorCount;
             _prunedPageCount = counters._prunedPageCount;
             _garbagePageCount = counters._garbagePageCount;
@@ -135,6 +140,7 @@ void difference(final Counters counters) {
             _mvvCount = counters._mvvCount - _mvvCount;
             _mvvOverhead = counters._mvvOverhead - _mvvOverhead;
             _mvvAntiValues = counters._mvvAntiValues - _mvvAntiValues;
+            _markedVersionCount = counters._markedVersionCount - _markedVersionCount;
             _pruningErrorCount = counters._pruningErrorCount - _pruningErrorCount;
             _prunedPageCount = counters._prunedPageCount - _prunedPageCount;
             _garbagePageCount = counters._garbagePageCount - _garbagePageCount;
@@ -144,20 +150,21 @@ void difference(final Counters counters) {
         public String toString() {
             return String.format("Index pages/bytes: %,d / %,d Data pages/bytes: %,d / %,d"
                     + " LongRec pages/bytes: %,d / %,d  MVV pages/records/bytes/antivalues: "
-                    + "%,d / %,d / %,d / %,d  Holes %,d Pages pruned %,d", _indexPageCount, _indexBytesInUse,
-                    _dataPageCount, _dataBytesInUse, _longRecordPageCount, _longRecordBytesInUse, _mvvPageCount,
-                    _mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount);
+                    + "%,d / %,d / %,d / %,d  Holes %,d Pages pruned %,d Marked versions %,d", _indexPageCount,
+                    _indexBytesInUse, _dataPageCount, _dataBytesInUse, _longRecordPageCount, _longRecordBytesInUse,
+                    _mvvPageCount, _mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount,
+                    _markedVersionCount);
         }
 
         private String toCSV() {
-            return String.format("%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d", _indexPageCount, _indexBytesInUse,
+            return String.format("%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d", _indexPageCount, _indexBytesInUse,
                     _dataPageCount, _dataBytesInUse, _longRecordPageCount, _longRecordBytesInUse, _mvvPageCount,
-                    _mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount);
+                    _mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount, _markedVersionCount);
         }
 
         private final static String CSV_HEADERS = "IndexPages,IndexBytes,"
                 + "DataPages,DataBytes,LongRecordPages,LongRecordBytes,MvvPages,"
-                + "MvvRecords,MvvOverhead,MvvAntiValues,IndexHoles,PrunedPages";
+                + "MvvRecords,MvvOverhead,MvvAntiValues,IndexHoles,PrunedPages,MarkedVersions";
 
     }
 
@@ -186,6 +193,19 @@ public void sawVersion(final long version, final int offset, final int valueLeng
         protected void visitDataRecord(final Key key, final int foundAt, final int tail, final int klength,
                 final int offset, final int length, final byte[] bytes) throws PersistitException {
             MVV.visitAllVersions(_versionVisitor, bytes, offset, length);
+            /*
+             * Scanned only after visitAllVersions has validated the version
+             * structure. Leftover prune marks read normally (the length
+             * accessors strip the mark bit), so only this dedicated scan can
+             * detect and report them - see issue #290.
+             */
+            final int marked = MVV.countMarkedVersions(bytes, offset, length);
+            if (marked > 0) {
+                _counters._markedVersionCount += marked;
+                _markedVersionFaultCount++;
+                addFault("MVV has " + plural(marked, "version") + " with a leftover mark bit", _currentPage, 0,
+                        foundAt);
+            }
             if (_versionVisitor._count > 0) {
                 _counters._mvvCount++;
                 final int voffset = _versionVisitor._lastOffset;
@@ -289,7 +309,18 @@ protected void runTask() {
                 postMessage("Total " + toString(), LOG_NORMAL);
             }
             if (_pruneAndClear) {
-                if (_faults.isEmpty() && _counters._mvvPageCount == _counters._prunedPageCount
+                /*
+                 * Leftover mark-bit faults (issue #290) identify volumes
+                 * affected by a pre-#286 interrupted prune but do not block
+                 * clearing the TransactionIndex: the prune pass above has
+                 * already rewritten those marks (issue #292). Only other
+                 * faults and incomplete pruning count as failures here. The
+                 * comparison uses fault counts, not the _faults list: the
+                 * list is capped at MAX_FAULTS, so a genuine fault found
+                 * after marked-version faults fill the cap would be missing
+                 * from the list but must still block the clear.
+                 */
+                if (_faultCount == _markedVersionFaultCount && _counters._mvvPageCount == _counters._prunedPageCount
                         && _counters._pruningErrorCount == 0) {
                     final int count = _persistit.getTransactionIndex().resetMVVCounts(startTimestamp);
                     postMessage(String.format("%,d aborted transactions were cleared by pruning", count), LOG_NORMAL);
@@ -331,13 +362,15 @@ private String plural(final int n, final String m) {
 
     private void addFault(final String description, final long page, final int level, final int position) {
         final Fault fault = new Fault(resourceName(), this, description, page, _treeDepth, level, position);
+        _faultCount++;
         if (_faults.size() < MAX_FAULTS)
             _faults.add(fault);
         postMessage(fault.toString(), LOG_VERBOSE);
     }
 
     private void addGarbageFault(final String description, final long page, final int level, final int position) {
         final Fault fault = new Fault(resourceName(), this, description, page, 3, level, position);
+        _faultCount++;
         if (_faults.size() < MAX_FAULTS)
             _faults.add(fault);
         postMessage(fault.toString(), LOG_VERBOSE);
@@ -544,6 +577,16 @@ public long getMvvAntiValues() {
         return _counters._mvvAntiValues;
     }
 
+    /**
+     * @return Count of MVV versions whose length field still carries a
+     *         leftover prune mark bit, left behind by a prune interrupted
+     *         before the issue #286 fix. Such versions read normally but
+     *         indicate the volume was corrupted; pruning rewrites them.
+     */
+    public long getMarkedVersionCount() {
+        return _counters._markedVersionCount;
+    }
+
     /**
      * @return Count of pages for which an expected index pointer is missing
      */
@@ -1144,6 +1187,7 @@ private Buffer walkRight(final int level, final long toPage, final Key key, fina
     }
 
     private boolean verifyPage(final Buffer buffer, final long page, final int level, final Key key, final Tree tree) {
+        _currentPage = page;
         if (buffer.getPageAddress() != page) {
             addFault("Buffer contains wrong page " + buffer.getPageAddress(), page, level, 0);
             return false;

persistit/core/src/main/java/com/persistit/MVV.java

@@ -654,6 +654,37 @@ static boolean verify(final byte[] bytes, final int offset, final int length) {
         return true;
     }
 
+    /**
+     * Count versions whose length field still carries the mark bit. Marks are
+     * transient state private to {@link #prune} and must never be observable
+     * outside it; a non-zero count on a stored MVV means the value was
+     * corrupted by a prune interrupted before the issue #286 fix. The length
+     * accessors strip the mark bit silently, so such versions read normally
+     * and only this scan can detect them (issue #290).
+     *
+     * @param bytes
+     *            the byte array
+     * @param offset
+     *            the index of the first byte of the MVV within the byte array
+     * @param length
+     *            the count of bytes in the MVV
+     * @return the number of marked versions, 0 for a non-MVV value
+     */
+    static int countMarkedVersions(final byte[] bytes, final int offset, final int length) {
+        if (!isArrayMVV(bytes, offset, length)) {
+            return 0;
+        }
+        int marked = 0;
+        int from = offset + 1;
+        while (from + LENGTH_PER_VERSION <= offset + length) {
+            if (isMarked(bytes, from)) {
+                marked++;
+            }
+            from += getLength(bytes, from) + LENGTH_PER_VERSION;
+        }
+        return marked;
+    }
+
     static boolean verify(final TransactionIndex ti, final byte[] bytes, final int offset, final int length) {
         if (!isArrayMVV(bytes, offset, length)) {
             /*