diff --git a/persistit/core/src/main/java/com/persistit/IntegrityCheck.java b/persistit/core/src/main/java/com/persistit/IntegrityCheck.java index 06f56d7eb..65337a411 100644 --- a/persistit/core/src/main/java/com/persistit/IntegrityCheck.java +++ b/persistit/core/src/main/java/com/persistit/IntegrityCheck.java @@ -60,6 +60,7 @@ public class IntegrityCheck extends Task { private Volume _currentVolume; private Tree _currentTree; + private long _currentPage; private LongBitSet _usedPageBits = new LongBitSet(); private long _totalPages = 0; private long _pagesVisited = 0; @@ -79,6 +80,7 @@ public class IntegrityCheck extends Task { private boolean _csv; private final ArrayList _faults = new ArrayList(); + private int _markedVersionFaultCount; private final ArrayList _holes = new ArrayList(); // Used in checking long values @@ -98,6 +100,7 @@ private static class Counters { private long _mvvCount = 0; private long _mvvOverhead = 0; private long _mvvAntiValues = 0; + private long _markedVersionCount = 0; private long _pruningErrorCount = 0; private long _prunedPageCount = 0; private long _garbagePageCount = 0; @@ -118,6 +121,7 @@ private static class Counters { _mvvCount = counters._mvvCount; _mvvOverhead = counters._mvvOverhead; _mvvAntiValues = counters._mvvAntiValues; + _markedVersionCount = counters._markedVersionCount; _pruningErrorCount = counters._pruningErrorCount; _prunedPageCount = counters._prunedPageCount; _garbagePageCount = counters._garbagePageCount; @@ -135,6 +139,7 @@ void difference(final Counters counters) { _mvvCount = counters._mvvCount - _mvvCount; _mvvOverhead = counters._mvvOverhead - _mvvOverhead; _mvvAntiValues = counters._mvvAntiValues - _mvvAntiValues; + _markedVersionCount = counters._markedVersionCount - _markedVersionCount; _pruningErrorCount = counters._pruningErrorCount - _pruningErrorCount; _prunedPageCount = counters._prunedPageCount - _prunedPageCount; _garbagePageCount = counters._garbagePageCount - _garbagePageCount; @@ -144,20 +149,21 @@ void difference(final Counters counters) { public String toString() { return String.format("Index pages/bytes: %,d / %,d Data pages/bytes: %,d / %,d" + " LongRec pages/bytes: %,d / %,d MVV pages/records/bytes/antivalues: " - + "%,d / %,d / %,d / %,d Holes %,d Pages pruned %,d", _indexPageCount, _indexBytesInUse, - _dataPageCount, _dataBytesInUse, _longRecordPageCount, _longRecordBytesInUse, _mvvPageCount, - _mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount); + + "%,d / %,d / %,d / %,d Holes %,d Pages pruned %,d Marked versions %,d", _indexPageCount, + _indexBytesInUse, _dataPageCount, _dataBytesInUse, _longRecordPageCount, _longRecordBytesInUse, + _mvvPageCount, _mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount, + _markedVersionCount); } private String toCSV() { - return String.format("%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d", _indexPageCount, _indexBytesInUse, + return String.format("%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d", _indexPageCount, _indexBytesInUse, _dataPageCount, _dataBytesInUse, _longRecordPageCount, _longRecordBytesInUse, _mvvPageCount, - _mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount); + _mvvCount, _mvvOverhead, _mvvAntiValues, _indexHoleCount, _prunedPageCount, _markedVersionCount); } private final static String CSV_HEADERS = "IndexPages,IndexBytes," + "DataPages,DataBytes,LongRecordPages,LongRecordBytes,MvvPages," - + "MvvRecords,MvvOverhead,MvvAntiValues,IndexHoles,PrunedPages"; + + "MvvRecords,MvvOverhead,MvvAntiValues,IndexHoles,PrunedPages,MarkedVersions"; } @@ -186,6 +192,20 @@ public void sawVersion(final long version, final int offset, final int valueLeng protected void visitDataRecord(final Key key, final int foundAt, final int tail, final int klength, final int offset, final int length, final byte[] bytes) throws PersistitException { MVV.visitAllVersions(_versionVisitor, bytes, offset, length); + /* + * Scanned only after visitAllVersions has validated the version + * structure. Leftover prune marks read normally (the length + * accessors strip the mark bit), so only this dedicated scan can + * detect and report them - see issue #290. + */ + final int marked = MVV.countMarkedVersions(bytes, offset, length); + if (marked > 0) { + _counters._markedVersionCount += marked; + if (addFault("MVV has " + plural(marked, "version") + " with a leftover mark bit", _currentPage, 0, + foundAt)) { + _markedVersionFaultCount++; + } + } if (_versionVisitor._count > 0) { _counters._mvvCount++; final int voffset = _versionVisitor._lastOffset; @@ -289,7 +309,14 @@ protected void runTask() { postMessage("Total " + toString(), LOG_NORMAL); } if (_pruneAndClear) { - if (_faults.isEmpty() && _counters._mvvPageCount == _counters._prunedPageCount + /* + * Leftover mark-bit faults (issue #290) identify volumes + * affected by a pre-#286 interrupted prune but do not block + * clearing the TransactionIndex: the prune pass above has + * already rewritten those marks (issue #292). Only other + * faults and incomplete pruning count as failures here. + */ + if (_faults.size() == _markedVersionFaultCount && _counters._mvvPageCount == _counters._prunedPageCount && _counters._pruningErrorCount == 0) { final int count = _persistit.getTransactionIndex().resetMVVCounts(startTimestamp); postMessage(String.format("%,d aborted transactions were cleared by pruning", count), LOG_NORMAL); @@ -329,11 +356,13 @@ private String plural(final int n, final String m) { } } - private void addFault(final String description, final long page, final int level, final int position) { + private boolean addFault(final String description, final long page, final int level, final int position) { final Fault fault = new Fault(resourceName(), this, description, page, _treeDepth, level, position); - if (_faults.size() < MAX_FAULTS) + final boolean recorded = _faults.size() < MAX_FAULTS; + if (recorded) _faults.add(fault); postMessage(fault.toString(), LOG_VERBOSE); + return recorded; } private void addGarbageFault(final String description, final long page, final int level, final int position) { @@ -544,6 +573,16 @@ public long getMvvAntiValues() { return _counters._mvvAntiValues; } + /** + * @return Count of MVV versions whose length field still carries a + * leftover prune mark bit, left behind by a prune interrupted + * before the issue #286 fix. Such versions read normally but + * indicate the volume was corrupted; pruning rewrites them. + */ + public long getMarkedVersionCount() { + return _counters._markedVersionCount; + } + /** * @return Count of pages for which an expected index pointer is missing */ @@ -1144,6 +1183,7 @@ private Buffer walkRight(final int level, final long toPage, final Key key, fina } private boolean verifyPage(final Buffer buffer, final long page, final int level, final Key key, final Tree tree) { + _currentPage = page; if (buffer.getPageAddress() != page) { addFault("Buffer contains wrong page " + buffer.getPageAddress(), page, level, 0); return false; diff --git a/persistit/core/src/main/java/com/persistit/MVV.java b/persistit/core/src/main/java/com/persistit/MVV.java index e370a8bdd..2236404eb 100644 --- a/persistit/core/src/main/java/com/persistit/MVV.java +++ b/persistit/core/src/main/java/com/persistit/MVV.java @@ -381,7 +381,13 @@ else if (target[targetOffset] != TYPE_MVV_BYTE) { *

*

* This method leaves the byte array unchanged if any of its checked - * Exceptions is thrown. + * Exceptions is thrown, with one exception: on a well-formed MVV, stale + * mark bits left on disk by a prune interrupted before the issue #286 fix + * are cleared up front regardless of outcome (issue #292). Mark bits are + * transient state private to this method and invisible to the read paths, + * so clearing them repairs the corruption without altering any version's + * value. A malformed MVV is never swept — it is rejected with a + * CorruptValueException and left unchanged. *

*

* This method adds {@link PrunedVersion} instances to the supplied list. @@ -420,7 +426,8 @@ static int prune(final byte[] bytes, final int offset, final int length, final T return length; } - Debug.$assert0.t(verify(bytes, offset, length)); + final boolean wellFormed = verify(bytes, offset, length); + Debug.$assert0.t(wellFormed); boolean primordial = convertToPrimordial; int marked = 0; @@ -437,6 +444,28 @@ static int prune(final byte[] bytes, final int offset, final int length, final T long lastVersionHandle = Long.MIN_VALUE; long lastVersionTc = UNCOMMITTED; long uncommittedTransactionTs = 0; + /* + * The passes below trust isMarked() while the marked counter only + * counts marks set by this run, so a stale mark left on disk by a + * prune interrupted before the issue #286 fix would win the + * primordial-conversion scan — promoting an obsolete version and + * silently dropping the current one with no PrunedVersion + * accounting (issue #292). Sweep the region clean before the + * first pass marks the real keepers. This is a separate pass, not + * an unmark() at the first pass's loop head, so the whole region + * is repaired even when that pass exits early via an exception. + * Only a well-formed region may be swept: a corrupt length field + * would misalign the traversal and unmark() would clear a bit + * inside a value payload. On a malformed region skip the sweep — + * the first pass's guard throws for it anyway. + */ + if (wellFormed) { + int sweep = offset + 1; + while (sweep + LENGTH_PER_VERSION <= offset + length) { + unmark(bytes, sweep); + sweep += getLength(bytes, sweep) + LENGTH_PER_VERSION; + } + } /* * First pass - mark all the versions to keep. Keep every * UNCOMMITTED version (there may be more than one created by the @@ -625,6 +654,37 @@ static boolean verify(final byte[] bytes, final int offset, final int length) { return true; } + /** + * Count versions whose length field still carries the mark bit. Marks are + * transient state private to {@link #prune} and must never be observable + * outside it; a non-zero count on a stored MVV means the value was + * corrupted by a prune interrupted before the issue #286 fix. The length + * accessors strip the mark bit silently, so such versions read normally + * and only this scan can detect them (issue #290). + * + * @param bytes + * the byte array + * @param offset + * the index of the first byte of the MVV within the byte array + * @param length + * the count of bytes in the MVV + * @return the number of marked versions, 0 for a non-MVV value + */ + static int countMarkedVersions(final byte[] bytes, final int offset, final int length) { + if (!isArrayMVV(bytes, offset, length)) { + return 0; + } + int marked = 0; + int from = offset + 1; + while (from + LENGTH_PER_VERSION <= offset + length) { + if (isMarked(bytes, from)) { + marked++; + } + from += getLength(bytes, from) + LENGTH_PER_VERSION; + } + return marked; + } + static boolean verify(final TransactionIndex ti, final byte[] bytes, final int offset, final int length) { if (!isArrayMVV(bytes, offset, length)) { /* diff --git a/persistit/core/src/test/java/com/persistit/IntegrityCheckTest.java b/persistit/core/src/test/java/com/persistit/IntegrityCheckTest.java index 4e688f198..331e39986 100644 --- a/persistit/core/src/test/java/com/persistit/IntegrityCheckTest.java +++ b/persistit/core/src/test/java/com/persistit/IntegrityCheckTest.java @@ -23,8 +23,12 @@ import org.junit.Test; import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.SortedMap; +import java.util.TreeMap; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; @@ -99,6 +103,88 @@ public void testBrokenMVVs() throws Exception { assertTrue(icheck.getFaults().length > 0); } + /** + * Issue #290: a leftover prune mark bit (pre-#288 interrupted-prune + * corruption) reads normally through all fetch paths, so IntegrityCheck + * must detect it explicitly and report it as a fault. + */ + @Test + public void testLeftoverMarkBitsReported() throws Exception { + final Exchange ex = _persistit.getExchange(_volumeName, "mvv", true); + disableBackgroundCleanup(); + transactionalStore(ex); + + IntegrityCheck icheck = icheck(); + icheck.checkTree(ex.getTree()); + assertEquals(0, icheck.getFaults().length); + assertEquals(0, icheck.getMarkedVersionCount()); + + final int[] corrupted = corrupt5(ex); + final int corruptedValues = corrupted[0]; + final int markedVersions = corrupted[1]; + assertTrue(corruptedValues > 0); + assertTrue("some values must carry more than one mark", markedVersions > corruptedValues); + + icheck = icheck(); + icheck.checkTree(ex.getTree()); + assertEquals(corruptedValues, icheck.getFaults().length); + assertEquals(markedVersions, icheck.getMarkedVersionCount()); + } + + /** + * Issue #290: pruning rewrites marked versions, so icheck with pruning + * enabled is the remediation path - a subsequent icheck must be clean. + */ + @Test + public void testPruneClearsLeftoverMarkBits() throws Exception { + final Exchange ex = _persistit.getExchange(_volumeName, "mvv", true); + disableBackgroundCleanup(); + transactionalStore(ex); + _persistit.getTransactionIndex().updateActiveTransactionCache(); + + final SortedMap expectedContents = treeContents(ex); + assertTrue(corrupt5(ex)[0] > 0); + + IntegrityCheck icheck = icheck(); + icheck.setPruneEnabled(true); + icheck.checkTree(ex.getTree()); + assertTrue(icheck.getMarkedVersionCount() > 0); + assertTrue(icheck.getPrunedPagesCount() > 0); + + assertEquals("remediation must not change the visible contents", expectedContents, treeContents(ex)); + + icheck = icheck(); + icheck.checkTree(ex.getTree()); + assertEquals(0, icheck.getMarkedVersionCount()); + assertEquals(0, icheck.getFaults().length); + } + + /** + * Issues #290/#292: icheck -P reports leftover mark bits as faults but + * must still clear the TransactionIndex - the prune pass has already + * rewritten the marks, so they are not a pruning failure. + */ + @Test + public void testPruneAndClearWithLeftoverMarkBits() throws Exception { + final Exchange ex = _persistit.getExchange(_volumeName, "mvv", true); + disableBackgroundCleanup(); + transactionalStore(ex); + _persistit.getTransactionIndex().updateActiveTransactionCache(); + + assertTrue(corrupt5(ex)[0] > 0); + + final IntegrityCheck icheck = (IntegrityCheck) CLI.parseTask(_persistit, "icheck trees=* -u -P"); + final StringWriter output = new StringWriter(); + icheck.setMessageWriter(new PrintWriter(output)); + icheck.setup(1, "icheck", "cli", 0, 5); + icheck.run(); + + assertTrue(icheck.getMarkedVersionCount() > 0); + assertTrue(icheck.hasFaults()); + assertTrue(output.toString(), output.toString().contains("aborted transactions were cleared by pruning")); + assertFalse(output.toString(), output.toString().contains("PruneAndClear failed")); + } + @Test public void testIndexFixHoles() throws Exception { final Exchange ex = _persistit.getExchange(_volumeName, "mvv", true); @@ -283,26 +369,52 @@ private void corrupt1(final Exchange ex) throws PersistitException { buffer.release(); } + private interface MVVCorruption { + /** Corrupts one stored MVV value; returns its contribution to the total */ + int corrupt(byte[] bytes, int length); + } + /** - * Corrupts some MVV values on a page by damaging a version length field - * + * Applies a corruption to the raw bytes of up to ten stored MVV values + * * @param ex + * @param corruption + * @return the number of values corrupted and the summed corruption + * contributions, as a two-element array * @throws PersistitException */ - private void corrupt2(final Exchange ex) throws PersistitException { + private int[] corruptMVVs(final Exchange ex, final MVVCorruption corruption) throws PersistitException { ex.ignoreMVCCFetch(true); - ex.clear().to(key(500)); - int corrupted = 0; - while (corrupted < 10 && ex.next()) { - final byte[] bytes = ex.getValue().getEncodedBytes(); - final int length = ex.getValue().getEncodedSize(); - if (MVV.isArrayMVV(bytes, 0, length)) { - bytes[9]++; - ex.store(); - corrupted++; + try { + ex.clear().to(key(500)); + int corrupted = 0; + int total = 0; + while (corrupted < 10 && ex.next()) { + final byte[] bytes = ex.getValue().getEncodedBytes(); + final int length = ex.getValue().getEncodedSize(); + if (MVV.isArrayMVV(bytes, 0, length)) { + total += corruption.corrupt(bytes, length); + ex.store(); + corrupted++; + } } + return new int[] { corrupted, total }; + } finally { + ex.ignoreMVCCFetch(false); } - ex.ignoreMVCCFetch(false); + } + + /** + * Corrupts some MVV values on a page by damaging a version length field + * + * @param ex + * @throws PersistitException + */ + private void corrupt2(final Exchange ex) throws PersistitException { + corruptMVVs(ex, (bytes, length) -> { + bytes[9]++; + return 1; + }); } /** @@ -338,6 +450,45 @@ private void corrupt4(final Exchange ex) throws PersistitException { buffer.release(); } + /** + * Simulates the pre-#288 interrupted-prune corruption (issue #286) by + * setting the leftover mark bit on every version of some MVV values + * + * @param ex + * @return the number of values corrupted and the total number of versions + * marked, as a two-element array + * @throws PersistitException + */ + private int[] corrupt5(final Exchange ex) throws PersistitException { + return corruptMVVs(ex, (bytes, length) -> { + int marked = 0; + int from = 1; + while (from + MVV.LENGTH_PER_VERSION <= length) { + MVV.mark(bytes, from); + marked++; + from += MVV.getLength(bytes, from) + MVV.LENGTH_PER_VERSION; + } + return marked; + }); + } + + /** + * Returns the visible key/value contents of the tree as seen by a + * non-transactional traversal + * + * @param ex + * @return the visible contents, keyed by decoded key string + * @throws PersistitException + */ + private SortedMap treeContents(final Exchange ex) throws PersistitException { + final SortedMap contents = new TreeMap(); + ex.clear().append(Key.BEFORE); + while (ex.next()) { + contents.put(ex.getKey().decodeString(), ex.getValue().getString()); + } + return contents; + } + private IntegrityCheck icheck() { final IntegrityCheck icheck = new IntegrityCheck(_persistit); icheck.setMessageLogVerbosity(Task.LOG_VERBOSE); diff --git a/persistit/core/src/test/java/com/persistit/MVVTest.java b/persistit/core/src/test/java/com/persistit/MVVTest.java index c68b99342..c2e05416d 100644 --- a/persistit/core/src/test/java/com/persistit/MVVTest.java +++ b/persistit/core/src/test/java/com/persistit/MVVTest.java @@ -33,6 +33,7 @@ import static com.persistit.MVV.TYPE_MVV; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -585,7 +586,7 @@ public void pruneExceptionUnmarksVersionsAtNonZeroOffset() throws Exception { fail("expected CorruptValueException"); } catch (final CorruptValueException expected) { } - assertArrayEquals("prune must leave the byte array unchanged when it throws", before, bytes); + assertArrayEquals("prune must leave an MVV without stale marks unchanged when it throws", before, bytes); } /** @@ -624,10 +625,223 @@ public void pruneExceptionMustNotUnmarkPastRegionEnd() throws Exception { assertArrayEquals("prune must not write outside the MVV region when it throws", before, bytes); } + /** + * Issue #290: leftover mark bits are invisible to the read paths (the + * length accessors strip them), so IntegrityCheck needs a dedicated scan + * to detect them. + */ + @Test + public void countMarkedVersions() { + final byte[] source = newArray(TYPE_MVV, 0, 0, 0, 0, 0, 0, 0, 10, 0, 2, 0xA, 0xB, 0, 0, 0, 0, 0, 0, 0, 11, 0, + 3, 0xC, 0xD, 0xE); + assertEquals(0, MVV.countMarkedVersions(source, 0, source.length)); + MVV.mark(source, 1); + assertEquals(1, MVV.countMarkedVersions(source, 0, source.length)); + MVV.mark(source, 13); + assertEquals(2, MVV.countMarkedVersions(source, 0, source.length)); + MVV.unmark(source, 1); + assertEquals(1, MVV.countMarkedVersions(source, 0, source.length)); + MVV.unmark(source, 13); + assertEquals(0, MVV.countMarkedVersions(source, 0, source.length)); + } + + @Test + public void countMarkedVersionsNonMVV() { + final byte[] source = newArray(0xA, 0xB, 0xC); + assertEquals(0, MVV.countMarkedVersions(source, 0, source.length)); + assertEquals(0, MVV.countMarkedVersions(source, 0, 0)); + assertEquals(0, MVV.countMarkedVersions(source, 0, -1)); + } + + // + // Issue #292: prune uses the mark bit as private transient state and + // assumes no version is marked on entry. A stale mark left on disk by a + // prune interrupted before the issue #286 fix violated that assumption: + // prune could promote the wrong version and skip the PrunedVersion + // accounting. Prune now clears all mark bits up front. + // + + /** + * A stale mark on an obsolete committed version used to win the + * primordial-conversion scan: the obsolete value was resurrected while the + * most recent committed version was silently dropped — and neither showed + * up in the PrunedVersion list, leaking the MVV count (and the page chain, + * had the dropped version been a long record). + */ + @Test + public void pruneStaleMarkedVersionPromotesMostRecentCommitted() throws Exception { + final TimestampAllocator tsa = new TimestampAllocator(); + final TransactionIndex ti = new TransactionIndex(tsa, 1); + + final int offset = 7; + final byte[] bytes = new byte[offset + 100]; + final byte[] oldValue = { 0xA, 0xB }; + final byte[] newValue = { 0xC, 0xD, 0xE }; + int length = storeCommittedVersion(tsa, ti, bytes, offset, -1, oldValue); + final long oldVersionHandle = MVV.getVersion(bytes, offset + 1); + length = storeCommittedVersion(tsa, ti, bytes, offset, length, newValue); + ti.updateActiveTransactionCache(); + + /* The stale mark an interrupted pre-#286 prune leaves behind. */ + MVV.mark(bytes, offset + 1); + + final ArrayList pruned = new ArrayList(); + final int newLength = MVV.prune(bytes, offset, length, ti, true, pruned); + + assertEquals(newValue.length, newLength); + assertArrayEquals(newValue, Arrays.copyOfRange(bytes, offset, offset + newLength)); + assertEquals("obsolete version must be accounted for", 1, pruned.size()); + assertEquals(oldVersionHandle, pruned.get(0).getVersionHandle()); + } + + /** + * The same scenario with the stale mark on the zero-length initial version + * of a value that was undefined before the MVV was created: prune used to + * resurrect "undefined", discarding the committed value entirely. + */ + @Test + public void pruneStaleMarkedUndefinedVersionNotPromoted() throws Exception { + final TimestampAllocator tsa = new TimestampAllocator(); + final TransactionIndex ti = new TransactionIndex(tsa, 1); + + final int offset = 7; + final byte[] bytes = new byte[offset + 100]; + final byte[] value = { 0xC, 0xD, 0xE }; + final int length = storeCommittedVersion(tsa, ti, bytes, offset, 0, value); + ti.updateActiveTransactionCache(); + + /* The stale mark sits on the zero-length undefined initial version. */ + MVV.mark(bytes, offset + 1); + + final ArrayList pruned = new ArrayList(); + final int newLength = MVV.prune(bytes, offset, length, ti, true, pruned); + + assertEquals(value.length, newLength); + assertArrayEquals(value, Arrays.copyOfRange(bytes, offset, offset + newLength)); + assertEquals("the primordial version carries no accounting", 0, pruned.size()); + } + + /** + * Milder variant in the multi-version path: a stale-marked dead version + * was treated as a keeper — it survived the prune and was left out of the + * PrunedVersion list, deferring its removal and accounting to the next + * prune while this one reported a clean result. + */ + @Test + public void pruneStaleMarkedVersionPrunedInMultiVersionPath() throws Exception { + final TimestampAllocator tsa = new TimestampAllocator(); + final TransactionIndex ti = new TransactionIndex(tsa, 1); + + final int offset = 7; + final byte[] bytes = new byte[offset + 100]; + final byte[] v1 = { 0xA }; + final byte[] v2 = { 0xB, 0xC }; + final byte[] v3 = { 0xD, 0xE, 0xF }; + int length = storeCommittedVersion(tsa, ti, bytes, offset, -1, v1); + final long vh1 = MVV.getVersion(bytes, offset + 1); + length = storeCommittedVersion(tsa, ti, bytes, offset, length, v2); + final int v2At = offset + 1 + MVV.LENGTH_PER_VERSION + v1.length; + final long vh2 = MVV.getVersion(bytes, v2At); + length = storeCommittedVersion(tsa, ti, bytes, offset, length, v3); + final int v3At = v2At + MVV.LENGTH_PER_VERSION + v2.length; + final long vh3 = MVV.getVersion(bytes, v3At); + ti.updateActiveTransactionCache(); + + MVV.mark(bytes, offset + 1); + + final ArrayList pruned = new ArrayList(); + final int newLength = MVV.prune(bytes, offset, length, ti, false, pruned); + + assertEquals("both dead versions must be pruned in one round", MVV.overheadLength(1) + v3.length, newLength); + assertEquals(vh3, MVV.getVersion(bytes, offset + 1)); + assertEquals(v3.length, MVV.getLength(bytes, offset + 1)); + assertArrayEquals(v3, + Arrays.copyOfRange(bytes, offset + 1 + MVV.LENGTH_PER_VERSION, offset + 1 + MVV.LENGTH_PER_VERSION + + v3.length)); + assertEquals("both dead versions must be accounted for", 2, pruned.size()); + assertEquals(vh1, pruned.get(0).getVersionHandle()); + assertEquals(vh2, pruned.get(1).getVersionHandle()); + int from = offset + 1; + while (from + MVV.LENGTH_PER_VERSION <= offset + newLength) { + assertFalse("no version may remain marked", MVV.isMarked(bytes, from)); + from += MVV.getLength(bytes, from) + MVV.LENGTH_PER_VERSION; + } + } + + /** + * The up-front sweep must not follow a corrupted length field: a + * misaligned traversal lands inside a value and unmark() clears bit 15 of + * a payload byte — inside the MVV region, past what + * {@link #pruneExceptionMustNotUnmarkPastRegionEnd} guards. Prune must + * reject the malformed region without writing anything. + */ + @Test + public void pruneMustNotSweepMisalignedMvv() throws Exception { + final TimestampAllocator tsa = new TimestampAllocator(); + final TransactionIndex ti = new TransactionIndex(tsa, 1); + + /* + * Two aborted versions, so no pass marks anything and any byte that + * differs after prune was written by the sweep. The 0xFF fill of the + * second value makes a misaligned unmark() visible (bit 15 cleared). + */ + final int offset = 7; + final byte[] bytes = new byte[offset + 100]; + final byte[] v1 = { 0x1, 0x2, 0x3, 0x4 }; + final byte[] v2 = new byte[20]; + Arrays.fill(v2, (byte) 0xFF); + int length = storeAbortedVersion(tsa, ti, bytes, offset, -1, v1); + length = storeAbortedVersion(tsa, ti, bytes, offset, length, v2); + ti.updateActiveTransactionCache(); + + /* Corrupt the first version's length field: it claims 9 bytes, not 4. */ + MVV.putLength(bytes, offset + 1, 9); + + final byte[] before = bytes.clone(); + try { + MVV.prune(bytes, offset, length, ti, true, new ArrayList()); + fail("expected CorruptValueException"); + } catch (final CorruptValueException expected) { + } + assertArrayEquals("prune must not write into a misaligned MVV region", before, bytes); + } + // // Test helper methods // + /** + * Store value as a new version created by a registered + * transaction and commit it, so that {@link MVV#prune} sees a committed, + * non-concurrent version. + */ + private static int storeCommittedVersion(final TimestampAllocator tsa, final TransactionIndex ti, + final byte[] bytes, final int offset, final int length, final byte[] value) throws Exception { + final TransactionStatus status = ti.registerTransaction(); + final int newLength = MVV.storeVersion(bytes, offset, length, bytes.length, + TransactionIndex.ts2vh(status.getTs()), value, 0, value.length) & STORE_LENGTH_MASK; + final long tc = tsa.updateTimestamp(); + status.commit(tc); + ti.notifyCompleted(status, tc); + return newLength; + } + + /** + * Store value as a new version created by a registered + * transaction and abort it, so that {@link MVV#prune} sees an aborted + * version and marks nothing. + */ + private static int storeAbortedVersion(final TimestampAllocator tsa, final TransactionIndex ti, + final byte[] bytes, final int offset, final int length, final byte[] value) throws Exception { + final TransactionStatus status = ti.registerTransaction(); + final int newLength = MVV.storeVersion(bytes, offset, length, bytes.length, + TransactionIndex.ts2vh(status.getTs()), value, 0, value.length) & STORE_LENGTH_MASK; + status.incrementMvvCount(); + status.abort(); + ti.notifyCompleted(status, tsa.updateTimestamp()); + return newLength; + } + private static int writeArray(final byte[] array, final int... contents) { assert contents.length <= array.length : "Too many values for array"; for (int i = 0; i < contents.length; ++i) { diff --git a/persistit/doc/Management.rst b/persistit/doc/Management.rst index bf02959b1..420b94f7b 100644 --- a/persistit/doc/Management.rst +++ b/persistit/doc/Management.rst @@ -133,9 +133,9 @@ programmatically or through JMX. (You may specify any valid, available port.) Th Where ``classpath`` includes the Persistit library. Assuming the name of the script is ``pcli`` you can then invoke commands from a shell as shown in this example:: /home/akiban:~$ pcli icheck -v -c "trees=*:Acc*" - Volume,Tree,Faults,IndexPages,IndexBytes,DataPages,DataBytes,LongRecordPages,LongRecordBytes,MvvPages,MvvRecords,MvvOverhead,MvvAntiValues,IndexHoles,PrunedPages - "persistit","AccumulatorRecoveryTest",0,3,24296,1519,15560788,0,0,1506,52192,721521,2397,0,0 - "*","*",0,3,24296,1519,15560788,0,0,1506,52192,721521,2397,0,0 + Volume,Tree,Faults,IndexPages,IndexBytes,DataPages,DataBytes,LongRecordPages,LongRecordBytes,MvvPages,MvvRecords,MvvOverhead,MvvAntiValues,IndexHoles,PrunedPages,MarkedVersions + "persistit","AccumulatorRecoveryTest",0,3,24296,1519,15560788,0,0,1506,52192,721521,2397,0,0,0 + "*","*",0,3,24296,1519,15560788,0,0,1506,52192,721521,2397,0,0,0 /home/akiban:~$ Alternatively, you can use ``curl`` as follows::