Merge lp:~pbeaman/akiban-persistit/warn-missing-volume into lp:akiban-persistit

[Peter Beaman]

Modification inspired by https://bugs.launchpad.net/akiban-persistit/+bug/1028016 to issue warnings when a previously existing volume is missing, and an option to allow a system to continue by ignoring pages and transactions in the journal from missing volumes.

This branch adds code in several places to deal with the possibility that a formerly open volume is no longer present during recovery:

JournalManager#readForCopy
JournalManager#writeForCopy
TransactionPlayer#applyTransactionUpdates

In each case new logic posts warning messages via the AlertMonitor and then checks a new JournalManagerMXBean property isIgnoreMissingVolumes() to determine whether the problem is fatal or should be ignored.

To allow setting the property before recovery begins there is also a new Configuration setting "ignoremissingvolumes" (suitably long and obscure).

New tests in JournalManagerTest verify intended behavior.

[Nathan Williams (nwilliams)]

One minor thing:
javadoc on setIgnoreMissingVolumes() references appendOnly parameter

Otherwise this makes sense I think.

review: Needs Fixing

[lp:~pbeaman/akiban-persistit/warn-missing-volume]

345. By Peter Beaman on 2012-07-26

Small Javadoc issue

[Peter Beaman (pbeaman)]

Fixed. Thanks.

[Nathan Williams (nwilliams)]

Looks good.

review: Approve

=== modified file 'src/main/java/com/persistit/Configuration.java'
--- src/main/java/com/persistit/Configuration.java	2012-06-08 19:14:52 +0000
+++ src/main/java/com/persistit/Configuration.java	2012-07-26 18:54:32 +0000
@@ -38,12 +38,12 @@
 import java.util.TreeMap;
 
 import com.persistit.Transaction.CommitPolicy;
+import com.persistit.exception.CorruptJournalException;
 import com.persistit.exception.InvalidVolumeSpecificationException;
 import com.persistit.exception.PersistitException;
 import com.persistit.exception.PersistitIOException;
 import com.persistit.exception.PropertiesNotFoundException;
 import com.persistit.logging.DefaultPersistitLogger;
-import com.persistit.logging.PersistitLevel;
 import com.persistit.policy.JoinPolicy;
 import com.persistit.policy.SplitPolicy;
 import com.persistit.util.Util;
@@ -271,6 +271,10 @@
     public final static String APPEND_ONLY_PROPERTY = "appendonly";
 
     /**
+     * Property name for the "ignore missing volumes" property.
+     */
+    public final static String IGNORE_MISSING_VOLUMES_PROPERTY = "ignoremissingvolumes";
+    /**
      * Property name to specify the default {@link SplitPolicy}.
      */
     public final static String SPLIT_POLICY_PROPERTY_NAME = "splitpolicy";
@@ -403,9 +407,10 @@
         public void setMaximumCount(int maximumCount) {
             this.maximumCount = maximumCount;
         }
-        
+
         /**
          * Set the minimum and maximum buffer count.
+         * 
          * @param count
          */
         public void setCount(int count) {
@@ -493,8 +498,8 @@
             int bufferSizeWithOverhead = Buffer.bufferSizeWithOverhead(bufferSize);
             int buffers = Math.max(minimumCount, Math.min(maximumCount, (int) (allocation / bufferSizeWithOverhead)));
             if (buffers < BufferPool.MINIMUM_POOL_COUNT || buffers > BufferPool.MAXIMUM_POOL_COUNT
-                    || (long)buffers * (long)bufferSizeWithOverhead > maximumAvailable
-                    || (long)(buffers + 1) * (long)bufferSizeWithOverhead < minimumMemory) {
+                    || (long) buffers * (long) bufferSizeWithOverhead > maximumAvailable
+                    || (long) (buffers + 1) * (long) bufferSizeWithOverhead < minimumMemory) {
                 throw new IllegalArgumentException(String.format(
                         "Invalid buffer pool configuration: %,d buffers in %sb of maximum available memory", buffers,
                         displayableLongValue(maximumAvailable)));
@@ -624,6 +629,7 @@
     private int rmiServerPort;
     private boolean jmx = true;
     private boolean appendOnly;
+    private boolean ignoreMissingVolumes;
     private String tmpVolDir;
     private int tmpVolPageSize;
     private long tmpVolMaxSize;
@@ -704,6 +710,7 @@
         setAppendOnly(getBooleanProperty(APPEND_ONLY_PROPERTY, false));
         setCommitPolicy(getProperty(COMMIT_POLICY_PROPERTY_NAME));
         setConstructorOverride(getBooleanProperty(CONSTRUCTOR_OVERRIDE_PROPERTY_NAME, false));
+        setIgnoreMissingVolumes(getBooleanProperty(IGNORE_MISSING_VOLUMES_PROPERTY, false));
         setJmxEnabled(getBooleanProperty(ENABLE_JMX_PROPERTY_NAME, true));
         setJoinPolicy(getProperty(JOIN_POLICY_PROPERTY_NAME));
         setJournalPath(getProperty(JOURNAL_PATH_PROPERTY_NAME, DEFAULT_JOURNAL_PATH));
@@ -1772,6 +1779,7 @@
 
     /**
      * Return the value defined by {@link #setAppendOnly}
+     * 
      * @return the whether to start Persistit in append-only mode
      */
     public boolean isAppendOnly() {
@@ -1780,7 +1788,7 @@
 
     /**
      * <p>
-     * Control whether Persistit starts in <i>append-only</i>. In this mode
+     * Control whether Persistit starts in <i>append-only</i> mode. In this mode
      * Persistit accumulates database updates in the journal without copying
      * changes back into the volume files. This method changes only the initial
      * state; use {@link Management#setAppendOnly(boolean)} method to change
@@ -1792,10 +1800,39 @@
      * </p>
      * 
      * @param appendOnly
-     *            the appendOnly to set
+     *            <code>true</code> to start Persistit in append-only only
      */
     public void setAppendOnly(boolean appendOnly) {
         this.appendOnly = appendOnly;
     }
 
+    /**
+     * Return the value defined by {@link #setIgnoreMissingVolumes(boolean)}
+     * 
+     * @return the whether to start Persistit in ignore-missing-volumes mode
+     */
+    public boolean isIgnoreMissingVolumes() {
+        return ignoreMissingVolumes;
+    }
+
+    /**
+     * <p>
+     * Control whether Persistit starts in <i>ignore-missing-volumes</i> mode.
+     * In this mode references in the journal to unknown volumes are ignored
+     * rather than noted as {@link CorruptJournalException}s. Almost always this
+     * mode should be disabled; the setting is available to enable recovery of a
+     * journal into a subset of formerly existing volumes and should be used
+     * only with care.
+     * </p>
+     * <p>
+     * Default value is <code>false</code><br />
+     * Property name is {@value #IGNORE_MISSING_VOLUMES_PROPERTY}
+     * </p>
+     * 
+     * @param ignoreMissingVolumes
+     *            <code>true</code> to ignore missing volumes
+     */
+    public void setIgnoreMissingVolumes(boolean ignoreMissingVolumes) {
+        this.ignoreMissingVolumes = ignoreMissingVolumes;
+    }
 }

=== modified file 'src/main/java/com/persistit/JournalManager.java'
--- src/main/java/com/persistit/JournalManager.java	2012-06-29 21:43:10 +0000
+++ src/main/java/com/persistit/JournalManager.java	2012-07-26 18:54:32 +0000
@@ -135,6 +135,8 @@
 
     private AtomicBoolean _appendOnly = new AtomicBoolean();
 
+    private AtomicBoolean _ignoreMissingVolume = new AtomicBoolean();
+
     private String _journalFilePath;
 
     /**
@@ -375,6 +377,11 @@
     }
 
     @Override
+    public boolean isIgnoreMissingVolumes() {
+        return _ignoreMissingVolume.get();
+    }
+
+    @Override
     public boolean isCopyingFast() {
         return _copyFast.get();
     }
@@ -385,6 +392,11 @@
     }
 
     @Override
+    public void setIgnoreMissingVolumes(boolean ignore) {
+        _ignoreMissingVolume.set(ignore);
+    }
+
+    @Override
     public void setCopyingFast(boolean fast) {
         _copyFast.set(fast);
     }
@@ -2362,6 +2374,8 @@
         int handle = -1;
 
         for (final Iterator<PageNode> iterator = list.iterator(); iterator.hasNext();) {
+            Volume volumeRef = null;
+
             if (_closed.get() && !_copyFast.get() || _appendOnly.get()) {
                 list.clear();
                 break;
@@ -2373,23 +2387,20 @@
             }
             if (pageNode.getVolumeHandle() != handle) {
                 handle = -1;
-                volume = _handleToVolumeMap.get(pageNode.getVolumeHandle());
-                if (volume == null) {
-                    // TODO
-                } else {
-                    volume = _persistit.getVolume(volume.getName());
+                volume = null;
+                // Possibly hollow volume
+                volumeRef = _handleToVolumeMap.get(pageNode.getVolumeHandle());
+                if (volumeRef != null) {
+                    // Opened volume, if present
+                    volume = _persistit.getVolume(volumeRef.getName());
                     handle = pageNode.getVolumeHandle();
                 }
             }
-            if (volume == null || volume.isClosed()) {
-                // Remove from the List so that below we won't remove it from
-                // from the pageMap.
-                iterator.remove();
+            if (volume == null) {
+                // Deal with this in writeForCopy
                 continue;
             }
 
-            volume.verifyId(volume.getId());
-
             final int at = bb.position();
             final long pageAddress;
             try {
@@ -2400,8 +2411,9 @@
                 }
                 pageAddress = readPageBufferFromJournal(stablePageNode, bb);
             } catch (PersistitException ioe) {
-                _persistit.getLogBase().copyException.log(ioe, volume, pageNode.getPageAddress(), pageNode
-                        .getJournalAddress());
+                _persistit.getAlertMonitor().post(
+                        new Event(AlertLevel.ERROR, _persistit.getLogBase().copyException, ioe, volume, pageNode
+                                .getPageAddress(), pageNode.getJournalAddress()), AlertMonitor.JOURNAL_CATEGORY);
                 throw ioe;
             }
 
@@ -2420,10 +2432,12 @@
 
     void writeForCopy(final List<PageNode> list, final ByteBuffer bb) throws PersistitException {
         Collections.sort(list, PageNode.WRITE_COMPARATOR);
+        Volume volumeRef = null;
         Volume volume = null;
         int handle = -1;
 
         final HashSet<Volume> volumes = new HashSet<Volume>();
+
         for (final Iterator<PageNode> iterator = list.iterator(); iterator.hasNext();) {
             if (_closed.get() && !_copyFast.get() || _appendOnly.get()) {
                 list.clear();
@@ -2434,15 +2448,28 @@
 
             if (pageNode.getVolumeHandle() != handle) {
                 handle = -1;
-                volume = _handleToVolumeMap.get(pageNode.getVolumeHandle());
-                if (volume == null) {
-                    // TODO
-                } else {
-                    volume = _persistit.getVolume(volume.getName());
+                volume = null;
+                // Possibly hollow volume
+                volumeRef = _handleToVolumeMap.get(pageNode.getVolumeHandle());
+                if (volumeRef != null) {
+                    // Opened volume, if present
+                    volume = _persistit.getVolume(volumeRef.getName());
                     handle = pageNode.getVolumeHandle();
                 }
             }
-
+            if (volume == null) {
+                _persistit.getAlertMonitor().post(
+                        new Event(AlertLevel.WARN, _persistit.getLogBase().missingVolume, volumeRef, pageNode
+                                .getJournalAddress()), AlertMonitor.MISSING_VOLUME_CATEGORY);
+                if (_ignoreMissingVolume.get()) {
+                    _persistit.getLogBase().lostPageFromMissingVolume.log(pageNode.getPageAddress(), volumeRef,
+                            pageNode.getJournalAddress());
+                    // Not removing the page from the List here will cause
+                    // cleanupForCopy to remove it from
+                    // the page map.
+                    continue;
+                }
+            }
             if (volume == null || volume.isClosed()) {
                 // Remove from the List so that below we won't remove it from
                 // from the pageMap.
@@ -2450,6 +2477,8 @@
                 continue;
             }
 
+            volumeRef.verifyId(volume.getId());
+
             final long pageAddress = pageNode.getPageAddress();
             volume.getStorage().extend(pageAddress);
             final int pageSize = volume.getPageSize();

=== modified file 'src/main/java/com/persistit/Persistit.java'
--- src/main/java/com/persistit/Persistit.java	2012-07-25 23:58:46 +0000
+++ src/main/java/com/persistit/Persistit.java	2012-07-26 18:54:32 +0000
@@ -647,6 +647,7 @@
 
         _journalManager.init(_recoveryManager, journalPath, journalSize);
         _journalManager.setAppendOnly(_configuration.isAppendOnly());
+        _journalManager.setIgnoreMissingVolumes(_configuration.isIgnoreMissingVolumes());
     }
 
     void initializeBufferPools() {

=== modified file 'src/main/java/com/persistit/RecoveryManager.java'
--- src/main/java/com/persistit/RecoveryManager.java	2012-06-28 14:48:47 +0000
+++ src/main/java/com/persistit/RecoveryManager.java	2012-07-26 18:54:32 +0000
@@ -699,12 +699,16 @@
         return sb.toString();
     }
 
-    public String addressToString(final long address) {
-        return _player.addressToString(address);
+    String addressToString(final long address) {
+        return TransactionPlayer.addressToString(address);
     }
 
     private String addressToString(final long address, final long timestamp) {
-        return _player.addressToString(address, timestamp);
+        return TransactionPlayer.addressToString(address, timestamp);
+    }
+    
+    TransactionPlayer getPlayer() {
+        return _player;
     }
 
     // ----------------------------Phase 1----------------------------

=== modified file 'src/main/java/com/persistit/TransactionPlayer.java'
--- src/main/java/com/persistit/TransactionPlayer.java	2012-05-25 18:50:59 +0000
+++ src/main/java/com/persistit/TransactionPlayer.java	2012-07-26 18:54:32 +0000
@@ -28,7 +28,10 @@
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
 
+import com.persistit.AlertMonitor.AlertLevel;
+import com.persistit.AlertMonitor.Event;
 import com.persistit.JournalManager.TransactionMapItem;
 import com.persistit.JournalManager.TreeDescriptor;
 import com.persistit.JournalRecord.D0;
@@ -38,9 +41,14 @@
 import com.persistit.JournalRecord.SR;
 import com.persistit.JournalRecord.TX;
 import com.persistit.exception.CorruptJournalException;
+import com.persistit.exception.MissingVolumeException;
 import com.persistit.exception.PersistitException;
 
 class TransactionPlayer {
+    
+    private final AtomicLong appliedUpdates = new AtomicLong();
+    private final AtomicLong ignoredUpdates = new AtomicLong();
+    private final AtomicLong failedUpdates = new AtomicLong();
 
     interface TransactionPlayerListener {
 
@@ -61,7 +69,7 @@
         void endTransaction(long address, long timestamp) throws PersistitException;
 
         void endRecovery(long address, long timestamp) throws PersistitException;
-        
+
         boolean requiresLongRecordConversion();
 
     }
@@ -72,7 +80,7 @@
         _support = support;
     }
 
-    public void applyTransaction(final TransactionMapItem item, final TransactionPlayerListener listener)
+    void applyTransaction(final TransactionMapItem item, final TransactionPlayerListener listener)
             throws PersistitException {
 
         final List<Long> chainedAddress = new ArrayList<Long>();
@@ -83,7 +91,6 @@
         long startTimestamp;
         long commitTimestamp;
         long backchainAddress;
-        int appliedUpdates = 0;
 
         for (;;) {
             _support.read(address, TX.OVERHEAD);
@@ -114,8 +121,8 @@
         }
 
         listener.startTransaction(address, startTimestamp, commitTimestamp);
-        appliedUpdates += applyTransactionUpdates(_support.getReadBuffer(), address, recordSize, startTimestamp, commitTimestamp,
-                listener);
+        applyTransactionUpdates(_support.getReadBuffer(), address, recordSize, startTimestamp,
+                commitTimestamp, listener);
 
         for (Long continuation : chainedAddress) {
             address = continuation.longValue();
@@ -127,124 +134,140 @@
                         + " has invalid length " + recordSize + " or type " + type);
             }
             _support.read(address, recordSize);
-            appliedUpdates += applyTransactionUpdates(_support.getReadBuffer(), address, recordSize, startTimestamp, commitTimestamp,
-                    listener);
+            applyTransactionUpdates(_support.getReadBuffer(), address, recordSize, startTimestamp,
+                    commitTimestamp, listener);
         }
         listener.endTransaction(address, startTimestamp);
 
     }
 
-    int applyTransactionUpdates(final ByteBuffer byteBuffer, final long address, final int recordSize,
+    void applyTransactionUpdates(final ByteBuffer byteBuffer, final long address, final int recordSize,
             final long startTimestamp, final long commitTimestamp, final TransactionPlayerListener listener)
             throws PersistitException {
         ByteBuffer bb = byteBuffer;
         final int start = bb.position();
         int end = start + recordSize;
         int position = start + TX.OVERHEAD;
-        int appliedUpdates = 0;
 
         while (position < end) {
             bb.position(position);
             final int innerSize = JournalRecord.getLength(bb);
             final int type = JournalRecord.getType(bb);
-            switch (type) {
-            case SR.TYPE: {
-                final int keySize = SR.getKeySize(bb);
-                final int treeHandle = SR.getTreeHandle(bb);
-                final Exchange exchange = getExchange(treeHandle, address, startTimestamp);
-                exchange.ignoreTransactions();
-                final Key key = exchange.getKey();
-                final Value value = exchange.getValue();
-                System.arraycopy(bb.array(), bb.position() + SR.OVERHEAD, key.getEncodedBytes(), 0, keySize);
-                key.setEncodedSize(keySize);
-                final int valueSize = innerSize - SR.OVERHEAD - keySize;
-                value.ensureFit(valueSize);
-                System.arraycopy(bb.array(), bb.position() + SR.OVERHEAD + keySize, value.getEncodedBytes(), 0,
-                        valueSize);
-                value.setEncodedSize(valueSize);
-
-                if (value.getEncodedSize() >= Buffer.LONGREC_SIZE
-                        && (value.getEncodedBytes()[0] & 0xFF) == Buffer.LONGREC_TYPE) {
+            try {
+                switch (type) {
+                case SR.TYPE: {
+                    final int keySize = SR.getKeySize(bb);
+                    final int treeHandle = SR.getTreeHandle(bb);
+                    final Exchange exchange = getExchange(treeHandle, address, startTimestamp);
+                    exchange.ignoreTransactions();
+                    final Key key = exchange.getKey();
+                    final Value value = exchange.getValue();
+                    System.arraycopy(bb.array(), bb.position() + SR.OVERHEAD, key.getEncodedBytes(), 0, keySize);
+                    key.setEncodedSize(keySize);
+                    final int valueSize = innerSize - SR.OVERHEAD - keySize;
+                    value.ensureFit(valueSize);
+                    System.arraycopy(bb.array(), bb.position() + SR.OVERHEAD + keySize, value.getEncodedBytes(), 0,
+                            valueSize);
+                    value.setEncodedSize(valueSize);
+
+                    if (value.getEncodedSize() >= Buffer.LONGREC_SIZE
+                            && (value.getEncodedBytes()[0] & 0xFF) == Buffer.LONGREC_TYPE) {
+                        /*
+                         * convertToLongRecord will pollute the getReadBuffer().
+                         * Therefore before calling it we need to copy the TX
+                         * record to a fresh ByteBuffer.
+                         */
+                        if (bb == _support.getReadBuffer()) {
+                            end = recordSize - (position - start);
+                            bb = ByteBuffer.allocate(end);
+                            bb.put(_support.getReadBuffer().array(), position, end);
+                            bb.flip();
+                            position = 0;
+                        }
+                        if (listener.requiresLongRecordConversion()) {
+                            _support.convertToLongRecord(value, treeHandle, address, commitTimestamp);
+                        }
+                    }
+
+                    listener.store(address, startTimestamp, exchange);
+                    appliedUpdates.incrementAndGet();
+                    // Don't keep exchanges with enlarged value - let them be
+                    // GC'd
+                    if (exchange.getValue().getMaximumSize() < Value.DEFAULT_MAXIMUM_SIZE) {
+                        releaseExchange(exchange);
+                    }
+                    break;
+                }
+
+                case DR.TYPE: {
+                    final int key1Size = DR.getKey1Size(bb);
+                    final int elisionCount = DR.getKey2Elision(bb);
+                    final Exchange exchange = getExchange(DR.getTreeHandle(bb), address, startTimestamp);
+                    exchange.ignoreTransactions();
+                    final Key key1 = exchange.getAuxiliaryKey1();
+                    final Key key2 = exchange.getAuxiliaryKey2();
+                    System.arraycopy(bb.array(), bb.position() + DR.OVERHEAD, key1.getEncodedBytes(), 0, key1Size);
+                    key1.setEncodedSize(key1Size);
+                    final int key2Size = innerSize - DR.OVERHEAD - key1Size;
+                    System.arraycopy(key1.getEncodedBytes(), 0, key2.getEncodedBytes(), 0, elisionCount);
+                    System.arraycopy(bb.array(), bb.position() + DR.OVERHEAD + key1Size, key2.getEncodedBytes(),
+                            elisionCount, key2Size);
+                    key2.setEncodedSize(key2Size + elisionCount);
+                    listener.removeKeyRange(address, startTimestamp, exchange, exchange.getAuxiliaryKey1(), exchange
+                            .getAuxiliaryKey2());
+                    appliedUpdates.incrementAndGet();
+                    releaseExchange(exchange);
+                    break;
+                }
+
+                case DT.TYPE: {
+                    final Exchange exchange = getExchange(DT.getTreeHandle(bb), address, startTimestamp);
+                    listener.removeTree(address, startTimestamp, exchange);
+                    appliedUpdates.incrementAndGet();
+                    releaseExchange(exchange);
+                    break;
+                }
+
+                case D0.TYPE: {
+                    final Exchange exchange = getExchange(D0.getTreeHandle(bb), address, startTimestamp);
+                    listener.delta(address, startTimestamp, exchange.getTree(), D0.getIndex(bb), D0
+                            .getAccumulatorTypeOrdinal(bb), 1);
+                    appliedUpdates.incrementAndGet();
+                    break;
+                }
+
+                case D1.TYPE: {
+                    final Exchange exchange = getExchange(D1.getTreeHandle(bb), address, startTimestamp);
+                    listener.delta(address, startTimestamp, exchange.getTree(), D1.getIndex(bb), D1
+                            .getAccumulatorTypeOrdinal(bb), D1.getValue(bb));
+                    appliedUpdates.incrementAndGet();
+                    break;
+                }
+
+                default: {
+                    throw new CorruptJournalException("Invalid record type " + type + " at journal address "
+                            + addressToString(address + position - start) + " index of transaction record at "
+                            + addressToString(address));
+                }
+                }
+            } catch (MissingVolumeException mve) {
+                final Persistit db = _support.getPersistit();
+                if (db.getJournalManager().isIgnoreMissingVolumes()) {
                     /*
-                     * convertToLongRecord will pollute the getReadBuffer().
-                     * Therefore before calling it we need to copy the TX record
-                     * to a fresh ByteBuffer.
+                     * If ignoreMissingVolumes is enabled, then issue a warning
+                     * Alert, but allow recovery or rollback to continue.
                      */
-                    if (bb == _support.getReadBuffer()) {
-                        end = recordSize - (position - start);
-                        bb = ByteBuffer.allocate(end);
-                        bb.put(_support.getReadBuffer().array(), position, end);
-                        bb.flip();
-                        position = 0;
-                    }
-                    if (listener.requiresLongRecordConversion()) {
-                        _support.convertToLongRecord(value, treeHandle, address, commitTimestamp);
-                    }
-                }
-
-                listener.store(address, startTimestamp, exchange);
-                appliedUpdates++;
-                // Don't keep exchanges with enlarged value - let them be GC'd
-                if (exchange.getValue().getMaximumSize() < Value.DEFAULT_MAXIMUM_SIZE) {
-                    releaseExchange(exchange);
-                }
-                break;
-            }
-
-            case DR.TYPE: {
-                final int key1Size = DR.getKey1Size(bb);
-                final int elisionCount = DR.getKey2Elision(bb);
-                final Exchange exchange = getExchange(DR.getTreeHandle(bb), address, startTimestamp);
-                exchange.ignoreTransactions();
-                final Key key1 = exchange.getAuxiliaryKey1();
-                final Key key2 = exchange.getAuxiliaryKey2();
-                System.arraycopy(bb.array(), bb.position() + DR.OVERHEAD, key1.getEncodedBytes(), 0, key1Size);
-                key1.setEncodedSize(key1Size);
-                final int key2Size = innerSize - DR.OVERHEAD - key1Size;
-                System.arraycopy(key1.getEncodedBytes(), 0, key2.getEncodedBytes(), 0, elisionCount);
-                System.arraycopy(bb.array(), bb.position() + DR.OVERHEAD + key1Size, key2.getEncodedBytes(),
-                        elisionCount, key2Size);
-                key2.setEncodedSize(key2Size + elisionCount);
-                listener.removeKeyRange(address, startTimestamp, exchange, exchange.getAuxiliaryKey1(), exchange
-                        .getAuxiliaryKey2());
-                appliedUpdates++;
-                releaseExchange(exchange);
-                break;
-            }
-
-            case DT.TYPE: {
-                final Exchange exchange = getExchange(DT.getTreeHandle(bb), address, startTimestamp);
-                listener.removeTree(address, startTimestamp, exchange);
-                appliedUpdates++;
-                releaseExchange(exchange);
-                break;
-            }
-
-            case D0.TYPE: {
-                final Exchange exchange = getExchange(D0.getTreeHandle(bb), address, startTimestamp);
-                listener.delta(address, startTimestamp, exchange.getTree(), D0.getIndex(bb), D0
-                        .getAccumulatorTypeOrdinal(bb), 1);
-                appliedUpdates++;
-                break;
-            }
-
-            case D1.TYPE: {
-                final Exchange exchange = getExchange(D1.getTreeHandle(bb), address, startTimestamp);
-                listener.delta(address, startTimestamp, exchange.getTree(), D1.getIndex(bb), D1
-                        .getAccumulatorTypeOrdinal(bb), D1.getValue(bb));
-                appliedUpdates++;
-                break;
-            }
-
-            default: {
-                throw new CorruptJournalException("Invalid record type " + type + " at journal address "
-                        + addressToString(address + position - start) + " index of transaction record at "
-                        + addressToString(address));
-            }
+                    db.getAlertMonitor().post(
+                            new Event(AlertLevel.WARN, db.getLogBase().missingVolume, mve.getVolumeName(), address
+                                    + position - start), AlertMonitor.MISSING_VOLUME_CATEGORY);
+                    ignoredUpdates.incrementAndGet();
+                } else {
+                    failedUpdates.incrementAndGet();
+                    throw mve;
+                }
             }
             position += innerSize;
         }
-        return appliedUpdates;
     }
 
     public static String addressToString(final long address) {
@@ -261,17 +284,20 @@
             throw new CorruptJournalException("Tree handle " + treeHandle + " is undefined at "
                     + addressToString(from, timestamp));
         }
-        Volume volume = _support.handleToVolume(td.getVolumeHandle());
-        if (volume == null) {
+        Volume volumeRef = _support.handleToVolume(td.getVolumeHandle());
+        Volume volume;
+        if (volumeRef == null) {
             throw new CorruptJournalException("Volume handle " + td.getVolumeHandle() + " is undefined at "
                     + addressToString(from, timestamp));
         }
 
-        if (!volume.isOpened()) {
-            volume = _support.getPersistit().getVolume(volume.getName());
+        if (volumeRef.isOpened()) {
+            volume = volumeRef;
+        } else {
+            volume = _support.getPersistit().getVolume(volumeRef.getName());
             if (volume == null) {
-                throw new CorruptJournalException("No matching Volume found for journal reference " + volume + " at "
-                        + addressToString(from, timestamp));
+                throw new MissingVolumeException("No matching Volume found for journal reference " + volumeRef + " at "
+                        + addressToString(from, timestamp), volumeRef.getName());
             }
         }
         volume.verifyId(volume.getId());
@@ -286,4 +312,16 @@
     private void releaseExchange(final Exchange exchange) {
         _support.getPersistit().releaseExchange(exchange);
     }
+    
+    long getAppliedUpdates() {
+        return appliedUpdates.get();
+    }
+    
+    long getIgnoredUpdates() {
+        return ignoredUpdates.get();
+    }
+    
+    long getFailedUpdates() {
+        return failedUpdates.get();
+    }
 }

=== added file 'src/main/java/com/persistit/exception/MissingVolumeException.java'
--- src/main/java/com/persistit/exception/MissingVolumeException.java	1970-01-01 00:00:00 +0000
+++ src/main/java/com/persistit/exception/MissingVolumeException.java	2012-07-26 18:54:32 +0000
@@ -0,0 +1,45 @@
+/**
+ * Copyright © 2011-2012 Akiban Technologies, Inc.  All rights reserved.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, version 3 (only) of the
+ * License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ * 
+ * This program may also be available under different license terms. For more
+ * information, see www.akiban.com or contact [e-mail].
+ */
+
+package com.persistit.exception;
+
+/**
+ * Thrown if the journal files refer to a volume that is no longer present in
+ * the system. Generally this condition is irrecoverable because without the
+ * missing volume a consistent database state cannot be restored. However, in
+ * the event the removal of the volume is intentional, it is possible to specify
+ * a mode in which pages and transactions destined for missing volumes are
+ * ignored.
+ * 
+ * @version 1.0
+ */
+public class MissingVolumeException extends CorruptJournalException {
+    private static final long serialVersionUID = -9014051945087375523L;
+    private final String _volumeName;
+
+    public MissingVolumeException(final String msg, final String volumeName) {
+        super(msg);
+        _volumeName = volumeName;
+    }
+    
+    public String getVolumeName() {
+        return _volumeName;
+    }
+}

=== modified file 'src/main/java/com/persistit/logging/LogBase.java'
--- src/main/java/com/persistit/logging/LogBase.java	2012-07-25 23:58:46 +0000
+++ src/main/java/com/persistit/logging/LogBase.java	2012-07-26 18:54:32 +0000
@@ -174,6 +174,12 @@
     @Message("ERROR|Exception %s while writing volume %s page %,d")
     public final LogItem writeException = PersistitLogMessage.empty();
 
+    @Message("WARNING|Missing volume %s referenced at journal address %,d")
+    public final LogItem missingVolume = PersistitLogMessage.empty();
+    
+    @Message("WARNING|Lost page %,d from missing volume %s referenced at journal address %,d")
+    public final LogItem lostPageFromMissingVolume = PersistitLogMessage.empty();
+
     @Message("WARNING|Exception %s while copying volume %s page %,d from journal address %,d")
     public final LogItem copyException = PersistitLogMessage.empty();
 

=== modified file 'src/main/java/com/persistit/mxbeans/AlertMonitorMXBean.java'
--- src/main/java/com/persistit/mxbeans/AlertMonitorMXBean.java	2012-05-25 18:50:59 +0000
+++ src/main/java/com/persistit/mxbeans/AlertMonitorMXBean.java	2012-07-26 18:54:32 +0000
@@ -41,6 +41,7 @@
     final static String EXTEND_VOLUME_CATEGORY = "ExtendVolume";
     final static String FLUSH_STATISTICS_CATEGORY = "FlushStatistics";
     final static String CLEANUP_CATEGORY = "Cleanup";
+    final static String MISSING_VOLUME_CATEGORY = "MissingVolume";
 
     /**
      * Current maximum AlertLevel in this monitor as a String: one of NORMAL,

=== modified file 'src/main/java/com/persistit/mxbeans/JournalManagerMXBean.java'
--- src/main/java/com/persistit/mxbeans/JournalManagerMXBean.java	2012-05-25 18:50:59 +0000
+++ src/main/java/com/persistit/mxbeans/JournalManagerMXBean.java	2012-07-26 18:54:32 +0000
@@ -148,12 +148,18 @@
 
     @Description("True if copying of pages from the journal to their destination volumes is disabled")
     boolean isAppendOnly();
+    
+    @Description("True to allow journal to lose pages from missing volumes")
+    boolean isIgnoreMissingVolumes();
 
     @Description("True if copy-fast mode has been enabled")
     boolean isCopyingFast();
 
     @Description("True if copying of pages from the journal to their destination volumes is disabled")
     void setAppendOnly(boolean appendOnly);
+    
+    @Description("True to allow journal to lose pages from missing volumes")
+    void setIgnoreMissingVolumes(boolean ignore);
 
     @Description("True if copy-fast mode has been enabled")
     void setCopyingFast(boolean fast);

=== modified file 'src/test/java/com/persistit/JournalManagerTest.java'
--- src/test/java/com/persistit/JournalManagerTest.java	2012-05-25 18:50:59 +0000
+++ src/test/java/com/persistit/JournalManagerTest.java	2012-07-26 18:54:32 +0000
@@ -21,19 +21,19 @@
 package com.persistit;
 
 import static com.persistit.unit.ConcurrentUtil.createThread;
-import static com.persistit.unit.ConcurrentUtil.ThrowingRunnable;
 import static com.persistit.unit.ConcurrentUtil.startAndJoinAssertSuccess;
 import static com.persistit.util.SequencerConstants.PAGE_MAP_READ_INVALIDATE_B;
 import static com.persistit.util.SequencerConstants.PAGE_MAP_READ_INVALIDATE_C;
+import static com.persistit.util.SequencerConstants.PAGE_MAP_READ_INVALIDATE_SCHEDULE;
 import static com.persistit.util.ThreadSequencer.addSchedules;
 import static com.persistit.util.ThreadSequencer.disableSequencer;
 import static com.persistit.util.ThreadSequencer.enableSequencer;
-import static com.persistit.util.ThreadSequencer.PAGE_MAP_READ_INVALIDATE_SCHEDULE;
 import static com.persistit.util.ThreadSequencer.sequence;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -50,6 +50,7 @@
 import com.persistit.JournalManager.PageNode;
 import com.persistit.TransactionPlayer.TransactionPlayerListener;
 import com.persistit.exception.PersistitException;
+import com.persistit.unit.ConcurrentUtil.ThrowingRunnable;
 import com.persistit.unit.PersistitUnitTestCase;
 import com.persistit.unit.UnitTestProperties;
 import com.persistit.util.Util;
@@ -405,7 +406,7 @@
          * Remove from the left end
          */
         {
-            final List<PageNode> source =testCleanupPageListSource(10);
+            final List<PageNode> source = testCleanupPageListSource(10);
             for (int i = 0; i < 4; i++) {
                 source.get(i).invalidate();
             }
@@ -416,7 +417,7 @@
          * Remove from the right end
          */
         {
-            final List<PageNode> source =testCleanupPageListSource(10);
+            final List<PageNode> source = testCleanupPageListSource(10);
             for (int i = 10; --i >= 7;) {
                 source.get(i).invalidate();
             }
@@ -427,7 +428,7 @@
          * Remove from the middle
          */
         {
-            final List<PageNode> source =testCleanupPageListSource(10);
+            final List<PageNode> source = testCleanupPageListSource(10);
             for (int i = 2; i < 8; i++) {
                 source.get(i).invalidate();
             }
@@ -454,6 +455,88 @@
         }
     }
 
+    @Test
+    public void missingVolumePageHandling() throws Exception {
+        final Configuration config = _persistit.getConfiguration();
+        Volume volume1 = new Volume(config.volumeSpecification("${datapath}/missing1,create,"
+                + "pageSize:16384,initialPages:1,extensionPages:1,maximumPages:25000"));
+        volume1.open(_persistit);
+        Volume volume2 = new Volume(config.volumeSpecification("${datapath}/missing2,create,"
+                + "pageSize:16384,initialPages:1,extensionPages:1,maximumPages:25000"));
+        volume2.open(_persistit);
+        _persistit.getExchange(volume1, "test1", true);
+        _persistit.getExchange(volume2, "test2", true);
+        _persistit.close();
+        new File(volume1.getPath()).delete();
+        new File(volume2.getPath()).delete();
+        volume1 = null;
+        volume2 = null;
+        _persistit = new Persistit();
+        _persistit.initialize(config);
+        AlertMonitor am = _persistit.getAlertMonitor();
+        assertTrue("Startup with missing volumes should have generated alerts", am
+                .getHistory(AlertMonitor.MISSING_VOLUME_CATEGORY) != null);
+
+        _persistit.getJournalManager().setIgnoreMissingVolumes(true);
+        // Should add more alerts
+        _persistit.copyBackPages();
+        int alertCount1 = am.getHistory(AlertMonitor.MISSING_VOLUME_CATEGORY).getCount();
+        _persistit.copyBackPages();
+        int alertCount2 = am.getHistory(AlertMonitor.MISSING_VOLUME_CATEGORY).getCount();
+        assertEquals("No more alerts after setting ignoreMissingVolumes", alertCount1, alertCount2);
+    }
+
+    @Test
+    public void missingVolumeTransactionHandlingNotIgnored() throws Exception {
+        final Configuration config = _persistit.getConfiguration();
+        Volume volume = new Volume(config.volumeSpecification("${datapath}/missing1,create,"
+                + "pageSize:16384,initialPages:1,extensionPages:1,maximumPages:25000"));
+        volume.open(_persistit);
+        Exchange ex = _persistit.getExchange(volume, "test1", true);
+        final Transaction txn = ex.getTransaction();
+        txn.begin();
+        ex.getValue().put(RED_FOX);
+        ex.to(0).store();
+        txn.commit();
+        txn.end();
+        _persistit.getJournalManager().flush();
+        _persistit.crash();
+        new File(volume.getPath()).delete();
+        volume = null;
+        _persistit = new Persistit();
+        _persistit.initialize(config);
+        assertTrue("Should have failed updates during recovery", _persistit.getRecoveryManager().getPlayer()
+                .getFailedUpdates() > 0);
+    }
+    
+    @Test
+    public void missingVolumeTransactionHandlingIgnored() throws Exception {
+        final Configuration config = _persistit.getConfiguration();
+        Volume volume = new Volume(config.volumeSpecification("${datapath}/missing1,create,"
+                + "pageSize:16384,initialPages:1,extensionPages:1,maximumPages:25000"));
+        volume.open(_persistit);
+        Exchange ex = _persistit.getExchange(volume, "test1", true);
+        final Transaction txn = ex.getTransaction();
+        txn.begin();
+        ex.getValue().put(RED_FOX);
+        ex.to(0).store();
+        txn.commit();
+        txn.end();
+        _persistit.getJournalManager().flush();
+        _persistit.crash();
+        new File(volume.getPath()).delete();
+        volume = null;
+        config.setIgnoreMissingVolumes(true);
+        _persistit = new Persistit();
+        _persistit.getJournalManager().setIgnoreMissingVolumes(true);
+        _persistit.initialize(config);
+        AlertMonitor am = _persistit.getAlertMonitor();
+        assertTrue("Startup with missing volumes should have generated alerts", am
+                .getHistory(AlertMonitor.MISSING_VOLUME_CATEGORY) != null);
+        assertTrue("Should have failed updates during recovery", _persistit.getRecoveryManager().getPlayer()
+                .getIgnoredUpdates() > 0);
+    }
+
     private List<PageNode> testCleanupPageListSource(final int size) {
         final List<PageNode> source = new ArrayList<PageNode>(size);
         for (int index = 0; index < 1000000; index++) {
@@ -461,8 +544,7 @@
         }
         return source;
     }
-    
-    
+
     private void testCleanupPageListHelper(final List<PageNode> source) throws Exception {
         final List<PageNode> cleaned = new ArrayList<PageNode>(source);
         for (Iterator<PageNode> iterator = cleaned.iterator(); iterator.hasNext();) {
@@ -512,8 +594,8 @@
         final int COUNT = 5000;
         final String TREE_NAME = "JournalManagerTest1";
         /*
-         * Test sequence points in the JOURNAL_COPIER path, don't
-         * want to hit unintentionally hit them.
+         * Test sequence points in the JOURNAL_COPIER path, don't want to hit
+         * unintentionally hit them.
          */
         _persistit.getJournalManager().setCopierInterval(50000);
         /*
@@ -522,7 +604,7 @@
         Transaction txn = _persistit.getTransaction();
         txn.begin();
         Exchange ex = _persistit.getExchange(_volumeName, TREE_NAME, true);
-        for(int i = 0; i < COUNT; ++i) {
+        for (int i = 0; i < COUNT; ++i) {
             ex.to(i);
             ex.getValue().put(RED_FOX);
             ex.store();
@@ -530,8 +612,8 @@
         txn.commit();
         txn.end();
         /*
-         * Thread will read over everything that is inserted, hopefully going
-         * to the journal for each required page.
+         * Thread will read over everything that is inserted, hopefully going to
+         * the journal for each required page.
          */
         Thread thread1 = createThread("READ_THREAD", new ThrowingRunnable() {
             @Override
@@ -541,7 +623,7 @@
                 Exchange ex = _persistit.getExchange(_volumeName, TREE_NAME, false);
                 ex.to(Key.BEFORE);
                 int count = 0;
-                while(ex.next(true)) {
+                while (ex.next(true)) {
                     ++count;
                 }
                 assertEquals("Traversed count", COUNT, count);
@@ -549,8 +631,8 @@
             }
         });
         /*
-         * Thread will copy pages out of the journal and into the volume, hopefully
-         * invalidating pageMap entries during the cleanupForCopy.
+         * Thread will copy pages out of the journal and into the volume,
+         * hopefully invalidating pageMap entries during the cleanupForCopy.
          */
         Thread thread2 = createThread("COPY_BACK_THREAD", new ThrowingRunnable() {
             @Override