Merge lp:~pbeaman/akiban-persistit/parameterize-tests into lp:akiban-persistit

[Peter Beaman]

This branch makes it easier to global modify configuration parameters for stress tests so that we can easily scale the the buffer pool size, checkpoint interval and commit policy for different hardware environments. Adds a Configuration property to set the default checkpoint interval. Adds logic to AbstractSuite to scale buffer pool configuration up or down, set checkpoint interval and default commit policy.

This branch is in preparation for a Matrix Jenkins test configuration. In the future we may want to do something even more general, but for now these settings are sufficient.

[Nathan Williams (nwilliams)]

As described.

review: Approve

=== modified file 'src/main/java/com/persistit/CheckpointManager.java'
--- src/main/java/com/persistit/CheckpointManager.java	2012-10-11 14:38:30 +0000
+++ src/main/java/com/persistit/CheckpointManager.java	2012-10-23 19:13:24 +0000
@@ -93,17 +93,11 @@
     /**
      * Default interval in nanoseconds between checkpoints - two minutes.
      */
-    private final static long DEFAULT_CHECKPOINT_INTERVAL = 120;
-
-    private final static long MINIMUM_CHECKPOINT_INTERVAL = 10;
-
-    private final static long MAXIMUM_CHECKPOINT_INTERVAL = 1800;
-
     private final static Checkpoint UNAVALABLE_CHECKPOINT = new Checkpoint(0, 0);
 
     private final SessionId _checkpointTxnSessionId = new SessionId();
 
-    private volatile long _checkpointIntervalNanos = DEFAULT_CHECKPOINT_INTERVAL * NS_PER_S;
+    private volatile long _checkpointIntervalNanos;
 
     private volatile long _lastCheckpointNanos = Long.MAX_VALUE;
 
@@ -125,6 +119,7 @@
 
     public void start() {
         _closed.set(false);
+        _checkpointIntervalNanos = _persistit.getConfiguration().getCheckpointInterval() * NS_PER_S;
         start("CHECKPOINT_WRITER", FLUSH_CHECKPOINT_INTERVAL);
     }
 
@@ -161,7 +156,7 @@
 
     @Override
     public void setCheckpointInterval(final long interval) {
-        Util.rangeCheck(interval, MINIMUM_CHECKPOINT_INTERVAL, MAXIMUM_CHECKPOINT_INTERVAL);
+        Util.rangeCheck(interval, MINIMUM_CHECKPOINT_INTERVAL_S, MAXIMUM_CHECKPOINT_INTERVAL_S);
         _checkpointIntervalNanos = interval * NS_PER_S;
     }
 

=== modified file 'src/main/java/com/persistit/Configuration.java'
--- src/main/java/com/persistit/Configuration.java	2012-10-15 21:22:32 +0000
+++ src/main/java/com/persistit/Configuration.java	2012-10-23 19:13:24 +0000
@@ -15,6 +15,10 @@
 
 package com.persistit;
 
+import static com.persistit.mxbeans.CheckpointManagerMXBean.DEFAULT_CHECKPOINT_INTERVAL_S;
+import static com.persistit.mxbeans.CheckpointManagerMXBean.MAXIMUM_CHECKPOINT_INTERVAL_S;
+import static com.persistit.mxbeans.CheckpointManagerMXBean.MINIMUM_CHECKPOINT_INTERVAL_S;
+
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
@@ -39,6 +43,7 @@
 import com.persistit.exception.PersistitIOException;
 import com.persistit.exception.PropertiesNotFoundException;
 import com.persistit.logging.DefaultPersistitLogger;
+import com.persistit.mxbeans.CheckpointManagerMXBean;
 import com.persistit.policy.JoinPolicy;
 import com.persistit.policy.SplitPolicy;
 import com.persistit.util.Util;
@@ -168,6 +173,11 @@
     public final static String SYSTEM_VOLUME_PROPERTY_NAME = "sysvolume";
 
     /**
+     * Property name for checkpoint interval in seconds
+     */
+    public final static String CHECKPOINT_INTERVAL_PROPERTY_NAME = "checkpointinterval";
+
+    /**
      * Property name for specifying default temporary volume page size
      */
     public final static String TEMPORARY_VOLUME_PAGE_SIZE_PROPERTY_NAME = "tmpvolpagesize";
@@ -628,6 +638,7 @@
     private final List<VolumeSpecification> volumeSpecifications = new ArrayList<VolumeSpecification>();
     private String journalPath = DEFAULT_JOURNAL_PATH;
     private long journalSize = JournalManager.DEFAULT_BLOCK_SIZE;
+    private long checkpointInterval = DEFAULT_CHECKPOINT_INTERVAL_S;
     private String sysVolume = DEFAULT_SYSTEM_VOLUME_NAME;
     private CommitPolicy commitPolicy = DEFAULT_TRANSACTION_COMMIT_POLICY;
     private JoinPolicy joinPolicy = DEFAULT_JOIN_POLICY;
@@ -1045,7 +1056,7 @@
      * @throws IllegalArgumentException
      *             if the supplied String is not a valid integer representation.
      */
-    static long parseLongProperty(final String propName, final String str) {
+    public static long parseLongProperty(final String propName, final String str) {
         if (str != null) {
             try {
                 long multiplier = 1;
@@ -1097,7 +1108,7 @@
      *             if the supplied String is not a valid floating point
      *             representation, or is outside the supplied bounds.
      */
-    static float parseFloatProperty(final String propName, final String str) {
+    public static float parseFloatProperty(final String propName, final String str) {
         if (str != null) {
             try {
                 return Float.parseFloat(str);
@@ -1117,7 +1128,7 @@
      * @param str
      * @return the boolean value
      */
-    static boolean parseBooleanValue(final String propName, final String str) {
+    public static boolean parseBooleanValue(final String propName, final String str) {
         if ("true".equalsIgnoreCase(str))
             return true;
         if ("false".equalsIgnoreCase(str))
@@ -1290,6 +1301,32 @@
     }
 
     /**
+     * Return the value defined by {@link #setCheckpointInterval(long)}
+     * 
+     * @return the checkpoint interval, in seconds
+     */
+    public long getCheckpointInterval() {
+        return checkpointInterval;
+    }
+
+    /**
+     * <p>
+     * Set the checkpoint interval, in seconds. This setting controls the
+     * elapsed time between attempts to write a checkpoint to the journal. A
+     * longer interval allows more updates to accumulate in buffers before they
+     * are required to be written to disk, but also potentially causes recovery
+     * from an abrupt termination (crash) to take more time.
+     * </p>
+     * Default size is
+     * {@value CheckpointManagerMXBean#DEFAULT_CHECKPOINT_INTERVAL_S} <br/>
+     * Property name is {@value #CHECKPOINT_INTERVAL_PROPERTY_NAME}
+     */
+    public void setCheckpointInterval(final long checkpointInterval) {
+        this.checkpointInterval = Util.rangeCheck(checkpointInterval, MINIMUM_CHECKPOINT_INTERVAL_S,
+                MAXIMUM_CHECKPOINT_INTERVAL_S);
+    }
+
+    /**
      * Return the value defined by {@link #setSysVolume}
      * 
      * @return the system volume name

=== modified file 'src/main/java/com/persistit/mxbeans/CheckpointManagerMXBean.java'
--- src/main/java/com/persistit/mxbeans/CheckpointManagerMXBean.java	2012-08-02 04:45:28 +0000
+++ src/main/java/com/persistit/mxbeans/CheckpointManagerMXBean.java	2012-10-23 19:13:24 +0000
@@ -24,6 +24,12 @@
 @MXBean
 public interface CheckpointManagerMXBean {
 
+    public final static long DEFAULT_CHECKPOINT_INTERVAL_S = 120;
+
+    public final static long MINIMUM_CHECKPOINT_INTERVAL_S = 10;
+
+    public final static long MAXIMUM_CHECKPOINT_INTERVAL_S = 3600;
+
     public final static String MXBEAN_NAME = "com.persistit:type=Persistit,class=CheckpointManager";
 
     @Description("Checkpoint most recently proposed")

=== modified file 'src/test/java/com/persistit/stress/AbstractSuite.java'
--- src/test/java/com/persistit/stress/AbstractSuite.java	2012-10-04 20:40:40 +0000
+++ src/test/java/com/persistit/stress/AbstractSuite.java	2012-10-23 19:13:24 +0000
@@ -42,7 +42,9 @@
 
     private final static String[] ARGS_TEMPLATE = { "duration|int::10|Maximum duration in seconds",
             "datapath|String:/tmp/persistit_test_data|Data path",
-            "progress|int:60:1:|Progress message interval in seconds", "_flag|S|Save on failure", };
+            "progress|int:60:1:|Progress message interval in seconds", "_flag|S|Save on failure",
+            "checkpointinterval|int:120:10:3600|Checkpoint interval in seconds",
+            "commitpolicy|String:|Default commit policy", "memoryadjustment|String:|Memory adjustment" };
 
     protected final static long PROGRESS_LOG_INTERVAL = 600000;
 
@@ -56,6 +58,9 @@
     final private String _dataPath;
     final private long _progressLogInterval;
     final private boolean _saveOnFailure;
+    private final int _checkpointInterval;
+    private final String _commitPolicyOverride;
+    private final String _memoryAdjustment;
 
     private long _duration;
     private boolean _untilStopped;
@@ -76,6 +81,9 @@
         _progressLogInterval = ap.getLongValue("progress");
         _untilStopped = ap.isSpecified("duration");
         _saveOnFailure = ap.isFlag('S');
+        _checkpointInterval = ap.isSpecified("checkpointinterval") ? ap.getIntValue("checkpointinterval") : 0;
+        _commitPolicyOverride = ap.getStringValue("commitpolicy");
+        _memoryAdjustment = ap.getStringValue("memoryadjustment");
     }
 
     public String getName() {
@@ -292,7 +300,7 @@
 
     protected Configuration makeConfiguration(final int pageSize, final String mem, final CommitPolicy policy) {
         final Configuration c = new Configuration();
-        c.setCommitPolicy(policy);
+        c.setCommitPolicy(overrideCommitPolicy(policy, _commitPolicyOverride));
         c.setJmxEnabled(true);
         c.setJournalPath(substitute("$datapath$/persistit_journal"));
         c.setLogFile(substitute("$datapath$/persistit_$timestamp$.log"));
@@ -306,6 +314,46 @@
         c.getVolumeList().add(
                 new VolumeSpecification(substitute("$datapath$/persistit,create,pageSize:" + pageSize
                         + ",initialSize:100M,extensionSize:100M,maximumSize:500G")));
+        if (_checkpointInterval > 0) {
+            c.setCheckpointInterval(_checkpointInterval);
+        }
+        adjustMemory(bpc);
         return c;
     }
+
+    private CommitPolicy overrideCommitPolicy(final CommitPolicy policy, final String override) {
+        return override != null && !override.isEmpty() ? CommitPolicy.valueOf(override) : policy;
+    }
+
+    private void adjustMemory(final BufferPoolConfiguration bpc) {
+        final int minc = bpc.getMinimumCount();
+        final int maxc = bpc.getMaximumCount();
+        final long mins = bpc.getMinimumMemory();
+        final long maxs = bpc.getMaximumMemory();
+        if (_memoryAdjustment == null || _memoryAdjustment.isEmpty()) {
+            return;
+        }
+        if (_memoryAdjustment.startsWith("x")) {
+            final float fraction = Float.parseFloat(_memoryAdjustment.substring(1));
+            if (minc > 0 && maxc < Integer.MAX_VALUE) {
+                bpc.setMinimumCount((int) (minc * fraction));
+                bpc.setMaximumCount((int) (maxc * fraction));
+            } else {
+                if (mins > 0 && maxs < Long.MAX_VALUE) {
+                    bpc.setMinimumMemory((long) (mins * fraction));
+                    bpc.setMaximumMemory((long) (maxs * fraction));
+                }
+            }
+        } else if (_memoryAdjustment.startsWith("c")) {
+            final int count = Integer.parseInt(_memoryAdjustment.substring(1));
+            bpc.setMinimumCount(count);
+            bpc.setMaximumCount(count);
+        } else {
+            final long size = Configuration.parseLongProperty("MemoryAdjustment", _memoryAdjustment);
+            bpc.setMinimumCount(0);
+            bpc.setMaximumCount(Integer.MAX_VALUE);
+            bpc.setMinimumMemory(0);
+            bpc.setMaximumMemory(size);
+        }
+    }
 }