Merge lp:~pbeaman/akiban-persistit/tree-builder3 into lp:akiban-persistit

[Peter Beaman]

More small tweaks to support multi-threaded use of a TreeBuilder. This branch makes VolumeStorageT2#allocNewPage synchronized, which is necessary for concurrent use of a temporary volume.

In addition, the InsertBigLoad stress test has been modified to run five current sort threads, loading a total of 50M random keys in about three minutes.

[Nathan Williams (nwilliams)]

As described.

review: Approve

[Akiban Build User (build-akiban)]

There were 2 failures during build/test:

* job system-tests-mtr failed at build number 3074: http://172.16.20.104:8080/job/system-tests-mtr/3074/

* view must-pass failed: system-tests-mtr is red

review: Needs Fixing

[Nathan Williams (nwilliams)]

Unrelated.

=== modified file 'src/main/java/com/persistit/TreeBuilder.java'
--- src/main/java/com/persistit/TreeBuilder.java	2013-01-03 17:13:07 +0000
+++ src/main/java/com/persistit/TreeBuilder.java	2013-01-04 00:17:20 +0000
@@ -162,7 +162,6 @@
         }
     };
 
-
     private final Comparator<Tree> _defaultTreeComparator = new Comparator<Tree>() {
         @Override
         public int compare(final Tree a, final Tree b) {
@@ -551,7 +550,7 @@
         for (final Volume volume : _sortVolumes) {
             try {
                 volume.close();
-            } catch (PersistitException e) {
+            } catch (final PersistitException e) {
                 if (exception == null) {
                     exception = e;
                 }
 
=== modified file 'src/main/java/com/persistit/VolumeStorageT2.java'
--- src/main/java/com/persistit/VolumeStorageT2.java	2012-12-31 01:47:18 +0000
+++ src/main/java/com/persistit/VolumeStorageT2.java	2013-01-04 00:17:20 +0000
@@ -342,7 +342,7 @@
     }
 
     @Override
-    long allocNewPage() throws PersistitException {
+    synchronized long allocNewPage() throws PersistitException {
         if (_nextAvailablePage >= _maxPages) {
             throw new VolumeFullException(_volume.getName());
         }
 
=== modified file 'src/test/java/com/persistit/stress/InsertBigLoad.java'
--- src/test/java/com/persistit/stress/InsertBigLoad.java	2012-12-31 20:58:07 +0000
+++ src/test/java/com/persistit/stress/InsertBigLoad.java	2013-01-04 00:17:20 +0000
@@ -18,7 +18,9 @@
 import com.persistit.Configuration;
 import com.persistit.Persistit;
 import com.persistit.Transaction.CommitPolicy;
+import com.persistit.TreeBuilder;
 import com.persistit.stress.unit.BigLoad;
+import com.persistit.stress.unit.BigLoad.BigLoadTreeBuilder;
 
 public class InsertBigLoad extends AbstractSuite {
 
@@ -39,14 +41,21 @@
 
         deleteFiles(substitute("$datapath$/persistit*"));
 
-        add(new BigLoad("records=10000000"));
-
         final Configuration config = makeConfiguration(16384, "50000", CommitPolicy.SOFT);
-        config.setTmpVolMaxSize(100000000000l);
+        config.setTmpVolMaxSize(10000000000l);
         final Persistit persistit = new Persistit(config);
+        final TreeBuilder tb = new BigLoadTreeBuilder(persistit);
+
+        add(new BigLoad(tb, "records=10000000"));
+        add(new BigLoad(tb, "records=10000000"));
+        add(new BigLoad(tb, "records=10000000"));
+        add(new BigLoad(tb, "records=10000000"));
+        add(new BigLoad(tb, "records=10000000"));
 
         try {
             execute(persistit);
+            tb.merge();
+
         } finally {
             persistit.close();
         }
 
=== modified file 'src/test/java/com/persistit/stress/unit/BigLoad.java'
--- src/test/java/com/persistit/stress/unit/BigLoad.java	2012-12-31 21:03:59 +0000
+++ src/test/java/com/persistit/stress/unit/BigLoad.java	2013-01-04 00:17:20 +0000
@@ -47,33 +47,19 @@
 public class BigLoad extends AbstractStressTest {
 
     private static final Random RANDOM = new Random();
+    private final TreeBuilder tb;
 
     private int totalRecords;
 
-    public BigLoad(final int totalRecords, final int buckets) {
+    public BigLoad(final TreeBuilder tb, final int totalRecords, final int buckets) {
         super("");
         this.totalRecords = totalRecords;
+        this.tb = tb;
     }
 
-    public void load(final Persistit db) throws Exception {
+    public long load(final Persistit db) throws Exception {
         final long startLoadTime = System.nanoTime();
-        final TreeBuilder tb = new TreeBuilder(db) {
-            @Override
-            protected void reportSorted(final long count) {
-                System.out.printf("Sorted %,15d records\n", count);
-            }
-
-            @Override
-            protected void reportMerged(final long count) {
-                System.out.printf("Merged %,15d records\n", count);
-            }
-
-            @Override
-            protected boolean duplicateKeyDetected(final Tree tree, final Key key, final Value v1, final Value v2) {
-                System.out.println("Duplicate  key detected: " + key);
-                return true;
-            }
-        };
+
         final Exchange resultExchange = db.getExchange("persistit", "sorted", true);
         System.out.printf("Loading %,d records\n", totalRecords);
 
@@ -84,14 +70,26 @@
         final long endLoadTime = System.nanoTime();
         System.out.printf("Loaded %,d records into %,d buckets in %,dms\n", totalRecords, tb.getSortVolumeCount(),
                 (endLoadTime - startLoadTime) / Util.NS_PER_MS);
+        return endLoadTime - startLoadTime;
+    }
+
+    public long merge(final Persistit db) throws Exception {
+        final long startMergeTime = System.nanoTime();
 
         System.out.printf("Merging %,d records into main database\n", totalRecords);
 
         tb.merge();
         final long endMergeTime = System.nanoTime();
-        System.out.printf("Merged %,d records in %,dms\n", totalRecords, (endMergeTime - endLoadTime) / Util.NS_PER_MS);
-
+        System.out.printf("Merged %,d records in %,dms\n", totalRecords, (endMergeTime - startMergeTime)
+                / Util.NS_PER_MS);
+
+        return endMergeTime - startMergeTime;
+    }
+
+    public long count(final Persistit db) throws Exception {
+        final long startCountTime = System.nanoTime();
         System.out.printf("Counting keys in main database (100M keys per dot) ");
+        final Exchange resultExchange = db.getExchange("persistit", "sorted", false);
         resultExchange.clear().append(Key.BEFORE);
         long count = 0;
         while (resultExchange.next()) {
@@ -102,10 +100,9 @@
             }
         }
         final long endCountTime = System.nanoTime();
-        System.out.printf("\nCounted %,d keys in the main database in %,dms\n", count, (endCountTime - endMergeTime)
+        System.out.printf("\nCounted %,d keys in the main database in %,dms\n", count, (endCountTime - startCountTime)
                 / Util.NS_PER_MS);
-        System.out.printf("Total time to load, merge and count %,d records is %,dms", totalRecords,
-                (endCountTime - startLoadTime) / Util.NS_PER_MS);
+        return endCountTime - startCountTime;
     }
 
     final StringBuilder sb = new StringBuilder(
@@ -136,7 +133,6 @@
     public static void main(final String[] args) throws Exception {
         final int records = args.length > 0 ? Integer.parseInt(args[0]) : 1000000;
         final int buckets = args.length > 1 ? Integer.parseInt(args[1]) : 100;
-        final BigLoad bl = new BigLoad(records, buckets);
         final Persistit db = new Persistit();
         if (args.length > 2) {
             db.setPropertiesFromFile(args[2]);
@@ -144,13 +140,39 @@
         } else {
             db.initialize();
         }
+        final BigLoad bl = new BigLoad(new BigLoadTreeBuilder(db), records, buckets);
         try {
             bl.load(db);
+            bl.merge(db);
+            bl.clone();
         } finally {
             db.close();
         }
     }
 
+    public static class BigLoadTreeBuilder extends TreeBuilder {
+
+        public BigLoadTreeBuilder(final Persistit db) {
+            super(db);
+        }
+
+        @Override
+        protected void reportSorted(final long count) {
+            System.out.printf("Sorted %,15d records\n", count);
+        }
+
+        @Override
+        protected void reportMerged(final long count) {
+            System.out.printf("Merged %,15d records\n", count);
+        }
+
+        @Override
+        protected boolean duplicateKeyDetected(final Tree tree, final Key key, final Value v1, final Value v2) {
+            System.out.println("Duplicate  key detected: " + key);
+            return true;
+        }
+    }
+
     /*
      * ----------------------------------------------------------------------
      * 
@@ -159,8 +181,9 @@
      * ----------------------------------------------------------------------
      */
 
-    public BigLoad(final String argsString) {
+    public BigLoad(final TreeBuilder tb, final String argsString) {
         super(argsString);
+        this.tb = tb;
     }
 
     private final static String[] ARGS_TEMPLATE = { "records|int:1000000:1:1000000000|Total records to create",