Merge lp:~nwilliams/akiban-server/full-text-background-rewrite into lp:~akiban-technologies/akiban-server/trunk

[Nathan Williams]

Rewrite, and simplify, full text background worker processes.

As shown by our long string of intermittent failures, the background Timer tasks in FullTextIndexServiceImpl weren't stable. In particular, it was possible for multiple instances to run at a single time (e.g. see associated bug), for a test wait to not actually wait for work to complete, and lack of control on when task start and stop. Part of the trouble was using Timer/Task and a larger part was just incorrect code.

Instead, replace with a simple BackgroundRunner helper that has well defined, and checked, state transitions with explicit wait/notify support. Important changes including ensuring that (at most) one runner of each type will be active, a specific sleep object for trivial 'kicking', and a proper waiter method to ensure a complete cycle is observed.

While in the area, remove the more complicated than necessary BackgroundWork/Observer/WaitFunction classes and replace the SQL wait function with a system routine.

To help convince myself of the correctness, this branch 'mvn verify' in a loop on sleepy for most of the weekend. Zero errors were observed in 320 individual runs.

[Thomas Jones-Low (tjoneslo)]

Looks much better. Any time we end up with less code for a bug fix than when we started is a bonus.

review: Approve

=== removed file 'src/main/java/com/akiban/server/service/BackgroundObserver.java'
--- src/main/java/com/akiban/server/service/BackgroundObserver.java	2013-07-02 19:16:02 +0000
+++ src/main/java/com/akiban/server/service/BackgroundObserver.java	1970-01-01 00:00:00 +0000
@@ -1,24 +0,0 @@
-/**
- * Copyright (C) 2009-2013 Akiban Technologies, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package com.akiban.server.service;
-
-public interface BackgroundObserver
-{
-    public void update(BackgroundWork event);
-    public boolean backgroundFinished();
-}

=== removed file 'src/main/java/com/akiban/server/service/BackgroundObserverImpl.java'
--- src/main/java/com/akiban/server/service/BackgroundObserverImpl.java	2013-04-17 16:23:40 +0000
+++ src/main/java/com/akiban/server/service/BackgroundObserverImpl.java	1970-01-01 00:00:00 +0000
@@ -1,44 +0,0 @@
-/**
- * Copyright (C) 2009-2013 Akiban Technologies, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package com.akiban.server.service;
-
-
-/**
- * 
- * Set the hasFinished flag to true when the background has done
- */
-public class BackgroundObserverImpl implements BackgroundObserver
-{
-    private  volatile boolean hasFinished = false;
-    public BackgroundObserverImpl ()
-    {
-    }
-
-    @Override
-    public boolean backgroundFinished()
-    {
-        return hasFinished;
-    }
-
-    @Override
-    public void update(BackgroundWork event)
-    {
-         hasFinished = true;
-    }
-
-}

=== removed file 'src/main/java/com/akiban/server/service/BackgroundWork.java'
--- src/main/java/com/akiban/server/service/BackgroundWork.java	2013-07-02 19:16:02 +0000
+++ src/main/java/com/akiban/server/service/BackgroundWork.java	1970-01-01 00:00:00 +0000
@@ -1,46 +0,0 @@
-/**
- * Copyright (C) 2009-2013 Akiban Technologies, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package com.akiban.server.service;
-
-import java.util.Collection;
-
-
-public interface BackgroundWork
-{
-    public void addObserver(BackgroundObserver observer);
-
-    public void removeObservers(Collection<BackgroundObserver> os);
-    
-    public void notifyObservers();
-
-    /**
-     * @return the minimum time interval that one has to wait to be guaranteed that
-     * the work has been executed at least once.
-     */
-    public abstract long getMinimumWaitTime();
-    
-    /**
-     * Force the work to be executed immediately if it is not already running.
-     * 
-     * (This is only a request, not a command. The handler might choose not 
-     *  to carry it out, if that would cause conflicts)
-     * 
-     * @return  true if the request is executed, false otherwise
-     */
-    public boolean forceExecution();
-}

=== removed file 'src/main/java/com/akiban/server/service/BackgroundWorkBase.java'
--- src/main/java/com/akiban/server/service/BackgroundWorkBase.java	2013-07-02 19:16:02 +0000
+++ src/main/java/com/akiban/server/service/BackgroundWorkBase.java	1970-01-01 00:00:00 +0000
@@ -1,51 +0,0 @@
-/**
- * Copyright (C) 2009-2013 Akiban Technologies, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package com.akiban.server.service;
-
-import java.util.Collection;
-import java.util.concurrent.ConcurrentHashMap;
-
-public abstract class BackgroundWorkBase implements BackgroundWork
-{
-    private final ConcurrentHashMap<BackgroundObserver,Boolean> observers = new ConcurrentHashMap<>();
-    
-    public BackgroundWorkBase()
-    {
-    }
-
-    @Override
-    public void addObserver(BackgroundObserver observer)
-    {
-        // two observers are equal IFF they are the same object.
-        observers.put(observer, true);
-    }
-
-    public void removeObservers(Collection<BackgroundObserver> os)
-    {
-        for (BackgroundObserver key : os) {
-            observers.remove(key);
-        }
-    }
-
-    @Override
-    public void notifyObservers()
-    {
-        for (BackgroundObserver o : observers.keySet())
-            o.update(this);
-    }
-}

=== modified file 'src/main/java/com/akiban/server/service/text/FullTextIndexService.java'
--- src/main/java/com/akiban/server/service/text/FullTextIndexService.java	2013-07-10 21:42:55 +0000
+++ src/main/java/com/akiban/server/service/text/FullTextIndexService.java	2013-07-29 13:55:36 +0000
@@ -20,18 +20,13 @@
 import com.akiban.ais.model.IndexName;
 import com.akiban.qp.operator.RowCursor;
 import com.akiban.qp.operator.QueryContext;
-import com.akiban.server.service.BackgroundWork;
 
-import java.util.List;
 import org.apache.lucene.search.Query;
 
 /** Full service that does index maintenance and querying. */
 public interface FullTextIndexService extends FullTextIndexInfos {
-    /**
-     * @return An array of available background works
-     */
-    public List<? extends BackgroundWork> getBackgroundWorks();
+    public RowCursor searchIndex(QueryContext context, IndexName name, Query query, int limit);
 
-    public RowCursor searchIndex(QueryContext context, IndexName name, 
-                                 Query query, int limit);
+    /** Wait for a complete run of background workers */
+    public void backgroundWait();
 }

=== modified file 'src/main/java/com/akiban/server/service/text/FullTextIndexServiceImpl.java'
--- src/main/java/com/akiban/server/service/text/FullTextIndexServiceImpl.java	2013-07-25 00:11:16 +0000
+++ src/main/java/com/akiban/server/service/text/FullTextIndexServiceImpl.java	2013-07-29 13:55:36 +0000
@@ -22,6 +22,7 @@
 import com.akiban.ais.model.Index;
 import com.akiban.ais.model.Index.IndexType;
 import com.akiban.ais.model.IndexName;
+import com.akiban.ais.model.Routine;
 import com.akiban.ais.model.TableName;
 import com.akiban.ais.model.UserTable;
 import com.akiban.ais.model.aisb2.AISBBasedBuilder;
@@ -47,8 +48,6 @@
 import com.akiban.server.error.NoSuchRowException;
 import com.akiban.server.error.QueryCanceledException;
 import com.akiban.server.rowdata.RowData;
-import com.akiban.server.service.BackgroundWork;
-import com.akiban.server.service.BackgroundWorkBase;
 import com.akiban.server.service.Service;
 import com.akiban.server.service.config.ConfigurationService;
 import com.akiban.server.service.listener.ListenerService;
@@ -59,8 +58,9 @@
 import com.akiban.server.service.transaction.TransactionService;
 import com.akiban.server.store.SchemaManager;
 import com.akiban.server.store.Store;
-import com.akiban.server.types3.mcompat.mfuncs.WaitFunctionHelpers;
 
+import com.akiban.sql.server.ServerCallContextStack;
+import com.akiban.sql.server.ServerQueryContext;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.search.Query;
 
@@ -73,23 +73,21 @@
 import java.io.*;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.Iterator;
-import java.util.List;
 import java.util.NoSuchElementException;
-import java.util.Timer;
-import java.util.TimerTask;
 import java.util.concurrent.ConcurrentHashMap;
 
 
 public class FullTextIndexServiceImpl extends FullTextIndexInfosImpl implements FullTextIndexService, Service, TableListener, RowListener
 {
+    private static final Logger logger = LoggerFactory.getLogger(FullTextIndexServiceImpl.class);
+
     public static final String INDEX_PATH_PROPERTY = "akserver.text.indexpath";
-    public static final String UPDATE_INTERVAL = "akserver.text.maintenanceInterval";
-    public static final String POPULATE_DELAY_INTERVAL = "akserver.text.populateDelayInterval";
+    public static final String BACKGROUND_INTERVAL_PROPERTY = "akserver.text.backgroundInterval";
 
     private static final TableName POPULATE_TABLE = new TableName(TableName.INFORMATION_SCHEMA, "full_text_populate");
     private static final TableName CHANGES_TABLE = new TableName(TableName.INFORMATION_SCHEMA, "full_text_changes");
+    private static final TableName BACKGROUND_WAIT_PROC_NAME = new TableName(TableName.SYS_SCHEMA, "full_text_background_wait");
 
 
     private final ConfigurationService configService;
@@ -98,18 +96,15 @@
     private final SchemaManager schemaManager;
     private final Store store;
     private final TransactionService transactionService;
+    private final ConcurrentHashMap<IndexName,Session> populating;
+    private final Object BACKGROUND_CHANGE_LOCK = new Object();
 
+    private volatile IndexName updatingIndex;
+    private BackgroundRunner backgroundPopulate;
+    private BackgroundRunner backgroundUpdate;
+    private long backgroundInterval;
     private File indexPath;
 
-    private Timer maintenanceTimer;
-    private long maintenanceInterval;
-    private volatile TimerTask updateWorker;
-
-    private Timer populateTimer;
-    private long populateDelayInterval;
-    private ConcurrentHashMap<IndexName,Session> populating;
-    
-    private static final Logger logger = LoggerFactory.getLogger(FullTextIndexServiceImpl.class);
 
     @Inject
     public FullTextIndexServiceImpl(ConfigurationService configService,
@@ -141,12 +136,7 @@
         if (populatingSession != null) {
             // if the population process is running, cancel it
             populatingSession.cancelCurrentQuery(true);
-            // wait for the thread to complete 
-            try {
-                WaitFunctionHelpers.waitOn(Collections.singleton(POPULATE_BACKGROUND));
-            } catch (InterruptedException e) {
-                logger.error("waitOn populate during dropIndex failed", e);
-            }
+            waitPopulateCycle();
         } else {
             // delete 'promise' for population, if any
             try {
@@ -156,18 +146,12 @@
             }
         }
         
-        // This deals with the update thread. If the update
-        // is running, wait for it to complete. Since the update
-        // process mixes different indexes in the same update, there's
-        // no good way to tell if our index is being updated. 
-        if (updateRunning) {
-            try {
-                WaitFunctionHelpers.waitOn(Collections.singleton(UPDATE_BACKGROUND));
-            } catch (InterruptedException e) {
-                logger.error("waitOn update during dropIndex failed", e);
-            }
+        // If updatingIndex is currently equal, wait for it to change.
+        // If it isn't, or once it does, it can never become equal as it now exists in the populating map.
+        if(idx.getIndexName().equals(updatingIndex)) {
+            waitUpdateCycle();
         }
-        
+
         // delete documents
         FullTextIndexInfo idxInfo = getIndex(session, idx.getIndexName(), idx.getIndexedTable().getAIS());
         idxInfo.deletePath();
@@ -190,36 +174,54 @@
     }
 
     @Override
-    public List<? extends BackgroundWork> getBackgroundWorks()
-    {
-        return Arrays.asList(POPULATE_BACKGROUND, UPDATE_BACKGROUND);
+    public void backgroundWait() {
+        waitPopulateCycle();
+        waitUpdateCycle();
     }
-    
-    /* Service */
+
+
+    //
+    // Service
+    //
     
     @Override
     public void start() {
+        indexPath = new File(configService.getProperty(INDEX_PATH_PROPERTY));
+        boolean success = indexPath.mkdirs();
+        if(!success && !indexPath.exists()) {
+            throw new AkibanInternalException("Could not create indexPath directories: " + indexPath);
+        }
+
         registerSystemTables();
+        listenerService.registerTableListener(this);
+        listenerService.registerRowListener(this);
+
+        backgroundInterval =  Long.parseLong(configService.getProperty(BACKGROUND_INTERVAL_PROPERTY));
         enableUpdateWorker();
         enablePopulateWorker();
-        listenerService.registerTableListener(this);
-        listenerService.registerRowListener(this);
     }
 
     @Override
     public void stop() {
+        disableUpdateWorker();
+        disablePopulateWorker();
+
         listenerService.deregisterTableListener(this);
         listenerService.deregisterRowListener(this);
+
         try {
             for (FullTextIndexShared index : indexes.values()) {
                 index.close();
             }
-            disableUpdateWorker();
-            disablePopulateWorker();
         }
         catch (IOException ex) {
             throw new AkibanInternalException("Error closing index", ex);
         }
+
+        populating.clear();
+
+        backgroundInterval = 0;
+        indexPath = null;
     }
 
     @Override
@@ -230,14 +232,7 @@
     /* FullTextIndexInfosImpl */
 
     @Override
-    protected synchronized File getIndexPath() {
-        if (indexPath == null) {
-            indexPath = new File(configService.getProperty(INDEX_PATH_PROPERTY));
-            boolean success = indexPath.mkdirs();
-            if (!success && !indexPath.exists()) {
-                throw new AkibanInternalException("Could not create indexPath directories: " + indexPath);
-            }
-        }
+    protected File getIndexPath() {
         return indexPath;
     }
 
@@ -389,116 +384,64 @@
     }
 
 
-    private volatile boolean updateRunning = false;
-    private class DefaultUpdateWorker extends TimerTask
-    {
-        @Override
-        public void run()
-        {
-            runUpdate();
-        }
-    }
-
-    final BackgroundWork POPULATE_BACKGROUND = new BackgroundWorkBase()
-    {
-        @Override
-        public boolean forceExecution()
-        {
-            return forcePopulate();
-        }
-
-        @Override
-        public long getMinimumWaitTime()
-        {
-            return populateEnabled && hasScheduled
-                    ? populateDelayInterval
-                    : 0;
-        }
-
-        @Override
-        public String toString()
-        {
-            return "POPULATE";
-        }
-    };
-
-    final BackgroundWork UPDATE_BACKGROUND = new BackgroundWorkBase()
-    {
-        @Override
-        public boolean forceExecution()
-        {
-            return forceUpdate();
-        }
-
-        @Override
-        public long getMinimumWaitTime()
-        {
-            return updateWorker == null
-                    ? 0 // worker is disabled.
-                    : maintenanceInterval;
-        }
-
-        @Override
-        public String toString()
-        {
-            return "UPDATE";
-        }
-    };
-
-    private volatile boolean populateRunning = false;
-    private class DefaultPopulateWorker extends TimerTask
-    {
-        @Override
-        public void run()
-        {
-            runPopulate();
-        }
-    }
-
     // ---------- mostly for testing ---------------
-    void disableUpdateWorker()
-    {
-        if (maintenanceTimer == null)
-        {
-            logger.debug("maintenance worker ALREADY disabled");
-            return;
-        }
-        updateWorker.cancel();
-        maintenanceTimer.cancel();
-        maintenanceTimer.purge();
-        maintenanceTimer = null;
-        updateWorker = null;
-    }
-    
-    protected void enableUpdateWorker()
-    {
-        maintenanceInterval = Long.parseLong(configService.getProperty(UPDATE_INTERVAL));
-        maintenanceTimer = new Timer();
-        updateWorker = new DefaultUpdateWorker();
-        maintenanceTimer.scheduleAtFixedRate(updateWorker, maintenanceInterval, maintenanceInterval);
-    }
-    
-    protected void enablePopulateWorker()
-    {
-        populateDelayInterval = Long.parseLong(configService.getProperty(POPULATE_DELAY_INTERVAL));
-        populateTimer = new Timer();
-        populateTimer.schedule(populateWorker(), populateDelayInterval);
-        populateEnabled = true;
-        hasScheduled = true;
-    }
-    
-    void disablePopulateWorker()
-    {
-        if (populateTimer == null)
-        {
-            logger.debug("populate worker already disabled");
-            return;
-        }
-        populateTimer.cancel();
-        populateTimer.purge();
-        hasScheduled = false;
-        populateEnabled = false;
-        populateTimer = null;
+
+    public void disableUpdateWorker() {
+        synchronized(BACKGROUND_CHANGE_LOCK) {
+            assert backgroundUpdate != null;
+            backgroundUpdate.toFinished();
+            backgroundUpdate = null;
+        }
+    }
+
+    public void enableUpdateWorker() {
+        synchronized(BACKGROUND_CHANGE_LOCK) {
+            assert backgroundUpdate == null;
+            backgroundUpdate = new BackgroundRunner("FullText_Update", backgroundInterval, new Runnable() {
+                @Override
+                public void run() {
+                    runUpdate();
+                }
+            });
+            backgroundUpdate.start();
+        }
+    }
+
+    public void disablePopulateWorker() {
+        synchronized(BACKGROUND_CHANGE_LOCK) {
+            assert backgroundPopulate != null;
+            backgroundPopulate.toFinished();
+            backgroundPopulate = null;
+        }
+    }
+
+    public void enablePopulateWorker() {
+        synchronized(BACKGROUND_CHANGE_LOCK) {
+            assert backgroundPopulate == null;
+            backgroundPopulate = new BackgroundRunner("FullText_Populate", backgroundInterval, new Runnable() {
+                @Override
+                public void run() {
+                    runPopulate();
+                }
+            });
+            backgroundPopulate.start();
+        }
+    }
+
+    public void waitPopulateCycle() {
+        synchronized(BACKGROUND_CHANGE_LOCK) {
+            if(backgroundPopulate != null) {
+                backgroundPopulate.waitForCycle();
+            }
+        }
+    }
+
+    public void waitUpdateCycle() {
+        synchronized(BACKGROUND_CHANGE_LOCK) {
+            if(backgroundUpdate != null) {
+                backgroundUpdate.waitForCycle();
+            }
+        }
     }
 
     Cursor populateTableCursor(Session session) {
@@ -510,12 +453,16 @@
         return API.cursor(plan, context, context.createBindings());
     }
 
-    protected boolean populateNextIndex(Session session) throws PersistitException
+    protected boolean populateNextIndex(Session session)
     {
         transactionService.beginTransaction(session);
         Cursor cursor = null;
         IndexName toPopulate = null;
         try {
+            // Quick exit if we wont' see any this transaction
+            if(populateRowCount(session) == 0) {
+                return false;
+            }
             cursor = populateTableCursor(session);
             cursor.open();
             toPopulate = nextInQueue(session, cursor, false);
@@ -523,56 +470,34 @@
                 FullTextIndexInfo index = getIndex(session, toPopulate, null);
                 populateIndex(session, index);
                 store.deleteRow(session, createPopulateRow(session, toPopulate), true, false);
-                transactionService.commitTransaction(session);
                 populating.remove(toPopulate);
+                transactionService.commitTransaction(session);
                 return true;
             }
         } catch (QueryCanceledException e2) {
             // The query could be canceled if the user drops the index  
-            // while this thread is populating the index
-            // The Lock Obtained failed exception occurs for the same reason.
-            //  we're trying to delete the index at the same time as the 
-            // populate is running, but with slightly different timing. 
-            // Clean up after ourselves. 
+            // while this thread is populating the index.
+            // Clean up after ourselves.
             if(cursor != null) {
                 cursor.close();
             }
             populating.remove(toPopulate);
-            transactionService.commitTransaction(session);
-            // start another thread to make sure we're not missing anything
-            populateTimer.schedule(populateWorker(), populateDelayInterval);
-            hasScheduled = true;
             logger.warn("populateNextIndex aborted : {}", e2.getMessage());
-        } catch (IOException ioex) {
-            throw new AkibanInternalException ("Failed to populate index ", ioex);
+        } catch(IOException e) {
+            throw new AkibanInternalException("Failed to populate index ", e);
         } finally {
             transactionService.rollbackTransactionIfOpen(session);
         }
         return false;
     }
     
-    protected synchronized void runPopulate()
-    {
-        populateRunning = true;
-        Session session = sessionService.createSession();
-        try
-        {
+    private void runPopulate() {
+        try(Session session = sessionService.createSession()) {
             boolean more = true;
             while(more) {
                 more = populateNextIndex(session);
             }
         }
-        catch (PersistitException ex1)
-        {
-            throw PersistitAdapter.wrapPersistitException(session, ex1);
-        }
-        finally
-        {
-            hasScheduled = false;
-            POPULATE_BACKGROUND.notifyObservers();
-            populateRunning = false;
-            session.close();
-        }
     }
     
     private boolean stillExists(Session session, IndexName indexName)
@@ -582,98 +507,44 @@
         return !(table == null ||  table.getFullTextIndex(indexName.getName()) == null);
     }
     
-    private synchronized void runUpdate()
+    private void runUpdate()
     {
-        updateRunning = true;
         Session session = sessionService.createSession();
         HKeyBytesStream rows = null;
-        try
-        {
+        try {
             // Consume and commit updates to each index in distinct blocks to keep r/w window small-ish
             boolean done = false;
             while(!done) {
                 transactionService.beginTransaction(session);
-                rows = getChangedRows(session);
-                if(rows.hasStream()) {
-                    if(populating.get(rows.getIndexName()) == null) {
-                        updateIndex(session, rows.getIndexName(), rows);
+                // Quick exit if we won't see any
+                if(changesRowCount(session) == 0) {
+                    done = true;
+                } else {
+                    rows = getChangedRows(session);
+                    if(rows.hasStream()) {
+                        IndexName name = rows.getIndexName();
+                        if(populating.get(name) == null) {
+                            updatingIndex = name;
+                            updateIndex(session, name, rows);
+                        }
+                        rows.cursor.close();
+                        rows = null;
+                    } else {
+                        done = true;
                     }
-                } else {
-                    done = true;
                 }
                 transactionService.commitTransaction(session);
             }
-
-        }
-        catch(Exception e)
-        {
-            logger.error("Error while maintaining full_text indices", e);
-        }
-        finally
-        {
+        } finally {
+            updatingIndex = null;
             if(rows != null && rows.cursor != null) {
                 rows.cursor.close();
             }
             transactionService.rollbackTransactionIfOpen(session);
             session.close();
-            UPDATE_BACKGROUND.notifyObservers();
-            updateRunning = false;
         }
     }
 
-    private boolean forceUpdate()
-    {
-        // worker is disabled or is already running
-        if (updateWorker == null || updateRunning)
-            return false;
-
-        // execute the thread in different 
-        // thread (so it has a new session)
-        new Thread(updateWorker).start();
-        return true;
-    }
-    
-    
-    /**
-     * If the populate job is not already running, force the worker to wake
-     * up and do its job immediately.
-     * 
-     * If the worker is disabled, return false and do nothing.
-     */
-    private boolean forcePopulate()
-    {
-        // no work to do
-        // or it is already being done
-        if (!populateEnabled || !hasScheduled || populateRunning)
-            return false;
-
-        // block the timer (so other threads would have to wait)
-        // Unlike the update case, this is needed because population does not
-        // have to be done  periodically
-        // So unless there are new index created, we don't need to
-        // execute the task again after this execution
-
-        // cancel scheduled task
-        populateTimer.cancel();
-
-        // execute the task
-        // in a different thread
-        // because we'd otherwise get "transaction already began" exception
-        //  as each thread only has one session)
-        new Thread(populateWorker()).start();
-        hasScheduled = true;
-        // get a new timer
-        // (So schedulePopulate can schedule new task if new index is created)
-        populateTimer = new Timer();
-       
-        return true;
-    }
-
-    private synchronized TimerTask populateWorker ()
-    {
-        return new DefaultPopulateWorker();
-    }
-
     IndexName nextInQueue(Session session, Cursor cursor, boolean traversing) {
         Row row;
         while((row = cursor.next()) != null) {
@@ -694,9 +565,6 @@
         return null;
     }
 
-    private volatile boolean hasScheduled = false;
-    private volatile boolean populateEnabled = false;
-
     private HKeyRow toHKeyRow(byte rowBytes[], HKeyRowType hKeyRowType,
                               StoreAdapter store)
     {
@@ -834,13 +702,6 @@
 
     private void trackPopulate(Session session, IndexName indexName) {
         store.writeRow(session, createPopulateRow(session, indexName), null);
-
-        // TODO: What if this fires before this row is committed?
-        // if there are no scheduled populate workers running, add one to run shortly.
-        if(populateEnabled && !hasScheduled) {
-            populateTimer.schedule(populateWorker(), populateDelayInterval);
-            hasScheduled = true;
-        }
     }
 
     private void trackChange(Session session, UserTable table, Key hKey) {
@@ -860,6 +721,14 @@
         }
     }
 
+    private long populateRowCount(Session session) {
+        return store.getAIS(session).getUserTable(POPULATE_TABLE).rowDef().getTableStatus().getRowCount(session);
+    }
+
+    private long changesRowCount(Session session) {
+        return store.getAIS(session).getUserTable(CHANGES_TABLE).rowDef().getTableStatus().getRowCount(session);
+    }
+
     private void registerSystemTables() {
         final int identMax = 128;
         final int tableVersion = 1;
@@ -877,8 +746,126 @@
                .colString("index_name", identMax, false)
                .colLong("index_id", false)
                .colVarBinary("hkey", 4096, false);
+        builder.procedure(BACKGROUND_WAIT_PROC_NAME)
+               .language("java", Routine.CallingConvention.JAVA)
+               .externalName(Routines.class.getName(), "backgroundWait");
         AkibanInformationSchema ais = builder.ais();
         schemaManager.registerStoredInformationSchemaTable(ais.getUserTable(POPULATE_TABLE), tableVersion);
         schemaManager.registerStoredInformationSchemaTable(ais.getUserTable(CHANGES_TABLE), tableVersion);
+        schemaManager.registerSystemRoutine(ais.getRoutine(BACKGROUND_WAIT_PROC_NAME));
+    }
+
+    @SuppressWarnings("unused") // Called reflectively
+    public static class Routines {
+        public static void backgroundWait() {
+            ServerQueryContext context = ServerCallContextStack.current().getContext();
+            FullTextIndexService ft = context.getServer().getServiceManager().getServiceByClass(FullTextIndexService.class);
+            ft.backgroundWait();
+        }
+    }
+
+
+    public enum STATE {
+        NOT_STARTED,
+        RUNNING,
+        STOPPING,
+        FINISHED
+    }
+
+    public class BackgroundRunner extends Thread {
+        private final Object SLEEP_MONITOR = new Object();
+        private final Runnable runnable;
+        private final long sleepMillis;
+        private volatile STATE state;
+        private long runCount;
+
+        public BackgroundRunner(String name, long sleepMillis, Runnable runnable) {
+            super(name);
+            this.runnable = runnable;
+            this.sleepMillis = sleepMillis;
+            this.state = STATE.NOT_STARTED;
+        }
+
+        @Override
+        public void run() {
+            state = STATE.RUNNING;
+            while(state != STATE.STOPPING) {
+                try {
+                    runInternal();
+                } catch(Exception e) {
+                    logger.error("{} run failed with exception", getName(), e);
+                }
+                sleep();
+            }
+        }
+
+        public synchronized void toFinished() {
+            checkRunning();
+            state = STATE.STOPPING;
+            sleepNotify();
+            waitWhile(STATE.STOPPING, STATE.FINISHED);
+            notifyAll();
+        }
+
+        public synchronized void waitForCycle() {
+            checkRunning();
+            // +2 ensures that we've observed one full run no matter where we initially started
+            long target = runCount + 2;
+            while(runCount < target) {
+                sleepNotify();
+                waitThenSet(null);
+            }
+        }
+
+        public void sleepNotify() {
+            synchronized(SLEEP_MONITOR) {
+                SLEEP_MONITOR.notify();
+            }
+        }
+
+        //
+        // Helpers
+        //
+
+        private void runInternal() {
+            runnable.run();
+            synchronized(this) {
+                ++runCount;
+                notifyAll();
+            }
+        }
+
+        private void checkRunning() {
+            if(state == STATE.STOPPING || state == STATE.FINISHED) {
+                throw new IllegalStateException("Not RUNNING: " + state);
+            }
+        }
+
+        private void waitWhile(STATE whileState, STATE newState) {
+            while(state == whileState) {
+                waitThenSet(newState);
+            }
+        }
+
+        private void waitThenSet(STATE newState) {
+            try {
+                wait();
+                if(newState != null) {
+                    state = newState;
+                }
+            } catch(InterruptedException e) {
+                // None
+            }
+        }
+
+        private void sleep() {
+            try {
+                synchronized(SLEEP_MONITOR) {
+                    SLEEP_MONITOR.wait(sleepMillis);
+                }
+            } catch(InterruptedException e) {
+                // None
+            }
+        }
     }
 }

=== removed file 'src/main/java/com/akiban/server/types3/mcompat/mfuncs/FulltextMaintenanceWait.java'
--- src/main/java/com/akiban/server/types3/mcompat/mfuncs/FulltextMaintenanceWait.java	2013-04-08 19:26:06 +0000
+++ src/main/java/com/akiban/server/types3/mcompat/mfuncs/FulltextMaintenanceWait.java	1970-01-01 00:00:00 +0000
@@ -1,86 +0,0 @@
-/**
- * Copyright (C) 2009-2013 Akiban Technologies, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package com.akiban.server.types3.mcompat.mfuncs;
-
-import com.akiban.qp.operator.QueryContext;
-import com.akiban.server.error.AkibanInternalException;
-import com.akiban.server.service.text.FullTextIndexService;
-import com.akiban.server.types3.LazyList;
-import com.akiban.server.types3.TExecutionContext;
-import com.akiban.server.types3.TOverloadResult;
-import com.akiban.server.types3.TScalar;
-import com.akiban.server.types3.mcompat.mtypes.MNumeric;
-import com.akiban.server.types3.pvalue.PValueSource;
-import com.akiban.server.types3.pvalue.PValueTarget;
-import com.akiban.server.types3.texpressions.TInputSetBuilder;
-import com.akiban.server.types3.texpressions.TScalarBase;
-
-
-public class FulltextMaintenanceWait extends TScalarBase
-{
-
-    public static final TScalar INSTANCE = new FulltextMaintenanceWait();
-    
-    private FulltextMaintenanceWait() {}
-
-    @Override
-    protected void buildInputSets(TInputSetBuilder builder)
-    {
-        // no argument
-    }
-
-    @Override
-    protected void doEvaluate(TExecutionContext context, LazyList<? extends PValueSource> inputs, PValueTarget output)
-    {
-        QueryContext qc = context.getQueryContext();
-        if (qc == null)
-        {
-            context.logError ("No querycontext for retreiving full-text service");
-            output.putInt32(-1);
-        }
-        else
-        {
-            FullTextIndexService service = qc.getServiceManager().getServiceByClass(FullTextIndexService.class);
-            try
-            {
-                WaitFunctionHelpers.waitOn(service.getBackgroundWorks());
-                output.putInt32(0);
-            }
-            catch (InterruptedException ex)
-            {
-                context.logError(ex.getMessage());
-                output.putInt32(-2);
-            }
-        }
-    }
-
-    @Override
-    public String displayName()
-    {
-        return "fulltext_maintenance_wait";
-    }
-
-    @Override
-    public TOverloadResult resultType()
-    {
-        // TODO: return value probably doesn't mean anything! to anyone not having
-        // access to the code.
-        // Maybe return a STRING/msg instead?
-        return TOverloadResult.fixed(MNumeric.INT);
-    }
-}

=== removed file 'src/main/java/com/akiban/server/types3/mcompat/mfuncs/WaitFunctionHelpers.java'
--- src/main/java/com/akiban/server/types3/mcompat/mfuncs/WaitFunctionHelpers.java	2013-07-02 21:57:02 +0000
+++ src/main/java/com/akiban/server/types3/mcompat/mfuncs/WaitFunctionHelpers.java	1970-01-01 00:00:00 +0000
@@ -1,79 +0,0 @@
-/**
- * Copyright (C) 2009-2013 Akiban Technologies, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- */
-
-package com.akiban.server.types3.mcompat.mfuncs;
-
-import com.akiban.server.service.BackgroundObserver;
-import com.akiban.server.service.BackgroundObserverImpl;
-import com.akiban.server.service.BackgroundWork;
-
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-
-
-public class WaitFunctionHelpers
-{
-    /**
-     * Blocks until all `works` have finished
-     * @param works
-     * @throws InterruptedException 
-     */
-    public static void waitOn(Collection<? extends BackgroundWork> works) throws InterruptedException
-    {
-        if (works != null)
-        {
-            List<BackgroundObserver> waiters = new LinkedList<>();
-            for (BackgroundWork wk : works)
-            {
-                // This work doesn't require wait time
-                if (wk.getMinimumWaitTime() <= 0)
-                    continue;
-
-                // add observer
-                BackgroundObserverImpl w = new BackgroundObserverImpl();
-                wk.addObserver(w);
-
-                // request the task to be executed
-                wk.forceExecution();
-                
-                waiters.add(w);
-            }
-
-            // busy-loop waiting for all the works to be done
-            boolean allAwaken;
-            while (true)
-            {
-                allAwaken = true;
-                for (BackgroundObserver w : waiters)
-                {
-                    allAwaken &= w.backgroundFinished();
-                }
-                
-                if (allAwaken)
-                    break;
-            }
-            
-            // clean up
-            for (BackgroundWork wk : works)
-            {
-                wk.removeObservers(waiters);
-            }
-        }
-    }
-    
-}

=== modified file 'src/main/resources/com/akiban/server/service/config/configuration-defaults.properties'
--- src/main/resources/com/akiban/server/service/config/configuration-defaults.properties	2013-07-12 01:35:28 +0000
+++ src/main/resources/com/akiban/server/service/config/configuration-defaults.properties	2013-07-29 13:55:36 +0000
@@ -34,8 +34,7 @@
 akserver.index_statistics.bucket_count=32
 akserver.restrict_user_schema=false
 akserver.text.indexpath=/tmp/aktext
-akserver.text.maintenanceInterval=3000
-akserver.text.populateDelayInterval=1000
+akserver.text.backgroundInterval=3000
 # EXPERIMENTAL
 akserver.indexRowPooling = false
 akserver.pt.osc.hook=disabled

=== modified file 'src/test/java/com/akiban/rest/RestServiceFilesIT.java'
--- src/test/java/com/akiban/rest/RestServiceFilesIT.java	2013-04-21 02:07:31 +0000
+++ src/test/java/com/akiban/rest/RestServiceFilesIT.java	2013-07-29 13:55:36 +0000
@@ -25,7 +25,6 @@
 import com.akiban.server.service.is.BasicInfoSchemaTablesServiceImpl;
 import com.akiban.server.service.servicemanager.GuicedServiceManager;
 import com.akiban.server.test.it.ITBase;
-import com.akiban.server.types3.mcompat.mfuncs.WaitFunctionHelpers;
 import com.akiban.sql.RegexFilenameFilter;
 import com.akiban.util.Strings;
 import com.fasterxml.jackson.core.JsonParseException;
@@ -204,28 +203,9 @@
             loadDataFile(SCHEMA_NAME, data);
         }
 
-        String postURI = dumpFileIfExists(new File(subDir, "schema.prepost"));
-        if (postURI != null) {
-            HttpExchange httpConn = openConnection(getRestURL(postURI.trim()), "POST");
-            postContents(httpConn, "[]".getBytes());
-            httpClient.send(httpConn);
-            fullyDisconnect(httpConn);
-        }
-        
-        // The file should  contain only the name of the wait function
-        // (Don't need to make a SELECT node here)
-        String waitFunc = dumpFileIfExists(new File(subDir, "background.wait"));
-        if (waitFunc != null)
-        {
-            switch(waitFunc.trim().toLowerCase())
-            {
-                case "fulltext_maintenance_wait":
-                    WaitFunctionHelpers.waitOn(ftService.getBackgroundWorks());
-                    break;
-
-                default:
-                    throw new UnsupportedOperationException("Unknown Wait Function: " + waitFunc);
-            }
+        String waitNeeded = dumpFileIfExists(new File(subDir, "full_text_background_wait"));
+        if(waitNeeded != null) {
+            ftService.backgroundWait();
         }
     }
     

=== modified file 'src/test/java/com/akiban/server/service/config/TestConfigService.java'
--- src/test/java/com/akiban/server/service/config/TestConfigService.java	2013-06-14 11:23:11 +0000
+++ src/test/java/com/akiban/server/service/config/TestConfigService.java	2013-07-29 13:55:36 +0000
@@ -59,8 +59,7 @@
     protected Map<String, String> loadProperties() {
         Map<String, String> ret = new HashMap<>(super.loadProperties());
         makeDataDirectory();
-        ret.put(FullTextIndexServiceImpl.UPDATE_INTERVAL, Long.toString(1000));
-        ret.put(FullTextIndexServiceImpl.POPULATE_DELAY_INTERVAL, Long.toString(1000));
+        ret.put(FullTextIndexServiceImpl.BACKGROUND_INTERVAL_PROPERTY, "1000");
         ret.put(DATA_PATH_KEY, dataDirectory.getAbsolutePath());
         ret.put(TEXT_INDEX_PATH_KEY, dataDirectory.getAbsolutePath());
         final int bufferSize = Integer.parseInt(ret.get(BUFFER_SIZE_KEY));

=== modified file 'src/test/java/com/akiban/server/service/text/FullTextIndexServiceBug1172013IT.java'
--- src/test/java/com/akiban/server/service/text/FullTextIndexServiceBug1172013IT.java	2013-07-13 17:24:25 +0000
+++ src/test/java/com/akiban/server/service/text/FullTextIndexServiceBug1172013IT.java	2013-07-29 13:55:36 +0000
@@ -21,47 +21,21 @@
 import java.util.Collections;
 
 import com.akiban.qp.operator.Cursor;
-import com.akiban.qp.operator.StoreAdapter;
 import com.akiban.server.service.transaction.TransactionService.CloseableTransaction;
 import org.junit.Before;
 import org.junit.Test;
 
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import com.akiban.ais.model.IndexName;
 import com.akiban.ais.model.TableName;
-import com.akiban.qp.operator.QueryContext;
-import com.akiban.qp.persistitadapter.PersistitAdapter;
-import com.akiban.qp.rowtype.Schema;
 import com.akiban.qp.util.SchemaCache;
 import com.akiban.server.error.DuplicateIndexException;
-import com.akiban.server.service.servicemanager.GuicedServiceManager;
 import com.akiban.server.service.session.Session;
-import com.akiban.server.test.it.ITBase;
-import com.akiban.server.types3.mcompat.mfuncs.WaitFunctionHelpers;
-import com.akiban.sql.embedded.EmbeddedJDBCService;
-import com.akiban.sql.embedded.EmbeddedJDBCServiceImpl;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
-public class FullTextIndexServiceBug1172013IT extends ITBase {
-    public static final String SCHEMA = "test";
-    protected FullTextIndexService fullText;
-    protected Schema schema;
-    protected StoreAdapter adapter;
-    protected QueryContext queryContext;
-    private int c;
-    private int o;
-    private int i;
-    private int a;
+public class FullTextIndexServiceBug1172013IT extends FullTextIndexServiceITBase {
     private static final Logger logger = LoggerFactory.getLogger(FullTextIndexServiceBug1172013IT.class);
     private final Object lock = new Object();
-    private FullTextIndexServiceImpl fullTextImpl = null;
-    @Override
-    protected GuicedServiceManager.BindingsConfigurationProvider serviceBindingsProvider() {
-        return super.serviceBindingsProvider()
-                .bindAndRequire(FullTextIndexService.class, FullTextIndexServiceImpl.class)
-                .bindAndRequire(EmbeddedJDBCService.class, EmbeddedJDBCServiceImpl.class);
-    }
 
     @Before
     public void createData() {
@@ -100,16 +74,9 @@
         writeRow(a, 301, 3, "MA");
         writeRow(a, 302, 3, "ME");
 
-        fullText = serviceManager().getServiceByClass(FullTextIndexService.class);
-
         schema = SchemaCache.globalSchema(ais());
         adapter = newStoreAdapter(schema);
         queryContext = queryContext(adapter);
-        
-        // This test is specifically for FullTextIndexServiceImpl.java
-        assertEquals(FullTextIndexServiceImpl.class, fullText.getClass());
-        fullTextImpl = (FullTextIndexServiceImpl)fullText;
-
     }
 
     /** Race concurrent drop vs create. */
@@ -138,7 +105,7 @@
             new DropIndex().run();
         }
 
-        verifyClean(fullTextImpl);
+        verifyClean();
     }
 
     /** Race concurrent drop vs create, with lined up start. */
@@ -148,9 +115,8 @@
         createFullTextIndex(
                 SCHEMA, "o", "idx3_o",
                 "oid", "c1", "c2", "c3", "c4");
-        fullTextImpl.POPULATE_BACKGROUND.forceExecution();
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
-        
+        waitPopulate();
+
         Thread t = new Thread(new DropIndex());
         t.start();
 
@@ -173,7 +139,7 @@
             new DropIndex().run();
         }
 
-        verifyClean(fullTextImpl);
+        verifyClean();
     }
 
     /** Serial create and drop. */
@@ -185,10 +151,10 @@
                 "oid", "c1", "c2", "c3", "c4");
 
         // Prevents potential rollback for deterministic test
-        WaitFunctionHelpers.waitOn(Collections.singleton(fullTextImpl.POPULATE_BACKGROUND));
+        waitPopulate();
 
         new DropIndex().run();
-        verifyClean(fullTextImpl);
+        verifyClean();
     }
 
     @Test
@@ -199,20 +165,17 @@
         createFullTextIndex(
                 SCHEMA, "o", "idx3_o",
                 "oid", "c1", "c2", "c3", "c4");
-        fullTextImpl.POPULATE_BACKGROUND.forceExecution();
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
-        
+        waitPopulate();
+
         writeRow(o, 103, 1, "c1", "c2", "c3", "c4", "2012-12-12");
         writeRow(o, 104, 1, "c1", "c2", "c3", "c4", "2012-12-12");
         writeRow(o, 105, 1, "c1", "c2", "c3", "c4", "2012-12-12");
 
-        new Thread(new DropIndex()).start();
-        synchronized (lock) {
-            lock.wait();
-        }
-        
-        // kick start the background updater
-        fullTextImpl.UPDATE_BACKGROUND.forceExecution();
+        // Race update vs drop
+        Thread t = new Thread(new DropIndex());
+        t.start();
+        // But don't let it fall off the end
+        t.join();
     }
     
     private static interface Visitor
@@ -235,10 +198,9 @@
         }
     }
 
-    private void verifyClean(FullTextIndexServiceImpl fullTextImpl) throws InterruptedException {
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
-        traverse(fullTextImpl,
-                new Visitor()
+    private void verifyClean() throws InterruptedException {
+        waitPopulateAndUpdate();
+        traverse(new Visitor()
                 {
                     int n = 0;
                    
@@ -257,16 +219,16 @@
 
     }
     
-    private void traverse(FullTextIndexServiceImpl serv, Visitor visitor)
+    private void traverse(Visitor visitor)
     {
         Cursor cursor = null;
         try(Session session = createNewSession();
             CloseableTransaction txn = txnService().beginCloseableTransaction(session))
         {
-            cursor = serv.populateTableCursor(session);
+            cursor = fullTextImpl.populateTableCursor(session);
             cursor.open();
             IndexName toPopulate;
-            while((toPopulate = serv.nextInQueue(session, cursor, true)) != null) {
+            while((toPopulate = fullTextImpl.nextInQueue(session, cursor, true)) != null) {
                 visitor.visit(toPopulate);
             }
             visitor.endOfTree();

=== modified file 'src/test/java/com/akiban/server/service/text/FullTextIndexServiceIT.java'
--- src/test/java/com/akiban/server/service/text/FullTextIndexServiceIT.java	2013-07-10 19:34:13 +0000
+++ src/test/java/com/akiban/server/service/text/FullTextIndexServiceIT.java	2013-07-29 13:55:36 +0000
@@ -21,45 +21,25 @@
 import com.akiban.ais.model.IndexName;
 import com.akiban.qp.operator.Cursor;
 import com.akiban.qp.operator.Operator;
-import com.akiban.qp.operator.QueryBindings;
-import com.akiban.qp.operator.QueryContext;
 import static com.akiban.qp.operator.API.cursor;
 
-import com.akiban.qp.operator.StoreAdapter;
 import com.akiban.qp.row.RowBase;
 import com.akiban.qp.rowtype.RowType;
-import com.akiban.qp.rowtype.Schema;
 import com.akiban.qp.util.SchemaCache;
 import com.akiban.server.service.servicemanager.GuicedServiceManager;
 import com.akiban.server.service.session.Session;
 import com.akiban.server.service.session.SessionServiceImpl;
 import com.akiban.server.service.transaction.TransactionService.CloseableTransaction;
-import com.akiban.server.test.it.ITBase;
 import com.akiban.server.test.it.qp.TestRow;
 
-import com.akiban.server.types3.mcompat.mfuncs.WaitFunctionHelpers;
 import org.junit.Before;
 import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import static org.junit.Assert.*;
 
-public class FullTextIndexServiceIT extends ITBase
+public class FullTextIndexServiceIT extends FullTextIndexServiceITBase
 {
     public static final String SCHEMA = "test";
-    protected FullTextIndexService fullText;
-    protected Schema schema;
-    protected StoreAdapter adapter;
-    protected QueryContext queryContext;
-    protected QueryBindings queryBindings;
-    private static final Logger logger = LoggerFactory.getLogger(FullTextIndexServiceIT.class);
-
-
-    private int c;
-    private int o;
-    private int i;
-    private int a;
     
     @Override
     protected GuicedServiceManager.BindingsConfigurationProvider serviceBindingsProvider() {
@@ -100,21 +80,16 @@
         writeRow(a, 301, 3, "MA");
         writeRow(a, 302, 3, "ME");
 
-        fullText = serviceManager().getServiceByClass(FullTextIndexService.class);
-
         schema = SchemaCache.globalSchema(ais());
         adapter = newStoreAdapter(schema);
         queryContext = queryContext(adapter);
         queryBindings = queryContext.createBindings();
     }
 
+
     @Test
     public void testPopulateScheduling() throws InterruptedException
     {
-        // This test is specifically for FullTextIndexServiceImpl.java
-        assertEquals(FullTextIndexServiceImpl.class, fullText.getClass());
-        FullTextIndexServiceImpl fullTextImpl = (FullTextIndexServiceImpl)fullText;
-        
         // disable the populate worker (so it doesn't read all the entries
         // out before we get a chance to look at the tree.
         fullTextImpl.disablePopulateWorker();
@@ -155,7 +130,7 @@
         // let the worker do its job.
         // (After it is done, the tree had better be empty)
         fullTextImpl.enablePopulateWorker();
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
+        waitPopulate();
 
         traverse(fullTextImpl,
                  new Visitor()
@@ -180,11 +155,6 @@
     @Test
     public void testDeleteIndex() throws InterruptedException
     {
-     
-        // This test is specifically for FullTextIndexServiceImpl.java
-        assertEquals(FullTextIndexServiceImpl.class, fullText.getClass());
-        FullTextIndexServiceImpl fullTextImpl = (FullTextIndexServiceImpl)fullText;
-         
         // <1> disable worker
         fullTextImpl.disablePopulateWorker();
 
@@ -209,7 +179,7 @@
         deleteFullTextIndex(expecteds[0].getIndexName());
         deleteFullTextIndex(expecteds[1].getIndexName());
 
-        // <4> check that the tree only has one entry now (ie., epxecteds2[2]
+        // <4> check that the tree only has one entry now (ie. expected[2])
         traverse(fullTextImpl,
                  new Visitor()
                  {
@@ -231,8 +201,8 @@
         
         // wake the worker up to do its job
         fullTextImpl.enablePopulateWorker();
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
-        
+        waitPopulate();
+
         session.close();
     }
 
@@ -287,7 +257,7 @@
                                                   SCHEMA, "c", "idx_c", 
                                                   "name", "i.sku", "a.state");
 
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
+        waitPopulate();
         RowType rowType = rowType("c");
         RowBase[] expected1 = new RowBase[]
         {
@@ -304,8 +274,8 @@
         writeRow(c, 5, "Sherlock Flintstone");
         writeRow(c, 6, "Mycroft Holmes");
         writeRow(c, 7, "Flintstone Lestrade");
-        
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
+
+        waitUpdate();
         RowBase expected2[] = new RowBase[]
         {
             row(rowType, 1L),
@@ -318,20 +288,20 @@
         ftScanAndCompare(builder, "flintstone", 15, expected2);
 
         // part 3
-        ((FullTextIndexServiceImpl)fullText).disableUpdateWorker();
+        fullTextImpl.disableUpdateWorker();
         
         writeRow(c, 8, "Flintstone Hudson");
         writeRow(c, 9, "Jim Flintstone");
         
         // The worker has been disabled, waitOn should return immediately
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
-        
+        waitUpdate();
+
         // confirm that new rows are not found (ie., expected2 still works)
         ftScanAndCompare(builder, "flintstone", 15, expected2);
 
-        ((FullTextIndexServiceImpl)fullText).enableUpdateWorker();
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
-        
+        fullTextImpl.enableUpdateWorker();
+        waitUpdate();
+
         // now the rows should be seen.
         // (Because disabling the worker does not stop the changes fron being recorded)
         RowBase expected3[] = new RowBase[]
@@ -352,7 +322,7 @@
         FullTextIndex index = createFullTextIndex(
                                                   SCHEMA, "c", "idx_c", 
                                                   "name", "i.sku", "a.state");
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
+        waitPopulate();
         RowType rowType = rowType("c");
         RowBase[] expected = new RowBase[] {
             row(rowType, 1L),
@@ -368,7 +338,7 @@
         FullTextIndex index = createFullTextIndex(
                                                   SCHEMA, "o", "idx_o",
                                                   "c.name", "i.sku");
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
+        waitPopulate();
         RowType rowType = rowType("o");
         RowBase[] expected = new RowBase[] {
             row(rowType, 1L, 101L)
@@ -380,7 +350,7 @@
     @Test
     public void testTruncate() throws InterruptedException {
         FullTextIndex index = createFullTextIndex(SCHEMA, "c", "idx_c", "name", "i.sku", "a.state");
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
+        waitPopulate();
         
         final int limit = 15;
         RowType rowType = rowType("c");
@@ -398,25 +368,25 @@
         ftScanAndCompare(builder, skuQuery, limit, skuExpected);
 
         dml().truncateTable(session(), a);
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
+        waitUpdate();
         ftScanAndCompare(builder, nameQuery, limit, nameExpected);
         ftScanAndCompare(builder, stateQuery, limit, emptyExpected);
         ftScanAndCompare(builder, skuQuery, limit, skuExpected);
 
         dml().truncateTable(session(), o);
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
+        waitUpdate();
         ftScanAndCompare(builder, nameQuery, limit, nameExpected);
         ftScanAndCompare(builder, stateQuery, limit, emptyExpected);
         ftScanAndCompare(builder, skuQuery, limit, emptyExpected); // Non-cascading key, connection to c1 is unknown
 
         dml().truncateTable(session(), i);
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
+        waitUpdate();
         ftScanAndCompare(builder, nameQuery, limit, nameExpected);
         ftScanAndCompare(builder, stateQuery, limit, emptyExpected);
         ftScanAndCompare(builder, skuQuery, limit, emptyExpected);
 
         dml().truncateTable(session(), c);
-        WaitFunctionHelpers.waitOn(fullText.getBackgroundWorks());
+        waitUpdate();
         ftScanAndCompare(builder, nameQuery, limit, emptyExpected);
         ftScanAndCompare(builder, stateQuery, limit, emptyExpected);
         ftScanAndCompare(builder, skuQuery, limit, emptyExpected);

=== added file 'src/test/java/com/akiban/server/service/text/FullTextIndexServiceITBase.java'
--- src/test/java/com/akiban/server/service/text/FullTextIndexServiceITBase.java	1970-01-01 00:00:00 +0000
+++ src/test/java/com/akiban/server/service/text/FullTextIndexServiceITBase.java	2013-07-29 13:55:36 +0000
@@ -0,0 +1,73 @@
+/**
+ * Copyright (C) 2009-2013 Akiban Technologies, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+package com.akiban.server.service.text;
+
+import com.akiban.qp.operator.QueryBindings;
+import com.akiban.qp.operator.QueryContext;
+import com.akiban.qp.operator.StoreAdapter;
+import com.akiban.qp.rowtype.Schema;
+import com.akiban.server.service.servicemanager.GuicedServiceManager;
+import com.akiban.server.test.it.ITBase;
+import org.junit.Before;
+
+import java.util.Map;
+
+public class FullTextIndexServiceITBase extends ITBase
+{
+    public static final String SCHEMA = "test";
+    protected FullTextIndexServiceImpl fullTextImpl;
+    protected Schema schema;
+    protected StoreAdapter adapter;
+    protected QueryContext queryContext;
+    protected QueryBindings queryBindings;
+    protected int c;
+    protected int o;
+    protected int i;
+    protected int a;
+
+
+    @Override
+    protected GuicedServiceManager.BindingsConfigurationProvider serviceBindingsProvider() {
+        return super.serviceBindingsProvider()
+                    .bindAndRequire(FullTextIndexService.class, FullTextIndexServiceImpl.class);
+    }
+
+    @Override
+    public Map<String, String> startupConfigProperties() {
+        return uniqueStartupConfigProperties(getClass());
+    }
+
+
+    @Before
+    public final void castService() {
+        fullTextImpl = (FullTextIndexServiceImpl)serviceManager().getServiceByClass(FullTextIndexService.class);
+    }
+
+    protected void waitPopulate() {
+        fullTextImpl.waitPopulateCycle();
+    }
+
+    protected void waitUpdate() {
+        fullTextImpl.waitUpdateCycle();
+    }
+
+    protected void waitPopulateAndUpdate() {
+        waitPopulate();
+        waitUpdate();
+    }
+}

=== renamed file 'src/test/resources/com/akiban/rest/text/background.wait' => 'src/test/resources/com/akiban/rest/text/full_text_background_wait'
--- src/test/resources/com/akiban/rest/text/background.wait	2013-04-17 16:23:40 +0000
+++ src/test/resources/com/akiban/rest/text/full_text_background_wait	2013-07-29 13:55:36 +0000
@@ -1,1 +0,0 @@
-fulltext_maintenance_wait
\ No newline at end of file

=== modified file 'src/test/resources/com/akiban/sql/pg/yaml/functional/test-drop-fulltext.yaml'
--- src/test/resources/com/akiban/sql/pg/yaml/functional/test-drop-fulltext.yaml	2013-07-04 06:00:37 +0000
+++ src/test/resources/com/akiban/sql/pg/yaml/functional/test-drop-fulltext.yaml	2013-07-29 13:55:36 +0000
@@ -4,11 +4,11 @@
 ---
 - Statement: CREATE INDEX idx on t(full_text(name));
 ---
-- Statement: SELECT fulltext_maintenance_wait();
+- Statement: CALL sys.full_text_background_wait()
 ---
 - Statement: INSERT INTO t values (1, 'bar1'), (2, 'bar2'), (3, 'bar3'), (4, 'bar1');
 ---
-- Statement: SELECT fulltext_maintenance_wait();
+- Statement: CALL sys.full_text_background_wait()
 ---
 - Statement: SELECT id from t where full_text_search(name='bar1');
 - output: [[1], [4]]
@@ -23,7 +23,7 @@
 ---
 - Statement: CREATE INDEX t2_idx on t2(full_text(name));
 ---
-- Statement: SELECT fulltext_maintenance_wait();
+- Statement: CALL sys.full_text_background_wait()
 ---
 - Statement: SELECT * FROM t2 WHERE FULL_TEXT_SEARCH(name, 'Jo*');
 - row_count: 0
@@ -36,7 +36,7 @@
 ---
 - Statement: CREATE INDEX t2_idx on t2(full_text(name));
 ---
-- Statement: SELECT fulltext_maintenance_wait()
+- Statement: CALL sys.full_text_background_wait()
 ---
 - Statement: SELECT * FROM t2 WHERE FULL_TEXT_SEARCH(name, 'Jo*');
 - output: [[1, 'John']]

=== modified file 'src/test/resources/com/akiban/sql/pg/yaml/functional/test-ft-with-inherited-key.yaml'
--- src/test/resources/com/akiban/sql/pg/yaml/functional/test-ft-with-inherited-key.yaml	2013-07-02 19:16:02 +0000
+++ src/test/resources/com/akiban/sql/pg/yaml/functional/test-ft-with-inherited-key.yaml	2013-07-29 13:55:36 +0000
@@ -14,16 +14,14 @@
 ---
 - Statement: CREATE INDEX t3_ft ON t3(FULL_TEXT(name));
 ---
-- Statement: SELECT fulltext_maintenance_wait();
-- output: [[0]]
+- Statement: CALL sys.full_text_background_wait()
 ---
 - Statement: SELECT * FROM t3 WHERE FULL_TEXT_SEARCH(name, 'fred');
 - output: [[111, 11, 'Fred']]
 ---
 - Statement: UPDATE t2 SET pid = 2 WHERE pid = 1;
 ---
-- Statement: SELECT fulltext_maintenance_wait();
-- output: [[0]]
+- Statement: CALL sys.full_text_background_wait()
 ---
 - Statement: SELECT * FROM t3 WHERE FULL_TEXT_SEARCH(name, 'fred');
 - output: [[111, 11, 'Fred']]

=== modified file 'src/test/resources/com/akiban/sql/pg/yaml/functional/test-ft-with-pk.yaml'
--- src/test/resources/com/akiban/sql/pg/yaml/functional/test-ft-with-pk.yaml	2013-07-02 19:16:02 +0000
+++ src/test/resources/com/akiban/sql/pg/yaml/functional/test-ft-with-pk.yaml	2013-07-29 13:55:36 +0000
@@ -8,19 +8,14 @@
 ---
 - Statement: create index idx1 on t (full_text(name));
 ---
-- Statement: SELECT fulltext_maintenance_wait();
-- output: [[0]]
+- Statement: CALL sys.full_text_background_wait()
 ---
 - Statement: SELECT id FROM t where full_text_search(name = 'foo1');
 - output: [[1]]
 ---
 - Statement: UPDATE t SET id = 3 where id = 1;
 ---
-- Statement: SELECT fulltext_maintenance_wait();
-- output: [[0]]
----
-- Statement: SELECT fulltext_maintenance_wait();
-- output: [[0]]
+- Statement: CALL sys.full_text_background_wait()
 ---
 - Statement: SELECT id FROM t where full_text_search(name = 'foo1');
 - output: [[3]]

=== modified file 'src/test/resources/com/akiban/sql/pg/yaml/functional/test-fulltext-maintenance.yaml'
--- src/test/resources/com/akiban/sql/pg/yaml/functional/test-fulltext-maintenance.yaml	2013-07-04 06:00:37 +0000
+++ src/test/resources/com/akiban/sql/pg/yaml/functional/test-fulltext-maintenance.yaml	2013-07-29 13:55:36 +0000
@@ -17,14 +17,12 @@
 - Statement: INSERT INTO  t VALUES (4, 'bar4');
 ---
 - Statement: CREATE INDEX idx ON t (full_text(name));
---- # blocks until the maintenance is done. Return value of 0 means nothing wrong happened!
-- Statement: SELECT fulltext_maintenance_wait();
-- output: [[0]]
---- # create a yet another index (to ensure the scheduler still works after forced execution)
+---
+- Statement: CALL sys.full_text_background_wait()
+---
 - Statement: CREATE INDEX idx2 on t2(full_text(name2));
 ---
-- Statement: SELECT fulltext_maintenance_wait();
-- output: [[0]]
+- Statement: CALL sys.full_text_background_wait()
 ---
 - Statement: SELECT id from t2 where full_text_search(name2 = 'foo1');
 - output: [[1]]
@@ -34,8 +32,7 @@
 ---
 - Statement: UPDATE t set name='foo' WHERE id = 1;
 ---
-- Statement: SELECT fulltext_maintenance_wait();
-- output: [[0]]
+- Statement: CALL sys.full_text_background_wait()
 ---
 - Statement: select id from t where full_text_search(name = 'bar1');
 - output: [[2]]
@@ -45,8 +42,7 @@
 ---
 - Statement: DELETE FROM t where id = 4;
 ---
-- Statement: SELECT fulltext_maintenance_wait();
-- output: [[0]]
+- Statement: CALL sys.full_text_background_wait()
 ---
 - Statement: SELECT id from t where full_text_search(name = 'bar4');
 - row_count: 0