Merge lp:~clay-gerrard/swift/run_daemon into lp:~hudson-openstack/swift/trunk

[clayg]

updated daemonize process, added option for servers/daemons to log to console

added helper/util to parse command line args
removed some duplicated code in server/daemon bin scripts
more standardized python/linux daemonization procedures
fixed lp:666957 "devauth server creates auth.db with the wrong
 privileges"
new run_daemon helper based on run_wsgi simplifies daemon
 launching/testing
new - all servers/daemons support verbose option when started
 interactivity which will log to the console
fixed lp:667839 "can't start servers with relative paths to configs"
added tests

[gholt (gholt)]

review: Approve

[gholt (gholt)]

This one's a wee bit hairy so another should take a peek as well.

review: Approve

[Chuck Thier (cthier)]

The only question I have is why the daemonize function ended up in swift.common.utils rather than swift.common.daemon? Otherwise, I think everything else looks good.

[clayg (clay-gerrard)]

thanks!

heh, funny story. When redbo and I talked about it - the plan was to expand utils.drop_privileges to also handle the capture_stdio stuff and rename it daemonize (for use in both run_wsgi and my new run_daemon) - but when I got into run_wsgi I decided that I want to redirect errors before the the port bindings... which needs elevated privileges for reserved ports. So I had to split up utils.daemonize back into utils.drop_privileges and the new utils.capture_stdio, which left utils.daemonize as pretty thin wrapper that was only called in common.daemon and the tests which I had already wrote in test.unit.common.test_utils :P

tl;dr - daemonize is dumb, run_daemon can call drop_privileges and capture_stdio same as run_wsgi; good catch - I'll remove it and rework the tests.

[John Dickinson (notmyname)]

run_daemon looks good, and it all works well with the stats processing code

however, the unit tests seem to print something out that needs to be removed (see test_utils.py:180)

review: Needs Fixing

[clayg (clay-gerrard)]

awesome, works great on nose 0.11.4 - it's way worse on 0.11.1 :P

[clayg (clay-gerrard)]

John, gholt - if you guys could pull down the latest changes and see if the unittest output got cleaned up?

[lp:~clay-gerrard/swift/run_daemon]

134. By clayg on 2010-11-19

remove capture_io stuff from db_replicator.__init__

[clayg (clay-gerrard)]

after discussion with redbo and gholt - we decided to pull out the stdio redirection from common.db_replicator.Replicator.__init__

[John Dickinson (notmyname)]

works now! yay!

review: Approve

=== modified file 'bin/swift-account-auditor'
--- bin/swift-account-auditor	2010-08-31 23:12:59 +0000
+++ bin/swift-account-auditor	2010-11-19 22:38:08 +0000
@@ -14,15 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
 from swift.account.auditor import AccountAuditor
-from swift.common import utils
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print "Usage: swift-account-auditor CONFIG_FILE [once]"
-        sys.exit()
-    once = len(sys.argv) > 2 and sys.argv[2] == 'once'
-    conf = utils.readconf(sys.argv[1], 'account-auditor')
-    auditor = AccountAuditor(conf).run(once)
+    conf_file, options = parse_options(once=True)
+    run_daemon(AccountAuditor, conf_file, **options)

=== modified file 'bin/swift-account-reaper'
--- bin/swift-account-reaper	2010-08-31 23:12:59 +0000
+++ bin/swift-account-reaper	2010-11-19 22:38:08 +0000
@@ -14,15 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
 from swift.account.reaper import AccountReaper
-from swift.common import utils
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print "Usage: account-reaper CONFIG_FILE [once]"
-        sys.exit()
-    once = len(sys.argv) > 2 and sys.argv[2] == 'once'
-    conf = utils.readconf(sys.argv[1], 'account-reaper')
-    reaper = AccountReaper(conf).run(once)
+    conf_file, options = parse_options(once=True)
+    run_daemon(AccountReaper, conf_file, **options)

=== modified file 'bin/swift-account-replicator'
--- bin/swift-account-replicator	2010-08-31 23:12:59 +0000
+++ bin/swift-account-replicator	2010-11-19 22:38:08 +0000
@@ -14,15 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
-from swift.common import utils
 from swift.account.replicator import AccountReplicator
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print "Usage: swift-account-replicator CONFIG_FILE [once]"
-        sys.exit(1)
-    once = len(sys.argv) > 2 and sys.argv[2] == 'once'
-    conf = utils.readconf(sys.argv[1], 'account-replicator')
-    AccountReplicator(conf).run(once)
+    conf_file, options = parse_options(once=True)
+    run_daemon(AccountReplicator, conf_file, **options)

=== modified file 'bin/swift-account-server'
--- bin/swift-account-server	2010-08-24 14:04:44 +0000
+++ bin/swift-account-server	2010-11-19 22:38:08 +0000
@@ -14,12 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
+from swift.common.utils import parse_options
 from swift.common.wsgi import run_wsgi
 
 if __name__ == '__main__':
-    if len(sys.argv) != 2:
-        sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
-    run_wsgi(sys.argv[1], 'account-server', default_port=6002)
-
+    conf_file, options = parse_options()
+    run_wsgi(conf_file, 'account-server', default_port=6002, **options)

=== modified file 'bin/swift-account-stats-logger'
--- bin/swift-account-stats-logger	2010-09-13 16:11:12 +0000
+++ bin/swift-account-stats-logger	2010-11-19 22:38:08 +0000
@@ -14,14 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
 from swift.stats.account_stats import AccountStat
-from swift.common import utils
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print "Usage: swift-account-stats-logger CONFIG_FILE"
-        sys.exit()
-    stats_conf = utils.readconf(sys.argv[1], 'log-processor-stats')
-    stats = AccountStat(stats_conf).run(once=True)
+    conf_file, options = parse_options()
+    # currently AccountStat only supports run_once
+    options['once'] = True
+    run_daemon(AccountStat, conf_file, section_name='log-processor-stats',
+               **options)

=== modified file 'bin/swift-auth-server'
--- bin/swift-auth-server	2010-08-24 14:08:16 +0000
+++ bin/swift-auth-server	2010-11-19 22:38:08 +0000
@@ -14,11 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
+from swift.common.utils import parse_options
 from swift.common.wsgi import run_wsgi
 
 if __name__ == '__main__':
-    if len(sys.argv) != 2:
-        sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
-    run_wsgi(sys.argv[1], 'auth-server', default_port=11000)
+    conf_file, options = parse_options()
+    run_wsgi(conf_file, 'auth-server', default_port=11000, **options)

=== modified file 'bin/swift-container-auditor'
--- bin/swift-container-auditor	2010-08-31 23:12:59 +0000
+++ bin/swift-container-auditor	2010-11-19 22:38:08 +0000
@@ -14,15 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
 from swift.container.auditor import ContainerAuditor
-from swift.common import utils
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print "Usage: swift-container-auditor CONFIG_FILE [once]"
-        sys.exit()
-    once = len(sys.argv) > 2 and sys.argv[2] == 'once'
-    conf = utils.readconf(sys.argv[1], 'container-auditor')
-    ContainerAuditor(conf).run(once)
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerAuditor, conf_file, **options)

=== modified file 'bin/swift-container-replicator'
--- bin/swift-container-replicator	2010-08-31 23:12:59 +0000
+++ bin/swift-container-replicator	2010-11-19 22:38:08 +0000
@@ -14,16 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
-from swift.common import db, utils
 from swift.container.replicator import ContainerReplicator
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print "Usage: swift-container-replicator CONFIG_FILE [once]"
-        sys.exit(1)
-    once = len(sys.argv) > 2 and sys.argv[2] == 'once'
-    conf = utils.readconf(sys.argv[1], 'container-replicator')
-    ContainerReplicator(conf).run(once)
-
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerReplicator, conf_file, **options)

=== modified file 'bin/swift-container-server'
--- bin/swift-container-server	2010-08-24 14:04:44 +0000
+++ bin/swift-container-server	2010-11-19 22:38:08 +0000
@@ -14,11 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
+from swift.common.utils import parse_options
 from swift.common.wsgi import run_wsgi
 
 if __name__ == '__main__':
-    if len(sys.argv) != 2:
-        sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
-    run_wsgi(sys.argv[1], 'container-server', default_port=6001)
+    conf_file, options = parse_options()
+    run_wsgi(conf_file, 'container-server', default_port=6001, **options)

=== modified file 'bin/swift-container-updater'
--- bin/swift-container-updater	2010-08-31 23:12:59 +0000
+++ bin/swift-container-updater	2010-11-19 22:38:08 +0000
@@ -14,15 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
 from swift.container.updater import ContainerUpdater
-from swift.common import utils
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print "Usage: swift-container-updater CONFIG_FILE [once]"
-        sys.exit()
-    once = len(sys.argv) > 2 and sys.argv[2] == 'once'
-    conf = utils.readconf(sys.argv[1], 'container-updater')
-    ContainerUpdater(conf).run(once)
+    conf_file, options = parse_options(once=True)
+    run_daemon(ContainerUpdater, conf_file, **options)

=== modified file 'bin/swift-log-stats-collector'
--- bin/swift-log-stats-collector	2010-09-22 14:32:40 +0000
+++ bin/swift-log-stats-collector	2010-11-19 22:38:08 +0000
@@ -14,14 +14,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
 from swift.stats.log_processor import LogProcessorDaemon
-from swift.common import utils
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print "Usage: swift-log-stats-collector CONFIG_FILE"
-        sys.exit()
-    conf = utils.readconf(sys.argv[1], log_name='log-stats-collector')
-    stats = LogProcessorDaemon(conf).run(once=True)
+    conf_file, options = parse_options()
+    # currently the LogProcessorDaemon only supports run_once
+    options['once'] = True
+    run_daemon(LogProcessorDaemon, conf_file, section_name=None,
+               log_name='log-stats-collector', **options)

=== modified file 'bin/swift-log-uploader'
--- bin/swift-log-uploader	2010-09-10 20:08:06 +0000
+++ bin/swift-log-uploader	2010-11-19 22:38:08 +0000
@@ -17,15 +17,25 @@
 import sys
 
 from swift.stats.log_uploader import LogUploader
+from swift.common.utils import parse_options
 from swift.common import utils
 
 if __name__ == '__main__':
-    if len(sys.argv) < 3:
-        print "Usage: swift-log-uploader CONFIG_FILE plugin"
-        sys.exit()
-    uploader_conf = utils.readconf(sys.argv[1], 'log-processor')
-    plugin = sys.argv[2]
+    conf_file, options = parse_options(usage="Usage: %prog CONFIG_FILE PLUGIN")
+    try:
+        plugin = options['extra_args'][0]
+    except IndexError:
+        print "Error: missing plugin name"
+        sys.exit(1)
+
+    uploader_conf = utils.readconf(conf_file, 'log-processor')
     section_name = 'log-processor-%s' % plugin
-    plugin_conf = utils.readconf(sys.argv[1], section_name)
+    plugin_conf = utils.readconf(conf_file, section_name)
     uploader_conf.update(plugin_conf)
-    uploader = LogUploader(uploader_conf, plugin).run(once=True)
+
+    # pre-configure logger
+    logger = utils.get_logger(uploader_conf, plugin,
+                              log_to_console=options.get('verbose', False))
+    # currently LogUploader only supports run_once
+    options['once'] = True
+    uploader = LogUploader(uploader_conf, plugin).run(**options)

=== modified file 'bin/swift-object-auditor'
--- bin/swift-object-auditor	2010-08-31 23:12:59 +0000
+++ bin/swift-object-auditor	2010-11-19 22:38:08 +0000
@@ -14,16 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
 from swift.obj.auditor import ObjectAuditor
-from swift.common import utils
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print "Usage: swift-object-auditor CONFIG_FILE [once]"
-        sys.exit()
-
-    once = len(sys.argv) > 2 and sys.argv[2] == 'once'
-    conf = utils.readconf(sys.argv[1], 'object-auditor')
-    ObjectAuditor(conf).run(once)
+    conf_file, options = parse_options(once=True)
+    run_daemon(ObjectAuditor, conf_file, **options)

=== modified file 'bin/swift-object-replicator'
--- bin/swift-object-replicator	2010-08-31 23:12:59 +0000
+++ bin/swift-object-replicator	2010-11-19 22:38:08 +0000
@@ -14,16 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
 from swift.obj.replicator import ObjectReplicator
-from swift.common import utils
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print "Usage: swift-object-replicator CONFIG_FILE [once]"
-        sys.exit()
-    conf = utils.readconf(sys.argv[1], "object-replicator")
-    once = (len(sys.argv) > 2 and sys.argv[2] == 'once') or \
-            conf.get('daemonize', 'true') not in utils.TRUE_VALUES
-    ObjectReplicator(conf).run(once)
+    conf_file, options = parse_options(once=True)
+    run_daemon(ObjectReplicator, conf_file, **options)

=== modified file 'bin/swift-object-server'
--- bin/swift-object-server	2010-08-24 14:04:44 +0000
+++ bin/swift-object-server	2010-11-19 22:38:08 +0000
@@ -14,11 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
+from swift.common.utils import parse_options
 from swift.common.wsgi import run_wsgi
 
 if __name__ == '__main__':
-    if len(sys.argv) != 2:
-        sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
-    run_wsgi(sys.argv[1], 'object-server', default_port=6000)
+    conf_file, options = parse_options()
+    run_wsgi(conf_file, 'object-server', default_port=6000, **options)

=== modified file 'bin/swift-object-updater'
--- bin/swift-object-updater	2010-08-31 23:12:59 +0000
+++ bin/swift-object-updater	2010-11-19 22:38:08 +0000
@@ -14,15 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
 from swift.obj.updater import ObjectUpdater
-from swift.common import utils
+from swift.common.utils import parse_options
+from swift.common.daemon import run_daemon
 
 if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print "Usage: swift-object-updater CONFIG_FILE [once]"
-        sys.exit(1)
-    once = len(sys.argv) > 2 and sys.argv[2] == 'once'
-    conf = utils.readconf(sys.argv[1], 'object-updater')
-    ObjectUpdater(conf).run(once)
+    conf_file, options = parse_options(once=True)
+    run_daemon(ObjectUpdater, conf_file, **options)

=== modified file 'bin/swift-proxy-server'
--- bin/swift-proxy-server	2010-08-24 14:04:44 +0000
+++ bin/swift-proxy-server	2010-11-19 22:38:08 +0000
@@ -14,11 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import sys
-
+from swift.common.utils import parse_options
 from swift.common.wsgi import run_wsgi
 
 if __name__ == '__main__':
-    if len(sys.argv) != 2:
-        sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0])
-    run_wsgi(sys.argv[1], 'proxy-server', default_port=8080)
+    conf_file, options = parse_options()
+    run_wsgi(conf_file, 'proxy-server', default_port=8080, **options)

=== modified file 'swift/common/daemon.py'
--- swift/common/daemon.py	2010-10-15 19:28:38 +0000
+++ swift/common/daemon.py	2010-11-19 22:38:08 +0000
@@ -16,6 +16,7 @@
 import os
 import sys
 import signal
+from re import sub
 from swift.common import utils
 
 
@@ -34,23 +35,11 @@
         """Override this to run forever"""
         raise NotImplementedError('run_forever not implemented')
 
-    def run(self, once=False, capture_stdout=True, capture_stderr=True):
+    def run(self, once=False, **kwargs):
         """Run the daemon"""
-        # log uncaught exceptions
-        sys.excepthook = lambda *exc_info: \
-            self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
-        if capture_stdout:
-            sys.stdout = utils.LoggerFileObject(self.logger)
-        if capture_stderr:
-            sys.stderr = utils.LoggerFileObject(self.logger)
-
+        utils.validate_configuration()
+        utils.capture_stdio(self.logger, **kwargs)
         utils.drop_privileges(self.conf.get('user', 'swift'))
-        utils.validate_configuration()
-
-        try:
-            os.setsid()
-        except OSError:
-            pass
 
         def kill_children(*args):
             signal.signal(signal.SIGTERM, signal.SIG_IGN)
@@ -63,3 +52,40 @@
             self.run_once()
         else:
             self.run_forever()
+
+
+def run_daemon(klass, conf_file, section_name='',
+               once=False, **kwargs):
+    """
+    Loads settings from conf, then instantiates daemon "klass" and runs the
+    daemon with the specified once kwarg.  The section_name will be derived
+    from the daemon "klass" if not provided (e.g. ObjectReplicator =>
+    object-replicator).
+
+    :param klass: Class to instantiate, subclass of common.daemon.Daemon
+    :param conf_file: Path to configuration file
+    :param section_name: Section name from conf file to load config from
+    :param once: Passed to daemon run method
+    """
+    # very often the config section_name is based on the class name
+    # the None singleton will be passed through to readconf as is
+    if section_name is '':
+        section_name = sub(r'([a-z])([A-Z])', r'\1-\2',
+                           klass.__name__).lower()
+    conf = utils.readconf(conf_file, section_name,
+                          log_name=kwargs.get('log_name'))
+
+    # once on command line (i.e. daemonize=false) will over-ride config
+    once = once or conf.get('daemonize', 'true') not in utils.TRUE_VALUES
+
+    # pre-configure logger
+    if 'logger' in kwargs:
+        logger = kwargs.pop('logger')
+    else:
+        logger = utils.get_logger(conf, conf.get('log_name', section_name),
+                                  log_to_console=kwargs.pop('verbose', False))
+    try:
+        klass(conf).run(once=once, **kwargs)
+    except KeyboardInterrupt:
+        logger.info('User quit')
+    logger.info('Exited')

=== modified file 'swift/common/db_replicator.py'
--- swift/common/db_replicator.py	2010-09-01 15:56:37 +0000
+++ swift/common/db_replicator.py	2010-11-19 22:38:08 +0000
@@ -93,10 +93,6 @@
     def __init__(self, conf):
         self.conf = conf
         self.logger = get_logger(conf)
-        # log uncaught exceptions
-        sys.excepthook = lambda * exc_info: \
-                self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
-        sys.stdout = sys.stderr = LoggerFileObject(self.logger)
         self.root = conf.get('devices', '/srv/node')
         self.mount_check = conf.get('mount_check', 'true').lower() in \
                               ('true', 't', '1', 'on', 'yes', 'y')

=== modified file 'swift/common/utils.py'
--- swift/common/utils.py	2010-11-16 19:52:18 +0000
+++ swift/common/utils.py	2010-11-19 22:38:08 +0000
@@ -31,6 +31,7 @@
 import ctypes.util
 import struct
 from ConfigParser import ConfigParser, NoSectionError, NoOptionError
+from optparse import OptionParser
 from tempfile import mkstemp
 import cPickle as pickle
 
@@ -283,17 +284,6 @@
         return self
 
 
-def drop_privileges(user):
-    """
-    Sets the userid of the current process
-
-    :param user: User id to change privileges to
-    """
-    user = pwd.getpwnam(user)
-    os.setgid(user[3])
-    os.setuid(user[2])
-
-
 class NamedLogger(object):
     """Cheesy version of the LoggerAdapter available in Python 3"""
 
@@ -343,7 +333,7 @@
         call('%s %s: %s' % (self.server, msg, emsg), *args)
 
 
-def get_logger(conf, name=None):
+def get_logger(conf, name=None, log_to_console=False):
     """
     Get the current system logger using config settings.
 
@@ -355,11 +345,18 @@
 
     :param conf: Configuration dict to read settings from
     :param name: Name of the logger
+    :param log_to_console: Add handler which writes to console on stderr
     """
     root_logger = logging.getLogger()
     if hasattr(get_logger, 'handler') and get_logger.handler:
         root_logger.removeHandler(get_logger.handler)
         get_logger.handler = None
+    if log_to_console:
+        # check if a previous call to get_logger already added a console logger
+        if hasattr(get_logger, 'console') and get_logger.console:
+            root_logger.removeHandler(get_logger.console)
+        get_logger.console = logging.StreamHandler(sys.__stderr__)
+        root_logger.addHandler(get_logger.console)
     if conf is None:
         root_logger.setLevel(logging.INFO)
         return NamedLogger(root_logger, name)
@@ -375,6 +372,99 @@
     return NamedLogger(root_logger, name)
 
 
+def drop_privileges(user):
+    """
+    Sets the userid/groupid of the current process, get session leader, etc.
+
+    :param user: User name to change privileges to
+    """
+    user = pwd.getpwnam(user)
+    os.setgid(user[3])
+    os.setuid(user[2])
+    try:
+        os.setsid()
+    except OSError:
+        pass
+    os.chdir('/')  # in case you need to rmdir on where you started the daemon
+    os.umask(0)  # ensure files are created with the correct privileges
+
+
+def capture_stdio(logger, **kwargs):
+    """
+    Log unhandled exceptions, close stdio, capture stdout and stderr.
+
+    param logger: Logger object to use
+    """
+    # log uncaught exceptions
+    sys.excepthook = lambda * exc_info: \
+        logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
+
+    # collect stdio file desc not in use for logging
+    stdio_fds = [0, 1, 2]
+    if hasattr(get_logger, 'console'):
+        stdio_fds.remove(get_logger.console.stream.fileno())
+
+    with open(os.devnull, 'r+b') as nullfile:
+        # close stdio (excludes fds open for logging)
+        for desc in stdio_fds:
+            try:
+                os.dup2(nullfile.fileno(), desc)
+            except OSError:
+                pass
+
+    # redirect stdio
+    if kwargs.pop('capture_stdout', True):
+        sys.stdout = LoggerFileObject(logger)
+    if kwargs.pop('capture_stderr', True):
+        sys.stderr = LoggerFileObject(logger)
+
+
+def parse_options(usage="%prog CONFIG [options]", once=False, test_args=None):
+    """
+    Parse standard swift server/daemon options with optparse.OptionParser.
+
+    :param usage: String describing usage
+    :param once: Boolean indicating the "once" option is available
+    :param test_args: Override sys.argv; used in testing
+
+    :returns : Tuple of (config, options); config is an absolute path to the
+               config file, options is the parser options as a dictionary.
+
+    :raises SystemExit: First arg (CONFIG) is required, file must exist
+    """
+    parser = OptionParser(usage)
+    parser.add_option("-v", "--verbose", default=False, action="store_true",
+                      help="log to console")
+    if once:
+        parser.add_option("-o", "--once", default=False, action="store_true",
+                          help="only run one pass of daemon")
+
+    # if test_args is None, optparse will use sys.argv[:1]
+    options, args = parser.parse_args(args=test_args)
+
+    if not args:
+        parser.print_usage()
+        print "Error: missing config file argument"
+        sys.exit(1)
+    config = os.path.abspath(args.pop(0))
+    if not os.path.exists(config):
+        parser.print_usage()
+        print "Error: unable to locate %s" % config
+        sys.exit(1)
+
+    extra_args = []
+    # if any named options appear in remaining args, set the option to True
+    for arg in args:
+        if arg in options.__dict__:
+            setattr(options, arg, True)
+        else:
+            extra_args.append(arg)
+
+    options = vars(options)
+    options['extra_args'] = extra_args
+    return config, options
+
+
 def whataremyips():
     """
     Get the machine's ip addresses using ifconfig

=== modified file 'swift/common/wsgi.py'
--- swift/common/wsgi.py	2010-10-15 19:28:38 +0000
+++ swift/common/wsgi.py	2010-11-19 22:38:08 +0000
@@ -34,7 +34,7 @@
 from eventlet.green import socket, ssl
 
 from swift.common.utils import get_logger, drop_privileges, \
-    validate_configuration, LoggerFileObject, NullLogger
+    validate_configuration, capture_stdio, NullLogger
 
 
 def monkey_patch_mimetools():
@@ -56,41 +56,17 @@
 
     mimetools.Message.parsetype = parsetype
 
-
-# We might be able to pull pieces of this out to test, but right now it seems
-# like more work than it's worth.
-def run_wsgi(conf_file, app_section, *args, **kwargs):   # pragma: no cover
-    """
-    Loads common settings from conf, then instantiates app and runs
-    the server using the specified number of workers.
-
-    :param conf_file: Path to paste.deploy style configuration file
-    :param app_section: App name from conf file to load config from
-    """
-
-    try:
-        conf = appconfig('config:%s' % conf_file, name=app_section)
-        log_name = conf.get('log_name', app_section)
-        app = loadapp('config:%s' % conf_file,
-            global_conf={'log_name': log_name})
-    except Exception, e:
-        print "Error trying to load config %s: %s" % (conf_file, e)
-        return
-    if 'logger' in kwargs:
-        logger = kwargs['logger']
-    else:
-        logger = get_logger(conf, log_name)
-    # log uncaught exceptions
-    sys.excepthook = lambda * exc_info: \
-        logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info)
-    sys.stdout = sys.stderr = LoggerFileObject(logger)
-
-    try:
-        os.setsid()
-    except OSError:
-        no_cover = True     # pass
+def get_socket(conf, default_port=8080):
+    """Bind socket to bind ip:port in conf
+
+    :param conf: Configuration dict to read settings from
+    :param default_port: port to use if not specified in conf
+
+    :returns : a socket object as returned from socket.listen or ssl.wrap_socket
+               if conf specifies cert_file
+    """
     bind_addr = (conf.get('bind_ip', '0.0.0.0'),
-                 int(conf.get('bind_port', kwargs.get('default_port', 8080))))
+                 int(conf.get('bind_port', default_port)))
     sock = None
     retry_until = time.time() + 30
     while not sock and time.time() < retry_until:
@@ -110,9 +86,43 @@
     # in my experience, sockets can hang around forever without keepalive
     sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
     sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600)
-    worker_count = int(conf.get('workers', '1'))
+    return sock
+
+
+# TODO: pull pieces of this out to test
+def run_wsgi(conf_file, app_section, *args, **kwargs):
+    """
+    Loads common settings from conf, then instantiates app and runs
+    the server using the specified number of workers.
+
+    :param conf_file: Path to paste.deploy style configuration file
+    :param app_section: App name from conf file to load config from
+    """
+
+    try:
+        conf = appconfig('config:%s' % conf_file, name=app_section)
+    except Exception, e:
+        print "Error trying to load config %s: %s" % (conf_file, e)
+        return
+    validate_configuration()
+
+    # pre-configure logger
+    log_name = conf.get('log_name', app_section)
+    if 'logger' in kwargs:
+        logger = kwargs.pop('logger')
+    else:
+        logger = get_logger(conf, log_name,
+                            log_to_console=kwargs.pop('verbose', False))
+
+    # redirect errors to logger and close stdio
+    capture_stdio(logger)
+    # bind to address and port
+    sock = get_socket(conf, default_port=kwargs.get('default_port', 8080))
+    # remaining tasks should not require elevated privileges
     drop_privileges(conf.get('user', 'swift'))
-    validate_configuration()
+
+    # finally after binding to ports and privilege drop, run app __init__ code
+    app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name})
 
     def run_server():
         wsgi.HttpProtocol.default_request_version = "HTTP/1.0"
@@ -127,6 +137,7 @@
                 raise
         pool.waitall()
 
+    worker_count = int(conf.get('workers', '1'))
     # Useful for profiling [no forks].
     if worker_count == 0:
         run_server()
@@ -169,6 +180,9 @@
         except OSError, err:
             if err.errno not in (errno.EINTR, errno.ECHILD):
                 raise
+        except KeyboardInterrupt:
+            logger.info('User quit')
+            break
     greenio.shutdown_safe(sock)
     sock.close()
     logger.info('Exited')

=== modified file 'test/unit/__init__.py'
--- test/unit/__init__.py	2010-07-29 20:06:01 +0000
+++ test/unit/__init__.py	2010-11-19 22:38:08 +0000
@@ -1,5 +1,8 @@
 """ Swift tests """
 
+import os
+from contextlib import contextmanager
+from tempfile import NamedTemporaryFile
 from eventlet.green import socket
 
 
@@ -23,6 +26,18 @@
     rv.connect(hostport)
     return rv
 
+
+@contextmanager
+def tmpfile(content):
+    with NamedTemporaryFile('w', delete=False) as f:
+        file_name = f.name
+        f.write(str(content))
+    try:
+        yield file_name
+    finally:
+        os.unlink(file_name)
+
+
 class MockTrue(object):
     """
     Instances of MockTrue evaluate like True

=== modified file 'test/unit/common/test_daemon.py'
--- test/unit/common/test_daemon.py	2010-10-07 15:23:17 +0000
+++ test/unit/common/test_daemon.py	2010-11-19 22:38:08 +0000
@@ -13,16 +13,95 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO: Tests
+# TODO: Test kill_children signal handlers
 
 import unittest
-from swift.common import daemon
+from getpass import getuser
+import logging
+from StringIO import StringIO
+from test.unit import tmpfile
+
+from swift.common import daemon, utils
+
+
+class MyDaemon(daemon.Daemon):
+
+    def __init__(self, conf):
+        self.conf = conf
+        self.logger = utils.get_logger(None)
+        MyDaemon.forever_called = False
+        MyDaemon.once_called = False
+
+    def run_forever(self):
+        MyDaemon.forever_called = True
+
+    def run_once(self):
+        MyDaemon.once_called = True
+
+    def run_raise(self):
+        raise OSError
+
+    def run_quit(self):
+        raise KeyboardInterrupt
 
 
 class TestDaemon(unittest.TestCase):
 
-    def test_placeholder(self):
-        pass
+    def test_create(self):
+        d = daemon.Daemon({})
+        self.assertEquals(d.conf, {})
+        self.assert_(isinstance(d.logger, utils.NamedLogger))
+
+    def test_stubs(self):
+        d = daemon.Daemon({})
+        self.assertRaises(NotImplementedError, d.run_once)
+        self.assertRaises(NotImplementedError, d.run_forever)
+
+
+class TestRunDaemon(unittest.TestCase):
+
+    def setUp(self):
+        utils.HASH_PATH_SUFFIX = 'endcap'
+        utils.drop_privileges = lambda *args: None
+        utils.capture_stdio = lambda *args: None
+
+    def tearDown(self):
+        reload(utils)
+
+    def test_run(self):
+        d = MyDaemon({})
+        self.assertFalse(MyDaemon.forever_called)
+        self.assertFalse(MyDaemon.once_called)
+        # test default
+        d.run()
+        self.assertEquals(d.forever_called, True)
+        # test once
+        d.run(once=True)
+        self.assertEquals(d.once_called, True)
+
+    def test_run_daemon(self):
+        sample_conf = """[my-daemon]
+user = %s
+""" % getuser()
+        with tmpfile(sample_conf) as conf_file:
+            daemon.run_daemon(MyDaemon, conf_file)
+            self.assertEquals(MyDaemon.forever_called, True)
+            daemon.run_daemon(MyDaemon, conf_file, once=True)
+            self.assertEquals(MyDaemon.once_called, True)
+
+            # test raise in daemon code
+            MyDaemon.run_once = MyDaemon.run_raise
+            self.assertRaises(OSError, daemon.run_daemon, MyDaemon,
+                              conf_file, once=True)
+
+            # test user quit
+            MyDaemon.run_forever = MyDaemon.run_quit
+            sio = StringIO()
+            logger = logging.getLogger()
+            logger.addHandler(logging.StreamHandler(sio))
+            logger = utils.get_logger(None, 'server')
+            daemon.run_daemon(MyDaemon, conf_file, logger=logger)
+            self.assert_('user quit' in sio.getvalue().lower())
 
 
 if __name__ == '__main__':

=== modified file 'test/unit/common/test_utils.py'
--- test/unit/common/test_utils.py	2010-11-01 21:47:48 +0000
+++ test/unit/common/test_utils.py	2010-11-19 22:38:08 +0000
@@ -25,12 +25,55 @@
 from getpass import getuser
 from shutil import rmtree
 from StringIO import StringIO
+from functools import partial
+from tempfile import NamedTemporaryFile
 
 from eventlet import sleep
 
 from swift.common import utils
 
 
+class MockOs():
+    def __init__(self, pass_funcs=[], called_funcs=[], raise_funcs=[]):
+        self.closed_fds = []
+        for func in pass_funcs:
+            setattr(self, func, self.pass_func)
+        self.called_funcs = {}
+        for func in called_funcs:
+            c_func = partial(self.called_func, func)
+            setattr(self, func, c_func)
+        for func in raise_funcs:
+            r_func = partial(self.raise_func, func)
+            setattr(self, func, r_func)
+
+    def pass_func(self, *args, **kwargs):
+        pass
+
+    chdir = setsid = setgid = setuid = umask = pass_func
+
+    def called_func(self, name, *args, **kwargs):
+        self.called_funcs[name] = True
+
+    def raise_func(self, name, *args, **kwargs):
+        self.called_funcs[name] = True
+        raise OSError()
+
+    def dup2(self, source, target):
+        self.closed_fds.append(target)
+
+    def __getattr__(self, name):
+        # I only over-ride portions of the os module
+        try:
+            return object.__getattr__(self, name)
+        except AttributeError:
+            return getattr(os, name)
+
+
+class MockSys():
+
+    __stderr__ = sys.__stderr__
+
+
 class TestUtils(unittest.TestCase):
     """ Tests for swift.common.utils """
 
@@ -182,10 +225,63 @@
         self.assertRaises(IOError, lfo.readline, 1024)
         lfo.tell()
 
-    def test_drop_privileges(self):
-        # Note that this doesn't really drop privileges as it just sets them to
-        # what they already are; but it exercises the code at least.
-        utils.drop_privileges(getuser())
+    def test_parse_options(self):
+        # use mkstemp to get a file that is definately on disk
+        with NamedTemporaryFile() as f:
+            conf_file = f.name
+            conf, options = utils.parse_options(test_args=[conf_file])
+            self.assertEquals(conf, conf_file)
+            # assert defaults
+            self.assertEquals(options['verbose'], False)
+            self.assert_('once' not in options)
+            # assert verbose as option
+            conf, options = utils.parse_options(test_args=[conf_file, '-v'])
+            self.assertEquals(options['verbose'], True)
+            # check once option
+            conf, options = utils.parse_options(test_args=[conf_file],
+                                                once=True)
+            self.assertEquals(options['once'], False)
+            test_args = [conf_file, '--once']
+            conf, options = utils.parse_options(test_args=test_args, once=True)
+            self.assertEquals(options['once'], True)
+            # check options as arg parsing
+            test_args = [conf_file, 'once', 'plugin_name', 'verbose']
+            conf, options = utils.parse_options(test_args=test_args, once=True)
+            self.assertEquals(options['verbose'], True)
+            self.assertEquals(options['once'], True)
+            self.assertEquals(options['extra_args'], ['plugin_name'])
+
+    def test_parse_options_errors(self):
+        orig_stdout = sys.stdout
+        orig_stderr = sys.stderr
+        stdo = StringIO()
+        stde = StringIO()
+        utils.sys.stdout = stdo
+        utils.sys.stderr = stde
+        err_msg = """Usage: test usage
+
+Error: missing config file argument
+"""
+        test_args = []
+        self.assertRaises(SystemExit, utils.parse_options, 'test usage', True,
+                          test_args)
+        self.assertEquals(stdo.getvalue(), err_msg)
+
+        # verify conf file must exist, context manager will delete temp file
+        with NamedTemporaryFile() as f:
+            conf_file = f.name
+        err_msg += """Usage: test usage
+
+Error: unable to locate %s
+""" % conf_file
+        test_args = [conf_file]
+        self.assertRaises(SystemExit, utils.parse_options, 'test usage', True,
+                          test_args)
+        self.assertEquals(stdo.getvalue(), err_msg)
+
+        # reset stdio
+        utils.sys.stdout = orig_stdout
+        utils.sys.stderr = orig_stderr
 
     def test_NamedLogger(self):
         sio = StringIO()
@@ -275,5 +371,80 @@
         self.assertEquals(result, expected)
         os.unlink('/tmp/test')
 
+    def test_drop_privileges(self):
+        user = getuser()
+        # over-ride os with mock
+        required_func_calls = ('setgid', 'setuid', 'setsid', 'chdir', 'umask')
+        utils.os = MockOs(called_funcs=required_func_calls)
+        # exercise the code
+        utils.drop_privileges(user)
+        for func in required_func_calls:
+            self.assert_(utils.os.called_funcs[func])
+
+        # reset; test same args, OSError trying to get session leader
+        utils.os = MockOs(called_funcs=required_func_calls,
+                          raise_funcs=('setsid',))
+        for func in required_func_calls:
+            self.assertFalse(utils.os.called_funcs.get(func, False))
+        utils.drop_privileges(user)
+        for func in required_func_calls:
+            self.assert_(utils.os.called_funcs[func])
+
+    def test_capture_stdio(self):
+        # stubs
+        logger = utils.get_logger(None, 'dummy')
+
+        # mock utils system modules
+        utils.sys = MockSys()
+        utils.os = MockOs()
+
+        # basic test
+        utils.capture_stdio(logger)
+        self.assert_(utils.sys.excepthook is not None)
+        self.assertEquals(utils.os.closed_fds, [0, 1, 2])
+        self.assert_(utils.sys.stdout is not None)
+        self.assert_(utils.sys.stderr is not None)
+
+        # reset; test same args, but exc when trying to close stdio
+        utils.os = MockOs(raise_funcs=('dup2',))
+        utils.sys = MockSys()
+
+        # test unable to close stdio
+        utils.capture_stdio(logger)
+        self.assert_(utils.sys.excepthook is not None)
+        self.assertEquals(utils.os.closed_fds, [])
+        self.assert_(utils.sys.stdout is not None)
+        self.assert_(utils.sys.stderr is not None)
+
+        # reset; test some other args
+        logger = utils.get_logger(None, log_to_console=True)
+        utils.os = MockOs()
+        utils.sys = MockSys()
+
+        # test console log
+        utils.capture_stdio(logger, capture_stdout=False,
+                            capture_stderr=False)
+        self.assert_(utils.sys.excepthook is not None)
+        # when logging to console, stderr remains open
+        self.assertEquals(utils.os.closed_fds, [0, 1])
+        logger.logger.removeHandler(utils.get_logger.console)
+        # stdio not captured
+        self.assertFalse(hasattr(utils.sys, 'stdout'))
+        self.assertFalse(hasattr(utils.sys, 'stderr'))
+
+    def test_get_logger_console(self):
+        reload(utils)  # reset get_logger attrs
+        logger = utils.get_logger(None)
+        self.assertFalse(hasattr(utils.get_logger, 'console'))
+        logger = utils.get_logger(None, log_to_console=True)
+        self.assert_(hasattr(utils.get_logger, 'console'))
+        self.assert_(isinstance(utils.get_logger.console,
+                                logging.StreamHandler))
+        # make sure you can't have two console handlers
+        old_handler = utils.get_logger.console
+        logger = utils.get_logger(None, log_to_console=True)
+        self.assertNotEquals(utils.get_logger.console, old_handler)
+        logger.logger.removeHandler(utils.get_logger.console)
+
 if __name__ == '__main__':
     unittest.main()

=== modified file 'test/unit/common/test_wsgi.py'
--- test/unit/common/test_wsgi.py	2010-07-19 16:25:18 +0000
+++ test/unit/common/test_wsgi.py	2010-11-19 22:38:08 +0000
@@ -25,12 +25,12 @@
 from getpass import getuser
 from shutil import rmtree
 from StringIO import StringIO
+from collections import defaultdict
 
 from eventlet import sleep
 
 from swift.common import wsgi
 
-
 class TestWSGI(unittest.TestCase):
     """ Tests for swift.common.wsgi """
 
@@ -72,5 +72,107 @@
         sio = StringIO('Content-Type: text/html; charset=ISO-8859-4')
         self.assertEquals(mimetools.Message(sio).subtype, 'html')
 
+    def test_get_socket(self):
+        # stubs
+        conf = {}
+        ssl_conf = {
+            'cert_file': '',
+            'key_file': '',
+        }
+
+        # mocks
+        class MockSocket():
+            def __init__(self):
+                self.opts = defaultdict(dict)
+
+            def setsockopt(self, level, optname, value):
+                self.opts[level][optname] = value
+
+        def mock_listen(*args, **kwargs):
+            return MockSocket()
+
+        class MockSsl():
+            def __init__(self):
+                self.wrap_socket_called = []
+
+            def wrap_socket(self, sock, **kwargs):
+                self.wrap_socket_called.append(kwargs)
+                return sock
+
+        # patch
+        old_listen = wsgi.listen
+        old_ssl = wsgi.ssl
+        try:
+            wsgi.listen = mock_listen
+            wsgi.ssl = MockSsl()
+            # test
+            sock = wsgi.get_socket(conf)
+            # assert
+            self.assert_(isinstance(sock, MockSocket))
+            expected_socket_opts = {
+                socket.SOL_SOCKET: {
+                    socket.SO_REUSEADDR: 1,
+                    socket.SO_KEEPALIVE: 1,
+                },
+                socket.IPPROTO_TCP: {
+                    socket.TCP_KEEPIDLE: 600,
+                },
+            }
+            self.assertEquals(sock.opts, expected_socket_opts)
+            # test ssl
+            sock = wsgi.get_socket(ssl_conf)
+            expected_kwargs = {
+                'certfile': '',
+                'keyfile': '',
+            }
+            self.assertEquals(wsgi.ssl.wrap_socket_called, [expected_kwargs])
+        finally:
+            wsgi.listen = old_listen
+            wsgi.ssl = old_ssl
+
+    def test_address_in_use(self):
+        # stubs
+        conf = {}
+
+        # mocks
+        def mock_listen(*args, **kwargs):
+            raise socket.error(errno.EADDRINUSE)
+
+        def value_error_listen(*args, **kwargs):
+            raise ValueError('fake')
+
+        def mock_sleep(*args):
+            pass
+
+        class MockTime():
+            """Fast clock advances 10 seconds after every call to time
+            """
+            def __init__(self):
+                self.current_time = old_time.time()
+
+            def time(self, *args, **kwargs):
+                rv = self.current_time
+                # advance for next call
+                self.current_time += 10
+                return rv
+
+        old_listen = wsgi.listen
+        old_sleep = wsgi.sleep
+        old_time = wsgi.time
+        try:
+            wsgi.listen = mock_listen
+            wsgi.sleep = mock_sleep
+            wsgi.time = MockTime()
+            # test error
+            self.assertRaises(Exception, wsgi.get_socket, conf)
+            # different error
+            wsgi.listen = value_error_listen
+            self.assertRaises(ValueError, wsgi.get_socket, conf)
+        finally:
+            wsgi.listen = old_listen
+            wsgi.sleep = old_sleep
+            wsgi.time = old_time
+
+
 if __name__ == '__main__':
     unittest.main()

=== modified file 'test/unit/stats/test_log_processor.py'
--- test/unit/stats/test_log_processor.py	2010-11-17 15:36:21 +0000
+++ test/unit/stats/test_log_processor.py	2010-11-19 22:38:08 +0000
@@ -14,25 +14,12 @@
 # limitations under the License.
 
 import unittest
-import os
-from contextlib import contextmanager
-from tempfile import NamedTemporaryFile
+from test.unit import tmpfile
 
 from swift.common import internal_proxy
 from swift.stats import log_processor
 
 
-@contextmanager
-def tmpfile(content):
-    with NamedTemporaryFile('w', delete=False) as f:
-        file_name = f.name
-        f.write(str(content))
-    try:
-        yield file_name
-    finally:
-        os.unlink(file_name)
-
-
 class FakeUploadApp(object):
     def __init__(self, *args, **kwargs):
         pass