Merge lp:~renamer-developers/renamer/config-file into lp:renamer

[Jonathan Jacobs]

[lp:~renamer-developers/renamer/config-file]

126. By Jonathan Jacobs on 2010-10-24

Tweak function name.

127. By Jonathan Jacobs on 2010-10-24

Merge trunk.

[Tristan Seligmann (mithrandi)]

> 36 ('concurrent', 'l', 10,

I think a better long name for this option might be "concurrency".

review: Approve

=== modified file 'renamer/_version.py'
--- renamer/_version.py	2009-04-29 00:06:49 +0000
+++ renamer/_version.py	2010-10-24 17:47:46 +0000
@@ -1,3 +1,3 @@
 from twisted.python.versions import Version
 
-version = Version('Renamer', 1, 0, 0)
+version = Version('Renamer', 0, 1, 0)

=== modified file 'renamer/application.py'
--- renamer/application.py	2010-10-20 00:38:33 +0000
+++ renamer/application.py	2010-10-24 17:47:46 +0000
@@ -9,10 +9,10 @@
 from axiom.store import Store
 
 from twisted.internet import defer
-from twisted.python import usage, log
+from twisted.python import usage, log, versions
 from twisted.python.filepath import FilePath
 
-from renamer import logging, plugin, util
+from renamer import config, logging, plugin, util, version
 from renamer.irenamer import IRenamingCommand
 from renamer.history import History
 
@@ -28,12 +28,14 @@
 
 
     optParameters = [
+        ('config', 'c', '~/.renamer/renamer.conf',
+         'Configuration file path.'),
         ('name',   'e', None,
          'Formatted filename.', string.Template),
         ('prefix', 'p', None,
          'Formatted path to prefix to files before renaming.', string.Template),
         ('concurrent', 'l',  10,
-         'Maximum number of concurrent tasks to perform at a time.', int)]
+         'Maximum number of asynchronous tasks to perform concurrently.', int)]
 
 
     @property
@@ -42,14 +44,19 @@
             plugin.getRenamingCommands(), plugin.getCommands())
         for plg in commands:
             try:
-                yield plg.name, None, plg, plg.description
+                yield (
+                    plg.name,
+                    None,
+                    config.defaultsFromConfigFactory(self.config, plg),
+                    plg.description)
             except AttributeError:
                 raise RuntimeError('Malformed plugin: %r' % (plg,))
 
 
-    def __init__(self):
-        usage.Options.__init__(self)
+    def __init__(self, config):
+        super(Options, self).__init__()
         self['verbosity'] = 1
+        self.config = config
 
 
     @property
@@ -76,6 +83,14 @@
     opt_q = opt_quiet
 
 
+    def opt_version(self):
+        """
+        Display version information.
+        """
+        print versions.getVersionString(version)
+        sys.exit(0)
+
+
     def parseArgs(self, *args):
         args = (self.decodeCommandLine(arg) for arg in args)
         if self['glob']:
@@ -101,24 +116,53 @@
     @ivar command: Renamer command being executed.
     """
     def __init__(self):
-        obs = logging.RenamerObserver()
-        log.startLoggingWithObserver(obs.emit, setStdout=False)
+        self._obs = logging.RenamerObserver()
+        log.startLoggingWithObserver(self._obs.emit, setStdout=False)
 
+        self.options = self.parseOptions()
         self.store = Store(os.path.expanduser('~/.renamer/renamer.axiom'))
         # XXX: One day there might be more than one History item.
         self.history = self.store.findOrCreate(History)
 
-        self.options = Options()
-        self.options.parseOptions()
-        obs.verbosity = self.options['verbosity']
-        self.command = self.getCommand()
-
-
-    def getCommand(self):
+        self.args = getattr(self.options, 'args', [])
+        self.command = self.getCommand(self.options)
+
+
+    def parseOptions(self):
+        """
+        Parse configuration file and command-line options.
+        """
+        _options = Options({})
+        _options.parseOptions()
+        self._obs.verbosity = _options['verbosity']
+
+        self._configFile = config.ConfigFile(
+            FilePath(os.path.expanduser(_options['config'])))
+        command = self.getCommand(_options)
+
+        options = Options(self._configFile)
+        # Apply global defaults.
+        options.update(self._configFile.get('renamer', options))
+        # Apply command-specific overrides for the global config.
+        options.update(
+            (k, v) for k, v in
+            self._configFile.get(command.name, options).iteritems()
+            if k in options)
+        # Command-line options trump the config file.
+        options.parseOptions()
+
+        logging.msg(
+            'Global options: %r' % (options,),
+            verbosity=5)
+
+        return options
+
+
+    def getCommand(self, options):
         """
         Get the L{twisted.python.usage.Options} command that was invoked.
         """
-        command = getattr(self.options, 'subOptions', None)
+        command = getattr(options, 'subOptions', None)
         if command is None:
             raise usage.UsageError('At least one command must be specified')
 
@@ -132,8 +176,7 @@
         """
         Perform a file rename.
         """
-        options = self.options
-        if options['no-act']:
+        if self.options['no-act']:
             logging.msg('Simulating: %s => %s' % (src.path, dst.path))
             return
 
@@ -141,25 +184,31 @@
             logging.msg('Skipping noop "%s"' % (src.path,), verbosity=2)
             return
 
-        if options['link-dst']:
+        if self.options['link-dst']:
             self.changeset.do(
                 self.changeset.newAction(u'symlink', src, dst),
-                options)
+                self.options)
         else:
             self.changeset.do(
                 self.changeset.newAction(u'move', src, dst),
-                options)
-            if options['link-src']:
+                self.options)
+            if self.options['link-src']:
                 self.changeset.do(
                     self.changeset.newAction(u'symlink', dst, src),
-                    options)
+                    self.options)
 
 
     def runCommand(self, command):
         """
         Run a generic command.
         """
-        return defer.maybeDeferred(command.process, self)
+        logging.msg(
+            'Using command "%s"' % (command.name,),
+            verbosity=4)
+        logging.msg(
+            'Command options: %r' % (command,),
+            verbosity=5)
+        return defer.maybeDeferred(command.process, self, self.options)
 
 
     def runRenamingCommand(self, command):
@@ -178,7 +227,7 @@
                 self.options['concurrent'],),
             verbosity=3)
         return util.parallel(
-            self.options.args, self.options['concurrent'], _processOne)
+            self.args, self.options['concurrent'], _processOne)
 
 
     def run(self):

=== added file 'renamer/config.py'
--- renamer/config.py	1970-01-01 00:00:00 +0000
+++ renamer/config.py	2010-10-24 17:47:46 +0000
@@ -0,0 +1,132 @@
+from ConfigParser import SafeConfigParser
+
+from renamer import util
+
+
+
+_identity = lambda x: x
+
+
+
+def flag(value):
+    """
+    Transform the flag C{value} into a C{bool}.
+    """
+    if isinstance(value, str):
+        value = unicode(value, 'ascii')
+    if isinstance(value, unicode):
+        return value.lower() in (u'true', u'yes', u'1')
+    return bool(value)
+
+
+
+def transformersFromOptions(options):
+    """
+    Extract option transformers from an C{Options} subclass.
+
+    @rtype: C{iterable} of C{(str, callable)}
+    @return: Iterable of C{(longOption, transformer)}.
+    """
+    flags = getattr(options, 'optFlags', [])
+    for long, short, desc in flags:
+        yield long, flag
+
+    parameters = getattr(options, 'optParameters', [])
+    parameters = (util.padIterable(ps, _identity, 5) for ps in parameters)
+    global _identity
+    for long, short, default, desc, coerce in parameters:
+        if coerce is None:
+            coerce = _identity
+        yield long, coerce
+
+
+
+def defaultsFromConfigFactory(config, commandClass):
+    """
+    Create a factory function that will create an C{ICommand} provider instance
+    and apply defaults from a config file.
+
+    @type  config: L{renamer.config.ConfigFile}
+    @param config: Config file to use defaults from.
+
+    @type  commandClass: C{type}
+    @param commandClass: L{renamer.irenamer.ICommand} provider to apply config
+        defaults to.
+
+    @return: C{callable} suitable for passing as the third (parser class)
+        argument to C{"subCommands"}.
+    """
+    def initWrapper():
+        conf = config.get(commandClass.name, {})
+
+        def _params():
+            parameters = getattr(commandClass, 'optParameters', [])
+            parameters = (util.padIterable(p, None, 5) for p in parameters)
+            for long, short, default, desc, coerce in parameters:
+                if long in conf:
+                    default = conf.get(long)
+                    if default is not None and coerce is not None:
+                        default = coerce(default)
+                yield long, short, default, desc, coerce
+
+        commandClass.optParameters = list(_params())
+        cmd = commandClass()
+
+        # Since we can't set defaults for flags, we have to apply the
+        # defaults after __init__ and before parseOptions.
+        flags = getattr(commandClass, 'optFlags', [])
+        for long, short, desc in flags:
+            if long in conf:
+                cmd[long] = flag(conf[long])
+
+        return cmd
+
+    return initWrapper
+
+
+
+class ConfigFile(object):
+    """
+    Configuration file.
+
+    @type sections: C{dict} mapping C{str} to C{dict} mapping C{str} to C{str}.
+    @ivar sections: Mapping of section names to mappings of configuration
+        options to values.
+    """
+    def __init__(self, path):
+        config = SafeConfigParser()
+        if path.exists():
+            fd = path.open()
+            config.readfp(fd)
+            fd.close()
+
+        self.sections = {}
+        for section in config.sections():
+            options = self.sections.setdefault(section, {})
+            for option in config.options(section):
+                options[option] = unicode(config.get(section, option))
+
+
+    def get(self, section, options=None):
+        """
+        Get and coerce configuration values.
+
+        @type  section: C{str}
+        @param section: Section name to retrieve configuration values for.
+
+        @type  options: L{twisted.python.usage.Options}
+        @param options: Options to use for determining coercion types, or
+            C{None} to perform no coercions.
+
+        @return: Mapping of configuration option names to their coerced values.
+        """
+        section = self.sections.get(section, {})
+        if options is not None:
+            for opt, optType in transformersFromOptions(options):
+                value = section.get(opt)
+                if value is not None:
+                    if optType is not None:
+                        value = optType(value)
+                    section[opt] = value
+
+        return section

=== modified file 'renamer/irenamer.py'
--- renamer/irenamer.py	2010-10-16 16:35:10 +0000
+++ renamer/irenamer.py	2010-10-24 17:47:46 +0000
@@ -16,9 +16,13 @@
     """)
 
 
-    def process(renamer):
+    def process(renamer, options):
         """
         Called once command line parsing is complete.
+
+        @type renamer: L{renamer.application.Renamer}
+
+        @type options: C{dict}
         """
 
 
@@ -67,7 +71,7 @@
         """
         Perform the action.
 
-        @type  options: L{twisted.python.usage.Options}
+        @type options: C{dict}
         """
 
 
@@ -75,5 +79,5 @@
         """
         Perform the reverse action.
 
-        @type  options: L{twisted.python.usage.Options}
+        @type options: C{dict}
         """

=== modified file 'renamer/plugin.py'
--- renamer/plugin.py	2010-10-16 16:35:10 +0000
+++ renamer/plugin.py	2010-10-24 17:47:46 +0000
@@ -72,7 +72,7 @@
 
     # ICommand
 
-    def process(self, renamer):
+    def process(self, renamer, options):
         raise NotImplementedError('Commands must implement "process"')
 
 
@@ -137,7 +137,7 @@
         @param mapping: Mapping of template variables, used for template
             substitution.
 
-        @type  options: L{twisted.python.usage.Options}
+        @type  options: C{dict}
 
         @type  src: L{twisted.python.filepath.FilePath}
         @param src: Source path.
@@ -179,12 +179,12 @@
 
     # ICommand
 
-    def process(self, renamer):
+    def process(self, renamer, options):
         arg = renamer.currentArgument
         logging.msg('Processing "%s"' % (arg.path,),
                     verbosity=3)
         d = defer.maybeDeferred(self.processArgument, arg)
-        d.addCallback(self.buildDestination, renamer.options, arg)
+        d.addCallback(self.buildDestination, options, arg)
         return d
 
 

=== modified file 'renamer/plugins/undo.py'
--- renamer/plugins/undo.py	2010-10-18 15:12:54 +0000
+++ renamer/plugins/undo.py	2010-10-24 17:47:46 +0000
@@ -30,19 +30,19 @@
         ('ignore-errors', None, 'Do not stop the process when encountering OS errors')]
 
 
-    def undoActions(self, renamer, changeset, actions):
+    def undoActions(self, options, changeset, actions):
         """
         Undo specific actions from a changeset.
         """
         for action in actions:
             msg = 'Simulating undo'
-            if not renamer.options['no-act']:
+            if not options['no-act']:
                 msg = 'Undo'
 
             logging.msg('%s: %s' % (msg, action.asHumanly()), verbosity=3)
-            if not renamer.options['no-act']:
+            if not options['no-act']:
                 try:
-                    changeset.undo(action, renamer.options)
+                    changeset.undo(action, options)
                 except OSError, e:
                     if not self['ignore-errors']:
                         raise e
@@ -67,10 +67,9 @@
         self['action'] = action
 
 
-    def process(self, renamer):
+    def process(self, renamer, options):
         action = getItem(renamer.store, self['action'], Action)
-        self.undoActions(
-            renamer, action.changeset, [action])
+        self.undoActions(options, action.changeset, [action])
 
 
 
@@ -90,13 +89,12 @@
         self['changeset'] = changeset
 
 
-    def process(self, renamer):
+    def process(self, renamer, options):
         changeset = getItem(renamer.store, self['changeset'], Changeset)
         logging.msg('Undoing: %s' % (changeset.asHumanly(),),
                     verbosity=3)
         actions = list(changeset.getActions())
-        self.undoActions(
-            renamer, changeset, reversed(actions))
+        self.undoActions(options, changeset, reversed(actions))
 
 
 
@@ -109,7 +107,7 @@
     """
 
 
-    def process(self, renamer):
+    def process(self, renamer, options):
         changesets = list(renamer.history.getChangesets())
         for cs in changesets:
             print 'Changeset ID=%d:  %s' % (cs.storeID, cs.asHumanly())
@@ -139,9 +137,9 @@
         self['identifier'] = identifier
 
 
-    def process(self, renamer):
+    def process(self, renamer, options):
         item = getItem(renamer.store, self['identifier'], (Action, Changeset))
-        if not renamer.options['no-act']:
+        if not options['no-act']:
             logging.msg('Forgetting: %s' % (item.asHumanly(),), verbosity=2)
             item.deleteFromStore()
 

=== added file 'renamer/test/data/test.conf'
--- renamer/test/data/test.conf	1970-01-01 00:00:00 +0000
+++ renamer/test/data/test.conf	2010-10-24 17:47:46 +0000
@@ -0,0 +1,8 @@
+[foo]
+bar=1
+baz=apple
+
+[test]
+aardvark=hello
+bobcat=3
+donut=no

=== modified file 'renamer/test/test_actions.py'
--- renamer/test/test_actions.py	2010-10-16 20:46:40 +0000
+++ renamer/test/test_actions.py	2010-10-24 17:47:46 +0000
@@ -14,7 +14,7 @@
     def setUp(self):
         self.path = FilePath(self.mktemp())
         self.path.makedirs()
-        self.options = Options()
+        self.options = Options(None)
         self.src, self.dst = self.createFiles()
 
 

=== added file 'renamer/test/test_config.py'
--- renamer/test/test_config.py	1970-01-01 00:00:00 +0000
+++ renamer/test/test_config.py	2010-10-24 17:47:46 +0000
@@ -0,0 +1,127 @@
+from twisted.python.filepath import FilePath
+from twisted.trial.unittest import TestCase
+
+from renamer import config, plugin
+
+
+
+class TestCommand(plugin.Command):
+    name = 'test'
+
+
+    optParameters = [
+        ('aardvark',  'a', None, 'desc'),
+        ('bobcat',    'b', 5,    'desc', int),
+        ('chocolate', 'c', None, 'desc')]
+
+
+    optFlags = [
+        ('donut', 'd', 'desc')]
+
+
+
+class ConfigTests(TestCase):
+    """
+    Tests for L{renamer.config}.
+    """
+    def setUp(self):
+        path = FilePath(__file__).sibling('data').child('test.conf')
+        self.config = config.ConfigFile(path)
+
+
+    def test_flag(self):
+        """
+        L{renamer.config.flag} returns C{True} for true flag values such as
+        C{'yes'}, C{'true'}, C{'1'} and otherwise C{False}.
+        """
+        truths = [
+            'yes', u'yes', 'YES', u'true', '1', 1]
+        for value in truths:
+            self.assertTrue(config.flag(value))
+
+        falsehoods = [
+            'no', u'NO', 'arst', 0, []]
+        for value in falsehoods:
+            self.assertFalse(config.flag(value))
+
+
+    def test_transformersFromOptions(self):
+        """
+        L{renamer.config.transformersFromOptions} extracts coercion functions
+        from a L{renamer.irenamer.ICommand} provider.
+        """
+        expected = {
+            'aardvark':  config._identity,
+            'bobcat':    int,
+            'chocolate': config._identity,
+            'donut':     config.flag}
+        res = dict(config.transformersFromOptions(TestCommand()))
+        self.assertEquals(expected, res)
+
+
+    def test_defaultsFromConfigFactory(self):
+        """
+        L{renamer.config.defaultsFromConfigFactory} creates a wrapper function
+        around a L{renamer.irenamer.ICommand} that, when called, instantiates
+        the L{renamer.irenamer.ICommand} and sets default values from a config
+        file from a section with a name matched
+        L{renamer.irenamer.ICommand.name}. Options that are explicitly provided
+        to the command trump values from the config file.
+        """
+        wrapper = config.defaultsFromConfigFactory(self.config, TestCommand)
+        cmd = wrapper()
+        cmd.parseOptions([])
+        self.assertEquals(cmd['aardvark'], u'hello')
+        self.assertEquals(cmd['bobcat'], 3)
+        self.assertIdentical(cmd['chocolate'], None)
+        self.assertEquals(cmd['donut'], False)
+
+        cmd = wrapper()
+        cmd.parseOptions(['--bobcat=7', '--donut'])
+        self.assertEquals(cmd['aardvark'], u'hello')
+        self.assertEquals(cmd['bobcat'], 7)
+        self.assertIdentical(cmd['chocolate'], None)
+        self.assertEquals(cmd['donut'], True)
+
+
+
+class ConfigFileTests(TestCase):
+    """
+    Tests for L{renamer.config.ConfigFile}.
+    """
+    def setUp(self):
+        path = FilePath(__file__).sibling('data').child('test.conf')
+        self.config = config.ConfigFile(path)
+
+
+    def test_nonexistentConfig(self):
+        """
+        Specifying a nonexistent path to L{renamer.config.ConfigFile} results
+        in an empty config object.
+        """
+        conf = config.ConfigFile(FilePath('no_such_config'))
+        self.assertEquals(conf.sections, {})
+
+
+    def test_getNoCoercion(self):
+        """
+        Getting a config section without specifying an C{'options'} argument
+        simply returns a mapping of strings to strings.
+        """
+        expected = {
+            'bar': u'1',
+            'baz': u'apple'}
+        self.assertEquals(expected, self.config.get('foo'))
+
+
+    def test_getCoercion(self):
+        """
+        Specifying an C{'options'} argument when getting a config section will
+        coerce recognized options to their corresponding type in the
+        C{'options'} argument.
+        """
+        expected = {
+            'aardvark': u'hello',
+            'bobcat':   3,
+            'donut':    False}
+        self.assertEquals(expected, self.config.get('test', TestCommand))

=== modified file 'renamer/test/test_util.py'
--- renamer/test/test_util.py	2010-10-16 16:16:58 +0000
+++ renamer/test/test_util.py	2010-10-24 17:47:46 +0000
@@ -140,3 +140,53 @@
         Interfaces are not provided by subclasses.
         """
         self.assertTrue(IThing.providedBy(Thing))
+
+
+
+class PadIterableTests(TestCase):
+    """
+    Tests for L{renamer.util.util.padIterable}.
+    """
+    def test_needsPadding(self):
+        """
+        Iterables that need padding are padded to the specified length with the
+        padding value.
+        """
+        padded = list(util.padIterable(xrange(3), -1, 5))
+        self.assertEquals(len(padded), 5)
+        self.assertEquals(padded, [0, 1, 2, -1, -1])
+
+
+    def test_exceedsPadding(self):
+        """
+        Iterables that exceed the padding amount are clipped to the specified
+        length.
+        """
+        padded = list(util.padIterable(xrange(10), -1, 5))
+        self.assertEquals(len(padded), 5)
+        self.assertEquals(padded, [0, 1, 2, 3, 4])
+
+
+    def test_negativeCount(self):
+        """
+        Negative padding counts raise C{ValueError}.
+        """
+        self.assertRaises(ValueError, util.padIterable, xrange(5), -1, -1)
+
+
+    def test_exact(self):
+        """
+        Iterables that are the same length as the padding are not padded.
+        """
+        padded = list(util.padIterable(xrange(5), -1, 5))
+        self.assertEquals(len(padded), 5)
+        self.assertEquals(padded, [0, 1, 2, 3, 4])
+
+
+    def test_empty(self):
+        """
+        Empty iterables are filled entirely with the padding value.
+        """
+        padded = list(util.padIterable([], -1, 5))
+        self.assertEquals(len(padded), 5)
+        self.assertEquals(padded, [-1, -1, -1, -1, -1])

=== modified file 'renamer/util.py'
--- renamer/util.py	2010-10-24 11:06:45 +0000
+++ renamer/util.py	2010-10-24 17:47:46 +0000
@@ -97,3 +97,23 @@
     if platform == 'win32':
         globbing = _iglobWin32
     return _glob(globbing)
+
+
+
+def padIterable(iterable, padding, count):
+    """
+    Pad C{iterable}, with C{padding}, to C{count} elements.
+
+    Iterables containing more than C{count} elements are clipped to C{count}
+    elements.
+
+    @param iterable: The iterable to iterate.
+
+    @param padding: Padding object.
+
+    @param count: The padded length.
+
+    @return: An iterable.
+    """
+    return itertools.islice(
+        itertools.chain(iterable, itertools.repeat(padding)), count)