Merge ~cjwatson/launchpad:sourcedeps-codetree into launchpad:master

[Colin Watson]

Replace most of devscripts.sourcecode with codetree

This drops some bespoke code and potentially lets us migrate individual sourcedeps to git.

We'll need to install python-codetree in various places first (this is too early for us to be able to rely on a Python dependency rather than a system dependency).

This is essentially the same as https://code.launchpad.net/~cjwatson/launchpad/sourcedeps-codetree/+merge/333073, converted to git and rebased on master.

[Colin Watson (cjwatson)]

Superseded by https://code.launchpad.net/~cjwatson/launchpad/+git/launchpad/+merge/428729.

Unmerged commits

8c94b91... by Colin Watson on 2017-11-01

Replace most of devscripts.sourcecode with codetree.

dc26ef8... by Colin Watson on 2017-10-31

Drop support for private/optional sourcedeps, unused since r12871.

c6b5d56... by Colin Watson on 2017-10-31

Remove utilities/sourcedeps.filter, unused since r8510.

diff --git a/lib/devscripts/sourcecode.py b/lib/devscripts/sourcecode.py
index 5cb6686..22bc84e 100644
--- a/lib/devscripts/sourcecode.py
+++ b/lib/devscripts/sourcecode.py
@@ -1,136 +1,37 @@
-# Copyright 2009 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Tools for maintaining the Launchpad source code."""
 
+from __future__ import absolute_import, print_function, unicode_literals
+
 __metaclass__ = type
 __all__ = [
-    'interpret_config',
-    'parse_config_file',
-    'plan_update',
+    'main',
     ]
 
-import errno
-import json
+import logging
 import optparse
 import os
 import shutil
-import sys
 
-from bzrlib import ui
-from bzrlib.branch import Branch
-from bzrlib.errors import (
-    BzrError,
-    IncompatibleRepositories,
-    NotBranchError,
-    )
-from bzrlib.plugin import load_plugins
-from bzrlib.revisionspec import RevisionSpec
-from bzrlib.trace import (
-    enable_default_logging,
-    report_exception,
-    )
-from bzrlib.upgrade import upgrade
-from bzrlib.workingtree import WorkingTree
+from codetree.config import Config
+from codetree.handlers.bzr import BzrSourceHandler
 
 from devscripts import get_launchpad_root
 
 
-def parse_config_file(file_handle):
-    """Parse the source code config file 'file_handle'.
-
-    :param file_handle: A file-like object containing sourcecode
-        configuration.
-    :return: A sequence of lines of either '[key, value]' or
-        '[key, value, optional]'.
-    """
-    for line in file_handle:
-        if line == '\n' or line.startswith('#'):
-            continue
-        yield line.split()
-
-
-def interpret_config_entry(entry, use_http=False):
-    """Interpret a single parsed line from the config file."""
-    branch_name = entry[0]
-    components = entry[1].split(';revno=')
-    branch_url = components[0]
-    if use_http:
-        branch_url = branch_url.replace('lp:', 'http://bazaar.launchpad.net/')
-    if len(components) == 1:
-        revision = None
-    else:
-        assert len(components) == 2, 'Bad branch URL: ' + entry[1]
-        revision = components[1] or None
-    if len(entry) > 2:
-        assert len(entry) == 3 and entry[2].lower() == 'optional', (
-            'Bad configuration line: should be space delimited values of '
-            'sourcecode directory name, branch URL [, "optional"]\n' +
-            ' '.join(entry))
-        optional = True
-    else:
-        optional = False
-    return branch_name, branch_url, revision, optional
-
-
-def load_cache(cache_filename):
-    try:
-        cache_file = open(cache_filename, 'rb')
-    except IOError as e:
-        if e.errno == errno.ENOENT:
-            return {}
-        else:
-            raise
-    with cache_file:
-        return json.load(cache_file)
-
-
-def interpret_config(config_entries, public_only, use_http=False):
-    """Interpret a configuration stream, as parsed by 'parse_config_file'.
-
-    :param configuration: A sequence of parsed configuration entries.
-    :param public_only: If true, ignore private/optional branches.
-    :param use_http: If True, force all branch URLs to use http://
-    :return: A dict mapping the names of the sourcecode dependencies to a
-        2-tuple of their branches and whether or not they are optional.
-    """
-    config = {}
-    for entry in config_entries:
-        branch_name, branch_url, revision, optional = interpret_config_entry(
-            entry, use_http)
-        if not optional or not public_only:
-            config[branch_name] = (branch_url, revision, optional)
-    return config
-
-
-def _subset_dict(d, keys):
-    """Return a dict that's a subset of 'd', based on the keys in 'keys'."""
-    return dict((key, d[key]) for key in keys)
-
-
-def plan_update(existing_branches, configuration):
+def plan_update(existing_branches, config):
     """Plan the update to existing branches based on 'configuration'.
 
     :param existing_branches: A sequence of branches that already exist.
-    :param configuration: A dictionary of sourcecode configuration, such as is
-        returned by `interpret_config`.
-    :return: (new_branches, update_branches, removed_branches), where
-        'new_branches' are the branches in the configuration that don't exist
-        yet, 'update_branches' are the branches in the configuration that do
-        exist, and 'removed_branches' are the branches that exist locally, but
-        not in the configuration. 'new_branches' and 'update_branches' are
-        dicts of the same form as 'configuration', 'removed_branches' is a
-        set of the same form as 'existing_branches'.
+    :param config: An instance of `codetree.Config`.
+    :return: a set of the branches that exist locally but not in the
+        configuration.
     """
     existing_branches = set(existing_branches)
-    config_branches = set(configuration.keys())
-    new_branches = config_branches - existing_branches
-    removed_branches = existing_branches - config_branches
-    update_branches = config_branches.intersection(existing_branches)
-    return (
-        _subset_dict(configuration, new_branches),
-        _subset_dict(configuration, update_branches),
-        removed_branches)
+    config_branches = set(config.directive_map)
+    return existing_branches - config_branches
 
 
 def find_branches(directory):
@@ -139,200 +40,50 @@ def find_branches(directory):
     for name in os.listdir(directory):
         if name in ('.', '..'):
             continue
-        try:
-            Branch.open(os.path.join(directory, name))
-            branches.append(name)
-        except NotBranchError:
-            pass
+        for subdir in ('.bzr', '.git'):
+            if os.path.exists(os.path.join(directory, name, subdir)):
+                branches.append(name)
     return branches
 
 
-def get_revision_id(revision, from_branch, tip=False):
-    """Return revision id for a revision number and a branch.
-
-    If the revision is empty, the revision_id will be None.
-
-    If ``tip`` is True, the revision value will be ignored.
-    """
-    if not tip and revision:
-        spec = RevisionSpec.from_string(revision)
-        return spec.as_revision_id(from_branch)
-    # else return None
-
-
-def _format_revision_name(revision, tip=False):
-    """Formatting helper to return human-readable identifier for revision.
-
-    If ``tip`` is True, the revision value will be ignored.
-    """
-    if not tip and revision:
-        return 'revision %s' % (revision,)
-    else:
-        return 'tip'
-
-
-def get_branches(sourcecode_directory, new_branches,
-                 possible_transports=None, tip=False, quiet=False):
-    """Get the new branches into sourcecode."""
-    for project, (branch_url, revision, optional) in new_branches.iteritems():
-        destination = os.path.join(sourcecode_directory, project)
-        try:
-            remote_branch = Branch.open(
-                branch_url, possible_transports=possible_transports)
-        except BzrError:
-            if optional:
-                report_exception(sys.exc_info(), sys.stderr)
-                continue
-            else:
-                raise
-        possible_transports.append(
-            remote_branch.bzrdir.root_transport)
-        if not quiet:
-            print 'Getting %s from %s at %s' % (
-                    project, branch_url, _format_revision_name(revision, tip))
-        # If the 'optional' flag is set, then it's a branch that shares
-        # history with Launchpad, so we should share repositories. Otherwise,
-        # we should avoid sharing repositories to avoid format
-        # incompatibilities.
-        force_new_repo = not optional
-        revision_id = get_revision_id(revision, remote_branch, tip)
-        remote_branch.bzrdir.sprout(
-            destination, revision_id=revision_id, create_tree_if_local=True,
-            source_branch=remote_branch, force_new_repo=force_new_repo,
-            possible_transports=possible_transports)
-
-
-def find_stale(updated, cache, sourcecode_directory, quiet):
-    """Find branches whose revision info doesn't match the cache."""
-    new_updated = dict(updated)
-    for project, (branch_url, revision, optional) in updated.iteritems():
-        cache_revision_info = cache.get(project)
-        if cache_revision_info is None:
-            continue
-        if cache_revision_info[0] != int(revision):
-            continue
-        destination = os.path.join(sourcecode_directory, project)
-        try:
-            branch = Branch.open(destination)
-        except BzrError:
-            continue
-        if list(branch.last_revision_info()) != cache_revision_info:
-            continue
-        if not quiet:
-            print '%s is already up to date.' % project
-        del new_updated[project]
-    return new_updated
-
-
-def update_cache(cache, cache_filename, changed, sourcecode_directory, quiet):
-    """Update the cache with the changed branches."""
-    old_cache = dict(cache)
-    for project, (branch_url, revision, optional) in changed.iteritems():
-        destination = os.path.join(sourcecode_directory, project)
-        branch = Branch.open(destination)
-        cache[project] = list(branch.last_revision_info())
-    if cache == old_cache:
-        return
-    with open(cache_filename, 'wb') as cache_file:
-        json.dump(cache, cache_file, indent=4, sort_keys=True)
-    if not quiet:
-        print 'Cache updated.  Please commit "%s".' % cache_filename
-
-
-def update_branches(sourcecode_directory, update_branches,
-                    possible_transports=None, tip=False, quiet=False):
-    """Update the existing branches in sourcecode."""
-    if possible_transports is None:
-        possible_transports = []
-    # XXX: JonathanLange 2009-11-09: Rather than updating one branch after
-    # another, we could instead try to get them in parallel.
-    for project, (branch_url, revision, optional) in (
-        update_branches.iteritems()):
-        # Update project from branch_url.
-        destination = os.path.join(sourcecode_directory, project)
-        if not quiet:
-            print 'Updating %s to %s' % (
-                    project, _format_revision_name(revision, tip))
-        local_tree = WorkingTree.open(destination)
-        try:
-            remote_branch = Branch.open(
-                branch_url, possible_transports=possible_transports)
-        except BzrError:
-            if optional:
-                report_exception(sys.exc_info(), sys.stderr)
-                continue
-            else:
-                raise
-        possible_transports.append(
-            remote_branch.bzrdir.root_transport)
-        revision_id = get_revision_id(revision, remote_branch, tip)
-        try:
-            result = local_tree.pull(
-                remote_branch, stop_revision=revision_id, overwrite=True,
-                possible_transports=possible_transports)
-        except IncompatibleRepositories:
-            # XXX JRV 20100407: Ideally remote_branch.bzrdir._format
-            # should be passed into upgrade() to ensure the format is the same
-            # locally and remotely. Unfortunately smart server branches
-            # have their _format set to RemoteFormat rather than an actual
-            # format instance.
-            upgrade(destination)
-            # Upgraded, repoen working tree
-            local_tree = WorkingTree.open(destination)
-            result = local_tree.pull(
-                remote_branch, stop_revision=revision_id, overwrite=True,
-                possible_transports=possible_transports)
-        if result.old_revid == result.new_revid:
-            if not quiet:
-                print '  (No change)'
-        else:
-            if result.old_revno < result.new_revno:
-                change = 'Updated'
-            else:
-                change = 'Reverted'
-            if not quiet:
-                print '  (%s from %s to %s)' % (
-                    change, result.old_revno, result.new_revno)
-
-
-def remove_branches(sourcecode_directory, removed_branches, quiet=False):
+def remove_branches(sourcecode_directory, removed_branches):
     """Remove sourcecode that's no longer there."""
     for project in removed_branches:
         destination = os.path.join(sourcecode_directory, project)
-        if not quiet:
-            print 'Removing %s' % project
+        logging.info('Removing %s', project)
         try:
             shutil.rmtree(destination)
         except OSError:
             os.unlink(destination)
 
 
-def update_sourcecode(sourcecode_directory, config_filename, cache_filename,
-                      public_only, tip, dry_run, quiet=False, use_http=False):
-    """Update the sourcecode."""
-    config_file = open(config_filename)
-    config = interpret_config(
-        parse_config_file(config_file), public_only, use_http)
-    config_file.close()
-    cache = load_cache(cache_filename)
+def mangle_config(sourcecode_directory, config, tip=False, use_http=False):
+    for directive in config.directive_map.values():
+        directive.location = os.path.join(
+            sourcecode_directory, directive.location)
+        if tip:
+            directive.source_options.pop('revno', None)
+        if use_http:
+            handler = directive.source
+            if (isinstance(handler, BzrSourceHandler) and
+                    handler.source.startswith('lp:')):
+                handler.source = handler.source.replace(
+                    'lp:', 'http://bazaar.launchpad.net/')
+
+
+def update_sourcecode(sourcecode_directory, config_filename,
+                      tip=False, dry_run=False, use_http=False):
+    config = Config([config_filename])
+    mangle_config(sourcecode_directory, config, tip=tip, use_http=use_http)
     branches = find_branches(sourcecode_directory)
-    new, updated, removed = plan_update(branches, config)
-    possible_transports = []
+    removed = plan_update(branches, config)
+    # XXX cjwatson 2017-10-31: If we start pulling sourcedeps from git, then
+    # we need to remove old bzr branches first.
+    config.build(dry_run=dry_run)
     if dry_run:
-        print 'Branches to fetch:', new.keys()
-        print 'Branches to update:', updated.keys()
-        print 'Branches to remove:', list(removed)
+        logging.info('Branches to remove: %s', list(removed))
     else:
-        get_branches(
-            sourcecode_directory, new, possible_transports, tip, quiet)
-        updated = find_stale(updated, cache, sourcecode_directory, quiet)
-        update_branches(
-            sourcecode_directory, updated, possible_transports, tip, quiet)
-        changed = dict(updated)
-        changed.update(new)
-        update_cache(
-            cache, cache_filename, changed, sourcecode_directory, quiet)
-        remove_branches(sourcecode_directory, removed, quiet)
+        remove_branches(sourcecode_directory, removed)
 
 
 # XXX: JonathanLange 2009-09-11: By default, the script will operate on the
@@ -349,9 +100,6 @@ def update_sourcecode(sourcecode_directory, config_filename, cache_filename,
 def main(args):
     parser = optparse.OptionParser("usage: %prog [options] [root [conffile]]")
     parser.add_option(
-        '--public-only', action='store_true',
-        help='Only fetch/update the public sourcecode branches.')
-    parser.add_option(
         '--tip', action='store_true',
         help='Ignore revision constraints for all branches and pull tip')
     parser.add_option(
@@ -374,20 +122,14 @@ def main(args):
         config_filename = args[2]
     else:
         config_filename = os.path.join(root, 'utilities', 'sourcedeps.conf')
-    cache_filename = os.path.join(
-        root, 'utilities', 'sourcedeps.cache')
     if len(args) > 3:
         parser.error("Too many arguments.")
-    if not options.quiet:
-        print 'Sourcecode: %s' % (sourcecode_directory,)
-        print 'Config: %s' % (config_filename,)
-    enable_default_logging()
-    # Tell bzr to use the terminal (if any) to show progress bars
-    ui.ui_factory = ui.make_ui_for_terminal(
-        sys.stdin, sys.stdout, sys.stderr)
-    load_plugins()
+    logging.basicConfig(
+        format='%(message)s',
+        level=logging.CRITICAL if options.quiet else logging.INFO)
+    logging.info('Sourcecode: %s', sourcecode_directory)
+    logging.info('Config: %s', config_filename)
     update_sourcecode(
-        sourcecode_directory, config_filename, cache_filename,
-        options.public_only, options.tip, options.dry_run, options.quiet,
-        options.use_http)
+        sourcecode_directory, config_filename,
+        tip=options.tip, dry_run=options.dry_run, use_http=options.use_http)
     return 0
diff --git a/lib/devscripts/tests/test_sourcecode.py b/lib/devscripts/tests/test_sourcecode.py
index a675333..01ab8f9 100644
--- a/lib/devscripts/tests/test_sourcecode.py
+++ b/lib/devscripts/tests/test_sourcecode.py
@@ -1,221 +1,164 @@
-# Copyright 2009 Canonical Ltd.  This software is licensed under the
+# Copyright 2009-2017 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Module docstring goes here."""
 
+from __future__ import absolute_import, print_function, unicode_literals
+
 __metaclass__ = type
 
 import os
-import shutil
-from StringIO import StringIO
 import tempfile
-import unittest
 
-from bzrlib.bzrdir import BzrDir
-from bzrlib.tests import TestCase
-from bzrlib.transport import get_transport
+from codetree.config import Config
+from fixtures import (
+    FakeLogger,
+    MonkeyPatch,
+    TempDir,
+    )
+import six
+from testtools import TestCase
 
 from devscripts import get_launchpad_root
 from devscripts.sourcecode import (
     find_branches,
-    interpret_config,
-    parse_config_file,
+    mangle_config,
     plan_update,
+    update_sourcecode,
     )
 
 
-class TestParseConfigFile(unittest.TestCase):
-    """Tests for the config file parser."""
+def make_config(lines):
+    with tempfile.NamedTemporaryFile('w') as config_file:
+        for line in lines:
+            print(line, file=config_file)
+        config_file.flush()
+        return Config([config_file.name])
 
-    def makeFile(self, contents):
-        return StringIO(contents)
 
-    def test_empty(self):
-        # Parsing an empty config file returns an empty sequence.
-        empty_file = self.makeFile("")
-        self.assertEqual([], list(parse_config_file(empty_file)))
+class TestMangleConfig(TestCase):
+    """Tests for mangling configuration after codetree has parsed it."""
 
-    def test_single_value(self):
-        # Parsing a file containing a single key=value pair returns a sequence
-        # containing the (key, value) as a list.
-        config_file = self.makeFile("key value")
+    def setUp(self):
+        super(TestMangleConfig, self).setUp()
+        self.tempdir = self.useFixture(TempDir()).path
+
+    def test_location(self):
+        # All locations are considered to be relative to the given
+        # sourcecode directory.
+        config = make_config(['key lp:~sabdfl/foo/trunk;revno=1'])
+        mangle_config(self.tempdir, config)
         self.assertEqual(
-            [['key', 'value']], list(parse_config_file(config_file)))
-
-    def test_comment_ignored(self):
-        # If a line begins with a '#', then its a comment.
-        comment_only = self.makeFile('# foo')
-        self.assertEqual([], list(parse_config_file(comment_only)))
-
-    def test_optional_value(self):
-        # Lines in the config file can have a third optional entry.
-        config_file = self.makeFile('key value optional')
+            os.path.join(self.tempdir, 'key'),
+            config.directive_map['key'].location)
+
+    def test_tip_false(self):
+        # If tip=False is passed to mangle_config, revno options are left
+        # untouched.
+        config = make_config(['key lp:~sabdfl/foo/trunk;revno=1'])
+        mangle_config(self.tempdir, config, tip=False)
         self.assertEqual(
-            [['key', 'value', 'optional']],
-            list(parse_config_file(config_file)))
-
-    def test_whitespace_stripped(self):
-        # Any whitespace around any of the tokens in the config file are
-        # stripped out.
-        config_file = self.makeFile('  key   value    optional   ')
+            {'revno': '1'}, config.directive_map['key'].source_options)
+
+    def test_tip_true(self):
+        # If tip=True is passed to mangle_config, revno options are removed.
+        config = make_config(['key lp:~sabdfl/foo/trunk;revno=1'])
+        mangle_config(self.tempdir, config, tip=True)
+        self.assertEqual({}, config.directive_map['key'].source_options)
+
+    def test_use_http_false(self):
+        # If use_http=False is passed to mangle_config, lp: branch URLs are
+        # left untouched.
+        url_path = '~sabdfl/foo/trunk'
+        config = make_config(['key lp:%s' % url_path])
+        mangle_config(self.tempdir, config, use_http=False)
         self.assertEqual(
-            [['key', 'value', 'optional']],
-            list(parse_config_file(config_file)))
-
-
-class TestInterpretConfiguration(unittest.TestCase):
-    """Tests for the configuration interpreter."""
-
-    def test_empty(self):
-        # An empty configuration stream means no configuration.
-        config = interpret_config([], False)
-        self.assertEqual({}, config)
-
-    def test_key_value(self):
-        # A (key, value) pair without a third optional value is returned in
-        # the configuration as a dictionary entry under 'key' with '(value,
-        # None, False)' as its value.
-        config = interpret_config([['key', 'value']], False)
-        self.assertEqual({'key': ('value', None, False)}, config)
-
-    def test_key_value_public_only(self):
-        # A (key, value) pair without a third optional value is returned in
-        # the configuration as a dictionary entry under 'key' with '(value,
-        # None, False)' as its value when public_only is true.
-        config = interpret_config([['key', 'value']], True)
-        self.assertEqual({'key': ('value', None, False)}, config)
-
-    def test_key_value_optional(self):
-        # A (key, value, optional) entry is returned in the configuration as a
-        # dictionary entry under 'key' with '(value, True)' as its value.
-        config = interpret_config([['key', 'value', 'optional']], False)
-        self.assertEqual({'key': ('value', None, True)}, config)
-
-    def test_key_value_optional_public_only(self):
-        # A (key, value, optional) entry is not returned in the configuration
-        # when public_only is true.
-        config = interpret_config([['key', 'value', 'optional']], True)
-        self.assertEqual({}, config)
-
-    def test_key_value_revision(self):
-        # A (key, value) pair without a third optional value when the
-        # value has a suffix of ``;revno=[REVISION]`` is returned in the
-        # configuration as a dictionary entry under 'key' with '(value,
-        # None, False)' as its value.
-        config = interpret_config([['key', 'value;revno=45']], False)
-        self.assertEqual({'key': ('value', '45', False)}, config)
-
-    def test_key_value_revision(self):
-        # A (key, value) pair without a third optional value when the
-        # value has multiple suffixes of ``;revno=[REVISION]`` raises an
-        # error.
-        self.assertRaises(
-            AssertionError,
-            interpret_config, [['key', 'value;revno=45;revno=47']], False)
-
-    def test_too_many_values(self):
-        # A line with too many values raises an error.
-        self.assertRaises(
-            AssertionError,
-            interpret_config, [['key', 'value', 'optional', 'extra']], False)
-
-    def test_bad_optional_value(self):
-        # A third value that is not the "optional" string raises an error.
-        self.assertRaises(
-            AssertionError,
-            interpret_config, [['key', 'value', 'extra']], False)
-
-    def test_use_http(self):
-        # If use_http=True is passed to interpret_config, all lp: branch
-        # URLs will be transformed into http:// URLs.
-        config = interpret_config(
-            [['key', 'lp:~sabdfl/foo/trunk']], False, use_http=True)
-        expected_url = 'http://bazaar.launchpad.net/~sabdfl/foo/trunk'
-        self.assertEqual(expected_url, config['key'][0])
-
-
-class TestPlanUpdate(unittest.TestCase):
+            'lp:%s' % url_path, config.directive_map['key'].source.source)
+
+    def test_use_http_true(self):
+        # If use_http=True is passed to mangle_config, lp: branch URLs are
+        # transformed into http:// URLs.
+        url_path = '~sabdfl/foo/trunk'
+        config = make_config(['key lp:%s' % url_path])
+        mangle_config(self.tempdir, config, use_http=True)
+        self.assertEqual(
+            'http://bazaar.launchpad.net/%s' % url_path,
+            config.directive_map['key'].source.source)
+
+
+class TestPlanUpdate(TestCase):
     """Tests for how to plan the update."""
 
     def test_trivial(self):
         # In the trivial case, there are no existing branches and no
-        # configured branches, so there are no branches to add, none to
-        # update, and none to remove.
-        new, existing, removed = plan_update([], {})
-        self.assertEqual({}, new)
-        self.assertEqual({}, existing)
+        # configured branches, so there are none to remove.
+        removed = plan_update([], make_config([]))
         self.assertEqual(set(), removed)
 
     def test_all_new(self):
-        # If there are no existing branches, then the all of the configured
-        # branches are new, none are existing and none have been removed.
-        new, existing, removed = plan_update([], {'a': ('b', False)})
-        self.assertEqual({'a': ('b', False)}, new)
-        self.assertEqual({}, existing)
+        # If there are no existing branches, then none have been removed.
+        removed = plan_update([], make_config(['a lp:a']))
         self.assertEqual(set(), removed)
 
     def test_all_old(self):
-        # If there configuration is now empty, but there are existing
-        # branches, then that means all the branches have been removed from
-        # the configuration, none are new and none are updated.
-        new, existing, removed = plan_update(['a', 'b', 'c'], {})
-        self.assertEqual({}, new)
-        self.assertEqual({}, existing)
+        # If the configuration is now empty but there are existing branches,
+        # then that means all the branches have been removed from the
+        # configuration.
+        removed = plan_update(['a', 'b', 'c'], make_config([]))
         self.assertEqual(set(['a', 'b', 'c']), removed)
 
     def test_all_same(self):
         # If the set of existing branches is the same as the set of
-        # non-existing branches, then they all need to be updated.
-        config = {'a': ('b', False), 'c': ('d', True)}
-        new, existing, removed = plan_update(config.keys(), config)
-        self.assertEqual({}, new)
-        self.assertEqual(config, existing)
+        # non-existing branches, then none have been removed.
+        config = make_config(['a lp:a', 'b lp:b'])
+        removed = plan_update(config.directive_map.keys(), config)
         self.assertEqual(set(), removed)
 
-    def test_smoke_the_default_config(self):
-        # Make sure we can parse, interpret and plan based on the default
-        # config file.
-        root = get_launchpad_root()
-        config_filename = os.path.join(root, 'utilities', 'sourcedeps.conf')
-        config_file = open(config_filename)
-        config = interpret_config(parse_config_file(config_file), False)
-        config_file.close()
-        plan_update([], config)
+    def test_some_old(self):
+        # If there are existing branches not in the configuration, then
+        # those branches have been removed.
+        removed = plan_update(['a', 'b', 'c'], make_config(['a lp:a']))
+        self.assertEqual(set(['b', 'c']), removed)
 
 
 class TestFindBranches(TestCase):
     """Tests the way that we find branches."""
 
     def setUp(self):
-        TestCase.setUp(self)
-        self.disable_directory_isolation()
-
-    def makeBranch(self, path):
-        transport = get_transport(path)
-        transport.ensure_base()
-        BzrDir.create_branch_convenience(
-            transport.base, possible_transports=[transport])
-
-    def makeDirectory(self):
-        directory = tempfile.mkdtemp()
-        self.addCleanup(shutil.rmtree, directory)
-        return directory
+        super(TestFindBranches, self).setUp()
+        self.tempdir = self.useFixture(TempDir()).path
 
     def test_empty_directory_has_no_branches(self):
         # An empty directory has no branches.
-        empty = self.makeDirectory()
-        self.assertEqual([], list(find_branches(empty)))
+        self.assertEqual(set(), set(find_branches(self.tempdir)))
 
     def test_directory_with_branches(self):
         # find_branches finds branches in the directory.
-        directory = self.makeDirectory()
-        self.makeBranch('%s/a' % directory)
-        self.assertEqual(['a'], list(find_branches(directory)))
+        os.makedirs(os.path.join(self.tempdir, 'a', '.bzr'))
+        os.makedirs(os.path.join(self.tempdir, 'b', '.git'))
+        self.assertEqual(set(['a', 'b']), set(find_branches(self.tempdir)))
+
+    def test_ignores_non_branch_directory(self):
+        # find_branches ignores any subdirectories in the directory which
+        # are not branches.
+        os.mkdir(os.path.join(self.tempdir, 'a'))
+        self.assertEqual(set(), set(find_branches(self.tempdir)))
 
     def test_ignores_files(self):
         # find_branches ignores any files in the directory.
-        directory = self.makeDirectory()
-        some_file = open('%s/a' % directory, 'w')
-        some_file.write('hello\n')
-        some_file.close()
-        self.assertEqual([], list(find_branches(directory)))
+        with open(os.path.join(self.tempdir, 'a'), 'w') as some_file:
+            some_file.write('hello\n')
+        self.assertEqual(set(), set(find_branches(self.tempdir)))
+
+
+class TestSmoke(TestCase):
+    """Smoke tests."""
+
+    def test_smoke_the_default_config(self):
+        # Make sure we can do a dry run based on the default config file.
+        self.useFixture(FakeLogger())
+        self.useFixture(MonkeyPatch('sys.stdout', six.StringIO()))
+        root = get_launchpad_root()
+        config_filename = os.path.join(root, 'utilities', 'sourcedeps.conf')
+        fake_sourcecode = self.useFixture(TempDir()).path
+        update_sourcecode(fake_sourcecode, config_filename, dry_run=True)
diff --git a/utilities/sourcedeps.cache b/utilities/sourcedeps.cache
deleted file mode 100644
index 1360489..0000000
--- a/utilities/sourcedeps.cache
+++ /dev/null
@@ -1,42 +0,0 @@
-{
-    "bzr-builder": [
-        70, 
-        "[e-mail]-20111114140506-6bmt9isw6lcud7yt"
-    ], 
-    "bzr-git": [
-        280, 
-        "[e-mail]-20171222005919-u98ut0f5z2g618um"
-    ], 
-    "bzr-loom": [
-        55, 
-        "[e-mail]-20120830090804-cg49kky93htwax7s"
-    ], 
-    "bzr-svn": [
-        2725, 
-        "[e-mail]-20130816045016-wzr810hu2z459t4y"
-    ], 
-    "cscvs": [
-        433, 
-        "[e-mail]-20130816043319-bts3l3bckmx431q1"
-    ], 
-    "difftacular": [
-        11, 
-        "[e-mail]-20190614154330-091l9edcnubsjmsx"
-    ], 
-    "loggerhead": [
-        493, 
-        "[e-mail]-20190621112125-3aaxj3hrmty19lr6"
-    ], 
-    "mailman": [
-        977, 
-        "[e-mail]-20130405041235-9ud0xancja2eefd7"
-    ], 
-    "old_xmlplus": [
-        4, 
-        "sinzui-20090526164636-1swugzupwvjgomo4"
-    ], 
-    "pygettextpo": [
-        25, 
-        "[e-mail]-20140116030912-lqm1dtb6a0y4femq"
-    ]
-}
\ No newline at end of file
diff --git a/utilities/sourcedeps.filter b/utilities/sourcedeps.filter
deleted file mode 100644
index d81ca05..0000000
--- a/utilities/sourcedeps.filter
+++ /dev/null
@@ -1,3 +0,0 @@
-P *.o
-P *.pyc
-P *.so