Merge lp:~jml/pkgme-devportal/remove-database-code into lp:pkgme-devportal

[Jonathan Lange]

Remove fetch-symbol-files and all of the code it uses

This is the first branch of many.

It removes fetch-symbol-files and all of the code that it calls that
is not used by others.

Other things that are now probably safe to remove are marked up with
XXX comments, and should be deleted in future branches.

I have tried to be careful and remove absolutely everything.

[James Westby (james-w)]

review: Approve

=== modified file 'NEWS'
--- NEWS	2012-10-29 16:31:07 +0000
+++ NEWS	2012-11-13 12:03:23 +0000
@@ -2,6 +2,16 @@
 NEWS for pkgme-devportal
 ========================
 
+NEXT
+====
+
+Changes
+-------
+
+  * ``fetch-symbol-files`` and everything needed to parse symbol files and
+    maintain a database of library dependencies based on published packages
+    from Launchpad has been removed.  (Jonathan Lange)
+
 0.4.11 (2012-10-29)
 ===================
 

=== removed file 'bin/fetch-symbol-files'
--- bin/fetch-symbol-files	2012-08-30 12:35:31 +0000
+++ bin/fetch-symbol-files	1970-01-01 00:00:00 +0000
@@ -1,26 +0,0 @@
-#!/usr/bin/python2.7 -S
-
-import os
-
-join = os.path.join
-base = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
-base = os.path.dirname(base)
-
-import sys
-sys.path[0:0] = [
-    join(base, 'parts/scripts'),
-    ]
-
-
-import os
-path = sys.path[0]
-if os.environ.get('PYTHONPATH'):
-    os.environ['BUILDOUT_ORIGINAL_PYTHONPATH'] = os.environ['PYTHONPATH']
-    path = os.pathsep.join([path, os.environ['PYTHONPATH']])
-os.environ['PYTHONPATH'] = path
-import site # imports custom buildout-generated site.py
-
-import devportalbinary.database
-
-if __name__ == '__main__':
-    devportalbinary.database.main()

=== modified file 'devportalbinary/acceptance/tests/__init__.py'
--- devportalbinary/acceptance/tests/__init__.py	2012-09-17 16:20:42 +0000
+++ devportalbinary/acceptance/tests/__init__.py	2012-11-13 12:03:23 +0000
@@ -15,7 +15,10 @@
 from pkgme.debuild import build_source_package
 from pkgme.run_script import ScriptUserError
 
-from devportalbinary.testing import DatabaseFixture, IsImage
+from devportalbinary.testing import (
+    IsImage,
+    LibdepFixture,
+    )
 
 
 class TestData(Fixture):
@@ -76,20 +79,18 @@
 
     def test_gtk(self):
         """Runs successfully for a basic GTK+ application."""
-        dep_db = self.useFixture(DatabaseFixture())
-        dep_db.db.update_package("pthreads",
-                {'i386': {"libpthread.so.0": "libpthread0"}})
-        dep_db.db.update_package("eglibc",
-                {'i386': {"libc.so.6": "libc6"}})
+        self.useFixture(
+            LibdepFixture(
+                [("libpthread0", {'i386': {"libpthread.so.0": "libpthread0"}}),
+                 ("libc6", {'i386': {"libc.so.6": "libc6"}})]))
         test_data = self.useFixture(TestData("gtk"))
         run_pkgme(test_data)
         self.assertThat(test_data.path, HasFileTree({'debian/control': {}}))
 
     def test_bundled_library(self):
         """Runs successfully for a basic bundled libary."""
-        dep_db = self.useFixture(DatabaseFixture())
-        dep_db.db.update_package("eglibc",
-                {'i386': {"libc.so.6": "libc6"}})
+        self.useFixture(
+            LibdepFixture([("libc6", {'i386': {"libc.so.6": "libc6"}})]))
         test_data = self.useFixture(TestData("bundled_lib"))
         run_pkgme(test_data)
         self.assertThat(

=== modified file 'devportalbinary/aptfile.py'
--- devportalbinary/aptfile.py	2012-10-26 14:38:49 +0000
+++ devportalbinary/aptfile.py	2012-11-13 12:03:23 +0000
@@ -1,6 +1,8 @@
 # Copyright 2012 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
+# XXX: Remove this.  No longer used.
+
 __all__ = [
     'AptFilePackageDatabase',
     ]

=== modified file 'devportalbinary/database.py'
--- devportalbinary/database.py	2012-10-26 14:38:49 +0000
+++ devportalbinary/database.py	2012-11-13 12:03:23 +0000
@@ -1,24 +1,6 @@
 # Copyright 2011 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
-from contextlib import closing, contextmanager
-import errno
-from itertools import chain
-import os
-import shutil
-import tempfile
-
-from bzrlib import urlutils
-from fixtures import (
-    Fixture,
-    TempDir,
-    )
-
-from launchpadlib import (
-    uris,
-    )
-from launchpadlib.launchpad import Launchpad
-from pkgme.run_script import run_subprocess
 from storm.expr import And, Column, Select, Table
 from storm.locals import create_database, Store
 from storm.uri import URI as StormURI
@@ -28,338 +10,11 @@
     CONF_FILE_ENV_VAR,
     get_config_file_path,
     load_configuration,
-    load_configuration_with_command_line,
     )
-from .utils import download_file
 
 from libdep_service_client.client import Client
 
 
-# XXX: Historic name of this package.  Update to 'pkgme-devportal' and
-# re-authorize.
-APPLICATION_NAME = 'pkgme-binary'
-SERVICE_ROOT = uris.LPNET_SERVICE_ROOT
-
-SHLIBS_FILENAME = 'shlibs'
-SYMBOLS_FILENAME = 'symbols'
-
-
-# A list of package names that don't match lib*, but which we want
-# to scan anyway.
-PACKAGE_NAME_WHITELIST = [
-    "e2fslibs",
-    "odbcinst1debian2",
-    "python2.7-dbg",
-    "uno-libs3",
-    "zlib1g",
-    ]
-
-
-def is_library_package(url):
-    """Is ``url`` likely to contain libraries?"""
-    filename = os.path.splitext(urlutils.basename(url))[0]
-    if filename.startswith('lib'):
-        return True
-    for prefix in PACKAGE_NAME_WHITELIST:
-        if filename.startswith(prefix):
-            return True
-    return False
-
-
-class LaunchpadFixture(Fixture):
-
-    def __init__(self, application_name, service_root):
-        super(LaunchpadFixture, self).__init__()
-        self._app_name = application_name
-        self._service_root = service_root
-
-    def setUp(self):
-        super(LaunchpadFixture, self).setUp()
-        tempdir = self.useFixture(TempDir())
-        self.anonymous = Launchpad.login_anonymously(
-            self._app_name, self._service_root, tempdir.path)
-
-
-def iter_published_binaries(lp, since=None, name=None, exact_match=True):
-    architectures = load_configuration().architectures_supported
-    ubuntu = lp.distributions['ubuntu']
-    archive = ubuntu.main_archive
-    # XXX: oneiric is a puppy that is just for christmas. Specifically, it's a
-    # bug that this is looking in oneiric, should instead be looking in
-    # ... well, we don't know.
-    oneiric = ubuntu.getSeries(name_or_version='oneiric')
-    our_series = (
-        oneiric.getDistroArchSeries(archtag=tag) for tag in architectures)
-    filters = dict(status='Published')
-    if since:
-        filters['created_since_date'] = since
-    if name:
-        filters['binary_name'] = name
-    filters['exact_match'] = exact_match
-    return chain(
-        *[archive.getPublishedBinaries(distro_arch_series=series, **filters)
-          for series in our_series])
-
-
-def possible_sonames_for_shared_object(libname, version):
-    """Return the possible sonames gives info about a shared object.
-
-    :return: a set of candidate sonames (as strings).
-    """
-    candidates = set(['{0}-{1}.so'.format(libname, version)])
-    if '.' not in version:
-        candidates.add('{0}.so.{1}'.format(libname, version))
-    return candidates
-
-
-def find_file_under_dir(filename, directory):
-    """Find `filename` under `directory`.
-
-    :return: the path of the first matching file, or None if the file
-        wasn't found.
-    """
-    for dirpath, dirnames, filenames in os.walk(directory):
-        if filename in filenames:
-            return os.path.join(dirpath, filename)
-
-
-class NoSharedObject(Exception):
-
-    def __init__(self, libname, version, candidates):
-        super(NoSharedObject, self).__init__(
-            "No shared object matching {0} {1} could be found, "
-            "looked for {2}".format(libname, version, candidates))
-
-
-class TooManySharedObjects(Exception):
-
-    def __init__(self, libname, version, candidates):
-        super(TooManySharedObjects, self).__init__(
-            "Too many objects matching {0} {1} could be found, "
-            "found {2}".format(libname, version, candidates))
-
-
-
-def shared_object_info_to_soname(libname, version, directory):
-    """Resolve a (libname, version) tuple to a soname.
-
-    Using the files from a package decide on the soname that
-    the shared object info refers to. E.g. if the info is ("libfoo", "1")
-    and there is a file named "libfoo.so.1" then that is the soname.
-
-    :param directory: a directory of files to use to resolve.
-    :return: the soname as a string
-    :raises NoSharedObject: if a file corresponding to the info can't be
-        found.
-    :raises TooManySharedObjects: if multiple files matching the shared object
-        info in different ways are found (mutliple files matching in
-        the same manner are ignored.)
-    """
-    candidates = possible_sonames_for_shared_object(libname, version)
-    files = {soname: find_file_under_dir(soname, directory) for soname in  candidates}
-    found_files = dict(filter(lambda (x, y): y is not None, files.items()))
-    if len(found_files) < 1:
-        raise NoSharedObject(libname, version, candidates)
-    elif len(found_files) > 1:
-        raise TooManySharedObjects(libname, version, found_files.values())
-    return found_files.keys()[0]
-
-
-
-def libdep_mapping_for_deb(deb_file):
-    """Returns the library -> dependency mapping information from a package.
-
-    The symbols file will be read in preference to the shlibs file.
-    If neither is present an empty dict will be returned indicating
-    that there are no libraries.
-
-    If there are libraries the dict will map from the library names
-    to the package dependencies that should be used to get the
-    library, e.g.
-
-        {'libfoo.so.1': 'libfoo1', ...}
-    """
-    with extract_deb_control(deb_file) as control_dir:
-        # Try symbols
-        symbols_path = os.path.join(control_dir, SYMBOLS_FILENAME)
-        symbols = get_file_contents(symbols_path)
-        if symbols is not None:
-            return libdep_mapping_from_symbols(symbols)
-
-        # Try shlibs
-        shlibs_path = os.path.join(control_dir, SHLIBS_FILENAME)
-        shlibs = get_file_contents(shlibs_path)
-        if shlibs is not None:
-            shared_object_mapping = shared_objects_from_shlibs(shlibs)
-            if shared_object_mapping:
-                with extract_deb_content(deb_file) as deb_content:
-                    def resolve_shared_objects(map_entry):
-                        (libname, version), dependency = map_entry
-                        soname = shared_object_info_to_soname(libname, version,
-                                deb_content)
-                        return soname, dependency
-                    return dict(
-                            map(resolve_shared_objects,
-                                shared_object_mapping.items())
-                            )
-        return {}
-
-
-def deb_file_url_for_publication(bpph):
-    """Get the download URL for a binary package."""
-    version = bpph.binary_package_version
-    if ':' in version:
-        version = version[version.index(':')+1:]
-    arch = bpph.distro_arch_series.architecture_tag
-    if not bpph.architecture_specific:
-        arch = 'all'
-    return '%s/+files/%s_%s_%s.deb' % (
-        bpph.archive.web_link,
-        bpph.binary_package_name,
-        version,
-        arch,
-        )
-
-
-def get_package_info_from_publication(bpph):
-    arch = bpph.distro_arch_series.architecture_tag
-    url = deb_file_url_for_publication(bpph)
-    return url, arch
-
-
-def get_libdep_mapping_for_package(url):
-    """Return the library -> dependency mapping for the package at url.
-
-    :return: a dict mapping library names to dependencies, e.g.
-            {'libfoo.so.1': 'libfoo1', ...}
-    """
-    directory = tempfile.mkdtemp()
-    try:
-        deb_file = download_file(url, directory)
-        return libdep_mapping_for_deb(deb_file)
-    finally:
-        shutil.rmtree(directory)
-
-
-@contextmanager
-def extract_deb_control(binary_package_path):
-    """Extract a deb control archive, returning the extracted path.
-
-    This is a context manager, and the tempdir will be cleaned up when closed.
-    """
-    temp_dir = tempfile.mkdtemp()
-    try:
-        run_subprocess(['dpkg-deb', '-e', binary_package_path, temp_dir])
-        yield temp_dir
-    finally:
-        shutil.rmtree(temp_dir)
-
-
-@contextmanager
-def extract_deb_content(binary_package_path):
-    """Extract the files from a .deb package to a tempdir.
-
-    This is a context manager, the path to the tempdir will be yielded
-    to the caller, and the tempdir will be cleaned up when closed.
-    """
-    temp_dir = tempfile.mkdtemp()
-    try:
-        run_subprocess(['dpkg-deb', '-x', binary_package_path, temp_dir])
-        yield temp_dir
-    finally:
-        shutil.rmtree(temp_dir)
-
-
-def get_file_contents(path):
-    """Read the contents of the file at `path`.
-
-    :return: the contents of the file, or None if there was
-        no such file.
-    :raises: Any errors opening or reading the file.
-    """
-    try:
-        return open(path).read()
-    except (OSError, IOError), e:
-        if e.errno == errno.ENOENT:
-            return
-        raise
-
-
-def libdep_mapping_from_symbols(symbol_contents):
-    """Returns a dict mapping libraries to dependencies based on the symbols
-
-    Ignores versions and a whole bunch of other stuff and is probably the
-    wrong API even.
-    """
-
-    # XXX: This is going to yield sonames and package names for now. Really,
-    # there should be something parsing the symbols file and another thing to
-    # somehow turn those into dependencies with versions.
-
-    # Doesn't know how to handle lines like this:
-    #
-    # libformw.so.5 #PACKAGE#.
-    #
-    # This is OK, as we're only processing symbols files from binary packages
-    # rather than source packages.
-
-    mapping = {}
-    for line in symbol_contents.splitlines():
-        if not line:
-            # Blank lines are skipped
-            continue
-        if line.startswith('|'):
-            # Alternative dependency template
-            # e.g. | libgl1-mesa-glx #MINVER#
-            continue
-        if line.startswith('*'):
-            # Meta-information
-            # e.g. * Build-Depends-Package: libgl1-mesa-dev
-            continue
-        if line.startswith(' '):
-            # Symbol
-            # e.g.  gdk_add_client_message_filter@Base 2.8.0
-            continue
-        if line.startswith('#'):
-            # Lines starting with '#' are comments.  There are also DEPRECATED
-            # and MISSING, and jml doesn't really know what they mean
-            continue
-        library, dependency = tuple(line.split()[:2])
-        if '#include' in library:
-            # To skip lines that are includes.  XXX: Properly ought to process
-            # the tags that might appear before the include
-            # line.
-            #
-            # e.g. (arch=!armel)#include "libdiagnostics0.symbols.backtrace"
-            continue
-        mapping[library] = dependency
-    return mapping
-
-
-def shared_objects_from_shlibs(shlibs_contents):
-    """Get the shared object info from the shlibs file.
-
-    http://www.debian.org/doc/debian-policy/ch-sharedlibs.html#s-sharedlibs-shlibdeps
-    defines the format.
-
-    :return: a dict mapping (libname, soname) to dependencies.
-    """
-    mapping = {}
-    for line in shlibs_contents.splitlines():
-        line = line.strip()
-        if not line:
-            continue
-        if line.startswith("#"):
-            continue
-        libname, soname, dependency = line.split(' ', 2)
-        if libname.endswith(':'):
-            # This is a 'type' marker, currently only udeb is used, and
-            # we don't want those
-            continue
-        mapping[(libname, soname)] = dependency
-    return mapping
-
-
 class URI(StormURI):
     """A stand-in for Storm's URI class.
 
@@ -369,6 +24,8 @@
     have a parsed version and just need to create the object.
     """
 
+    # XXX: Only used by PackageDatabase, which is flagged for deletion.
+
     def __init__(self, scheme=None, host=None, port=None, username=None,
                  password=None, database=None, options=None):
         self.scheme = scheme
@@ -384,6 +41,8 @@
 
 class PackageDatabase(object):
 
+    # XXX: No longer used within pkgme-devportal
+
     SQLITE = 'sqlite'
     POSTGRES = 'postgres'
 
@@ -468,40 +127,6 @@
                 found[lib] = set([dependency])
         return found
 
-    def insert_new_library(self, package_name, library_name,
-                           dependency, arch):
-        """Insert a library and its needed dependency into the database.
-
-        :param library_name: A full soname, e.g. libfoo.so.1.
-        :param dependency: A binary package dependency, possibly including
-            version.
-        """
-        self._store.execute(
-            "INSERT INTO libdep VALUES (?, ?, ?, ?)",
-            (unicode(package_name),
-             unicode(library_name),
-             unicode(dependency),
-             unicode(arch)))
-
-    def update_package(self, package_name, arch_libdep_mapping):
-        """Update the database with the libdep info from 'package_name'.
-
-        :param package_name: The name of the package where the
-            symbols came from.
-        :param arch_libdep_mapping: a dict mapping architecture tags to dicts
-            mapping library names to dependencies, e.g.
-                {'amd64': {'libfoo.so.1': 'libfoo1', ...}, ...}
-        """
-        for arch, libdep_mapping in arch_libdep_mapping.items():
-            self._store.execute(
-                    "DELETE FROM libdep WHERE source_package_name = ? "
-                    "AND architecture = ?",
-                    (unicode(package_name), unicode(arch)))
-            for library, dependency in libdep_mapping.items():
-                self.insert_new_library(
-                    package_name, library, dependency, arch)
-        self._store.commit()
-
     def close(self):
         self._store.close()
 
@@ -534,57 +159,10 @@
 
 def get_dependency_database():
     """Return an object that can get dependencies."""
-    # TODO: Change this to return LibdepServiceClient sometimes
+    # XXX: Remove AptFilePackageDatabase from here and simplify the method.
     databases = {
         AptFilePackageDatabase.DB_TYPE: AptFilePackageDatabase.from_options,
-        PackageDatabase.POSTGRES: PackageDatabase.from_options,
-        PackageDatabase.SQLITE: PackageDatabase.from_options,
         LibdepServiceClient.DB_TYPE: LibdepServiceClient.from_options,
         }
     options = load_configuration()
     return databases[options.database_db_type](options)
-
-
-def dict_add(*dicts):
-    """Add dicts, with later dicts taking precedence."""
-    result = dict()
-    for d in dicts:
-        result.update(d)
-    return result
-
-
-def publishings_to_package_info(publishings):
-    """Takes a list of publishings and returns the info for the library packages.
-
-    :param publishings: an iterable of launchpadlib
-        binary_package_publishing_history objects.
-    :return: an iterable of (.deb url, arch tag) for the publishings
-        that represent a library package.
-    """
-    packages_info = map(get_package_info_from_publication, publishings)
-    return filter(
-            lambda (url, arch): is_library_package(url),
-            packages_info)
-
-
-def fetch_symbol_files(scan_mode, package_name, db):
-    """Insert the libdep info for ``package_name`` into ``db``."""
-    if scan_mode != 'binary':
-        raise AssertionError("Unsupported scan mode: {0}".format(scan_mode))
-    with LaunchpadFixture(APPLICATION_NAME, SERVICE_ROOT) as lp:
-        publishings = iter_published_binaries(lp.anonymous, name=package_name)
-        lib_packages_info = publishings_to_package_info(publishings)
-        def arch_libdep_for_package((url, arch)):
-            return {arch: get_libdep_mapping_for_package(url)}
-        arch_libdep_mapping = dict_add(
-                *map(arch_libdep_for_package, lib_packages_info))
-    db.update_package(package_name, arch_libdep_mapping)
-
-
-def main():
-    """Import the symbol files for 'package_name'."""
-    glue = load_configuration_with_command_line()
-    with closing(PackageDatabase.get_store_from_config(glue.options)) as store:
-        package_name = glue.args[0]
-        db = PackageDatabase(store)
-        fetch_symbol_files(glue.options.scan_mode, package_name, db)

=== modified file 'devportalbinary/testing.py'
--- devportalbinary/testing.py	2012-09-12 08:45:38 +0000
+++ devportalbinary/testing.py	2012-11-13 12:03:23 +0000
@@ -33,10 +33,17 @@
     )
 
 from devportalbinary.binary import MetadataBackend
-from devportalbinary.database import PackageDatabase, URI
+from devportalbinary.database import (
+    LibdepServiceClient,
+    PackageDatabase,
+    URI,
+    )
 
 from devportalbinary.configuration import CONF_FILE_ENV_VAR
 
+from djlibdep.test_double import LibdepServiceDouble
+from libdep_service_client.client import Client
+
 
 class IsChildPath(Matcher):
 
@@ -235,6 +242,23 @@
         self.useFixture(ConfigFileFixture(config_file_path))
 
 
+class LibdepFixture(Fixture):
+
+    def __init__(self, data):
+        super(LibdepFixture, self).__init__()
+        self._data = data
+
+    def setUp(self):
+        super(LibdepFixture, self).setUp()
+        double = self.useFixture(LibdepServiceDouble(self._data))
+        self.useFixture(
+            ConfigSettings(
+                ('database', {'db_type': 'libdep-service',
+                              'base_url': double.base_url,
+                              })))
+        self.client = LibdepServiceClient(Client(double.base_url))
+
+
 def LibsConfigSettings(test_libs):
     """Create a lib_overrides config file."""
     return ConfigSettings(
@@ -355,3 +379,8 @@
         for section, values in settings:
             f.write(make_config_section(section, values))
             f.write('\n')
+
+
+def get_libdep_service_client(fixture, test_data):
+    double = fixture.useFixture(LibdepServiceDouble(test_data))
+    return LibdepServiceClient(Client(double.base_url))

=== modified file 'devportalbinary/tests/test_binary.py'
--- devportalbinary/tests/test_binary.py	2012-10-26 11:47:52 +0000
+++ devportalbinary/tests/test_binary.py	2012-11-13 12:03:23 +0000
@@ -1,12 +1,10 @@
 # Copyright 2011-2012 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
-from contextlib import closing
 import os
 import shutil
 
 from fixtures import TempDir
-from testresources import ResourcedTestCase
 from testtools import TestCase
 from testtools.matchers import (
     SameMembers,
@@ -39,19 +37,17 @@
     UnknownDependency,
     )
 from devportalbinary.configuration import load_configuration
-from devportalbinary.database import PackageDatabase
 from devportalbinary.metadata import (
     MetadataBackend,
     )
 from devportalbinary.testing import (
     BackendTests,
     BinaryFileFixture,
-    DatabaseConfig,
     get_test_data_dir_path,
     get_test_data_file_path,
+    LibdepFixture,
     LibsConfigSettings,
     MetadataFixture,
-    postgres_db_resource,
     )
 
 
@@ -258,21 +254,16 @@
             { "libfoo.so.1" : ["."], "libbar.so.2" : ["."]}, found)
 
 
-class GuessDependenciesTests(TestCase, ResourcedTestCase):
-
-    resources = [
-        ('db_fixture', postgres_db_resource),
-        ]
+class GuessDependenciesTests(TestCase):
 
     def test_guess_dependencies(self):
-        self.useFixture(DatabaseConfig(self.db_fixture))
-        with closing(PackageDatabase(self.db_fixture.conn)) as db:
-            db.update_package('eglibc', {'i386': {'libc.so.6': 'libc6'}})
+        self.useFixture(
+            LibdepFixture([('libc6', {'i386': {'libc.so.6': 'libc6'}})]))
         deps, arch = guess_dependencies(get_test_data_dir_path('hello'))
         self.assertEqual(set(['libc6']), deps)
 
     def test_guess_dependencies_error_on_unknown_dependency(self):
-        self.useFixture(DatabaseConfig(self.db_fixture))
+        self.useFixture(LibdepFixture([]))
         e = self.assertRaises(UnknownDependency,
                 guess_dependencies, get_test_data_dir_path('hello'))
         self.assertEqual('Can\'t find dependency for "libc.so.6".', str(e))
@@ -286,17 +277,15 @@
         self.assertEqual(set(["."]), paths)
 
 
-class BinaryBackendTests(BackendTests, ResourcedTestCase):
+class BinaryBackendTests(BackendTests):
 
     BACKEND = BinaryBackend
 
-    resources = [
-        ('db_fixture', postgres_db_resource),
-        ]
-
     def setUp(self):
         super(BinaryBackendTests, self).setUp()
-        self.useFixture(DatabaseConfig(self.db_fixture))
+        self.useFixture(
+            LibdepFixture([('libc6', {'i386': {'libc.so.6': 'libc6'}}),
+                           ('libfoo', {'i386': {'libasound.so.2': 'libfoo'}})]))
 
     def test_want_with_metadata_and_binaries(self):
         # If we detect a binary, then we score 10. The way we determine if
@@ -338,8 +327,6 @@
         backend = self.make_backend()
         shutil.copy(
             get_test_data_file_path('hello', 'hello'), backend.path)
-        with closing(PackageDatabase(self.db_fixture.conn)) as db:
-            db.update_package('eglibc', {'i386': {'libc.so.6': 'libc6'}})
         deps, arch = guess_dependencies(backend.path)
         expected_deps = ', '.join(deps)
         build_deps = backend.get_build_depends()
@@ -375,8 +362,6 @@
 
     def test_get_lib_overrides_for_packages_for_libraries(self):
         # The configuration file overrides the found library dependencies.
-        with closing(PackageDatabase(self.db_fixture.conn)) as db:
-            db.update_package('foo', {'i386': {'libasound.so.2': 'libfoo'}})
         self.assertEqual(
             get_packages_for_libraries(set(["libasound.so.2"]), "i386"),
             set(["libasound2"]))
@@ -389,8 +374,6 @@
         path = self.useFixture(MetadataFixture({})).path
         self.useFixture(BinaryFileFixture(path))
         backend = self.make_backend(path)
-        with closing(PackageDatabase(self.db_fixture.conn)) as db:
-            db.update_package('eglibc', {'i386': {'libc.so.6': 'libc6'}})
         self.assertEqual(backend.get_architecture(), "i386")
 
     def test_get_extra_targets_makes_executable_executable(self):

=== modified file 'devportalbinary/tests/test_database.py'
--- devportalbinary/tests/test_database.py	2012-10-26 14:38:49 +0000
+++ devportalbinary/tests/test_database.py	2012-11-13 12:03:23 +0000
@@ -1,47 +1,27 @@
-from collections import namedtuple
 import os
 
 from fixtures import TempDir
-from storm.databases.postgres import psycopg2
-from storm.exceptions import ClosedError
 from testresources import ResourcedTestCase
 from testtools import TestCase
 from testtools.matchers import (
     Equals,
     Matcher,
     )
-from treeshape import (
-    CONTENT,
-    FileTree,
-    )
 
 from devportalbinary.database import (
     AptFilePackageDatabase,
-    deb_file_url_for_publication,
-    dict_add,
-    find_file_under_dir,
     get_dependency_database,
-    get_file_contents,
-    get_package_info_from_publication,
-    is_library_package,
     LibdepServiceClient,
-    libdep_mapping_from_symbols,
     load_configuration,
-    NoSharedObject,
-    possible_sonames_for_shared_object,
     PackageDatabase,
-    publishings_to_package_info,
-    shared_object_info_to_soname,
-    shared_objects_from_shlibs,
-    TooManySharedObjects,
     )
 from devportalbinary.testing import (
     ConfigFileFixture,
     ConfigSettings,
+    get_libdep_service_client,
     postgres_db_resource,
     )
 
-from djlibdep.test_double import LibdepServiceDouble
 from libdep_service_client.client import Client
 
 
@@ -68,76 +48,15 @@
         self.addCleanup(db.close)
         return db
 
-    def test_insert_new_library(self):
-        db = self.get_package_db()
-        db.insert_new_library('foo-src', 'libfoo.so.0', 'foo', 'i386')
-        self.assertThat(
-            "SELECT source_package_name, library, dependency, architecture FROM libdep",
-            ResultsIn(db, [('foo-src', 'libfoo.so.0', 'foo', 'i386')]))
-
-    def test_double_insert(self):
-        db = self.get_package_db()
-        db.insert_new_library('foo-src', 'libfoo.so.0', 'foo', 'i386')
-        self.assertRaises(
-            psycopg2.IntegrityError,
-            db.insert_new_library, 'foo-src', 'libfoo.so.0', 'foo', 'i386')
-
-    def test_differing_dependencies(self):
-        db = self.get_package_db()
-        db.insert_new_library('foo-src', 'libfoo.so.0', 'foo', 'i386')
-        db.insert_new_library('foo-src', 'libfoo.so.0', 'bar', 'i386')
-        deps = db.get_multiple_dependencies(['libfoo.so.0'], 'i386')
-        self.assertEqual(deps, {'libfoo.so.0': set(['foo', 'bar'])})
-
-    def test_get_dependencies(self):
-        db = self.get_package_db()
-        db.insert_new_library('foo-src', 'libfoo.so.0', 'foo', 'i386')
-        deps = db.get_multiple_dependencies(['libfoo.so.0'], 'i386')
-        self.assertEqual(deps, {'libfoo.so.0': set(['foo'])})
-
-    def test_respects_architecture(self):
-        db = self.get_package_db()
-        db.insert_new_library('foo-src', 'libfoo.so.0', 'foo', 'i386')
-        db.insert_new_library('foo-src', 'libfoo.so.0', 'foo-amd64', 'amd64')
-        deps = db.get_multiple_dependencies(['libfoo.so.0'], arch='amd64')
-        self.assertEqual(deps, {'libfoo.so.0': set(['foo-amd64'])})
-
     def test_unknown_library(self):
         db = self.get_package_db()
         deps = db.get_multiple_dependencies(['libfoo.so.0'], 'i386')
         self.assertEqual(deps, {})
 
-    def test_update_package(self):
-        db = self.get_package_db()
-        db.update_package(
-            'foo', {'i386': {'libfoo.so.1': 'foo-bin'}})
-        deps = db.get_multiple_dependencies(['libfoo.so.1'], 'i386')
-        self.assertEqual(deps, {'libfoo.so.1': set(['foo-bin'])})
-
-    def test_update_existing_package_no_libraries(self):
-        db = self.get_package_db()
-        db.update_package('foo', {'i386': {'libfoo.so.1': 'foo-bin'}})
-        # Run again, this time with no symbols, representing that a newer
-        # version of the package no longer exports any libraries.
-        db.update_package('foo', {'i386': {}})
-        deps = db.get_multiple_dependencies(['libfoo.so.1'], 'i386')
-        self.assertEqual(deps, {})
-
-    def test_update_package_two_architectures(self):
-        # If two architectures are updated separately then they
-        # shouldn't interfere
-        db = self.get_package_db()
-        db.update_package('foo', {'i386': {'libfoo.so.1': 'foo-bin'}})
-        db.update_package('foo', {'amd64': {'libfoo.so.1': 'foo-bin-amd64'}})
-        deps = db.get_multiple_dependencies(['libfoo.so.1'], arch='i386')
-        self.assertEqual(deps, {'libfoo.so.1': set(['foo-bin'])})
-
     def test_close(self):
         # Test that we can close the package db.
         db = PackageDatabase(self.db_fixture.conn)
         db.close()
-        self.assertRaises(ClosedError, db.insert_new_library, 'foo',
-                'libfoo.so.1', 'foo-bin', 'i386')
 
     def test_close_twice(self):
         # Test that we can close the package db twice with no exception.
@@ -200,264 +119,12 @@
         self.assertEqual(expected_username, uri.username)
 
 
-class FakeBPPH(object):
-
-    def __init__(self):
-        self.archive = namedtuple(
-                'Archive', 'web_link')('http://lp.net/archive')
-        self.distro_arch_series = namedtuple(
-                'DistroArchSeries', 'architecture_tag')('i386')
-        self.binary_package_name = 'foo'
-        self.binary_package_version = '1'
-        self.architecture_specific = True
-
-
-class TestDebFileUrlForPublication(TestCase):
-
-    def test_get_url(self):
-        bpph = FakeBPPH()
-        expected_url = '%s/+files/%s_%s_%s.deb' % (
-            bpph.archive.web_link,
-            bpph.binary_package_name,
-            bpph.binary_package_version,
-            bpph.distro_arch_series.architecture_tag,
-            )
-        self.assertEqual(expected_url, deb_file_url_for_publication(bpph))
-
-    def test_get_url_with_epoch(self):
-        # epochs are stripped from the version number
-        bpph = FakeBPPH()
-        bpph.binary_package_version = '1:1'
-        expected_url = '%s/+files/%s_%s_%s.deb' % (
-            bpph.archive.web_link,
-            bpph.binary_package_name,
-            '1',
-            bpph.distro_arch_series.architecture_tag,
-            )
-        self.assertEqual(expected_url, deb_file_url_for_publication(bpph))
-
-    def test_get_url_for_arch_indep(self):
-        # epochs are stripped from the version number
-        bpph = FakeBPPH()
-        bpph.architecture_specific = False
-        expected_url = '%s/+files/%s_%s_all.deb' % (
-            bpph.archive.web_link,
-            bpph.binary_package_name,
-            '1',
-            )
-        self.assertEqual(expected_url, deb_file_url_for_publication(bpph))
-
-
-class TestShlibs(TestCase):
-
-    def test_empty(self):
-        self.assertEqual({}, shared_objects_from_shlibs(""))
-
-    def test_comments(self):
-        self.assertEqual({}, shared_objects_from_shlibs("# aaaaa\n"))
-
-    def test_blank_line(self):
-        self.assertEqual({}, shared_objects_from_shlibs("\n"))
-
-    def test_whitespace_line(self):
-        self.assertEqual({}, shared_objects_from_shlibs("  \n"))
-
-    def test_udeb_skipped(self):
-        self.assertEqual({},
-                shared_objects_from_shlibs("udeb: libfoo 1 libfoo\n"))
-
-    def test_simple_soname(self):
-        self.assertEqual({('libfoo', '1'): 'libfoo'},
-                shared_objects_from_shlibs("libfoo 1 libfoo\n"))
-
-    def test_other_type_of_soname(self):
-        self.assertEqual({('libfoo', '4.8'): 'libfoo'},
-                shared_objects_from_shlibs("libfoo 4.8 libfoo\n"))
-
-
-class TestPossibleSonamesForSharedObject(TestCase):
-
-    def test_no_dot(self):
-        self.assertEqual(
-            set(['libfoo.so.1', 'libfoo-1.so']),
-            possible_sonames_for_shared_object('libfoo', '1'))
-
-    def test_dot(self):
-        self.assertEqual(
-            set(['libfoo-1.0.so']),
-            possible_sonames_for_shared_object('libfoo', '1.0'))
-
-
-class TestFindFileUnderDir(TestCase):
-
-    def test_file_missing(self):
-        tree = {}
-        path = self.useFixture(FileTree(tree)).path
-        self.assertEqual(None, find_file_under_dir('nothere', path))
-
-    def test_file_in_basedir(self):
-        filename = 'here'
-        tree = {filename: {}}
-        built_tree = self.useFixture(FileTree(tree))
-        self.assertEqual(built_tree.join(filename),
-                find_file_under_dir(filename, built_tree.path))
-
-    def test_file_in_subdir(self):
-        filename = 'here'
-        relpath = 'there/' + filename
-        tree = {relpath: {}}
-        built_tree = self.useFixture(FileTree(tree))
-        self.assertEqual(built_tree.join(relpath),
-                find_file_under_dir(filename, built_tree.path))
-
-    def test_handles_multiple_matches(self):
-        filename = 'here'
-        relpath = 'there/' + filename
-        tree = {filename: {}, relpath: {}}
-        built_tree = self.useFixture(FileTree(tree))
-        self.assertEqual(built_tree.join(filename),
-                find_file_under_dir(filename, built_tree.path))
-
-
-class TestSharedObjectInfoToSoname(TestCase):
-
-    def test_no_files(self):
-        tree = {}
-        path = self.useFixture(FileTree(tree)).path
-        self.assertRaises(NoSharedObject,
-                shared_object_info_to_soname, 'libfoo', '1', path)
-
-    def test_too_many_files(self):
-        libname = 'libfoo'
-        version = '1'
-        possible_sonames = possible_sonames_for_shared_object(libname, version)
-        tree = dict((name, {}) for name in possible_sonames)
-        path = self.useFixture(FileTree(tree)).path
-        self.assertRaises(TooManySharedObjects,
-                shared_object_info_to_soname, libname, version, path)
-
-    def test_file_found(self):
-        libname = 'libfoo'
-        version = '1'
-        expected_soname = '{0}.so.{1}'.format(libname, version)
-        tree = {expected_soname: {}}
-        path = self.useFixture(FileTree(tree)).path
-        self.assertEqual(expected_soname,
-                shared_object_info_to_soname(libname, version, path))
-
-
-class TestDictAdd(TestCase):
-
-    def test_no_dicts(self):
-        self.assertEqual({}, dict_add())
-
-    def test_one_dict(self):
-        self.assertEqual(dict(a=1), dict_add(dict(a=1)))
-
-    def test_two_dicts(self):
-        self.assertEqual(dict(a=1, b=2), dict_add(dict(a=1), dict(b=2)))
-
-    def test_precedence(self):
-        self.assertEqual(dict(a=2), dict_add(dict(a=1), dict(a=2)))
-
-
-class TestIsLibraryPackage(TestCase):
-
-    def test_lib_filename(self):
-        self.assertEqual(True,
-                is_library_package('http://launchpad.net/libfoo.deb'))
-
-    def test_whitelisted(self):
-        self.assertEqual(True,
-                is_library_package('http://launchpad.net/zlib1g.deb'))
-
-    def test_other(self):
-        self.assertEqual(False,
-                is_library_package('http://launchpad.net/bzr.deb'))
-
-
-class TestGetPackageInfoFromPublication(TestCase):
-
-    def test_returns_url_and_arch(self):
-        bpph = FakeBPPH()
-        self.assertEqual(
-            (deb_file_url_for_publication(bpph),
-                bpph.distro_arch_series.architecture_tag),
-            get_package_info_from_publication(bpph))
-
-
-class TestPublishingsToPackageInfo(TestCase):
-
-    def test_integration(self):
-        # This function is split out as it is the testable part of
-        # fetch symbols files. This tests that the functions glue
-        # together
-        publishings = [FakeBPPH(), FakeBPPH()]
-        expected_bpph = publishings[1]
-        expected_bpph.binary_package_name = 'libfoo'
-        expected_info = (deb_file_url_for_publication(expected_bpph),
-                expected_bpph.distro_arch_series.architecture_tag)
-        self.assertEqual([expected_info],
-                publishings_to_package_info(publishings))
-
-
-class TestGetFileContents(TestCase):
-
-    def test_exists(self):
-        expected_content = 'boring content\n'
-        filename = 'foo.txt'
-        tree = {filename: {CONTENT: expected_content}}
-        path = self.useFixture(FileTree(tree)).join(filename)
-        self.assertEqual(expected_content, get_file_contents(path))
-
-    def test_not_exists(self):
-        tree = {}
-        path = self.useFixture(FileTree(tree)).join('nothere')
-        self.assertEqual(None, get_file_contents(path))
-
-    def test_directory(self):
-        tree = {}
-        path = self.useFixture(FileTree(tree)).path
-        self.assertRaises(IOError, get_file_contents, path)
-
-
-class TestLibdepMappingFromSymbols(TestCase):
-
-    def test_empty(self):
-        self.assertEqual({}, libdep_mapping_from_symbols(''))
-
-    def test_blank_line_ignored(self):
-        self.assertEqual({}, libdep_mapping_from_symbols('\n'))
-
-    def test_alternate_template_ignored(self):
-        self.assertEqual({}, libdep_mapping_from_symbols('| foo\n'))
-
-    def test_meta_information_ignored(self):
-        self.assertEqual({}, libdep_mapping_from_symbols('* foo\n'))
-
-    def test_symbols_ignored(self):
-        self.assertEqual({}, libdep_mapping_from_symbols(' foo\n'))
-
-    def test_comments_ignored(self):
-        self.assertEqual({}, libdep_mapping_from_symbols('# foo\n'))
-
-    def test_include_ignored(self):
-        self.assertEqual({},
-                libdep_mapping_from_symbols('(arch=!armel)#include foo\n'))
-
-    def test_includes_mapping(self):
-        self.assertEqual({'libfoo.so.1': 'libfoo'},
-                libdep_mapping_from_symbols('libfoo.so.1 libfoo #MINVER#\n'))
-
-
 class TestLibdepServiceClient(TestCase):
 
     TEST_DATA = [('libfoo', {'i386': {'libfoo': 'libfoo-bin'}})]
 
     def test_wraps_libdep_service(self):
-        double = self.useFixture(LibdepServiceDouble(self.TEST_DATA))
-        client = Client(double.base_url)
-        wrapper = LibdepServiceClient(client)
+        wrapper = get_libdep_service_client(self, self.TEST_DATA)
         self.assertEqual(
             {'libfoo': set(['libfoo-bin'])},
             wrapper.get_multiple_dependencies(['libfoo'], 'i386'))

=== modified file 'devportalbinary/utils.py'
--- devportalbinary/utils.py	2012-09-12 15:07:16 +0000
+++ devportalbinary/utils.py	2012-11-13 12:03:23 +0000
@@ -68,6 +68,7 @@
 def download_file(url, directory=None, name=None, working_dir=None,
                   bufsize=4 * 2 ** 10, headers=None):
     """Download 'url' into 'directory'."""
+    # XXX: download_file is no longer used within pkgme-devportal
     request = Request(url)
     if headers is not None:
         for h in headers:

=== modified file 'setup.py'
--- setup.py	2012-10-18 13:36:37 +0000
+++ setup.py	2012-11-13 12:03:23 +0000
@@ -46,7 +46,6 @@
     entry_points = {
         'console_scripts': [
             'dump-apt-file-db=devportalbinary.aptfile:dump_apt_file_db',
-            'fetch-symbol-files=devportalbinary.database:main',
             'guess-executable=devportalbinary.binary:print_executable',
             'guess-deps=devportalbinary.binary:print_dependencies',
             ],