Merge lp:~jml/pkgme-devportal/remove-database-code into lp:pkgme-devportal

[Jonathan Lange]

Remove aptfile backend

Removes the apt-file backend, which we don't need any more.

[lp:~jml/pkgme-devportal/remove-database-code]

175. By Jonathan Lange on 2012-11-13

Merge versions.cfg change.

[James Westby (james-w)]

review: Approve

=== modified file 'NEWS'
--- NEWS	2012-11-13 11:25:44 +0000
+++ NEWS	2012-11-13 14:34:20 +0000
@@ -12,6 +12,9 @@
     maintain a database of library dependencies based on published packages
     from Launchpad has been removed.  (Jonathan Lange)
 
+  * ``aptfile`` backend has been removed.  (Jonathan Lange)
+
+
 0.4.11 (2012-10-29)
 ===================
 

=== removed file 'devportalbinary/aptfile.py'
--- devportalbinary/aptfile.py	2012-11-13 11:25:44 +0000
+++ devportalbinary/aptfile.py	1970-01-01 00:00:00 +0000
@@ -1,226 +0,0 @@
-# Copyright 2012 Canonical Ltd.  This software is licensed under the
-# GNU Affero General Public License version 3 (see the file LICENSE).
-
-# XXX: Remove this.  No longer used.
-
-__all__ = [
-    'AptFilePackageDatabase',
-    ]
-
-import gzip
-import urllib
-import argparse
-import os
-import re
-
-
-def make_arg_parser():
-    p = argparse.ArgumentParser()
-    p.add_argument('--cache-dir', type=str, default='cache')
-    p.add_argument('output_file', type=argparse.FileType('w'))
-    return p
-
-
-so_filename_re = re.compile(r'\.so(\.[0-9]+)*$')
-def export_database(db, stream):
-    for library, package, arch in db.iter_database():
-        if so_filename_re.search(library):
-            stream.write(','.join([package, library, package, arch]))
-            stream.write('\n')
-            stream.flush()
-
-
-def dump_apt_file_db():
-    parser = make_arg_parser()
-    args = parser.parse_args()
-    if not os.path.isdir(args.cache_dir):
-        os.path.makedirs(args.cache_dir)
-    db = AptFilePackageDatabase(args.cache_dir)
-    export_database(db, args.output_file)
-    return 0
-
-
-def iter_contents_file(contents):
-    """ Yield (full-library-path, set-of-pkgnames) from a Contents file.
-
-    It expects a line starting with "FILE" that tells it when the header ends
-    and the actual content starts.
-    """
-    found_start_marker = False
-    for line in contents:
-        if not found_start_marker:
-            if line.startswith("FILE"):
-                found_start_marker = True
-            continue
-        (path, sep, pkgs) = [s.strip() for s in line.rpartition(" ")]
-        # pkgs is formated a bit funny, e.g. universe/pkgname
-        pkgs = set([os.path.basename(pkg) for pkg in pkgs.split(",")])
-        yield (path, pkgs)
-
-
-class AptFilePackageDatabase(object):
-    """Really dumb database that just uses apt-file for local testing """
-
-    # we could also read /etc/ld.so.conf.d/*.conf but this maybe different on
-    # different distroseries especially if
-    #    server-distroseries != target-distroseries
-    #  (I wish there was ldconfig --print-search-dirs)
-    LD_SEARCH_PATH = [
-        # standards
-        "lib",
-        "usr/lib",
-        "usr/local/lib",
-        # old biarch
-        "lib32",
-        "usr/lib32",
-        # new multiarch
-        "lib/i686-linux-gnu",
-        "lib/i386-linux-gnu",
-        "lib/x86_64-linux-gnu",
-        "usr/lib/i386-linux-gnu",
-        "usr/lib/i686-linux-gnu",
-        "usr/lib/x86_64-linux-gnu",
-        # ?
-        "usr/lib/x86_64-linux-gnu/fakechroot",
-        "usr/lib/x86_64-linux-gnu/mesa",
-        "usr/lib/x86_64-linux-gnu/mesa-egl",
-        "usr/lib/i386-linux-gnu/mesa",
-        ]
-
-    DISTROSERIES = "oneiric"
-
-    # If db_type is set to this in the config, that means use this database.
-    DB_TYPE =  'aptfile'
-
-    CONTENTS_FILE_URL_LOCATION = (
-        "http://archive.ubuntu.com/ubuntu/dists/%(distroseries)s/"
-        "Contents-%(arch)s.gz")
-
-    CONTENTS_FILE = "Contents-%(distroseries)s-%(arch)s"
-
-    def __init__(self, cachedir):
-        self.cachedir = os.path.expanduser(cachedir)
-        self._distroseries_arch_cache = {}
-
-    @classmethod
-    def from_options(cls, options):
-        return cls(options.database_aptfile_cachedir)
-
-    def _get_lib_to_pkgs_mapping(self, distroseries, arch):
-        """Returns a dict of { library-name : set([pkg1,pkg2])
-
-        This function will return a dict to lookup library-name to package
-        dependencies for the given distroseries and architecture
-        """
-        if not (distroseries, arch) in self._distroseries_arch_cache:
-            self._distroseries_arch_cache[(distroseries, arch)] = \
-                      self._get_mapping_from_contents_file(distroseries, arch)
-        return self._distroseries_arch_cache[(distroseries, arch)]
-
-    def _get_contents_file_cache_path(self, distroseries, arch):
-        """Return the path in the cache for the given distroseries, arch """
-        return os.path.join(
-            self.cachedir, self.CONTENTS_FILE % {
-                'distroseries': distroseries, 'arch': arch})
-
-    def _get_contents_file_server_url(self, distroseries, arch):
-        """Return the remote server URL for the given distroseries, arch """
-        return self.CONTENTS_FILE_URL_LOCATION % {
-            'distroseries': distroseries, 'arch': arch}
-
-    def _get_mapping_from_contents_file(self, distroseries, arch):
-        """Return lib,pkgs mapping from contents file for distroseries, arch
-
-        This expects the contents file to be in the cachedir already.
-        """
-        lib_to_pkgs = {}
-        path = self._get_contents_file_cache_path(distroseries, arch)
-        with open(path) as f:
-            for path, pkgs in self._iter_contents_file(f):
-                basename = os.path.basename(path)
-                if not basename in lib_to_pkgs:
-                    lib_to_pkgs[basename] = set()
-                lib_to_pkgs[basename] |= pkgs
-        return lib_to_pkgs
-
-    def _download_contents_file_compressed(self, distroseries, arch):
-        """Downloads the content file for distroseries, arch into target """
-        # XXX: we may eventually want to merge the Contents files from
-        #      the -updates repository too in addition to the main archive
-        url = self._get_contents_file_server_url(distroseries, arch)
-        target = self._get_contents_file_cache_path(distroseries, arch)
-        compressed_target = target + os.path.splitext(url)[1]
-        # download
-        urllib.urlretrieve(url, compressed_target)
-        return compressed_target
-
-    def _iter_contents_file(self, in_file):
-        for path, pkgs in iter_contents_file(in_file):
-            if os.path.dirname(path) in self.LD_SEARCH_PATH:
-                yield path, pkgs
-
-    def _prune_contents_gz_file(self, infile, outfile):
-        """Read a compressed Contents.gz and write out a pruned version.
-
-        This will use iter_contents_file to go over infile and write
-        the relevant lines that are in the LD_SEARCH_PATH to outfile.
-        """
-        with open(outfile, "w") as outf, gzip.open(infile) as inf:
-            # first write the header
-            outf.write("FILE  LOCATION\n")
-            # then iter over all relevant lines and write them out
-            for path, pkgs in self._iter_contents_file(inf):
-                outf.write("%s %s\n" % (path, ",".join(pkgs)))
-
-    def _download_and_prepare_contents_file_if_needed(self, distroseries, arch):
-        """Ensure there is a usable Contents file in the cachedir
-
-        This will download, uncompress and prune a Conents file for
-        distroseries, arch so that get_dependencies works.
-        """
-        # mvo: We can (and should eventually) do etag/if-modified-since
-        #      matching here. But its not really important as long as
-        #      we package for stable distroseries as the Contents file
-        #      will not change
-        path = self._get_contents_file_cache_path(distroseries, arch)
-        if not os.path.exists(path):
-            compressed_contents = self._download_contents_file_compressed(
-                distroseries, arch)
-            # and prune from ~300mb to 1mb uncompressed as we are only
-            # interested in the library path parts
-            self._prune_contents_gz_file(compressed_contents, path)
-            os.remove(compressed_contents)
-
-    def iter_database(self, architectures=('i386', 'amd64'),
-                      distroseries=None):
-        """Export the database.
-
-        Yields (library, package, arch) tuples for everything that we can
-        find.
-        """
-        # XXX: Untested
-        if distroseries is None:
-            distroseries = self.DISTROSERIES
-        for arch in architectures:
-            self._download_and_prepare_contents_file_if_needed(
-                distroseries, arch)
-            mapping = self._get_lib_to_pkgs_mapping(distroseries, arch)
-            for library in mapping:
-                for package in mapping[library]:
-                    yield library, package, arch
-
-    def get_multiple_dependencies(self, library_names, arch):
-        """Get the binary packages that provide libraries.
-
-        :return: (deps, missing), where ``deps`` is a dict mapping library
-            names to sets of packages that provide them, and ``missing`` is a
-            set of library names for which no dependencies could be found.
-        """
-        self._download_and_prepare_contents_file_if_needed(
-            self.DISTROSERIES, arch)
-        lib_to_pkgs = self._get_lib_to_pkgs_mapping(self.DISTROSERIES, arch)
-        deps = ((lib, lib_to_pkgs.get(lib)) for lib in library_names)
-        return dict((lib, dep) for (lib, dep) in deps if dep)
-
-    def close(self):
-        pass

=== modified file 'devportalbinary/database.py'
--- devportalbinary/database.py	2012-11-13 11:28:53 +0000
+++ devportalbinary/database.py	2012-11-13 14:34:20 +0000
@@ -5,7 +5,6 @@
 from storm.locals import create_database, Store
 from storm.uri import URI as StormURI
 
-from .aptfile import AptFilePackageDatabase
 from .configuration import (
     CONF_FILE_ENV_VAR,
     get_config_file_path,
@@ -159,10 +158,5 @@
 
 def get_dependency_database():
     """Return an object that can get dependencies."""
-    # XXX: Remove AptFilePackageDatabase from here and simplify the method.
-    databases = {
-        AptFilePackageDatabase.DB_TYPE: AptFilePackageDatabase.from_options,
-        LibdepServiceClient.DB_TYPE: LibdepServiceClient.from_options,
-        }
     options = load_configuration()
-    return databases[options.database_db_type](options)
+    return LibdepServiceClient.from_options(options)

=== removed file 'devportalbinary/tests/test_aptfile.py'
--- devportalbinary/tests/test_aptfile.py	2012-10-26 12:21:03 +0000
+++ devportalbinary/tests/test_aptfile.py	1970-01-01 00:00:00 +0000
@@ -1,53 +0,0 @@
-import gzip
-import os
-
-from mock import patch
-from fixtures import TempDir
-from testtools import TestCase
-
-from ..aptfile import AptFilePackageDatabase
-
-
-class AptFilePackageDatabaseTestCase(TestCase):
-
-    # point to our local contents file version that is a tad smaller
-    CONTENTS_CACHE = os.path.join(
-        os.path.dirname(__file__), "data", "apt-file-backend")
-
-    def setUp(self):
-        super(AptFilePackageDatabaseTestCase, self).setUp()
-        self.db = AptFilePackageDatabase(self.CONTENTS_CACHE)
-
-    def test_read_fixture_contents_worked(self):
-        """ test that our fixture Contents file works as expected """
-        # our test DB has 4 entries in the default search path
-        self.assertEqual(
-            len(self.db._get_lib_to_pkgs_mapping("oneiric", "i386")), 4)
-
-    def test_get_dependencies(self):
-        """ Test that data from the fixture dependencies file works """
-        self.assertEqual(
-            self.db.get_multiple_dependencies(["libz.so.1"], 'i386'),
-            ({'libz.so.1': 'zlib1g'}, set()))
-
-    @patch("urllib.urlretrieve")
-    def test_lazy_downloading(self, mock_urlretrieve):
-        """ test that lazy downloading works """
-        def _put_fixture_contents_file_in_place(url, target):
-            with gzip.open(target, "w") as f:
-                f.write("""
-Some header text that is ignored
-FILE                 LOCATION
-usr/lib/libfoo.so.2  pkgfoo,pkgbar
-""")
-        tempdir = self.useFixture(TempDir())
-        db = AptFilePackageDatabase(tempdir.path)
-        mock_urlretrieve.side_effect = _put_fixture_contents_file_in_place
-        self.assertEqual(
-            db.get_multiple_dependencies(["libfoo.so.2"], arch="i386"),
-            ({'libfoo.so.2': set(["pkgfoo", "pkgbar"])}, set()))
-        self.assertEqual(len(db._get_lib_to_pkgs_mapping("oneiric", "i386")), 1)
-
-    def test_close(self):
-        # Test that there is a close method we can call
-        self.db.close()

=== modified file 'devportalbinary/tests/test_database.py'
--- devportalbinary/tests/test_database.py	2012-11-13 11:22:16 +0000
+++ devportalbinary/tests/test_database.py	2012-11-13 14:34:20 +0000
@@ -9,7 +9,6 @@
     )
 
 from devportalbinary.database import (
-    AptFilePackageDatabase,
     get_dependency_database,
     LibdepServiceClient,
     load_configuration,
@@ -78,17 +77,6 @@
         self.assertRaises(ValueError, PackageDatabase.get_db_info_from_config,
                 options)
 
-    def test_default_create_no_config(self):
-        nonexistent = self.getUniqueString()
-        self.useFixture(ConfigFileFixture(nonexistent))
-        self.assertIsInstance(
-            get_dependency_database(), AptFilePackageDatabase)
-
-    def test_default_create_empty_config(self):
-        self.useFixture(ConfigSettings())
-        self.assertIsInstance(
-            get_dependency_database(), AptFilePackageDatabase)
-
     def test_remote_service(self):
         base_url = 'http://example.com/libdep-service/'
         self.use_database_config(db_type='libdep-service', base_url=base_url)

=== modified file 'setup.py'
--- setup.py	2012-11-13 09:14:39 +0000
+++ setup.py	2012-11-13 14:34:20 +0000
@@ -36,7 +36,6 @@
     install_requires = [
         'bzr',
         'configglue',
-        'launchpadlib',
         'libdep-service-python>=0.0.5',
         'PIL',
         'pkgme>=0.4.1',
@@ -45,7 +44,6 @@
         ],
     entry_points = {
         'console_scripts': [
-            'dump-apt-file-db=devportalbinary.aptfile:dump_apt_file_db',
             'guess-executable=devportalbinary.binary:print_executable',
             'guess-deps=devportalbinary.binary:print_dependencies',
             ],