Merge lp:~jml/pkgme-devportal/apt-file-db into lp:pkgme-devportal

[Jonathan Lange]

Script to dump out the apt-file database as CSV

This script dumps out the apt-file database.

Also PEP8's some things and makes the code a little bit more understandable
to me.

[James Westby (james-w)]

Hi,

I would suggest making the script a console entry_point. Then buildout will
fill in the paths so you can just run the script. We don't need this script
in production, so we aren't forced to avoid buildout.

Thanks,

James

[lp:~jml/pkgme-devportal/apt-file-db]

128. By Jonathan Lange on 2012-09-17

Make it an entry point. Thanks James.

129. By Jonathan Lange on 2012-09-17

Whitespace

130. By Jonathan Lange on 2012-09-17

Move AptFilePackageDatabase out of database.py module.

131. By Jonathan Lange on 2012-09-17

Move apt file tests to a separate module.

[Jonathan Lange (jml)]

Good idea. Done.

While at it, I moved the aptfilepackagedatabase code into a separate module.

jml

[lp:~jml/pkgme-devportal/apt-file-db]

132. By Jonathan Lange on 2012-09-17

Remove the file in bin/ again.

[James Westby (james-w)]

review: Approve

=== added file 'devportalbinary/aptfile.py'
--- devportalbinary/aptfile.py	1970-01-01 00:00:00 +0000
+++ devportalbinary/aptfile.py	2012-09-17 15:39:29 +0000
@@ -0,0 +1,212 @@
+# Copyright 2012 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__all__ = [
+    'AptFilePackageDatabase',
+    ]
+
+import gzip
+import urllib
+import argparse
+import os
+import re
+
+
+def make_arg_parser():
+    p = argparse.ArgumentParser()
+    p.add_argument('--cache-dir', type=str, default='cache')
+    p.add_argument('output_file', type=argparse.FileType('w'))
+    return p
+
+
+so_filename_re = re.compile(r'\.so(\.[0-9]+)*$')
+def export_database(db, stream):
+    for library, package, arch in db.iter_database():
+        if so_filename_re.search(library):
+            stream.write(','.join([package, library, package, arch]))
+            stream.write('\n')
+            stream.flush()
+
+
+def dump_apt_file_db():
+    parser = make_arg_parser()
+    args = parser.parse_args()
+    if not os.path.isdir(args.cache_dir):
+        os.path.makedirs(args.cache_dir)
+    db = AptFilePackageDatabase(args.cache_dir)
+    export_database(db, args.output_file)
+    return 0
+
+
+def iter_contents_file(contents):
+    """ Yield (full-library-path, set-of-pkgnames) from a Contents file.
+
+    It expects a line starting with "FILE" that tells it when the header ends
+    and the actual content starts.
+    """
+    found_start_marker = False
+    for line in contents:
+        if not found_start_marker:
+            if line.startswith("FILE"):
+                found_start_marker = True
+            continue
+        (path, sep, pkgs) = [s.strip() for s in line.rpartition(" ")]
+        # pkgs is formated a bit funny, e.g. universe/pkgname
+        pkgs = set([os.path.basename(pkg) for pkg in pkgs.split(",")])
+        yield (path, pkgs)
+
+
+class AptFilePackageDatabase(object):
+    """Really dumb database that just uses apt-file for local testing """
+
+    # we could also read /etc/ld.so.conf.d/*.conf but this maybe different on
+    # different distroseries especially if
+    #    server-distroseries != target-distroseries
+    #  (I wish there was ldconfig --print-search-dirs)
+    LD_SEARCH_PATH = [
+        # standards
+        "lib",
+        "usr/lib",
+        "usr/local/lib",
+        # old biarch
+        "lib32",
+        "usr/lib32",
+        # new multiarch
+        "lib/i686-linux-gnu",
+        "lib/i386-linux-gnu",
+        "lib/x86_64-linux-gnu",
+        "usr/lib/i386-linux-gnu",
+        "usr/lib/i686-linux-gnu",
+        "usr/lib/x86_64-linux-gnu",
+        # ?
+        "usr/lib/x86_64-linux-gnu/fakechroot",
+        "usr/lib/x86_64-linux-gnu/mesa",
+        "usr/lib/x86_64-linux-gnu/mesa-egl",
+        "usr/lib/i386-linux-gnu/mesa",
+        ]
+
+    DISTROSERIES = "oneiric"
+
+    CONTENTS_FILE_URL_LOCATION = (
+        "http://archive.ubuntu.com/ubuntu/dists/%(distroseries)s/"
+        "Contents-%(arch)s.gz")
+
+    CONTENTS_FILE = "Contents-%(distroseries)s-%(arch)s"
+
+    def __init__(self, cachedir):
+        self.cachedir = os.path.expanduser(cachedir)
+        self._distroseries_arch_cache = {}
+
+    def _get_lib_to_pkgs_mapping(self, distroseries, arch):
+        """Returns a dict of { library-name : set([pkg1,pkg2])
+
+        This function will return a dict to lookup library-name to package
+        dependencies for the given distroseries and architecture
+        """
+        if not (distroseries, arch) in self._distroseries_arch_cache:
+            self._distroseries_arch_cache[(distroseries, arch)] = \
+                      self._get_mapping_from_contents_file(distroseries, arch)
+        return self._distroseries_arch_cache[(distroseries, arch)]
+
+    def _get_contents_file_cache_path(self, distroseries, arch):
+        """Return the path in the cache for the given distroseries, arch """
+        return os.path.join(
+            self.cachedir, self.CONTENTS_FILE % {
+                'distroseries': distroseries, 'arch': arch})
+
+    def _get_contents_file_server_url(self, distroseries, arch):
+        """Return the remote server URL for the given distroseries, arch """
+        return self.CONTENTS_FILE_URL_LOCATION % {
+            'distroseries': distroseries, 'arch': arch}
+
+    def _get_mapping_from_contents_file(self, distroseries, arch):
+        """Return lib,pkgs mapping from contents file for distroseries, arch
+
+        This expects the contents file to be in the cachedir already.
+        """
+        lib_to_pkgs = {}
+        path = self._get_contents_file_cache_path(distroseries, arch)
+        with open(path) as f:
+            for path, pkgs in self._iter_contents_file(f):
+                basename = os.path.basename(path)
+                if not basename in lib_to_pkgs:
+                    lib_to_pkgs[basename] = set()
+                lib_to_pkgs[basename] |= pkgs
+        return lib_to_pkgs
+
+    def _download_contents_file_compressed(self, distroseries, arch):
+        """Downloads the content file for distroseries, arch into target """
+        # XXX: we may eventually want to merge the Contents files from
+        #      the -updates repository too in addition to the main archive
+        url = self._get_contents_file_server_url(distroseries, arch)
+        target = self._get_contents_file_cache_path(distroseries, arch)
+        compressed_target = target + os.path.splitext(url)[1]
+        # download
+        urllib.urlretrieve(url, compressed_target)
+        return compressed_target
+
+    def _iter_contents_file(self, in_file):
+        for path, pkgs in iter_contents_file(in_file):
+            if os.path.dirname(path) in self.LD_SEARCH_PATH:
+                yield path, pkgs
+
+    def _prune_contents_gz_file(self, infile, outfile):
+        """Read a compressed Contents.gz and write out a pruned version.
+
+        This will use iter_contents_file to go over infile and write
+        the relevant lines that are in the LD_SEARCH_PATH to outfile.
+        """
+        with open(outfile, "w") as outf, gzip.open(infile) as inf:
+            # first write the header
+            outf.write("FILE  LOCATION\n")
+            # then iter over all relevant lines and write them out
+            for path, pkgs in self._iter_contents_file(inf):
+                outf.write("%s %s\n" % (path, ",".join(pkgs)))
+
+    def _download_and_prepare_contents_file_if_needed(self, distroseries, arch):
+        """Ensure there is a usable Contents file in the cachedir
+
+        This will download, uncompress and prune a Conents file for
+        distroseries, arch so that get_dependencies works.
+        """
+        # mvo: We can (and should eventually) do etag/if-modified-since
+        #      matching here. But its not really important as long as
+        #      we package for stable distroseries as the Contents file
+        #      will not change
+        path = self._get_contents_file_cache_path(distroseries, arch)
+        if not os.path.exists(path):
+            compressed_contents = self._download_contents_file_compressed(
+                distroseries, arch)
+            # and prune from ~300mb to 1mb uncompressed as we are only
+            # interested in the library path parts
+            self._prune_contents_gz_file(compressed_contents, path)
+            os.remove(compressed_contents)
+
+    def iter_database(self, architectures=('i386', 'amd64'),
+                      distroseries=None):
+        """Export the database.
+
+        Yields (library, package, arch) tuples for everything that we can
+        find.
+        """
+        # XXX: Untested
+        if distroseries is None:
+            distroseries = self.DISTROSERIES
+        for arch in architectures:
+            self._download_and_prepare_contents_file_if_needed(
+                distroseries, arch)
+            mapping = self._get_lib_to_pkgs_mapping(distroseries, arch)
+            for library in mapping:
+                for package in mapping[library]:
+                    yield library, package, arch
+
+    def get_dependencies(self, lib, arch="i386"):
+        # do lazy downloading for now, we could also make this part
+        # of bin/fetch-symbols I guess(?)
+        self._download_and_prepare_contents_file_if_needed(
+            self.DISTROSERIES, arch)
+        lib_to_pkgs = self._get_lib_to_pkgs_mapping(self.DISTROSERIES, arch)
+        return lib_to_pkgs.get(lib)
+
+    def close(self):
+        pass

=== modified file 'devportalbinary/database.py'
--- devportalbinary/database.py	2012-09-13 15:04:20 +0000
+++ devportalbinary/database.py	2012-09-17 15:39:29 +0000
@@ -3,12 +3,10 @@
 
 from contextlib import closing, contextmanager
 import errno
-import gzip
 from itertools import chain
 import os
 import shutil
 import tempfile
-import urllib
 
 from bzrlib import urlutils
 from fixtures import (
@@ -24,6 +22,7 @@
 from storm.locals import create_database, Store
 from storm.uri import URI as StormURI
 
+from .aptfile import AptFilePackageDatabase
 from .configuration import (
     CONF_FILE_ENV_VAR,
     get_config_file_path,
@@ -357,24 +356,6 @@
     return mapping
 
 
-def iter_contents_file(contents):
-    """ Yield (full-library-path, set-of-pkgnames) from a Contents file.
-
-    It expects a line starting with "FILE" that tells it when the header ends
-    and the actual content starts.
-    """
-    found_start_marker = False
-    for line in contents:
-        if not found_start_marker:
-            if line.startswith("FILE"):
-                found_start_marker = True
-            continue
-        (path, sep, pkgs) = [s.strip() for s in line.rpartition(" ")]
-        # pkgs is formated a bit funny, e.g. universe/pkgname
-        pkgs = set([os.path.basename(pkg) for pkg in pkgs.split(",")])
-        yield (path, pkgs)
-
-
 class URI(StormURI):
     """A stand-in for Storm's URI class.
 
@@ -397,143 +378,6 @@
             self.options = dict()
 
 
-class AptFilePackageDatabase(object):
-    """ Really dumb database that just uses apt-file for local testing """
-
-    # we could also read /etc/ld.so.conf.d/*.conf but this maybe different on
-    # different distroseries especially if
-    #    server-distroseries != target-distroseries
-    #  (I wish there was ldconfig --print-search-dirs)
-    LD_SEARCH_PATH = [
-        # standards
-        "lib",
-        "usr/lib",
-        "usr/local/lib",
-        # old biarch
-        "lib32",
-        "usr/lib32",
-        # new multiarch
-        "lib/i686-linux-gnu",
-        "lib/i386-linux-gnu",
-        "lib/x86_64-linux-gnu",
-        "usr/lib/i386-linux-gnu",
-        "usr/lib/i686-linux-gnu",
-        "usr/lib/x86_64-linux-gnu",
-        # ?
-        "usr/lib/x86_64-linux-gnu/fakechroot",
-        "usr/lib/x86_64-linux-gnu/mesa",
-        "usr/lib/x86_64-linux-gnu/mesa-egl",
-        "usr/lib/i386-linux-gnu/mesa",
-        ]
-
-    DISTROSERIES = "oneiric"
-
-    CONTENTS_FILE_URL_LOCATION = (
-        "http://archive.ubuntu.com/ubuntu/dists/%(distroseries)s/"
-        "Contents-%(arch)s.gz")
-
-    CONTENTS_FILE = "Contents-%(distroseries)s-%(arch)s"
-
-    def __init__(self, cachedir=None):
-        self.cachedir = os.path.expanduser(cachedir)
-        self._distroseries_arch_cache = {}
-
-    def _get_lib_to_pkgs_mapping(self, distroseries, arch):
-        """ Returns a dict of { library-name : set([pkg1,pkg2])
-
-        This function will return a dict to lookup library-name to package
-        dependencies for the given distroseries and architecture
-        """
-        if not (distroseries, arch) in self._distroseries_arch_cache:
-            self._distroseries_arch_cache[(distroseries, arch)] = \
-                      self._get_mapping_from_contents_file(distroseries, arch)
-        return self._distroseries_arch_cache[(distroseries, arch)]
-
-    def _get_contents_file_cache_path(self, distroseries, arch):
-        """ Return the path in the cache for the given distroseries, arch """
-        return os.path.join(
-            self.cachedir, self.CONTENTS_FILE % {
-                'distroseries' : distroseries, 'arch' : arch })
-
-    def _get_contents_file_server_url(self, distroseries, arch):
-        """ Return the remote server URL for the given distroseries, arch """
-        return self.CONTENTS_FILE_URL_LOCATION % {
-            'distroseries' : distroseries, 'arch' : arch }
-
-    def _get_mapping_from_contents_file(self, distroseries, arch):
-        """ Return lib,pkgs mapping from contents file for distroseries, arch
-
-        This expects the contents file to be in the cachedir already.
-        """
-        lib_to_pkgs = {}
-        path = self._get_contents_file_cache_path(distroseries, arch)
-        with open(path) as f:
-            for (path, pkgs) in filter(
-                lambda (p, pkgs): os.path.dirname(p) in self.LD_SEARCH_PATH,
-                iter_contents_file(f)):
-                basename = os.path.basename(path)
-                if not basename in lib_to_pkgs:
-                    lib_to_pkgs[basename] = set()
-                lib_to_pkgs[basename] |= pkgs
-        return lib_to_pkgs
-
-    def _download_contents_file_compressed(self, distroseries, arch):
-        """ Downloads the content file for distroseries, arch into target """
-        # XXX: we may eventually want to merge the Contents files from
-        #      the -updates repository too in addition to the main archive
-        url = self._get_contents_file_server_url(distroseries, arch)
-        target = self._get_contents_file_cache_path(distroseries, arch)
-        compressed_target = target + os.path.splitext(url)[1]
-        # download
-        urllib.urlretrieve(url, compressed_target)
-        return compressed_target
-
-    def _prune_contents_gz_file(self, infile, outfile):
-        """ Read a compressed Contents.gz and write out a pruned version.
-
-        This will use iter_contents_file to go over infile and write
-        the relevant lines that are in the LD_SEARCH_PATH to outfile.
-        """
-        with open(outfile, "w") as outf, gzip.open(infile) as inf:
-            # first write the header
-            outf.write("FILE  LOCATION\n")
-            # then iter over all relevant lines and write them out
-            for (path, pkgs) in filter(
-                lambda (p,pkgs): os.path.dirname(p) in self.LD_SEARCH_PATH,
-                iter_contents_file(inf)):
-                outf.write("%s %s\n" % (path, ",".join(pkgs)))
-
-    def _download_and_prepare_contents_file_if_needed(self, distroseries, arch):
-        """ Ensure there is a usable Contents file in the cachedir
-
-        This will download, uncompress and prune a Conents file for
-        distroseries, arch so that get_dependencies works.
-        """
-        # mvo: We can (and should eventually) do etag/if-modified-since
-        #      matching here. But its not really important as long as
-        #      we package for stable distroseries as the Contents file
-        #      will not change
-        path = self._get_contents_file_cache_path(distroseries, arch)
-        if not os.path.exists(path):
-            compressed_contents = self._download_contents_file_compressed(
-                distroseries, arch)
-            # and prune from ~300mb to 1mb uncompressed as we are only
-            # interested in the library path parts
-            self._prune_contents_gz_file(compressed_contents, path)
-            os.remove(compressed_contents)
-
-    def get_dependencies(self, lib, arch="i386"):
-        # do lazy downloading for now, we could also make this part
-        # of bin/fetch-symbols I guess(?)
-        self._download_and_prepare_contents_file_if_needed(
-            self.DISTROSERIES, arch)
-        lib_to_pkgs = self._get_lib_to_pkgs_mapping(self.DISTROSERIES, arch)
-        return lib_to_pkgs.get(lib)
-
-    def close(self):
-        pass
-
-
 class PackageDatabase(object):
 
     # the various db backends, aptfile is a bit special

=== added file 'devportalbinary/tests/test_aptfile.py'
--- devportalbinary/tests/test_aptfile.py	1970-01-01 00:00:00 +0000
+++ devportalbinary/tests/test_aptfile.py	2012-09-17 15:39:29 +0000
@@ -0,0 +1,52 @@
+import gzip
+import os
+
+from mock import patch
+from fixtures import TempDir
+from testtools import TestCase
+
+from ..aptfile import AptFilePackageDatabase
+
+
+class AptFilePackageDatabaseTestCase(TestCase):
+
+    # point to our local contents file version that is a tad smaller
+    CONTENTS_CACHE = os.path.join(
+        os.path.dirname(__file__), "data", "apt-file-backend")
+
+    def setUp(self):
+        super(AptFilePackageDatabaseTestCase, self).setUp()
+        self.db = AptFilePackageDatabase(self.CONTENTS_CACHE)
+
+    def test_read_fixture_contents_worked(self):
+        """ test that our fixture Contents file works as expected """
+        # our test DB has 4 entries in the default search path
+        self.assertEqual(
+            len(self.db._get_lib_to_pkgs_mapping("oneiric", "i386")), 4)
+
+    def test_get_dependencies(self):
+        """ Test that data from the fixture dependencies file works """
+        self.assertEqual(
+            self.db.get_dependencies("libz.so.1"), set(["zlib1g"]))
+
+    @patch("urllib.urlretrieve")
+    def test_lazy_downloading(self, mock_urlretrieve):
+        """ test that lazy downloading works """
+        def _put_fixture_contents_file_in_place(url, target):
+            with gzip.open(target, "w") as f:
+                f.write("""
+Some header text that is ignored
+FILE                 LOCATION
+usr/lib/libfoo.so.2  pkgfoo,pkgbar
+""")
+        tempdir = self.useFixture(TempDir())
+        db = AptFilePackageDatabase(tempdir.path)
+        mock_urlretrieve.side_effect = _put_fixture_contents_file_in_place
+        self.assertEqual(
+            db.get_dependencies("libfoo.so.2", arch="i386"),
+            set(["pkgfoo", "pkgbar"]))
+        self.assertEqual(len(db._get_lib_to_pkgs_mapping("oneiric", "i386")), 1)
+
+    def test_close(self):
+        # Test that there is a close method we can call
+        self.db.close()

=== modified file 'devportalbinary/tests/test_database.py'
--- devportalbinary/tests/test_database.py	2012-09-07 19:15:08 +0000
+++ devportalbinary/tests/test_database.py	2012-09-17 15:39:29 +0000
@@ -1,11 +1,7 @@
 from collections import namedtuple
-import gzip
 import os
 
-from fixtures import (
-    TempDir,
-    )
-from mock import patch
+from fixtures import TempDir
 from storm.databases.postgres import psycopg2
 from storm.exceptions import ClosedError
 from testresources import ResourcedTestCase
@@ -192,50 +188,6 @@
         self.assertEqual(expected_username, uri.username)
 
 
-class AptFilePackageDatabaseTestCase(TestCase):
-
-    # point to our local contents file version that is a tad smaller
-    CONTENTS_CACHE = os.path.join(
-        os.path.dirname(__file__), "data", "apt-file-backend")
-
-    def setUp(self):
-        super(AptFilePackageDatabaseTestCase, self).setUp()
-        self.db = AptFilePackageDatabase(self.CONTENTS_CACHE)
-
-    def test_read_fixture_contents_worked(self):
-        """ test that our fixture Contents file works as expected """
-        # our test DB has 4 entries in the default search path
-        self.assertEqual(
-            len(self.db._get_lib_to_pkgs_mapping("oneiric", "i386")), 4)
-
-    def test_get_dependencies(self):
-        """ Test that data from the fixture dependencies file works """
-        self.assertEqual(
-            self.db.get_dependencies("libz.so.1"), set(["zlib1g"]))
-
-    @patch("urllib.urlretrieve")
-    def test_lazy_downloading(self, mock_urlretrieve):
-        """ test that lazy downloading works """
-        def _put_fixture_contents_file_in_place(url, target):
-            with gzip.open(target, "w") as f:
-                f.write("""
-Some header text that is ignored
-FILE                 LOCATION
-usr/lib/libfoo.so.2  pkgfoo,pkgbar
-""")
-        tempdir = self.useFixture(TempDir())
-        db = AptFilePackageDatabase(tempdir.path)
-        mock_urlretrieve.side_effect = _put_fixture_contents_file_in_place
-        self.assertEqual(
-            db.get_dependencies("libfoo.so.2", arch="i386"),
-            set(["pkgfoo", "pkgbar"]))
-        self.assertEqual(len(db._get_lib_to_pkgs_mapping("oneiric", "i386")), 1)
-
-    def test_close(self):
-        # Test that there is a close method we can call
-        self.db.close()
-
-
 class FakeBPPH(object):
 
     def __init__(self):

=== modified file 'setup.py'
--- setup.py	2012-09-12 18:31:59 +0000
+++ setup.py	2012-09-17 15:39:29 +0000
@@ -44,6 +44,7 @@
         ],
     entry_points = {
         'console_scripts': [
+            'dump-apt-file-db=devportalbinary.aptfile:dump_apt_file_db',
             'fetch-symbol-files=devportalbinary.database:main',
             'guess-executable=devportalbinary.binary:print_executable',
             'guess-deps=devportalbinary.binary:print_dependencies',