Merge lp:~cjwatson/ddeb-retriever/lp-fetch into lp:ddeb-retriever

[Colin Watson]

Fetch ddebs from the Launchpad librarian if possible.

Fetch ddebs from the Launchpad librarian if possible.

[Colin Watson (cjwatson)]

20:36 <pitti> cjwatson: I didn't look through it with a fine comb again, but this looks very similar to the changes I've reviewed some two weeks ago?
20:36 <cjwatson> pitti: Yes, it's just bug-fixes since then
20:36 <cjwatson> pitti: And the order_by_date=True stuff which was a result of LP reviews
20:36 <pitti> cjwatson: so if you feel good about it, please feel free to merge
20:37 <pitti> or rather, push, for cleaner history
20:38 <cjwatson> pitti: will do, thanks!

review: Approve

=== modified file 'ddeb_retriever.py'
--- ddeb_retriever.py	2014-08-07 09:03:46 +0000
+++ ddeb_retriever.py	2015-04-14 19:16:49 +0000
@@ -10,6 +10,9 @@
 import tarfile
 import argparse
 import fcntl
+import datetime
+
+import wadllib.iso_strptime
 
 import lpinfo
 import archive_tools
@@ -61,6 +64,75 @@
     return result
 
 
+distroarchseries_links = {}
+distroseries_links = {}
+
+def get_suite_for_publication(pub):
+    '''Return the suite for a binary_package_publishing_history object.
+
+    We memoise the series objects, as otherwise this is very slow for lots
+    of publications.
+    '''
+    # Don't be tempted to shorten this using setdefault; fetching
+    # pub.distro_arch_series and das.distroseries is expensive.
+    distroarchseries_link = pub.distro_arch_series_link
+    if distroarchseries_link not in distroarchseries_links:
+        distroarchseries_links[distroarchseries_link] = pub.distro_arch_series
+    distroarchseries = distroarchseries_links[distroarchseries_link]
+    distroseries_link = distroarchseries.distroseries_link
+    if distroseries_link not in distroseries_links:
+        distroseries_links[distroseries_link] = distroarchseries.distroseries
+    distroseries = distroseries_links[distroseries_link]
+    return lpinfo.make_suite(distroseries, pub.pocket)
+
+
+def install_from_librarian(pub, url, ddeb_archive_root):
+    '''Fetch ddeb from the librarian and install it into ddeb_archive_root
+
+    If the file already exists in the appropriate place in pool, it will not
+    be downloaded again.
+
+    Return True if it was installed successfully or already present,
+    otherwise False.
+    '''
+    ddeb = urllib.unquote(os.path.basename(url))
+    try:
+        (dbgsymname, version, arch) = ddeb.split('_')
+        assert arch.endswith('.ddeb')
+        (arch, _) = arch.split('.')
+        assert dbgsymname.endswith('-dbgsym')
+    except (ValueError, AssertionError):
+        logging.warning('Ignoring non-ddeb file %s in %s', ddeb, pub)
+        return False
+
+    source_pub = pub.build.getLatestSourcePublication()
+    if source_pub is None:
+        logging.warning('Ignoring %s which has no source publication', pub)
+        return False
+
+    source_name = source_pub.source_package_name
+    if source_name.startswith('lib'):
+        prefix = source_name[:4]
+    else:
+        prefix = source_name[0]
+
+    destdir = os.path.join(
+        ddeb_archive_root, 'pool', pub.component_name, prefix, source_name)
+    dest = os.path.join(destdir, ddeb)
+    if os.path.exists(dest):
+        logging.debug('%s already exists, skipping', dest)
+    else:
+        logging.debug('Downloading %s into %s', ddeb, os.path.dirname(dest))
+        try:
+            os.makedirs(destdir)
+        except OSError:
+            pass
+        urllib.urlretrieve(url, dest)
+        logging.debug('Downloaded %s into %s', ddeb, os.path.dirname(dest))
+
+    return True
+
+
 def get_ddeb_index(url):
     '''Parse the ddeb index file at the given URL
 
@@ -251,9 +323,39 @@
         logging.error('Cannot acquire lock, another ddeb-retriever instance is already running')
         return
 
+    UTC = wadllib.iso_strptime.TimeZone('+00:00')
+    lp_threshold_path = os.path.join(args.archive_root, '.lp-threshold')
+    try:
+        with open(lp_threshold_path) as lp_threshold_file:
+            lp_threshold = datetime.datetime.fromtimestamp(
+                int(lp_threshold_file.readline()), tz=UTC)
+    except IOError:
+        lp_threshold = None
+
     series_info = lpinfo.get_series(args.distribution)
     logging.debug('series info: %s', series_info)
 
+    if lp_threshold is not None:
+        # Allow a grace period to cope with publications arriving out of
+        # order during long transactions.
+        real_threshold = lp_threshold - datetime.timedelta(hours=1)
+        logging.info(
+            'Retrieving Launchpad publications since %s' % real_threshold)
+    else:
+        real_threshold = None
+        logging.info('Retrieving all Launchpad publications')
+    binary_pubs = lpinfo.get_binary_publications(args.distribution)
+    debug_pubs = []
+    latest_date_created = lp_threshold
+    for pub in binary_pubs:
+        if pub.date_created < real_threshold:
+            break
+        if (latest_date_created is None or
+                pub.date_created > latest_date_created):
+            latest_date_created = pub.date_created
+        if pub.is_debug:
+            debug_pubs.append(pub)
+
     # Download ddebs into temporary dir; all in parallel, as buildd Apaches are
     # very slow
     for date in args.date:
@@ -275,6 +377,8 @@
     queue_dir = os.path.join(args.archive_root, 'queue')
     if os.path.isdir(queue_dir):
         updated_pockets.update(os.listdir(queue_dir))
+    for pub in debug_pubs:
+        updated_pockets.add(get_suite_for_publication(pub))
     logging.info('got downloaded/queued ddebs for pockets: %s', ' '.join(updated_pockets))
 
     # complete the pocket list
@@ -295,8 +399,39 @@
                                             pockets_filter=updated_pockets,
                                             strip_epoch=True)
 
+    logging.info('Building ddeb map, this can take a while')
+    ddeb_map = archive_tools.archive_map(series_info, [args.archive_root],
+                                         suffix='')
+
+    # try to install ddebs from publishing history records
+    installed_pockets = set()
+    for pub in debug_pubs:
+        already_present = False
+        pub_installed = False
+        if pub.binary_package_name in ddeb_map:
+            existing = ddeb_map[pub.binary_package_name]
+            for _, series_map in existing.items():
+                if pub.binary_package_version in series_map:
+                    already_present = True
+                    break
+        if already_present:
+            # We already have this ddeb on disk, so just note that indexes
+            # for this pocket need to be rebuilt.
+            pub_installed = True
+        else:
+            for url in pub.binaryFileUrls():
+                if install_from_librarian(pub, url, args.archive_root):
+                    pub_installed = True
+        if pub_installed:
+            installed_pockets.add(get_suite_for_publication(pub))
+
+    if latest_date_created is not None:
+        epoch = datetime.datetime.fromtimestamp(0, tz=UTC)
+        new_threshold = (latest_date_created - epoch).total_seconds()
+        with open(lp_threshold_path, 'w') as lp_threshold_file:
+            lp_threshold_file.write("%d\n" % new_threshold)
+
     # try to install downloaded ddebs
-    installed_pockets = set()
     for pocket in updated_pockets:
         d = os.path.join(download_dir, pocket)
         if not os.path.isdir(d):

=== modified file 'lpinfo.py'
--- lpinfo.py	2014-07-25 06:50:12 +0000
+++ lpinfo.py	2015-04-14 19:16:49 +0000
@@ -10,7 +10,9 @@
     global lp
     if lp is None:
         lp = Launchpad.login_anonymously(
-            'ddeb-retriever', os.environ.get('LAUNCHPAD_INSTANCE', 'production'))
+            'ddeb-retriever',
+            os.environ.get('LAUNCHPAD_INSTANCE', 'production'),
+            version='devel')
 
 
 def get_series(distribution):
@@ -45,3 +47,27 @@
         buildds.append(builder.name)
 
     return buildds
+
+
+def get_binary_publications(distribution, created_since_date=None):
+    '''Get recently-published binaries for the given distribution'''
+
+    _get_lp()
+    archive = lp.distributions[distribution].main_archive
+    # It's important to omit the status filter here, even if we later decide
+    # that we only care about the Published status (although at the moment
+    # it seems reasonable to scan everything and let the garbage-collection
+    # pass clean up superseded ddebs later).  This is because the collection
+    # may change as we're iterating over it.  Without any filtering, this is
+    # OK because entries can never be removed from the collection: the worst
+    # case is that we encounter the same publication twice.  With filtering
+    # on mutable properties, it would be possible to lose entries between
+    # two successive batches.
+    return archive.getPublishedBinaries(order_by_date=True)
+
+
+def make_suite(distroseries, pocket):
+    if pocket == 'Release':
+        return distroseries.name
+    else:
+        return '%s-%s' % (distroseries.name, pocket.lower())