Merge ~racb/usd-importer:queue into usd-importer:master

[Robie Basak]

Add import of Launchpad "Unapproved" and "New" queues.

19:41 <rbasak> I want it to autodetect the package name from debian/changelog so it doesn't need to be specified

19:41 <rbasak> And then replace the existing "usd queue <package>" with "usd queue sync". Then I think it'll be ready to land.

19:41 <nacc> rbasak: ok, i can probably do that locally if you want to propose a MR?

[Robie Basak (racb)]

I just noticed that FIX_SERIES_LIST should probably be fixed before landing, too.

diff --git a/usd/__main__.py b/usd/__main__.py
index 2cad560..ba602ad 100644
--- a/usd/__main__.py
+++ b/usd/__main__.py
@@ -13,6 +13,7 @@ def main():
         from usd.merge import USDMerge
         from usd.clone import USDClone
         from usd.tag import USDTag
+        from usd.queue import USDQueueImporter
 
         try:
             pkg = 'python3-pkg-resources'
@@ -28,10 +29,11 @@ def main():
                              'build-source':USDBuildSource,
                              'merge':USDMerge,
                              'clone':USDClone,
-                             'tag':USDTag
+                             'tag':USDTag,
+                             'queue':USDQueueImporter,
                             }
 
-        known_network_subcommands = {'import', 'clone', 'build', 'build-source'}
+        known_network_subcommands = {'import', 'clone', 'build', 'build-source', 'queue'}
 
         parser = argparse.ArgumentParser(
             description='Ubuntu Server Dev git tool',
diff --git a/usd/queue.py b/usd/queue.py
new file mode 100644
index 0000000..5166c4e
--- /dev/null
+++ b/usd/queue.py
@@ -0,0 +1,172 @@
+import argparse
+import itertools
+import logging
+import os
+import re
+import tempfile
+import urllib
+
+import pygit2
+
+from usd.git_repository import USDGitRepository
+import usd.importer
+import usd.source_information
+
+FIX_SERIES_LIST = ['precise', 'trusty', 'xenial', 'yakkety']
+
+class USDQueueImporter:
+    MATCH_TAG_URI = re.compile(r'^Launchpad-URI:\s*(.*)$', flags=re.MULTILINE)
+    DOWNLOAD_MATCHES = {
+        'changes': re.compile(r'_source\.changes$'),
+        'dsc': re.compile(r'\.dsc$'),
+    }
+
+    def parse_args(self, subparsers=None, base_subparsers=None):
+        kwargs = dict(description='Import unapproved and new uploads',
+                      formatter_class=argparse.RawTextHelpFormatter,
+                     )
+        if base_subparsers:
+            kwargs['parents'] = base_subparsers
+        if subparsers:
+            parser = subparsers.add_parser('queue', **kwargs)
+            parser.set_defaults(func=self.main)
+        else:
+            parser = argparse.ArgumentParser(**kwargs)
+
+        parser.add_argument('package')
+        parser.add_argument('-d', '--directory', type=str,
+                            help='Use git repository at specified location.',
+                            default=os.path.abspath(os.getcwd())
+                           )
+
+        if not subparsers:
+            return parser.parse_args()
+        return 'queue - %s' % kwargs['description']
+
+    @staticmethod
+    def _fetch_lp_queue_items(lp, series, package):
+        '''Fetch unapproved and new entries from Launchpad
+
+        Returns a dictionary of Launchpad upload objects keyed by persistent
+        unique ID.
+        '''
+        s = lp.distributions['ubuntu'].getSeries(name_or_version=series)
+        us = itertools.chain(*(
+            s.getPackageUploads(
+                pocket='Proposed',
+                status=status,
+                exact_match=True,
+                name=package,
+            ) for status in ['Unapproved', 'New']
+        ))
+        lp_queue_items = {x.self_link: x for x in us}
+        return lp_queue_items
+
+    @staticmethod
+    def _fetch_queue_tag_names(repo, series):
+        regexp = re.compile(r'^refs/tags/queue/%s/' % series)
+        return repo.listall_references_matching_regexp(regexp)
+
+    @classmethod
+    def _get_queue_tag_uri(self, tag):
+        match = self.MATCH_TAG_URI.search(tag.message)
+        assert match, "Tag URI not found"
+        return match.group(1)
+
+    @staticmethod
+    def _download_url(url, destdir):
+        parsed_url = urllib.parse.urlparse(url)
+        # Require secure URL
+        if parsed_url.scheme != 'https':
+            raise ValueError("URL scheme is not HTTPS: %s" % url)
+        # Find and require a basename to store the file locally with a sensible
+        # filename
+        basename = parsed_url.path.split('/')[-1]
+        if not basename:
+            raise ValueError("Provided URL %s has no basename component" % url)
+        with urllib.request.urlopen(url) as response:
+            with open(os.path.join(destdir, basename), 'wb') as f:
+                f.write(response.read())
+
+    @classmethod
+    def _commit_upload(self, repo, upload, parent_commit):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            self._download_url(upload.changes_file_url, tmpdir)
+            for url in upload.sourceFileUrls():
+                self._download_url(url, tmpdir)
+
+            download_key = {}
+            for filename in os.listdir(tmpdir):
+                for file_type, regexp in self.DOWNLOAD_MATCHES.items():
+                    if regexp.search(filename):
+                        download_key[file_type] = filename
+
+            # We should have both a changes file and a dsc file
+            assert set(download_key.keys()) == {'changes', 'dsc'}
+
+            # Import the source package into a git tree object
+            tree_hash = usd.importer.dsc_to_tree_hash(
+                repo,
+                os.path.join(tmpdir, download_key['dsc']),
+            )
+
+        # Ensure parents contains a string if specified
+        parents = [str(parent_commit)] if parent_commit else []
+        commit_hash = repo.commit_tree_hash(
+            tree_hash=tree_hash,
+            parents=parents,
+            log_message='Queue import'.encode(),
+        )
+
+        return commit_hash
+
+    def main(self, args):
+        repo = USDGitRepository(args.directory)
+        lp = usd.source_information.launchpad_login()
+        for series in FIX_SERIES_LIST:
+            devel_head_ref = repo.get_head_by_name('lpusip/ubuntu/%s-devel' % series)
+            devel_head = devel_head_ref.target if devel_head_ref else None
+            lp_queue_items = self._fetch_lp_queue_items(lp, series, args.package)
+            all_tag_names = self._fetch_queue_tag_names(repo, series)
+            found_uris_in_repo = set()
+            for tag_name in all_tag_names:
+                logging.debug('Considering %s', tag_name)
+                tag_ref = repo.lookup_reference(tag_name)
+                try:
+                    tag = tag_ref.peel(pygit2.Tag)
+                except ValueError:
+                    print('%s is invalid; deleting' % tag_name)
+                    tag_ref.delete()
+                    continue
+                tag_uri = self._get_queue_tag_uri(tag)
+                if tag_uri not in lp_queue_items:
+                    print('%s is no longer in the queue; deleting' % tag_name)
+                    tag_ref.delete()
+                    continue
+                if lp_queue_items[tag_uri].status == 'Unapproved' and tag.peel(pygit2.Commit).parent_ids != [devel_head]:
+                    print('%s is no longer based correctly; deleting' % tag_name)
+                    tag_ref.delete()
+                    continue
+                found_uris_in_repo.add(tag_uri)
+            for uri, upload in lp_queue_items.items():
+                if upload.status == 'Unapproved':
+                    assert devel_head
+                    parent_commit = devel_head
+                    tag_type = 'unapproved'
+                elif upload.status == 'New':
+                    parent_commit = None
+                    tag_type = 'new'
+                else:
+                    raise RuntimeError('Unknown status for %s: %s' % (uri, upload.status))
+                if uri in found_uris_in_repo:
+                    logging.debug('Skipping %s: already imported', uri)
+                    continue  # already imported
+                logging.debug('Importing %s', uri)
+                commit_hash = self._commit_upload(repo, upload, parent_commit)
+                unique_id = repo.get_short_hash(commit_hash)
+                repo.annotated_tag(
+                    'queue/%s/%s/%s' % (series, tag_type, unique_id),
+                    commit_hash,
+                    force=False,
+                    msg=('Launchpad-URI: %s' % uri),
+                )