Merge ~nacc/git-ubuntu:refactor-import-pristine-tarv2 into git-ubuntu:master

[Nish Aravamudan]

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:3ba561d5c6816954393999a27e6487b5b31c4fe7
No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want a jenkins rebuild you need to trigger it yourself):
https://code.launchpad.net/~nacc/usd-importer/+git/usd-importer/+merge/329554/+edit-commit-message

https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/9/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Tests
    SUCCESS: Build
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/9/rebuild

review: Needs Fixing (continuous-integration)

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:32ba12fd7c43f48e55fa435e07553654ffc88212
No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want a jenkins rebuild you need to trigger it yourself):
https://code.launchpad.net/~nacc/usd-importer/+git/usd-importer/+merge/329646/+edit-commit-message

https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/13/
Executed test runs:
    SUCCESS: Checkout
    FAILED: Tests

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/13/rebuild

review: Needs Fixing (continuous-integration)

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:76ce4ba27f30b7a02eca4130c40d3737a0fa43ff
No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want a jenkins rebuild you need to trigger it yourself):
https://code.launchpad.net/~nacc/usd-importer/+git/usd-importer/+merge/329646/+edit-commit-message

https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/17/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Tests
    SUCCESS: Build
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/17/rebuild

review: Needs Fixing (continuous-integration)

[Server Team CI bot (server-team-bot)]

FAILED: Continuous integration, rev:29e03d74bfa4a708c9e0cf4a57683ea7d08dca3b
No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want a jenkins rebuild you need to trigger it yourself):
https://code.launchpad.net/~nacc/usd-importer/+git/usd-importer/+merge/329646/+edit-commit-message

https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/21/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Tests
    SUCCESS: Build
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/21/rebuild

review: Needs Fixing (continuous-integration)

[Robie Basak (racb)]

review: Approve

diff --git a/gitubuntu/git_repository.py b/gitubuntu/git_repository.py
index 335a14c..91b865a 100644
--- a/gitubuntu/git_repository.py
+++ b/gitubuntu/git_repository.py
@@ -2,6 +2,7 @@
 ### XXX: is any of this data in lp already?
 
 import collections
+from contextlib import contextmanager
 from copy import copy
 from functools import lru_cache
 import itertools
@@ -361,6 +362,15 @@ def test_changelog_utf8():
 class GitUbuntuChangelogError(Exception):
     pass
 
+class PristineTarError(Exception):
+    pass
+
+class PristineTarNotFoundError(PristineTarError):
+    pass
+
+class MultiplePristineTarFoundError(PristineTarError):
+    pass
+
 
 def git_dep14_tag(version):
     """Munge a version string according to taken from
@@ -469,6 +479,145 @@ class GitUbuntuRepository:
                                   'Initial Debian pristine-tar branch.'
                                  )
 
+    @contextmanager
+    def pristine_tar_branches(self, dist, namespace='pkg', move=False):
+        """Context manager wrapping pristine-tar branch manipulation
+
+        In this context, the repository pristine-tar branch will point to
+        the pristine-tar branch for @dist distribution in @namespace.
+        If a local branch named pristine-tar exists already, it will be
+        moved to a (hopefully) safe name.
+        If @move is True, then the pristine-tar branch will be moved
+        rather than simply started from the parameter-specified point.
+        """
+        pt_branch = '%s/importer/%s/pristine-tar' % (namespace, dist)
+        old_pt_branch = self.raw_repo.lookup_branch('pristine-tar')
+        if old_pt_branch:
+            self.git_run(['branch', '-M', 'pristine-tar', 'pristine-tar.bak'])
+        cmd = ['branch']
+        if move:
+            cmd.extend(['-M', pt_branch, 'pristine-tar'])
+        else:
+            cmd.extend(['pristine-tar', pt_branch])
+        self.git_run(cmd)
+        yield
+        cmd = ['branch']
+        if move:
+            cmd.extend(['-M', 'pristine-tar', pt_branch])
+        else:
+            cmd.extend(['-D', 'pristine-tar'])
+        self.git_run(cmd)
+        if old_pt_branch:
+            self.git_run(['branch', '-M', 'pristine-tar.bak', 'pristine-tar'])
+
+    def pristine_tar_list(self, dist, namespace='pkg'):
+        with self.pristine_tar_branches(dist, namespace):
+            cp = run(['pristine-tar', 'list'])
+            return decode_binary(cp.stdout).strip().splitlines()
+
+    def pristine_tar_extract(self, pkgname, version, dist=None, namespace='pkg'):
+        '''Extract orig tarballs for a given package and upstream version
+
+        returns: list of tarball paths that are now present on the filesystem
+        throws:
+        - PristineTarNotFoundError if no suitable tarballs are found
+        - MultiplePristineTarFoundError if multiple distinct suitable tarballs
+          are found
+
+        '''
+        dists = [dist] if dist else ['debian', 'ubuntu']
+        for dist in dists:
+            main_tarball = '%s_%s.orig.tar' % (pkgname, version)
+
+            all_tarballs = self.pristine_tar_list(dist, namespace)
+
+            potential_main_tarballs = [tarball for tarball
+                in all_tarballs if tarball.startswith(main_tarball)]
+            if len(potential_main_tarballs) == 0:
+                continue
+            if len(potential_main_tarballs) > 1:
+                # This will need some extension/flag for the case of there
+                # being multiple imports with varying compression
+                raise MultiplePristineTarFoundError(
+                    'More than one pristine-tar tarball found for %s: %s' %
+                    (version, potential_main_tarballs)
+                )
+            ext = os.path.splitext(potential_main_tarballs[0])[1]
+            tarballs = []
+            tarballs.append(os.path.join('..', potential_main_tarballs[0]))
+            cmd = ['gbp', 'buildpackage', '--git-builder=/bin/true',
+                   '--git-pristine-tar', '--git-ignore-branch',
+                   '--git-upstream-tag=%s/upstream/%s/%%(version)s%s' %
+                   (namespace, dist, ext)]
+            orig_re = re.compile(
+                r'^%s_%s\.orig-(?P<component>[^.]+)\.tar\.[^.]+$' %
+                (re.escape(pkgname), re.escape(version))
+            )
+            # This will probably break if the component tarballs get
+            # compressed differently, as each component tarball will show up
+            # multiple times
+            # Breaks may be too strong -- we will 'over cache' tarballs, and
+            # then it's up to dpkg-buildpackage to use the 'correct' one
+            potential_component_tarballs = {
+                orig_re.match(tarball).group('component') : tarball for tarball
+                in all_tarballs if orig_re.match(tarball)
+            }
+            tarballs.extend(map(lambda x : os.path.join('..', x),
+                list(potential_component_tarballs.values()))
+            )
+            cmd.extend(map(lambda x : '--git-component=%s' % x,
+                list(potential_component_tarballs.keys()))
+            )
+            with self.pristine_tar_branches(dist, namespace):
+                run(cmd)
+            return tarballs
+
+        raise PristineTarNotFoundError(
+            'No pristine-tar tarball found for %s' % version
+        )
+
+    def pristine_tar_exists(self, pkgname, version, namespace='pkg'):
+        '''Report distributions that contain pristine-tar data for @version
+        '''
+        results = []
+        for dist in ('debian', 'ubuntu'):
+            main_tarball = '%s_%s.orig.tar' % (pkgname, version)
+
+            all_tarballs = self.pristine_tar_list(dist, namespace)
+
+            potential_main_tarballs = [tarball for tarball
+                in all_tarballs if tarball.startswith(main_tarball)]
+            if len(potential_main_tarballs) == 0:
+                continue
+            if len(potential_main_tarballs) > 1:
+                # This will need some extension/flag for the case of there
+                # being multiple imports with varying compression
+                raise MultiplePristineTarFoundError(
+                    'More than one pristine-tar tarball found for %s: %s' %
+                    (version, potential_main_tarballs)
+                )
+            results.append(dist)
+
+        return results
+
+    def verify_pristine_tar(self, tarball_paths, dist, namespace='pkg'):
+        '''Verify the pristine-tar data matches for a set of paths
+        '''
+        all_tarballs = self.pristine_tar_list(dist, namespace)
+        for path in tarball_paths:
+            if os.path.basename(path) not in all_tarballs:
+                break
+            try:
+                with self.pristine_tar_branches(dist, namespace):
+                    # need to handle this not existing
+                    run(['pristine-tar', 'verify', path])
+            except CalledProcessError as e:
+                raise PristineTarError('Tarball at %s failed to verify')
+        else:
+            return True
+
+        return False
+
     def set_git_attributes(self):
         git_attr_path = os.path.join(self.raw_repo.path,
                                      'info',
diff --git a/gitubuntu/importer.py b/gitubuntu/importer.py
index 7667ed2..3f4125a 100644
--- a/gitubuntu/importer.py
+++ b/gitubuntu/importer.py
@@ -36,7 +36,10 @@ import sys
 import tempfile
 import time
 from gitubuntu.cache import CACHE_PATH
-from gitubuntu.dsc import GitUbuntuDsc
+from gitubuntu.dsc import (
+    GitUbuntuDsc,
+    GitUbuntuDscError,
+)
 from gitubuntu.git_repository import (
     GitUbuntuRepository,
     GitUbuntuRepositoryFetchError,
@@ -44,6 +47,7 @@ from gitubuntu.git_repository import (
     applied_tag,
     import_tag,
     upstream_tag,
+    PristineTarError,
 )
 from gitubuntu.run import decode_binary, run, runq
 from gitubuntu.source_information import GitUbuntuSourceInformation, NoPublicationHistoryException, SourceExtractionException, launchpad_login_auth
@@ -341,14 +345,19 @@ class GitUbuntuImport:
 
         self.local_repo.clean_repository_state()
 
-    def orig_imported(self, orig_tarball):
-        """Determines if a given tarball has already been imported
+    def orig_imported(self, orig_tarball_paths, dist):
+        """Determines if a list of paths to tarballs have already been imported to @dist
 
         Assumes that the pristine-tar branch exists
         """
-        cp = run(['pristine-tar', 'list'])
-        tarballs = decode_binary(cp.stdout).strip()
-        return orig_tarball in tarballs
+        try:
+            return self.local_repo.verify_pristine_tar(orig_tarball_paths, dist,
+                self.namespace
+            )
+        except PristineTarError as e:
+            raise GitUbuntuImportOrigError('Tarball has already been imported to '
+                '%s with different contents: %s' % (dist, e)
+            )
 
     def import_orig(self, spi):
         """Imports the orig-tarball using gbp import-org --pristine-tar
@@ -361,23 +370,13 @@ class GitUbuntuImport:
         version = str(spi.version)
         dist = spi.distribution.name.lower()
 
-        # adapted from ubuntutools.SourcePackage.verify_orig
-        orig_tarball = None
-        components = []
-        orig_re = re.compile(r'.*\.orig(-(?P<component>[^.]+))?\.tar\.[^.]+$')
-        for entry in dsc['Files']:
-            m = orig_re.match(entry['name'])
-            if m is not None:
-                if m.group('component') is not None:
-                    components.append(m.group('component'))
-                else:
-                    if orig_tarball is None:
-                        orig_tarball = entry['name']
-                    else:
-                        raise GitUbuntuImportOrigError('Multiple base orig '
-                            'tarballs in DSC for version %s' % version)
+        try:
+            orig_tarball_path = dsc.orig_tarball_path
+        except GitUbuntuDscError as e:
+            raise GitUbuntuImportOrigError('Multiple base orig '
+                'tarballs in DSC for version %s' % version) from e
 
-        if orig_tarball is None:
+        if orig_tarball_path is None:
             # native packages only have a .tar.*
             native_re = re.compile(r'.*\.tar\.[^.]+$')
             for entry in dsc['Files']:
@@ -386,41 +385,42 @@ class GitUbuntuImport:
             raise GitUbuntuImportOrigError('No orig tarball in '
                 'DSC for version %s.' % version)
 
-        # update the branch name so that gbp is happy
-        self.local_repo.git_run(['branch', '-M',
-                      '%s/importer/%s/pristine-tar' % (self.namespace, dist),
-                      'pristine-tar'
-                     ]
-                    )
+        try:
+            orig_component_paths = dsc.component_tarball_paths
+        except GitUbuntuDscError as e:
+            raise GitUbuntuImportOrigError('Unable to import component '
+                'tarballs in DSC for version %s' % version) from e
+
+        orig_paths = [orig_tarball_path] + list(orig_component_paths.values())
+        if (not orig_tarball_path or
+            self.orig_imported(orig_paths, dist)):
+            return
+        if not all(map(os.path.exists, orig_paths)):
+            raise GitUbuntuImportOrigError('Unable to find tarball: '
+                '%s' % [p for p in orig_paths if not os.path.exists(p)])
 
         try:
-            oldcwd = os.getcwd()
-            # gbp fails if the tag already exist
-            if (not orig_tarball or
-                self.orig_imported(orig_tarball)):
-                return
-            if not os.path.exists(dsc.orig_tarball_path):
-                raise GitUbuntuImportOrigError('Unable to find tarball: '
-                    '%s' % dsc.orig_tarball_path)
-
-            # gbp does not support running from arbitrary git trees or
-            # working directories
-            # https://github.com/agx/git-buildpackage/pull/16
-            os.chdir(self.local_repo.local_dir)
-
-            ext = os.path.splitext(dsc.orig_tarball_path)[1]
-
-            # make this non-fatal if upstream already exist as tagged?
-            cmd = ['gbp', 'import-orig', '--no-merge',
-                 '--upstream-branch', 'do-not-push',
-                 '--pristine-tar', '--no-interactive',
-                 '--no-symlink-orig',
-                 '--upstream-tag=%s/upstream/%s/%%(version)s%s' %
-                     (self.namespace, dist, ext)]
-            if len(components) > 0:
-                cmd.extend(map(lambda x: '--component=%s' % x, components))
-            cmd.extend([dsc.orig_tarball_path,])
-            run(cmd, env=self.local_repo.env)
+            with self.local_repo.pristine_tar_branches(dist, self.namespace, move=True):
+                oldcwd = os.getcwd()
+                # gbp does not support running from arbitrary git trees or
+                # working directories
+                # https://github.com/agx/git-buildpackage/pull/16
+                os.chdir(self.local_repo.local_dir)
+
+                ext = os.path.splitext(dsc.orig_tarball_path)[1]
+
+                # make this non-fatal if upstream already exist as tagged?
+                cmd = ['gbp', 'import-orig', '--no-merge',
+                     '--upstream-branch', 'do-not-push',
+                     '--pristine-tar', '--no-interactive',
+                     '--no-symlink-orig',
+                     '--upstream-tag=%s/upstream/%s/%%(version)s%s' %
+                         (self.namespace, dist, ext)]
+                cmd.extend(map(lambda x: '--component=%s' % x,
+                    list(orig_component_paths.keys()))
+                )
+                cmd.extend([orig_tarball_path,])
+                run(cmd, env=self.local_repo.env)
         except CalledProcessError as e:
             raise GitUbuntuImportOrigError('Unable to import tarballs: '
                 '%s' % orig_paths) from e
@@ -429,13 +429,6 @@ class GitUbuntuImport:
 
             self.local_repo.clean_repository_state()
 
-            # update the branch name so that the importer is happy
-            self.local_repo.git_run(['branch', '-M',
-                         'pristine-tar',
-                         '%s/importer/%s/pristine-tar' % (self.namespace, dist),
-                        ]
-                       )
-
     def import_patches_unapplied_tree(self, dsc_pathname):
         """Imports the patches-unapplied source package and writes the
         corresponding working tree for a given srcpkg