Merge ~racb/usd-importer:nofetch.v2 into usd-importer:master

[Robie Basak]

[Robie Basak (racb)]

Something has screwed up here. 5cfd5ea and parents are already in master, but Launchpad doesn't seem to think so. https://code.launchpad.net/~usd-import-team/usd-importer/+git/usd-importer/+ref/master is behind compared to https://git.launchpad.net/usd-importer/log/. So Launchpad is showing more in its preview diff than it should.

[Nish Aravamudan (nacc)]

review: Approve

diff --git a/usd/build.py b/usd/build.py
index dc689e1..69574f7 100644
--- a/usd/build.py
+++ b/usd/build.py
@@ -211,7 +211,9 @@ class USDBuild:
                                     pkgname, distname)
 
         if not upstream_version_found:
-            raise NoUpstreamFoundException
+            raise NoUpstreamFoundException("Unable to find any prior "
+                                           "publish with a matching upstream "
+                                           "version to %s" % changelog.full_version)
 
         # symlink orig tarballs
         for entry in dsc['Files']:
diff --git a/usd/clone.py b/usd/clone.py
index 91edc6a..d1f5882 100644
--- a/usd/clone.py
+++ b/usd/clone.py
@@ -152,13 +152,16 @@ Example:
         if self.lp_user == LP_USER_NONE:
             usd_proto = self.proto
 
-        self.local_repo.add_and_fetch_lpusip(self.package, self.lp_user)
+        self.local_repo.add_lpusip(self.package, self.lp_user)
+        self.local_repo.fetch_remote('lpusip', ignore_nonexistent=True)
         if self.lp_user != LP_USER_NONE:
-            self.local_repo.add_and_fetch_remote(self.package,
-                                                 self.lp_user,
-                                                 'lpmep',
-                                                 self.lp_user
-                                                )
+            self.local_repo.add_remote(
+                self.package,
+                self.lp_user,
+                'lpmep',
+                self.lp_user
+            )
+            self.local_repo.fetch_remote('lpmep', ignore_nonexistent=True)
 
             logging.debug("added remote 'lpmep' -> %s",
                           self.local_repo.remotes['lpmep'].url
diff --git a/usd/git_repository.py b/usd/git_repository.py
index 93aa853..ed95fcd 100644
--- a/usd/git_repository.py
+++ b/usd/git_repository.py
@@ -203,7 +203,18 @@ class USDGitRepository:
                                      )
                     sys.exit(1)
 
-    def add_and_fetch_remote(self, pkgname, repo_owner, remote_name, lp_user):
+    def fetch_remote(self, remote_name, ignore_nonexistent=False):
+        try:
+            # Does not seem to be working with https
+            # self.local_repo.remotes[remote_name].fetch()
+            self.git_run(['fetch', remote_name])
+        except CalledProcessError:
+            if ignore_nonexistent:
+                logging.info('No objects found in remote %s', remote_name)
+            else:
+                raise
+
+    def add_remote(self, pkgname, repo_owner, remote_name, lp_user):
         if not self.fetch_proto:
             raise Exception('Cannot fetch using an object without a protocol')
         remote_url = ('git.launchpad.net/~%s/ubuntu/+source/%s' %
@@ -243,12 +254,6 @@ class USDGitRepository:
                           '--no-tags'
                          ]
                         )
-        try:
-            # Does not seem to be working with https
-            # self.local_repo.remotes[remote_name].fetch()
-            self.git_run(['fetch', remote_name])
-        except:
-            logging.info('No objects found in remote %s', remote_url)
 
         self.local_repo.remotes.set_push_url(remote_name,
                                              push_url
@@ -265,8 +270,8 @@ class USDGitRepository:
 
         self.remote_name = remote_name
 
-    def add_and_fetch_lpusip(self, pkgname, lp_user):
-        self.add_and_fetch_remote(pkgname, 'usd-import-team', 'lpusip', lp_user)
+    def add_lpusip(self, pkgname, lp_user):
+        self.add_remote(pkgname, 'usd-import-team', 'lpusip', lp_user)
 
     @property
     def env(self):
diff --git a/usd/importer.py b/usd/importer.py
index 73de6a6..f0b12f9 100755
--- a/usd/importer.py
+++ b/usd/importer.py
@@ -77,6 +77,10 @@ class USDImport:
                                   CACHE_PATH
                                  ),
                             default=argparse.SUPPRESS)
+        parser.add_argument('--no-fetch', action='store_true',
+                            help='Do not fetch from the remote (DANGEROUS; '
+                                 'only useful for debugging); implies '
+                                 '--no-push')
         parser.add_argument('--no-push', action='store_true',
                             help='Do not push to the remote')
         parser.add_argument('--no-clean', action='store_true',
@@ -374,7 +378,8 @@ class USDImport:
                 break
         if extracted_dir is None:
             logging.exception('No source extracted?')
-            raise SourceExtractionException
+            raise SourceExtractionException("Failed to find an extracted "
+                                            "directory from dpkg-source -x")
 
         self.local_repo.git_run(['--work-tree', extracted_dir, 'add', '-f', '-A'])
         cp = self.local_repo.git_run(['--work-tree', extracted_dir, 'write-tree'])
@@ -411,7 +416,8 @@ class USDImport:
                 break
         if extracted_dir is None:
             logging.exception('No source extracted?')
-            raise SourceExtractionException
+            raise SourceExtractionException("Failed to find an extracted "
+                                            "directory from dpkg-source -x")
 
         if os.path.isdir(os.path.join(extracted_dir, '.pc')):
             self.local_repo.git_run(['--work-tree', extracted_dir, 'add', '-f', '-A'])
@@ -425,7 +431,7 @@ class USDImport:
                 cp = run(['dpkg-source', '--print-format', extracted_dir])
                 fmt = decode_binary(cp.stdout).strip()
                 if '3.0 (quilt)' not in fmt:
-                    raise StopIteration
+                    raise StopIteration()
             except CalledProcessError as e:
                 try:
                     with open('debian/source/format', 'r') as f:
@@ -433,10 +439,10 @@ class USDImport:
                              if re.match(r'3.0 (.*)', line):
                                  break
                         else:
-                             raise StopIteration
+                             raise StopIteration()
                 # `man dpkg-source` indicates no d/s/format implies 1.0
                 except OSError:
-                    raise StopIteration
+                    raise StopIteration()
 
             while True:
                 try:
@@ -467,7 +473,7 @@ class USDImport:
                     # quilt returns 2 when done pushing
                     if e.returncode != 2:
                         raise
-                    raise StopIteration
+                    raise StopIteration()
         finally:
             os.chdir(oldcwd)
             shutil.rmtree(extract_dir)
@@ -880,7 +886,7 @@ class USDImport:
             no_clean = True
         except AttributeError:
             directory = None
-        no_push = args.no_push
+        no_push = args.no_push or args.no_fetch  # --no-fetch implies --no-push
         try:
             dl_cache = args.dl_cache
         except AttributeError:
@@ -897,10 +903,14 @@ class USDImport:
         atexit.register(self.cleanup, no_clean, self.local_repo.local_dir)
 
 
-        if owner != 'usd-import-team':
-            self.local_repo.add_and_fetch_remote(pkgname, owner, owner, user)
+        if owner == 'usd-import-team':
+            self.local_repo.add_lpusip(pkgname, user)
+            if not args.no_fetch:
+                self.local_repo.fetch_remote('lpusip', ignore_nonexistent=True)
         else:
-            self.local_repo.add_and_fetch_lpusip(pkgname, user)
+            self.local_repo.add_remote(pkgname, owner, owner, user)
+            if not args.no_fetch:
+                self.local_repo.fetch_remote(owner, ignore_nonexistent=True)
 
         debian_source_information = USDSourceInformation(
             'debian',
diff --git a/usd/source_information.py b/usd/source_information.py
index c6a74be..f766ddf 100644
--- a/usd/source_information.py
+++ b/usd/source_information.py
@@ -197,6 +197,7 @@ class USDSourceInformation(object):
                  retry_backoffs
                 ):
         self.launchpad = launchpad_login()
+        self.dist_name = dist_name
         self.dist = self.launchpad.distributions[dist_name]
         self.pkgname = pkgname
         self.parse_pullfile(pull_overrides_filename)
@@ -249,7 +250,8 @@ class USDSourceInformation(object):
 
         spph = self.dist.main_archive.getPublishedSources(**args)
         if len(spph) == 0:
-            raise NoPublicationHistoryException
+            raise NoPublicationHistoryException("Is %s published in %s?" %
+                                                (self.pkgname, self.dist_name))
 
         for srcpkg in spph:
             yield self.get_corrected_spi(srcpkg, workdir)
@@ -280,7 +282,8 @@ class USDSourceInformation(object):
         # Sanity check that the passed in srcpkg name has a publication
         # history
         if len(spph) == 0:
-            raise NoPublicationHistoryException
+            raise NoPublicationHistoryException("Is %s published in %s?" %
+                                                (self.pkgname, self.dist_name))
         if len(head_versions) > 0:
             _spph = list()
             for spphr in spph:
@@ -311,7 +314,7 @@ class USDSourceInformation(object):
                 )
                )
             yield self.get_corrected_spi(srcpkg, workdir)
-        raise StopIteration
+        raise StopIteration()
 
     def parse_pullfile(self, pullfile):
         """Extract source file overrides from a file