Merge ~nacc/git-ubuntu:lp1730655-fix-importer-devel-branches into git-ubuntu:master

[Nish Aravamudan]

Make jenkins happy.

02e3e7d... by Nish Aravamudan on 2017-11-07

source_information: fix incorrect API comments

f53eda9... by Nish Aravamudan on 2017-11-07

source_information: cleanup formatting per style

9507164... by Nish Aravamudan on 2017-11-07

source_information: add API to obtain all series objects/names

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:69c9b3397ceec3843bc1706b065b2fe4475d5e5b
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/197/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Style Check
    SUCCESS: Unit Tests
    SUCCESS: Integration Tests
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/197/rebuild

review: Approve (continuous-integration)

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:853cb2e401a1122295562357225c63c127426aa5
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/211/
Executed test runs:
    SUCCESS: Checkout
    SUCCESS: Style Check
    SUCCESS: Unit Tests
    SUCCESS: Integration Tests
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/211/rebuild

review: Approve (continuous-integration)

[Robie Basak (racb)]

Looks good!

I refactored and added unit tests in https://code.launchpad.net/~racb/usd-importer/+git/usd-importer/+ref/fix-importer-devel-branches

I have not however tested integration (ie. that my update_devel_branches() function is still correct or even works at all).

Please could you review and pull in my commits if you're happy?

diff --git a/gitubuntu/importer.py b/gitubuntu/importer.py
index f7dfe92..2674f4b 100644
--- a/gitubuntu/importer.py
+++ b/gitubuntu/importer.py
@@ -355,6 +355,13 @@ def main(
             allow_applied_failures=allow_applied_failures,
         )
 
+    update_devel_branches(
+        repo=repo,
+        namespace=namespace,
+        pkgname=pkgname,
+        ubuntu_sinfo=ubuntu_sinfo,
+    )
+
     os.chdir(oldcwd)
 
     repo.garbage_collect()
@@ -905,62 +912,111 @@ def override_parents(repo, spi, namespace):
         applied_changelog_parent_commit,
     )
 
-def _update_devel_branches(
+def update_devel_branches(
     repo,
     namespace,
     pkgname,
     ubuntu_sinfo,
-    spi,
 ):
-    # spi previously had a default argument value of None, but the
-    # functionality has since been dropped. Let's ensure that no caller
-    # depending on this functionality remains.
-    assert spi
+    """update_devel_branches - move the 'meta' -devel branches to the
+    latest publication in each series
 
-    # we do not maintain -devel pointers for debian
-    if str(spi.distribution.name.lower()) != "ubuntu":
-        return
-    for devel_head in (
-        '%s/ubuntu/%s-devel' % (namespace, spi.series.name.lower()),
-        '%s/ubuntu/devel' % namespace
-                      ):
-        logging.debug("Updating %s to %s" % (devel_head,
-                                             spi.head_name(namespace)))
-        repo.update_head_to_commit(
-            devel_head,
-            repo.head_to_commit(spi.head_name(namespace)),
+    For all known series in @ubuntu_sinfo, update the
+    ubuntu/series-devel branch to the latest version in the series'
+    pocket branches.
+
+    Update the ubuntu/devel branch to the latest ubuntu/series-devel
+    branch.
+
+    Do this for both the unapplied and applied branches.
+
+    Arguments:
+    repo - gitubuntu.git_repository.GitUbuntuRepository object
+    pkgname - string source package name
+    namespace - string namespace under which the relevant branch refs
+    can be found
+    ubuntu_sinfo - GitUbuntuSourceInformation object for the ubuntu
+    archive
+    """
+
+    for applied_prefix in ('', 'applied/',):
+        head_versions = repo.get_heads_and_versions(
+            '%subuntu' % applied_prefix,
+            namespace=namespace,
+        )
+        devel_hash = None
+        devel_name = None
+        devel_branch_name = '%s/%subuntu/devel' % (
+            namespace,
+            applied_prefix,
         )
 
+        for series_name in ubuntu_sinfo.all_series_name_list:
+            series_devel_hash = None
+            series_devel_name = None
+            series_devel_version = None
+            # Find highest version publish in these pockets, favoring this
+            # order of pockets
+            for suff in ("-proposed", "-updates", "-security", "",):
+                name = "%s/%subuntu/%s%s" % (
+                    namespace,
+                    applied_prefix,
+                    series_name,
+                    suff,
+                )
+                if name in head_versions and version_compare(
+                    head_versions[name]['version'],
+                    series_devel_version
+                ) > 0:
+                    series_devel_name = name
+                    series_devel_version = head_versions[name]['version']
+                    series_devel_hash = str(
+                        head_versions[name]['head'].peel().id
+                    )
+            if series_devel_hash:
+                series_devel_branch_name = '%s/%subuntu/%s-devel' % (
+                    namespace,
+                    applied_prefix,
+                    series_name,
+                )
+                logging.debug(
+                    "Setting %s branch to '%s' (%s)",
+                    series_devel_branch_name,
+                    series_devel_name,
+                    series_devel_hash,
+                )
+                repo.update_head_to_commit(
+                    series_devel_branch_name,
+                    series_devel_hash,
+                )
 
-def update_devel_branches(
-    repo,
-    namespace,
-    pkgname,
-    ubuntu_sinfo,
-    spi,
-):
-    _update_devel_branches(
-        repo=repo,
-        namespace=namespace,
-        pkgname=pkgname,
-        ubuntu_sinfo=ubuntu_sinfo,
-        spi=spi,
-    )
+                # the first series devel pointer we find is the most recent
+                # XXX: if a source package was published, but no longer is,
+                # do we want to have a ubuntu-devel pointer? This was the
+                # old behavior, but we need to decide on the semantics of
+                # the devel branch (most recent or development release)
+                if not devel_hash:
+                    devel_name = series_devel_name
+                    devel_hash = series_devel_hash
 
-def update_applied_devel_branches(
-    repo,
-    namespace,
-    pkgname,
-    ubuntu_sinfo,
-    spi,
-):
-    _update_devel_branches(
-        repo=repo,
-        namespace='%s/applied' % namespace,
-        pkgname=pkgname,
-        ubuntu_sinfo=ubuntu_sinfo,
-        spi=spi,
-    )
+        if devel_hash is None:
+            logging.warn(
+                "Source package '%s' does not appear to have been published "
+                "in Ubuntu. No %s branch created.",
+                pkgname,
+                devel_branch_name,
+            )
+        else:
+            logging.debug(
+                "Setting %s branch to '%s' (%s)",
+                devel_branch_name,
+                devel_name,
+                devel_hash,
+            )
+            repo.update_head_to_commit(
+                "%s/%subuntu/devel" % (namespace, applied_prefix),
+                devel_hash,
+            )
 
 # imports a package based upon source package information
 def import_unapplied_spi(repo, spi, namespace, skip_orig, ubuntu_sinfo):
@@ -1159,14 +1215,6 @@ def import_unapplied_spi(repo, spi, namespace, skip_orig, ubuntu_sinfo):
         upload_parent_commit,
     )
 
-    update_devel_branches(
-        repo=repo,
-        namespace=namespace,
-        pkgname=spi.name,
-        ubuntu_sinfo=ubuntu_sinfo,
-        spi=spi,
-    )
-
 def import_applied_spi(
     repo,
     spi,
@@ -1337,14 +1385,6 @@ def import_applied_spi(
         unapplied_parent_commit,
     )
 
-    update_applied_devel_branches(
-        repo=repo,
-        pkgname=spi.name,
-        namespace=namespace,
-        ubuntu_sinfo=ubuntu_sinfo,
-        spi=spi,
-    )
-
 def import_publishes(
     repo,
     pkgname,
diff --git a/gitubuntu/source_information.py b/gitubuntu/source_information.py
index 3fef4ec..b97bc8a 100644
--- a/gitubuntu/source_information.py
+++ b/gitubuntu/source_information.py
@@ -290,6 +290,7 @@ class NoPublicationHistoryException(Exception):
 
 # An abstraction of an information source about source packages
 class GitUbuntuSourceInformation(object):
+    _all_series_list = None
     _active_series_list = None
     _stable_series_list = None
     _current_series = None
@@ -345,11 +346,12 @@ class GitUbuntuSourceInformation(object):
     def active_series(self):
         if self.dist_name.startswith('ppa:'):
             return []
-        # return a list of series names sorted with newest first.
+        # return a list of series objects sorted with newest first.
         if self._active_series_list is None:
             self._active_series_list = sorted(
                 [r for r in self.dist.series if r.active],
-                key=lambda s: float(s.version), reverse=True
+                key=lambda s: float(s.version),
+                reverse=True,
             )
         return self._active_series_list
 
@@ -357,16 +359,32 @@ class GitUbuntuSourceInformation(object):
     def stable_series(self):
         if self.dist_name.startswith('ppa:'):
             return []
-        # return a list of released series names sorted with newest first.
+        # return a list of released series objects sorted with newest first.
         if self._stable_series_list is None:
             self._stable_series_list = sorted(
-                [r for r in self.dist.series if r.active and
-                 r.status in ('Current Stable Release', 'Supported')],
-                key=lambda s: float(s.version), reverse=True
+                [
+                    r for r in self.dist.series if r.active and
+                    r.status in ('Current Stable Release', 'Supported')
+                ],
+                key=lambda s: float(s.version),
+                reverse=True,
             )
         return self._stable_series_list
 
     @property
+    def all_series(self):
+        if self.dist_name.startswith('ppa:'):
+            return []
+        # return a list of all series objects sorted with newest first
+        if self._all_series_list is None:
+            self._all_series_list = sorted(
+                self.dist.series,
+                key=lambda s: float(s.version),
+                reverse=True,
+            )
+        return self._all_series_list
+
+    @property
     def current_series_name(self):
         if not self.current_series:
              return None
@@ -380,6 +398,10 @@ class GitUbuntuSourceInformation(object):
     def stable_series_name_list(self):
         return [r.name for r in self.stable_series]
 
+    @property
+    def all_series_name_list(self):
+        return [r.name for r in self.all_series]
+
     def get_corrected_spi(self, srcpkg, workdir=None):
        try:
            pull_override = self.pull_overrides[srcpkg.source_package_version]