Merge ~vorlon/ubuntu-dev-tools:ubuntu-build-revamp into ubuntu-dev-tools:main

[Steve Langasek]

[Gianfranco Costamagna (costamagnagianfranco)]

review: Approve

[Michael Hudson-Doyle (mwhudson)]

The script still feels a bit poorly factored to me somehow (like it does a lot of slow look-before-you-leap checking?) but also this change is an improvement I suppose.

review: Approve

[Michael Hudson-Doyle (mwhudson)]

Going in a better direction indeed!

review: Approve

[Gianfranco Costamagna (costamagnagianfranco)]

For me it worked just nicely, really faster than before.

diff --git a/ubuntu-build b/ubuntu-build
index acbf140..88ec52f 100755
--- a/ubuntu-build
+++ b/ubuntu-build
@@ -2,16 +2,16 @@
 #
 #   ubuntu-build - command line interface for Launchpad buildd operations.
 #
-#   Copyright (C) 2007 Canonical Ltd.
+#   Copyright (C) 2007-2024 Canonical Ltd.
 #   Authors:
 #    - Martin Pitt <[e-mail]>
 #    - Jonathan Davies <[e-mail]>
 #    - Michael Bienia <[e-mail]>
+#    - Steve Langasek <[e-mail]>
 #
 #   This program is free software: you can redistribute it and/or modify
 #   it under the terms of the GNU General Public License as published by
-#   the Free Software Foundation, either version 3 of the License, or
-#   (at your option) any later version.
+#   the Free Software Foundation, version 3 of the License.
 #
 #   This program is distributed in the hope that it will be useful,
 #   but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -29,19 +29,63 @@ import argparse
 import sys
 
 from launchpadlib.credentials import TokenAuthorizationException
+from launchpadlib.launchpad import Launchpad
+import lazr.restfulclient.errors
 
 from ubuntutools import getLogger
-from ubuntutools.lp.lpapicache import Distribution, Launchpad, PersonTeam
-from ubuntutools.lp.udtexceptions import (
-    PackageNotFoundException,
-    PocketDoesNotExistError,
-    SeriesNotFoundException,
-)
+from ubuntutools.lp.udtexceptions import PocketDoesNotExistError
 from ubuntutools.misc import split_release_pocket
 
 Logger = getLogger()
 
 
+def getBuildStates(pkg, archs):
+    res = []
+
+    for build in pkg.getBuilds():
+        if build.arch_tag in archs:
+            res.append(f"  {build.arch_tag}: {build.buildstate}")
+    msg = "\n".join(res)
+    return f"Build state(s) for '{pkg.source_package_name}':\n{msg}"
+
+def rescoreBuilds(pkg, archs, score):
+    res = []
+
+    for build in pkg.getBuilds():
+        arch = build.arch_tag
+        if arch in archs:
+            if not build.can_be_rescored:
+                continue
+            try:
+                build.rescore(score=score)
+                res.append(f"  {arch}: done")
+            except lazr.restfulclient.errors.Unauthorized:
+                Logger.error(
+                    "You don't have the permissions to rescore builds. Ignoring your rescore request."
+                )
+                return None
+            except lazr.restfulclient.errors.BadRequest:
+                Logger.info("Cannot rescore build of %s on %s.",
+                    build.source_package_name, arch)
+                res.append(f"  {arch}: failed")
+
+    msg = "\n".join(res)
+    return f"Rescoring builds of '{pkg.source_package_name}' to {score}:\n{msg}"
+
+def retryBuilds(pkg, archs):
+    res = []
+    for build in pkg.getBuilds():
+        arch = build.arch_tag
+        if arch in archs:
+            try:
+                build.retry()
+                res.append(f"  {arch}: done")
+            except lazr.restfulclient.errors.BadRequest:
+                res.append(f"  {arch}: failed")
+    msg = "\n".join(res)
+    return f"Retrying builds of '{pkg.source_package_name}':\n{msg}"
+
+
 def main():
     # Usage.
     usage = "%(prog)s <srcpackage> <release> <operation>\n\n"
@@ -65,12 +109,7 @@ def main():
     # Prepare our option parser.
     parser = argparse.ArgumentParser(usage=usage)
 
-    # Retry options
-    retry_rescore_options = parser.add_argument_group(
-        "Retry and rescore options",
-        "These options may only be used with the 'retry' and 'rescore' operations.",
-    )
-    retry_rescore_options.add_argument(
+    parser.add_argument(
         "-a",
         "--arch",
         action="append",
@@ -79,6 +118,9 @@ def main():
         f"include: {', '.join(valid_archs)}.",
     )
 
+    parser.add_argument("-A", "--archive", help="operate on ARCHIVE",
+                        default="ubuntu")
+
     # Batch processing options
     batch_options = parser.add_argument_group(
         "Batch processing",
@@ -106,25 +148,35 @@ def main():
         help="Rescore builds to <priority>.",
     )
     batch_options.add_argument(
-        "--arch2",
-        action="append",
-        dest="architecture",
-        help=f"Affect only 'architecture' (can be used several times)."
-        f" Valid architectures are: {', '.join(valid_archs)}.",
+        "--state", action="store", dest="state",
+        help="Act on builds that are in the specified state",
     )
-    parser.add_argument("packages", metavar="package", nargs="+", help=argparse.SUPPRESS)
+
+    parser.add_argument("packages", metavar="package", nargs="*", help=argparse.SUPPRESS)
 
     # Parse our options.
     args = parser.parse_args()
 
-    try:
-        # Will fail here if we have no credentials, bail out
-        Launchpad.login()
-    except TokenAuthorizationException:
-        sys.exit(1)
-    me = PersonTeam.me
+    launchpad = Launchpad.login_with("ubuntu-dev-tools", "production",
+                                     version="devel")
+    me = launchpad.me
 
-    if not args.batch:
+    ubuntu = launchpad.distributions['ubuntu']
+
+    if args.batch:
+        release = args.series
+        if not release:
+            # ppas don't have a proposed pocket so just use the release pocket;
+            # but for the main archive we default to -proposed
+            release = ubuntu.getDevelopmentSeries()[0].name
+            if args.archive == 'ubuntu':
+                release = release + "-proposed"
+        try:
+            (release, pocket) = split_release_pocket(release)
+        except PocketDoesNotExistError as error:
+            Logger.error(error)
+            sys.exit(1)
+    else:
         # Check we have the correct number of arguments.
         if len(args.packages) < 3:
             parser.error("Incorrect number of arguments.")
@@ -137,6 +189,14 @@ def main():
             parser.print_help()
             sys.exit(1)
 
+    archive = launchpad.archives.getByReference(reference=args.archive)
+    try:
+        distroseries = ubuntu.getSeries(name_or_version=release)
+    except lazr.restfulclient.errors.NotFound as error:
+        Logger.error(error)
+        sys.exit(1)
+
+    if not args.batch:
         # Check our operation.
         if operation not in ("rescore", "retry", "status"):
             Logger.error("Invalid operation: %s.", operation)
@@ -160,36 +220,37 @@ def main():
             Logger.error(error)
             sys.exit(1)
 
-        ubuntu_archive = Distribution("ubuntu").getArchive()
         # Get list of published sources for package in question.
         try:
-            sources = ubuntu_archive.getSourcePackage(package, release, pocket)
-            distroseries = Distribution("ubuntu").getSeries(release)
-        except (SeriesNotFoundException, PackageNotFoundException) as error:
-            Logger.error(error)
+            sources = archive.getPublishedSources(
+                          distro_series=distroseries,
+                          exact_match=True,
+                          pocket=pocket,
+                          source_name=package,
+                          status='Published')[0]
+        except IndexError as error:
+            Logger.error("No publication found for package %s", package)
             sys.exit(1)
         # Get list of builds for that package.
         builds = sources.getBuilds()
 
         # Find out the version and component in given release.
-        version = sources.getVersion()
-        component = sources.getComponent()
+        version = sources.source_package_version
+        component = sources.component_name
 
         # Operations that are remaining may only be done by Ubuntu developers
         # (retry) or buildd admins (rescore). Check if the proper permissions
         # are in place.
-        if operation == "rescore":
-            necessary_privs = me.isLpTeamMember("launchpad-buildd-admins")
         if operation == "retry":
-            necessary_privs = me.canUploadPackage(
-                ubuntu_archive,
-                distroseries,
-                sources.getPackageName(),
-                sources.getComponent(),
-                pocket=pocket,
+            necessary_privs = archive.checkUpload(
+                    component=sources.getComponent(),
+                    distroseries=distroseries,
+                    person=launchpad.me,
+                    pocket=pocket,
+                    sourcepackagename=sources.getPackageName(),
             )
 
-        if operation in ("rescore", "retry") and not necessary_privs:
+        if operation == "retry" and not necessary_privs:
             Logger.error(
                 "You cannot perform the %s operation on a %s package as you"
                 " do not have the permissions to do this action.",
@@ -223,7 +284,13 @@ def main():
                     # FIXME: make priority an option
                     priority = 5000
                     Logger.info("Rescoring build %s to %d...", build.arch_tag, priority)
-                    build.rescore(score=priority)
+                    try:
+                        build.rescore(score=priority)
+                    except lazr.restfulclient.errors.Unauthorized:
+                        Logger.error(
+                            "You don't have the permissions to rescore builds. Ignoring your rescore request."
+                        )
+                        break
                 else:
                     Logger.info("Cannot rescore build on %s.", build.arch_tag)
             if operation == "retry":
@@ -252,64 +319,131 @@ def main():
     # filter out duplicate and invalid architectures
     archs.intersection_update(valid_archs)
 
-    release = args.series
-    if not release:
-        release = Distribution("ubuntu").getDevelopmentSeries().name + "-proposed"
-    try:
-        (release, pocket) = split_release_pocket(release)
-    except PocketDoesNotExistError as error:
-        Logger.error(error)
-        sys.exit(1)
+    if not args.packages:
+        retry_count = 0
+        can_rescore = True
+
+        if not args.state:
+            if args.retry:
+                args.state='Failed to build'
+            elif args.priority:
+                args.state='Needs building'
+        # there is no equivalent to series.getBuildRecords() for a ppa.
+        # however, we don't want to have to traverse all build records for
+        # all series when working on the main archive, so we use
+        # series.getBuildRecords() for ubuntu and handle ppas separately
+        series = ubuntu.getSeries(name_or_version=release)
+        if args.archive == 'ubuntu':
+            builds = series.getBuildRecords(build_state=args.state,
+                                            pocket=pocket)
+        else:
+            builds = []
+            for build in archive.getBuildRecords(build_state=args.state,
+                                                 pocket=pocket):
+                if not build.current_source_publication:
+                    continue
+                if build.current_source_publication.distro_series==series:
+                    builds.append(build)
+        for build in builds:
+            if build.arch_tag not in archs:
+                continue
+            if not build.current_source_publication:
+                continue
+            # fixme: refactor
+            # Check permissions (part 2): check upload permissions for the
+            # source package
+            can_retry = args.retry and archive.checkUpload(
+                component=build.current_source_publication.component_name,
+                distroseries=series,
+                person=launchpad.me,
+                pocket=pocket,
+                sourcepackagename=build.source_package_name,
+            )
+            if args.retry and not can_retry:
+                Logger.error(
+                    "You don't have the permissions to retry the "
+                    "build of '%s', skipping.",
+                    build.source_package_name
+                )
+                continue
+            Logger.info(
+                "The source version for '%s' in '%s' (%s) is: %s",
+                 build.source_package_name,
+                 release,
+                 pocket,
+                 build.source_package_version
+            )
 
-    ubuntu_archive = Distribution("ubuntu").getArchive()
-    try:
-        distroseries = Distribution("ubuntu").getSeries(release)
-    except SeriesNotFoundException as error:
-        Logger.error(error)
-        sys.exit(1)
+            if args.retry:
+                Logger.info("Retrying build of %s on %s...",
+                            build.source_package_name, build.arch_tag)
+                retry_count += 1
+                build.retry()
+
+            if args.priority and can_rescore:
+                if build.can_be_rescored:
+                    try:
+                        build.rescore(score=args.priority)
+                    except lazr.restfulclient.errors.Unauthorized:
+                        Logger.error(
+                            "You don't have the permissions to rescore builds. Ignoring your rescore request."
+                        )
+                        can_rescore = False
+                    except lazr.restfulclient.errors.BadRequest:
+                        Logger.info("Cannot rescore build of %s on %s.",
+                            build.source_package_name, build.arch_tag)
+
+            Logger.info("")
+        if args.retry:
+            Logger.info("%d package builds retried", retry_count)
+        sys.exit(0)
 
-    # Check permisions (part 1): Rescoring can only be done by buildd admins
-    can_rescore = args.priority and me.isLpTeamMember("launchpad-buildd-admins")
-    if args.priority and not can_rescore:
-        Logger.error(
-            "You don't have the permissions to rescore builds. Ignoring your rescore request."
-        )
 
     for pkg in args.packages:
         try:
-            pkg = ubuntu_archive.getSourcePackage(pkg, release, pocket)
-        except PackageNotFoundException as error:
-            Logger.error(error)
+            pkg = archive.getPublishedSources(
+                          distro_series=distroseries,
+                          exact_match=True,
+                          pocket=pocket,
+                          source_name=pkg,
+                          status='Published')[0]
+        except IndexError as error:
+            Logger.error("No publication found for package %s", pkg)
             continue
 
         # Check permissions (part 2): check upload permissions for the source
         # package
-        can_retry = args.retry and me.canUploadPackage(
-            ubuntu_archive, distroseries, pkg.getPackageName(), pkg.getComponent()
+        can_retry = args.retry and archive.checkUpload(
+            component=pkg.component_name,
+            distroseries=distroseries,
+            person=launchpad.me,
+            pocket=pocket,
+            sourcepackagename=pkg.source_package_name,
         )
         if args.retry and not can_retry:
             Logger.error(
                 "You don't have the permissions to retry the "
                 "build of '%s'. Ignoring your request.",
-                pkg.getPackageName(),
+                pkg.source_package_name,
             )
 
         Logger.info(
             "The source version for '%s' in '%s' (%s) is: %s",
-            pkg.getPackageName(),
+            pkg.source_package_name,
             release,
             pocket,
-            pkg.getVersion(),
+            pkg.source_package_version,
         )
 
-        Logger.info(pkg.getBuildStates(archs))
+        Logger.info(getBuildStates(pkg, archs))
         if can_retry:
-            Logger.info(pkg.retryBuilds(archs))
-        if args.priority and can_rescore:
-            Logger.info(pkg.rescoreBuilds(archs, args.priority))
+            Logger.info(retryBuilds(pkg, archs))
+        if args.priority:
+            Logger.info(rescoreBuilds(pkg, archs, args.priority))
 
         Logger.info("")
 
 
 if __name__ == "__main__":
     main()
+
diff --git a/ubuntutools/lp/lpapicache.py b/ubuntutools/lp/lpapicache.py
index 589b332..1404809 100644
--- a/ubuntutools/lp/lpapicache.py
+++ b/ubuntutools/lp/lpapicache.py
@@ -1097,51 +1097,6 @@ class SourcePackagePublishingHistory(BaseWrapper):
         for build in builds:
             self._builds[build.arch_tag] = Build(build)
 
-    def getBuildStates(self, archs):
-        res = []
-
-        if not self._builds:
-            self._fetch_builds()
-
-        for arch in archs:
-            build = self._builds.get(arch)
-            if build:
-                res.append(f"  {build}")
-        msg = "\n".join(res)
-        return f"Build state(s) for '{self.getPackageName()}':\n{msg}"
-
-    def rescoreBuilds(self, archs, score):
-        res = []
-
-        if not self._builds:
-            self._fetch_builds()
-
-        for arch in archs:
-            build = self._builds.get(arch)
-            if build:
-                if build.rescore(score):
-                    res.append(f"  {arch}: done")
-                else:
-                    res.append(f"  {arch}: failed")
-        msg = "\n".join(res)
-        return f"Rescoring builds of '{self.getPackageName()}' to {score}:\n{msg}"
-
-    def retryBuilds(self, archs):
-        res = []
-
-        if not self._builds:
-            self._fetch_builds()
-
-        for arch in archs:
-            build = self._builds.get(arch)
-            if build:
-                if build.retry():
-                    res.append(f"  {arch}: done")
-                else:
-                    res.append(f"  {arch}: failed")
-        msg = "\n".join(res)
-        return f"Retrying builds of '{self.getPackageName()}':\n{msg}"
-
 
 class BinaryPackagePublishingHistory(BaseWrapper):
     """