Merge ~cjwatson/launchpad:cibuild-create-reports into launchpad:master

[Colin Watson]

Create RevisionStatusReports when requesting CIBuilds

This ensures that jobs that have been scheduled as part of these builds show up as pending in the web UI as soon as they're scheduled, rather than only becoming visible once the build completes and the resulting upload is processed.

[Jürgen Gmach (jugmac00)]

review: Approve

diff --git a/lib/lp/archiveuploader/ciupload.py b/lib/lp/archiveuploader/ciupload.py
index 6f8dea0..7cbd698 100644
--- a/lib/lp/archiveuploader/ciupload.py
+++ b/lib/lp/archiveuploader/ciupload.py
@@ -61,10 +61,11 @@ class CIUpload:
             report = getUtility(IRevisionStatusReportSet).getByCIBuildAndTitle(
                 build, job_name)
             if not report:
-                # the report should normally exist, since the build request
-                # logic will eventually create report rows for the jobs it
-                # expects to run, but for robustness it's a good idea to
-                # ensure its existence here
+                # The report should normally exist, since
+                # lp.code.model.cibuild.CIBuildSet._tryToRequestBuild
+                # creates report rows for the jobs it expects to run, but
+                # for robustness it's a good idea to ensure its existence
+                # here.
                 report = getUtility(IRevisionStatusReportSet).new(
                     creator=build.git_repository.owner,
                     title=job_name,
diff --git a/lib/lp/code/model/cibuild.py b/lib/lp/code/model/cibuild.py
index bdc570c..4da34fb 100644
--- a/lib/lp/code/model/cibuild.py
+++ b/lib/lp/code/model/cibuild.py
@@ -49,6 +49,7 @@ from lp.code.interfaces.cibuild import (
     MissingConfiguration,
     )
 from lp.code.interfaces.githosting import IGitHostingClient
+from lp.code.interfaces.revisionstatus import IRevisionStatusReportSet
 from lp.code.model.gitref import GitRef
 from lp.code.model.lpcraft import load_configuration
 from lp.registry.interfaces.pocket import PackagePublishingPocket
@@ -427,14 +428,34 @@ class CIBuildSet(SpecificBuildFarmJobSourceMixin):
         notify(ObjectCreatedEvent(build))
         return build
 
-    def _tryToRequestBuild(self, git_repository, commit_sha1, das, logger):
+    def _tryToRequestBuild(self, git_repository, commit_sha1, configuration,
+                           das, logger):
         try:
             if logger is not None:
                 logger.info(
                     "Requesting CI build for %s on %s/%s",
                     commit_sha1, das.distroseries.name, das.architecturetag,
                 )
-            self.requestBuild(git_repository, commit_sha1, das)
+            build = self.requestBuild(git_repository, commit_sha1, das)
+            # Create reports for each individual job in this build so that
+            # they show up as pending in the web UI.  The job names
+            # generated here should match those generated by
+            # lpbuildd.ci._make_job_id in launchpad-buildd;
+            # lp.archiveuploader.ciupload looks for this report and attaches
+            # artifacts to it.
+            rsr_set = getUtility(IRevisionStatusReportSet)
+            for stage in configuration.pipeline:
+                for job_name in stage:
+                    for i in range(len(configuration.jobs.get(job_name, []))):
+                        # XXX cjwatson 2022-03-17: It would be better if we
+                        # could set some kind of meaningful description as
+                        # well.
+                        rsr_set.new(
+                            creator=git_repository.owner,
+                            title="%s:%s" % (job_name, i),
+                            git_repository=git_repository,
+                            commit_sha1=commit_sha1,
+                            ci_build=build)
         except CIBuildAlreadyRequested:
             pass
         except Exception as e:
@@ -463,9 +484,9 @@ class CIBuildSet(SpecificBuildFarmJobSourceMixin):
                 if logger is not None:
                     logger.error(e)
                 continue
-            for das in determine_DASes_to_build(configuration):
+            for das in determine_DASes_to_build(configuration, logger=logger):
                 self._tryToRequestBuild(
-                    git_repository, commit["sha1"], das,  logger)
+                    git_repository, commit["sha1"], configuration, das, logger)
 
     def getByID(self, build_id):
         """See `ISpecificBuildFarmJobSource`."""
diff --git a/lib/lp/code/model/tests/test_cibuild.py b/lib/lp/code/model/tests/test_cibuild.py
index bd2c5e9..6118379 100644
--- a/lib/lp/code/model/tests/test_cibuild.py
+++ b/lib/lp/code/model/tests/test_cibuild.py
@@ -16,6 +16,7 @@ import pytz
 from storm.locals import Store
 from testtools.matchers import (
     Equals,
+    MatchesSetwise,
     MatchesStructure,
     )
 from zope.component import getUtility
@@ -43,6 +44,7 @@ from lp.code.interfaces.cibuild import (
     ICIBuildSet,
     MissingConfiguration,
     )
+from lp.code.interfaces.revisionstatus import IRevisionStatusReportSet
 from lp.code.model.cibuild import (
     determine_DASes_to_build,
     get_all_commits_for_paths,
@@ -484,9 +486,17 @@ class TestCIBuildSet(TestCaseWithFactory):
         )
         configuration = dedent("""\
             pipeline:
-            - test
+                - build
+                - test
 
             jobs:
+                build:
+                    matrix:
+                        - series: bionic
+                          architectures: amd64
+                        - series: focal
+                          architectures: amd64
+                    run: pyproject-build
                 test:
                     series: focal
                     architectures: amd64
@@ -512,11 +522,21 @@ class TestCIBuildSet(TestCaseWithFactory):
         )
 
         build = getUtility(ICIBuildSet).findByGitRepository(repository).one()
+        reports = list(
+            getUtility(IRevisionStatusReportSet).findByRepository(repository))
 
-        # check that a build was created
+        # check that a build and some reports were created
         self.assertEqual(ref.commit_sha1, build.commit_sha1)
         self.assertEqual("focal", build.distro_arch_series.distroseries.name)
         self.assertEqual("amd64", build.distro_arch_series.architecturetag)
+        self.assertThat(reports, MatchesSetwise(*(
+            MatchesStructure.byEquality(
+                creator=repository.owner,
+                title=title,
+                git_repository=repository,
+                commit_sha1=ref.commit_sha1,
+                ci_build=build)
+            for title in ("build:0", "build:1", "test:0"))))
 
     def test_requestBuildsForRefs_no_commits_at_all(self):
         repository = self.factory.makeGitRepository()
@@ -534,6 +554,10 @@ class TestCIBuildSet(TestCaseWithFactory):
         self.assertTrue(
             getUtility(ICIBuildSet).findByGitRepository(repository).is_empty()
         )
+        self.assertTrue(
+            getUtility(IRevisionStatusReportSet).findByRepository(
+                repository).is_empty()
+        )
 
     def test_requestBuildsForRefs_no_matching_commits(self):
         repository = self.factory.makeGitRepository()
@@ -554,6 +578,10 @@ class TestCIBuildSet(TestCaseWithFactory):
         self.assertTrue(
             getUtility(ICIBuildSet).findByGitRepository(repository).is_empty()
         )
+        self.assertTrue(
+            getUtility(IRevisionStatusReportSet).findByRepository(
+                repository).is_empty()
+        )
 
     def test_requestBuildsForRefs_configuration_parse_error(self):
         ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
@@ -592,6 +620,10 @@ class TestCIBuildSet(TestCaseWithFactory):
         self.assertTrue(
             getUtility(ICIBuildSet).findByGitRepository(repository).is_empty()
         )
+        self.assertTrue(
+            getUtility(IRevisionStatusReportSet).findByRepository(
+                repository).is_empty()
+        )
 
         self.assertEqual(
             "ERROR Cannot parse .launchpad.yaml from %s: "
@@ -646,6 +678,10 @@ class TestCIBuildSet(TestCaseWithFactory):
         self.assertTrue(
             getUtility(ICIBuildSet).findByGitRepository(repository).is_empty()
         )
+        self.assertTrue(
+            getUtility(IRevisionStatusReportSet).findByRepository(
+                repository).is_empty()
+        )
         self.assertEqual(
             "INFO Requesting CI build "
             "for %s on focal/amd64\n" % ref.commit_sha1,
@@ -698,6 +734,10 @@ class TestCIBuildSet(TestCaseWithFactory):
         self.assertTrue(
             getUtility(ICIBuildSet).findByGitRepository(repository).is_empty()
         )
+        self.assertTrue(
+            getUtility(IRevisionStatusReportSet).findByRepository(
+                repository).is_empty()
+        )
 
         log_line1, log_line2 = logger.getLogBuffer().splitlines()
         self.assertEqual(