Merge ~cjwatson/launchpad-buildd:ci-individual-results into launchpad-buildd:master

[Colin Watson]

Return results from individual CI jobs

It's otherwise difficult for Launchpad to know how to record the results.

[Ioana Lasc (ilasc)]

LGTM!

review: Approve

diff --git a/debian/changelog b/debian/changelog
index c35ca7e..b2bcf77 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+launchpad-buildd (207) UNRELEASED; urgency=medium
+
+  * Return results from individual CI jobs.
+
+ -- Colin Watson <[e-mail]>  Thu, 13 Jan 2022 14:51:09 +0000
+
 launchpad-buildd (206) bionic; urgency=medium
 
   * Fix flake8 violations.
diff --git a/lpbuildd/ci.py b/lpbuildd/ci.py
index 9c70451..14ac7c6 100644
--- a/lpbuildd/ci.py
+++ b/lpbuildd/ci.py
@@ -24,6 +24,11 @@ RETCODE_SUCCESS = 0
 RETCODE_FAILURE_INSTALL = 200
 RETCODE_FAILURE_BUILD = 201
 
+# These must match the names of `RevisionStatusResult` enumeration items in
+# Launchpad.
+RESULT_SUCCEEDED = "SUCCEEDED"
+RESULT_FAILED = "FAILED"
+
 
 class CIBuildState(DebianBuildState):
     PREPARE = "PREPARE"
@@ -125,18 +130,20 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
         This state is repeated for each CI job in the pipeline.
         """
         if retcode == RETCODE_SUCCESS:
-            pass
-        elif (retcode >= RETCODE_FAILURE_INSTALL and
-              retcode <= RETCODE_FAILURE_BUILD):
-            if not self.alreadyfailed:
-                self._builder.log("Job %s failed." % self.current_job_id)
-                self._builder.buildFail()
-            self.alreadyfailed = True
+            result = RESULT_SUCCEEDED
         else:
-            if not self.alreadyfailed:
-                self._builder.builderFail()
+            result = RESULT_FAILED
+            if (retcode >= RETCODE_FAILURE_INSTALL and
+                    retcode <= RETCODE_FAILURE_BUILD):
+                self._builder.log("Job %s failed." % self.current_job_id)
+                if not self.alreadyfailed:
+                    self._builder.buildFail()
+            else:
+                if not self.alreadyfailed:
+                    self._builder.builderFail()
             self.alreadyfailed = True
         yield self.deferGatherResults(reap=False)
+        self.job_status[self.current_job_id]["result"] = result
         if self.remaining_jobs and not self.alreadyfailed:
             self.runNextJob()
         else:
diff --git a/lpbuildd/tests/test_ci.py b/lpbuildd/tests/test_ci.py
index 1009a36..2cc1387 100644
--- a/lpbuildd/tests/test_ci.py
+++ b/lpbuildd/tests/test_ci.py
@@ -18,6 +18,10 @@ from lpbuildd.builder import get_build_path
 from lpbuildd.ci import (
     CIBuildManager,
     CIBuildState,
+    RESULT_SUCCEEDED,
+    RESULT_FAILED,
+    RETCODE_FAILURE_BUILD,
+    RETCODE_SUCCESS,
     )
 from lpbuildd.tests.fakebuilder import FakeBuilder
 from lpbuildd.tests.matchers import HasWaitingFiles
@@ -94,8 +98,9 @@ class TestCIBuildManagerIteration(TestCase):
         self.assertFalse(self.builder.wasCalled("chrootFail"))
 
     @defer.inlineCallbacks
-    def expectRunJob(self, job_name, job_index, options=None):
-        yield self.buildmanager.iterate(0)
+    def expectRunJob(self, job_name, job_index, options=None,
+                     retcode=RETCODE_SUCCESS):
+        yield self.buildmanager.iterate(retcode)
         self.assertEqual(CIBuildState.RUN_JOB, self.getState())
         expected_command = [
             "sharepath/bin/in-target", "in-target", "run-ci",
@@ -110,7 +115,7 @@ class TestCIBuildManagerIteration(TestCase):
         self.assertFalse(self.builder.wasCalled("chrootFail"))
 
     @defer.inlineCallbacks
-    def test_iterate(self):
+    def test_iterate_success(self):
         # The build manager iterates multiple CI jobs from start to finish.
         args = {
             "git_repository": "https://git.launchpad.test/~example/+git/ci",
@@ -148,6 +153,7 @@ class TestCIBuildManagerIteration(TestCase):
                     "output": {
                         "ci.whl": self.builder.waitingfiles["build:0/ci.whl"],
                         },
+                    "result": RESULT_SUCCEEDED,
                     },
                 },
             extra_status["jobs"])
@@ -179,6 +185,7 @@ class TestCIBuildManagerIteration(TestCase):
                     "output": {
                         "ci.whl": self.builder.waitingfiles["build:0/ci.whl"],
                         },
+                    "result": RESULT_SUCCEEDED,
                     },
                 "test:0": {
                     "log": self.builder.waitingfiles["test:0.log"],
@@ -186,6 +193,7 @@ class TestCIBuildManagerIteration(TestCase):
                         "ci.tar.gz":
                             self.builder.waitingfiles["test:0/ci.tar.gz"],
                         },
+                    "result": RESULT_SUCCEEDED,
                     },
                 },
             extra_status["jobs"])
@@ -221,3 +229,80 @@ class TestCIBuildManagerIteration(TestCase):
         shutil.rmtree(get_build_path(
             self.buildmanager.home, self.buildmanager._buildid))
         self.assertIn("jobs", self.buildmanager.status())
+
+    @defer.inlineCallbacks
+    def test_iterate_failure(self):
+        # The build manager records CI jobs that fail.
+        args = {
+            "git_repository": "https://git.launchpad.test/~example/+git/ci",
+            "git_path": "main",
+            "jobs": [("build", "0"), ("test", "0")],
+            }
+        expected_options = [
+            "--git-repository", "https://git.launchpad.test/~example/+git/ci",
+            "--git-path", "main",
+            ]
+        yield self.startBuild(args, expected_options)
+
+        # After preparation, start running the first job.
+        yield self.expectRunJob("build", "0")
+        self.buildmanager.backend.add_file(
+            "/build/output/build:0.log", b"I am a failing CI build job log.")
+
+        # If the first job fails, then the build fails here.
+        yield self.buildmanager.iterate(RETCODE_FAILURE_BUILD)
+        expected_command = [
+            "sharepath/bin/in-target", "in-target", "scan-for-processes",
+            "--backend=lxd", "--series=focal", "--arch=amd64", self.buildid,
+            ]
+        self.assertEqual(CIBuildState.RUN_JOB, self.getState())
+        self.assertEqual(expected_command, self.buildmanager.commands[-1])
+        self.assertNotEqual(
+            self.buildmanager.iterate, self.buildmanager.iterators[-1])
+        self.assertTrue(self.builder.wasCalled("buildFail"))
+        self.assertThat(self.builder, HasWaitingFiles.byEquality({
+            "build:0.log": b"I am a failing CI build job log.",
+            }))
+
+        # Output from the first job is visible in the status response.
+        extra_status = self.buildmanager.status()
+        self.assertEqual(
+            {
+                "build:0": {
+                    "log": self.builder.waitingfiles["build:0.log"],
+                    "result": RESULT_FAILED,
+                    },
+                },
+            extra_status["jobs"])
+
+        # Control returns to the DebianBuildManager in the UMOUNT state.
+        self.buildmanager.iterateReap(self.getState(), 0)
+        expected_command = [
+            "sharepath/bin/in-target", "in-target", "umount-chroot",
+            "--backend=lxd", "--series=focal", "--arch=amd64", self.buildid,
+            ]
+        self.assertEqual(CIBuildState.UMOUNT, self.getState())
+        self.assertEqual(expected_command, self.buildmanager.commands[-1])
+        self.assertEqual(
+            self.buildmanager.iterate, self.buildmanager.iterators[-1])
+        self.assertTrue(self.builder.wasCalled("buildFail"))
+
+        # If we iterate to the end of the build, then the extra status
+        # information is still present.
+        self.buildmanager.iterate(0)
+        expected_command = [
+            'sharepath/bin/in-target', 'in-target', 'remove-build',
+            '--backend=lxd', '--series=focal', '--arch=amd64', self.buildid,
+            ]
+        self.assertEqual(CIBuildState.CLEANUP, self.getState())
+        self.assertEqual(expected_command, self.buildmanager.commands[-1])
+        self.assertEqual(
+            self.buildmanager.iterate, self.buildmanager.iterators[-1])
+
+        self.buildmanager.iterate(0)
+        self.assertFalse(self.builder.wasCalled('buildOK'))
+        self.assertTrue(self.builder.wasCalled('buildComplete'))
+        # remove-build would remove this in a non-test environment.
+        shutil.rmtree(get_build_path(
+            self.buildmanager.home, self.buildmanager._buildid))
+        self.assertIn("jobs", self.buildmanager.status())