Merge ~cjwatson/launchpad-buildd:lpcraft into launchpad-buildd:master

[Colin Watson]

Add CI job support

Running jobs is handled via lpcraft.

The main complexity here is that we need to be able to return the output of each individual lpcraft job separately, and so we need a novel mechanism for returning the status of parts of a build farm job. I built this on top of the extra status file mechanism, previously used to return VCS revision IDs. This also means that we need a slightly more complicated state machine than usual, since we have an initial preparation step followed by iterating over a "run job" state for each job. Most of the rest of this is fairly typical for new build types.

At present there's a fair amount of overhead, since we're using standard buildd containers and installing lpcraft in them, and then lpcraft itself creates containers and sets them up according to `.launchpad.yaml`. We should be able to optimize this later by providing pre-built containers.

[Ioana Lasc (ilasc)]

LGTM!

review: Approve

[Jürgen Gmach (jugmac00)]

Tough one :-) Two comments, which don't need to be addressed, just wondering.

When trying to setup a dev environment to run the tests I got a:

```
$ lxc list
+---------------+---------+----------------------+-----------------------------------------------+-----------------+-----------+
| NAME | STATE | IPV4 | IPV6 | TYPE | SNAPSHOTS |
+---------------+---------+----------------------+-----------------------------------------------+-----------------+-----------+
| lp-builddev | RUNNING | | fd42:dbc9:151f:7e68:216:3eff:fe2d:e662 (eth0) | VIRTUAL-MACHINE | 0 |
+---------------+---------+----------------------+-----------------------------------------------+-----------------+-----------+
| lpdev | RUNNING | 10.67.244.209 (eth0) | fd42:dbc9:151f:7e68:216:3eff:fe8f:4b1d (eth0) | CONTAINER | 0 |
+---------------+---------+----------------------+-----------------------------------------------+-----------------+-----------+
| turnip-bionic | RUNNING | 10.67.244.191 (eth0) | fd42:dbc9:151f:7e68:216:3eff:fe56:ea7a (eth0) | CONTAINER | 0 |
+---------------+---------+----------------------+-----------------------------------------------+-----------------+-----------+
$ lxc shell lp-builddev
Error: Failed to connect to lxd-agent
```

Any clue what is going on?

[Colin Watson (cjwatson)]

You'd probably be best off asking LXD folks about your environment woes. It rings a faint bell somewhere but I don't remember the details - they'll probably be more on top of this sort of thing.

diff --git a/debian/changelog b/debian/changelog
index cf63e39..5a758a6 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ launchpad-buildd (206) UNRELEASED; urgency=medium
   * Fix flake8 violations.
   * Refactor extra status handling to be common to all build types.
   * Fix handling of empty output in Backend.find.
+  * Add CI job support.
 
  -- Colin Watson <[e-mail]>  Wed, 08 Dec 2021 15:42:26 +0000
 
diff --git a/lpbuildd/buildd-slave.tac b/lpbuildd/buildd-slave.tac
index 7c1947e..24d60ef 100644
--- a/lpbuildd/buildd-slave.tac
+++ b/lpbuildd/buildd-slave.tac
@@ -24,6 +24,7 @@ from twisted.web import (
 from lpbuildd.binarypackage import BinaryPackageBuildManager
 from lpbuildd.builder import XMLRPCBuilder
 from lpbuildd.charm import CharmBuildManager
+from lpbuildd.ci import CIBuildManager
 from lpbuildd.oci import OCIBuildManager
 from lpbuildd.livefs import LiveFilesystemBuildManager
 from lpbuildd.log import RotatableFileLogObserver
@@ -49,6 +50,7 @@ builder.registerManager(LiveFilesystemBuildManager, "livefs")
 builder.registerManager(SnapBuildManager, "snap")
 builder.registerManager(OCIBuildManager, "oci")
 builder.registerManager(CharmBuildManager, "charm")
+builder.registerManager(CIBuildManager, "ci")
 
 application = service.Application('Builder')
 application.addComponent(
diff --git a/lpbuildd/builder.py b/lpbuildd/builder.py
index 194f749..1257fce 100644
--- a/lpbuildd/builder.py
+++ b/lpbuildd/builder.py
@@ -151,6 +151,8 @@ class BuildManager(object):
         self.is_archive_private = False
         self.home = os.environ['HOME']
         self.abort_timeout = 120
+        self.status_path = get_build_path(self.home, self._buildid, "status")
+        self._final_extra_status = None
 
     @property
     def needs_sanitized_logs(self):
@@ -214,6 +216,9 @@ class BuildManager(object):
 
     def doCleanup(self):
         """Remove the build tree etc."""
+        # Fetch a final snapshot of manager-specific extra status.
+        self._final_extra_status = self.status()
+
         if not self.fast_cleanup:
             self.runTargetSubProcess("remove-build")
 
@@ -276,9 +281,10 @@ class BuildManager(object):
         This may be used to return manager-specific information from the
         XML-RPC status call.
         """
-        status_path = get_build_path(self.home, self._buildid, "status")
+        if self._final_extra_status is not None:
+            return self._final_extra_status
         try:
-            with open(status_path) as status_file:
+            with open(self.status_path) as status_file:
                 return json.load(status_file)
         except IOError:
             pass
@@ -359,12 +365,12 @@ class BuildManager(object):
         self._subprocess.ignore = True
         self._subprocess.transport.loseConnection()
 
-    def addWaitingFileFromBackend(self, path):
+    def addWaitingFileFromBackend(self, path, name=None):
         fetched_dir = tempfile.mkdtemp()
         try:
             fetched_path = os.path.join(fetched_dir, os.path.basename(path))
             self.backend.copy_out(path, fetched_path)
-            self._builder.addWaitingFile(fetched_path)
+            self._builder.addWaitingFile(fetched_path, name=name)
         finally:
             shutil.rmtree(fetched_dir)
 
@@ -501,9 +507,11 @@ class Builder(object):
         os.rename(tmppath, self.cachePath(sha1sum))
         return sha1sum
 
-    def addWaitingFile(self, path):
+    def addWaitingFile(self, path, name=None):
         """Add a file to the cache and store its details for reporting."""
-        self.waitingfiles[os.path.basename(path)] = self.storeFile(path)
+        if name is None:
+            name = os.path.basename(path)
+        self.waitingfiles[name] = self.storeFile(path)
 
     def abort(self):
         """Abort the current build."""
diff --git a/lpbuildd/ci.py b/lpbuildd/ci.py
new file mode 100644
index 0000000..9c70451
--- /dev/null
+++ b/lpbuildd/ci.py
@@ -0,0 +1,187 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import print_function
+
+__metaclass__ = type
+
+import os
+
+from six.moves.configparser import (
+    NoOptionError,
+    NoSectionError,
+    )
+from twisted.internet import defer
+
+from lpbuildd.debian import (
+    DebianBuildManager,
+    DebianBuildState,
+    )
+from lpbuildd.proxy import BuildManagerProxyMixin
+
+
+RETCODE_SUCCESS = 0
+RETCODE_FAILURE_INSTALL = 200
+RETCODE_FAILURE_BUILD = 201
+
+
+class CIBuildState(DebianBuildState):
+    PREPARE = "PREPARE"
+    RUN_JOB = "RUN_JOB"
+
+
+class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
+    """Run CI jobs."""
+
+    backend_name = "lxd"
+    initial_build_state = CIBuildState.PREPARE
+
+    @property
+    def needs_sanitized_logs(self):
+        return True
+
+    def initiate(self, files, chroot, extra_args):
+        """Initiate a build with a given set of files and chroot."""
+        self.jobs = extra_args["jobs"]
+        if not self.jobs:
+            raise ValueError("Must request at least one job")
+        self.branch = extra_args.get("branch")
+        self.git_repository = extra_args.get("git_repository")
+        self.git_path = extra_args.get("git_path")
+        self.channels = extra_args.get("channels", {})
+        self.proxy_url = extra_args.get("proxy_url")
+        self.revocation_endpoint = extra_args.get("revocation_endpoint")
+        self.proxy_service = None
+        self.job_status = {}
+
+        super(CIBuildManager, self).initiate(files, chroot, extra_args)
+
+    def doRunBuild(self):
+        """Start running CI jobs."""
+        self.proxy_args = self.startProxy()
+        if self.revocation_endpoint:
+            self.proxy_args.extend(
+                ["--revocation-endpoint", self.revocation_endpoint])
+        args = list(self.proxy_args)
+        for snap, channel in sorted(self.channels.items()):
+            args.extend(["--channel", "%s=%s" % (snap, channel)])
+        if self.branch is not None:
+            args.extend(["--branch", self.branch])
+        if self.git_repository is not None:
+            args.extend(["--git-repository", self.git_repository])
+        if self.git_path is not None:
+            args.extend(["--git-path", self.git_path])
+        try:
+            snap_store_proxy_url = self._builder._config.get(
+                "proxy", "snapstore")
+            args.extend(["--snap-store-proxy-url", snap_store_proxy_url])
+        except (NoSectionError, NoOptionError):
+            pass
+        self.runTargetSubProcess("run-ci-prepare", *args)
+
+    def iterate_PREPARE(self, retcode):
+        """Finished preparing for running CI jobs."""
+        self.remaining_jobs = list(self.jobs)
+        if retcode == RETCODE_SUCCESS:
+            pass
+        elif (retcode >= RETCODE_FAILURE_INSTALL and
+              retcode <= RETCODE_FAILURE_BUILD):
+            if not self.alreadyfailed:
+                self._builder.log("Preparation failed.")
+                self._builder.buildFail()
+            self.alreadyfailed = True
+        else:
+            if not self.alreadyfailed:
+                self._builder.builderFail()
+            self.alreadyfailed = True
+        if self.remaining_jobs and not self.alreadyfailed:
+            self._state = CIBuildState.RUN_JOB
+            self.runNextJob()
+        else:
+            self.stopProxy()
+            self.revokeProxyToken()
+            self.doReapProcesses(self._state)
+
+    def iterateReap_PREPARE(self, retcode):
+        """Finished reaping after preparing for running CI jobs.
+
+        This only happens if preparation failed or there were no jobs to run.
+        """
+        self._state = DebianBuildState.UMOUNT
+        self.doUnmounting()
+
+    def runNextJob(self):
+        """Run the next CI job."""
+        args = list(self.proxy_args)
+        job_name, job_index = self.remaining_jobs.pop(0)
+        self.current_job_id = "%s:%s" % (job_name, job_index)
+        args.extend([job_name, str(job_index)])
+        self.runTargetSubProcess("run-ci", *args)
+
+    @defer.inlineCallbacks
+    def iterate_RUN_JOB(self, retcode):
+        """Finished running a CI job.
+
+        This state is repeated for each CI job in the pipeline.
+        """
+        if retcode == RETCODE_SUCCESS:
+            pass
+        elif (retcode >= RETCODE_FAILURE_INSTALL and
+              retcode <= RETCODE_FAILURE_BUILD):
+            if not self.alreadyfailed:
+                self._builder.log("Job %s failed." % self.current_job_id)
+                self._builder.buildFail()
+            self.alreadyfailed = True
+        else:
+            if not self.alreadyfailed:
+                self._builder.builderFail()
+            self.alreadyfailed = True
+        yield self.deferGatherResults(reap=False)
+        if self.remaining_jobs and not self.alreadyfailed:
+            self.runNextJob()
+        else:
+            self.stopProxy()
+            self.revokeProxyToken()
+            self.doReapProcesses(self._state)
+
+    def iterateReap_RUN_JOB(self, retcode):
+        """Finished reaping after running a CI job.
+
+        This only happens if the job failed or there were no more jobs to run.
+        """
+        self.iterateReap_PREPARE(retcode)
+
+    def status(self):
+        """See `BuildManager.status`."""
+        status = super(CIBuildManager, self).status()
+        status["jobs"] = dict(self.job_status)
+        return status
+
+    def gatherResults(self):
+        """Gather the results of the CI job that just completed.
+
+        This is called once for each CI job in the pipeline.
+        """
+        job_status = {}
+        output_path = os.path.join("/build", "output", self.current_job_id)
+        log_path = "%s.log" % output_path
+        if self.backend.path_exists(log_path):
+            log_name = "%s.log" % self.current_job_id
+            self.addWaitingFileFromBackend(log_path, log_name)
+            job_status["log"] = self._builder.waitingfiles[log_name]
+        if self.backend.path_exists(output_path):
+            for entry in sorted(self.backend.find(
+                    output_path, include_directories=False)):
+                path = os.path.join(output_path, entry)
+                if self.backend.islink(path):
+                    continue
+                entry_base = os.path.basename(entry)
+                name = os.path.join(self.current_job_id, entry_base)
+                self.addWaitingFileFromBackend(path, name=name)
+                job_status.setdefault("output", {})[entry_base] = (
+                    self._builder.waitingfiles[name])
+
+        # Save a file map for this job in the extra status file.  This
+        # allows buildd-manager to fetch job logs/output incrementally
+        # rather than having to wait for the entire CI job to finish.
+        self.job_status[self.current_job_id] = job_status
diff --git a/lpbuildd/debian.py b/lpbuildd/debian.py
index a3ac2ef..da37eb1 100644
--- a/lpbuildd/debian.py
+++ b/lpbuildd/debian.py
@@ -128,7 +128,7 @@ class DebianBuildManager(BuildManager):
                 self._builder.addWaitingFile(
                     get_build_path(self.home, self._buildid, fn))
 
-    def deferGatherResults(self):
+    def deferGatherResults(self, reap=True):
         """Gather the results of the build in a thread."""
         # XXX cjwatson 2018-10-04: Refactor using inlineCallbacks once we're
         # on Twisted >= 18.7.0 (https://twistedmatrix.com/trac/ticket/4632).
@@ -143,11 +143,14 @@ class DebianBuildManager(BuildManager):
                 self._builder.buildFail()
             self.alreadyfailed = True
 
-        def reap(ignored):
+        def reap_processes(ignored):
             self.doReapProcesses(self._state)
 
-        return threads.deferToThread(self.gatherResults).addErrback(
-            failed_to_gather).addCallback(reap)
+        d = threads.deferToThread(self.gatherResults).addErrback(
+            failed_to_gather)
+        if reap:
+            d.addCallback(reap_processes)
+        return d
 
     @defer.inlineCallbacks
     def iterate(self, success, quiet=False):
diff --git a/lpbuildd/target/cli.py b/lpbuildd/target/cli.py
index 85e5b26..e114f35 100644
--- a/lpbuildd/target/cli.py
+++ b/lpbuildd/target/cli.py
@@ -28,6 +28,10 @@ from lpbuildd.target.lifecycle import (
     Start,
     Stop,
     )
+from lpbuildd.target.run_ci import (
+    RunCI,
+    RunCIPrepare,
+    )
 
 
 def configure_logging():
@@ -59,6 +63,8 @@ operations = {
     "override-sources-list": OverrideSourcesList,
     "mount-chroot": Start,
     "remove-build": Remove,
+    "run-ci": RunCI,
+    "run-ci-prepare": RunCIPrepare,
     "scan-for-processes": KillProcesses,
     "umount-chroot": Stop,
     "unpack-chroot": Create,
diff --git a/lpbuildd/target/run_ci.py b/lpbuildd/target/run_ci.py
new file mode 100644
index 0000000..e74cbc7
--- /dev/null
+++ b/lpbuildd/target/run_ci.py
@@ -0,0 +1,123 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+import logging
+import os
+
+from lpbuildd.target.build_snap import SnapChannelsAction
+from lpbuildd.target.operation import Operation
+from lpbuildd.target.proxy import BuilderProxyOperationMixin
+from lpbuildd.target.snapstore import SnapStoreOperationMixin
+from lpbuildd.target.vcs import VCSOperationMixin
+from lpbuildd.util import shell_escape
+
+
+RETCODE_FAILURE_INSTALL = 200
+RETCODE_FAILURE_BUILD = 201
+
+
+logger = logging.getLogger(__name__)
+
+
+class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin,
+                   SnapStoreOperationMixin, Operation):
+
+    description = "Prepare for running CI jobs."
+    buildd_path = "/build/tree"
+
+    @classmethod
+    def add_arguments(cls, parser):
+        super(RunCIPrepare, cls).add_arguments(parser)
+        parser.add_argument(
+            "--channel", action=SnapChannelsAction, metavar="SNAP=CHANNEL",
+            dest="channels", default={}, help="install SNAP from CHANNEL")
+
+    def install(self):
+        logger.info("Running install phase...")
+        deps = []
+        if self.args.proxy_url:
+            deps.extend(self.proxy_deps)
+            self.install_git_proxy()
+        if self.args.backend == "lxd":
+            for dep in "snapd", "fuse", "squashfuse":
+                if self.backend.is_package_available(dep):
+                    deps.append(dep)
+        deps.extend(self.vcs_deps)
+        self.backend.run(["apt-get", "-y", "install"] + deps)
+        if self.args.backend in ("lxd", "fake"):
+            self.snap_store_set_proxy()
+        for snap_name, channel in sorted(self.args.channels.items()):
+            if snap_name not in ("lxd", "lpcraft"):
+                self.backend.run(
+                    ["snap", "install", "--channel=%s" % channel, snap_name])
+        for snap_name, classic in (("lxd", False), ("lpcraft", True)):
+            cmd = ["snap", "install"]
+            if classic:
+                cmd.append("--classic")
+            if snap_name in self.args.channels:
+                cmd.append("--channel=%s" % self.args.channels[snap_name])
+            cmd.append(snap_name)
+            self.backend.run(cmd)
+        self.backend.run(["lxd", "init", "--auto"])
+
+    def repo(self):
+        """Collect VCS branch."""
+        logger.info("Running repo phase...")
+        env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
+        self.vcs_fetch("tree", cwd="/build", env=env)
+        self.vcs_update_status(self.buildd_path)
+
+    def run(self):
+        try:
+            self.install()
+        except Exception:
+            logger.exception("Install failed")
+            return RETCODE_FAILURE_INSTALL
+        try:
+            self.repo()
+        except Exception:
+            logger.exception("VCS setup failed")
+            return RETCODE_FAILURE_BUILD
+        return 0
+
+
+class RunCI(BuilderProxyOperationMixin, Operation):
+
+    description = "Run a CI job."
+    buildd_path = "/build/tree"
+
+    @classmethod
+    def add_arguments(cls, parser):
+        super(RunCI, cls).add_arguments(parser)
+        parser.add_argument("job_name", help="job name to run")
+        parser.add_argument(
+            "job_index", type=int, help="index within job name to run")
+
+    def run_job(self):
+        logger.info("Running job phase...")
+        env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
+        job_id = "%s:%s" % (self.args.job_name, self.args.job_index)
+        logger.info("Running %s" % job_id)
+        output_path = os.path.join("/build", "output", job_id)
+        self.backend.run(["mkdir", "-p", output_path])
+        lpcraft_args = [
+            "lpcraft", "-v", "run-one", "--output", output_path,
+            self.args.job_name, str(self.args.job_index),
+            ]
+        tee_args = ["tee", "%s.log" % output_path]
+        args = [
+            "/bin/bash", "-o", "pipefail", "-c", "%s 2>&1 | %s" % (
+                " ".join(shell_escape(arg) for arg in lpcraft_args),
+                " ".join(shell_escape(arg) for arg in tee_args)),
+            ]
+        self.run_build_command(args, env=env)
+
+    def run(self):
+        try:
+            self.run_job()
+        except Exception:
+            logger.exception("Job failed")
+            return RETCODE_FAILURE_BUILD
+        return 0
diff --git a/lpbuildd/target/tests/matchers.py b/lpbuildd/target/tests/matchers.py
index fbecb05..e8e01a1 100644
--- a/lpbuildd/target/tests/matchers.py
+++ b/lpbuildd/target/tests/matchers.py
@@ -46,8 +46,8 @@ class RanAptGet(RanCommand):
 
 class RanSnap(RanCommand):
 
-    def __init__(self, *args):
-        super(RanSnap, self).__init__(["snap"] + list(args))
+    def __init__(self, *args, **kwargs):
+        super(RanSnap, self).__init__(["snap"] + list(args), **kwargs)
 
 
 class RanBuildCommand(RanCommand):
diff --git a/lpbuildd/target/tests/test_run_ci.py b/lpbuildd/target/tests/test_run_ci.py
new file mode 100644
index 0000000..aad9e54
--- /dev/null
+++ b/lpbuildd/target/tests/test_run_ci.py
@@ -0,0 +1,373 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+import json
+import os
+import stat
+import subprocess
+from textwrap import dedent
+
+from fixtures import (
+    FakeLogger,
+    TempDir,
+    )
+import responses
+from systemfixtures import FakeFilesystem
+from testtools import TestCase
+from testtools.matchers import (
+    AnyMatch,
+    MatchesAll,
+    MatchesListwise,
+    )
+
+from lpbuildd.target.cli import parse_args
+from lpbuildd.target.run_ci import (
+    RETCODE_FAILURE_BUILD,
+    RETCODE_FAILURE_INSTALL,
+    )
+from lpbuildd.target.tests.matchers import (
+    RanAptGet,
+    RanBuildCommand,
+    RanCommand,
+    RanSnap,
+    )
+from lpbuildd.tests.fakebuilder import FakeMethod
+
+
+class FakeRevisionID(FakeMethod):
+
+    def __init__(self, revision_id):
+        super(FakeRevisionID, self).__init__()
+        self.revision_id = revision_id
+
+    def __call__(self, run_args, *args, **kwargs):
+        super(FakeRevisionID, self).__call__(run_args, *args, **kwargs)
+        if run_args[0] == "git" and "rev-parse" in run_args:
+            return "%s\n" % self.revision_id
+
+
+class TestRunCIPrepare(TestCase):
+
+    def test_install_git(self):
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--git-repository", "lp:foo",
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.install()
+        self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
+            RanAptGet("install", "git"),
+            RanSnap("install", "lxd"),
+            RanSnap("install", "--classic", "lpcraft"),
+            RanCommand(["lxd", "init", "--auto"]),
+            ]))
+
+    @responses.activate
+    def test_install_snap_store_proxy(self):
+        store_assertion = dedent("""\
+            type: store
+            store: store-id
+            url: http://snap-store-proxy.example
+
+            body
+            """)
+
+        def respond(request):
+            return 200, {"X-Assertion-Store-Id": "store-id"}, store_assertion
+
+        responses.add_callback(
+            "GET", "http://snap-store-proxy.example/v2/auth/store/assertions",
+            callback=respond)
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--git-repository", "lp:foo",
+            "--snap-store-proxy-url", "http://snap-store-proxy.example/",
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.install()
+        self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
+            RanAptGet("install", "git"),
+            RanSnap("ack", "/dev/stdin", input_text=store_assertion),
+            RanSnap("set", "core", "proxy.store=store-id"),
+            RanSnap("install", "lxd"),
+            RanSnap("install", "--classic", "lpcraft"),
+            RanCommand(["lxd", "init", "--auto"]),
+            ]))
+
+    def test_install_proxy(self):
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--git-repository", "lp:foo",
+            "--proxy-url", "http://proxy.example:3128/",
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.bin = "/builderbin"
+        self.useFixture(FakeFilesystem()).add("/builderbin")
+        os.mkdir("/builderbin")
+        with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
+            proxy_script.write("proxy script\n")
+            os.fchmod(proxy_script.fileno(), 0o755)
+        run_ci_prepare.install()
+        self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
+            RanAptGet("install", "python3", "socat", "git"),
+            RanSnap("install", "lxd"),
+            RanSnap("install", "--classic", "lpcraft"),
+            RanCommand(["lxd", "init", "--auto"]),
+            ]))
+        self.assertEqual(
+            (b"proxy script\n", stat.S_IFREG | 0o755),
+            run_ci_prepare.backend.backend_fs[
+                "/usr/local/bin/lpbuildd-git-proxy"])
+
+    def test_install_channels(self):
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--channel=core=candidate", "--channel=core20=beta",
+            "--channel=lxd=beta", "--channel=lpcraft=edge",
+            "--git-repository", "lp:foo",
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.install()
+        self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
+            RanAptGet("install", "git"),
+            RanSnap("install", "--channel=candidate", "core"),
+            RanSnap("install", "--channel=beta", "core20"),
+            RanSnap("install", "--channel=beta", "lxd"),
+            RanSnap("install", "--classic", "--channel=edge", "lpcraft"),
+            RanCommand(["lxd", "init", "--auto"]),
+            ]))
+
+    def test_repo_git(self):
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--git-repository", "lp:foo",
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.backend.build_path = self.useFixture(TempDir()).path
+        run_ci_prepare.backend.run = FakeRevisionID("0" * 40)
+        run_ci_prepare.repo()
+        self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
+            RanBuildCommand(["git", "clone", "lp:foo", "tree"], cwd="/build"),
+            RanBuildCommand(
+                ["git", "submodule", "update", "--init", "--recursive"],
+                cwd="/build/tree"),
+            RanBuildCommand(
+                ["git", "rev-parse", "HEAD^{}"],
+                cwd="/build/tree", get_output=True, universal_newlines=True),
+            ]))
+        status_path = os.path.join(run_ci_prepare.backend.build_path, "status")
+        with open(status_path) as status:
+            self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
+    def test_repo_git_with_path(self):
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--git-repository", "lp:foo", "--git-path", "next",
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.backend.build_path = self.useFixture(TempDir()).path
+        run_ci_prepare.backend.run = FakeRevisionID("0" * 40)
+        run_ci_prepare.repo()
+        self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
+            RanBuildCommand(
+                ["git", "clone", "-b", "next", "lp:foo", "tree"],
+                cwd="/build"),
+            RanBuildCommand(
+                ["git", "submodule", "update", "--init", "--recursive"],
+                cwd="/build/tree"),
+            RanBuildCommand(
+                ["git", "rev-parse", "next^{}"],
+                cwd="/build/tree", get_output=True, universal_newlines=True),
+            ]))
+        status_path = os.path.join(run_ci_prepare.backend.build_path, "status")
+        with open(status_path) as status:
+            self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
+    def test_repo_git_with_tag_path(self):
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--git-repository", "lp:foo", "--git-path", "refs/tags/1.0",
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.backend.build_path = self.useFixture(TempDir()).path
+        run_ci_prepare.backend.run = FakeRevisionID("0" * 40)
+        run_ci_prepare.repo()
+        self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
+            RanBuildCommand(
+                ["git", "clone", "-b", "1.0", "lp:foo", "tree"], cwd="/build"),
+            RanBuildCommand(
+                ["git", "submodule", "update", "--init", "--recursive"],
+                cwd="/build/tree"),
+            RanBuildCommand(
+                ["git", "rev-parse", "refs/tags/1.0^{}"],
+                cwd="/build/tree", get_output=True, universal_newlines=True),
+            ]))
+        status_path = os.path.join(run_ci_prepare.backend.build_path, "status")
+        with open(status_path) as status:
+            self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
+    def test_repo_proxy(self):
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--git-repository", "lp:foo",
+            "--proxy-url", "http://proxy.example:3128/",
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.backend.build_path = self.useFixture(TempDir()).path
+        run_ci_prepare.backend.run = FakeRevisionID("0" * 40)
+        run_ci_prepare.repo()
+        env = {
+            "http_proxy": "http://proxy.example:3128/",
+            "https_proxy": "http://proxy.example:3128/",
+            "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
+            "SNAPPY_STORE_NO_CDN": "1",
+            }
+        self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
+            RanBuildCommand(
+                ["git", "clone", "lp:foo", "tree"], cwd="/build", **env),
+            RanBuildCommand(
+                ["git", "submodule", "update", "--init", "--recursive"],
+                cwd="/build/tree", **env),
+            RanBuildCommand(
+                ["git", "rev-parse", "HEAD^{}"],
+                cwd="/build/tree", get_output=True, universal_newlines=True),
+            ]))
+        status_path = os.path.join(run_ci_prepare.backend.build_path, "status")
+        with open(status_path) as status:
+            self.assertEqual({"revision_id": "0" * 40}, json.load(status))
+
+    def test_run_succeeds(self):
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--git-repository", "lp:foo",
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.backend.build_path = self.useFixture(TempDir()).path
+        run_ci_prepare.backend.run = FakeRevisionID("0" * 40)
+        self.assertEqual(0, run_ci_prepare.run())
+        # Just check that it did something in each step, not every detail.
+        self.assertThat(run_ci_prepare.backend.run.calls, MatchesAll(
+            AnyMatch(RanSnap("install", "--classic", "lpcraft")),
+            AnyMatch(
+                RanBuildCommand(
+                    ["git", "clone", "lp:foo", "tree"], cwd="/build")),
+            ))
+
+    def test_run_install_fails(self):
+        class FailInstall(FakeMethod):
+            def __call__(self, run_args, *args, **kwargs):
+                super(FailInstall, self).__call__(run_args, *args, **kwargs)
+                if run_args[0] == "apt-get":
+                    raise subprocess.CalledProcessError(1, run_args)
+
+        self.useFixture(FakeLogger())
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--git-repository", "lp:foo",
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.backend.run = FailInstall()
+        self.assertEqual(RETCODE_FAILURE_INSTALL, run_ci_prepare.run())
+
+    def test_run_repo_fails(self):
+        class FailRepo(FakeMethod):
+            def __call__(self, run_args, *args, **kwargs):
+                super(FailRepo, self).__call__(run_args, *args, **kwargs)
+                if run_args[0] == "git":
+                    raise subprocess.CalledProcessError(1, run_args)
+
+        self.useFixture(FakeLogger())
+        args = [
+            "run-ci-prepare",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--git-repository", "lp:foo",
+            ]
+        run_ci_prepare = parse_args(args=args).operation
+        run_ci_prepare.backend.run = FailRepo()
+        self.assertEqual(RETCODE_FAILURE_BUILD, run_ci_prepare.run())
+
+
+class TestRunCI(TestCase):
+
+    def test_run_job(self):
+        args = [
+            "run-ci",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "test", "0",
+            ]
+        run_ci = parse_args(args=args).operation
+        run_ci.run_job()
+        self.assertThat(run_ci.backend.run.calls, MatchesListwise([
+            RanCommand(["mkdir", "-p", "/build/output/test:0"]),
+            RanBuildCommand([
+                "/bin/bash", "-o", "pipefail", "-c",
+                "lpcraft -v run-one --output /build/output/test:0 test 0 2>&1 "
+                "| tee /build/output/test:0.log",
+                ], cwd="/build/tree"),
+            ]))
+
+    def test_run_job_proxy(self):
+        args = [
+            "run-ci",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "--proxy-url", "http://proxy.example:3128/",
+            "test", "0",
+            ]
+        run_ci = parse_args(args=args).operation
+        run_ci.run_job()
+        env = {
+            "http_proxy": "http://proxy.example:3128/",
+            "https_proxy": "http://proxy.example:3128/",
+            "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
+            "SNAPPY_STORE_NO_CDN": "1",
+            }
+        self.assertThat(run_ci.backend.run.calls, MatchesListwise([
+            RanCommand(["mkdir", "-p", "/build/output/test:0"]),
+            RanBuildCommand([
+                "/bin/bash", "-o", "pipefail", "-c",
+                "lpcraft -v run-one --output /build/output/test:0 test 0 2>&1 "
+                "| tee /build/output/test:0.log",
+                ], cwd="/build/tree", **env),
+            ]))
+
+    def test_run_succeeds(self):
+        args = [
+            "run-ci",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "test", "0",
+            ]
+        run_ci = parse_args(args=args).operation
+        self.assertEqual(0, run_ci.run())
+        # Just check that it did something in each step, not every detail.
+        self.assertThat(
+            run_ci.backend.run.calls,
+            AnyMatch(RanCommand(["mkdir", "-p", "/build/output/test:0"])))
+
+    def test_run_install_fails(self):
+        class FailInstall(FakeMethod):
+            def __call__(self, run_args, *args, **kwargs):
+                super(FailInstall, self).__call__(run_args, *args, **kwargs)
+                if run_args[0] == "/bin/bash":
+                    raise subprocess.CalledProcessError(1, run_args)
+
+        self.useFixture(FakeLogger())
+        args = [
+            "run-ci",
+            "--backend=fake", "--series=focal", "--arch=amd64", "1",
+            "test", "0",
+            ]
+        run_ci = parse_args(args=args).operation
+        run_ci.backend.run = FailInstall()
+        self.assertEqual(RETCODE_FAILURE_BUILD, run_ci.run())
diff --git a/lpbuildd/tests/fakebuilder.py b/lpbuildd/tests/fakebuilder.py
index ef20810..4013c4b 100644
--- a/lpbuildd/tests/fakebuilder.py
+++ b/lpbuildd/tests/fakebuilder.py
@@ -110,19 +110,21 @@ class FakeBuilder:
         for fake_method in (
                 "emptyLog", "log",
                 "chrootFail", "buildFail", "builderFail", "depFail", "buildOK",
-                "buildComplete",
+                "buildComplete", "sanitizeBuildlog",
                 ):
             setattr(self, fake_method, FakeMethod())
 
     def cachePath(self, file):
         return os.path.join(self._cachepath, file)
 
-    def addWaitingFile(self, path):
+    def addWaitingFile(self, path, name=None):
+        if name is None:
+            name = os.path.basename(path)
         with open(path, "rb") as f:
             contents = f.read()
         sha1sum = hashlib.sha1(contents).hexdigest()
         shutil.copy(path, self.cachePath(sha1sum))
-        self.waitingfiles[os.path.basename(path)] = sha1sum
+        self.waitingfiles[name] = sha1sum
 
     def anyMethod(self, *args, **kwargs):
         pass
@@ -199,9 +201,10 @@ class FakeBackend(Backend):
 
     def find(self, path, max_depth=None, include_directories=True, name=None):
         def match(backend_path, mode):
-            rel_path = os.path.relpath(backend_path, path)
-            if rel_path == os.sep or os.path.dirname(rel_path) == os.pardir:
+            prefix = path + os.sep
+            if os.path.commonprefix((backend_path, prefix)) != prefix:
                 return False
+            rel_path = os.path.relpath(backend_path, path)
             if max_depth is not None:
                 if rel_path.count(os.sep) + 1 > max_depth:
                     return False
diff --git a/lpbuildd/tests/test_ci.py b/lpbuildd/tests/test_ci.py
new file mode 100644
index 0000000..1009a36
--- /dev/null
+++ b/lpbuildd/tests/test_ci.py
@@ -0,0 +1,223 @@
+# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+__metaclass__ = type
+
+import os
+import shutil
+
+from fixtures import (
+    EnvironmentVariable,
+    TempDir,
+    )
+from testtools import TestCase
+from testtools.deferredruntest import AsynchronousDeferredRunTest
+from twisted.internet import defer
+
+from lpbuildd.builder import get_build_path
+from lpbuildd.ci import (
+    CIBuildManager,
+    CIBuildState,
+    )
+from lpbuildd.tests.fakebuilder import FakeBuilder
+from lpbuildd.tests.matchers import HasWaitingFiles
+
+
+class MockBuildManager(CIBuildManager):
+    def __init__(self, *args, **kwargs):
+        super(MockBuildManager, self).__init__(*args, **kwargs)
+        self.commands = []
+        self.iterators = []
+
+    def runSubProcess(self, path, command, iterate=None, env=None):
+        self.commands.append([path] + command)
+        if iterate is None:
+            iterate = self.iterate
+        self.iterators.append(iterate)
+        return 0
+
+
+class TestCIBuildManagerIteration(TestCase):
+    """Run CIBuildManager through its iteration steps."""
+
+    run_tests_with = AsynchronousDeferredRunTest.make_factory(timeout=5)
+
+    def setUp(self):
+        super(TestCIBuildManagerIteration, self).setUp()
+        self.working_dir = self.useFixture(TempDir()).path
+        builder_dir = os.path.join(self.working_dir, "builder")
+        home_dir = os.path.join(self.working_dir, "home")
+        for dir in (builder_dir, home_dir):
+            os.mkdir(dir)
+        self.useFixture(EnvironmentVariable("HOME", home_dir))
+        self.builder = FakeBuilder(builder_dir)
+        self.buildid = "123"
+        self.buildmanager = MockBuildManager(self.builder, self.buildid)
+        self.buildmanager._cachepath = self.builder._cachepath
+
+    def getState(self):
+        """Retrieve build manager's state."""
+        return self.buildmanager._state
+
+    @defer.inlineCallbacks
+    def startBuild(self, args=None, options=None):
+        # The build manager's iterate() kicks off the consecutive states
+        # after INIT.
+        extra_args = {
+            "series": "focal",
+            "arch_tag": "amd64",
+            "name": "test",
+            }
+        if args is not None:
+            extra_args.update(args)
+        original_backend_name = self.buildmanager.backend_name
+        self.buildmanager.backend_name = "fake"
+        self.buildmanager.initiate({}, "chroot.tar.gz", extra_args)
+        self.buildmanager.backend_name = original_backend_name
+
+        # Skip states that are done in DebianBuildManager to the state
+        # directly before PREPARE.
+        self.buildmanager._state = CIBuildState.UPDATE
+
+        # PREPARE: Run the builder's payload to prepare for running CI jobs.
+        yield self.buildmanager.iterate(0)
+        self.assertEqual(CIBuildState.PREPARE, self.getState())
+        expected_command = [
+            "sharepath/bin/in-target", "in-target", "run-ci-prepare",
+            "--backend=lxd", "--series=focal", "--arch=amd64", self.buildid,
+            ]
+        if options is not None:
+            expected_command.extend(options)
+        self.assertEqual(expected_command, self.buildmanager.commands[-1])
+        self.assertEqual(
+            self.buildmanager.iterate, self.buildmanager.iterators[-1])
+        self.assertFalse(self.builder.wasCalled("chrootFail"))
+
+    @defer.inlineCallbacks
+    def expectRunJob(self, job_name, job_index, options=None):
+        yield self.buildmanager.iterate(0)
+        self.assertEqual(CIBuildState.RUN_JOB, self.getState())
+        expected_command = [
+            "sharepath/bin/in-target", "in-target", "run-ci",
+            "--backend=lxd", "--series=focal", "--arch=amd64", self.buildid,
+            ]
+        if options is not None:
+            expected_command.extend(options)
+        expected_command.extend([job_name, job_index])
+        self.assertEqual(expected_command, self.buildmanager.commands[-1])
+        self.assertEqual(
+            self.buildmanager.iterate, self.buildmanager.iterators[-1])
+        self.assertFalse(self.builder.wasCalled("chrootFail"))
+
+    @defer.inlineCallbacks
+    def test_iterate(self):
+        # The build manager iterates multiple CI jobs from start to finish.
+        args = {
+            "git_repository": "https://git.launchpad.test/~example/+git/ci",
+            "git_path": "main",
+            "jobs": [("build", "0"), ("test", "0")],
+            }
+        expected_options = [
+            "--git-repository", "https://git.launchpad.test/~example/+git/ci",
+            "--git-path", "main",
+            ]
+        yield self.startBuild(args, expected_options)
+
+        # After preparation, start running the first job.
+        yield self.expectRunJob("build", "0")
+        self.buildmanager.backend.add_file(
+            "/build/output/build:0.log", b"I am a CI build job log.")
+        self.buildmanager.backend.add_file(
+            "/build/output/build:0/ci.whl",
+            b"I am output from a CI build job.")
+
+        # Collect the output of the first job and start running the second.
+        yield self.expectRunJob("test", "0")
+        self.buildmanager.backend.add_file(
+            "/build/output/test:0.log", b"I am a CI test job log.")
+        self.buildmanager.backend.add_file(
+            "/build/output/test:0/ci.tar.gz",
+            b"I am output from a CI test job.")
+
+        # Output from the first job is visible in the status response.
+        extra_status = self.buildmanager.status()
+        self.assertEqual(
+            {
+                "build:0": {
+                    "log": self.builder.waitingfiles["build:0.log"],
+                    "output": {
+                        "ci.whl": self.builder.waitingfiles["build:0/ci.whl"],
+                        },
+                    },
+                },
+            extra_status["jobs"])
+
+        # After running the final job, reap processes.
+        yield self.buildmanager.iterate(0)
+        expected_command = [
+            "sharepath/bin/in-target", "in-target", "scan-for-processes",
+            "--backend=lxd", "--series=focal", "--arch=amd64", self.buildid,
+            ]
+        self.assertEqual(CIBuildState.RUN_JOB, self.getState())
+        self.assertEqual(expected_command, self.buildmanager.commands[-1])
+        self.assertNotEqual(
+            self.buildmanager.iterate, self.buildmanager.iterators[-1])
+        self.assertFalse(self.builder.wasCalled("buildFail"))
+        self.assertThat(self.builder, HasWaitingFiles.byEquality({
+            "build:0.log": b"I am a CI build job log.",
+            "build:0/ci.whl": b"I am output from a CI build job.",
+            "test:0.log": b"I am a CI test job log.",
+            "test:0/ci.tar.gz": b"I am output from a CI test job.",
+            }))
+
+        # Output from both jobs is visible in the status response.
+        extra_status = self.buildmanager.status()
+        self.assertEqual(
+            {
+                "build:0": {
+                    "log": self.builder.waitingfiles["build:0.log"],
+                    "output": {
+                        "ci.whl": self.builder.waitingfiles["build:0/ci.whl"],
+                        },
+                    },
+                "test:0": {
+                    "log": self.builder.waitingfiles["test:0.log"],
+                    "output": {
+                        "ci.tar.gz":
+                            self.builder.waitingfiles["test:0/ci.tar.gz"],
+                        },
+                    },
+                },
+            extra_status["jobs"])
+
+        # Control returns to the DebianBuildManager in the UMOUNT state.
+        self.buildmanager.iterateReap(self.getState(), 0)
+        expected_command = [
+            "sharepath/bin/in-target", "in-target", "umount-chroot",
+            "--backend=lxd", "--series=focal", "--arch=amd64", self.buildid,
+            ]
+        self.assertEqual(CIBuildState.UMOUNT, self.getState())
+        self.assertEqual(expected_command, self.buildmanager.commands[-1])
+        self.assertEqual(
+            self.buildmanager.iterate, self.buildmanager.iterators[-1])
+        self.assertFalse(self.builder.wasCalled("buildFail"))
+
+        # If we iterate to the end of the build, then the extra status
+        # information is still present.
+        self.buildmanager.iterate(0)
+        expected_command = [
+            'sharepath/bin/in-target', 'in-target', 'remove-build',
+            '--backend=lxd', '--series=focal', '--arch=amd64', self.buildid,
+            ]
+        self.assertEqual(CIBuildState.CLEANUP, self.getState())
+        self.assertEqual(expected_command, self.buildmanager.commands[-1])
+        self.assertEqual(
+            self.buildmanager.iterate, self.buildmanager.iterators[-1])
+
+        self.buildmanager.iterate(0)
+        self.assertTrue(self.builder.wasCalled('buildOK'))
+        self.assertTrue(self.builder.wasCalled('buildComplete'))
+        # remove-build would remove this in a non-test environment.
+        shutil.rmtree(get_build_path(
+            self.buildmanager.home, self.buildmanager._buildid))
+        self.assertIn("jobs", self.buildmanager.status())