Merge ~cjwatson/launchpad:ci-build-webhooks into launchpad:master

[Colin Watson]

Add webhooks for CI builds

This was mostly just cribbed from charm recipe build webhooks, with adjustments as needed for the different data model here. I expect we'll probably need to add some more fields to the payload once people start using this in practice, but this should be enough to get started.

[Ines Almeida (ines-almeida)]

Looks good to me!

Was wondering why only trigger "status-changed" actions when the build is modified, but looking into other similar webhooks and thinking about the use case, that seems consistent.

review: Approve

[Ines Almeida (ines-almeida)]

One small note, we should update the webhooks documentation page

diff --git a/lib/lp/code/configure.zcml b/lib/lp/code/configure.zcml
index eaf4020..cf2e52e 100644
--- a/lib/lp/code/configure.zcml
+++ b/lib/lp/code/configure.zcml
@@ -1,4 +1,4 @@
-<!-- Copyright 2009-2021 Canonical Ltd.  This software is licensed under the
+<!-- Copyright 2009-2023 Canonical Ltd.  This software is licensed under the
      GNU Affero General Public License version 3 (see the file LICENSE).
 -->
 
@@ -1297,6 +1297,14 @@
         permission="launchpad.Admin"
         interface="lp.code.interfaces.cibuild.ICIBuildAdmin" />
   </class>
+  <subscriber
+      for="lp.code.interfaces.cibuild.ICIBuild
+           lazr.lifecycle.interfaces.IObjectCreatedEvent"
+      handler="lp.code.subscribers.cibuild.ci_build_created" />
+  <subscriber
+      for="lp.code.interfaces.cibuild.ICIBuild
+           lazr.lifecycle.interfaces.IObjectModifiedEvent"
+      handler="lp.code.subscribers.cibuild.ci_build_modified" />
 
   <!-- CIBuildSet -->
   <lp:securedutility
diff --git a/lib/lp/code/interfaces/cibuild.py b/lib/lp/code/interfaces/cibuild.py
index 9599b75..bac6181 100644
--- a/lib/lp/code/interfaces/cibuild.py
+++ b/lib/lp/code/interfaces/cibuild.py
@@ -1,9 +1,10 @@
-# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# Copyright 2022-2023 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Interfaces for CI builds."""
 
 __all__ = [
+    "CI_WEBHOOKS_FEATURE_FLAG",
     "CannotFetchConfiguration",
     "CannotParseConfiguration",
     "CIBuildAlreadyRequested",
@@ -41,6 +42,8 @@ from lp.code.interfaces.gitrepository import IGitRepository
 from lp.services.database.constants import DEFAULT
 from lp.soyuz.interfaces.distroarchseries import IDistroArchSeries
 
+CI_WEBHOOKS_FEATURE_FLAG = "ci.webhooks.enabled"
+
 
 class MissingConfiguration(Exception):
     """The repository for this CI build does not have a .launchpad.yaml."""
diff --git a/lib/lp/code/model/cibuild.py b/lib/lp/code/model/cibuild.py
index ad5bbfb..e1b7450 100644
--- a/lib/lp/code/model/cibuild.py
+++ b/lib/lp/code/model/cibuild.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# Copyright 2022-2023 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """CI builds."""
@@ -77,6 +77,7 @@ from lp.services.macaroons.interfaces import (
 )
 from lp.services.macaroons.model import MacaroonIssuerBase
 from lp.services.propertycache import cachedproperty
+from lp.services.webapp.snapshot import notify_modified
 from lp.soyuz.model.binarypackagename import BinaryPackageName
 from lp.soyuz.model.binarypackagerelease import BinaryPackageRelease
 from lp.soyuz.model.distroarchseries import DistroArchSeries
@@ -504,6 +505,31 @@ class CIBuild(PackageBuildMixin, StormBase):
         # We have no interesting checks to perform here.
         return True
 
+    def updateStatus(
+        self,
+        status,
+        builder=None,
+        worker_status=None,
+        date_started=None,
+        date_finished=None,
+        force_invalid_transition=False,
+    ):
+        """See `IBuildFarmJob`."""
+        edited_fields = set()
+        with notify_modified(
+            self, edited_fields, snapshot_names=("status",)
+        ) as previous_obj:
+            super().updateStatus(
+                status,
+                builder=builder,
+                worker_status=worker_status,
+                date_started=date_started,
+                date_finished=date_finished,
+                force_invalid_transition=force_invalid_transition,
+            )
+            if self.status != previous_obj.status:
+                edited_fields.add("status")
+
     def notify(self, extra_info=None):
         """See `IPackageBuild`."""
         from lp.code.mail.cibuild import CIBuildMailer
diff --git a/lib/lp/code/model/gitrepository.py b/lib/lp/code/model/gitrepository.py
index 0ffae31..0384d58 100644
--- a/lib/lp/code/model/gitrepository.py
+++ b/lib/lp/code/model/gitrepository.py
@@ -417,7 +417,7 @@ class GitRepository(
 
     @property
     def valid_webhook_event_types(self):
-        return ["git:push:0.1", "merge-proposal:0.1"]
+        return ["ci:build:0.1", "git:push:0.1", "merge-proposal:0.1"]
 
     @property
     def default_webhook_event_types(self):
diff --git a/lib/lp/code/model/tests/test_cibuild.py b/lib/lp/code/model/tests/test_cibuild.py
index ec452d6..ae95fbd 100644
--- a/lib/lp/code/model/tests/test_cibuild.py
+++ b/lib/lp/code/model/tests/test_cibuild.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Canonical Ltd.  This software is licensed under the
+# Copyright 2022-2023 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Test CI builds."""
@@ -9,13 +9,14 @@ from textwrap import dedent
 from unittest.mock import Mock
 from urllib.request import urlopen
 
-from fixtures import MockPatchObject
+from fixtures import FakeLogger, MockPatchObject
 from pymacaroons import Macaroon
 from storm.locals import Store
 from testtools.matchers import (
     ContainsDict,
     Equals,
     Is,
+    MatchesDict,
     MatchesListwise,
     MatchesSetwise,
     MatchesStructure,
@@ -35,6 +36,7 @@ from lp.buildmaster.model.buildfarmjob import BuildFarmJob
 from lp.buildmaster.model.buildqueue import BuildQueue
 from lp.code.errors import GitRepositoryBlobNotFound, GitRepositoryScanFault
 from lp.code.interfaces.cibuild import (
+    CI_WEBHOOKS_FEATURE_FLAG,
     CannotFetchConfiguration,
     CannotParseConfiguration,
     CIBuildAlreadyRequested,
@@ -55,12 +57,15 @@ from lp.registry.interfaces.sourcepackage import SourcePackageType
 from lp.services.authserver.xmlrpc import AuthServerAPIView
 from lp.services.config import config
 from lp.services.database.sqlbase import flush_database_caches
+from lp.services.features.testing import FeatureFixture
 from lp.services.librarian.browser import ProxiedLibraryFileAlias
 from lp.services.log.logger import BufferLogger
 from lp.services.macaroons.interfaces import IMacaroonIssuer
 from lp.services.macaroons.testing import MacaroonTestMixin
 from lp.services.propertycache import clear_property_cache
 from lp.services.webapp.interfaces import OAuthPermission
+from lp.services.webapp.publisher import canonical_url
+from lp.services.webhooks.testing import LogsScheduledWebhooks
 from lp.soyuz.enums import BinaryPackageFormat
 from lp.testing import (
     ANONYMOUS,
@@ -73,6 +78,7 @@ from lp.testing import (
     person_logged_in,
     pop_notifications,
 )
+from lp.testing.dbuser import dbuser
 from lp.testing.layers import LaunchpadFunctionalLayer, LaunchpadZopelessLayer
 from lp.testing.matchers import HasQueryCount
 from lp.testing.pages import webservice_for_person
@@ -320,6 +326,95 @@ class TestCIBuild(TestCaseWithFactory):
         build = self.factory.makeCIBuild()
         self.assertTrue(build.verifySuccessfulUpload())
 
+    def test_updateStatus_triggers_webhooks(self):
+        # Updating the status of a CIBuild triggers webhooks on the
+        # corresponding GitRepository.
+        self.useFixture(FeatureFixture({CI_WEBHOOKS_FEATURE_FLAG: "on"}))
+        logger = self.useFixture(FakeLogger())
+        build = self.factory.makeCIBuild()
+        hook = self.factory.makeWebhook(
+            target=build.git_repository, event_types=["ci:build:0.1"]
+        )
+        build.updateStatus(BuildStatus.FULLYBUILT)
+        expected_payload = {
+            "build": Equals(canonical_url(build, force_local_path=True)),
+            "action": Equals("status-changed"),
+            "git_repository": Equals(
+                canonical_url(build.git_repository, force_local_path=True)
+            ),
+            "commit_sha1": Equals(build.commit_sha1),
+            "status": Equals("Successfully built"),
+        }
+        delivery = hook.deliveries.one()
+        self.assertThat(
+            delivery,
+            MatchesStructure(
+                event_type=Equals("ci:build:0.1"),
+                payload=MatchesDict(expected_payload),
+            ),
+        )
+        with dbuser(config.IWebhookDeliveryJobSource.dbuser):
+            self.assertEqual(
+                "<WebhookDeliveryJob for webhook %d on %r>"
+                % (hook.id, hook.target),
+                repr(delivery),
+            )
+            self.assertThat(
+                logger.output,
+                LogsScheduledWebhooks(
+                    [(hook, "ci:build:0.1", MatchesDict(expected_payload))]
+                ),
+            )
+
+    def test_updateStatus_no_change_does_not_trigger_webhooks(self):
+        # An updateStatus call that changes details of the worker status but
+        # that doesn't change the build's status attribute does not trigger
+        # webhooks.
+        self.useFixture(FeatureFixture({CI_WEBHOOKS_FEATURE_FLAG: "on"}))
+        logger = self.useFixture(FakeLogger())
+        build = self.factory.makeCIBuild()
+        hook = self.factory.makeWebhook(
+            target=build.git_repository, event_types=["ci:build:0.1"]
+        )
+        builder = self.factory.makeBuilder()
+        build.updateStatus(BuildStatus.BUILDING)
+        expected_logs = [
+            (
+                hook,
+                "ci:build:0.1",
+                ContainsDict(
+                    {
+                        "action": Equals("status-changed"),
+                        "status": Equals("Currently building"),
+                    }
+                ),
+            )
+        ]
+        self.assertEqual(1, hook.deliveries.count())
+        self.assertThat(logger.output, LogsScheduledWebhooks(expected_logs))
+        build.updateStatus(
+            BuildStatus.BUILDING,
+            builder=builder,
+            worker_status={"revision_id": build.commit_sha1},
+        )
+        self.assertEqual(1, hook.deliveries.count())
+        self.assertThat(logger.output, LogsScheduledWebhooks(expected_logs))
+        build.updateStatus(BuildStatus.UPLOADING)
+        expected_logs.append(
+            (
+                hook,
+                "ci:build:0.1",
+                ContainsDict(
+                    {
+                        "action": Equals("status-changed"),
+                        "status": Equals("Uploading build"),
+                    }
+                ),
+            )
+        )
+        self.assertEqual(2, hook.deliveries.count())
+        self.assertThat(logger.output, LogsScheduledWebhooks(expected_logs))
+
     def addFakeBuildLog(self, build):
         build.setLog(self.factory.makeLibraryFileAlias("mybuildlog.txt"))
 
diff --git a/lib/lp/code/model/tests/test_gitrepository.py b/lib/lp/code/model/tests/test_gitrepository.py
index 2aeb2ca..ecf5663 100644
--- a/lib/lp/code/model/tests/test_gitrepository.py
+++ b/lib/lp/code/model/tests/test_gitrepository.py
@@ -12,7 +12,7 @@ from textwrap import dedent
 
 import transaction
 from breezy import urlutils
-from fixtures import MockPatch
+from fixtures import FakeLogger, MockPatch
 from lazr.lifecycle.event import ObjectModifiedEvent
 from pymacaroons import Macaroon
 from storm.exceptions import LostObjectError
@@ -76,7 +76,11 @@ from lp.code.event.git import GitRefsUpdatedEvent
 from lp.code.interfaces.branchmergeproposal import (
     BRANCH_MERGE_PROPOSAL_FINAL_STATES as FINAL_STATES,
 )
-from lp.code.interfaces.cibuild import ICIBuild, ICIBuildSet
+from lp.code.interfaces.cibuild import (
+    CI_WEBHOOKS_FEATURE_FLAG,
+    ICIBuild,
+    ICIBuildSet,
+)
 from lp.code.interfaces.codeimport import ICodeImportSet
 from lp.code.interfaces.defaultgit import ICanHasDefaultGitRepository
 from lp.code.interfaces.gitjob import (
@@ -170,7 +174,9 @@ from lp.services.propertycache import clear_property_cache
 from lp.services.utils import seconds_since_epoch
 from lp.services.webapp.authorization import check_permission
 from lp.services.webapp.interfaces import OAuthPermission
+from lp.services.webapp.publisher import canonical_url
 from lp.services.webapp.snapshot import notify_modified
+from lp.services.webhooks.testing import LogsScheduledWebhooks
 from lp.snappy.interfaces.snap import SNAP_TESTING_FLAGS
 from lp.testing import (
     ANONYMOUS,
@@ -4171,6 +4177,83 @@ class TestGitRepositoryRequestCIBuilds(TestCaseWithFactory):
         )
         self.assertEqual("", logger.getLogBuffer())
 
+    def test_triggers_webhooks(self):
+        # Requesting CI builds triggers any relevant webhooks.
+        self.useFixture(FeatureFixture({CI_WEBHOOKS_FEATURE_FLAG: "on"}))
+        logger = self.useFixture(FakeLogger())
+        repository = self.factory.makeGitRepository()
+        hook = self.factory.makeWebhook(
+            target=repository, event_types=["ci:build:0.1"]
+        )
+        ubuntu = getUtility(ILaunchpadCelebrities).ubuntu
+        distroseries = self.factory.makeDistroSeries(distribution=ubuntu)
+        das = self.factory.makeBuildableDistroArchSeries(
+            distroseries=distroseries
+        )
+        configuration = dedent(
+            """\
+            pipeline: [test]
+            jobs:
+                test:
+                    series: {series}
+                    architectures: [{architecture}]
+            """.format(
+                series=distroseries.name, architecture=das.architecturetag
+            )
+        ).encode()
+        new_commit = hashlib.sha1(self.factory.getUniqueBytes()).hexdigest()
+        self.useFixture(
+            GitHostingFixture(
+                commits=[
+                    {
+                        "sha1": new_commit,
+                        "blobs": {".launchpad.yaml": configuration},
+                    },
+                ]
+            )
+        )
+        with dbuser("branchscanner"):
+            repository.createOrUpdateRefs(
+                {
+                    "refs/heads/test": {
+                        "sha1": new_commit,
+                        "type": GitObjectType.COMMIT,
+                    }
+                }
+            )
+
+        [build] = getUtility(ICIBuildSet).findByGitRepository(repository)
+        delivery = hook.deliveries.one()
+        payload_matcher = MatchesDict(
+            {
+                "build": Equals(canonical_url(build, force_local_path=True)),
+                "action": Equals("created"),
+                "git_repository": Equals(
+                    canonical_url(repository, force_local_path=True)
+                ),
+                "commit_sha1": Equals(new_commit),
+                "status": Equals("Needs building"),
+            }
+        )
+        self.assertThat(
+            delivery,
+            MatchesStructure(
+                event_type=Equals("ci:build:0.1"), payload=payload_matcher
+            ),
+        )
+        with dbuser(config.IWebhookDeliveryJobSource.dbuser):
+            self.assertEqual(
+                "<WebhookDeliveryJob for webhook %d on %r>"
+                % (hook.id, hook.target),
+                repr(delivery),
+            )
+            self.assertThat(
+                logger.output,
+                LogsScheduledWebhooks(
+                    [(hook, "ci:build:0.1", payload_matcher)]
+                ),
+            )
+
 
 class TestGitRepositoryGetBlob(TestCaseWithFactory):
     """Tests for retrieving files from a Git repository."""
diff --git a/lib/lp/code/subscribers/cibuild.py b/lib/lp/code/subscribers/cibuild.py
new file mode 100644
index 0000000..83d3363
--- /dev/null
+++ b/lib/lp/code/subscribers/cibuild.py
@@ -0,0 +1,41 @@
+# Copyright 2023 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""Event subscribers for CI builds."""
+
+from lazr.lifecycle.interfaces import IObjectCreatedEvent, IObjectModifiedEvent
+from zope.component import getUtility
+
+from lp.code.interfaces.cibuild import CI_WEBHOOKS_FEATURE_FLAG, ICIBuild
+from lp.services.features import getFeatureFlag
+from lp.services.webapp.publisher import canonical_url
+from lp.services.webhooks.interfaces import IWebhookSet
+from lp.services.webhooks.payload import compose_webhook_payload
+
+
+def _trigger_ci_build_webhook(build: ICIBuild, action: str) -> None:
+    if getFeatureFlag(CI_WEBHOOKS_FEATURE_FLAG):
+        payload = {
+            "build": canonical_url(build, force_local_path=True),
+            "action": action,
+        }
+        payload.update(
+            compose_webhook_payload(
+                ICIBuild, build, ["git_repository", "commit_sha1", "status"]
+            )
+        )
+        getUtility(IWebhookSet).trigger(
+            build.git_repository, "ci:build:0.1", payload
+        )
+
+
+def ci_build_created(build: ICIBuild, event: IObjectCreatedEvent) -> None:
+    """Trigger events when a new CI build is created."""
+    _trigger_ci_build_webhook(build, "created")
+
+
+def ci_build_modified(build: ICIBuild, event: IObjectModifiedEvent) -> None:
+    """Trigger events when a CI build is modified."""
+    if event.edited_fields is not None:
+        if "status" in event.edited_fields:
+            _trigger_ci_build_webhook(build, "status-changed")
diff --git a/lib/lp/services/webhooks/interfaces.py b/lib/lp/services/webhooks/interfaces.py
index b31f29d..4f0a1e8 100644
--- a/lib/lp/services/webhooks/interfaces.py
+++ b/lib/lp/services/webhooks/interfaces.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2021 Canonical Ltd.  This software is licensed under the
+# Copyright 2015-2023 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Webhook interfaces."""
@@ -50,6 +50,7 @@ from lp.services.webservice.apihelpers import (
 WEBHOOK_EVENT_TYPES = {
     "bzr:push:0.1": "Bazaar push",
     "charm-recipe:build:0.1": "Charm recipe build",
+    "ci:build:0.1": "CI build",
     "git:push:0.1": "Git push",
     "livefs:build:0.1": "Live filesystem build",
     "merge-proposal:0.1": "Merge proposal",
diff --git a/lib/lp/services/webhooks/tests/test_browser.py b/lib/lp/services/webhooks/tests/test_browser.py
index 65ba0c3..b3f28fe 100644
--- a/lib/lp/services/webhooks/tests/test_browser.py
+++ b/lib/lp/services/webhooks/tests/test_browser.py
@@ -1,4 +1,4 @@
-# Copyright 2015-2021 Canonical Ltd.  This software is licensed under the
+# Copyright 2015-2023 Canonical Ltd.  This software is licensed under the
 # GNU Affero General Public License version 3 (see the file LICENSE).
 
 """Unit tests for Webhook views."""
@@ -63,6 +63,7 @@ class GitRepositoryTestHelpers:
 
     event_type = "git:push:0.1"
     expected_event_types = [
+        ("ci:build:0.1", "CI build"),
         ("git:push:0.1", "Git push"),
         ("merge-proposal:0.1", "Merge proposal"),
     ]
diff --git a/tox.ini b/tox.ini
index 9054ea2..dbf21ce 100644
--- a/tox.ini
+++ b/tox.ini
@@ -27,7 +27,7 @@ commands_pre =
     {toxinidir}/scripts/update-version-info.sh
 commands =
     mypy --follow-imports=silent \
-    {posargs:lib/lp/answers lib/lp/app lib/lp/archivepublisher lib/lp/archiveuploader lib/lp/buildmaster lib/lp/charms/model/charmrecipebuildbehaviour.py lib/lp/code/model/cibuildbehaviour.py lib/lp/code/model/recipebuilder.py lib/lp/oci/model/ocirecipebuildbehaviour.py lib/lp/snappy/model/snapbuildbehaviour.py lib/lp/soyuz/model/binarypackagebuildbehaviour.py lib/lp/soyuz/model/livefsbuildbehaviour.py lib/lp/testing lib/lp/translations/model/translationtemplatesbuildbehaviour.py}
+    {posargs:lib/lp/answers lib/lp/app lib/lp/archivepublisher lib/lp/archiveuploader lib/lp/buildmaster lib/lp/charms/model/charmrecipebuildbehaviour.py lib/lp/code/model/cibuildbehaviour.py lib/lp/code/model/recipebuilder.py lib/lp/code/subscribers lib/lp/oci/model/ocirecipebuildbehaviour.py lib/lp/snappy/model/snapbuildbehaviour.py lib/lp/soyuz/model/binarypackagebuildbehaviour.py lib/lp/soyuz/model/livefsbuildbehaviour.py lib/lp/testing lib/lp/translations/model/translationtemplatesbuildbehaviour.py}
 
 [testenv:docs]
 basepython = python3