Merge ~cjwatson/launchpad:dispatch-builder-constraints into launchpad:master

[Colin Watson]

Dispatch builder constraints to builders

It seems reasonable to tell the builder which constraints the build has requested. There are some concrete GPU-related situations where this will be useful, since we'll need to set up LXD containers in particular ways.

[Jürgen Gmach (jugmac00)]

review: Approve

diff --git a/lib/lp/buildmaster/interfaces/buildfarmjobbehaviour.py b/lib/lp/buildmaster/interfaces/buildfarmjobbehaviour.py
index 9f847a7..0ef7da0 100644
--- a/lib/lp/buildmaster/interfaces/buildfarmjobbehaviour.py
+++ b/lib/lp/buildmaster/interfaces/buildfarmjobbehaviour.py
@@ -8,7 +8,7 @@ __all__ = [
     "IBuildFarmJobBehaviour",
 ]
 
-from typing import Any, Dict, Generator, List, Union
+from typing import Any, Dict, Generator, List, Tuple, Union
 
 from typing_extensions import TypedDict
 from zope.interface import Attribute, Interface
@@ -71,6 +71,8 @@ BuildArgs = TypedDict(
         "build_source_tarball": bool,
         # The URL of this build.
         "build_url": str,
+        # Builder resource tags required by this build farm job.
+        "builder_constraints": Tuple[str],
         # Source snap channels to use for this build [charm, ci, snap].
         "channels": Dict[str, str],
         # The date stamp to set in the built image [livefs].
diff --git a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
index e2bc1d8..15b860e 100644
--- a/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
+++ b/lib/lp/buildmaster/model/buildfarmjobbehaviour.py
@@ -18,6 +18,7 @@ import transaction
 from twisted.internet import defer
 from twisted.web import xmlrpc
 from zope.component import getUtility
+from zope.security.proxy import removeSecurityProxy
 
 from lp.buildmaster.enums import (
     BuildBaseImageType,
@@ -100,6 +101,9 @@ class BuildFarmJobBehaviourBase:
             "arch_tag": self.distro_arch_series.architecturetag,
             "archive_private": self.archive.private,
             "build_url": canonical_url(self.build),
+            "builder_constraints": removeSecurityProxy(
+                self.build.builder_constraints
+            ),
             "fast_cleanup": self._builder.virtualized,
             "series": self.distro_arch_series.distroseries.name,
         }
diff --git a/lib/lp/charms/tests/test_charmrecipebuildbehaviour.py b/lib/lp/charms/tests/test_charmrecipebuildbehaviour.py
index 045b416..9f21dcf 100644
--- a/lib/lp/charms/tests/test_charmrecipebuildbehaviour.py
+++ b/lib/lp/charms/tests/test_charmrecipebuildbehaviour.py
@@ -304,6 +304,7 @@ class TestAsyncCharmRecipeBuildBehaviour(
                     "archives": Equals(expected_archives),
                     "arch_tag": Equals("i386"),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "channels": Equals({}),
                     "fast_cleanup": Is(True),
                     "git_repository": Equals(ref.repository.git_https_url),
@@ -345,6 +346,7 @@ class TestAsyncCharmRecipeBuildBehaviour(
                     "archives": Equals(expected_archives),
                     "arch_tag": Equals("i386"),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "channels": Equals({}),
                     "fast_cleanup": Is(True),
                     "git_repository": Equals(ref.repository.git_https_url),
diff --git a/lib/lp/code/model/tests/test_cibuildbehaviour.py b/lib/lp/code/model/tests/test_cibuildbehaviour.py
index 20c85c2..52b3139 100644
--- a/lib/lp/code/model/tests/test_cibuildbehaviour.py
+++ b/lib/lp/code/model/tests/test_cibuildbehaviour.py
@@ -299,6 +299,7 @@ class TestAsyncCIBuildBehaviour(StatsMixin, TestCIBuildBehaviourBase):
                     "archives": Equals(expected_archives),
                     "arch_tag": Equals("i386"),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "git_path": Equals(job.build.commit_sha1),
                     "git_repository": Equals(
@@ -434,6 +435,7 @@ class TestAsyncCIBuildBehaviour(StatsMixin, TestCIBuildBehaviourBase):
                     "archives": Equals(expected_archives),
                     "arch_tag": Equals("i386"),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "git_path": Equals(job.build.commit_sha1),
                     "git_repository": Equals(
@@ -608,6 +610,7 @@ class TestAsyncCIBuildBehaviour(StatsMixin, TestCIBuildBehaviourBase):
                     "archives": Equals(expected_archives),
                     "arch_tag": Equals("i386"),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "git_path": Equals(job.build.commit_sha1),
                     "git_repository": AfterPreprocessing(
@@ -652,6 +655,18 @@ class TestAsyncCIBuildBehaviour(StatsMixin, TestCIBuildBehaviourBase):
         )
 
     @defer.inlineCallbacks
+    def test_extraBuildArgs_builder_constraints(self):
+        git_repository = self.factory.makeGitRepository(
+            builder_constraints=["gpu"]
+        )
+        job = self.makeJob(
+            stages=[[("test", 0)]], git_repository=git_repository
+        )
+        with dbuser(config.builddmaster.dbuser):
+            args = yield job.extraBuildArgs()
+        self.assertEqual(["gpu"], args["builder_constraints"])
+
+    @defer.inlineCallbacks
     def test_composeBuildRequest_proxy_url_set(self):
         job = self.makeJob()
         build_request = yield job.composeBuildRequest(None)
diff --git a/lib/lp/code/model/tests/test_recipebuilder.py b/lib/lp/code/model/tests/test_recipebuilder.py
index 582443b..1fe01d9 100644
--- a/lib/lp/code/model/tests/test_recipebuilder.py
+++ b/lib/lp/code/model/tests/test_recipebuilder.py
@@ -212,6 +212,7 @@ class TestAsyncRecipeBuilder(TestRecipeBuilderBase):
                 "author_email": "[e-mail]",
                 "author_name": "Joe User",
                 "build_url": canonical_url(job.build),
+                "builder_constraints": None,
                 "distroseries_name": job.build.distroseries.name,
                 "fast_cleanup": True,
                 "ogrecomponent": "universe",
@@ -314,6 +315,7 @@ class TestAsyncRecipeBuilder(TestRecipeBuilderBase):
                 "author_email": "[e-mail]",
                 "author_name": "Joe User",
                 "build_url": canonical_url(job.build),
+                "builder_constraints": None,
                 "distroseries_name": job.build.distroseries.name,
                 "fast_cleanup": True,
                 "ogrecomponent": "universe",
@@ -363,6 +365,7 @@ class TestAsyncRecipeBuilder(TestRecipeBuilderBase):
                 "author_email": "[e-mail]",
                 "author_name": "Joe User",
                 "build_url": canonical_url(job.build),
+                "builder_constraints": None,
                 "distroseries_name": job.build.distroseries.name,
                 "fast_cleanup": True,
                 "git": True,
diff --git a/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
index 2388df3..9897717 100644
--- a/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
+++ b/lib/lp/oci/tests/test_ocirecipebuildbehaviour.py
@@ -498,6 +498,7 @@ class TestAsyncOCIRecipeBuildBehaviour(
                     ),
                     "build_path": Equals(job.build.recipe.build_path),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "git_repository": Equals(ref.repository.git_https_url),
                     "git_path": Equals(ref.name),
@@ -573,6 +574,7 @@ class TestAsyncOCIRecipeBuildBehaviour(
                     ),
                     "build_path": Equals(job.build.recipe.build_path),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "git_repository": AfterPreprocessing(
                         urlsplit,
@@ -660,6 +662,7 @@ class TestAsyncOCIRecipeBuildBehaviour(
                     ),
                     "build_path": Equals(job.build.recipe.build_path),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "git_repository": Equals(ref.repository.git_https_url),
                     "name": Equals(job.build.recipe.name),
diff --git a/lib/lp/snappy/tests/test_snapbuildbehaviour.py b/lib/lp/snappy/tests/test_snapbuildbehaviour.py
index 5d76217..054df94 100644
--- a/lib/lp/snappy/tests/test_snapbuildbehaviour.py
+++ b/lib/lp/snappy/tests/test_snapbuildbehaviour.py
@@ -404,6 +404,7 @@ class TestAsyncSnapBuildBehaviour(StatsMixin, TestSnapBuildBehaviourBase):
                     "branch": Equals(branch.bzr_identity),
                     "build_source_tarball": Is(False),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "name": Equals("test-snap"),
                     "private": Is(False),
@@ -454,6 +455,7 @@ class TestAsyncSnapBuildBehaviour(StatsMixin, TestSnapBuildBehaviourBase):
                     "arch_tag": Equals("i386"),
                     "build_source_tarball": Is(False),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "git_repository": Equals(ref.repository.git_https_url),
                     "git_path": Equals(ref.name),
@@ -496,6 +498,7 @@ class TestAsyncSnapBuildBehaviour(StatsMixin, TestSnapBuildBehaviourBase):
                     "arch_tag": Equals("i386"),
                     "build_source_tarball": Is(False),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "git_repository": Equals(ref.repository.git_https_url),
                     "name": Equals("test-snap"),
@@ -543,6 +546,7 @@ class TestAsyncSnapBuildBehaviour(StatsMixin, TestSnapBuildBehaviourBase):
                     "arch_tag": Equals("i386"),
                     "build_source_tarball": Is(False),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "git_repository": AfterPreprocessing(
                         urlsplit,
@@ -612,6 +616,7 @@ class TestAsyncSnapBuildBehaviour(StatsMixin, TestSnapBuildBehaviourBase):
                     "arch_tag": Equals("i386"),
                     "build_source_tarball": Is(False),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "git_repository": Equals(url),
                     "git_path": Equals("master"),
@@ -654,6 +659,7 @@ class TestAsyncSnapBuildBehaviour(StatsMixin, TestSnapBuildBehaviourBase):
                     "arch_tag": Equals("i386"),
                     "build_source_tarball": Is(False),
                     "build_url": Equals(canonical_url(job.build)),
+                    "builder_constraints": Is(None),
                     "fast_cleanup": Is(True),
                     "git_repository": Equals(url),
                     "name": Equals("test-snap"),
diff --git a/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py b/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py
index 8af763b..581b5b6 100644
--- a/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py
+++ b/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py
@@ -167,6 +167,7 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory):
             "archives": archives,
             "build_debug_symbols": archive.build_debug_symbols,
             "build_url": canonical_url(build),
+            "builder_constraints": None,
             "distribution": das.distroseries.distribution.name,
             "fast_cleanup": builder.virtualized,
             "image_type": "chroot",
diff --git a/lib/lp/soyuz/tests/test_livefsbuildbehaviour.py b/lib/lp/soyuz/tests/test_livefsbuildbehaviour.py
index 25d759b..42e736b 100644
--- a/lib/lp/soyuz/tests/test_livefsbuildbehaviour.py
+++ b/lib/lp/soyuz/tests/test_livefsbuildbehaviour.py
@@ -208,6 +208,7 @@ class TestAsyncLiveFSBuildBehaviour(TestLiveFSBuildBehaviourBase):
                 "archives": expected_archives,
                 "arch_tag": "i386",
                 "build_url": canonical_url(job.build),
+                "builder_constraints": None,
                 "datestamp": "20140425-103800",
                 "fast_cleanup": True,
                 "pocket": "release",
diff --git a/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py b/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py
index ef9d543..be9c1cd 100644
--- a/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py
+++ b/lib/lp/translations/tests/test_translationtemplatesbuildbehaviour.py
@@ -113,6 +113,7 @@ class TestTranslationTemplatesBuildBehaviour(
                     "archive_private": False,
                     "branch_url": behaviour.build.branch.composePublicURL(),
                     "build_url": canonical_url(behaviour.build),
+                    "builder_constraints": None,
                     "fast_cleanup": True,
                     "series": das.distroseries.name,
                 },