Merge lp:~cjwatson/launchpad/snap-pending-import into lp:launchpad

[Colin Watson]

Don't dispatch SnapBuilds whose code object is empty, perhaps because an associated code import hasn't finished.

This should make it easier to create a code import, a snap, and some builds in quick succession (via automation) without needing to worry about whether the builds will be dispatched before the import has completed.

[William Grant (wgrant)]

This seems risky. The set of snap builds that have to be considered in each dispatch will grow without bound as imports fail or refs to non-commits attempt to be built. I suppose we could just have graphs of that and check them occasionally.

[Colin Watson (cjwatson)]

I'm withdrawing this since I have no current plans to work on the problem William raised, and the code import workflow has been mostly superseded for automation purposes these days by building directly from external repositories instead.

Unmerged revisions

18280. By Colin Watson on 2016-11-21

Don't dispatch SnapBuilds whose code object is empty, perhaps because an associated code import hasn't finished.

=== modified file 'lib/lp/snappy/model/snapbuild.py'
--- lib/lp/snappy/model/snapbuild.py	2016-08-16 16:24:51 +0000
+++ lib/lp/snappy/model/snapbuild.py	2016-11-21 14:12:14 +0000
@@ -48,6 +48,7 @@
     IMasterStore,
     IStore,
     )
+from lp.services.database.sqlbase import sqlvalues
 from lp.services.job.interfaces.job import JobStatus
 from lp.services.job.model.job import Job
 from lp.services.librarian.browser import ProxiedLibraryFileAlias
@@ -469,3 +470,27 @@
             SnapBuild, SnapBuild.build_farm_job_id.is_in(
                 bfj.id for bfj in build_farm_jobs))
         return DecoratedResultSet(rows, pre_iter_hook=self.preloadBuildsData)
+
+    def addCandidateSelectionCriteria(self, processor, virtualized):
+        """See `ISpecificBuildFarmJobSource`."""
+        return """
+            SELECT 1 FROM Snap, SnapBuild
+            WHERE
+                SnapBuild.build_farm_job = BuildQueue.build_farm_job AND
+                SnapBuild.snap = Snap.id AND
+                ((Snap.branch IS NOT NULL AND
+                  EXISTS (
+                      SELECT 1 FROM Branch
+                      WHERE
+                          Branch.id = Snap.branch AND
+                          Branch.revision_count != 0))
+                 OR
+                 (Snap.git_repository IS NOT NULL AND
+                  EXISTS (
+                      SELECT 1 FROM GitRef
+                      WHERE
+                          GitRef.repository = Snap.git_repository AND
+                          GitRef.path = Snap.git_path AND
+                          GitRef.commit_message IS NOT NULL))) AND
+                SnapBuild.status = %s
+            """ % sqlvalues(BuildStatus.NEEDSBUILD)
 
=== modified file 'lib/lp/snappy/tests/test_snapbuild.py'
--- lib/lp/snappy/tests/test_snapbuild.py	2016-08-16 16:24:51 +0000
+++ lib/lp/snappy/tests/test_snapbuild.py	2016-11-21 14:12:14 +0000
@@ -15,6 +15,10 @@
     )
 
 import pytz
+from testscenarios import (
+    load_tests_apply_scenarios,
+    WithScenarios,
+    )
 from testtools.matchers import (
     Equals,
     MatchesDict,
@@ -29,6 +33,8 @@
 from lp.buildmaster.interfaces.buildqueue import IBuildQueue
 from lp.buildmaster.interfaces.packagebuild import IPackageBuild
 from lp.buildmaster.interfaces.processor import IProcessorSet
+from lp.buildmaster.tests.mock_slaves import make_publisher
+from lp.code.interfaces.revision import IRevisionSet
 from lp.registry.enums import PersonVisibility
 from lp.services.config import config
 from lp.services.features.testing import FeatureFixture
@@ -466,6 +472,69 @@
             [], getUtility(ISnapBuildSet).getByBuildFarmJobs([]))
 
 
+class TestSnapBuildFindBuildCandidate(WithScenarios, TestCaseWithFactory):
+
+    layer = LaunchpadZopelessLayer
+
+    scenarios = [
+        ("Branch", {"context_type": "branch"}),
+        ("GitRef", {"context_type": "gitref"}),
+        ]
+
+    def setUp(self):
+        super(TestSnapBuildFindBuildCandidate, self).setUp()
+        self.publisher = make_publisher()
+        self.publisher.prepareBreezyAutotest()
+        self.builder = self.factory.makeBuilder(
+            processors=[self.publisher.breezy_autotest_i386.processor],
+            virtualized=True)
+
+    def makeSnap(self, with_revisions=True):
+        if self.context_type == "branch":
+            branch = self.factory.makeBranch()
+            if with_revisions:
+                self.factory.makeRevisionsForBranch(branch)
+            return self.factory.makeSnap(branch=branch)
+        elif self.context_type == "gitref":
+            [ref] = self.factory.makeGitRefs()
+            if with_revisions:
+                person = self.factory.makePerson()
+                email = removeSecurityProxy(person).preferredemail.email
+                author = getUtility(IRevisionSet).acquireRevisionAuthors(
+                    [email])[email]
+                now = datetime.now(pytz.UTC)
+                naked_ref = removeSecurityProxy(ref)
+                naked_ref.author = naked_ref.committer = author
+                naked_ref.author_date = naked_ref.committer_date = now
+                naked_ref.commit_message = u"something"
+            return self.factory.makeSnap(git_ref=ref)
+        else:
+            raise AssertionError("unknown context_type %s" % self.context_type)
+
+    def test_findBuildCandidate_ready(self):
+        # Builder._findBuildCandidate finds a SnapBuild with a usable code
+        # object.
+        snap = self.makeSnap()
+        build = self.factory.makeSnapBuild(
+            snap=snap, distroarchseries=self.publisher.breezy_autotest_i386)
+        build.queueBuild()
+        next_job = removeSecurityProxy(self.builder)._findBuildCandidate()
+        self.assertEqual(
+            build,
+            getUtility(ISnapBuildSet).getByBuildFarmJob(
+                removeSecurityProxy(next_job)._build_farm_job))
+
+    def test_findBuildCandidate_no_revisions(self):
+        # Builder._findBuildCandidate skips SnapBuilds without a usable code
+        # object, such as an imported branch whose import hasn't run yet.
+        snap = self.makeSnap(with_revisions=False)
+        build = self.factory.makeSnapBuild(
+            snap=snap, distroarchseries=self.publisher.breezy_autotest_i386)
+        build.queueBuild()
+        self.assertIsNone(
+            removeSecurityProxy(self.builder)._findBuildCandidate())
+
+
 class TestSnapBuildWebservice(TestCaseWithFactory):
 
     layer = LaunchpadFunctionalLayer
@@ -621,3 +690,6 @@
         browser = self.getNonRedirectingBrowser(user=self.person)
         for file_url in file_urls:
             self.assertCanOpenRedirectedUrl(browser, file_url)
+
+
+load_tests = load_tests_apply_scenarios