Merge lp:~jml/launchpad/buildd-slavescanner-bustage into lp:launchpad

[Jonathan Lange]

Bust up much of buildd-slavescanner.txt into better, more focused tests

This branch takes the tests in buildd-slavescanner.txt that were exercising the BinaryPackageBuildBehavior class and turns them into more-focused Python tests.

Some of the tests were moved to archive-dependencies.txt, since they were really testing something at a much lower level.

To do this we had to update the mocks that we use and tweak the factory a little.

There are a few incidental cleanups & comments in the actual code, but nothing that puts the massive test refactoring at risk.

[Jeroen T. Vermeulen (jtv)]

Thanks for the work on this. We discussed many superficial changes on IRC, and you've made corresponding improvements: separate creation of complex expected values from comparisons to actual values. Better not import twisted.trial.unittest under that name since it leads to confusion with the global unittest. Use a list comprehension instead of a loop calling a function. Use transaction instead of layer.txn. Bear our syntax guidelines in mind when initializing dicts.

Discussed but not changed:
 * Testing the logic without Twisted. Complete the overhaul first, then be clever.
 * Extracting more of the unit test setup into helpers to make the tests shorter. First see what parameterization would be needed as the tests grow.

review: Approve

=== modified file 'lib/lp/buildmaster/model/builder.py'
--- lib/lp/buildmaster/model/builder.py	2010-09-21 09:05:34 +0000
+++ lib/lp/buildmaster/model/builder.py	2010-09-22 10:37:47 +0000
@@ -115,6 +115,11 @@
 class BuilderSlave(object):
     """Add in a few useful methods for the XMLRPC slave."""
 
+    # WARNING: If you change the API for this, you should also change the APIs
+    # of the mocks in soyuzbuilderhelpers to match. Otherwise, you will have
+    # many false positives in your test run and will most likely break
+    # production.
+
     # XXX: This (BuilderSlave) should use composition, rather than
     # inheritance.
 

=== modified file 'lib/lp/services/twistedsupport/processmonitor.py'
--- lib/lp/services/twistedsupport/processmonitor.py	2010-08-20 20:31:18 +0000
+++ lib/lp/services/twistedsupport/processmonitor.py	2010-09-22 10:37:47 +0000
@@ -300,7 +300,7 @@
 
     def spawnProcess(self, *args, **kwargs):
         """Start a process.
-        
+
         See reactor.spawnProcess.
         """
         self._process_transport = reactor.spawnProcess(

=== modified file 'lib/lp/soyuz/doc/archive-dependencies.txt'
--- lib/lp/soyuz/doc/archive-dependencies.txt	2010-08-24 15:29:01 +0000
+++ lib/lp/soyuz/doc/archive-dependencies.txt	2010-09-22 10:37:47 +0000
@@ -250,18 +250,27 @@
     deb http://ftpmaster.internal/ubuntu hoary-updates
         main restricted universe multiverse
 
-Similarly, populated PPA dependencies are listed in the building
+Similarly, unpopulated PPA dependencies are *not* listed in the building
 'sources_list'.
 
     >>> mark = getUtility(IPersonSet).getByName('mark')
+    >>> archive_dependency = cprov.archive.addArchiveDependency(
+    ...     mark.archive, PackagePublishingPocket.RELEASE,
+    ...     getUtility(IComponentSet)['main'])
+    >>> print_building_sources_list(a_build)
+    deb http://ppa.launchpad.dev/cprov/ppa/ubuntu hoary main
+    deb http://ftpmaster.internal/ubuntu hoary
+        main restricted universe multiverse
+    deb http://ftpmaster.internal/ubuntu hoary-security
+        main restricted universe multiverse
+    deb http://ftpmaster.internal/ubuntu hoary-updates
+        main restricted universe multiverse
+
+But *populated* PPA dependencies *are* listed in the building 'sources_list'.
+
     >>> pub_binaries = test_publisher.getPubBinaries(
     ...     binaryname='dep-bin', archive=mark.archive,
     ...     status=PackagePublishingStatus.PUBLISHED)
-
-    >>> archive_dependency = cprov.archive.addArchiveDependency(
-    ...     mark.archive, PackagePublishingPocket.RELEASE,
-    ...     getUtility(IComponentSet)['main'])
-
     >>> print_building_sources_list(a_build)
     deb http://ppa.launchpad.dev/cprov/ppa/ubuntu hoary main
     deb http://ppa.launchpad.dev/mark/ppa/ubuntu hoary main

=== modified file 'lib/lp/soyuz/doc/buildd-slave.txt'
--- lib/lp/soyuz/doc/buildd-slave.txt	2010-04-30 10:00:34 +0000
+++ lib/lp/soyuz/doc/buildd-slave.txt	2010-09-22 10:37:47 +0000
@@ -13,43 +13,6 @@
   >>> from canonical.buildd.tests import BuilddSlaveTestSetup
   >>> BuilddSlaveTestSetup().setUp()
 
-Use simple xmlrpclib client to certify the BuildSlave is running
-
-  >>> import xmlrpclib
-  >>> slave = xmlrpclib.Server('http://localhost:8221/rpc/')
-  >>> slave.echo('Hello World')
-  ['Hello World']
-
-With slave protocol v1.0new, the only way to get files to the slave is to
-put them in the librarian first...
-
-  >>> from canonical.librarian.client import LibrarianClient
-  >>> from StringIO import StringIO
-  >>> from canonical.launchpad.database import LibraryFileAlias
-  >>> import transaction
-  >>> lc = LibrarianClient()
-  >>> helloworld = "Hello World"
-  >>> hw_sio = StringIO(helloworld)
-  >>> alias = lc.addFile("HelloWorld.txt", len(helloworld),
-  ...                    hw_sio, "text/plain")
-  >>> transaction.commit()
-  >>> lf = LibraryFileAlias.get(alias)
-  >>> present, info = slave.ensurepresent(
-  ...     lf.content.sha1, lf.http_url, "", "")
-  >>> present, info
-  (True, 'Download')
-
-As of slave protocol v1.0new, /filecache/SHA1SUM is *THE* way
-to retrieve files from the slave. Verify it works...
-
-  >>> from urllib2 import urlopen
-  >>> f = urlopen("http://localhost:8221/filecache/" + lf.content.sha1)
-  >>> hw_str = f.read()
-  >>> f.close()
-  >>> hw_str == helloworld
-  True
-
-
 == BuilderSet polling operations ==
 
     >>> import logging
@@ -90,6 +53,7 @@
 
 At this point the buildd-slave is not accessible anymore.
 
+  >>> import xmlrpclib
   >>> s = xmlrpclib.Server('http://localhost:8221/rpc/')
   >>> s.info()
   Traceback (most recent call last):

=== modified file 'lib/lp/soyuz/doc/buildd-slavescanner.txt'
--- lib/lp/soyuz/doc/buildd-slavescanner.txt	2010-09-16 14:36:47 +0000
+++ lib/lp/soyuz/doc/buildd-slavescanner.txt	2010-09-22 10:37:47 +0000
@@ -112,12 +112,10 @@
 
 Make sure that a_builder has no active builds:
 
-    >>> from canonical.launchpad.ftests import syncUpdate
     >>> if a_builder.currentjob is not None:
     ...     currentjob = a_builder.currentjob
     ...     currentjob.setDateStarted(None)
     ...     currentjob.builder = None
-    ...     syncUpdate(currentjob)
 
 Force the test builder to be 'ok' as the code required to do this
 automatically is not yet factored into the content class.
@@ -595,11 +593,9 @@
     >>> resurrect_build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(
     ...     current_job)
     >>> resurrect_build.status = BuildStatus.NEEDSBUILD
-    >>> syncUpdate(resurrect_build)
     >>> current_job.builder = None
     >>> current_job.setDateStarted(None)
     >>> current_job.lastscore = 0
-    >>> syncUpdate(current_job)
 
 IBuilder.findCandidate also identifies if there are builds for
 superseded source package releases in the queue and marks the
@@ -740,66 +736,6 @@
     >>> commit()
     >>> LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser)
 
-For building a candidate in the release pocket for the main component
-and the primary archive It will pass an 'archives' argument to the
-slave that contains sources.list entries for each pocket required in
-the primary archive dependency tree.
-
-We also pass arguments called 'suite' which is the current distroseries and
-pocket, (e.g. edgy-updates) and 'archive_purpose' which contains the build's
-archive.purpose (e.g. PRIMARY or PPA).  These latter two arguments are
-used in the chroot to determine whether it needs to turn on some features
-or not (like pkgstriptranslations and pkgmaintainermangler).
-
-Please note also that the 'archive_private' flag is passed to the slave
-builder.  It is True for private archives and False otherwise.
-
-    >>> a_builder.setSlaveForTesting(OkSlave())
-    >>> a_builder.is_available
-    True
-    >>> candidate = a_build.queueBuild()
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
-    ensurepresent called, url=...
-    ensurepresent called,
-        url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
-    OkSlave BUILDING
-    Archives:
-     deb http://ftpmaster.internal/ubuntu hoary main
-    Suite: hoary
-    Ogre-component: main
-    Archive Purpose: PRIMARY
-    Archive Private: False
-
-    >>> candidate.destroySelf()
-
-Currently we can theoretically dispatch a build candidate for a
-builder in 'manual' mode.
-
-Although this will not be optimal, because we can only
-do it once the manual builder has been collected (due to the
-BuildQueue.builder constraint). Also because we don't yet provide a
-API/UI method to request the dispatch in advance.
-
-    >>> a_builder.manual = True
-    >>> commit()
-    >>> a_builder.setSlaveForTesting(OkSlave())
-    >>> a_builder.is_available
-    True
-    >>> candidate = a_build.queueBuild()
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
-    ensurepresent called, url=...
-    ensurepresent called,
-        url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
-    OkSlave BUILDING
-    Archives:
-     deb http://ftpmaster.internal/ubuntu hoary main
-    Suite: hoary
-    Ogre-component: main
-    Archive Purpose: PRIMARY
-    Archive Private: False
-
-    >>> candidate.destroySelf()
-
 Partner archive builds will set up the 'archives' argument such that it
 references all the required pockets/components in the primary archive, in
 addition to a reference to the release pocket in the partner archive itself.
@@ -812,10 +748,6 @@
     >>> a_builder.is_available
     True
 
-    >>> candidate = a_build.queueBuild()
-    >>> setupBuildQueue(candidate, a_builder)
-    >>> last_stub_mail_count = len(stub.test_emails)
-
 The partner archive won't be passed to the builder unless it has at
 least one published binary availble in the target distroarchseries.
 This feature fixes bug #196782, when archive/suites got passed to
@@ -823,229 +755,12 @@
 any PPA/suite will fail during the first 20 minutes because no empty
 indexes are published.
 
-Note that only a published binary in the right context will make the
-archive relevant, anything PENDING or published in another context
-wouldn't work.
-
-    >>> warty = getUtility(IDistributionSet)['ubuntu']['warty']
-    >>> create_binary_publication_for(
-    ...    partner_archive, warty, PackagePublishingStatus.PUBLISHED)
-
-    >>> hoary = getUtility(IDistributionSet)['ubuntu']['hoary']
-    >>> create_binary_publication_for(
-    ...    partner_archive, hoary, PackagePublishingStatus.PENDING)
-
-So, at moment, partner archive is still not relevant for builds in
-hoary/i386. It's not passed to the builder.
-
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
-    ensurepresent called, url=...
-    ensurepresent called,
-        url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
-    OkSlave BUILDING
-    Archives:
-     deb http://ftpmaster.internal/ubuntu hoary main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-security main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-updates main restricted universe multiverse
-    Suite: hoary
-    Ogre-component: main
-    Archive Purpose: PARTNER
-    Archive Private: False
-
-Let's try it again.
-
-    >>> candidate.destroySelf()
-    >>> a_builder.setSlaveForTesting(OkSlave())
-    >>> a_builder.is_available
-    True
-
-    >>> candidate = a_build.queueBuild()
-    >>> setupBuildQueue(candidate, a_builder)
-    >>> last_stub_mail_count = len(stub.test_emails)
-
-    >>> removeSecurityProxy(a_build).archive = ubuntu.main_archive
-    >>> candidate.destroySelf()
-
-But this time We will create a valid publication on partner hoary/i386.
-
-    >>> from lp.soyuz.interfaces.component import IComponentSet
-    >>> commit()
-    >>> LaunchpadZopelessLayer.switchDbUser('launchpad')
-    >>> login('[e-mail]')
-    >>> pub_source = test_publisher.getPubSource(
-    ...    archive=partner_archive, distroseries=hoary,
-    ...    status=PackagePublishingStatus.PUBLISHED,
-    ...    component='partner')
-    >>> pub_binaries = test_publisher.getPubBinaries(
-    ...     archive=partner_archive, pub_source=pub_source,
-    ...     distroseries=hoary, status=PackagePublishingStatus.PUBLISHED)
-    >>> partner_build = pub_binaries[0].binarypackagerelease.build
-    >>> partner_candidate = partner_build.buildqueue_record
-    >>> commit()
-    >>> LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser)
-
-Now when we dispatch the partner build, since it has one published
-binary in hoary/i386, the partner archive gets included in the builder
-sources_list.
-
-    >>> removeSecurityProxy(
-    ...     a_builder)._dispatchBuildCandidate(partner_candidate)
-    ensurepresent called, url=...
-    ensurepresent called, url=http://localhost:58000/.../foo_666.dsc
-    OkSlave BUILDING
-    Archives:
-     deb http://ftpmaster.internal/ubuntu hoary
-         main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-security
-         main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-updates
-         main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu-partner hoary partner
-    Suite: hoary
-    Ogre-component: partner
-    Archive Purpose: PARTNER
-    Archive Private: False
-
-    >>> partner_candidate.destroySelf()
-
-Similarly, PPA builds pass the 'archives' arguments:
-
-    >>> from canonical.launchpad.interfaces import IPersonSet
-    >>> cprov_archive = getUtility(IPersonSet).getByName('cprov').archive
-    >>> removeSecurityProxy(a_build).archive = cprov_archive
-    >>> a_builder.virtualized = True
-    >>> a_builder.vm_host = 'localhost.ppa'
-    >>> commit()
-    >>> a_builder.setSlaveForTesting(OkSlave())
-    >>> a_builder.is_available
-    True
-
-    >>> candidate = a_build.queueBuild()
-    >>> setupBuildQueue(candidate, a_builder)
-    >>> last_stub_mail_count = len(stub.test_emails)
-
-Exactly as Partner, Celso's PPA won't be included if it doesn't
-contain any published binary in hoary/i386. We will create it before
-dispatching.
-
-    >>> create_binary_publication_for(
-    ...    cprov_archive, hoary, PackagePublishingStatus.PUBLISHED)
-
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
-    ensurepresent called, url=...
-    ensurepresent called,
-        url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
-    OkSlave BUILDING
-    Archives:
-     deb http://ftpmaster.internal/ubuntu hoary main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-security main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-updates main restricted universe multiverse
-     deb http://ppa.launchpad.dev/cprov/ppa/ubuntu hoary main
-    Suite: hoary
-    Ogre-component: main
-    Archive Purpose: PPA
-    Archive Private: False
-
-If the build is for a private PPA, the slave scanner will pass a
-sources.list entry that contains a password to access the archive.
-
-    >>> from canonical.testing import LaunchpadZopelessLayer
-    >>> commit()
-    >>> LaunchpadZopelessLayer.switchDbUser('launchpad')
-    >>> login('[e-mail]')
-    >>> build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(
-    ...     candidate)
-    >>> for build_file in build.source_package_release.files:
-    ...     removeSecurityProxy(build_file).libraryfile.restricted = True
-    >>> private_ppa = factory.makeArchive(
-    ...     owner=cprov_archive.owner, name='pppa', private=True,
-    ...     virtualized=False, distribution=ubuntu)
-
-It's necessary to publish some binaries into the private PPA, otherwise
-the PPA won't be included as a dependency in the sources list below.
-
-    >>> binaries = test_publisher.getPubBinaries(
-    ...     distroseries=ubuntu['hoary'], archive=private_ppa,
-    ...     status=PackagePublishingStatus.PUBLISHED)
-    >>> removeSecurityProxy(build).archive = private_ppa
-    >>> commit()
-    >>> LaunchpadZopelessLayer.switchDbUser(test_dbuser)
-    >>> login(ANONYMOUS)
-
-Dispatch the build again. Celso's archive sources.list entry now has the
-buildd:secret@ part in the URL.
-
-Also note that when ensurepresent() is called, it receives a URL that
-points to the private archive rather than the librarian for the private
-firefox file.  This is because the build slaves are not allowed to
-access the restricted librarian as it cannot provide access via
-credentials, unlike the archive itself.
-
-Finally, the archive purpose is overridden to PRIMARY instead of PPA
-for any archives that have require_virtualized as False.
-
-In this circumstance, it also uses the component override from the PRIMARY
-archive and not the one from the PPA, which on the absence of ancestry
-defaults to 'universe'.
-
-    >>> build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(candidate)
-    >>> print build.current_component.name
-    main
-
-This is so that the mangling tools will run over the built packages.
-
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
-    ensurepresent called, url=...
-    ensurepresent called,
-     url=http://private-ppa.../cprov/pppa/.../firefox_0.9.2.orig.tar.gz
-    URL authorisation with buildd/sekrit
-    OkSlave BUILDING
-    Archives:
-     deb http://buildd:sekrit@private-ppa.../cprov/pppa/ubuntu hoary main
-     deb http://ftpmaster.internal/ubuntu hoary
-         main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-security
-         main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-updates
-         main restricted universe multiverse
-    Suite: hoary
-    Ogre-component: universe
-    Archive Purpose: PRIMARY
-    Archive Private: True
-
-We will create an ancestry in the primary archive target to the 'main'
-component and this time the dispatching will follow that component.
-
-    >>> sourcename = build.source_package_release.name
-
-    >>> LaunchpadZopelessLayer.switchDbUser('launchpad')
-    >>> login('[e-mail]')
-
-    >>> ancestry = test_publisher.getPubSource(
-    ...     sourcename=sourcename, version='0.1', distroseries=hoary)
-
-    >>> print ancestry.displayname
-    mozilla-firefox 0.1 in hoary
-
-    >>> print ancestry.component.name
-    main
-
-    >>> commit()
-    >>> LaunchpadZopelessLayer.switchDbUser(config.builddmaster.dbuser)
-    >>> login(ANONYMOUS)
-
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
-    ensurepresent called, ...
-    ...
-    Ogre-component: main
-    ...
-
-    >>> candidate.destroySelf()
-
 Since this is a build in a private archive, the log was uploaded to
 the restricted librarian.
 
-    >>> candidate = a_build.queueBuild()
+    >>> removeSecurityProxy(build).archive = private_ppa
+    >>> commit()
+    >>> candidate = build.queueBuild()
     >>> setupBuildQueue(candidate, a_builder)
     >>> build.upload_log = None
     >>> candidate.builder.setSlaveForTesting(WaitingSlave('BuildStatus.OK'))
@@ -1059,7 +774,7 @@
     >>> lfa.restricted
     True
     >>> print lfa.filename
-    buildlog_ubuntu-hoary-i386.mozilla-firefox_0.9_BUILDING.txt.gz
+    buildlog_ubuntu-warty-i386.mozilla-firefox_0.9_BUILDING.txt.gz
 
 The attempt to fetch the buildlog from the common librarian will fail
 since this is a build in a private archive and the buildlog was thus
@@ -1079,7 +794,7 @@
     ...     getUtility(IRestrictedLibrarianClient).getFileByAlias(lfa.id))
     >>> url_parts = urlparse.urlsplit(lfa2.file.geturl())
     >>> print os.path.basename(url_parts[2])
-    buildlog_ubuntu-hoary-i386.mozilla-firefox_0.9_BUILDING.txt.gz
+    buildlog_ubuntu-warty-i386.mozilla-firefox_0.9_BUILDING.txt.gz
 
 A PPA can depend on another PPA. We can make Celso's PPA depend on
 Mark's PPA:
@@ -1088,218 +803,13 @@
     >>> LaunchpadZopelessLayer.switchDbUser('launchpad')
     >>> login('[e-mail]')
 
-We'll switch the build's archive back to Celso's PPA and set the PPA to
-virtualized before adding the dependency on Mark's PPA.
-
-    >>> removeSecurityProxy(build).archive = cprov_archive
-    >>> cprov_archive.require_virtualized = True
-    >>> for build_file in a_build.source_package_release.files:
-    ...     removeSecurityProxy(build_file).libraryfile.restricted = False
-    >>> mark_archive = getUtility(IPersonSet).getByName('mark').archive
-
-    >>> unused_dep = cprov_archive.addArchiveDependency(
-    ...      mark_archive, PackagePublishingPocket.RELEASE,
-    ...      getUtility(IComponentSet)['main'])
-
-    >>> commit()
-    >>> LaunchpadZopelessLayer.switchDbUser(test_dbuser)
-    >>> login(ANONYMOUS)
-
-Now we can see that a build from Celso's PPA will be able to install
-dependencies from Mark's PPA, if Mark's PPA has at least one binary
-published in hoary/i386, which is not the case.
-
-    >>> a_builder.setSlaveForTesting(OkSlave())
-    >>> a_builder.is_available
-    True
-
-    >>> candidate = a_build.queueBuild()
-    >>> setupBuildQueue(candidate, a_builder)
-    >>> last_stub_mail_count = len(stub.test_emails)
-
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
-    ensurepresent called, url=...
-    ensurepresent called,
-        url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
-    OkSlave BUILDING
-    Archives:
-     deb http://ftpmaster.internal/ubuntu hoary main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-security main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-updates main restricted universe multiverse
-     deb http://ppa.launchpad.dev/cprov/ppa/ubuntu hoary main
-    Suite: hoary
-    Ogre-component: main
-    Archive Purpose: PPA
-    Archive Private: False
-
-We will create the required publication in Mark's PPA and try again.
-
-    >>> candidate.destroySelf()
-    >>> a_builder.setSlaveForTesting(OkSlave())
-    >>> a_builder.is_available
-    True
-
-    >>> candidate = a_build.queueBuild()
-    >>> setupBuildQueue(candidate, a_builder)
-    >>> last_stub_mail_count = len(stub.test_emails)
-
-    >>> create_binary_publication_for(
-    ...    mark_archive, hoary, PackagePublishingStatus.PUBLISHED)
-
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(candidate)
-    ensurepresent called, url=...
-    ensurepresent called,
-        url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
-    OkSlave BUILDING
-    Archives:
-     deb http://ftpmaster.internal/ubuntu hoary main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-security main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-updates main restricted universe multiverse
-     deb http://ppa.launchpad.dev/cprov/ppa/ubuntu hoary main
-     deb http://ppa.launchpad.dev/mark/ppa/ubuntu hoary main
-    Suite: hoary
-    Ogre-component: main
-    Archive Purpose: PPA
-    Archive Private: False
 
 Clean up before continuing:
 
-    >>> candidate.destroySelf()
     >>> a_builder.virtualized = False
     >>> removeSecurityProxy(a_build).archive = ubuntu.main_archive
     >>> commit()
 
-Builddmaster stops before starting to build a denied build.
-Since hoary is in development, we are not able to dispatch
-builds for post-release pockets:
-
-    >>> candidate = a_build.queueBuild()
-    >>> setupBuildQueue(candidate, a_builder)
-    >>> last_stub_mail_count = len(stub.test_emails)
-
-Make a build in the updates pocket:
-
-    >>> hoary = hoary_i386.distroseries
-    >>> hoary_evo = hoary.getSourcePackage(
-    ...    'evolution').currentrelease.sourcepackagerelease
-    >>> updates_build = hoary_evo.createBuild(
-    ...     distro_arch_series=hoary_i386,
-    ...     pocket=PackagePublishingPocket.UPDATES,
-    ...     processor=hoary_i386.default_processor,
-    ...     archive=hoary_i386.main_archive)
-    >>> updates_bqItem = updates_build.queueBuild()
-
-    >>> hoary_i386.distroseries.status.name
-    'DEVELOPMENT'
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(updates_bqItem)
-    Traceback (most recent call last):
-    ...
-    AssertionError: i386 build of evolution 1.0 in ubuntu hoary UPDATES (...) can not be built for pocket UPDATES: invalid pocket due to the series status of hoary.
-
-== Pocket dependencies ==
-
-Change the distroseries status for testing. FROZEN allows building in
-all pockets:
-
-    >>> from canonical.launchpad.interfaces import SeriesStatus
-    >>> hoary_i386.distroseries.status = SeriesStatus.FROZEN
-
-Now we can start a build in other pockets, and see what archives are
-passed to the slave.
-
-A build in the updates pocket:
-
-    >>> a_builder.currentjob.destroySelf()
-
-    >>> bqItem3 = a_build.queueBuild()
-    >>> build = getUtility(IBinaryPackageBuildSet).getByQueueEntry(bqItem3)
-    >>> removeSecurityProxy(build).pocket = (
-    ...     PackagePublishingPocket.UPDATES)
-    >>> last_stub_mail_count = len(stub.test_emails)
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
-    ensurepresent called, url=...
-    ensurepresent called,
-        url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
-    OkSlave BUILDING
-    Archives:
-     deb http://ftpmaster.internal/ubuntu hoary main
-     deb http://ftpmaster.internal/ubuntu hoary-security main
-     deb http://ftpmaster.internal/ubuntu hoary-updates main
-    Suite: hoary-updates
-    Ogre-component: main
-    Archive Purpose: PRIMARY
-    Archive Private: False
-
-A build in the proposed pocket:
-
-    >>> a_builder.currentjob.destroySelf()
-
-    >>> bqItem3 = a_build.queueBuild()
-    >>> removeSecurityProxy(build).pocket = (
-    ...     PackagePublishingPocket.PROPOSED)
-    >>> last_stub_mail_count = len(stub.test_emails)
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
-    ensurepresent called, url=...
-    ensurepresent called,
-        url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
-    OkSlave BUILDING
-    Archives:
-     deb http://ftpmaster.internal/ubuntu hoary main
-     deb http://ftpmaster.internal/ubuntu hoary-proposed main
-     deb http://ftpmaster.internal/ubuntu hoary-security main
-     deb http://ftpmaster.internal/ubuntu hoary-updates main
-    Suite: hoary-proposed
-    Ogre-component: main
-    Archive Purpose: PRIMARY
-    Archive Private: False
-
-A build in the backports pocket:
-
-    >>> a_builder.currentjob.destroySelf()
-
-    >>> bqItem3 = a_build.queueBuild()
-    >>> removeSecurityProxy(build).pocket = (
-    ...     PackagePublishingPocket.BACKPORTS)
-    >>> last_stub_mail_count = len(stub.test_emails)
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
-    ensurepresent called, url=...
-    ensurepresent called,
-        url=http://localhost:58000/3/firefox_0.9.2.orig.tar.gz
-    OkSlave BUILDING
-    Archives:
-     deb http://ftpmaster.internal/ubuntu hoary main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-backports main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-security main restricted universe multiverse
-     deb http://ftpmaster.internal/ubuntu hoary-updates main restricted universe multiverse
-    Suite: hoary-backports
-    Ogre-component: main
-    Archive Purpose: PRIMARY
-    Archive Private: False
-
-A build in the security pocket:
-
-    >>> a_builder.currentjob.destroySelf()
-
-    >>> bqItem3 = a_build.queueBuild()
-    >>> removeSecurityProxy(build).status = (
-    ...     BuildStatus.NEEDSBUILD)
-    >>> removeSecurityProxy(build).pocket = (
-    ...     PackagePublishingPocket.SECURITY)
-    >>> last_stub_mail_count = len(stub.test_emails)
-
-The pocket-dependency infrastructure is ready to deal with SECURITY
-pocket, however we explicitly skip security builds when dispatching
-because Embargoed-Archives and Restricted-UI implementations are not
-yet ready.
-
-    >>> removeSecurityProxy(a_builder)._dispatchBuildCandidate(bqItem3)
-    Traceback (most recent call last):
-    ...
-    AssertionError: Soyuz is not yet capable of building SECURITY uploads.
-
-Builds for security pocket are marked as FAILEDTOBUILD inside the
-_findBuildCandidate() method, see doc/buildd-dispatching.txt
-
 
 == Builder Status Handler ==
 

=== modified file 'lib/lp/soyuz/model/binarypackagebuildbehavior.py'
--- lib/lp/soyuz/model/binarypackagebuildbehavior.py	2010-08-23 16:51:11 +0000
+++ lib/lp/soyuz/model/binarypackagebuildbehavior.py	2010-09-22 10:37:47 +0000
@@ -99,8 +99,9 @@
            distroseries state.
         """
         build = self.build
-        assert not (not self._builder.virtualized and build.is_virtualized), (
-            "Attempt to build non-virtual item on a virtual builder.")
+        if build.is_virtualized and not self._builder.virtualized:
+            raise AssertionError(
+                "Attempt to build non-virtual item on a virtual builder.")
 
         # Assert that we are not silently building SECURITY jobs.
         # See findBuildCandidates. Once we start building SECURITY

=== modified file 'lib/lp/soyuz/scripts/buildd.py'
--- lib/lp/soyuz/scripts/buildd.py	2010-08-27 11:19:54 +0000
+++ lib/lp/soyuz/scripts/buildd.py	2010-09-22 10:37:47 +0000
@@ -276,6 +276,9 @@
             self.txn.commit()
 
 
+# XXX: JonathanLange 2010-09-22 bug=645046: This is the old slave
+# scanner. Julian says it's not running on production. We should either delete
+# it or update it to use the async apis.
 class SlaveScanner(LaunchpadCronScript):
 
     def main(self):

=== modified file 'lib/lp/soyuz/tests/soyuzbuilddhelpers.py'
--- lib/lp/soyuz/tests/soyuzbuilddhelpers.py	2010-09-20 11:52:51 +0000
+++ lib/lp/soyuz/tests/soyuzbuilddhelpers.py	2010-09-22 10:37:47 +0000
@@ -17,10 +17,8 @@
     ]
 
 from StringIO import StringIO
-import subprocess
 import xmlrpclib
 
-from canonical.config import config
 from lp.buildmaster.interfaces.builder import CannotFetchFile
 from lp.buildmaster.model.builder import (
     rescueBuilderIfLost,
@@ -85,68 +83,53 @@
         updateBuilderStatus(self, logger)
 
 
+# XXX: It would be *really* nice to run some set of tests against the real
+# BuilderSlave and this one to prevent interface skew.
 class OkSlave:
     """An idle mock slave that prints information about itself.
 
     The architecture tag can be customised during initialisation."""
 
     def __init__(self, arch_tag=I386_ARCHITECTURE_NAME):
+        self.call_log = []
         self.arch_tag = arch_tag
 
     def status(self):
         return ('BuilderStatus.IDLE', '')
 
     def ensurepresent(self, sha1, url, user=None, password=None):
-        print "ensurepresent called, url=%s" % url
-        if user is not None and user != "":
-            print "URL authorisation with %s/%s" % (user, password)
+        self.call_log.append(('ensurepresent', url, user, password))
         return True, None
 
     def build(self, buildid, buildtype, chroot, filemap, args):
+        self.call_log.append(
+            ('build', buildid, buildtype, chroot, filemap.keys(), args))
         info = 'OkSlave BUILDING'
-        print info
-        if 'archives' in args:
-            print "Archives:"
-            for archive_line in sorted(args['archives']):
-                print " %s" % archive_line
-        else:
-            print "No archives set."
-        print "Suite: %s" % args['suite']
-        print "Ogre-component: %s" % args['ogrecomponent']
-        print "Archive Purpose: %s" % args['archive_purpose']
-        print "Archive Private: %s" % args['archive_private']
         return ('BuildStatus.Building', info)
 
-    def fetchlogtail(self, size):
-        return 'BOGUS'
-
     def echo(self, *args):
+        self.call_log.append(('echo',) + args)
         return args
 
     def clean(self):
-        pass
+        self.call_log.append('clean')
 
     def abort(self):
-        pass
+        self.call_log.append('abort')
 
     def info(self):
+        self.call_log.append('info')
         return ('1.0', self.arch_tag, 'debian')
 
-    def resume(self):
-        resume_argv = config.builddmaster.vm_resume_command.split()
-        resume_process = subprocess.Popen(
-            resume_argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        stdout, stderr = resume_process.communicate()
-
-        return (stdout, stderr, resume_process.returncode)
-
     def sendFileToSlave(self, sha1, url, username="", password=""):
+        self.call_log.append('sendFileToSlave')
         present, info = self.ensurepresent(sha1, url, username, password)
         if not present:
             raise CannotFetchFile(url, info)
 
     def cacheFile(self, logger, libraryfilealias):
-        self.sendFileToSlave(
+        self.call_log.append('cacheFile')
+        return self.sendFileToSlave(
             libraryfilealias.content.sha1, libraryfilealias.http_url)
 
 
@@ -158,10 +141,12 @@
         self.build_id = build_id
 
     def status(self):
+        self.call_log.append('status')
         buildlog = xmlrpclib.Binary("This is a build log")
         return ('BuilderStatus.BUILDING', self.build_id, buildlog)
 
     def getFile(self, sum):
+        self.call_log.append('getFile')
         if sum == "buildlog":
             s = StringIO("This is a build log")
             s.headers = {'content-length': 19}
@@ -183,10 +168,12 @@
         self.valid_file_hashes = ['buildlog']
 
     def status(self):
+        self.call_log.append('status')
         return ('BuilderStatus.WAITING', self.state, self.build_id, {},
                 self.dependencies)
 
     def getFile(self, hash):
+        self.call_log.append('getFile')
         if hash in self.valid_file_hashes:
             content = "This is a %s" % hash
             s = StringIO(content)
@@ -198,6 +185,7 @@
     """A mock slave that looks like it's in the process of aborting."""
 
     def status(self):
+        self.call_log.append('status')
         return ('BuilderStatus.ABORTING', '1-1')
 
 
@@ -205,6 +193,7 @@
     """A mock slave that looks like it's aborted."""
 
     def status(self):
+        self.call_log.append('status')
         return ('BuilderStatus.ABORTED', '1-1')
 
 
@@ -214,10 +203,15 @@
     When 'aborted' it raises an xmlrpclib.Fault(8002, 'Could not abort')
     """
 
+    def __init__(self):
+        self.call_log = []
+
     def status(self):
+        self.call_log.append('status')
         return ('BuilderStatus.BUILDING', '1000-10000')
 
     def abort(self):
+        self.call_log.append('abort')
         raise xmlrpclib.Fault(8002, "Could not abort")
 
 
@@ -225,4 +219,5 @@
     """A mock slave that reports that it is broken."""
 
     def status(self):
+        self.call_log.append('status')
         raise xmlrpclib.Fault(8001, "Broken slave")

=== added file 'lib/lp/soyuz/tests/test_binarypackagebuildbehavior.py'
--- lib/lp/soyuz/tests/test_binarypackagebuildbehavior.py	1970-01-01 00:00:00 +0000
+++ lib/lp/soyuz/tests/test_binarypackagebuildbehavior.py	2010-09-22 10:37:47 +0000
@@ -0,0 +1,204 @@
+# Copyright 2010 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from __future__ import with_statement
+
+"""Tests for BinaryPackageBuildBehavior."""
+
+__metaclass__ = type
+
+import transaction
+
+from twisted.internet import defer
+from twisted.trial import unittest as trialtest
+
+from zope.security.proxy import removeSecurityProxy
+
+from canonical.launchpad.scripts.logger import QuietFakeLogger
+from canonical.testing import TwistedLaunchpadZopelessLayer
+
+from lp.registry.interfaces.pocket import (
+    PackagePublishingPocket,
+    pocketsuffix,
+    )
+from lp.registry.interfaces.series import SeriesStatus
+from lp.soyuz.adapters.archivedependencies import (
+    get_sources_list_for_building,
+    )
+from lp.soyuz.enums import (
+    ArchivePurpose,
+    )
+from lp.soyuz.tests.soyuzbuilddhelpers import OkSlave
+from lp.testing import (
+    ANONYMOUS,
+    login_as,
+    logout,
+    )
+from lp.testing.factory import LaunchpadObjectFactory
+
+
+class TestBinaryBuildPackageBehavior(trialtest.TestCase):
+    """Tests for the BinaryPackageBuildBehavior.
+
+    In particular, these tests are about how the BinaryPackageBuildBehavior
+    interacts with the build slave.  We test this by using a test double that
+    implements the same interface as `BuilderSlave` but instead of actually
+    making XML-RPC calls, just records any method invocations along with
+    interesting parameters.
+    """
+
+    layer = TwistedLaunchpadZopelessLayer
+
+    def setUp(self):
+        super(TestBinaryBuildPackageBehavior, self).setUp()
+        self.factory = LaunchpadObjectFactory()
+        login_as(ANONYMOUS)
+        self.addCleanup(logout)
+        self.layer.switchDbUser('testadmin')
+
+    def assertExpectedInteraction(self, ignored, call_log, builder, build,
+                                  chroot, archive, archive_purpose, component,
+                                  extra_urls=None, filemap_names=None):
+        expected = self.makeExpectedInteraction(
+            builder, build, chroot, archive, archive_purpose, component,
+            extra_urls, filemap_names)
+        self.assertEqual(call_log, expected)
+
+    def makeExpectedInteraction(self, builder, build, chroot, archive,
+                                archive_purpose, component,
+                                extra_urls=None, filemap_names=None):
+        """Build the log of calls that we expect to be made to the slave.
+
+        :param builder: The builder we are using to build the binary package.
+        :param build: The build being done on the builder.
+        :param chroot: The `LibraryFileAlias` for the chroot in which we are
+            building.
+        :param archive: The `IArchive` into which we are building.
+        :param archive_purpose: The ArchivePurpose we are sending to the
+            builder. We specify this separately from the archive because
+            sometimes the behavior object has to give a different purpose
+            in order to trick the slave into building correctly.
+        :return: A list of the calls we expect to be made.
+        """
+        job = removeSecurityProxy(builder.current_build_behavior).buildfarmjob
+        build_id = job.generateSlaveBuildCookie()
+        ds_name = build.distro_arch_series.distroseries.name
+        suite = ds_name + pocketsuffix[build.pocket]
+        archives = get_sources_list_for_building(
+            build, build.distro_arch_series,
+            build.source_package_release.name)
+        arch_indep = build.distro_arch_series.isNominatedArchIndep
+        if filemap_names is None:
+            filemap_names = []
+        if extra_urls is None:
+            extra_urls = []
+
+        upload_logs = [
+            ['cacheFile', 'sendFileToSlave', ('ensurepresent', url, '', '')]
+            for url in [chroot.http_url] + extra_urls]
+
+        extra_args = {
+            'arch_indep': arch_indep,
+            'arch_tag': build.distro_arch_series.architecturetag,
+            'archive_private': archive.private,
+            'archive_purpose': archive_purpose.name,
+            'archives': archives,
+            'build_debug_symbols': archive.build_debug_symbols,
+            'ogrecomponent': component,
+            'suite': suite,
+            }
+        build_log = [
+            ('build', build_id, 'binarypackage', chroot.content.sha1,
+             filemap_names, extra_args)]
+        return sum(upload_logs, []) + build_log
+
+    def startBuild(self, builder, candidate):
+        builder = removeSecurityProxy(builder)
+        candidate = removeSecurityProxy(candidate)
+        return defer.maybeDeferred(
+            builder.startBuild, candidate, QuietFakeLogger())
+
+    def test_non_virtual_ppa_dispatch(self):
+        # When the BinaryPackageBuildBehavior dispatches PPA builds to
+        # non-virtual builders, it stores the chroot on the server and
+        # requests a binary package build, lying to say that the archive
+        # purpose is "PRIMARY" because this ensures that the package mangling
+        # tools will run over the built packages.
+        archive = self.factory.makeArchive(virtualized=False)
+        slave = OkSlave()
+        builder = self.factory.makeBuilder(virtualized=False)
+        builder.setSlaveForTesting(slave)
+        build = self.factory.makeBinaryPackageBuild(
+            builder=builder, archive=archive)
+        lf = self.factory.makeLibraryFileAlias()
+        transaction.commit()
+        build.distro_arch_series.addOrUpdateChroot(lf)
+        candidate = build.queueBuild()
+        d = self.startBuild(builder, candidate)
+        d.addCallback(
+            self.assertExpectedInteraction, slave.call_log,
+            builder, build, lf, archive, ArchivePurpose.PRIMARY, 'universe')
+        return d
+
+    def test_partner_dispatch_no_publishing_history(self):
+        archive = self.factory.makeArchive(
+            virtualized=False, purpose=ArchivePurpose.PARTNER)
+        slave = OkSlave()
+        builder = self.factory.makeBuilder(virtualized=False)
+        builder.setSlaveForTesting(slave)
+        build = self.factory.makeBinaryPackageBuild(
+            builder=builder, archive=archive)
+        lf = self.factory.makeLibraryFileAlias()
+        transaction.commit()
+        build.distro_arch_series.addOrUpdateChroot(lf)
+        candidate = build.queueBuild()
+        d = self.startBuild(builder, candidate)
+        d.addCallback(
+            self.assertExpectedInteraction, slave.call_log,
+            builder, build, lf, archive,
+            ArchivePurpose.PARTNER, build.current_component.name)
+        return d
+
+    def test_dont_dispatch_release_builds(self):
+        archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY)
+        builder = self.factory.makeBuilder()
+        distroseries = self.factory.makeDistroSeries(
+            status=SeriesStatus.CURRENT, distribution=archive.distribution)
+        distro_arch_series = self.factory.makeDistroArchSeries(
+            distroseries=distroseries)
+        build = self.factory.makeBinaryPackageBuild(
+            builder=builder, archive=archive,
+            distroarchseries=distro_arch_series,
+            pocket=PackagePublishingPocket.RELEASE)
+        lf = self.factory.makeLibraryFileAlias()
+        transaction.commit()
+        build.distro_arch_series.addOrUpdateChroot(lf)
+        candidate = build.queueBuild()
+        behavior = candidate.required_build_behavior
+        behavior.setBuilder(build)
+        e = self.assertRaises(
+            AssertionError, behavior.verifyBuildRequest, QuietFakeLogger())
+        expected_message = (
+            "%s (%s) can not be built for pocket %s: invalid pocket due "
+            "to the series status of %s." % (
+                build.title, build.id, build.pocket.name,
+                build.distro_series.name))
+        self.assertEqual(expected_message, str(e))
+
+    def test_dont_dispatch_security_builds(self):
+        archive = self.factory.makeArchive(purpose=ArchivePurpose.PRIMARY)
+        builder = self.factory.makeBuilder()
+        build = self.factory.makeBinaryPackageBuild(
+            builder=builder, archive=archive,
+            pocket=PackagePublishingPocket.SECURITY)
+        lf = self.factory.makeLibraryFileAlias()
+        transaction.commit()
+        build.distro_arch_series.addOrUpdateChroot(lf)
+        candidate = build.queueBuild()
+        behavior = candidate.required_build_behavior
+        behavior.setBuilder(build)
+        e = self.assertRaises(
+            AssertionError, behavior.verifyBuildRequest, QuietFakeLogger())
+        self.assertEqual(
+            'Soyuz is not yet capable of building SECURITY uploads.',
+            str(e))

=== modified file 'lib/lp/testing/factory.py'
--- lib/lp/testing/factory.py	2010-09-21 11:45:15 +0000
+++ lib/lp/testing/factory.py	2010-09-22 10:37:47 +0000
@@ -1944,6 +1944,9 @@
             processorfamily = ProcessorFamilySet().getByName('powerpc')
         if owner is None:
             owner = self.makePerson()
+        # XXX: architecturetag & processerfamily are tightly coupled. It's
+        # wrong to just make a fresh architecture tag without also making a
+        # processor family to go with it (ideally with processors!)
         if architecturetag is None:
             architecturetag = self.getUniqueString('arch')
         return distroseries.newArch(
@@ -2625,7 +2628,7 @@
 
     def makeBinaryPackageBuild(self, source_package_release=None,
             distroarchseries=None, archive=None, builder=None,
-            status=None):
+            status=None, pocket=None):
         """Create a BinaryPackageBuild.
 
         If archive is not supplied, the source_package_release is used
@@ -2656,13 +2659,15 @@
                 processorfamily=processor.family)
         if status is None:
             status = BuildStatus.NEEDSBUILD
+        if pocket is None:
+            pocket = PackagePublishingPocket.RELEASE
         binary_package_build = getUtility(IBinaryPackageBuildSet).new(
             source_package_release=source_package_release,
             processor=processor,
             distro_arch_series=distroarchseries,
             status=status,
             archive=archive,
-            pocket=PackagePublishingPocket.RELEASE,
+            pocket=pocket,
             date_created=self.getUniqueDate())
         naked_build = removeSecurityProxy(binary_package_build)
         naked_build.builder = builder

=== modified file 'lib/lp/testing/tests/test_factory.py'
--- lib/lp/testing/tests/test_factory.py	2010-09-21 11:08:26 +0000
+++ lib/lp/testing/tests/test_factory.py	2010-09-22 10:37:47 +0000
@@ -113,6 +113,11 @@
             status=BuildStatus.FULLYBUILT)
         self.assertEqual(BuildStatus.FULLYBUILT, bpb.status)
 
+    def test_makeBinaryPackageBuild_uses_pocket(self):
+        bpb = self.factory.makeBinaryPackageBuild(
+            pocket=PackagePublishingPocket.UPDATES)
+        self.assertEqual(PackagePublishingPocket.UPDATES, bpb.pocket)
+
     def test_makeBinaryPackageBuild_can_be_queued(self):
         build = self.factory.makeBinaryPackageBuild()
         # Just check that makeBinaryPackageBuild returns a build that can be