Merge lp:~al-maisan/launchpad/bq-platform-columns-505725 into lp:launchpad/db-devel

[Muharem Hrnjadovic]

[Muharem Hrnjadovic (al-maisan)]

Hello there!

In order to be able to formulate reasonably straightforward queries that find
candidate jobs for idle builders we'd like to add NULLable 'processor' and
'virtualized' columns to BuildQueue.

In case of binary builds these will replicate the respective Build.processor
and Build.is_virtualized values. For processor-independent jobs they will
remain NULL.

Please note also that these columns will only be initialed once, when the
BuildQueue record is inserted.

Pre-implementation chat with jml. Bug #505725.

[Stuart Bishop (stub)]

Looks fine. patch-2207-24-0.sql

review: Approve (db)

[Jonathan Lange (jml)]

Fine by me.

review: Approve

[Paul Hummer (rockstar)]

Please add whitespace around the test's "layer =..." line. Other than that, it looks good.

review: Approve (code)

=== modified file 'database/sampledata/current-dev.sql'
--- database/sampledata/current-dev.sql	2009-12-14 13:49:03 +0000
+++ database/sampledata/current-dev.sql	2010-01-12 02:26:13 +0000
@@ -1715,8 +1715,8 @@
 
 ALTER TABLE buildqueue DISABLE TRIGGER ALL;
 
-INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration) VALUES (1, 1, 'Dummy sampledata entry, not processing', 1, false, 1, 1, '00:00:00');
-INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration) VALUES (2, NULL, NULL, 10, false, 2, 1, '00:01:00');
+INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized) VALUES (1, 1, 'Dummy sampledata entry, not processing', 1, false, 1, 1, '00:00:00', 1, FALSE);
+INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized) VALUES (2, NULL, NULL, 10, false, 2, 1, '00:01:00', 1, FALSE);
 
 
 ALTER TABLE buildqueue ENABLE TRIGGER ALL;

=== modified file 'database/sampledata/current.sql'
--- database/sampledata/current.sql	2009-12-14 13:49:03 +0000
+++ database/sampledata/current.sql	2010-01-12 02:26:13 +0000
@@ -1697,8 +1697,8 @@
 
 ALTER TABLE buildqueue DISABLE TRIGGER ALL;
 
-INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration) VALUES (1, 1, 'Dummy sampledata entry, not processing', 1, false, 1, 1, '00:00:00');
-INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration) VALUES (2, NULL, NULL, 10, false, 2, 1, '00:01:00');
+INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized) VALUES (1, 1, 'Dummy sampledata entry, not processing', 1, false, 1, 1, '00:00:00', 1, FALSE);
+INSERT INTO buildqueue (id, builder, logtail, lastscore, manual, job, job_type, estimated_duration, processor, virtualized) VALUES (2, NULL, NULL, 10, false, 2, 1, '00:01:00', 1, FALSE);
 
 
 ALTER TABLE buildqueue ENABLE TRIGGER ALL;

=== modified file 'database/schema/comments.sql'
--- database/schema/comments.sql	2009-12-01 13:45:58 +0000
+++ database/schema/comments.sql	2010-01-12 02:26:13 +0000
@@ -1537,6 +1537,8 @@
 COMMENT ON COLUMN BuildQueue.job IS 'Foreign key to the `Job` table row with the generic job data.';
 COMMENT ON COLUMN BuildQueue.job_type IS 'Type of job (enumeration value), enables us to find/query the correct table with the data specific to this type of job.';
 COMMENT ON COLUMN BuildQueue.estimated_duration IS 'Estimated job duration, based on previous running times of comparable jobs.';
+COMMENT ON COLUMN BuildQueue.processor IS 'The processor required by the associated build farm job.';
+COMMENT ON COLUMN BuildQueue.virtualized IS 'The virtualization setting required by the associated build farm job.';
 
 -- Mirrors
 

=== added file 'database/schema/patch-2207-24-0.sql'
--- database/schema/patch-2207-24-0.sql	1970-01-01 00:00:00 +0000
+++ database/schema/patch-2207-24-0.sql	2010-01-12 02:26:13 +0000
@@ -0,0 +1,20 @@
+-- Copyright 2009 Canonical Ltd.  This software is licensed under the
+-- GNU Affero General Public License version 3 (see the file LICENSE).
+
+SET client_min_messages=ERROR;
+
+-- Another schema patch required for the Soyuz buildd generalisation, see
+-- https://dev.launchpad.net/Soyuz/Specs/BuilddGeneralisation for details.
+-- Bug #505725.
+
+-- Changes needed to the `BuildQueue` table.
+
+-- The 'processor' and the 'virtualized' columns will enable us to formulate
+-- more straightforward queries for finding candidate jobs when builders
+-- become idle.
+ALTER TABLE ONLY buildqueue ADD COLUMN processor integer;
+ALTER TABLE ONLY buildqueue ADD COLUMN virtualized boolean;
+
+CREATE INDEX buildqueue__processor__virtualized__idx ON buildqueue USING btree (processor, virtualized) WHERE (processor IS NOT NULL);
+
+INSERT INTO LaunchpadDatabaseRevision VALUES (2207, 24, 0);

=== modified file 'lib/lp/soyuz/interfaces/buildqueue.py'
--- lib/lp/soyuz/interfaces/buildqueue.py	2010-01-07 04:07:09 +0000
+++ lib/lp/soyuz/interfaces/buildqueue.py	2010-01-12 02:26:13 +0000
@@ -13,7 +13,7 @@
     ]
 
 from zope.interface import Interface, Attribute
-from zope.schema import Choice, Datetime, Field, Timedelta
+from zope.schema import Bool, Choice, Datetime, Field, Int, Text, Timedelta
 
 from lazr.restful.fields import Reference
 
@@ -21,6 +21,8 @@
 from lp.buildmaster.interfaces.buildfarmjob import (
     IBuildFarmJob, BuildFarmJobType)
 from lp.services.job.interfaces.job import IJob
+from lp.soyuz.interfaces.builder import IBuilder
+from lp.soyuz.interfaces.processor import IProcessor
 
 
 class IBuildQueue(Interface):
@@ -38,10 +40,21 @@
     """
 
     id = Attribute("Job identifier")
-    builder = Attribute("The IBuilder instance processing this job")
-    logtail = Attribute("The current tail of the log of the build")
-    lastscore = Attribute("Last score to be computed for this job")
-    manual = Attribute("Whether or not the job was manually scored")
+    builder = Reference(
+        IBuilder, title=_("Builder"), required=True, readonly=True,
+        description=_("The IBuilder instance processing this job"))
+    logtail = Text(
+        description=_("The current tail of the log of the job"))
+    lastscore = Int(description=_("This job's score."))
+    manual = Bool(
+        description=_("Whether or not the job was manually scored."))
+    processor = Reference(
+        IProcessor, title=_("Processor"), required=False, readonly=True,
+        description=_("The processor required by this build farm job."))
+    virtualized = Bool(
+        required=False,
+        description=_(
+            "The virtualization setting required by this build farm job."))
 
     job = Reference(
         IJob, title=_("Job"), required=True, readonly=True,

=== modified file 'lib/lp/soyuz/model/build.py'
--- lib/lp/soyuz/model/build.py	2009-12-10 16:51:18 +0000
+++ lib/lp/soyuz/model/build.py	2010-01-12 02:26:13 +0000
@@ -682,7 +682,8 @@
         queue_entry = BuildQueue(
             estimated_duration=duration_estimate,
             job_type=BuildFarmJobType.PACKAGEBUILD,
-            job=job.id)
+            job=job.id, processor=self.processor,
+            virtualized=self.is_virtualized)
         store.add(queue_entry)
         return queue_entry
 

=== modified file 'lib/lp/soyuz/model/buildqueue.py'
--- lib/lp/soyuz/model/buildqueue.py	2010-01-08 08:55:10 +0000
+++ lib/lp/soyuz/model/buildqueue.py	2010-01-12 02:26:13 +0000
@@ -52,6 +52,9 @@
     lastscore = IntCol(dbName='lastscore', default=0)
     manual = BoolCol(dbName='manual', default=False)
     estimated_duration = IntervalCol()
+    processor = ForeignKey(
+        dbName='processor', foreignKey='Processor', notNull=True)
+    virtualized = BoolCol(dbName='virtualized')
 
     @property
     def required_build_behavior(self):

=== modified file 'lib/lp/soyuz/tests/test_buildqueue.py'
--- lib/lp/soyuz/tests/test_buildqueue.py	2010-01-07 20:12:52 +0000
+++ lib/lp/soyuz/tests/test_buildqueue.py	2010-01-12 02:26:13 +0000
@@ -740,3 +740,50 @@
         self.assertEqual(
             bq.specific_job_classes[BuildFarmJobType.BRANCHBUILD],
             FakeBranchBuild)
+
+
+class TestPlatformData(TestCaseWithFactory):
+    """Tests covering the processor/virtualized properties."""
+
+    layer = LaunchpadZopelessLayer
+
+    def setUp(self):
+        """Set up a native x86 build for the test archive."""
+        super(TestPlatformData, self).setUp()
+
+        self.publisher = SoyuzTestPublisher()
+        self.publisher.prepareBreezyAutotest()
+
+        # First mark all builds in the sample data as already built.
+        store = getUtility(IStoreSelector).get(MAIN_STORE, DEFAULT_FLAVOR)
+        sample_data = store.find(Build)
+        for build in sample_data:
+            build.buildstate = BuildStatus.FULLYBUILT
+        store.flush()
+
+        # We test builds that target a primary archive.
+        self.non_ppa = self.factory.makeArchive(
+            name="primary", purpose=ArchivePurpose.PRIMARY)
+        self.non_ppa.require_virtualized = False
+
+        self.builds = []
+        self.builds.extend(
+            self.publisher.getPubSource(
+                sourcename="gedit", status=PackagePublishingStatus.PUBLISHED,
+                archive=self.non_ppa).createMissingBuilds())
+
+    def test_JobPlatformSettings(self):
+        """The `BuildQueue` instance shares the processor/virtualized
+        properties with the associated `Build`."""
+        build, bq = find_job(self, 'gedit')
+
+        # Make sure the 'processor' properties are the same.
+        self.assertEqual(
+            bq.processor, build.processor,
+            "The 'processor' property deviates.")
+
+        # Make sure the 'virtualized' properties are the same.
+        self.assertEqual(
+            bq.virtualized, build.is_virtualized,
+            "The 'virtualized' property deviates.")
+