Merge lp:~abentley/launchpad/build-duration into lp:launchpad/db-devel

[Aaron Bentley]

= Summary =
Fix bug #612899: Some recipe builds have NULL date_started but non-NULL date_finished

== Proposed fix ==
Fix the broken production data before migration by using the
date_first_dispatched (which we have determined is not broken) to synthesize a
build_duration.

Add a constraint to ensure this bug doesn't re-surface.

== Pre-implementation notes ==
Discussed with bigjools

== Implementation details ==
Some of the sampledata was broken, so I fixed it.

== Tests ==
None

== Demo and Q/A ==
None

= Launchpad lint =

Checking for conflicts and issues in changed files.

Linting changed files:
  database/sampledata/current.sql
  database/sampledata/current-dev.sql
  database/schema/patch-2207-75-0.sql

./database/schema/patch-2207-75-0.sql
       7: Line exceeds 78 characters.
      10: Line exceeds 78 characters.
      12: Line exceeds 78 characters.
      15: Line exceeds 78 characters.
      18: Line exceeds 78 characters.

[Stuart Bishop (stub)]

The change looks fine. Note that updating the existing db patch means it will not have effect on staging until the next full database restore (hopefully this weekend).

review: Approve (db)

[Aaron Bentley (abentley)]

Addendum: I have had to fix several tests that were assuming the presence of bad data (which is now impossible). I deleted test_estimateDuration_with_bad_history because its only purpose was to test behaviour with bad data.

[Edwin Grubbs (edwin-grubbs)]

Hi Aaron,

This branch looks good.

-Edwin

review: Approve (code)

=== modified file 'database/sampledata/current-dev.sql'
--- database/sampledata/current-dev.sql	2010-07-30 23:01:04 +0000
+++ database/sampledata/current-dev.sql	2010-08-04 15:21:28 +0000
@@ -1,5 +1,6 @@
 -- Copyright 2010 Canonical Ltd.  This software is licensed under the
 -- GNU Affero General Public License version 3 (see the file LICENSE).
+-- Created using pg_dump (PostgreSQL) 8.4.4
 
 SET check_function_bodies = false;
 SET client_encoding = 'UTF8';
@@ -2313,7 +2314,7 @@
 INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (26, 1, true, '2007-07-08 00:00:00', '2007-07-07 23:58:41', '2007-07-08 00:00:01', NULL, 1, 2, 1, 1);
 INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (27, 1, true, '2007-07-24 00:00:00', '2007-07-23 23:58:41', '2007-07-24 00:00:01', NULL, 1, 1, 1, 1);
 INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (28, 3, true, '2007-08-10 00:00:00', '2007-08-10 00:00:00', '2007-08-10 00:00:13', NULL, 1, 1, 1, 1);
-INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (29, 1, false, '2007-08-09 21:54:18.553132', NULL, '2007-08-09 23:59:59', NULL, NULL, 1, NULL, 1);
+INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (29, 1, false, '2007-08-09 21:54:18.553132', '2007-08-09 23:49:59', '2007-08-09 23:59:59', NULL, NULL, 1, NULL, 1);
 INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (30, 3, false, '2007-08-10 00:00:01', '2007-08-10 00:00:01', '2007-08-10 00:00:14', NULL, 1, 1, 1, 1);
 
 
@@ -10543,6 +10544,13 @@
 ALTER TABLE structuralsubscription ENABLE TRIGGER ALL;
 
 
+ALTER TABLE suggestivepotemplate DISABLE TRIGGER ALL;
+
+
+
+ALTER TABLE suggestivepotemplate ENABLE TRIGGER ALL;
+
+
 ALTER TABLE teammembership DISABLE TRIGGER ALL;
 
 INSERT INTO teammembership (id, person, team, status, date_joined, date_expires, last_changed_by, last_change_comment, proposed_by, acknowledged_by, reviewed_by, date_proposed, date_last_changed, date_acknowledged, date_reviewed, proponent_comment, acknowledger_comment, reviewer_comment, date_created) VALUES (1, 1, 17, 3, '2005-03-03 10:02:53.830191', '2050-03-03 10:02:53.830191', NULL, NULL, 1, NULL, 1, '2005-03-03 10:02:53.830191', NULL, NULL, '2005-03-03 10:02:53.830191', NULL, NULL, NULL, '2008-02-15 13:59:16.98918');

=== modified file 'database/sampledata/current.sql'
--- database/sampledata/current.sql	2010-07-30 23:01:04 +0000
+++ database/sampledata/current.sql	2010-08-04 15:21:28 +0000
@@ -1,5 +1,6 @@
 -- Copyright 2010 Canonical Ltd.  This software is licensed under the
 -- GNU Affero General Public License version 3 (see the file LICENSE).
+-- Created using pg_dump (PostgreSQL) 8.4.4
 
 SET check_function_bodies = false;
 SET client_encoding = 'UTF8';
@@ -2270,7 +2271,7 @@
 INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (26, 1, true, '2007-07-08 00:00:00', '2007-07-07 23:58:41', '2007-07-08 00:00:01', NULL, 1, 2, 1, 1);
 INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (27, 1, true, '2007-07-24 00:00:00', '2007-07-23 23:58:41', '2007-07-24 00:00:01', NULL, 1, 1, 1, 1);
 INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (28, 3, true, '2007-08-10 00:00:00', '2007-08-10 00:00:00', '2007-08-10 00:00:13', NULL, 1, 1, 1, 1);
-INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (29, 1, false, '2007-08-09 21:54:18.553132', NULL, '2007-08-09 23:59:59', NULL, NULL, 1, NULL, 1);
+INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (29, 1, false, '2007-08-09 21:54:18.553132', '2007-08-09 23:49:59', '2007-08-09 23:59:59', NULL, NULL, 1, NULL, 1);
 INSERT INTO buildfarmjob (id, processor, virtualized, date_created, date_started, date_finished, date_first_dispatched, builder, status, log, job_type) VALUES (30, 3, false, '2007-08-10 00:00:01', '2007-08-10 00:00:01', '2007-08-10 00:00:14', NULL, 1, 1, 1, 1);
 
 
@@ -10413,6 +10414,13 @@
 ALTER TABLE structuralsubscription ENABLE TRIGGER ALL;
 
 
+ALTER TABLE suggestivepotemplate DISABLE TRIGGER ALL;
+
+
+
+ALTER TABLE suggestivepotemplate ENABLE TRIGGER ALL;
+
+
 ALTER TABLE teammembership DISABLE TRIGGER ALL;
 
 INSERT INTO teammembership (id, person, team, status, date_joined, date_expires, last_changed_by, last_change_comment, proposed_by, acknowledged_by, reviewed_by, date_proposed, date_last_changed, date_acknowledged, date_reviewed, proponent_comment, acknowledger_comment, reviewer_comment, date_created) VALUES (1, 1, 17, 3, '2005-03-03 10:02:53.830191', '2050-03-03 10:02:53.830191', NULL, NULL, 1, NULL, 1, '2005-03-03 10:02:53.830191', NULL, NULL, '2005-03-03 10:02:53.830191', NULL, NULL, NULL, '2008-02-15 13:59:16.98918');

=== modified file 'database/schema/patch-2207-75-0.sql'
--- database/schema/patch-2207-75-0.sql	2010-07-30 10:08:40 +0000
+++ database/schema/patch-2207-75-0.sql	2010-08-04 15:21:28 +0000
@@ -3,6 +3,9 @@
 
 SET client_min_messages=ERROR;
 
+-- Fix broken production data
+UPDATE SourcePackageRecipeBuild SET build_duration = date_built - date_first_dispatched WHERE build_duration is NULL;
+
 CREATE TEMPORARY TABLE NewBuildFarmJob AS SELECT
   nextval('buildfarmjob_id_seq') AS id, 1 AS processor, True AS virtualized, date_created, date_built - build_duration AS date_started, date_built AS date_finished, date_first_dispatched, builder, build_state, build_log, 3 AS job_type, id AS sprb_id FROM SourcePackageRecipeBuild;
 
@@ -29,4 +32,10 @@
   DROP COLUMN build_state, DROP COLUMN build_log, DROP COLUMN archive,
   DROP COLUMN pocket, DROP COLUMN upload_log, DROP COLUMN dependencies;
 
+
+ALTER TABLE BuildFarmJob
+  ADD CONSTRAINT started_if_finished CHECK (
+    date_finished IS NULL OR date_started IS NOT NULL);
+
+
 INSERT INTO LaunchpadDatabaseRevision VALUES (2207, 75, 0);

=== modified file 'lib/lp/buildmaster/tests/test_buildfarmjob.py'
--- lib/lp/buildmaster/tests/test_buildfarmjob.py	2010-07-18 17:43:23 +0000
+++ lib/lp/buildmaster/tests/test_buildfarmjob.py	2010-08-04 15:21:28 +0000
@@ -49,6 +49,7 @@
         build_farm_job = getUtility(IBuildFarmJobSource).new(
             job_type=job_type, status=status)
         removeSecurityProxy(build_farm_job).builder = builder
+        removeSecurityProxy(build_farm_job).date_started = date_finished
         removeSecurityProxy(build_farm_job).date_finished = date_finished
         return build_farm_job
 

=== modified file 'lib/lp/code/browser/tests/test_sourcepackagerecipe.py'
--- lib/lp/code/browser/tests/test_sourcepackagerecipe.py	2010-07-31 20:55:46 +0000
+++ lib/lp/code/browser/tests/test_sourcepackagerecipe.py	2010-08-04 15:21:28 +0000
@@ -515,6 +515,7 @@
         build = removeSecurityProxy(self.factory.makeSourcePackageRecipeBuild(
             recipe=recipe, distroseries=self.squirrel, archive=self.ppa))
         build.status = BuildStatus.FULLYBUILT
+        build.date_started = datetime(2010, 03, 16, tzinfo=utc)
         build.date_finished = datetime(2010, 03, 16, tzinfo=utc)
 
         self.assertTextMatchesExpressionIgnoreWhitespace("""\
@@ -596,8 +597,9 @@
             set(view.builds))
 
         def set_day(build, day):
-            removeSecurityProxy(build).date_finished = datetime(
-                2010, 03, day, tzinfo=utc)
+            naked_build = removeSecurityProxy(build)
+            naked_build.date_started = datetime(2010, 03, day, tzinfo=utc)
+            naked_build.date_finished = datetime(2010, 03, day, tzinfo=utc)
         set_day(build1, 16)
         set_day(build2, 15)
         # When there are 4+ pending builds, only the the most

=== modified file 'lib/lp/code/model/tests/test_sourcepackagerecipebuild.py'
--- lib/lp/code/model/tests/test_sourcepackagerecipebuild.py	2010-08-03 08:49:19 +0000
+++ lib/lp/code/model/tests/test_sourcepackagerecipebuild.py	2010-08-04 15:21:28 +0000
@@ -326,7 +326,9 @@
         def prepare_build():
             queue_record = self.factory.makeSourcePackageRecipeBuildJob()
             build = queue_record.specific_job.build
-            removeSecurityProxy(build).status = BuildStatus.FULLYBUILT
+            naked_build = removeSecurityProxy(build)
+            naked_build.status = BuildStatus.FULLYBUILT
+            naked_build.date_started = self.factory.getUniqueDate()
             queue_record.builder = self.factory.makeBuilder()
             slave = WaitingSlave('BuildStatus.OK')
             queue_record.builder.setSlaveForTesting(slave)

=== modified file 'lib/lp/soyuz/doc/binarypackagebuild.txt'
--- lib/lp/soyuz/doc/binarypackagebuild.txt	2010-07-23 09:04:18 +0000
+++ lib/lp/soyuz/doc/binarypackagebuild.txt	2010-08-04 15:21:28 +0000
@@ -1200,7 +1200,7 @@
     >>> properties = {
     ...     'log': 1,
     ...     'date_finished': UTC_NOW,
-    ...     'date_started': None,
+    ...     'date_started': UTC_NOW,
     ...     'builder': bob,
     ...     'status': BuildStatus.FAILEDTOUPLOAD,
     ...     'dependencies': u'whatever',
@@ -1417,10 +1417,12 @@
     >>> earlier_builds = src_pkg_earlier.createMissingBuilds()
     >>> eg_build_date = earlier_builds[0].date_created
     >>> for build in earlier_builds:
+    ...     build.date_started = eg_build_date - timedelta(1)
     ...     build.date_finished = eg_build_date - timedelta(1)
 
     >>> later_builds = src_pkg_later.createMissingBuilds()
     >>> for build in later_builds:
+    ...     build.date_started = eg_build_date
     ...     build.date_finished = eg_build_date
 
     Ensure that the i386 builds are created by the 'frog' builder,

=== modified file 'lib/lp/soyuz/stories/soyuz/xx-builds-pages.txt'
--- lib/lp/soyuz/stories/soyuz/xx-builds-pages.txt	2010-04-14 17:34:35 +0000
+++ lib/lp/soyuz/stories/soyuz/xx-builds-pages.txt	2010-08-04 15:21:28 +0000
@@ -143,8 +143,8 @@
     ------------------------------
     Successfully built
     i386 build of commercialpackage 1.0-1 in ubuntu breezy-autotest RELEASE
-    Build started at an unknown time on an unknown build machine and finished
-    on 2007-08-09 taking
+    Build started on 2007-08-09 on an unknown build machine and finished
+    on 2007-08-09 taking ten minutes
     ------------------------------
     Failed to upload
     i386 build of cdrkit 1.0 in ubuntu breezy-autotest RELEASE
@@ -200,8 +200,8 @@
     ------------------------------
     Successfully built
     i386 build of commercialpackage 1.0-1 in ubuntu breezy-autotest RELEASE
-    Build started at an unknown time on an unknown build machine and finished
-    on 2007-08-09 taking
+    Build started on 2007-08-09 on an unknown build machine and finished
+    on 2007-08-09 taking ten minutes
     ------------------------------
     Successfully built
     i386 build of pmount 0.1-1 in ubuntu hoary RELEASE

=== modified file 'lib/lp/soyuz/tests/test_binarypackagebuild.py'
--- lib/lp/soyuz/tests/test_binarypackagebuild.py	2010-07-21 07:45:50 +0000
+++ lib/lp/soyuz/tests/test_binarypackagebuild.py	2010-08-04 15:21:28 +0000
@@ -10,7 +10,6 @@
 from zope.component import getUtility
 from zope.security.proxy import removeSecurityProxy
 
-from canonical.database.constants import UTC_NOW
 from canonical.testing import LaunchpadZopelessLayer
 from lp.services.job.model.job import Job
 from lp.buildmaster.interfaces.buildbase import BuildStatus
@@ -83,13 +82,6 @@
         self.create_previous_build(335)
         self.assertEqual(335, self.build.estimateDuration().seconds)
 
-    def test_estimateDuration_with_bad_history(self):
-        # If the latest matching build has bad data, ignore it.
-        # See bug 589068.
-        previous_build = self.create_previous_build(335)
-        previous_build.date_started = None
-        self.assertEqual(60, self.build.estimateDuration().seconds)
-
     def addFakeBuildLog(self):
         lfa = self.factory.makeLibraryFileAlias('mybuildlog.txt')
         removeSecurityProxy(self.build).log = lfa
@@ -418,8 +410,7 @@
 
         self.builder = self.factory.makeBuilder()
         self.builder.setSlaveForTesting(WaitingSlave('BuildStatus.OK'))
-        self.build.buildqueue_record.builder = self.builder
-        self.build.buildqueue_record.setDateStarted(UTC_NOW)
+        self.build.buildqueue_record.markAsBuilding(self.builder)
 
     def testDependencies(self):
         """Verify that storeBuildInfo sets any dependencies."""