Merge lp:~notnownikki/offspring/exclude-failing-builders into lp:offspring

[Nicola Heald]

Adds contiguous_errors field to Lexbuilder, increments this when a build fails. Resets to 0 on a successful build. Will not use builders that have failed more that X times in a row (configurable)

[lp:~notnownikki/offspring/exclude-failing-builders]

165. By Nicola Heald on 2013-05-24

Removed TODO comments

[Kevin McDermott (bigkevmcd)]

Looks good, nothing major, couple of minors (and the first is a personal preference).

=== modified file 'config/offspring.cfg'
+builder_upper_error_limit: 5

Given that we don't have a lower error limit, "upper" seems redundant here?

=== modified file 'lib/offspring/master/master.py'
+ return self.db_store.find(
+ Lexbuilder, Lexbuilder.is_retired == False,
+ Lexbuilder.contiguous_errors < int(
+ self.config.get("master", "builder_upper_error_limit")))

self.config is a ConfigParser, so you could replace this with

+ Lexbuilder.contiguous_errors < self.config.getint(
+ "master", "builder_upper_error_limit"))

What I was wondering about, is should we expose failing builders through the monitoring API? Or should we send a notification when we knock a builder offline (i.e. when we increment the contiguous_errors above the limit?)

review: Needs Fixing

[Kevin McDermott (bigkevmcd)]

review: Needs Information

[Kevin McDermott (bigkevmcd)]

Oh, and as the configuration variable is "limit" then we the query should probably be...

+ Lexbuilder.contiguous_errors <= self.config.getint(
+ "master", "builder_upper_error_limit"))

:-)

[lp:~notnownikki/offspring/exclude-failing-builders]

166. By Nicola Heald on 2013-05-24

error limit behaves like a limit now, config option changed to remove _upper from the name.

[Kevin McDermott (bigkevmcd)]

Looks good now, thanks for the changes :-)

review: Approve

=== modified file 'config/offspring.cfg'
--- config/offspring.cfg	2012-02-16 06:44:19 +0000
+++ config/offspring.cfg	2013-05-24 16:54:37 +0000
@@ -140,6 +140,7 @@
 poll_frequency: 20
 logfile: %(log_dir)s/offspring-master.log
 launchpad_oauth: %(config_dir)s/launchpad.oauth
+builder_error_limit: 5
 
 #===============================================================================
 # Offspring publisher

=== modified file 'lib/offspring/master/master.py'
--- lib/offspring/master/master.py	2013-05-23 04:18:49 +0000
+++ lib/offspring/master/master.py	2013-05-24 16:54:37 +0000
@@ -11,6 +11,7 @@
 from storm.locals import create_database, Desc, Store
 
 from offspring.daemon import LexbuilderDaemon
+from offspring.enums import ProjectBuildStates
 from offspring.master.exceptions import SlaveCommunicationError
 from offspring.master.models import BuildRequest, Lexbuilder
 from offspring.master import notifications
@@ -97,7 +98,10 @@
         self.is_online = False
 
     def _get_builders(self):
-        return self.db_store.find(Lexbuilder, Lexbuilder.is_retired == False)
+        return self.db_store.find(
+            Lexbuilder, Lexbuilder.is_retired == False,
+            Lexbuilder.contiguous_errors <= self.config.getint(
+                "master", "builder_error_limit"))
 
     def scanSlaves(self):
         """
@@ -146,6 +150,10 @@
                                 logging.error(
                                     "PROBLEM DETECTED: A non-fatal problem has"
                                     " been detected. See above for details.")
+                        if builder.current_job.result == ProjectBuildStates.FAILED:
+                            builder.contiguous_errors += 1
+                        else:
+                            builder.contiguous_errors = 0
                         builder.current_job = None
                         self.db_store.commit()
                     else:

=== modified file 'lib/offspring/master/models.py'
--- lib/offspring/master/models.py	2013-05-23 04:18:49 +0000
+++ lib/offspring/master/models.py	2013-05-24 16:54:37 +0000
@@ -35,6 +35,7 @@
     is_active = Bool(default=True)
     is_okay = Bool(default=True)
     is_retired = Bool(default=False)
+    contiguous_errors = Int(default=0)
     current_job_id = Int()
     current_job = Reference(current_job_id, "BuildResult.id")
     notes = Unicode()

=== modified file 'lib/offspring/master/tests/helpers.py'
--- lib/offspring/master/tests/helpers.py	2013-05-23 04:18:49 +0000
+++ lib/offspring/master/tests/helpers.py	2013-05-24 16:54:37 +0000
@@ -91,6 +91,7 @@
         self.log_dir = self.makeDir()
         self.config.set("master", "log_dir", self.log_dir)
         self.config.set("master", "db", self.database_uri)
+        self.config.set("master", "builder_error_limit", "5")
 
 
 class BaseOffspringMasterTestCase(LexbuilderMasterTestMixin,
@@ -178,7 +179,8 @@
             "current_job_id INTEGER, "
             "is_okay BOOLEAN NOT NULL, "
             "is_active BOOLEAN NOT NULL DEFAULT TRUE, "
-            "is_retired BOOLEAN NOT NULL DEFAULT FALSE)")
+            "is_retired BOOLEAN NOT NULL DEFAULT FALSE,"
+            "contiguous_errors INTEGER NOT NULL DEFAULT 0)")
 
         self.connection.execute(
             "CREATE TABLE buildrequests ("

=== modified file 'lib/offspring/master/tests/test_master.py'
--- lib/offspring/master/tests/test_master.py	2013-05-23 13:55:56 +0000
+++ lib/offspring/master/tests/test_master.py	2013-05-24 16:54:37 +0000
@@ -7,6 +7,7 @@
 import storm
 from storm.exceptions import DisconnectionError
 
+from offspring.enums import ProjectBuildStates
 from offspring.master.master import LexbuilderMaster
 from offspring.master.models import (
     Lexbuilder, BuildResult)
@@ -40,6 +41,7 @@
         builder.is_active = kwargs.get("is_active", True)
         builder.is_okay = kwargs.get("is_okay", True)
         builder.is_retired = kwargs.get("is_retired", False)
+        builder.contiguous_errors = kwargs.get("contiguous_errors", 0)
         builder.machine_type = kwargs.get("arch", self.project.arch)
         self.db_store.add(builder)
         return builder
@@ -118,12 +120,33 @@
                 "Scan cycle completed\n",
             log_file.getvalue())
 
-    def build_builder_and_slave_with_job_and_result(self):
+    def test_scan_slaves_error_increments_error_count(self):
+        """
+        When a builder has a failed build, the contiguous_errors count
+        is incremented.
+        """
+        master, builder = self.build_builder_and_slave_with_job_and_result(
+            result=ProjectBuildStates.FAILED)
+        master.scanSlaves()
+        self.assertEqual(1, builder.contiguous_errors)
+
+    def test_scan_slaves_non_error_resets_error_count(self):
+        """
+        When a builder has a non-erroring build, the contiguous_errors
+        field is reset to 0.
+        """
+        master, builder = self.build_builder_and_slave_with_job_and_result(
+            result=ProjectBuildStates.SUCCESS, contiguous_errors=1)
+        master.scanSlaves()
+        self.assertEqual(0, builder.contiguous_errors)
+
+    def build_builder_and_slave_with_job_and_result(
+            self, result=None, contiguous_errors=0):
         """
         This builds and returns a LexbuilderMaster with a single slave builder,
         and a job result.
         """
-        builder = self.create_builder()
+        builder = self.create_builder(contiguous_errors=contiguous_errors)
         builder_mock = self.mocker.patch(builder)
         builder_mock.scan()
         self.mocker.result((Slave.STATE_IDLE, Slave.STATE_IDLE))
@@ -131,6 +154,7 @@
         build_request = self.create_build_request()
         build_result = BuildResult(build_request, builder)
         build_result.name = u"20111115-testing"
+        build_result.result = result
         self.db_store.add(build_result)
 
         builder_mock.getBuildResult()
@@ -232,6 +256,20 @@
         self.assertEqual(2, builders.count())
         self.assertEqual(["BuilderB", "BuilderC"], list(builders.values(Lexbuilder.name)))
 
+    def test_get_builders_does_not_return_errors_over_threshold(self):
+        """
+        Test that builders with contiguous_errors over the values defined in settings
+        are not returned.
+        """
+        master = self.get_master()
+        builder1 = self.create_builder(name="BuilderA", contiguous_errors=5)
+        builder2 = self.create_builder(name="BuilderB", contiguous_errors=6)
+        builder3 = self.create_builder(name="BuilderC")
+        builders = master._get_builders()
+        self.assertEqual(2, builders.count())
+        self.assertEqual(["BuilderA", "BuilderC"], list(builders.values(Lexbuilder.name)))
+
+
     def test_dispatch_builds(self):
         """
         dispatchBuilds sends BuildRequests to a slave.

=== modified file 'lib/offspring/web/queuemanager/admin.py'
--- lib/offspring/web/queuemanager/admin.py	2013-05-23 04:18:49 +0000
+++ lib/offspring/web/queuemanager/admin.py	2013-05-24 16:54:37 +0000
@@ -20,8 +20,8 @@
 
 class LexbuilderAdmin(admin.ModelAdmin):
     list_display = ('name', 'machine_type', 'is_active', 'is_okay',
-                    'is_retired', 'current_state', 'current_build',
-                    'updated_at')
+                    'is_retired', 'current_state', 'contiguous_errors',
+                    'current_build', 'updated_at')
     list_filter = ['current_state', 'machine_type', 'is_active',
                    'is_okay', 'is_retired']
     actions=['make_enabled', 'make_disabled']
@@ -36,7 +36,7 @@
     make_disabled.short_description = "Mark selected builders as disabled"
 
     def make_enabled(self, request, queryset):
-        rows_updated = queryset.update(is_active=True)
+        rows_updated = queryset.update(is_active=True, contiguous_errors=0)
         if rows_updated == 1:
             message_bit = "1 lexbuilder was"
         else:

=== added file 'lib/offspring/web/queuemanager/migrations/0004_auto__add_field_lexbuilder_contiguous_errors.py'
--- lib/offspring/web/queuemanager/migrations/0004_auto__add_field_lexbuilder_contiguous_errors.py	1970-01-01 00:00:00 +0000
+++ lib/offspring/web/queuemanager/migrations/0004_auto__add_field_lexbuilder_contiguous_errors.py	2013-05-24 16:54:37 +0000
@@ -0,0 +1,174 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding field 'Lexbuilder.contiguous_errors'
+        db.add_column('lexbuilders', 'contiguous_errors', self.gf('django.db.models.fields.IntegerField')(default=0), keep_default=False)
+
+
+    def backwards(self, orm):
+        
+        # Deleting field 'Lexbuilder.contiguous_errors'
+        db.delete_column('lexbuilders', 'contiguous_errors')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2013, 5, 24, 9, 34, 39, 946400)'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime(2013, 5, 24, 9, 34, 39, 946337)'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'django_group_access.accessgroup': {
+            'Meta': {'ordering': "('name',)", 'object_name': 'AccessGroup'},
+            'auto_share_groups': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'auto_share_groups_rel_+'", 'blank': 'True', 'to': "orm['django_group_access.AccessGroup']"}),
+            'can_be_shared_with': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'can_share_with': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['django_group_access.AccessGroup']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'members': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.User']", 'symmetrical': 'False', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '64'}),
+            'supergroup': ('django.db.models.fields.BooleanField', [], {'default': 'False'})
+        },
+        'queuemanager.buildrequest': {
+            'Meta': {'object_name': 'BuildRequest', 'db_table': "'buildrequests'"},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['queuemanager.Project']", 'db_column': "'project_name'"}),
+            'reason': ('django.db.models.fields.TextField', [], {'max_length': '200', 'blank': 'True'}),
+            'requestor': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'db_column': "'requestor_id'", 'blank': 'True'}),
+            'score': ('django.db.models.fields.IntegerField', [], {'default': '10'})
+        },
+        'queuemanager.buildresult': {
+            'Meta': {'object_name': 'BuildResult', 'db_table': "'buildresults'"},
+            'builder': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['queuemanager.Lexbuilder']", 'null': 'True', 'blank': 'True'}),
+            'dispatched_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'finished_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'notes': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['queuemanager.Project']", 'db_column': "'project_name'"}),
+            'reason': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'requested_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'null': 'True', 'blank': 'True'}),
+            'requestor': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'db_column': "'requestor_id'", 'blank': 'True'}),
+            'result': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True'}),
+            'started_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'})
+        },
+        'queuemanager.dailybuildorder': {
+            'Meta': {'object_name': 'DailyBuildOrder', 'db_table': "'dailybuildorders'"},
+            'hour': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'projects': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['queuemanager.Project']", 'symmetrical': 'False'})
+        },
+        'queuemanager.launchpadproject': {
+            'Meta': {'object_name': 'LaunchpadProject', 'db_table': "'launchpad_projects'"},
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200', 'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'})
+        },
+        'queuemanager.launchpadprojectmilestone': {
+            'Meta': {'object_name': 'LaunchpadProjectMilestone', 'db_table': "'launchpad_project_milestones'"},
+            'active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'date_targeted': ('django.db.models.fields.DateField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'launchpad_project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['queuemanager.LaunchpadProject']", 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'})
+        },
+        'queuemanager.lexbuilder': {
+            'Meta': {'object_name': 'Lexbuilder', 'db_table': "'lexbuilders'"},
+            'contiguous_errors': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'current_job': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['queuemanager.BuildResult']", 'null': 'True', 'db_column': "'current_job_id'", 'blank': 'True'}),
+            'current_state': ('django.db.models.fields.CharField', [], {'default': "'UNKNOWN'", 'max_length': '200'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_okay': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_retired': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'machine_type': ('django.db.models.fields.CharField', [], {'default': "'x86_64'", 'max_length': '200'}),
+            'name': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '200', 'db_index': 'True'}),
+            'notes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'previous_state': ('django.db.models.fields.CharField', [], {'default': "'UNKNOWN'", 'max_length': '200'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'uri': ('django.db.models.fields.CharField', [], {'max_length': '200'})
+        },
+        'queuemanager.project': {
+            'Meta': {'object_name': 'Project', 'db_table': "'projects'"},
+            'access_groups': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'to': "orm['django_group_access.AccessGroup']", 'null': 'True', 'blank': 'True'}),
+            'arch': ('django.db.models.fields.CharField', [], {'max_length': '10'}),
+            'config_url': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'launchpad_project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['queuemanager.LaunchpadProject']", 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '200', 'primary_key': 'True', 'db_index': 'True'}),
+            'notes': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'owner': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'project_owner'", 'null': 'True', 'to': "orm['auth.User']"}),
+            'priority': ('django.db.models.fields.IntegerField', [], {'default': '50'}),
+            'project_group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['queuemanager.ProjectGroup']", 'null': 'True', 'blank': 'True'}),
+            'series': ('django.db.models.fields.CharField', [], {'default': "'lucid'", 'max_length': '200'}),
+            'status': ('django.db.models.fields.CharField', [], {'max_length': '200', 'null': 'True'}),
+            'suite': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '30'})
+        },
+        'queuemanager.projectgroup': {
+            'Meta': {'object_name': 'ProjectGroup', 'db_table': "'projectgroups'"},
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200', 'primary_key': 'True'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '200'})
+        },
+        'queuemanager.projectnotificationsubscription': {
+            'Meta': {'object_name': 'ProjectNotificationSubscription', 'db_table': "'project_notification_subscriptions'"},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'project': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['queuemanager.Project']", 'db_column': "'project_name'"}),
+            'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'db_column': "'user_id'"})
+        },
+        'queuemanager.release': {
+            'Meta': {'object_name': 'Release', 'db_table': "'releases'"},
+            'build': ('django.db.models.fields.related.OneToOneField', [], {'to': "orm['queuemanager.BuildResult']", 'unique': 'True', 'primary_key': 'True'}),
+            'checklist_url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'blank': 'True'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'creator': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'milestone': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['queuemanager.LaunchpadProjectMilestone']", 'null': 'True', 'blank': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '200'}),
+            'notes': ('django.db.models.fields.TextField', [], {'blank': 'True'}),
+            'published_at': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}),
+            'status': ('django.db.models.fields.PositiveSmallIntegerField', [], {'default': '0'}),
+            'tag': ('django.db.models.fields.CharField', [], {'max_length': '5', 'null': 'True', 'blank': 'True'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['queuemanager']

=== modified file 'lib/offspring/web/queuemanager/models.py'
--- lib/offspring/web/queuemanager/models.py	2013-05-23 04:18:49 +0000
+++ lib/offspring/web/queuemanager/models.py	2013-05-24 16:54:37 +0000
@@ -165,6 +165,7 @@
     is_active = models.BooleanField(default=True)
     is_okay = models.BooleanField(default=True, editable=False)
     is_retired = models.BooleanField(default=False)
+    contiguous_errors = models.IntegerField(default=0)
     machine_type = models.CharField(max_length=200, default="x86_64")
     current_job = models.ForeignKey("BuildResult", db_column="current_job_id", editable=False, blank=True, null=True)
     notes = models.TextField('whiteboard', blank=True, null=True)