Merge lp:~mwhudson/lava-scheduler/device-tags into lp:lava-scheduler

[Michael Hudson-Doyle]

Hey,

This branch adds support for tags to the scheduler. The idea is that a job will not be dispatched to a board if the job has an tag the board does not. The core query change is pretty hard to understand and convince oneself of correctness of, which is why it is extensively tested.

The UI aspects are very very simple -- we can fix that up later.

Cheers,
mwh

[lp:~mwhudson/lava-scheduler/device-tags]

120. By Michael Hudson-Doyle on 2011-12-15

merge trunk

=== modified file 'lava_scheduler_app/admin.py'
--- lava_scheduler_app/admin.py	2011-06-17 03:55:23 +0000
+++ lava_scheduler_app/admin.py	2011-12-15 20:59:23 +0000
@@ -1,6 +1,7 @@
 from django.contrib import admin
-from lava_scheduler_app.models import Device, DeviceType, TestJob
+from lava_scheduler_app.models import Device, DeviceType, TestJob, Tag
 
 admin.site.register(Device)
 admin.site.register(DeviceType)
 admin.site.register(TestJob)
+admin.site.register(Tag)

=== added file 'lava_scheduler_app/migrations/0011_auto__add_tag.py'
--- lava_scheduler_app/migrations/0011_auto__add_tag.py	1970-01-01 00:00:00 +0000
+++ lava_scheduler_app/migrations/0011_auto__add_tag.py	2011-12-15 20:59:23 +0000
@@ -0,0 +1,122 @@
+# encoding: utf-8
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        
+        # Adding model 'Tag'
+        db.create_table('lava_scheduler_app_tag', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('name', self.gf('django.db.models.fields.SlugField')(unique=True, max_length=50, db_index=True)),
+            ('description', self.gf('django.db.models.fields.TextField')(null=True, blank=True)),
+        ))
+        db.send_create_signal('lava_scheduler_app', ['Tag'])
+
+        # Adding M2M table for field tags on 'Device'
+        db.create_table('lava_scheduler_app_device_tags', (
+            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
+            ('device', models.ForeignKey(orm['lava_scheduler_app.device'], null=False)),
+            ('tag', models.ForeignKey(orm['lava_scheduler_app.tag'], null=False))
+        ))
+        db.create_unique('lava_scheduler_app_device_tags', ['device_id', 'tag_id'])
+
+        # Adding M2M table for field tags on 'TestJob'
+        db.create_table('lava_scheduler_app_testjob_tags', (
+            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
+            ('testjob', models.ForeignKey(orm['lava_scheduler_app.testjob'], null=False)),
+            ('tag', models.ForeignKey(orm['lava_scheduler_app.tag'], null=False))
+        ))
+        db.create_unique('lava_scheduler_app_testjob_tags', ['testjob_id', 'tag_id'])
+
+
+    def backwards(self, orm):
+        
+        # Deleting model 'Tag'
+        db.delete_table('lava_scheduler_app_tag')
+
+        # Removing M2M table for field tags on 'Device'
+        db.delete_table('lava_scheduler_app_device_tags')
+
+        # Removing M2M table for field tags on 'TestJob'
+        db.delete_table('lava_scheduler_app_testjob_tags')
+
+
+    models = {
+        'auth.group': {
+            'Meta': {'object_name': 'Group'},
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}),
+            'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'auth.permission': {
+            'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'},
+            'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '50'})
+        },
+        'auth.user': {
+            'Meta': {'object_name': 'User'},
+            'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}),
+            'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}),
+            'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}),
+            'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}),
+            'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}),
+            'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}),
+            'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'})
+        },
+        'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        'lava_scheduler_app.device': {
+            'Meta': {'object_name': 'Device'},
+            'current_job': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['lava_scheduler_app.TestJob']", 'unique': 'True', 'null': 'True', 'blank': 'True'}),
+            'device_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['lava_scheduler_app.DeviceType']"}),
+            'hostname': ('django.db.models.fields.CharField', [], {'max_length': '200', 'primary_key': 'True'}),
+            'status': ('django.db.models.fields.IntegerField', [], {'default': '1'}),
+            'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['lava_scheduler_app.Tag']", 'symmetrical': 'False', 'blank': 'True'})
+        },
+        'lava_scheduler_app.devicetype': {
+            'Meta': {'object_name': 'DeviceType'},
+            'name': ('django.db.models.fields.SlugField', [], {'max_length': '50', 'primary_key': 'True', 'db_index': 'True'})
+        },
+        'lava_scheduler_app.tag': {
+            'Meta': {'object_name': 'Tag'},
+            'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'name': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50', 'db_index': 'True'})
+        },
+        'lava_scheduler_app.testjob': {
+            'Meta': {'object_name': 'TestJob'},
+            'actual_device': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.Device']"}),
+            'definition': ('django.db.models.fields.TextField', [], {}),
+            'description': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}),
+            'end_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'log_file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True', 'blank': 'True'}),
+            'requested_device': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.Device']"}),
+            'requested_device_type': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.DeviceType']"}),
+            'results_link': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '400', 'null': 'True', 'blank': 'True'}),
+            'start_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}),
+            'status': ('django.db.models.fields.IntegerField', [], {'default': '0'}),
+            'submit_time': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'submitter': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}),
+            'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['lava_scheduler_app.Tag']", 'symmetrical': 'False', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['lava_scheduler_app']

=== modified file 'lava_scheduler_app/models.py'
--- lava_scheduler_app/models.py	2011-10-28 00:24:13 +0000
+++ lava_scheduler_app/models.py	2011-12-15 20:59:23 +0000
@@ -9,6 +9,16 @@
     """Error raised when JSON is syntactically valid but ill-formed."""
 
 
+class Tag(models.Model):
+
+    name = models.SlugField(unique=True)
+
+    description = models.TextField(null=True, blank=True)
+
+    def __unicode__(self):
+        return self.name
+
+
 class DeviceType(models.Model):
     """
     A class of device, for example a pandaboard or a snowball.
@@ -51,6 +61,8 @@
     current_job = models.ForeignKey(
         "TestJob", blank=True, unique=True, null=True)
 
+    tags = models.ManyToManyField(Tag, blank=True)
+
     status = models.IntegerField(
         choices = STATUS_CHOICES,
         default = IDLE,
@@ -137,6 +149,8 @@
     requested_device_type = models.ForeignKey(
         DeviceType, null=True, default=None, related_name='+', blank=True)
 
+    tags = models.ManyToManyField(Tag, blank=True)
+
     # This is set once the job starts.
     actual_device = models.ForeignKey(
         Device, null=True, default=None, related_name='+', blank=True)
@@ -206,6 +220,8 @@
             definition=json_data, submitter=user, requested_device=target,
             requested_device_type=device_type, description=job_name)
         job.save()
+        for tag_name in job_data.get('device_tags', []):
+            job.tags.add(Tag.objects.get_or_create(name=tag_name)[0])
         return job
 
     def can_cancel(self, user):

=== modified file 'lava_scheduler_app/templates/lava_scheduler_app/device.html'
--- lava_scheduler_app/templates/lava_scheduler_app/device.html	2011-10-28 00:24:13 +0000
+++ lava_scheduler_app/templates/lava_scheduler_app/device.html	2011-12-15 20:59:23 +0000
@@ -37,6 +37,13 @@
 
     <dt>Device type:</dt>
     <dd>{{ device.device_type }}</dd>
+
+    <dt>Device Tags</dt>
+    {% for tag in device.tags.all %}
+    <dd>{{ tag.name }}</dd>
+    {% empty %}
+    <dd><i>None</i></dd>
+    {% endfor %}
   </div>
   <div class="column">
     <dt>Status:</dt>

=== modified file 'lava_scheduler_app/templates/lava_scheduler_app/job_sidebar.html'
--- lava_scheduler_app/templates/lava_scheduler_app/job_sidebar.html	2011-12-09 03:55:33 +0000
+++ lava_scheduler_app/templates/lava_scheduler_app/job_sidebar.html	2011-12-15 20:59:23 +0000
@@ -25,6 +25,17 @@
     <dd>{{ job.requested_device_type }}</dd>
     {% endif %}
 
+    {% for tag in job.tags.all %}
+    {% if forloop.first %}
+    {% if forloop.revcounter > 1 %}
+    <dt>Required Device Tags</dt>
+    {% else %}
+    <dt>Required Device Tag</dt>
+    {% endif %}
+    {% endif %}
+    <dd>{{ tag.name }}</dd>
+    {% endfor %}
+
     {% if job.description %}
     <dt>Description:</dt>
     <dd>{{ job.description }}</dd>

=== modified file 'lava_scheduler_app/tests.py'
--- lava_scheduler_app/tests.py	2011-12-14 02:10:04 +0000
+++ lava_scheduler_app/tests.py	2011-12-15 20:59:23 +0000
@@ -9,7 +9,7 @@
 
 from django_testscenarios.ubertest import TestCase
 
-from lava_scheduler_app.models import Device, DeviceType, TestJob
+from lava_scheduler_app.models import Device, DeviceType, Tag, TestJob
 
 
 
@@ -135,6 +135,41 @@
             json.dumps({'device_type':'panda'}), self.factory.make_user())
         self.assertEqual(job.status, TestJob.SUBMITTED)
 
+    def test_from_json_and_user_sets_no_tags_if_no_tags(self):
+        self.factory.ensure_device_type(name='panda')
+        job = TestJob.from_json_and_user(
+            json.dumps({'device_type':'panda', 'device_tags':[]}),
+            self.factory.make_user())
+        self.assertEqual(set(job.tags.all()), set([]))
+
+    def test_from_json_and_user_sets_tag_from_device_tags(self):
+        self.factory.ensure_device_type(name='panda')
+        job = TestJob.from_json_and_user(
+            json.dumps({'device_type':'panda', 'device_tags':['tag']}),
+            self.factory.make_user())
+        self.assertEqual(
+            set(tag.name for tag in job.tags.all()), set(['tag']))
+
+    def test_from_json_and_user_sets_multiple_tag_from_device_tags(self):
+        self.factory.ensure_device_type(name='panda')
+        job = TestJob.from_json_and_user(
+            json.dumps({'device_type':'panda', 'device_tags':['tag1', 'tag2']}),
+            self.factory.make_user())
+        self.assertEqual(
+            set(tag.name for tag in job.tags.all()), set(['tag1', 'tag2']))
+
+    def test_from_json_and_user_reuses_tag_objects(self):
+        self.factory.ensure_device_type(name='panda')
+        job1 = TestJob.from_json_and_user(
+            json.dumps({'device_type':'panda', 'device_tags':['tag']}),
+            self.factory.make_user())
+        job2 = TestJob.from_json_and_user(
+            json.dumps({'device_type':'panda', 'device_tags':['tag']}),
+            self.factory.make_user())
+        self.assertEqual(
+            set(tag.pk for tag in job1.tags.all()),
+            set(tag.pk for tag in job2.tags.all()))
+
 
 class TestSchedulerAPI(TestCaseWithFactory):
 
@@ -298,6 +333,53 @@
             None,
             DatabaseJobSource().getJobForBoard_impl('panda02'))
 
+    def _makeBoardWithTags(self, tags):
+        board = self.factory.make_device()
+        for tag_name in tags:
+            board.tags.add(Tag.objects.get_or_create(name=tag_name)[0])
+        return board
+
+    def _makeJobWithTagsForBoard(self, tags, board):
+        job = self.factory.make_testjob(requested_device=board)
+        for tag_name in tags:
+            job.tags.add(Tag.objects.get_or_create(name=tag_name)[0])
+        return job
+
+    def assertBoardWithTagsGetsJobWithTags(self, board_tags, job_tags):
+        board = self._makeBoardWithTags(board_tags)
+        self._makeJobWithTagsForBoard(job_tags, board)
+        self.assertEqual(
+            board.hostname,
+            DatabaseJobSource().getJobForBoard_impl(board.hostname)['target'])
+
+    def assertBoardWithTagsDoesNotGetJobWithTags(self, board_tags, job_tags):
+        board = self._makeBoardWithTags(board_tags)
+        self._makeJobWithTagsForBoard(job_tags, board)
+        self.assertEqual(
+            None,
+            DatabaseJobSource().getJobForBoard_impl(board.hostname))
+
+    def test_getJobForBoard_does_not_return_job_if_board_lacks_tag(self):
+        self.assertBoardWithTagsDoesNotGetJobWithTags([], ['tag'])
+
+    def test_getJobForBoard_returns_job_if_board_has_tag(self):
+        self.assertBoardWithTagsGetsJobWithTags(['tag'], ['tag'])
+
+    def test_getJobForBoard_returns_job_if_board_has_both_tags(self):
+        self.assertBoardWithTagsGetsJobWithTags(['tag1', 'tag2'], ['tag1', 'tag2'])
+
+    def test_getJobForBoard_returns_job_if_board_has_extra_tags(self):
+        self.assertBoardWithTagsGetsJobWithTags(['tag1', 'tag2'], ['tag1'])
+
+    def test_getJobForBoard_does_not_return_job_if_board_has_only_one_tag(self):
+        self.assertBoardWithTagsDoesNotGetJobWithTags(['tag1'], ['tag1', 'tag2'])
+
+    def test_getJobForBoard_does_not_return_job_if_board_has_unrelated_tag(self):
+        self.assertBoardWithTagsDoesNotGetJobWithTags(['tag1'], ['tag2'])
+
+    def test_getJobForBoard_does_not_return_job_if_only_one_tag_matches(self):
+        self.assertBoardWithTagsDoesNotGetJobWithTags(['tag1', 'tag2'], ['tag1', 'tag3'])
+
     def test_getJobForBoard_sets_start_time(self):
         device = self.factory.make_device(hostname='panda01')
         job = self.factory.make_testjob(requested_device=device)

=== modified file 'lava_scheduler_daemon/dbjobsource.py'
--- lava_scheduler_daemon/dbjobsource.py	2011-12-01 03:58:08 +0000
+++ lava_scheduler_daemon/dbjobsource.py	2011-12-15 20:59:23 +0000
@@ -87,7 +87,17 @@
                 | Q(requested_device_type=device.device_type),
                 status=TestJob.SUBMITTED)
             jobs_for_device = jobs_for_device.extra(
-                select={'is_targeted': 'requested_device_id is not NULL'},
+                select={
+                    'is_targeted': 'requested_device_id is not NULL',
+                    'missing_tags': '''
+                        select count(*) from lava_scheduler_app_testjob_tags
+                                       where testjob_id = lava_scheduler_app_testjob.id
+                                       and tag_id not in (select tag_id
+                                                            from lava_scheduler_app_device_tags
+                                                           where device_id = '%s')
+                        ''' % device.hostname,
+                    },
+                where=['missing_tags = 0'],
                 order_by=['-is_targeted', 'submit_time'])
             jobs = jobs_for_device[:1]
             if jobs: