Merge lp:~stylesen/lava-scheduler/multinode into lp:lava-scheduler/multinode
- multinode
- Merge into multinode
Proposed by
Senthil Kumaran S
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | Neil Williams | ||||
Approved revision: | no longer in the source branch. | ||||
Merged at revision: | 248 | ||||
Proposed branch: | lp:~stylesen/lava-scheduler/multinode | ||||
Merge into: | lp:lava-scheduler/multinode | ||||
Diff against target: |
540 lines (+376/-12) 8 files modified
lava_scheduler_app/api.py (+1/-1) lava_scheduler_app/management/commands/scheduler.py (+2/-2) lava_scheduler_app/migrations/0030_auto__add_field_testjob_sub_id.py (+160/-0) lava_scheduler_app/models.py (+14/-4) lava_scheduler_app/views.py (+11/-5) lava_scheduler_daemon/dbjobsource.py (+76/-0) lava_scheduler_daemon/job.py (+79/-0) lava_scheduler_daemon/service.py (+33/-0) |
||||
To merge this branch: | bzr merge lp:~stylesen/lava-scheduler/multinode | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Neil Williams | Approve | ||
Review via email: mp+170590@code.launchpad.net |
Commit message
Description of the change
Initial bits with a job based scheduler for multi-node support.
We do not accept multi-node jobs yet.
To post a comment you must log in.
- 248. By Neil Williams
-
Merge Senthil's initial MultiNode changes for lava-scheduler.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lava_scheduler_app/api.py' | |||
2 | --- lava_scheduler_app/api.py 2013-05-02 07:35:44 +0000 | |||
3 | +++ lava_scheduler_app/api.py 2013-06-20 11:53:35 +0000 | |||
4 | @@ -36,7 +36,7 @@ | |||
5 | 36 | raise xmlrpclib.Fault(404, "Specified device not found.") | 36 | raise xmlrpclib.Fault(404, "Specified device not found.") |
6 | 37 | except DeviceType.DoesNotExist: | 37 | except DeviceType.DoesNotExist: |
7 | 38 | raise xmlrpclib.Fault(404, "Specified device type not found.") | 38 | raise xmlrpclib.Fault(404, "Specified device type not found.") |
9 | 39 | return job.id | 39 | return job |
10 | 40 | 40 | ||
11 | 41 | def resubmit_job(self, job_id): | 41 | def resubmit_job(self, job_id): |
12 | 42 | try: | 42 | try: |
13 | 43 | 43 | ||
14 | === modified file 'lava_scheduler_app/management/commands/scheduler.py' | |||
15 | --- lava_scheduler_app/management/commands/scheduler.py 2012-03-27 05:41:09 +0000 | |||
16 | +++ lava_scheduler_app/management/commands/scheduler.py 2013-06-20 11:53:35 +0000 | |||
17 | @@ -42,7 +42,7 @@ | |||
18 | 42 | 42 | ||
19 | 43 | from twisted.internet import reactor | 43 | from twisted.internet import reactor |
20 | 44 | 44 | ||
22 | 45 | from lava_scheduler_daemon.service import BoardSet | 45 | from lava_scheduler_daemon.service import BoardSet, JobQueue |
23 | 46 | from lava_scheduler_daemon.dbjobsource import DatabaseJobSource | 46 | from lava_scheduler_daemon.dbjobsource import DatabaseJobSource |
24 | 47 | 47 | ||
25 | 48 | daemon_options = self._configure(options) | 48 | daemon_options = self._configure(options) |
26 | @@ -57,7 +57,7 @@ | |||
27 | 57 | 'fake-dispatcher') | 57 | 'fake-dispatcher') |
28 | 58 | else: | 58 | else: |
29 | 59 | dispatcher = options['dispatcher'] | 59 | dispatcher = options['dispatcher'] |
31 | 60 | service = BoardSet( | 60 | service = JobQueue( |
32 | 61 | source, dispatcher, reactor, daemon_options=daemon_options) | 61 | source, dispatcher, reactor, daemon_options=daemon_options) |
33 | 62 | reactor.callWhenRunning(service.startService) | 62 | reactor.callWhenRunning(service.startService) |
34 | 63 | reactor.run() | 63 | reactor.run() |
35 | 64 | 64 | ||
36 | === added file 'lava_scheduler_app/migrations/0030_auto__add_field_testjob_sub_id.py' | |||
37 | --- lava_scheduler_app/migrations/0030_auto__add_field_testjob_sub_id.py 1970-01-01 00:00:00 +0000 | |||
38 | +++ lava_scheduler_app/migrations/0030_auto__add_field_testjob_sub_id.py 2013-06-20 11:53:35 +0000 | |||
39 | @@ -0,0 +1,160 @@ | |||
40 | 1 | # -*- coding: utf-8 -*- | ||
41 | 2 | import datetime | ||
42 | 3 | from south.db import db | ||
43 | 4 | from south.v2 import SchemaMigration | ||
44 | 5 | from django.db import models | ||
45 | 6 | |||
46 | 7 | |||
47 | 8 | class Migration(SchemaMigration): | ||
48 | 9 | |||
49 | 10 | def forwards(self, orm): | ||
50 | 11 | # Adding field 'TestJob.sub_id' | ||
51 | 12 | db.add_column('lava_scheduler_app_testjob', 'sub_id', | ||
52 | 13 | self.gf('django.db.models.fields.CharField')(default='', max_length=200, blank=True), | ||
53 | 14 | keep_default=False) | ||
54 | 15 | |||
55 | 16 | |||
56 | 17 | def backwards(self, orm): | ||
57 | 18 | # Deleting field 'TestJob.sub_id' | ||
58 | 19 | db.delete_column('lava_scheduler_app_testjob', 'sub_id') | ||
59 | 20 | |||
60 | 21 | |||
61 | 22 | models = { | ||
62 | 23 | 'auth.group': { | ||
63 | 24 | 'Meta': {'object_name': 'Group'}, | ||
64 | 25 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
65 | 26 | 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), | ||
66 | 27 | 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}) | ||
67 | 28 | }, | ||
68 | 29 | 'auth.permission': { | ||
69 | 30 | 'Meta': {'ordering': "('content_type__app_label', 'content_type__model', 'codename')", 'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, | ||
70 | 31 | 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), | ||
71 | 32 | 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), | ||
72 | 33 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
73 | 34 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) | ||
74 | 35 | }, | ||
75 | 36 | 'auth.user': { | ||
76 | 37 | 'Meta': {'object_name': 'User'}, | ||
77 | 38 | 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), | ||
78 | 39 | 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), | ||
79 | 40 | 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), | ||
80 | 41 | 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'symmetrical': 'False', 'blank': 'True'}), | ||
81 | 42 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
82 | 43 | 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), | ||
83 | 44 | 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
84 | 45 | 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
85 | 46 | 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), | ||
86 | 47 | 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), | ||
87 | 48 | 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), | ||
88 | 49 | 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'symmetrical': 'False', 'blank': 'True'}), | ||
89 | 50 | 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) | ||
90 | 51 | }, | ||
91 | 52 | 'contenttypes.contenttype': { | ||
92 | 53 | 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, | ||
93 | 54 | 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), | ||
94 | 55 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
95 | 56 | 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), | ||
96 | 57 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) | ||
97 | 58 | }, | ||
98 | 59 | 'dashboard_app.bundle': { | ||
99 | 60 | 'Meta': {'ordering': "['-uploaded_on']", 'object_name': 'Bundle'}, | ||
100 | 61 | '_gz_content': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'db_column': "'gz_content'"}), | ||
101 | 62 | '_raw_content': ('django.db.models.fields.files.FileField', [], {'max_length': '100', 'null': 'True', 'db_column': "'content'"}), | ||
102 | 63 | 'bundle_stream': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'bundles'", 'to': "orm['dashboard_app.BundleStream']"}), | ||
103 | 64 | 'content_filename': ('django.db.models.fields.CharField', [], {'max_length': '256'}), | ||
104 | 65 | 'content_sha1': ('django.db.models.fields.CharField', [], {'max_length': '40', 'unique': 'True', 'null': 'True'}), | ||
105 | 66 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
106 | 67 | 'is_deserialized': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
107 | 68 | 'uploaded_by': ('django.db.models.fields.related.ForeignKey', [], {'blank': 'True', 'related_name': "'uploaded_bundles'", 'null': 'True', 'to': "orm['auth.User']"}), | ||
108 | 69 | 'uploaded_on': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.utcnow'}) | ||
109 | 70 | }, | ||
110 | 71 | 'dashboard_app.bundlestream': { | ||
111 | 72 | 'Meta': {'object_name': 'BundleStream'}, | ||
112 | 73 | 'group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}), | ||
113 | 74 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
114 | 75 | 'is_anonymous': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
115 | 76 | 'is_public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
116 | 77 | 'name': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), | ||
117 | 78 | 'pathname': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '128'}), | ||
118 | 79 | 'slug': ('django.db.models.fields.CharField', [], {'max_length': '64', 'blank': 'True'}), | ||
119 | 80 | 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}) | ||
120 | 81 | }, | ||
121 | 82 | 'lava_scheduler_app.device': { | ||
122 | 83 | 'Meta': {'object_name': 'Device'}, | ||
123 | 84 | 'current_job': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['lava_scheduler_app.TestJob']", 'blank': 'True', 'unique': 'True'}), | ||
124 | 85 | 'device_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['lava_scheduler_app.DeviceType']"}), | ||
125 | 86 | 'device_version': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}), | ||
126 | 87 | 'health_status': ('django.db.models.fields.IntegerField', [], {'default': '0'}), | ||
127 | 88 | 'hostname': ('django.db.models.fields.CharField', [], {'max_length': '200', 'primary_key': 'True'}), | ||
128 | 89 | 'last_health_report_job': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'null': 'True', 'on_delete': 'models.SET_NULL', 'to': "orm['lava_scheduler_app.TestJob']", 'blank': 'True', 'unique': 'True'}), | ||
129 | 90 | 'status': ('django.db.models.fields.IntegerField', [], {'default': '1'}), | ||
130 | 91 | 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['lava_scheduler_app.Tag']", 'symmetrical': 'False', 'blank': 'True'}) | ||
131 | 92 | }, | ||
132 | 93 | 'lava_scheduler_app.devicestatetransition': { | ||
133 | 94 | 'Meta': {'object_name': 'DeviceStateTransition'}, | ||
134 | 95 | 'created_by': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}), | ||
135 | 96 | 'created_on': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), | ||
136 | 97 | 'device': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'transitions'", 'to': "orm['lava_scheduler_app.Device']"}), | ||
137 | 98 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
138 | 99 | 'job': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['lava_scheduler_app.TestJob']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}), | ||
139 | 100 | 'message': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), | ||
140 | 101 | 'new_state': ('django.db.models.fields.IntegerField', [], {}), | ||
141 | 102 | 'old_state': ('django.db.models.fields.IntegerField', [], {}) | ||
142 | 103 | }, | ||
143 | 104 | 'lava_scheduler_app.devicetype': { | ||
144 | 105 | 'Meta': {'object_name': 'DeviceType'}, | ||
145 | 106 | 'display': ('django.db.models.fields.BooleanField', [], {'default': 'True'}), | ||
146 | 107 | 'health_check_job': ('django.db.models.fields.TextField', [], {'default': 'None', 'null': 'True', 'blank': 'True'}), | ||
147 | 108 | 'name': ('django.db.models.fields.SlugField', [], {'max_length': '50', 'primary_key': 'True'}) | ||
148 | 109 | }, | ||
149 | 110 | 'lava_scheduler_app.jobfailuretag': { | ||
150 | 111 | 'Meta': {'object_name': 'JobFailureTag'}, | ||
151 | 112 | 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), | ||
152 | 113 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
153 | 114 | 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '256'}) | ||
154 | 115 | }, | ||
155 | 116 | 'lava_scheduler_app.tag': { | ||
156 | 117 | 'Meta': {'object_name': 'Tag'}, | ||
157 | 118 | 'description': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), | ||
158 | 119 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
159 | 120 | 'name': ('django.db.models.fields.SlugField', [], {'unique': 'True', 'max_length': '50'}) | ||
160 | 121 | }, | ||
161 | 122 | 'lava_scheduler_app.testjob': { | ||
162 | 123 | 'Meta': {'object_name': 'TestJob'}, | ||
163 | 124 | '_results_bundle': ('django.db.models.fields.related.OneToOneField', [], {'null': 'True', 'db_column': "'results_bundle_id'", 'on_delete': 'models.SET_NULL', 'to': "orm['dashboard_app.Bundle']", 'blank': 'True', 'unique': 'True'}), | ||
164 | 125 | '_results_link': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '400', 'null': 'True', 'db_column': "'results_link'", 'blank': 'True'}), | ||
165 | 126 | 'actual_device': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.Device']"}), | ||
166 | 127 | 'definition': ('django.db.models.fields.TextField', [], {}), | ||
167 | 128 | 'description': ('django.db.models.fields.CharField', [], {'default': 'None', 'max_length': '200', 'null': 'True', 'blank': 'True'}), | ||
168 | 129 | 'end_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), | ||
169 | 130 | 'failure_comment': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), | ||
170 | 131 | 'failure_tags': ('django.db.models.fields.related.ManyToManyField', [], {'symmetrical': 'False', 'related_name': "'failure_tags'", 'blank': 'True', 'to': "orm['lava_scheduler_app.JobFailureTag']"}), | ||
171 | 132 | 'group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.Group']", 'null': 'True', 'blank': 'True'}), | ||
172 | 133 | 'health_check': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
173 | 134 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
174 | 135 | 'is_public': ('django.db.models.fields.BooleanField', [], {'default': 'False'}), | ||
175 | 136 | 'log_file': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '100', 'null': 'True', 'blank': 'True'}), | ||
176 | 137 | 'priority': ('django.db.models.fields.IntegerField', [], {'default': '50'}), | ||
177 | 138 | 'requested_device': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.Device']"}), | ||
178 | 139 | 'requested_device_type': ('django.db.models.fields.related.ForeignKey', [], {'default': 'None', 'related_name': "'+'", 'null': 'True', 'blank': 'True', 'to': "orm['lava_scheduler_app.DeviceType']"}), | ||
179 | 140 | 'start_time': ('django.db.models.fields.DateTimeField', [], {'null': 'True', 'blank': 'True'}), | ||
180 | 141 | 'status': ('django.db.models.fields.IntegerField', [], {'default': '0'}), | ||
181 | 142 | 'sub_id': ('django.db.models.fields.CharField', [], {'max_length': '200', 'blank': 'True'}), | ||
182 | 143 | 'submit_time': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), | ||
183 | 144 | 'submit_token': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['linaro_django_xmlrpc.AuthToken']", 'null': 'True', 'on_delete': 'models.SET_NULL', 'blank': 'True'}), | ||
184 | 145 | 'submitter': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'+'", 'to': "orm['auth.User']"}), | ||
185 | 146 | 'tags': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['lava_scheduler_app.Tag']", 'symmetrical': 'False', 'blank': 'True'}), | ||
186 | 147 | 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']", 'null': 'True', 'blank': 'True'}) | ||
187 | 148 | }, | ||
188 | 149 | 'linaro_django_xmlrpc.authtoken': { | ||
189 | 150 | 'Meta': {'object_name': 'AuthToken'}, | ||
190 | 151 | 'created_on': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), | ||
191 | 152 | 'description': ('django.db.models.fields.TextField', [], {'default': "''", 'blank': 'True'}), | ||
192 | 153 | 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), | ||
193 | 154 | 'last_used_on': ('django.db.models.fields.DateTimeField', [], {'null': 'True'}), | ||
194 | 155 | 'secret': ('django.db.models.fields.CharField', [], {'default': "'ynu6yihw337isktsrzd0ocr83r0huox1b5y4qs1c0ktat7s3089d2xgiz3ll0n68fr6q026mep0t5xwg1coxnl2aoknolgowx2779uluan8quez0row0jnk5j2qsxlle'", 'unique': 'True', 'max_length': '128'}), | ||
195 | 156 | 'user': ('django.db.models.fields.related.ForeignKey', [], {'related_name': "'auth_tokens'", 'to': "orm['auth.User']"}) | ||
196 | 157 | } | ||
197 | 158 | } | ||
198 | 159 | |||
199 | 160 | complete_apps = ['lava_scheduler_app'] | ||
200 | 0 | \ No newline at end of file | 161 | \ No newline at end of file |
201 | 1 | 162 | ||
202 | === modified file 'lava_scheduler_app/models.py' | |||
203 | --- lava_scheduler_app/models.py 2013-02-10 21:26:06 +0000 | |||
204 | +++ lava_scheduler_app/models.py 2013-06-20 11:53:35 +0000 | |||
205 | @@ -2,6 +2,7 @@ | |||
206 | 2 | import os | 2 | import os |
207 | 3 | import simplejson | 3 | import simplejson |
208 | 4 | import urlparse | 4 | import urlparse |
209 | 5 | import copy | ||
210 | 5 | 6 | ||
211 | 6 | from django.conf import settings | 7 | from django.conf import settings |
212 | 7 | from django.contrib.auth.models import User | 8 | from django.contrib.auth.models import User |
213 | @@ -249,6 +250,12 @@ | |||
214 | 249 | 250 | ||
215 | 250 | id = models.AutoField(primary_key=True) | 251 | id = models.AutoField(primary_key=True) |
216 | 251 | 252 | ||
217 | 253 | sub_id = models.CharField( | ||
218 | 254 | verbose_name = _(u"Sub ID"), | ||
219 | 255 | blank = True, | ||
220 | 256 | max_length = 200 | ||
221 | 257 | ) | ||
222 | 258 | |||
223 | 252 | submitter = models.ForeignKey( | 259 | submitter = models.ForeignKey( |
224 | 253 | User, | 260 | User, |
225 | 254 | verbose_name = _(u"Submitter"), | 261 | verbose_name = _(u"Submitter"), |
226 | @@ -398,9 +405,13 @@ | |||
227 | 398 | elif 'device_type' in job_data: | 405 | elif 'device_type' in job_data: |
228 | 399 | target = None | 406 | target = None |
229 | 400 | device_type = DeviceType.objects.get(name=job_data['device_type']) | 407 | device_type = DeviceType.objects.get(name=job_data['device_type']) |
230 | 408 | elif 'device_group' in job_data: | ||
231 | 409 | target = None | ||
232 | 410 | device_type = None | ||
233 | 401 | else: | 411 | else: |
234 | 402 | raise JSONDataError( | 412 | raise JSONDataError( |
236 | 403 | "Neither 'target' nor 'device_type' found in job data.") | 413 | "No 'target' or 'device_type' or 'device_group' are found " |
237 | 414 | "in job data.") | ||
238 | 404 | 415 | ||
239 | 405 | priorities = dict([(j.upper(), i) for i, j in cls.PRIORITY_CHOICES]) | 416 | priorities = dict([(j.upper(), i) for i, j in cls.PRIORITY_CHOICES]) |
240 | 406 | priority = cls.MEDIUM | 417 | priority = cls.MEDIUM |
241 | @@ -463,15 +474,14 @@ | |||
242 | 463 | tags.append(Tag.objects.get(name=tag_name)) | 474 | tags.append(Tag.objects.get(name=tag_name)) |
243 | 464 | except Tag.DoesNotExist: | 475 | except Tag.DoesNotExist: |
244 | 465 | raise JSONDataError("tag %r does not exist" % tag_name) | 476 | raise JSONDataError("tag %r does not exist" % tag_name) |
245 | 477 | |||
246 | 466 | job = TestJob( | 478 | job = TestJob( |
247 | 467 | definition=json_data, submitter=submitter, | 479 | definition=json_data, submitter=submitter, |
248 | 468 | requested_device=target, requested_device_type=device_type, | 480 | requested_device=target, requested_device_type=device_type, |
249 | 469 | description=job_name, health_check=health_check, user=user, | 481 | description=job_name, health_check=health_check, user=user, |
250 | 470 | group=group, is_public=is_public, priority=priority) | 482 | group=group, is_public=is_public, priority=priority) |
251 | 471 | job.save() | 483 | job.save() |
255 | 472 | for tag in tags: | 484 | return job.id |
253 | 473 | job.tags.add(tag) | ||
254 | 474 | return job | ||
256 | 475 | 485 | ||
257 | 476 | def _can_admin(self, user): | 486 | def _can_admin(self, user): |
258 | 477 | """ used to check for things like if the user can cancel or annotate | 487 | """ used to check for things like if the user can cancel or annotate |
259 | 478 | 488 | ||
260 | === modified file 'lava_scheduler_app/views.py' | |||
261 | --- lava_scheduler_app/views.py 2013-05-02 09:00:04 +0000 | |||
262 | +++ lava_scheduler_app/views.py 2013-06-20 11:53:35 +0000 | |||
263 | @@ -74,10 +74,16 @@ | |||
264 | 74 | 74 | ||
265 | 75 | 75 | ||
266 | 76 | def pklink(record): | 76 | def pklink(record): |
267 | 77 | job_id = record.pk | ||
268 | 78 | try: | ||
269 | 79 | if record.sub_id: | ||
270 | 80 | job_id = record.sub_id | ||
271 | 81 | except: | ||
272 | 82 | pass | ||
273 | 77 | return mark_safe( | 83 | return mark_safe( |
274 | 78 | '<a href="%s">%s</a>' % ( | 84 | '<a href="%s">%s</a>' % ( |
275 | 79 | record.get_absolute_url(), | 85 | record.get_absolute_url(), |
277 | 80 | escape(record.pk))) | 86 | escape(job_id))) |
278 | 81 | 87 | ||
279 | 82 | 88 | ||
280 | 83 | class IDLinkColumn(Column): | 89 | class IDLinkColumn(Column): |
281 | @@ -100,13 +106,13 @@ | |||
282 | 100 | 106 | ||
283 | 101 | 107 | ||
284 | 102 | def all_jobs_with_device_sort(): | 108 | def all_jobs_with_device_sort(): |
286 | 103 | return TestJob.objects.select_related( | 109 | jobs = TestJob.objects.select_related( |
287 | 104 | "actual_device", "requested_device", "requested_device_type", | 110 | "actual_device", "requested_device", "requested_device_type", |
288 | 105 | "submitter", "user", "group").extra( | 111 | "submitter", "user", "group").extra( |
289 | 106 | select={ | 112 | select={ |
290 | 107 | 'device_sort': 'coalesce(actual_device_id, requested_device_id, requested_device_type_id)' | 113 | 'device_sort': 'coalesce(actual_device_id, requested_device_id, requested_device_type_id)' |
291 | 108 | }).all() | 114 | }).all() |
293 | 109 | 115 | return jobs.order_by('submit_time') | |
294 | 110 | 116 | ||
295 | 111 | 117 | ||
296 | 112 | class JobTable(DataTablesTable): | 118 | class JobTable(DataTablesTable): |
297 | @@ -126,7 +132,7 @@ | |||
298 | 126 | else: | 132 | else: |
299 | 127 | return '' | 133 | return '' |
300 | 128 | 134 | ||
302 | 129 | id = RestrictedIDLinkColumn() | 135 | sub_id = RestrictedIDLinkColumn() |
303 | 130 | status = Column() | 136 | status = Column() |
304 | 131 | priority = Column() | 137 | priority = Column() |
305 | 132 | device = Column(accessor='device_sort') | 138 | device = Column(accessor='device_sort') |
306 | @@ -137,7 +143,7 @@ | |||
307 | 137 | duration = Column() | 143 | duration = Column() |
308 | 138 | 144 | ||
309 | 139 | datatable_opts = { | 145 | datatable_opts = { |
311 | 140 | 'aaSorting': [[0, 'desc']], | 146 | 'aaSorting': [[6, 'desc']], |
312 | 141 | } | 147 | } |
313 | 142 | searchable_columns=['description'] | 148 | searchable_columns=['description'] |
314 | 143 | 149 | ||
315 | 144 | 150 | ||
316 | === modified file 'lava_scheduler_daemon/dbjobsource.py' | |||
317 | --- lava_scheduler_daemon/dbjobsource.py 2013-06-13 14:53:25 +0000 | |||
318 | +++ lava_scheduler_daemon/dbjobsource.py 2013-06-20 11:53:35 +0000 | |||
319 | @@ -103,6 +103,68 @@ | |||
320 | 103 | def getBoardList(self): | 103 | def getBoardList(self): |
321 | 104 | return self.deferForDB(self.getBoardList_impl) | 104 | return self.deferForDB(self.getBoardList_impl) |
322 | 105 | 105 | ||
323 | 106 | def _fix_device(self, device, job): | ||
324 | 107 | """Associate an available/idle DEVICE to the given JOB. | ||
325 | 108 | |||
326 | 109 | Returns the job with actual_device set to DEVICE. | ||
327 | 110 | |||
328 | 111 | If we are unable to grab the DEVICE then we return None. | ||
329 | 112 | """ | ||
330 | 113 | DeviceStateTransition.objects.create( | ||
331 | 114 | created_by=None, device=device, old_state=device.status, | ||
332 | 115 | new_state=Device.RUNNING, message=None, job=job).save() | ||
333 | 116 | device.status = Device.RUNNING | ||
334 | 117 | device.current_job = job | ||
335 | 118 | try: | ||
336 | 119 | # The unique constraint on current_job may cause this to | ||
337 | 120 | # fail in the case of concurrent requests for different | ||
338 | 121 | # boards grabbing the same job. If there are concurrent | ||
339 | 122 | # requests for the *same* board they may both return the | ||
340 | 123 | # same job -- this is an application level bug though. | ||
341 | 124 | device.save() | ||
342 | 125 | except IntegrityError: | ||
343 | 126 | self.logger.info( | ||
344 | 127 | "job %s has been assigned to another board -- rolling back", | ||
345 | 128 | job.id) | ||
346 | 129 | transaction.rollback() | ||
347 | 130 | return None | ||
348 | 131 | else: | ||
349 | 132 | job.actual_device = device | ||
350 | 133 | job.save() | ||
351 | 134 | transaction.commit() | ||
352 | 135 | return job | ||
353 | 136 | |||
354 | 137 | def getJobList_impl(self): | ||
355 | 138 | jobs = TestJob.objects.all().filter(status=TestJob.SUBMITTED) | ||
356 | 139 | job_list = [] | ||
357 | 140 | devices = None | ||
358 | 141 | |||
359 | 142 | for job in jobs: | ||
360 | 143 | if job.actual_device: | ||
361 | 144 | job_list.append(job) | ||
362 | 145 | elif job.requested_device: | ||
363 | 146 | self.logger.info("Checking Requested Device") | ||
364 | 147 | devices = Device.objects.all().filter( | ||
365 | 148 | hostname=job.requested_device.hostname, | ||
366 | 149 | status=Device.IDLE) | ||
367 | 150 | elif job.requested_device_type: | ||
368 | 151 | self.logger.info("Checking Requested Device Type") | ||
369 | 152 | devices = Device.objects.all().filter( | ||
370 | 153 | device_type=job.requested_device_type, | ||
371 | 154 | status=Device.IDLE) | ||
372 | 155 | else: | ||
373 | 156 | continue | ||
374 | 157 | if devices: | ||
375 | 158 | device = devices[0] | ||
376 | 159 | job = self._fix_device(device, job) | ||
377 | 160 | if job: | ||
378 | 161 | job_list.append(job) | ||
379 | 162 | |||
380 | 163 | return job_list | ||
381 | 164 | |||
382 | 165 | def getJobList(self): | ||
383 | 166 | return self.deferForDB(self.getJobList_impl) | ||
384 | 167 | |||
385 | 106 | def _get_json_data(self, job): | 168 | def _get_json_data(self, job): |
386 | 107 | json_data = simplejson.loads(job.definition) | 169 | json_data = simplejson.loads(job.definition) |
387 | 108 | json_data['target'] = job.actual_device.hostname | 170 | json_data['target'] = job.actual_device.hostname |
388 | @@ -229,6 +291,20 @@ | |||
389 | 229 | def getJobForBoard(self, board_name): | 291 | def getJobForBoard(self, board_name): |
390 | 230 | return self.deferForDB(self.getJobForBoard_impl, board_name) | 292 | return self.deferForDB(self.getJobForBoard_impl, board_name) |
391 | 231 | 293 | ||
392 | 294 | def getJobDetails_impl(self, job): | ||
393 | 295 | job.status = TestJob.RUNNING | ||
394 | 296 | job.start_time = datetime.datetime.utcnow() | ||
395 | 297 | shutil.rmtree(job.output_dir, ignore_errors=True) | ||
396 | 298 | job.log_file.save('job-%s.log' % job.id, ContentFile(''), save=False) | ||
397 | 299 | job.submit_token = AuthToken.objects.create(user=job.submitter) | ||
398 | 300 | job.save() | ||
399 | 301 | json_data = self._get_json_data(job) | ||
400 | 302 | transaction.commit() | ||
401 | 303 | return json_data | ||
402 | 304 | |||
403 | 305 | def getJobDetails(self, job): | ||
404 | 306 | return self.deferForDB(self.getJobDetails_impl, job) | ||
405 | 307 | |||
406 | 232 | def getOutputDirForJobOnBoard_impl(self, board_name): | 308 | def getOutputDirForJobOnBoard_impl(self, board_name): |
407 | 233 | device = Device.objects.get(hostname=board_name) | 309 | device = Device.objects.get(hostname=board_name) |
408 | 234 | job = device.current_job | 310 | job = device.current_job |
409 | 235 | 311 | ||
410 | === added file 'lava_scheduler_daemon/job.py' | |||
411 | --- lava_scheduler_daemon/job.py 1970-01-01 00:00:00 +0000 | |||
412 | +++ lava_scheduler_daemon/job.py 2013-06-20 11:53:35 +0000 | |||
413 | @@ -0,0 +1,79 @@ | |||
414 | 1 | # Copyright (C) 2013 Linaro Limited | ||
415 | 2 | # | ||
416 | 3 | # Author: Senthil Kumaran <senthil.kumaran@linaro.org> | ||
417 | 4 | # | ||
418 | 5 | # This file is part of LAVA Scheduler. | ||
419 | 6 | # | ||
420 | 7 | # LAVA Scheduler is free software: you can redistribute it and/or modify it | ||
421 | 8 | # under the terms of the GNU Affero General Public License version 3 as | ||
422 | 9 | # published by the Free Software Foundation | ||
423 | 10 | # | ||
424 | 11 | # LAVA Scheduler is distributed in the hope that it will be useful, but | ||
425 | 12 | # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY | ||
426 | 13 | # or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | ||
427 | 14 | # more details. | ||
428 | 15 | # | ||
429 | 16 | # You should have received a copy of the GNU Affero General Public License | ||
430 | 17 | # along with LAVA Scheduler. If not, see <http://www.gnu.org/licenses/>. | ||
431 | 18 | |||
432 | 19 | import logging | ||
433 | 20 | |||
434 | 21 | from twisted.internet import defer | ||
435 | 22 | from lava_scheduler_daemon.board import MonitorJob, catchall_errback | ||
436 | 23 | |||
437 | 24 | class NewJob(object): | ||
438 | 25 | job_cls = MonitorJob | ||
439 | 26 | |||
440 | 27 | def __init__(self, source, job, dispatcher, reactor, daemon_options, | ||
441 | 28 | job_cls=None): | ||
442 | 29 | self.source = source | ||
443 | 30 | self.dispatcher = dispatcher | ||
444 | 31 | self.reactor = reactor | ||
445 | 32 | self.daemon_options = daemon_options | ||
446 | 33 | self.job = job | ||
447 | 34 | self.board_name = job.actual_device.hostname | ||
448 | 35 | if job_cls is not None: | ||
449 | 36 | self.job_cls = job_cls | ||
450 | 37 | self.running_job = None | ||
451 | 38 | self.logger = logging.getLogger(__name__ + '.NewJob.' + str(job.id)) | ||
452 | 39 | |||
453 | 40 | def start(self): | ||
454 | 41 | self.logger.debug("processing job") | ||
455 | 42 | if self.job is None: | ||
456 | 43 | self.logger.debug("no job found for processing") | ||
457 | 44 | return | ||
458 | 45 | self.source.getJobDetails(self.job).addCallbacks( | ||
459 | 46 | self._startJob, self._ebStartJob) | ||
460 | 47 | |||
461 | 48 | def _startJob(self, job_data): | ||
462 | 49 | if job_data is None: | ||
463 | 50 | self.logger.debug("no job found") | ||
464 | 51 | return | ||
465 | 52 | self.logger.info("starting job %r", job_data) | ||
466 | 53 | |||
467 | 54 | self.running_job = self.job_cls( | ||
468 | 55 | job_data, self.dispatcher, self.source, self.board_name, | ||
469 | 56 | self.reactor, self.daemon_options) | ||
470 | 57 | d = self.running_job.run() | ||
471 | 58 | d.addCallbacks(self._cbJobFinished, self._ebJobFinished) | ||
472 | 59 | |||
473 | 60 | def _ebStartJob(self, result): | ||
474 | 61 | self.logger.error( | ||
475 | 62 | '%s: %s\n%s', result.type.__name__, result.value, | ||
476 | 63 | result.getTraceback()) | ||
477 | 64 | return | ||
478 | 65 | |||
479 | 66 | def stop(self): | ||
480 | 67 | self.logger.debug("stopping") | ||
481 | 68 | |||
482 | 69 | if self.running_job is not None: | ||
483 | 70 | self.logger.debug("job running; deferring stop") | ||
484 | 71 | else: | ||
485 | 72 | self.logger.debug("stopping immediately") | ||
486 | 73 | return defer.succeed(None) | ||
487 | 74 | |||
488 | 75 | def _ebJobFinished(self, result): | ||
489 | 76 | self.logger.exception(result.value) | ||
490 | 77 | |||
491 | 78 | def _cbJobFinished(self, result): | ||
492 | 79 | self.running_job = None | ||
493 | 0 | 80 | ||
494 | === modified file 'lava_scheduler_daemon/service.py' | |||
495 | --- lava_scheduler_daemon/service.py 2012-12-03 05:03:38 +0000 | |||
496 | +++ lava_scheduler_daemon/service.py 2013-06-20 11:53:35 +0000 | |||
497 | @@ -5,6 +5,7 @@ | |||
498 | 5 | from twisted.internet.task import LoopingCall | 5 | from twisted.internet.task import LoopingCall |
499 | 6 | 6 | ||
500 | 7 | from lava_scheduler_daemon.board import Board, catchall_errback | 7 | from lava_scheduler_daemon.board import Board, catchall_errback |
501 | 8 | from lava_scheduler_daemon.job import NewJob | ||
502 | 8 | 9 | ||
503 | 9 | 10 | ||
504 | 10 | class BoardSet(Service): | 11 | class BoardSet(Service): |
505 | @@ -56,3 +57,35 @@ | |||
506 | 56 | self.logger.info( | 57 | self.logger.info( |
507 | 57 | "waiting for %s boards", len(self.boards) - len(dead_boards)) | 58 | "waiting for %s boards", len(self.boards) - len(dead_boards)) |
508 | 58 | return defer.gatherResults(ds) | 59 | return defer.gatherResults(ds) |
509 | 60 | |||
510 | 61 | |||
511 | 62 | class JobQueue(Service): | ||
512 | 63 | |||
513 | 64 | def __init__(self, source, dispatcher, reactor, daemon_options): | ||
514 | 65 | self.logger = logging.getLogger(__name__ + '.JobQueue') | ||
515 | 66 | self.source = source | ||
516 | 67 | self.dispatcher = dispatcher | ||
517 | 68 | self.reactor = reactor | ||
518 | 69 | self.daemon_options = daemon_options | ||
519 | 70 | self._check_job_call = LoopingCall(self._checkJobs) | ||
520 | 71 | self._check_job_call.clock = reactor | ||
521 | 72 | |||
522 | 73 | def _checkJobs(self): | ||
523 | 74 | self.logger.debug("Refreshing jobs") | ||
524 | 75 | return self.source.getJobList().addCallback( | ||
525 | 76 | self._cbCheckJobs).addErrback(catchall_errback(self.logger)) | ||
526 | 77 | |||
527 | 78 | def _cbCheckJobs(self, job_list): | ||
528 | 79 | for job in job_list: | ||
529 | 80 | self.logger.debug("Found job: %d" % job.id) | ||
530 | 81 | new_job = NewJob(self.source, job, self.dispatcher, self.reactor, | ||
531 | 82 | self.daemon_options) | ||
532 | 83 | self.logger.info("Starting Job: %d " % job.id) | ||
533 | 84 | new_job.start() | ||
534 | 85 | |||
535 | 86 | def startService(self): | ||
536 | 87 | self._check_job_call.start(20) | ||
537 | 88 | |||
538 | 89 | def stopService(self): | ||
539 | 90 | self._check_job_call.stop() | ||
540 | 91 | return None |
Approved