Merge lp:~cjohnston/qa-dashboard/smoke-artifact-admin into lp:qa-dashboard

[Chris Johnston]

Adds a smoke only artifact view in admin

Unmerged revisions

576. By Chris Johnston on 2013-08-16

Add 'smoke only' artifacts using common.models Artifact

=== modified file 'common/admin.py'
--- common/admin.py	2013-08-09 03:20:35 +0000
+++ common/admin.py	2013-08-16 01:01:50 +0000
@@ -17,6 +17,7 @@
     Bug,
     JenkinsJob,
     JenkinsBuild,
+    Artifact,
 )
 
 from django.contrib import admin
@@ -50,6 +51,13 @@
     ]
     date_hierarchy = 'ran_at'
 
+
+class ArtifactAdmin(admin.ModelAdmin):
+    list_display = ('name', 'jenkins_build', 'publish')
+    search_fields = ['name', 'jenkins_build']
+
+
 admin.site.register(Bug, BugAdmin)
 admin.site.register(JenkinsJob, JenkinsJobAdmin)
 admin.site.register(JenkinsBuild, JenkinsBuildAdmin)
+admin.site.register(Artifact, ArtifactAdmin)

=== modified file 'common/management/__init__.py'
--- common/management/__init__.py	2013-08-02 18:56:36 +0000
+++ common/management/__init__.py	2013-08-16 01:01:50 +0000
@@ -327,6 +327,9 @@
     def _process_logs(self, build, build_date, install_data, func):
         raise NotImplementedError
 
+    def process_artifacts(self, build, install_data):
+        pass
+
     def add_result(self):
         pass
 
@@ -615,6 +618,7 @@
                 if self.install_data['process_build']:
                     self.add_job_bugs(build)
                     self.process_build(build)
+                    self.process_artifacts(build, self.install_data)
                     self.process_add_result(build)
                 self.post_process_build(build)
 

=== added file 'common/migrations/0006_add_artifacts.py'
--- common/migrations/0006_add_artifacts.py	1970-01-01 00:00:00 +0000
+++ common/migrations/0006_add_artifacts.py	2013-08-16 01:01:50 +0000
@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Artifact'
+        db.create_table('common_artifacts', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('created_at', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('updated_at', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+            ('internal', self.gf('django.db.models.fields.BooleanField')(default=True)),
+            ('publish', self.gf('django.db.models.fields.BooleanField')(default=True)),
+            ('url', self.gf('django.db.models.fields.URLField')(max_length=200)),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=4096)),
+            ('jenkins_build', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['common.JenkinsBuild'])),
+        ))
+        db.send_create_signal('common', ['Artifact'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'Artifact'
+        db.delete_table('common_artifacts')
+
+
+    models = {
+        'common.artifact': {
+            'Meta': {'object_name': 'Artifact', 'db_table': "'common_artifacts'"},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'internal': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'jenkins_build': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['common.JenkinsBuild']"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '4096'}),
+            'publish': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '200'})
+        },
+        'common.bug': {
+            'Meta': {'object_name': 'Bug', 'db_table': "'bugs'"},
+            'assignee': ('django.db.models.fields.CharField', [], {'max_length': '4096', 'null': 'True'}),
+            'bug_no': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'importance': ('django.db.models.fields.CharField', [], {'default': "u'unknown'", 'max_length': '4096'}),
+            'internal': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'project': ('django.db.models.fields.CharField', [], {'max_length': '4096', 'null': 'True'}),
+            'publish': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "u'unknown'", 'max_length': '4096'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '4096', 'null': 'True'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        'common.jenkinsbuild': {
+            'Meta': {'unique_together': "(('job', 'build_number'),)", 'object_name': 'JenkinsBuild', 'db_table': "'jenkins_builds'"},
+            'bugs': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'build_bugs'", 'symmetrical': 'False', 'to': "orm['common.Bug']"}),
+            'build_description': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024'}),
+            'build_number': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'failed': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'internal': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'job': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['common.JenkinsJob']"}),
+            'publish': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'ran_at': ('django.db.models.fields.DateTimeField', [], {}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        'common.jenkinsjob': {
+            'Meta': {'object_name': 'JenkinsJob', 'db_table': "'jenkins_jobs'"},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'internal': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'}),
+            'publish': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '200'})
+        }
+    }
+
+    complete_apps = ['common']
\ No newline at end of file

=== modified file 'common/models.py'
--- common/models.py	2013-07-02 21:04:09 +0000
+++ common/models.py	2013-08-16 01:01:50 +0000
@@ -150,3 +150,14 @@
         """ Get number of bugs for this build. """
 
         return self.bugs.count()
+
+
+class Artifact(DashboardBaseModel):
+    """ Add artifacts """
+
+    class Meta:
+        db_table = "common_artifacts"
+
+    url = models.URLField()
+    name = models.CharField(max_length=4096)
+    jenkins_build = models.ForeignKey(JenkinsBuild)

=== modified file 'smokeng/admin.py'
--- smokeng/admin.py	2013-07-10 16:56:19 +0000
+++ smokeng/admin.py	2013-08-16 01:01:50 +0000
@@ -13,11 +13,14 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from common.models import Artifact
+
 from smokeng.models import (
     SmokeImage,
     SmokeResult,
     SmokeResultBug,
     SmokeTestResult,
+    SmokeArtifact,
 )
 
 from django.contrib import admin
@@ -59,7 +62,21 @@
     list_display = ('result', 'testcase', 'returncode')
     ordering = ['result']
 
+
+class SmokeArtifactAdmin(admin.ModelAdmin):
+    list_display = ('name', 'jenkins_build', 'publish')
+    search_fields = ['name', 'jenkins_build']
+
+    def queryset(self, request):
+        results = SmokeResult.objects.all().values('jenkins_build')
+        builds = []
+        for result in results:
+            builds.append(result['jenkins_build'])
+        artifacts = SmokeArtifact.objects.filter(jenkins_build__in=builds)
+        return artifacts
+
 admin.site.register(SmokeImage, SmokeImageAdmin)
 admin.site.register(SmokeResult, SmokeResultAdmin)
 admin.site.register(SmokeResultBug, SmokeResultBugAdmin)
 admin.site.register(SmokeTestResult, SmokeTestResultAdmin)
+admin.site.register(SmokeArtifact, SmokeArtifactAdmin)

=== modified file 'smokeng/management/commands/jenkins_pull_smokeng.py'
--- smokeng/management/commands/jenkins_pull_smokeng.py	2013-08-07 19:05:49 +0000
+++ smokeng/management/commands/jenkins_pull_smokeng.py	2013-08-16 01:01:50 +0000
@@ -25,7 +25,7 @@
 
 from common.utah_parser import UTAHParser, ParserError
 from common.utils import regexes
-from common.models import Bug
+from common.models import Bug, Artifact
 
 from smokeng.models import (
     SmokeImage,
@@ -107,6 +107,31 @@
             total=total_count,
         )
 
+    def process_artifacts(self, build, install_data):
+        artifact_regex = re.compile(
+            ur'^(clientlogs\/|)(.*.yaml|.*.log$|media-info|.*.props|.*.crash)'
+        )
+        for artifact in build['artifacts']:
+            path = artifact['relativePath']
+            if artifact_regex.match(path):
+                logfile_path = "{}{}/artifact/{}".format(
+                    self.install_data['jenkins_build'].job.url,
+                    self.install_data['jenkins_build'].build_number,
+                    artifact['relativePath'],
+                )
+                artifact, new_artifact = Artifact.objects.get_or_create(
+                    url=logfile_path,
+                    name=artifact['fileName'],
+                    jenkins_build=install_data['jenkins_build'],
+                )
+
+                if new_artifact:
+                    logging.info(
+                        "Adding %s for jenkins build %s",
+                        artifact['fileName'],
+                        install_data['jenkins_build'],
+                    )
+
     def _process_logs(self, build, build_date, install_data, func):
 
         if not hasattr(func, '__call__'):

=== modified file 'smokeng/models.py'
--- smokeng/models.py	2013-07-30 21:05:41 +0000
+++ smokeng/models.py	2013-08-16 01:01:50 +0000
@@ -19,6 +19,7 @@
     DashboardBaseModel,
     Bug,
     JenkinsBuild,
+    Artifact,
 )
 
 
@@ -164,3 +165,9 @@
     returncode = models.IntegerField()
     stderr = models.TextField(null=True)
     stdout = models.TextField(null=True)
+
+
+class SmokeArtifact(Artifact):
+    class Meta:
+        proxy = True
+        app_label = 'smokeng'