Merge lp:~cjohnston/qa-dashboard/store-artifacts into lp:qa-dashboard

[Chris Johnston]

Creates 'artifacts' in the dashboard; pulls artifacts from smokeng jobs

There are an increasing number of requests that are being made for the dashboard which all point to the dashboard replacing Jenkins as the viewable front end for QA testing. Many of these requests require that the dashboard produce some sort of data or result based upon the artifacts from a job. In order to satisfy these requests more easily, it would be a good idea to give the dashboard the notion of artifacts.

[PS Jenkins bot (ps-jenkins)]

PASSED: Continuous integration, rev:575
http://s-jenkins:8080/job/dashboard-ci/154/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins:8080/job/dashboard-ci/154/rebuild

review: Approve (continuous-integration)

[lp:~cjohnston/qa-dashboard/store-artifacts]

575. By Chris Johnston on 2013-08-15

Add common.artifacts, start saving smokeng artifacts

[Andy Doan (doanac)]

review: Approve

=== modified file 'common/admin.py'
--- common/admin.py	2013-08-09 03:20:35 +0000
+++ common/admin.py	2013-08-15 21:24:23 +0000
@@ -17,6 +17,7 @@
     Bug,
     JenkinsJob,
     JenkinsBuild,
+    Artifact,
 )
 
 from django.contrib import admin
@@ -50,6 +51,13 @@
     ]
     date_hierarchy = 'ran_at'
 
+
+class ArtifactAdmin(admin.ModelAdmin):
+    list_display = ('name', 'jenkins_build', 'publish')
+    search_fields = ['name', 'jenkins_build']
+
+
 admin.site.register(Bug, BugAdmin)
 admin.site.register(JenkinsJob, JenkinsJobAdmin)
 admin.site.register(JenkinsBuild, JenkinsBuildAdmin)
+admin.site.register(Artifact, ArtifactAdmin)

=== modified file 'common/management/__init__.py'
--- common/management/__init__.py	2013-08-02 18:56:36 +0000
+++ common/management/__init__.py	2013-08-15 21:24:23 +0000
@@ -327,6 +327,9 @@
     def _process_logs(self, build, build_date, install_data, func):
         raise NotImplementedError
 
+    def process_artifacts(self, build, install_data):
+        pass
+
     def add_result(self):
         pass
 
@@ -615,6 +618,7 @@
                 if self.install_data['process_build']:
                     self.add_job_bugs(build)
                     self.process_build(build)
+                    self.process_artifacts(build, self.install_data)
                     self.process_add_result(build)
                 self.post_process_build(build)
 

=== added file 'common/migrations/0006_add_artifacts.py'
--- common/migrations/0006_add_artifacts.py	1970-01-01 00:00:00 +0000
+++ common/migrations/0006_add_artifacts.py	2013-08-15 21:24:23 +0000
@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'Artifact'
+        db.create_table('common_artifacts', (
+            ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('created_at', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)),
+            ('updated_at', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)),
+            ('internal', self.gf('django.db.models.fields.BooleanField')(default=True)),
+            ('publish', self.gf('django.db.models.fields.BooleanField')(default=True)),
+            ('url', self.gf('django.db.models.fields.URLField')(max_length=200)),
+            ('name', self.gf('django.db.models.fields.CharField')(max_length=4096)),
+            ('jenkins_build', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['common.JenkinsBuild'])),
+        ))
+        db.send_create_signal('common', ['Artifact'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'Artifact'
+        db.delete_table('common_artifacts')
+
+
+    models = {
+        'common.artifact': {
+            'Meta': {'object_name': 'Artifact', 'db_table': "'common_artifacts'"},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'internal': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'jenkins_build': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['common.JenkinsBuild']"}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '4096'}),
+            'publish': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '200'})
+        },
+        'common.bug': {
+            'Meta': {'object_name': 'Bug', 'db_table': "'bugs'"},
+            'assignee': ('django.db.models.fields.CharField', [], {'max_length': '4096', 'null': 'True'}),
+            'bug_no': ('django.db.models.fields.CharField', [], {'max_length': '255'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'importance': ('django.db.models.fields.CharField', [], {'default': "u'unknown'", 'max_length': '4096'}),
+            'internal': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'project': ('django.db.models.fields.CharField', [], {'max_length': '4096', 'null': 'True'}),
+            'publish': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'status': ('django.db.models.fields.CharField', [], {'default': "u'unknown'", 'max_length': '4096'}),
+            'title': ('django.db.models.fields.CharField', [], {'max_length': '4096', 'null': 'True'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        'common.jenkinsbuild': {
+            'Meta': {'unique_together': "(('job', 'build_number'),)", 'object_name': 'JenkinsBuild', 'db_table': "'jenkins_builds'"},
+            'bugs': ('django.db.models.fields.related.ManyToManyField', [], {'related_name': "'build_bugs'", 'symmetrical': 'False', 'to': "orm['common.Bug']"}),
+            'build_description': ('django.db.models.fields.CharField', [], {'default': "''", 'max_length': '1024'}),
+            'build_number': ('django.db.models.fields.CharField', [], {'max_length': '50'}),
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'failed': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'internal': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'job': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['common.JenkinsJob']"}),
+            'publish': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'ran_at': ('django.db.models.fields.DateTimeField', [], {}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'})
+        },
+        'common.jenkinsjob': {
+            'Meta': {'object_name': 'JenkinsJob', 'db_table': "'jenkins_jobs'"},
+            'created_at': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'internal': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '200'}),
+            'publish': ('django.db.models.fields.BooleanField', [], {'default': 'True'}),
+            'updated_at': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}),
+            'url': ('django.db.models.fields.URLField', [], {'max_length': '200'})
+        }
+    }
+
+    complete_apps = ['common']
\ No newline at end of file

=== modified file 'common/models.py'
--- common/models.py	2013-07-02 21:04:09 +0000
+++ common/models.py	2013-08-15 21:24:23 +0000
@@ -150,3 +150,14 @@
         """ Get number of bugs for this build. """
 
         return self.bugs.count()
+
+
+class Artifact(DashboardBaseModel):
+    """ Add artifacts """
+
+    class Meta:
+        db_table = "common_artifacts"
+
+    url = models.URLField()
+    name = models.CharField(max_length=4096)
+    jenkins_build = models.ForeignKey(JenkinsBuild)

=== modified file 'smokeng/management/commands/jenkins_pull_smokeng.py'
--- smokeng/management/commands/jenkins_pull_smokeng.py	2013-08-07 19:05:49 +0000
+++ smokeng/management/commands/jenkins_pull_smokeng.py	2013-08-15 21:24:23 +0000
@@ -25,7 +25,7 @@
 
 from common.utah_parser import UTAHParser, ParserError
 from common.utils import regexes
-from common.models import Bug
+from common.models import Bug, Artifact
 
 from smokeng.models import (
     SmokeImage,
@@ -107,6 +107,31 @@
             total=total_count,
         )
 
+    def process_artifacts(self, build, install_data):
+        artifact_regex = re.compile(
+            ur'^(clientlogs\/|)(.*.yaml|.*.log$|media-info|.*.props|.*.crash)'
+        )
+        for artifact in build['artifacts']:
+            path = artifact['relativePath']
+            if artifact_regex.match(path):
+                logfile_path = "{}{}/artifact/{}".format(
+                    self.install_data['jenkins_build'].job.url,
+                    self.install_data['jenkins_build'].build_number,
+                    artifact['relativePath'],
+                )
+                artifact, new_artifact = Artifact.objects.get_or_create(
+                    url=logfile_path,
+                    name=artifact['fileName'],
+                    jenkins_build=install_data['jenkins_build'],
+                )
+
+                if new_artifact:
+                    logging.info(
+                        "Adding %s for jenkins build %s",
+                        artifact['fileName'],
+                        install_data['jenkins_build'],
+                    )
+
     def _process_logs(self, build, build_date, install_data, func):
 
         if not hasattr(func, '__call__'):