Merge lp:~chris.gagnon/qa-coverage-dashboard/only-get-last-build-from-jenkins into lp:qa-coverage-dashboard

[Chris Gagnon]

[lp:~chris.gagnon/qa-coverage-dashboard/only-get-last-build-from-jenkins]

770. By Chris Gagnon on 2014-05-13

clean up if statement

771. By Chris Gagnon on 2014-05-13

Update doc string

[Allan LeSage (allanlesage)]

I like the tests, they're good. I personally like when assertions go like this: assertEqual(42, my_var) , but apparently there's no consensus on that. A couple of things:

* Can we not test those integers as strings?
* For ll. 68-71, I think you can not do the conditional and just pass through the var.

Otherwise looks good.

review: Needs Fixing

[lp:~chris.gagnon/qa-coverage-dashboard/only-get-last-build-from-jenkins]

772. By Chris Gagnon on 2014-05-19

dont convert decimal to string in tests, don't use if statment in option

773. By Chris Gagnon on 2014-05-19

fix text conflict

[Allan LeSage (allanlesage)]

I certify this commit.

review: Approve

=== modified file 'gaps/management/commands/c2dconfigutils/cu2dOutputCi.py'
--- gaps/management/commands/c2dconfigutils/cu2dOutputCi.py	2014-03-09 01:05:53 +0000
+++ gaps/management/commands/c2dconfigutils/cu2dOutputCi.py	2014-05-19 13:24:17 +0000
@@ -114,9 +114,10 @@
                          'parents': None})
         self.master_list.append(job_list)
 
-    def __call__(self, default_config_path):
+    def __call__(self, default_config_path, just_last_build=False):
         """Entry point for cu2d-output-ci"""
         self.default_config_path = default_config_path
+        self.just_last_build = just_last_build
         set_logging(False)
         local_path = os.path.dirname(os.path.abspath(__file__))
         pocket_list = os.listdir('stacks')
@@ -159,10 +160,13 @@
                                                 job in s for s in added_job)
                                                 and not str(job).startswith(
                                                     'generic')):
-                                            add.records(stack['name'],
-                                                        param.value,
-                                                        job,
-                                                        'coverage.xml',
-                                                        )
+                                            add.records(
+                                                stack['name'],
+                                                param.value,
+                                                job,
+                                                'coverage.xml',
+                                                self.just_last_build,
+                                            )
+
                                             added_job.append(job)
                                             print('job {}'.format(job))

=== modified file 'gaps/management/commands/jenkins_pull_coverage.py'
--- gaps/management/commands/jenkins_pull_coverage.py	2014-02-25 15:31:32 +0000
+++ gaps/management/commands/jenkins_pull_coverage.py	2014-05-19 13:24:17 +0000
@@ -3,6 +3,7 @@
 import os
 
 from django.core.management.base import BaseCommand
+from optparse import make_option
 
 logger = logging.getLogger('qa_dashboard')
 
@@ -10,6 +11,15 @@
 class Command(BaseCommand):
     help = 'add coverage artifacts to database'
 
+    option_list = BaseCommand.option_list + (
+        make_option(
+            '--last',
+            action='store_true',
+            dest='just_last_build',
+            help='pull in just the last build',
+        ),
+    )
+
     def handle(self, *args, **options):
         BASEDIR = os.path.dirname(__file__)
 
@@ -20,4 +30,7 @@
         from c2dconfigutils.cu2dOutputCi import OutputCi
 
         get_coverage = OutputCi()
-        get_coverage(BASEDIR)
+        #if options['just_last_build']:
+        #    get_coverage(BASEDIR, True)
+        #else:
+        get_coverage(BASEDIR, options['just_last_build'])

=== modified file 'gaps/tests/test_jenkins_pull_coverage.py'
--- gaps/tests/test_jenkins_pull_coverage.py	2014-04-18 22:55:02 +0000
+++ gaps/tests/test_jenkins_pull_coverage.py	2014-05-19 13:24:17 +0000
@@ -1,7 +1,3 @@
-import datetime
-
-import common
-
 from django.test import TestCase
 
 from gaps.models import (
@@ -10,22 +6,39 @@
     CoverageProject,
     CoverageStack,
 )
-from gaps.util import add, jenkins_pull
+from gaps.util import add
 
-from mock import Mock
+import datetime
+import decimal
+import mock
 from os import path
 
 
+def make_decimal(number):
+    c = decimal.getcontext()
+    c.prec = 3
+    dec = c.create_decimal(number)
+    return dec
+
+
 class TestJenkinsPullCoverage(TestCase):
     def setUp(self):
         self.url_list = [
             {
-                'url': 'http://jenkins.qa.ubuntu.com/job/libqtdbustest-trusty'
+                'url': 'http://jenkins.fake/job/libqtdbustest-trusty'
+                '-i386-ci/11/artifact/work/results/coverage.xml',
+                'build_id': 11,
+                'ran_at': datetime.datetime(
+                    2014, 4, 18, 17, 31, 24)
+            },
+            {
+                'url': 'http://jenkins.fake/job/libqtdbustest-trusty'
                 '-i386-ci/10/artifact/work/results/coverage.xml',
                 'build_id': 10,
                 'ran_at': datetime.datetime(
                     2014, 3, 18, 17, 31, 24)
             }
+
         ]
 
         localdir = path.dirname(__file__)
@@ -38,37 +51,99 @@
         """Verify jenkins_get method is in common.managment"""
         from common.management import jenkins_get  # noqa
 
-    def test_jenkins_pull_coverage_add_to_database(self):
-        jenkins_pull.url_artifact_list = Mock(return_value=self.url_list)
-        common.management.jenkins_get = Mock(return_value=self.jdata)
-        add.records(
-            'qa',
-            'libqtdbustest',
-            'libqtdbustest-trusty-amd64-ci',
-            'coverage.xml',
-        )
-
-        data_result = CoverageData.objects.filter(
-            url__contains='coverage.xml').latest('coverage_build')
-
-        self.assertEquals(
-            data_result.url,
-            'http://jenkins.qa.ubuntu.com/job/libqtdbustest-trusty-i386-ci'
-            '/10/artifact/work/results/coverage.xml'
-        )
-        self.assertEquals(data_result.name, 'coverage.xml')
-        self.assertEquals(data_result.xml_file, '')
-        self.assertEquals(str(data_result.branch_coverage), '0.514')
-        self.assertEquals(str(data_result.line_coverage), '0.884')
-        self.assertEquals(str(data_result.hit_count), '107')
-        self.assertEquals(str(data_result.line_count), '121')
-        self.assertEquals(str(data_result.taken_count), '75')
-        self.assertEquals(str(data_result.total_count), '146')
-
-        build_result = CoverageBuild.objects.filter(
-            build_number='10').latest('ran_at')
-
-        self.assertEquals(str(build_result.build_number), '10')
+    @mock.patch('gaps.util.add.jenkins_get')
+    @mock.patch('gaps.util.add.jenkins_pull')
+    def test_jenkins_pull_coverage_add_to_database(self, mock_pull, mock_get):
+        mock_get.return_value = self.jdata
+        mock_pull.url_artifact_list.return_value = self.url_list
+        add.records(
+            'qa',
+            'libqtdbustest',
+            'libqtdbustest-trusty-amd64-ci',
+            'coverage.xml',
+        )
+
+        # There must be 2 records when getting all the builds
+        self.assertEquals(
+            CoverageData.objects.count(),
+            2
+        )
+
+        data_result = CoverageData.objects.filter(
+            url__contains='coverage.xml').latest('coverage_build')
+
+        self.assertEquals(
+            data_result.url,
+            'http://jenkins.fake/job/libqtdbustest-trusty-i386-ci'
+            '/11/artifact/work/results/coverage.xml'
+        )
+        self.assertEquals(data_result.name, 'coverage.xml')
+        self.assertEquals(data_result.xml_file, '')
+
+        self.assertEquals(data_result.branch_coverage, make_decimal(0.514))
+        self.assertEquals(data_result.line_coverage, make_decimal(.884))
+        self.assertEquals(data_result.hit_count, 107)
+        self.assertEquals(data_result.line_count, 121)
+        self.assertEquals(data_result.taken_count, 75)
+        self.assertEquals(data_result.total_count, 146)
+
+        build_result = CoverageBuild.objects.filter(
+            build_number='11').latest('ran_at')
+
+        self.assertEquals(str(build_result.build_number), '11')
+        self.assertEqual(build_result.build_description, 'None')
+
+        project_result = CoverageProject.objects.filter(
+            name='libqtdbustest').latest('name')
+
+        self.assertEquals(project_result.name, 'libqtdbustest')
+
+        stack_result = CoverageStack.objects.filter(
+            name='qa').latest('name')
+
+        self.assertEquals(stack_result.name, 'qa')
+
+    @mock.patch('gaps.util.add.jenkins_get')
+    @mock.patch('gaps.util.add.jenkins_pull')
+    def test_jenkins_pull_coverage_last_add_to_database(self,
+                                                        mock_pull,
+                                                        mock_get):
+        mock_get.return_value = self.jdata
+        mock_pull.url_artifact_list.return_value = self.url_list
+        add.records(
+            'qa',
+            'libqtdbustest',
+            'libqtdbustest-trusty-amd64-ci',
+            'coverage.xml',
+            True
+        )
+        # There must be only one record when getting the last build
+        self.assertEquals(
+            CoverageData.objects.count(),
+            1
+        )
+
+        data_result = CoverageData.objects.filter(
+            url__contains='coverage.xml').latest('coverage_build')
+
+        self.assertEquals(
+            data_result.url,
+            'http://jenkins.fake/job/libqtdbustest-trusty-i386-ci'
+            '/11/artifact/work/results/coverage.xml'
+        )
+        self.assertEquals(data_result.name, 'coverage.xml')
+        self.assertEquals(data_result.xml_file, '')
+        self.assertEquals(data_result.branch_coverage, make_decimal(0.514))
+        self.assertEquals(data_result.line_coverage, make_decimal(0.884))
+        self.assertEquals(data_result.hit_count, 107)
+        self.assertEquals(data_result.line_count, 121)
+        self.assertEquals(data_result.taken_count, 75)
+        self.assertEquals(data_result.total_count, 146)
+
+        build_result = CoverageBuild.objects.filter(
+            build_number='11').latest('ran_at')
+
+        self.assertEquals(str(build_result.build_number), '11')
         self.assertEqual(build_result.build_description, 'None')
 
         project_result = CoverageProject.objects.filter(

=== modified file 'gaps/util/add.py'
--- gaps/util/add.py	2014-04-18 21:15:24 +0000
+++ gaps/util/add.py	2014-05-19 13:24:17 +0000
@@ -13,13 +13,15 @@
 logger = logging.getLogger('qa_dashboard')
 
 
-def records(stack_name, project_name, jenkins_job_name, file_name):
+def records(stack_name, project_name, jenkins_job_name,
+            file_name, just_last_build=False):
     """Adds coverage data to the database
 
     :param stack_name: stack to get artifacts from
     :param project name: name of project to get artifact from
     :param jenkins_job_name: job to get artifacts from
     :param file_name: file name of artifact
+    :param just_last_build: get just the last build from jenkins
     """
     logger.debug("getting last_build")
     last_build = [
@@ -30,14 +32,17 @@
     if last_build == []:
         last_build.append(0)
     logger.debug("last build in our database is {}".format(last_build[-1]))
-    url_list = jenkins_pull.url_artifact_list(
+    pulled_list = jenkins_pull.url_artifact_list(
         jenkins_job_name,
         file_name,
         last_build[-1]
     )
 
-    if url_list:
-        for build_info in url_list:
+    if pulled_list:
+        pulled_list.sort()
+        if just_last_build:
+            pulled_list = [pulled_list[-1]]
+        for build_info in pulled_list:
             logger.debug('build info{}'.format(build_info))
             if build_info['url'].endswith('xml'):
                 url_file = build_info['url']