Merge lp:~allanlesage/qa-coverage-dashboard/simplify-extractor-split-out-get-last-build into lp:qa-coverage-dashboard

[Allan LeSage]

Split out the get_last_build function and add a test for it individually, also chop down the olde extractor.

[lp:~allanlesage/qa-coverage-dashboard/simplify-extractor-split-out-get-last-build]

768. By Allan LeSage on 2014-05-12

Merged trunk, resolving conflicts.

[Chris Gagnon (chris.gagnon)]

looks good

[Chris Gagnon (chris.gagnon)]

review: Approve

=== modified file 'gaps/tests/__init__.py'
--- gaps/tests/__init__.py	2014-04-09 19:08:42 +0000
+++ gaps/tests/__init__.py	2014-05-13 14:26:02 +0000
@@ -4,3 +4,6 @@
     GetArtifactUrlsTestCase,
     UrlArtifactListTestCase,
 )
+from gaps.tests.test_add import (  # noqa
+    GetLastBuildTestCase,
+)

=== added file 'gaps/tests/test_add.py'
--- gaps/tests/test_add.py	1970-01-01 00:00:00 +0000
+++ gaps/tests/test_add.py	2014-05-13 14:26:02 +0000
@@ -0,0 +1,44 @@
+
+from django.test import TestCase
+
+from mock import patch, MagicMock
+
+from gaps.util.add import (
+    get_last_build_number_for_jenkins_job,
+    CoverageData,
+)
+
+
+class GetLastBuildTestCase(TestCase):
+
+    def setUp(self):
+        super(GetLastBuildTestCase, self).setUp()
+        self.coveragedata_mock = MagicMock(
+            spec=CoverageData)
+        coveragedata_patch = patch(
+            'gaps.util.add.CoverageData',
+            new=self.coveragedata_mock)
+        coveragedata_patch.start()
+        self.addCleanup(coveragedata_patch.stop)
+
+    def test_get_last_build_number_for_jenkins_job(self):
+        """The list of builds is [1, 2]."""
+        first_build = MagicMock(build_number=1)
+        second_build = MagicMock(build_number=2)
+        # ok I agree this is evil
+        self.coveragedata_mock.objects = MagicMock(
+            filter=MagicMock(
+                return_value=MagicMock(
+                    order_by=MagicMock(
+                        return_value=[
+                            first_build,
+                            second_build,
+                        ],
+                    )
+                )
+            )
+        )
+        self.assertEqual(
+            [1],
+            get_last_build_number_for_jenkins_job("what_me_workie?"),
+        )

=== modified file 'gaps/util/add.py'
--- gaps/util/add.py	2014-04-18 21:15:24 +0000
+++ gaps/util/add.py	2014-05-13 14:26:02 +0000
@@ -13,6 +13,22 @@
 logger = logging.getLogger('qa_dashboard')
 
 
+def get_last_build_number_for_jenkins_job(jenkins_job_name):
+    logger.debug("getting the number of the last_build")
+    try:
+        last_build = CoverageData.objects.filter(
+            coverage_build__job__name__exact=jenkins_job_name
+        ).order_by(
+            '-coverage_build__build_number'
+        )[0]
+        logger.debug("last build in our database is {}".format(
+            last_build.build_number)
+        )
+        return [last_build.build_number]
+    except IndexError:
+        return [0]
+
+
 def records(stack_name, project_name, jenkins_job_name, file_name):
     """Adds coverage data to the database
 
@@ -21,15 +37,7 @@
     :param jenkins_job_name: job to get artifacts from
     :param file_name: file name of artifact
     """
-    logger.debug("getting last_build")
-    last_build = [
-        j_build.coverage_build.build_number for j_build in
-        CoverageData.objects.filter(
-            coverage_build__job__name__exact=jenkins_job_name)]
-    sorted(last_build)
-    if last_build == []:
-        last_build.append(0)
-    logger.debug("last build in our database is {}".format(last_build[-1]))
+    last_build = get_last_build_number_for_jenkins_job(jenkins_job_name)
     url_list = jenkins_pull.url_artifact_list(
         jenkins_job_name,
         file_name,

=== modified file 'gaps/util/extractor.py'
--- gaps/util/extractor.py	2014-02-21 01:05:01 +0000
+++ gaps/util/extractor.py	2014-05-13 14:26:02 +0000
@@ -1,5 +1,5 @@
 # QA Dashboard
-# Copyright 2012-2013 Canonical Ltd.
+# Copyright 2012-2014 Canonical Ltd.
 
 # This program is free software: you can redistribute it and/or modify it
 # under the terms of the GNU Affero General Public License version 3, as
@@ -13,362 +13,20 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import datetime
-import re
 from lxml import etree
-from urlparse import urlsplit
 
 from common.management import jenkins_get
 
-
 """Classes and utilities to extract QA data from jenkins"""
 
 
-class BuildUrl(object):
-    def __init__(self, number=None, url=None):
-        self.number = number
-        self.url = url
-
-    def __repr__(self):
-        return '<BuildUrl: [{}, {}]>'.format(self.number, self.url)
-
-    def __cmp__(self, other):
-        return (self.number == other.number) and (self.url == other.url)
-
-
-class JobUrl(object):
-    def __init__(self, name=None, url=None, color=None):
-        self.name = name
-        self.url = url
-        self.color = color
-
-
-class Parameter(object):
-    def __init__(self, name=None, value=None, description=None):
-        self.name = name
-        self.value = value
-        self.description = description
-
-
-class JenkinsObject(object):
-    """The base class for other jenkins objects with common attributes"""
-
-    def __init__(self, url, data=None):
-        self.url = url
-        if data is None:
-            data = get_json_from_url(self.url)
-        self.data = data
-
-    def refresh(self):
-        self.data = get_json_from_url(self.url)
-
-    @property
-    def name(self):
-        return self.data['displayName']
-
-
-class JenkinsJob(JenkinsObject):
-
-    """Provides data access for a jenkins job"""
-
-    def __init__(self, url, data=None):
-        """Creates the job object"""
-        super(JenkinsJob, self).__init__(url, data)
-        self.builds = self._get_builds()
-        self.downstream_jobs = self._get_downstream_jobs()
-
-    def __str__(self):
-        return "<JenkinsJob: %s>" % (self.data)
-
-    def _get_builds(self):
-        builds = list()
-        for b in self.data['builds']:
-            build = BuildUrl(**b)
-            builds.append(build)
-        return builds
-
-    def _get_downstream_jobs(self):
-        downstream_jobs = list()
-        for j in self.data['downstreamProjects']:
-            job = JobUrl(**j)
-            downstream_jobs.append(job)
-        return downstream_jobs
-
-    def _get_build_number(self, build):
-        # A build will be None when the last*Build has not occurred yet
-        if build is None:
-            return None
-        return int(build['number'])
-
-    def refresh(self):
-        super(JenkinsJob, self).refresh()
-        self.builds = self._get_builds()
-        self.downstream_jobs = self._get_downstream_jobs()
-
-    @property
-    def buildable(self):
-        return self.data['buildable']
-
-    @property
-    def in_queue(self):
-        return self.data['inQueue']
-
-    @property
-    def last_build(self):
-        return self._get_build_number(self.data['lastBuild'])
-
-    @property
-    def last_completed_build(self):
-        return self._get_build_number(self.data['lastCompletedBuild'])
-
-    @property
-    def last_failed_build(self):
-        return self._get_build_number(self.data['lastFailedBuild'])
-
-    @property
-    def last_successful_build(self):
-        return self._get_build_number(self.data['lastSuccessfulBuild'])
-
-    def get_build_url_from_number(self, number):
-        url = self.url.strip('/')
-        return "/".join([url, "%d" % (number)])
-
-
-class JenkinsBuild(JenkinsObject):
-
-    """Provides data access for a jenkins build"""
-
-    FLAVORS = ['armel', 'armhf', 'amd64', 'i386']
-    SERIES = ['precise', 'quantal', 'raring']
-
-    def __init__(self, url, data=None):
-        """Creates the build object"""
-        super(JenkinsBuild, self).__init__(url, data)
-        self.duration = self._get_duration()
-        self.flavor = self._get_flavor()
-        self.runs = self._get_runs()
-        self.series = self._get_series()
-        self.timestamp = self._get_timestamp()
-
-    def __str__(self):
-        return "<JenkinsBuild: %s>" % (self.data)
-
-    def _get_duration(self):
-        return datetime.timedelta(milliseconds=int(self.data['duration']))
-
-    def _get_timestamp(self):
-        timestamp = int(self.data['timestamp']) / 1000
-        return datetime.datetime.fromtimestamp(timestamp)
-
-    def _get_flavor(self):
-        """Returns the flavor found in the build"""
-        for flavor in self.FLAVORS:
-            if flavor in self.url:
-                return flavor
-        return None
-
-    def _get_runs(self):
-        """Returns a list of multiconfiguration sub builds"""
-        if not self.is_multiconfig():
-            return []
-        runs = list()
-        for r in self.data['runs']:
-            # The api may return a run for ever configuration in the
-            # build matrix, throw out those that don't match the build
-            # number.
-            if r['number'] == self.data['number']:
-                run = BuildUrl(**r)
-                runs.append(run)
-        return runs
-
-    def _get_series(self):
-        """Returns a set containing all flavors found in the build"""
-        for series in self.SERIES:
-            if series in self.url:
-                return series
-        return None
-
-    def refresh(self):
-        super(JenkinsBuild, self).refresh()
-        self.duration = self._get_duration()
-        self.flavors = self._get_flavors()
-        self.runs = self._get_runs()
-        self.series = self._get_series()
-        self.timestamp = self._get_timestamp()
-
-    @property
-    def name(self):
-        return self.data['fullDisplayName']
-
-    @property
-    def job_name(self):
-        return self.name.split()[0]
-
-    @property
-    def number(self):
-        """Returns the number of this build"""
-        return int(self.data['number'])
-
-    @property
-    def building(self):
-        return self.data['building']
-
-    @property
-    def result(self):
-        return self.data['result']
-
-    def is_multiconfig(self):
-        """Indicates if this build is a multiconfiguration build"""
-        return 'runs' in self.data
-
-    def is_multiconfig_run(self):
-        """Indicates that this build is not a multiconfiguration run"""
-        return False
-
-    def get_console_url(self):
-        url = self.url.strip('/')
-        return '/'.join([url, 'consoleText'])
-
-    def get_coverage_report_url(self):
-        """Retrieves the build coverage report"""
-        url = self.url.strip('/')
-        return '/'.join([url, 'artifact/work/results/coverage.xml'])
-
-    def get_test_report_url(self):
-        """Retrieves the build junit test report"""
-        url = self.url.strip('/')
-        return '/'.join([url, 'testReport'])
-
-    def get_downstream_builds(self):
-        return get_downstream_builds_from_build(self)
-
-
-class JenkinsRun(JenkinsBuild):
-
-    def __init__(self, url, data=None):
-        """Creates the run object"""
-        super(JenkinsRun, self).__init__(url, data)
-
-    def __str__(self):
-        return "<JenkinsRun: %s>" % (self.data)
-
-    def is_mulitconfig_build(self):
-        """Indicates that this is not a multiconfiguration build"""
-        return False
-
-    def is_mulitconfig_run(self):
-        """Indicates that this is a multiconfiguration run"""
-        return True
-
-
-class JenkinsTestReport(JenkinsObject):
-
-    """Provides methods and attributes from a jenkins test report."""
-
-    def __init__(self, url, data=None):
-        """Creates a test report object"""
-        super(JenkinsTestReport, self).__init__(url, data)
-        # TODO Parse child reports
-        self.duration = self._get_duration()
-        self.suites = self._get_suites()
-
-    def _get_duration(self):
-        try:
-            return datetime.timedelta(milliseconds=int(self.data['duration']))
-        except KeyError:
-            return datetime.timedelta(0)
-
-    def _get_suites(self):
-        suites = list()
-        try:
-            for s in self.data['suites']:
-                cases = s['cases']
-                suite = type("JenkinsTestSuite", (object,), s)
-                suite.cases = list()
-                for c in cases:
-                    case = type("JenkinsTestCase", (object,), c)
-                    suite.cases.append(case)
-                suites.append(suite)
-        except KeyError:
-            pass
-        return suites
-
-    def refresh(self):
-        super(JenkinsBuild, self).refresh()
-        self.duration = self._get_duration()
-        self.suites = self._get_suites()
-
-    @property
-    def fail_count(self):
-        try:
-            return int(self.data['failCount'])
-        except KeyError:
-            return 0
-
-    @property
-    def pass_count(self):
-        # If jenkins aggregates multiple test reports, it will use
-        # totalCount, otherwise it should use passCount.
-        try:
-            return int(self.data['passCount'])
-        except KeyError:
-            try:
-                total = int(self.data['totalCount'])
-                total -= self.fail_count
-                total -= self.skip_count
-                return total
-            except KeyError:
-                return 0
-
-    @property
-    def skip_count(self):
-        try:
-            return int(self.data['skipCount'])
-        except KeyError:
-            return 0
-
-
-class JenkinsTestReportSet(object):
-
-    """Provides aggregation for jenkins test reports."""
-
-    def __init__(self, report=None):
-        """Creates a test report aggregate object"""
-        self.report_set = set()
-        self.fail_count = 0
-        self.pass_count = 0
-        self.skip_count = 0
-        self.duration = datetime.timedelta(0)
-        if report is None:
-            pass
-        elif isinstance(report, JenkinsTestReport):
-            self.add(report)
-        elif isinstance(report, JenkinsTestReportSet):
-            for r in report.report_set:
-                self.add(r)
-        else:
-            raise TypeError("Need JenkinsTestReport, JenkinsTestReportSet "
-                            "or None")
-
-    def add(self, report):
-        if report not in self.report_set:
-            self.report_set.add(report)
-            self.fail_count += report.fail_count
-            self.pass_count += report.pass_count
-            self.skip_count += report.skip_count
-            if self.duration < report.duration:
-                self.duration = report.duration
-
-
-coverage_url_part = {
-    'arch':
-    '/build=pbuilder,distribution=%s,flavor=%s'
-    '/lastSuccessfulBuild/artifact/work/results/coverage.xml',
-    'noarch': '/lastSuccessfulBuild/artifact/work/results/coverage.xml',
-}
-
-
-# TODO: Create a base class for all the Coverage classes
+def get_text_from_url(url):
+    response = jenkins_get(url, as_json=False)
+    if response:
+        return response
+    raise EnvironmentError("No data at %s" % (url))
+
+
 class JenkinsCoverage(object):
 
     """Provides methods and attributes for a jenkins code coverage report."""
@@ -472,202 +130,3 @@
         self.taken = taken_count
         self.total = total_count
         return self
-
-
-class JenkinsCoverageSet(object):
-
-    """Provides aggregation for jenkins code coverage reports."""
-
-    def __init__(self, report=None):
-        """Creates the coverage aggregate object"""
-        self.report_set = set()
-        self.branches = 0
-        self.hits = 0
-        self.lines = 0
-        self.taken = 0
-        self.total = 0
-        if report is None:
-            pass
-        elif isinstance(report, JenkinsCoverage):
-            self.add(report)
-        elif isinstance(report, JenkinsCoverageSet):
-            for r in report.report_set:
-                self.add(r)
-        else:
-            raise TypeError("Need JenkinsCoverage, JenkinsCoverageSet or None")
-
-    def add(self, coverage):
-        """Aggragetes a coverage report with the current set"""
-        if coverage not in self.report_set:
-            self.report_set.add(coverage)
-            self.branches += coverage.branches
-            self.hits += coverage.hits
-            self.lines += coverage.lines
-            self.taken += coverage.taken
-            self.total += coverage.total
-
-    @property
-    def line_rate(self):
-        try:
-            return float(self.hits) / float(self.lines)
-        except ZeroDivisionError:
-            return 0
-
-    @property
-    def branch_rate(self):
-        try:
-            return float(self.taken) / float(self.total)
-        except ZeroDivisionError:
-            return 0
-
-
-def get_server_url(url):
-    """Extracts the jenkins server url from any job, build, etc. url"""
-    server_url = urlsplit(url)
-    return '://'.join([server_url.scheme, server_url.netloc])
-
-
-def get_json_from_url(url):
-    url = "/".join([url, "api", "json"])
-    response = jenkins_get(url, as_json=True)
-    if response:
-        return response
-    raise EnvironmentError("No json data at %s" % (url))
-
-
-def get_text_from_url(url):
-    response = jenkins_get(url, as_json=False)
-    if response:
-        return response
-    raise EnvironmentError("No data at %s" % (url))
-
-
-def get_job_url(server_url, job_name):
-    return "/".join([server_url, "job", job_name])
-
-
-def get_job(server_url, job_name):
-    return JenkinsJob(get_job_url(server_url, job_name))
-
-
-def get_build_url(server_url, job_name, build_number):
-    return "/".join([server_url, "job", job_name, '%s' % build_number])
-
-
-def get_build(server_url, job_name, build_number):
-    return JenkinsBuild(get_build_url(server_url, job_name, build_number))
-
-
-def get_test_report_url(server_url, job_name, build_number):
-    return "/".join([server_url, "job", job_name, '%s' % build_number,
-                     'testReport'])
-
-
-def get_test_report(server_url, job_name, build_number):
-    return JenkinsTestReport(get_test_report_url(server_url, job_name,
-                                                 build_number))
-
-
-def get_console_text_url(server_url, job_name, build_number):
-    return "/".join([server_url, "job", job_name, '%s' % build_number,
-                     'consoleText'])
-
-
-def get_console_text(server_url, job_name, build_number):
-    return get_text_from_url(get_console_text_url(server_url, job_name,
-                                                  build_number))
-
-
-def get_coverage_report_url(server_url, job_name, build_number):
-    return "/".join([server_url, "job", job_name, '%s' % build_number,
-                     'artifact/work/results/coverage.xml'])
-
-
-def get_coverage_report(server_url, job_name, build_number):
-    return JenkinsCoverage(
-        get_coverage_report_url(
-            server_url, job_name,
-            build_number,
-        ),
-    )
-
-
-## BEGIN SERGIO
-## The following are a carry over from sergio's original work
-## and should be deprecated
-def get_coverage_for_arch(job_url, series, arch_order):
-    url = None
-    for arch in arch_order:
-        url = job_url + coverage_url_part['arch'] % (series, arch)
-        coverage = jenkins_get(url, as_json=False)
-    return JenkinsCoverage(url, "%s" % (coverage))
-
-
-def get_coverage_from_daily(job_url):
-    url = job_url + coverage_url_part['noarch']
-    coverage = jenkins_get(url, as_json=False)
-
-    return JenkinsCoverage(url, "%s" % coverage)
-
-
-def get_coverage_from_url(job_url, series=None, arch_order=None):
-    if arch_order:
-        coverage = get_coverage_for_arch(job_url, series, arch_order)
-    else:
-        coverage = get_coverage_from_daily(job_url)
-    return coverage
-
-
-def get_coverage_from_job(server, job_name, series=None, arch_order=None):
-    """Retreives the coverage data from a jenkins server and job"""
-    job = server.get_job(job_name)
-    return get_coverage_from_url(job.baseurl, series, arch_order)
-## END SERGIO
-
-
-def get_console_text_from_build(build):
-    """Return the console text from a Jenkins build."""
-    return get_text_from_url(build.get_console_url())
-
-
-def get_console_text_from_url(build_url):
-    """Return a block of text from a Jenkins build."""
-    url = build_url.strip('/')
-    url = '/'.join([url, "consoleText"])
-    return get_text_from_url(url)
-
-
-def get_downstream_builds_from_console_text(base_url, text):
-    """Return a list of BuildUrls for the downstream jobs in the console"""
-
-    # Extracts build identifiers from the following formats:
-    # autopilot-raring-amd64-autolanding #5 completed. Result was SUCCESS
-    # Triggering a new build of autopilot-docs-upload #9
-    format_list = [
-        "^(?P<name>.*) #(?P<number>\d+) completed\. Result was .*$",
-        "Triggering a new build of (?P<name>.*) #(?P<number>\d+)$"]
-    downstream_builds = []
-    for f in format_list:
-        build_regex = re.compile(f, re.MULTILINE)
-        results = build_regex.finditer(text)
-        for result in results:
-            number = result.group('number')
-            url = get_build_url(base_url, result.group('name'), number)
-            downstream_builds.append(BuildUrl(number=number, url=url))
-    return downstream_builds
-
-
-def get_downstream_builds_from_build(build):
-    """Return a dict of build names, numbers, and urls."""
-
-    base_url = get_server_url(build.url)
-    text = get_console_text_from_build(build)
-    return get_downstream_builds_from_console_text(base_url, text)
-
-
-def get_downstream_builds_from_build_url(url):
-    """Return a dict of build names, numbers, and urls."""
-
-    base_url = get_server_url(url)
-    text = get_console_text_from_url(url)
-    return get_downstream_builds_from_console_text(base_url, text)