Merge lp:~allanlesage/qa-coverage-dashboard/simplify-extractor-split-out-get-last-build into lp:qa-coverage-dashboard

Proposed by Allan LeSage on 2014-04-23
Status: Merged
Approved by: Chris Gagnon on 2014-05-16
Approved revision: 768
Merged at revision: 775
Proposed branch: lp:~allanlesage/qa-coverage-dashboard/simplify-extractor-split-out-get-last-build
Merge into: lp:qa-coverage-dashboard
Diff against target: 687 lines (+72/-558)
4 files modified
gaps/tests/__init__.py (+3/-0)
gaps/tests/test_add.py (+44/-0)
gaps/util/add.py (+17/-9)
gaps/util/extractor.py (+8/-549)
To merge this branch: bzr merge lp:~allanlesage/qa-coverage-dashboard/simplify-extractor-split-out-get-last-build
Reviewer Review Type Date Requested Status
Chris Gagnon (community) 2014-04-23 Approve on 2014-05-16
Review via email: mp+216934@code.launchpad.net

Description of the change

Split out the get_last_build function and add a test for it individually, also chop down the olde extractor.

To post a comment you must log in.
768. By Allan LeSage on 2014-05-12

Merged trunk, resolving conflicts.

Chris Gagnon (chris.gagnon) wrote :

looks good

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'gaps/tests/__init__.py'
2--- gaps/tests/__init__.py 2014-04-09 19:08:42 +0000
3+++ gaps/tests/__init__.py 2014-05-13 14:26:02 +0000
4@@ -4,3 +4,6 @@
5 GetArtifactUrlsTestCase,
6 UrlArtifactListTestCase,
7 )
8+from gaps.tests.test_add import ( # noqa
9+ GetLastBuildTestCase,
10+)
11
12=== added file 'gaps/tests/test_add.py'
13--- gaps/tests/test_add.py 1970-01-01 00:00:00 +0000
14+++ gaps/tests/test_add.py 2014-05-13 14:26:02 +0000
15@@ -0,0 +1,44 @@
16+
17+from django.test import TestCase
18+
19+from mock import patch, MagicMock
20+
21+from gaps.util.add import (
22+ get_last_build_number_for_jenkins_job,
23+ CoverageData,
24+)
25+
26+
27+class GetLastBuildTestCase(TestCase):
28+
29+ def setUp(self):
30+ super(GetLastBuildTestCase, self).setUp()
31+ self.coveragedata_mock = MagicMock(
32+ spec=CoverageData)
33+ coveragedata_patch = patch(
34+ 'gaps.util.add.CoverageData',
35+ new=self.coveragedata_mock)
36+ coveragedata_patch.start()
37+ self.addCleanup(coveragedata_patch.stop)
38+
39+ def test_get_last_build_number_for_jenkins_job(self):
40+ """The list of builds is [1, 2]."""
41+ first_build = MagicMock(build_number=1)
42+ second_build = MagicMock(build_number=2)
43+ # ok I agree this is evil
44+ self.coveragedata_mock.objects = MagicMock(
45+ filter=MagicMock(
46+ return_value=MagicMock(
47+ order_by=MagicMock(
48+ return_value=[
49+ first_build,
50+ second_build,
51+ ],
52+ )
53+ )
54+ )
55+ )
56+ self.assertEqual(
57+ [1],
58+ get_last_build_number_for_jenkins_job("what_me_workie?"),
59+ )
60
61=== modified file 'gaps/util/add.py'
62--- gaps/util/add.py 2014-04-18 21:15:24 +0000
63+++ gaps/util/add.py 2014-05-13 14:26:02 +0000
64@@ -13,6 +13,22 @@
65 logger = logging.getLogger('qa_dashboard')
66
67
68+def get_last_build_number_for_jenkins_job(jenkins_job_name):
69+ logger.debug("getting the number of the last_build")
70+ try:
71+ last_build = CoverageData.objects.filter(
72+ coverage_build__job__name__exact=jenkins_job_name
73+ ).order_by(
74+ '-coverage_build__build_number'
75+ )[0]
76+ logger.debug("last build in our database is {}".format(
77+ last_build.build_number)
78+ )
79+ return [last_build.build_number]
80+ except IndexError:
81+ return [0]
82+
83+
84 def records(stack_name, project_name, jenkins_job_name, file_name):
85 """Adds coverage data to the database
86
87@@ -21,15 +37,7 @@
88 :param jenkins_job_name: job to get artifacts from
89 :param file_name: file name of artifact
90 """
91- logger.debug("getting last_build")
92- last_build = [
93- j_build.coverage_build.build_number for j_build in
94- CoverageData.objects.filter(
95- coverage_build__job__name__exact=jenkins_job_name)]
96- sorted(last_build)
97- if last_build == []:
98- last_build.append(0)
99- logger.debug("last build in our database is {}".format(last_build[-1]))
100+ last_build = get_last_build_number_for_jenkins_job(jenkins_job_name)
101 url_list = jenkins_pull.url_artifact_list(
102 jenkins_job_name,
103 file_name,
104
105=== modified file 'gaps/util/extractor.py'
106--- gaps/util/extractor.py 2014-02-21 01:05:01 +0000
107+++ gaps/util/extractor.py 2014-05-13 14:26:02 +0000
108@@ -1,5 +1,5 @@
109 # QA Dashboard
110-# Copyright 2012-2013 Canonical Ltd.
111+# Copyright 2012-2014 Canonical Ltd.
112
113 # This program is free software: you can redistribute it and/or modify it
114 # under the terms of the GNU Affero General Public License version 3, as
115@@ -13,362 +13,20 @@
116 # You should have received a copy of the GNU Affero General Public License
117 # along with this program. If not, see <http://www.gnu.org/licenses/>.
118
119-import datetime
120-import re
121 from lxml import etree
122-from urlparse import urlsplit
123
124 from common.management import jenkins_get
125
126-
127 """Classes and utilities to extract QA data from jenkins"""
128
129
130-class BuildUrl(object):
131- def __init__(self, number=None, url=None):
132- self.number = number
133- self.url = url
134-
135- def __repr__(self):
136- return '<BuildUrl: [{}, {}]>'.format(self.number, self.url)
137-
138- def __cmp__(self, other):
139- return (self.number == other.number) and (self.url == other.url)
140-
141-
142-class JobUrl(object):
143- def __init__(self, name=None, url=None, color=None):
144- self.name = name
145- self.url = url
146- self.color = color
147-
148-
149-class Parameter(object):
150- def __init__(self, name=None, value=None, description=None):
151- self.name = name
152- self.value = value
153- self.description = description
154-
155-
156-class JenkinsObject(object):
157- """The base class for other jenkins objects with common attributes"""
158-
159- def __init__(self, url, data=None):
160- self.url = url
161- if data is None:
162- data = get_json_from_url(self.url)
163- self.data = data
164-
165- def refresh(self):
166- self.data = get_json_from_url(self.url)
167-
168- @property
169- def name(self):
170- return self.data['displayName']
171-
172-
173-class JenkinsJob(JenkinsObject):
174-
175- """Provides data access for a jenkins job"""
176-
177- def __init__(self, url, data=None):
178- """Creates the job object"""
179- super(JenkinsJob, self).__init__(url, data)
180- self.builds = self._get_builds()
181- self.downstream_jobs = self._get_downstream_jobs()
182-
183- def __str__(self):
184- return "<JenkinsJob: %s>" % (self.data)
185-
186- def _get_builds(self):
187- builds = list()
188- for b in self.data['builds']:
189- build = BuildUrl(**b)
190- builds.append(build)
191- return builds
192-
193- def _get_downstream_jobs(self):
194- downstream_jobs = list()
195- for j in self.data['downstreamProjects']:
196- job = JobUrl(**j)
197- downstream_jobs.append(job)
198- return downstream_jobs
199-
200- def _get_build_number(self, build):
201- # A build will be None when the last*Build has not occurred yet
202- if build is None:
203- return None
204- return int(build['number'])
205-
206- def refresh(self):
207- super(JenkinsJob, self).refresh()
208- self.builds = self._get_builds()
209- self.downstream_jobs = self._get_downstream_jobs()
210-
211- @property
212- def buildable(self):
213- return self.data['buildable']
214-
215- @property
216- def in_queue(self):
217- return self.data['inQueue']
218-
219- @property
220- def last_build(self):
221- return self._get_build_number(self.data['lastBuild'])
222-
223- @property
224- def last_completed_build(self):
225- return self._get_build_number(self.data['lastCompletedBuild'])
226-
227- @property
228- def last_failed_build(self):
229- return self._get_build_number(self.data['lastFailedBuild'])
230-
231- @property
232- def last_successful_build(self):
233- return self._get_build_number(self.data['lastSuccessfulBuild'])
234-
235- def get_build_url_from_number(self, number):
236- url = self.url.strip('/')
237- return "/".join([url, "%d" % (number)])
238-
239-
240-class JenkinsBuild(JenkinsObject):
241-
242- """Provides data access for a jenkins build"""
243-
244- FLAVORS = ['armel', 'armhf', 'amd64', 'i386']
245- SERIES = ['precise', 'quantal', 'raring']
246-
247- def __init__(self, url, data=None):
248- """Creates the build object"""
249- super(JenkinsBuild, self).__init__(url, data)
250- self.duration = self._get_duration()
251- self.flavor = self._get_flavor()
252- self.runs = self._get_runs()
253- self.series = self._get_series()
254- self.timestamp = self._get_timestamp()
255-
256- def __str__(self):
257- return "<JenkinsBuild: %s>" % (self.data)
258-
259- def _get_duration(self):
260- return datetime.timedelta(milliseconds=int(self.data['duration']))
261-
262- def _get_timestamp(self):
263- timestamp = int(self.data['timestamp']) / 1000
264- return datetime.datetime.fromtimestamp(timestamp)
265-
266- def _get_flavor(self):
267- """Returns the flavor found in the build"""
268- for flavor in self.FLAVORS:
269- if flavor in self.url:
270- return flavor
271- return None
272-
273- def _get_runs(self):
274- """Returns a list of multiconfiguration sub builds"""
275- if not self.is_multiconfig():
276- return []
277- runs = list()
278- for r in self.data['runs']:
279- # The api may return a run for ever configuration in the
280- # build matrix, throw out those that don't match the build
281- # number.
282- if r['number'] == self.data['number']:
283- run = BuildUrl(**r)
284- runs.append(run)
285- return runs
286-
287- def _get_series(self):
288- """Returns a set containing all flavors found in the build"""
289- for series in self.SERIES:
290- if series in self.url:
291- return series
292- return None
293-
294- def refresh(self):
295- super(JenkinsBuild, self).refresh()
296- self.duration = self._get_duration()
297- self.flavors = self._get_flavors()
298- self.runs = self._get_runs()
299- self.series = self._get_series()
300- self.timestamp = self._get_timestamp()
301-
302- @property
303- def name(self):
304- return self.data['fullDisplayName']
305-
306- @property
307- def job_name(self):
308- return self.name.split()[0]
309-
310- @property
311- def number(self):
312- """Returns the number of this build"""
313- return int(self.data['number'])
314-
315- @property
316- def building(self):
317- return self.data['building']
318-
319- @property
320- def result(self):
321- return self.data['result']
322-
323- def is_multiconfig(self):
324- """Indicates if this build is a multiconfiguration build"""
325- return 'runs' in self.data
326-
327- def is_multiconfig_run(self):
328- """Indicates that this build is not a multiconfiguration run"""
329- return False
330-
331- def get_console_url(self):
332- url = self.url.strip('/')
333- return '/'.join([url, 'consoleText'])
334-
335- def get_coverage_report_url(self):
336- """Retrieves the build coverage report"""
337- url = self.url.strip('/')
338- return '/'.join([url, 'artifact/work/results/coverage.xml'])
339-
340- def get_test_report_url(self):
341- """Retrieves the build junit test report"""
342- url = self.url.strip('/')
343- return '/'.join([url, 'testReport'])
344-
345- def get_downstream_builds(self):
346- return get_downstream_builds_from_build(self)
347-
348-
349-class JenkinsRun(JenkinsBuild):
350-
351- def __init__(self, url, data=None):
352- """Creates the run object"""
353- super(JenkinsRun, self).__init__(url, data)
354-
355- def __str__(self):
356- return "<JenkinsRun: %s>" % (self.data)
357-
358- def is_mulitconfig_build(self):
359- """Indicates that this is not a multiconfiguration build"""
360- return False
361-
362- def is_mulitconfig_run(self):
363- """Indicates that this is a multiconfiguration run"""
364- return True
365-
366-
367-class JenkinsTestReport(JenkinsObject):
368-
369- """Provides methods and attributes from a jenkins test report."""
370-
371- def __init__(self, url, data=None):
372- """Creates a test report object"""
373- super(JenkinsTestReport, self).__init__(url, data)
374- # TODO Parse child reports
375- self.duration = self._get_duration()
376- self.suites = self._get_suites()
377-
378- def _get_duration(self):
379- try:
380- return datetime.timedelta(milliseconds=int(self.data['duration']))
381- except KeyError:
382- return datetime.timedelta(0)
383-
384- def _get_suites(self):
385- suites = list()
386- try:
387- for s in self.data['suites']:
388- cases = s['cases']
389- suite = type("JenkinsTestSuite", (object,), s)
390- suite.cases = list()
391- for c in cases:
392- case = type("JenkinsTestCase", (object,), c)
393- suite.cases.append(case)
394- suites.append(suite)
395- except KeyError:
396- pass
397- return suites
398-
399- def refresh(self):
400- super(JenkinsBuild, self).refresh()
401- self.duration = self._get_duration()
402- self.suites = self._get_suites()
403-
404- @property
405- def fail_count(self):
406- try:
407- return int(self.data['failCount'])
408- except KeyError:
409- return 0
410-
411- @property
412- def pass_count(self):
413- # If jenkins aggregates multiple test reports, it will use
414- # totalCount, otherwise it should use passCount.
415- try:
416- return int(self.data['passCount'])
417- except KeyError:
418- try:
419- total = int(self.data['totalCount'])
420- total -= self.fail_count
421- total -= self.skip_count
422- return total
423- except KeyError:
424- return 0
425-
426- @property
427- def skip_count(self):
428- try:
429- return int(self.data['skipCount'])
430- except KeyError:
431- return 0
432-
433-
434-class JenkinsTestReportSet(object):
435-
436- """Provides aggregation for jenkins test reports."""
437-
438- def __init__(self, report=None):
439- """Creates a test report aggregate object"""
440- self.report_set = set()
441- self.fail_count = 0
442- self.pass_count = 0
443- self.skip_count = 0
444- self.duration = datetime.timedelta(0)
445- if report is None:
446- pass
447- elif isinstance(report, JenkinsTestReport):
448- self.add(report)
449- elif isinstance(report, JenkinsTestReportSet):
450- for r in report.report_set:
451- self.add(r)
452- else:
453- raise TypeError("Need JenkinsTestReport, JenkinsTestReportSet "
454- "or None")
455-
456- def add(self, report):
457- if report not in self.report_set:
458- self.report_set.add(report)
459- self.fail_count += report.fail_count
460- self.pass_count += report.pass_count
461- self.skip_count += report.skip_count
462- if self.duration < report.duration:
463- self.duration = report.duration
464-
465-
466-coverage_url_part = {
467- 'arch':
468- '/build=pbuilder,distribution=%s,flavor=%s'
469- '/lastSuccessfulBuild/artifact/work/results/coverage.xml',
470- 'noarch': '/lastSuccessfulBuild/artifact/work/results/coverage.xml',
471-}
472-
473-
474-# TODO: Create a base class for all the Coverage classes
475+def get_text_from_url(url):
476+ response = jenkins_get(url, as_json=False)
477+ if response:
478+ return response
479+ raise EnvironmentError("No data at %s" % (url))
480+
481+
482 class JenkinsCoverage(object):
483
484 """Provides methods and attributes for a jenkins code coverage report."""
485@@ -472,202 +130,3 @@
486 self.taken = taken_count
487 self.total = total_count
488 return self
489-
490-
491-class JenkinsCoverageSet(object):
492-
493- """Provides aggregation for jenkins code coverage reports."""
494-
495- def __init__(self, report=None):
496- """Creates the coverage aggregate object"""
497- self.report_set = set()
498- self.branches = 0
499- self.hits = 0
500- self.lines = 0
501- self.taken = 0
502- self.total = 0
503- if report is None:
504- pass
505- elif isinstance(report, JenkinsCoverage):
506- self.add(report)
507- elif isinstance(report, JenkinsCoverageSet):
508- for r in report.report_set:
509- self.add(r)
510- else:
511- raise TypeError("Need JenkinsCoverage, JenkinsCoverageSet or None")
512-
513- def add(self, coverage):
514- """Aggragetes a coverage report with the current set"""
515- if coverage not in self.report_set:
516- self.report_set.add(coverage)
517- self.branches += coverage.branches
518- self.hits += coverage.hits
519- self.lines += coverage.lines
520- self.taken += coverage.taken
521- self.total += coverage.total
522-
523- @property
524- def line_rate(self):
525- try:
526- return float(self.hits) / float(self.lines)
527- except ZeroDivisionError:
528- return 0
529-
530- @property
531- def branch_rate(self):
532- try:
533- return float(self.taken) / float(self.total)
534- except ZeroDivisionError:
535- return 0
536-
537-
538-def get_server_url(url):
539- """Extracts the jenkins server url from any job, build, etc. url"""
540- server_url = urlsplit(url)
541- return '://'.join([server_url.scheme, server_url.netloc])
542-
543-
544-def get_json_from_url(url):
545- url = "/".join([url, "api", "json"])
546- response = jenkins_get(url, as_json=True)
547- if response:
548- return response
549- raise EnvironmentError("No json data at %s" % (url))
550-
551-
552-def get_text_from_url(url):
553- response = jenkins_get(url, as_json=False)
554- if response:
555- return response
556- raise EnvironmentError("No data at %s" % (url))
557-
558-
559-def get_job_url(server_url, job_name):
560- return "/".join([server_url, "job", job_name])
561-
562-
563-def get_job(server_url, job_name):
564- return JenkinsJob(get_job_url(server_url, job_name))
565-
566-
567-def get_build_url(server_url, job_name, build_number):
568- return "/".join([server_url, "job", job_name, '%s' % build_number])
569-
570-
571-def get_build(server_url, job_name, build_number):
572- return JenkinsBuild(get_build_url(server_url, job_name, build_number))
573-
574-
575-def get_test_report_url(server_url, job_name, build_number):
576- return "/".join([server_url, "job", job_name, '%s' % build_number,
577- 'testReport'])
578-
579-
580-def get_test_report(server_url, job_name, build_number):
581- return JenkinsTestReport(get_test_report_url(server_url, job_name,
582- build_number))
583-
584-
585-def get_console_text_url(server_url, job_name, build_number):
586- return "/".join([server_url, "job", job_name, '%s' % build_number,
587- 'consoleText'])
588-
589-
590-def get_console_text(server_url, job_name, build_number):
591- return get_text_from_url(get_console_text_url(server_url, job_name,
592- build_number))
593-
594-
595-def get_coverage_report_url(server_url, job_name, build_number):
596- return "/".join([server_url, "job", job_name, '%s' % build_number,
597- 'artifact/work/results/coverage.xml'])
598-
599-
600-def get_coverage_report(server_url, job_name, build_number):
601- return JenkinsCoverage(
602- get_coverage_report_url(
603- server_url, job_name,
604- build_number,
605- ),
606- )
607-
608-
609-## BEGIN SERGIO
610-## The following are a carry over from sergio's original work
611-## and should be deprecated
612-def get_coverage_for_arch(job_url, series, arch_order):
613- url = None
614- for arch in arch_order:
615- url = job_url + coverage_url_part['arch'] % (series, arch)
616- coverage = jenkins_get(url, as_json=False)
617- return JenkinsCoverage(url, "%s" % (coverage))
618-
619-
620-def get_coverage_from_daily(job_url):
621- url = job_url + coverage_url_part['noarch']
622- coverage = jenkins_get(url, as_json=False)
623-
624- return JenkinsCoverage(url, "%s" % coverage)
625-
626-
627-def get_coverage_from_url(job_url, series=None, arch_order=None):
628- if arch_order:
629- coverage = get_coverage_for_arch(job_url, series, arch_order)
630- else:
631- coverage = get_coverage_from_daily(job_url)
632- return coverage
633-
634-
635-def get_coverage_from_job(server, job_name, series=None, arch_order=None):
636- """Retreives the coverage data from a jenkins server and job"""
637- job = server.get_job(job_name)
638- return get_coverage_from_url(job.baseurl, series, arch_order)
639-## END SERGIO
640-
641-
642-def get_console_text_from_build(build):
643- """Return the console text from a Jenkins build."""
644- return get_text_from_url(build.get_console_url())
645-
646-
647-def get_console_text_from_url(build_url):
648- """Return a block of text from a Jenkins build."""
649- url = build_url.strip('/')
650- url = '/'.join([url, "consoleText"])
651- return get_text_from_url(url)
652-
653-
654-def get_downstream_builds_from_console_text(base_url, text):
655- """Return a list of BuildUrls for the downstream jobs in the console"""
656-
657- # Extracts build identifiers from the following formats:
658- # autopilot-raring-amd64-autolanding #5 completed. Result was SUCCESS
659- # Triggering a new build of autopilot-docs-upload #9
660- format_list = [
661- "^(?P<name>.*) #(?P<number>\d+) completed\. Result was .*$",
662- "Triggering a new build of (?P<name>.*) #(?P<number>\d+)$"]
663- downstream_builds = []
664- for f in format_list:
665- build_regex = re.compile(f, re.MULTILINE)
666- results = build_regex.finditer(text)
667- for result in results:
668- number = result.group('number')
669- url = get_build_url(base_url, result.group('name'), number)
670- downstream_builds.append(BuildUrl(number=number, url=url))
671- return downstream_builds
672-
673-
674-def get_downstream_builds_from_build(build):
675- """Return a dict of build names, numbers, and urls."""
676-
677- base_url = get_server_url(build.url)
678- text = get_console_text_from_build(build)
679- return get_downstream_builds_from_console_text(base_url, text)
680-
681-
682-def get_downstream_builds_from_build_url(url):
683- """Return a dict of build names, numbers, and urls."""
684-
685- base_url = get_server_url(url)
686- text = get_console_text_from_url(url)
687- return get_downstream_builds_from_console_text(base_url, text)

Subscribers

People subscribed via source and target branches

to all changes: