Merge lp:~cjohnston/qa-dashboard/sru-attention into lp:qa-dashboard

[Chris Johnston]

Fixes the pulling bugs in SRU

[lp:~cjohnston/qa-dashboard/sru-attention]

282. By Chris Johnston on 2013-03-13

remove code

[Andy Doan (doanac)]

On 03/12/2013 08:40 PM, Chris Johnston wrote:
> + bug = None
> + try:
> + bug = Bug.objects.get(bug_no=bug_no)
> + except Bug.DoesNotExist:
> + bug = Bug.objects.create(bug_no=bug_no, status='unknown')
> + if bug not in list(sruresult.bugs.all()):
> + sruresult.bugs.add(bug)
> + sruresult.save()

I noticed that pattern used a bit in this module. Is there a reason you
aren't using "get_or_create"[1]? I think that method is more fool-proof
and race-condition safe.

1:
<https://docs.djangoproject.com/en/dev/ref/models/querysets/#get-or-create>

[lp:~cjohnston/qa-dashboard/sru-attention]

283. By Chris Johnston on 2013-03-15

Changes from feedback

284. By Chris Johnston on 2013-03-22

More sru work

285. By Chris Johnston on 2013-03-23

Switches all instances of get to get_or_create

[Andy Doan (doanac)]

+1

[Joe Talbott (joetalbott)]

Looks good to me.

review: Approve

=== modified file 'sru/management/commands/jenkins_pull_sru.py'
--- sru/management/commands/jenkins_pull_sru.py	2013-03-21 18:59:36 +0000
+++ sru/management/commands/jenkins_pull_sru.py	2013-03-23 16:02:22 +0000
@@ -13,27 +13,22 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import re
 import logging
 from datetime import datetime, timedelta
 from optparse import make_option
-from django.core.management.base import BaseCommand, CommandError
+from django.core.management.base import BaseCommand
 
 from common.management import (
-        jenkins_get,
-        skip_build,
-        BUILD_COUNT_MAX,
-        JOB_COUNT_MAX,
-        DEBUG_LIMITS,
-        )
+    jenkins_get,
+    skip_build,
+    BUILD_COUNT_MAX,
+    JOB_COUNT_MAX,
+    DEBUG_LIMITS,
+)
 
 from common.utils import regexes
-
-from sru.models import (
-        Kernel,
-        SRUResult,
-        SRUResultLog,
-        )
+from common.models import Bug
+from sru.models import Kernel
 
 
 def _get_kernel_version(description, default="Unknown"):
@@ -54,15 +49,35 @@
 
     return version
 
+
+def _parse_bugs(lp_bugs, result):
+    """
+    Add bugs to the sruresult.
+    """
+    sruresult = result
+    status = 'unknown'
+    for bug_no in lp_bugs:
+        bug = None
+        bug, new_bug = Bug.objects.get_or_create(
+            bug_no=bug_no,
+            defaults={'status': status}
+        )
+        if bug not in list(sruresult.bugs.all()):
+            sruresult.bugs.add(bug)
+            sruresult.save()
+
+
 class Command(BaseCommand):
     option_list = BaseCommand.option_list + (
-            make_option('-I', '--initial-run',
-                action='store_true',
-                dest='initial_run',
-                default=False,
-                help='Initial run processes all data (takes longer)'),
-            )
-                
+        make_option(
+            '-I', '--initial-run',
+            action='store_true',
+            dest='initial_run',
+            default=False,
+            help='Initial run processes all data (takes longer)'
+        ),
+    )
+
     help = 'Pull data from jenkins'
 
     def handle(self, *args, **options):
@@ -77,7 +92,7 @@
 
         jobs = jobs_top['jobs']
 
-        job_count = 0 # XXX: DEBUG
+        job_count = 0  # XXX: DEBUG
         for job in jobs:
             name = job['name']
             m = regexes['kernel_sru'].match(name)
@@ -92,12 +107,11 @@
 
                 variant, arch, gpu = m.group(5, 6, 7)
 
-
                 # XXX: debugging limit
                 job_count += 1
                 if DEBUG_LIMITS and job_count > JOB_COUNT_MAX:
                     break
-                
+
                 logging.info("job: {}\n".format(name))
                 logging.info("\tjob_type: {}\n".format(job_type))
                 logging.info("\trelease: {}\n".format(release))
@@ -112,18 +126,24 @@
                 # ignore jobs that haven't been run yet.
                 if job_info['lastBuild'] is None:
                     continue
-                last_build = datetime.fromtimestamp(job_info['lastBuild']['timestamp']/1000)
+                last_build = datetime.fromtimestamp(
+                    job_info['lastBuild']['timestamp']/1000
+                )
                 # for update runs only check jobs that have built in the
                 # last week.
-                if not initial_run and datetime.now() - last_build > timedelta(days=7):
+                if not initial_run and datetime.now() - last_build > timedelta(
+                    days=7
+                ):
                     continue
 
-                build_count = 0 # XXX: DEBUG
+                build_count = 0  # XXX: DEBUG
                 for build in builds:
                     if skip_build(build):
-                        logging.info("skipping build: {} {}".format(
-                            job['name'], build['number']
-                        ))
+                        logging.info(
+                            "skipping build: %s %s",
+                            job['name'],
+                            build['number']
+                        )
                         continue
 
                     # XXX: debugging limit
@@ -132,76 +152,80 @@
                         break
 
                     build_number = build['number']
-                    build_url    = build['url']
-                    #build_info   = jenkins_get("%s/api/json" % build_url)
-                    building     = build['building']
-                    build_desc   = build['description']
-                    build_date   = datetime.fromtimestamp(build['timestamp']/1000)
-                    artifacts    = build['artifacts']
-                    actions      = build['actions']
+                    build_url = build['url']
+                    building = build['building']
+                    build_desc = build['description']
+                    build_date = datetime.fromtimestamp(
+                        build['timestamp']/1000
+                    )
+                    artifacts = build['artifacts']
+                    actions = build['actions']
 
                     if build_desc is None:
                         build_desc = ''
 
-                    kernel_version = _get_kernel_version(build_desc,
-                            "Unknown. Ran on {}".format(build_date.strftime("%Y-%m-%d %H:%M")))
+                    kernel_version = _get_kernel_version(
+                        build_desc,
+                        "Unknown. Ran on {}".format(
+                            build_date.strftime("%Y-%m-%d %H:%M")
+                        )
+                    )
 
                     # Skip this Jenkins build if it hasn't finished yet
                     if building:
                         continue
 
-                    # XXX: find out what TZ jenkins uses and use correct one here
+                    # XXX: find out what TZ jenkins uses
+                    # and use correct one here
                     if not initial_run and datetime.now() - build_date > timedelta(days=7):
                         continue
 
-                    lp_bugs  = []
+                    lp_bugs = []
 
-                    if build_desc != None:
+                    if build_desc is not None:
                         b = build_desc.split(',')
                         b = regexes['build_desc'].findall(build_desc)
                         for word in b:
                             word = word.strip()
-                            if regexes['buildid'].match(word) and build_no is None:
+                            regex = regexes['buildid'].match(word)
+                            if regex and build_no is None:
                                 # It's a build number
                                 build_no = "{} ?".format(word)
                             elif regexes['lpbug'].match(word):
                                 lp_bugs.append(word[4:])
 
-                    try:
-                        kernel = Kernel.objects.get(release=release,
-                                                    variant=variant,
-                                                    arch=arch,
-                                                    gpu=gpu,
-                                                    version=kernel_version,
-                                                    job_type=job_type)
-                    except Kernel.DoesNotExist:
-                        kernel = Kernel.objects.create(release=release,
-                                                       variant=variant,
-                                                       arch=arch,
-                                                       gpu=gpu,
-                                                       version=kernel_version,
-                                                       job_type=job_type)
-
-                    result = None
-                    try:
-                        result = kernel.sruresult_set.get(name=name, jenkins_build=build_number)
-                        # break
-                    except SRUResult.DoesNotExist:
-                        fail_count, skip_count, pass_count, total_count = (0, 0, 0, 0)
-                        for action in actions:
-                            try:
-                                fail_count, skip_count, total_count = (action['failCount'], action['skipCount'], action['totalCount'])
-                                pass_count = total_count - fail_count - skip_count
-                                break
-                            except KeyError:
-                                continue
-
-                        result = kernel.sruresult_set.create(name=name,
-                                                             jenkins_build=build_number,
-                                                             ran_at=build_date,
-                                                             jenkins_url=build_url,
-                                                             fail_count=fail_count,
-                                                             skip_count=skip_count,
-                                                             pass_count=pass_count,
-                                                             total_count=total_count
-                        )
+                    kernel, new_kernel = Kernel.objects.get_or_create(
+                        release=release,
+                        variant=variant,
+                        arch=arch,
+                        gpu=gpu,
+                        version=kernel_version,
+                        job_type=job_type,
+                    )
+
+                    fail_count = 0
+                    skip_count = 0
+                    pass_count = 0
+                    total_count = 0
+                    for action in actions:
+                        try:
+                            fail_count = action['failCount']
+                            skip_count = action['skipCount']
+                            total_count = action['totalCount']
+                            pass_count = total_count - fail_count - skip_count
+                            break
+                        except KeyError:
+                            continue
+                    result, new_result = kernel.sruresult_set.get_or_create(
+                        name=name,
+                        jenkins_build=build_number,
+                        defaults={
+                            'ran_at': build_date,
+                            'jenkins_url': build_url,
+                            'fail_count': fail_count,
+                            'skip_count': skip_count,
+                            'pass_count': pass_count,
+                            'total_count': total_count,
+                        }
+                    )
+                    _parse_bugs(lp_bugs, result)