Merge lp:~joetalbott/qa-dashboard/utah_log_parser into lp:qa-dashboard

[Joe Talbott]

Add support for parsing utah logs to jenkins_pull.

This branch add utah log parsing to the jenkins_pull command.

This is to sync my current work and contains a few new minor features.

* jenkins_pull now supports '-w <jobname>' to skip all jenkins jobs until the named one is encountered. This greatly speeds up debugging initial pulls.
* jenkins_pull now supports '... [jobname] [jobname] ...' at the end of the command line to process only certain jobs whose names are supplied.
* adds proper logging support and uses the verbosity command line option. -v N where N is > 1 will turn on debugging.
* skips processing UTAH jobs as regular jenkins jobs.

[Chris Johnston (cjohnston)]

This is looking good.

[Chris Johnston (cjohnston)]

review: Approve

=== modified file 'bootspeed/management/commands/fakeup_bootspeed.py'
--- bootspeed/management/commands/fakeup_bootspeed.py	2012-11-15 02:13:32 +0000
+++ bootspeed/management/commands/fakeup_bootspeed.py	2012-11-28 17:34:20 +0000
@@ -18,60 +18,80 @@
         variants = ['server', 'desktop', 'mini', 'alternate']
         build_numbers = ['20120909', '20120910', '20120911']
 
-        machine, created = Machine.objects.get_or_create(
-                name='Dell Optiplex 242',
-                mac_address='01:02:03:0a:0b',
-                )
-
-        releases = ['quantal']
-        arches = ['amd64', 'i386']
-        variants = ['server']
-        build_numbers = ['2012{:0>2}{:0>2}'.format(y,x) for y in [4,5,6,7,8,9] for x in range(1, 31)]
-        build_numbers.append('20120917.1')
-        build_numbers.append('20120917.2')
-
-        """
-                '20120701',
-                '20120914',
-                '20120915',
-                '20120916',
-                '20120917',
-                '20120918',
-                ]
-        """
-
-        for release in releases:
-            for arch in arches:
-                for variant in variants:
-                    for build_number in build_numbers:
-                        image, created = Image.objects.get_or_create(
-                                    release=release,
-                                    variant=variant,
-                                    arch=arch,
-                                    build_number=build_number,
-                                    md5='fake{}{}{}{}'.format(build_number, release, variant, arch),
-                                    )
-
-                        now = datetime.datetime.now()
-                        for i in range(100):
-                            for j in range(10):
-                                kernel_init=random.random() * 20
-                                kernel=random.random() * 20
-                                plumbing=random.random() * 20
-                                xorg=random.random() * 20
-                                desktop=random.random() * 20
-
-                                boot = kernel_init + kernel + plumbing + xorg + desktop
-                                ran_at = now - datetime.timedelta(days=i)
-
-                                imageresult = ImageResult.objects.create(
-                                        image=image,
-                                        machine=machine,
-                                        kernel=kernel,
-                                        kernel_init=kernel_init,
-                                        plumbing=plumbing,
-                                        xorg=xorg,
-                                        desktop=desktop,
-                                        boot=boot,
-                                        ran_at=ran_at,
+        machines = [
+                {
+                    'name': 'Dell Optiplex 242',
+                    'mac': '01:02:03:0a:0b',
+                },
+                {
+                    'name': 'Nexus 7',
+                    'mac': '01:02:03:0a:0c',
+                },
+                {
+                    'name': 'Dell 242',
+                    'mac': '01:02:03:0a:0d',
+                },
+                {
+                    'name': 'HP 242',
+                    'mac': '01:02:03:0a:0e',
+                },
+        ]
+
+        for m in machines:
+            machine, created = Machine.objects.get_or_create(
+                    name=m['name'],
+                    mac_address=m['mac']
+                    )
+
+            releases = ['quantal']
+            arches = ['amd64', 'i386']
+            variants = ['server']
+            build_numbers = ['2012{:0>2}{:0>2}'.format(y,x) for y in [4,5,6,7,8,9] for x in range(1, 31)]
+            build_numbers.append('20120917.1')
+            build_numbers.append('20120917.2')
+
+            """
+                    '20120701',
+                    '20120914',
+                    '20120915',
+                    '20120916',
+                    '20120917',
+                    '20120918',
+                    ]
+            """
+
+            for release in releases:
+                for arch in arches:
+                    for variant in variants:
+                        for build_number in build_numbers:
+                            image, created = Image.objects.get_or_create(
+                                        release=release,
+                                        variant=variant,
+                                        arch=arch,
+                                        build_number=build_number,
+                                        md5='fake{}{}{}{}'.format(build_number, release, variant, arch),
                                         )
+
+                            now = datetime.datetime.now()
+                            for i in range(10):
+                                for j in range(10):
+                                    kernel_init=random.random() * 20
+                                    kernel=random.random() * 20
+                                    plumbing=random.random() * 20
+                                    xorg=random.random() * 20
+                                    desktop=random.random() * 20
+
+                                    boot = kernel_init + kernel + plumbing + xorg + desktop
+                                    ran_at = now - datetime.timedelta(days=i)
+
+                                    imageresult = ImageResult.objects.create(
+                                            image=image,
+                                            machine=machine,
+                                            kernel=kernel,
+                                            kernel_init=kernel_init,
+                                            plumbing=plumbing,
+                                            xorg=xorg,
+                                            desktop=desktop,
+                                            boot=boot,
+                                            ran_at=ran_at,
+                                            )

=== modified file 'dashboard/management/__init__.py'
--- dashboard/management/__init__.py	2012-08-29 17:33:22 +0000
+++ dashboard/management/__init__.py	2012-11-28 17:34:20 +0000
@@ -11,13 +11,15 @@
     try_count = 0
 
     while try_count < 5:
-      try:
-        f = urllib2.urlopen(req)
-        response = f.read()
-        f.close()
-        break
-      except urllib2.HTTPError:
-        try_count += 1
+        try:
+            f = urllib2.urlopen(req)
+            response = f.read()
+            f.close()
+            break
+        except urllib2.HTTPError:
+            try_count += 1
+        except urllib2.URLError as e:
+            try_count += 1
 
     if response != '' and as_json:
         return json.loads(response)

=== modified file 'dashboard/management/commands/jenkins_pull.py'
--- dashboard/management/commands/jenkins_pull.py	2012-09-19 17:54:56 +0000
+++ dashboard/management/commands/jenkins_pull.py	2012-11-28 17:34:20 +0000
@@ -23,12 +23,57 @@
         DEBUG_LIMITS,
         )
 
+from dashboard.utah_utils import process_smoke_log
+
 #l = logging.getLogger('django.db.backends')
 #l.setLevel(logging.DEBUG)
 #l.addHandler(logging.StreamHandler())
 
 from dashboard.utils import regexes
 
+def _get_utah_logs(artifacts):
+    """
+    Check if there are files in 'clientlogs'.
+    
+    Returns a list of utah logs found.
+    
+    """
+
+    logs = []
+
+    for artifact in artifacts:
+        logging.debug(artifact['relativePath'])
+        if 'clientlogs' in artifact['relativePath']:
+            logs.append(artifact)
+
+    return logs
+
+def _process_utah_logs(logs, jenkins_build=None, jenkins_url=None, 
+        name=None):
+
+    runs = []
+    builds = []
+    results = []
+
+    for log in logs:
+        logging.debug("log: {}".format(log))
+        run, build, result = process_smoke_log(
+                log['relativePath'],
+                jenkins_build=jenkins_build,
+                jenkins_url=jenkins_url,
+                name=name,
+        )
+
+        if run:
+            runs.append(run)
+        if build:
+            builds.append(build)
+        if result:
+            results.append(result)
+
+    return runs, builds, results
+
+
 class Command(BaseCommand):
     option_list = BaseCommand.option_list + (
             make_option('-I', '--initial-run',
@@ -36,8 +81,12 @@
                 dest='initial_run',
                 default=False,
                 help='Initial run processes all data (takes longer)'),
+            make_option('-w', '--wait-for',
+                dest='waitfor',
+                help='Skip all jobs until the named job is found'),
             )
                 
+    args = "[job name] [job name]..."
     help = 'Pull data from jenkins'
 
     def get_build_number(self, build_url):
@@ -55,48 +104,87 @@
             return None
 
     def handle(self, *args, **options):
+        verbosity = int(options.get('verbosity'))
+        log_level = logging.DEBUG if verbosity > 1 else logging.INFO
+
+        logging.basicConfig(level=log_level)
+
+        job_list = list(args)
+        logging.debug("job_list: {}".format(job_list))
+
         starttime = datetime.now()
         initial_run = options.get('initial_run')
+        waitfor = options.get('waitfor')
+
+        logging.debug("waitfor: {}".format(waitfor))
 
         # for the initial run include older releases as well.
         if initial_run:
             regexes['smoke'] = re.compile(ur"^(lucid|natty|oneiric|precise|quantal)-(desktop|server|alternate)")
 
+        #regexes['smoke'] = re.compile(ur"^(raring)-(desktop)-i386_default")
+        #regexes['smoke'] = re.compile(ur"^(quantal)-(desktop|server|alternate)-")
+        #regexes['utah-smoke'] = re.compile(ur"^(raring)-(desktop)-i386-smoke-default")
+
+
         jobs_top = jenkins_get('https://jenkins.qa.ubuntu.com/api/json')
         jobs = jobs_top['jobs']
 
 
         job_count = 0 # XXX: DEBUG
+        waiting = waitfor is not None
+        logging.info("waiting for job {}".format(waitfor))
         for job in jobs:
+            utah_job = False
             name = job['name']
             url  = job['url']
-            self.stdout.write('Name: %s\n' % name)
-            if regexes['bootspeed'].match(name):
-                self.stdout.write("Bootspeed test %s\n" % name)
-            elif regexes['upgrade'].match(name):
-                self.stdout.write("Upgrade test %s\n" % name)
-            elif regexes['smoke'].match(name):
-                self.stdout.write("Smoke test %s\n" % name)
+
+            # skip jobs until the one we're waiting for
+            if waiting:
+                if name == waitfor:
+                    logging.info("found job {}".format(waitfor))
+                    waiting = False
+                else:
+                    continue
+
+            # if passed a list of jobs skip those that don't match
+            if len(job_list) > 0 and name not in job_list:
+                continue
+
+            if regexes['smoke'].match(name) or regexes['utah-smoke'].match(name):
+                logging.info("Smoke test %s" % name)
                 flavor = 'ubuntu'
                 if regexes['smoke-ec2'].match(name):
                     if regexes['smoke-ec2-daily'].match(name):
                         m = regexes['smoke-ec2-daily-data'].match(name)
                         release, variant = m.group(1,2)
                 else:
-                    m = regexes['smoke-data'].match(name)
+                    # first check if it's a utah based job
+                    m = regexes['utah-smoke-data'].match(name)
                     orig_name = name
-                    release, variant, arch, name = m.group(1,2,3,4)
+                    if m:
+                        release, variant, arch, name = m.group(1,2,3,4)
+                        logging.debug(("Got a UTAH job: ", release, variant, arch, name))
+                        utah_job = True
+                    else:
+                        m = regexes['smoke-data'].match(name)
+                        if m:
+                            release, variant, arch, name = m.group(1,2,3,4)
+                        else:
+                            logging.warn("Strange smoke name: {}"
+                                    .format(name))
+                            continue
 
                     # XXX: debugging limit
                     job_count += 1
                     if DEBUG_LIMITS and job_count > JOB_COUNT_MAX:
                         break
 
-                    self.stdout.write("Release: %s, variant: %s, arch: %s, name: %s\n" % (release, variant, arch, name))
+                    logging.info("Release: %s, variant: %s, arch: %s, name: %s" % (release, variant, arch, name))
 
                     # Ignore static validation per Gema's request via IRC.
                     if name == "static_validation":
-                        self.stdout.write("Skipping static validation test: {}".format(orig_name))
+                        logging.info("Skipping static validation test: {}".format(orig_name))
                         continue
                     
 
@@ -131,6 +219,17 @@
                         if building:
                             continue
 
+                        if utah_job: # A utah job
+                            utah_logs = _get_utah_logs(artifacts)
+
+                            if len(utah_logs) > 0:
+                                runs, builds, results = _process_utah_logs(
+                                        utah_logs,
+                                        jenkins_build=build_number,
+                                        jenkins_url=url,
+                                        name=name,
+                                )
+
                         # XXX: find out what TZ jenkins uses and use correct one here
                         if not initial_run and datetime.now() - build_date > timedelta(days=7):
                             continue
@@ -154,45 +253,56 @@
                         if build_no is None:
                             build_no = build_date.strftime("%Y%m%d ?")
 
-                        self.stdout.write("Build #%s, date: %s, build_no: %s\n" % (build_number, build_date, build_no))
-
-                        run = None
-                        try:
-                            run = Run.objects.get(release=release, flavor=flavor, build_no=build_no)
-                        except Run.DoesNotExist:
-                            run = Run.objects.create(release=release, flavor=flavor, build_no=build_no, ran_at=build_date, test_type=0)
-
-                        build = None
-                        try:
-                            build = run.build_set.get(variant=variant, arch=arch, flavor=flavor)
-                        except Build.DoesNotExist:
-                            build = run.build_set.create(variant=variant, arch=arch, flavor=flavor)
+                        logging.info("Build #%s, date: %s, build_no: %s" % (build_number, build_date, build_no))
 
                         result = None
-                        try:
-                            result = build.result_set.get(name=name, jenkins_build=build_number)
-                            # break
-                        except Result.DoesNotExist:
-                            fail_count, skip_count, pass_count, total_count = (0, 0, 0, 0)
-                            for action in actions:
+                        if not utah_job:
+                            logging.debug("Processing results as jenkins job")
+                            # Start adding objects here.
+
+                            run, new_run = Run.objects.get_or_create(
+                                    release=release,
+                                    flavor=flavor,
+                                    build_no=build_no,
+                                    defaults={
+                                        'ran_at': build_date,
+                                        'test_type': 0,
+                                    },
+                            )
+
+                            build, new_build = run.build_set.get_or_create(
+                                    variant=variant,
+                                    flavor=flavor,
+                                    arch=arch,
+                            )
+
+                            try:
+                                result = build.result_set.get(name=name, jenkins_build=build_number)
+                                # break
+                            except Result.DoesNotExist:
+                                fail_count, skip_count, pass_count, total_count = (0, 0, 0, 0)
+                                for action in actions:
+                                    try:
+                                        fail_count, skip_count, total_count = (action['failCount'], action['skipCount'], action['totalCount'])
+                                        pass_count = total_count - fail_count - skip_count
+                                        break
+                                    except KeyError:
+                                        continue
+
+                                result = build.result_set.create(name=name, jenkins_build=build_number, ran_at=build_date, jenkins_url=build_url, fail_count=fail_count, skip_count=skip_count, pass_count=pass_count, total_count=total_count)
+
+                            for bug_no in lp_bugs:
+                                bug = None
                                 try:
-                                    fail_count, skip_count, total_count = (action['failCount'], action['skipCount'], action['totalCount'])
-                                    pass_count = total_count - fail_count - skip_count
-                                    break
-                                except KeyError:
-                                    continue
-
-                            result = build.result_set.create(name=name, jenkins_build=build_number, ran_at=build_date, jenkins_url=build_url, fail_count=fail_count, skip_count=skip_count, pass_count=pass_count, total_count=total_count)
-
-                        for bug_no in lp_bugs:
-                            bug = None
-                            try:
-                                bug = Bug.objects.get(bug_no = bug_no)
-                            except Bug.DoesNotExist:
-                                bug = Bug.objects.create(bug_no=bug_no, status='unknown')
-                            if bug not in list(result.bugs.all()):
-                                result.bugs.add(bug)
-                                result.save()
+                                    bug = Bug.objects.get(bug_no = bug_no)
+                                except Bug.DoesNotExist:
+                                    bug = Bug.objects.create(bug_no=bug_no, status='unknown')
+                                if bug not in list(result.bugs.all()):
+                                    result.bugs.add(bug)
+                                    result.save()
+
+                        if result:
+                            results = [result]
 
                         for artifact in artifacts:
                             path = artifact['relativePath']
@@ -207,15 +317,11 @@
                                     'remote_url': remote_url,
                                     'path': path,
                                     }
-                            try:
-                                res_log = result.resultlog_set.get(**art_args)
-                            except ResultLog.DoesNotExist:
-                                result.resultlog_set.create(**art_args)
-
-            elif regexes['kernel_sru'].match(name):
-                self.stdout.write("Kernel SRU test %s\n" %name)
+                            for result in results:
+                                res_log, new_res = result.resultlog_set.get_or_create(
+                                            **art_args)
 
 
         endtime = datetime.now()
 
-        self.stdout.write("time: {}\n".format(endtime - starttime))
+        logging.info("time: {}".format(endtime - starttime))

=== modified file 'dashboard/utah_utils.py'
--- dashboard/utah_utils.py	2012-11-19 21:06:18 +0000
+++ dashboard/utah_utils.py	2012-11-28 17:34:20 +0000
@@ -4,7 +4,7 @@
 
 import logging
 
-from utah.parser import UTAHParser
+from utah.parser import UTAHParser, ParserError
 from dashboard.models import (
         Build,
         Run,
@@ -13,11 +13,15 @@
 
 FLAVOR='ubuntu'
 
-def process_smoke_log(logfile, jenkins_build=None, jenkins_url=None):
+def process_smoke_log(logfile, jenkins_build=None, jenkins_url=None,
+        name=None):
     """
     Parse a utah client log for smoke test results.
     """
 
+    run = None
+    build = None
+    result = None
     logfile_path = logfile
 
     if jenkins_url is not None and jenkins_build is not None:
@@ -35,10 +39,24 @@
         jenkins_url = "http://jenkins.qa.ubuntu.com/"
 
     parser = UTAHParser()
-    data = parser.parse(logfile_path)
+
+    try:
+        data = parser.parse(logfile_path)
+    except ParserError as e:
+        logging.error(e)
+        return run, build, result
+
+    if data is None:
+        logging.warn("Unable to parse {}".format(logfile_path))
+        return run, build, result
 
     build_number = data['build_number']
-    name = data['name']
+
+    # Use the name from the logs if there is one, if there isn't one
+    # and one is passed in use that.
+    if name is None or data['name'] != 'unnamed':
+        name = data['name']
+
     arch = data['arch']
     ran_at = data['ran_at']
     release = data['release']
@@ -61,6 +79,9 @@
             build_no=build_number,
             release=release,
             flavor=FLAVOR,
+            defaults={
+                'ran_at': ran_at,
+            },
     )
 
     logging.debug("run: {}".format(run))
@@ -69,8 +90,8 @@
     build, new_build = Build.objects.get_or_create(
             run=run,
             flavor=FLAVOR,
+            variant=install_type,
             arch=arch,
-            variant=install_type,
     )
 
     logging.debug("build: {}".format(build))
@@ -86,8 +107,10 @@
             #XXX: need to be provided from outside
             skip_count=0,
             jenkins_build=jenkins_build,
-            jenkins_url=jenkins_url,
+            jenkins_url="{}{}".format(jenkins_url, jenkins_build),
     )
 
     logging.debug("result: {}".format(result))
     logging.debug("new_result: {}".format(new_result))
+
+    return run, build, result

=== modified file 'dashboard/utils.py'
--- dashboard/utils.py	2012-09-19 17:54:14 +0000
+++ dashboard/utils.py	2012-11-28 17:34:20 +0000
@@ -14,14 +14,17 @@
         'bootspeed': re.compile(ur".*-bootspeed-.*"),
         'upgrade': re.compile(ur".*-upgrade-.*"),
         'smoke': re.compile(ur"^(precise|quantal)-(desktop|server|alternate)"),
+        'utah-smoke': re.compile(ur"^(raring)-(desktop|server)-.*-smoke-.*"),
+        'utah-smoke-data': re.compile(ur"^(raring)-(desktop|server)-([^-]*)-smoke-(.*)"),
+
         'smoke-ec2': re.compile(ur"(.*)-(.*)-ec2.*"),
         'smoke-ec2-daily': re.compile(ur".*-daily$"),
         'smoke-ec2-daily-data': re.compile(ur"^(.*)-(.*).*"),
-        'smoke-data': re.compile(ur"^(.*)-(.*)-([^_]*)_(.*)"),
+        'smoke-data': re.compile(ur"^(precise|quantal)-(.*)-([^_]*)_(.*)"),
         'smoke-buildid': re.compile(r"^[0-9]{8}(\.[0-9]+)?$"),
         'smoke-lpbug': re.compile(ur"LP:#[0-9]+$"),
         'smoke-build_desc': re.compile(ur"[^,\s]+"),
-        'kernel_sru': re.compile(ur"^(sru_kernel)(|_backport)-(lucid|maveric|natty|oneiric|precise|quantal)(|_lts_hwe)-(.*)-(.*)-(.*).*"),
+        'kernel_sru': re.compile(ur"^(sru_kernel)(|_backport)-(lucid|maveric|natty|oneiric|precise|quantal|raring)(|_lts_hwe)-(.*)-(.*)-(.*).*"),
         'kernel_sru-version': re.compile(ur"^([^,\s]+)(\s+|,)"),
         'lpbug': re.compile(ur"LP:#[0-9]+$"),
         'buildid': re.compile(r"^[0-9]{8}(\.[0-9]+)?$"),