Merge lp:~javier.collado/ubuntu-test-runlists/jenkins-smoketest-setup-iso-validation-artifacts into lp:~canonical-ci-engineering/ubuntu-test-runlists/jenkins-smoketest-setup

[Javier Collado]

This branch gets the YAML file from the ISO validation script so that it can be
retrieved by the QA dashboard.

At first I tried to use `utah_logs.sh` in the jenkins job, but not everything
applies to the ISO validation results, so I decided to use the parts that apply
directly in the job.

I've successfully tested locally both cases (job passes/fails), so it should
work fine the lab as well.

[lp:~javier.collado/ubuntu-test-runlists/jenkins-smoketest-setup-iso-validation-artifacts]

46. By Javier Collado on 2012-12-14

Updated job relationships to what have been agreed in today's standup.

- ISO validation jobs triggers default job unconditionally.
- Default job triggers all other jobs only if it passes.

[Javier Collado (javier.collado)]

Since this change has been used to generate the jobs that have been in used for
a quite a long time now, I'm approving and merging this change myself.

review: Approve

=== modified file 'setup-jobs.py'
--- setup-jobs.py	2012-12-12 15:46:05 +0000
+++ setup-jobs.py	2012-12-14 16:16:25 +0000
@@ -111,28 +111,6 @@
     return instance
 
 
-def get_downstream_projects(instance, jobname):
-    """Get downstream projects from a given job name
-
-    :param instance: jenkins instance where job should be available
-    :type instance: jenkins.Jenkins
-    :param jobname: job name for which information should be retrieved
-    :type jobname: str
-    :returns: Downstream project names
-    :rtype: list(str)
-
-    """
-    if not instance.job_exists(jobname):
-        return []
-
-    job_info = instance.get_job_info(jobname)
-    projects = job_info.get('downstreamProjects', [])
-    project_names = [project['name'] for project in projects]
-    project_names = [name for name in project_names
-                     if instance.job_exists(jobname)]
-    return project_names
-
-
 def update_jenkins(instance, dryrun, jobname, config):
     """Create/update a jenkins job.
 
@@ -274,15 +252,25 @@
             logging.info('Processing architecture ' + arch)
             isopath = branch_isopath.format(arch=arch)
 
+            def get_job_name(name):
+                """Map runlist/script name to job name."""
+                jobname = '-'.join((args.series, installtype, arch,
+                                    'smoke', name))
+                return jobname
+            jobnames = {get_job_name(name)
+                        for name, _command in commands.iteritems()}
+
             # Script/runlist job generation
-            job_names = []
             for name, command in commands.iteritems():
+                jobname = get_job_name(name)
+                logging.info('Processing job ' + jobname)
+                if name == 'default':
+                    downstream_jobs = ','.join(jobnames - set((jobname, )))
+                else:
+                    downstream_jobs = None
+
                 command = command.format(arch=arch)
-
                 jobtype = 'script' if scripts_url else 'runlist'
-                jobname = '-'.join((args.series, installtype, arch,
-                                    'smoke', name))
-                logging.info('Processing job ' + jobname)
 
                 template = environment.get_template('smoke.xml.jinja')
                 smokexml = template.render(name=name,
@@ -290,23 +278,23 @@
                                            installtype=installtype,
                                            arch=arch,
                                            isopath=isopath,
+                                           downstream_jobs=downstream_jobs,
                                            scripts_url=scripts_url,
                                            command=command,
                                            node=args.node,
                                            jobtype=jobtype)
                 result, text = update_jenkins(instance, args.dryrun, jobname,
                                               smokexml)
-                if result:
-                    job_names.append(jobname)
-                else:
+                if not result:
                     logging.error(jobname + ' failed to configure: ' + text)
                     failures.append(jobname)
 
             # Static validation job generation
+            default_jobname = get_job_name('default')
+            assert default_jobname in jobnames
             jobname = '-'.join((args.series, installtype, arch, 'smoke',
                                 'iso-validation'))
 
-            project_names = get_downstream_projects(instance, jobname)
             logging.info('Processing job ' + jobname)
             template = environment.get_template('iso_validation.xml.jinja')
             defaultxml = template.render(
@@ -314,7 +302,7 @@
                 installtype=installtype,
                 arch=arch,
                 isopath=isopath,
-                jobs=','.join(project_names + job_names),
+                jobs=default_jobname,
                 imageurl=args.imageurl,
                 node=args.node)
             result, text = update_jenkins(instance, args.dryrun, jobname,

=== modified file 'templates/base.xml.jinja'
--- templates/base.xml.jinja	2012-12-04 15:52:20 +0000
+++ templates/base.xml.jinja	2012-12-14 16:16:25 +0000
@@ -45,7 +45,12 @@
       <publishFailedBuilds>true</publishFailedBuilds>
       <postActions class="vector"/>
     </hudson.plugins.build__publisher.BuildPublisher>
-{% block artifacts %}{% endblock %}
+{% block artifacts %}
+    <hudson.tasks.ArtifactArchiver>
+      <artifacts>**/*</artifacts>
+      <latestOnly>false</latestOnly>
+    </hudson.tasks.ArtifactArchiver>
+{% endblock %}
 {% block publishers %}{% endblock %}
   </publishers>
   <buildWrappers>

=== modified file 'templates/iso_validation.xml.jinja'
--- templates/iso_validation.xml.jinja	2012-12-04 15:55:43 +0000
+++ templates/iso_validation.xml.jinja	2012-12-14 16:16:25 +0000
@@ -20,9 +20,18 @@
 wget -q {{imageurl}}/{{series}}/{{series}}-{{installtype}}-{{arch}}.iso -O $TMPISO
 {{ media_info_cmd('$TMPISO') }}
 chmod 644 $TMPISO
-sudo -u utah -i /usr/share/utah/isotest/iso_static_validation.py --name $TMPISO
+LOG_FILE=iso_validation.log
+set +e
+sudo -u utah -i /usr/share/utah/isotest/iso_static_validation.py --name $TMPISO > $LOG_FILE
+RETCODE=$?
+set -e
+YAML_FILE=$(grep -P "^\t/" $LOG_FILE | sed 's/^\t//')
+if [ -n "$YAML_FILE" ]; then
+    cp $YAML_FILE .
+fi
 sudo -u utah -i cp $TMPISO {{isopath}}
 rm $TMPISO
+exit $RETCODE
         </command>
     </hudson.tasks.Shell>
 {% endblock %}
@@ -31,9 +40,9 @@
     <hudson.tasks.BuildTrigger>
       <childProjects>{{jobs}}</childProjects>
       <threshold>
-        <name>SUCCESS</name>
-        <ordinal>0</ordinal>
-        <color>BLUE</color>
+        <name>FAILURE</name>
+        <ordinal>2</ordinal>
+        <color>RED</color>
       </threshold>
     </hudson.tasks.BuildTrigger>
 {% endblock %}

=== modified file 'templates/smoke.xml.jinja'
--- templates/smoke.xml.jinja	2012-12-12 15:46:05 +0000
+++ templates/smoke.xml.jinja	2012-12-14 16:16:25 +0000
@@ -26,9 +26,15 @@
     </hudson.tasks.Shell>
 {% endblock %}
 
-{% block artifacts %}
-    <hudson.tasks.ArtifactArchiver>
-      <artifacts>**/*</artifacts>
-      <latestOnly>false</latestOnly>
-    </hudson.tasks.ArtifactArchiver>
+{% block publishers %}
+{% if downstream_jobs %}
+    <hudson.tasks.BuildTrigger>
+      <childProjects>{{downstream_jobs}}</childProjects>
+      <threshold>
+        <name>SUCCESS</name>
+        <ordinal>0</ordinal>
+        <color>BLUE</color>
+      </threshold>
+    </hudson.tasks.BuildTrigger>
+{% endif %}
 {% endblock %}