Merge lp:~pfalcon/linaro-android-build-tools/xpath-jobs into lp:linaro-android-build-tools

[Paul Sokolovsky]

query-job allows to execute an XPath query across all Jenkins config, something which we really need to have a global overview of our Jenkins job sets.

[Paul Sokolovsky (pfalcon)]

Please see README in the patch for usage examples.

[lp:~pfalcon/linaro-android-build-tools/xpath-jobs]

444. By Paul Sokolovsky on 2012-04-26

Remove stray copy&paste content.

445. By Paul Sokolovsky on 2012-04-26

Add a helper function to add/replace a node easily.

446. By Paul Sokolovsky on 2012-04-26

Solve long-standing issue with showing XML changes for review.

Well, we can't operate on raw Jenkins XML files, because it appears to write
it in different styles in different circumstances, so we always get some kind
of spurious changes. All XML-specific diff tools I saw are not human-friendly
(none I tested shows context for example), not helping with the task of easy
sanity review. So, instead normalize both old and new XMLs to common format,
and diff that.

447. By Paul Sokolovsky on 2012-04-26

Polish metavar name in --help output.

448. By Paul Sokolovsky on 2012-04-26

Add mangle script to set expiration policy for a job.

449. By Paul Sokolovsky on 2012-04-26

Use new diffing algorithm for runs against remote Jenkins too.

=== modified file 'utils/mangle-jobs/mangle-jobs'
--- utils/mangle-jobs/mangle-jobs	2012-03-09 10:31:08 +0000
+++ utils/mangle-jobs/mangle-jobs	2012-04-26 11:24:21 +0000
@@ -16,6 +16,7 @@
 import json
 import os
 import sys
+import copy
 import re
 from tempfile import NamedTemporaryFile
 import urllib2
@@ -31,6 +32,8 @@
                      help="File holding Jenkins password")
 optparser.add_option("--really", action="store_true",
                      help="Actually perform changes")
+optparser.add_option("--filter-jobname",
+                     help="Process only jobs matching regex pattern")
 optparser.add_option("--limit", type="int", default=-1,
                      help="Change at most LIMIT jobs")
 optparser.add_option("--file",
@@ -108,36 +111,79 @@
         os.system('diff -u %s %s' % (a.name, b.name))
     print
 
+def indent_tree(elem, level=0):
+    "Indent XML tree for pretty-printing"
+    i = "\n" + level*"  "
+    if len(elem):
+        if not elem.text or not elem.text.strip():
+            elem.text = i + "  "
+        if not elem.tail or not elem.tail.strip():
+            elem.tail = i
+        for elem in elem:
+            indent_tree(elem, level+1)
+        if not elem.tail or not elem.tail.strip():
+            elem.tail = i
+    else:
+        if level and (not elem.tail or not elem.tail.strip()):
+            elem.tail = i
+
+def normalize2text(tree):
+    """Return normalized text representation of XML tree, suitable for
+    diffing with normal diff tool."""
+    normalized = copy.deepcopy(tree)
+    indent_tree(normalized)
+    return tostring(normalized)
+
+def match_job_name(job_name):
+    "Check if job name matches filters which may be specified on command line."
+    if not options.filter_jobname:
+        return True
+    neg = False
+    r = options.filter_jobname
+    if r[0] == "-":
+        neg = True
+        r = r[1:]
+    return bool(re.search(r, job_name)) ^ neg
+
 def process_remote_jenkins():
     jobs = json.load(urllib2.urlopen('http://localhost:9090/jenkins/api/json?tree=jobs[name]'))
     names = [job['name'] for job in jobs['jobs']]
     names = [name for name in names if name == 'blank' or '_' in name]
     limit = options.limit
     for name in names:
+        if not match_job_name(name):
+            continue
         if limit == 0:
             break
         limit -= 1
         print "Processing:" + name
         sys.stdout.flush()
-        text = getJobConfig(name)
-        tree = fromstring(text)
+        org_text = getJobConfig(name)
+        tree = fromstring(org_text)
+        org_normalized = normalize2text(tree)
+
         if mangler(tree) == False:
             continue
-        new_text = render_xml(tree)
+
         if not options.really:
-            show_diff(text, new_text)
+            new_normalized = normalize2text(tree)
+            show_diff(org_normalized, new_normalized)
         else:
+            new_text = render_xml(tree)
             if type(new_text) == type(u""):
                 new_text = new_text.encode("utf8")
             postConfig(str('job/' + name + '/config.xml'), new_text)
 
+
 def main():
     if options.file:
         text = open(options.file).read()
         tree = fromstring(text)
+
+        org_normalized = normalize2text(tree)
         mangler(tree)
-        new_text = render_xml(tree)
-        show_diff(text, new_text)
+        new_normalized = normalize2text(tree)
+        show_diff(org_normalized, new_normalized)
     else:
         process_remote_jenkins()
 

=== modified file 'utils/mangle-jobs/mangle_helper.py'
--- utils/mangle-jobs/mangle_helper.py	2011-12-21 19:48:46 +0000
+++ utils/mangle-jobs/mangle_helper.py	2012-04-26 11:24:21 +0000
@@ -1,5 +1,6 @@
-# Disable Jenkins builtin artifact archiving (part of migration to snapshots.linaro.org)
 import base64
+import lxml.etree
+
 
 def get_build_config(tree):
     tag = tree.xpath("/project/properties//defaultValue")
@@ -14,11 +15,17 @@
         d[k] = v
     return d
 
-def mangle(tree):
-    cfg = get_build_config(tree)
-    if cfg.get("BUILD_TYPE", "build-android") != "build-android":
-        return
-    tags = tree.xpath('//hudson.tasks.ArtifactArchiver')
-    if not tags:
-        return
-    tags[0].getparent().remove(tags[0])
+
+def add_or_replace_node(tree, node_xpath, node_text):
+    new_node = lxml.etree.fromstring(node_text)
+    nodes = tree.xpath(node_xpath)
+    assert len(nodes) < 2, "Please use more selective XPath expression"
+    if nodes:
+        # First, add new node after existing
+        nodes[0].addnext(new_node)
+        # Then, delete the original node
+        nodes[0].getparent().remove(nodes[0])
+    else:
+        parent_xpath, _ = node_xpath.rsplit("/", 1)
+        parent = tree.xpath(parent_xpath)[0]
+        parent.append(new_node)

=== added file 'utils/mangle-jobs/set-build-expiration.mangle'
--- utils/mangle-jobs/set-build-expiration.mangle	1970-01-01 00:00:00 +0000
+++ utils/mangle-jobs/set-build-expiration.mangle	2012-04-26 11:24:21 +0000
@@ -0,0 +1,17 @@
+# Set expiration policy for a job
+
+from lxml.etree import fromstring
+from mangle_helper import *
+
+
+new_node = """\
+  <logRotator>
+    <daysToKeep>90</daysToKeep>
+    <numToKeep>100</numToKeep>
+    <artifactDaysToKeep>-1</artifactDaysToKeep>
+    <artifactNumToKeep>-1</artifactNumToKeep>
+  </logRotator>
+"""
+
+def mangle(tree):
+    add_or_replace_node(tree, "/project/logRotator", new_node)

=== added directory 'utils/query-jobs'
=== added file 'utils/query-jobs/README'
--- utils/query-jobs/README	1970-01-01 00:00:00 +0000
+++ utils/query-jobs/README	2012-04-26 11:24:21 +0000
@@ -0,0 +1,20 @@
+This is a script to query Jenkins job configs, on the level of raw XML files.
+
+Run ./query-jobs --help for list of optoins.
+
+
+Example usage:
+
+./query-jobs //hudson.triggers.TimerTrigger --filter
+
+    List all regularly repeating jobs.
+
+./query-jobs //hudson.triggers.TimerTrigger/spec --filter -v
+
+    List crontab specs for all regularly repeating jobs.
+
+./query-jobs //daysToKeep --filter-not --filter-jobname='-(release|toolchain)'
+
+    List daily (not toolchain and not release) jobs which don't have build
+    expriation configured (i.e. would use up disk space without limit).
+

=== added file 'utils/query-jobs/query-jobs'
--- utils/query-jobs/query-jobs	1970-01-01 00:00:00 +0000
+++ utils/query-jobs/query-jobs	2012-04-26 11:24:21 +0000
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+import sys
+import glob
+import re
+import optparse
+
+from lxml import etree
+
+
+JENKINS_HOME = "/var/lib/jenkins/"
+
+def get_xpath(el):
+    path = []
+    while el is not None:
+        path.insert(0, el.tag)
+        el = el.getparent()
+    return "/" + "/".join(path)
+
+def match_job_name(job_name):
+    "Check if job name matches filters which may be specified on command line."
+    if not options.filter_jobname:
+        return True
+    neg = False
+    r = options.filter_jobname
+    if r[0] == "-":
+        neg = True
+        r = r[1:]
+    return bool(re.search(r, job_name)) ^ neg
+
+def process(query):
+    for fname in glob.glob(options.home + "jobs/*/config.xml"):
+        job_name = fname.split('/')[-2]
+        if not match_job_name(job_name):
+            continue
+
+        doc = etree.parse(fname)
+        results = doc.getroot().xpath(query)
+        if options.filter and len(results) == 0:
+            continue
+        if options.filter_not and len(results) > 0:
+            continue
+
+        if options.verbose:
+            print "===== %s =====" % (fname if options.show_file else job_name)
+        else:
+            print fname if options.show_file else job_name
+        if options.verbose:
+            first = True
+            for el in results:
+                if not first:
+                    print "-" * 20
+                print "Element XPath: %s" % get_xpath(el)
+                print etree.tostring(el)
+                first = False
+
+optparser = optparse.OptionParser(usage="%prog")
+optparser.add_option("-v", "--verbose", action="store_true", help="Verbose")
+optparser.add_option("--show-file", action="store_true", help="Show job config path (default is job name)")
+optparser.add_option("--home", default=JENKINS_HOME, help="Jenkins home dir (%default)")
+optparser.add_option("--filter-jobname", metavar="REGEX", help="Process only jobs matching regex pattern (prefix with '-' for not matching)")
+optparser.add_option("--filter", action="store_true", help="Show jobs matching query")
+optparser.add_option("--filter-not", action="store_true", help="Show jobs not matching query")
+options, args = optparser.parse_args(sys.argv[1:])
+if len(args) != 1:
+    optparser.error("Wrong number of arguments")
+
+process(args[0])