Merge lp:~didrocks/tarmac/jenkins-plugins into lp:tarmac

[Didier Roche]

Add a jenkins plugin to be able to run a job and accept/reject a merge from this job result.

[lp:~didrocks/tarmac/jenkins-plugins]

400. By Didier Roche on 2011-11-23

add a jenkins plugin, to be able to connect tarmac to a jenkins job for a project

Unmerged revisions

400. By Didier Roche on 2011-11-23

add a jenkins plugin, to be able to connect tarmac to a jenkins job for a project

399. By Didier Roche on 2011-11-23

add the new ignore exception

=== added file 'tarmac/plugins/jenkins.py'
--- tarmac/plugins/jenkins.py	1970-01-01 00:00:00 +0000
+++ tarmac/plugins/jenkins.py	2011-11-23 13:31:50 +0000
@@ -0,0 +1,191 @@
+# This file is part of Tarmac.
+#
+# Copyright 2011 Canonical, Ltd.
+#
+# Tarmac is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Tarmac is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Tarmac.  If not, see <http://www.gnu.org/licenses/>.
+"""Tarmac plug-in for integrating with Jenkins."""
+
+from tarmac.exceptions import TarmacMergeError, TarmacMergeIgnore
+from tarmac.hooks import tarmac_hooks
+from tarmac.plugins import TarmacPlugin
+
+import os
+import urllib2
+import stat
+from time import sleep
+
+WAIT_BEFORE_NETWORK_RETRIAL = 1
+NUMBER_OF_RETRY = 3
+
+class JenkinsReject(TarmacMergeError):
+    """Error for when a jenkins rejects the branch."""
+    
+class JenkinsTimeout(TarmacMergeIgnore):
+    """Error for when a jenkins is not reachable."""
+
+class JenkinsCheck(TarmacPlugin):
+    """Tarmac plug-in for integrating with jenkins.
+
+    This plug-in run a jenkins job on the target branch, and then wait
+    on the result (this can take some time). When the job ended, it checks
+    its status and will accordingly will cause the branch merge to fail or
+    not.
+    Set jenkins_job_url to your configuration for the current target to use
+    it.
+    """
+
+    def run(self, command, target, source, proposal):
+        """Start and wait the jenkins job to end."""
+        
+        try:
+            # do not use jenkins if no jenkins_job_url
+            target.config.jenkins_job_url
+        except AttributeError:
+            return
+        try:
+            jenkins_job = JenkinsJob(target.config, target, self.logger)
+            self.jenkins_job = jenkins_job
+        except AttributeError:
+            message = (u'You set a jenkins_job_url configuration, but didn\'t setup other'
+                       u'mandatory settings for jenkins plugin to run. Those are:\n'
+                       u'jenkins_job_url, packaging_branch, jenkins_token')
+            raise JenkinsTimeout(message)
+        
+        # start server job
+        (status, response) = jenkins_job.start()
+        if not status:
+            message = (u'The Jenkins server %(jenkins_url)s is not reachable'
+                       u'(message)s.' %
+                       {'jenkins_url': jenkins_job.url, 'message': response})
+            raise JenkinsTimeout(message)
+        
+        # check it is still running
+        sleep(10)
+        while(jenkins_job.is_running() and jenkins_job.timeout < NUMBER_OF_RETRY):
+            self.logger.debug('Jenkins job %(jenkins_job_url)s/%(job_number)s/console running.' % {
+                'jenkins_job_url': jenkins_job.url,
+                'job_number': jenkins_job.number})
+            sleep(3)
+            running = False
+            
+        if jenkins_job.timeout == NUMBER_OF_RETRY:
+            message = (u'The Jenkins server %(jenkins_url)s is not reachable'
+                       u'for retrieving run status.' % {'jenkins_url': jenkins_job.url})
+            raise JenkinsTimeout(message)
+        
+        # get latest result
+        (status, response) = jenkins_job.last_job_passed()
+        if not status:
+            message = u'The Jenkins job failed.'
+            comment = (u'The Jenkins job %(jenkins_job_url)s/%(job_number)s/console reported an error '
+                       u'when processing this %(source)s branch.\nNot merging it.' % {
+                'source': proposal.source_branch.display_name,
+                'jenkins_job_url': jenkins_job.public_url,
+                'job_number': jenkins_job.number})
+            raise JenkinsReject(message, comment)
+        
+class JenkinsJob(object):
+    """Jenkins Job Server communication object binding"""
+
+    def __init__(self, config, target, logger):
+        super(JenkinsJob, self).__init__()
+        self.logger = logger
+        self.target = target
+        self.tree_path = target.tree.abspath('')
+        self.number = None
+        # let other like jenkins see the tree!
+        os.chmod(self.tree_path, stat.S_IRWXU + stat.S_IROTH + stat.S_IXOTH)
+        self.timeout=0
+        self.url = config.jenkins_job_url
+        self.packaging_branch = config.packaging_branch
+        try:
+            self.public_url = config.jenkins_public_url
+        except AttributeError:
+            self.public_url = self.url
+        self.token = config.jenkins_token
+    
+    def _get_json_from_service(self, data):
+        true = True
+        false = False
+        null = None
+        return eval(data)
+        
+    def start(self):
+        '''start a jenkins job'''
+        revno = self.target.bzr_branch.revno()
+        req = urllib2.Request("%(url)s/buildWithParameters?token=%(token)s&dir=%(dir)s&branch=%(packaging_branch)s&trunkrev=%(revno)s"
+               % {'url': self.url, 'token': self.token, 'dir': self.tree_path, 'revno': revno,
+                  'packaging_branch':  self.packaging_branch})
+        trial_inc = 0
+        while(trial_inc < NUMBER_OF_RETRY):
+            try:
+                response = urllib2.urlopen(req)
+                error_msg = None
+                break
+            except urllib2.URLError, e:
+                error_msg = e
+                trial_inc += 1
+                sleep(WAIT_BEFORE_NETWORK_RETRIAL)
+        if error_msg:
+            message = (u'The Jenkins server %(jenkins_url)s is not reachable'
+                       u'(message)s.' % {'jenkins_url': self.url, 'message': error_msg})
+            raise JenkinsTimeout(message)
+        data = response.read()
+        #if data == 'OK':
+        return (True, 'Job started')
+        #return (False, data)
+        
+    def is_running(self):
+        '''check if the current job is still running'''
+        req = urllib2.Request("%(url)s/lastBuild/api/json" % {'url': self.url})
+        try:
+            response = urllib2.urlopen(req)
+        except urllib2.URLError, e:
+            self.logger.debug('Can\'t get an answer for %(url)s, incrementing the '
+                              'timeout counter.')
+            self.timeout += 1
+            return True # Assuming still busy
+        self.timeout=0 # got an answer, reset the timeout
+        data = response.read()
+        if not self.number:
+            self.number = self._get_json_from_service(data)['number']
+        if self.number != self._get_json_from_service(data)['number']:
+            return False
+        return self._get_json_from_service(data)['building']
+
+        
+    def last_job_passed(self):
+        '''check if the last jenkins job passed, return as well the response if not'''
+        req = urllib2.Request("%(url)s/lastBuild/api/json" % {'url': self.url})
+        trial_inc = 0
+        while(trial_inc < NUMBER_OF_RETRY):
+            try:
+                response = urllib2.urlopen(req)
+                error_msg = None
+                break
+            except urllib2.URLError, e:
+                error_msg = e
+                trial_inc += 1
+                sleep(WAIT_BEFORE_NETWORK_RETRIAL)
+        if error_msg:
+            message = (u'The Jenkins server %(jenkins_url)s is not reachable'
+                       u'(message)s.' % {'jenkins_url': self.url, 'message': error_msg})
+            raise JenkinsTimeout(message)
+        data = response.read()
+        if self._get_json_from_service(data)['result'] == 'SUCCESS':
+            return (True, 'Everything is ok')
+        return (False, data)
+
+tarmac_hooks['tarmac_pre_commit'].hook(
+    JenkinsCheck(), 'Jenkins integration check plug-in')
+

=== added file 'tarmac/plugins/tests/test_jenkins.py'
--- tarmac/plugins/tests/test_jenkins.py	1970-01-01 00:00:00 +0000
+++ tarmac/plugins/tests/test_jenkins.py	2011-11-23 13:31:50 +0000
@@ -0,0 +1,119 @@
+# Copyright 2011 Canonical, Ltd.
+#
+# This file is part of Tarmac.
+#
+# Tarmac is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3 as
+# published by the Free Software Foundation.
+#
+# Tarmac is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Tarmac.  If not, see <http://www.gnu.org/licenses/>.
+"""Tests for the Jenkins Integration plug-in."""
+
+from tarmac.plugins.jenkins import JenkinsCheck, JenkinsTimeout, JenkinsReject
+from tarmac.tests import TarmacTestCase
+from tarmac.tests.mock import Thing
+
+import os
+import shutil
+import tempfile
+
+class JenkinsCheckTests(TarmacTestCase):
+    """Test the jenkins integration."""
+
+    def setUp(self):
+        super(JenkinsCheckTests, self).setUp()
+        self.jenkins_dir = tempfile.mkdtemp()
+        self.jenkins_token = "footoken"
+        config=Thing(jenkins_job_url="file://%s" % self.jenkins_dir,
+                     packaging_branch='lp:~foo/test/ubuntu',
+                     jenkins_token=self.jenkins_token,
+                     jenkins_public_url="fooo")
+        self.target = Thing(config=config, tree=Thing(abspath=os.path.abspath),
+                            bzr_branch=Thing(revno=lambda: 10))
+        self.proposal = Thing(
+            source_branch = Thing(display_name='lp:foo/bar'))
+        self.command = Thing()
+        self.plugin = JenkinsCheck()
+        
+    def tearDown(self):
+        super(JenkinsCheckTests, self).tearDown()
+        shutil.rmtree(self.jenkins_dir)
+        
+    def generate_jenkins_mock_server(self, run_status=None, building=None, last_job_status=None):
+        if run_status:
+            run_test_file = ('%(url)s/buildWithParameters?token=%(token)s&dir=%(dir)s&branch=%(branch)s&trunkrev=%(revno)s'
+                   % {'url': self.jenkins_dir, 'token': self.jenkins_token, 'dir': self.target.tree.abspath(''),
+                      'revno': self.target.bzr_branch.revno(), 'branch': self.target.config.packaging_branch})
+            os.makedirs(os.path.dirname(run_test_file))
+            with open(run_test_file, 'w') as f:
+                f.write("")
+        if building or last_job_status:
+            os.makedirs(self.jenkins_dir + '/lastBuild/api/')
+            with open(self.jenkins_dir + '/lastBuild/api/json', 'w') as f:
+                f.write('{"actions":[{"parameters":[{"name":"RELEASE","value":"oneiric"},{"name":"VARIANT","value":"alternate"},{"name":"ARCH","value":"i386"},{"name":"PRESEED","value":"oem"},{"name":"FLAVOR","value":"ubuntu"}]},{"causes":[]},{}],"artifacts":[],"building":%(building)s,"description":null,"duration":2701176,"fullDisplayName":"oneiric-alternate-i386_oem #65","id":"2011-11-07_07-57-49","keepLog":false,"number":65,"result":"%(last_job_status)s","timestamp":1320652669000,"url":"http://jenkins.qa.ubuntu.com/job/oneiric-alternate-i386_oem/65/","builtOn":"aldebaran","changeSet":{"items":[],"kind":null},"culprits":[]}' % {'building': building, 'last_job_status': last_job_status})
+        
+    def test_fail_no_server(self):
+        """Test that the merge actually fail when there is no server."""
+        # no file
+        self.assertRaises(JenkinsTimeout, self.plugin.run, command=self.command,
+                          target=self.target, source=None,
+                          proposal=self.proposal)
+                            
+    def test_fail_running_doesnt_answer(self):
+        """Test that the merge actually fail when is_running() doesn't answer
+           and don't stall."""
+        # no "running" file
+        self.generate_jenkins_mock_server(run_status='OK')
+        self.assertRaises(JenkinsTimeout, self.plugin.run, command=self.command,
+                          target=self.target, source=None,
+                          proposal=self.proposal)
+                          
+    def test_pass(self):
+        """Test that nothing is raised if all finished as excpected."""
+        self.generate_jenkins_mock_server(run_status='OK', building = 'false',
+                                          last_job_status='SUCCESS')
+        self.plugin.run(command=self.command,
+                        target=self.target, source=None,
+                        proposal=self.proposal)
+                        
+    def test_jenkins_not_passed(self):
+        """Test that the merge actually fail if jenkins last job failed"""
+        self.generate_jenkins_mock_server(run_status='OK', building = 'false',
+                                          last_job_status='FAILURE')
+        self.assertRaises(JenkinsReject, self.plugin.run, command=self.command,
+                          target=self.target, source=None,
+                          proposal=self.proposal)
+                          
+    def test_no_jenkins_parameter(self):
+        """Test that nothing happens if there is no jenkins parameter"""
+        self.target = Thing(config=Thing())
+        self.plugin.run(command=self.command, target=self.target, source=None,
+                        proposal=self.proposal)
+
+    def test_jenkins_fail_if_only_jenkins_url(self):
+        """Test that jenkins fails if there is only the jenkins url but no other mandatory parameters"""
+        self.target = Thing(config=Thing(jenkins_job_url="file://%s" % self.jenkins_dir))
+        self.assertRaises(JenkinsTimeout, self.plugin.run, command=self.command,
+                          target=self.target, source=None,
+                          proposal=self.proposal)
+
+    def test_no_jenkins_public_url(self):
+        """Test that if no public url is given to jenkins, the jenkins job is used"""
+        config = Thing(jenkins_job_url="file://%s" % self.jenkins_dir,
+                       packaging_branch='lp:~foo/test/ubuntu',
+                      jenkins_token=self.jenkins_token)
+        self.target = Thing(config=config, tree=Thing(abspath=os.path.abspath),
+                            bzr_branch=Thing(revno=lambda: 10))
+        self.generate_jenkins_mock_server(run_status='OK', building = 'false',
+                                          last_job_status='SUCCESS')
+        self.plugin.run(command=self.command,
+                        target=self.target, source=None,
+                        proposal=self.proposal)
+        self.assertEqual(self.plugin.jenkins_job.url, self.plugin.jenkins_job.public_url)
+