Merge lp:~fginther/auto-package-testing/remove-boottestjob into lp:~canonical-ci-engineering/auto-package-testing/add-boottest-requests

[Francis Ginther]

Remove boottestjob.py, replacing differences with the boottest flag.

Remove boottestjob.py, replacing differences with the boottest flag.

=== modified file 'jenkins/adtjob.py'
--- jenkins/adtjob.py	2015-01-22 19:12:09 +0000
+++ jenkins/adtjob.py	2015-01-29 20:22:15 +0000
@@ -219,32 +219,39 @@
             causes[self.pkgname] = version
             ret = True
 
-        # Verification of the dependencies
-        # Run a test if:
-        #  - No package list is specified and
-        #    - this is a new dependency
-        #    - or version of a dependency in the archive is newer than previous run
-        #  - Or if a package list is specified the following conditions must
-        #  also be met:
-        #    - and the dependency is in the list
-        #    - the version in the archive is newer or equal to the version requested
-        #
-        #  Note that request files use source package names while dependency
-        #  lists returned by get_package_info use binary package names
-        for depname, depver in depends.iteritems():
-            depsrcname = self._get_sourcename(depname)
-            if (not pkglist or
-                (pkglist and depsrcname in pkglist.keys() and
-                 apt_pkg.version_compare(pkglist[depsrcname]['version'], depver) <= 0)):
-                if not depname in self.depends:
-                    logging.info("== New dependency '%s'.", depname)
-                    causes[depsrcname] = depver
-                    ret = True
-                elif apt_pkg.version_compare(self.depends[depname], depver) < 0:
-                    logging.info("== New version of dependency '%s %s'.",
-                                 depname, depver)
-                    causes[depsrcname] = depver
-                    ret = True
+        if self.boottest:
+            # Verification of the dependencies
+            # Run a test if:
+            #  - No package list is specified and
+            #    - this is a new dependency
+            #    - or version of a dependency in the archive is newer than previous run
+            #  - Or if a package list is specified the following conditions must
+            #  also be met:
+            #    - and the dependency is in the list
+            #    - the version in the archive is newer or equal to the version requested
+            #
+            #  Note that request files use source package names while dependency
+            #  lists returned by get_package_info use binary package names
+            # XXX - fginther 20150129
+            # Skip this for boottesting because the reverse dependences
+            # will almost always already be a part of the image being tested
+            # The only exception is when the reverse dependence is also in
+            # the proposed pocket. We'll just acknowledge that these are
+            # skipped for now.
+            for depname, depver in depends.iteritems():
+                depsrcname = self._get_sourcename(depname)
+                if (not pkglist or
+                    (pkglist and depsrcname in pkglist.keys() and
+                     apt_pkg.version_compare(pkglist[depsrcname]['version'], depver) <= 0)):
+                    if not depname in self.depends:
+                        logging.info("== New dependency '%s'.", depname)
+                        causes[depsrcname] = depver
+                        ret = True
+                    elif apt_pkg.version_compare(self.depends[depname], depver) < 0:
+                        logging.info("== New version of dependency '%s %s'.",
+                                     depname, depver)
+                        causes[depsrcname] = depver
+                        ret = True
 
         if ret:
             self.causes = causes

=== modified file 'jenkins/boottest-britney'
--- jenkins/boottest-britney	2015-01-29 01:55:34 +0000
+++ jenkins/boottest-britney	2015-01-29 20:22:15 +0000
@@ -27,7 +27,7 @@
 import subprocess
 import yaml
 import argparse
-from boottestjob import BoottestJob
+from adtjob import AdtJob
 from aptcache import AptCache
 from time import strftime
 from shutil import copyfile
@@ -321,8 +321,9 @@
             deps_path = self.status_pattern % (
                 self.config['release'], self.pocket,
                 self.config['arch'], pkgname)
-            job = BoottestJob(self.cache, deps_path,
-                              use_proposed=self.args.use_proposed)
+            job = AdtJob(self.cache, deps_path,
+                         use_proposed=self.args.use_proposed,
+                         boottest=True)
             if not job.package:
                 job.release = self.config['release']
                 job.pkgname = pkgname
@@ -368,8 +369,9 @@
             deps_path = self.status_pattern % (
                 self.config['release'], self.pocket,
                 self.config['arch'], pkgname)
-            job = BoottestJob(self.cache, deps_path,
-                              use_proposed=self.args.use_proposed)
+            job = AdtJob(self.cache, deps_path,
+                         use_proposed=self.args.use_proposed,
+                         boottest=True)
             if ('causes' in pkgprops and
                     pkgname in pkgprops['causes'] and
                     pkgprops['causes'][pkgname] is None and job.version):

=== removed file 'jenkins/boottestjob.py'
--- jenkins/boottestjob.py	2015-01-29 01:55:34 +0000
+++ jenkins/boottestjob.py	1970-01-01 00:00:00 +0000
@@ -1,430 +0,0 @@
-#! /usr/bin/python
-""" Interface between britney and autopkgtest running on Jenkins
-
-"""
-# Copyright (C) 2012, Canonical Ltd (http://www.canonical.com/)
-#
-# Author: Jean-Baptiste Lallement <[e-mail]>
-#
-# This software is free software: you can redistribute it
-# and/or modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation, either version 3 of
-# the License, or (at your option) any later version.
-#
-# This software is distributed in the hope that it will
-# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty
-# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this software.  If not, see <http://www.gnu.org/licenses/>.
-#
-import logging
-import os
-import sys
-import json
-import apt_pkg
-import subprocess
-import yaml
-from xml.sax.saxutils import escape
-
-if os.path.exists(os.path.join(os.path.dirname(__file__), "../.bzr")):
-    sys.path.insert(0, os.path.dirname(__file__))
-from adtnotify import AdtNotify
-
-try:
-    import jenkins
-except ImportError:
-    sys.stderr.write('python-jenkins is not installed. Jenkins support is '
-                     'disabled\n')
-try:
-    import jinja2
-except ImportError:
-    sys.stderr.write('python-jinja2 is not installed. Jinja support is '
-                     'disabled\n')
-from urllib2 import urlopen
-
-
-ARCHS = ('i386', 'amd64', 'all')
-BINDIR = os.path.dirname(__file__)
-JENKINS_TMPL = 'jenkins_config.xml.tmpl'
-JENKINS_TMPL_ARMHF = 'jenkins_config_armhf.xml.tmpl'
-JENKINS_TMPL_PPC64EL = 'jenkins_config_ppc64el.xml.tmpl'
-DEFAULT_RECIPIENTS = ["[e-mail]",
-                      "[e-mail]",
-                      "[e-mail]"]
-
-
-class BoottestJob(object):
-    '''Class that manages jobs'''
-    def __init__(self, cache, state_path, use_proposed=False, credfile=None):
-        global ARCHS
-
-        self.job_status = {
-            'status': dict([(arch, None) for arch in ARCHS]),
-            'release': None,
-            'package': None,
-            'version': None,
-            'depends': {},  # package:version
-            'causes': {},  # package:version
-        }
-
-        self.state_path = state_path
-        self.cache = cache
-
-        self.status = dict([(arch, None) for arch in ARCHS])
-        self.release = None
-        self.package = None
-        self.pkgname = None
-        self.version = None
-        self.depends = {}
-        self.causes = {}
-
-        self.use_proposed = use_proposed
-        self.load_status(state_path)
-
-        if credfile:
-            self.credentials = self.load_jenkins_credentials(credfile)
-            if self.credentials:
-                # "http://10.189.74.2:8080/job/%s%s-adt-%s/build?token=TOKEN"
-                self.jenkins_url = "%s/job/%%s/build?token=%s" \
-                    % (self.credentials['url'], self.credentials['token'])
-
-    def load_jenkins_credentials(self, path):
-        """ Load Credentials from credentials configuration file """
-        if not os.path.exists(path):
-            return False
-
-        logging.debug('Loading credentials from %s', path)
-        cred = yaml.load(file(path, 'r'))
-        return False if not 'jenkins' in cred else cred['jenkins']
-
-    def load_status(self, state_path):
-        ''' Loads status of a job from configuration file'''
-        logging.debug("Config path: '%s'", state_path)
-        if not os.path.exists(state_path):
-            logging.debug("'%s' doesn't exist", state_path)
-        else:
-            logging.debug("Loading current status")
-            with open(state_path, 'r') as fp:
-                self.job_status = json.load(fp)
-
-            self.status = self.job_status['status']
-            self.release = self.job_status['release']
-            self.pkgname = self.job_status['package']
-            self.package = apt_pkg.SourceRecords()
-            self.package.restart()
-            if not self.package.lookup(self.pkgname):
-                logging.warning('No source package named: %s' % self.pkgname)
-            self.version = self.job_status['version']
-            self.depends = self.job_status['depends']
-            if 'causes' in self.job_status:
-                self.causes = self.job_status['causes']
-
-    def write_status(self):
-        '''Write status to a file'''
-        with open(self.state_path, 'w') as fp:
-            logging.debug("Writing status file to '%s'", self.state_path)
-            self.job_status = {
-                'status': self.status,
-                'release': self.release,
-                'package': self.pkgname,
-                'version': self.version,
-                'depends': self.depends,
-                'causes': self.causes,
-            }
-            json.dump(self.job_status, fp)
-
-    def update_status(self):
-        '''Update package information'''
-        logging.debug('Updating job status')
-        if not self.status:
-            self.status = dict([(arch, 'NEEDRUN') for arch in ARCHS])
-
-        (self.version, pocket, self.depends) = self.get_package_info()
-
-    def get_package_info(self):
-        '''Get package info for the newest version available in the
-           repository from apt
-
-        :return: version, pocket, binary dependencies
-        '''
-        pkg = apt_pkg.SourceRecords()
-        pkg.restart()
-
-        pocket = None
-        version = None
-        depends = None
-
-        while pkg.lookup(self.pkgname):
-            if not version or apt_pkg.version_compare(version, pkg.version) < 0:
-                pocket = pkg.index.describe.split(' ', 4)[1]
-                version = pkg.version
-                binaries = pkg.binaries
-
-        if version:
-            depends = self.binaries_depends(binaries)
-        return (version, pocket, depends)
-
-    def run_required(self, pkglist={}):
-        '''Return True if the job must run
-
-        Conditions are:
-            - No version for the package (never ran before)
-            - Version of the package changed
-            - Version of a dependency changed
-
-        @pkglist: If specified the conditions to run a job are:
-            - package is in the list and no test has been run before (no
-              version)
-            - package is in the list and the version is higher than previous run
-            - package is not in the list but a dependency is and the version is
-              higher than previous run or package has never been tested
-        '''
-        logging.info('Checking status for: %s %s', self.release, self.pkgname)
-        # Find the newest package in enabled repositories
-        (version, pocket, depends) = self.get_package_info()
-
-        if not version:
-            logging.warning("Source package not found: %s", self.pkgname)
-            return False
-
-        logging.debug('Found package version: %s %s', version, pocket)
-
-        # Never ran the test before
-        # If a package list is specified the package must be in the list
-        if (not self.version and
-            (not pkglist or
-             (pkglist and self.pkgname in pkglist.keys()))):
-            logging.info("== New package '%s %s'", self.pkgname, version)
-            self.causes[self.pkgname] = version
-            return True
-
-        # New version of the package
-        # Trigger a test if the version in the archive is newer than last run
-        # Or if a package list is specified, the version in the archive must be
-        # new or equal to the version requested which must be newer than the
-        # last version of the package that have been tested
-        ret = False
-        causes = {}
-
-        if (self.version is not None and apt_pkg.version_compare(self.version, version) < 0 and
-            (not pkglist or
-             (pkglist and self.pkgname in pkglist.keys() and
-              apt_pkg.version_compare(pkglist[self.pkgname]['version'], version) <= 0))):
-            logging.info("== New version of package '%s %s'",
-                         self.pkgname, version)
-            causes[self.pkgname] = version
-            ret = True
-
-        if ret:
-            self.causes = causes
-        else:
-            logging.info('Same version. Skipped')
-        return ret
-
-    def _get_sourcename(self, pkgname):
-        '''Returns a source package name
-
-        :param pkgname: Package name which we want to find source name'''
-
-        try:
-            pkg = self.cache[pkgname]
-            return pkg.candidate.source_name
-        except:
-            logging.error('Unexpected error: %s', sys.exc_info()[0])
-            return pkgname
-
-    def submit(self, dest, force=False):
-        '''Run a job is needed
-
-        :param force: Force run even if there is no dependency update
-        '''
-        if force or self.run_required():
-            logging.debug('Starting job')
-            self.status = dict([(arch, 'RUNNING') for arch in ARCHS])
-            self.update_status()
-            self.write_status()
-            logging.info('Triggering remote job for package %s', self.pkgname)
-            cmd = ['rsync', '-a', self.state_path, dest]
-            logging.debug('Running %s', cmd)
-            try:
-                subprocess.check_call(cmd)
-            except subprocess.CalledProcessError as exc:
-                logging.error('Command failed: %s', exc)
-                return False
-        return True
-
-    def binaries_depends(self, binaries):
-        '''
-        Return a dict with the list of dependencies for all the binary
-        packages built from this source
-        '''
-        depends = {}
-        if not self.pkgname:
-            return None
-
-        virtpkgs = []
-        for pkgname in binaries:
-            try:
-                pkgrec = self.cache[pkgname]
-                for dep in pkgrec.candidate.dependencies:
-                    for basedep in dep.or_dependencies:
-                        try:
-                            if self.cache.is_virtual_package(basedep.name):
-                                virtpkgs.append(basedep)
-                            else:
-                                basedeprec = self.cache[basedep.name]
-                                depends[basedeprec.name] = \
-                                    basedeprec.candidate.version
-                        except KeyError, exc:
-                            logging.warning(exc)
-
-            except KeyError, exc:
-                logging.warning(exc)
-
-        # Process virtual packages
-        # providing packages are only added if there is only 1 package providing
-        # this virtual dependency (for dependency on a version of an ABI) and
-        # if it is not already in the list of binary dependencies
-        deps = set(depends.keys())
-        for vpkg in virtpkgs:
-            ppkgs = self.cache.get_providing_packages(vpkg.name)
-            if (len(ppkgs) > 1):
-                continue
-            vdeps = set([bpkg.name for bpkg in ppkgs])
-            if not deps.intersection(vdeps):
-                # No providing package is in the list of dependencies
-                logging.debug("Adding providing package to list of dependencies: '%s'", vdeps)
-                for bpkg in ppkgs:
-                    depends[bpkg.name] = bpkg.candidate.version
-        logging.debug('Dependency list: %s', depends)
-        return depends
-
-    def execute(self, update=False, dryrun=False):
-        '''Execute a jenkins job'''
-        logging.debug('Executing job')
-
-        # Creates job from template if it doesn't exist
-        # or update it if it already exists
-        # Executes it
-        if not os.path.exists(os.path.join(BINDIR, JENKINS_TMPL)):
-            logging.warning('Template file doesn\'t exist: %s', JENKINS_TMPL)
-            return False
-
-        templates = jinja2.Environment(loader=jinja2.FileSystemLoader(BINDIR))
-
-        opts = ''  # Args to pass to run-adt-test
-        jobname = self.release
-        if self.use_proposed:
-            opts += '-P'
-        jobname += '-adt-' + self.pkgname.replace('+', '-')
-        jobname_armhf = jobname + '-armhf'
-        jobname_ppc64el = jobname + '-ppc64el'
-
-        # Who should be notified
-        notify = AdtNotify(package=self.pkgname, release=self.release)
-        notify.collect()
-        # Space separated list of recipients
-        email_recipients = ",".join(DEFAULT_RECIPIENTS)
-        email_subject = "$DEFAULT_SUBJECT"
-        email_content = "$DEFAULT_CONTENT\n\n"
-
-        for notification in notify.notifications:
-            email_content += "%s %s uploaded on %s by %s <%s>\n" % notification
-            if notification[4] is not None:
-                email_recipients += ',%s' % notification[4]
-
-        ctxt = {'release': self.release,
-                'release_url': "",
-                'test': self.pkgname,
-                'opts': opts,
-                'jobname': jobname,
-                'email_recipients': email_recipients,
-                'email_subject': escape(email_subject),
-                'email_content': escape(email_content)
-                }
-
-        logging.debug('Generating job: %s', jobname)
-        tmpl = templates.get_template(JENKINS_TMPL)
-        tmpl_armhf = templates.get_template(JENKINS_TMPL_ARMHF)
-        tmpl_ppc64el = templates.get_template(JENKINS_TMPL_PPC64EL)
-        config = tmpl.render(ctxt)
-        config_armhf = tmpl_armhf.render(ctxt)
-        config_ppc64el = tmpl_ppc64el.render(ctxt)
-        self.__update_jenkins_job(jobname_armhf, config_armhf, update, dryrun)
-        self.__update_jenkins_job(jobname_ppc64el, config_ppc64el, update, dryrun)
-        if not self.__update_jenkins_job(jobname, config, update, dryrun):
-            return False
-
-        job_url = self.jenkins_url % jobname
-        logging.debug('Job URL: %s', job_url)
-        if not dryrun:
-            urlopen(job_url)
-        else:
-            logging.warning('dry run enabled, job won\'t be executed')
-        return True
-
-    def __update_jenkins_job(self, jobname, jobconfig, update=False,
-                             dryrun=False):
-        """Update a jenkins job"""
-        settings = self.credentials
-        if not settings['url']:
-            logging.error("Please provide a URL to the jenkins instance.")
-            sys.exit(1)
-
-        if 'username' in settings:
-            logging.debug('Logging to jenkins with user %s',
-                          settings['username'])
-            jkh = jenkins.Jenkins(settings['url'],
-                                  username=settings['username'],
-                                  password=settings['password'])
-        else:
-            logging.debug('Logging to jenkins anonymously')
-            jkh = jenkins.Jenkins(settings['url'])
-        if not jkh.job_exists(jobname):
-            logging.info("Creating Jenkins Job %s ", jobname)
-            if dryrun:
-                logging.warning('dry run enabled, job won\'t be created')
-            else:
-                jkh.create_job(jobname, jobconfig)
-        else:
-            # Do not requeue or reconfigure running/queued jobs
-            try:
-                job = jkh.get_job_info(jobname)
-                if job['inQueue']:
-                    logging.debug("Job '%s' already queued. skipped!", jobname)
-                    return False
-                elif job['color'].endswith('_anime'):
-                    logging.debug("Job '%s' already running. skipped!", jobname)
-                    return False
-            except jenkins.JenkinsException as exc:
-                logging.warning('get_job_info failed with exception: %s', exc)
-                return False
-            if update:
-                logging.info("Reconfiguring Jenkins Job %s ", jobname)
-                if dryrun:
-                    logging.warning('dry run enabled, job won\'t be '
-                                    'updated')
-                else:
-                    jkh.reconfig_job(jobname, jobconfig)
-            else:
-                logging.debug('update set to %s. Skipping reconfiguration '
-                              'of %s', update, jobname)
-        return True
-
-    def is_newer_dependency(self, pkgname, pkgversion):
-        ''' Returns true if package in argument is newer than one of the
-        dependency
-        '''
-        if not self.depends:
-            return True
-
-        for depname, depversion in self.depends.iteritems():
-            srcname = self._get_sourcename(depname)
-            if srcname == pkgname:
-                if apt_pkg.version_compare(depversion, pkgversion) < 0:
-                    logging.debug('New dependency')
-                    return True
-
-        return False