Merge lp:~abentley/ci-director/remove-ci-director into lp:ci-director

[Aaron Bentley]

Start removing obsolete ci-director functionality.

This branch begins removing obsolete functionality from CI Director.

Now that cidirector.py is no longer being run, only functionality imported by start_builds.py or update_outcomes.py is still needed.

This branch removes the execute bit, the shebang, and the symlink, since ci-director will no longer be executible.

It removes main(), build_revision(), schedule_builds(), and their associated tests.

[Curtis Hovey (sinzui)]

Thank you.

review: Approve (code)

=== removed symlink 'ci-director'
=== target was u'cidirector/cidirector.py'
=== modified file 'cidirector/cidirector.py' (properties changed: +x to -x)
--- cidirector/cidirector.py	2017-01-23 15:37:35 +0000
+++ cidirector/cidirector.py	2017-02-02 17:57:29 +0000
@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 from argparse import ArgumentParser
 from collections import namedtuple
 from contextlib import contextmanager
@@ -15,8 +14,6 @@
 import re
 import shlex
 from smtplib import SMTP
-from socket import setdefaulttimeout
-import sys
 from time import sleep
 import urllib2
 
@@ -47,11 +44,9 @@
     PENDING,
     SUCCEEDED,
     StateFile,
-    StateFileInUse,
     TERMINAL_STATUS,
 )
 from utility import (
-    log_exceptions,
     S3Storage,
     )
 
@@ -865,61 +860,6 @@
         active.extend(item['task']['name'] for item in queued)
         return active
 
-    def schedule_builds(self, server_info):
-        """Request builds for appropriate jobs.
-
-        Can schedule BuildRevisionJob, ResourcefulJob, RecordResultJob.
-        :param server_info: ServerInfo for the current Jenkins.
-        """
-        self.adopt_builds(server_info)
-        active_jobs = set(self.get_active_jobs(server_info))
-        self.logger.info('Active jobs: %s', ', '.join(sorted(active_jobs)))
-        candidate_jobs = []
-        self.update_builds()
-        judge = ResultJudge(self.logger, self.state_file, self.jenkins)
-        all_results_ready = judge.finalize(self.mailer)
-        job = RecordResultJob(
-            self.logger, self.state_file, self.jenkins, self.mailer)
-        judged = job.maybe_build()
-        if all_results_ready or judged:
-            self.state_file.finish_revision(datetime.utcnow())
-            # Disabled in favour of update-outcomes.
-            # self.state_file.build_info_to_s3()
-        resourcefuljobs = ResourcefulJob.get_available_jobs(
-            server_info, self.jenkins, self.state_file, self.logger)
-        for r_job in resourcefuljobs:
-                candidate_jobs.append(r_job)
-        candidate_jobs.append(BuildRevisionJob(
-            self.repo_path, self.branches, self.ignore_branches, self.logger,
-            self.state_file, self.jenkins))
-        for job in candidate_jobs:
-            if job.maybe_build(active_jobs):
-                # We create a new set so that tests that access the parameters
-                # of maybe_build don't get mutated results.
-                active_jobs = active_jobs.union({job.job_id})
-        if should_update_health(self.jenkins, active_jobs):
-            jobs = list_cloud_health(server_info, self.jenkins, self.logger)
-            self.update_cloud_health(jobs)
-
-    def adopt_builds(self, server_info):
-        """Adopt any builds for the build-revision not already recorded."""
-        # We happen to know that branch will not be used here.
-        job_instances = [BuildRevisionJob(self.repo_path, None, None,
-                         self.logger, self.state_file, self.jenkins)]
-        job_instances.extend(
-            Job.make_subclass(si_job['name'], self.logger, self.state_file,
-                              self.jenkins)
-            for si_job in server_info.info['jobs'])
-        for job in job_instances:
-            if job is None:
-                continue
-            job_info = JobInfo.from_jenkins(self.jenkins, job.job_id)
-            if job_info.current_build_number is None:
-                continue
-            build_info = self.jenkins.get_build_info(
-                job.job_id, job_info.current_build_number)
-            job.adopt_build(build_info)
-
     def update_builds(self):
         """Record the status of jobs that are not currently building."""
         self.logger.info('Updating job outcomes.')
@@ -958,19 +898,6 @@
                 version, job, status, info['timestamp'], info['duration'])
 
 
-def build_revision(branches, ignore_branches):
-    """Top-level logic for building jobs.
-
-    For a supplied list of branches, it
-    - Uses the config to access Jenkins, and loads the current state.
-    - Runs jobs using CIDirector.
-    - Saves the updated state.
-    """
-    with CIDirector.stateful(branches, ignore_branches) as director:
-        server_info = ServerInfo(director.jenkins.get_info())
-        director.schedule_builds(server_info)
-
-
 def list_cloud_health(si, jenkins, logger):
     """Iterate through CloudHealthJobs for every relevent job on Jenkins."""
     for job in si.info['jobs']:
@@ -1028,21 +955,3 @@
     root_logger.addHandler(s_handler)
     if verbose:
         root_logger.setLevel(logging.INFO)
-
-
-def main(argv=None):
-    setdefaulttimeout(30)
-    args = get_arg_parser().parse_args(argv)
-    setup_logging(args.log_path, args.log_count, args.verbose)
-    logger = logging.getLogger('cidirector')
-    with log_exceptions(logger) as result:
-        try:
-            build_revision(args.branch, args.ignore)
-        except StateFileInUse:
-            logger.warning('State file already in use.')
-            return 1
-    return result.exit_status
-
-
-if __name__ == '__main__':
-    sys.exit(main())

=== modified file 'cidirector/tests/test_cidirector.py'
--- cidirector/tests/test_cidirector.py	2017-01-23 15:37:35 +0000
+++ cidirector/tests/test_cidirector.py	2017-02-02 17:57:29 +0000
@@ -6,7 +6,6 @@
     )
 import logging
 import os
-import sys
 from StringIO import StringIO
 from textwrap import dedent
 from unittest import TestCase
@@ -30,7 +29,6 @@
     get_buildvars_file,
     Job,
     JobInfo,
-    main,
     Mailer,
     NoSuchBranch,
     list_cloud_health,
@@ -306,154 +304,6 @@
         self.assertEqual(['foo', 'bar', 'baz', 'qux'],
                          director.get_active_jobs(si))
 
-    def make_schedule_builds_director(self, branches, active=True,
-                                      published=True, server_info=None):
-        state_file = StateFile()
-        state_file.start_revision('a', 1, 'b', '1.6', 2)
-        if published:
-            state_file.publication_job().update_from_build_result('SUCCESS')
-        version = 2 if active else 3
-        jenkins_dict = {
-            PUBLISH_REVISION: {
-                'build_number': 5,
-                'description': '[ci-director]\n requires: build-revision'},
-            REVISION_RESULTS: {'build_number': 1},
-            'aws-deploy': {
-                'build_number': 5,
-                'description': '[ci-director]\n requires: build-revision'},
-        }
-        if server_info is not None:
-            for entry in server_info.info['jobs']:
-                jenkins_dict.setdefault(entry['name'], {
-                    'build_number': 277,
-                })
-        jenkins = FakeJenkins(jenkins_dict, current_version=version)
-        if not active:
-            jenkins.set_build_result(BUILD_REVISION, 2, 'SUCCESS')
-        return CIDirector(jenkins, state_file, branches,
-                          repo_path='hubgit.com/mumu/mumu')
-
-    def test_schedule_builds_uses_get_active_jobs(self):
-        director = CIDirector(FakeJenkins({
-            REVISION_RESULTS: {'build_number': 2}
-        }, current_version=5), StateFile())
-        server_info = make_server_info('idle')
-        director.state_file.start_revision('a', 54, 'b', '1.25', 5)
-        # get_determine_result calls get_available_jobs, but doesn't count
-        # because it doesn't run those jobs.
-        with patch.object(
-                ResultJudge, 'get_candidate_determine_result_jobs',
-                return_value=[]):
-            with patch.object(
-                    director, 'get_active_jobs',
-                    return_value=[BUILD_REVISION]) as gaj_mock:
-                with patch.object(director.state_file, 'build_info_to_s3'):
-                    director.schedule_builds(server_info)
-        gaj_mock.assert_called_with(server_info)
-
-    def test_schedule_builds_respects_branch_order(self):
-        bar_url = 'gitbranch:bar:hubgit.com/mumu/mumu'
-        foo_url = 'gitbranch:foo:hubgit.com/mumu/mumu'
-        director = self.make_schedule_builds_director([bar_url, foo_url])
-        ls_remote_out = dedent("""\
-            rev-1\trefs/heads/bar
-            rev-2\trefs/heads/foo
-            """)
-        with patch('subprocess.check_output', return_value=ls_remote_out):
-            director.schedule_builds(make_server_info('idle'))
-        self.assertEqual(
-            director.jenkins.calls['build_job'],
-            [((BUILD_REVISION, {
-                'branch': bar_url, 'revision': 'rev-1'
-            }), {})])
-        director = self.make_schedule_builds_director([foo_url, bar_url])
-        with patch('subprocess.check_output', return_value=ls_remote_out):
-            director.schedule_builds(make_server_info('idle'))
-        self.assertEqual(
-            director.jenkins.calls['build_job'],
-            [((BUILD_REVISION, {
-                'branch': foo_url, 'revision': 'rev-2'
-            }), {})])
-
-    def test_schedule_builds_updates_builds(self):
-        director = self.make_schedule_builds_director([])
-        with patch.object(director, 'update_builds') as ub_mock:
-            with patch('subprocess.check_output'):
-                director.schedule_builds(make_server_info('idle'))
-        ub_mock.assert_called_with()
-
-    def test_schedule_builds_records_results(self):
-        director = self.make_schedule_builds_director([])
-        with patch.object(RecordResultJob, 'maybe_build') as mb_mock:
-            with patch.object(director.state_file, 'build_info_to_s3'):
-                with patch('subprocess.check_output'):
-                    director.schedule_builds(make_server_info('idle'))
-        mb_mock.assert_called_with()
-
-    def test_schedule_builds_publishes(self):
-        director = self.make_schedule_builds_director([], active=True)
-        with patch.object(ResourcefulJob, 'maybe_build') as mb_mock:
-            si = make_server_info('idle', buildable=[PUBLISH_REVISION])
-            director.schedule_builds(si)
-        mb_mock.assert_called_with(set())
-
-    def test_schedule_builds_inactive_not_publishes(self):
-        director = self.make_schedule_builds_director([], active=False)
-        with patch.object(ResourcefulJob, 'maybe_build') as mb_mock:
-            with patch('subprocess.check_output'):
-                director.schedule_builds(make_server_info('idle'))
-        self.assertEqual(mb_mock.call_count, 0)
-
-    def test_schedule_builds_schedules_resourceful_tests_unique(self):
-        server_info = make_server_info('idle', buildable=['package-foo'])
-        director = self.make_schedule_builds_director(
-            [], server_info=server_info)
-        description = "[ci-director]\n requires: build-revision"
-        director.jenkins.job_info['package-foo']['description'] = description
-        with patch.object(ResourcefulJob, 'maybe_build') as mb_mock:
-            director.schedule_builds(server_info)
-        mb_mock.assert_called_once_with(set())
-
-    def test_schedule_builds_adopts_builds(self):
-        director = self.make_schedule_builds_director([])
-        server_info = make_server_info('idle')
-        with patch.object(director, 'adopt_builds') as ab_mock:
-            with patch('subprocess.check_output'):
-                director.schedule_builds(server_info)
-        ab_mock.assert_called_with(server_info)
-
-    def test_schedule_builds_calls_send_final_mail(self):
-        director = self.make_schedule_builds_director([])
-        server_info = make_server_info('idle')
-        with patch.object(ResultJudge, 'finalize') as finalize_mock:
-            with patch.object(director.state_file, 'build_info_to_s3'):
-                with patch('subprocess.check_output'):
-                    director.schedule_builds(server_info)
-        finalize_mock.assert_called_with(director.mailer)
-
-    def test_schedule_builds_no_write_results_to_s3_without_final_data(self):
-        director = self.make_schedule_builds_director([])
-        with patch.object(ResultJudge, 'finalize', return_value=False):
-            with patch.object(RecordResultJob, 'maybe_build',
-                              return_value=False):
-                with patch.object(director.state_file,
-                                  'build_info_to_s3') as s3_mock:
-                    with patch('subprocess.check_output'):
-                        director.schedule_builds(make_server_info('idle'))
-        self.assertEqual(0, s3_mock.call_count)
-
-    def test_schedule_builds_completes_unfinished_tests_with_new_rev(self):
-        # A test suite is finished even when a new revision is available.
-        server_info = make_server_info(
-            'idle', buildable=['aws-deploy'])
-        director = self.make_schedule_builds_director(
-            ['branch-url'], server_info=server_info, published=True)
-        director.schedule_builds(server_info)
-        # build_job is not called for the build-revision job.
-        self.assertEqual(
-            [(('aws-deploy', {'revision_build': 2}), {})],
-            director.jenkins.calls['build_job'])
-
     def test_from_config(self):
         director = CIDirector.from_config({
             'jenkins_url': 'http://192.168.1.1:8080',
@@ -646,57 +496,6 @@
         director.update_builds()
         self.assertEqual(sfj.get_status(), SUCCEEDED)
 
-    def test_adopt_builds(self):
-        jenkins = FakeJenkins({
-            'foo-deploy': {
-                'build_number': 5,
-                'description': '[ci-director]\n requires: build-revision'},
-            BuildRevisionJob.job_id: {'build_number': 2},
-        })
-        sf = self.get_state_file()
-        director = CIDirector(jenkins, sf)
-        with patch.object(ResourcefulJob, 'adopt_build') as ab_mock:
-            director.adopt_builds(make_server_info('idle'))
-            self.assertEqual(ab_mock.call_count, 0)
-            director.adopt_builds(
-                make_server_info(
-                    'idle', buildable=['foo-deploy']))
-        ab_mock.assert_called_with({
-            'result': 'SUCCESS',
-            'number': 5,
-            'building': False,
-            'artifacts': [],
-            'timestamp': 0,
-            'url': 'fakettp://fake.fake/job/foo-deploy/5'
-        })
-
-    def test_adopt_builds_unbuilt(self):
-        info = ServerInfo({'jobs': [{
-            'name': 'aws-deploy',
-        }]})
-        jenkins = FakeJenkins({
-            BuildRevisionJob.job_id: {'build_number': 2},
-            'aws-deploy': {
-                'build_number': 2,
-                'description': '[ci-director]\n requires: build-revision',
-                'lastBuild': None}
-        })
-        jenkins.job_info['aws-deploy'] = {'lastBuild': None}
-        sf = StateFile()
-        director = CIDirector(jenkins, sf)
-        director.adopt_builds(info)
-        self.assertEqual(
-            [c[0][0] for c in jenkins.calls['get_build_info']],
-            ['build-revision'])
-        jenkins.job_info['aws-deploy'] = {
-            'lastBuild': {'number': 1},
-            'description': '[ci-director]\n requires: build-revision'}
-        jenkins.build_info['aws-deploy'] = {1: {}}
-        director.adopt_builds(info)
-        self.assertEqual(
-            [c[0][0] for c in jenkins.calls['get_build_info']],
-            ['build-revision', 'build-revision', 'aws-deploy'])
-
     def update_cloud_health_check(self, result, status):
         foo = CloudHealthJob(name='foo', substrate='bar', last_completed=4)
         jenkins = make_cloud_health_jenkins([foo])
@@ -2295,61 +2094,6 @@
         })
 
 
-class TestMain(TestCase):
-
-    def setUp(self):
-        self.logger = logging.getLogger()
-        self.orig_handlers = self.logger.handlers
-        self.logger.handlers = []
-        self.orig_level = self.logger.level
-
-    def tearDown(self):
-        self.logger.handlers = self.orig_handlers
-        self.logger.level = self.orig_level
-
-    def test_main_default(self):
-        with patch('cidirector.cidirector.build_revision',
-                   autospec=True) as br_mock:
-            with patch('cidirector.cidirector.RotatingFileHandler'
-                       ) as rfh_mock:
-                ret = main(['b1', 'b2'])
-                self.assertEqual(ret, 0)
-            br_mock.assert_called_with(['b1', 'b2'], [])
-            self.assertEqual(1, br_mock.call_count)
-        root_logger = logging.getLogger()
-        self.assertEqual(logging.WARNING, root_logger.level)
-        rfh_mock.assert_called_once_with(
-            'ci-director.log', backupCount=2, maxBytes=1024 * 1024)
-        self.assertIs(sys.stderr, root_logger.handlers[1].stream)
-
-    def test_main_logging_config(self):
-        with patch('cidirector.cidirector.build_revision'):
-            with temp_dir() as log_dir:
-                log_path = '{}/ci-director.log'.format(log_dir)
-                ret = main(['-v', '--log-path', log_path, '--log-count',
-                           '5', 'b1'])
-                self.assertEqual(ret, 0)
-        root_logger = logging.getLogger()
-        self.assertEqual(logging.INFO, root_logger.level)
-        self.assertEqual(5, root_logger.handlers[0].backupCount)
-        self.assertEqual(log_path, root_logger.handlers[0].baseFilename)
-
-    def test_main_logs_exception(self):
-        with test_logger() as (log, log_stream):
-            with patch('cidirector.cidirector.setup_logging', autospec=True):
-                with patch('cidirector.cidirector.build_revision',
-                           side_effect=ValueError("mishap")):
-                    ret = main(["branch"])
-                    self.assertEqual(ret, 1)
-        log_pattern = (
-            "(?s)\\Aexception during build revision:\n"
-            "Traceback \\(most recent call last\\):\n"
-            ".*\n"
-            "ValueError: mishap\n\\Z"
-        )
-        self.assertRegexpMatches(log_stream.getvalue(), log_pattern)
-
-
 def make_description(tags=None, section='ci-director'):
     lines = ['[{}]\n'.format(section)]
     if tags is not None: