Merge lp:~sseman/juju-ci-tools/schedule-all into lp:juju-ci-tools

[Seman]

This branch updates schedule_hetero_control adding "--all" option to schedule all candidates for client-server testing. This will allow us to retest releases in candidates directory with the correct version information.

Since 'candidate' parameter contains version info rather than branch, the compatibility-control job may not able to find the path to the candidate directory. I updated schedule_hetero_control so that it returns 'candidate_path' parameter. This parameter will contain the path to the candidate directory

When this branch gets approved, I will add "candidate_path" parameter to the http://juju-ci.vapour.ws/job/compatibility-control/configure job. I will also update the shell script.
from:
  new_juju=$(find $HOME/candidate/$candidate -name juju)
to:
  new_juju=$(find $HOME/candidate/$candidate_path -name juju)

[Curtis Hovey (sinzui)]

Thank you.

review: Approve (code)

=== modified file 'schedule_hetero_control.py'
--- schedule_hetero_control.py	2015-07-16 15:57:13 +0000
+++ schedule_hetero_control.py	2015-07-20 18:05:04 +0000
@@ -21,6 +21,9 @@
     parser = ArgumentParser()
     parser.add_argument(
         'root_dir', help='Directory containing releases and candidates dir')
+    parser.add_argument(
+        '--all', action='store_true', default=False,
+        help='Schedule all candidates for client-server testing.')
     add_credential_args(parser)
     args = parser.parse_args(argv)
     return args, get_credentials(args)
@@ -39,10 +42,10 @@
         return json.load(fp)['version']
 
 
-def calculate_jobs(root):
+def calculate_jobs(root, schedule_all=False):
     releases = list(get_releases(root))
     candidates_path = get_candidates_path(root)
-    for candidate_path in find_candidates(root):
+    for candidate_path in find_candidates(root, schedule_all):
         parent, candidate = os.path.split(candidate_path)
         if parent != candidates_path:
             raise ValueError('Wrong path')
@@ -51,12 +54,14 @@
             yield {
                 'old_version': release,
                 'candidate': candidate_version,
-                'new_to_old': 'true'
+                'new_to_old': 'true',
+                'candidate_path': candidate
             }
             yield {
                 'old_version': release,
                 'candidate': candidate_version,
-                'new_to_old': 'false'
+                'new_to_old': 'false',
+                'candidate_path': candidate
             }
 
 
@@ -69,7 +74,8 @@
 
 def main():
     args, credentials = get_args()
-    build_jobs(credentials, args.root_dir, calculate_jobs(args.root_dir))
+    build_jobs(
+        credentials, args.root_dir, calculate_jobs(args.root_dir, args.all))
 
 
 if __name__ == '__main__':

=== modified file 'test_schedule_hetero_control.py'
--- test_schedule_hetero_control.py	2015-07-16 17:34:29 +0000
+++ test_schedule_hetero_control.py	2015-07-20 18:05:04 +0000
@@ -2,8 +2,10 @@
 
 __metaclass__ = type
 
+from datetime import timedelta
 import json
 import os
+from time import time
 from unittest import TestCase
 
 from mock import patch
@@ -41,32 +43,71 @@
             jenkins_mock.assert_called_once_with(
                 'http://localhost:8080', 'jrandom', 'password1')
 
+
+class TestGetCandidateVersion(TestCase):
     def test_get_candidate_version(self):
         with temp_dir() as dir_path:
-            self.make_build_var_file(dir_path)
+            make_build_var_file(dir_path, version='1.24.3')
             version = get_candidate_version(dir_path)
         self.assertEqual(version, '1.24.3')
 
+
+class CalculateJobs(TestCase):
     def test_calculate_jobs(self):
         with temp_dir() as root:
             release_path = os.path.join(root, 'old-juju', '1.20.11')
             os.makedirs(release_path)
-            candidate_path = os.path.join(root, 'candidate', '1.22')
+            candidate_path = os.path.join(root, 'candidate', '1.24')
             os.makedirs(candidate_path)
-            jobs = []
-            self.make_build_var_file(candidate_path)
-            for job in calculate_jobs(root):
-                jobs.append(job)
+            make_build_var_file(candidate_path, version='1.24.3')
+            jobs = list(calculate_jobs(root))
         expected = [{'new_to_old': 'true',
                      'old_version': '1.20.11',
-                     'candidate': '1.24.3'},
+                     'candidate': '1.24.3',
+                     'candidate_path': '1.24'},
                     {'new_to_old': 'false',
                      'old_version': '1.20.11',
-                     'candidate': '1.24.3'}]
+                     'candidate': '1.24.3',
+                     'candidate_path': '1.24'}]
         self.assertItemsEqual(jobs, expected)
 
-    def make_build_var_file(self, dir_path):
-        build_vars = {"version": "1.24.3", "revision_build": "2870"}
-        file_path = os.path.join(dir_path, 'buildvars.json')
-        with open(file_path, 'w') as json_file:
-            json.dump(build_vars, json_file)
+    def test_calculate_jobs_schedule_all(self):
+        with temp_dir() as root:
+            release_path = os.path.join(root, 'old-juju', '1.20.11')
+            os.makedirs(release_path)
+            candidate_path = os.path.join(root, 'candidate', '1.24')
+            os.makedirs(candidate_path)
+            make_build_var_file(candidate_path, '1.24.3')
+            candidate_path_2 = os.path.join(root, 'candidate', '1.23')
+            os.makedirs(candidate_path_2)
+            buildvars_path = make_build_var_file(candidate_path_2, '1.23.3')
+            a_week_ago = time() - timedelta(days=7, seconds=1).total_seconds()
+            os.utime(buildvars_path, (time(), a_week_ago))
+            jobs = list(calculate_jobs(root, schedule_all=False))
+            jobs_schedule_all = list(calculate_jobs(root, schedule_all=True))
+        expected = [{'new_to_old': 'true',
+                     'old_version': '1.20.11',
+                     'candidate': '1.23.3',
+                     'candidate_path': '1.23'},
+                    {'new_to_old': 'false',
+                     'old_version': '1.20.11',
+                     'candidate': '1.23.3',
+                     'candidate_path': '1.23'},
+                    {'new_to_old': 'true',
+                     'old_version': '1.20.11',
+                     'candidate': '1.24.3',
+                     'candidate_path': '1.24'},
+                    {'new_to_old': 'false',
+                     'old_version': '1.20.11',
+                     'candidate': '1.24.3',
+                     'candidate_path': '1.24'}]
+        self.assertItemsEqual(jobs, expected[2:])
+        self.assertItemsEqual(jobs_schedule_all, expected)
+
+
+def make_build_var_file(dir_path, version):
+    build_vars = {"version": version, "revision_build": "2870"}
+    file_path = os.path.join(dir_path, 'buildvars.json')
+    with open(file_path, 'w') as json_file:
+        json.dump(build_vars, json_file)
+    return file_path

=== modified file 'test_utility.py'
--- test_utility.py	2015-07-16 18:23:21 +0000
+++ test_utility.py	2015-07-20 18:05:04 +0000
@@ -120,24 +120,36 @@
 
     def test_find_candidates_old_buildvars(self):
         with temp_dir() as root:
-            candidates_path = get_candidates_path(root)
-            os.mkdir(candidates_path)
-            master_path = os.path.join(candidates_path, 'master')
-            os.mkdir(master_path)
-            buildvars_path = os.path.join(master_path, 'buildvars.json')
-            open(buildvars_path, 'w')
+            _, buildvars_path = self.make_candidates_dir(root, 'master')
             a_week_ago = time() - timedelta(days=7, seconds=1).total_seconds()
             os.utime(buildvars_path, (time(), a_week_ago))
             self.assertEqual(list(find_candidates(root)), [])
 
     def test_find_candidates_artifacts(self):
         with temp_dir() as root:
-            candidates_path = get_candidates_path(root)
+            self.make_candidates_dir(root, 'master-artifacts')
+            self.assertEqual(list(find_candidates(root)), [])
+
+    def test_find_candidates_find_all(self):
+        with temp_dir() as root:
+            master_path, buildvars_path = self.make_candidates_dir(
+                root, '1.23')
+            master_path_2, _ = self.make_candidates_dir(root, '1.24')
+            a_week_ago = time() - timedelta(days=7, seconds=1).total_seconds()
+            os.utime(buildvars_path, (time(), a_week_ago))
+            self.assertItemsEqual(list(find_candidates(root)), [master_path_2])
+            self.assertItemsEqual(list(find_candidates(root, find_all=True)),
+                                  [master_path, master_path_2])
+
+    def make_candidates_dir(self, root, master_name):
+        candidates_path = get_candidates_path(root)
+        if not os.path.isdir(candidates_path):
             os.mkdir(candidates_path)
-            master_path = os.path.join(candidates_path, 'master-artifacts')
-            os.mkdir(master_path)
-            open(os.path.join(master_path, 'buildvars.json'), 'w')
-            self.assertEqual(list(find_candidates(root)), [])
+        master_path = os.path.join(candidates_path, master_name)
+        os.mkdir(master_path)
+        buildvars_path = os.path.join(master_path, 'buildvars.json')
+        open(buildvars_path, 'w')
+        return master_path, buildvars_path
 
 
 class TestWaitForPort(TestCase):

=== modified file 'utility.py'
--- utility.py	2015-07-17 21:43:36 +0000
+++ utility.py	2015-07-20 18:05:04 +0000
@@ -286,7 +286,7 @@
     return os.path.join(root_dir, 'candidate')
 
 
-def find_candidates(root_dir):
+def find_candidates(root_dir, find_all=False):
     candidates_path = get_candidates_path(root_dir)
     a_week_ago = time() - timedelta(days=7).total_seconds()
     for candidate_dir in os.listdir(candidates_path):
@@ -300,7 +300,7 @@
             if e.errno in (errno.ENOENT, errno.ENOTDIR):
                 continue
             raise
-        if stat.st_mtime < a_week_ago:
+        if not find_all and stat.st_mtime < a_week_ago:
             continue
         yield candidate_path