Merge lp:~allanlesage/helipad/slurp-one into lp:helipad/ci

[Allan LeSage]

Enable pulling just one stack.

Here we can slurp just one stack, e.g. head.indicators ; I've tested a pull but have found that we're missing the integration_job being set in Cu2dStack, am addressing in another MP.

[Allan LeSage (allanlesage)]

Syntax for this is ./manage.py jenkins_pull_ci -I head.indicators (or substitute your fav). cjohnston, fginther, what do you think of this? It seems like there would be other options available, e.g. update all builds relevant to a project (launchpad_project)--which is what we're doing for the update script. Maybe these two should be brought into alignment, e.g.

[PS Jenkins bot (ps-jenkins)]

FAILED: Continuous integration, rev:34
No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want a jenkins rebuild you need to trigger it yourself):
https://code.launchpad.net/~allanlesage/helipad/slurp-one/+merge/177956/+edit-commit-message

http://s-jenkins:8080/job/helipad-ci-ci/60/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins:8080/job/helipad-ci-ci/60/rebuild

review: Needs Fixing (continuous-integration)

Unmerged revisions

34. By Allan LeSage on 2013-07-31

Add command-line option and tests.

33. By Allan LeSage on 2013-07-30

Use patch.stopall.

32. By Allan LeSage on 2013-07-30

Improve tests, having split function.

31. By Allan LeSage on 2013-07-30

Move the stack update to its own function.

30. By Allan LeSage on 2013-07-29

Add a comment and shift artifact to config for clarity.

=== modified file 'management/commands/jenkins_pull_ci.py'
--- management/commands/jenkins_pull_ci.py	2013-07-29 17:12:18 +0000
+++ management/commands/jenkins_pull_ci.py	2013-07-31 22:16:40 +0000
@@ -121,7 +121,7 @@
         page = jenkins_get(url, as_json=False)
         return yaml.safe_load(page)
 
-    def _get_artifact_url_list(self):
+    def _get_config_url_list(self):
         # Get the jenkins job which contains the data dump
         jenkapi = jenkinsapi.api.Jenkins(settings.JENKINS_URL)
         job = jenkapi.get_job(CUPSTREAM2DISTRO_CONFIG_DATA_SOURCE)
@@ -138,40 +138,69 @@
                 break
         return artifact_list
 
-    def initial_run(self):
-        url_list = self._get_artifact_url_list()
+    def _retrieve_config_url(self, stack_name, url_list):
+        """Find a regex match for stack_name in url_list."""
 
-        # For each artifact
         for url in url_list:
-            # Parse the stack release and name from the url
-            (release, config_name) = self._extract_release_and_config_name(url)
-            if config_is_on_blacklist(config_name):
-                logger.debug("Skipping cu2d config at {} due to blacklist "
-                             "match.".format(config_name))
-                continue
-
-            data = self._load_artifact(url)
-            # Process the stack
-            if not data:
-                # Stack has no contents
-                logger.debug("No projects found in {}".format(url))
-                continue
-            if 'projects' not in data:
-                # Stack has no projects
-                logger.debug("No projects found in {}".format(url))
-                continue
-            stack_name = self._create_stack(data, release)
-
-            # Process the projects
-            if stack_name:
-                for project_name in data['projects']:
-                    self._create_project(data, release, stack_name,
-                                         project_name)
-        return
+            if re.search("{}.yaml".format(stack_name), url):
+                return url
+        return None
+
+    def stack_initial_run(self, stack_config_url):
+        # Parse the stack release and name from the url
+        (release, config_name) = self._extract_release_and_config_name(
+            stack_config_url)
+        if config_is_on_blacklist(config_name):
+            logger.info("Skipping cu2d config at {} due to blacklist "
+                        "match.".format(config_name))
+            return
+        data = self._load_artifact(stack_config_url)
+
+        # Process the stack
+        if not data:
+            # Stack has no contents
+            logger.debug("No projects found in {}".format(stack_config_url))
+            return
+
+        if 'projects' not in data:
+            # Stack has no projects
+            logger.debug("No projects found in {}".format(stack_config_url))
+            return
+        stack_name = self._create_stack(data, release)
+
+        if not stack_name:
+            logger.debug("Failed to create stack.")
+            return
+
+        # Process the projects
+        for project_name in data['projects']:
+            self._create_project(
+                data, release, stack_name, project_name)
+
+    def initial_run(self, stack_list=[]):
+        """Pull Jenkins builds and jobs relevant to stacks in given list.
+
+        Stacks as 'release.project', e.g. 'head.indicators', 'saucy.mir'.
+        """
+
+        # A Jenkins job dumps all .yaml configs from lp:cupstream2distro-config
+        # as artifacts in a Jenkins workspace, so we can wget 'em, e.g.
+        config_url_list = self._get_config_url_list()
+        if len(stack_list) == 0:
+            pull_url_list = config_url_list
+        else:
+            pull_url_list = []
+            for stack_name in stack_list:
+                config_url = self._retrieve_config_url(
+                    stack_name, config_url_list)
+                if config_url is not None:
+                    pull_url_list.append(config_url)
+        for config_url in pull_url_list:
+            self.stack_initial_run(config_url)
 
     def handle(self, *args, **options):
         if options.get('initial_run'):
-            self.initial_run()
+            self.initial_run(stack_list=args)
         else:
             self.update_projects(project_list=args)
             self.update_cu2d_stacks()
 
=== modified file 'tests/test_jenkins_pull_ci.py'
--- tests/test_jenkins_pull_ci.py	2013-07-28 21:26:11 +0000
+++ tests/test_jenkins_pull_ci.py	2013-07-31 22:16:40 +0000
@@ -55,14 +55,23 @@
             Command,
             '__init__',
             new=command_mock)
+        self.load_artifact_patch = patch.object(
+            jenkins_pull_ci.Command,
+            '_load_artifact')
+        self.load_artifact_mock = self.load_artifact_patch.start()
+        self.get_config_url_list_patch = patch.object(
+            jenkins_pull_ci.Command,
+            '_get_config_url_list')
+        self.get_config_url_list_mock = self.get_config_url_list_patch.start()
         self.command_patch.start()
         self.command = Command()
         self.logger_patch = patch('ci.models.logger')
         self.logger_patch.start()
 
-
     def tearDown(self):
         self.command_patch.stop()
+        self.get_config_url_list_patch.stop()
+        self.load_artifact_patch.stop()
         self.logger_patch.stop()
 
 
@@ -73,6 +82,11 @@
             result = self.command.handle(initial_run=True)
         self.assertTrue(initial_run.called)
 
+    def test_handle_initial_run_some_stacks(self):
+        with patch.object(self.command, 'initial_run') as initial_run:
+            result = self.command.handle('fake_stack', initial_run=True)
+        initial_run.assert_called_with(stack_list=('fake_stack',))
+
     def test_handle_update_projects_none(self):
         with patch.object(self.command, 'update_projects') as update_projects:
             result = self.command.handle()
@@ -85,36 +99,130 @@
 
 
 class TestInitialRun(JenkinsPullCiCommandTestCase):
-    def test_no_urls(self):
-        self.command._get_artifact_url_list = MagicMock(return_value=[])
-        self.command._load_artifact = MagicMock()
-        result = self.command.initial_run()
-        self.assertFalse(self.command._load_artifact.called)
+
+    def setUp(self):
+        super(TestInitialRun, self).setUp()
+        self.stack_initial_run_patch = patch.object(
+            jenkins_pull_ci.Command,
+            'stack_initial_run')
+        self.stack_initial_run_mock = self.stack_initial_run_patch.start()
+        self.create_stack_patch = patch.object(
+            jenkins_pull_ci.Command,
+            '_create_stack')
+        self.create_stack_mock = self.create_stack_patch.start()
+        self.create_project_patch = patch.object(
+            jenkins_pull_ci.Command,
+            '_create_project')
+        self.create_project_mock = self.create_project_patch.start()
+
+    def tearDown(self):
+        super(TestInitialRun, self).tearDown()
+        patch.stopall()
+
+    def test_no_stack_given_no_stacks_initted(self):
+        self.get_config_url_list_mock.return_value = []
+        result = self.command.initial_run()
+        self.assertFalse(self.stack_initial_run_mock.called)
+
+    def test_no_stack_given_some_stacks_initted(self):
+        self.get_config_url_list_mock.return_value = [
+            'http://fake_stack_url/one']
+        result = self.command.initial_run()
+        self.stack_initial_run_mock.assert_called_with(
+            'http://fake_stack_url/one')
+
+    def test_one_stack_given_one_stack_initted(self):
+        fake_config_url = 'http://fake_jenkins_bla/head.indicators.yaml'
+        self.get_config_url_list_mock.return_value = [
+            fake_config_url]
+        result = self.command.initial_run(["head.indicators"])
+        self.stack_initial_run_mock.assert_called_with(fake_config_url)
+
+    def test_gibberish_stack_inits_nothing(self):
+        fake_config_url = 'http://fake_jenkins_bla/head.indicators.yaml'
+        self.get_config_url_list_mock.return_value = [
+            fake_config_url]
+        result = self.command.initial_run(["bla.indicators"])
+        self.stack_initial_run_mock.assert_not_called()
+
+
+class TestRetrieveConfigUrlFromList(JenkinsPullCiCommandTestCase):
+
+    def test_return_matching_config(self):
+        url_list = ['http://jenkins.url/artifacts/head.mir.yaml',
+                    'http://jenkins.url/artifacts/head.indicators.yaml']
+        result = self.command._retrieve_config_url(
+            'head.indicators',
+            url_list)
+        self.assertEqual(
+            'http://jenkins.url/artifacts/head.indicators.yaml',
+            result)
+
+    def test_return_none_on_gibberish(self):
+        url_list = ['http://jenkins.url/artifacts/head.mir.yaml',
+                    'http://jenkins.url/artifacts/head.indicators.yaml']
+        result = self.command._retrieve_config_url(
+            'head.flobnicator',
+            url_list)
+        self.assertEqual(None, result)
+
+
+class TestStackInitialRun(JenkinsPullCiCommandTestCase):
+
+    def setUp(self):
+        super(TestStackInitialRun, self).setUp()
+        self.stack_initial_run_patch = patch.object(
+            jenkins_pull_ci.Command,
+            'stack_initial_run')
+        self.create_stack_patch = patch.object(
+            jenkins_pull_ci.Command,
+            '_create_stack')
+        self.create_stack_mock = self.create_stack_patch.start()
+        self.create_project_patch = patch.object(
+            jenkins_pull_ci.Command,
+            '_create_project')
+        self.create_project_mock = self.create_project_patch.start()
+        self.extract_release_and_config_name_patch = patch.object(
+            jenkins_pull_ci.Command,
+            '_extract_release_and_config_name')
+        self.extract_release_and_config_name_mock = \
+            self.extract_release_and_config_name_patch.start()
+        self.extract_release_and_config_name_mock.return_value = (
+            'fake_release_name', 'fake_config_name')
+        self.config_is_on_blacklist_patch = patch(
+            'ci.management.commands.jenkins_pull_ci.config_is_on_blacklist')
+        self.config_is_on_blacklist_mock = \
+            self.config_is_on_blacklist_patch.start()
+
+    def tearDown(self):
+        super(TestStackInitialRun, self).tearDown()
+        patch.stopall()
+
+    def test_config_is_on_blacklist(self):
+        self.config_is_on_blacklist_mock.return_value = True
+        result = self.command.stack_initial_run("fake_url")
+        self.command._load_artifact.assert_not_called()
+        self.assertFalse(self.create_stack_mock.called)
+        self.assertFalse(self.create_project_mock.called)
 
     def test_no_data(self):
-        self.command._get_artifact_url_list = MagicMock(
-            return_value=['http://jenkins/blah/blah/release.stack.yaml'])
-        self.command._load_artifact = MagicMock(return_value=None)
-        self.command._create_stack = MagicMock()
-        self.command._create_project = MagicMock()
-        result = self.command.initial_run()
+        self.config_is_on_blacklist_mock.return_value = False
+        self.load_artifact_mock.return_value = []
+        result = self.command.stack_initial_run("fake_url")
         self.command._load_artifact.assert_called()
-        self.assertFalse(self.command._create_stack.called)
-        self.assertFalse(self.command._create_project.called)
+        self.assertFalse(self.create_stack_mock.called)
+        self.assertFalse(self.create_project_mock.called)
 
     def test_no_projects(self):
-        stack = {'name': 'fake'}
-        self.command._get_artifact_url_list = MagicMock(
-            return_value=['http://jenkins/blah/blah/release.stack.yaml'])
-        self.command._load_artifact = MagicMock(return_value=stack)
-        self.command._create_stack = MagicMock()
-        self.command._create_project = MagicMock()
-        result = self.command.initial_run()
+        self.config_is_on_blacklist_mock.return_value = False
+        self.load_artifact_mock.return_value = MagicMock(name='fake')
+        result = self.command.stack_initial_run("fake_url")
         self.command._load_artifact.assert_called()
-        self.assertFalse(self.command._create_stack.called)
-        self.assertFalse(self.command._create_project.called)
+        self.assertFalse(self.create_stack_mock.called)
+        self.assertFalse(self.create_project_mock.called)
 
     def test_simple_stack(self):
+        self.config_is_on_blacklist_mock.return_value = False
         stack = {'name': 'fake',
                  'ppa': 'ppa:project/ppa',
                  'series': 'saucy',
@@ -122,15 +230,12 @@
                      'project-1': {
                          'ci': 'project-1-ci',
                          'autolanding': 'project-1-autolanding'}}}
-        self.command._get_artifact_url_list = MagicMock(
-            return_value=['http://jenkins/blah/blah/release.stack.yaml'])
         self.command._load_artifact = MagicMock(return_value=stack)
-        self.command._create_stack = MagicMock()
-        self.command._create_project = MagicMock()
-        result = self.command.initial_run()
-        self.command._load_artifact.assert_called()
-        self.command._create_stack.assert_called()
-        self.command._create_project.assert_called()
+        result = self.command.stack_initial_run("fake_url")
+        self.load_artifact_mock.assert_called()
+        self.create_stack_mock.assert_called_with(
+            stack, "fake_release_name")
+        self.create_project_mock.assert_called()
 
 
 class TestExtractReleaseAndConfigName(JenkinsPullCiCommandTestCase):
@@ -189,3 +294,5 @@
         self.command.update_projects(
             [MagicMock(name="fake_project")])
         update_jenkins_job_mock.assert_called_with(fake_jenkins_job)
+
+