Merge lp:~allanlesage/helipad/slurp-one into lp:helipad/ci

Proposed by Allan LeSage
Status: Needs review
Proposed branch: lp:~allanlesage/helipad/slurp-one
Merge into: lp:helipad/ci
Diff against target: 332 lines (+197/-61)
2 files modified
management/commands/jenkins_pull_ci.py (+59/-30)
tests/test_jenkins_pull_ci.py (+138/-31)
To merge this branch: bzr merge lp:~allanlesage/helipad/slurp-one
Reviewer Review Type Date Requested Status
PS Jenkins bot continuous-integration Needs Fixing
helipad-team Pending
Review via email: mp+177956@code.launchpad.net

Commit message

Enable pulling just one stack.

Description of the change

Here we can slurp just one stack, e.g. head.indicators ; I've tested a pull but have found that we're missing the integration_job being set in Cu2dStack, am addressing in another MP.

To post a comment you must log in.
Revision history for this message
Allan LeSage (allanlesage) wrote :

Syntax for this is ./manage.py jenkins_pull_ci -I head.indicators (or substitute your fav). cjohnston, fginther, what do you think of this? It seems like there would be other options available, e.g. update all builds relevant to a project (launchpad_project)--which is what we're doing for the update script. Maybe these two should be brought into alignment, e.g.

Revision history for this message
PS Jenkins bot (ps-jenkins) wrote :

FAILED: Continuous integration, rev:34
No commit message was specified in the merge proposal. Click on the following link and set the commit message (if you want a jenkins rebuild you need to trigger it yourself):
https://code.launchpad.net/~allanlesage/helipad/slurp-one/+merge/177956/+edit-commit-message

http://s-jenkins:8080/job/helipad-ci-ci/60/
Executed test runs:

Click here to trigger a rebuild:
http://s-jenkins:8080/job/helipad-ci-ci/60/rebuild

review: Needs Fixing (continuous-integration)

Unmerged revisions

34. By Allan LeSage

Add command-line option and tests.

33. By Allan LeSage

Use patch.stopall.

32. By Allan LeSage

Improve tests, having split function.

31. By Allan LeSage

Move the stack update to its own function.

30. By Allan LeSage

Add a comment and shift artifact to config for clarity.

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'management/commands/jenkins_pull_ci.py'
2--- management/commands/jenkins_pull_ci.py 2013-07-29 17:12:18 +0000
3+++ management/commands/jenkins_pull_ci.py 2013-07-31 22:16:40 +0000
4@@ -121,7 +121,7 @@
5 page = jenkins_get(url, as_json=False)
6 return yaml.safe_load(page)
7
8- def _get_artifact_url_list(self):
9+ def _get_config_url_list(self):
10 # Get the jenkins job which contains the data dump
11 jenkapi = jenkinsapi.api.Jenkins(settings.JENKINS_URL)
12 job = jenkapi.get_job(CUPSTREAM2DISTRO_CONFIG_DATA_SOURCE)
13@@ -138,40 +138,69 @@
14 break
15 return artifact_list
16
17- def initial_run(self):
18- url_list = self._get_artifact_url_list()
19+ def _retrieve_config_url(self, stack_name, url_list):
20+ """Find a regex match for stack_name in url_list."""
21
22- # For each artifact
23 for url in url_list:
24- # Parse the stack release and name from the url
25- (release, config_name) = self._extract_release_and_config_name(url)
26- if config_is_on_blacklist(config_name):
27- logger.debug("Skipping cu2d config at {} due to blacklist "
28- "match.".format(config_name))
29- continue
30-
31- data = self._load_artifact(url)
32- # Process the stack
33- if not data:
34- # Stack has no contents
35- logger.debug("No projects found in {}".format(url))
36- continue
37- if 'projects' not in data:
38- # Stack has no projects
39- logger.debug("No projects found in {}".format(url))
40- continue
41- stack_name = self._create_stack(data, release)
42-
43- # Process the projects
44- if stack_name:
45- for project_name in data['projects']:
46- self._create_project(data, release, stack_name,
47- project_name)
48- return
49+ if re.search("{}.yaml".format(stack_name), url):
50+ return url
51+ return None
52+
53+ def stack_initial_run(self, stack_config_url):
54+ # Parse the stack release and name from the url
55+ (release, config_name) = self._extract_release_and_config_name(
56+ stack_config_url)
57+ if config_is_on_blacklist(config_name):
58+ logger.info("Skipping cu2d config at {} due to blacklist "
59+ "match.".format(config_name))
60+ return
61+ data = self._load_artifact(stack_config_url)
62+
63+ # Process the stack
64+ if not data:
65+ # Stack has no contents
66+ logger.debug("No projects found in {}".format(stack_config_url))
67+ return
68+
69+ if 'projects' not in data:
70+ # Stack has no projects
71+ logger.debug("No projects found in {}".format(stack_config_url))
72+ return
73+ stack_name = self._create_stack(data, release)
74+
75+ if not stack_name:
76+ logger.debug("Failed to create stack.")
77+ return
78+
79+ # Process the projects
80+ for project_name in data['projects']:
81+ self._create_project(
82+ data, release, stack_name, project_name)
83+
84+ def initial_run(self, stack_list=[]):
85+ """Pull Jenkins builds and jobs relevant to stacks in given list.
86+
87+ Stacks as 'release.project', e.g. 'head.indicators', 'saucy.mir'.
88+ """
89+
90+ # A Jenkins job dumps all .yaml configs from lp:cupstream2distro-config
91+ # as artifacts in a Jenkins workspace, so we can wget 'em, e.g.
92+ config_url_list = self._get_config_url_list()
93+ if len(stack_list) == 0:
94+ pull_url_list = config_url_list
95+ else:
96+ pull_url_list = []
97+ for stack_name in stack_list:
98+ config_url = self._retrieve_config_url(
99+ stack_name, config_url_list)
100+ if config_url is not None:
101+ pull_url_list.append(config_url)
102+ for config_url in pull_url_list:
103+ self.stack_initial_run(config_url)
104
105 def handle(self, *args, **options):
106 if options.get('initial_run'):
107- self.initial_run()
108+ self.initial_run(stack_list=args)
109 else:
110 self.update_projects(project_list=args)
111 self.update_cu2d_stacks()
112
113=== modified file 'tests/test_jenkins_pull_ci.py'
114--- tests/test_jenkins_pull_ci.py 2013-07-28 21:26:11 +0000
115+++ tests/test_jenkins_pull_ci.py 2013-07-31 22:16:40 +0000
116@@ -55,14 +55,23 @@
117 Command,
118 '__init__',
119 new=command_mock)
120+ self.load_artifact_patch = patch.object(
121+ jenkins_pull_ci.Command,
122+ '_load_artifact')
123+ self.load_artifact_mock = self.load_artifact_patch.start()
124+ self.get_config_url_list_patch = patch.object(
125+ jenkins_pull_ci.Command,
126+ '_get_config_url_list')
127+ self.get_config_url_list_mock = self.get_config_url_list_patch.start()
128 self.command_patch.start()
129 self.command = Command()
130 self.logger_patch = patch('ci.models.logger')
131 self.logger_patch.start()
132
133-
134 def tearDown(self):
135 self.command_patch.stop()
136+ self.get_config_url_list_patch.stop()
137+ self.load_artifact_patch.stop()
138 self.logger_patch.stop()
139
140
141@@ -73,6 +82,11 @@
142 result = self.command.handle(initial_run=True)
143 self.assertTrue(initial_run.called)
144
145+ def test_handle_initial_run_some_stacks(self):
146+ with patch.object(self.command, 'initial_run') as initial_run:
147+ result = self.command.handle('fake_stack', initial_run=True)
148+ initial_run.assert_called_with(stack_list=('fake_stack',))
149+
150 def test_handle_update_projects_none(self):
151 with patch.object(self.command, 'update_projects') as update_projects:
152 result = self.command.handle()
153@@ -85,36 +99,130 @@
154
155
156 class TestInitialRun(JenkinsPullCiCommandTestCase):
157- def test_no_urls(self):
158- self.command._get_artifact_url_list = MagicMock(return_value=[])
159- self.command._load_artifact = MagicMock()
160- result = self.command.initial_run()
161- self.assertFalse(self.command._load_artifact.called)
162+
163+ def setUp(self):
164+ super(TestInitialRun, self).setUp()
165+ self.stack_initial_run_patch = patch.object(
166+ jenkins_pull_ci.Command,
167+ 'stack_initial_run')
168+ self.stack_initial_run_mock = self.stack_initial_run_patch.start()
169+ self.create_stack_patch = patch.object(
170+ jenkins_pull_ci.Command,
171+ '_create_stack')
172+ self.create_stack_mock = self.create_stack_patch.start()
173+ self.create_project_patch = patch.object(
174+ jenkins_pull_ci.Command,
175+ '_create_project')
176+ self.create_project_mock = self.create_project_patch.start()
177+
178+ def tearDown(self):
179+ super(TestInitialRun, self).tearDown()
180+ patch.stopall()
181+
182+ def test_no_stack_given_no_stacks_initted(self):
183+ self.get_config_url_list_mock.return_value = []
184+ result = self.command.initial_run()
185+ self.assertFalse(self.stack_initial_run_mock.called)
186+
187+ def test_no_stack_given_some_stacks_initted(self):
188+ self.get_config_url_list_mock.return_value = [
189+ 'http://fake_stack_url/one']
190+ result = self.command.initial_run()
191+ self.stack_initial_run_mock.assert_called_with(
192+ 'http://fake_stack_url/one')
193+
194+ def test_one_stack_given_one_stack_initted(self):
195+ fake_config_url = 'http://fake_jenkins_bla/head.indicators.yaml'
196+ self.get_config_url_list_mock.return_value = [
197+ fake_config_url]
198+ result = self.command.initial_run(["head.indicators"])
199+ self.stack_initial_run_mock.assert_called_with(fake_config_url)
200+
201+ def test_gibberish_stack_inits_nothing(self):
202+ fake_config_url = 'http://fake_jenkins_bla/head.indicators.yaml'
203+ self.get_config_url_list_mock.return_value = [
204+ fake_config_url]
205+ result = self.command.initial_run(["bla.indicators"])
206+ self.stack_initial_run_mock.assert_not_called()
207+
208+
209+class TestRetrieveConfigUrlFromList(JenkinsPullCiCommandTestCase):
210+
211+ def test_return_matching_config(self):
212+ url_list = ['http://jenkins.url/artifacts/head.mir.yaml',
213+ 'http://jenkins.url/artifacts/head.indicators.yaml']
214+ result = self.command._retrieve_config_url(
215+ 'head.indicators',
216+ url_list)
217+ self.assertEqual(
218+ 'http://jenkins.url/artifacts/head.indicators.yaml',
219+ result)
220+
221+ def test_return_none_on_gibberish(self):
222+ url_list = ['http://jenkins.url/artifacts/head.mir.yaml',
223+ 'http://jenkins.url/artifacts/head.indicators.yaml']
224+ result = self.command._retrieve_config_url(
225+ 'head.flobnicator',
226+ url_list)
227+ self.assertEqual(None, result)
228+
229+
230+class TestStackInitialRun(JenkinsPullCiCommandTestCase):
231+
232+ def setUp(self):
233+ super(TestStackInitialRun, self).setUp()
234+ self.stack_initial_run_patch = patch.object(
235+ jenkins_pull_ci.Command,
236+ 'stack_initial_run')
237+ self.create_stack_patch = patch.object(
238+ jenkins_pull_ci.Command,
239+ '_create_stack')
240+ self.create_stack_mock = self.create_stack_patch.start()
241+ self.create_project_patch = patch.object(
242+ jenkins_pull_ci.Command,
243+ '_create_project')
244+ self.create_project_mock = self.create_project_patch.start()
245+ self.extract_release_and_config_name_patch = patch.object(
246+ jenkins_pull_ci.Command,
247+ '_extract_release_and_config_name')
248+ self.extract_release_and_config_name_mock = \
249+ self.extract_release_and_config_name_patch.start()
250+ self.extract_release_and_config_name_mock.return_value = (
251+ 'fake_release_name', 'fake_config_name')
252+ self.config_is_on_blacklist_patch = patch(
253+ 'ci.management.commands.jenkins_pull_ci.config_is_on_blacklist')
254+ self.config_is_on_blacklist_mock = \
255+ self.config_is_on_blacklist_patch.start()
256+
257+ def tearDown(self):
258+ super(TestStackInitialRun, self).tearDown()
259+ patch.stopall()
260+
261+ def test_config_is_on_blacklist(self):
262+ self.config_is_on_blacklist_mock.return_value = True
263+ result = self.command.stack_initial_run("fake_url")
264+ self.command._load_artifact.assert_not_called()
265+ self.assertFalse(self.create_stack_mock.called)
266+ self.assertFalse(self.create_project_mock.called)
267
268 def test_no_data(self):
269- self.command._get_artifact_url_list = MagicMock(
270- return_value=['http://jenkins/blah/blah/release.stack.yaml'])
271- self.command._load_artifact = MagicMock(return_value=None)
272- self.command._create_stack = MagicMock()
273- self.command._create_project = MagicMock()
274- result = self.command.initial_run()
275+ self.config_is_on_blacklist_mock.return_value = False
276+ self.load_artifact_mock.return_value = []
277+ result = self.command.stack_initial_run("fake_url")
278 self.command._load_artifact.assert_called()
279- self.assertFalse(self.command._create_stack.called)
280- self.assertFalse(self.command._create_project.called)
281+ self.assertFalse(self.create_stack_mock.called)
282+ self.assertFalse(self.create_project_mock.called)
283
284 def test_no_projects(self):
285- stack = {'name': 'fake'}
286- self.command._get_artifact_url_list = MagicMock(
287- return_value=['http://jenkins/blah/blah/release.stack.yaml'])
288- self.command._load_artifact = MagicMock(return_value=stack)
289- self.command._create_stack = MagicMock()
290- self.command._create_project = MagicMock()
291- result = self.command.initial_run()
292+ self.config_is_on_blacklist_mock.return_value = False
293+ self.load_artifact_mock.return_value = MagicMock(name='fake')
294+ result = self.command.stack_initial_run("fake_url")
295 self.command._load_artifact.assert_called()
296- self.assertFalse(self.command._create_stack.called)
297- self.assertFalse(self.command._create_project.called)
298+ self.assertFalse(self.create_stack_mock.called)
299+ self.assertFalse(self.create_project_mock.called)
300
301 def test_simple_stack(self):
302+ self.config_is_on_blacklist_mock.return_value = False
303 stack = {'name': 'fake',
304 'ppa': 'ppa:project/ppa',
305 'series': 'saucy',
306@@ -122,15 +230,12 @@
307 'project-1': {
308 'ci': 'project-1-ci',
309 'autolanding': 'project-1-autolanding'}}}
310- self.command._get_artifact_url_list = MagicMock(
311- return_value=['http://jenkins/blah/blah/release.stack.yaml'])
312 self.command._load_artifact = MagicMock(return_value=stack)
313- self.command._create_stack = MagicMock()
314- self.command._create_project = MagicMock()
315- result = self.command.initial_run()
316- self.command._load_artifact.assert_called()
317- self.command._create_stack.assert_called()
318- self.command._create_project.assert_called()
319+ result = self.command.stack_initial_run("fake_url")
320+ self.load_artifact_mock.assert_called()
321+ self.create_stack_mock.assert_called_with(
322+ stack, "fake_release_name")
323+ self.create_project_mock.assert_called()
324
325
326 class TestExtractReleaseAndConfigName(JenkinsPullCiCommandTestCase):
327@@ -189,3 +294,5 @@
328 self.command.update_projects(
329 [MagicMock(name="fake_project")])
330 update_jenkins_job_mock.assert_called_with(fake_jenkins_job)
331+
332+

Subscribers

People subscribed via source and target branches

to all changes: