Merge lp:~hazmat/juju-deployer/unit-placement into lp:juju-deployer
- unit-placement
- Merge into trunk
Status: | Merged |
---|---|
Approved by: | Adam Gandelman |
Approved revision: | 91 |
Merged at revision: | 80 |
Proposed branch: | lp:~hazmat/juju-deployer/unit-placement |
Merge into: | lp:juju-deployer |
Diff against target: |
731 lines (+436/-42) 14 files modified
configs/export.yml (+17/-0) deployer/action/diff.py (+1/-1) deployer/action/importer.py (+55/-23) deployer/deployment.py (+111/-1) deployer/env/base.py (+3/-0) deployer/env/go.py (+68/-6) deployer/service.py (+11/-3) deployer/tests/test_config.py (+5/-6) deployer/tests/test_data/stack-placement-invalid-2.yaml (+13/-0) deployer/tests/test_data/stack-placement-invalid.yaml (+13/-0) deployer/tests/test_data/stack-placement.yaml (+18/-0) deployer/tests/test_deployment.py (+66/-1) doc/config.rst (+54/-0) setup.py (+1/-1) |
To merge this branch: | bzr merge lp:~hazmat/juju-deployer/unit-placement |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Adam Gandelman (community) | Needs Fixing | ||
Review via email:
|
Commit message
Description of the change
Support for unit placement using deployer configuration. Also improve terminate/reset support with containers, and a fix for --diff output.
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Kapil Thangavelu (hazmat) wrote : | # |
The second one is due to a need for 0.13 for jujuclient, i'll update the
dep spec for that and add some test for the first case.
On Thu, Oct 31, 2013 at 1:41 PM, Adam Gandelman <email address hidden> wrote:
> Review: Needs Fixing
>
> Testing this now, couple of issues so far:
>
> If a service is configured as:
>
> fooservice:
> to: [0]
>
> deployer blows up:
>
> 70, in get_unit_placement
> if ':' in unit_placement:
> TypeError: argument of type 'int' is not iterable
>
> Only running that code if unit_placement is a string seems to fix that.
>
> Adding units fails:
>
> 2013-10-31 17:34:26 [INFO] deployer.import: Adding 2 more units to
> nova-compute
> Traceback (most recent call last):
> File "/usr/local/
> load_entry_
> 'juju-deployer')()
> File
> "/usr/local/
> line 118, in main
> run()
> File
> "/usr/local/
> line 209, in run
> importer.
> File
> "/usr/local/
> line 186, in run
> self.add_units()
> File
> "/usr/local/
> line 54, in add_units
> self.env.
> File
> "/usr/local/
> line 28, in add_unit
> return self.client.
> File
> "/usr/local/
> line 465, in add_unit
> "Params": params})
> File
> "/usr/local/
> line 135, in _rpc
> raise EnvError(result)
> jujuclient.
> { u'Error': u'must add at least one unit', u'RequestId': 1,
> u'Response': { }}
>
> --
>
> https:/
> You are the owner of lp:~hazmat/juju-deployer/unit-placement.
>
- 89. By Kapil Thangavelu
-
fix for machine 0 placement
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Kapil Thangavelu (hazmat) wrote : | # |
Fixes pushed
- 90. By Kapil Thangavelu
-
require newer version of jujuclient
- 91. By Kapil Thangavelu
-
by request do a wait for units before adding relations.
Preview Diff
1 | === added file 'configs/export.yml' |
2 | --- configs/export.yml 1970-01-01 00:00:00 +0000 |
3 | +++ configs/export.yml 2013-11-01 15:46:48 +0000 |
4 | @@ -0,0 +1,17 @@ |
5 | +envExport: |
6 | + services: |
7 | + mysql: |
8 | + charm: "cs:precise/mysql-27" |
9 | + annotations: |
10 | + "gui-x": "-251" |
11 | + "gui-y": "-203" |
12 | + wordpress: |
13 | + charm: "cs:precise/wordpress-20" |
14 | + num_units: 3 |
15 | + to: ['lxc:mysql', 'lxc:mysql'] |
16 | + annotations: |
17 | + "gui-x": "116" |
18 | + "gui-y": "-206" |
19 | + relations: |
20 | + - - "wordpress:db" |
21 | + - "mysql:db" |
22 | |
23 | === modified file 'deployer/action/diff.py' |
24 | --- deployer/action/diff.py 2013-07-22 15:29:31 +0000 |
25 | +++ deployer/action/diff.py 2013-11-01 15:46:48 +0000 |
26 | @@ -134,7 +134,7 @@ |
27 | if e_v != v: |
28 | mod['config'] = {k: e_v} |
29 | if e_s['unit_count'] != d_s.get('num_units', 1): |
30 | - mod['num_units'] = e_s['num_units'] |
31 | + mod['num_units'] = e_s['unit_count'] - d_s['num_units'] |
32 | return mod |
33 | |
34 | def run(self): |
35 | |
36 | === modified file 'deployer/action/importer.py' |
37 | --- deployer/action/importer.py 2013-10-02 17:50:05 +0000 |
38 | +++ deployer/action/importer.py 2013-11-01 15:46:48 +0000 |
39 | @@ -19,25 +19,41 @@ |
40 | self.log.debug("Adding units...") |
41 | # Add units to existing services that don't match count. |
42 | env_status = self.env.status() |
43 | - added = set() |
44 | + reloaded = False |
45 | + |
46 | for svc in self.deployment.get_services(): |
47 | - delta = (svc.num_units - |
48 | - len(env_status['services'][svc.name].get('units', ()))) |
49 | - if delta > 0: |
50 | - charm = self.deployment.get_charm_for(svc.name) |
51 | - if charm.is_subordinate(): |
52 | - self.log.warning( |
53 | - "Config specifies num units for subordinate: %s", |
54 | - svc.name) |
55 | - continue |
56 | - self.log.info( |
57 | - "Adding %d more units to %s" % (abs(delta), svc.name)) |
58 | - for u in self.env.add_units(svc.name, abs(delta)): |
59 | - added.add(u) |
60 | - else: |
61 | + cur_units = len(env_status['services'][svc.name].get('units', ())) |
62 | + delta = (svc.num_units - cur_units) |
63 | + |
64 | + if delta <= 0: |
65 | self.log.debug( |
66 | " Service %r does not need any more units added.", |
67 | svc.name) |
68 | + continue |
69 | + |
70 | + charm = self.deployment.get_charm_for(svc.name) |
71 | + if charm.is_subordinate(): |
72 | + self.log.warning( |
73 | + "Config specifies num units for subordinate: %s", |
74 | + svc.name) |
75 | + continue |
76 | + |
77 | + self.log.info( |
78 | + "Adding %d more units to %s" % (abs(delta), svc.name)) |
79 | + if svc.unit_placement: |
80 | + # Reload status once after non placed services units are done. |
81 | + if reloaded is False: |
82 | + # Crappy workaround juju-core api inconsistency |
83 | + time.sleep(5.1) |
84 | + env_status = self.env.status() |
85 | + reloaded = True |
86 | + |
87 | + for mid in range(cur_units, svc.num_units): |
88 | + mspec = self.deployment.get_unit_placement( |
89 | + svc, mid, env_status) |
90 | + self.env.add_unit(svc.name, mspec) |
91 | + else: |
92 | + self.env.add_units(svc.name, abs(delta)) |
93 | |
94 | def get_charms(self): |
95 | # Get Charms |
96 | @@ -53,6 +69,8 @@ |
97 | def deploy_services(self): |
98 | self.log.info("Deploying services...") |
99 | env_status = self.env.status() |
100 | + reloaded = False |
101 | + |
102 | for svc in self.deployment.get_services(): |
103 | if svc.name in env_status['services']: |
104 | self.log.debug( |
105 | @@ -62,14 +80,28 @@ |
106 | charm = self.deployment.get_charm_for(svc.name) |
107 | self.log.info( |
108 | " Deploying service %s using %s", svc.name, charm.charm_url) |
109 | + |
110 | + if svc.unit_placement: |
111 | + # We sorted all the non placed services first, so we only |
112 | + # need to update status once after we're done with them. |
113 | + if not reloaded: |
114 | + self.log.debug( |
115 | + " Refetching status for placement deploys") |
116 | + time.sleep(5.1) |
117 | + env_status = self.env.status() |
118 | + reloaded = True |
119 | + num_units = 1 |
120 | + else: |
121 | + num_units = svc.num_units |
122 | + |
123 | self.env.deploy( |
124 | svc.name, |
125 | charm.charm_url, |
126 | self.deployment.repo_path, |
127 | svc.config, |
128 | svc.constraints, |
129 | - svc.num_units, |
130 | - svc.force_machine) |
131 | + num_units, |
132 | + self.deployment.get_unit_placement(svc, 0, env_status)) |
133 | |
134 | if svc.expose: |
135 | self.log.info(" Exposing service %r" % svc.name) |
136 | @@ -93,8 +125,8 @@ |
137 | self.env.add_relation(end_a, end_b) |
138 | created = True |
139 | # per the original, not sure the use case. |
140 | - self.log.debug(" Waiting 5s before next relation") |
141 | - time.sleep(5) |
142 | + #self.log.debug(" Waiting 5s before next relation") |
143 | + #time.sleep(5) |
144 | return created |
145 | |
146 | def _rel_exists(self, status, end_a, end_b): |
147 | @@ -149,17 +181,17 @@ |
148 | # to be consistent to subsequent watch api interactions, see |
149 | # http://pad.lv/1203105 which will obviate this wait. |
150 | time.sleep(5.1) |
151 | + self.add_units() |
152 | |
153 | + self.log.debug("Waiting for units before adding relations") |
154 | self.wait_for_units() |
155 | - self.add_units() |
156 | |
157 | rels_created = self.add_relations() |
158 | |
159 | # Wait for the units to be up before waiting for rel stability. |
160 | - self.log.debug("Waiting for units to be started") |
161 | - self.wait_for_units(self.options.retry_count) |
162 | if rels_created: |
163 | - self.log.debug("Waiting for relations %d", self.options.rel_wait) |
164 | + self.log.debug( |
165 | + "Waiting for relation convergence %ds", self.options.rel_wait) |
166 | time.sleep(self.options.rel_wait) |
167 | self.wait_for_units(self.options.retry_count) |
168 | |
169 | |
170 | === modified file 'deployer/deployment.py' |
171 | --- deployer/deployment.py 2013-07-24 23:10:15 +0000 |
172 | +++ deployer/deployment.py 2013-11-01 15:46:48 +0000 |
173 | @@ -40,8 +40,74 @@ |
174 | return Service(name, self.data['services'][name]) |
175 | |
176 | def get_services(self): |
177 | + services = [] |
178 | for name, svc_data in self.data.get('services', {}).items(): |
179 | - yield Service(name, svc_data) |
180 | + services.append(Service(name, svc_data)) |
181 | + services.sort(self._placement_sort) |
182 | + return services |
183 | + |
184 | + @staticmethod |
185 | + def _placement_sort(svc_a, svc_b): |
186 | + if svc_a.unit_placement: |
187 | + if svc_b.unit_placement: |
188 | + return cmp(svc_a.name, svc_b.name) |
189 | + return 1 |
190 | + if svc_b.unit_placement: |
191 | + return -1 |
192 | + return cmp(svc_a.name, svc_b.name) |
193 | + |
194 | + @staticmethod |
195 | + def _format_placement(machine, container=None): |
196 | + if container: |
197 | + return "%s:%s" % (container, machine) |
198 | + else: |
199 | + return machine |
200 | + |
201 | + def get_unit_placement(self, svc, unit_number, status): |
202 | + unit_mapping = svc.unit_placement |
203 | + if not unit_mapping: |
204 | + return None |
205 | + if len(unit_mapping) <= unit_number: |
206 | + return None |
207 | + |
208 | + unit_placement = placement = str(unit_mapping[unit_number]) |
209 | + container = None |
210 | + u_idx = unit_number |
211 | + |
212 | + if ':' in unit_placement: |
213 | + container, placement = unit_placement.split(":") |
214 | + if '=' in placement: |
215 | + placement, u_idx = placement.split("=") |
216 | + |
217 | + if placement.isdigit() and placement == "0": |
218 | + return self._format_placement(placement, container) |
219 | + |
220 | + with_service = status['services'].get(placement) |
221 | + if with_service is None: |
222 | + # Should be caught in validate relations but sanity check |
223 | + # for concurrency. |
224 | + self.log.error( |
225 | + "Service %s to be deployed with non existant service %s", |
226 | + svc.name, placement) |
227 | + # Prefer continuing deployment with a new machine rather |
228 | + # than an in-progress abort. |
229 | + return None |
230 | + |
231 | + svc_units = with_service['units'] |
232 | + if len(svc_units) <= unit_number: |
233 | + self.log.warning( |
234 | + "Service:%s deploy-with Service:%s, but no with unit found", |
235 | + svc.name, placement) |
236 | + return None |
237 | + unit_names = svc_units.keys() |
238 | + unit_names.sort() |
239 | + machine = svc_units[unit_names[int(u_idx)]].get('machine') |
240 | + if not machine: |
241 | + self.log.warning( |
242 | + "Service:%s deploy-with unit missing machine %s", |
243 | + svc.name, unit_names[unit_number]) |
244 | + return None |
245 | + return self._format_placement(machine, container) |
246 | |
247 | def get_relations(self): |
248 | if 'relations' not in self.data: |
249 | @@ -117,6 +183,7 @@ |
250 | self.load_overrides(cli_overides) |
251 | self.resolve_config() |
252 | self.validate_relations() |
253 | + self.validate_placement() |
254 | |
255 | def load_overrides(self, cli_overrides=()): |
256 | """Load overrides.""" |
257 | @@ -195,6 +262,49 @@ |
258 | ep.service, "%s <-> %s" % (e_a, e_b)) |
259 | raise ErrorExit() |
260 | |
261 | + def validate_placement(self): |
262 | + services = dict([(s.name, s) for s in self.get_services()]) |
263 | + for name, s in services.items(): |
264 | + unit_placement = s.unit_placement |
265 | + if unit_placement is None: |
266 | + continue |
267 | + if not isinstance(unit_placement, list): |
268 | + unit_placement = [unit_placement] |
269 | + unit_placement = map(str, unit_placement) |
270 | + for idx, p in enumerate(unit_placement): |
271 | + if ':' in p: |
272 | + container, p = p.split(':') |
273 | + if container not in ('lxc', 'kvm'): |
274 | + self.log.error( |
275 | + "Invalid service:%s placement: %s", |
276 | + name, unit_placement[idx]) |
277 | + raise ErrorExit() |
278 | + if '=' in p: |
279 | + p, u_idx = p.split("=") |
280 | + if not u_idx.isdigit(): |
281 | + self.log.error( |
282 | + "Invalid service:%s placement: %s", |
283 | + name, unit_placement[idx]) |
284 | + raise ErrorExit() |
285 | + if p.isdigit() and p == '0': |
286 | + continue |
287 | + elif p.isdigit(): |
288 | + self.log.error( |
289 | + "Service placement to machine not supported %s to %s", |
290 | + name, unit_placement[idx]) |
291 | + raise ErrorExit() |
292 | + elif p in services: |
293 | + if services[p].unit_placement: |
294 | + self.log.error( |
295 | + "Nested placement not supported %s -> %s -> %s" % ( |
296 | + name, p, services[p].unit_placement)) |
297 | + raise ErrorExit() |
298 | + else: |
299 | + self.log.error( |
300 | + "Invalid service placement %s to %s" % ( |
301 | + name, unit_placement[idx])) |
302 | + raise ErrorExit() |
303 | + |
304 | def save(self, path): |
305 | with open(path, "w") as fh: |
306 | fh.write(yaml_dump(self.data)) |
307 | |
308 | === modified file 'deployer/env/base.py' |
309 | --- deployer/env/base.py 2013-10-17 03:16:24 +0000 |
310 | +++ deployer/env/base.py 2013-11-01 15:46:48 +0000 |
311 | @@ -152,3 +152,6 @@ |
312 | stderr=fh) |
313 | status = yaml_load(output) |
314 | return status |
315 | + |
316 | + def add_unit(self, service_name, machine_spec): |
317 | + raise NotImplementedError() |
318 | |
319 | === modified file 'deployer/env/go.py' |
320 | --- deployer/env/go.py 2013-10-08 20:13:39 +0000 |
321 | +++ deployer/env/go.py 2013-11-01 15:46:48 +0000 |
322 | @@ -5,7 +5,11 @@ |
323 | from .base import BaseEnvironment |
324 | from ..utils import ErrorExit |
325 | |
326 | -from jujuclient import Environment as EnvironmentClient, UnitErrors, EnvError |
327 | +from jujuclient import ( |
328 | + Environment as EnvironmentClient, |
329 | + UnitErrors, |
330 | + EnvError, |
331 | + WatchWrapper) |
332 | |
333 | |
334 | class GoEnvironment(BaseEnvironment): |
335 | @@ -20,6 +24,9 @@ |
336 | config = self._get_env_config() |
337 | return config['admin-secret'] |
338 | |
339 | + def add_unit(self, service_name, machine_spec): |
340 | + return self.client.add_unit(service_name, machine_spec) |
341 | + |
342 | def add_units(self, service_name, num_units): |
343 | return self.client.add_units(service_name, num_units) |
344 | |
345 | @@ -101,17 +108,50 @@ |
346 | if len(status['machines']) == 1: |
347 | return |
348 | |
349 | - for mid in status['machines'].keys(): |
350 | - if mid == "0": |
351 | - continue |
352 | - self.log.debug(" Terminating machine %s", mid) |
353 | - self.terminate_machine(mid) |
354 | + # containers before machines, container hosts post wait. |
355 | + machines = status['machines'].keys() |
356 | + |
357 | + container_hosts = set() |
358 | + containers = set() |
359 | + |
360 | + def machine_sort(x, y): |
361 | + for ctype in ('lxc', 'kvm'): |
362 | + for m in (x, y): |
363 | + if ctype in m: |
364 | + container_hosts.add(m.split('/', 1)[0]) |
365 | + containers.add(m) |
366 | + if m == x: |
367 | + return -1 |
368 | + if m == y: |
369 | + return 1 |
370 | + return cmp(x, y) |
371 | + |
372 | + machines.sort(machine_sort) |
373 | + |
374 | + for mid in machines: |
375 | + self._terminate_machine(mid, container_hosts) |
376 | + |
377 | + if containers: |
378 | + watch = self.client.get_watch(120) |
379 | + WaitForMachineTermination( |
380 | + watch, containers).run(self._delta_event_log) |
381 | + |
382 | + for mid in container_hosts: |
383 | + self._terminate_machine(mid) |
384 | |
385 | if terminate_wait: |
386 | self.log.info(" Waiting for machine termination") |
387 | callback = watch and self._delta_event_log or None |
388 | self.client.wait_for_no_machines(None, callback) |
389 | |
390 | + def _terminate_machine(self, mid, container_hosts=()): |
391 | + if mid == "0": |
392 | + return |
393 | + if mid in container_hosts: |
394 | + return |
395 | + self.log.debug(" Terminating machine %s", mid) |
396 | + self.terminate_machine(mid) |
397 | + |
398 | def _check_timeout(self, etime): |
399 | w_timeout = etime - time.time() |
400 | if w_timeout < 0: |
401 | @@ -211,3 +251,25 @@ |
402 | eps[0]['Relation']['Name'], |
403 | eps[1]['ServiceName'], |
404 | eps[1]['Relation']['Name']) |
405 | + |
406 | + |
407 | +class WaitForMachineTermination(WatchWrapper): |
408 | + |
409 | + def __init__(self, watch, machines): |
410 | + super(WaitForMachineTermination, self).__init__(watch) |
411 | + self.machines = set(machines) |
412 | + self.known = set() |
413 | + |
414 | + def process(self, entity_type, change, data): |
415 | + if entity_type != 'machine': |
416 | + return |
417 | + if change == 'remove' and data['Id'] in self.machines: |
418 | + self.machines.remove(data['Id']) |
419 | + else: |
420 | + self.known.add(data['Id']) |
421 | + |
422 | + def complete(self): |
423 | + for m in self.machines: |
424 | + if m in self.known: |
425 | + return False |
426 | + return True |
427 | |
428 | === modified file 'deployer/service.py' |
429 | --- deployer/service.py 2013-09-13 13:36:22 +0000 |
430 | +++ deployer/service.py 2013-11-01 15:46:48 +0000 |
431 | @@ -4,6 +4,9 @@ |
432 | self.svc_data = svc_data |
433 | self.name = name |
434 | |
435 | + def __repr__(self): |
436 | + return "<Service %s>" % (self.name) |
437 | + |
438 | @property |
439 | def config(self): |
440 | return self.svc_data.get('options', None) |
441 | @@ -17,9 +20,14 @@ |
442 | return int(self.svc_data.get('num_units', 1)) |
443 | |
444 | @property |
445 | - def force_machine(self): |
446 | - return self.svc_data.get('to') or self.svc_data.get( |
447 | - 'force-machine') |
448 | + def unit_placement(self): |
449 | + # Separate checks to support machine 0 placement. |
450 | + value = self.svc_data.get('to') |
451 | + if value is None: |
452 | + value = self.svc_data.get('force-machine') |
453 | + if value is not None and not isinstance(value, list): |
454 | + value = [value] |
455 | + return value or [] |
456 | |
457 | @property |
458 | def expose(self): |
459 | |
460 | === modified file 'deployer/tests/test_config.py' |
461 | --- deployer/tests/test_config.py 2013-10-10 21:18:38 +0000 |
462 | +++ deployer/tests/test_config.py 2013-11-01 15:46:48 +0000 |
463 | @@ -54,7 +54,6 @@ |
464 | config.get('my-files-frontend-dev').get_services()] |
465 | self.assertTrue(set(wordpress).issubset(set(my_app))) |
466 | |
467 | - |
468 | def test_inherits_config_overridden(self): |
469 | config = ConfigStack([ |
470 | os.path.join(self.test_data_dir, "stack-default.cfg"), |
471 | @@ -66,12 +65,12 @@ |
472 | # over-ridden |
473 | self.assertEquals(db.config.get('tuning-level'), 'fastest') |
474 | |
475 | - |
476 | def test_multi_inheritance_multi_files(self): |
477 | config = ConfigStack([ |
478 | os.path.join(self.test_data_dir, "openstack", "openstack.cfg"), |
479 | os.path.join(self.test_data_dir, "openstack", "ubuntu_base.cfg"), |
480 | - os.path.join(self.test_data_dir, "openstack", "openstack_base.cfg"), |
481 | + os.path.join( |
482 | + self.test_data_dir, "openstack", "openstack_base.cfg"), |
483 | ]) |
484 | self._test_multiple_inheritance(config) |
485 | |
486 | @@ -107,7 +106,7 @@ |
487 | |
488 | deployment = config.get('precise-grizzly') |
489 | services = [s.name for s in list(deployment.get_services())] |
490 | - self.assertEquals(['nova-cloud-controller', 'mysql'], services) |
491 | + self.assertEquals(['mysql', 'nova-cloud-controller'], services) |
492 | |
493 | nova = deployment.get_service('nova-cloud-controller') |
494 | self.assertEquals(nova.config['openstack-origin'], |
495 | @@ -116,8 +115,8 @@ |
496 | deployment = config.get('precise-grizzly-quantum') |
497 | services = [s.name for s in list(deployment.get_services())] |
498 | self.assertEquals(services, |
499 | - ['quantum-gateway', 'nova-cloud-controller', |
500 | - 'mysql']) |
501 | + ['mysql', 'nova-cloud-controller', |
502 | + 'quantum-gateway']) |
503 | nova = deployment.get_service('nova-cloud-controller') |
504 | self.assertEquals(nova.config['network-manager'], 'Quantum') |
505 | self.assertEquals(nova.config['openstack-origin'], |
506 | |
507 | === added file 'deployer/tests/test_data/stack-placement-invalid-2.yaml' |
508 | --- deployer/tests/test_data/stack-placement-invalid-2.yaml 1970-01-01 00:00:00 +0000 |
509 | +++ deployer/tests/test_data/stack-placement-invalid-2.yaml 2013-11-01 15:46:48 +0000 |
510 | @@ -0,0 +1,13 @@ |
511 | +stack: |
512 | + series: precise |
513 | + services: |
514 | + nova-compute: |
515 | + charm: cs:precise/nova-compute |
516 | + units: 3 |
517 | + ceph: |
518 | + units: 3 |
519 | + to: [nova-compute, nova-compute, nova-compute] |
520 | + mysql: |
521 | + to: lxc:nova-compute |
522 | + wordpress: |
523 | + to: lxc:foobar |
524 | |
525 | === added file 'deployer/tests/test_data/stack-placement-invalid.yaml' |
526 | --- deployer/tests/test_data/stack-placement-invalid.yaml 1970-01-01 00:00:00 +0000 |
527 | +++ deployer/tests/test_data/stack-placement-invalid.yaml 2013-11-01 15:46:48 +0000 |
528 | @@ -0,0 +1,13 @@ |
529 | +stack: |
530 | + series: precise |
531 | + services: |
532 | + nova-compute: |
533 | + charm: cs:precise/nova-compute |
534 | + units: 3 |
535 | + ceph: |
536 | + units: 3 |
537 | + to: [nova-compute, nova-compute, nova-compute] |
538 | + mysql: |
539 | + to: lxc:nova-compute |
540 | + wordpress: |
541 | + to: lxc:mysql |
542 | |
543 | === added file 'deployer/tests/test_data/stack-placement.yaml' |
544 | --- deployer/tests/test_data/stack-placement.yaml 1970-01-01 00:00:00 +0000 |
545 | +++ deployer/tests/test_data/stack-placement.yaml 2013-11-01 15:46:48 +0000 |
546 | @@ -0,0 +1,18 @@ |
547 | +stack: |
548 | + series: precise |
549 | + services: |
550 | + nova-compute: |
551 | + charm: cs:precise/nova-compute |
552 | + units: 3 |
553 | + ceph: |
554 | + units: 3 |
555 | + to: [nova-compute, nova-compute] |
556 | + mysql: |
557 | + to: 0 |
558 | + quantum: |
559 | + units: 4 |
560 | + to: ["lxc:nova-compute", "lxc:nova-compute", "lxc:nova-compute", "lxc:nova-compute"] |
561 | + verity: |
562 | + to: lxc:nova-compute=2 |
563 | + semper: |
564 | + to: nova-compute=2 |
565 | |
566 | === modified file 'deployer/tests/test_deployment.py' |
567 | --- deployer/tests/test_deployment.py 2013-07-24 23:10:15 +0000 |
568 | +++ deployer/tests/test_deployment.py 2013-11-01 15:46:48 +0000 |
569 | @@ -5,7 +5,7 @@ |
570 | |
571 | from deployer.config import ConfigStack |
572 | from deployer.deployment import Deployment |
573 | -from deployer.utils import setup_logging |
574 | +from deployer.utils import setup_logging, ErrorExit |
575 | |
576 | from .base import Base |
577 | |
578 | @@ -16,6 +16,11 @@ |
579 | self.output = setup_logging( |
580 | debug=True, verbose=True, stream=StringIO.StringIO()) |
581 | |
582 | + def get_named_deployment(self, file_name, stack_name): |
583 | + return ConfigStack( |
584 | + [os.path.join( |
585 | + self.test_data_dir, file_name)]).get(stack_name) |
586 | + |
587 | def test_deployer(self): |
588 | d = ConfigStack( |
589 | [os.path.join( |
590 | @@ -51,6 +56,66 @@ |
591 | list(d.get_relations()), |
592 | [('blog', 'db'), ('blog', 'cache'), ('blog', 'haproxy')]) |
593 | |
594 | + def test_validate_placement_sorting(self): |
595 | + d = self.get_named_deployment("stack-placement.yaml", "stack") |
596 | + services = d.get_services() |
597 | + self.assertEqual(services[0].name, 'nova-compute') |
598 | + try: |
599 | + d.validate_placement() |
600 | + except ErrorExit: |
601 | + self.fail("Should not fail") |
602 | + |
603 | + def test_validate_invalid_placement_nested(self): |
604 | + d = self.get_named_deployment("stack-placement-invalid.yaml", "stack") |
605 | + services = d.get_services() |
606 | + self.assertEqual(services[0].name, 'nova-compute') |
607 | + try: |
608 | + d.validate_placement() |
609 | + except ErrorExit: |
610 | + pass |
611 | + else: |
612 | + self.fail("Should fail") |
613 | + |
614 | + def test_validate_invalid_placement_no_with_service(self): |
615 | + d = self.get_named_deployment( |
616 | + "stack-placement-invalid-2.yaml", "stack") |
617 | + services = d.get_services() |
618 | + self.assertEqual(services[0].name, 'nova-compute') |
619 | + try: |
620 | + d.validate_placement() |
621 | + except ErrorExit: |
622 | + pass |
623 | + else: |
624 | + self.fail("Should fail") |
625 | + |
626 | + def test_get_unit_placement(self): |
627 | + d = self.get_named_deployment("stack-placement.yaml", "stack") |
628 | + status = { |
629 | + 'services': { |
630 | + 'nova-compute': { |
631 | + 'units': { |
632 | + 'nova-compute/2': {'machine': '1'}, |
633 | + 'nova-compute/3': {'machine': '2'}, |
634 | + 'nova-compute/4': {'machine': '3'}}}}} |
635 | + svc = d.get_service('ceph') |
636 | + self.assertEqual(d.get_unit_placement(svc, 0, status), '1') |
637 | + self.assertEqual(d.get_unit_placement(svc, 1, status), '2') |
638 | + self.assertEqual(d.get_unit_placement(svc, 2, status), None) |
639 | + |
640 | + svc = d.get_service('quantum') |
641 | + self.assertEqual(d.get_unit_placement(svc, 0, status), 'lxc:1') |
642 | + self.assertEqual(d.get_unit_placement(svc, 2, status), 'lxc:3') |
643 | + self.assertEqual(d.get_unit_placement(svc, 3, status), None) |
644 | + |
645 | + svc = d.get_service('verity') |
646 | + self.assertEqual(d.get_unit_placement(svc, 0, status), 'lxc:3') |
647 | + |
648 | + svc = d.get_service('mysql') |
649 | + self.assertEqual(d.get_unit_placement(svc, 0, status), '0') |
650 | + |
651 | + svc = d.get_service('semper') |
652 | + self.assertEqual(d.get_unit_placement(svc, 0, status), '3') |
653 | + |
654 | def test_multiple_relations_no_weight(self): |
655 | data = {"relations": {"wordpress": {"consumes": ["mysql"]}, |
656 | "nginx": {"consumes": ["wordpress"]}}} |
657 | |
658 | === modified file 'doc/config.rst' |
659 | --- doc/config.rst 2013-05-16 03:20:42 +0000 |
660 | +++ doc/config.rst 2013-11-01 15:46:48 +0000 |
661 | @@ -94,3 +94,57 @@ |
662 | constraints: mem=16 |
663 | options: |
664 | tuning: optimized |
665 | + |
666 | + |
667 | +Placement |
668 | +========= |
669 | + |
670 | +Flexible unit placement can be specified via deployer. Primarily this is via |
671 | +specifying service units deployments alongside those of another service. |
672 | + |
673 | +Each unit's placement must be specified individually, absence of placement for |
674 | +a unit results in juju default behavior for the given constraints. |
675 | + |
676 | +One special placement form is machine placement, which only allowed to machine 0, |
677 | +as other machine identities are ambigious for most usage scenarios. |
678 | + |
679 | +Both container and hulk-smash placement are supported. Different |
680 | +containerization and deploy with services can be mixed. |
681 | + |
682 | +The deployed-with service must have enough units to hold # # Nested |
683 | +to: specifications are not supported (ie in the below wordpress can't |
684 | +# be deployed to mysql because mysql already specifies a 'to' |
685 | +placement) |
686 | + |
687 | +Example:: |
688 | + envExport: |
689 | + services: |
690 | + mysql: |
691 | + # The only machine id supported is machine 0 |
692 | + to: 0 |
693 | + wordpress: |
694 | + units: 3 |
695 | + redis-server: |
696 | + units: 3 |
697 | + to: [lxc:wordpress, wordpress] |
698 | + ceph: |
699 | + units: 4 |
700 | + to: [wordpress, wordpress, wordpress, wordpress] |
701 | + serenade: |
702 | + to: lxc:wordpress=2 |
703 | + |
704 | +In this case the first unit of redis-server is deployed in a container |
705 | +on wordpress/0 The second unit of redis-server is deployed hulk smash |
706 | +onto wordpress/1 The third unit of redis-server gets a full machine |
707 | +allocated to itself. |
708 | + |
709 | +For ceph, we deploy hulk smash of the first 3 units, the final unit doesn't |
710 | +have a corresponding unit of wordpress and is deployed (along with a console |
711 | +warning) to a separate machine per its default constraints. |
712 | + |
713 | +The serenade service is overriding the default deploy-with unit by |
714 | +explicitly specifying a unit index for the deployment. These are not |
715 | +unit id based but rather zero based offsets into a sorted list of |
716 | +units. |
717 | + |
718 | + |
719 | |
720 | === modified file 'setup.py' |
721 | --- setup.py 2013-10-11 17:01:16 +0000 |
722 | +++ setup.py 2013-11-01 15:46:48 +0000 |
723 | @@ -12,7 +12,7 @@ |
724 | author="Kapil Thangavelu", |
725 | author_email="kapil.foss@gmail.com", |
726 | url="http://launchpad.net/juju-deployer", |
727 | - install_requires=["jujuclient >= 0.0.7"], |
728 | + install_requires=["jujuclient >= 0.13"], |
729 | packages=find_packages(), |
730 | classifiers=[ |
731 | "Development Status :: 2 - Pre-Alpha", |
Testing this now, couple of issues so far:
If a service is configured as:
fooservice:
to: [0]
deployer blows up:
70, in get_unit_placement
if ':' in unit_placement:
TypeError: argument of type 'int' is not iterable
Only running that code if unit_placement is a string seems to fix that.
Adding units fails:
2013-10-31 17:34:26 [INFO] deployer.import: Adding 2 more units to nova-compute bin/juju- deployer" , line 9, in <module> entry_point( 'juju-deployer= =0.2.8' , 'console_scripts', 'juju-deployer')() lib/python2. 7/dist- packages/ juju_deployer- 0.2.8-py2. 7.egg/deployer/ cli.py" , line 118, in main lib/python2. 7/dist- packages/ juju_deployer- 0.2.8-py2. 7.egg/deployer/ cli.py" , line 209, in run Importer( env, deployment, options).run() lib/python2. 7/dist- packages/ juju_deployer- 0.2.8-py2. 7.egg/deployer/ action/ importer. py", line 186, in run add_units( ) lib/python2. 7/dist- packages/ juju_deployer- 0.2.8-py2. 7.egg/deployer/ action/ importer. py", line 54, in add_units env.add_ unit(svc. name, mspec) lib/python2. 7/dist- packages/ juju_deployer- 0.2.8-py2. 7.egg/deployer/ env/go. py", line 28, in add_unit add_unit( service_ name, machine_spec) lib/python2. 7/dist- packages/ jujuclient- 0.12-py2. 7.egg/jujuclien t.py", line 465, in add_unit lib/python2. 7/dist- packages/ jujuclient- 0.12-py2. 7.egg/jujuclien t.py", line 135, in _rpc EnvError: <Env Error - Details:
Traceback (most recent call last):
File "/usr/local/
load_
File "/usr/local/
run()
File "/usr/local/
importer.
File "/usr/local/
self.
File "/usr/local/
self.
File "/usr/local/
return self.client.
File "/usr/local/
"Params": params})
File "/usr/local/
raise EnvError(result)
jujuclient.
{ u'Error': u'must add at least one unit', u'RequestId': 1, u'Response': { }}