Merge lp:~frankban/juju-deployer/unit-placement-fix into lp:juju-deployer
- unit-placement-fix
- Merge into trunk
Status: | Merged |
---|---|
Merged at revision: | 152 |
Proposed branch: | lp:~frankban/juju-deployer/unit-placement-fix |
Merge into: | lp:juju-deployer |
Diff against target: |
593 lines (+156/-107) 11 files modified
deployer/action/importer.py (+15/-24) deployer/config.py (+5/-5) deployer/deployment.py (+3/-3) deployer/service.py (+46/-45) deployer/tests/base.py (+36/-0) deployer/tests/test_data/v4/container-existing-machine.yaml (+1/-1) deployer/tests/test_data/v4/placement-invalid-number.yaml (+8/-0) deployer/tests/test_deployment.py (+23/-10) deployer/tests/test_guiserver.py (+7/-3) deployer/tests/test_importer.py (+11/-15) setup.py (+1/-1) |
To merge this branch: | bzr merge lp:~frankban/juju-deployer/unit-placement-fix |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Madison Scott-Clary (community) | code and qa | Approve | |
Tim Van Steenburgh (community) | Approve | ||
Brad Crittenden (community) | code | Approve | |
Review via email: mp+267187@code.launchpad.net |
Commit message
Description of the change
Fix deployments of bundles v4.
Fix some incorrect behavior of new v4 bundles, mostly
concerning validation and ordering of services.
Improve status retrieval while adding unit.
Fix checking for existing machines: it did work
properly only with top level machines.
Fix a problem with machines mapping while
co-locating units to other services, units or
to declared machines.
Improve the logic used for detecting v4 vs v3
bundle syntax: the machines section in v4 can
be safely omitted accotding to the spec.
Fix sleep loops and memory leaks while running
tests exercising the importer.
Some lint/code style clean up.
Bump version up a little.
Tim Van Steenburgh (tvansteenburgh) wrote : | # |
Brad Crittenden (bac) wrote : | # |
Thanks for this fix Francesco. It LGTM.
Francesco Banconi (frankban) wrote : | # |
As mentioned on IRC, the sleep loops are solved by setting up the wnv mock with patch_env_status.
I also removed the reloading logic, as requested.
Thank you both for the reviews!
- 159. By Francesco Banconi
-
Remove the reloading logic in importer.add_unit.
Tim Van Steenburgh (tvansteenburgh) wrote : | # |
LGTM!
Madison Scott-Clary (makyo) wrote : | # |
Thanks Francesco, much appreciated, this LGTM. QA okay on local.
Preview Diff
1 | === modified file 'deployer/action/importer.py' | |||
2 | --- deployer/action/importer.py 2015-08-04 12:50:12 +0000 | |||
3 | +++ deployer/action/importer.py 2015-08-06 14:43:20 +0000 | |||
4 | @@ -14,13 +14,11 @@ | |||
5 | 14 | self.options = options | 14 | self.options = options |
6 | 15 | self.env = env | 15 | self.env = env |
7 | 16 | self.deployment = deployment | 16 | self.deployment = deployment |
8 | 17 | self.machines = {} | ||
9 | 18 | 17 | ||
10 | 19 | def add_units(self): | 18 | def add_units(self): |
11 | 20 | self.log.debug("Adding units...") | 19 | self.log.debug("Adding units...") |
12 | 21 | # Add units to existing services that don't match count. | 20 | # Add units to existing services that don't match count. |
13 | 22 | env_status = self.env.status() | 21 | env_status = self.env.status() |
14 | 23 | reloaded = False | ||
15 | 24 | 22 | ||
16 | 25 | # Workaround an issue where watch output doesn't include subordinate | 23 | # Workaround an issue where watch output doesn't include subordinate |
17 | 26 | # services right away, and add_unit would fail while attempting to add | 24 | # services right away, and add_unit would fail while attempting to add |
18 | @@ -56,18 +54,9 @@ | |||
19 | 56 | self.log.info( | 54 | self.log.info( |
20 | 57 | "Adding %d more units to %s" % (abs(delta), svc.name)) | 55 | "Adding %d more units to %s" % (abs(delta), svc.name)) |
21 | 58 | if svc.unit_placement: | 56 | if svc.unit_placement: |
34 | 59 | # Reload status once after non placed services units are done. | 57 | # Reload status before each placed unit is deployed, so that |
35 | 60 | if reloaded is False: | 58 | # co-location to other units can take place properly. |
36 | 61 | # Improved crappy workaround juju-core api inconsistency | 59 | env_status = self.env.status() |
25 | 62 | delay = time.time() + 60 | ||
26 | 63 | while delay > time.time(): | ||
27 | 64 | time.sleep(5.1) | ||
28 | 65 | env_status = self.env.status() | ||
29 | 66 | if svc.name in env_status['services']: | ||
30 | 67 | break | ||
31 | 68 | |||
32 | 69 | reloaded = True | ||
33 | 70 | |||
37 | 71 | placement = self.deployment.get_unit_placement(svc, env_status) | 60 | placement = self.deployment.get_unit_placement(svc, env_status) |
38 | 72 | for mid in range(cur_units, svc.num_units): | 61 | for mid in range(cur_units, svc.num_units): |
39 | 73 | self.env.add_unit(svc.name, placement.get(mid)) | 62 | self.env.add_unit(svc.name, placement.get(mid)) |
40 | @@ -76,8 +65,7 @@ | |||
41 | 76 | 65 | ||
42 | 77 | def machine_exists(self, id): | 66 | def machine_exists(self, id): |
43 | 78 | """Checks if the given id exists on the current environment.""" | 67 | """Checks if the given id exists on the current environment.""" |
46 | 79 | return int(id) in map(int, | 68 | return str(id) in map(str, self.env.status().get('machines', {})) |
45 | 80 | self.env.status().get('machines', {}).keys()) | ||
47 | 81 | 69 | ||
48 | 82 | def create_machines(self): | 70 | def create_machines(self): |
49 | 83 | """Create machines as specified in the machine spec in the bundle. | 71 | """Create machines as specified in the machine spec in the bundle. |
50 | @@ -104,19 +92,23 @@ | |||
51 | 104 | creation is skipped. | 92 | creation is skipped. |
52 | 105 | """ | 93 | """ |
53 | 106 | machines = self.deployment.get_machines() | 94 | machines = self.deployment.get_machines() |
54 | 95 | machines_map = {} | ||
55 | 107 | 96 | ||
56 | 108 | if machines: | 97 | if machines: |
57 | 109 | self.log.info("Creating machines...") | 98 | self.log.info("Creating machines...") |
58 | 110 | for machine_name, spec in machines.items(): | 99 | for machine_name, spec in machines.items(): |
59 | 111 | if self.machine_exists(machine_name): | 100 | if self.machine_exists(machine_name): |
60 | 101 | # XXX frankban: do we really want this? The machine | ||
61 | 102 | # identifiers as included in v4 bundles are not intended | ||
62 | 103 | # to refer to real machines. A mapping could be provided | ||
63 | 104 | # but this kind of implicit mapping seems weak. | ||
64 | 112 | self.log.info( | 105 | self.log.info( |
65 | 113 | """Machine %s already exists on environment, """ | 106 | """Machine %s already exists on environment, """ |
66 | 114 | """skipping creation""" % machine_name) | 107 | """skipping creation""" % machine_name) |
69 | 115 | self.machines[machine_name] = self.deployment.get_machine( | 108 | machines_map[machine_name] = str(machine_name) |
68 | 116 | machine_name) | ||
70 | 117 | else: | 109 | else: |
71 | 118 | self.log.info("Machine %s will be created" % machine_name) | 110 | self.log.info("Machine %s will be created" % machine_name) |
73 | 119 | self.machines[machine_name] = self.env.add_machine( | 111 | machines_map[machine_name] = self.env.add_machine( |
74 | 120 | series=spec.get('series', | 112 | series=spec.get('series', |
75 | 121 | self.deployment.data['series']), | 113 | self.deployment.data['series']), |
76 | 122 | constraints=spec.get('constraints')) | 114 | constraints=spec.get('constraints')) |
77 | @@ -125,7 +117,7 @@ | |||
78 | 125 | annotations = spec.get('annotations') | 117 | annotations = spec.get('annotations') |
79 | 126 | if annotations: | 118 | if annotations: |
80 | 127 | self.env.set_annotation( | 119 | self.env.set_annotation( |
82 | 128 | self.machines[machine_name], | 120 | machines_map[machine_name], |
83 | 129 | annotations, | 121 | annotations, |
84 | 130 | entity_type='machine') | 122 | entity_type='machine') |
85 | 131 | 123 | ||
86 | @@ -139,13 +131,12 @@ | |||
87 | 139 | if self.machine_exists(machine_name): | 131 | if self.machine_exists(machine_name): |
88 | 140 | self.log.info("Machine %s already exists," | 132 | self.log.info("Machine %s already exists," |
89 | 141 | "skipping creation" % machine_name) | 133 | "skipping creation" % machine_name) |
92 | 142 | self.machines[container_host] = self.deployment.get_machine( | 134 | machines_map[container_host] = str(container_host) |
91 | 143 | container_host) | ||
93 | 144 | else: | 135 | else: |
94 | 145 | self.log.info("A new container will be created" | 136 | self.log.info("A new container will be created" |
95 | 146 | "on machine: %s" % container_host) | 137 | "on machine: %s" % container_host) |
98 | 147 | self.machines[container_host] = self.env.add_machine() | 138 | machines_map[container_host] = self.env.add_machine() |
99 | 148 | self.deployment.set_machines(machines) | 139 | self.deployment.set_machines(machines_map) |
100 | 149 | 140 | ||
101 | 150 | def get_charms(self): | 141 | def get_charms(self): |
102 | 151 | # Get Charms | 142 | # Get Charms |
103 | 152 | 143 | ||
104 | === modified file 'deployer/config.py' | |||
105 | --- deployer/config.py 2015-03-16 20:00:02 +0000 | |||
106 | +++ deployer/config.py 2015-08-06 14:43:20 +0000 | |||
107 | @@ -51,9 +51,9 @@ | |||
108 | 51 | raise | 51 | raise |
109 | 52 | 52 | ||
110 | 53 | # Check if this is a v4 bundle. | 53 | # Check if this is a v4 bundle. |
114 | 54 | if 'services' in yaml_result: | 54 | services = yaml_result.get('services') |
115 | 55 | if 'machines' in yaml_result: | 55 | if isinstance(services, dict) and 'services' not in services: |
116 | 56 | self.version = 4 | 56 | self.version = 4 |
117 | 57 | yaml_result = {config_file: yaml_result} | 57 | yaml_result = {config_file: yaml_result} |
118 | 58 | 58 | ||
119 | 59 | self.yaml[config_file] = yaml_result | 59 | self.yaml[config_file] = yaml_result |
120 | @@ -64,7 +64,7 @@ | |||
121 | 64 | return sorted(self.data) | 64 | return sorted(self.data) |
122 | 65 | 65 | ||
123 | 66 | def get(self, key): | 66 | def get(self, key): |
125 | 67 | if not key in self.data: | 67 | if key not in self.data: |
126 | 68 | self.log.warning("Deployment %r not found. Available %s", | 68 | self.log.warning("Deployment %r not found. Available %s", |
127 | 69 | key, ", ".join(self.keys())) | 69 | key, ", ".join(self.keys())) |
128 | 70 | raise ErrorExit() | 70 | raise ErrorExit() |
129 | @@ -99,7 +99,7 @@ | |||
130 | 99 | return parents | 99 | return parents |
131 | 100 | 100 | ||
132 | 101 | def _resolve_inherited(self, deploy_data): | 101 | def _resolve_inherited(self, deploy_data): |
134 | 102 | if not 'inherits' in deploy_data: | 102 | if 'inherits' not in deploy_data: |
135 | 103 | return deploy_data | 103 | return deploy_data |
136 | 104 | inherits = parents = self._inherits(deploy_data) | 104 | inherits = parents = self._inherits(deploy_data) |
137 | 105 | for parent_name in parents: | 105 | for parent_name in parents: |
138 | 106 | 106 | ||
139 | === modified file 'deployer/deployment.py' | |||
140 | --- deployer/deployment.py 2015-05-13 14:16:46 +0000 | |||
141 | +++ deployer/deployment.py 2015-08-06 14:43:20 +0000 | |||
142 | @@ -38,7 +38,7 @@ | |||
143 | 38 | pprint.pprint(self.data) | 38 | pprint.pprint(self.data) |
144 | 39 | 39 | ||
145 | 40 | def get_service(self, name): | 40 | def get_service(self, name): |
147 | 41 | if not name in self.data['services']: | 41 | if name not in self.data['services']: |
148 | 42 | return | 42 | return |
149 | 43 | return Service(name, self.data['services'][name]) | 43 | return Service(name, self.data['services'][name]) |
150 | 44 | 44 | ||
151 | @@ -95,9 +95,9 @@ | |||
152 | 95 | # Check for colocation. This naively assumes that there is no | 95 | # Check for colocation. This naively assumes that there is no |
153 | 96 | # circularity in placements. | 96 | # circularity in placements. |
154 | 97 | if x_in_y(svc_b, svc_a): | 97 | if x_in_y(svc_b, svc_a): |
155 | 98 | return 1 | ||
156 | 99 | if x_in_y(svc_a, svc_b): | ||
157 | 98 | return -1 | 100 | return -1 |
158 | 99 | if x_in_y(svc_a, svc_b): | ||
159 | 100 | return 1 | ||
160 | 101 | # If no colocation exists, simply compare names. | 101 | # If no colocation exists, simply compare names. |
161 | 102 | return cmp(svc_a.name, svc_b.name) | 102 | return cmp(svc_a.name, svc_b.name) |
162 | 103 | return 1 | 103 | return 1 |
163 | 104 | 104 | ||
164 | === modified file 'deployer/service.py' | |||
165 | --- deployer/service.py 2015-03-18 18:19:43 +0000 | |||
166 | +++ deployer/service.py 2015-08-06 14:43:20 +0000 | |||
167 | @@ -220,7 +220,7 @@ | |||
168 | 220 | 220 | ||
169 | 221 | def _fill_placement(self): | 221 | def _fill_placement(self): |
170 | 222 | """Fill the placement spec with necessary data. | 222 | """Fill the placement spec with necessary data. |
172 | 223 | 223 | ||
173 | 224 | From the spec: | 224 | From the spec: |
174 | 225 | A unit placement may be specified with a service name only, in which | 225 | A unit placement may be specified with a service name only, in which |
175 | 226 | case its unit number is assumed to be one more than the unit number of | 226 | case its unit number is assumed to be one more than the unit number of |
176 | @@ -238,7 +238,7 @@ | |||
177 | 238 | return | 238 | return |
178 | 239 | 239 | ||
179 | 240 | self.service.svc_data['to'] = ( | 240 | self.service.svc_data['to'] = ( |
181 | 241 | unit_mapping + | 241 | unit_mapping + |
182 | 242 | list(itertools.repeat(unit_mapping[-1], unit_count - len(unit_mapping))) | 242 | list(itertools.repeat(unit_mapping[-1], unit_count - len(unit_mapping))) |
183 | 243 | ) | 243 | ) |
184 | 244 | unit_mapping = self.service.unit_placement | 244 | unit_mapping = self.service.unit_placement |
185 | @@ -256,9 +256,9 @@ | |||
186 | 256 | 256 | ||
187 | 257 | def _parse_placement(self, placement): | 257 | def _parse_placement(self, placement): |
188 | 258 | """Parse a unit placement statement. | 258 | """Parse a unit placement statement. |
190 | 259 | 259 | ||
191 | 260 | In version 4 bundles, unit placement statements take the form of | 260 | In version 4 bundles, unit placement statements take the form of |
193 | 261 | 261 | ||
194 | 262 | (<containertype>:)?(<unit>|<machine>|new) | 262 | (<containertype>:)?(<unit>|<machine>|new) |
195 | 263 | 263 | ||
196 | 264 | This splits the placement into a container, a placement, and a unit | 264 | This splits the placement into a container, a placement, and a unit |
197 | @@ -278,7 +278,7 @@ | |||
198 | 278 | containers, and/or services must be internally consistent, consistent | 278 | containers, and/or services must be internally consistent, consistent |
199 | 279 | with other services in the deployment, and consistent with any machines | 279 | with other services in the deployment, and consistent with any machines |
200 | 280 | specified in the 'machines' block of the deployment. | 280 | specified in the 'machines' block of the deployment. |
202 | 281 | 281 | ||
203 | 282 | A feedback object is returned, potentially with errors and warnings | 282 | A feedback object is returned, potentially with errors and warnings |
204 | 283 | inside it. | 283 | inside it. |
205 | 284 | """ | 284 | """ |
206 | @@ -294,55 +294,51 @@ | |||
207 | 294 | 294 | ||
208 | 295 | services = dict([(s.name, s) for s in self.deployment.get_services()]) | 295 | services = dict([(s.name, s) for s in self.deployment.get_services()]) |
209 | 296 | machines = self.deployment.get_machines() | 296 | machines = self.deployment.get_machines() |
212 | 297 | container = None | 297 | service_name = self.service.name |
211 | 298 | unit_number = None | ||
213 | 299 | 298 | ||
214 | 300 | for i, placement in enumerate(unit_placement): | 299 | for i, placement in enumerate(unit_placement): |
216 | 301 | container, placement, unit_number = self._parse_placement(placement) | 300 | container, target, unit_number = self._parse_placement(placement) |
217 | 302 | 301 | ||
218 | 302 | # Validate the container type. | ||
219 | 303 | if container and container not in ('lxc', 'kvm'): | 303 | if container and container not in ('lxc', 'kvm'): |
220 | 304 | feedback.error( | 304 | feedback.error( |
230 | 305 | "Invalid container type: %s service: %s placement: %s" \ | 305 | 'Invalid container type: {} service: {} placement: {}' |
231 | 306 | % (container, self.service.name, unit_placement[i])) | 306 | ''.format(container, service_name, placement)) |
223 | 307 | # XXX Nesting containers not supported yet. | ||
224 | 308 | # Makyo - 2015-03-01 | ||
225 | 309 | if container is not None and not (placement.isdigit() | ||
226 | 310 | or placement == 'new'): | ||
227 | 311 | feedback.error( | ||
228 | 312 | "Invalid target for container: %s" % ( | ||
229 | 313 | unit_placement[i])) | ||
232 | 314 | # Specify an existing machine (or, if the number is in the | 307 | # Specify an existing machine (or, if the number is in the |
233 | 315 | # list of machine specs, one of those). | 308 | # list of machine specs, one of those). |
244 | 316 | if placement.isdigit(): | 309 | if str(target) in machines: |
235 | 317 | if placement in machines: | ||
236 | 318 | continue | ||
237 | 319 | else: | ||
238 | 320 | feedback.error( | ||
239 | 321 | ("Service placement to machine " | ||
240 | 322 | "not supported %s to %s") % ( | ||
241 | 323 | self.service.name, unit_placement[i])) | ||
242 | 324 | # Specify a machine from the machine spec. | ||
243 | 325 | elif placement in self.deployment.get_machines(): | ||
245 | 326 | continue | 310 | continue |
258 | 327 | # Specify a service for colocation. | 311 | if target.isdigit(): |
259 | 328 | elif placement in services: | 312 | feedback.error( |
260 | 329 | # Specify a particular unit for colocation. | 313 | 'Service placement to machine not supported: {} to {}' |
261 | 330 | if unit_number is not None and \ | 314 | ''.format(service_name, placement)) |
262 | 331 | unit_number > services[placement].num_units: | 315 | # Specify a service for co-location. |
263 | 332 | feedback.error( | 316 | elif target in services: |
264 | 333 | "Service unit does not exist, %s to %s/%s" % ( | 317 | # Specify a particular unit for co-location. |
265 | 334 | self.service.name, placement, unit_number)) | 318 | if unit_number is not None: |
266 | 335 | elif self.deployment.get_charm_for(placement).is_subordinate(): | 319 | try: |
267 | 336 | feedback.error( | 320 | unit_number = int(unit_number) |
268 | 337 | "Cannot place to a subordinate service: %s -> %s" % ( | 321 | except (TypeError, ValueError): |
269 | 338 | self.service.name, placement)) | 322 | feedback.error( |
270 | 323 | 'Invalid unit number for placement: {} to {}' | ||
271 | 324 | ''.format(service_name, unit_number)) | ||
272 | 325 | continue | ||
273 | 326 | if unit_number > services[target].num_units: | ||
274 | 327 | feedback.error( | ||
275 | 328 | 'Service unit does not exist: {} to {}/{}' | ||
276 | 329 | ''.format(service_name, target, unit_number)) | ||
277 | 330 | continue | ||
278 | 331 | if self.deployment.get_charm_for(target).is_subordinate(): | ||
279 | 332 | feedback.error( | ||
280 | 333 | 'Cannot place to a subordinate service: {} -> {}' | ||
281 | 334 | ''.format(service_name, target)) | ||
282 | 339 | # Create a new machine or container. | 335 | # Create a new machine or container. |
284 | 340 | elif placement == 'new': | 336 | elif target == 'new': |
285 | 341 | continue | 337 | continue |
286 | 342 | else: | 338 | else: |
287 | 343 | feedback.error( | 339 | feedback.error( |
290 | 344 | "Invalid service placement %s to %s" % ( | 340 | 'Invalid service placement: {} to {}' |
291 | 345 | self.service.name, unit_placement[i])) | 341 | ''.format(service_name, placement)) |
292 | 346 | return feedback | 342 | return feedback |
293 | 347 | 343 | ||
294 | 348 | def get_new_machines_for_containers(self): | 344 | def get_new_machines_for_containers(self): |
295 | @@ -367,6 +363,8 @@ | |||
296 | 367 | svc = self.service | 363 | svc = self.service |
297 | 368 | 364 | ||
298 | 369 | unit_mapping = svc.unit_placement | 365 | unit_mapping = svc.unit_placement |
299 | 366 | if not unit_mapping: | ||
300 | 367 | return None | ||
301 | 370 | unit_placement = placement = str(unit_mapping[unit_number]) | 368 | unit_placement = placement = str(unit_mapping[unit_number]) |
302 | 371 | container = None | 369 | container = None |
303 | 372 | u_idx = unit_number | 370 | u_idx = unit_number |
304 | @@ -375,14 +373,17 @@ | |||
305 | 375 | if placement == 'new': | 373 | if placement == 'new': |
306 | 376 | return None | 374 | return None |
307 | 377 | 375 | ||
309 | 378 | container, placement, unit_number = self._parse_placement(unit_placement) | 376 | container, placement, unit_number = self._parse_placement( |
310 | 377 | unit_placement) | ||
311 | 379 | 378 | ||
312 | 380 | if placement in self.machines_map: | 379 | if placement in self.machines_map: |
314 | 381 | return self._format_placement(self.machines_map[placement], container) | 380 | return self._format_placement( |
315 | 381 | self.machines_map[placement], container) | ||
316 | 382 | 382 | ||
317 | 383 | # Handle <container_type>:new | 383 | # Handle <container_type>:new |
318 | 384 | if placement == 'new': | 384 | if placement == 'new': |
319 | 385 | return self._format_placement( | 385 | return self._format_placement( |
321 | 386 | self.machines_map['%s/%d' % (self.service.name, u_idx)], container) | 386 | self.machines_map['%s/%d' % (self.service.name, u_idx)], |
322 | 387 | container) | ||
323 | 387 | 388 | ||
324 | 388 | return self.colocate(status, placement, u_idx, container, svc) | 389 | return self.colocate(status, placement, u_idx, container, svc) |
325 | 389 | 390 | ||
326 | === modified file 'deployer/tests/base.py' | |||
327 | --- deployer/tests/base.py 2015-03-18 18:19:43 +0000 | |||
328 | +++ deployer/tests/base.py 2015-08-06 14:43:20 +0000 | |||
329 | @@ -6,6 +6,8 @@ | |||
330 | 6 | import StringIO | 6 | import StringIO |
331 | 7 | import tempfile | 7 | import tempfile |
332 | 8 | 8 | ||
333 | 9 | import mock | ||
334 | 10 | |||
335 | 9 | import deployer | 11 | import deployer |
336 | 10 | from deployer.config import ConfigStack | 12 | from deployer.config import ConfigStack |
337 | 11 | 13 | ||
338 | @@ -77,3 +79,37 @@ | |||
339 | 77 | os.environ.update(original_environ) | 79 | os.environ.update(original_environ) |
340 | 78 | 80 | ||
341 | 79 | os.environ.update(kw) | 81 | os.environ.update(kw) |
342 | 82 | |||
343 | 83 | |||
344 | 84 | def patch_env_status(env, services, machines=()): | ||
345 | 85 | """Simulate that the given mock env has the status described in services. | ||
346 | 86 | |||
347 | 87 | This function is used so that tests do not have to wait minutes for | ||
348 | 88 | service units presence when the importer is used with the given env. | ||
349 | 89 | |||
350 | 90 | The services argument is a dict mapping service names with the number of | ||
351 | 91 | their units. This will be reflected by the status returned when the | ||
352 | 92 | importer adds the units (see "deployer.action.importer.Importer.add_unit"). | ||
353 | 93 | |||
354 | 94 | The machines argument can be used to simulate that the given machines are | ||
355 | 95 | present in the Juju environment. | ||
356 | 96 | """ | ||
357 | 97 | services_status = dict( | ||
358 | 98 | (k, {'units': dict((i, {}) for i in range(v))}) | ||
359 | 99 | for k, v in services.items() | ||
360 | 100 | ) | ||
361 | 101 | machines_status = dict((i, {}) for i in machines) | ||
362 | 102 | env.status.side_effect = [ | ||
363 | 103 | # There are no services initially. | ||
364 | 104 | {'services': {}, 'machines': machines_status}, | ||
365 | 105 | {'services': {}, 'machines': machines_status}, | ||
366 | 106 | # This is the workaround check for subordinate charms presence: | ||
367 | 107 | # see lp:1421315 for details. | ||
368 | 108 | {'services': services_status, 'machines': machines_status}, | ||
369 | 109 | {'services': services_status, 'machines': machines_status}, | ||
370 | 110 | # After we exited the workaround loop, we can just mock further | ||
371 | 111 | # status results. | ||
372 | 112 | mock.MagicMock(), | ||
373 | 113 | mock.MagicMock(), | ||
374 | 114 | mock.MagicMock(), | ||
375 | 115 | ] | ||
376 | 80 | 116 | ||
377 | === modified file 'deployer/tests/test_data/v4/container-existing-machine.yaml' | |||
378 | --- deployer/tests/test_data/v4/container-existing-machine.yaml 2015-05-13 14:06:34 +0000 | |||
379 | +++ deployer/tests/test_data/v4/container-existing-machine.yaml 2015-08-06 14:43:20 +0000 | |||
380 | @@ -39,4 +39,4 @@ | |||
381 | 39 | - - mediawiki:db | 39 | - - mediawiki:db |
382 | 40 | - mysql:db | 40 | - mysql:db |
383 | 41 | machines: | 41 | machines: |
385 | 42 | 1: 1 | 42 | 1: {} |
386 | 43 | 43 | ||
387 | === added file 'deployer/tests/test_data/v4/placement-invalid-number.yaml' | |||
388 | --- deployer/tests/test_data/v4/placement-invalid-number.yaml 1970-01-01 00:00:00 +0000 | |||
389 | +++ deployer/tests/test_data/v4/placement-invalid-number.yaml 2015-08-06 14:43:20 +0000 | |||
390 | @@ -0,0 +1,8 @@ | |||
391 | 1 | series: trusty | ||
392 | 2 | services: | ||
393 | 3 | django: | ||
394 | 4 | charm: cs:trusty/django | ||
395 | 5 | to: | ||
396 | 6 | - lxc:haproxy/bad-wolf | ||
397 | 7 | haproxy: | ||
398 | 8 | charm: cs:trusty/haproxy | ||
399 | 0 | 9 | ||
400 | === modified file 'deployer/tests/test_deployment.py' | |||
401 | --- deployer/tests/test_deployment.py 2015-03-18 18:19:43 +0000 | |||
402 | +++ deployer/tests/test_deployment.py 2015-08-06 14:43:20 +0000 | |||
403 | @@ -106,12 +106,17 @@ | |||
404 | 106 | d._machines_placement_sort( | 106 | d._machines_placement_sort( |
405 | 107 | FauxService(name="x", unit_placement=['asdf']), | 107 | FauxService(name="x", unit_placement=['asdf']), |
406 | 108 | FauxService(name="y", unit_placement=['lxc:x/1']) | 108 | FauxService(name="y", unit_placement=['lxc:x/1']) |
408 | 109 | ), 1) | 109 | ), -1) |
409 | 110 | self.assertEqual( | 110 | self.assertEqual( |
410 | 111 | d._machines_placement_sort( | 111 | d._machines_placement_sort( |
411 | 112 | FauxService(name="y", unit_placement=['lxc:x/1']), | 112 | FauxService(name="y", unit_placement=['lxc:x/1']), |
412 | 113 | FauxService(name="x", unit_placement=['asdf']) | 113 | FauxService(name="x", unit_placement=['asdf']) |
414 | 114 | ), -1) | 114 | ), 1) |
415 | 115 | self.assertEqual( | ||
416 | 116 | d._machines_placement_sort( | ||
417 | 117 | FauxService(name="x", unit_placement=['y']), | ||
418 | 118 | FauxService(name="y") | ||
419 | 119 | ), 1) | ||
420 | 115 | self.assertEqual( | 120 | self.assertEqual( |
421 | 116 | d._machines_placement_sort( | 121 | d._machines_placement_sort( |
422 | 117 | FauxService(name="x", unit_placement=['asdf']), | 122 | FauxService(name="x", unit_placement=['asdf']), |
423 | @@ -195,12 +200,12 @@ | |||
424 | 195 | self.assertIn( | 200 | self.assertIn( |
425 | 196 | 'Cannot place to a subordinate service: nova-compute -> nrpe\n', | 201 | 'Cannot place to a subordinate service: nova-compute -> nrpe\n', |
426 | 197 | output) | 202 | output) |
428 | 198 | 203 | ||
429 | 199 | @skip_if_offline | 204 | @skip_if_offline |
430 | 200 | def test_validate_invalid_placement_subordinate_v4(self): | 205 | def test_validate_invalid_placement_subordinate_v4(self): |
431 | 201 | # Placement validation fails if a subordinate charm is provided. | 206 | # Placement validation fails if a subordinate charm is provided. |
432 | 202 | deployment = self.get_deployment_and_fetch_v4( | 207 | deployment = self.get_deployment_and_fetch_v4( |
434 | 203 | 'placement-invalid-subordinate.yaml') | 208 | 'placement-invalid-subordinate.yaml') |
435 | 204 | with self.assertRaises(ErrorExit): | 209 | with self.assertRaises(ErrorExit): |
436 | 205 | deployment.validate_placement() | 210 | deployment.validate_placement() |
437 | 206 | output = self.output.getvalue() | 211 | output = self.output.getvalue() |
438 | @@ -208,6 +213,15 @@ | |||
439 | 208 | 'Cannot place to a subordinate service: nova-compute -> nrpe\n', | 213 | 'Cannot place to a subordinate service: nova-compute -> nrpe\n', |
440 | 209 | output) | 214 | output) |
441 | 210 | 215 | ||
442 | 216 | def test_validate_invalid_unit_number(self): | ||
443 | 217 | # Placement validation fails if an invalid unit number is provided. | ||
444 | 218 | deployment = self.get_deployment_v4('placement-invalid-number.yaml') | ||
445 | 219 | with self.assertRaises(ErrorExit): | ||
446 | 220 | deployment.validate_placement() | ||
447 | 221 | output = self.output.getvalue() | ||
448 | 222 | self.assertIn( | ||
449 | 223 | 'Invalid unit number for placement: django to bad-wolf\n', output) | ||
450 | 224 | |||
451 | 211 | def test_get_unit_placement_v3(self): | 225 | def test_get_unit_placement_v3(self): |
452 | 212 | d = self.get_named_deployment_v3("stack-placement.yaml", "stack") | 226 | d = self.get_named_deployment_v3("stack-placement.yaml", "stack") |
453 | 213 | status = { | 227 | status = { |
454 | @@ -281,12 +295,11 @@ | |||
455 | 281 | feedback = placement.validate() | 295 | feedback = placement.validate() |
456 | 282 | self.assertEqual(feedback.get_errors(), [ | 296 | self.assertEqual(feedback.get_errors(), [ |
457 | 283 | 'Invalid container type: asdf service: mysql placement: asdf:0', | 297 | 'Invalid container type: asdf service: mysql placement: asdf:0', |
464 | 284 | 'Service placement to machine not supported mysql to asdf:0', | 298 | 'Service placement to machine not supported: mysql to asdf:0', |
465 | 285 | 'Invalid target for container: lxc:asdf', | 299 | 'Invalid service placement: mysql to lxc:asdf', |
466 | 286 | 'Invalid service placement mysql to lxc:asdf', | 300 | 'Service placement to machine not supported: mysql to 1', |
467 | 287 | 'Service placement to machine not supported mysql to 1', | 301 | 'Service unit does not exist: mysql to wordpress/3', |
468 | 288 | 'Service unit does not exist, mysql to wordpress/3', | 302 | 'Invalid service placement: mysql to asdf']) |
463 | 289 | 'Invalid service placement mysql to asdf']) | ||
469 | 290 | 303 | ||
470 | 291 | def test_get_unit_placement_v4_simple(self): | 304 | def test_get_unit_placement_v4_simple(self): |
471 | 292 | d = self.get_deployment_v4('simple.yaml') | 305 | d = self.get_deployment_v4('simple.yaml') |
472 | 293 | 306 | ||
473 | === modified file 'deployer/tests/test_guiserver.py' | |||
474 | --- deployer/tests/test_guiserver.py 2015-03-17 10:40:34 +0000 | |||
475 | +++ deployer/tests/test_guiserver.py 2015-08-06 14:43:20 +0000 | |||
476 | @@ -11,7 +11,10 @@ | |||
477 | 11 | 11 | ||
478 | 12 | from deployer import guiserver | 12 | from deployer import guiserver |
479 | 13 | from deployer.feedback import Feedback | 13 | from deployer.feedback import Feedback |
481 | 14 | from deployer.tests.base import skip_if_offline | 14 | from deployer.tests.base import ( |
482 | 15 | patch_env_status, | ||
483 | 16 | skip_if_offline, | ||
484 | 17 | ) | ||
485 | 15 | 18 | ||
486 | 16 | 19 | ||
487 | 17 | class TestGetDefaultGuiserverOptions(unittest.TestCase): | 20 | class TestGetDefaultGuiserverOptions(unittest.TestCase): |
488 | @@ -268,6 +271,8 @@ | |||
489 | 268 | def test_importer_behavior(self, mock_sleep, mock_environment): | 271 | def test_importer_behavior(self, mock_sleep, mock_environment): |
490 | 269 | # The importer executes the expected environment calls. | 272 | # The importer executes the expected environment calls. |
491 | 270 | self.addCleanup(self.cleanup_series_path) | 273 | self.addCleanup(self.cleanup_series_path) |
492 | 274 | patch_env_status(mock_environment(), {'mysql': 1, 'wordpress': 1}) | ||
493 | 275 | mock_environment.reset_mock() | ||
494 | 271 | with self.patch_juju_home(): | 276 | with self.patch_juju_home(): |
495 | 272 | self.import_bundle() | 277 | self.import_bundle() |
496 | 273 | mock_sleep.assert_has_calls([mock.call(5.1), mock.call(60)]) | 278 | mock_sleep.assert_has_calls([mock.call(5.1), mock.call(60)]) |
497 | @@ -289,8 +294,7 @@ | |||
498 | 289 | None, 1, None), | 294 | None, 1, None), |
499 | 290 | mock.call.set_annotation( | 295 | mock.call.set_annotation( |
500 | 291 | 'wordpress', {'gui-y': '414.547', 'gui-x': '425.347'}), | 296 | 'wordpress', {'gui-y': '414.547', 'gui-x': '425.347'}), |
503 | 292 | mock.call.add_units('mysql', 2), | 297 | mock.call.add_units('mysql', 1), |
502 | 293 | mock.call.add_units('wordpress', 1), | ||
504 | 294 | mock.call.add_relation('mysql:db', 'wordpress:db'), | 298 | mock.call.add_relation('mysql:db', 'wordpress:db'), |
505 | 295 | mock.call.close(), | 299 | mock.call.close(), |
506 | 296 | ], any_order=True) | 300 | ], any_order=True) |
507 | 297 | 301 | ||
508 | === modified file 'deployer/tests/test_importer.py' | |||
509 | --- deployer/tests/test_importer.py 2015-05-13 14:06:34 +0000 | |||
510 | +++ deployer/tests/test_importer.py 2015-08-06 14:43:20 +0000 | |||
511 | @@ -6,7 +6,11 @@ | |||
512 | 6 | from deployer.action.importer import Importer | 6 | from deployer.action.importer import Importer |
513 | 7 | from deployer.utils import yaml_dump, yaml_load | 7 | from deployer.utils import yaml_dump, yaml_load |
514 | 8 | 8 | ||
516 | 9 | from base import Base, skip_if_offline | 9 | from base import ( |
517 | 10 | Base, | ||
518 | 11 | patch_env_status, | ||
519 | 12 | skip_if_offline, | ||
520 | 13 | ) | ||
521 | 10 | 14 | ||
522 | 11 | 15 | ||
523 | 12 | class Options(dict): | 16 | class Options(dict): |
524 | @@ -51,6 +55,7 @@ | |||
525 | 51 | stack = ConfigStack(self.options.configs) | 55 | stack = ConfigStack(self.options.configs) |
526 | 52 | deploy = stack.get('wiki') | 56 | deploy = stack.get('wiki') |
527 | 53 | env = mock.MagicMock() | 57 | env = mock.MagicMock() |
528 | 58 | patch_env_status(env, {'wiki': 1, 'db': 1}) | ||
529 | 54 | importer = Importer(env, deploy, self.options) | 59 | importer = Importer(env, deploy, self.options) |
530 | 55 | importer.run() | 60 | importer.run() |
531 | 56 | 61 | ||
532 | @@ -60,10 +65,6 @@ | |||
533 | 60 | 'wiki', 'cs:precise/mediawiki', '', config, None, 1, None)) | 65 | 'wiki', 'cs:precise/mediawiki', '', config, None, 1, None)) |
534 | 61 | env.add_relation.assert_called_once_with('wiki', 'db') | 66 | env.add_relation.assert_called_once_with('wiki', 'db') |
535 | 62 | 67 | ||
536 | 63 | self.assertEqual( | ||
537 | 64 | yaml_load(yaml_dump(yaml_load(yaml_dump(config)))), | ||
538 | 65 | config) | ||
539 | 66 | |||
540 | 67 | @skip_if_offline | 68 | @skip_if_offline |
541 | 68 | @mock.patch('deployer.action.importer.time') | 69 | @mock.patch('deployer.action.importer.time') |
542 | 69 | def test_importer_no_relations(self, mock_time): | 70 | def test_importer_no_relations(self, mock_time): |
543 | @@ -71,10 +72,10 @@ | |||
544 | 71 | stack = ConfigStack(self.options.configs) | 72 | stack = ConfigStack(self.options.configs) |
545 | 72 | deploy = stack.get('wiki') | 73 | deploy = stack.get('wiki') |
546 | 73 | env = mock.MagicMock() | 74 | env = mock.MagicMock() |
547 | 75 | patch_env_status(env, {'wiki': 1, 'db': 1}) | ||
548 | 74 | importer = Importer(env, deploy, self.options) | 76 | importer = Importer(env, deploy, self.options) |
549 | 75 | importer.run() | 77 | importer.run() |
552 | 76 | 78 | self.assertFalse(env.add_relation.called) | |
551 | 77 | self.assertFalse(env.add_relation.called, False) | ||
553 | 78 | 79 | ||
554 | 79 | @skip_if_offline | 80 | @skip_if_offline |
555 | 80 | @mock.patch('deployer.action.importer.time') | 81 | @mock.patch('deployer.action.importer.time') |
556 | @@ -84,6 +85,7 @@ | |||
557 | 84 | stack = ConfigStack(self.options.configs) | 85 | stack = ConfigStack(self.options.configs) |
558 | 85 | deploy = stack.get(self.options.configs[0]) | 86 | deploy = stack.get(self.options.configs[0]) |
559 | 86 | env = mock.MagicMock() | 87 | env = mock.MagicMock() |
560 | 88 | patch_env_status(env, {'mediawiki': 1, 'mysql': 1}) | ||
561 | 87 | importer = Importer(env, deploy, self.options) | 89 | importer = Importer(env, deploy, self.options) |
562 | 88 | importer.run() | 90 | importer.run() |
563 | 89 | 91 | ||
564 | @@ -104,14 +106,8 @@ | |||
565 | 104 | stack = ConfigStack(self.options.configs) | 106 | stack = ConfigStack(self.options.configs) |
566 | 105 | deploy = stack.get(self.options.configs[0]) | 107 | deploy = stack.get(self.options.configs[0]) |
567 | 106 | env = mock.MagicMock() | 108 | env = mock.MagicMock() |
574 | 107 | 109 | patch_env_status(env, {'mediawiki': 1, 'mysql': 1}, machines=[1]) | |
569 | 108 | config = {'status.return_value.get.return_value': { | ||
570 | 109 | 1: {} | ||
571 | 110 | }} | ||
572 | 111 | |||
573 | 112 | env.configure_mock(**config) | ||
575 | 113 | 110 | ||
576 | 114 | importer = Importer(env, deploy, self.options) | 111 | importer = Importer(env, deploy, self.options) |
577 | 115 | importer.run() | 112 | importer.run() |
580 | 116 | 113 | self.assertFalse(env.add_machine.called) | |
579 | 117 | env.add_machine.assert_not_called() | ||
581 | 118 | 114 | ||
582 | === modified file 'setup.py' | |||
583 | --- setup.py 2015-08-04 17:57:46 +0000 | |||
584 | +++ setup.py 2015-08-06 14:43:20 +0000 | |||
585 | @@ -3,7 +3,7 @@ | |||
586 | 3 | 3 | ||
587 | 4 | setup( | 4 | setup( |
588 | 5 | name="juju-deployer", | 5 | name="juju-deployer", |
590 | 6 | version="0.5.0", | 6 | version="0.5.1", |
591 | 7 | description="A tool for deploying complex stacks with juju.", | 7 | description="A tool for deploying complex stacks with juju.", |
592 | 8 | long_description=open("README").read(), | 8 | long_description=open("README").read(), |
593 | 9 | author="Kapil Thangavelu", | 9 | author="Kapil Thangavelu", |
One inline comment, plus I wonder if you could point out which change fixes the test memory leak problem? Otherwise looks good.