Merge lp:~gz/juju-ci-tools/endpoints_bindings into lp:juju-ci-tools
- endpoints_bindings
- Merge into trunk
Status: | Merged |
---|---|
Approved by: | Martin Packman |
Approved revision: | 1660 |
Merged at revision: | 1675 |
Proposed branch: | lp:~gz/juju-ci-tools/endpoints_bindings |
Merge into: | lp:juju-ci-tools |
Diff against target: |
708 lines (+586/-6) 6 files modified
assess_endpoint_bindings.py (+344/-0) jujupy.py (+23/-5) substrate.py (+22/-0) tests/test_assess_endpoint_bindings.py (+177/-0) tests/test_jujupy.py (+2/-1) tests/test_substrate.py (+18/-0) |
To merge this branch: | bzr merge lp:~gz/juju-ci-tools/endpoints_bindings |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Curtis Hovey (community) | code | Approve | |
Review via email: mp+308923@code.launchpad.net |
Commit message
Description of the change
Work in progress on making endpoint bindings test that can be run on shared maas
Basic outline is that the networking parts are set up and torn down using the underlying maas commands each run, and that machines that need interfaces reconfigured are constrained from being used on other maas jobs. There are still some details here that need discussing in more depth, but the basic outline here should provide a start.
I have chopped out most of the test complexity that I was adding locally for ease of review. Also, had to make some changes to how the network configuration was done in light of how finfolk is actually set up, so for now have deleted the tests based on real data for the maas configuration, will update with the new behaviour and re-add.
There's a bunch more to talk about here as well, happy to answer any questions.
Martin Packman (gz) wrote : | # |
Thanks for all the comments, have replied in line and with push some changes.
- 1659. By Martin Packman
-
Address review comments from sinzui and use persistent spaces
- 1660. By Martin Packman
-
Give gateway_ip when creating subnets
Preview Diff
1 | === added file 'assess_endpoint_bindings.py' | |||
2 | --- assess_endpoint_bindings.py 1970-01-01 00:00:00 +0000 | |||
3 | +++ assess_endpoint_bindings.py 2016-10-21 22:18:30 +0000 | |||
4 | @@ -0,0 +1,344 @@ | |||
5 | 1 | #!/usr/bin/env python | ||
6 | 2 | """Validate endpoint bindings functionality on MAAS.""" | ||
7 | 3 | |||
8 | 4 | from __future__ import print_function | ||
9 | 5 | |||
10 | 6 | import argparse | ||
11 | 7 | import contextlib | ||
12 | 8 | import logging | ||
13 | 9 | import os | ||
14 | 10 | import sys | ||
15 | 11 | import yaml | ||
16 | 12 | |||
17 | 13 | from deploy_stack import ( | ||
18 | 14 | BootstrapManager, | ||
19 | 15 | ) | ||
20 | 16 | from jujucharm import ( | ||
21 | 17 | Charm, | ||
22 | 18 | ) | ||
23 | 19 | from substrate import ( | ||
24 | 20 | maas_account_from_config, | ||
25 | 21 | ) | ||
26 | 22 | from utility import ( | ||
27 | 23 | add_basic_testing_arguments, | ||
28 | 24 | configure_logging, | ||
29 | 25 | temp_dir, | ||
30 | 26 | ) | ||
31 | 27 | |||
32 | 28 | |||
33 | 29 | log = logging.getLogger("assess_endpoint_bindings") | ||
34 | 30 | |||
35 | 31 | |||
36 | 32 | script_identifier = "endpoint-bindings" | ||
37 | 33 | |||
38 | 34 | # To avoid clashes with other tests these space names must be seperately | ||
39 | 35 | # registered in jujupy to populate constraints. | ||
40 | 36 | space_control = script_identifier + "-control" | ||
41 | 37 | space_data = script_identifier + "-data" | ||
42 | 38 | space_public = script_identifier + "-public" | ||
43 | 39 | |||
44 | 40 | |||
45 | 41 | def _generate_vids(start=10): | ||
46 | 42 | """ | ||
47 | 43 | Generate a series of vid values beginning with start. | ||
48 | 44 | |||
49 | 45 | Ideally these values would be carefully chosen to not clash with existing | ||
50 | 46 | vlans, but for now just hardcode. | ||
51 | 47 | """ | ||
52 | 48 | for vid in range(start, 4096): | ||
53 | 49 | yield vid | ||
54 | 50 | |||
55 | 51 | |||
56 | 52 | def _generate_cidrs(start=40, inc=10, block_pattern="10.0.{}.0/24"): | ||
57 | 53 | """ | ||
58 | 54 | Generate a series of cidrs based on block_pattern beginning with start. | ||
59 | 55 | |||
60 | 56 | Would be good not to hardcode but inspecting network for free ranges is | ||
61 | 57 | also non-trivial. | ||
62 | 58 | """ | ||
63 | 59 | for n in range(start, 255, inc): | ||
64 | 60 | yield block_pattern.format(n) | ||
65 | 61 | |||
66 | 62 | |||
67 | 63 | def ensure_spaces(manager, required_spaces): | ||
68 | 64 | """Return details for each given required_spaces creating spaces as needed. | ||
69 | 65 | |||
70 | 66 | :param manager: MAAS account manager. | ||
71 | 67 | :param required_spaces: List of space names that may need to be created. | ||
72 | 68 | """ | ||
73 | 69 | existing_spaces = manager.spaces() | ||
74 | 70 | log.info("Have spaces: %s", ", ".join(s["name"] for s in existing_spaces)) | ||
75 | 71 | spaces_map = dict((s["name"], s) for s in existing_spaces) | ||
76 | 72 | spaces = [] | ||
77 | 73 | for space_name in required_spaces: | ||
78 | 74 | space = spaces_map.get(space_name) | ||
79 | 75 | if space is None: | ||
80 | 76 | space = manager.create_space(space_name) | ||
81 | 77 | log.info("Created space: %r", space) | ||
82 | 78 | spaces.append(space) | ||
83 | 79 | return spaces | ||
84 | 80 | |||
85 | 81 | |||
86 | 82 | @contextlib.contextmanager | ||
87 | 83 | def reconfigure_networking(manager, required_spaces): | ||
88 | 84 | """Create new MAAS networking primitives to prepare for testing. | ||
89 | 85 | |||
90 | 86 | :param manager: MAAS account manager. | ||
91 | 87 | :param required_spaces: List of spaces to make with vlans and subnets. | ||
92 | 88 | """ | ||
93 | 89 | new_subnets = [] | ||
94 | 90 | new_vlans = [] | ||
95 | 91 | fabrics = manager.fabrics() | ||
96 | 92 | log.info("Have fabrics: %s", ", ".join(f["name"] for f in fabrics)) | ||
97 | 93 | new_fabric = manager.create_fabric(script_identifier) | ||
98 | 94 | try: | ||
99 | 95 | log.info("Created fabric: %r", new_fabric) | ||
100 | 96 | |||
101 | 97 | spaces = ensure_spaces(manager, required_spaces) | ||
102 | 98 | |||
103 | 99 | for vid, space_name in zip(_generate_vids(), required_spaces): | ||
104 | 100 | name = space_name + "-vlan" | ||
105 | 101 | new_vlans.append(manager.create_vlan(new_fabric["id"], vid, name)) | ||
106 | 102 | log.info("Created vlan: %r", new_vlans[-1]) | ||
107 | 103 | |||
108 | 104 | for cidr, vlan, space in zip(_generate_cidrs(), new_vlans, spaces): | ||
109 | 105 | new_subnets.append(manager.create_subnet( | ||
110 | 106 | cidr, fabric_id=new_fabric["id"], vlan_id=vlan["id"], | ||
111 | 107 | space=space["id"], gateway_ip=cidr.replace(".0/24", ".1"))) | ||
112 | 108 | log.info("Created subnet: %r", new_subnets[-1]) | ||
113 | 109 | |||
114 | 110 | yield new_fabric, spaces, list(new_vlans), list(new_subnets) | ||
115 | 111 | |||
116 | 112 | finally: | ||
117 | 113 | for subnet in new_subnets: | ||
118 | 114 | manager.delete_subnet(subnet["id"]) | ||
119 | 115 | log.info("Deleted subnet: %s", subnet["name"]) | ||
120 | 116 | |||
121 | 117 | for vlan in new_vlans: | ||
122 | 118 | manager.delete_vlan(new_fabric["id"], vlan["vid"]) | ||
123 | 119 | log.info("Deleted vlan: %s", vlan["name"]) | ||
124 | 120 | |||
125 | 121 | try: | ||
126 | 122 | manager.delete_fabric(new_fabric["id"]) | ||
127 | 123 | except Exception: | ||
128 | 124 | log.exception("Failed to delete fabric: %s", new_fabric["id"]) | ||
129 | 125 | raise | ||
130 | 126 | else: | ||
131 | 127 | log.info("Deleted fabric: %s", new_fabric["id"]) | ||
132 | 128 | |||
133 | 129 | |||
134 | 130 | @contextlib.contextmanager | ||
135 | 131 | def reconfigure_machines(manager, fabric, required_machine_subnets): | ||
136 | 132 | """ | ||
137 | 133 | Reconfigure MAAS machines with new interfaces to prepare for testing. | ||
138 | 134 | |||
139 | 135 | There are some unavoidable races if multiple jobs attempt to reconfigure | ||
140 | 136 | machines at the same time. Also, in heterogenous environments an | ||
141 | 137 | inadequate machine may be reserved at this time. | ||
142 | 138 | |||
143 | 139 | Ideally this function would just allocate some machines before operating | ||
144 | 140 | on them. Alas, MAAS doesn't allow interface changes on allocated machines | ||
145 | 141 | and Juju will not select them for deployment. | ||
146 | 142 | |||
147 | 143 | :param manager: MAAS account manager. | ||
148 | 144 | :param fabric: Data from MAAS about the fabric to be used. | ||
149 | 145 | :param required_machine_subnets: List of list of vlan and subnet ids. | ||
150 | 146 | """ | ||
151 | 147 | |||
152 | 148 | # Find all machines not currently being used | ||
153 | 149 | all_machines = manager.machines() | ||
154 | 150 | candidate_machines = [ | ||
155 | 151 | m for m in all_machines if m["status"] == manager.STATUS_READY] | ||
156 | 152 | # Take the id of the default vlan on the new fabric | ||
157 | 153 | default_vlan = fabric["vlans"][0]["id"] | ||
158 | 154 | |||
159 | 155 | configured_machines = [] | ||
160 | 156 | machine_interfaces = {} | ||
161 | 157 | try: | ||
162 | 158 | for machine_subnets in required_machine_subnets: | ||
163 | 159 | if not candidate_machines: | ||
164 | 160 | raise Exception("No ready maas machines to configure") | ||
165 | 161 | |||
166 | 162 | machine = candidate_machines.pop() | ||
167 | 163 | system_id = machine["system_id"] | ||
168 | 164 | # TODO(gz): Add logic to pick sane parent? | ||
169 | 165 | existing_interface = [ | ||
170 | 166 | interface for interface in machine["interface_set"] | ||
171 | 167 | if not any("subnet" in link for link in interface["links"]) | ||
172 | 168 | ][0] | ||
173 | 169 | previous_vlan_id = existing_interface["vlan"]["id"] | ||
174 | 170 | new_interfaces = [] | ||
175 | 171 | machine_interfaces[system_id] = ( | ||
176 | 172 | existing_interface, previous_vlan_id, new_interfaces) | ||
177 | 173 | manager.interface_update( | ||
178 | 174 | system_id, existing_interface["id"], vlan_id=default_vlan) | ||
179 | 175 | log.info("Changed existing interface: %s %s", system_id, | ||
180 | 176 | existing_interface["name"]) | ||
181 | 177 | parent = existing_interface["id"] | ||
182 | 178 | |||
183 | 179 | for vlan_id, subnet_id in machine_subnets: | ||
184 | 180 | links = [] | ||
185 | 181 | interface = manager.interface_create_vlan( | ||
186 | 182 | system_id, parent, vlan_id) | ||
187 | 183 | new_interfaces.append(interface) | ||
188 | 184 | log.info("Created interface: %r", interface) | ||
189 | 185 | |||
190 | 186 | updated_subnet = manager.interface_link_subnet( | ||
191 | 187 | system_id, interface["id"], "AUTO", subnet_id) | ||
192 | 188 | # TODO(gz): Need to pick out right link if multiple are added. | ||
193 | 189 | links.append(updated_subnet["links"][0]) | ||
194 | 190 | log.info("Created link: %r", links[-1]) | ||
195 | 191 | |||
196 | 192 | configured_machines.append(machine) | ||
197 | 193 | yield configured_machines | ||
198 | 194 | finally: | ||
199 | 195 | log.info("About to reset machine interfaces to original states.") | ||
200 | 196 | for system_id in machine_interfaces: | ||
201 | 197 | parent, vlan, children = machine_interfaces[system_id] | ||
202 | 198 | for child in children: | ||
203 | 199 | manager.delete_interface(system_id, child["id"]) | ||
204 | 200 | log.info("Deleted interface: %s %s", system_id, child["id"]) | ||
205 | 201 | manager.interface_update(system_id, parent["id"], vlan_id=vlan) | ||
206 | 202 | log.info("Reset original interface: %s", parent["name"]) | ||
207 | 203 | |||
208 | 204 | |||
209 | 205 | def create_test_charms(): | ||
210 | 206 | """Create charms for testing and bundle using them.""" | ||
211 | 207 | charm_datastore = Charm("datastore", "Testing datastore charm.") | ||
212 | 208 | charm_datastore.metadata["provides"] = { | ||
213 | 209 | "datastore": {"interface": "data"}, | ||
214 | 210 | } | ||
215 | 211 | |||
216 | 212 | charm_frontend = Charm("frontend", "Testing frontend charm.") | ||
217 | 213 | charm_frontend.metadata["provides"] = { | ||
218 | 214 | "website": {"interface": "http"}, | ||
219 | 215 | } | ||
220 | 216 | charm_frontend.metadata["requires"] = { | ||
221 | 217 | "datastore": {"interface": "data"}, | ||
222 | 218 | } | ||
223 | 219 | |||
224 | 220 | bundle = { | ||
225 | 221 | "services": { | ||
226 | 222 | "datastore": { | ||
227 | 223 | "charm": "./xenial/datastore", | ||
228 | 224 | "series": "xenial", | ||
229 | 225 | "num-units": 1, | ||
230 | 226 | "bindings": { | ||
231 | 227 | "datastore": space_data, | ||
232 | 228 | }, | ||
233 | 229 | }, | ||
234 | 230 | "frontend": { | ||
235 | 231 | "charm": "./xenial/frontend", | ||
236 | 232 | "series": "xenial", | ||
237 | 233 | "num-units": 1, | ||
238 | 234 | "bindings": { | ||
239 | 235 | "website": space_public, | ||
240 | 236 | "datastore": space_data, | ||
241 | 237 | }, | ||
242 | 238 | }, | ||
243 | 239 | }, | ||
244 | 240 | "relations": [ | ||
245 | 241 | ["datastore:datastore", "frontend:datastore"], | ||
246 | 242 | ], | ||
247 | 243 | } | ||
248 | 244 | return bundle, [charm_datastore, charm_frontend] | ||
249 | 245 | |||
250 | 246 | |||
251 | 247 | @contextlib.contextmanager | ||
252 | 248 | def using_bundle_and_charms(bundle, charms, bundle_name="bundle.yaml"): | ||
253 | 249 | """Commit bundle and charms to disk and gives path to bundle.""" | ||
254 | 250 | with temp_dir() as working_dir: | ||
255 | 251 | for charm in charms: | ||
256 | 252 | charm.to_repo_dir(working_dir) | ||
257 | 253 | |||
258 | 254 | # TODO(gz): Create a bundle abstration in jujucharm module | ||
259 | 255 | bundle_path = os.path.join(working_dir, bundle_name) | ||
260 | 256 | with open(bundle_path, "w") as f: | ||
261 | 257 | yaml.safe_dump(bundle, f) | ||
262 | 258 | |||
263 | 259 | yield bundle_path | ||
264 | 260 | |||
265 | 261 | |||
266 | 262 | def machine_spaces_for_bundle(bundle): | ||
267 | 263 | """Return a list of sets of spaces required for machines in bundle.""" | ||
268 | 264 | machines = [] | ||
269 | 265 | for service in bundle["services"].values(): | ||
270 | 266 | spaces = frozenset(service.get("bindings", {}).values()) | ||
271 | 267 | num_units = service.get("num_units", 1) | ||
272 | 268 | machines.extend([spaces] * num_units) | ||
273 | 269 | return machines | ||
274 | 270 | |||
275 | 271 | |||
276 | 272 | def bootstrap_and_test(bootstrap_manager, bundle_path, machine): | ||
277 | 273 | with bootstrap_manager.booted_context(False, to=machine): | ||
278 | 274 | client = bootstrap_manager.client | ||
279 | 275 | log.info("Deploying bundle.") | ||
280 | 276 | client.deploy(bundle_path) | ||
281 | 277 | log.info("Waiting for all units to start.") | ||
282 | 278 | client.wait_for_started() | ||
283 | 279 | client.wait_for_workloads() | ||
284 | 280 | log.info("Validating bindings.") | ||
285 | 281 | validate(client) | ||
286 | 282 | |||
287 | 283 | |||
288 | 284 | def validate(client): | ||
289 | 285 | """Ensure relations are bound to the correct spaces.""" | ||
290 | 286 | |||
291 | 287 | |||
292 | 288 | def assess_endpoint_bindings(maas_manager, bootstrap_manager): | ||
293 | 289 | required_spaces = [space_data, space_public] | ||
294 | 290 | |||
295 | 291 | bundle, charms = create_test_charms() | ||
296 | 292 | |||
297 | 293 | machine_spaces = machine_spaces_for_bundle(bundle) | ||
298 | 294 | # Add a bootstrap machine in all spaces | ||
299 | 295 | machine_spaces.insert(0, frozenset().union(*machine_spaces)) | ||
300 | 296 | |||
301 | 297 | log.info("About to write charms to disk.") | ||
302 | 298 | with using_bundle_and_charms(bundle, charms) as bundle_path: | ||
303 | 299 | log.info("About to reconfigure maas networking.") | ||
304 | 300 | with reconfigure_networking(maas_manager, required_spaces) as nets: | ||
305 | 301 | |||
306 | 302 | fabric, spaces, vlans, subnets = nets | ||
307 | 303 | # Derive the vlans and subnets that need to be added to machines | ||
308 | 304 | vlan_subnet_per_machine = [] | ||
309 | 305 | for spaces in machine_spaces: | ||
310 | 306 | idxs = sorted(required_spaces.index(space) for space in spaces) | ||
311 | 307 | vlans_subnets = [ | ||
312 | 308 | (vlans[i]["id"], subnets[i]["id"]) for i in idxs] | ||
313 | 309 | vlan_subnet_per_machine.append(vlans_subnets) | ||
314 | 310 | |||
315 | 311 | log.info("About to add new interfaces to machines.") | ||
316 | 312 | with reconfigure_machines( | ||
317 | 313 | maas_manager, fabric, vlan_subnet_per_machine) as machines: | ||
318 | 314 | |||
319 | 315 | bootstrap_manager.client.use_reserved_spaces(required_spaces) | ||
320 | 316 | |||
321 | 317 | base_machine = machines[0]["hostname"] | ||
322 | 318 | |||
323 | 319 | log.info("About to bootstrap.") | ||
324 | 320 | bootstrap_and_test( | ||
325 | 321 | bootstrap_manager, bundle_path, base_machine) | ||
326 | 322 | |||
327 | 323 | |||
328 | 324 | def parse_args(argv): | ||
329 | 325 | """Parse all arguments.""" | ||
330 | 326 | parser = argparse.ArgumentParser(description="assess endpoint bindings") | ||
331 | 327 | add_basic_testing_arguments(parser) | ||
332 | 328 | args = parser.parse_args(argv) | ||
333 | 329 | if args.upload_tools: | ||
334 | 330 | parser.error("giving --upload-tools meaningless on 2.0 only test") | ||
335 | 331 | return args | ||
336 | 332 | |||
337 | 333 | |||
338 | 334 | def main(argv=None): | ||
339 | 335 | args = parse_args(argv) | ||
340 | 336 | configure_logging(args.verbose) | ||
341 | 337 | bs_manager = BootstrapManager.from_args(args) | ||
342 | 338 | with maas_account_from_config(bs_manager.client.env.config) as account: | ||
343 | 339 | assess_endpoint_bindings(account, bs_manager) | ||
344 | 340 | return 0 | ||
345 | 341 | |||
346 | 342 | |||
347 | 343 | if __name__ == '__main__': | ||
348 | 344 | sys.exit(main()) | ||
349 | 0 | 345 | ||
350 | === modified file 'jujupy.py' | |||
351 | --- jujupy.py 2016-10-21 19:50:45 +0000 | |||
352 | +++ jujupy.py 2016-10-21 22:18:30 +0000 | |||
353 | @@ -940,6 +940,9 @@ | |||
354 | 940 | 940 | ||
355 | 941 | controller_permissions = frozenset(['login', 'addmodel', 'superuser']) | 941 | controller_permissions = frozenset(['login', 'addmodel', 'superuser']) |
356 | 942 | 942 | ||
357 | 943 | reserved_spaces = frozenset([ | ||
358 | 944 | 'endpoint-bindings-data', 'endpoint-bindings-public']) | ||
359 | 945 | |||
360 | 943 | @classmethod | 946 | @classmethod |
361 | 944 | def preferred_container(cls): | 947 | def preferred_container(cls): |
362 | 945 | for container_type in [LXD_MACHINE, LXC_MACHINE]: | 948 | for container_type in [LXD_MACHINE, LXC_MACHINE]: |
363 | @@ -1032,6 +1035,7 @@ | |||
364 | 1032 | feature_flags = self.feature_flags.intersection(cls.used_feature_flags) | 1035 | feature_flags = self.feature_flags.intersection(cls.used_feature_flags) |
365 | 1033 | backend = self._backend.clone(full_path, version, debug, feature_flags) | 1036 | backend = self._backend.clone(full_path, version, debug, feature_flags) |
366 | 1034 | other = cls.from_backend(backend, env) | 1037 | other = cls.from_backend(backend, env) |
367 | 1038 | other.excluded_spaces = set(self.excluded_spaces) | ||
368 | 1035 | return other | 1039 | return other |
369 | 1036 | 1040 | ||
370 | 1037 | @classmethod | 1041 | @classmethod |
371 | @@ -1108,6 +1112,7 @@ | |||
372 | 1108 | env.juju_home = get_juju_home() | 1112 | env.juju_home = get_juju_home() |
373 | 1109 | else: | 1113 | else: |
374 | 1110 | env.juju_home = juju_home | 1114 | env.juju_home = juju_home |
375 | 1115 | self.excluded_spaces = set(self.reserved_spaces) | ||
376 | 1111 | 1116 | ||
377 | 1112 | @property | 1117 | @property |
378 | 1113 | def version(self): | 1118 | def version(self): |
379 | @@ -1141,6 +1146,12 @@ | |||
380 | 1141 | return self._backend.shell_environ(self.used_feature_flags, | 1146 | return self._backend.shell_environ(self.used_feature_flags, |
381 | 1142 | self.env.juju_home) | 1147 | self.env.juju_home) |
382 | 1143 | 1148 | ||
383 | 1149 | def use_reserved_spaces(self, spaces): | ||
384 | 1150 | """Allow machines in given spaces to be allocated and used.""" | ||
385 | 1151 | if not self.reserved_spaces.issuperset(spaces): | ||
386 | 1152 | raise ValueError('Space not reserved: {}'.format(spaces)) | ||
387 | 1153 | self.excluded_spaces.difference_update(spaces) | ||
388 | 1154 | |||
389 | 1144 | def add_ssh_machines(self, machines): | 1155 | def add_ssh_machines(self, machines): |
390 | 1145 | for count, machine in enumerate(machines): | 1156 | for count, machine in enumerate(machines): |
391 | 1146 | try: | 1157 | try: |
392 | @@ -1163,11 +1174,7 @@ | |||
393 | 1163 | credential=None, auto_upgrade=False, metadata_source=None, | 1174 | credential=None, auto_upgrade=False, metadata_source=None, |
394 | 1164 | to=None, no_gui=False, agent_version=None): | 1175 | to=None, no_gui=False, agent_version=None): |
395 | 1165 | """Return the bootstrap arguments for the substrate.""" | 1176 | """Return the bootstrap arguments for the substrate.""" |
401 | 1166 | if self.env.joyent: | 1177 | constraints = self._get_substrate_constraints() |
397 | 1167 | # Only accept kvm packages by requiring >1 cpu core, see lp:1446264 | ||
398 | 1168 | constraints = 'mem=2G cpu-cores=1' | ||
399 | 1169 | else: | ||
400 | 1170 | constraints = 'mem=2G' | ||
402 | 1171 | cloud_region = self.get_cloud_region(self.env.get_cloud(), | 1178 | cloud_region = self.get_cloud_region(self.env.get_cloud(), |
403 | 1172 | self.env.get_region()) | 1179 | self.env.get_region()) |
404 | 1173 | # Note cloud_region before controller name | 1180 | # Note cloud_region before controller name |
405 | @@ -1523,6 +1530,10 @@ | |||
406 | 1523 | if self.env.joyent: | 1530 | if self.env.joyent: |
407 | 1524 | # Only accept kvm packages by requiring >1 cpu core, see lp:1446264 | 1531 | # Only accept kvm packages by requiring >1 cpu core, see lp:1446264 |
408 | 1525 | return 'mem=2G cpu-cores=1' | 1532 | return 'mem=2G cpu-cores=1' |
409 | 1533 | elif self.env.maas: | ||
410 | 1534 | # For now only maas support spaces in a meaningful way. | ||
411 | 1535 | return 'mem=2G spaces={}'.format( ','.join( | ||
412 | 1536 | '^' + space for space in sorted(self.excluded_spaces))) | ||
413 | 1526 | else: | 1537 | else: |
414 | 1527 | return 'mem=2G' | 1538 | return 'mem=2G' |
415 | 1528 | 1539 | ||
416 | @@ -2559,6 +2570,13 @@ | |||
417 | 2559 | else: | 2570 | else: |
418 | 2560 | return unqualified_model_name(self.model_name) | 2571 | return unqualified_model_name(self.model_name) |
419 | 2561 | 2572 | ||
420 | 2573 | def _get_substrate_constraints(self): | ||
421 | 2574 | if self.env.joyent: | ||
422 | 2575 | # Only accept kvm packages by requiring >1 cpu core, see lp:1446264 | ||
423 | 2576 | return 'mem=2G cpu-cores=1' | ||
424 | 2577 | else: | ||
425 | 2578 | return 'mem=2G' | ||
426 | 2579 | |||
427 | 2562 | def get_bootstrap_args(self, upload_tools, bootstrap_series=None, | 2580 | def get_bootstrap_args(self, upload_tools, bootstrap_series=None, |
428 | 2563 | credential=None): | 2581 | credential=None): |
429 | 2564 | """Return the bootstrap arguments for the substrate.""" | 2582 | """Return the bootstrap arguments for the substrate.""" |
430 | 2565 | 2583 | ||
431 | === modified file 'substrate.py' | |||
432 | --- substrate.py 2016-09-28 12:51:00 +0000 | |||
433 | +++ substrate.py 2016-10-21 22:18:30 +0000 | |||
434 | @@ -440,6 +440,8 @@ | |||
435 | 440 | 440 | ||
436 | 441 | _API_PATH = 'api/2.0/' | 441 | _API_PATH = 'api/2.0/' |
437 | 442 | 442 | ||
438 | 443 | STATUS_READY = 4 | ||
439 | 444 | |||
440 | 443 | SUBNET_CONNECTION_MODES = frozenset(('AUTO', 'DHCP', 'STATIC', 'LINK_UP')) | 445 | SUBNET_CONNECTION_MODES = frozenset(('AUTO', 'DHCP', 'STATIC', 'LINK_UP')) |
441 | 444 | 446 | ||
442 | 445 | def __init__(self, profile, url, oauth): | 447 | def __init__(self, profile, url, oauth): |
443 | @@ -493,6 +495,10 @@ | |||
444 | 493 | if v['ip_addresses']} | 495 | if v['ip_addresses']} |
445 | 494 | return ips | 496 | return ips |
446 | 495 | 497 | ||
447 | 498 | def machines(self): | ||
448 | 499 | """Return list of all machines.""" | ||
449 | 500 | return self._maas(self.profile, 'machines', 'read') | ||
450 | 501 | |||
451 | 496 | def fabrics(self): | 502 | def fabrics(self): |
452 | 497 | """Return list of all fabrics.""" | 503 | """Return list of all fabrics.""" |
453 | 498 | return self._maas(self.profile, 'fabrics', 'read') | 504 | return self._maas(self.profile, 'fabrics', 'read') |
454 | @@ -538,6 +544,22 @@ | |||
455 | 538 | """Return list of interfaces belonging to node with given system_id.""" | 544 | """Return list of interfaces belonging to node with given system_id.""" |
456 | 539 | return self._maas(self.profile, 'interfaces', 'read', system_id) | 545 | return self._maas(self.profile, 'interfaces', 'read', system_id) |
457 | 540 | 546 | ||
458 | 547 | def interface_update(self, system_id, interface_id, name=None, | ||
459 | 548 | mac_address=None, tags=None, vlan_id=None): | ||
460 | 549 | """Update fields of existing interface on node with given system_id.""" | ||
461 | 550 | args = [ | ||
462 | 551 | self.profile, 'interface', 'update', system_id, str(interface_id), | ||
463 | 552 | ] | ||
464 | 553 | if name is not None: | ||
465 | 554 | args.append('name=' + name) | ||
466 | 555 | if mac_address is not None: | ||
467 | 556 | args.append('mac_address=' + mac_address) | ||
468 | 557 | if tags is not None: | ||
469 | 558 | args.append('tags=' + tags) | ||
470 | 559 | if vlan_id is not None: | ||
471 | 560 | args.append('vlan=' + str(vlan_id)) | ||
472 | 561 | return self._maas(*args) | ||
473 | 562 | |||
474 | 541 | def interface_create_vlan(self, system_id, parent, vlan_id): | 563 | def interface_create_vlan(self, system_id, parent, vlan_id): |
475 | 542 | """Create a vlan interface on machine with given system_id.""" | 564 | """Create a vlan interface on machine with given system_id.""" |
476 | 543 | args = [ | 565 | args = [ |
477 | 544 | 566 | ||
478 | === added file 'tests/test_assess_endpoint_bindings.py' | |||
479 | --- tests/test_assess_endpoint_bindings.py 1970-01-01 00:00:00 +0000 | |||
480 | +++ tests/test_assess_endpoint_bindings.py 2016-10-21 22:18:30 +0000 | |||
481 | @@ -0,0 +1,177 @@ | |||
482 | 1 | """Tests for assess_endpoint_bindings module.""" | ||
483 | 2 | |||
484 | 3 | import logging | ||
485 | 4 | from mock import Mock, patch | ||
486 | 5 | import StringIO | ||
487 | 6 | |||
488 | 7 | from assess_endpoint_bindings import ( | ||
489 | 8 | assess_endpoint_bindings, | ||
490 | 9 | ensure_spaces, | ||
491 | 10 | parse_args, | ||
492 | 11 | machine_spaces_for_bundle, | ||
493 | 12 | main, | ||
494 | 13 | ) | ||
495 | 14 | from tests import ( | ||
496 | 15 | parse_error, | ||
497 | 16 | TestCase, | ||
498 | 17 | ) | ||
499 | 18 | from tests.test_jujupy import fake_juju_client | ||
500 | 19 | |||
501 | 20 | |||
502 | 21 | class TestParseArgs(TestCase): | ||
503 | 22 | |||
504 | 23 | def test_common_args(self): | ||
505 | 24 | args = parse_args(["an-env", "/bin/juju", "/tmp/logs", "an-env-mod"]) | ||
506 | 25 | self.assertEqual("an-env", args.env) | ||
507 | 26 | self.assertEqual("/bin/juju", args.juju_bin) | ||
508 | 27 | self.assertEqual("/tmp/logs", args.logs) | ||
509 | 28 | self.assertEqual("an-env-mod", args.temp_env_name) | ||
510 | 29 | self.assertEqual(False, args.debug) | ||
511 | 30 | |||
512 | 31 | def test_no_upload_tools(self): | ||
513 | 32 | with parse_error(self) as fake_stderr: | ||
514 | 33 | parse_args(["an-env", "/bin/juju", "--upload-tools"]) | ||
515 | 34 | self.assertIn( | ||
516 | 35 | "error: giving --upload-tools meaningless on 2.0 only test", | ||
517 | 36 | fake_stderr.getvalue()) | ||
518 | 37 | |||
519 | 38 | def test_help(self): | ||
520 | 39 | fake_stdout = StringIO.StringIO() | ||
521 | 40 | with parse_error(self) as fake_stderr: | ||
522 | 41 | with patch("sys.stdout", fake_stdout): | ||
523 | 42 | parse_args(["--help"]) | ||
524 | 43 | self.assertEqual("", fake_stderr.getvalue()) | ||
525 | 44 | self.assertIn("endpoint bindings", fake_stdout.getvalue()) | ||
526 | 45 | |||
527 | 46 | |||
528 | 47 | class TestEnsureSpaces(TestCase): | ||
529 | 48 | |||
530 | 49 | default_space = { | ||
531 | 50 | "name": "Default space", | ||
532 | 51 | "id": 0, | ||
533 | 52 | "resource_uri": "/MAAS/api/2.0/spaces/0/", | ||
534 | 53 | "subnets": [ | ||
535 | 54 | { | ||
536 | 55 | "space": "Default space", | ||
537 | 56 | "id": 2, | ||
538 | 57 | }, | ||
539 | 58 | ], | ||
540 | 59 | } | ||
541 | 60 | alpha_space = { | ||
542 | 61 | "name": "alpha-space", | ||
543 | 62 | "id": 60, | ||
544 | 63 | "resource_uri": "/MAAS/api/2.0/spaces/60/", | ||
545 | 64 | "subnets": [], | ||
546 | 65 | } | ||
547 | 66 | beta_space = { | ||
548 | 67 | "name": "beta-space", | ||
549 | 68 | "id": 61, | ||
550 | 69 | "resource_uri": "/MAAS/api/2.0/spaces/61/", | ||
551 | 70 | "subnets": [], | ||
552 | 71 | } | ||
553 | 72 | |||
554 | 73 | def test_all_existing(self): | ||
555 | 74 | manager = Mock(spec=["spaces"]) | ||
556 | 75 | manager.spaces.return_value = [self.default_space, self.alpha_space] | ||
557 | 76 | spaces = ensure_spaces(manager, ["alpha-space"]) | ||
558 | 77 | self.assertEqual(spaces, [self.alpha_space]) | ||
559 | 78 | manager.spaces.assert_called_once_with() | ||
560 | 79 | self.assertEqual( | ||
561 | 80 | "INFO Have spaces: Default space, alpha-space\n", | ||
562 | 81 | self.log_stream.getvalue()) | ||
563 | 82 | |||
564 | 83 | def test_some_existing(self): | ||
565 | 84 | manager = Mock(spec=["create_space", "spaces"]) | ||
566 | 85 | manager.create_space.return_value = self.alpha_space | ||
567 | 86 | manager.spaces.return_value = [self.default_space, self.beta_space] | ||
568 | 87 | spaces = ensure_spaces(manager, ["alpha-space", "beta-space"]) | ||
569 | 88 | self.assertEqual(spaces, [self.alpha_space, self.beta_space]) | ||
570 | 89 | manager.spaces.assert_called_once_with() | ||
571 | 90 | manager.create_space.assert_called_once_with("alpha-space") | ||
572 | 91 | self.assertRegexpMatches( | ||
573 | 92 | self.log_stream.getvalue(), | ||
574 | 93 | r"^INFO Have spaces: Default space, beta-space\n" | ||
575 | 94 | r"INFO Created space: \{.*\}\n$") | ||
576 | 95 | |||
577 | 96 | |||
578 | 97 | class TestMachineSpacesForBundle(TestCase): | ||
579 | 98 | |||
580 | 99 | def test_no_bindings(self): | ||
581 | 100 | bundle_without_bindings = { | ||
582 | 101 | "services": { | ||
583 | 102 | "ubuntu": { | ||
584 | 103 | "charm": "cs:ubuntu", | ||
585 | 104 | "num_units": 3, | ||
586 | 105 | }, | ||
587 | 106 | }, | ||
588 | 107 | } | ||
589 | 108 | machines = machine_spaces_for_bundle(bundle_without_bindings) | ||
590 | 109 | self.assertEqual(machines, [frozenset(), frozenset(), frozenset()]) | ||
591 | 110 | |||
592 | 111 | def test_single_binding(self): | ||
593 | 112 | bundle_without_bindings = { | ||
594 | 113 | "services": { | ||
595 | 114 | "anapp": { | ||
596 | 115 | "charm": "./anapp", | ||
597 | 116 | "series": "xenial", | ||
598 | 117 | "num_units": 1, | ||
599 | 118 | "bindings": { | ||
600 | 119 | "website": "space-public", | ||
601 | 120 | }, | ||
602 | 121 | }, | ||
603 | 122 | }, | ||
604 | 123 | } | ||
605 | 124 | machines = machine_spaces_for_bundle(bundle_without_bindings) | ||
606 | 125 | self.assertEqual(machines, [frozenset(["space-public"])]) | ||
607 | 126 | |||
608 | 127 | def test_multiple_bindings(self): | ||
609 | 128 | bundle_without_bindings = { | ||
610 | 129 | "services": { | ||
611 | 130 | "anapp": { | ||
612 | 131 | "charm": "./anapp", | ||
613 | 132 | "series": "xenial", | ||
614 | 133 | "num_units": 1, | ||
615 | 134 | "bindings": { | ||
616 | 135 | "website": "space-public", | ||
617 | 136 | "data": "space-data", | ||
618 | 137 | "monitoring": "space-ctl", | ||
619 | 138 | }, | ||
620 | 139 | }, | ||
621 | 140 | "adb": { | ||
622 | 141 | "charm": "./adb", | ||
623 | 142 | "series": "xenial", | ||
624 | 143 | "num_units": 2, | ||
625 | 144 | "bindings": { | ||
626 | 145 | "data": "space-data", | ||
627 | 146 | "monitoring": "space-ctl", | ||
628 | 147 | }, | ||
629 | 148 | }, | ||
630 | 149 | }, | ||
631 | 150 | } | ||
632 | 151 | machines = machine_spaces_for_bundle(bundle_without_bindings) | ||
633 | 152 | app_spaces = frozenset(["space-data", "space-ctl", "space-public"]) | ||
634 | 153 | db_spaces = frozenset(["space-data", "space-ctl"]) | ||
635 | 154 | self.assertEqual(machines, [app_spaces, db_spaces, db_spaces]) | ||
636 | 155 | |||
637 | 156 | |||
638 | 157 | class TestMain(TestCase): | ||
639 | 158 | |||
640 | 159 | def test_main(self): | ||
641 | 160 | argv = ["an-env", "/bin/juju", "/tmp/logs", "an-env-mod", "--verbose"] | ||
642 | 161 | env = Mock(spec=["config"]) | ||
643 | 162 | client = Mock(spec=["env", "is_jes_enabled"]) | ||
644 | 163 | client.env = env | ||
645 | 164 | with patch("assess_endpoint_bindings.configure_logging", | ||
646 | 165 | autospec=True) as mock_cl: | ||
647 | 166 | with patch("assess_endpoint_bindings.maas_account_from_config", | ||
648 | 167 | autospec=True) as mock_ma: | ||
649 | 168 | with patch("deploy_stack.client_from_config", | ||
650 | 169 | return_value=client) as mock_c: | ||
651 | 170 | with patch("assess_endpoint_bindings.assess_endpoint_bindings", | ||
652 | 171 | autospec=True) as mock_assess: | ||
653 | 172 | main(argv) | ||
654 | 173 | mock_cl.assert_called_once_with(logging.DEBUG) | ||
655 | 174 | mock_c.assert_called_once_with( | ||
656 | 175 | "an-env", "/bin/juju", debug=False, soft_deadline=None) | ||
657 | 176 | self.assertEqual(mock_ma.call_count, 1) | ||
658 | 177 | self.assertEqual(mock_assess.call_count, 1) | ||
659 | 0 | 178 | ||
660 | === modified file 'tests/test_jujupy.py' | |||
661 | --- tests/test_jujupy.py 2016-10-21 20:02:20 +0000 | |||
662 | +++ tests/test_jujupy.py 2016-10-21 22:18:30 +0000 | |||
663 | @@ -796,7 +796,8 @@ | |||
664 | 796 | client.bootstrap() | 796 | client.bootstrap() |
665 | 797 | mock.assert_called_with( | 797 | mock.assert_called_with( |
666 | 798 | 'bootstrap', ( | 798 | 'bootstrap', ( |
668 | 799 | '--constraints', 'mem=2G', | 799 | '--constraints', 'mem=2G spaces=^endpoint_bindings_data,' |
669 | 800 | '^endpoint_bindings_public', | ||
670 | 800 | 'foo/asdf', 'maas', | 801 | 'foo/asdf', 'maas', |
671 | 801 | '--config', config_file.name, '--default-model', 'maas', | 802 | '--config', config_file.name, '--default-model', 'maas', |
672 | 802 | '--agent-version', '2.0'), | 803 | '--agent-version', '2.0'), |
673 | 803 | 804 | ||
674 | === modified file 'tests/test_substrate.py' | |||
675 | --- tests/test_substrate.py 2016-10-14 19:07:39 +0000 | |||
676 | +++ tests/test_substrate.py 2016-10-21 22:18:30 +0000 | |||
677 | @@ -877,6 +877,14 @@ | |||
678 | 877 | ('maas', 'mas', 'machines', 'list-allocated')) | 877 | ('maas', 'mas', 'machines', 'list-allocated')) |
679 | 878 | self.assertEqual({}, ips) | 878 | self.assertEqual({}, ips) |
680 | 879 | 879 | ||
681 | 880 | def test_machines(self): | ||
682 | 881 | account = self.get_account() | ||
683 | 882 | with patch('subprocess.check_output', autospec=True, | ||
684 | 883 | return_value='[]') as co_mock: | ||
685 | 884 | machines = account.machines() | ||
686 | 885 | co_mock.assert_called_once_with(('maas', 'mas', 'machines', 'read')) | ||
687 | 886 | self.assertEqual([], machines) | ||
688 | 887 | |||
689 | 880 | def test_fabrics(self): | 888 | def test_fabrics(self): |
690 | 881 | account = self.get_account() | 889 | account = self.get_account() |
691 | 882 | with patch('subprocess.check_output', autospec=True, | 890 | with patch('subprocess.check_output', autospec=True, |
692 | @@ -967,6 +975,16 @@ | |||
693 | 967 | 'maas', 'mas', 'interfaces', 'read', 'node-xyz')) | 975 | 'maas', 'mas', 'interfaces', 'read', 'node-xyz')) |
694 | 968 | self.assertEqual([], interfaces) | 976 | self.assertEqual([], interfaces) |
695 | 969 | 977 | ||
696 | 978 | def test_interface_update(self): | ||
697 | 979 | account = self.get_account() | ||
698 | 980 | with patch('subprocess.check_output', autospec=True, | ||
699 | 981 | return_value='{"id": 10}') as co_mock: | ||
700 | 982 | interface = account.interface_update('node-xyz', 10, vlan=5000) | ||
701 | 983 | co_mock.assert_called_once_with(( | ||
702 | 984 | 'maas', 'mas', 'interface', 'update', 'node-xyz', '10', | ||
703 | 985 | 'vlan=5000')) | ||
704 | 986 | self.assertEqual({'id': 10}, interface) | ||
705 | 987 | |||
706 | 970 | def test_interface_create_vlan(self): | 988 | def test_interface_create_vlan(self): |
707 | 971 | account = self.get_account() | 989 | account = self.get_account() |
708 | 972 | with patch('subprocess.check_output', autospec=True, | 990 | with patch('subprocess.check_output', autospec=True, |
Thank you. I have some advice, questions, and comments inline. Nothing blocks this branch from merging.