Merge ~chad.smith/cloud-init:feature/openstack-network-v2-multi-nic into cloud-init:master
- Git
- lp:~chad.smith/cloud-init
- feature/openstack-network-v2-multi-nic
- Merge into master
Proposed by
Chad Smith
Status: | Work in progress |
---|---|
Proposed branch: | ~chad.smith/cloud-init:feature/openstack-network-v2-multi-nic |
Merge into: | cloud-init:master |
Diff against target: |
526 lines (+356/-28) 5 files modified
cloudinit/cmd/devel/net_convert.py (+5/-5) cloudinit/net/network_state.py (+89/-17) cloudinit/net/sysconfig.py (+2/-2) cloudinit/sources/helpers/openstack.py (+257/-1) tests/unittests/test_datasource/test_configdrive.py (+3/-3) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
cloud-init Commiters | Pending | ||
Review via email: mp+372009@code.launchpad.net |
Commit message
Work in progress for initial review:
openstack: return network v2 from parsed network_data.json
TODO: sort promotion of global dns
Description of the change
To post a comment you must log in.
Revision history for this message
Chad Smith (chad.smith) : | # |
Unmerged commits
- 5fac7a8... by Chad Smith
-
wip: generate openstack v2 networking config
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/cloudinit/cmd/devel/net_convert.py b/cloudinit/cmd/devel/net_convert.py | |||
2 | index 1ad7e0b..7b4ae3f 100755 | |||
3 | --- a/cloudinit/cmd/devel/net_convert.py | |||
4 | +++ b/cloudinit/cmd/devel/net_convert.py | |||
5 | @@ -81,12 +81,8 @@ def handle_args(name, args): | |||
6 | 81 | pre_ns = yaml.load(net_data) | 81 | pre_ns = yaml.load(net_data) |
7 | 82 | if 'network' in pre_ns: | 82 | if 'network' in pre_ns: |
8 | 83 | pre_ns = pre_ns.get('network') | 83 | pre_ns = pre_ns.get('network') |
9 | 84 | if args.debug: | ||
10 | 85 | sys.stderr.write('\n'.join( | ||
11 | 86 | ["Input YAML", | ||
12 | 87 | yaml.dump(pre_ns, default_flow_style=False, indent=4), ""])) | ||
13 | 88 | elif args.kind == 'network_data.json': | 84 | elif args.kind == 'network_data.json': |
15 | 89 | pre_ns = openstack.convert_net_json( | 85 | pre_ns = openstack.convert_net_json_v2( |
16 | 90 | json.loads(net_data), known_macs=known_macs) | 86 | json.loads(net_data), known_macs=known_macs) |
17 | 91 | elif args.kind == 'azure-imds': | 87 | elif args.kind == 'azure-imds': |
18 | 92 | pre_ns = azure.parse_network_config(json.loads(net_data)) | 88 | pre_ns = azure.parse_network_config(json.loads(net_data)) |
19 | @@ -94,6 +90,10 @@ def handle_args(name, args): | |||
20 | 94 | config = ovf.Config(ovf.ConfigFile(args.network_data.name)) | 90 | config = ovf.Config(ovf.ConfigFile(args.network_data.name)) |
21 | 95 | pre_ns = ovf.get_network_config_from_conf(config, False) | 91 | pre_ns = ovf.get_network_config_from_conf(config, False) |
22 | 96 | 92 | ||
23 | 93 | if args.debug: | ||
24 | 94 | sys.stderr.write('\n'.join( | ||
25 | 95 | ["Input YAML", | ||
26 | 96 | yaml.dump(pre_ns, default_flow_style=False, indent=4), ""])) | ||
27 | 97 | ns = network_state.parse_net_config_data(pre_ns) | 97 | ns = network_state.parse_net_config_data(pre_ns) |
28 | 98 | if not ns: | 98 | if not ns: |
29 | 99 | raise RuntimeError("No valid network_state object created from" | 99 | raise RuntimeError("No valid network_state object created from" |
30 | diff --git a/cloudinit/net/network_state.py b/cloudinit/net/network_state.py | |||
31 | index c0c415d..a7454f7 100644 | |||
32 | --- a/cloudinit/net/network_state.py | |||
33 | +++ b/cloudinit/net/network_state.py | |||
34 | @@ -4,6 +4,7 @@ | |||
35 | 4 | # | 4 | # |
36 | 5 | # This file is part of cloud-init. See LICENSE file for license information. | 5 | # This file is part of cloud-init. See LICENSE file for license information. |
37 | 6 | 6 | ||
38 | 7 | from collections import defaultdict | ||
39 | 7 | import copy | 8 | import copy |
40 | 8 | import functools | 9 | import functools |
41 | 9 | import logging | 10 | import logging |
42 | @@ -159,6 +160,10 @@ class NetworkState(object): | |||
43 | 159 | return self._version | 160 | return self._version |
44 | 160 | 161 | ||
45 | 161 | @property | 162 | @property |
46 | 163 | def _global_dns_counts(self): | ||
47 | 164 | return self._network_state['global_dns_counts'] | ||
48 | 165 | |||
49 | 166 | @property | ||
50 | 162 | def dns_nameservers(self): | 167 | def dns_nameservers(self): |
51 | 163 | try: | 168 | try: |
52 | 164 | return self._network_state['dns']['nameservers'] | 169 | return self._network_state['dns']['nameservers'] |
53 | @@ -234,6 +239,10 @@ class NetworkStateInterpreter(object): | |||
54 | 234 | self._network_state = copy.deepcopy(self.initial_network_state) | 239 | self._network_state = copy.deepcopy(self.initial_network_state) |
55 | 235 | self._network_state['config'] = config | 240 | self._network_state['config'] = config |
56 | 236 | self._parsed = False | 241 | self._parsed = False |
57 | 242 | # Reference counters to promote to global | ||
58 | 243 | self._global_dns_refs = { | ||
59 | 244 | 'nameserver': defaultdict(list), 'search': defaultdict(list)} | ||
60 | 245 | self._network_state['global_dns_refs'] = self._global_dns_refs | ||
61 | 237 | 246 | ||
62 | 238 | @property | 247 | @property |
63 | 239 | def network_state(self): | 248 | def network_state(self): |
64 | @@ -318,7 +327,7 @@ class NetworkStateInterpreter(object): | |||
65 | 318 | " command '%s'" % command_type) | 327 | " command '%s'" % command_type) |
66 | 319 | try: | 328 | try: |
67 | 320 | handler(self, command) | 329 | handler(self, command) |
69 | 321 | self._v2_common(command) | 330 | self._maybe_promote_v2_common(command) |
70 | 322 | except InvalidCommand: | 331 | except InvalidCommand: |
71 | 323 | if not skip_broken: | 332 | if not skip_broken: |
72 | 324 | raise | 333 | raise |
73 | @@ -326,6 +335,41 @@ class NetworkStateInterpreter(object): | |||
74 | 326 | LOG.warning("Skipping invalid command: %s", command, | 335 | LOG.warning("Skipping invalid command: %s", command, |
75 | 327 | exc_info=True) | 336 | exc_info=True) |
76 | 328 | LOG.debug(self.dump_network_state()) | 337 | LOG.debug(self.dump_network_state()) |
77 | 338 | # Post-process v2 dns promotions if needed | ||
78 | 339 | # count interfaces with ip, compare unpromoted global dns | ||
79 | 340 | self._cleanup_v2_common_from_interfaces() | ||
80 | 341 | |||
81 | 342 | def _cleanup_v2_common_from_interfaces(self): | ||
82 | 343 | """Strip any promoted global dns/search from specific interfaces.""" | ||
83 | 344 | interfaces = self._network_state.get('interfaces') | ||
84 | 345 | global_dns = set(self._network_state['dns'].get('nameservers', [])) | ||
85 | 346 | global_search = set(self._network_state['dns'].get('search', [])) | ||
86 | 347 | dns_refs = self._global_dns_refs['nameserver'] | ||
87 | 348 | search_refs = self._global_dns_refs['search'] | ||
88 | 349 | promoted_dns = global_dns.intersection(dns_refs) | ||
89 | 350 | promoted_search = global_dns.intersection(search_refs) | ||
90 | 351 | for intf_name, intf_cfg in interfaces.items(): | ||
91 | 352 | for subnet in intf_cfg['subnets']: | ||
92 | 353 | promote_dns = bool(not promoted_dns and len(interfaces) == 1) | ||
93 | 354 | subnet_dns = subnet.get('dns_nameservers', []) | ||
94 | 355 | if promote_dns and (subnet_dns or subnet.get('dns_search')): | ||
95 | 356 | name_cmd = {'type': 'nameserver', | ||
96 | 357 | 'search': subnet.get('dns_search', []), | ||
97 | 358 | 'address': subnet_dns} | ||
98 | 359 | self.handle_nameserver(name_cmd) | ||
99 | 360 | subnet.pop('dns_search', None) | ||
100 | 361 | subnet.pop('dns_nameservers', None) | ||
101 | 362 | continue | ||
102 | 363 | for dns_ip in subnet_dns: | ||
103 | 364 | if dns_ip in promoted_dns: | ||
104 | 365 | subnet['dns_nameservers'].remove(dns_ip) | ||
105 | 366 | if not subnet['dns_nameservers']: | ||
106 | 367 | subnet.pop('dns_nameservers') | ||
107 | 368 | for search in subnet.get('dns_search', []): | ||
108 | 369 | if search in promoted_search: | ||
109 | 370 | subnet['dns_search'].remove(search) | ||
110 | 371 | if not subnet['dns_search']: | ||
111 | 372 | subnet.pop('dns_search') | ||
112 | 329 | 373 | ||
113 | 330 | @ensure_command_keys(['name']) | 374 | @ensure_command_keys(['name']) |
114 | 331 | def handle_loopback(self, command): | 375 | def handle_loopback(self, command): |
115 | @@ -372,7 +416,6 @@ class NetworkStateInterpreter(object): | |||
116 | 372 | 'subnets': subnets, | 416 | 'subnets': subnets, |
117 | 373 | }) | 417 | }) |
118 | 374 | self._network_state['interfaces'].update({command.get('name'): iface}) | 418 | self._network_state['interfaces'].update({command.get('name'): iface}) |
119 | 375 | self.dump_network_state() | ||
120 | 376 | 419 | ||
121 | 377 | @ensure_command_keys(['name', 'vlan_id', 'vlan_link']) | 420 | @ensure_command_keys(['name', 'vlan_id', 'vlan_link']) |
122 | 378 | def handle_vlan(self, command): | 421 | def handle_vlan(self, command): |
123 | @@ -520,13 +563,15 @@ class NetworkStateInterpreter(object): | |||
124 | 520 | if not type(addrs) == list: | 563 | if not type(addrs) == list: |
125 | 521 | addrs = [addrs] | 564 | addrs = [addrs] |
126 | 522 | for addr in addrs: | 565 | for addr in addrs: |
128 | 523 | dns['nameservers'].append(addr) | 566 | if addr not in dns['nameservers']: |
129 | 567 | dns['nameservers'].append(addr) | ||
130 | 524 | if 'search' in command: | 568 | if 'search' in command: |
131 | 525 | paths = command['search'] | 569 | paths = command['search'] |
132 | 526 | if not isinstance(paths, list): | 570 | if not isinstance(paths, list): |
133 | 527 | paths = [paths] | 571 | paths = [paths] |
134 | 528 | for path in paths: | 572 | for path in paths: |
136 | 529 | dns['search'].append(path) | 573 | if path not in dns['search']: |
137 | 574 | dns['search'].append(path) | ||
138 | 530 | 575 | ||
139 | 531 | @ensure_command_keys(['destination']) | 576 | @ensure_command_keys(['destination']) |
140 | 532 | def handle_route(self, command): | 577 | def handle_route(self, command): |
141 | @@ -689,18 +734,45 @@ class NetworkStateInterpreter(object): | |||
142 | 689 | LOG.warning('Wifi configuration is only available to distros with' | 734 | LOG.warning('Wifi configuration is only available to distros with' |
143 | 690 | 'netplan rendering support.') | 735 | 'netplan rendering support.') |
144 | 691 | 736 | ||
157 | 692 | def _v2_common(self, cfg): | 737 | def _maybe_promote_v2_common(self, cfg): |
158 | 693 | LOG.debug('v2_common: handling config:\n%s', cfg) | 738 | """Possibly promote v2 common/global services from specific devices. |
159 | 694 | if 'nameservers' in cfg: | 739 | |
160 | 695 | search = cfg.get('nameservers').get('search', []) | 740 | Since network v2 only supports per-interface DNS config settings, there |
161 | 696 | dns = cfg.get('nameservers').get('addresses', []) | 741 | is no 'global' dns service that can be expressed, unless we set |
162 | 697 | name_cmd = {'type': 'nameserver'} | 742 | the same dns values on every interface. If v2 config has the same |
163 | 698 | if len(search) > 0: | 743 | dns config on every configured interface, it will be assumed that |
164 | 699 | name_cmd.update({'search': search}) | 744 | the common dns setting needs to be written to the distribution's |
165 | 700 | if len(dns) > 0: | 745 | 'global (read /etc/resolv.conf)' dns config. |
166 | 701 | name_cmd.update({'addresses': dns}) | 746 | |
167 | 702 | LOG.debug('v2(nameserver) -> v1(nameserver):\n%s', name_cmd) | 747 | Track reference counts in _global_dns_refs so net/sysconfig renderer |
168 | 703 | self.handle_nameserver(name_cmd) | 748 | can determine whether to use /etc/resolv.conf of not for specific |
169 | 749 | device dns configuration. | ||
170 | 750 | """ | ||
171 | 751 | LOG.debug('maybe_promote_v2_common: handling config:\n%s', cfg) | ||
172 | 752 | for if_name, iface_cfg in cfg.items(): | ||
173 | 753 | if 'nameservers' in iface_cfg: | ||
174 | 754 | search = iface_cfg.get('nameservers').get('search', []) | ||
175 | 755 | if not search: | ||
176 | 756 | search = [] | ||
177 | 757 | elif not isinstance(search, list): | ||
178 | 758 | search = [search] | ||
179 | 759 | dns = iface_cfg.get('nameservers').get('addresses') | ||
180 | 760 | if not dns: | ||
181 | 761 | dns = [] | ||
182 | 762 | elif not isinstance(dns, list): | ||
183 | 763 | dns = [dns] | ||
184 | 764 | name_cmd = {'type': 'nameserver', 'search': [], 'address': []} | ||
185 | 765 | for sname in search: | ||
186 | 766 | if self._global_dns_refs['search'][sname]: | ||
187 | 767 | name_cmd['search'].append[sname] | ||
188 | 768 | self._global_dns_refs['search'][sname].append(if_name) | ||
189 | 769 | for dns_ip in dns: | ||
190 | 770 | if self._global_dns_refs['nameserver'][dns_ip]: | ||
191 | 771 | name_cmd['address'].append[dns_ip] | ||
192 | 772 | self._global_dns_refs['nameserver'][dns_ip].append(if_name) | ||
193 | 773 | if any([name_cmd['search'], name_cmd['address']]): | ||
194 | 774 | # promote DNS config seen by multiple interfaces | ||
195 | 775 | self.handle_nameserver(name_cmd) | ||
196 | 704 | 776 | ||
197 | 705 | def _handle_bond_bridge(self, command, cmd_type=None): | 777 | def _handle_bond_bridge(self, command, cmd_type=None): |
198 | 706 | """Common handler for bond and bridge types""" | 778 | """Common handler for bond and bridge types""" |
199 | @@ -827,7 +899,7 @@ def _normalize_net_keys(network, address_keys=()): | |||
200 | 827 | 899 | ||
201 | 828 | @returns: A dict containing normalized prefix and matching addr_key. | 900 | @returns: A dict containing normalized prefix and matching addr_key. |
202 | 829 | """ | 901 | """ |
204 | 830 | net = dict((k, v) for k, v in network.items() if v) | 902 | net = dict((k, v) for k, v in network.items() if v is not None) |
205 | 831 | addr_key = None | 903 | addr_key = None |
206 | 832 | for key in address_keys: | 904 | for key in address_keys: |
207 | 833 | if net.get(key): | 905 | if net.get(key): |
208 | diff --git a/cloudinit/net/sysconfig.py b/cloudinit/net/sysconfig.py | |||
209 | index be5dede..0fa0508 100644 | |||
210 | --- a/cloudinit/net/sysconfig.py | |||
211 | +++ b/cloudinit/net/sysconfig.py | |||
212 | @@ -444,9 +444,9 @@ class Renderer(renderer.Renderer): | |||
213 | 444 | 444 | ||
214 | 445 | if _is_default_route(route): | 445 | if _is_default_route(route): |
215 | 446 | if ( | 446 | if ( |
217 | 447 | (subnet.get('ipv4') and | 447 | (not is_ipv6 and |
218 | 448 | route_cfg.has_set_default_ipv4) or | 448 | route_cfg.has_set_default_ipv4) or |
220 | 449 | (subnet.get('ipv6') and | 449 | (is_ipv6 and |
221 | 450 | route_cfg.has_set_default_ipv6) | 450 | route_cfg.has_set_default_ipv6) |
222 | 451 | ): | 451 | ): |
223 | 452 | raise ValueError("Duplicate declaration of default " | 452 | raise ValueError("Duplicate declaration of default " |
224 | diff --git a/cloudinit/sources/helpers/openstack.py b/cloudinit/sources/helpers/openstack.py | |||
225 | index 8f06911..ceaf868 100644 | |||
226 | --- a/cloudinit/sources/helpers/openstack.py | |||
227 | +++ b/cloudinit/sources/helpers/openstack.py | |||
228 | @@ -17,6 +17,7 @@ import six | |||
229 | 17 | from cloudinit import ec2_utils | 17 | from cloudinit import ec2_utils |
230 | 18 | from cloudinit import log as logging | 18 | from cloudinit import log as logging |
231 | 19 | from cloudinit import net | 19 | from cloudinit import net |
232 | 20 | from cloudinit.net import network_state | ||
233 | 20 | from cloudinit import sources | 21 | from cloudinit import sources |
234 | 21 | from cloudinit import url_helper | 22 | from cloudinit import url_helper |
235 | 22 | from cloudinit import util | 23 | from cloudinit import util |
236 | @@ -82,6 +83,36 @@ KNOWN_PHYSICAL_TYPES = ( | |||
237 | 82 | 'vif', | 83 | 'vif', |
238 | 83 | ) | 84 | ) |
239 | 84 | 85 | ||
240 | 86 | LINK_TYPE_TO_NETWORK_V2_KEYS = { | ||
241 | 87 | 'bond': 'bonds', | ||
242 | 88 | 'vlan': 'vlans', | ||
243 | 89 | } | ||
244 | 90 | |||
245 | 91 | NET_V2_PHYSICAL_TYPES = ['ethernets', 'vlans', 'bonds', 'bridges', 'wifi'] | ||
246 | 92 | |||
247 | 93 | NETWORK_DATA_TO_V2 = { | ||
248 | 94 | 'bond': { | ||
249 | 95 | 'mtu': 'mtu', | ||
250 | 96 | 'bond_links': 'interfaces', | ||
251 | 97 | 'ethernet_mac_address': 'macaddress', | ||
252 | 98 | }, | ||
253 | 99 | 'ethernets': { | ||
254 | 100 | 'mtu': 'mtu', | ||
255 | 101 | 'ethernet_mac_address': 'match.macaddress', | ||
256 | 102 | 'id': 'set-name', | ||
257 | 103 | }, | ||
258 | 104 | 'vlan': { | ||
259 | 105 | 'key_rename': '{link}.{id}', # override top level key for object | ||
260 | 106 | 'mtu': 'mtu', | ||
261 | 107 | 'vlan_mac_address': 'macaddress', | ||
262 | 108 | 'vlan_id': 'id', | ||
263 | 109 | 'vlan_link': 'link', | ||
264 | 110 | }, | ||
265 | 111 | } | ||
266 | 112 | |||
267 | 113 | # network_data.json keys for which converted net v2 values should be lowercase | ||
268 | 114 | LOWERCASE_KEY_VALUE_V2 = ('macaddress',) | ||
269 | 115 | |||
270 | 85 | 116 | ||
271 | 86 | class NonReadable(IOError): | 117 | class NonReadable(IOError): |
272 | 87 | pass | 118 | pass |
273 | @@ -496,8 +527,233 @@ class MetadataReader(BaseReader): | |||
274 | 496 | retries=self.retries) | 527 | retries=self.retries) |
275 | 497 | 528 | ||
276 | 498 | 529 | ||
278 | 499 | # Convert OpenStack ConfigDrive NetworkData json to network_config yaml | 530 | def _find_v2_device_type(name, net_v2): |
279 | 531 | """Return the netv2 physical device type containing matching name.""" | ||
280 | 532 | for device_type in NET_V2_PHYSICAL_TYPES: | ||
281 | 533 | if name in net_v2.get(device_type, {}): | ||
282 | 534 | return device_type | ||
283 | 535 | return None | ||
284 | 536 | |||
285 | 537 | |||
286 | 538 | def _convert_network_json_network_to_net_v2(src_json): | ||
287 | 539 | """Parse a single network item from the networks list in network_data.json | ||
288 | 540 | |||
289 | 541 | @param src_json: One network item from network_data.json 'networks' key. | ||
290 | 542 | |||
291 | 543 | @return: Tuple of <interface_name>, network v2 configuration dict for the | ||
292 | 544 | src_json. For example: eth0, {'addresses': [...], 'dhcp4': True} | ||
293 | 545 | """ | ||
294 | 546 | net_v2 = {'addresses': []} | ||
295 | 547 | ignored_keys = set() | ||
296 | 548 | |||
297 | 549 | # In Liberty spec https://specs.openstack.org/openstack/nova-specs/ | ||
298 | 550 | # specs/liberty/implemented/metadata-service-network-info.html | ||
299 | 551 | if src_json['type'] == 'ipv4_dhcp': | ||
300 | 552 | net_v2['dhcp4'] = True | ||
301 | 553 | elif src_json['type'] == 'ipv6_dhcp': | ||
302 | 554 | net_v2['dhcp6'] = True | ||
303 | 555 | |||
304 | 556 | for service in src_json.get('services', []): | ||
305 | 557 | if service['type'] != 'dns': | ||
306 | 558 | ignored_keys.update(['services.type(%s)' % service['type']]) | ||
307 | 559 | continue | ||
308 | 560 | if 'nameservers' not in net_v2: | ||
309 | 561 | net_v2['nameservers'] = {'addresses': [], 'search': []} | ||
310 | 562 | net_v2['nameservers']['addresses'].append(service['address']) | ||
311 | 563 | # In Rocky spec https://specs.openstack.org/openstack/nova-specs/specs/ | ||
312 | 564 | # rocky/approved/multiple-fixed-ips-network-information.html | ||
313 | 565 | dns_nameservers = src_json.get('dns_nameservers', []) | ||
314 | 566 | if dns_nameservers: | ||
315 | 567 | if 'nameservers' not in net_v2: | ||
316 | 568 | net_v2['nameservers'] = {'addresses': [], 'search': []} | ||
317 | 569 | net_v2['nameservers']['addresses'] = copy.copy(dns_nameservers) | ||
318 | 570 | |||
319 | 571 | # Parse routes for network, prefix and gateway | ||
320 | 572 | route_keys = set(['netmask', 'network', 'gateway']) | ||
321 | 573 | for route in src_json.get('routes', []): | ||
322 | 574 | ignored_route_keys = (set(route.keys()).difference(route_keys)) | ||
323 | 575 | ignored_keys.update(['route.%s' % key for key in ignored_route_keys]) | ||
324 | 576 | route_cfg = { | ||
325 | 577 | 'to': '{network}/{prefix}'.format( | ||
326 | 578 | network=route['network'], | ||
327 | 579 | prefix=net.network_state.mask_to_net_prefix(route['netmask'])), | ||
328 | 580 | 'via': route['gateway']} | ||
329 | 581 | if route.get('metric'): | ||
330 | 582 | route_cfg['metric'] = route.get('metric') | ||
331 | 583 | if 'routes' not in net_v2: | ||
332 | 584 | net_v2['routes'] = [] | ||
333 | 585 | net_v2['routes'].append(route_cfg) | ||
334 | 586 | |||
335 | 587 | # Parse ip addresses on Rocky and Liberty | ||
336 | 588 | for ip_cfg in src_json.get('ip_addresses', []): | ||
337 | 589 | if ip_cfg.get('netmask'): | ||
338 | 590 | prefix = net.network_state.mask_to_net_prefix(ip_cfg['netmask']) | ||
339 | 591 | cidr_fmt = '{ip}/{prefix}' | ||
340 | 592 | else: | ||
341 | 593 | cidr_fmt = '{ip}' | ||
342 | 594 | prefix = None | ||
343 | 595 | net_v2['addresses'].append( | ||
344 | 596 | cidr_fmt.format(ip=ip_cfg['address'], prefix=prefix)) | ||
345 | 597 | liberty_ip = src_json.get('ip_address') | ||
346 | 598 | if liberty_ip: | ||
347 | 599 | if src_json.get('netmask'): | ||
348 | 600 | prefix = net.network_state.mask_to_net_prefix(src_json['netmask']) | ||
349 | 601 | cidr_fmt = '{ip}/{prefix}' | ||
350 | 602 | else: | ||
351 | 603 | cidr_fmt = '{ip}' | ||
352 | 604 | prefix = None | ||
353 | 605 | liberty_cidr = cidr_fmt.format(ip=liberty_ip, prefix=prefix) | ||
354 | 606 | if liberty_cidr not in net_v2['addresses']: | ||
355 | 607 | net_v2['addresses'].append(liberty_cidr) | ||
356 | 608 | if not net_v2['addresses']: | ||
357 | 609 | net_v2.pop('addresses') | ||
358 | 610 | if ignored_keys: | ||
359 | 611 | LOG.debug( | ||
360 | 612 | 'Ignoring the network_data.json %s config keys %s', | ||
361 | 613 | src_json['id'], ', '.join(ignored_keys)) | ||
362 | 614 | return src_json['link'], net_v2 | ||
363 | 615 | |||
364 | 616 | |||
365 | 617 | def _convert_network_json_to_net_v2(src_json, var_map): | ||
366 | 618 | """Return network v2 for an element of OpenStack NetworkData json. | ||
367 | 619 | |||
368 | 620 | @param src_json: Dict of network_data.json for a single src_json object | ||
369 | 621 | @param var_map: Dict with a variable name map from network_data.json to | ||
370 | 622 | network v2 | ||
371 | 623 | |||
372 | 624 | @return Tuple of the interface name and the converted network v2 for the | ||
373 | 625 | src_json object. For example: eth0, {'match': {'macaddress': 'AA:BB'}} | ||
374 | 626 | """ | ||
375 | 627 | net_v2 = {} | ||
376 | 628 | # Map openstack bond keys to network v2 | ||
377 | 629 | # Copy key values | ||
378 | 630 | current_keys = set(src_json) | ||
379 | 631 | for key in current_keys.intersection(set(var_map)): | ||
380 | 632 | keyparts = var_map[key].split('.') | ||
381 | 633 | tmp_cfg = net_v2 # allow traversing net_v2 dict | ||
382 | 634 | while keyparts: | ||
383 | 635 | keypart = keyparts.pop(0) | ||
384 | 636 | if keyparts: | ||
385 | 637 | if keypart not in tmp_cfg: | ||
386 | 638 | tmp_cfg[keypart] = {} | ||
387 | 639 | tmp_cfg = tmp_cfg[keypart] | ||
388 | 640 | elif isinstance(src_json[key], list): | ||
389 | 641 | tmp_cfg[keypart] = copy.copy(src_json[key]) | ||
390 | 642 | elif src_json[key]: | ||
391 | 643 | if keypart in LOWERCASE_KEY_VALUE_V2: | ||
392 | 644 | tmp_cfg[keypart] = src_json[key].lower() | ||
393 | 645 | else: | ||
394 | 646 | tmp_cfg[keypart] = src_json[key] | ||
395 | 647 | if 'key_rename' in var_map: | ||
396 | 648 | net_v2['key_rename'] = var_map['key_rename'].format(**net_v2) | ||
397 | 649 | return src_json['id'], net_v2 | ||
398 | 650 | |||
399 | 651 | |||
400 | 500 | def convert_net_json(network_json=None, known_macs=None): | 652 | def convert_net_json(network_json=None, known_macs=None): |
401 | 653 | """Parse OpenStack ConfigDrive NetworkData json, returning network cfg v2. | ||
402 | 654 | |||
403 | 655 | OpenStack network_data.json provides a 3 element dictionary | ||
404 | 656 | - "links" (links are network devices, physical or virtual) | ||
405 | 657 | - "networks" (networks are ip network configurations for one or more | ||
406 | 658 | links) | ||
407 | 659 | - services (non-ip services, like dns) | ||
408 | 660 | |||
409 | 661 | networks and links are combined via network items referencing specific | ||
410 | 662 | links via a 'link_id' which maps to a links 'id' field. | ||
411 | 663 | """ | ||
412 | 664 | if network_json is None: | ||
413 | 665 | return None | ||
414 | 666 | net_config = {'version': 2} | ||
415 | 667 | for link in network_json.get('links', []): | ||
416 | 668 | link_type = link['type'] | ||
417 | 669 | v2_key = LINK_TYPE_TO_NETWORK_V2_KEYS.get(link_type) | ||
418 | 670 | if not v2_key: | ||
419 | 671 | v2_key = 'ethernets' | ||
420 | 672 | if link_type not in KNOWN_PHYSICAL_TYPES: | ||
421 | 673 | LOG.warning('Unknown network_data link type (%s); treating as' | ||
422 | 674 | ' physical ethernet', link_type) | ||
423 | 675 | if v2_key not in net_config: | ||
424 | 676 | net_config[v2_key] = {} | ||
425 | 677 | var_map = copy.deepcopy(NETWORK_DATA_TO_V2.get(v2_key)) | ||
426 | 678 | if not var_map: | ||
427 | 679 | var_map = copy.deepcopy(NETWORK_DATA_TO_V2.get(link_type)) | ||
428 | 680 | |||
429 | 681 | # Add v2 config parameters map for this link_type if present | ||
430 | 682 | if link_type in network_state.NET_CONFIG_TO_V2: | ||
431 | 683 | var_map.update(dict( | ||
432 | 684 | (k.replace('-', '_'), 'parameters.{v}'.format(v=v)) | ||
433 | 685 | for k, v in network_state.NET_CONFIG_TO_V2[link_type].items())) | ||
434 | 686 | intf_id, intf_cfg = _convert_network_json_to_net_v2(link, var_map) | ||
435 | 687 | if v2_key in ('ethernets', 'bonds') and 'name' not in intf_cfg: | ||
436 | 688 | if known_macs is None: | ||
437 | 689 | known_macs = net.get_interfaces_by_mac() | ||
438 | 690 | lower_known_macs = dict( | ||
439 | 691 | (k.lower(), v) for k, v in known_macs.items()) | ||
440 | 692 | mac = intf_cfg.get( # top-level macaddress and match.macaddress | ||
441 | 693 | 'macaddress', intf_cfg.get('match', {}).get('macaddress')) | ||
442 | 694 | if not mac: | ||
443 | 695 | raise ValueError("No mac_address or name entry for %s" % d) | ||
444 | 696 | mac = mac.lower() | ||
445 | 697 | if mac not in lower_known_macs: | ||
446 | 698 | raise ValueError( | ||
447 | 699 | 'Unable to find a system nic for %s' % mac) | ||
448 | 700 | intf_cfg['key_rename'] = lower_known_macs[mac] | ||
449 | 701 | net_config[v2_key].update({intf_id: intf_cfg}) | ||
450 | 702 | for network in network_json.get('networks', []): | ||
451 | 703 | v2_key = _find_v2_device_type(network['link'], net_config) | ||
452 | 704 | intf_id, network_cfg = _convert_network_json_network_to_net_v2(network) | ||
453 | 705 | for key, val in network_cfg.items(): | ||
454 | 706 | if isinstance(val, list): | ||
455 | 707 | if key not in net_config[v2_key][intf_id]: | ||
456 | 708 | net_config[v2_key][intf_id][key] = [] | ||
457 | 709 | net_config[v2_key][intf_id][key].extend(val) | ||
458 | 710 | else: | ||
459 | 711 | net_config[v2_key][intf_id][key] = val | ||
460 | 712 | |||
461 | 713 | # Inject global nameserver values under each all interface which | ||
462 | 714 | # has addresses and do not already have a DNS configuration | ||
463 | 715 | ignored_keys = set() | ||
464 | 716 | global_dns = [] | ||
465 | 717 | for service in network_json.get('services', []): | ||
466 | 718 | if service['type'] != 'dns': | ||
467 | 719 | ignored_keys.update('services.type(%s)' % service['type']) | ||
468 | 720 | continue | ||
469 | 721 | global_dns.append(service['address']) | ||
470 | 722 | |||
471 | 723 | # Handle renames and global_dns | ||
472 | 724 | for dev_type in NET_V2_PHYSICAL_TYPES: | ||
473 | 725 | if dev_type not in net_config: | ||
474 | 726 | continue | ||
475 | 727 | renames = {} | ||
476 | 728 | for dev in net_config[dev_type]: | ||
477 | 729 | renames[dev] = net_config[dev_type][dev].pop('key_rename', None) | ||
478 | 730 | if not global_dns: | ||
479 | 731 | continue | ||
480 | 732 | dev_keys = set(net_config[dev_type][dev].keys()) | ||
481 | 733 | if set(['nameservers', 'dhcp4', 'dhcp6']).intersection(dev_keys): | ||
482 | 734 | # Do not add nameservers if we already have dns config | ||
483 | 735 | continue | ||
484 | 736 | if 'addresses' not in net_config[dev_type][dev]: | ||
485 | 737 | # No configured address, needs no nameserver | ||
486 | 738 | continue | ||
487 | 739 | net_config[dev_type][dev]['nameservers'] = { | ||
488 | 740 | 'addresses': copy.copy(global_dns), 'search': []} | ||
489 | 741 | for dev, rename in renames.items(): | ||
490 | 742 | if rename: | ||
491 | 743 | net_config[dev_type][rename] = net_config[dev_type].pop(dev) | ||
492 | 744 | if 'set-name' in net_config[dev_type][rename]: | ||
493 | 745 | net_config[dev_type][rename]['set-name'] = rename | ||
494 | 746 | |||
495 | 747 | if ignored_keys: | ||
496 | 748 | LOG.debug( | ||
497 | 749 | 'Ignoring the network_data.json config keys %s', | ||
498 | 750 | ', '.join(ignored_keys)) | ||
499 | 751 | |||
500 | 752 | return net_config | ||
501 | 753 | |||
502 | 754 | |||
503 | 755 | # Convert OpenStack ConfigDrive NetworkData json to network_config yaml | ||
504 | 756 | def convert_net_json1(network_json=None, known_macs=None): | ||
505 | 501 | """Return a dictionary of network_config by parsing provided | 757 | """Return a dictionary of network_config by parsing provided |
506 | 502 | OpenStack ConfigDrive NetworkData json format | 758 | OpenStack ConfigDrive NetworkData json format |
507 | 503 | 759 | ||
508 | diff --git a/tests/unittests/test_datasource/test_configdrive.py b/tests/unittests/test_datasource/test_configdrive.py | |||
509 | index 520c50f..f4181f5 100644 | |||
510 | --- a/tests/unittests/test_datasource/test_configdrive.py | |||
511 | +++ b/tests/unittests/test_datasource/test_configdrive.py | |||
512 | @@ -719,11 +719,11 @@ class TestConvertNetworkData(CiTestCase): | |||
513 | 719 | for link in my_netdata['links']: | 719 | for link in my_netdata['links']: |
514 | 720 | link['ethernet_mac_address'] = link['ethernet_mac_address'].upper() | 720 | link['ethernet_mac_address'] = link['ethernet_mac_address'].upper() |
515 | 721 | 721 | ||
516 | 722 | import pdb; pdb.set_trace() | ||
517 | 722 | ncfg = openstack.convert_net_json(my_netdata, known_macs=KNOWN_MACS) | 723 | ncfg = openstack.convert_net_json(my_netdata, known_macs=KNOWN_MACS) |
518 | 723 | config_name2mac = {} | 724 | config_name2mac = {} |
522 | 724 | for n in ncfg['config']: | 725 | for name, if_cfg in ncfg['ethernets'].items(): |
523 | 725 | if n['type'] == 'physical': | 726 | config_name2mac[name] = if_cfg['match']['macaddress'] |
521 | 726 | config_name2mac[n['name']] = n['mac_address'] | ||
524 | 727 | 727 | ||
525 | 728 | expected = {'nic0': 'fa:16:3e:05:30:fe', 'enp0s1': 'fa:16:3e:69:b0:58', | 728 | expected = {'nic0': 'fa:16:3e:05:30:fe', 'enp0s1': 'fa:16:3e:69:b0:58', |
526 | 729 | 'enp0s2': 'fa:16:3e:d4:57:ad'} | 729 | 'enp0s2': 'fa:16:3e:d4:57:ad'} |