Merge lp:~jtv/maas/netifaces into lp:~maas-committers/maas/trunk
- netifaces
- Merge into trunk
Proposed by
Jeroen T. Vermeulen
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | Jeroen T. Vermeulen | ||||
Approved revision: | no longer in the source branch. | ||||
Merged at revision: | 1125 | ||||
Proposed branch: | lp:~jtv/maas/netifaces | ||||
Merge into: | lp:~maas-committers/maas/trunk | ||||
Diff against target: |
450 lines (+88/-304) 3 files modified
required-packages/base (+1/-0) src/provisioningserver/network.py (+25/-54) src/provisioningserver/tests/test_network.py (+62/-250) |
||||
To merge this branch: | bzr merge lp:~jtv/maas/netifaces | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Julian Edwards (community) | Approve | ||
Review via email: mp+127420@code.launchpad.net |
Commit message
Replace ifconfig-parser with netifaces. We need this to avoid fork()ing off ifconfig, which confuses Upstart's fork tracking.
Description of the change
Briefly discussed with Julian. This one's needed to get tests passing on the branch that will make celery run as the maas user: we were forking once to get our networks information *before* the crucial fork of the celery "daemon" which we wanted upstart to track.
Jeroen
To post a comment you must log in.
Revision history for this message
Gavin Panella (allenap) wrote : | # |
I like this too. An alternative approach, should we have problems with other forks, would be to fork, then exec celery in the *parent* and continue doing whatever it is we need to do in the child process. Then upstart wouldn't need to follow forks, and there would be no chance of it getting confused.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'required-packages/base' |
2 | --- required-packages/base 2012-09-22 15:38:25 +0000 |
3 | +++ required-packages/base 2012-10-02 06:34:21 +0000 |
4 | @@ -23,6 +23,7 @@ |
5 | python-httplib2 |
6 | python-lockfile |
7 | python-netaddr |
8 | +python-netifaces |
9 | python-oauth |
10 | python-oops |
11 | python-oops-amqp |
12 | |
13 | === modified file 'src/provisioningserver/network.py' |
14 | --- src/provisioningserver/network.py 2012-09-26 17:21:53 +0000 |
15 | +++ src/provisioningserver/network.py 2012-10-02 06:34:21 +0000 |
16 | @@ -18,17 +18,21 @@ |
17 | 'discover_networks', |
18 | ] |
19 | |
20 | -import os |
21 | -from subprocess import check_output |
22 | + |
23 | +from netifaces import ( |
24 | + AF_INET, |
25 | + ifaddresses, |
26 | + interfaces, |
27 | + ) |
28 | |
29 | |
30 | class InterfaceInfo: |
31 | """The details of a network interface we are interested in.""" |
32 | |
33 | - def __init__(self, interface): |
34 | + def __init__(self, interface, ip=None, subnet_mask=None): |
35 | self.interface = interface |
36 | - self.ip = None |
37 | - self.subnet_mask = None |
38 | + self.ip = ip |
39 | + self.subnet_mask = subnet_mask |
40 | |
41 | def may_be_subnet(self): |
42 | """Could this be a subnet that MAAS is interested in?""" |
43 | @@ -51,57 +55,24 @@ |
44 | } |
45 | |
46 | |
47 | -def run_ifconfig(): |
48 | - """Run `ifconfig` to list active interfaces. Return output.""" |
49 | - env = dict(os.environ, LC_ALL='C') |
50 | - output = check_output(['/sbin/ifconfig'], env=env) |
51 | - return output.decode('ascii') |
52 | - |
53 | - |
54 | -def extract_ip_and_subnet_mask(line): |
55 | - """Get IP address and subnet mask from an inet address line.""" |
56 | - # This line consists of key:value pairs separated by double spaces. |
57 | - # The "inet addr" key contains a space. There is typically a |
58 | - # trailing separator. |
59 | - settings = dict( |
60 | - tuple(pair.split(':', 1)) |
61 | - for pair in line.split(' ')) |
62 | - return settings.get('inet addr'), settings.get('Mask') |
63 | - |
64 | - |
65 | -def parse_stanza(stanza): |
66 | - """Return a :class:`InterfaceInfo` representing this ifconfig stanza. |
67 | - |
68 | - May return `None` if it's immediately clear that the interface is not |
69 | - relevant for MAAS. |
70 | - """ |
71 | - lines = [line.strip() for line in stanza.splitlines()] |
72 | - header = lines[0] |
73 | - if 'Link encap:Ethernet' not in header: |
74 | - return None |
75 | - info = InterfaceInfo(header.split()[0]) |
76 | - for line in lines[1:]: |
77 | - if line.split()[0] == 'inet': |
78 | - info.ip, info.subnet_mask = extract_ip_and_subnet_mask(line) |
79 | - return info |
80 | - |
81 | - |
82 | -def split_stanzas(output): |
83 | - """Split `ifconfig` output into stanzas, one per interface.""" |
84 | - stanzas = [stanza.strip() for stanza in output.strip().split('\n\n')] |
85 | - return [stanza for stanza in stanzas if len(stanza) > 0] |
86 | - |
87 | - |
88 | -def parse_ifconfig(output): |
89 | - """List `InterfaceInfo` for each interface found in `ifconfig` output.""" |
90 | - infos = [parse_stanza(stanza) for stanza in split_stanzas(output)] |
91 | - return [info for info in infos if info is not None] |
92 | +def get_interface_info(interface): |
93 | + """Return a list of `InterfaceInfo` for the named `interface`.""" |
94 | + addresses = ifaddresses(interface).get(AF_INET) |
95 | + if addresses is None: |
96 | + return [] |
97 | + else: |
98 | + return [ |
99 | + InterfaceInfo( |
100 | + interface, ip=address.get('addr'), |
101 | + subnet_mask=address.get('netmask')) |
102 | + for address in addresses] |
103 | |
104 | |
105 | def discover_networks(): |
106 | """Find the networks attached to this system.""" |
107 | - output = run_ifconfig() |
108 | + infos = sum( |
109 | + [get_interface_info(interface) for interface in interfaces()], []) |
110 | return [ |
111 | - interface.as_dict() |
112 | - for interface in parse_ifconfig(output) |
113 | - if interface.may_be_subnet()] |
114 | + info.as_dict() |
115 | + for info in infos |
116 | + if info.may_be_subnet()] |
117 | |
118 | === modified file 'src/provisioningserver/tests/test_network.py' |
119 | --- src/provisioningserver/tests/test_network.py 2012-09-26 17:21:53 +0000 |
120 | +++ src/provisioningserver/tests/test_network.py 2012-10-02 06:34:21 +0000 |
121 | @@ -12,267 +12,79 @@ |
122 | __metaclass__ = type |
123 | __all__ = [] |
124 | |
125 | -from random import ( |
126 | - choice, |
127 | - randint, |
128 | - ) |
129 | - |
130 | from maastesting.factory import factory |
131 | from maastesting.testcase import TestCase |
132 | +from netaddr import IPNetwork |
133 | +from netifaces import AF_INET |
134 | from provisioningserver import network |
135 | |
136 | |
137 | -def make_address_line(**kwargs): |
138 | - """Create an inet address line.""" |
139 | - # First word on this line is inet or inet6. |
140 | - kwargs.setdefault('inet', 'inet') |
141 | - kwargs.setdefault('broadcast', '10.255.255.255') |
142 | - kwargs.setdefault('subnet_mask', '255.0.0.0') |
143 | - items = [ |
144 | - "%(inet)s addr:%(ip)s" |
145 | - ] |
146 | - if len(kwargs['broadcast']) > 0: |
147 | - items.append("Bcast:%(broadcast)s") |
148 | - items.append("Mask:%(subnet_mask)s") |
149 | - return ' '.join(items) % kwargs |
150 | - |
151 | - |
152 | -def make_stats_line(direction, **kwargs): |
153 | - """Create one of the incoming/outcoming packet-count lines.""" |
154 | - assert direction in {'RX', 'TX'} |
155 | - if direction == 'RX': |
156 | - variable_field = 'frame' |
157 | - else: |
158 | - variable_field = 'carrier' |
159 | - kwargs.setdefault('variable_field', variable_field) |
160 | - kwargs.setdefault('packets', randint(0, 100000)) |
161 | - kwargs.setdefault('errors', randint(0, 100)) |
162 | - kwargs.setdefault('dropped', randint(0, 100)) |
163 | - kwargs.setdefault('overruns', randint(0, 100)) |
164 | - kwargs.setdefault('variable', randint(0, 100)) |
165 | - |
166 | - return " ".join([ |
167 | - direction, |
168 | - "packets:%(packets)d", |
169 | - "errors:%(errors)d", |
170 | - "dropped:%(dropped)d", |
171 | - "overruns:%(overruns)d", |
172 | - "%(variable_field)s:%(variable)d" |
173 | - ]) % kwargs |
174 | - |
175 | - |
176 | -def make_payload_stats(direction, **kwargs): |
177 | - assert direction in {'RX', 'TX'} |
178 | - kwargs.setdefault('bytes', randint(0, 1000000)) |
179 | - kwargs.setdefault('bigger_unit', randint(10, 10240) / 10.0) |
180 | - kwargs.setdefault('unit', choice(['B', 'KB', 'GB'])) |
181 | - return " ".join([ |
182 | - direction, |
183 | - "bytes:%(bytes)s", |
184 | - "(%(bigger_unit)d %(unit)s)", |
185 | - ]) % kwargs |
186 | - |
187 | - |
188 | -def make_stanza(**kwargs): |
189 | - """Create an ifconfig output stanza. |
190 | - |
191 | - Variable values can be specified, but will be given random values by |
192 | - default. Values that interfaces may not have, such as broadcast |
193 | - address or allocated interrupt, may be set to the empty string to |
194 | - indicate that they should be left out of the output. |
195 | - """ |
196 | - kwargs.setdefault('interface', factory.make_name('eth')) |
197 | - kwargs.setdefault('encapsulation', 'Ethernet') |
198 | - kwargs.setdefault('mac', factory.getRandomMACAddress()) |
199 | - kwargs.setdefault('ip', factory.getRandomIPAddress()) |
200 | - kwargs.setdefault('broadcast', factory.getRandomIPAddress()) |
201 | - kwargs.setdefault('mtu', randint(100, 10000)) |
202 | - kwargs.setdefault('rxline', make_stats_line('RX', **kwargs)) |
203 | - kwargs.setdefault('txline', make_stats_line('TX', **kwargs)) |
204 | - kwargs.setdefault('collisions', randint(0, 100)) |
205 | - kwargs.setdefault('txqueuelen', randint(0, 100)) |
206 | - kwargs.setdefault('rxbytes', make_payload_stats('RX', **kwargs)) |
207 | - kwargs.setdefault('txbytes', make_payload_stats('TX', **kwargs)) |
208 | - kwargs.setdefault('interrupt', randint(1, 30)) |
209 | - |
210 | - # The real-life output seems to have two trailing spaces here. |
211 | - header = "%(interface)s Link encap:%(encapsulation)s HWaddr %(mac)s " |
212 | - body_lines = [ |
213 | - "UP BROADCAST MULTICAST MTU:%(mtu)d Metric:1", |
214 | - ] |
215 | - if len(kwargs['ip']) > 0: |
216 | - body_lines.append(make_address_line(inet='inet', **kwargs)) |
217 | - body_lines += [ |
218 | - "%(rxline)s", |
219 | - "%(txline)s", |
220 | - # This line has a trailing space in the real-life output. |
221 | - "collisions:%(collisions)d txqueuelen:%(txqueuelen)d ", |
222 | - "%(rxbytes)s %(txbytes)s", |
223 | - ] |
224 | - if kwargs['interrupt'] != '': |
225 | - body_lines.append("Interrupt:%(interrupt)d") |
226 | - |
227 | - text = '\n'.join( |
228 | - [header] + |
229 | - [(10 * " ") + line for line in body_lines]) |
230 | - return (text + "\n") % kwargs |
231 | - |
232 | - |
233 | -def join_stanzas(stanzas): |
234 | - """Format a sequence of interface stanzas like ifconfig does.""" |
235 | - return '\n'.join(stanzas) + '\n' |
236 | - |
237 | - |
238 | -# Tragically can't afford to indent and then dedent() this. This output |
239 | -# isn't entirely realistic: the real output has trailing spaces here and |
240 | -# there, which we don't tolerate in our source code. |
241 | -sample_output = """\ |
242 | -eth0 Link encap:Ethernet HWaddr 00:25:bc:e6:0b:c2 |
243 | - UP BROADCAST MULTICAST MTU:1500 Metric:1 |
244 | - RX packets:0 errors:0 dropped:0 overruns:0 frame:0 |
245 | - TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 |
246 | - collisions:0 txqueuelen:1000 |
247 | - RX bytes:0 (0.0 B) TX bytes:0 (0.0 B) |
248 | - |
249 | -eth1 Link encap:Ethernet HWaddr 00:14:73:ad:29:62 |
250 | - inet addr:192.168.12.103 Bcast:192.168.12.255 Mask:255.255.255.0 |
251 | - inet6 addr: fe81::210:9ff:fcd3:6120/64 Scope:Link |
252 | - UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 |
253 | - RX packets:5272 errors:1 dropped:0 overruns:0 frame:3274 |
254 | - TX packets:5940 errors:2 dropped:0 overruns:0 carrier:0 |
255 | - collisions:0 txqueuelen:1000 |
256 | - RX bytes:2254714 (2.2 MB) TX bytes:4045385 (4.0 MB) |
257 | - Interrupt:22 |
258 | - |
259 | -lo Link encap:Local Loopback |
260 | - inet addr:127.0.0.1 Mask:255.0.0.0 |
261 | - inet6 addr: ::1/128 Scope:Host |
262 | - UP LOOPBACK RUNNING MTU:16436 Metric:1 |
263 | - RX packets:297493 errors:0 dropped:0 overruns:0 frame:0 |
264 | - TX packets:297493 errors:0 dropped:0 overruns:0 carrier:0 |
265 | - collisions:0 txqueuelen:0 |
266 | - RX bytes:43708 (43.7 KB) TX bytes:43708 (43.7 KB) |
267 | - |
268 | -maasbr0 Link encap:Ethernet HWaddr 46:a1:20:8b:77:14 |
269 | - inet addr:192.168.64.1 Bcast:192.168.64.255 Mask:255.255.255.0 |
270 | - UP BROADCAST MULTICAST MTU:1500 Metric:1 |
271 | - RX packets:0 errors:0 dropped:0 overruns:0 frame:0 |
272 | - TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 |
273 | - collisions:0 txqueuelen:0 |
274 | - RX bytes:0 (0.0 B) TX bytes:0 (0.0 B) |
275 | - |
276 | -virbr0 Link encap:Ethernet HWaddr 68:14:23:c0:6d:bf |
277 | - inet addr:192.168.80.1 Bcast:192.168.80.255 Mask:255.255.255.0 |
278 | - UP BROADCAST MULTICAST MTU:1500 Metric:1 |
279 | - RX packets:0 errors:0 dropped:0 overruns:0 frame:0 |
280 | - TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 |
281 | - collisions:0 txqueuelen:0 |
282 | - RX bytes:0 (0.0 B) TX bytes:0 (0.0 B) |
283 | - |
284 | - """ |
285 | +def make_inet_address(subnet=None): |
286 | + """Fake an AF_INET address.""" |
287 | + if subnet is None: |
288 | + subnet = factory.getRandomNetwork() |
289 | + return { |
290 | + 'broadcast': subnet.broadcast, |
291 | + 'netmask': subnet.netmask, |
292 | + 'addr': factory.getRandomIPInNetwork(subnet), |
293 | + } |
294 | + |
295 | + |
296 | +def make_loopback(): |
297 | + """Fake a loopback AF_INET address.""" |
298 | + return make_inet_address(IPNetwork('127.0.0.0/8')) |
299 | + |
300 | + |
301 | +def make_interface(inet_address=None): |
302 | + """Minimally fake up an interface definition as returned by netifaces.""" |
303 | + if inet_address is None: |
304 | + inet_address = make_inet_address() |
305 | + return {AF_INET: [inet_address]} |
306 | |
307 | |
308 | class TestNetworks(TestCase): |
309 | |
310 | - def test_run_ifconfig_returns_ifconfig_output(self): |
311 | - text = join_stanzas([make_stanza()]) |
312 | - self.patch(network, 'check_output').return_value = text.encode('ascii') |
313 | - self.assertEqual(text, network.run_ifconfig()) |
314 | - |
315 | - def test_parse_ifconfig_produces_interface_info(self): |
316 | - num_interfaces = randint(1, 3) |
317 | - text = join_stanzas([ |
318 | - make_stanza() |
319 | - for counter in range(num_interfaces)]) |
320 | - info = network.parse_ifconfig(text) |
321 | - self.assertEqual(num_interfaces, len(info)) |
322 | - self.assertIsInstance(info[0], network.InterfaceInfo) |
323 | - |
324 | - def test_parse_stanza_reads_interface_with_ip_and_interrupt(self): |
325 | - parms = { |
326 | - 'interface': factory.make_name('eth'), |
327 | - 'ip': factory.getRandomIPAddress(), |
328 | - 'subnet_mask': '255.255.255.128', |
329 | - } |
330 | - info = network.parse_stanza(make_stanza(**parms)) |
331 | - self.assertEqual(parms, info.as_dict()) |
332 | - |
333 | - def test_parse_stanza_reads_interface_without_interrupt(self): |
334 | - parms = { |
335 | - 'interface': factory.make_name('eth'), |
336 | - 'ip': factory.getRandomIPAddress(), |
337 | - 'subnet_mask': '255.255.255.128', |
338 | - 'interrupt': '', |
339 | - } |
340 | - info = network.parse_stanza(make_stanza(**parms)) |
341 | - expected = parms.copy() |
342 | - del expected['interrupt'] |
343 | - self.assertEqual(expected, info.as_dict()) |
344 | - |
345 | - def test_parse_stanza_reads_interface_without_ip(self): |
346 | - parms = { |
347 | - 'interface': factory.make_name('eth'), |
348 | - 'ip': '', |
349 | - } |
350 | - info = network.parse_stanza(make_stanza(**parms)) |
351 | - expected = parms.copy() |
352 | - expected['ip'] = None |
353 | - expected['subnet_mask'] = None |
354 | - self.assertEqual(expected, info.as_dict()) |
355 | - |
356 | - def test_parse_stanza_returns_nothing_for_loopback(self): |
357 | - parms = { |
358 | - 'interface': 'lo', |
359 | - 'ip': '127.1.2.3', |
360 | - 'subnet_mask': '255.0.0.0', |
361 | - 'encapsulation': 'Local Loopback', |
362 | - 'broadcast': '', |
363 | - 'interrupt': '', |
364 | - } |
365 | - self.assertIsNone(network.parse_stanza(make_stanza(**parms))) |
366 | - |
367 | - def test_split_stanzas_returns_empty_for_empty_input(self): |
368 | - self.assertEqual([], network.split_stanzas('')) |
369 | - |
370 | - def test_split_stanzas_returns_single_stanza(self): |
371 | - stanza = make_stanza() |
372 | - self.assertEqual([stanza.strip()], network.split_stanzas(stanza)) |
373 | - |
374 | - def test_split_stanzas_splits_multiple_stanzas(self): |
375 | - stanzas = [make_stanza() for counter in range(3)] |
376 | - full_output = join_stanzas(stanzas) |
377 | + def patch_netifaces(self, interfaces): |
378 | + """Patch up netifaces to pretend we have given `interfaces`. |
379 | + |
380 | + :param interfaces: A dict mapping each interface's name to its |
381 | + definition as `netifaces` would return it. |
382 | + """ |
383 | + self.patch(network, 'interfaces').return_value = interfaces.keys() |
384 | + self.patch( |
385 | + network, 'ifaddresses', lambda interface: interfaces[interface]) |
386 | + |
387 | + def test_discover_networks_ignores_interface_without_IP_address(self): |
388 | + self.patch_netifaces({factory.make_name('eth'): {}}) |
389 | + self.assertEqual([], network.discover_networks()) |
390 | + |
391 | + def test_discover_networks_ignores_loopback(self): |
392 | + self.patch_netifaces({'lo': make_interface(make_loopback())}) |
393 | + self.assertEqual([], network.discover_networks()) |
394 | + |
395 | + def test_discover_networks_represents_interface(self): |
396 | + eth = factory.make_name('eth') |
397 | + interface = make_interface() |
398 | + self.patch_netifaces({eth: interface}) |
399 | + self.assertEqual([{ |
400 | + 'interface': eth, |
401 | + 'ip': interface[AF_INET][0]['addr'], |
402 | + 'subnet_mask': interface[AF_INET][0]['netmask'], |
403 | + }], |
404 | + network.discover_networks()) |
405 | + |
406 | + def test_discover_networks_returns_suitable_interfaces(self): |
407 | + eth = factory.make_name('eth') |
408 | + self.patch_netifaces({ |
409 | + eth: make_interface(), |
410 | + 'lo': make_interface(make_loopback()), |
411 | + factory.make_name('dummy'): make_interface({}), |
412 | + }) |
413 | self.assertEqual( |
414 | - [stanza.strip() for stanza in stanzas], |
415 | - network.split_stanzas(full_output)) |
416 | + [eth], [ |
417 | + interface['interface'] |
418 | + for interface in network.discover_networks()]) |
419 | |
420 | def test_discover_networks_runs_in_real_life(self): |
421 | interfaces = network.discover_networks() |
422 | self.assertIsInstance(interfaces, list) |
423 | - |
424 | - def test_discover_networks_returns_suitable_interfaces(self): |
425 | - params = { |
426 | - 'interface': factory.make_name('eth'), |
427 | - 'ip': factory.getRandomIPAddress(), |
428 | - 'subnet_mask': '255.255.255.0', |
429 | - } |
430 | - regular_interface = make_stanza(**params) |
431 | - loopback = make_stanza( |
432 | - interface='lo', encapsulation='Local loopback', broadcast='', |
433 | - interrupt='') |
434 | - disabled_interface = make_stanza(ip='', broadcast='', subnet_mask='') |
435 | - |
436 | - text = join_stanzas([regular_interface, loopback, disabled_interface]) |
437 | - self.patch(network, 'run_ifconfig').return_value = text |
438 | - |
439 | - interfaces = network.discover_networks() |
440 | - |
441 | - self.assertEqual( |
442 | - [params], |
443 | - [interface for interface in interfaces]) |
444 | - |
445 | - def test_discover_networks_processes_real_ifconfig_output(self): |
446 | - self.patch(network, 'run_ifconfig').return_value = sample_output |
447 | - info = network.discover_networks() |
448 | - self.assertEqual( |
449 | - ['eth1', 'maasbr0', 'virbr0'], |
450 | - [interface['interface'] for interface in info]) |
Nice.