Merge lp:~allenap/maas/dhcp-leases-parsing--1.5 into lp:maas/1.5
- dhcp-leases-parsing--1.5
- Merge into 1.5
Proposed by
Gavin Panella
Status: | Merged |
---|---|
Approved by: | Gavin Panella |
Approved revision: | no longer in the source branch. |
Merged at revision: | 2273 |
Proposed branch: | lp:~allenap/maas/dhcp-leases-parsing--1.5 |
Merge into: | lp:maas/1.5 |
Diff against target: |
773 lines (+407/-212) 2 files modified
src/provisioningserver/dhcp/leases_parser_fast.py (+87/-0) src/provisioningserver/dhcp/tests/test_leases_parser.py (+320/-212) |
To merge this branch: | bzr merge lp:~allenap/maas/dhcp-leases-parsing--1.5 |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Gavin Panella (community) | Approve | ||
Review via email: mp+220086@code.launchpad.net |
Commit message
Backport trunk r2335: Faster DHCP leases parser.
Description of the change
To post a comment you must log in.
Revision history for this message
Gavin Panella (allenap) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === added file 'src/provisioningserver/dhcp/leases_parser_fast.py' | |||
2 | --- src/provisioningserver/dhcp/leases_parser_fast.py 1970-01-01 00:00:00 +0000 | |||
3 | +++ src/provisioningserver/dhcp/leases_parser_fast.py 2014-05-19 15:16:18 +0000 | |||
4 | @@ -0,0 +1,87 @@ | |||
5 | 1 | # Copyright 2013 Canonical Ltd. This software is licensed under the | ||
6 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | ||
7 | 3 | |||
8 | 4 | """A speedier version of `leases_parser`. | ||
9 | 5 | |||
10 | 6 | This extracts the relevant stanzas from a leases file, keeping only the | ||
11 | 7 | most recent "host" and "lease" entries, then uses the existing and | ||
12 | 8 | properly defined but slow parser to parse them. This massively speeds up | ||
13 | 9 | parsing a leases file that contains a modest number of unique host and | ||
14 | 10 | lease entries, but has become very large because of churn. | ||
15 | 11 | """ | ||
16 | 12 | |||
17 | 13 | from __future__ import ( | ||
18 | 14 | absolute_import, | ||
19 | 15 | print_function, | ||
20 | 16 | unicode_literals, | ||
21 | 17 | ) | ||
22 | 18 | |||
23 | 19 | str = None | ||
24 | 20 | |||
25 | 21 | __metaclass__ = type | ||
26 | 22 | __all__ = [ | ||
27 | 23 | 'parse_leases', | ||
28 | 24 | ] | ||
29 | 25 | |||
30 | 26 | from collections import defaultdict | ||
31 | 27 | from datetime import datetime | ||
32 | 28 | from itertools import chain | ||
33 | 29 | import re | ||
34 | 30 | |||
35 | 31 | from provisioningserver.dhcp.leases_parser import ( | ||
36 | 32 | get_host_mac, | ||
37 | 33 | has_expired, | ||
38 | 34 | is_host, | ||
39 | 35 | is_lease, | ||
40 | 36 | lease_parser, | ||
41 | 37 | ) | ||
42 | 38 | |||
43 | 39 | |||
44 | 40 | re_entry = re.compile( | ||
45 | 41 | r''' | ||
46 | 42 | ^\s* # Ignore leading whitespace on each line. | ||
47 | 43 | (host|lease) # Look only for host or lease stanzas. | ||
48 | 44 | \s+ # Mandatory whitespace. | ||
49 | 45 | ([0-9a-fA-F.:]+) # Capture the IP/MAC address for this stanza. | ||
50 | 46 | \s*{ # Optional whitespace then an opening brace. | ||
51 | 47 | ''', | ||
52 | 48 | re.MULTILINE | re.DOTALL | re.VERBOSE) | ||
53 | 49 | |||
54 | 50 | |||
55 | 51 | def find_lease_starts(leases_contents): | ||
56 | 52 | results = defaultdict(dict) | ||
57 | 53 | for match in re_entry.finditer(leases_contents): | ||
58 | 54 | stanza, address = match.groups() | ||
59 | 55 | results[stanza][address] = match.start() | ||
60 | 56 | return chain.from_iterable( | ||
61 | 57 | mapping.itervalues() for mapping in results.itervalues()) | ||
62 | 58 | |||
63 | 59 | |||
64 | 60 | def extract_leases(leases_contents): | ||
65 | 61 | starts = find_lease_starts(leases_contents) | ||
66 | 62 | for start in sorted(starts): | ||
67 | 63 | record = lease_parser.scanString(leases_contents[start:]) | ||
68 | 64 | try: | ||
69 | 65 | token, _, _ = next(record) | ||
70 | 66 | except StopIteration: | ||
71 | 67 | pass | ||
72 | 68 | else: | ||
73 | 69 | yield token | ||
74 | 70 | |||
75 | 71 | |||
76 | 72 | def parse_leases(leases_contents): | ||
77 | 73 | results = {} | ||
78 | 74 | now = datetime.utcnow() | ||
79 | 75 | for entry in extract_leases(leases_contents): | ||
80 | 76 | if is_lease(entry): | ||
81 | 77 | if not has_expired(entry, now): | ||
82 | 78 | results[entry.ip] = entry.hardware.mac | ||
83 | 79 | elif is_host(entry): | ||
84 | 80 | mac = get_host_mac(entry) | ||
85 | 81 | if mac is None: | ||
86 | 82 | # TODO: Test this. | ||
87 | 83 | if entry.ip in results: | ||
88 | 84 | del results[entry.ip] | ||
89 | 85 | else: | ||
90 | 86 | results[entry.ip] = mac | ||
91 | 87 | return results | ||
92 | 0 | 88 | ||
93 | === modified file 'src/provisioningserver/dhcp/tests/test_leases_parser.py' | |||
94 | --- src/provisioningserver/dhcp/tests/test_leases_parser.py 2013-10-07 09:12:40 +0000 | |||
95 | +++ src/provisioningserver/dhcp/tests/test_leases_parser.py 2014-05-19 15:16:18 +0000 | |||
96 | @@ -20,6 +20,10 @@ | |||
97 | 20 | 20 | ||
98 | 21 | from maastesting.factory import factory | 21 | from maastesting.factory import factory |
99 | 22 | from maastesting.testcase import MAASTestCase | 22 | from maastesting.testcase import MAASTestCase |
100 | 23 | from provisioningserver.dhcp import ( | ||
101 | 24 | leases_parser, | ||
102 | 25 | leases_parser_fast, | ||
103 | 26 | ) | ||
104 | 23 | from provisioningserver.dhcp.leases_parser import ( | 27 | from provisioningserver.dhcp.leases_parser import ( |
105 | 24 | combine_entries, | 28 | combine_entries, |
106 | 25 | gather_hosts, | 29 | gather_hosts, |
107 | @@ -30,197 +34,51 @@ | |||
108 | 30 | is_host, | 34 | is_host, |
109 | 31 | is_lease, | 35 | is_lease, |
110 | 32 | lease_parser, | 36 | lease_parser, |
292 | 33 | parse_leases, | 37 | ) |
293 | 34 | ) | 38 | |
294 | 35 | 39 | ||
295 | 36 | 40 | def fake_parsed_lease(ip=None, mac=None, ends=None, | |
296 | 37 | class TestLeasesParser(MAASTestCase): | 41 | entry_type='lease'): |
297 | 38 | 42 | """Fake a lease as produced by the parser.""" | |
298 | 39 | def fake_parsed_lease(self, ip=None, mac=None, ends=None, | 43 | if ip is None: |
299 | 40 | entry_type='lease'): | 44 | ip = factory.getRandomIPAddress() |
300 | 41 | """Fake a lease as produced by the parser.""" | 45 | if mac is None: |
301 | 42 | if ip is None: | 46 | mac = factory.getRandomMACAddress() |
302 | 43 | ip = factory.getRandomIPAddress() | 47 | Hardware = namedtuple('Hardware', ['mac']) |
303 | 44 | if mac is None: | 48 | Lease = namedtuple( |
304 | 45 | mac = factory.getRandomMACAddress() | 49 | 'Lease', ['lease_or_host', 'ip', 'hardware', 'ends']) |
305 | 46 | Hardware = namedtuple('Hardware', ['mac']) | 50 | return Lease(entry_type, ip, Hardware(mac), ends) |
306 | 47 | Lease = namedtuple( | 51 | |
307 | 48 | 'Lease', ['lease_or_host', 'ip', 'hardware', 'ends']) | 52 | |
308 | 49 | return Lease(entry_type, ip, Hardware(mac), ends) | 53 | def fake_parsed_host(ip=None, mac=None): |
309 | 50 | 54 | """Fake a host declaration as produced by the parser.""" | |
310 | 51 | def fake_parsed_host(self, ip=None, mac=None): | 55 | return fake_parsed_lease(ip=ip, mac=mac, entry_type='host') |
311 | 52 | """Fake a host declaration as produced by the parser.""" | 56 | |
312 | 53 | return self.fake_parsed_lease(ip=ip, mac=mac, entry_type='host') | 57 | |
313 | 54 | 58 | def fake_parsed_rubout(ip=None): | |
314 | 55 | def fake_parsed_rubout(self, ip=None): | 59 | """Fake a "rubout" host declaration.""" |
315 | 56 | """Fake a "rubout" host declaration.""" | 60 | if ip is None: |
316 | 57 | if ip is None: | 61 | ip = factory.getRandomIPAddress() |
317 | 58 | ip = factory.getRandomIPAddress() | 62 | Rubout = namedtuple('Rubout', ['lease_or_host', 'ip']) |
318 | 59 | Rubout = namedtuple('Rubout', ['lease_or_host', 'ip']) | 63 | return Rubout('host', ip) |
319 | 60 | return Rubout('host', ip) | 64 | |
320 | 61 | 65 | ||
321 | 62 | def test_get_expiry_date_parses_expiry_date(self): | 66 | class TestLeasesParsers(MAASTestCase): |
322 | 63 | lease = self.fake_parsed_lease(ends='0 2011/01/02 03:04:05') | 67 | |
323 | 64 | self.assertEqual( | 68 | scenarios = ( |
324 | 65 | datetime( | 69 | ("original", dict(parse=leases_parser.parse_leases)), |
325 | 66 | year=2011, month=01, day=02, | 70 | ("fast", dict(parse=leases_parser_fast.parse_leases)), |
326 | 67 | hour=03, minute=04, second=05), | 71 | ) |
146 | 68 | get_expiry_date(lease)) | ||
147 | 69 | |||
148 | 70 | def test_get_expiry_date_returns_None_for_never(self): | ||
149 | 71 | self.assertIsNone( | ||
150 | 72 | get_expiry_date(self.fake_parsed_lease(ends='never'))) | ||
151 | 73 | |||
152 | 74 | def test_get_expiry_date_returns_None_if_no_expiry_given(self): | ||
153 | 75 | self.assertIsNone(get_expiry_date(self.fake_parsed_lease(ends=None))) | ||
154 | 76 | |||
155 | 77 | def test_has_expired_returns_False_for_eternal_lease(self): | ||
156 | 78 | now = datetime.utcnow() | ||
157 | 79 | self.assertFalse(has_expired(self.fake_parsed_lease(ends=None), now)) | ||
158 | 80 | |||
159 | 81 | def test_has_expired_returns_False_for_future_expiry_date(self): | ||
160 | 82 | now = datetime.utcnow() | ||
161 | 83 | later = '1 2035/12/31 23:59:59' | ||
162 | 84 | self.assertFalse(has_expired(self.fake_parsed_lease(ends=later), now)) | ||
163 | 85 | |||
164 | 86 | def test_has_expired_returns_True_for_past_expiry_date(self): | ||
165 | 87 | now = datetime.utcnow() | ||
166 | 88 | earlier = '1 2001/01/01 00:00:00' | ||
167 | 89 | self.assertTrue( | ||
168 | 90 | has_expired(self.fake_parsed_lease(ends=earlier), now)) | ||
169 | 91 | |||
170 | 92 | def test_gather_leases_finds_current_leases(self): | ||
171 | 93 | lease = self.fake_parsed_lease() | ||
172 | 94 | self.assertEqual( | ||
173 | 95 | {lease.ip: lease.hardware.mac}, | ||
174 | 96 | gather_leases([lease])) | ||
175 | 97 | |||
176 | 98 | def test_gather_leases_ignores_expired_leases(self): | ||
177 | 99 | earlier = '1 2001/01/01 00:00:00' | ||
178 | 100 | lease = self.fake_parsed_lease(ends=earlier) | ||
179 | 101 | self.assertEqual({}, gather_leases([lease])) | ||
180 | 102 | |||
181 | 103 | def test_gather_leases_combines_expired_and_current_leases(self): | ||
182 | 104 | earlier = '1 2001/01/01 00:00:00' | ||
183 | 105 | ip = factory.getRandomIPAddress() | ||
184 | 106 | old_owner = factory.getRandomMACAddress() | ||
185 | 107 | new_owner = factory.getRandomMACAddress() | ||
186 | 108 | leases = [ | ||
187 | 109 | self.fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier), | ||
188 | 110 | self.fake_parsed_lease(ip=ip, mac=new_owner), | ||
189 | 111 | ] | ||
190 | 112 | self.assertEqual({ip: new_owner}, gather_leases(leases)) | ||
191 | 113 | |||
192 | 114 | def test_gather_leases_ignores_ordering(self): | ||
193 | 115 | earlier = '1 2001/01/01 00:00:00' | ||
194 | 116 | ip = factory.getRandomIPAddress() | ||
195 | 117 | old_owner = factory.getRandomMACAddress() | ||
196 | 118 | new_owner = factory.getRandomMACAddress() | ||
197 | 119 | leases = [ | ||
198 | 120 | self.fake_parsed_lease(ip=ip, mac=new_owner), | ||
199 | 121 | self.fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier), | ||
200 | 122 | ] | ||
201 | 123 | self.assertEqual({ip: new_owner}, gather_leases(leases)) | ||
202 | 124 | |||
203 | 125 | def test_gather_leases_ignores_host_declarations(self): | ||
204 | 126 | self.assertEqual({}, gather_leases([self.fake_parsed_host()])) | ||
205 | 127 | |||
206 | 128 | def test_gather_hosts_finds_hosts(self): | ||
207 | 129 | host = self.fake_parsed_host() | ||
208 | 130 | self.assertEqual({host.ip: host.hardware.mac}, gather_hosts([host])) | ||
209 | 131 | |||
210 | 132 | def test_gather_hosts_ignores_unaccompanied_rubouts(self): | ||
211 | 133 | self.assertEqual({}, gather_hosts([self.fake_parsed_rubout()])) | ||
212 | 134 | |||
213 | 135 | def test_gather_hosts_ignores_rubbed_out_entries(self): | ||
214 | 136 | ip = factory.getRandomIPAddress() | ||
215 | 137 | hosts = [ | ||
216 | 138 | self.fake_parsed_host(ip=ip), | ||
217 | 139 | self.fake_parsed_rubout(ip=ip), | ||
218 | 140 | ] | ||
219 | 141 | self.assertEqual({}, gather_hosts(hosts)) | ||
220 | 142 | |||
221 | 143 | def test_gather_hosts_follows_reassigned_host(self): | ||
222 | 144 | ip = factory.getRandomIPAddress() | ||
223 | 145 | new_owner = factory.getRandomMACAddress() | ||
224 | 146 | hosts = [ | ||
225 | 147 | self.fake_parsed_host(ip=ip), | ||
226 | 148 | self.fake_parsed_rubout(ip=ip), | ||
227 | 149 | self.fake_parsed_host(ip=ip, mac=new_owner), | ||
228 | 150 | ] | ||
229 | 151 | self.assertEqual({ip: new_owner}, gather_hosts(hosts)) | ||
230 | 152 | |||
231 | 153 | def test_is_lease_and_is_host_recognize_lease(self): | ||
232 | 154 | params = { | ||
233 | 155 | 'ip': factory.getRandomIPAddress(), | ||
234 | 156 | 'mac': factory.getRandomMACAddress(), | ||
235 | 157 | } | ||
236 | 158 | [parsed_lease] = lease_parser.searchString(dedent("""\ | ||
237 | 159 | lease %(ip)s { | ||
238 | 160 | hardware ethernet %(mac)s; | ||
239 | 161 | } | ||
240 | 162 | """ % params)) | ||
241 | 163 | self.assertEqual( | ||
242 | 164 | (True, False), | ||
243 | 165 | (is_lease(parsed_lease), is_host(parsed_lease))) | ||
244 | 166 | |||
245 | 167 | def test_is_lease_and_is_host_recognize_host(self): | ||
246 | 168 | params = { | ||
247 | 169 | 'ip': factory.getRandomIPAddress(), | ||
248 | 170 | 'mac': factory.getRandomMACAddress(), | ||
249 | 171 | } | ||
250 | 172 | [parsed_host] = lease_parser.searchString(dedent("""\ | ||
251 | 173 | host %(ip)s { | ||
252 | 174 | hardware ethernet %(mac)s; | ||
253 | 175 | } | ||
254 | 176 | """ % params)) | ||
255 | 177 | self.assertEqual( | ||
256 | 178 | (False, True), | ||
257 | 179 | (is_lease(parsed_host), is_host(parsed_host))) | ||
258 | 180 | |||
259 | 181 | def test_get_host_mac_returns_None_for_host(self): | ||
260 | 182 | params = { | ||
261 | 183 | 'ip': factory.getRandomIPAddress(), | ||
262 | 184 | 'mac': factory.getRandomMACAddress(), | ||
263 | 185 | } | ||
264 | 186 | [parsed_host] = lease_parser.searchString(dedent("""\ | ||
265 | 187 | host %(ip)s { | ||
266 | 188 | hardware ethernet %(mac)s; | ||
267 | 189 | } | ||
268 | 190 | """ % params)) | ||
269 | 191 | self.assertEqual(params['mac'], get_host_mac(parsed_host)) | ||
270 | 192 | |||
271 | 193 | def test_get_host_mac_returns_None_for_rubout(self): | ||
272 | 194 | ip = factory.getRandomIPAddress() | ||
273 | 195 | [parsed_host] = lease_parser.searchString(dedent("""\ | ||
274 | 196 | host %s { | ||
275 | 197 | deleted; | ||
276 | 198 | } | ||
277 | 199 | """ % ip)) | ||
278 | 200 | self.assertIsNone(get_host_mac(parsed_host)) | ||
279 | 201 | |||
280 | 202 | def test_get_host_mac_returns_None_for_rubout_even_with_mac(self): | ||
281 | 203 | params = { | ||
282 | 204 | 'ip': factory.getRandomIPAddress(), | ||
283 | 205 | 'mac': factory.getRandomMACAddress(), | ||
284 | 206 | } | ||
285 | 207 | [parsed_host] = lease_parser.searchString(dedent("""\ | ||
286 | 208 | host %(ip)s { | ||
287 | 209 | deleted; | ||
288 | 210 | hardware ethernet %(mac)s; | ||
289 | 211 | } | ||
290 | 212 | """ % params)) | ||
291 | 213 | self.assertIsNone(get_host_mac(parsed_host)) | ||
327 | 214 | 72 | ||
328 | 215 | def test_parse_leases_copes_with_empty_file(self): | 73 | def test_parse_leases_copes_with_empty_file(self): |
330 | 216 | self.assertEqual({}, parse_leases("")) | 74 | self.assertEqual({}, self.parse("")) |
331 | 217 | 75 | ||
332 | 218 | def test_parse_leases_parses_lease(self): | 76 | def test_parse_leases_parses_lease(self): |
333 | 219 | params = { | 77 | params = { |
334 | 220 | 'ip': factory.getRandomIPAddress(), | 78 | 'ip': factory.getRandomIPAddress(), |
335 | 221 | 'mac': factory.getRandomMACAddress(), | 79 | 'mac': factory.getRandomMACAddress(), |
336 | 222 | } | 80 | } |
338 | 223 | leases = parse_leases(dedent("""\ | 81 | leases = self.parse(dedent("""\ |
339 | 224 | lease %(ip)s { | 82 | lease %(ip)s { |
340 | 225 | starts 5 2010/01/01 00:00:01; | 83 | starts 5 2010/01/01 00:00:01; |
341 | 226 | ends never; | 84 | ends never; |
342 | @@ -254,7 +112,7 @@ | |||
343 | 254 | 'ip': factory.getRandomIPAddress(), | 112 | 'ip': factory.getRandomIPAddress(), |
344 | 255 | 'mac': factory.getRandomMACAddress(), | 113 | 'mac': factory.getRandomMACAddress(), |
345 | 256 | } | 114 | } |
347 | 257 | leases = parse_leases(dedent("""\ | 115 | leases = self.parse(dedent("""\ |
348 | 258 | host %(ip)s { | 116 | host %(ip)s { |
349 | 259 | dynamic; | 117 | dynamic; |
350 | 260 | hardware ethernet %(mac)s; | 118 | hardware ethernet %(mac)s; |
351 | @@ -263,8 +121,36 @@ | |||
352 | 263 | """ % params)) | 121 | """ % params)) |
353 | 264 | self.assertEqual({params['ip']: params['mac']}, leases) | 122 | self.assertEqual({params['ip']: params['mac']}, leases) |
354 | 265 | 123 | ||
355 | 124 | def test_parse_leases_copes_with_misleading_values(self): | ||
356 | 125 | params = { | ||
357 | 126 | 'ip1': factory.getRandomIPAddress(), | ||
358 | 127 | 'mac1': factory.getRandomMACAddress(), | ||
359 | 128 | 'ip2': factory.getRandomIPAddress(), | ||
360 | 129 | 'mac2': factory.getRandomMACAddress(), | ||
361 | 130 | } | ||
362 | 131 | leases = self.parse(dedent("""\ | ||
363 | 132 | host %(ip1)s { | ||
364 | 133 | dynamic; | ||
365 | 134 | ### NOTE the following value has a closing brace, and | ||
366 | 135 | ### also looks like a host record. | ||
367 | 136 | uid "foo}host 12.34.56.78 { }"; | ||
368 | 137 | hardware ethernet %(mac1)s; | ||
369 | 138 | fixed-address %(ip1)s; | ||
370 | 139 | } | ||
371 | 140 | ### NOTE the extra indent on the line below. | ||
372 | 141 | host %(ip2)s { | ||
373 | 142 | dynamic; | ||
374 | 143 | hardware ethernet %(mac2)s; | ||
375 | 144 | fixed-address %(ip2)s; | ||
376 | 145 | } | ||
377 | 146 | """ % params)) | ||
378 | 147 | self.assertEqual( | ||
379 | 148 | {params['ip1']: params['mac1'], | ||
380 | 149 | params['ip2']: params['mac2']}, | ||
381 | 150 | leases) | ||
382 | 151 | |||
383 | 266 | def test_parse_leases_parses_host_rubout(self): | 152 | def test_parse_leases_parses_host_rubout(self): |
385 | 267 | leases = parse_leases(dedent("""\ | 153 | leases = self.parse(dedent("""\ |
386 | 268 | host %s { | 154 | host %s { |
387 | 269 | deleted; | 155 | deleted; |
388 | 270 | } | 156 | } |
389 | @@ -277,7 +163,7 @@ | |||
390 | 277 | 'mac': factory.getRandomMACAddress(), | 163 | 'mac': factory.getRandomMACAddress(), |
391 | 278 | 'incomplete_ip': factory.getRandomIPAddress(), | 164 | 'incomplete_ip': factory.getRandomIPAddress(), |
392 | 279 | } | 165 | } |
394 | 280 | leases = parse_leases(dedent("""\ | 166 | leases = self.parse(dedent("""\ |
395 | 281 | lease %(ip)s { | 167 | lease %(ip)s { |
396 | 282 | hardware ethernet %(mac)s; | 168 | hardware ethernet %(mac)s; |
397 | 283 | } | 169 | } |
398 | @@ -291,7 +177,7 @@ | |||
399 | 291 | 'ip': factory.getRandomIPAddress(), | 177 | 'ip': factory.getRandomIPAddress(), |
400 | 292 | 'mac': factory.getRandomMACAddress(), | 178 | 'mac': factory.getRandomMACAddress(), |
401 | 293 | } | 179 | } |
403 | 294 | leases = parse_leases(dedent("""\ | 180 | leases = self.parse(dedent("""\ |
404 | 295 | # Top comment (ignored). | 181 | # Top comment (ignored). |
405 | 296 | lease %(ip)s { # End-of-line comment (ignored). | 182 | lease %(ip)s { # End-of-line comment (ignored). |
406 | 297 | # Comment in lease block (ignored). | 183 | # Comment in lease block (ignored). |
407 | @@ -306,7 +192,7 @@ | |||
408 | 306 | 'ip': factory.getRandomIPAddress(), | 192 | 'ip': factory.getRandomIPAddress(), |
409 | 307 | 'mac': factory.getRandomMACAddress(), | 193 | 'mac': factory.getRandomMACAddress(), |
410 | 308 | } | 194 | } |
412 | 309 | leases = parse_leases(dedent("""\ | 195 | leases = self.parse(dedent("""\ |
413 | 310 | lease %(ip)s { | 196 | lease %(ip)s { |
414 | 311 | hardware ethernet %(mac)s; | 197 | hardware ethernet %(mac)s; |
415 | 312 | ends 1 2001/01/01 00:00:00; | 198 | ends 1 2001/01/01 00:00:00; |
416 | @@ -319,7 +205,7 @@ | |||
417 | 319 | 'ip': factory.getRandomIPAddress(), | 205 | 'ip': factory.getRandomIPAddress(), |
418 | 320 | 'mac': factory.getRandomMACAddress(), | 206 | 'mac': factory.getRandomMACAddress(), |
419 | 321 | } | 207 | } |
421 | 322 | leases = parse_leases(dedent("""\ | 208 | leases = self.parse(dedent("""\ |
422 | 323 | lease %(ip)s { | 209 | lease %(ip)s { |
423 | 324 | hardware ethernet %(mac)s; | 210 | hardware ethernet %(mac)s; |
424 | 325 | ends never; | 211 | ends never; |
425 | @@ -332,7 +218,7 @@ | |||
426 | 332 | 'ip': factory.getRandomIPAddress(), | 218 | 'ip': factory.getRandomIPAddress(), |
427 | 333 | 'mac': factory.getRandomMACAddress(), | 219 | 'mac': factory.getRandomMACAddress(), |
428 | 334 | } | 220 | } |
430 | 335 | leases = parse_leases(dedent("""\ | 221 | leases = self.parse(dedent("""\ |
431 | 336 | lease %(ip)s { | 222 | lease %(ip)s { |
432 | 337 | hardware ethernet %(mac)s; | 223 | hardware ethernet %(mac)s; |
433 | 338 | } | 224 | } |
434 | @@ -345,7 +231,7 @@ | |||
435 | 345 | 'old_owner': factory.getRandomMACAddress(), | 231 | 'old_owner': factory.getRandomMACAddress(), |
436 | 346 | 'new_owner': factory.getRandomMACAddress(), | 232 | 'new_owner': factory.getRandomMACAddress(), |
437 | 347 | } | 233 | } |
439 | 348 | leases = parse_leases(dedent("""\ | 234 | leases = self.parse(dedent("""\ |
440 | 349 | lease %(ip)s { | 235 | lease %(ip)s { |
441 | 350 | hardware ethernet %(old_owner)s; | 236 | hardware ethernet %(old_owner)s; |
442 | 351 | } | 237 | } |
443 | @@ -360,7 +246,7 @@ | |||
444 | 360 | 'ip': factory.getRandomIPAddress(), | 246 | 'ip': factory.getRandomIPAddress(), |
445 | 361 | 'mac': factory.getRandomMACAddress(), | 247 | 'mac': factory.getRandomMACAddress(), |
446 | 362 | } | 248 | } |
448 | 363 | leases = parse_leases(dedent("""\ | 249 | leases = self.parse(dedent("""\ |
449 | 364 | host %(ip)s { | 250 | host %(ip)s { |
450 | 365 | dynamic; | 251 | dynamic; |
451 | 366 | hardware ethernet %(mac)s; | 252 | hardware ethernet %(mac)s; |
452 | @@ -375,7 +261,7 @@ | |||
453 | 375 | 'ip': factory.getRandomIPAddress(), | 261 | 'ip': factory.getRandomIPAddress(), |
454 | 376 | 'mac': factory.getRandomMACAddress(), | 262 | 'mac': factory.getRandomMACAddress(), |
455 | 377 | } | 263 | } |
457 | 378 | leases = parse_leases(dedent("""\ | 264 | leases = self.parse(dedent("""\ |
458 | 379 | host %(ip)s { | 265 | host %(ip)s { |
459 | 380 | hardware ethernet %(mac)s; | 266 | hardware ethernet %(mac)s; |
460 | 381 | fixed-address %(ip)s; | 267 | fixed-address %(ip)s; |
461 | @@ -383,13 +269,235 @@ | |||
462 | 383 | """ % params)) | 269 | """ % params)) |
463 | 384 | self.assertEqual({params['ip']: params['mac']}, leases) | 270 | self.assertEqual({params['ip']: params['mac']}, leases) |
464 | 385 | 271 | ||
465 | 272 | |||
466 | 273 | class TestLeasesParserFast(MAASTestCase): | ||
467 | 274 | |||
468 | 275 | def test_expired_lease_does_not_shadow_earlier_host_stanza(self): | ||
469 | 276 | params = { | ||
470 | 277 | 'ip': factory.getRandomIPAddress(), | ||
471 | 278 | 'mac1': factory.getRandomMACAddress(), | ||
472 | 279 | 'mac2': factory.getRandomMACAddress(), | ||
473 | 280 | } | ||
474 | 281 | leases = leases_parser_fast.parse_leases(dedent("""\ | ||
475 | 282 | host %(ip)s { | ||
476 | 283 | dynamic; | ||
477 | 284 | hardware ethernet %(mac1)s; | ||
478 | 285 | fixed-address %(ip)s; | ||
479 | 286 | } | ||
480 | 287 | lease %(ip)s { | ||
481 | 288 | starts 5 2010/01/01 00:00:01; | ||
482 | 289 | ends 1 2010/01/01 00:00:02; | ||
483 | 290 | hardware ethernet %(mac2)s; | ||
484 | 291 | } | ||
485 | 292 | """ % params)) | ||
486 | 293 | # The lease has expired so it doesn't shadow the host stanza, | ||
487 | 294 | # and so the MAC returned is from the host stanza. | ||
488 | 295 | self.assertEqual({params["ip"]: params["mac1"]}, leases) | ||
489 | 296 | |||
490 | 297 | def test_active_lease_shadows_earlier_host_stanza(self): | ||
491 | 298 | params = { | ||
492 | 299 | 'ip': factory.getRandomIPAddress(), | ||
493 | 300 | 'mac1': factory.getRandomMACAddress(), | ||
494 | 301 | 'mac2': factory.getRandomMACAddress(), | ||
495 | 302 | } | ||
496 | 303 | leases = leases_parser_fast.parse_leases(dedent("""\ | ||
497 | 304 | host %(ip)s { | ||
498 | 305 | dynamic; | ||
499 | 306 | hardware ethernet %(mac1)s; | ||
500 | 307 | fixed-address %(ip)s; | ||
501 | 308 | } | ||
502 | 309 | lease %(ip)s { | ||
503 | 310 | starts 5 2010/01/01 00:00:01; | ||
504 | 311 | hardware ethernet %(mac2)s; | ||
505 | 312 | } | ||
506 | 313 | """ % params)) | ||
507 | 314 | # The lease hasn't expired, so shadows the earlier host stanza. | ||
508 | 315 | self.assertEqual({params["ip"]: params["mac2"]}, leases) | ||
509 | 316 | |||
510 | 317 | def test_host_stanza_shadows_earlier_active_lease(self): | ||
511 | 318 | params = { | ||
512 | 319 | 'ip': factory.getRandomIPAddress(), | ||
513 | 320 | 'mac1': factory.getRandomMACAddress(), | ||
514 | 321 | 'mac2': factory.getRandomMACAddress(), | ||
515 | 322 | } | ||
516 | 323 | leases = leases_parser_fast.parse_leases(dedent("""\ | ||
517 | 324 | lease %(ip)s { | ||
518 | 325 | starts 5 2010/01/01 00:00:01; | ||
519 | 326 | hardware ethernet %(mac2)s; | ||
520 | 327 | } | ||
521 | 328 | host %(ip)s { | ||
522 | 329 | dynamic; | ||
523 | 330 | hardware ethernet %(mac1)s; | ||
524 | 331 | fixed-address %(ip)s; | ||
525 | 332 | } | ||
526 | 333 | """ % params)) | ||
527 | 334 | # The lease hasn't expired, but the host entry is later, so it | ||
528 | 335 | # shadows the earlier lease stanza. | ||
529 | 336 | self.assertEqual({params["ip"]: params["mac1"]}, leases) | ||
530 | 337 | |||
531 | 338 | |||
532 | 339 | class TestLeasesParserFunctions(MAASTestCase): | ||
533 | 340 | |||
534 | 341 | def test_get_expiry_date_parses_expiry_date(self): | ||
535 | 342 | lease = fake_parsed_lease(ends='0 2011/01/02 03:04:05') | ||
536 | 343 | self.assertEqual( | ||
537 | 344 | datetime( | ||
538 | 345 | year=2011, month=01, day=02, | ||
539 | 346 | hour=03, minute=04, second=05), | ||
540 | 347 | get_expiry_date(lease)) | ||
541 | 348 | |||
542 | 349 | def test_get_expiry_date_returns_None_for_never(self): | ||
543 | 350 | self.assertIsNone( | ||
544 | 351 | get_expiry_date(fake_parsed_lease(ends='never'))) | ||
545 | 352 | |||
546 | 353 | def test_get_expiry_date_returns_None_if_no_expiry_given(self): | ||
547 | 354 | self.assertIsNone(get_expiry_date(fake_parsed_lease(ends=None))) | ||
548 | 355 | |||
549 | 356 | def test_has_expired_returns_False_for_eternal_lease(self): | ||
550 | 357 | now = datetime.utcnow() | ||
551 | 358 | self.assertFalse(has_expired(fake_parsed_lease(ends=None), now)) | ||
552 | 359 | |||
553 | 360 | def test_has_expired_returns_False_for_future_expiry_date(self): | ||
554 | 361 | now = datetime.utcnow() | ||
555 | 362 | later = '1 2035/12/31 23:59:59' | ||
556 | 363 | self.assertFalse(has_expired(fake_parsed_lease(ends=later), now)) | ||
557 | 364 | |||
558 | 365 | def test_has_expired_returns_True_for_past_expiry_date(self): | ||
559 | 366 | now = datetime.utcnow() | ||
560 | 367 | earlier = '1 2001/01/01 00:00:00' | ||
561 | 368 | self.assertTrue( | ||
562 | 369 | has_expired(fake_parsed_lease(ends=earlier), now)) | ||
563 | 370 | |||
564 | 371 | def test_gather_leases_finds_current_leases(self): | ||
565 | 372 | lease = fake_parsed_lease() | ||
566 | 373 | self.assertEqual( | ||
567 | 374 | {lease.ip: lease.hardware.mac}, | ||
568 | 375 | gather_leases([lease])) | ||
569 | 376 | |||
570 | 377 | def test_gather_leases_ignores_expired_leases(self): | ||
571 | 378 | earlier = '1 2001/01/01 00:00:00' | ||
572 | 379 | lease = fake_parsed_lease(ends=earlier) | ||
573 | 380 | self.assertEqual({}, gather_leases([lease])) | ||
574 | 381 | |||
575 | 382 | def test_gather_leases_combines_expired_and_current_leases(self): | ||
576 | 383 | earlier = '1 2001/01/01 00:00:00' | ||
577 | 384 | ip = factory.getRandomIPAddress() | ||
578 | 385 | old_owner = factory.getRandomMACAddress() | ||
579 | 386 | new_owner = factory.getRandomMACAddress() | ||
580 | 387 | leases = [ | ||
581 | 388 | fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier), | ||
582 | 389 | fake_parsed_lease(ip=ip, mac=new_owner), | ||
583 | 390 | ] | ||
584 | 391 | self.assertEqual({ip: new_owner}, gather_leases(leases)) | ||
585 | 392 | |||
586 | 393 | def test_gather_leases_ignores_ordering(self): | ||
587 | 394 | earlier = '1 2001/01/01 00:00:00' | ||
588 | 395 | ip = factory.getRandomIPAddress() | ||
589 | 396 | old_owner = factory.getRandomMACAddress() | ||
590 | 397 | new_owner = factory.getRandomMACAddress() | ||
591 | 398 | leases = [ | ||
592 | 399 | fake_parsed_lease(ip=ip, mac=new_owner), | ||
593 | 400 | fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier), | ||
594 | 401 | ] | ||
595 | 402 | self.assertEqual({ip: new_owner}, gather_leases(leases)) | ||
596 | 403 | |||
597 | 404 | def test_gather_leases_ignores_host_declarations(self): | ||
598 | 405 | self.assertEqual({}, gather_leases([fake_parsed_host()])) | ||
599 | 406 | |||
600 | 407 | def test_gather_hosts_finds_hosts(self): | ||
601 | 408 | host = fake_parsed_host() | ||
602 | 409 | self.assertEqual({host.ip: host.hardware.mac}, gather_hosts([host])) | ||
603 | 410 | |||
604 | 411 | def test_gather_hosts_ignores_unaccompanied_rubouts(self): | ||
605 | 412 | self.assertEqual({}, gather_hosts([fake_parsed_rubout()])) | ||
606 | 413 | |||
607 | 414 | def test_gather_hosts_ignores_rubbed_out_entries(self): | ||
608 | 415 | ip = factory.getRandomIPAddress() | ||
609 | 416 | hosts = [ | ||
610 | 417 | fake_parsed_host(ip=ip), | ||
611 | 418 | fake_parsed_rubout(ip=ip), | ||
612 | 419 | ] | ||
613 | 420 | self.assertEqual({}, gather_hosts(hosts)) | ||
614 | 421 | |||
615 | 422 | def test_gather_hosts_follows_reassigned_host(self): | ||
616 | 423 | ip = factory.getRandomIPAddress() | ||
617 | 424 | new_owner = factory.getRandomMACAddress() | ||
618 | 425 | hosts = [ | ||
619 | 426 | fake_parsed_host(ip=ip), | ||
620 | 427 | fake_parsed_rubout(ip=ip), | ||
621 | 428 | fake_parsed_host(ip=ip, mac=new_owner), | ||
622 | 429 | ] | ||
623 | 430 | self.assertEqual({ip: new_owner}, gather_hosts(hosts)) | ||
624 | 431 | |||
625 | 432 | def test_is_lease_and_is_host_recognize_lease(self): | ||
626 | 433 | params = { | ||
627 | 434 | 'ip': factory.getRandomIPAddress(), | ||
628 | 435 | 'mac': factory.getRandomMACAddress(), | ||
629 | 436 | } | ||
630 | 437 | [parsed_lease] = lease_parser.searchString(dedent("""\ | ||
631 | 438 | lease %(ip)s { | ||
632 | 439 | hardware ethernet %(mac)s; | ||
633 | 440 | } | ||
634 | 441 | """ % params)) | ||
635 | 442 | self.assertEqual( | ||
636 | 443 | (True, False), | ||
637 | 444 | (is_lease(parsed_lease), is_host(parsed_lease))) | ||
638 | 445 | |||
639 | 446 | def test_is_lease_and_is_host_recognize_host(self): | ||
640 | 447 | params = { | ||
641 | 448 | 'ip': factory.getRandomIPAddress(), | ||
642 | 449 | 'mac': factory.getRandomMACAddress(), | ||
643 | 450 | } | ||
644 | 451 | [parsed_host] = lease_parser.searchString(dedent("""\ | ||
645 | 452 | host %(ip)s { | ||
646 | 453 | hardware ethernet %(mac)s; | ||
647 | 454 | } | ||
648 | 455 | """ % params)) | ||
649 | 456 | self.assertEqual( | ||
650 | 457 | (False, True), | ||
651 | 458 | (is_lease(parsed_host), is_host(parsed_host))) | ||
652 | 459 | |||
653 | 460 | def test_get_host_mac_returns_None_for_host(self): | ||
654 | 461 | params = { | ||
655 | 462 | 'ip': factory.getRandomIPAddress(), | ||
656 | 463 | 'mac': factory.getRandomMACAddress(), | ||
657 | 464 | } | ||
658 | 465 | [parsed_host] = lease_parser.searchString(dedent("""\ | ||
659 | 466 | host %(ip)s { | ||
660 | 467 | hardware ethernet %(mac)s; | ||
661 | 468 | } | ||
662 | 469 | """ % params)) | ||
663 | 470 | self.assertEqual(params['mac'], get_host_mac(parsed_host)) | ||
664 | 471 | |||
665 | 472 | def test_get_host_mac_returns_None_for_rubout(self): | ||
666 | 473 | ip = factory.getRandomIPAddress() | ||
667 | 474 | [parsed_host] = lease_parser.searchString(dedent("""\ | ||
668 | 475 | host %s { | ||
669 | 476 | deleted; | ||
670 | 477 | } | ||
671 | 478 | """ % ip)) | ||
672 | 479 | self.assertIsNone(get_host_mac(parsed_host)) | ||
673 | 480 | |||
674 | 481 | def test_get_host_mac_returns_None_for_rubout_even_with_mac(self): | ||
675 | 482 | params = { | ||
676 | 483 | 'ip': factory.getRandomIPAddress(), | ||
677 | 484 | 'mac': factory.getRandomMACAddress(), | ||
678 | 485 | } | ||
679 | 486 | [parsed_host] = lease_parser.searchString(dedent("""\ | ||
680 | 487 | host %(ip)s { | ||
681 | 488 | deleted; | ||
682 | 489 | hardware ethernet %(mac)s; | ||
683 | 490 | } | ||
684 | 491 | """ % params)) | ||
685 | 492 | self.assertIsNone(get_host_mac(parsed_host)) | ||
686 | 493 | |||
687 | 386 | def test_combine_entries_accepts_host_followed_by_expired_lease(self): | 494 | def test_combine_entries_accepts_host_followed_by_expired_lease(self): |
688 | 387 | ip = factory.getRandomIPAddress() | 495 | ip = factory.getRandomIPAddress() |
689 | 388 | mac = factory.getRandomMACAddress() | 496 | mac = factory.getRandomMACAddress() |
690 | 389 | earlier = '1 2001/01/01 00:00:00' | 497 | earlier = '1 2001/01/01 00:00:00' |
691 | 390 | entries = [ | 498 | entries = [ |
694 | 391 | self.fake_parsed_host(ip=ip, mac=mac), | 499 | fake_parsed_host(ip=ip, mac=mac), |
695 | 392 | self.fake_parsed_lease(ip=ip, ends=earlier), | 500 | fake_parsed_lease(ip=ip, ends=earlier), |
696 | 393 | ] | 501 | ] |
697 | 394 | self.assertEqual({ip: mac}, combine_entries(entries)) | 502 | self.assertEqual({ip: mac}, combine_entries(entries)) |
698 | 395 | 503 | ||
699 | @@ -398,8 +506,8 @@ | |||
700 | 398 | mac = factory.getRandomMACAddress() | 506 | mac = factory.getRandomMACAddress() |
701 | 399 | earlier = '1 2001/01/01 00:00:00' | 507 | earlier = '1 2001/01/01 00:00:00' |
702 | 400 | entries = [ | 508 | entries = [ |
705 | 401 | self.fake_parsed_lease(ip=ip, ends=earlier), | 509 | fake_parsed_lease(ip=ip, ends=earlier), |
706 | 402 | self.fake_parsed_host(ip=ip, mac=mac), | 510 | fake_parsed_host(ip=ip, mac=mac), |
707 | 403 | ] | 511 | ] |
708 | 404 | self.assertEqual({ip: mac}, combine_entries(entries)) | 512 | self.assertEqual({ip: mac}, combine_entries(entries)) |
709 | 405 | 513 | ||
710 | @@ -407,9 +515,9 @@ | |||
711 | 407 | ip = factory.getRandomIPAddress() | 515 | ip = factory.getRandomIPAddress() |
712 | 408 | mac = factory.getRandomMACAddress() | 516 | mac = factory.getRandomMACAddress() |
713 | 409 | entries = [ | 517 | entries = [ |
717 | 410 | self.fake_parsed_host(ip=ip), | 518 | fake_parsed_host(ip=ip), |
718 | 411 | self.fake_parsed_rubout(ip=ip), | 519 | fake_parsed_rubout(ip=ip), |
719 | 412 | self.fake_parsed_lease(ip=ip, mac=mac), | 520 | fake_parsed_lease(ip=ip, mac=mac), |
720 | 413 | ] | 521 | ] |
721 | 414 | self.assertEqual({ip: mac}, combine_entries(entries)) | 522 | self.assertEqual({ip: mac}, combine_entries(entries)) |
722 | 415 | 523 | ||
723 | @@ -418,9 +526,9 @@ | |||
724 | 418 | mac = factory.getRandomMACAddress() | 526 | mac = factory.getRandomMACAddress() |
725 | 419 | earlier = '1 2001/01/01 00:00:00' | 527 | earlier = '1 2001/01/01 00:00:00' |
726 | 420 | entries = [ | 528 | entries = [ |
730 | 421 | self.fake_parsed_host(ip=ip), | 529 | fake_parsed_host(ip=ip), |
731 | 422 | self.fake_parsed_rubout(ip=ip), | 530 | fake_parsed_rubout(ip=ip), |
732 | 423 | self.fake_parsed_lease(ip=ip, mac=mac, ends=earlier), | 531 | fake_parsed_lease(ip=ip, mac=mac, ends=earlier), |
733 | 424 | ] | 532 | ] |
734 | 425 | self.assertEqual({}, combine_entries(entries)) | 533 | self.assertEqual({}, combine_entries(entries)) |
735 | 426 | 534 | ||
736 | @@ -429,9 +537,9 @@ | |||
737 | 429 | mac = factory.getRandomMACAddress() | 537 | mac = factory.getRandomMACAddress() |
738 | 430 | earlier = '1 2001/01/01 00:00:00' | 538 | earlier = '1 2001/01/01 00:00:00' |
739 | 431 | entries = [ | 539 | entries = [ |
743 | 432 | self.fake_parsed_host(ip=ip), | 540 | fake_parsed_host(ip=ip), |
744 | 433 | self.fake_parsed_lease(ip=ip, mac=mac, ends=earlier), | 541 | fake_parsed_lease(ip=ip, mac=mac, ends=earlier), |
745 | 434 | self.fake_parsed_rubout(ip=ip), | 542 | fake_parsed_rubout(ip=ip), |
746 | 435 | ] | 543 | ] |
747 | 436 | self.assertEqual({}, combine_entries(entries)) | 544 | self.assertEqual({}, combine_entries(entries)) |
748 | 437 | 545 | ||
749 | @@ -439,9 +547,9 @@ | |||
750 | 439 | ip = factory.getRandomIPAddress() | 547 | ip = factory.getRandomIPAddress() |
751 | 440 | mac = factory.getRandomMACAddress() | 548 | mac = factory.getRandomMACAddress() |
752 | 441 | entries = [ | 549 | entries = [ |
756 | 442 | self.fake_parsed_host(ip=ip), | 550 | fake_parsed_host(ip=ip), |
757 | 443 | self.fake_parsed_lease(ip=ip, mac=mac), | 551 | fake_parsed_lease(ip=ip, mac=mac), |
758 | 444 | self.fake_parsed_rubout(ip=ip), | 552 | fake_parsed_rubout(ip=ip), |
759 | 445 | ] | 553 | ] |
760 | 446 | self.assertEqual({ip: mac}, combine_entries(entries)) | 554 | self.assertEqual({ip: mac}, combine_entries(entries)) |
761 | 447 | 555 | ||
762 | @@ -449,8 +557,8 @@ | |||
763 | 449 | ip = factory.getRandomIPAddress() | 557 | ip = factory.getRandomIPAddress() |
764 | 450 | mac = factory.getRandomMACAddress() | 558 | mac = factory.getRandomMACAddress() |
765 | 451 | entries = [ | 559 | entries = [ |
769 | 452 | self.fake_parsed_host(ip=ip), | 560 | fake_parsed_host(ip=ip), |
770 | 453 | self.fake_parsed_rubout(ip=ip), | 561 | fake_parsed_rubout(ip=ip), |
771 | 454 | self.fake_parsed_host(ip=ip, mac=mac), | 562 | fake_parsed_host(ip=ip, mac=mac), |
772 | 455 | ] | 563 | ] |
773 | 456 | self.assertEqual({ip: mac}, combine_entries(entries)) | 564 | self.assertEqual({ip: mac}, combine_entries(entries)) |