Merge lp:~allenap/maas/dhcp-leases-parsing--1.5 into lp:maas/1.5

[Gavin Panella]

Backport trunk r2335: Faster DHCP leases parser.

[Gavin Panella (allenap)]

review: Approve

=== added file 'src/provisioningserver/dhcp/leases_parser_fast.py'
--- src/provisioningserver/dhcp/leases_parser_fast.py	1970-01-01 00:00:00 +0000
+++ src/provisioningserver/dhcp/leases_parser_fast.py	2014-05-19 15:16:18 +0000
@@ -0,0 +1,87 @@
+# Copyright 2013 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+"""A speedier version of `leases_parser`.
+
+This extracts the relevant stanzas from a leases file, keeping only the
+most recent "host" and "lease" entries, then uses the existing and
+properly defined but slow parser to parse them. This massively speeds up
+parsing a leases file that contains a modest number of unique host and
+lease entries, but has become very large because of churn.
+"""
+
+from __future__ import (
+    absolute_import,
+    print_function,
+    unicode_literals,
+    )
+
+str = None
+
+__metaclass__ = type
+__all__ = [
+    'parse_leases',
+    ]
+
+from collections import defaultdict
+from datetime import datetime
+from itertools import chain
+import re
+
+from provisioningserver.dhcp.leases_parser import (
+    get_host_mac,
+    has_expired,
+    is_host,
+    is_lease,
+    lease_parser,
+    )
+
+
+re_entry = re.compile(
+    r'''
+    ^\s*              # Ignore leading whitespace on each line.
+    (host|lease)      # Look only for host or lease stanzas.
+    \s+               # Mandatory whitespace.
+    ([0-9a-fA-F.:]+)  # Capture the IP/MAC address for this stanza.
+    \s*{              # Optional whitespace then an opening brace.
+    ''',
+    re.MULTILINE | re.DOTALL | re.VERBOSE)
+
+
+def find_lease_starts(leases_contents):
+    results = defaultdict(dict)
+    for match in re_entry.finditer(leases_contents):
+        stanza, address = match.groups()
+        results[stanza][address] = match.start()
+    return chain.from_iterable(
+        mapping.itervalues() for mapping in results.itervalues())
+
+
+def extract_leases(leases_contents):
+    starts = find_lease_starts(leases_contents)
+    for start in sorted(starts):
+        record = lease_parser.scanString(leases_contents[start:])
+        try:
+            token, _, _ = next(record)
+        except StopIteration:
+            pass
+        else:
+            yield token
+
+
+def parse_leases(leases_contents):
+    results = {}
+    now = datetime.utcnow()
+    for entry in extract_leases(leases_contents):
+        if is_lease(entry):
+            if not has_expired(entry, now):
+                results[entry.ip] = entry.hardware.mac
+        elif is_host(entry):
+            mac = get_host_mac(entry)
+            if mac is None:
+                # TODO: Test this.
+                if entry.ip in results:
+                    del results[entry.ip]
+            else:
+                results[entry.ip] = mac
+    return results

=== modified file 'src/provisioningserver/dhcp/tests/test_leases_parser.py'
--- src/provisioningserver/dhcp/tests/test_leases_parser.py	2013-10-07 09:12:40 +0000
+++ src/provisioningserver/dhcp/tests/test_leases_parser.py	2014-05-19 15:16:18 +0000
@@ -20,6 +20,10 @@
 
 from maastesting.factory import factory
 from maastesting.testcase import MAASTestCase
+from provisioningserver.dhcp import (
+    leases_parser,
+    leases_parser_fast,
+    )
 from provisioningserver.dhcp.leases_parser import (
     combine_entries,
     gather_hosts,
@@ -30,197 +34,51 @@
     is_host,
     is_lease,
     lease_parser,
-    parse_leases,
-    )
-
-
-class TestLeasesParser(MAASTestCase):
-
-    def fake_parsed_lease(self, ip=None, mac=None, ends=None,
-                          entry_type='lease'):
-        """Fake a lease as produced by the parser."""
-        if ip is None:
-            ip = factory.getRandomIPAddress()
-        if mac is None:
-            mac = factory.getRandomMACAddress()
-        Hardware = namedtuple('Hardware', ['mac'])
-        Lease = namedtuple(
-            'Lease', ['lease_or_host', 'ip', 'hardware', 'ends'])
-        return Lease(entry_type, ip, Hardware(mac), ends)
-
-    def fake_parsed_host(self, ip=None, mac=None):
-        """Fake a host declaration as produced by the parser."""
-        return self.fake_parsed_lease(ip=ip, mac=mac, entry_type='host')
-
-    def fake_parsed_rubout(self, ip=None):
-        """Fake a "rubout" host declaration."""
-        if ip is None:
-            ip = factory.getRandomIPAddress()
-        Rubout = namedtuple('Rubout', ['lease_or_host', 'ip'])
-        return Rubout('host', ip)
-
-    def test_get_expiry_date_parses_expiry_date(self):
-        lease = self.fake_parsed_lease(ends='0 2011/01/02 03:04:05')
-        self.assertEqual(
-            datetime(
-                year=2011, month=01, day=02,
-                hour=03, minute=04, second=05),
-            get_expiry_date(lease))
-
-    def test_get_expiry_date_returns_None_for_never(self):
-        self.assertIsNone(
-            get_expiry_date(self.fake_parsed_lease(ends='never')))
-
-    def test_get_expiry_date_returns_None_if_no_expiry_given(self):
-        self.assertIsNone(get_expiry_date(self.fake_parsed_lease(ends=None)))
-
-    def test_has_expired_returns_False_for_eternal_lease(self):
-        now = datetime.utcnow()
-        self.assertFalse(has_expired(self.fake_parsed_lease(ends=None), now))
-
-    def test_has_expired_returns_False_for_future_expiry_date(self):
-        now = datetime.utcnow()
-        later = '1 2035/12/31 23:59:59'
-        self.assertFalse(has_expired(self.fake_parsed_lease(ends=later), now))
-
-    def test_has_expired_returns_True_for_past_expiry_date(self):
-        now = datetime.utcnow()
-        earlier = '1 2001/01/01 00:00:00'
-        self.assertTrue(
-            has_expired(self.fake_parsed_lease(ends=earlier), now))
-
-    def test_gather_leases_finds_current_leases(self):
-        lease = self.fake_parsed_lease()
-        self.assertEqual(
-            {lease.ip: lease.hardware.mac},
-            gather_leases([lease]))
-
-    def test_gather_leases_ignores_expired_leases(self):
-        earlier = '1 2001/01/01 00:00:00'
-        lease = self.fake_parsed_lease(ends=earlier)
-        self.assertEqual({}, gather_leases([lease]))
-
-    def test_gather_leases_combines_expired_and_current_leases(self):
-        earlier = '1 2001/01/01 00:00:00'
-        ip = factory.getRandomIPAddress()
-        old_owner = factory.getRandomMACAddress()
-        new_owner = factory.getRandomMACAddress()
-        leases = [
-            self.fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier),
-            self.fake_parsed_lease(ip=ip, mac=new_owner),
-            ]
-        self.assertEqual({ip: new_owner}, gather_leases(leases))
-
-    def test_gather_leases_ignores_ordering(self):
-        earlier = '1 2001/01/01 00:00:00'
-        ip = factory.getRandomIPAddress()
-        old_owner = factory.getRandomMACAddress()
-        new_owner = factory.getRandomMACAddress()
-        leases = [
-            self.fake_parsed_lease(ip=ip, mac=new_owner),
-            self.fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier),
-            ]
-        self.assertEqual({ip: new_owner}, gather_leases(leases))
-
-    def test_gather_leases_ignores_host_declarations(self):
-        self.assertEqual({}, gather_leases([self.fake_parsed_host()]))
-
-    def test_gather_hosts_finds_hosts(self):
-        host = self.fake_parsed_host()
-        self.assertEqual({host.ip: host.hardware.mac}, gather_hosts([host]))
-
-    def test_gather_hosts_ignores_unaccompanied_rubouts(self):
-        self.assertEqual({}, gather_hosts([self.fake_parsed_rubout()]))
-
-    def test_gather_hosts_ignores_rubbed_out_entries(self):
-        ip = factory.getRandomIPAddress()
-        hosts = [
-            self.fake_parsed_host(ip=ip),
-            self.fake_parsed_rubout(ip=ip),
-            ]
-        self.assertEqual({}, gather_hosts(hosts))
-
-    def test_gather_hosts_follows_reassigned_host(self):
-        ip = factory.getRandomIPAddress()
-        new_owner = factory.getRandomMACAddress()
-        hosts = [
-            self.fake_parsed_host(ip=ip),
-            self.fake_parsed_rubout(ip=ip),
-            self.fake_parsed_host(ip=ip, mac=new_owner),
-            ]
-        self.assertEqual({ip: new_owner}, gather_hosts(hosts))
-
-    def test_is_lease_and_is_host_recognize_lease(self):
-        params = {
-            'ip': factory.getRandomIPAddress(),
-            'mac': factory.getRandomMACAddress(),
-        }
-        [parsed_lease] = lease_parser.searchString(dedent("""\
-            lease %(ip)s {
-                hardware ethernet %(mac)s;
-            }
-            """ % params))
-        self.assertEqual(
-            (True, False),
-            (is_lease(parsed_lease), is_host(parsed_lease)))
-
-    def test_is_lease_and_is_host_recognize_host(self):
-        params = {
-            'ip': factory.getRandomIPAddress(),
-            'mac': factory.getRandomMACAddress(),
-        }
-        [parsed_host] = lease_parser.searchString(dedent("""\
-            host %(ip)s {
-                hardware ethernet %(mac)s;
-            }
-            """ % params))
-        self.assertEqual(
-            (False, True),
-            (is_lease(parsed_host), is_host(parsed_host)))
-
-    def test_get_host_mac_returns_None_for_host(self):
-        params = {
-            'ip': factory.getRandomIPAddress(),
-            'mac': factory.getRandomMACAddress(),
-        }
-        [parsed_host] = lease_parser.searchString(dedent("""\
-            host %(ip)s {
-                hardware ethernet %(mac)s;
-            }
-            """ % params))
-        self.assertEqual(params['mac'], get_host_mac(parsed_host))
-
-    def test_get_host_mac_returns_None_for_rubout(self):
-        ip = factory.getRandomIPAddress()
-        [parsed_host] = lease_parser.searchString(dedent("""\
-            host %s {
-                deleted;
-            }
-            """ % ip))
-        self.assertIsNone(get_host_mac(parsed_host))
-
-    def test_get_host_mac_returns_None_for_rubout_even_with_mac(self):
-        params = {
-            'ip': factory.getRandomIPAddress(),
-            'mac': factory.getRandomMACAddress(),
-        }
-        [parsed_host] = lease_parser.searchString(dedent("""\
-            host %(ip)s {
-                deleted;
-                hardware ethernet %(mac)s;
-            }
-            """ % params))
-        self.assertIsNone(get_host_mac(parsed_host))
+    )
+
+
+def fake_parsed_lease(ip=None, mac=None, ends=None,
+                      entry_type='lease'):
+    """Fake a lease as produced by the parser."""
+    if ip is None:
+        ip = factory.getRandomIPAddress()
+    if mac is None:
+        mac = factory.getRandomMACAddress()
+    Hardware = namedtuple('Hardware', ['mac'])
+    Lease = namedtuple(
+        'Lease', ['lease_or_host', 'ip', 'hardware', 'ends'])
+    return Lease(entry_type, ip, Hardware(mac), ends)
+
+
+def fake_parsed_host(ip=None, mac=None):
+    """Fake a host declaration as produced by the parser."""
+    return fake_parsed_lease(ip=ip, mac=mac, entry_type='host')
+
+
+def fake_parsed_rubout(ip=None):
+    """Fake a "rubout" host declaration."""
+    if ip is None:
+        ip = factory.getRandomIPAddress()
+    Rubout = namedtuple('Rubout', ['lease_or_host', 'ip'])
+    return Rubout('host', ip)
+
+
+class TestLeasesParsers(MAASTestCase):
+
+    scenarios = (
+        ("original", dict(parse=leases_parser.parse_leases)),
+        ("fast", dict(parse=leases_parser_fast.parse_leases)),
+    )
 
     def test_parse_leases_copes_with_empty_file(self):
-        self.assertEqual({}, parse_leases(""))
+        self.assertEqual({}, self.parse(""))
 
     def test_parse_leases_parses_lease(self):
         params = {
             'ip': factory.getRandomIPAddress(),
             'mac': factory.getRandomMACAddress(),
         }
-        leases = parse_leases(dedent("""\
+        leases = self.parse(dedent("""\
             lease %(ip)s {
                 starts 5 2010/01/01 00:00:01;
                 ends never;
@@ -254,7 +112,7 @@
             'ip': factory.getRandomIPAddress(),
             'mac': factory.getRandomMACAddress(),
         }
-        leases = parse_leases(dedent("""\
+        leases = self.parse(dedent("""\
             host %(ip)s {
                 dynamic;
                 hardware ethernet %(mac)s;
@@ -263,8 +121,36 @@
             """ % params))
         self.assertEqual({params['ip']: params['mac']}, leases)
 
+    def test_parse_leases_copes_with_misleading_values(self):
+        params = {
+            'ip1': factory.getRandomIPAddress(),
+            'mac1': factory.getRandomMACAddress(),
+            'ip2': factory.getRandomIPAddress(),
+            'mac2': factory.getRandomMACAddress(),
+        }
+        leases = self.parse(dedent("""\
+            host %(ip1)s {
+                dynamic;
+              ### NOTE the following value has a closing brace, and
+              ### also looks like a host record.
+                uid "foo}host 12.34.56.78 { }";
+                hardware ethernet %(mac1)s;
+                fixed-address %(ip1)s;
+            }
+              ### NOTE the extra indent on the line below.
+                host %(ip2)s {
+                dynamic;
+                hardware ethernet %(mac2)s;
+                fixed-address %(ip2)s;
+            }
+            """ % params))
+        self.assertEqual(
+            {params['ip1']: params['mac1'],
+             params['ip2']: params['mac2']},
+            leases)
+
     def test_parse_leases_parses_host_rubout(self):
-        leases = parse_leases(dedent("""\
+        leases = self.parse(dedent("""\
             host %s {
                 deleted;
             }
@@ -277,7 +163,7 @@
             'mac': factory.getRandomMACAddress(),
             'incomplete_ip': factory.getRandomIPAddress(),
         }
-        leases = parse_leases(dedent("""\
+        leases = self.parse(dedent("""\
             lease %(ip)s {
                 hardware ethernet %(mac)s;
             }
@@ -291,7 +177,7 @@
             'ip': factory.getRandomIPAddress(),
             'mac': factory.getRandomMACAddress(),
         }
-        leases = parse_leases(dedent("""\
+        leases = self.parse(dedent("""\
             # Top comment (ignored).
             lease %(ip)s { # End-of-line comment (ignored).
                 # Comment in lease block (ignored).
@@ -306,7 +192,7 @@
             'ip': factory.getRandomIPAddress(),
             'mac': factory.getRandomMACAddress(),
         }
-        leases = parse_leases(dedent("""\
+        leases = self.parse(dedent("""\
             lease %(ip)s {
                 hardware ethernet %(mac)s;
                 ends 1 2001/01/01 00:00:00;
@@ -319,7 +205,7 @@
             'ip': factory.getRandomIPAddress(),
             'mac': factory.getRandomMACAddress(),
         }
-        leases = parse_leases(dedent("""\
+        leases = self.parse(dedent("""\
             lease %(ip)s {
                 hardware ethernet %(mac)s;
                 ends never;
@@ -332,7 +218,7 @@
             'ip': factory.getRandomIPAddress(),
             'mac': factory.getRandomMACAddress(),
         }
-        leases = parse_leases(dedent("""\
+        leases = self.parse(dedent("""\
             lease %(ip)s {
                 hardware ethernet %(mac)s;
             }
@@ -345,7 +231,7 @@
             'old_owner': factory.getRandomMACAddress(),
             'new_owner': factory.getRandomMACAddress(),
         }
-        leases = parse_leases(dedent("""\
+        leases = self.parse(dedent("""\
             lease %(ip)s {
                 hardware ethernet %(old_owner)s;
             }
@@ -360,7 +246,7 @@
             'ip': factory.getRandomIPAddress(),
             'mac': factory.getRandomMACAddress(),
         }
-        leases = parse_leases(dedent("""\
+        leases = self.parse(dedent("""\
             host %(ip)s {
                 dynamic;
                 hardware ethernet %(mac)s;
@@ -375,7 +261,7 @@
             'ip': factory.getRandomIPAddress(),
             'mac': factory.getRandomMACAddress(),
         }
-        leases = parse_leases(dedent("""\
+        leases = self.parse(dedent("""\
             host %(ip)s {
                 hardware ethernet %(mac)s;
                 fixed-address %(ip)s;
@@ -383,13 +269,235 @@
             """ % params))
         self.assertEqual({params['ip']: params['mac']}, leases)
 
+
+class TestLeasesParserFast(MAASTestCase):
+
+    def test_expired_lease_does_not_shadow_earlier_host_stanza(self):
+        params = {
+            'ip': factory.getRandomIPAddress(),
+            'mac1': factory.getRandomMACAddress(),
+            'mac2': factory.getRandomMACAddress(),
+        }
+        leases = leases_parser_fast.parse_leases(dedent("""\
+            host %(ip)s {
+                dynamic;
+                hardware ethernet %(mac1)s;
+                fixed-address %(ip)s;
+            }
+            lease %(ip)s {
+                starts 5 2010/01/01 00:00:01;
+                ends 1 2010/01/01 00:00:02;
+                hardware ethernet %(mac2)s;
+            }
+            """ % params))
+        # The lease has expired so it doesn't shadow the host stanza,
+        # and so the MAC returned is from the host stanza.
+        self.assertEqual({params["ip"]: params["mac1"]}, leases)
+
+    def test_active_lease_shadows_earlier_host_stanza(self):
+        params = {
+            'ip': factory.getRandomIPAddress(),
+            'mac1': factory.getRandomMACAddress(),
+            'mac2': factory.getRandomMACAddress(),
+        }
+        leases = leases_parser_fast.parse_leases(dedent("""\
+            host %(ip)s {
+                dynamic;
+                hardware ethernet %(mac1)s;
+                fixed-address %(ip)s;
+            }
+            lease %(ip)s {
+                starts 5 2010/01/01 00:00:01;
+                hardware ethernet %(mac2)s;
+            }
+            """ % params))
+        # The lease hasn't expired, so shadows the earlier host stanza.
+        self.assertEqual({params["ip"]: params["mac2"]}, leases)
+
+    def test_host_stanza_shadows_earlier_active_lease(self):
+        params = {
+            'ip': factory.getRandomIPAddress(),
+            'mac1': factory.getRandomMACAddress(),
+            'mac2': factory.getRandomMACAddress(),
+        }
+        leases = leases_parser_fast.parse_leases(dedent("""\
+            lease %(ip)s {
+                starts 5 2010/01/01 00:00:01;
+                hardware ethernet %(mac2)s;
+            }
+            host %(ip)s {
+                dynamic;
+                hardware ethernet %(mac1)s;
+                fixed-address %(ip)s;
+            }
+            """ % params))
+        # The lease hasn't expired, but the host entry is later, so it
+        # shadows the earlier lease stanza.
+        self.assertEqual({params["ip"]: params["mac1"]}, leases)
+
+
+class TestLeasesParserFunctions(MAASTestCase):
+
+    def test_get_expiry_date_parses_expiry_date(self):
+        lease = fake_parsed_lease(ends='0 2011/01/02 03:04:05')
+        self.assertEqual(
+            datetime(
+                year=2011, month=01, day=02,
+                hour=03, minute=04, second=05),
+            get_expiry_date(lease))
+
+    def test_get_expiry_date_returns_None_for_never(self):
+        self.assertIsNone(
+            get_expiry_date(fake_parsed_lease(ends='never')))
+
+    def test_get_expiry_date_returns_None_if_no_expiry_given(self):
+        self.assertIsNone(get_expiry_date(fake_parsed_lease(ends=None)))
+
+    def test_has_expired_returns_False_for_eternal_lease(self):
+        now = datetime.utcnow()
+        self.assertFalse(has_expired(fake_parsed_lease(ends=None), now))
+
+    def test_has_expired_returns_False_for_future_expiry_date(self):
+        now = datetime.utcnow()
+        later = '1 2035/12/31 23:59:59'
+        self.assertFalse(has_expired(fake_parsed_lease(ends=later), now))
+
+    def test_has_expired_returns_True_for_past_expiry_date(self):
+        now = datetime.utcnow()
+        earlier = '1 2001/01/01 00:00:00'
+        self.assertTrue(
+            has_expired(fake_parsed_lease(ends=earlier), now))
+
+    def test_gather_leases_finds_current_leases(self):
+        lease = fake_parsed_lease()
+        self.assertEqual(
+            {lease.ip: lease.hardware.mac},
+            gather_leases([lease]))
+
+    def test_gather_leases_ignores_expired_leases(self):
+        earlier = '1 2001/01/01 00:00:00'
+        lease = fake_parsed_lease(ends=earlier)
+        self.assertEqual({}, gather_leases([lease]))
+
+    def test_gather_leases_combines_expired_and_current_leases(self):
+        earlier = '1 2001/01/01 00:00:00'
+        ip = factory.getRandomIPAddress()
+        old_owner = factory.getRandomMACAddress()
+        new_owner = factory.getRandomMACAddress()
+        leases = [
+            fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier),
+            fake_parsed_lease(ip=ip, mac=new_owner),
+            ]
+        self.assertEqual({ip: new_owner}, gather_leases(leases))
+
+    def test_gather_leases_ignores_ordering(self):
+        earlier = '1 2001/01/01 00:00:00'
+        ip = factory.getRandomIPAddress()
+        old_owner = factory.getRandomMACAddress()
+        new_owner = factory.getRandomMACAddress()
+        leases = [
+            fake_parsed_lease(ip=ip, mac=new_owner),
+            fake_parsed_lease(ip=ip, mac=old_owner, ends=earlier),
+            ]
+        self.assertEqual({ip: new_owner}, gather_leases(leases))
+
+    def test_gather_leases_ignores_host_declarations(self):
+        self.assertEqual({}, gather_leases([fake_parsed_host()]))
+
+    def test_gather_hosts_finds_hosts(self):
+        host = fake_parsed_host()
+        self.assertEqual({host.ip: host.hardware.mac}, gather_hosts([host]))
+
+    def test_gather_hosts_ignores_unaccompanied_rubouts(self):
+        self.assertEqual({}, gather_hosts([fake_parsed_rubout()]))
+
+    def test_gather_hosts_ignores_rubbed_out_entries(self):
+        ip = factory.getRandomIPAddress()
+        hosts = [
+            fake_parsed_host(ip=ip),
+            fake_parsed_rubout(ip=ip),
+            ]
+        self.assertEqual({}, gather_hosts(hosts))
+
+    def test_gather_hosts_follows_reassigned_host(self):
+        ip = factory.getRandomIPAddress()
+        new_owner = factory.getRandomMACAddress()
+        hosts = [
+            fake_parsed_host(ip=ip),
+            fake_parsed_rubout(ip=ip),
+            fake_parsed_host(ip=ip, mac=new_owner),
+            ]
+        self.assertEqual({ip: new_owner}, gather_hosts(hosts))
+
+    def test_is_lease_and_is_host_recognize_lease(self):
+        params = {
+            'ip': factory.getRandomIPAddress(),
+            'mac': factory.getRandomMACAddress(),
+        }
+        [parsed_lease] = lease_parser.searchString(dedent("""\
+            lease %(ip)s {
+                hardware ethernet %(mac)s;
+            }
+            """ % params))
+        self.assertEqual(
+            (True, False),
+            (is_lease(parsed_lease), is_host(parsed_lease)))
+
+    def test_is_lease_and_is_host_recognize_host(self):
+        params = {
+            'ip': factory.getRandomIPAddress(),
+            'mac': factory.getRandomMACAddress(),
+        }
+        [parsed_host] = lease_parser.searchString(dedent("""\
+            host %(ip)s {
+                hardware ethernet %(mac)s;
+            }
+            """ % params))
+        self.assertEqual(
+            (False, True),
+            (is_lease(parsed_host), is_host(parsed_host)))
+
+    def test_get_host_mac_returns_None_for_host(self):
+        params = {
+            'ip': factory.getRandomIPAddress(),
+            'mac': factory.getRandomMACAddress(),
+        }
+        [parsed_host] = lease_parser.searchString(dedent("""\
+            host %(ip)s {
+                hardware ethernet %(mac)s;
+            }
+            """ % params))
+        self.assertEqual(params['mac'], get_host_mac(parsed_host))
+
+    def test_get_host_mac_returns_None_for_rubout(self):
+        ip = factory.getRandomIPAddress()
+        [parsed_host] = lease_parser.searchString(dedent("""\
+            host %s {
+                deleted;
+            }
+            """ % ip))
+        self.assertIsNone(get_host_mac(parsed_host))
+
+    def test_get_host_mac_returns_None_for_rubout_even_with_mac(self):
+        params = {
+            'ip': factory.getRandomIPAddress(),
+            'mac': factory.getRandomMACAddress(),
+        }
+        [parsed_host] = lease_parser.searchString(dedent("""\
+            host %(ip)s {
+                deleted;
+                hardware ethernet %(mac)s;
+            }
+            """ % params))
+        self.assertIsNone(get_host_mac(parsed_host))
+
     def test_combine_entries_accepts_host_followed_by_expired_lease(self):
         ip = factory.getRandomIPAddress()
         mac = factory.getRandomMACAddress()
         earlier = '1 2001/01/01 00:00:00'
         entries = [
-            self.fake_parsed_host(ip=ip, mac=mac),
-            self.fake_parsed_lease(ip=ip, ends=earlier),
+            fake_parsed_host(ip=ip, mac=mac),
+            fake_parsed_lease(ip=ip, ends=earlier),
             ]
         self.assertEqual({ip: mac}, combine_entries(entries))
 
@@ -398,8 +506,8 @@
         mac = factory.getRandomMACAddress()
         earlier = '1 2001/01/01 00:00:00'
         entries = [
-            self.fake_parsed_lease(ip=ip, ends=earlier),
-            self.fake_parsed_host(ip=ip, mac=mac),
+            fake_parsed_lease(ip=ip, ends=earlier),
+            fake_parsed_host(ip=ip, mac=mac),
             ]
         self.assertEqual({ip: mac}, combine_entries(entries))
 
@@ -407,9 +515,9 @@
         ip = factory.getRandomIPAddress()
         mac = factory.getRandomMACAddress()
         entries = [
-            self.fake_parsed_host(ip=ip),
-            self.fake_parsed_rubout(ip=ip),
-            self.fake_parsed_lease(ip=ip, mac=mac),
+            fake_parsed_host(ip=ip),
+            fake_parsed_rubout(ip=ip),
+            fake_parsed_lease(ip=ip, mac=mac),
             ]
         self.assertEqual({ip: mac}, combine_entries(entries))
 
@@ -418,9 +526,9 @@
         mac = factory.getRandomMACAddress()
         earlier = '1 2001/01/01 00:00:00'
         entries = [
-            self.fake_parsed_host(ip=ip),
-            self.fake_parsed_rubout(ip=ip),
-            self.fake_parsed_lease(ip=ip, mac=mac, ends=earlier),
+            fake_parsed_host(ip=ip),
+            fake_parsed_rubout(ip=ip),
+            fake_parsed_lease(ip=ip, mac=mac, ends=earlier),
             ]
         self.assertEqual({}, combine_entries(entries))
 
@@ -429,9 +537,9 @@
         mac = factory.getRandomMACAddress()
         earlier = '1 2001/01/01 00:00:00'
         entries = [
-            self.fake_parsed_host(ip=ip),
-            self.fake_parsed_lease(ip=ip, mac=mac, ends=earlier),
-            self.fake_parsed_rubout(ip=ip),
+            fake_parsed_host(ip=ip),
+            fake_parsed_lease(ip=ip, mac=mac, ends=earlier),
+            fake_parsed_rubout(ip=ip),
             ]
         self.assertEqual({}, combine_entries(entries))
 
@@ -439,9 +547,9 @@
         ip = factory.getRandomIPAddress()
         mac = factory.getRandomMACAddress()
         entries = [
-            self.fake_parsed_host(ip=ip),
-            self.fake_parsed_lease(ip=ip, mac=mac),
-            self.fake_parsed_rubout(ip=ip),
+            fake_parsed_host(ip=ip),
+            fake_parsed_lease(ip=ip, mac=mac),
+            fake_parsed_rubout(ip=ip),
             ]
         self.assertEqual({ip: mac}, combine_entries(entries))
 
@@ -449,8 +557,8 @@
         ip = factory.getRandomIPAddress()
         mac = factory.getRandomMACAddress()
         entries = [
-            self.fake_parsed_host(ip=ip),
-            self.fake_parsed_rubout(ip=ip),
-            self.fake_parsed_host(ip=ip, mac=mac),
+            fake_parsed_host(ip=ip),
+            fake_parsed_rubout(ip=ip),
+            fake_parsed_host(ip=ip, mac=mac),
             ]
         self.assertEqual({ip: mac}, combine_entries(entries))