Merge lp:~ptman/hipl/cleanup into lp:hipl

[Paul Tötterman]

This branch consists of various polishing and cleanup commits.

- Fix (retry and backoff) for a race condition
- Fix logging to syslog
- Fix upstream DNS server change when moving from one network to another
- Source format improvement as reported by pylint and pep8
- Split huge mainloop into smaller, testable pieces and added tests

[lp:~ptman/hipl/cleanup]

6441. By Paul Tötterman on 2013-10-10

Fix several crashes when hipconf daemon cannot reach hipd

[Miika Komu (miika-iki)]

Nice work! I have been testing this and it seems to work fine especially after the last fix in 6441.

review: Approve

[lp:~ptman/hipl/cleanup]

6442. By Paul Tötterman on 2013-10-11

Merged trunk

=== modified file 'debian/hipl-dnsproxy.init'
--- debian/hipl-dnsproxy.init	2013-03-06 14:07:36 +0000
+++ debian/hipl-dnsproxy.init	2013-10-11 15:24:31 +0000
@@ -9,7 +9,7 @@
 # Short-Description:    HIP DNS proxy
 ### END INIT INFO
 
-DNSPROXY_OPTS='-k'
+DNSPROXY_OPTS='-kS'
 PID_FILE=/var/run/hipdnsproxy.pid
 EXEC=/usr/sbin/hipdnsproxy
 

=== modified file 'tools/hipdnsproxy/dnsproxy.py'
--- tools/hipdnsproxy/dnsproxy.py	2013-07-11 15:08:01 +0000
+++ tools/hipdnsproxy/dnsproxy.py	2013-10-11 15:24:31 +0000
@@ -67,6 +67,7 @@
 import logging
 import logging.handlers
 import os
+import pprint
 import re
 import select
 import signal
@@ -79,10 +80,10 @@
 
 # prepending (instead of appending) to make sure hosts.py does not
 # collide with the system default
+import DNS
 import hosts
 import resolvconf
 import util
-from DNS import Serialize, DeSerialize, Type
 
 
 DEFAULT_HOSTS = '/etc/hosts'
@@ -94,6 +95,7 @@
     sys.stderr.write('Usage: %s\n' % os.path.split(sys.argv[0])[1])
     if msg:
         sys.stderr.write('Error: %r\n' % msg)
+
     sys.exit(1)
 
 
@@ -103,44 +105,58 @@
 def add_hit_ip_map(hit, addr):
     """Add IP for HIT."""
     logging.info('Associating HIT %s with IP %s', hit, addr)
-    subprocess.check_call(['hipconf', 'daemon', 'add', 'map', hit, addr],
-                          stdout=open(os.devnull, 'w'),
-                          stderr=subprocess.STDOUT)
+    try:
+        subprocess.check_call(['hipconf', 'daemon', 'add', 'map', hit, addr],
+                              stdout=open(os.devnull, 'w'),
+                              stderr=subprocess.STDOUT)
+    except subprocess.CalledProcessError:
+        logging.error('Got error from `hipconf daemon ...`. Is hipd up?')
 
 
 def hit_to_lsi(hit):
     """Return LSI for HIT if found."""
-    output = subprocess.Popen(['hipconf', 'daemon', 'hit-to-lsi', hit],
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.STDOUT).stdout
-
-    for line in output:
-        match = LSI_RE.search(line)
-        if match:
-            return match.group('lsi')
+    proc = subprocess.Popen(['hipconf', 'daemon', 'hit-to-lsi', hit],
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+
+    if proc.returncode == 254:
+        logging.error('Cannot contact hipd. Is it running?')
+        return
+
+    output = proc.stdout
+
+    try:
+        for line in output:
+            match = LSI_RE.search(line)
+            if match:
+                return match.group('lsi')
+    except IOError:
+        logging.error('Cannot read from `hipconf daemon ...`. Is hipd up?')
 
 
 def lsi_to_hit(lsi):
     """Return HIT for LSI if found."""
-    output = subprocess.Popen(['hipconf', 'daemon', 'lsi-to-hit', lsi],
-                              stdout=subprocess.PIPE,
-                              stderr=subprocess.STDOUT).stdout
-
-    for line in output:
-        match = hosts.HIT_RE.search(line)
-        if match:
-            return match.group('hit')
+    proc = subprocess.Popen(['hipconf', 'daemon', 'lsi-to-hit', lsi],
+                            stdout=subprocess.PIPE,
+                            stderr=subprocess.STDOUT)
+
+    if proc.returncode == 254:
+        logging.error('Cannot contact hipd. Is it running?')
+        return
+
+    output = proc.stdout
+
+    try:
+        for line in output:
+            match = hosts.HIT_RE.search(line)
+            if match:
+                return match.group('hit')
+    except IOError:
+        logging.error('Cannot read from `hipconf daemon ...`. Is hipd up?')
 
 
 def is_reverse_hit_query(name):
-    """Check if the query is a reverse query to a HIT.
-
-    >>> is_reverse_hit_query('::1')
-    False
-    >>> is_reverse_hit_query('8.e.b.8.b.3.c.9.1.a.0.c.e.e.2.c.c.e.d.0.9.c.'
-    ...                      '9.a.e.1.0.0.1.0.0.2.hit-to-ip.infrahip.net')
-    True
-    """
+    """Check if the query is a reverse query to a HIT."""
     if (name.endswith('.1.0.0.1.0.0.2.hit-to-ip.infrahip.net') and
         len(name) == 86):
         return True
@@ -156,6 +172,7 @@
                  pidfile=None, prefix=None, server_ip=None, server_port=None):
         self.bind_ip = bind_ip
         self.bind_port = bind_port
+        self.connected = False
         self.disable_lsi = disable_lsi
         self.dns_timeout = dns_timeout
         self.fork = fork
@@ -171,10 +188,13 @@
         if self.hostsnames is None:
             self.hostsnames = []
 
+        self.clisock = None
+        self.servsock = None
         self.app_timeout = 1
         self.hosts_ttl = 122
         self.sent_queue = []
         self.hosts = None
+        self.query_id = 1
         # Keyed by ('server_ip',server_port,query_id) tuple
         self.sent_queue_d = {}
         # required for ifconfig and hipconf in Fedora
@@ -217,20 +237,26 @@
         env = os.environ
         if self.server_ip is None:
             self.server_ip = env.get('SERVER', None)
+
         if self.server_port is None:
             server_port = env.get('SERVERPORT', None)
             if server_port is not None:
                 self.server_port = int(server_port)
+
         if self.server_port is None:
             self.server_port = 53
+
         if self.bind_ip is None:
             self.bind_ip = env.get('IP', None)
+
         if self.bind_ip is None:
             self.bind_ip = '127.0.0.53'
+
         if self.bind_port is None:
             bind_port = env.get('PORT', None)
             if bind_port is not None:
                 self.bind_port = int(bind_port)
+
         if self.bind_port is None:
             self.bind_port = 53
 
@@ -238,7 +264,6 @@
         """Recheck all hosts files."""
         for hostsdb in self.hosts:
             hostsdb.recheck()
-        return
 
     def getaddr(self, ahn):
         """Get a hostname matching address."""
@@ -273,7 +298,6 @@
             if result:
                 return result
 
-
     def killold(self):
         """Kill process with PID from pidfile."""
         try:
@@ -284,6 +308,7 @@
             else:
                 logging.error('Error opening pid file: %s', ioe)
                 sys.exit(1)
+
         try:
             os.kill(int(ifile.readline().rstrip()), signal.SIGTERM)
         except OSError, ose:
@@ -293,6 +318,7 @@
             else:
                 logging.error('Error terminating old process: %s', ose)
                 sys.exit(1)
+
         time.sleep(3)
         ifile.close()
 
@@ -306,6 +332,7 @@
             else:
                 logging.error('Error opening pid file: %s', ioe)
                 sys.exit(1)
+
         ifile.readline()
         global MYID
         MYID = ifile.readline().rstrip()
@@ -319,6 +346,7 @@
         except IOError, ioe:
             logging.error('Error opening pid file for writing: %s', ioe)
             sys.exit(1)
+
         global MYID
         MYID = '%d-%d' % (time.time(), os.getpid())
         ofile.write('%d\n' % (os.getpid(),))
@@ -338,19 +366,22 @@
         proc = subprocess.Popen(['ifconfig', 'dummy0'],
                                 stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT).stdout
-        result = proc.readline()
-        while result:
-            start = result.find('2001:1')
-            end = result.find('/28')
+        line = proc.readline()
+        while line:
+            start = line.find('2001:1')
+            end = line.find('/28')
             if start != -1 and end != -1:
-                hit = result[start:end]
+                hit = line[start:end]
                 if not self.getaddr(hit):
                     localhit.append(hit)
-            result = proc.readline()
+
+            line = proc.readline()
+
         proc.close()
         ofile = open(self.hiphosts, 'a')
         for i in range(len(localhit)):
             ofile.write('%s\tlocalhit%s\n' % (localhit[i], i + 1))
+
         ofile.close()
 
     def hip_cache_lookup(self, packet):
@@ -364,12 +395,13 @@
 
         # convert 1.2....1.0.0.1.0.0.2.ip6.arpa to a HIT and
         # map host name to address from cache
-        if qtype == Type.PTR:
+        if qtype == DNS.Type.PTR:
             lr_ptr = None
             addr_str = hosts.ptr_to_addr(qname)
             if (not self.disable_lsi and addr_str is not None and
                 hosts.valid_lsi(addr_str)):
                 addr_str = lsi_to_hit(addr_str)
+
             lr_ptr = self.getaddr(addr_str)
             lr_aaaa_hit = None
         else:
@@ -382,26 +414,29 @@
              packet['questions'][0][0].startswith(self.prefix))):
             if lr_a is not None:
                 add_hit_ip_map(lr_aaaa_hit[0], lr_a[0])
+
             if lr_aaaa is not None:
                 add_hit_ip_map(lr_aaaa_hit[0], lr_aaaa[0])
-            if qtype == Type.AAAA:
+
+            if qtype == DNS.Type.AAAA:
                 result = lr_aaaa_hit
-            elif qtype == Type.A and not self.disable_lsi:
+            elif qtype == DNS.Type.A and not self.disable_lsi:
                 lsi = hit_to_lsi(lr_aaaa_hit[0])
                 if lsi is not None:
                     result = (lsi, lr_aaaa_hit[1])
+
         elif self.prefix and packet['questions'][0][0].startswith(self.prefix):
             result = None
-        elif qtype == Type.AAAA:
+        elif qtype == DNS.Type.AAAA:
             result = lr_aaaa
-        elif qtype == Type.A:
+        elif qtype == DNS.Type.A:
             result = lr_a
-        elif qtype == Type.PTR and lr_ptr is not None:
+        elif qtype == DNS.Type.PTR and lr_ptr is not None:
             result = (lr_ptr, self.hosts_ttl)
 
         if result is not None:
             packet['answers'].append([packet['questions'][0][0], qtype, 1,
-                                     result[1], result[0]])
+                                      result[1], result[0]])
             packet['ancount'] = len(packet['answers'])
             packet['qr'] = 1
             return True
@@ -415,7 +450,7 @@
 
         dns_hit_found = False
         for answer in packet['answers']:
-            if answer[1] == Type.HIP:
+            if answer[1] == DNS.Type.HIP:
                 dns_hit_found = True
                 break
 
@@ -426,20 +461,20 @@
             lsi_ans = []
 
             for answer in packet['answers']:
-                if answer[1] != Type.HIP:
+                if answer[1] != DNS.Type.HIP:
                     continue
 
                 hit = socket.inet_ntop(socket.AF_INET6, answer[7])
-                hit_ans.append([qname, Type.AAAA, 1, answer[3], hit])
+                hit_ans.append([qname, DNS.Type.AAAA, 1, answer[3], hit])
 
-                if qtype == Type.A and not self.disable_lsi:
+                if qtype == DNS.Type.A and not self.disable_lsi:
                     lsi = hit_to_lsi(hit)
                     if lsi is not None:
                         lsi_ans.append([qname, 1, 1, self.hosts_ttl, lsi])
 
                 self.cache_name(qname, hit, answer[3])
 
-        if qtype == Type.AAAA and hit_found:
+        if qtype == DNS.Type.AAAA and hit_found:
             packet['answers'] = hit_ans
         elif lsi is not None:
             packet['answers'] = lsi_ans
@@ -448,10 +483,211 @@
 
         packet['ancount'] = len(packet['answers'])
 
+    def handle_query(self, packet, sender):
+        """Handle DNS query from downstream client."""
+        qtype = packet['questions'][0][1]
+
+        sent_answer = False
+
+        if qtype in (DNS.Type.A, DNS.Type.AAAA, DNS.Type.PTR):
+            if self.hip_cache_lookup(packet):
+                try:
+                    outbuf = DNS.Serialize(packet).get_packet()
+                    self.servsock.sendto(outbuf, sender)
+                    sent_answer = True
+                except socket.error:
+                    logging.exception('Exception:')
+
+        elif (self.prefix and
+              packet['questions'][0][0].startswith(
+                  self.prefix)):
+            # Query with HIP prefix for unsupported RR type.
+            # Send empty response.
+            packet['qr'] = 1
+            try:
+                outbuf = DNS.Serialize(packet).get_packet()
+                self.servsock.sendto(outbuf, sender)
+                sent_answer = True
+            except socket.error:
+                logging.exception('Exception:')
+
+        if self.connected and not sent_answer:
+            logging.info('Query type %d for %s from %s',
+                         qtype, packet['questions'][0][0],
+                         (self.server_ip, self.server_port))
+
+            query = (packet, sender[0], sender[1], qtype)
+            # FIXME: Should randomize for security
+            self.query_id = (self.query_id % 65535) + 1
+            pckt = copy.copy(packet)
+            pckt['id'] = self.query_id
+            if ((qtype == DNS.Type.AAAA or
+                 (qtype == DNS.Type.A and
+                  not self.disable_lsi)) and
+                not is_reverse_hit_query(
+                    packet['questions'][0][0])):
+
+                if not self.prefix:
+                    pckt['questions'][0][1] = DNS.Type.HIP
+
+                if (self.prefix and
+                    pckt['questions'][0][0].startswith(
+                        self.prefix)):
+                    pckt['questions'][0][0] = pckt[
+                        'questions'][0][0][len(self.prefix):]
+                    pckt['questions'][0][1] = DNS.Type.HIP
+
+            if qtype == DNS.Type.PTR and not self.disable_lsi:
+                qname = packet['questions'][0][0]
+                addr_str = hosts.ptr_to_addr(qname)
+                if (addr_str is not None and
+                    hosts.valid_lsi(addr_str)):
+                    query = (packet, sender[0], sender[1],
+                             qname)
+                    hit_str = lsi_to_hit(addr_str)
+                    if hit_str is not None:
+                        pckt['questions'][0][0] = hosts.addr_to_ptr(hit_str)
+
+            outbuf = DNS.Serialize(pckt).get_packet()
+            self.clisock.sendto(outbuf, (self.server_ip,
+                                         self.server_port))
+
+            self.add_query(self.server_ip, self.server_port,
+                           self.query_id, query)
+
+    def handle_response(self, packet, sender):
+        """Handle DNS response from upstream server."""
+        if packet['qdcount'] == 0:
+            logging.warn('Bad response from upstream server: %s',
+                         pprint.pformat(packet))
+            return
+
+        # Find original query
+        query_id_o = packet['id']
+        query_o = self.find_query(sender[0], sender[1],
+                                  query_id_o)
+        if query_o and packet['qdcount'] > 0:
+            qname = packet['questions'][0][0]
+            qtype = packet['questions'][0][1]
+            send_reply = True
+            query_again = False
+            hit_found = False
+            packet_o = query_o[0]
+            # Replace with the original query id
+            packet['id'] = packet_o['id']
+
+            if qtype == DNS.Type.HIP and query_o[3] in (DNS.Type.A,
+                                                        DNS.Type.AAAA):
+                # Restore qtype
+                packet['questions'][0][1] = query_o[3]
+                self.hip_lookup(packet)
+                if packet['ancount'] > 0:
+                    hit_found = True
+
+                if (not self.prefix or
+                    (hit_found and not (self.getaaaa(qname) or
+                                        self.geta(qname)))):
+                    query_again = True
+                    send_reply = False
+                elif self.prefix:
+                    hit_found = True
+                    packet['questions'][0][0] = (
+                        self.prefix + packet['questions'][0][0])
+                    for answer in packet['answers']:
+                        answer[0] = self.prefix + answer[0]
+
+            elif qtype in (DNS.Type.A, DNS.Type.AAAA):
+                hit = self.getaaaa_hit(qname)
+                ip6 = self.getaaaa(qname)
+                ip4 = self.geta(qname)
+                for answer in packet['answers']:
+                    if answer[1] in (DNS.Type.A, DNS.Type.AAAA):
+                        self.cache_name(qname, answer[4],
+                                        answer[3])
+
+                if hit is not None:
+                    for answer in packet['answers']:
+                        if (answer[1] == DNS.Type.A or
+                            (answer[1] == DNS.Type.AAAA and not
+                             hosts.valid_hit(answer[4]))):
+                            add_hit_ip_map(hit[0], answer[4])
+
+                    # Reply with HIT/LSI once it's been mapped
+                    # to an IP
+                    if ip6 is None and ip4 is None:
+                        if (packet_o['ancount'] == 0 and
+                            not self.prefix):
+                            # No LSI available. Return IPv4
+                            tmp = packet['answers']
+                            packet = packet_o
+                            packet['answers'] = tmp
+                            packet['ancount'] = len(
+                                packet['answers'])
+                        else:
+                            packet = packet_o
+                            if self.prefix:
+                                packet['questions'][0][0] = (
+                                    self.prefix + packet['questions'][0][0])
+                                for answer in packet['answers']:
+                                    answer[0] = (self.prefix +
+                                                 answer[0])
+
+                    else:
+                        send_reply = False
+
+                elif query_o[3] == 0:
+                    # Prefix is in use
+                    # IP was queried for cache only
+                    send_reply = False
+
+            elif qtype == DNS.Type.PTR and isinstance(query_o[3],
+                                                      str):
+                packet['questions'][0][0] = query_o[3]
+                for answer in packet['answers']:
+                    answer[0] = query_o[3]
+
+            if query_again:
+                if hit_found:
+                    qtypes = [DNS.Type.AAAA, DNS.Type.A]
+                    pckt = copy.deepcopy(packet)
+                else:
+                    qtypes = [query_o[3]]
+                    pckt = copy.copy(packet)
+
+                pckt['qr'] = 0
+                pckt['answers'] = []
+                pckt['ancount'] = 0
+                pckt['nslist'] = []
+                pckt['nscount'] = 0
+                pckt['additional'] = []
+                pckt['arcount'] = 0
+                for qtype in qtypes:
+                    if self.prefix:
+                        query = (packet, query_o[1], query_o[2],
+                                 0)
+                    else:
+                        query = (packet, query_o[1], query_o[2],
+                                 qtype)
+
+                    self.query_id = (self.query_id % 65535) + 1
+                    pckt['id'] = self.query_id
+                    pckt['questions'][0][1] = qtype
+                    outbuf = DNS.Serialize(pckt).get_packet()
+                    self.clisock.sendto(outbuf, (self.server_ip,
+                                                 self.server_port))
+                    self.add_query(self.server_ip,
+                                   self.server_port,
+                                   self.query_id, query)
+
+                packet['questions'][0][1] = query_o[3]
+
+            if send_reply:
+                outbuf = DNS.Serialize(packet).get_packet()
+                self.servsock.sendto(outbuf, (query_o[1],
+                                              query_o[2]))
+
     def mainloop(self, unused_args):
         """HIP DNS proxy main loop."""
-        connected = False
-
         logging.info('Dns proxy for HIP started')
 
         self.parameter_defaults()
@@ -460,16 +696,15 @@
         # avoid problems with other dns forwarders (e.g. dnsmasq)
         os.system('ifconfig lo:53 %s' % (self.bind_ip,))
 
-        servsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        self.servsock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
         try:
-            servsock.bind((self.bind_ip, self.bind_port))
+            self.servsock.bind((self.bind_ip, self.bind_port))
         except socket.error:
             logging.error('Port %d already in use. See HOWTO.',
                           self.bind_port)
             return
 
-
-        servsock.settimeout(self.app_timeout)
+        self.servsock.settimeout(self.app_timeout)
 
         self.hosts = []
         if self.hostsnames:
@@ -484,8 +719,6 @@
 
         self.write_local_hits_to_hosts()
 
-        query_id = 1
-
         util.init_wantdown()  # Initialize signal handler for shutdown
         util.init_hup()  # Initialize signal handler for reload
 
@@ -496,232 +729,54 @@
                 self.server_ip = self.resolvconf.nameserver
 
             if util.wanthup():
-                logging.info('Received SIGHUP. Picking %s as new upstream '
-                             'DNS server', self.server_ip)
+                logging.info('Received SIGHUP. Picking new upstream server')
+                self.server_ip = self.resolvconf.nameserver
                 util.wanthup(False)
 
-            logging.info('Connecting to upstream DNS server...')
+            logging.info('Connecting to upstream DNS server %s ...',
+                         self.server_ip)
             if ':' not in self.server_ip:
                 server_family = socket.AF_INET
             else:
                 server_family = socket.AF_INET6
-            clisock = socket.socket(server_family, socket.SOCK_DGRAM)
-            clisock.settimeout(self.dns_timeout)
+
+            self.clisock = socket.socket(server_family, socket.SOCK_DGRAM)
+            self.clisock.settimeout(self.dns_timeout)
             try:
-                clisock.connect((self.server_ip, self.server_port))
-                connected = True
+                self.clisock.connect((self.server_ip, self.server_port))
+                self.connected = True
                 logging.debug('... connected!')
                 self.resolvconf.nameserver = self.bind_ip
             except socket.error:
                 logging.error('Connecting to upstream DNS server failed!')
                 time.sleep(3)
-                connected = False
+                self.connected = False
 
-            while connected and (not util.wantdown()) and (not util.wanthup()):
+            while self.connected and (not util.wantdown()) and (
+                not util.wanthup()):
                 try:
                     self.hosts_recheck()
 
-                    if connected:
-                        rlist, _, _ = select.select([servsock, clisock], [], [],
-                                                    5.0)
+                    if self.connected:
+                        rlist, _, _ = select.select([self.servsock,
+                                                     self.clisock],
+                                                    [], [], 5.0)
                     else:
-                        rlist, _, _ = select.select([servsock], [], [], 5.0)
+                        rlist, _, _ = select.select([self.servsock],
+                                                    [], [], 5.0)
+
                     self.clean_queries()
-                    if servsock in rlist:  # Incoming DNS request
-                        inbuf, from_a = servsock.recvfrom(2048)
-
-                        packet = DeSerialize(inbuf).get_dict()
-                        qtype = packet['questions'][0][1]
-
-                        sent_answer = False
-
-                        if qtype in (Type.A, Type.AAAA, Type.PTR):
-                            if self.hip_cache_lookup(packet):
-                                try:
-                                    outbuf = Serialize(packet).get_packet()
-                                    servsock.sendto(outbuf, from_a)
-                                    sent_answer = True
-                                except socket.error:
-                                    logging.exception('Exception:')
-                        elif (self.prefix and
-                              packet['questions'][0][0].startswith(
-                                  self.prefix)):
-                            # Query with HIP prefix for unsupported RR type.
-                            # Send empty response.
-                            packet['qr'] = 1
-                            try:
-                                outbuf = Serialize(packet).get_packet()
-                                servsock.sendto(outbuf, from_a)
-                                sent_answer = True
-                            except socket.error:
-                                logging.exception('Exception:')
-
-                        if connected and not sent_answer:
-                            logging.info('Query type %d for %s from %s',
-                                qtype, packet['questions'][0][0],
-                                (self.server_ip, self.server_port))
-
-                            query = (packet, from_a[0], from_a[1], qtype)
-                            # FIXME: Should randomize for security
-                            query_id = (query_id % 65535) + 1
-                            pckt = copy.copy(packet)
-                            pckt['id'] = query_id
-                            if ((qtype == Type.AAAA or
-                                 (qtype == Type.A and
-                                  not self.disable_lsi)) and
-                                not is_reverse_hit_query(
-                                    packet['questions'][0][0])):
-
-                                if not self.prefix:
-                                    pckt['questions'][0][1] = Type.HIP
-                                if (self.prefix and
-                                    pckt['questions'][0][0].startswith(
-                                        self.prefix)):
-                                    pckt['questions'][0][0] = pckt[
-                                        'questions'][0][0][len(self.prefix):]
-                                    pckt['questions'][0][1] = Type.HIP
-
-                            if qtype == Type.PTR and not self.disable_lsi:
-                                qname = packet['questions'][0][0]
-                                addr_str = hosts.ptr_to_addr(qname)
-                                if (addr_str is not None and
-                                    hosts.valid_lsi(addr_str)):
-                                    query = (packet, from_a[0], from_a[1],
-                                             qname)
-                                    hit_str = lsi_to_hit(addr_str)
-                                    if hit_str is not None:
-                                        pckt['questions'][0][0] = \
-                                                hosts.addr_to_ptr(hit_str)
-
-                            outbuf = Serialize(pckt).get_packet()
-                            clisock.sendto(outbuf, (self.server_ip,
-                                                    self.server_port))
-
-                            self.add_query(self.server_ip, self.server_port,
-                                           query_id, query)
-
-                    if connected and clisock in rlist:   # Incoming DNS reply
-                        inbuf, from_a = clisock.recvfrom(2048)
+                    if self.servsock in rlist:
+                        payload, sender = self.servsock.recvfrom(2048)
+                        packet = DNS.DeSerialize(payload).get_dict()
+                        self.handle_query(packet, sender)
+
+                    if self.connected and self.clisock in rlist:
+                        payload, sender = self.clisock.recvfrom(2048)
                         logging.info('Packet from DNS server %d bytes from %s',
-                            len(inbuf), from_a)
-                        packet = DeSerialize(inbuf).get_dict()
-
-                        # Find original query
-                        query_id_o = packet['id']
-                        query_o = self.find_query(from_a[0], from_a[1],
-                                                  query_id_o)
-                        if query_o:
-                            qname = packet['questions'][0][0]
-                            qtype = packet['questions'][0][1]
-                            send_reply = True
-                            query_again = False
-                            hit_found = False
-                            packet_o = query_o[0]
-                            # Replace with the original query id
-                            packet['id'] = packet_o['id']
-
-                            if qtype == Type.HIP and query_o[3] in (Type.A,
-                                                                    Type.AAAA):
-                                # Restore qtype
-                                packet['questions'][0][1] = query_o[3]
-                                self.hip_lookup(packet)
-                                if packet['ancount'] > 0:
-                                    hit_found = True
-                                if (not self.prefix or
-                                    (hit_found and not (self.getaaaa(qname) or
-                                                        self.geta(qname)))):
-                                    query_again = True
-                                    send_reply = False
-                                elif self.prefix:
-                                    hit_found = True
-                                    packet['questions'][0][0] = (
-                                        self.prefix + packet['questions'][0][0])
-                                    for answer in packet['answers']:
-                                        answer[0] = self.prefix + answer[0]
-
-                            elif qtype in (Type.A, Type.AAAA):
-                                hit = self.getaaaa_hit(qname)
-                                ip6 = self.getaaaa(qname)
-                                ip4 = self.geta(qname)
-                                for answer in packet['answers']:
-                                    if answer[1] in (Type.A, Type.AAAA):
-                                        self.cache_name(qname, answer[4],
-                                                        answer[3])
-                                if hit is not None:
-                                    for answer in packet['answers']:
-                                        if (answer[1] == Type.A or
-                                            (answer[1] == Type.AAAA and not
-                                             hosts.valid_hit(answer[4]))):
-                                            add_hit_ip_map(hit[0], answer[4])
-                                    # Reply with HIT/LSI once it's been mapped
-                                    # to an IP
-                                    if ip6 is None and ip4 is None:
-                                        if (packet_o['ancount'] == 0 and
-                                            not self.prefix):
-                                            # No LSI available. Return IPv4
-                                            tmp = packet['answers']
-                                            packet = packet_o
-                                            packet['answers'] = tmp
-                                            packet['ancount'] = len(
-                                                packet['answers'])
-                                        else:
-                                            packet = packet_o
-                                            if self.prefix:
-                                                packet['questions'][0][0] = \
-                                                    (self.prefix +
-                                                     packet['questions'][0][0])
-                                                for answer in packet['answers']:
-                                                    answer[0] = (self.prefix +
-                                                                 answer[0])
-                                    else:
-                                        send_reply = False
-                                elif query_o[3] == 0:
-                                    # Prefix is in use
-                                    # IP was queried for cache only
-                                    send_reply = False
-
-                            elif qtype == Type.PTR and isinstance(query_o[3],
-                                                                  str):
-                                packet['questions'][0][0] = query_o[3]
-                                for answer in packet['answers']:
-                                    answer[0] = query_o[3]
-
-                            if query_again:
-                                if hit_found:
-                                    qtypes = [Type.AAAA, Type.A]
-                                    pckt = copy.deepcopy(packet)
-                                else:
-                                    qtypes = [query_o[3]]
-                                    pckt = copy.copy(packet)
-                                pckt['qr'] = 0
-                                pckt['answers'] = []
-                                pckt['ancount'] = 0
-                                pckt['nslist'] = []
-                                pckt['nscount'] = 0
-                                pckt['additional'] = []
-                                pckt['arcount'] = 0
-                                for qtype in qtypes:
-                                    if self.prefix:
-                                        query = (packet, query_o[1], query_o[2],
-                                                 0)
-                                    else:
-                                        query = (packet, query_o[1], query_o[2],
-                                                 qtype)
-                                    query_id = (query_id % 65535) + 1
-                                    pckt['id'] = query_id
-                                    pckt['questions'][0][1] = qtype
-                                    outbuf = Serialize(pckt).get_packet()
-                                    clisock.sendto(outbuf, (self.server_ip,
-                                                            self.server_port))
-                                    self.add_query(self.server_ip,
-                                                   self.server_port,
-                                                   query_id, query)
-                                packet['questions'][0][1] = query_o[3]
-
-                            if send_reply:
-                                outbuf = Serialize(packet).get_packet()
-                                servsock.sendto(outbuf, (query_o[1],
-                                                         query_o[2]))
+                                     len(payload), sender)
+                        packet = DNS.DeSerialize(payload).get_dict()
+                        self.handle_response(packet, sender)
                 except (select.error, OSError), exc:
                     if exc[0] == errno.EINTR:
                         pass
@@ -729,7 +784,7 @@
                         logging.exception('Exception:')
                 except socket.error, exc:
                     logging.info('Connection to upstream DNS server lost')
-                    connected = False
+                    self.connected = False
 
         logging.info('Wants down')
         self.resolvconf.restore()

=== added file 'tools/hipdnsproxy/dnsproxy_test.py'
--- tools/hipdnsproxy/dnsproxy_test.py	1970-01-01 00:00:00 +0000
+++ tools/hipdnsproxy/dnsproxy_test.py	2013-10-11 15:24:31 +0000
@@ -0,0 +1,182 @@
+#! /usr/bin/env python
+
+# Copyright (c) 2012 Aalto University and RWTH Aachen University.
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation
+# files (the "Software"), to deal in the Software without
+# restriction, including without limitation the rights to use,
+# copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following
+# conditions:
+#
+# The above copyright notice and this permission notice shall be
+# included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+
+# pylint: disable-msg=C0111,R0903,C0103,R0904
+
+import DNS
+import dnsproxy
+import logging
+import subprocess
+import StringIO
+import unittest
+
+
+# Template DNS packet for used as basis of other packets in testing (rfc2929)
+TEMPLATE_PACKET = {'aa': 0,  # authoritativy answer
+                   'additional': [],  # additional records
+                   'ancount': 0,  # answer count, len(answers)
+                   'answers': [],  # answer records
+                   'arcount': 0,  # additional count, len(additional)
+                   'id': 0,  # Transaction id
+                   'nscount': 0,  # ns count, len(nslist)
+                   'nslist': [],  # NS records
+                   'opcode': 0,  # 0 = query (rfc1035)
+                                 # 1 = inverse query (rfc1035)
+                                 # 2 = status (rfc1035)
+                                 # 4 = notify (rfc1996)
+                                 # 5 = update (rfc2136)
+                   'qdcount': 0,  # query count, len(questions)
+                   'qr': 0,  # 0 = query, 1 = response
+                   'questions': [],  # questions
+                   'ra': 1,  # recursion available flag
+                   'rcode': 0,  # 0 = no error (rfc1035)
+                                # 1 = format error (rfc1035)
+                                # 2 = server failure (rfc 1035)
+                                # 3 = nxdomain (rfc1035)
+                                # 4 = not implemented (rfc1035)
+                                # 5 = query refused (rfc1035)
+                                # 6 = yxdomain (rfc2136)
+                                # 7 = yxrrset (rfc2136)
+                                # 8 = nxrrset (rfc2136)
+                                # 9 = not authoritative (rfc2136)
+                                # 10 = not in zone (rfc2136)
+                                # 16 = bad tsig (rfc2845)
+                                # 17 = bad key (rfc2845)
+                                # 18 = bad time (rfc2845)
+                                # 19 = bad tkey mode (rfc2930)
+                                # 20 = bad key name (rfc2930)
+                                # 21 = bad algorithm (rfc2930)
+                   'rd': 0,  # recursion desired flag
+                   'tc': 0,  # message truncated flag
+                   'z': 0}  # no longer in use, ignore (rfc2929)
+SIMPLE_RESPONSE = TEMPLATE_PACKET.copy()
+SIMPLE_RESPONSE.update({'ancount': 1,
+                        'answers': (('example.com', DNS.Type.A, DNS.Class.IN,
+                                     60, '127.0.0.1'),),
+                        'qdcount': 1,
+                        'questions': (('example.com', DNS.Type.A,
+                                       DNS.Class.IN),),
+                        'qr': 1})
+
+SIMPLE_QUERY = TEMPLATE_PACKET.copy()
+SIMPLE_QUERY.update({'qdcount': 1,
+                     'questions': (('example.com', DNS.Type.A,
+                                    DNS.Class.IN),)})
+
+
+HIT = '2001:1b:a9be:c6a6:34e5:8361:c07f:a990'
+LSI = '1.0.0.1'
+
+
+class MockSocket(object):
+    def sendto(self, payload, sender):
+        pass
+
+
+class MockFile(object):
+    def __init__(self, exc=None, payload=None):
+        self.exc = exc
+        self.payload = StringIO.StringIO(payload)
+
+    def __iter__(self):
+        if self.exc is not None:
+            return self
+        return self.payload.__iter__()
+
+    def next(self):
+        if self.exc is not None:
+            raise self.exc
+        raise StopIteration
+
+
+class MockPopen(object):
+    def __init__(self, exc=None, stdout=None, returncode=0):
+        if exc is not None:
+            self.stdout = MockFile(exc=exc, payload=stdout)
+        else:
+            self.stdout = StringIO.StringIO(stdout)
+
+        self.returncode = returncode
+
+
+class DNSProxyTest(unittest.TestCase):
+    def setUp(self):
+        logging.basicConfig(level=logging.DEBUG)
+        self.dnsproxy = dnsproxy.DNSProxy()
+        self.dnsproxy.hosts = []
+        self.dnsproxy.clisock = MockSocket()
+        self.dnsproxy.servsock = MockSocket()
+
+    def test_handle_response(self):
+        self.dnsproxy.add_query('127.0.0.1', 53, 0,
+                                (SIMPLE_QUERY, '127.0.0.1', 53,
+                                 SIMPLE_QUERY['questions'][0][1]))
+        self.dnsproxy.handle_response(SIMPLE_RESPONSE, ('127.0.0.1', 53))
+
+    def test_handle_query(self):
+        self.dnsproxy.handle_query(SIMPLE_QUERY, ('127.0.0.1', 53))
+
+    def test_is_reverse_hit_query(self):
+        self.assertFalse(dnsproxy.is_reverse_hit_query('::1'))
+        name = ('8.e.b.8.b.3.c.9.1.a.0.c.e.e.2.c.c.e.d.0.9.c.9.a.e.1.0.0.1.0.'
+                '0.2.hit-to-ip.infrahip.net')
+        self.assertTrue(dnsproxy.is_reverse_hit_query(name))
+
+    def test_hit_to_lsi(self):
+        subprocess.Popen = lambda *x, **y: MockPopen(exc=IOError('hit_to_lsi'))
+        lsi = dnsproxy.hit_to_lsi(HIT)
+        self.assertIsNone(lsi)
+
+        subprocess.Popen = lambda *x, **y: MockPopen(returncode=254)
+        lsi = dnsproxy.hit_to_lsi(HIT)
+        self.assertIsNone(lsi)
+
+        subprocess.Popen = lambda *x, **y: MockPopen(stdout=LSI)
+        lsi = dnsproxy.hit_to_lsi(HIT)
+        self.assertEqual(lsi, LSI)
+
+    def test_lsi_to_hit(self):
+        subprocess.Popen = lambda *x, **y: MockPopen(exc=IOError('lsi_to_hit'))
+        hit = dnsproxy.lsi_to_hit(LSI)
+        self.assertIsNone(hit)
+
+        subprocess.Popen = lambda *x, **y: MockPopen(returncode=254)
+        hit = dnsproxy.lsi_to_hit(LSI)
+        self.assertIsNone(hit)
+
+        subprocess.Popen = lambda *x, **y: MockPopen(stdout=HIT)
+        hit = dnsproxy.lsi_to_hit(LSI)
+        self.assertEqual(hit, HIT)
+
+    def test_add_hit_ip_map(self):
+        # pylint: disable-msg=W0613
+        def check_call(*x, **y):
+            raise subprocess.CalledProcessError(254, 'hipconf daemon')
+        subprocess.check_call = check_call
+        self.assertIsNone(dnsproxy.add_hit_ip_map(HIT, LSI))
+
+
+if __name__ == '__main__':
+    unittest.main()

=== modified file 'tools/hipdnsproxy/hipdnsproxy.in'
--- tools/hipdnsproxy/hipdnsproxy.in	2013-02-22 15:16:12 +0000
+++ tools/hipdnsproxy/hipdnsproxy.in	2013-10-11 15:24:31 +0000
@@ -39,6 +39,8 @@
     optparser.add_option('-p', '--serverport', dest='server_port',
                          action='store', type='int', default=None,
                          help='DNS server port')
+    optparser.add_option('-S', '--syslog', dest='syslog', action='store_true',
+                         default=False, help='Log to syslog')
     optparser.add_option('-t', '--dns-timeout', dest='dns_timeout',
                          action='store', type='float', default=2.0,
                          help='DNS timeout')
@@ -58,15 +60,19 @@
 
     (options, args) = optparser.parse_args()
 
+    logging.getLogger().setLevel(options.loglevel)
+
     child = False
     if (options.background):
         child = util.daemonize()
     else:
-        logging.getLogger().setLevel(options.loglevel)
-        loghandler = logging.StreamHandler(sys.stderr)
-        loghandler.setFormatter(logging.Formatter(
-            '%(asctime)s %(levelname)-8s %(message)s'))
-        logging.getLogger().addHandler(loghandler)
+        if options.syslog:
+            util.log2syslog()
+        else:
+            loghandler = logging.StreamHandler(sys.stderr)
+            loghandler.setFormatter(logging.Formatter(
+                '%(asctime)s %(levelname)-8s %(message)s'))
+            logging.getLogger().addHandler(loghandler)
 
     dnsp = dnsproxy.DNSProxy(bind_ip=options.bind_ip,
                              bind_port=options.bind_port,

=== modified file 'tools/hipdnsproxy/resolvconf.py'
--- tools/hipdnsproxy/resolvconf.py	2013-08-11 15:03:36 +0000
+++ tools/hipdnsproxy/resolvconf.py	2013-10-11 15:24:31 +0000
@@ -31,6 +31,7 @@
 import os
 import subprocess
 import tempfile
+import time
 
 
 class FragmentFile(object):
@@ -40,8 +41,7 @@
     def __init__(self, fragments):
         self.fragments = fragments
         self.current = None
-        if self.fragments:
-            self.current = open(self.fragments.pop(0), 'rb')
+        self.nextfragment()
 
     def __iter__(self):
         return self
@@ -52,6 +52,23 @@
     def __exit__(self, exc_type, exc_value, trackeback):
         pass
 
+    def nextfragment(self):
+        """Advance current fragment. Returns True when successful."""
+        if self.fragments:
+            fragment = self.fragments.pop(0)
+            retries = 3
+            while retries > 0:
+                try:
+                    self.current = open(fragment, 'rb')
+                    return True
+                except IOError, ioe:
+                    if ioe.errno != errno.ENOENT:
+                        raise
+                    time.sleep(0.1)
+                retries -= 1
+        self.current = None
+        return False
+
     def next(self):
         """Returns next line in virtual file."""
         if self.current is None:
@@ -60,8 +77,7 @@
         try:
             return self.current.next()
         except StopIteration, sie:
-            if self.fragments:
-                self.current = open(self.fragments.pop(0), 'rb')
+            if self.nextfragment():
                 return self.next()
             else:
                 raise sie
@@ -95,10 +111,10 @@
         self.config = None
 
         if self.using_resolvconf and isinstance(ResolvConf.FRAGMENT_DIR, list):
+            # pylint: disable-msg=W0141
             ResolvConf.FRAGMENT_DIR = filter(os.path.exists,
                                              ResolvConf.FRAGMENT_DIR)[0]
 
-
     def realpath(self):
         """Return the real path to the actual resolv.conf."""
         return os.path.realpath(self.path)
@@ -126,13 +142,13 @@
 
                 frags = [x.strip() for x in out]
 
-                return [os.path.join(ResolvConf.FRAGMENT_DIR, x) for x in frags]
+                return [os.path.join(ResolvConf.FRAGMENT_DIR, x)
+                        for x in frags]
             except IOError, ioe:
                 if ioe.errno != errno.EINTR:
                     raise
                 retries -= 1
 
-
     @classmethod
     def resolvconf_add(cls, ifile):
         """Pass new fragment to resolvconf(8)."""

=== modified file 'tools/hipdnsproxy/util.py'
--- tools/hipdnsproxy/util.py	2013-07-11 15:08:01 +0000
+++ tools/hipdnsproxy/util.py	2013-10-11 15:24:31 +0000
@@ -87,6 +87,7 @@
     """Check if SIGINT or SIGTERM has been received."""
     return __FLAGS['down']
 
+
 def wanthup(reset=None):
     """Check if SIGHUP has been received."""
     previous = __FLAGS['hup']
@@ -173,6 +174,15 @@
     return '%dd%02dh%02dm%02ds' % (days, hours, mins, secs)
 
 
+def log2syslog():
+    """Configure logging to send messages to syslog."""
+    loghandler = logging.handlers.SysLogHandler(address='/dev/log',
+        facility=logging.handlers.SysLogHandler.LOG_DAEMON)
+    loghandler.setFormatter(logging.Formatter(
+        'hipdnsproxy[%(process)s] %(levelname)-8s %(message)s'))
+    logging.getLogger().addHandler(loghandler)
+
+
 def daemonize():
     """Daemonize current process."""
     pid = os.fork()
@@ -193,11 +203,7 @@
     if pid:
         sys.exit(0)
 
-    loghandler = logging.handlers.SysLogHandler(address='/dev/log',
-        facility=logging.handlers.SysLogHandler.LOG_DAEMON)
-    loghandler.setFormatter(logging.Formatter(
-        'hipdnsproxy[%(process)s] %(levelname)-8s %(message)s'))
-    logging.getLogger().addHandler(loghandler)
+    log2syslog()
 
     sys.stdout.flush()
     sys.stderr.flush()