Merge ~hloeung/content-cache-charm:haproxy-config into content-cache-charm:master

[Haw Loeung]

Use resolvers when hostname rather than IP provided

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Stuart Bishop (stub)]

All looks good.

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Failed to merge change (unable to merge source repository due to conflicts), setting status to needs review.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 4eba9f57939da382d43929801d51d9143f4c12f1

diff --git a/lib/haproxy.py b/lib/haproxy.py
index 6171d28..33c0c3b 100644
--- a/lib/haproxy.py
+++ b/lib/haproxy.py
@@ -207,7 +207,7 @@ listen {name}
 
         return rendered_output
 
-    def render_stanza_backend(self, config):
+    def render_stanza_backend(self, config):  # NOQA: C901
         backend_stanza = """
 backend backend-{name}
 {options}{indent}{httpchk}
@@ -252,11 +252,17 @@ backend backend-{name}
                     count += 1
 
                     name = 'server_{}'.format(count)
+                    use_resolvers = ''
+                    try:
+                        utils.ip_addr_port_split(backend)
+                    except utils.InvalidAddressPortError:
+                        use_resolvers = ' resolvers dns init-addr none'
                     backend_confs.append(
-                        '{indent}server {name} {backend} check inter {inter_time} rise 2 fall 5 '
+                        '{indent}server {name} {backend}{use_resolvers} check inter {inter_time} rise 2 fall 5 '
                         'maxconn {maxconn}{tls}'.format(
                             name=name,
                             backend=backend,
+                            use_resolvers=use_resolvers,
                             inter_time=inter_time,
                             maxconn=maxconn,
                             tls=tls_config,
@@ -335,6 +341,7 @@ backend backend-{name}
         return template.render(
             {
                 'backend': self.render_stanza_backend(config),
+                'dns_servers': utils.dns_servers(),
                 'global_max_connections': global_max_connections,
                 'hard_stop_after': self.hard_stop_after,
                 'listen': listen_stanzas,
diff --git a/lib/utils.py b/lib/utils.py
index 9bbb42f..f06effc 100644
--- a/lib/utils.py
+++ b/lib/utils.py
@@ -196,6 +196,19 @@ def process_rlimits(pid, res, limits_file=None):
     return None
 
 
+def dns_servers(resolvconf_file='/etc/resolv.conf'):
+    servers = []
+    with open(resolvconf_file, 'r', encoding='utf-8') as f:
+        resolvconf = f.read()
+    for line in resolvconf.split('\n'):
+        t = line.split()
+        if not t:
+            continue
+        if t[0] == 'nameserver':
+            servers.append(t[1])
+    return servers
+
+
 def package_version(package):
     cmd = ['dpkg-query', '--show', r'--showformat=${Version}\n', package]
     try:
diff --git a/templates/haproxy_cfg.tmpl b/templates/haproxy_cfg.tmpl
index 6d7f0b8..26242b2 100644
--- a/templates/haproxy_cfg.tmpl
+++ b/templates/haproxy_cfg.tmpl
@@ -59,6 +59,18 @@ defaults
 	errorfile 503 /etc/haproxy/errors/503.http
 	errorfile 504 /etc/haproxy/errors/504.http
 
+{%- if dns_servers %}
+
+resolvers dns
+{%- for resolver in dns_servers %}
+	nameserver dns{{loop.index}} {{resolver}}:53
+{%- endfor %}
+	resolve_retries 3
+	timeout resolve 3s
+	timeout retry   3s
+	accepted_payload_size 8192
+{%- endif %}
+
 listen  stats
 	bind 127.0.0.1:10000
 	acl allowed_cidr src 127.0.0.0/8
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output.txt
index 60c0b9e..775bc5f 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output.txt
@@ -54,6 +54,13 @@ defaults
 	errorfile 503 /etc/haproxy/errors/503.http
 	errorfile 504 /etc/haproxy/errors/504.http
 
+resolvers dns
+	nameserver dns1 127.0.0.53:53
+	resolve_retries 3
+	timeout resolve 3s
+	timeout retry   3s
+	accepted_payload_size 8192
+
 listen  stats
 	bind 127.0.0.1:10000
 	acl allowed_cidr src 127.0.0.0/8
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output2.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output2.txt
index 89506ab..0ef9fd0 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output2.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output2.txt
@@ -54,6 +54,13 @@ defaults
 	errorfile 503 /etc/haproxy/errors/503.http
 	errorfile 504 /etc/haproxy/errors/504.http
 
+resolvers dns
+	nameserver dns1 127.0.0.53:53
+	resolve_retries 3
+	timeout resolve 3s
+	timeout retry   3s
+	accepted_payload_size 8192
+
 listen  stats
 	bind 127.0.0.1:10000
 	acl allowed_cidr src 127.0.0.0/8
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output3.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output3.txt
index c82bf5c..43c80e7 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output3.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output3.txt
@@ -54,6 +54,13 @@ defaults
 	errorfile 503 /etc/haproxy/errors/503.http
 	errorfile 504 /etc/haproxy/errors/504.http
 
+resolvers dns
+	nameserver dns1 127.0.0.53:53
+	resolve_retries 3
+	timeout resolve 3s
+	timeout retry   3s
+	accepted_payload_size 8192
+
 listen  stats
 	bind 127.0.0.1:10000
 	acl allowed_cidr src 127.0.0.0/8
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt
index 79359af..222ccc7 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt
@@ -54,6 +54,13 @@ defaults
 	errorfile 503 /etc/haproxy/errors/503.http
 	errorfile 504 /etc/haproxy/errors/504.http
 
+resolvers dns
+	nameserver dns1 127.0.0.53:53
+	resolve_retries 3
+	timeout resolve 3s
+	timeout retry   3s
+	accepted_payload_size 8192
+
 listen  stats
 	bind 127.0.0.1:10000
 	acl allowed_cidr src 127.0.0.0/8
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt
index dbf58b0..91c8ca1 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt
@@ -55,6 +55,13 @@ defaults
 	errorfile 503 /etc/haproxy/errors/503.http
 	errorfile 504 /etc/haproxy/errors/504.http
 
+resolvers dns
+	nameserver dns1 127.0.0.53:53
+	resolve_retries 3
+	timeout resolve 3s
+	timeout retry   3s
+	accepted_payload_size 8192
+
 listen  stats
 	bind 127.0.0.1:10000
 	acl allowed_cidr src 127.0.0.0/8
diff --git a/tests/unit/files/haproxy_config_rendered_backends_stanzas_test_output2.txt b/tests/unit/files/haproxy_config_rendered_backends_stanzas_test_output2.txt
 new file mode 100644
index 0000000..44e1159
--- /dev/null
+++ b/tests/unit/files/haproxy_config_rendered_backends_stanzas_test_output2.txt
@@ -0,0 +1,6 @@
+
+backend backend-site1-local
+    option httpchk HEAD / HTTP/1.0\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
+    http-request set-header Host site1.local
+    balance leastconn
+    server server_1 archive.ubuntu.com:80 resolvers dns init-addr none check inter 5s rise 2 fall 5 maxconn 2048
diff --git a/tests/unit/files/haproxy_config_rendered_test_output.txt b/tests/unit/files/haproxy_config_rendered_test_output.txt
index 7b07b8f..9545d62 100644
--- a/tests/unit/files/haproxy_config_rendered_test_output.txt
+++ b/tests/unit/files/haproxy_config_rendered_test_output.txt
@@ -55,6 +55,13 @@ defaults
 	errorfile 503 /etc/haproxy/errors/503.http
 	errorfile 504 /etc/haproxy/errors/504.http
 
+resolvers dns
+	nameserver dns1 127.0.0.53:53
+	resolve_retries 3
+	timeout resolve 3s
+	timeout retry   3s
+	accepted_payload_size 8192
+
 listen  stats
 	bind 127.0.0.1:10000
 	acl allowed_cidr src 127.0.0.0/8
diff --git a/tests/unit/files/haproxy_config_rendered_test_output2.txt b/tests/unit/files/haproxy_config_rendered_test_output2.txt
index 196b6c3..3e44b41 100644
--- a/tests/unit/files/haproxy_config_rendered_test_output2.txt
+++ b/tests/unit/files/haproxy_config_rendered_test_output2.txt
@@ -55,6 +55,13 @@ defaults
 	errorfile 503 /etc/haproxy/errors/503.http
 	errorfile 504 /etc/haproxy/errors/504.http
 
+resolvers dns
+	nameserver dns1 127.0.0.53:53
+	resolve_retries 3
+	timeout resolve 3s
+	timeout retry   3s
+	accepted_payload_size 8192
+
 listen  stats
 	bind 127.0.0.1:10000
 	acl allowed_cidr src 127.0.0.0/8
diff --git a/tests/unit/files/resolv.conf b/tests/unit/files/resolv.conf
 new file mode 100644
index 0000000..e459d59
--- /dev/null
+++ b/tests/unit/files/resolv.conf
@@ -0,0 +1,3 @@
+nameserver 127.0.0.53
+domain local
+# comment
diff --git a/tests/unit/files/resolv.conf-none b/tests/unit/files/resolv.conf-none
 new file mode 100644
index 0000000..7fc2489
--- /dev/null
+++ b/tests/unit/files/resolv.conf-none
@@ -0,0 +1 @@
+domain local
diff --git a/tests/unit/files/resolv.conf2 b/tests/unit/files/resolv.conf2
 new file mode 100644
index 0000000..205859b
--- /dev/null
+++ b/tests/unit/files/resolv.conf2
@@ -0,0 +1,2 @@
+nameserver 127.0.0.53
+nameserver 127.0.0.153
diff --git a/tests/unit/test_content_cache.py b/tests/unit/test_content_cache.py
index 5b4e386..3b62cbb 100644
--- a/tests/unit/test_content_cache.py
+++ b/tests/unit/test_content_cache.py
@@ -70,6 +70,11 @@ class TestCharm(unittest.TestCase):
         self.mock_close_port = patcher.start()
         self.addCleanup(patcher.stop)
 
+        patcher = mock.patch('lib.utils.dns_servers')
+        self.mock_dns_servers = patcher.start()
+        self.addCleanup(patcher.stop)
+        self.mock_dns_servers.return_value = ['127.0.0.53']
+
         patcher = mock.patch('multiprocessing.cpu_count')
         self.mock_cpu_count = patcher.start()
         self.addCleanup(patcher.stop)
diff --git a/tests/unit/test_haproxy.py b/tests/unit/test_haproxy.py
index 60cf0d0..198879d 100644
--- a/tests/unit/test_haproxy.py
+++ b/tests/unit/test_haproxy.py
@@ -146,10 +146,20 @@ class TestLibHAProxy(unittest.TestCase):
             want = f.read()
         self.assertEqual(''.join(haproxy.render_stanza_backend(config)), want)
 
+    def test_haproxy_config_rendered_backend_stanzas_use_dns(self):
+        haproxy = HAProxy.HAProxyConf(self.tmpdir)
+        config = {'site1.local': {'locations': {'/': {'backends': ['archive.ubuntu.com:80']}}}}
+        output = 'tests/unit/files/haproxy_config_rendered_backends_stanzas_test_output2.txt'
+        with open(output, 'r', encoding='utf-8') as f:
+            want = f.read()
+        self.assertEqual(''.join(haproxy.render_stanza_backend(config)), want)
+
     @freezegun.freeze_time("2019-03-22", tz_offset=0)
+    @mock.patch('lib.utils.dns_servers')
     @mock.patch('lib.utils.package_version')
     @mock.patch('lib.utils.process_rlimits')
-    def test_haproxy_config_rendered_full_config(self, process_rlimits, package_version):
+    def test_haproxy_config_rendered_full_config(self, process_rlimits, package_version, dns_servers):
+        dns_servers.return_value = ['127.0.0.53']
         package_version.return_value = '1.8.8-1ubuntu0.10'
         haproxy = HAProxy.HAProxyConf(self.tmpdir, max_connections=5000)
         config = self.site_config
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index 60da5f6..b2d7348 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -186,6 +186,14 @@ class TestLibUtils(unittest.TestCase):
 
         self.assertEqual(None, utils.process_rlimits(1, 'NOMATCH'))
 
+    def test_dns_servers(self):
+        want = ['127.0.0.53']
+        self.assertEqual(want, utils.dns_servers('tests/unit/files/resolv.conf'))
+        want = ['127.0.0.53', '127.0.0.153']
+        self.assertEqual(want, utils.dns_servers('tests/unit/files/resolv.conf2'))
+        want = []
+        self.assertEqual(want, utils.dns_servers('tests/unit/files/resolv.conf-none'))
+
     def test_package_version(self):
         self.assertTrue(int(utils.package_version('apt').split('.')[0]) > 1)