Merge ~hloeung/content-cache-charm:haproxy-config into content-cache-charm:master

[Haw Loeung]

Improve resiliency with HAProxy redispatch

This is as suggested by Jay in an internal ticket. Note that retries already defaults to 3.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision f640753c7c52a81131bb70b1c34774c20ff4d542

diff --git a/lib/haproxy.py b/lib/haproxy.py
index a68068b..14d0fc7 100644
--- a/lib/haproxy.py
+++ b/lib/haproxy.py
@@ -303,7 +303,11 @@ backend backend-{name}
 
                 opts = []
                 for option in loc_conf.get('backend-options', []):
-                    opts.append('{indent}option {opt}'.format(opt=option, indent=INDENT))
+                    prefix = ''
+                    if option.split()[0] in ['allbackups', 'forceclose', 'forwardfor', 'redispatch']:
+                        prefix = 'option '
+                    opts.append('{indent}{prefix}{opt}'.format(opt=option, prefix=prefix, indent=INDENT))
+
                 options = ''
                 if opts:
                     options = '\n'.join(opts + [''])
diff --git a/reactive/content_cache.py b/reactive/content_cache.py
index 42c0997..5d6ddee 100644
--- a/reactive/content_cache.py
+++ b/reactive/content_cache.py
@@ -298,9 +298,14 @@ def configure_haproxy():  # NOQA: C901 LP#1825084
             if backend_check_path:
                 new_cached_loc_conf['backend-check-path'] = backend_check_path
                 new_loc_conf['backend-check-path'] = backend_check_path
+            new_loc_conf['backend-options'] = []
             backend_options = loc_conf.get('backend-options')
             if backend_options:
                 new_loc_conf['backend-options'] = backend_options
+
+            # Make it more resilient to failures and redispatch requests to different backends.
+            new_loc_conf['backend-options'].append('redispatch 1')
+
             new_cached_loc_conf['signed-url-hmac-key'] = loc_conf.get('signed-url-hmac-key')
             # Pass through selected backend location configs, if defined.
             for key in ('site-name', 'backend-inter-time', 'backend-tls'):
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output.txt
index bd1bd1e..6fa7004 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output.txt
@@ -147,6 +147,7 @@ backend backend-cached-site1-local
     server server_1 127.0.0.1:6080 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site1-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site1.local
     balance leastconn
@@ -162,6 +163,7 @@ backend backend-cached-site2-local
     server server_1 127.0.0.1:6081 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site2-local
+    option redispatch 1
     option httpchk GET /check/ HTTP/1.0\r\nHost:\ site2.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site2.local
     balance leastconn
@@ -179,6 +181,7 @@ backend backend-cached-site3-local
 backend backend-site3-local
     option forwardfor except 127.0.0.1
     option forceclose
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site3.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site3.local
     balance leastconn
@@ -201,12 +204,14 @@ backend backend-cached-site5
     server server_1 127.0.0.1:6084 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site5
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site5.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site5.local
     balance leastconn
     server server_1 127.0.1.10:80 check inter 5s rise 2 fall 5 maxconn 2048
 
 backend backend-site5-2
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site5.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site5.local
     balance leastconn
@@ -220,6 +225,7 @@ backend backend-cached-site6-local
     server server_1 127.0.0.1:6085 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site6-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site6.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site6.local
     balance leastconn
@@ -233,6 +239,7 @@ backend backend-cached-site7-local
     server server_1 127.0.0.1:6086 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site7-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site7.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site7.local
     balance leastconn
@@ -246,12 +253,14 @@ backend backend-cached-site8-local
     server server_1 127.0.0.1:6087 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site8-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site8.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site8.local
     balance leastconn
     server server_1 127.0.1.10:80 check inter 5s rise 2 fall 5 maxconn 2048
 
 backend backend-site8-local-2
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ auth.site8.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host auth.site8.local
     balance leastconn
@@ -265,6 +274,7 @@ backend backend-cached-site9-local
     server server_1 127.0.0.1:6088 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site9-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site9.local
     balance leastconn
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output2.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output2.txt
index ef8dfa0..fe8a37f 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output2.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output2.txt
@@ -93,6 +93,8 @@ backend backend-cached-site1-local
     server server_1 127.0.0.1:6080 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site1-local
+    retries 3
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site1.local
     balance leastconn
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output3.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output3.txt
index ad10667..0c134d3 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output3.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output3.txt
@@ -98,6 +98,7 @@ backend backend-cached-site1-local
     server server_1 127.0.0.1:6080 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site1-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site1.local
     balance leastconn
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt
index d7c2c0a..9bc7d2b 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt
@@ -147,6 +147,7 @@ backend backend-cached-site1-local
     server server_1 127.0.0.1:6080 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site1-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site1.local
     balance leastconn
@@ -162,6 +163,7 @@ backend backend-cached-site2-local
     server server_1 127.0.0.1:6081 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site2-local
+    option redispatch 1
     option httpchk GET /check/ HTTP/1.0\r\nHost:\ site2.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site2.local
     balance leastconn
@@ -179,6 +181,7 @@ backend backend-cached-site3-local
 backend backend-site3-local
     option forwardfor except 127.0.0.1
     option forceclose
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site3.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site3.local
     balance leastconn
@@ -201,12 +204,14 @@ backend backend-cached-site5
     server server_1 127.0.0.1:6084 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site5
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site5.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site5.local
     balance leastconn
     server server_1 127.0.1.10:80 check inter 5s rise 2 fall 5 maxconn 2048
 
 backend backend-site5-2
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site5.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site5.local
     balance leastconn
@@ -220,6 +225,7 @@ backend backend-cached-site6-local
     server server_1 127.0.0.1:6085 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site6-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site6.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site6.local
     balance leastconn
@@ -233,6 +239,7 @@ backend backend-cached-site7-local
     server server_1 127.0.0.1:6086 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site7-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site7.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site7.local
     balance leastconn
@@ -246,12 +253,14 @@ backend backend-cached-site8-local
     server server_1 127.0.0.1:6087 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site8-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site8.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site8.local
     balance leastconn
     server server_1 127.0.1.10:80 check inter 5s rise 2 fall 5 maxconn 2048
 
 backend backend-site8-local-2
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ auth.site8.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host auth.site8.local
     balance leastconn
@@ -265,6 +274,7 @@ backend backend-cached-site9-local
     server server_1 127.0.0.1:6088 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site9-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site9.local
     balance leastconn
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_backup.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_backup.txt
index 06670cd..7f6ab4d 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output_backup.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_backup.txt
@@ -98,6 +98,7 @@ backend backend-cached-site1-local
 
 backend backend-site1-local
     option allbackups
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site1.local
     balance leastconn
@@ -105,6 +106,7 @@ backend backend-site1-local
     server server_2 192.168.1.2:8080 backup check inter 5s rise 2 fall 5 maxconn 2048
 
 backend backend-site1-local-2
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site1.local
     balance leastconn
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_load_balancing_algorithm.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_load_balancing_algorithm.txt
index 486f3a2..645b39c 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output_load_balancing_algorithm.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_load_balancing_algorithm.txt
@@ -147,6 +147,7 @@ backend backend-cached-site1-local
     server server_1 127.0.0.1:6080 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site1-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site1.local
     balance roundrobin
@@ -162,6 +163,7 @@ backend backend-cached-site2-local
     server server_1 127.0.0.1:6081 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site2-local
+    option redispatch 1
     option httpchk GET /check/ HTTP/1.0\r\nHost:\ site2.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site2.local
     balance roundrobin
@@ -179,6 +181,7 @@ backend backend-cached-site3-local
 backend backend-site3-local
     option forwardfor except 127.0.0.1
     option forceclose
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site3.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site3.local
     balance roundrobin
@@ -201,12 +204,14 @@ backend backend-cached-site5
     server server_1 127.0.0.1:6084 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site5
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site5.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site5.local
     balance roundrobin
     server server_1 127.0.1.10:80 check inter 5s rise 2 fall 5 maxconn 2048
 
 backend backend-site5-2
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site5.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site5.local
     balance roundrobin
@@ -220,6 +225,7 @@ backend backend-cached-site6-local
     server server_1 127.0.0.1:6085 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site6-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site6.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site6.local
     balance roundrobin
@@ -233,6 +239,7 @@ backend backend-cached-site7-local
     server server_1 127.0.0.1:6086 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site7-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site7.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site7.local
     balance roundrobin
@@ -246,12 +253,14 @@ backend backend-cached-site8-local
     server server_1 127.0.0.1:6087 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site8-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site8.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site8.local
     balance roundrobin
     server server_1 127.0.1.10:80 check inter 5s rise 2 fall 5 maxconn 2048
 
 backend backend-site8-local-2
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ auth.site8.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host auth.site8.local
     balance roundrobin
@@ -265,6 +274,7 @@ backend backend-cached-site9-local
     server server_1 127.0.0.1:6088 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site9-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site9.local
     balance roundrobin
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt
index 4e64b08..8e65dbc 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt
@@ -148,6 +148,7 @@ backend backend-cached-site1-local
     server server_1 127.0.0.1:6080 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site1-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site1.local
     balance leastconn
@@ -163,6 +164,7 @@ backend backend-cached-site2-local
     server server_1 127.0.0.1:6081 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site2-local
+    option redispatch 1
     option httpchk GET /check/ HTTP/1.0\r\nHost:\ site2.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site2.local
     balance leastconn
@@ -180,6 +182,7 @@ backend backend-cached-site3-local
 backend backend-site3-local
     option forwardfor except 127.0.0.1
     option forceclose
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site3.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site3.local
     balance leastconn
@@ -202,12 +205,14 @@ backend backend-cached-site5
     server server_1 127.0.0.1:6084 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site5
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site5.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site5.local
     balance leastconn
     server server_1 127.0.1.10:80 check inter 5s rise 2 fall 5 maxconn 2048
 
 backend backend-site5-2
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site5.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site5.local
     balance leastconn
@@ -221,6 +226,7 @@ backend backend-cached-site6-local
     server server_1 127.0.0.1:6085 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site6-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site6.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site6.local
     balance leastconn
@@ -234,6 +240,7 @@ backend backend-cached-site7-local
     server server_1 127.0.0.1:6086 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site7-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site7.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site7.local
     balance leastconn
@@ -247,12 +254,14 @@ backend backend-cached-site8-local
     server server_1 127.0.0.1:6087 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site8-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site8.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site8.local
     balance leastconn
     server server_1 127.0.1.10:80 check inter 5s rise 2 fall 5 maxconn 2048
 
 backend backend-site8-local-2
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ auth.site8.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host auth.site8.local
     balance leastconn
@@ -266,6 +275,7 @@ backend backend-cached-site9-local
     server server_1 127.0.0.1:6088 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site9-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site9.local
     balance leastconn
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_srv_template.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_srv_template.txt
index 5391a03..9c3ead5 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output_srv_template.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_srv_template.txt
@@ -93,6 +93,7 @@ backend backend-cached-site1-local
     server server_1 127.0.0.1:6080 check inter 2s rise 2 fall 60 maxconn 2048
 
 backend backend-site1-local
+    option redispatch 1
     option httpchk HEAD / HTTP/1.0\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk\r\nCache-Control:\ no-cache
     http-request set-header Host site1.local
     balance leastconn
diff --git a/tests/unit/test_content_cache.py b/tests/unit/test_content_cache.py
index 1c11a30..ef464aa 100644
--- a/tests/unit/test_content_cache.py
+++ b/tests/unit/test_content_cache.py
@@ -421,6 +421,8 @@ site1.local:
     /:
       backend-tls: false
       backends: ['192.168.1.1:8080']
+      backend-options:
+      - retries 3
   tls-cert-bundle-path: /var/lib/haproxy/certs
 '''
         self.mock_config.return_value = {'haproxy_hard_stop_after': '15m', 'max_connections': 8192, 'sites': config}