Merge ~hloeung/content-cache-charm:rate-limiting into content-cache-charm:master

[Haw Loeung]

Add support for rate limiting on source IP and more - LP:1918439

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[James Simpson (jsimpso)]

LGTM

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 61cc22b969fc5555ec7e0d5c37f0fca78ce6b9f8

diff --git a/lib/haproxy.py b/lib/haproxy.py
index d666169..fa5c940 100644
--- a/lib/haproxy.py
+++ b/lib/haproxy.py
@@ -227,7 +227,7 @@ listen {name}
         backend_stanza = """
 backend backend-{name}
 {indent}{httpchk}
-{indent}http-request set-header Host {site_name}
+{ratelimit}{indent}http-request set-header Host {site_name}
 {options}{indent}balance {load_balancing_algorithm}
 {backends}
 """
@@ -314,6 +314,25 @@ backend backend-{name}
                         )
                     )
 
+                ratelimit = ''
+                if loc_conf.get('rate-limit'):
+                    loc_rl = loc_conf.get('rate-limit')
+                    condition = loc_rl.get('condition')
+                    sticky = loc_rl.get('sticky-table')
+                    track = loc_rl.get('track')
+
+                    if condition and sticky and track:
+                        rlconf = []
+                        # https://www.haproxy.com/blog/four-examples-of-haproxy-rate-limiting/
+                        rlconf.append('{indent}stick-table {sticky}'.format(sticky=sticky, indent=INDENT))
+                        rlconf.append('{indent}http-request track-sc0 {track}'.format(track=track, indent=INDENT))
+                        rlconf.append(
+                            '{indent}http-request deny deny_status 429 if {condition}'.format(
+                                condition=condition, indent=INDENT
+                            )
+                        )
+                        ratelimit = '\n'.join(rlconf + [''])
+
                 opts = []
                 for option in loc_conf.get('backend-options', []):
                     # retry-on only available from HAProxy 2.1.
@@ -344,6 +363,7 @@ backend backend-{name}
                     load_balancing_algorithm=self.load_balancing_algorithm,
                     backends='\n'.join(backend_confs),
                     options=options,
+                    ratelimit=ratelimit,
                     indent=INDENT,
                 )
 
diff --git a/reactive/content_cache.py b/reactive/content_cache.py
index 4ddeb7a..d811b14 100644
--- a/reactive/content_cache.py
+++ b/reactive/content_cache.py
@@ -322,6 +322,8 @@ def configure_haproxy():  # NOQA: C901 LP#1825084
             # the X-F-F header in case it's spoofed.
             new_cached_loc_conf['backend-options'].insert(0, 'http-request set-header X-Forwarded-For %[src]')
 
+            new_cached_loc_conf['rate-limit'] = loc_conf.get('rate-limit', '')
+
             # No backends
             if not site_conf['locations'][location].get('backends'):
                 if not new_conf[cached_site]['locations']:
diff --git a/tests/unit/files/config_test_config.txt b/tests/unit/files/config_test_config.txt
index 32ea46c..6c591e0 100644
--- a/tests/unit/files/config_test_config.txt
+++ b/tests/unit/files/config_test_config.txt
@@ -113,7 +113,7 @@ site8.local:
       cache-valid: ['200 1h', '401 15m']
       site-name: auth.site8.local
 
-# Test 9: Custom backend-inter-time
+# Test 9: Custom backend-inter-time and rate limiting
 site9.local:
   locations:
     /:
@@ -124,6 +124,11 @@ site9.local:
       cache-validity: ['200 1h', '401 30m']
       extra-configs:
         - proxy_force_ranges off
+      # https://www.haproxy.com/blog/four-examples-of-haproxy-rate-limiting/
+      rate-limit:
+        condition: "{ sc_http_req_rate(0) gt 20 }"
+        sticky-table: "type ip size 1m expire 30s store http_req_rate(10s)"
+        track: "src"
     /private/content:
       extra-configs:
         - root /srv/example1.com/content/
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output.txt
index f06fc0e..7d7d726 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output.txt
@@ -297,6 +297,9 @@ backend backend-site8-local-2
 
 backend backend-cached-site9-local
     option httpchk GET /_status/content-cache-check HTTP/1.1\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk
+    stick-table type ip size 1m expire 30s store http_req_rate(10s)
+    http-request track-sc0 src
+    http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 }
     http-request set-header Host site9.local
     http-request set-header X-Forwarded-For %[src]
     balance leastconn
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt
index 4435e25..93e026a 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_auto_maxconns.txt
@@ -297,6 +297,9 @@ backend backend-site8-local-2
 
 backend backend-cached-site9-local
     option httpchk GET /_status/content-cache-check HTTP/1.1\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk
+    stick-table type ip size 1m expire 30s store http_req_rate(10s)
+    http-request track-sc0 src
+    http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 }
     http-request set-header Host site9.local
     http-request set-header X-Forwarded-For %[src]
     balance leastconn
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_load_balancing_algorithm.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_load_balancing_algorithm.txt
index e3a0956..2cd7279 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output_load_balancing_algorithm.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_load_balancing_algorithm.txt
@@ -297,6 +297,9 @@ backend backend-site8-local-2
 
 backend backend-cached-site9-local
     option httpchk GET /_status/content-cache-check HTTP/1.1\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk
+    stick-table type ip size 1m expire 30s store http_req_rate(10s)
+    http-request track-sc0 src
+    http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 }
     http-request set-header Host site9.local
     http-request set-header X-Forwarded-For %[src]
     balance roundrobin
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt
index fc0c096..450faaa 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads.txt
@@ -298,6 +298,9 @@ backend backend-site8-local-2
 
 backend backend-cached-site9-local
     option httpchk GET /_status/content-cache-check HTTP/1.1\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk
+    stick-table type ip size 1m expire 30s store http_req_rate(10s)
+    http-request track-sc0 src
+    http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 }
     http-request set-header Host site9.local
     http-request set-header X-Forwarded-For %[src]
     balance leastconn
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads_haproxy2.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads_haproxy2.txt
index 1943512..8905d99 100644
--- a/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads_haproxy2.txt
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_processes_and_threads_haproxy2.txt
@@ -306,6 +306,9 @@ backend backend-site8-local-2
 
 backend backend-cached-site9-local
     option httpchk GET /_status/content-cache-check HTTP/1.1\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk
+    stick-table type ip size 1m expire 30s store http_req_rate(10s)
+    http-request track-sc0 src
+    http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 }
     http-request set-header Host site9.local
     http-request set-header X-Forwarded-For %[src]
     balance leastconn
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_rate_limiting.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_rate_limiting.txt
new file mode 100644
index 0000000..fa3404d
--- /dev/null
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_rate_limiting.txt
@@ -0,0 +1,95 @@
+global
+    nbthread 4
+    maxconn 8192
+    log /dev/log local0
+    log /dev/log local1 notice
+    chroot /var/lib/haproxy
+    stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
+    stats timeout 30s
+    server-state-file /run/haproxy/saved-server-state
+    user haproxy
+    group haproxy
+    daemon
+
+    # LP#1874386: Work around lingering HAProxy processes as per LP:1874386
+    # and kill them off.
+    hard-stop-after 15m
+
+    # Default SSL material locations
+    ca-base /etc/ssl/certs
+    crt-base /etc/ssl/private
+
+    # Default ciphers to use on SSL-enabled listening sockets.
+    # For more information, see ciphers(1SSL). This list is from:
+    #  https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
+    # An alternative list with additional directives can be obtained from
+    #  https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=haproxy
+    ssl-default-bind-ciphers ECDHE+AESGCM:ECDHE+AES256:ECDHE+AES128:!SSLv3:!TLSv1
+    ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11
+    # We'll eventually disable DHE (LP#1825321), but for now, bump DH params
+    tune.ssl.default-dh-param 2048
+
+    # Increase the SSL/TLS session cache from the default 20k. But
+    # rather than hardcode values, let's just set it to match
+    # global_max_connections (which by default is calculated using
+    # num. of CPU cores and num. of configured sites). Each entry
+    # requires ~200 bytes so on a host with say 32 CPUs, 10 sites,
+    # each with 2000 max conns will only consume around 122 Mbytes
+    # (32 * 10 * 2000 * 200), which is not much.
+    tune.ssl.cachesize 8192
+
+defaults
+    log global
+    maxconn 8192
+    mode http
+    option dontlognull
+    timeout connect 5s
+    timeout client  50s
+    timeout server  50s
+    errorfile 400 /etc/haproxy/errors/400.http
+    errorfile 403 /etc/haproxy/errors/403.http
+    errorfile 408 /etc/haproxy/errors/408.http
+    errorfile 500 /etc/haproxy/errors/500.http
+    errorfile 502 /etc/haproxy/errors/502.http
+    errorfile 503 /etc/haproxy/errors/503.http
+    errorfile 504 /etc/haproxy/errors/504.http
+    load-server-state-from-file global
+    unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid
+    unique-id-header X-Cache-Request-ID
+    log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r %ID"
+
+resolvers dns
+    nameserver dns1 127.0.0.53:53
+    resolve_retries 3
+    timeout resolve 3s
+    timeout retry   3s
+    accepted_payload_size 8192
+
+listen  stats
+    bind 127.0.0.1:10000
+    acl allowed_cidr src 127.0.0.0/8
+    http-request deny unless allowed_cidr
+
+    mode http
+    stats enable
+    stats uri /
+    stats realm Haproxy\ Statistics
+    stats auth haproxy:biometricsarenotsecret
+    stats refresh 3
+
+
+listen cached-site1-local
+    bind 0.0.0.0:80
+    bind :::80
+    capture request header X-Cache-Request-ID len 60
+    default_backend backend-cached-site1-local
+
+backend backend-cached-site1-local
+    option httpchk GET /_status/content-cache-check HTTP/1.1\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk
+    stick-table type ip size 1m expire 30s store http_req_rate(10s)
+    http-request track-sc0 src
+    http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 }
+    http-request set-header Host site1.local
+    http-request set-header X-Forwarded-For %[src]
+    balance leastconn
+    server server_1 127.0.0.1:6080 check inter 2s rise 2 fall 60 maxconn 200
diff --git a/tests/unit/files/content_cache_rendered_haproxy_test_output_rate_limiting_missing_condition.txt b/tests/unit/files/content_cache_rendered_haproxy_test_output_rate_limiting_missing_condition.txt
new file mode 100644
index 0000000..037cd9f
--- /dev/null
+++ b/tests/unit/files/content_cache_rendered_haproxy_test_output_rate_limiting_missing_condition.txt
@@ -0,0 +1,92 @@
+global
+    nbthread 4
+    maxconn 8192
+    log /dev/log local0
+    log /dev/log local1 notice
+    chroot /var/lib/haproxy
+    stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
+    stats timeout 30s
+    server-state-file /run/haproxy/saved-server-state
+    user haproxy
+    group haproxy
+    daemon
+
+    # LP#1874386: Work around lingering HAProxy processes as per LP:1874386
+    # and kill them off.
+    hard-stop-after 15m
+
+    # Default SSL material locations
+    ca-base /etc/ssl/certs
+    crt-base /etc/ssl/private
+
+    # Default ciphers to use on SSL-enabled listening sockets.
+    # For more information, see ciphers(1SSL). This list is from:
+    #  https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
+    # An alternative list with additional directives can be obtained from
+    #  https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=haproxy
+    ssl-default-bind-ciphers ECDHE+AESGCM:ECDHE+AES256:ECDHE+AES128:!SSLv3:!TLSv1
+    ssl-default-bind-options no-sslv3 no-tlsv10 no-tlsv11
+    # We'll eventually disable DHE (LP#1825321), but for now, bump DH params
+    tune.ssl.default-dh-param 2048
+
+    # Increase the SSL/TLS session cache from the default 20k. But
+    # rather than hardcode values, let's just set it to match
+    # global_max_connections (which by default is calculated using
+    # num. of CPU cores and num. of configured sites). Each entry
+    # requires ~200 bytes so on a host with say 32 CPUs, 10 sites,
+    # each with 2000 max conns will only consume around 122 Mbytes
+    # (32 * 10 * 2000 * 200), which is not much.
+    tune.ssl.cachesize 8192
+
+defaults
+    log global
+    maxconn 8192
+    mode http
+    option dontlognull
+    timeout connect 5s
+    timeout client  50s
+    timeout server  50s
+    errorfile 400 /etc/haproxy/errors/400.http
+    errorfile 403 /etc/haproxy/errors/403.http
+    errorfile 408 /etc/haproxy/errors/408.http
+    errorfile 500 /etc/haproxy/errors/500.http
+    errorfile 502 /etc/haproxy/errors/502.http
+    errorfile 503 /etc/haproxy/errors/503.http
+    errorfile 504 /etc/haproxy/errors/504.http
+    load-server-state-from-file global
+    unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid
+    unique-id-header X-Cache-Request-ID
+    log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r %ID"
+
+resolvers dns
+    nameserver dns1 127.0.0.53:53
+    resolve_retries 3
+    timeout resolve 3s
+    timeout retry   3s
+    accepted_payload_size 8192
+
+listen  stats
+    bind 127.0.0.1:10000
+    acl allowed_cidr src 127.0.0.0/8
+    http-request deny unless allowed_cidr
+
+    mode http
+    stats enable
+    stats uri /
+    stats realm Haproxy\ Statistics
+    stats auth haproxy:biometricsarenotsecret
+    stats refresh 3
+
+
+listen cached-site1-local
+    bind 0.0.0.0:80
+    bind :::80
+    capture request header X-Cache-Request-ID len 60
+    default_backend backend-cached-site1-local
+
+backend backend-cached-site1-local
+    option httpchk GET /_status/content-cache-check HTTP/1.1\r\nHost:\ site1.local\r\nUser-Agent:\ haproxy/httpchk
+    http-request set-header Host site1.local
+    http-request set-header X-Forwarded-For %[src]
+    balance leastconn
+    server server_1 127.0.0.1:6080 check inter 2s rise 2 fall 60 maxconn 200
diff --git a/tests/unit/files/haproxy_config_rendered_backends_stanzas_test_output.txt b/tests/unit/files/haproxy_config_rendered_backends_stanzas_test_output.txt
index bdb8064..565c469 100644
--- a/tests/unit/files/haproxy_config_rendered_backends_stanzas_test_output.txt
+++ b/tests/unit/files/haproxy_config_rendered_backends_stanzas_test_output.txt
@@ -63,6 +63,9 @@ backend backend-site8-local-2
 
 backend backend-site9-local
     option httpchk HEAD / HTTP/1.1\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk
+    stick-table type ip size 1m expire 30s store http_req_rate(10s)
+    http-request track-sc0 src
+    http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 }
     http-request set-header Host site9.local
     balance leastconn
     server server_1 127.0.1.15:80 check inter 1m rise 2 fall 5 maxconn 200
diff --git a/tests/unit/files/haproxy_config_rendered_test_output.txt b/tests/unit/files/haproxy_config_rendered_test_output.txt
index 4f3e3a7..5020ee0 100644
--- a/tests/unit/files/haproxy_config_rendered_test_output.txt
+++ b/tests/unit/files/haproxy_config_rendered_test_output.txt
@@ -172,6 +172,9 @@ backend backend-site8-local-2
 
 backend backend-site9-local
     option httpchk HEAD / HTTP/1.1\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk
+    stick-table type ip size 1m expire 30s store http_req_rate(10s)
+    http-request track-sc0 src
+    http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 }
     http-request set-header Host site9.local
     balance leastconn
     server server_1 127.0.1.15:80 check inter 1m rise 2 fall 5 maxconn 200
diff --git a/tests/unit/files/haproxy_config_rendered_test_output2.txt b/tests/unit/files/haproxy_config_rendered_test_output2.txt
index 0020879..4071e23 100644
--- a/tests/unit/files/haproxy_config_rendered_test_output2.txt
+++ b/tests/unit/files/haproxy_config_rendered_test_output2.txt
@@ -172,6 +172,9 @@ backend backend-site8-local-2
 
 backend backend-site9-local
     option httpchk HEAD / HTTP/1.1\r\nHost:\ site9.local\r\nUser-Agent:\ haproxy/httpchk
+    stick-table type ip size 1m expire 30s store http_req_rate(10s)
+    http-request track-sc0 src
+    http-request deny deny_status 429 if { sc_http_req_rate(0) gt 20 }
     http-request set-header Host site9.local
     balance leastconn
     server server_1 127.0.1.15:80 check inter 1m rise 2 fall 5 maxconn 200
diff --git a/tests/unit/test_content_cache.py b/tests/unit/test_content_cache.py
index f5f32a2..41e098c 100644
--- a/tests/unit/test_content_cache.py
+++ b/tests/unit/test_content_cache.py
@@ -685,6 +685,59 @@ site1.local:
     @mock.patch('charms.reactive.set_flag')
     @mock.patch('lib.haproxy.HAProxyConf.save_server_state')
     @mock.patch('reactive.content_cache.update_logrotate')
+    def test_configure_haproxy_sites_rate_limiting(self, logrotation, save_s_state, set_flag, opened_ports):
+        config = '''
+site1.local:
+  locations:
+    /:
+      rate-limit:
+        condition: "{ sc_http_req_rate(0) gt 20 }"
+        sticky-table: "type ip size 1m expire 30s store http_req_rate(10s)"
+        track: "src"
+'''
+        self.mock_config.return_value = {'haproxy_hard_stop_after': '15m', 'max_connections': 8192, 'sites': config}
+        with mock.patch('lib.haproxy.HAProxyConf.conf_file', new_callable=mock.PropertyMock) as mock_conf_file:
+            mock_conf_file.return_value = os.path.join(self.tmpdir, 'haproxy.cfg')
+            opened_ports.return_value = ['443/tcp']
+            content_cache.configure_haproxy()
+
+            with open(
+                'tests/unit/files/content_cache_rendered_haproxy_test_output_rate_limiting.txt', 'r', encoding='utf-8'
+            ) as f:
+                want = f.read()
+            with open(os.path.join(self.tmpdir, 'haproxy.cfg'), 'r', encoding='utf-8') as f:
+                got = f.read()
+            self.assertEqual(got, want)
+
+        # Missing 'condition'
+        config = '''
+site1.local:
+  locations:
+    /:
+      rate-limit:
+        sticky-table: "type ip size 1m expire 30s store http_req_rate(10s)"
+'''
+        self.mock_config.return_value = {'haproxy_hard_stop_after': '15m', 'max_connections': 8192, 'sites': config}
+        with mock.patch('lib.haproxy.HAProxyConf.conf_file', new_callable=mock.PropertyMock) as mock_conf_file:
+            mock_conf_file.return_value = os.path.join(self.tmpdir, 'haproxy.cfg')
+            opened_ports.return_value = ['443/tcp']
+            content_cache.configure_haproxy()
+
+            with open(
+                'tests/unit/files/content_cache_rendered_haproxy_test_output_rate_limiting_missing_condition.txt',
+                'r',
+                encoding='utf-8',
+            ) as f:
+                want = f.read()
+            with open(os.path.join(self.tmpdir, 'haproxy.cfg'), 'r', encoding='utf-8') as f:
+                got = f.read()
+            self.assertEqual(got, want)
+
+    @freezegun.freeze_time("2019-03-22", tz_offset=0)
+    @mock.patch('charmhelpers.core.hookenv.opened_ports')
+    @mock.patch('charms.reactive.set_flag')
+    @mock.patch('lib.haproxy.HAProxyConf.save_server_state')
+    @mock.patch('reactive.content_cache.update_logrotate')
     def test_configure_haproxy_processes_and_threads(self, logrotation, save_server_state, set_flag, opened_ports):
         with open('tests/unit/files/config_test_config.txt', 'r', encoding='utf-8') as f:
             ngx_config = f.read()