Merge ~cgrabowski/maas:dns_performance into maas:master

[Christian Grabowski]

add DNS performance metrics

add DNS full reload perf test

[MAAS Lander (maas-lander)]

UNIT TESTS
-b dns_performance lp:~cgrabowski/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas-tester/1747/consoleText
COMMIT: ce24390ca5d5beae5e75b41bee720ff21f128ac0

review: Needs Fixing

[MAAS Lander (maas-lander)]

UNIT TESTS
-b dns_performance lp:~cgrabowski/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas-tester/1757/consoleText
COMMIT: 667eaadd29b3238d269cd031a402403654c29ae6

review: Needs Fixing

[Alberto Donato (ack)]

+1

minor comments inline

review: Approve

036eb11... by Christian Grabowski on 2023-01-17

use monkeypatch to set env

[MAAS Lander (maas-lander)]

UNIT TESTS
-b dns_performance lp:~cgrabowski/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas-tester/1780/consoleText
COMMIT: e89b0ea208c551bcb61e65e3a85d832e2692f3c5

review: Needs Fixing

29d2a56... by Christian Grabowski on 2023-01-17

add missing metric update actions

[MAAS Lander (maas-lander)]

UNIT TESTS
-b dns_performance lp:~cgrabowski/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas-tester/1782/consoleText
COMMIT: 29d2a560306e1585b6f15fbbe2dad06a0a05d2fd

review: Needs Fixing

[Christian Grabowski (cgrabowski)]

jenkins: !test

[MAAS Lander (maas-lander)]

UNIT TESTS
-b dns_performance lp:~cgrabowski/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: FAILED
LOG: http://maas-ci.internal:8080/job/maas-tester/1785/consoleText
COMMIT: 29d2a560306e1585b6f15fbbe2dad06a0a05d2fd

review: Needs Fixing

[Christian Grabowski (cgrabowski)]

jenkins: !test

[MAAS Lander (maas-lander)]

UNIT TESTS
-b dns_performance lp:~cgrabowski/maas/+git/maas into -b master lp:~maas-committers/maas

STATUS: SUCCESS
COMMIT: 29d2a560306e1585b6f15fbbe2dad06a0a05d2fd

review: Approve

diff --git a/src/maasperf/tests/maasserver/dns/__init__.py b/src/maasperf/tests/maasserver/dns/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/maasperf/tests/maasserver/dns/__init__.py
diff --git a/src/maasperf/tests/maasserver/dns/conftest.py b/src/maasperf/tests/maasserver/dns/conftest.py
new file mode 100644
index 0000000..796bbd5
--- /dev/null
+++ b/src/maasperf/tests/maasserver/dns/conftest.py
@@ -0,0 +1,29 @@
+# Copyright 2023 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+from tempfile import mkdtemp
+
+import pytest
+
+from provisioningserver.testing.bindfixture import BINDServer
+
+
[e-mail]()
+def dns_config_path(monkeypatch):
+    path = mkdtemp(prefix="maas-dns-config")
+    monkeypatch.setenv("MAAS_BIND_CONFIG_DIR", path)
+    return path
+
+
[e-mail]()
+def zone_file_config_path(monkeypatch):
+    path = mkdtemp(prefix="maas-zonefile-config")
+    monkeypatch.setenv("MAAS_ZONE_FILE_CONFIG_DIR", path)
+    return path
+
+
[e-mail]()
+def bind_server():
+    bind = BINDServer()
+    bind.setUp()
+    return bind
diff --git a/src/maasperf/tests/maasserver/dns/test_config.py b/src/maasperf/tests/maasserver/dns/test_config.py
new file mode 100644
index 0000000..7609cae
--- /dev/null
+++ b/src/maasperf/tests/maasserver/dns/test_config.py
@@ -0,0 +1,52 @@
+# Copyright 2023 Canonical Ltd.  This software is licensed under the
+# GNU Affero General Public License version 3 (see the file LICENSE).
+
+import pytest
+
+from maasserver.dns.config import (
+    dns_update_all_zones,
+    process_dns_update_notify,
+)
+from provisioningserver.dns.config import DynamicDNSUpdate
+
+
[e-mail]("maasdb")
+def test_perf_full_dns_reload(
+    perf, dns_config_path, zone_file_config_path, bind_server, factory
+):
+    domains = [factory.make_Domain() for _ in range(5)]
+    subnet = factory.make_Subnet(cidr="10.0.0.0/24")
+    ips = [factory.make_StaticIPAddress(subnet=subnet) for _ in range(100)]
+    [
+        factory.make_DNSResource(domain=domains[i % 5], ip_addresses=[ips[i]])
+        for i in range(100)
+    ]
+
+    with perf.record("test_perf_full_dns_reload.zonefile_write"):
+        dns_update_all_zones()
+
+    # zonefile already written, sends dynamic update instead
+    with perf.record("test_perf_full_dns_reload.dynamic_update"):
+        dns_update_all_zones()
+
+
[e-mail]("maasdb")
+def test_perf_generate_dns_updates(perf, factory):
+    domain = factory.make_Domain()
+    subnet = factory.make_Subnet(cidr="10.0.0.0/24")
+    ips = [factory.make_StaticIPAddress(subnet=subnet) for _ in range(100)]
+    records = [
+        factory.make_DNSResource(domain=domain, ip_addresses=[ips[i]])
+        for i in range(100)
+    ]
+
+    with perf.record("test_perf_generate_dns_updates.forward"):
+        for record in records:
+            notify = f"INSERT {domain.name} {record.name} A 30 {record.ip_addresses.first()}"
+            process_dns_update_notify(notify)
+
+    with perf.record("test_perf_generate_dns_updates.reverse"):
+        for record in records:
+            notify = f"INSERT {domain.name} {record.name} A 30 {record.ip_addresses.first()}"
+            fwd, _ = process_dns_update_notify(notify)
+            DynamicDNSUpdate.as_reverse_record_update(fwd[0], subnet.cidr)
diff --git a/src/maasserver/dns/config.py b/src/maasserver/dns/config.py
index a0390c0..571caf1 100644
--- a/src/maasserver/dns/config.py
+++ b/src/maasserver/dns/config.py
@@ -33,6 +33,7 @@ from provisioningserver.dns.actions import (
 )
 from provisioningserver.dns.config import DynamicDNSUpdate
 from provisioningserver.logger import get_maas_logger
+from provisioningserver.prometheus.metrics import PROMETHEUS_METRICS
 from provisioningserver.utils.shell import ExternalProcessError
 
 maaslog = get_maas_logger("dns")
@@ -66,6 +67,12 @@ def forward_domains_to_forwarded_zones(forward_domains):
     ]
 
 
+@PROMETHEUS_METRICS.record_call_latency(
+    "maas_dns_update_latency",
+    get_labels=lambda *args, **kwargs: {
+        "update_type": "reload" if kwargs.get("requires_reload") else "dynamic"
+    },
+)
 def dns_update_all_zones(
     reload_retry=False,
     reload_timeout=2,
diff --git a/src/provisioningserver/dns/zoneconfig.py b/src/provisioningserver/dns/zoneconfig.py
index 8dec9b6..dec6029 100644
--- a/src/provisioningserver/dns/zoneconfig.py
+++ b/src/provisioningserver/dns/zoneconfig.py
@@ -18,6 +18,7 @@ from provisioningserver.dns.config import (
     render_dns_template,
     report_missing_config_dir,
 )
+from provisioningserver.prometheus.metrics import PROMETHEUS_METRICS
 from provisioningserver.utils.fs import incremental_write
 from provisioningserver.utils.network import (
     intersect_iprange,
@@ -330,6 +331,11 @@ class DNSForwardZoneConfig(DomainConfigBase):
             )
             if not self.force_config_write and self.zone_file_exists(zi):
                 self.dynamic_update(zi)
+                PROMETHEUS_METRICS.update(
+                    "maas_dns_dynamic_update_count",
+                    "inc",
+                    labels={"zone": self.domain},
+                )
             else:
                 Path(f"{zi.target_path}.jnl").unlink(missing_ok=True)
                 self.requires_reload = True
@@ -351,6 +357,11 @@ class DNSForwardZoneConfig(DomainConfigBase):
                         "generate_directives": {"A": generate_directives},
                     },
                 )
+                PROMETHEUS_METRICS.update(
+                    "maas_dns_full_zonefile_write_count",
+                    "inc",
+                    labels={"zone": self.domain},
+                )
 
 
 class DNSReverseZoneConfig(DomainConfigBase):
@@ -620,6 +631,11 @@ class DNSReverseZoneConfig(DomainConfigBase):
             )
             if not self.force_config_write and self.zone_file_exists(zi):
                 self.dynamic_update(zi)
+                PROMETHEUS_METRICS.update(
+                    "maas_dns_dynamic_update_count",
+                    "inc",
+                    labels={"zone": self.domain},
+                )
             else:
                 Path(f"{zi.target_path}.jnl").unlink(missing_ok=True)
                 self.requires_reload = True
@@ -643,3 +659,8 @@ class DNSReverseZoneConfig(DomainConfigBase):
                         },
                     },
                 )
+                PROMETHEUS_METRICS.update(
+                    "maas_dns_full_zonefile_write_count",
+                    "inc",
+                    labels={"zone": self.domain},
+                )
diff --git a/src/provisioningserver/prometheus/metrics.py b/src/provisioningserver/prometheus/metrics.py
index bc2e6bb..34fe068 100644
--- a/src/provisioningserver/prometheus/metrics.py
+++ b/src/provisioningserver/prometheus/metrics.py
@@ -117,6 +117,32 @@ METRICS_DEFINITIONS = [
         of connections
         """,
     ),
+    MetricDefinition(
+        "Counter",
+        "maas_dns_full_zonefile_write_count",
+        """
+        counts the number of times MAAS writes
+        to a zonefile rather than push a dynamic
+        update per DNS zone
+        """,
+        ["zone"],
+    ),
+    MetricDefinition(
+        "Counter",
+        "maas_dns_dynamic_update_count",
+        """
+        counts the number of times MAAS pushes
+        a dynamic update per DNS zone rather than
+        writing a zonefile
+        """,
+        ["zone"],
+    ),
+    MetricDefinition(
+        "Histogram",
+        "maas_dns_update_latency",
+        "the time it takes MAAS to update BIND",
+        ["update_type"],
+    ),
     # Common metrics
     *node_metrics_definitions(),
 ]