Merge ~troyanov/maas:backport-76e0fb0-3.5 into maas:3.5

[Anton Troyanov]

feat(temporal): enable mTLS

Use MAAS cluster key/cert and CA to configure Temporal mTLS [0]

[0]: https://docs.temporal.io/self-hosted-guide/security#encryption-in-transit-with-mtls

Resolves LP:2058332

(cherry picked from commit 76e0fb0351f31aa9e4f3c87a78792c8cec20ef0d)

[Anton Troyanov (troyanov)]

Self approving backport

review: Approve

diff --git a/debian/maas-region-api.maas-temporal.service b/debian/maas-region-api.maas-temporal.service
index f99026f..91aa21c 100644
--- a/debian/maas-region-api.maas-temporal.service
+++ b/debian/maas-region-api.maas-temporal.service
@@ -9,7 +9,8 @@ ConditionPathIsDirectory=/var/lib/maas/temporal
 User=maas
 Group=maas
 KillMode=mixed
-# XXX: usage of --allow-no-auth is temporary for experimental Temporal support
+
+# --allow-no-auth flag to allow noopAuthorizer
 ExecStart=/usr/sbin/temporal-server \
     -e production \
     -r "/var/lib/maas/temporal/" \
diff --git a/src/maasagent/cmd/maas-agent/main.go b/src/maasagent/cmd/maas-agent/main.go
index d2b6490..5ff28ea 100644
--- a/src/maasagent/cmd/maas-agent/main.go
+++ b/src/maasagent/cmd/maas-agent/main.go
@@ -7,6 +7,8 @@ package main
 
 import (
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"os"
 	"os/signal"
@@ -86,6 +88,22 @@ func Run() int {
 	clientBackoff := backoff.NewExponentialBackOff()
 	clientBackoff.MaxElapsedTime = 60 * time.Second
 
+	certsDir := getCertificatesDir()
+
+	cert, err := tls.LoadX509KeyPair(fmt.Sprintf("%s/cluster.pem", certsDir), fmt.Sprintf("%s/cluster.key", certsDir))
+	if err != nil {
+		log.Error().Err(err).Msg("Failed loading client cert and key")
+	}
+
+	ca := x509.NewCertPool()
+
+	b, err := os.ReadFile(fmt.Sprintf("%s/cacerts.pem", certsDir))
+	if err != nil {
+		log.Error().Err(err).Msg("Failed reading CA")
+	} else if !ca.AppendCertsFromPEM(b) {
+		log.Error().Err(err).Msg("CA PEM file is invalid")
+	}
+
 	temporalClient, err := backoff.RetryWithData(
 		func() (client.Client, error) {
 			return client.Dial(client.Options{
@@ -97,6 +115,14 @@ func Run() int {
 					converter.GetDefaultDataConverter(),
 					codec,
 				),
+				ConnectionOptions: client.ConnectionOptions{
+					TLS: &tls.Config{
+						MinVersion:   tls.VersionTLS12,
+						Certificates: []tls.Certificate{cert},
+						RootCAs:      ca,
+						ServerName:   "maas",
+					},
+				},
 			})
 		}, clientBackoff,
 	)
@@ -227,6 +253,16 @@ func getRunDir() string {
 	return "/run/maas"
 }
 
+func getCertificatesDir() string {
+	dataDir := os.Getenv("SNAP_DATA")
+
+	if dataDir != "" {
+		return fmt.Sprintf("%s/certificates", dataDir)
+	}
+
+	return "/var/lib/maas/certificates"
+}
+
 func main() {
 	os.Exit(Run())
 }
diff --git a/src/maasagent/cmd/maas-agent/main_test.go b/src/maasagent/cmd/maas-agent/main_test.go
index 505b348..8c5f0df 100644
--- a/src/maasagent/cmd/maas-agent/main_test.go
+++ b/src/maasagent/cmd/maas-agent/main_test.go
@@ -34,3 +34,33 @@ func TestGetRunDir(t *testing.T) {
 		})
 	}
 }
+
+func TestCertificatesDir(t *testing.T) {
+	testcases := map[string]struct {
+		in  func(t *testing.T)
+		out string
+	}{
+		"snap": {
+			in: func(t *testing.T) {
+				t.Setenv("SNAP_DATA", "/var/snap/maas/x1")
+			},
+			out: "/var/snap/maas/x1/certificates",
+		},
+		"deb": {
+			in: func(t *testing.T) {
+				t.Setenv("SNAP_DATA", "")
+			}, out: "/var/lib/maas/certificates",
+		},
+	}
+
+	for name, tc := range testcases {
+		tc := tc
+
+		t.Run(name, func(t *testing.T) {
+			tc.in(t)
+
+			res := getCertificatesDir()
+			assert.Equal(t, tc.out, res)
+		})
+	}
+}
diff --git a/src/maasserver/regiondservices/temporal.py b/src/maasserver/regiondservices/temporal.py
index 5220dac..c00eeb3 100644
--- a/src/maasserver/regiondservices/temporal.py
+++ b/src/maasserver/regiondservices/temporal.py
@@ -17,6 +17,7 @@ from twisted.internet.defer import inlineCallbacks
 from maasserver.config import RegionConfiguration
 from maasserver.service_monitor import service_monitor
 from maasserver.utils import load_template
+from provisioningserver.certificates import get_maas_cluster_cert_paths
 from provisioningserver.logger import get_maas_logger
 from provisioningserver.path import get_maas_data_path
 from provisioningserver.utils.env import MAAS_ID
@@ -33,6 +34,9 @@ class RegionTemporalService(Service):
     def _configure(self):
         """Update the Temporal configuration for the Temporal service."""
         template = load_template("temporal", "production.yaml.template")
+        dynamic_template = load_template(
+            "temporal", "production-dynamic.yaml.template"
+        )
 
         # Can't use the public attribute since it hits
         # maasserver.utils.orm.DisabledDatabaseConnection
@@ -64,6 +68,15 @@ class RegionTemporalService(Service):
                     f"Please consider setting it manually using regiond.conf"
                 )
 
+        temporal_config_dir = Path(
+            os.environ.get(
+                "MAAS_TEMPORAL_CONFIG_DIR", get_maas_data_path("temporal")
+            )
+        )
+        temporal_config_dir.mkdir(parents=True, exist_ok=True)
+
+        cert_file, key_file, cacert_file = get_maas_cluster_cert_paths()
+
         environ = {
             "database": dbconf["NAME"],
             "user": dbconf.get("USER", ""),
@@ -71,20 +84,27 @@ class RegionTemporalService(Service):
             "address": f"{host}:{dbconf['PORT']}",
             "connect_attributes": connection_attributes,
             "broadcast_address": broadcast_address,
+            "config_dir": str(temporal_config_dir),
+            "cert_file": cert_file,
+            "key_file": key_file,
+            "cacert_file": cacert_file,
         }
 
-        rendered = template.substitute(environ).encode()
+        rendered_template = template.substitute(environ).encode()
+        rendered_dynamic_template = dynamic_template.substitute(
+            environ
+        ).encode()
 
-        temporal_config_dir = Path(
-            os.environ.get(
-                "MAAS_TEMPORAL_CONFIG_DIR", get_maas_data_path("temporal")
-            )
+        atomic_write(
+            rendered_template,
+            temporal_config_dir / "production.yaml",
+            overwrite=True,
+            mode=0o600,
         )
-        temporal_config_dir.mkdir(parents=True, exist_ok=True)
 
         atomic_write(
-            rendered,
-            temporal_config_dir / "production.yaml",
+            rendered_dynamic_template,
+            temporal_config_dir / "production-dynamic.yaml",
             overwrite=True,
             mode=0o600,
         )
diff --git a/src/maasserver/templates/temporal/production-dynamic.yaml.template b/src/maasserver/templates/temporal/production-dynamic.yaml.template
new file mode 100644
index 0000000..177adfd
--- /dev/null
+++ b/src/maasserver/templates/temporal/production-dynamic.yaml.template
@@ -0,0 +1,4 @@
+# EnableRingpopTLS controls whether to use TLS for ringpop, using the same "internode" TLS
+# config as the other services.
+system.enableRingpopTLS:
+  - value: true
diff --git a/src/maasserver/templates/temporal/production.yaml.template b/src/maasserver/templates/temporal/production.yaml.template
index 7cf4ba7..ae8e478 100644
--- a/src/maasserver/templates/temporal/production.yaml.template
+++ b/src/maasserver/templates/temporal/production.yaml.template
@@ -1,6 +1,6 @@
 log:
   stdout: true
-  level: info
+  level: warn
 
 persistence:
   defaultStore: default
@@ -48,6 +48,40 @@ global:
   pprof:
     # disable pprof
     port: 0
+  tls:
+    refreshInterval: 0s
+    internode:
+      # This server section configures the TLS certificate that internal temporal
+      # cluster nodes (history, matching, and internal-frontend) present to other
+      # clients within the Temporal Cluster.
+      server:
+        requireClientAuth: true
+        certFile: {{ cert_file }}
+        keyFile: {{ key_file }}
+        clientCaFiles:
+          - {{ cacert_file }}
+      # This client section is used to configure the TLS clients within
+      # the Temporal Cluster that connect to an Internode (history, matching, or
+      # internal-frontend)
+      client:
+        serverName: maas
+        disableHostVerification: false
+        rootCaFiles:
+          - {{ cacert_file }}
+    frontend:
+      # This server section configures the TLS certificate that the Frontend
+      # server presents to external clients.
+      server:
+        requireClientAuth: true
+        certFile: {{ cert_file }}
+        keyFile: {{ key_file }}
+        clientCaFiles:
+          - {{ cacert_file }}
+      client:
+        serverName: maas
+        disableHostVerification: false
+        rootCaFiles:
+          - {{ cacert_file }}
   metrics:
     prometheus:
       framework: "tally"
@@ -92,3 +126,6 @@ clusterMetadata:
       rpcName: "frontend"
       rpcAddress: "localhost:7233"
 
+dynamicConfigClient:
+  filepath: "{{ config_dir }}/production-dynamic.yaml"
+  pollInterval: "60s"
diff --git a/src/maasserver/tests/test_commands_config_tls.py b/src/maasserver/tests/test_commands_config_tls.py
index f7bcd2f..be2c040 100644
--- a/src/maasserver/tests/test_commands_config_tls.py
+++ b/src/maasserver/tests/test_commands_config_tls.py
@@ -100,7 +100,7 @@ class TestConfigTLSCommand(MAASServerTestCase):
     def test_config_tls_enable_with_cacert(self):
         sample_cert = get_sample_cert_with_cacerts()
         cert_path, key_path = sample_cert.tempfiles()
-        cacert_path = Path(self.make_dir()) / "cacert.pem"
+        cacert_path = Path(self.make_dir()) / "cacerts.pem"
         cacert_path.write_text(sample_cert.ca_certificates_pem())
 
         call_command(
diff --git a/src/maasserver/workflow/worker/worker.py b/src/maasserver/workflow/worker/worker.py
index ec0391e..99743c8 100644
--- a/src/maasserver/workflow/worker/worker.py
+++ b/src/maasserver/workflow/worker/worker.py
@@ -12,13 +12,14 @@ from temporalio.api.workflowservice.v1 import (
     DescribeNamespaceRequest,
     RegisterNamespaceRequest,
 )
-from temporalio.client import Client
+from temporalio.client import Client, TLSConfig
 import temporalio.converter
 from temporalio.service import RPCError, RPCStatusCode
 from temporalio.worker import Worker as TemporalWorker
 
 from maasserver.utils.asynchronous import async_retry
 from maasserver.workflow.codec.encryptor import EncryptionCodec
+from provisioningserver.certificates import get_maas_cluster_cert_paths
 from provisioningserver.utils.env import MAAS_ID, MAAS_SHARED_SECRET
 
 REGION_TASK_QUEUE = "region-internal"
@@ -32,6 +33,15 @@ TEMPORAL_NAMESPACE = "default"
 async def get_client_async() -> Client:
     maas_id = MAAS_ID.get()
     pid = os.getpid()
+    cert_file, key_file, cacert_file = get_maas_cluster_cert_paths()
+
+    with open(cert_file, "rb") as f:
+        cert = f.read()
+    with open(key_file, "rb") as f:
+        key = f.read()
+    with open(cacert_file, "rb") as f:
+        cacert = f.read()
+
     return await Client.connect(
         f"{TEMPORAL_HOST}:{TEMPORAL_PORT}",
         identity=f"{maas_id}@region:{pid}",
@@ -39,6 +49,12 @@ async def get_client_async() -> Client:
             temporalio.converter.default(),
             payload_codec=EncryptionCodec(MAAS_SHARED_SECRET.get().encode()),
         ),
+        tls=TLSConfig(
+            domain="maas",
+            server_root_ca_cert=cacert,
+            client_cert=cert,
+            client_private_key=key,
+        ),
     )