Merge ~hloeung/smtp-relay-charm:mx into smtp-relay-charm:master

[Haw Loeung]

Add support for running as MX by allowing transport and virtual alias maps

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Colin Misare (cmisare)]

One nitpick but LGTM either way

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision c6c92aa26414b5c6f3f34f1daf5853f7af24ee73

[Haw Loeung (hloeung)]

Follow up fixes in https://code.launchpad.net/~hloeung/smtp-relay-charm/+git/smtp-relay-charm/+merge/416360

diff --git a/config.yaml b/config.yaml
index 19eab05..b3d6470 100644
--- a/config.yaml
+++ b/config.yaml
@@ -159,3 +159,21 @@ options:
       outbound to other MTAs as it may cause deliverability issues.
 
       http://www.postfix.org/postconf.5.html#smtpd_tls_security_level
+  transport_maps:
+    type: string
+    default: ''
+    description: |
+      Optional lookup tables with mappings from recipient address to
+      message delivery transport or next-hop destination (use 'MANUAL'
+      to indicate it is managed outside of juju configs).
+
+      http://www.postfix.org/postconf.5.html#transport_maps
+  virtual_alias_maps:
+    type: string
+    default: ''
+    description: |
+      Optional lookup tables that alias specific mail addresses or
+      domains to other local or remote addresses (use 'MANUAL' to
+      indicate it is managed outside of juju configs).
+
+      http://www.postfix.org/postconf.5.html#virtual_alias_maps
diff --git a/reactive/smtp_relay.py b/reactive/smtp_relay.py
index 8fb59c3..58ec8b3 100644
--- a/reactive/smtp_relay.py
+++ b/reactive/smtp_relay.py
@@ -122,6 +122,8 @@ def configure_smtp_auth(dovecot_config='/etc/dovecot/dovecot.conf', dovecot_user
     'config.changed.tls_policy_maps',
     'config.changed.tls_protocols',
     'config.changed.tls_security_level',
+    'config.changed.transport_maps',
+    'config.changed.virtual_alias_maps',
 )
 def config_changed():
     reactive.clear_flag('smtp-relay.configured')
@@ -152,7 +154,9 @@ def _create_update_map(content, postmap):
             os.utime(pmfname, None)
         changed = True
 
-    if not content.startswith('MANUAL'):
+    if content.startswith('MANUAL'):
+        hookenv.log('Map {} manually managed'.format(pmfname))
+    else:
         contents = JUJU_HEADER + content + '\n'
         changed = _write_file(contents, pmfname) or changed
 
@@ -220,6 +224,8 @@ def configure_smtp_relay(postfix_conf_dir='/etc/postfix', tls_dh_params='/etc/ss
         'tls_exclude_ciphers': config['tls_exclude_ciphers'],
         'tls_protocols': config['tls_protocols'],
         'tls_security_level': config['tls_security_level'],
+        'transport_maps': True if config['transport_maps'] else False,
+        'virtual_alias_maps': True if config['virtual_alias_maps'] else False,
     }
     base = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
     env = jinja2.Environment(loader=jinja2.FileSystemLoader(base))
@@ -237,6 +243,8 @@ def configure_smtp_relay(postfix_conf_dir='/etc/postfix', tls_dh_params='/etc/ss
         'sender_access': 'hash:{}'.format(os.path.join(postfix_conf_dir, 'access')),
         'sender_login_maps': 'hash:{}'.format(os.path.join(postfix_conf_dir, 'sender_login')),
         'tls_policy_maps': 'hash:{}'.format(os.path.join(postfix_conf_dir, 'tls_policy')),
+        'transport_maps': 'hash:{}'.format(os.path.join(postfix_conf_dir, 'transport')),
+        'virtual_alias_maps': 'hash:{}'.format(os.path.join(postfix_conf_dir, 'virtual_alias')),
     }
     sender_access_content = config['restrict_sender_access']
     if sender_access_content and not sender_access_content.startswith('MANUAL'):
@@ -251,6 +259,8 @@ def configure_smtp_relay(postfix_conf_dir='/etc/postfix', tls_dh_params='/etc/ss
         'sender_access': sender_access_content,
         'sender_login_maps': config['sender_login_maps'],
         'tls_policy_maps': config['tls_policy_maps'],
+        'transport_maps': config['transport_maps'],
+        'virtual_alias_maps': config['virtual_alias_maps'],
     }
 
     # Ensure various maps exists before starting/restarting postfix.
@@ -365,9 +375,7 @@ def _write_file(source, dest_path, perms=0o644, owner=None, group=None):
     """Write file only on changes and return True if changes written."""
     # Compare and only write out file on change.
     dest = ''
-    if not os.path.exists(dest_path):
-        with open(dest_path, 'a') as f:
-            os.utime(dest_path, None)
+
     try:
         with open(dest_path, 'r') as f:
             dest = f.read()
diff --git a/templates/postfix_main_cf.tmpl b/templates/postfix_main_cf.tmpl
index a555c4b..c92ee6f 100644
--- a/templates/postfix_main_cf.tmpl
+++ b/templates/postfix_main_cf.tmpl
@@ -98,6 +98,14 @@ restricted = check_recipient_access hash:/etc/postfix/restricted_recipients, rej
 smtpd_milters = {{milter}}
 non_smtpd_milters = {{milter}}
 {% endif -%}
-{% if header_checks %}
+
+{%- if header_checks %}
 header_checks = regexp:/etc/postfix/header_checks
+{% endif -%}
+
+{%- if transport_maps %}
+transport_maps = hash:/etc/postfix/transport
+{% endif %}
+{%- if virtual_alias_maps %}
+virtual_alias_maps = hash:/etc/postfix/virtual_alias
 {% endif %}
diff --git a/tests/unit/files/postfix_main_transport_maps.cf b/tests/unit/files/postfix_main_transport_maps.cf
new file mode 100644
index 0000000..a80d6b8
--- /dev/null
+++ b/tests/unit/files/postfix_main_transport_maps.cf
@@ -0,0 +1,59 @@
+## This file is Juju managed - do not edit by hand #
+
+
+smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu)
+biff = no
+
+append_dot_mydomain = no
+
+readme_directory = no
+
+# See http://www.postfix.org/COMPATIBILITY_README.html -- default to 2 on
+# fresh installs.
+compatibility_level = 2
+
+# TLS parameters
+tls_preempt_cipherlist = yes
+smtpd_tls_cert_file = /etc/ssl/certs/ssl-cert-snakeoil.pem
+smtpd_tls_key_file = /etc/ssl/private/ssl-cert-snakeoil.key
+smtpd_tls_dh1024_param_file = /etc/ssl/private/dhparams.pem
+smtpd_tls_ciphers = HIGH
+smtpd_tls_exclude_ciphers = aNULL, eNULL, DES, 3DES, MD5, RC4, CAMELLIA
+smtpd_tls_loglevel = 1
+smtpd_tls_mandatory_ciphers = HIGH
+smtpd_tls_mandatory_protocols = !SSLv2 !SSLv3
+smtpd_tls_protocols = !SSLv2 !SSLv3
+smtpd_tls_received_header = yes
+smtpd_tls_security_level = may
+smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
+
+smtp_tls_CApath = /etc/ssl/certs
+smtp_tls_loglevel = 1
+smtp_tls_security_level = may
+smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
+
+message_size_limit = 61440000
+strict_rfc821_envelopes = yes
+smtpd_client_connection_count_limit = 100
+smtpd_helo_required = yes
+smtpd_relay_restrictions = permit_mynetworks, permit_sasl_authenticated, defer_unauth_destination
+myhostname = juju-87625f-hloeung-94.openstacklocal
+alias_maps = hash:/etc/aliases
+alias_database = hash:/etc/aliases
+mydestination = $myhostname, juju-87625f-hloeung-94, localhost.localdomain, localhost
+relayhost =
+mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
+mailbox_size_limit = 0
+recipient_delimiter = +
+inet_interfaces = all
+inet_protocols = all
+
+broken_sasl_auth_clients = yes
+smtpd_sasl_auth_enable = yes
+smtpd_sasl_security_options = noanonymous
+smtpd_sasl_type = dovecot
+smtpd_sasl_path = private/auth
+smtpd_tls_auth_only = yes
+smtpd_sender_restrictions = reject_unknown_sender_domain, check_sender_access hash:/etc/postfix/access
+
+transport_maps = hash:/etc/postfix/transport
diff --git a/tests/unit/files/postfix_main_transport_maps_with_header_checks.cf b/tests/unit/files/postfix_main_transport_maps_with_header_checks.cf
new file mode 100644
index 0000000..34e055c
--- /dev/null
+++ b/tests/unit/files/postfix_main_transport_maps_with_header_checks.cf
@@ -0,0 +1,61 @@
+## This file is Juju managed - do not edit by hand #
+
+
+smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu)
+biff = no
+
+append_dot_mydomain = no
+
+readme_directory = no
+
+# See http://www.postfix.org/COMPATIBILITY_README.html -- default to 2 on
+# fresh installs.
+compatibility_level = 2
+
+# TLS parameters
+tls_preempt_cipherlist = yes
+smtpd_tls_cert_file = /etc/ssl/certs/ssl-cert-snakeoil.pem
+smtpd_tls_key_file = /etc/ssl/private/ssl-cert-snakeoil.key
+smtpd_tls_dh1024_param_file = /etc/ssl/private/dhparams.pem
+smtpd_tls_ciphers = HIGH
+smtpd_tls_exclude_ciphers = aNULL, eNULL, DES, 3DES, MD5, RC4, CAMELLIA
+smtpd_tls_loglevel = 1
+smtpd_tls_mandatory_ciphers = HIGH
+smtpd_tls_mandatory_protocols = !SSLv2 !SSLv3
+smtpd_tls_protocols = !SSLv2 !SSLv3
+smtpd_tls_received_header = yes
+smtpd_tls_security_level = may
+smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
+
+smtp_tls_CApath = /etc/ssl/certs
+smtp_tls_loglevel = 1
+smtp_tls_security_level = may
+smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
+
+message_size_limit = 61440000
+strict_rfc821_envelopes = yes
+smtpd_client_connection_count_limit = 100
+smtpd_helo_required = yes
+smtpd_relay_restrictions = permit_mynetworks, permit_sasl_authenticated, defer_unauth_destination
+myhostname = juju-87625f-hloeung-94.openstacklocal
+alias_maps = hash:/etc/aliases
+alias_database = hash:/etc/aliases
+mydestination = $myhostname, juju-87625f-hloeung-94, localhost.localdomain, localhost
+relayhost =
+mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
+mailbox_size_limit = 0
+recipient_delimiter = +
+inet_interfaces = all
+inet_protocols = all
+
+broken_sasl_auth_clients = yes
+smtpd_sasl_auth_enable = yes
+smtpd_sasl_security_options = noanonymous
+smtpd_sasl_type = dovecot
+smtpd_sasl_path = private/auth
+smtpd_tls_auth_only = yes
+smtpd_sender_restrictions = reject_unknown_sender_domain, check_sender_access hash:/etc/postfix/access
+
+header_checks = regexp:/etc/postfix/header_checks
+
+transport_maps = hash:/etc/postfix/transport
diff --git a/tests/unit/files/postfix_main_transport_maps_with_virtual_alias_maps.cf b/tests/unit/files/postfix_main_transport_maps_with_virtual_alias_maps.cf
new file mode 100644
index 0000000..6049e7c
--- /dev/null
+++ b/tests/unit/files/postfix_main_transport_maps_with_virtual_alias_maps.cf
@@ -0,0 +1,61 @@
+## This file is Juju managed - do not edit by hand #
+
+
+smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu)
+biff = no
+
+append_dot_mydomain = no
+
+readme_directory = no
+
+# See http://www.postfix.org/COMPATIBILITY_README.html -- default to 2 on
+# fresh installs.
+compatibility_level = 2
+
+# TLS parameters
+tls_preempt_cipherlist = yes
+smtpd_tls_cert_file = /etc/ssl/certs/ssl-cert-snakeoil.pem
+smtpd_tls_key_file = /etc/ssl/private/ssl-cert-snakeoil.key
+smtpd_tls_dh1024_param_file = /etc/ssl/private/dhparams.pem
+smtpd_tls_ciphers = HIGH
+smtpd_tls_exclude_ciphers = aNULL, eNULL, DES, 3DES, MD5, RC4, CAMELLIA
+smtpd_tls_loglevel = 1
+smtpd_tls_mandatory_ciphers = HIGH
+smtpd_tls_mandatory_protocols = !SSLv2 !SSLv3
+smtpd_tls_protocols = !SSLv2 !SSLv3
+smtpd_tls_received_header = yes
+smtpd_tls_security_level = may
+smtpd_tls_session_cache_database = btree:${data_directory}/smtpd_scache
+
+smtp_tls_CApath = /etc/ssl/certs
+smtp_tls_loglevel = 1
+smtp_tls_security_level = may
+smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache
+
+message_size_limit = 61440000
+strict_rfc821_envelopes = yes
+smtpd_client_connection_count_limit = 100
+smtpd_helo_required = yes
+smtpd_relay_restrictions = permit_mynetworks, permit_sasl_authenticated, defer_unauth_destination
+myhostname = juju-87625f-hloeung-94.openstacklocal
+alias_maps = hash:/etc/aliases
+alias_database = hash:/etc/aliases
+mydestination = $myhostname, juju-87625f-hloeung-94, localhost.localdomain, localhost
+relayhost =
+mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128
+mailbox_size_limit = 0
+recipient_delimiter = +
+inet_interfaces = all
+inet_protocols = all
+
+broken_sasl_auth_clients = yes
+smtpd_sasl_auth_enable = yes
+smtpd_sasl_security_options = noanonymous
+smtpd_sasl_type = dovecot
+smtpd_sasl_path = private/auth
+smtpd_tls_auth_only = yes
+smtpd_sender_restrictions = reject_unknown_sender_domain, check_sender_access hash:/etc/postfix/access
+
+transport_maps = hash:/etc/postfix/transport
+
+virtual_alias_maps = hash:/etc/postfix/virtual_alias
diff --git a/tests/unit/test_smtp_relay.py b/tests/unit/test_smtp_relay.py
index 366a24b..f9cdd61 100644
--- a/tests/unit/test_smtp_relay.py
+++ b/tests/unit/test_smtp_relay.py
@@ -77,6 +77,8 @@ class TestCharm(unittest.TestCase):
             'tls_policy_maps': '',
             'tls_protocols': '!SSLv2 !SSLv3',
             'tls_security_level': 'may',
+            'transport_maps': '',
+            'virtual_alias_maps': '',
         }
 
         patcher = mock.patch('charmhelpers.core.hookenv.close_port')
@@ -905,6 +907,80 @@ someplace.local encrypt
     @mock.patch('charms.reactive.set_flag')
     @mock.patch('reactive.smtp_relay._get_autocert_cn')
     @mock.patch('reactive.smtp_relay._get_milters')
+    @mock.patch('subprocess.call')
+    def test_configure_smtp_relay_config_transport_maps(self, call, get_milters, get_cn, set_flag, clear_flag):
+        postfix_main_cf = os.path.join(self.tmpdir, 'main.cf')
+        postfix_transport_maps = os.path.join(self.tmpdir, 'transport')
+        get_cn.return_value = ''
+        get_milters.return_value = ''
+        self.mock_config.return_value['transport_maps'] = '.mydomain.local  smtp:[smtp.mydomain.local]'
+        smtp_relay.configure_smtp_relay(self.tmpdir)
+        with open('tests/unit/files/postfix_main_transport_maps.cf', 'r', encoding='utf-8') as f:
+            want = f.read()
+        with open(postfix_main_cf, 'r', encoding='utf-8') as f:
+            got = f.read()
+        self.assertEqual(want, got)
+        want = smtp_relay.JUJU_HEADER + '.mydomain.local  smtp:[smtp.mydomain.local]' + "\n"
+        with open(postfix_transport_maps, 'r', encoding='utf-8') as f:
+            got = f.read()
+        self.assertEqual(want, got)
+
+    @mock.patch('charms.reactive.clear_flag')
+    @mock.patch('charms.reactive.set_flag')
+    @mock.patch('reactive.smtp_relay._get_autocert_cn')
+    @mock.patch('reactive.smtp_relay._get_milters')
+    @mock.patch('subprocess.call')
+    def test_configure_smtp_relay_config_transport_maps_with_header_checks(
+        self, call, get_milters, get_cn, set_flag, clear_flag
+    ):
+        postfix_main_cf = os.path.join(self.tmpdir, 'main.cf')
+        postfix_transport_maps = os.path.join(self.tmpdir, 'transport')
+        get_cn.return_value = ''
+        get_milters.return_value = ''
+        self.mock_config.return_value['header_checks'] = '/^Received:/ HOLD'
+        self.mock_config.return_value['transport_maps'] = '.mydomain.local  smtp:[smtp.mydomain.local]'
+        smtp_relay.configure_smtp_relay(self.tmpdir)
+        with open('tests/unit/files/postfix_main_transport_maps_with_header_checks.cf', 'r', encoding='utf-8') as f:
+            want = f.read()
+        with open(postfix_main_cf, 'r', encoding='utf-8') as f:
+            got = f.read()
+        self.assertEqual(want, got)
+        want = smtp_relay.JUJU_HEADER + '.mydomain.local  smtp:[smtp.mydomain.local]' + "\n"
+        with open(postfix_transport_maps, 'r', encoding='utf-8') as f:
+            got = f.read()
+        self.assertEqual(want, got)
+
+    @mock.patch('charms.reactive.clear_flag')
+    @mock.patch('charms.reactive.set_flag')
+    @mock.patch('reactive.smtp_relay._get_autocert_cn')
+    @mock.patch('reactive.smtp_relay._get_milters')
+    @mock.patch('subprocess.call')
+    def test_configure_smtp_relay_config_transport_maps_with_virtual_alias_maps(
+        self, call, get_milters, get_cn, set_flag, clear_flag
+    ):
+        postfix_main_cf = os.path.join(self.tmpdir, 'main.cf')
+        postfix_virtual_alias_maps = os.path.join(self.tmpdir, 'virtual_alias')
+        get_cn.return_value = ''
+        get_milters.return_value = ''
+        self.mock_config.return_value['transport_maps'] = '.mydomain.local  smtp:[smtp.mydomain.local]'
+        self.mock_config.return_value['virtual_alias_maps'] = '[e-mail] [e-mail]'
+        smtp_relay.configure_smtp_relay(self.tmpdir)
+        with open(
+            'tests/unit/files/postfix_main_transport_maps_with_virtual_alias_maps.cf', 'r', encoding='utf-8'
+        ) as f:
+            want = f.read()
+        with open(postfix_main_cf, 'r', encoding='utf-8') as f:
+            got = f.read()
+        self.assertEqual(want, got)
+        want = smtp_relay.JUJU_HEADER + '[e-mail] [e-mail]' + "\n"
+        with open(postfix_virtual_alias_maps, 'r', encoding='utf-8') as f:
+            got = f.read()
+        self.assertEqual(want, got)
+
+    @mock.patch('charms.reactive.clear_flag')
+    @mock.patch('charms.reactive.set_flag')
+    @mock.patch('reactive.smtp_relay._get_autocert_cn')
+    @mock.patch('reactive.smtp_relay._get_milters')
     @mock.patch('reactive.smtp_relay._write_file')
     @mock.patch('subprocess.call')
     def test_configure_smtp_relay_flags(self, call, write_file, get_milters, get_cn, set_flag, clear_flag):