Merge ~tcuthbert/mailman3-core-charm:mail-filtering into mailman3-core-charm:master

[Thomas Cuthbert]

Integrate Spamassassin and SPF with Postfix

This is currently working in lists.staging.c.c

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Tom Haddon (mthaddon)]

LGTM in terms of the charm changes. I'm not familiar with our spamassassin or SPF set up, but I'm assuming this is copied from existing implementations.

review: Approve

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

Change successfully merged at revision 7f909904566fae6fa882de28d08c638efefe99b5

diff --git a/config.yaml b/config.yaml
index 8265ce4..90a4ce5 100644
--- a/config.yaml
+++ b/config.yaml
@@ -24,6 +24,36 @@ options:
     type: string
     description: |
       Policy map to enforce TLS/SSL
+  enable_spamassassin:
+    default: true
+    type: boolean
+    description: |
+      Enable Spamassassin mail filter
+  enable_spf:
+    default: true
+    type: boolean
+    description: |
+      Enable Postfix SPF extensions
+  spamassassin_policy:
+    default: ""
+    type: string
+    description: |
+      Custom Spamassassin policies
+  spf_config:
+    default: |
+      debugLevel = 1 
+      defaultSeedOnly = 1
+
+      HELO_reject = SPF_Not_Pass
+      Mail_From_reject = Fail
+
+      PermError_reject = False
+      TempError_Defer = False
+
+      skip_addresses = 127.0.0.0/8,::ffff:127.0.0.0/104,::1/128
+    type: string
+    description: |
+      Custom SPF configuration
   nagios_context:
     default: "juju"
     type: string
diff --git a/files/spamassassin_master.cf b/files/spamassassin_master.cf
new file mode 100644
index 0000000..41a5ca9
--- /dev/null
+++ b/files/spamassassin_master.cf
@@ -0,0 +1,2 @@
+spamassassin unix -     n       n       -       -       pipe
+  flags=FR user=spamd argv=/usr/bin/spamc -e /usr/sbin/sendmail -G -i -f ${sender} -- ${recipient}
diff --git a/files/spf_master.cf b/files/spf_master.cf
new file mode 100644
index 0000000..fc349ca
--- /dev/null
+++ b/files/spf_master.cf
@@ -0,0 +1,2 @@
+policy-spf  unix  -       n       n       -       -       spawn
+  user=nobody argv=/usr/bin/policyd-spf
diff --git a/layer.yaml b/layer.yaml
index d297df3..c6e1d2c 100644
--- a/layer.yaml
+++ b/layer.yaml
@@ -10,3 +10,6 @@ options:
     packages:
       - mailman3
       - postfix
+      - postfix-policyd-spf-python
+      - spamassassin
+      - spamc
diff --git a/reactive/mailman3_core.py b/reactive/mailman3_core.py
index df82b1e..1eddf2a 100644
--- a/reactive/mailman3_core.py
+++ b/reactive/mailman3_core.py
@@ -1,4 +1,5 @@
 import os
+import pwd
 import random
 import string
 import subprocess
@@ -94,6 +95,11 @@ def configure_postfix():
     subprocess.check_output(['postconf', 'always_add_missing_headers=yes',
                                          'local_header_rewrite_clients=static:all'])
 
+    if config["enable_spamassassin"]:
+        configure_spamassassin()
+    if config["enable_spf"]:
+        configure_postfix_spf()
+
     host.service_reload('postfix')
     # Open the SMTP port
     hookenv.open_port(25)
@@ -101,6 +107,52 @@ def configure_postfix():
     hookenv.status_set('active', 'ready')
 
 
+def configure_spamassassin():
+    config = hookenv.config()
+    default_config = ('ENABLED=0\n'
+                      'OPTIONS="--create-prefs --max-children 5 --pidfile=/var/run/spamassassin.pid --username spamd --helper-home-dir /var/lib/spamd -s /var/log/spamd.log"\n'
+                      'CRON=1\n')
+    sa_policy = config["spamassassin_policy"]
+    render(source="sa-local.cf.tmpl",
+           target="/etc/spamassassin/local.cf",
+           owner="spamd",
+           group="spamd",
+           perms=0o640,
+           context={'policy': sa_policy})
+
+    host.write_file("/etc/default/spamassassin", default_config)
+    if not postconf(key="spamassassin", args=("-F",)):  # Check to see if Spamassassin is already configured.
+        postfix_master_cf = os.path.join(hookenv.charm_dir(), "files/spamassassin_master.cf")
+        with open(postfix_master_cf) as f:
+            append_to_file("/etc/postfix/master.cf", f.read())
+    postconf(key="smtp/inet/content_filter", value="spamassassin:dummy", args=("-P",))
+
+    useradd("spamd")
+    systemctl("reload", "postfix.service")
+    systemctl("enable", "spamassassin.service")
+    systemctl("start", "spamassassin.service")
+    set_flag('mailman.postfix.spamassassin.configured')
+
+
+def configure_postfix_spf():
+    config = hookenv.config()
+    postconf(key="policy-spf_time_limit", value="3600s")
+    postconf(key="smtpd_recipient_restrictions", value=(
+        "permit_sasl_authenticated "
+        "permit_mynetworks "
+        "reject_unauth_destination "
+        "check_policy_service unix:private/policy-spf"
+    ))
+    if not postconf(key="policy-spf", args=("-F",)):  # Check to see if spf is already configured.
+        spf_master_cf = os.path.join(hookenv.charm_dir(), "files/spf_master.cf")
+        with open(spf_master_cf) as f:
+            append_to_file("/etc/postfix/master.cf", f.read())
+
+    host.write_file("/etc/postfix-policyd-spf-python/policyd-spf.conf", config["spf_config"])
+    systemctl("reload", "postfix.service")
+    set_flag('mailman.postfix.spf.configured')
+
+
 # HyperKitty's recommended dependencies include, basically, all of Django so
 # we go ahead and purge that, we just want the mailman plugin, not the whole
 # thing.
@@ -234,3 +286,34 @@ def setup_nagios(nagios):
 
     nrpe_setup.write()
     hookenv.status_set('active', 'ready')
+
+
+def systemctl(action, service_name):
+    return subprocess.check_call(["systemctl", action, service_name])
+
+
+def useradd(username):
+    cmd = ["useradd",
+           "-r",
+           "-k", "/dev/null",
+           "-md", "/var/lib/spamd",
+           "-s", "/bin/false",
+           username]
+    try:
+        pwd.getpwnam(username)
+    except KeyError:
+        return subprocess.check_call(cmd)
+
+
+def postconf(key, value=None, args=None):
+    cmd = ['postconf', '{k}'.format(k=key)]
+    if value:
+        cmd = ['postconf', '{k}={v}'.format(k=key, v=value)]
+    if args:
+        [cmd.insert(1, a) for a in reversed(args)]
+    return subprocess.check_output(cmd)
+
+
+def append_to_file(filep, content):
+    with open(filep, "a") as f:
+        f.write(content)
diff --git a/templates/sa-local.cf.tmpl b/templates/sa-local.cf.tmpl
new file mode 100644
index 0000000..682699c
--- /dev/null
+++ b/templates/sa-local.cf.tmpl
@@ -0,0 +1,4 @@
+# This file is managed by Juju
+{{ policy }}
+ifplugin Mail::SpamAssassin::Plugin::Shortcircuit
+endif # Mail::SpamAssassin::Plugin::Shortcircuit