Merge ~kotodama/mojo/+git/mojo-specs:mailman3-core-charm into ~mojo-maintainers/mojo/+git/mojo-specs:master

[Loïc Gomez]

Initial collect and deploy for mailman3-core charm testing

[🤖 Canonical IS Merge Bot (canonical-is-mergebot)]

This merge proposal is being monitored by mergebot. Change the status to Approved to merge.

[Tom Haddon (mthaddon)]

A few comments inline

[Tom Haddon (mthaddon)]

Some comments inline

Unmerged commits

e35aaff... by Loïc Gomez on 2021-09-01

Initial collect and deploy for mailman3-core charm testing

diff --git a/charm-testing/mailman3-core/collect b/charm-testing/mailman3-core/collect
new file mode 100644
index 0000000..0a923cc
--- /dev/null
+++ b/charm-testing/mailman3-core/collect
@@ -0,0 +1,8 @@
+apache2                          cs:apache2
+mailman3-core                    cs:mailman3-core
+mailman3-web                     cs:mailman3-web
+postgresql                       cs:postgresql;overwrite=True
+
+# subordinates
+nrpe                             cs:nrpe
+ntp                              cs:ntp
diff --git a/charm-testing/mailman3-core/deploy b/charm-testing/mailman3-core/deploy
new file mode 100644
index 0000000..7bf9448
--- /dev/null
+++ b/charm-testing/mailman3-core/deploy
@@ -0,0 +1,58 @@
+mailman3-core:
+  series: {{series}}
+  services:
+    apache2:
+      charm: apache2
+      options:
+        servername: lists.ci.internal
+        enable_modules: headers proxy proxy_http rewrite ssl
+    mailman3-core:
+      charm: mailman3-core
+      options:
+        hostname: lists-mx.ci.internal
+        site_owner: [e-mail]
+        spf_config: include-file://{{spec_dir}}/{{stage}}/files/policyd-spf.conf
+        enable_strict_host_checks: "test"
+    mailman3-web:
+      charm: mailman3-web
+      num_units: 2
+      expose: true
+      options:
+        domain: lists-mx.ci.internal
+        site_name: CI Canonical Mailing Lists
+        extra_packages: "django-mailman-launchpad-teams python-contextlib2 python-django-openid-auth python-raven"
+        install_sources: '["ppa:canonical-sysadmins/django-mailman-launchpad-teams"]'
+        additional_django_apps: "'django_openid_auth', 'django_mailman_launchpad_teams'"
+        arbitrary_django_settings: include-file://{{spec_dir}}/{{stage}}/files/arbitrary_config
+        check-lists-command: include-base64://{{spec_dir}}/{{stage}}/templates/check_list_settings.py.tmpl
+        check-lists-conf: include-base64://{{spec_dir}}/{{stage}}/files/check-lists.conf
+    postgresql:
+      charm: postgresql
+
+    # subordinates
+    nrpe:
+      charm: nrpe
+      options:
+        nagios_master: nagios1.internal
+        nagios_host_context: ci-lists-canonical
+        swap: -w 40% -c 25% -n ok
+    ntp:
+      charm: ntp
+      options:
+        pools: ntp.ubuntu.com
+
+  relations:
+    - ["apache2:balancer", "mailman3-web:mailman3website"]
+    - ["mailman3-core:mailman3", "mailman3-web:mailman3"]
+    - ["mailman3-web:mailman3archiver", "mailman3-core:mailman3archiver"]
+    - ["nrpe:nrpe-external-master", "apache2"]
+    - ["nrpe:nrpe-external-master", "mailman3-core:nrpe-external-master"]
+    - ["nrpe:nrpe-external-master", "mailman3-web:nrpe-external-master"]
+    - ["nrpe:nrpe-external-master", "ntp:nrpe-external-master"]
+    - ["nrpe:nrpe-external-master", "postgresql:nrpe-external-master"]
+    - ["ntp:juju-info", "apache2"]
+    - ["ntp:juju-info", "mailman3-core"]
+    - ["ntp:juju-info", "mailman3-web"]
+    - ["ntp:juju-info", "postgresql"]
+    - ["postgresql:db", "mailman3-core:db"]
+    - ["postgresql:db", "mailman3-web:db"]
diff --git a/charm-testing/mailman3-core/files/arbitrary_config b/charm-testing/mailman3-core/files/arbitrary_config
new file mode 100644
index 0000000..8e5691f
--- /dev/null
+++ b/charm-testing/mailman3-core/files/arbitrary_config
@@ -0,0 +1,47 @@
+MIDDLEWARE_CLASSES = (
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.middleware.locale.LocaleMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+    'django.middleware.security.SecurityMiddleware',
+    'django_mailman3.middleware.TimezoneMiddleware',
+    'postorius.middleware.PostoriusMiddleware',
+    'django_mailman_launchpad_teams.middleware.RequireLoginMiddleware',
+    )
+
+AUTHENTICATION_BACKENDS = (
+    'django_openid_auth.auth.OpenIDBackend',
+    'django.contrib.auth.backends.ModelBackend',
+    )
+
+# Can OpenID create users?
+OPENID_CREATE_USERS = True
+# Can we reuser existing users?
+OPENID_REUSE_USERS = True
+# When logging in again, should we overwrite user details based on
+# data received via Simple Registration?
+OPENID_UPDATE_DETAILS_FROM_SREG = True
+# If set, always use this as the identity URL rather than asking the
+# user.  This only makes sense if it is a server URL.
+OPENID_SSO_SERVER_URL = 'https://login.ubuntu.com'
+OPENID_LAUNCHPAD_TEAMS_REQUIRED = ["mojo-maintainers"]
+OPENID_LAUNCHPAD_STAFF_TEAMS = ["mojo-maintainers"]
+OPENID_LAUNCHPAD_TEAMS_MAPPING_AUTO = True
+# Tell django.contrib.auth to use the OpenID signin URLs.
+LOGIN_URL = 'openid-login'
+LOGIN_REDIRECT_URL = '/'
+LOGIN_REQUIRED_URLS = (
+    r'(.*)',
+    )
+LOGIN_REQUIRED_URLS_EXCEPTIONS = (
+    r'/mailman3/openid(.*)$',
+    r'/mailman3/hyperkitty/api(.*)$',
+    )
+SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
+ALLOWED_EXTERNAL_OPENID_REDIRECT_DOMAINS = ['*']
+ROOT_URLCONF = 'django_mailman_launchpad_teams.urls'
+
diff --git a/charm-testing/mailman3-core/files/check-lists.conf b/charm-testing/mailman3-core/files/check-lists.conf
new file mode 100644
index 0000000..4ea13b0
--- /dev/null
+++ b/charm-testing/mailman3-core/files/check-lists.conf
@@ -0,0 +1,13 @@
+[list_sanity]
+
+external_subscriber_lists:
+    fake-external
+
+not_archived_lists:
+    fake-not-arch
+
+do_not_accept_all_canonical_posts:
+    fake-restricted
+
+private:
+    fake-private
diff --git a/charm-testing/mailman3-core/files/policyd-spf.conf b/charm-testing/mailman3-core/files/policyd-spf.conf
new file mode 100644
index 0000000..c20b46c
--- /dev/null
+++ b/charm-testing/mailman3-core/files/policyd-spf.conf
@@ -0,0 +1,10 @@
+debugLevel = 1
+defaultSeedOnly = 1
+
+HELO_reject = SPF_Not_Pass
+Mail_From_reject = Fail
+
+PermError_reject = False
+TempError_Defer = False
+
+skip_addresses = 127.0.0.0/8,::ffff:127.0.0.0/104,::1/128
diff --git a/charm-testing/mailman3-core/templates/check_list_settings.py.tmpl b/charm-testing/mailman3-core/templates/check_list_settings.py.tmpl
new file mode 100644
index 0000000..ef4365f
--- /dev/null
+++ b/charm-testing/mailman3-core/templates/check_list_settings.py.tmpl
@@ -0,0 +1,84 @@
+import sys
+
+from django_mailman3.lib.mailman import get_mailman_client
+from django.core.management.base import BaseCommand
+from django.utils.six.moves.urllib.error import HTTPError
+from mailmanclient import MailmanConnectionError
+
+from ConfigParser import ConfigParser
+
+NAGIOS_OK = 0
+NAGIOS_WARN = 1
+NAGIOS_FAIL = 2
+
+
+class nagios_warning:
+    def __init__(self):
+        self.warning = ""
+
+    def add(self, line):
+        self.warning += line + "\n"
+
+
+class SeriousConfigParser(ConfigParser):
+    def get_list(self, section, option, raw=False, vars=None):
+        """Look up OPTION in SECTION and return it as a list of words."""
+        return self.get(section, option, raw, vars).split()
+
+
+class Command(BaseCommand):
+
+    def get_mm_lists(self, client):
+        try:
+            mm_lists = client.get_lists()
+        except (MailmanConnectionError, HTTPError):
+            print("E: could not connect to mailman3-core to retrieve lists")
+            sys.exit(NAGIOS_FAIL)
+        if mm_lists is None:
+            print("E: could not retrieve lists from mailman3-core")
+            sys.exit(NAGIOS_FAIL)
+        return mm_lists
+
+    def check_list(self, ccc, mlist, warning):
+        ret_val = NAGIOS_OK
+
+        # Rules are as follows:
+        #   All lists must require subscription confirmation by the user
+        #   Must be archived or explicitly excepted in the check config
+        #   Either fully private or public
+        #   If they have external subscribers, must be named list-external or excepted in the config
+        #   Private lists must be explicitly listed in the check config
+
+        if mlist.settings['subscription_policy'] not in ['confirm', 'confirm_then_moderate']:
+            print("E: list '%s' doesn't require subscription confirmation" % (mlist.list_name))
+            ret_val = NAGIOS_FAIL
+
+        if 'never' in mlist.settings['archive_policy'] and mlist.list_name.lower() not in not_archived_lists:
+            warning.add("WARN: %s is not archived and not in not_archived_lists in /etc/nagios/check-lists.conf"
+                        % mlist.list_name.lower())
+
+        return ret_val
+
+    def add_arguments(self, parser):
+        parser.add_argument("--verbose", action="store_true", dest="verbose")
+
+    def handle(self, *args, **options):
+
+        warning = nagios_warning()
+
+        ccc = SeriousConfigParser()
+        ccc.readfp(file("/etc/nagios/check-lists.conf"))
+
+        client = get_mailman_client()
+        mlists = self.get_mm_lists(client)
+
+        ret_val = NAGIOS_OK
+        for mlist in mlists:
+            ret = self.check_list(ccc, mlist, warning)
+            ret_val = max([ret_val, ret])
+        if ret_val == NAGIOS_FAIL:
+            sys.exit(ret_val)
+
+        if options['verbose'] and warning.warning:
+            print(warning.warning,)
+            sys.exit(NAGIOS_WARN)