Merge lp:~jskrzeszewska/mailman/enablingArchivers into lp:mailman

[Joanna Skrzeszewska]

Changes for enabling/disabling archivers using postorius (bug 1158040).

[lp:~jskrzeszewska/mailman/enablingArchivers]

7220. By Joanna Skrzeszewska on 2013-09-18

Unit tests for enabling/disabling archivers.

[Nicki Hutchens (nhutch01)]

review: Approve

[Barry Warsaw (barry)]

Hi Joanna,

Apologies for the long delay, but I've finally managed to review this branch. I decided to make a number of changes to the semantics and API, but your branch was instrumental in getting this new feature landed. Thanks very much for your contribution!

The main change that I made was to the REST API. Rather than put the list of archivers inside the list's 'configuration' resource, I decided to make it a separate sub-resource of the mailing list. This helped map the semantics of getting the list of enabled archivers, and changing their state, better to HTTP commands.

r7229 in trunk contains the motified and merged branch.

review: Approve

=== modified file 'src/mailman/interfaces/mailinglist.py'
--- src/mailman/interfaces/mailinglist.py	2013-06-19 02:43:40 +0000
+++ src/mailman/interfaces/mailinglist.py	2013-09-18 10:39:39 +0000
@@ -24,6 +24,7 @@
     'IAcceptableAlias',
     'IAcceptableAliasSet',
     'IMailingList',
+    'IArchiverList',
     'Personalization',
     'ReplyToMunging',
     ]
@@ -54,6 +55,16 @@
     # An explicit Reply-To header is added
     explicit_header = 2
 
+class IArchiverList(Interface):    
+    mailing_list_id = Attribute("""List id""")
+    archiver_name = Attribute("""Archiver name""")
+    archiver_enabled = Attribute("""If is enabled.""")
+
+class IListArchiverSet(Interface):
+    def getAll():
+        """Return dict containing all archivers and their settings."""
+    def set(archiver, is_enabled):
+        """Set archiver for this list."""
 
 

 class IMailingList(Interface):

=== modified file 'src/mailman/model/mailinglist.py'
--- src/mailman/model/mailinglist.py	2013-01-01 14:05:42 +0000
+++ src/mailman/model/mailinglist.py	2013-09-18 10:39:39 +0000
@@ -28,8 +28,8 @@
 import os
 
 from storm.locals import (
-    And, Bool, DateTime, Float, Int, Pickle, RawStr, Reference, Store,
-    TimeDelta, Unicode)
+    And, Bool, DateTime, Float, Int, Pickle, RawStr, Reference, ReferenceSet,
+    Store, TimeDelta, Unicode)
 from urlparse import urljoin
 from zope.component import getUtility
 from zope.event import notify
@@ -47,8 +47,8 @@
 from mailman.interfaces.domain import IDomainManager
 from mailman.interfaces.languages import ILanguageManager
 from mailman.interfaces.mailinglist import (
-    IAcceptableAlias, IAcceptableAliasSet, IMailingList, Personalization,
-    ReplyToMunging)
+    IAcceptableAlias, IAcceptableAliasSet, IArchiverList, IListArchiverSet,
+    IMailingList, Personalization, ReplyToMunging)
 from mailman.interfaces.member import (
     AlreadySubscribedError, MemberRole, MissingPreferredAddressError,
     SubscriptionEvent)
@@ -67,6 +67,19 @@
 SPACE = ' '
 UNDERSCORE = '_'
 
+@implementer(IArchiverList)
+class ArchiverList(Model):    
+    __storm_primary__ = "mailing_list_id", "archiver_name"
+    mailing_list_id = Int()
+    archiver_name = Unicode()
+    archiver_enabled = Bool()
+
+    def __init__(self, mailing_list_id, archiver_name):
+        self.mailing_list_id = mailing_list_id
+        self.archiver_name = archiver_name
+        self.archiver_enabled = False
+
+
 
 

 @implementer(IMailingList)
@@ -78,6 +91,7 @@
     # XXX denotes attributes that should be part of the public interface but
     # are currently missing.
 
+    archivers = ReferenceSet(id, ArchiverList.mailing_list_id)
     # List identity
     list_name = Unicode()
     mail_host = Unicode()
@@ -538,3 +552,40 @@
             AcceptableAlias.mailing_list == self._mailing_list)
         for alias in aliases:
             yield alias.alias
+
+@implementer(IListArchiverSet)
+class ListArchiverSet:
+    def __init__(self, mailing_list):
+        self._mailing_list = mailing_list
+        self.lazyAdd()
+
+    def getAll(self):
+        entries = Store.of(self._mailing_list).find(ArchiverList, ArchiverList.mailing_list_id == self._mailing_list.id)
+        all_in_config = {archiver.name for archiver in config.archivers}
+        ret = {}
+        for entry in entries:
+            if entry.archiver_name in all_in_config:
+                ret[entry.archiver_name] = int(entry.archiver_enabled)
+        return ret
+
+    def set(self, archiver, is_enabled):
+        bool_enabled = (int(is_enabled) != 0)
+        self.get(archiver).set(archiver_enabled=bool_enabled)
+
+    def isEnabled(self, archiverName):
+        return self.get(archiverName).one().archiver_enabled
+
+    def get(self, archiverName):
+        return Store.of(self._mailing_list).find(ArchiverList,
+            (ArchiverList.mailing_list_id == self._mailing_list.id) & (ArchiverList.archiver_name == archiverName))
+
+    def lazyAdd(self):
+        names = []
+        for archiver in config.archivers:
+            count = self.get(archiver.name).count()
+            names.append((archiver.name, count))
+            if not count:
+                entry = ArchiverList(self._mailing_list.id, archiver.name)
+                Store.of(self._mailing_list).add(entry)
+        Store.of(self._mailing_list).commit()
+

=== modified file 'src/mailman/rest/configuration.py'
--- src/mailman/rest/configuration.py	2013-03-20 21:01:57 +0000
+++ src/mailman/rest/configuration.py	2013-09-18 10:39:39 +0000
@@ -34,9 +34,10 @@
 from mailman.interfaces.action import Action
 from mailman.interfaces.archiver import ArchivePolicy
 from mailman.interfaces.autorespond import ResponseAction
-from mailman.interfaces.mailinglist import IAcceptableAliasSet, ReplyToMunging
+from mailman.interfaces.mailinglist import IAcceptableAliasSet, IListArchiverSet, ReplyToMunging
 from mailman.rest.helpers import GetterSetter, PATCH, etag, no_content
 from mailman.rest.validator import PatchValidator, Validator, enum_validator
+from mailman.model.mailinglist import ListArchiverSet
 
 
 

@@ -64,6 +65,22 @@
         for alias in value:
             alias_set.add(unicode(alias))
 
+class ListArchivers(GetterSetter):
+
+    def get(self, mlist, attribute):
+        """Return the mailing list's acceptable aliases."""
+        assert attribute == 'archivers', (
+            'Unexpected attribute: {0}'.format(attribute))
+        archivers = ListArchiverSet(mlist)
+        return archivers.getAll()
+
+    def put(self, mlist, attribute, value):
+        assert attribute == 'archivers', (
+            'Unexpected attribute: {0}'.format(attribute))
+        archivers = ListArchiverSet(mlist)
+        for key, value in value.iteritems():
+            archivers.set(key, value)
+
 
 

 # Additional validators for converting from web request strings to internal
@@ -80,6 +97,15 @@
     """Turn a list of things into a list of unicodes."""
     return [unicode(value) for value in values]
 
+def list_of_pairs_to_dict(pairs):
+    dict = {}
+    # If pairs has only one element then it is not a list but a string.
+    if not isinstance(pairs, list):
+        pairs = [pairs]
+    for key_value in pairs:
+        parts = key_value.split('|')
+        dict[parts[0]] = parts[1]
+    return dict
 
 

 # This is the list of IMailingList attributes that are exposed through the
@@ -98,6 +124,7 @@
 
 ATTRIBUTES = dict(
     acceptable_aliases=AcceptableAliases(list_of_unicode),
+    archivers=ListArchivers(list_of_pairs_to_dict),
     admin_immed_notify=GetterSetter(as_boolean),
     admin_notify_mchanges=GetterSetter(as_boolean),
     administrivia=GetterSetter(as_boolean),

=== modified file 'src/mailman/rest/tests/test_lists.py'
--- src/mailman/rest/tests/test_lists.py	2013-01-01 14:05:42 +0000
+++ src/mailman/rest/tests/test_lists.py	2013-09-18 10:39:39 +0000
@@ -28,12 +28,16 @@
 
 import unittest
 
+from zope.component import getUtility
 from urllib2 import HTTPError
 from zope.component import getUtility
 
 from mailman.app.lifecycle import create_list
+from mailman.config import config
 from mailman.database.transaction import transaction
 from mailman.interfaces.usermanager import IUserManager
+from mailman.interfaces.listmanager import IListManager
+from mailman.model.mailinglist import ListArchiverSet
 from mailman.testing.helpers import call_api
 from mailman.testing.layers import RESTLayer
 
@@ -159,3 +163,25 @@
             call_api('http://localhost:9001/3.0/lists/ant.example.com',
                      method='DELETE')
         self.assertEqual(cm.exception.code, 404)
+
+    def test_prototype_in_list_archivers(self):
+        resource, response = call_api(
+            'http://localhost:9001/3.0/lists/[e-mail]/config')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(resource['archivers']['prototype'], 0)
+
+    def test_lazy_add_archivers(self):
+        call_api('http://localhost:9001/3.0/lists', {
+                 'fqdn_listname': '[e-mail]',
+                 })
+        resource, response = call_api(
+            'http://localhost:9001/3.0/lists/[e-mail]/config')
+        self.assertEqual(response.status, 200)
+        self.assertEqual(resource['archivers']['prototype'], 0)
+
+    def test_set_archiver_enabled(self):
+        mlist = getUtility(IListManager).create('[e-mail]')
+        lset = ListArchiverSet(mlist)
+        lset.set('prototype', 1)
+        self.assertEqual(lset.isEnabled('prototype'), 1)
+

=== modified file 'src/mailman/runners/archive.py'
--- src/mailman/runners/archive.py	2013-01-01 14:05:42 +0000
+++ src/mailman/runners/archive.py	2013-09-18 10:39:39 +0000
@@ -36,6 +36,7 @@
 from mailman.core.runner import Runner
 from mailman.interfaces.archiver import ClobberDate
 from mailman.utilities.datetime import RFC822_DATE_FMT, now
+from mailman.model.mailinglist import ListArchiverSet
 
 
 log = logging.getLogger('mailman.error')
@@ -91,17 +92,19 @@
     def _dispose(self, mlist, msg, msgdata):
         received_time = msgdata.get('received_time', now(strip_tzinfo=False))
         for archiver in config.archivers:
-            msg_copy = copy.deepcopy(msg)
-            if _should_clobber(msg, msgdata, archiver.name):
-                original_date = msg_copy['date']
-                del msg_copy['date']
-                del msg_copy['x-original-date']
-                msg_copy['Date'] = received_time.strftime(RFC822_DATE_FMT)
-                if original_date:
-                    msg_copy['X-Original-Date'] = original_date
-            # A problem in one archiver should not prevent other archivers
-            # from running.
-            try:
-                archiver.archive_message(mlist, msg_copy)
-            except Exception:
-                log.exception('Broken archiver: %s' % archiver.name)
+            archSet = ListArchiverSet(mlist)
+            if archSet.isEnabled(archiver.name):
+                msg_copy = copy.deepcopy(msg)
+                if _should_clobber(msg, msgdata, archiver.name):
+                    original_date = msg_copy['date']
+                    del msg_copy['date']
+                    del msg_copy['x-original-date']
+                    msg_copy['Date'] = received_time.strftime(RFC822_DATE_FMT)
+                    if original_date:
+                        msg_copy['X-Original-Date'] = original_date
+                # A problem in one archiver should not prevent other archivers
+                # from running.
+                try:
+                    archiver.archive_message(mlist, msg_copy)
+                except Exception:
+                    log.exception('Broken archiver: %s' % archiver.name)