Merge lp:~jskrzeszewska/mailman/mailman into lp:mailman

[Joanna Skrzeszewska]

Added archiver for rss feed and REST API for rss settings.

[lp:~jskrzeszewska/mailman/mailman]

7218. By Joanna Skrzeszewska on 2013-09-01

Changes in rssarchive.py

7219. By Joanna Skrzeszewska on 2013-09-01

Tests for rssarchive

[Nicki Hutchens (nhutch01)]

review: Approve

Unmerged revisions

7219. By Joanna Skrzeszewska on 2013-09-01

Tests for rssarchive

7218. By Joanna Skrzeszewska on 2013-09-01

Changes in rssarchive.py

7217. By Joanna Skrzeszewska on 2013-08-02

Adding REST API for rss archiver.

7216. By Joanna Skrzeszewska on 2013-08-02

Adding rss archiver.

=== added file 'src/mailman/archiving/rssarchive.py'
--- src/mailman/archiving/rssarchive.py	1970-01-01 00:00:00 +0000
+++ src/mailman/archiving/rssarchive.py	2013-09-01 16:37:50 +0000
@@ -0,0 +1,225 @@
+# Copyright (C) 2008-2013 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'RssArchiver',
+    ]
+
+import os
+import errno
+import logging
+import datetime
+import PyRSS2Gen
+import sqlite3
+
+from email.utils import parsedate
+from datetime import datetime
+from operator import itemgetter
+from datetime import timedelta
+from mailbox import Maildir
+from urlparse import urljoin
+
+from flufl.lock import Lock, TimeOutError
+from zope.interface import implementer
+from mailman.interfaces.listmanager import IListManager
+from zope.component import getUtility
+
+from mailman.config import config
+from mailman.interfaces.archiver import IArchiver
+
+log = logging.getLogger('mailman.error')
+
+@implementer(IArchiver)
+class RssArchiver:
+    name = 'rssarchive'
+    table_name = 'admin_features'
+
+    @staticmethod
+    def set_length_limit(mlist, length_limit):
+        rss_db_dir = os.path.join(config.ARCHIVE_DIR, 'rss.db')
+        is_enabled = RssArchiver.is_feed_enabled(mlist)
+        conn = sqlite3.connect(rss_db_dir)
+        c = conn.cursor()
+        sql = ('insert or replace into ' + RssArchiver.table_name + 
+              ' values (\'' + mlist.fqdn_listname + '\', ' + str(is_enabled) + 
+              ', ' + str(length_limit) + ')') 
+        c.execute(sql)
+        conn.commit()
+        c.close()
+        conn.close()
+
+    @staticmethod
+    def set_feed_enabled(mlist, is_enabled):
+        rss_db_dir = os.path.join(config.ARCHIVE_DIR, 'rss.db')
+        conn = sqlite3.connect(rss_db_dir)
+        c = conn.cursor()
+        length_limit = RssArchiver.get_length_limit(mlist)
+        sql = ('insert or replace into ' + RssArchiver.table_name + 
+              ' values (\'' + mlist.fqdn_listname + '\', ' + str(is_enabled) + 
+              ', ' +str(length_limit) + ')')
+        c.execute(sql)
+        conn.commit()
+        c.close()
+        conn.close()
+        if is_enabled:
+            RssArchiver.generate_rss_feed(mlist)
+   
+    @staticmethod
+    def get_length_limit(mlist):
+        rss_db_dir = os.path.join(config.ARCHIVE_DIR, 'rss.db')
+        RssArchiver.init_database()
+        conn = sqlite3.connect(rss_db_dir)
+        c = conn.cursor()
+        sql = ('select entries_no from ' + RssArchiver.table_name + 
+        ' where list_name = \'' + mlist.fqdn_listname + '\'')
+        is_enabled = c.execute(sql).fetchone()
+	if is_enabled == None:
+		is_enabled = [100]
+        conn.commit()
+        c.close()
+        conn.close()
+        return is_enabled[0]
+
+    @staticmethod
+    def is_feed_enabled(mlist):
+        rss_db_dir = os.path.join(config.ARCHIVE_DIR, 'rss.db')
+        RssArchiver.init_database()
+        conn = sqlite3.connect(rss_db_dir)
+        c = conn.cursor()
+        sql = ('select is_switched_on from ' + RssArchiver.table_name + 
+              ' where list_name = \'' + mlist.fqdn_listname + '\'')
+        is_enabled = c.execute(sql).fetchone()
+	if is_enabled == None:
+		is_enabled = [1]
+        conn.commit()
+        c.close()
+        conn.close() 
+        return is_enabled[0]
+    
+    @staticmethod
+    def init_database():
+        rss_db_dir = os.path.join(config.ARCHIVE_DIR, 'rss.db')
+        conn = sqlite3.connect(rss_db_dir)
+        c = conn.cursor()
+        sql = ('create table if not exists ' + RssArchiver.table_name + 
+              ' (list_name text unique, is_switched_on integer, entries_no integer)')
+        c.execute(sql)
+        conn.commit()
+        lists = []
+        list_manager = getUtility(IListManager)
+        for fqdn_name in sorted(list_manager.names):
+            mlist = list_manager.get(fqdn_name)
+            lists.append(mlist)
+        for list in lists:
+            log.error(list.fqdn_listname)
+            sql = ('insert or ignore into ' + RssArchiver.table_name + 
+                  ' values (\'' + list.fqdn_listname + '\', 1, 30)')
+            c.execute(sql)
+            conn.commit()
+        c.close()
+        conn.close()
+    
+    @staticmethod
+    def format_message(msg, msg_as_string):
+        string_msg = ''
+        string_msg += 'Date: ' + msg['date'] + '\n'
+        string_msg += 'From: ' + msg['from'] + '\n'
+        string_msg += 'To: ' + msg['to'] + '\n\n' + msg_as_string
+        return string_msg.replace('\n', '<br>').replace(' ', '&nbsp')
+
+    @staticmethod
+    def extract_content(msg):
+        if msg.is_multipart():
+            for part in msg.walk():
+                if part.get_content_type() == 'text/plain':
+                    msg_as_string = part.get_payload + msg_as_string
+        else:
+            msg_as_string =  msg.get_payload()
+        return RssArchiver.format_message(msg, msg_as_string) 
+    
+    @staticmethod
+    def generate_rss_feed(mlist):
+        archive_dir = os.path.join(config.ARCHIVE_DIR, 'rssarchive')
+        RssArchiver.create_if_none(archive_dir)
+        list_dir = os.path.join(archive_dir, mlist.fqdn_listname)
+        mailbox = Maildir(list_dir, create=True, factory=None)
+        messages = mailbox.values()
+        for mes in messages:
+            mes['parsedate'] = parsedate(mes['date'])
+        messages.sort(key=itemgetter('parsedate'), reverse=True)
+        length_limit = RssArchiver.get_length_limit(mlist)
+        messages = messages[0: length_limit]      
+        rss = PyRSS2Gen.RSS2(
+            title = mlist.fqdn_listname,
+            link = "",
+            description = "The latest messages from: " + mlist.fqdn_listname,
+            lastBuildDate = datetime.now(),
+            items = [
+                    PyRSS2Gen.RSSItem(
+                        title = mes['subject'],
+                        description = RssArchiver.extract_content(mes),
+                        pubDate = mes['date'])
+                    for mes in messages
+            ])
+        dirPath = os.path.abspath("feeds")
+        if not os.path.exists(dirPath):
+            os.makedirs(dirPath)
+        fileName = mlist.fqdn_listname + '.xml'
+        filePath = os.path.abspath(os.path.join(dirPath, fileName))
+        with open(filePath, "w") as f:
+            rss.write_xml(f)     
+
+    @staticmethod
+    def list_url(mlist):
+        return None
+
+    @staticmethod
+    def permalink(mlist, msg):
+        return None
+
+    @staticmethod
+    def create_if_none(archive_dir):
+        try:
+            os.makedirs(archive_dir, 0775)
+        except OSError as error:
+            if error.errno != errno.EEXIST:
+                raise
+
+    @staticmethod
+    def archive_message(mlist, message):
+        archive_dir = os.path.join(config.ARCHIVE_DIR, 'rssarchive')
+        RssArchiver.create_if_none(archive_dir)
+        list_dir = os.path.join(archive_dir, mlist.fqdn_listname)
+        mailbox = Maildir(list_dir, create=True, factory=None)
+        lock_file = os.path.join(
+            config.LOCK_DIR, '{0}-maildir.lock'.format(mlist.fqdn_listname))
+        lock = Lock(lock_file)
+        try:
+            lock.lock(timeout=timedelta(seconds=1))
+            mailbox.add(message)
+            if RssArchiver.is_feed_enabled(mlist):
+                RssArchiver.generate_rss_feed(mlist);
+        except TimeOutError:
+            log.error('Unable to acquire rss archiver lock for {0}, '
+                      'discarding: {1}'.format(
+                          mlist.fqdn_listname,
+                          message.get('message-id', 'n/a')))
+        finally:
+            lock.unlock(unconditionally=True)
 
=== added file 'src/mailman/archiving/tests/test_rssarchive.py'
--- src/mailman/archiving/tests/test_rssarchive.py	1970-01-01 00:00:00 +0000
+++ src/mailman/archiving/tests/test_rssarchive.py	2013-09-01 16:37:50 +0000
@@ -0,0 +1,203 @@
+# Copyright (C) 2012-2013 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Test the rssarchiver."""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'TestRssArchiver',
+    ]
+
+
+import os
+import shutil
+import tempfile
+import unittest
+import threading
+
+from xml.dom.minidom import parse, parseString
+from email import message_from_file
+from flufl.lock import Lock
+
+from mailman.app.lifecycle import create_list
+from mailman.archiving.rssarchive import RssArchiver
+from mailman.config import config
+from mailman.database.transaction import transaction
+from mailman.testing.helpers import LogFileMark
+from mailman.testing.helpers import (
+    specialized_message_from_string as mfs)
+from mailman.testing.layers import ConfigLayer
+from mailman.utilities.email import add_message_hash
+
+
+class TestRssArchiver(unittest.TestCase):
+    """Test the rss archiver."""
+
+    layer = ConfigLayer
+
+    def setUp(self):
+        # Create a fake mailing list and message object
+        self._msg = mfs("""\
+To: [e-mail]
+From: [e-mail]
+Date: 16-05-2012
+Subject: Testing the test list
+Message-ID: <ant>
+X-Message-ID-Hash: MS6QLWERIJLGCRF44J7USBFDELMNT2BW
+
+Tests are better than no tests
+but the water deserves to be swum.
+""")
+        with transaction():
+            self._mlist = create_list('[e-mail]')
+        # Set up a temporary directory for the prototype archiver so that it's
+        # easier to clean up.
+        self._tempdir = tempfile.mkdtemp()
+        config.push('rssarchive', """
+        [paths.testing]
+        archive_dir: {0}
+        """.format(self._tempdir))
+        # Capture the structure of a maildir.
+        self._expected_dir_structure = set(
+            (os.path.join(config.ARCHIVE_DIR, path) for path in (
+                'rssarchive',
+                os.path.join('rssarchive', self._mlist.fqdn_listname),
+                os.path.join('rssarchive', self._mlist.fqdn_listname, 'cur'),
+                os.path.join('rssarchive', self._mlist.fqdn_listname, 'new'),
+                os.path.join('rssarchive', self._mlist.fqdn_listname, 'tmp'),
+                )))
+        self._expected_dir_structure.add(config.ARCHIVE_DIR)
+
+    def tearDown(self):
+        shutil.rmtree(self._tempdir)
+        config.pop('rssarchive')
+
+    def _find(self, path):
+        all_filenames = set()
+        for dirpath, dirnames, filenames in os.walk(path):
+            if not isinstance(dirpath, unicode):
+                dirpath = unicode(dirpath)
+            all_filenames.add(dirpath)
+            for filename in filenames:
+                new_filename = filename
+                if not isinstance(filename, unicode):
+                    new_filename = unicode(filename)
+                all_filenames.add(os.path.join(dirpath, new_filename))
+        return all_filenames
+
+    def test_rssarchiver_xml_file(self):
+        RssArchiver.archive_message(self._mlist, self._msg)
+	xml_path = os.path.join('/root/mailman/parts/test/feeds/', (self._mlist.fqdn_listname + '.xml'))
+
+        datasource = open(xml_path)
+        dom = parse(datasource)
+	rssNode = dom.childNodes[0]
+	channelNode = rssNode.childNodes[0]
+
+	self.assertEqual(len(channelNode.childNodes), 7)
+
+	itemNode = channelNode.childNodes[6]
+	titleNode = itemNode.childNodes[0]
+	self.assertEqual(titleNode.nodeName, 'title')
+	self.assertEqual(titleNode.firstChild.nodeValue, 'Testing the test list')
+
+    def test_set_length_limit(self):
+        RssArchiver.set_length_limit(self._mlist, 2)
+        self.assertEqual(RssArchiver.get_length_limit(self._mlist), 2)
+
+    def test_set_feed_enabled(self):
+        RssArchiver.set_feed_enabled(self._mlist, 1)
+        self.assertEqual(RssArchiver.is_feed_enabled(self._mlist), 1)
+        RssArchiver.set_feed_enabled(self._mlist, 0)
+        self.assertEqual(RssArchiver.is_feed_enabled(self._mlist), 0)       
+ 
+    def test_archive_maildir_created(self):
+        # Archiving a message to the  rssarchiver should create the
+        # expected directory structure.
+        RssArchiver.archive_message(self._mlist, self._msg)
+        all_filenames = self._find(config.ARCHIVE_DIR)
+        # Check that the directory structure has been created and we have one
+        # more file (the archived message) than expected directories.
+        archived_messages = [x for x in (all_filenames - self._expected_dir_structure) if 'rss.db' not in x] 
+        self.assertEqual(len(archived_messages), 1)
+        self.assertTrue(
+            archived_messages.pop().startswith(
+                os.path.join(config.ARCHIVE_DIR, 'rssarchive',
+                             self._mlist.fqdn_listname, 'new')))
+
+    def test_archive_maildir_existence_does_not_raise(self):
+        # Archiving a second message does not cause an EEXIST to be raised
+        # when a second message is archived.
+        new_dir = None
+        RssArchiver.archive_message(self._mlist, self._msg)
+        for directory in ('cur', 'new', 'tmp'):
+            path = os.path.join(config.ARCHIVE_DIR, 'rssarchive',
+                                self._mlist.fqdn_listname, directory)
+            if directory == 'new':
+                new_dir = path
+            self.assertTrue(os.path.isdir(path))
+        # There should be one message in the 'new' directory.
+        self.assertEqual(len(os.listdir(new_dir)), 1)
+        # Archive a second message.  If an exception occurs, let it fail the
+        # test.  Afterward, two messages should be in the 'new' directory.
+        del self._msg['message-id']
+        del self._msg['x-message-id-hash']
+        self._msg['Message-ID'] = '<bee>'
+        add_message_hash(self._msg)
+        RssArchiver.archive_message(self._mlist, self._msg)
+        self.assertEqual(len(os.listdir(new_dir)), 2)
+
+    def test_archive_lock_used(self):
+        # Test that locking the maildir when adding works as a failure here
+        # could mean we lose mail.
+        lock_file = os.path.join(
+            config.LOCK_DIR, '{0}-maildir.lock'.format(
+                self._mlist.fqdn_listname))
+        with Lock(lock_file):
+            # Acquire the archiver lock, then make sure the archiver logs the
+            # fact that it could not acquire the lock.
+            archive_thread = threading.Thread(
+                target=RssArchiver.archive_message,
+                args=(self._mlist, self._msg))
+            mark = LogFileMark('mailman.error')
+            archive_thread.run()
+            # Test that the archiver output the correct error.
+            line = mark.readline()
+            # XXX 2012-03-15 BAW: we really should remove timestamp prefixes
+            # from the loggers when under test.
+            self.assertTrue(line.endswith(
+                'Unable to acquire rss archiver lock for {0}, '
+                'discarding: {1}\n'.format(
+                    self._mlist.fqdn_listname,
+                    self._msg.get('message-id'))))
+        # Check that the message didn't get archived.
+        created_files = self._find(config.ARCHIVE_DIR)
+        self.assertEqual(self._expected_dir_structure, created_files)
+
+    def test_rssarchiver_good_path(self):
+        # Verify the good path; the message gets archived.
+        RssArchiver.archive_message(self._mlist, self._msg)
+        new_path = os.path.join(
+            config.ARCHIVE_DIR, 'rssarchive', self._mlist.fqdn_listname, 'new')
+        archived_messages = list(os.listdir(new_path))
+        self.assertEqual(len(archived_messages), 1)
+        # Check that the email has been added.
+        with open(os.path.join(new_path, archived_messages[0])) as fp:
+            archived_message = message_from_file(fp)
+        self.assertEqual(self._msg.as_string(), archived_message.as_string())
 
=== modified file 'src/mailman/config/mailman.cfg'
--- src/mailman/config/mailman.cfg	2013-01-16 23:54:32 +0000
+++ src/mailman/config/mailman.cfg	2013-09-01 16:37:50 +0000
@@ -91,3 +91,7 @@
 class: mailman.runners.digest.DigestRunner
 
 [style.default]
+
+[archiver.rssarchive]
+class: mailman.archiving.rssarchive.RssArchiver
+enable: yes
 
=== modified file 'src/mailman/rest/root.py'
--- src/mailman/rest/root.py	2013-01-01 14:05:42 +0000
+++ src/mailman/rest/root.py	2013-09-01 16:37:50 +0000
@@ -42,6 +42,7 @@
 from mailman.rest.preferences import ReadOnlyPreferences
 from mailman.rest.templates import TemplateFinder
 from mailman.rest.users import AUser, AllUsers
+from mailman.rest.rss import RssFeed
 
 
 
 
@@ -136,6 +137,14 @@
             return AList(list_identifier), segments
 
     @resource.child()
+    def feeds(self, request, segments):
+        """/<api>/feeds/<list>/
+        """
+        #return http.ok([], etag({u'segments': segments[0]}))
+        list_identifier = segments.pop(0)
+        return RssFeed(list_identifier), segments
+
+    @resource.child()
     def members(self, request, segments):
         """/<api>/members"""
         if len(segments) == 0:
 
=== added file 'src/mailman/rest/rss.py'
--- src/mailman/rest/rss.py	1970-01-01 00:00:00 +0000
+++ src/mailman/rest/rss.py	2013-09-01 16:37:50 +0000
@@ -0,0 +1,72 @@
+# Copyright (C) 2010-2013 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""REST for members."""
+
+from __future__ import absolute_import, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'RssAdmin',
+    ]
+
+
+from uuid import UUID
+from operator import attrgetter
+from restish import http, resource
+from zope.component import getUtility
+
+from mailman.app.membership import delete_member
+from mailman.interfaces.address import InvalidEmailAddressError
+from mailman.interfaces.listmanager import IListManager, NoSuchListError
+from mailman.interfaces.member import (
+    AlreadySubscribedError, DeliveryMode, MemberRole, MembershipError,
+    NotAMemberError)
+from mailman.interfaces.subscriptions import ISubscriptionService
+from mailman.interfaces.user import UnverifiedAddressError
+from mailman.interfaces.usermanager import IUserManager
+from mailman.rest.helpers import (
+    CollectionMixin, PATCH, etag, no_content, paginate, path_to)
+from mailman.rest.preferences import Preferences, ReadOnlyPreferences
+from mailman.rest.validator import (
+    Validator, enum_validator, subscriber_validator)
+from mailman.archiving.rssarchive import RssArchiver
+
+class RssFeed(resource.Resource):
+    def __init__(self, list_identifier):
+        # list-id is preferred, but for backward compatibility, fqdn_listname
+        # is also accepted.  If the string contains '@', treat it as the
+        # latter.
+        manager = getUtility(IListManager)
+        if '@' in list_identifier:
+            self._mlist = manager.get(list_identifier)
+        else:
+            self._mlist = manager.get_by_list_id(list_identifier)
+
+    @resource.GET()
+    def getData(self, request):
+        is_feed_enabled = RssArchiver.is_feed_enabled(self._mlist)
+        size = RssArchiver.get_length_limit(self._mlist)
+        return http.ok([], etag({u'enabled': is_feed_enabled, u'sizeLimit': size}))
+
+    @resource.POST()
+    def setData(self, request):
+        length_limit = request.POST.get('size_limit')
+        is_enabled = request.POST.get('is_enabled')
+        RssArchiver.set_length_limit(self._mlist, length_limit)
+        RssArchiver.set_feed_enabled(self._mlist, is_enabled)
+        return no_content()