Merge lp:~abompard/mailman/bug-1060951 into lp:mailman

[Aurélien Bompard]

This branch solves bug #1060951 by removing the auto-conversion of headers to unicode in the Message class.

[Barry Warsaw (barry)]

Some comments and questions. Thanks for working on this!

review: Needs Fixing

[lp:~abompard/mailman/bug-1060951]

7271. By Aurélien Bompard on 2014-12-04

Convert unicode instance testing to bytes instance testing

[Aurélien Bompard (abompard)]

I fixed the isinstance calls and the exception, but I don't understand your comment about Python's base email class.

[Barry Warsaw (barry)]

review: Approve

[Barry Warsaw (barry)]

Committed, after fixing a few other minor style fixes. Thanks!

=== modified file 'src/mailman/app/bounces.py'
--- src/mailman/app/bounces.py	2014-01-07 03:43:59 +0000
+++ src/mailman/app/bounces.py	2014-12-04 09:39:12 +0000
@@ -200,10 +200,13 @@
                 optionsurl=member.options_url,
                 owneraddr=mlist.owner_address,
                 )
+    message_id = msg['message-id']
+    if isinstance(message_id, bytes):
+        message_id = message_id.decode("ascii")
     pendable = _ProbePendable(
         # We can only pend unicodes.
         member_id=member.member_id.hex,
-        message_id=msg['message-id'],
+        message_id=message_id,
         )
     token = getUtility(IPendings).add(pendable)
     mailbox, domain_parts = split_email(mlist.bounces_address)
 
=== modified file 'src/mailman/app/moderator.py'
--- src/mailman/app/moderator.py	2014-04-28 15:23:35 +0000
+++ src/mailman/app/moderator.py	2014-12-04 09:39:12 +0000
@@ -86,9 +86,9 @@
     # Message-ID header.
     message_id = msg.get('message-id')
     if message_id is None:
-        msg['Message-ID'] = message_id = unicode(make_msgid())
-    assert isinstance(message_id, unicode), (
-        'Message-ID is not a unicode: %s' % message_id)
+        msg['Message-ID'] = message_id = make_msgid().decode("ascii")
+    if isinstance(message_id, bytes):
+        message_id = message_id.decode("ascii")
     getUtility(IMessageStore).add(msg)
     # Prepare the message metadata with some extra information needed only by
     # the moderation interface.
 
=== modified file 'src/mailman/archiving/mailarchive.py'
--- src/mailman/archiving/mailarchive.py	2014-01-01 14:59:42 +0000
+++ src/mailman/archiving/mailarchive.py	2014-12-04 09:39:12 +0000
@@ -68,6 +68,8 @@
         message_id_hash = msg.get('x-message-id-hash')
         if message_id_hash is None:
             return None
+        if isinstance(message_id_hash, bytes):
+            message_id_hash = message_id_hash.decode("ascii")
         return urljoin(self.base_url, message_id_hash)
 
     def archive_message(self, mlist, msg):
 
=== modified file 'src/mailman/archiving/mhonarc.py'
--- src/mailman/archiving/mhonarc.py	2014-01-01 14:59:42 +0000
+++ src/mailman/archiving/mhonarc.py	2014-12-04 09:39:12 +0000
@@ -73,6 +73,8 @@
         message_id_hash = msg.get('x-message-id-hash')
         if message_id_hash is None:
             return None
+        if isinstance(message_id_hash, bytes):
+            message_id_hash = message_id_hash.decode("ascii")
         return urljoin(self.list_url(mlist), message_id_hash)
 
     def archive_message(self, mlist, msg):
 
=== modified file 'src/mailman/archiving/prototype.py'
--- src/mailman/archiving/prototype.py	2014-03-02 20:59:30 +0000
+++ src/mailman/archiving/prototype.py	2014-12-04 09:39:12 +0000
@@ -68,6 +68,8 @@
         message_id_hash = msg.get('x-message-id-hash')
         if message_id_hash is None:
             return None
+        if isinstance(message_id_hash, bytes):
+            message_id_hash = message_id_hash.decode("ascii")
         return urljoin(Prototype.list_url(mlist), message_id_hash)
 
     @staticmethod
 
=== modified file 'src/mailman/commands/docs/unshunt.rst'
--- src/mailman/commands/docs/unshunt.rst	2014-04-28 15:23:35 +0000
+++ src/mailman/commands/docs/unshunt.rst	2014-12-04 09:39:12 +0000
@@ -83,7 +83,7 @@
     2
 
     >>> sorted(item.msg['message-id'] for item in items)
-    [u'<badgers>', u'<crow>']
+    ['<badgers>', '<crow>']
 
 
 Return to the original queue
 
=== modified file 'src/mailman/commands/eml_membership.py'
--- src/mailman/commands/eml_membership.py	2014-01-01 14:59:42 +0000
+++ src/mailman/commands/eml_membership.py	2014-12-04 09:39:12 +0000
@@ -72,6 +72,8 @@
             print(_('$self.name: No valid address found to subscribe'),
                   file=results)
             return ContinueProcessing.no
+        if isinstance(address, bytes):
+            address = address.decode("ascii")
         # Have we already seen one join request from this user during the
         # processing of this email?
         joins = getattr(results, 'joins', set())
 
=== modified file 'src/mailman/email/message.py'
--- src/mailman/email/message.py	2014-04-28 15:23:35 +0000
+++ src/mailman/email/message.py	2014-12-04 09:39:12 +0000
@@ -53,29 +53,6 @@
         self.__version__ = VERSION
         email.message.Message.__init__(self)
 
-    def __getitem__(self, key):
-        # Ensure that header values are unicodes.
-        value = email.message.Message.__getitem__(self, key)
-        if isinstance(value, str):
-            return unicode(value, 'ascii')
-        return value
-
-    def get(self, name, failobj=None):
-        # Ensure that header values are unicodes.
-        value = email.message.Message.get(self, name, failobj)
-        if isinstance(value, str):
-            return unicode(value, 'ascii')
-        return value
-
-    def get_all(self, name, failobj=None):
-        # Ensure all header values are unicodes.
-        missing = object()
-        all_values = email.message.Message.get_all(self, name, missing)
-        if all_values is missing:
-            return failobj
-        return [(unicode(value, 'ascii') if isinstance(value, str) else value)
-                for value in all_values]
-
     # BAW: For debugging w/ bin/dumpdb.  Apparently pprint uses repr.
     def __repr__(self):
         return self.__str__()
@@ -144,18 +121,15 @@
                 field_values = self.get_all(header, [])
                 senders.extend(address.lower() for (display_name, address)
                                in email.utils.getaddresses(field_values))
-        # Filter out None and the empty string.
-        return [sender for sender in senders if sender]
-
-    def get_filename(self, failobj=None):
-        """Some MUA have bugs in RFC2231 filename encoding and cause
-        Mailman to stop delivery in Scrubber.py (called from ToDigest.py).
-        """
-        try:
-            filename = email.message.Message.get_filename(self, failobj)
-            return filename
-        except (UnicodeError, LookupError, ValueError):
-            return failobj
+        # Filter out None and the empty string, and convert to unicode.
+        clean_senders = []
+        for sender in senders:
+            if not sender:
+                continue
+            if isinstance(sender, bytes):
+                sender = sender.decode("ascii")
+            clean_senders.append(sender)
+        return clean_senders
 
 
 
 
 
=== modified file 'src/mailman/email/tests/test_message.py'
--- src/mailman/email/tests/test_message.py	2014-01-01 14:59:42 +0000
+++ src/mailman/email/tests/test_message.py	2014-12-04 09:39:12 +0000
@@ -26,9 +26,10 @@
 
 
 import unittest
+from email.parser import FeedParser
 
 from mailman.app.lifecycle import create_list
-from mailman.email.message import UserNotification
+from mailman.email.message import UserNotification, Message
 from mailman.testing.helpers import get_queue_messages
 from mailman.testing.layers import ConfigLayer
 
@@ -58,3 +59,34 @@
         self.assertEqual(len(messages), 1)
         self.assertEqual(messages[0].msg.get_all('precedence'), 
                          ['omg wtf bbq'])
+
+
+
 
+class TestMessageSubclass(unittest.TestCase):
+
+    def test_i18n_filenames(self):
+        parser = FeedParser(_factory=Message)
+        parser.feed(b"""Message-ID: <[e-mail]>
+Content-Type: multipart/mixed; boundary="------------050607040206050605060208"
+
+This is a multi-part message in MIME format.
+--------------050607040206050605060208
+Content-Type: text/plain; charset=UTF-8
+Content-Transfer-Encoding: quoted-printable
+
+Test message containing an attachment with an accented filename
+
+--------------050607040206050605060208
+Content-Disposition: attachment;
+	filename*=UTF-8''d%C3%A9jeuner.txt
+
+Test content
+--------------050607040206050605060208--
+""")
+        msg = parser.close()
+        attachment = msg.get_payload()[1]
+        try:
+            filename = attachment.get_filename()
+        except TypeError as e:
+            self.fail(e)
+        self.assertEqual(filename, u"d\xe9jeuner.txt")
 
=== modified file 'src/mailman/model/bounce.py'
--- src/mailman/model/bounce.py	2014-09-22 18:47:02 +0000
+++ src/mailman/model/bounce.py	2014-12-04 09:39:12 +0000
@@ -57,7 +57,10 @@
         self.list_id = list_id
         self.email = email
         self.timestamp = now()
-        self.message_id = msg['message-id']
+        msgid = msg['message-id']
+        if isinstance(msgid, bytes):
+            msgid = msgid.decode("ascii")
+        self.message_id = msgid
         self.context = (BounceContext.normal if context is None else context)
         self.processed = False
 
 
=== modified file 'src/mailman/model/messagestore.py'
--- src/mailman/model/messagestore.py	2014-09-28 00:17:05 +0000
+++ src/mailman/model/messagestore.py	2014-12-04 09:39:12 +0000
@@ -58,6 +58,8 @@
             raise ValueError('Exactly one Message-ID header required')
         # Calculate and insert the X-Message-ID-Hash.
         message_id = message_ids[0]
+        if isinstance(message_id, bytes):
+            message_id = message_id.decode("ascii")
         # Complain if the Message-ID already exists in the storage.
         existing = store.query(Message).filter(
             Message.message_id == message_id).first()
 
=== modified file 'src/mailman/rules/implicit_dest.py'
--- src/mailman/rules/implicit_dest.py	2014-01-01 14:59:42 +0000
+++ src/mailman/rules/implicit_dest.py	2014-12-04 09:39:12 +0000
@@ -73,6 +73,8 @@
         recipients = set()
         for header in ('to', 'cc', 'resent-to', 'resent-cc'):
             for fullname, address in getaddresses(msg.get_all(header, [])):
+                if isinstance(address, bytes):
+                    address = address.decode("ascii")
                 address = address.lower()
                 if address in aliases:
                     return False
 
=== modified file 'src/mailman/utilities/email.py'
--- src/mailman/utilities/email.py	2014-04-28 15:23:35 +0000
+++ src/mailman/utilities/email.py	2014-12-04 09:39:12 +0000
@@ -62,6 +62,8 @@
     message_id = msg.get('message-id')
     if message_id is None:
         return
+    if isinstance(message_id, bytes):
+        message_id = message_id.decode("ascii")
     # The angle brackets are not part of the Message-ID.  See RFC 2822
     # and http://wiki.list.org/display/DEV/Stable+URLs
     if message_id.startswith('<') and message_id.endswith('>'):