Merge lp:~abompard/mailman/import21 into lp:mailman

Proposed by Aurélien Bompard on 2013-10-22
Status: Merged
Merge reported by: Barry Warsaw
Merged at revision: not available
Proposed branch: lp:~abompard/mailman/import21
Merge into: lp:mailman
Diff against target: 1337 lines (+1121/-19)
8 files modified
src/mailman/app/templates.py (+1/-1)
src/mailman/app/tests/test_templates.py (+11/-0)
src/mailman/commands/cli_import.py (+6/-2)
src/mailman/handlers/decorate.py (+7/-2)
src/mailman/model/listmanager.py (+2/-0)
src/mailman/model/tests/test_listmanager.py (+15/-0)
src/mailman/utilities/importer.py (+331/-7)
src/mailman/utilities/tests/test_import.py (+748/-7)
To merge this branch: bzr merge lp:~abompard/mailman/import21
Reviewer Review Type Date Requested Status
Barry Warsaw 2013-10-22 Abstain on 2014-03-28
Review via email: mp+192146@code.launchpad.net

Description of the change

This branch contains my work on the import21 command.

Most list properties are imported, as well as their rosters.
There are a few exceptions that I don't know how to handle, such as umbrella_list & umbrella_member_suffix, or the fact that header_filter_rules can't be directly converted to header_matches because the former used to match the whole header while the latter only matches the header value, not the header name itself, so I would need to "split" the regexp, which seems very brittle.

There are currently 67 unit tests (which are great for conversions), and it's been tested on about 300 mailing-lists (the Fedora project's lists).

To post a comment you must log in.
Aurélien Bompard (abompard) wrote :

Oh by the way, I'm not very at ease with Bazaar yet, so I may have made a mistake with the merging, please make sure I did not do something funny there... Sorry.

Aurélien Bompard (abompard) wrote :

Another thing, it contains a fix on mailman.app.templates (and the associated test case) to expect templates to be UTF-8 encoded, as discussed on the mailing-list.

Aurélien Bompard (abompard) wrote :

Hey Barry! If there's anything I can do help you merging this branch, please feel free to ask.

lp:~abompard/mailman/import21 updated on 2014-01-27
7235. By Aurélien Bompard on 2014-01-27

Merge from the main branch

Kẏra (thekyriarchy) wrote :

Barry, is there a reason this hasn't been reviewed and approved/rejected yet?

Barry Warsaw (barry) wrote :

Just to let you know, I am working my way through a review/merge of this branch.

review: Abstain

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'src/mailman/app/templates.py'
2--- src/mailman/app/templates.py 2014-01-01 14:59:42 +0000
3+++ src/mailman/app/templates.py 2014-01-27 11:01:58 +0000
4@@ -103,4 +103,4 @@
5 def get(self, uri):
6 """See `ITemplateLoader`."""
7 with closing(urllib2.urlopen(uri)) as fp:
8- return fp.read()
9+ return unicode(fp.read(), "utf-8")
10
11=== modified file 'src/mailman/app/tests/test_templates.py'
12--- src/mailman/app/tests/test_templates.py 2014-01-01 14:59:42 +0000
13+++ src/mailman/app/tests/test_templates.py 2014-01-27 11:01:58 +0000
14@@ -126,3 +126,14 @@
15 with self.assertRaises(urllib2.URLError) as cm:
16 self._loader.get('mailman:///missing@example.com/en/foo/demo.txt')
17 self.assertEqual(cm.exception.reason, 'No such file')
18+
19+ def test_non_ascii(self):
20+ # mailman://demo.txt with non-ascii content
21+ test_text = b'\xe4\xb8\xad'
22+ path = os.path.join(self.var_dir, 'templates', 'site', 'it')
23+ os.makedirs(path)
24+ with open(os.path.join(path, 'demo.txt'), 'w') as fp:
25+ print(test_text, end='', file=fp)
26+ content = self._loader.get('mailman:///it/demo.txt')
27+ self.assertTrue(isinstance(content, unicode))
28+ self.assertEqual(content, test_text.decode("utf-8"))
29
30=== modified file 'src/mailman/commands/cli_import.py'
31--- src/mailman/commands/cli_import.py 2014-01-01 14:59:42 +0000
32+++ src/mailman/commands/cli_import.py 2014-01-27 11:01:58 +0000
33@@ -35,7 +35,7 @@
34 from mailman.database.transaction import transactional
35 from mailman.interfaces.command import ICLISubCommand
36 from mailman.interfaces.listmanager import IListManager
37-from mailman.utilities.importer import import_config_pck
38+from mailman.utilities.importer import import_config_pck, Import21Error
39
40
41
42
43@@ -93,4 +93,8 @@
44 print(_('Ignoring non-dictionary: {0!r}').format(
45 config_dict), file=sys.stderr)
46 continue
47- import_config_pck(mlist, config_dict)
48+ try:
49+ import_config_pck(mlist, config_dict)
50+ except Import21Error, e:
51+ print(e, file=sys.stderr)
52+ sys.exit(1)
53
54=== modified file 'src/mailman/handlers/decorate.py'
55--- src/mailman/handlers/decorate.py 2014-01-01 14:59:42 +0000
56+++ src/mailman/handlers/decorate.py 2014-01-27 11:01:58 +0000
57@@ -201,8 +201,8 @@
58
59
60
61 def decorate(mlist, uri, extradict=None):
62- """Expand the decoration template."""
63- if uri is None:
64+ """Expand the decoration template from its URI."""
65+ if uri is None or uri == '':
66 return ''
67 # Get the decorator template.
68 loader = getUtility(ITemplateLoader)
69@@ -211,6 +211,11 @@
70 language=mlist.preferred_language.code,
71 ))
72 template = loader.get(template_uri)
73+ return decorate_template(mlist, template, extradict)
74+
75+
76
77+def decorate_template(mlist, template, extradict=None):
78+ """Expand the decoration template."""
79 # Create a dictionary which includes the default set of interpolation
80 # variables allowed in headers and footers. These will be augmented by
81 # any key/value pairs in the extradict.
82
83=== modified file 'src/mailman/model/listmanager.py'
84--- src/mailman/model/listmanager.py 2014-01-01 14:59:42 +0000
85+++ src/mailman/model/listmanager.py 2014-01-27 11:01:58 +0000
86@@ -34,6 +34,7 @@
87 IListManager, ListAlreadyExistsError, ListCreatedEvent, ListCreatingEvent,
88 ListDeletedEvent, ListDeletingEvent)
89 from mailman.model.mailinglist import MailingList
90+from mailman.model.mime import ContentFilter
91 from mailman.utilities.datetime import now
92
93
94@@ -79,6 +80,7 @@
95 """See `IListManager`."""
96 fqdn_listname = mlist.fqdn_listname
97 notify(ListDeletingEvent(mlist))
98+ store.find(ContentFilter, ContentFilter.mailing_list == mlist).remove()
99 store.remove(mlist)
100 notify(ListDeletedEvent(fqdn_listname))
101
102
103=== modified file 'src/mailman/model/tests/test_listmanager.py'
104--- src/mailman/model/tests/test_listmanager.py 2014-01-01 14:59:42 +0000
105+++ src/mailman/model/tests/test_listmanager.py 2014-01-27 11:01:58 +0000
106@@ -30,6 +30,7 @@
107 import unittest
108
109 from zope.component import getUtility
110+from storm.locals import Store
111
112 from mailman.app.lifecycle import create_list
113 from mailman.app.moderator import hold_message
114@@ -40,6 +41,7 @@
115 from mailman.interfaces.requests import IListRequests
116 from mailman.interfaces.subscriptions import ISubscriptionService
117 from mailman.interfaces.usermanager import IUserManager
118+from mailman.model.mime import ContentFilter
119 from mailman.testing.helpers import (
120 event_subscribers, specialized_message_from_string)
121 from mailman.testing.layers import ConfigLayer
122@@ -129,6 +131,19 @@
123 saved_message = getUtility(IMessageStore).get_message_by_id('<argon>')
124 self.assertEqual(saved_message.as_string(), msg.as_string())
125
126+ def test_content_filters_are_deleted_when_mailing_list_is_deleted(self):
127+ # When a mailing list with content filters is deleted, the filters must
128+ # be deleted fist or an IntegrityError will be raised
129+ filter_names = ("filter_types", "pass_types",
130+ "filter_extensions", "pass_extensions")
131+ for fname in filter_names:
132+ setattr(self._ant, fname, ["test-filter-1", "test-filter-2"])
133+ getUtility(IListManager).delete(self._ant)
134+ store = Store.of(self._ant)
135+ filters = store.find(ContentFilter,
136+ ContentFilter.mailing_list == self._ant)
137+ self.assertEqual(filters.count(), 0)
138+
139
140
141
142 class TestListCreation(unittest.TestCase):
143
144=== modified file 'src/mailman/utilities/importer.py'
145--- src/mailman/utilities/importer.py 2014-01-01 14:59:42 +0000
146+++ src/mailman/utilities/importer.py 2014-01-27 11:01:58 +0000
147@@ -22,45 +22,183 @@
148 __metaclass__ = type
149 __all__ = [
150 'import_config_pck',
151+ 'Import21Error',
152 ]
153
154
155 import sys
156 import datetime
157+import os
158+from urllib2 import URLError
159
160-from mailman.interfaces.action import FilterAction
161-from mailman.interfaces.archiver import ArchivePolicy
162+from mailman.config import config
163+from mailman.core.errors import MailmanError
164+from mailman.interfaces.action import FilterAction, Action
165 from mailman.interfaces.autorespond import ResponseAction
166 from mailman.interfaces.digests import DigestFrequency
167 from mailman.interfaces.mailinglist import Personalization, ReplyToMunging
168 from mailman.interfaces.nntp import NewsgroupModeration
169-
170+from mailman.interfaces.archiver import ArchivePolicy
171+from mailman.interfaces.bans import IBanManager
172+from mailman.interfaces.mailinglist import IAcceptableAliasSet
173+from mailman.interfaces.bounce import UnrecognizedBounceDisposition
174+from mailman.interfaces.usermanager import IUserManager
175+from mailman.interfaces.member import DeliveryMode, DeliveryStatus, MemberRole
176+from mailman.interfaces.languages import ILanguageManager
177+from mailman.handlers.decorate import decorate, decorate_template
178+from mailman.utilities.i18n import search
179+from zope.component import getUtility
180+
181+
182+
183
184+class Import21Error(MailmanError):
185+ pass
186+
187+
188
189+def str_to_unicode(value):
190+ # Convert a string to unicode when the encoding is not declared
191+ if isinstance(value, unicode):
192+ return value
193+ for encoding in ("ascii", "utf-8"):
194+ try:
195+ return unicode(value, encoding)
196+ except UnicodeDecodeError:
197+ continue
198+ # we did our best, use replace
199+ return unicode(value, 'ascii', 'replace')
200
201
202
203 def seconds_to_delta(value):
204 return datetime.timedelta(seconds=value)
205
206+
207
208+def days_to_delta(value):
209+ return datetime.timedelta(days=value)
210+
211+
212
213+def list_members_to_unicode(value):
214+ return [ unicode(item) for item in value ]
215+
216+
217
218+def filter_action_mapping(value):
219+ # The filter_action enum values have changed. In Mailman 2.1 the order was
220+ # 'Discard', 'Reject', 'Forward to List Owner', 'Preserve'.
221+ # In 3.0 it's 'hold', 'reject', 'discard', 'accept', 'defer', 'forward',
222+ # 'preserve'
223+ if value == 0:
224+ return FilterAction.discard
225+ elif value == 1:
226+ return FilterAction.reject
227+ elif value == 2:
228+ return FilterAction.forward
229+ elif value == 3:
230+ return FilterAction.preserve
231+ else:
232+ raise ValueError("Unknown filter_action value: %s" % value)
233+
234+
235
236+def member_action_mapping(value):
237+ # The mlist.default_member_action and mlist.default_nonmember_action enum
238+ # values are different in Mailman 2.1, because they have been merged into a
239+ # single enum in Mailman 3
240+ # For default_member_action, which used to be called
241+ # member_moderation_action, the values were:
242+ # 0==Hold, 1=Reject, 2==Discard
243+ if value == 0:
244+ return Action.hold
245+ elif value == 1:
246+ return Action.reject
247+ elif value == 2:
248+ return Action.discard
249+def nonmember_action_mapping(value):
250+ # For default_nonmember_action, which used to be called
251+ # generic_nonmember_action, the values were:
252+ # 0==Accept, 1==Hold, 2==Reject, 3==Discard
253+ if value == 0:
254+ return Action.accept
255+ elif value == 1:
256+ return Action.hold
257+ elif value == 2:
258+ return Action.reject
259+ elif value == 3:
260+ return Action.discard
261+
262+
263
264+def unicode_to_string(value):
265+ return str(value) if value is not None else None
266+
267+
268
269+def check_language_code(code):
270+ if code is None:
271+ return None
272+ code = unicode(code)
273+ if code not in getUtility(ILanguageManager):
274+ msg = """Missing language: {0}
275+You must add a section describing this language in your mailman.cfg file.
276+This section should look like this:
277+[language.{0}]
278+# The English name for the language.
279+description: CHANGE ME
280+# And the default character set for the language.
281+charset: utf-8
282+# Whether the language is enabled or not.
283+enabled: yes
284+""".format(code)
285+ raise Import21Error(msg)
286+ return code
287+
288
289 # Attributes in Mailman 2 which have a different type in Mailman 3.
290 TYPES = dict(
291 autorespond_owner=ResponseAction,
292 autorespond_postings=ResponseAction,
293 autorespond_requests=ResponseAction,
294+ autoresponse_grace_period=days_to_delta,
295 bounce_info_stale_after=seconds_to_delta,
296 bounce_you_are_disabled_warnings_interval=seconds_to_delta,
297 digest_volume_frequency=DigestFrequency,
298- filter_action=FilterAction,
299+ filter_action=filter_action_mapping,
300 newsgroup_moderation=NewsgroupModeration,
301 personalize=Personalization,
302 reply_goes_to_list=ReplyToMunging,
303+ filter_types=list_members_to_unicode,
304+ pass_types=list_members_to_unicode,
305+ filter_extensions=list_members_to_unicode,
306+ pass_extensions=list_members_to_unicode,
307+ forward_unrecognized_bounces_to=UnrecognizedBounceDisposition,
308+ default_member_action=member_action_mapping,
309+ default_nonmember_action=nonmember_action_mapping,
310+ moderator_password=unicode_to_string,
311+ preferred_language=check_language_code,
312 )
313
314
315 # Attribute names in Mailman 2 which are renamed in Mailman 3.
316 NAME_MAPPINGS = dict(
317- host_name='mail_host',
318 include_list_post_header='allow_list_posts',
319 real_name='display_name',
320+ last_post_time='last_post_at',
321+ autoresponse_graceperiod='autoresponse_grace_period',
322+ autorespond_admin='autorespond_owner',
323+ autoresponse_admin_text='autoresponse_owner_text',
324+ filter_mime_types='filter_types',
325+ pass_mime_types='pass_types',
326+ filter_filename_extensions='filter_extensions',
327+ pass_filename_extensions='pass_extensions',
328+ bounce_processing='process_bounces',
329+ bounce_unrecognized_goes_to_list_owner='forward_unrecognized_bounces_to',
330+ mod_password='moderator_password',
331+ news_moderation='newsgroup_moderation',
332+ news_prefix_subject_too='nntp_prefix_subject_too',
333+ send_welcome_msg='send_welcome_message',
334+ send_goodbye_msg='send_goodbye_message',
335+ member_moderation_action='default_member_action',
336+ generic_nonmember_action='default_nonmember_action',
337+ )
338+
339+EXCLUDES = (
340+ "members",
341+ "digest_members",
342 )
343
344
345@@ -74,14 +212,19 @@
346 :type config_dict: dict
347 """
348 for key, value in config_dict.items():
349+ # Some attributes must not be directly imported
350+ if key in EXCLUDES:
351+ continue
352 # Some attributes from Mailman 2 were renamed in Mailman 3.
353 key = NAME_MAPPINGS.get(key, key)
354 # Handle the simple case where the key is an attribute of the
355 # IMailingList and the types are the same (modulo 8-bit/unicode
356 # strings).
357- if hasattr(mlist, key):
358+ # When attributes raise an exception, hasattr may think they don't
359+ # exist (see python issue 9666). Add them here.
360+ if hasattr(mlist, key) or key in ("preferred_language", ):
361 if isinstance(value, str):
362- value = unicode(value, 'ascii')
363+ value = str_to_unicode(value)
364 # Some types require conversion.
365 converter = TYPES.get(key)
366 if converter is not None:
367@@ -103,3 +246,184 @@
368 mlist.archive_policy = ArchivePolicy.public
369 else:
370 mlist.archive_policy = ArchivePolicy.never
371+ # Handle ban list
372+ for addr in config_dict.get('ban_list', []):
373+ IBanManager(mlist).ban(str_to_unicode(addr))
374+ # Handle acceptable aliases
375+ acceptable_aliases = config_dict.get('acceptable_aliases', '')
376+ if isinstance(acceptable_aliases, basestring):
377+ acceptable_aliases = acceptable_aliases.splitlines()
378+ for addr in acceptable_aliases:
379+ addr = addr.strip()
380+ if not addr:
381+ continue
382+ addr = str_to_unicode(addr)
383+ try:
384+ IAcceptableAliasSet(mlist).add(addr)
385+ except ValueError:
386+ IAcceptableAliasSet(mlist).add("^" + addr)
387+ # Handle conversion to URIs
388+ convert_to_uri = {
389+ "welcome_msg": "welcome_message_uri",
390+ "goodbye_msg": "goodbye_message_uri",
391+ "msg_header": "header_uri",
392+ "msg_footer": "footer_uri",
393+ "digest_header": "digest_header_uri",
394+ "digest_footer": "digest_footer_uri",
395+ }
396+ convert_placeholders = { # only the most common ones
397+ "%(real_name)s": "$display_name",
398+ "%(real_name)s@%(host_name)s": "$fqdn_listname",
399+ "%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s": "$listinfo_uri",
400+ }
401+ # Collect defaults
402+ defaults = {}
403+ for oldvar, newvar in convert_to_uri.iteritems():
404+ default_value = getattr(mlist, newvar)
405+ if not default_value:
406+ continue
407+ # Check if the value changed from the default
408+ try:
409+ default_text = decorate(mlist, default_value)
410+ except (URLError, KeyError):
411+ # Use case: importing the old a@ex.com into b@ex.com
412+ # We can't check if it changed from the default
413+ # -> don't import, we may do more harm than good and it's easy to
414+ # change if needed
415+ continue
416+ defaults[newvar] = (default_value, default_text)
417+ for oldvar, newvar in convert_to_uri.iteritems():
418+ if oldvar not in config_dict:
419+ continue
420+ text = config_dict[oldvar]
421+ text = unicode(text, "utf-8", "replace")
422+ for oldph, newph in convert_placeholders.iteritems():
423+ text = text.replace(oldph, newph)
424+ default_value, default_text = defaults.get(newvar, (None, None))
425+ if not text and not (default_value or default_text):
426+ continue # both are empty, leave it
427+ # Check if the value changed from the default
428+ try:
429+ expanded_text = decorate_template(mlist, text)
430+ except KeyError:
431+ # Use case: importing the old a@ex.com into b@ex.com
432+ # We can't check if it changed from the default
433+ # -> don't import, we may do more harm than good and it's easy to
434+ # change if needed
435+ continue
436+ if expanded_text and default_text \
437+ and expanded_text.strip() == default_text.strip():
438+ continue # keep the default
439+ # Write the custom value to the right file
440+ base_uri = "mailman:///$listname/$language/"
441+ if default_value:
442+ filename = default_value.rpartition("/")[2]
443+ else:
444+ filename = "%s.txt" % newvar[:-4]
445+ if not default_value or not default_value.startswith(base_uri):
446+ setattr(mlist, newvar, base_uri + filename)
447+ filepath = list(search(filename, mlist))[0]
448+ try:
449+ os.makedirs(os.path.dirname(filepath))
450+ except OSError, e:
451+ if e.errno != 17: # Already exists
452+ raise
453+ with open(filepath, "w") as template:
454+ template.write(text.encode('utf-8'))
455+ # Import rosters
456+ members = set(config_dict.get("members", {}).keys()
457+ + config_dict.get("digest_members", {}).keys())
458+ import_roster(mlist, config_dict, members, MemberRole.member)
459+ import_roster(mlist, config_dict, config_dict.get("owner", []),
460+ MemberRole.owner)
461+ import_roster(mlist, config_dict, config_dict.get("moderator", []),
462+ MemberRole.moderator)
463+
464+
465+
466
467+def import_roster(mlist, config_dict, members, role):
468+ """
469+ Import members lists from a config.pck configuration dictionary to a
470+ mailing list.
471+
472+ :param mlist: The mailing list.
473+ :type mlist: IMailingList
474+ :param config_dict: The Mailman 2.1 configuration dictionary.
475+ :type config_dict: dict
476+ :param members: The members list to import
477+ :type members: list
478+ :param role: The MemberRole to import them as
479+ :type role: MemberRole enum
480+ """
481+ usermanager = getUtility(IUserManager)
482+ for email in members:
483+ # for owners and members, the emails can have a mixed case, so
484+ # lowercase them all
485+ email = str_to_unicode(email).lower()
486+ roster = mlist.get_roster(role)
487+ if roster.get_member(email) is not None:
488+ print("%s is already imported with role %s" % (email, role),
489+ file=sys.stderr)
490+ continue
491+ address = usermanager.get_address(email)
492+ user = usermanager.get_user(email)
493+ if user is None:
494+ user = usermanager.create_user()
495+ if address is None:
496+ merged_members = {}
497+ merged_members.update(config_dict.get("members", {}))
498+ merged_members.update(config_dict.get("digest_members", {}))
499+ if merged_members.get(email, 0) != 0:
500+ original_email = str_to_unicode(merged_members[email])
501+ else:
502+ original_email = email
503+ address = usermanager.create_address(original_email)
504+ address.verified_on = datetime.datetime.now()
505+ user.link(address)
506+ mlist.subscribe(address, role)
507+ member = roster.get_member(email)
508+ assert member is not None
509+ prefs = config_dict.get("user_options", {}).get(email, 0)
510+ if email in config_dict.get("members", {}):
511+ member.preferences.delivery_mode = DeliveryMode.regular
512+ elif email in config_dict.get("digest_members", {}):
513+ if prefs & 8: # DisableMime
514+ member.preferences.delivery_mode = DeliveryMode.plaintext_digests
515+ else:
516+ member.preferences.delivery_mode = DeliveryMode.mime_digests
517+ else:
518+ # probably not adding a member role here
519+ pass
520+ if email in config_dict.get("language", {}):
521+ member.preferences.preferred_language = \
522+ check_language_code(config_dict["language"][email])
523+ # if the user already exists, display_name and password will be
524+ # overwritten
525+ if email in config_dict.get("usernames", {}):
526+ address.display_name = \
527+ str_to_unicode(config_dict["usernames"][email])
528+ user.display_name = \
529+ str_to_unicode(config_dict["usernames"][email])
530+ if email in config_dict.get("passwords", {}):
531+ user.password = config.password_context.encrypt(
532+ config_dict["passwords"][email])
533+ # delivery_status
534+ oldds = config_dict.get("delivery_status", {}).get(email, (0, 0))[0]
535+ if oldds == 0:
536+ member.preferences.delivery_status = DeliveryStatus.enabled
537+ elif oldds == 1:
538+ member.preferences.delivery_status = DeliveryStatus.unknown
539+ elif oldds == 2:
540+ member.preferences.delivery_status = DeliveryStatus.by_user
541+ elif oldds == 3:
542+ member.preferences.delivery_status = DeliveryStatus.by_moderator
543+ elif oldds == 4:
544+ member.preferences.delivery_status = DeliveryStatus.by_bounces
545+ # moderation
546+ if prefs & 128:
547+ member.moderation_action = Action.hold
548+ # other preferences
549+ member.preferences.acknowledge_posts = bool(prefs & 4) # AcknowledgePosts
550+ member.preferences.hide_address = bool(prefs & 16) # ConcealSubscription
551+ member.preferences.receive_own_postings = not bool(prefs & 2) # DontReceiveOwnPosts
552+ member.preferences.receive_list_copy = not bool(prefs & 256) # DontReceiveDuplicates
553
554=== modified file 'src/mailman/utilities/tests/test_import.py'
555--- src/mailman/utilities/tests/test_import.py 2014-01-01 14:59:42 +0000
556+++ src/mailman/utilities/tests/test_import.py 2014-01-27 11:01:58 +0000
557@@ -26,15 +26,41 @@
558 ]
559
560
561+import os
562 import cPickle
563 import unittest
564+from datetime import timedelta, datetime
565+from traceback import format_exc
566
567+from mailman.config import config
568 from mailman.app.lifecycle import create_list, remove_list
569+from mailman.testing.layers import ConfigLayer
570+from mailman.utilities.importer import import_config_pck, Import21Error
571 from mailman.interfaces.archiver import ArchivePolicy
572-from mailman.testing.layers import ConfigLayer
573-from mailman.utilities.importer import import_config_pck
574+from mailman.interfaces.action import Action, FilterAction
575+from mailman.interfaces.address import ExistingAddressError
576+from mailman.interfaces.bounce import UnrecognizedBounceDisposition
577+from mailman.interfaces.bans import IBanManager
578+from mailman.interfaces.mailinglist import IAcceptableAliasSet
579+from mailman.interfaces.nntp import NewsgroupModeration
580+from mailman.interfaces.autorespond import ResponseAction
581+from mailman.interfaces.templates import ITemplateLoader
582+from mailman.interfaces.usermanager import IUserManager
583+from mailman.interfaces.member import DeliveryMode, DeliveryStatus
584+from mailman.interfaces.languages import ILanguageManager
585+from mailman.model.address import Address
586+from mailman.handlers.decorate import decorate
587+from mailman.utilities.string import expand
588 from pkg_resources import resource_filename
589-
590+from enum import Enum
591+from zope.component import getUtility
592+from storm.locals import Store
593+
594+
595+
596
597+class DummyEnum(Enum):
598+ # For testing purposes
599+ val = 42
600
601
602
603 class TestBasicImport(unittest.TestCase):
604@@ -58,11 +84,12 @@
605 self._import()
606 self.assertEqual(self._mlist.display_name, 'Test')
607
608- def test_mail_host(self):
609- # The mlist.mail_host gets set.
610+ def test_mail_host_invariant(self):
611+ # The mlist.mail_host must not be updated when importing (it will
612+ # change the list_id property, which is supposed to be read-only)
613 self.assertEqual(self._mlist.mail_host, 'example.com')
614 self._import()
615- self.assertEqual(self._mlist.mail_host, 'heresy.example.org')
616+ self.assertEqual(self._mlist.mail_host, 'example.com')
617
618 def test_rfc2369_headers(self):
619 self._mlist.allow_list_posts = False
620@@ -71,6 +98,204 @@
621 self.assertTrue(self._mlist.allow_list_posts)
622 self.assertTrue(self._mlist.include_rfc2369_headers)
623
624+ def test_no_overwrite_rosters(self):
625+ # The mlist.members and mlist.digest_members rosters must not be
626+ # overwritten.
627+ for rname in ("members", "digest_members"):
628+ roster = getattr(self._mlist, rname)
629+ self.assertFalse(isinstance(roster, dict))
630+ self._import()
631+ self.assertFalse(isinstance(roster, dict),
632+ "The %s roster has been overwritten by the import" % rname)
633+
634+ def test_last_post_time(self):
635+ # last_post_time -> last_post_at
636+ self._pckdict["last_post_time"] = 1270420800.274485
637+ self.assertEqual(self._mlist.last_post_at, None)
638+ self._import()
639+ # convert 1270420800.2744851 to datetime
640+ expected = datetime(2010, 4, 4, 22, 40, 0, 274485)
641+ self.assertEqual(self._mlist.last_post_at, expected)
642+
643+ def test_autoresponse_grace_period(self):
644+ # autoresponse_graceperiod -> autoresponse_grace_period
645+ # must be a timedelta, not an int
646+ self._mlist.autoresponse_grace_period = timedelta(days=42)
647+ self._import()
648+ self.assertTrue(isinstance(
649+ self._mlist.autoresponse_grace_period, timedelta))
650+ self.assertEqual(self._mlist.autoresponse_grace_period,
651+ timedelta(days=90))
652+
653+ def test_autoresponse_admin_to_owner(self):
654+ # admin -> owner
655+ self._mlist.autorespond_owner = DummyEnum.val
656+ self._mlist.autoresponse_owner_text = 'DUMMY'
657+ self._import()
658+ self.assertEqual(self._mlist.autorespond_owner, ResponseAction.none)
659+ self.assertEqual(self._mlist.autoresponse_owner_text, '')
660+
661+ #def test_administrative(self):
662+ # # administrivia -> administrative
663+ # self._mlist.administrative = None
664+ # self._import()
665+ # self.assertTrue(self._mlist.administrative)
666+
667+ def test_filter_pass_renames(self):
668+ # mime_types -> types
669+ # filename_extensions -> extensions
670+ self._mlist.filter_types = ["dummy"]
671+ self._mlist.pass_types = ["dummy"]
672+ self._mlist.filter_extensions = ["dummy"]
673+ self._mlist.pass_extensions = ["dummy"]
674+ self._import()
675+ self.assertEqual(list(self._mlist.filter_types), [])
676+ self.assertEqual(list(self._mlist.filter_extensions),
677+ ['exe', 'bat', 'cmd', 'com', 'pif',
678+ 'scr', 'vbs', 'cpl'])
679+ self.assertEqual(list(self._mlist.pass_types),
680+ ['multipart/mixed', 'multipart/alternative', 'text/plain'])
681+ self.assertEqual(list(self._mlist.pass_extensions), [])
682+
683+ def test_process_bounces(self):
684+ # bounce_processing -> process_bounces
685+ self._mlist.process_bounces = None
686+ self._import()
687+ self.assertTrue(self._mlist.process_bounces)
688+
689+ def test_forward_unrecognized_bounces_to(self):
690+ # bounce_unrecognized_goes_to_list_owner -> forward_unrecognized_bounces_to
691+ self._mlist.forward_unrecognized_bounces_to = DummyEnum.val
692+ self._import()
693+ self.assertEqual(self._mlist.forward_unrecognized_bounces_to,
694+ UnrecognizedBounceDisposition.administrators)
695+
696+ def test_moderator_password(self):
697+ # mod_password -> moderator_password
698+ self._mlist.moderator_password = str("TESTDATA")
699+ self._import()
700+ self.assertEqual(self._mlist.moderator_password, None)
701+
702+ def test_moderator_password_str(self):
703+ # moderator_password must not be unicode
704+ self._pckdict[b"mod_password"] = b'TESTVALUE'
705+ self._import()
706+ self.assertFalse(isinstance(self._mlist.moderator_password, unicode))
707+ self.assertEqual(self._mlist.moderator_password, b'TESTVALUE')
708+
709+ def test_newsgroup_moderation(self):
710+ # news_moderation -> newsgroup_moderation
711+ # news_prefix_subject_too -> nntp_prefix_subject_too
712+ self._mlist.newsgroup_moderation = DummyEnum.val
713+ self._mlist.nntp_prefix_subject_too = None
714+ self._import()
715+ self.assertEqual(self._mlist.newsgroup_moderation,
716+ NewsgroupModeration.none)
717+ self.assertTrue(self._mlist.nntp_prefix_subject_too)
718+
719+ def test_msg_to_message(self):
720+ # send_welcome_msg -> send_welcome_message
721+ # send_goodbye_msg -> send_goodbye_message
722+ self._mlist.send_welcome_message = None
723+ self._mlist.send_goodbye_message = None
724+ self._import()
725+ self.assertTrue(self._mlist.send_welcome_message)
726+ self.assertTrue(self._mlist.send_goodbye_message)
727+
728+ def test_ban_list(self):
729+ banned = [
730+ ("anne@example.com", "anne@example.com"),
731+ ("^.*@example.com", "bob@example.com"),
732+ ("non-ascii-\xe8@example.com", "non-ascii-\ufffd@example.com"),
733+ ]
734+ self._pckdict["ban_list"] = [ b[0].encode("iso-8859-1") for b in banned ]
735+ try:
736+ self._import()
737+ except UnicodeDecodeError, e:
738+ print(format_exc())
739+ self.fail(e)
740+ for _pattern, addr in banned:
741+ self.assertTrue(IBanManager(self._mlist).is_banned(addr))
742+
743+ def test_acceptable_aliases(self):
744+ # it used to be a plain-text field (values are newline-separated)
745+ aliases = ["alias1@example.com",
746+ "alias2@exemple.com",
747+ "non-ascii-\xe8@example.com",
748+ ]
749+ self._pckdict[b"acceptable_aliases"] = \
750+ ("\n".join(aliases)).encode("utf-8")
751+ self._import()
752+ alias_set = IAcceptableAliasSet(self._mlist)
753+ self.assertEqual(sorted(alias_set.aliases), aliases)
754+
755+ def test_acceptable_aliases_invalid(self):
756+ # values without an '@' sign used to be matched against the local part,
757+ # now we need to add the '^' sign
758+ aliases = ["invalid-value", ]
759+ self._pckdict[b"acceptable_aliases"] = \
760+ ("\n".join(aliases)).encode("utf-8")
761+ try:
762+ self._import()
763+ except ValueError, e:
764+ print(format_exc())
765+ self.fail("Invalid value '%s' caused a crash" % e)
766+ alias_set = IAcceptableAliasSet(self._mlist)
767+ self.assertEqual(sorted(alias_set.aliases),
768+ [ ("^" + a) for a in aliases ])
769+
770+ def test_acceptable_aliases_as_list(self):
771+ # in some versions of the pickle, it can be a list, not a string
772+ # (seen in the wild)
773+ aliases = [b"alias1@example.com", b"alias2@exemple.com" ]
774+ self._pckdict[b"acceptable_aliases"] = aliases
775+ try:
776+ self._import()
777+ except AttributeError:
778+ print(format_exc())
779+ self.fail("Import does not handle acceptable_aliases as list")
780+ alias_set = IAcceptableAliasSet(self._mlist)
781+ self.assertEqual(sorted(alias_set.aliases), aliases)
782+
783+ def test_info_non_ascii(self):
784+ # info can contain non-ascii chars
785+ info = 'O idioma aceito \xe9 somente Portugu\xeas do Brasil'
786+ self._pckdict[b"info"] = info.encode("utf-8")
787+ self._import()
788+ self.assertEqual(self._mlist.info, info,
789+ "Encoding to UTF-8 is not handled")
790+ # test fallback to ascii with replace
791+ self._pckdict[b"info"] = info.encode("iso-8859-1")
792+ self._import()
793+ self.assertEqual(self._mlist.info,
794+ unicode(self._pckdict[b"info"], "ascii", "replace"),
795+ "We don't fall back to replacing non-ascii chars")
796+
797+ def test_preferred_language(self):
798+ self._pckdict[b"preferred_language"] = b'ja'
799+ english = getUtility(ILanguageManager).get('en')
800+ japanese = getUtility(ILanguageManager).get('ja')
801+ self.assertEqual(self._mlist.preferred_language, english)
802+ self._import()
803+ self.assertEqual(self._mlist.preferred_language, japanese)
804+
805+ def test_preferred_language_unknown_previous(self):
806+ # when the previous language is unknown, it should not fail
807+ self._mlist._preferred_language = 'xx' # non-existant
808+ self._import()
809+ english = getUtility(ILanguageManager).get('en')
810+ self.assertEqual(self._mlist.preferred_language, english)
811+
812+ def test_new_language(self):
813+ self._pckdict[b"preferred_language"] = b'xx_XX'
814+ try:
815+ self._import()
816+ except Import21Error, e:
817+ # check the message
818+ self.assertTrue("[language.xx_XX]" in str(e))
819+ else:
820+ self.fail("Import21Error was not raised")
821+
822
823
824
825 class TestArchiveImport(unittest.TestCase):
826@@ -83,7 +308,10 @@
827
828 def setUp(self):
829 self._mlist = create_list('blank@example.com')
830- self._mlist.archive_policy = 'INITIAL-TEST-VALUE'
831+ self._mlist.archive_policy = DummyEnum.val
832+
833+ def tearDown(self):
834+ remove_list(self._mlist)
835
836 def _do_test(self, pckdict, expected):
837 import_config_pck(self._mlist, pckdict)
838@@ -123,3 +351,516 @@
839 # For some reason, the old list was missing an `archive_private` key.
840 # For maximum safety, we treat this as private archiving.
841 self._do_test(dict(archive=True), ArchivePolicy.private)
842+
843+
844+
845
846+class TestFilterActionImport(unittest.TestCase):
847+ # The mlist.filter_action enum values have changed. In Mailman 2.1 the
848+ # order was 'Discard', 'Reject', 'Forward to List Owner', 'Preserve'.
849+
850+ layer = ConfigLayer
851+
852+ def setUp(self):
853+ self._mlist = create_list('blank@example.com')
854+ self._mlist.filter_action = DummyEnum.val
855+
856+ def tearDown(self):
857+ remove_list(self._mlist)
858+
859+ def _do_test(self, original, expected):
860+ import_config_pck(self._mlist, dict(filter_action=original))
861+ self.assertEqual(self._mlist.filter_action, expected)
862+
863+ def test_discard(self):
864+ self._do_test(0, FilterAction.discard)
865+
866+ def test_reject(self):
867+ self._do_test(1, FilterAction.reject)
868+
869+ def test_forward(self):
870+ self._do_test(2, FilterAction.forward)
871+
872+ def test_preserve(self):
873+ self._do_test(3, FilterAction.preserve)
874+
875+
876+
877
878+class TestMemberActionImport(unittest.TestCase):
879+ # The mlist.default_member_action and mlist.default_nonmember_action enum
880+ # values are different in Mailman 2.1, they have been merged into a
881+ # single enum in Mailman 3
882+ # For default_member_action, which used to be called
883+ # member_moderation_action, the values were:
884+ # 0==Hold, 1=Reject, 2==Discard
885+ # For default_nonmember_action, which used to be called
886+ # generic_nonmember_action, the values were:
887+ # 0==Accept, 1==Hold, 2==Reject, 3==Discard
888+
889+ layer = ConfigLayer
890+
891+ def setUp(self):
892+ self._mlist = create_list('blank@example.com')
893+ self._mlist.default_member_action = DummyEnum.val
894+ self._mlist.default_nonmember_action = DummyEnum.val
895+ self._pckdict = dict(
896+ member_moderation_action=DummyEnum.val,
897+ generic_nonmember_action=DummyEnum.val,
898+ )
899+
900+ def tearDown(self):
901+ remove_list(self._mlist)
902+
903+ def _do_test(self, expected):
904+ import_config_pck(self._mlist, self._pckdict)
905+ for key, value in expected.iteritems():
906+ self.assertEqual(getattr(self._mlist, key), value)
907+
908+ def test_member_hold(self):
909+ self._pckdict[b"member_moderation_action"] = 0
910+ self._do_test(dict(default_member_action=Action.hold))
911+
912+ def test_member_reject(self):
913+ self._pckdict[b"member_moderation_action"] = 1
914+ self._do_test(dict(default_member_action=Action.reject))
915+
916+ def test_member_discard(self):
917+ self._pckdict[b"member_moderation_action"] = 2
918+ self._do_test(dict(default_member_action=Action.discard))
919+
920+ def test_nonmember_accept(self):
921+ self._pckdict[b"generic_nonmember_action"] = 0
922+ self._do_test(dict(default_nonmember_action=Action.accept))
923+
924+ def test_nonmember_hold(self):
925+ self._pckdict[b"generic_nonmember_action"] = 1
926+ self._do_test(dict(default_nonmember_action=Action.hold))
927+
928+ def test_nonmember_reject(self):
929+ self._pckdict[b"generic_nonmember_action"] = 2
930+ self._do_test(dict(default_nonmember_action=Action.reject))
931+
932+ def test_nonmember_discard(self):
933+ self._pckdict[b"generic_nonmember_action"] = 3
934+ self._do_test(dict(default_nonmember_action=Action.discard))
935+
936+
937+
938
939+class TestConvertToURI(unittest.TestCase):
940+ # The following values were plain text, and are now URIs in Mailman 3:
941+ # - welcome_message_uri
942+ # - goodbye_message_uri
943+ # - header_uri
944+ # - footer_uri
945+ # - digest_header_uri
946+ # - digest_footer_uri
947+ #
948+ # The templates contain variables that must be replaced:
949+ # - %(real_name)s -> %(display_name)s
950+ # - %(real_name)s@%(host_name)s -> %(fqdn_listname)s
951+ # - %(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s -> %(listinfo_uri)s
952+
953+ layer = ConfigLayer
954+
955+ def setUp(self):
956+ self._mlist = create_list('blank@example.com')
957+ self._conf_mapping = dict(
958+ welcome_msg="welcome_message_uri",
959+ goodbye_msg="goodbye_message_uri",
960+ msg_header="header_uri",
961+ msg_footer="footer_uri",
962+ digest_header="digest_header_uri",
963+ digest_footer="digest_footer_uri",
964+ )
965+ self._pckdict = dict()
966+ #self._pckdict = {
967+ # "preferred_language": "XX", # templates are lang-specific
968+ #}
969+
970+ def tearDown(self):
971+ remove_list(self._mlist)
972+
973+ def test_text_to_uri(self):
974+ for oldvar, newvar in self._conf_mapping.iteritems():
975+ self._pckdict[str(oldvar)] = b"TEST VALUE"
976+ import_config_pck(self._mlist, self._pckdict)
977+ newattr = getattr(self._mlist, newvar)
978+ text = decorate(self._mlist, newattr)
979+ self.assertEqual(text, "TEST VALUE",
980+ "Old variable %s was not properly imported to %s"
981+ % (oldvar, newvar))
982+
983+ def test_substitutions(self):
984+ test_text = ("UNIT TESTING %(real_name)s mailing list\n"
985+ "%(real_name)s@%(host_name)s\n"
986+ "%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s")
987+ expected_text = ("UNIT TESTING $display_name mailing list\n"
988+ "$fqdn_listname\n"
989+ "$listinfo_uri")
990+ for oldvar, newvar in self._conf_mapping.iteritems():
991+ self._pckdict[str(oldvar)] = str(test_text)
992+ import_config_pck(self._mlist, self._pckdict)
993+ newattr = getattr(self._mlist, newvar)
994+ template_uri = expand(newattr, dict(
995+ listname=self._mlist.fqdn_listname,
996+ language=self._mlist.preferred_language.code,
997+ ))
998+ loader = getUtility(ITemplateLoader)
999+ text = loader.get(template_uri)
1000+ self.assertEqual(text, expected_text,
1001+ "Old variables were not converted for %s" % newvar)
1002+
1003+ def test_keep_default(self):
1004+ # If the value was not changed from MM2.1's default, don't import it
1005+ default_msg_footer = (
1006+ "_______________________________________________\n"
1007+ "%(real_name)s mailing list\n"
1008+ "%(real_name)s@%(host_name)s\n"
1009+ "%(web_page_url)slistinfo%(cgiext)s/%(_internal_name)s"
1010+ )
1011+ for oldvar in ("msg_footer", "digest_footer"):
1012+ newvar = self._conf_mapping[oldvar]
1013+ self._pckdict[str(oldvar)] = str(default_msg_footer)
1014+ old_value = getattr(self._mlist, newvar)
1015+ import_config_pck(self._mlist, self._pckdict)
1016+ new_value = getattr(self._mlist, newvar)
1017+ self.assertEqual(old_value, new_value,
1018+ "Default value was not preserved for %s" % newvar)
1019+
1020+ def test_keep_default_if_fqdn_changed(self):
1021+ # Use case: importing the old a@ex.com into b@ex.com
1022+ # We can't check if it changed from the default
1023+ # -> don't import, we may do more harm than good and it's easy to
1024+ # change if needed
1025+ test_value = b"TEST-VALUE"
1026+ for oldvar, newvar in self._conf_mapping.iteritems():
1027+ self._mlist.mail_host = "example.com"
1028+ self._pckdict[b"mail_host"] = b"test.example.com"
1029+ self._pckdict[str(oldvar)] = test_value
1030+ old_value = getattr(self._mlist, newvar)
1031+ import_config_pck(self._mlist, self._pckdict)
1032+ new_value = getattr(self._mlist, newvar)
1033+ self.assertEqual(old_value, new_value,
1034+ "Default value was not preserved for %s" % newvar)
1035+
1036+ def test_unicode(self):
1037+ # non-ascii templates
1038+ for oldvar in self._conf_mapping:
1039+ self._pckdict[str(oldvar)] = b"Ol\xe1!"
1040+ try:
1041+ import_config_pck(self._mlist, self._pckdict)
1042+ except UnicodeDecodeError, e:
1043+ print(format_exc())
1044+ self.fail(e)
1045+ for oldvar, newvar in self._conf_mapping.iteritems():
1046+ newattr = getattr(self._mlist, newvar)
1047+ text = decorate(self._mlist, newattr)
1048+ expected = u'Ol\ufffd!'
1049+ self.assertEqual(text, expected)
1050+
1051+ def test_unicode_in_default(self):
1052+ # What if the default template is already in UTF-8? (like if you import twice)
1053+ footer = b'\xe4\xb8\xad $listinfo_uri'
1054+ footer_path = os.path.join(config.VAR_DIR, "templates", "lists",
1055+ "blank@example.com", "en", "footer-generic.txt")
1056+ try:
1057+ os.makedirs(os.path.dirname(footer_path))
1058+ except OSError:
1059+ pass
1060+ with open(footer_path, "w") as footer_file:
1061+ footer_file.write(footer)
1062+ self._pckdict[b"msg_footer"] = b"NEW-VALUE"
1063+ import_config_pck(self._mlist, self._pckdict)
1064+ text = decorate(self._mlist, self._mlist.footer_uri)
1065+ self.assertEqual(text, 'NEW-VALUE')
1066+
1067+
1068
1069+class TestRosterImport(unittest.TestCase):
1070+
1071+ layer = ConfigLayer
1072+
1073+ def setUp(self):
1074+ self._mlist = create_list('blank@example.com')
1075+ self._pckdict = {
1076+ "members": {
1077+ "anne@example.com": 0,
1078+ "bob@example.com": b"bob@ExampLe.Com",
1079+ },
1080+ "digest_members": {
1081+ "cindy@example.com": 0,
1082+ "dave@example.com": b"dave@ExampLe.Com",
1083+ },
1084+ "passwords": {
1085+ "anne@example.com" : b"annepass",
1086+ "bob@example.com" : b"bobpass",
1087+ "cindy@example.com": b"cindypass",
1088+ "dave@example.com" : b"davepass",
1089+ },
1090+ "language": {
1091+ "anne@example.com" : b"fr",
1092+ "bob@example.com" : b"de",
1093+ "cindy@example.com": b"es",
1094+ "dave@example.com" : b"it",
1095+ },
1096+ "usernames": { # Usernames are unicode strings in the pickle
1097+ "anne@example.com" : "Anne",
1098+ "bob@example.com" : "Bob",
1099+ "cindy@example.com": "Cindy",
1100+ "dave@example.com" : "Dave",
1101+ },
1102+ "owner": [
1103+ "anne@example.com",
1104+ "emily@example.com",
1105+ ],
1106+ "moderator": [
1107+ "bob@example.com",
1108+ "fred@example.com",
1109+ ],
1110+ }
1111+ self._usermanager = getUtility(IUserManager)
1112+ language_manager = getUtility(ILanguageManager)
1113+ for code in self._pckdict["language"].values():
1114+ if code not in language_manager.codes:
1115+ language_manager.add(code, 'utf-8', code)
1116+
1117+ def tearDown(self):
1118+ remove_list(self._mlist)
1119+
1120+ def test_member(self):
1121+ import_config_pck(self._mlist, self._pckdict)
1122+ for name in ("anne", "bob", "cindy", "dave"):
1123+ addr = "%s@example.com" % name
1124+ self.assertTrue(
1125+ addr in [ a.email for a in self._mlist.members.addresses],
1126+ "Address %s was not imported" % addr)
1127+ self.assertTrue("anne@example.com" in [ a.email
1128+ for a in self._mlist.regular_members.addresses])
1129+ self.assertTrue("bob@example.com" in [ a.email
1130+ for a in self._mlist.regular_members.addresses])
1131+ self.assertTrue("cindy@example.com" in [ a.email
1132+ for a in self._mlist.digest_members.addresses])
1133+ self.assertTrue("dave@example.com" in [ a.email
1134+ for a in self._mlist.digest_members.addresses])
1135+
1136+ def test_original_email(self):
1137+ import_config_pck(self._mlist, self._pckdict)
1138+ bob = self._usermanager.get_address("bob@example.com")
1139+ self.assertEqual(bob.original_email, "bob@ExampLe.Com")
1140+ dave = self._usermanager.get_address("dave@example.com")
1141+ self.assertEqual(dave.original_email, "dave@ExampLe.Com")
1142+
1143+ def test_language(self):
1144+ import_config_pck(self._mlist, self._pckdict)
1145+ for name in ("anne", "bob", "cindy", "dave"):
1146+ addr = "%s@example.com" % name
1147+ member = self._mlist.members.get_member(addr)
1148+ self.assertTrue(member is not None,
1149+ "Address %s was not imported" % addr)
1150+ print(self._pckdict["language"])
1151+ print(member.preferred_language, member.preferred_language.code)
1152+ self.assertEqual(member.preferred_language.code,
1153+ self._pckdict["language"][addr])
1154+
1155+ def test_new_language(self):
1156+ self._pckdict[b"language"]["anne@example.com"] = b'xx_XX'
1157+ try:
1158+ import_config_pck(self._mlist, self._pckdict)
1159+ except Import21Error, e:
1160+ # check the message
1161+ self.assertTrue("[language.xx_XX]" in str(e))
1162+ else:
1163+ self.fail("Import21Error was not raised")
1164+
1165+ def test_username(self):
1166+ import_config_pck(self._mlist, self._pckdict)
1167+ for name in ("anne", "bob", "cindy", "dave"):
1168+ addr = "%s@example.com" % name
1169+ user = self._usermanager.get_user(addr)
1170+ address = self._usermanager.get_address(addr)
1171+ self.assertTrue(user is not None,
1172+ "User %s was not imported" % addr)
1173+ self.assertTrue(address is not None,
1174+ "Address %s was not imported" % addr)
1175+ display_name = self._pckdict["usernames"][addr]
1176+ self.assertEqual(user.display_name, display_name,
1177+ "The display name was not set for User %s" % addr)
1178+ self.assertEqual(address.display_name, display_name,
1179+ "The display name was not set for Address %s" % addr)
1180+
1181+ def test_owner(self):
1182+ import_config_pck(self._mlist, self._pckdict)
1183+ for name in ("anne", "emily"):
1184+ addr = "%s@example.com" % name
1185+ self.assertTrue(
1186+ addr in [ a.email for a in self._mlist.owners.addresses ],
1187+ "Address %s was not imported as owner" % addr)
1188+ self.assertFalse("emily@example.com" in
1189+ [ a.email for a in self._mlist.members.addresses ],
1190+ "Address emily@ was wrongly added to the members list")
1191+
1192+ def test_moderator(self):
1193+ import_config_pck(self._mlist, self._pckdict)
1194+ for name in ("bob", "fred"):
1195+ addr = "%s@example.com" % name
1196+ self.assertTrue(
1197+ addr in [ a.email for a in self._mlist.moderators.addresses ],
1198+ "Address %s was not imported as moderator" % addr)
1199+ self.assertFalse("fred@example.com" in
1200+ [ a.email for a in self._mlist.members.addresses ],
1201+ "Address fred@ was wrongly added to the members list")
1202+
1203+ def test_password(self):
1204+ #self.anne.password = config.password_context.encrypt('abc123')
1205+ import_config_pck(self._mlist, self._pckdict)
1206+ for name in ("anne", "bob", "cindy", "dave"):
1207+ addr = "%s@example.com" % name
1208+ user = self._usermanager.get_user(addr)
1209+ self.assertTrue(user is not None,
1210+ "Address %s was not imported" % addr)
1211+ self.assertEqual(user.password, b'{plaintext}%spass' % name,
1212+ "Password for %s was not imported" % addr)
1213+
1214+ def test_same_user(self):
1215+ # Adding the address of an existing User must not create another user
1216+ user = self._usermanager.create_user('anne@example.com', 'Anne')
1217+ user.register("bob@example.com") # secondary email
1218+ import_config_pck(self._mlist, self._pckdict)
1219+ member = self._mlist.members.get_member('bob@example.com')
1220+ self.assertEqual(member.user, user)
1221+
1222+ def test_owner_and_moderator_not_lowercase(self):
1223+ # In the v2.1 pickled dict, the owner and moderator lists are not
1224+ # necessarily lowercased already
1225+ self._pckdict["owner"] = [b"Anne@example.com"]
1226+ self._pckdict["moderator"] = [b"Anne@example.com"]
1227+ try:
1228+ import_config_pck(self._mlist, self._pckdict)
1229+ except AssertionError:
1230+ print(format_exc())
1231+ self.fail("The address was not lowercased")
1232+ self.assertTrue("anne@example.com" in
1233+ [ a.email for a in self._mlist.owners.addresses ])
1234+ self.assertTrue("anne@example.com" in
1235+ [ a.email for a in self._mlist.moderators.addresses])
1236+
1237+ def test_address_already_exists_but_no_user(self):
1238+ # An address already exists, but it is not linked to a user nor
1239+ # subscribed
1240+ anne_addr = Address("anne@example.com", "Anne")
1241+ Store.of(self._mlist).add(anne_addr)
1242+ try:
1243+ import_config_pck(self._mlist, self._pckdict)
1244+ except ExistingAddressError:
1245+ print(format_exc())
1246+ self.fail("existing address was not checked")
1247+ anne = self._usermanager.get_user("anne@example.com")
1248+ self.assertTrue(anne.controls("anne@example.com"))
1249+ self.assertTrue(anne_addr in self._mlist.regular_members.addresses)
1250+
1251+ def test_address_already_subscribed_but_no_user(self):
1252+ # An address is already subscribed, but it is not linked to a user
1253+ anne_addr = Address("anne@example.com", "Anne")
1254+ self._mlist.subscribe(anne_addr)
1255+ try:
1256+ import_config_pck(self._mlist, self._pckdict)
1257+ except ExistingAddressError:
1258+ print(format_exc())
1259+ self.fail("existing address was not checked")
1260+ anne = self._usermanager.get_user("anne@example.com")
1261+ self.assertTrue(anne.controls("anne@example.com"))
1262+
1263+
1264+
1265+
1266
1267+class TestPreferencesImport(unittest.TestCase):
1268+
1269+ layer = ConfigLayer
1270+
1271+ def setUp(self):
1272+ self._mlist = create_list('blank@example.com')
1273+ self._pckdict = dict(
1274+ members={ "anne@example.com": 0 },
1275+ user_options=dict(),
1276+ delivery_status=dict(),
1277+ )
1278+ self._usermanager = getUtility(IUserManager)
1279+
1280+ def tearDown(self):
1281+ remove_list(self._mlist)
1282+
1283+ def _do_test(self, oldvalue, expected):
1284+ self._pckdict["user_options"]["anne@example.com"] = oldvalue
1285+ import_config_pck(self._mlist, self._pckdict)
1286+ user = self._usermanager.get_user("anne@example.com")
1287+ self.assertTrue(user is not None, "User was not imported")
1288+ member = self._mlist.members.get_member("anne@example.com")
1289+ self.assertTrue(member is not None, "Address was not subscribed")
1290+ for exp_name, exp_val in expected.iteritems():
1291+ try:
1292+ currentval = getattr(member, exp_name)
1293+ except AttributeError:
1294+ # hide_address has no direct getter
1295+ currentval = getattr(member.preferences, exp_name)
1296+ self.assertEqual(currentval, exp_val,
1297+ "Preference %s was not imported" % exp_name)
1298+ # XXX: should I check that other params are still equal to
1299+ # mailman.core.constants.system_preferences ?
1300+
1301+ def test_acknowledge_posts(self):
1302+ # AcknowledgePosts
1303+ self._do_test(4, dict(acknowledge_posts=True))
1304+
1305+ def test_hide_address(self):
1306+ # ConcealSubscription
1307+ self._do_test(16, dict(hide_address=True))
1308+
1309+ def test_receive_own_postings(self):
1310+ # DontReceiveOwnPosts
1311+ self._do_test(2, dict(receive_own_postings=False))
1312+
1313+ def test_receive_list_copy(self):
1314+ # DontReceiveDuplicates
1315+ self._do_test(256, dict(receive_list_copy=False))
1316+
1317+ def test_digest_plain(self):
1318+ # Digests & DisableMime
1319+ self._pckdict["digest_members"] = self._pckdict["members"].copy()
1320+ self._pckdict["members"] = dict()
1321+ self._do_test(8, dict(delivery_mode=DeliveryMode.plaintext_digests))
1322+
1323+ def test_digest_mime(self):
1324+ # Digests & not DisableMime
1325+ self._pckdict["digest_members"] = self._pckdict["members"].copy()
1326+ self._pckdict["members"] = dict()
1327+ self._do_test(0, dict(delivery_mode=DeliveryMode.mime_digests))
1328+
1329+ def test_delivery_status(self):
1330+ # look for the pckdict["delivery_status"] key which will look like
1331+ # (status, time) where status is among the following:
1332+ # ENABLED = 0 # enabled
1333+ # UNKNOWN = 1 # legacy disabled
1334+ # BYUSER = 2 # disabled by user choice
1335+ # BYADMIN = 3 # disabled by admin choice
1336+ # BYBOUNCE = 4 # disabled by bounces
1337+ for oldval, expected in enumerate((DeliveryStatus.enabled,
1338+ DeliveryStatus.unknown, DeliveryStatus.by_user,
1339+ DeliveryStatus.by_moderator, DeliveryStatus.by_bounces)):
1340+ self._pckdict["delivery_status"]["anne@example.com"] = (oldval, 0)
1341+ import_config_pck(self._mlist, self._pckdict)
1342+ member = self._mlist.members.get_member("anne@example.com")
1343+ self.assertTrue(member is not None, "Address was not subscribed")
1344+ self.assertEqual(member.delivery_status, expected)
1345+ member.unsubscribe()
1346+
1347+ def test_moderate(self):
1348+ # Option flag Moderate is translated to
1349+ # member.moderation_action = Action.hold
1350+ self._do_test(128, dict(moderation_action=Action.hold))
1351+
1352+ def test_multiple_options(self):
1353+ # DontReceiveDuplicates & DisableMime & SuppressPasswordReminder
1354+ self._pckdict[b"digest_members"] = self._pckdict[b"members"].copy()
1355+ self._pckdict[b"members"] = dict()
1356+ self._do_test(296, dict(
1357+ receive_list_copy=False,
1358+ delivery_mode=DeliveryMode.plaintext_digests,
1359+ ))