Merge lp:~raj-abhilash1/mailman/sqlalchemy into lp:mailman

Proposed by Abhilash Raj
Status: Merged
Merged at revision: 7253
Proposed branch: lp:~raj-abhilash1/mailman/sqlalchemy
Merge into: lp:mailman
Diff against target: 6388 lines (+1161/-3567)
66 files modified
MANIFEST.in (+2/-0)
setup.py (+2/-1)
src/mailman/app/subscriptions.py (+7/-9)
src/mailman/bin/tests/test_master.py (+1/-1)
src/mailman/commands/docs/conf.rst (+3/-1)
src/mailman/config/config.py (+5/-8)
src/mailman/config/configure.zcml (+0/-20)
src/mailman/config/schema.cfg (+13/-4)
src/mailman/core/logging.py (+4/-0)
src/mailman/database/alembic/__init__.py (+32/-0)
src/mailman/database/alembic/env.py (+80/-0)
src/mailman/database/alembic/script.py.mako (+22/-0)
src/mailman/database/alembic/versions/51b7f92bd06c_initial.py (+34/-0)
src/mailman/database/base.py (+14/-126)
src/mailman/database/docs/migration.rst (+0/-207)
src/mailman/database/factory.py (+72/-25)
src/mailman/database/model.py (+30/-40)
src/mailman/database/postgresql.py (+6/-59)
src/mailman/database/schema/helpers.py (+0/-43)
src/mailman/database/schema/mm_00000000000000_base.py (+0/-35)
src/mailman/database/schema/mm_20120407000000.py (+0/-212)
src/mailman/database/schema/mm_20121015000000.py (+0/-95)
src/mailman/database/schema/mm_20130406000000.py (+0/-65)
src/mailman/database/schema/postgres.sql (+0/-349)
src/mailman/database/schema/sqlite.sql (+0/-327)
src/mailman/database/schema/sqlite_20120407000000_01.sql (+0/-280)
src/mailman/database/schema/sqlite_20121015000000_01.sql (+0/-230)
src/mailman/database/schema/sqlite_20130406000000_01.sql (+0/-46)
src/mailman/database/sqlite.py (+5/-41)
src/mailman/database/tests/data/migration_postgres_1.sql (+0/-133)
src/mailman/database/tests/data/migration_sqlite_1.sql (+0/-133)
src/mailman/database/tests/test_factory.py (+162/-0)
src/mailman/database/tests/test_migrations.py (+0/-506)
src/mailman/database/types.py (+57/-30)
src/mailman/interfaces/database.py (+1/-7)
src/mailman/interfaces/messages.py (+1/-1)
src/mailman/model/address.py (+17/-12)
src/mailman/model/autorespond.py (+25/-23)
src/mailman/model/bans.py (+16/-13)
src/mailman/model/bounce.py (+13/-10)
src/mailman/model/digests.py (+13/-10)
src/mailman/model/docs/messagestore.rst (+3/-2)
src/mailman/model/domain.py (+15/-14)
src/mailman/model/language.py (+7/-5)
src/mailman/model/listmanager.py (+12/-13)
src/mailman/model/mailinglist.py (+230/-216)
src/mailman/model/member.py (+19/-17)
src/mailman/model/message.py (+7/-5)
src/mailman/model/messagestore.py (+15/-14)
src/mailman/model/mime.py (+11/-8)
src/mailman/model/pending.py (+34/-27)
src/mailman/model/preferences.py (+11/-9)
src/mailman/model/requests.py (+26/-17)
src/mailman/model/roster.py (+19/-26)
src/mailman/model/tests/test_listmanager.py (+12/-4)
src/mailman/model/tests/test_requests.py (+2/-2)
src/mailman/model/uid.py (+9/-5)
src/mailman/model/user.py (+32/-17)
src/mailman/model/usermanager.py (+9/-9)
src/mailman/model/version.py (+0/-44)
src/mailman/rest/validator.py (+1/-1)
src/mailman/styles/base.py (+2/-6)
src/mailman/testing/layers.py (+10/-1)
src/mailman/testing/testing.cfg (+1/-1)
src/mailman/utilities/importer.py (+23/-1)
src/mailman/utilities/modules.py (+14/-1)
To merge this branch: bzr merge lp:~raj-abhilash1/mailman/sqlalchemy
Reviewer Review Type Date Requested Status
Mailman Coders Pending
Review via email: mp+235329@code.launchpad.net

Description of the change

Replace storm with sqlalchemy.

To post a comment you must log in.
lp:~raj-abhilash1/mailman/sqlalchemy updated
7266. By Abhilash Raj

merge test fix from Barry

7267. By Abhilash Raj

merge updates from to sqlalchemy branch

7268. By Abhilash Raj

merge updated with some removal from barry

7269. By Abhilash Raj

added support for migrations via alembic

7270. By Abhilash Raj

added license block for the new file

7271. By Abhilash Raj

no need to stamp the testing db

7272. By Abhilash Raj

add new command `mailman migrate` to migrate the new schema on the old database

7273. By Abhilash Raj

add autogenerate switch that generates to create migration scripts automatically

7274. By Abhilash Raj

* fixed a bug where alemnic could not find its migrations directory
* add a new method in base database to stamp with latest alembic version

7275. By Abhilash Raj

Add support for postgresql

* revert changes in message_id_has encoding by barry
* Change message_id_hash column to LargeBinary
  (from previously mistaken one i.e.unicode)
* add missing import in database/types.py
* fix a bug in database/Model.py, transaction has no
  method abort(), instead it is rollback()

7276. By Abhilash Raj

Merge alembic setup from abompard

7277. By Abhilash Raj

Merge barry\'s branch with test fixes and clean code

7278. By Abhilash Raj

add central alembic config

7279. By Abhilash Raj

fix database reset error due to foreign key constraint between user and address tables

7280. By Abhilash Raj

merge branch from abompard

7281. By Abhilash Raj

* remove migrate command
* remove alembic.cfg, move contents to schema.cfg
* fix import errors in src/mailman/model/language.py
* add indexes
* change the previously wrong written tablename autoresponserecord
* change alembic_cfg to use schema.cfg instead of alembic.cfg

7282. By Abhilash Raj

changes from abompard to fix the unit tests

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'MANIFEST.in'
--- MANIFEST.in 2011-06-11 18:16:04 +0000
+++ MANIFEST.in 2014-10-11 02:14:38 +0000
@@ -13,3 +13,5 @@
13prune parts13prune parts
14include MANIFEST.in14include MANIFEST.in
15include src/mailman/testing/config.pck15include src/mailman/testing/config.pck
16include src/mailman/database/alembic/script.py.mako
17include src/mailman/database/alembic/versions/*.py
1618
=== modified file 'setup.py'
--- setup.py 2014-04-15 16:06:01 +0000
+++ setup.py 2014-10-11 02:14:38 +0000
@@ -93,6 +93,7 @@
93 'console_scripts' : list(scripts),93 'console_scripts' : list(scripts),
94 },94 },
95 install_requires = [95 install_requires = [
96 'alembic',
96 'enum34',97 'enum34',
97 'flufl.bounce',98 'flufl.bounce',
98 'flufl.i18n',99 'flufl.i18n',
@@ -104,7 +105,7 @@
104 'nose2',105 'nose2',
105 'passlib',106 'passlib',
106 'restish',107 'restish',
107 'storm',108 'sqlalchemy',
108 'zope.component',109 'zope.component',
109 'zope.configuration',110 'zope.configuration',
110 'zope.event',111 'zope.event',
111112
=== modified file 'src/mailman/app/subscriptions.py'
--- src/mailman/app/subscriptions.py 2014-04-15 14:03:39 +0000
+++ src/mailman/app/subscriptions.py 2014-10-11 02:14:38 +0000
@@ -28,7 +28,7 @@
2828
29from operator import attrgetter29from operator import attrgetter
30from passlib.utils import generate_password as generate30from passlib.utils import generate_password as generate
31from storm.expr import And, Or31from sqlalchemy import and_, or_
32from uuid import UUID32from uuid import UUID
33from zope.component import getUtility33from zope.component import getUtility
34from zope.interface import implementer34from zope.interface import implementer
@@ -88,9 +88,7 @@
88 @dbconnection88 @dbconnection
89 def get_member(self, store, member_id):89 def get_member(self, store, member_id):
90 """See `ISubscriptionService`."""90 """See `ISubscriptionService`."""
91 members = store.find(91 members = store.query(Member).filter(Member._member_id == member_id)
92 Member,
93 Member._member_id == member_id)
94 if members.count() == 0:92 if members.count() == 0:
95 return None93 return None
96 else:94 else:
@@ -117,8 +115,8 @@
117 # This probably could be made more efficient.115 # This probably could be made more efficient.
118 if address is None or user is None:116 if address is None or user is None:
119 return []117 return []
120 query.append(Or(Member.address_id == address.id,118 query.append(or_(Member.address_id == address.id,
121 Member.user_id == user.id))119 Member.user_id == user.id))
122 else:120 else:
123 # subscriber is a user id.121 # subscriber is a user id.
124 user = user_manager.get_user_by_id(subscriber)122 user = user_manager.get_user_by_id(subscriber)
@@ -126,15 +124,15 @@
126 if address.id is not None)124 if address.id is not None)
127 if len(address_ids) == 0 or user is None:125 if len(address_ids) == 0 or user is None:
128 return []126 return []
129 query.append(Or(Member.user_id == user.id,127 query.append(or_(Member.user_id == user.id,
130 Member.address_id.is_in(address_ids)))128 Member.address_id.in_(address_ids)))
131 # Calculate the rest of the query expression, which will get And'd129 # Calculate the rest of the query expression, which will get And'd
132 # with the Or clause above (if there is one).130 # with the Or clause above (if there is one).
133 if list_id is not None:131 if list_id is not None:
134 query.append(Member.list_id == list_id)132 query.append(Member.list_id == list_id)
135 if role is not None:133 if role is not None:
136 query.append(Member.role == role)134 query.append(Member.role == role)
137 results = store.find(Member, And(*query))135 results = store.query(Member).filter(and_(*query))
138 return sorted(results, key=_membership_sort_key)136 return sorted(results, key=_membership_sort_key)
139137
140 def __iter__(self):138 def __iter__(self):
141139
=== modified file 'src/mailman/bin/tests/test_master.py'
--- src/mailman/bin/tests/test_master.py 2014-04-28 15:23:35 +0000
+++ src/mailman/bin/tests/test_master.py 2014-10-11 02:14:38 +0000
@@ -55,7 +55,7 @@
55 lock = master.acquire_lock_1(False, self.lock_file)55 lock = master.acquire_lock_1(False, self.lock_file)
56 is_locked = lock.is_locked56 is_locked = lock.is_locked
57 lock.unlock()57 lock.unlock()
58 self.failUnless(is_locked)58 self.assertTrue(is_locked)
5959
60 def test_master_state(self):60 def test_master_state(self):
61 my_lock = Lock(self.lock_file)61 my_lock = Lock(self.lock_file)
6262
=== modified file 'src/mailman/commands/docs/conf.rst'
--- src/mailman/commands/docs/conf.rst 2013-09-01 15:08:46 +0000
+++ src/mailman/commands/docs/conf.rst 2014-10-11 02:14:38 +0000
@@ -22,7 +22,7 @@
22command without any options.22command without any options.
2323
24 >>> command.process(FakeArgs)24 >>> command.process(FakeArgs)
25 [logging.archiver] path: mailman.log25 [alembic] script_location: mailman.database:alembic
26 ...26 ...
27 [passwords] password_length: 827 [passwords] password_length: 8
28 ...28 ...
@@ -43,12 +43,14 @@
43 >>> FakeArgs.section = None43 >>> FakeArgs.section = None
44 >>> FakeArgs.key = 'path'44 >>> FakeArgs.key = 'path'
45 >>> command.process(FakeArgs)45 >>> command.process(FakeArgs)
46 [logging.dbmigration] path: mailman.log
46 [logging.archiver] path: mailman.log47 [logging.archiver] path: mailman.log
47 [logging.locks] path: mailman.log48 [logging.locks] path: mailman.log
48 [logging.mischief] path: mailman.log49 [logging.mischief] path: mailman.log
49 [logging.config] path: mailman.log50 [logging.config] path: mailman.log
50 [logging.error] path: mailman.log51 [logging.error] path: mailman.log
51 [logging.smtp] path: smtp.log52 [logging.smtp] path: smtp.log
53 [logging.database] path: mailman.log
52 [logging.http] path: mailman.log54 [logging.http] path: mailman.log
53 [logging.root] path: mailman.log55 [logging.root] path: mailman.log
54 [logging.fromusenet] path: mailman.log56 [logging.fromusenet] path: mailman.log
5557
=== modified file 'src/mailman/config/config.py'
--- src/mailman/config/config.py 2014-03-02 22:59:30 +0000
+++ src/mailman/config/config.py 2014-10-11 02:14:38 +0000
@@ -33,7 +33,7 @@
33from ConfigParser import SafeConfigParser33from ConfigParser import SafeConfigParser
34from flufl.lock import Lock34from flufl.lock import Lock
35from lazr.config import ConfigSchema, as_boolean35from lazr.config import ConfigSchema, as_boolean
36from pkg_resources import resource_filename, resource_stream, resource_string36from pkg_resources import resource_stream, resource_string
37from string import Template37from string import Template
38from zope.component import getUtility38from zope.component import getUtility
39from zope.event import notify39from zope.event import notify
@@ -46,7 +46,7 @@
46 ConfigurationUpdatedEvent, IConfiguration, MissingConfigurationFileError)46 ConfigurationUpdatedEvent, IConfiguration, MissingConfigurationFileError)
47from mailman.interfaces.languages import ILanguageManager47from mailman.interfaces.languages import ILanguageManager
48from mailman.utilities.filesystem import makedirs48from mailman.utilities.filesystem import makedirs
49from mailman.utilities.modules import call_name49from mailman.utilities.modules import call_name, expand_path
5050
5151
52SPACE = ' '52SPACE = ' '
@@ -87,6 +87,7 @@
87 self.pipelines = {}87 self.pipelines = {}
88 self.commands = {}88 self.commands = {}
89 self.password_context = None89 self.password_context = None
90 self.initialized = False
9091
91 def _clear(self):92 def _clear(self):
92 """Clear the cached configuration variables."""93 """Clear the cached configuration variables."""
@@ -136,6 +137,7 @@
136 # Expand and set up all directories.137 # Expand and set up all directories.
137 self._expand_paths()138 self._expand_paths()
138 self.ensure_directories_exist()139 self.ensure_directories_exist()
140 self.initialized = True
139 notify(ConfigurationUpdatedEvent(self))141 notify(ConfigurationUpdatedEvent(self))
140142
141 def _expand_paths(self):143 def _expand_paths(self):
@@ -304,12 +306,7 @@
304 :return: A `ConfigParser` instance.306 :return: A `ConfigParser` instance.
305 """307 """
306 # Is the context coming from a file system or Python path?308 # Is the context coming from a file system or Python path?
307 if path.startswith('python:'):309 cfg_path = expand_path(path)
308 resource_path = path[7:]
309 package, dot, resource = resource_path.rpartition('.')
310 cfg_path = resource_filename(package, resource + '.cfg')
311 else:
312 cfg_path = path
313 parser = SafeConfigParser()310 parser = SafeConfigParser()
314 files = parser.read(cfg_path)311 files = parser.read(cfg_path)
315 if files != [cfg_path]:312 if files != [cfg_path]:
316313
=== modified file 'src/mailman/config/configure.zcml'
--- src/mailman/config/configure.zcml 2013-11-26 02:26:15 +0000
+++ src/mailman/config/configure.zcml 2014-10-11 02:14:38 +0000
@@ -40,20 +40,6 @@
40 factory="mailman.model.requests.ListRequests"40 factory="mailman.model.requests.ListRequests"
41 />41 />
4242
43 <adapter
44 for="mailman.interfaces.database.IDatabase"
45 provides="mailman.interfaces.database.ITemporaryDatabase"
46 factory="mailman.database.sqlite.make_temporary"
47 name="sqlite"
48 />
49
50 <adapter
51 for="mailman.interfaces.database.IDatabase"
52 provides="mailman.interfaces.database.ITemporaryDatabase"
53 factory="mailman.database.postgresql.make_temporary"
54 name="postgres"
55 />
56
57 <utility43 <utility
58 provides="mailman.interfaces.bounce.IBounceProcessor"44 provides="mailman.interfaces.bounce.IBounceProcessor"
59 factory="mailman.model.bounce.BounceProcessor"45 factory="mailman.model.bounce.BounceProcessor"
@@ -72,12 +58,6 @@
72 />58 />
7359
74 <utility60 <utility
75 provides="mailman.interfaces.database.IDatabaseFactory"
76 factory="mailman.database.factory.DatabaseTemporaryFactory"
77 name="temporary"
78 />
79
80 <utility
81 provides="mailman.interfaces.domain.IDomainManager"61 provides="mailman.interfaces.domain.IDomainManager"
82 factory="mailman.model.domain.DomainManager"62 factory="mailman.model.domain.DomainManager"
83 />63 />
8464
=== modified file 'src/mailman/config/schema.cfg'
--- src/mailman/config/schema.cfg 2014-01-01 14:59:42 +0000
+++ src/mailman/config/schema.cfg 2014-10-11 02:14:38 +0000
@@ -204,9 +204,6 @@
204url: sqlite:///$DATA_DIR/mailman.db204url: sqlite:///$DATA_DIR/mailman.db
205debug: no205debug: no
206206
207# The module path to the migrations modules.
208migrations_path: mailman.database.schema
209
210[logging.template]207[logging.template]
211# This defines various log settings. The options available are:208# This defines various log settings. The options available are:
212#209#
@@ -240,6 +237,8 @@
240# - smtp-failure -- Unsuccessful SMTP activity237# - smtp-failure -- Unsuccessful SMTP activity
241# - subscribe -- Information about leaves/joins238# - subscribe -- Information about leaves/joins
242# - vette -- Message vetting information239# - vette -- Message vetting information
240# - database -- Database activity
241# - dbmigration -- Database migrations
243format: %(asctime)s (%(process)d) %(message)s242format: %(asctime)s (%(process)d) %(message)s
244datefmt: %b %d %H:%M:%S %Y243datefmt: %b %d %H:%M:%S %Y
245propagate: no244propagate: no
@@ -304,6 +303,12 @@
304303
305[logging.vette]304[logging.vette]
306305
306[logging.database]
307level: warn
308
309[logging.dbmigration]
310level: warn
311
307312
308[webservice]313[webservice]
309# The hostname at which admin web service resources are exposed.314# The hostname at which admin web service resources are exposed.
@@ -532,7 +537,7 @@
532# following values.537# following values.
533538
534# The class implementing the IArchiver interface.539# The class implementing the IArchiver interface.
535class: 540class:
536541
537# Set this to 'yes' to enable the archiver.542# Set this to 'yes' to enable the archiver.
538enable: no543enable: no
@@ -640,3 +645,7 @@
640 CC X-Original-CC645 CC X-Original-CC
641 Content-Transfer-Encoding X-Original-Content-Transfer-Encoding646 Content-Transfer-Encoding X-Original-Content-Transfer-Encoding
642 MIME-Version X-MIME-Version647 MIME-Version X-MIME-Version
648
649[alembic]
650# path to migration scripts
651script_location = mailman.database:alembic
643652
=== modified file 'src/mailman/core/logging.py'
--- src/mailman/core/logging.py 2014-04-28 15:23:35 +0000
+++ src/mailman/core/logging.py 2014-10-11 02:14:38 +0000
@@ -126,6 +126,10 @@
126 continue126 continue
127 if sub_name == 'locks':127 if sub_name == 'locks':
128 log = logging.getLogger('flufl.lock')128 log = logging.getLogger('flufl.lock')
129 elif sub_name == 'database':
130 log = logging.getLogger('sqlalchemy')
131 elif sub_name == 'dbmigration':
132 log = logging.getLogger('alembic')
129 else:133 else:
130 logger_name = 'mailman.' + sub_name134 logger_name = 'mailman.' + sub_name
131 log = logging.getLogger(logger_name)135 log = logging.getLogger(logger_name)
132136
=== added directory 'src/mailman/database/alembic'
=== added file 'src/mailman/database/alembic/__init__.py'
--- src/mailman/database/alembic/__init__.py 1970-01-01 00:00:00 +0000
+++ src/mailman/database/alembic/__init__.py 2014-10-11 02:14:38 +0000
@@ -0,0 +1,32 @@
1# Copyright (C) 2014 by the Free Software Foundation, Inc.
2#
3# This file is part of GNU Mailman.
4#
5# GNU Mailman is free software: you can redistribute it and/or modify it under
6# the terms of the GNU General Public License as published by the Free
7# Software Foundation, either version 3 of the License, or (at your option)
8# any later version.
9#
10# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13# more details.
14#
15# You should have received a copy of the GNU General Public License along with
16# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
17
18"Alembic config init."
19
20from __future__ import absolute_import, print_function, unicode_literals
21
22__metaclass__ = type
23__all__ = [
24 'alembic_cfg'
25]
26
27
28from alembic.config import Config
29from mailman.utilities.modules import expand_path
30
31
32alembic_cfg=Config(expand_path("python:mailman.config.schema"))
033
=== added file 'src/mailman/database/alembic/env.py'
--- src/mailman/database/alembic/env.py 1970-01-01 00:00:00 +0000
+++ src/mailman/database/alembic/env.py 2014-10-11 02:14:38 +0000
@@ -0,0 +1,80 @@
1# Copyright (C) 2014 by the Free Software Foundation, Inc.
2#
3# This file is part of GNU Mailman.
4#
5# GNU Mailman is free software: you can redistribute it and/or modify it under
6# the terms of the GNU General Public License as published by the Free
7# Software Foundation, either version 3 of the License, or (at your option)
8# any later version.
9#
10# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13# more details.
14#
15# You should have received a copy of the GNU General Public License along with
16# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
17
18"""Alembic migration environment."""
19
20from __future__ import absolute_import, print_function, unicode_literals
21
22__metaclass__ = type
23__all__ = [
24 'run_migrations_offline',
25 'run_migrations_online',
26 ]
27
28
29from alembic import context
30from contextlib import closing
31from sqlalchemy import create_engine
32
33from mailman.core import initialize
34from mailman.config import config
35from mailman.database.alembic import alembic_cfg
36from mailman.database.model import Model
37from mailman.utilities.string import expand
38
39if not config.initialized:
40 initialize.initialize_1(context.config.config_file_name)
41
42
43
044
45def run_migrations_offline():
46 """Run migrations in 'offline' mode.
47
48 This configures the context with just a URL and not an Engine,
49 though an Engine is acceptable here as well. By skipping the Engine
50 creation we don't even need a DBAPI to be available.
51
52 Calls to context.execute() here emit the given string to the script
53 output.
54 """
55 url = expand(config.database.url, config.paths)
56 context.configure(url=url, target_metadata=Model.metadata)
57 with context.begin_transaction():
58 context.run_migrations()
59
60
61def run_migrations_online():
62 """Run migrations in 'online' mode.
63
64 In this scenario we need to create an Engine and associate a
65 connection with the context.
66 """
67 url = expand(config.database.url, config.paths)
68 engine = create_engine(url)
69
70 connection = engine.connect()
71 with closing(connection):
72 context.configure(
73 connection=connection, target_metadata=Model.metadata)
74 with context.begin_transaction():
75 context.run_migrations()
76
77
78if context.is_offline_mode():
79 run_migrations_offline()
80else:
81 run_migrations_online()
182
=== added file 'src/mailman/database/alembic/script.py.mako'
--- src/mailman/database/alembic/script.py.mako 1970-01-01 00:00:00 +0000
+++ src/mailman/database/alembic/script.py.mako 2014-10-11 02:14:38 +0000
@@ -0,0 +1,22 @@
1"""${message}
2
3Revision ID: ${up_revision}
4Revises: ${down_revision}
5Create Date: ${create_date}
6
7"""
8
9# revision identifiers, used by Alembic.
10revision = ${repr(up_revision)}
11down_revision = ${repr(down_revision)}
12
13from alembic import op
14import sqlalchemy as sa
15${imports if imports else ""}
16
17def upgrade():
18 ${upgrades if upgrades else "pass"}
19
20
21def downgrade():
22 ${downgrades if downgrades else "pass"}
023
=== added directory 'src/mailman/database/alembic/versions'
=== added file 'src/mailman/database/alembic/versions/51b7f92bd06c_initial.py'
--- src/mailman/database/alembic/versions/51b7f92bd06c_initial.py 1970-01-01 00:00:00 +0000
+++ src/mailman/database/alembic/versions/51b7f92bd06c_initial.py 2014-10-11 02:14:38 +0000
@@ -0,0 +1,34 @@
1"""initial
2
3Revision ID: 51b7f92bd06c
4Revises: None
5Create Date: 2014-10-10 09:53:35.624472
6
7"""
8
9# revision identifiers, used by Alembic.
10revision = '51b7f92bd06c'
11down_revision = None
12
13from alembic import op
14import sqlalchemy as sa
15
16
17def upgrade():
18 ### commands auto generated by Alembic - please adjust! ###
19 op.drop_table('version')
20 if op.get_bind().dialect.name != "sqlite":
21 # SQLite does not support dropping columns
22 op.drop_column('mailinglist', 'acceptable_aliases_id')
23 op.create_index(op.f('ix_user__user_id'), 'user', ['_user_id'], unique=False)
24 op.drop_index('ix_user_user_id', table_name='user')
25 ### end Alembic commands ###
26
27
28def downgrade():
29 ### commands auto generated by Alembic - please adjust! ###
30 op.create_table('version')
31 op.create_index('ix_user_user_id', 'user', ['_user_id'], unique=False)
32 op.drop_index(op.f('ix_user__user_id'), table_name='user')
33 op.add_column('mailinglist', sa.Column('acceptable_aliases_id', sa.INTEGER(), nullable=True))
34 ### end Alembic commands ###
035
=== modified file 'src/mailman/database/base.py'
--- src/mailman/database/base.py 2014-01-01 14:59:42 +0000
+++ src/mailman/database/base.py 2014-10-11 02:14:38 +0000
@@ -19,49 +19,40 @@
1919
20__metaclass__ = type20__metaclass__ = type
21__all__ = [21__all__ = [
22 'StormBaseDatabase',22 'SABaseDatabase',
23 ]23 ]
2424
2525
26import os
27import sys
28import logging26import logging
2927
30from lazr.config import as_boolean28from alembic import command
31from pkg_resources import resource_listdir, resource_string29from sqlalchemy import create_engine
32from storm.cache import GenerationalCache30from sqlalchemy.orm import sessionmaker
33from storm.locals import create_database, Store
34from zope.interface import implementer31from zope.interface import implementer
3532
36from mailman.config import config33from mailman.config import config
37from mailman.interfaces.database import IDatabase34from mailman.interfaces.database import IDatabase
38from mailman.model.version import Version
39from mailman.utilities.string import expand35from mailman.utilities.string import expand
4036
37
41log = logging.getLogger('mailman.config')38log = logging.getLogger('mailman.config')
42
43NL = '\n'39NL = '\n'
4440
4541
4642
4743
48@implementer(IDatabase)44@implementer(IDatabase)
49class StormBaseDatabase:45class SABaseDatabase:
50 """The database base class for use with the Storm ORM.46 """The database base class for use with SQLAlchemy.
5147
52 Use this as a base class for your DB-specific derived classes.48 Use this as a base class for your DB-Specific derived classes.
53 """49 """
54
55 # Tag used to distinguish the database being used. Override this in base
56 # classes.
57 TAG = ''
58
59 def __init__(self):50 def __init__(self):
60 self.url = None51 self.url = None
61 self.store = None52 self.store = None
6253
63 def begin(self):54 def begin(self):
64 """See `IDatabase`."""55 """See `IDatabase`."""
65 # Storm takes care of this for us.56 # SQLAlchemy does this for us.
66 pass57 pass
6758
68 def commit(self):59 def commit(self):
@@ -72,16 +63,6 @@
72 """See `IDatabase`."""63 """See `IDatabase`."""
73 self.store.rollback()64 self.store.rollback()
7465
75 def _database_exists(self):
76 """Return True if the database exists and is initialized.
77
78 Return False when Mailman needs to create and initialize the
79 underlying database schema.
80
81 Base classes *must* override this.
82 """
83 raise NotImplementedError
84
85 def _pre_reset(self, store):66 def _pre_reset(self, store):
86 """Clean up method for testing.67 """Clean up method for testing.
8768
@@ -113,6 +94,7 @@
113 """See `IDatabase`."""94 """See `IDatabase`."""
114 # Calculate the engine url.95 # Calculate the engine url.
115 url = expand(config.database.url, config.paths)96 url = expand(config.database.url, config.paths)
97 self._prepare(url)
116 log.debug('Database url: %s', url)98 log.debug('Database url: %s', url)
117 # XXX By design of SQLite, database file creation does not honor99 # XXX By design of SQLite, database file creation does not honor
118 # umask. See their ticket #1193:100 # umask. See their ticket #1193:
@@ -129,101 +111,7 @@
129 # engines, and yes, we could have chmod'd the file after the fact, but111 # engines, and yes, we could have chmod'd the file after the fact, but
130 # half dozen and all...112 # half dozen and all...
131 self.url = url113 self.url = url
132 self._prepare(url)114 self.engine = create_engine(url)
133 database = create_database(url)115 session = sessionmaker(bind=self.engine)
134 store = Store(database, GenerationalCache())116 self.store = session()
135 database.DEBUG = (as_boolean(config.database.debug)117 self.store.commit()
136 if debug is None else debug)
137 self.store = store
138 store.commit()
139
140 def load_migrations(self, until=None):
141 """Load schema migrations.
142
143 :param until: Load only the migrations up to the specified timestamp.
144 With default value of None, load all migrations.
145 :type until: string
146 """
147 migrations_path = config.database.migrations_path
148 if '.' in migrations_path:
149 parent, dot, child = migrations_path.rpartition('.')
150 else:
151 parent = migrations_path
152 child = ''
153 # If the database does not yet exist, load the base schema.
154 filenames = sorted(resource_listdir(parent, child))
155 # Find out which schema migrations have already been loaded.
156 if self._database_exists(self.store):
157 versions = set(version.version for version in
158 self.store.find(Version, component='schema'))
159 else:
160 versions = set()
161 for filename in filenames:
162 module_fn, extension = os.path.splitext(filename)
163 if extension != '.py':
164 continue
165 parts = module_fn.split('_')
166 if len(parts) < 2:
167 continue
168 version = parts[1].strip()
169 if len(version) == 0:
170 # Not a schema migration file.
171 continue
172 if version in versions:
173 log.debug('already migrated to %s', version)
174 continue
175 if until is not None and version > until:
176 # We're done.
177 break
178 module_path = migrations_path + '.' + module_fn
179 __import__(module_path)
180 upgrade = getattr(sys.modules[module_path], 'upgrade', None)
181 if upgrade is None:
182 continue
183 log.debug('migrating db to %s: %s', version, module_path)
184 upgrade(self, self.store, version, module_path)
185 self.commit()
186
187 def load_sql(self, store, sql):
188 """Load the given SQL into the store.
189
190 :param store: The Storm store to load the schema into.
191 :type store: storm.locals.Store`
192 :param sql: The possibly multi-line SQL to load.
193 :type sql: string
194 """
195 # Discard all blank and comment lines.
196 lines = (line for line in sql.splitlines()
197 if line.strip() != '' and line.strip()[:2] != '--')
198 sql = NL.join(lines)
199 for statement in sql.split(';'):
200 if statement.strip() != '':
201 store.execute(statement + ';')
202
203 def load_schema(self, store, version, filename, module_path):
204 """Load the schema from a file.
205
206 This is a helper method for migration classes to call.
207
208 :param store: The Storm store to load the schema into.
209 :type store: storm.locals.Store`
210 :param version: The schema version identifier of the form
211 YYYYMMDDHHMMSS.
212 :type version: string
213 :param filename: The file name containing the schema to load. Pass
214 `None` if there is no schema file to load.
215 :type filename: string
216 :param module_path: The fully qualified Python module path to the
217 migration module being loaded. This is used to record information
218 for use by the test suite.
219 :type module_path: string
220 """
221 if filename is not None:
222 contents = resource_string('mailman.database.schema', filename)
223 self.load_sql(store, contents)
224 # Add a marker that indicates the migration version being applied.
225 store.add(Version(component='schema', version=version))
226
227 @staticmethod
228 def _make_temporary():
229 raise NotImplementedError
230118
=== removed directory 'src/mailman/database/docs'
=== removed file 'src/mailman/database/docs/__init__.py'
=== removed file 'src/mailman/database/docs/migration.rst'
--- src/mailman/database/docs/migration.rst 2014-04-28 15:23:35 +0000
+++ src/mailman/database/docs/migration.rst 1970-01-01 00:00:00 +0000
@@ -1,207 +0,0 @@
1=================
2Schema migrations
3=================
4
5The SQL database schema will over time require upgrading to support new
6features. This is supported via schema migration.
7
8Migrations are embodied in individual Python classes, which themselves may
9load SQL into the database. The naming scheme for migration files is:
10
11 mm_YYYYMMDDHHMMSS_comment.py
12
13where `YYYYMMDDHHMMSS` is a required numeric year, month, day, hour, minute,
14and second specifier providing unique ordering for processing. Only this
15component of the file name is used to determine the ordering. The prefix is
16required due to Python module naming requirements, but it is actually
17ignored. `mm_` is reserved for Mailman's own use.
18
19The optional `comment` part of the file name can be used as a short
20description for the migration, although comments and docstrings in the
21migration files should be used for more detailed descriptions.
22
23Migrations are applied automatically when Mailman starts up, but can also be
24applied at any time by calling in the API directly. Once applied, a
25migration's version string is registered so it will not be applied again.
26
27We see that the base migration, as well as subsequent standard migrations, are
28already applied.
29
30 >>> from mailman.model.version import Version
31 >>> results = config.db.store.find(Version, component='schema')
32 >>> results.count()
33 4
34 >>> versions = sorted(result.version for result in results)
35 >>> for version in versions:
36 ... print(version)
37 00000000000000
38 20120407000000
39 20121015000000
40 20130406000000
41
42
43Migrations
44==========
45
46Migrations can be loaded at any time, and can be found in the migrations path
47specified in the configuration file.
48
49.. Create a temporary directory for the migrations::
50
51 >>> import os, sys, tempfile
52 >>> tempdir = tempfile.mkdtemp()
53 >>> path = os.path.join(tempdir, 'migrations')
54 >>> os.makedirs(path)
55 >>> sys.path.append(tempdir)
56 >>> config.push('migrations', """
57 ... [database]
58 ... migrations_path: migrations
59 ... """)
60
61.. Clean this up at the end of the doctest.
62 >>> def cleanup():
63 ... import shutil
64 ... from mailman.config import config
65 ... config.pop('migrations')
66 ... shutil.rmtree(tempdir)
67 >>> cleanups.append(cleanup)
68
69Here is an example migrations module. The key part of this interface is the
70``upgrade()`` method, which takes four arguments:
71
72 * `database` - The database class, as derived from `StormBaseDatabase`
73 * `store` - The Storm `Store` object.
74 * `version` - The version string as derived from the migrations module's file
75 name. This will include only the `YYYYMMDDHHMMSS` string.
76 * `module_path` - The dotted module path to the migrations module, suitable
77 for lookup in `sys.modules`.
78
79This migration module just adds a marker to the `version` table.
80
81 >>> with open(os.path.join(path, '__init__.py'), 'w') as fp:
82 ... pass
83 >>> with open(os.path.join(path, 'mm_20159999000000.py'), 'w') as fp:
84 ... print("""
85 ... from __future__ import unicode_literals
86 ... from mailman.model.version import Version
87 ... def upgrade(database, store, version, module_path):
88 ... v = Version(component='test', version=version)
89 ... store.add(v)
90 ... database.load_schema(store, version, None, module_path)
91 ... """, file=fp)
92
93This will load the new migration, since it hasn't been loaded before.
94
95 >>> config.db.load_migrations()
96 >>> results = config.db.store.find(Version, component='schema')
97 >>> for result in sorted(result.version for result in results):
98 ... print(result)
99 00000000000000
100 20120407000000
101 20121015000000
102 20130406000000
103 20159999000000
104 >>> test = config.db.store.find(Version, component='test').one()
105 >>> print(test.version)
106 20159999000000
107
108Migrations will only be loaded once.
109
110 >>> with open(os.path.join(path, 'mm_20159999000001.py'), 'w') as fp:
111 ... print("""
112 ... from __future__ import unicode_literals
113 ... from mailman.model.version import Version
114 ... _marker = 801
115 ... def upgrade(database, store, version, module_path):
116 ... global _marker
117 ... # Pad enough zeros on the left to reach 14 characters wide.
118 ... marker = '{0:=#014d}'.format(_marker)
119 ... _marker += 1
120 ... v = Version(component='test', version=marker)
121 ... store.add(v)
122 ... database.load_schema(store, version, None, module_path)
123 ... """, file=fp)
124
125The first time we load this new migration, we'll get the 801 marker.
126
127 >>> config.db.load_migrations()
128 >>> results = config.db.store.find(Version, component='schema')
129 >>> for result in sorted(result.version for result in results):
130 ... print(result)
131 00000000000000
132 20120407000000
133 20121015000000
134 20130406000000
135 20159999000000
136 20159999000001
137 >>> test = config.db.store.find(Version, component='test')
138 >>> for marker in sorted(marker.version for marker in test):
139 ... print(marker)
140 00000000000801
141 20159999000000
142
143We do not get an 802 marker because the migration has already been loaded.
144
145 >>> config.db.load_migrations()
146 >>> results = config.db.store.find(Version, component='schema')
147 >>> for result in sorted(result.version for result in results):
148 ... print(result)
149 00000000000000
150 20120407000000
151 20121015000000
152 20130406000000
153 20159999000000
154 20159999000001
155 >>> test = config.db.store.find(Version, component='test')
156 >>> for marker in sorted(marker.version for marker in test):
157 ... print(marker)
158 00000000000801
159 20159999000000
160
161
162Partial upgrades
163================
164
165It's possible (mostly for testing purposes) to only do a partial upgrade, by
166providing a timestamp to `load_migrations()`. To demonstrate this, we add two
167additional migrations, intended to be applied in sequential order.
168
169 >>> from shutil import copyfile
170 >>> from mailman.testing.helpers import chdir
171 >>> with chdir(path):
172 ... copyfile('mm_20159999000000.py', 'mm_20159999000002.py')
173 ... copyfile('mm_20159999000000.py', 'mm_20159999000003.py')
174 ... copyfile('mm_20159999000000.py', 'mm_20159999000004.py')
175
176Now, only migrate to the ...03 timestamp.
177
178 >>> config.db.load_migrations('20159999000003')
179
180You'll notice that the ...04 version is not present.
181
182 >>> results = config.db.store.find(Version, component='schema')
183 >>> for result in sorted(result.version for result in results):
184 ... print(result)
185 00000000000000
186 20120407000000
187 20121015000000
188 20130406000000
189 20159999000000
190 20159999000001
191 20159999000002
192 20159999000003
193
194
195.. cleanup:
196 Because the Version table holds schema migration data, it will not be
197 cleaned up by the standard test suite. This is generally not a problem
198 for SQLite since each test gets a new database file, but for PostgreSQL,
199 this will cause migration.rst to fail on subsequent runs. So let's just
200 clean up the database explicitly.
201
202 >>> if config.db.TAG != 'sqlite':
203 ... results = config.db.store.execute("""
204 ... DELETE FROM version WHERE version.version >= '201299990000'
205 ... OR version.component = 'test';
206 ... """)
207 ... config.db.commit()
2080
=== modified file 'src/mailman/database/factory.py'
--- src/mailman/database/factory.py 2014-01-01 14:59:42 +0000
+++ src/mailman/database/factory.py 2014-10-11 02:14:38 +0000
@@ -22,7 +22,6 @@
22__metaclass__ = type22__metaclass__ = type
23__all__ = [23__all__ = [
24 'DatabaseFactory',24 'DatabaseFactory',
25 'DatabaseTemporaryFactory',
26 'DatabaseTestingFactory',25 'DatabaseTestingFactory',
27 ]26 ]
2827
@@ -30,15 +29,19 @@
30import os29import os
31import types30import types
3231
32from alembic import command
33from alembic.migration import MigrationContext
34from alembic.script import ScriptDirectory
33from flufl.lock import Lock35from flufl.lock import Lock
34from zope.component import getAdapter36from sqlalchemy import MetaData
35from zope.interface import implementer37from zope.interface import implementer
36from zope.interface.verify import verifyObject38from zope.interface.verify import verifyObject
3739
38from mailman.config import config40from mailman.config import config
39from mailman.interfaces.database import (41from mailman.database.model import Model
40 IDatabase, IDatabaseFactory, ITemporaryDatabase)42from mailman.database.alembic import alembic_cfg
41from mailman.utilities.modules import call_name43from mailman.interfaces.database import IDatabase, IDatabaseFactory
44from mailman.utilities.modules import call_name, expand_path
4245
4346
4447
4548
@@ -54,18 +57,78 @@
54 database = call_name(database_class)57 database = call_name(database_class)
55 verifyObject(IDatabase, database)58 verifyObject(IDatabase, database)
56 database.initialize()59 database.initialize()
57 database.load_migrations()60 schema_mgr = SchemaManager(database)
61 schema_mgr.setup_db()
58 database.commit()62 database.commit()
59 return database63 return database
6064
6165
6266
6367
68class SchemaManager:
69
70 LAST_STORM_SCHEMA_VERSION = '20130406000000'
71
72 def __init__(self, database):
73 self.database = database
74 self.script = ScriptDirectory.from_config(alembic_cfg)
75
76 def get_storm_schema_version(self):
77 md = MetaData()
78 md.reflect(bind=self.database.engine)
79 if "version" not in md.tables:
80 return None
81 Version = md.tables["version"]
82 last_version = self.database.store.query(Version.c.version).filter(
83 Version.c.component == "schema"
84 ).order_by(Version.c.version.desc()).first()
85 # Don't leave open transactions or they will block any schema change
86 self.database.commit()
87 return last_version
88
89 def _create(self):
90 # initial DB creation
91 Model.metadata.create_all(self.database.engine)
92 self.database.commit()
93 command.stamp(alembic_cfg, "head")
94
95 def _upgrade(self):
96 command.upgrade(alembic_cfg, "head")
97
98 def setup_db(self):
99 context = MigrationContext.configure(self.database.store.connection())
100 current_rev = context.get_current_revision()
101 head_rev = self.script.get_current_head()
102 if current_rev == head_rev:
103 return head_rev # already at the latest revision, nothing to do
104 if current_rev == None:
105 # no alembic information
106 storm_version = self.get_storm_schema_version()
107 if storm_version is None:
108 # initial DB creation
109 self._create()
110 else:
111 # DB from a previous version managed by Storm
112 if storm_version.version < self.LAST_STORM_SCHEMA_VERSION:
113 raise RuntimeError(
114 "Upgrading while skipping beta version is "
115 "unsupported, please install the previous "
116 "Mailman beta release")
117 # Run migrations to remove the Storm-specific table and
118 # upgrade to SQLAlchemy & Alembic
119 self._upgrade()
120 elif current_rev != head_rev:
121 self._upgrade()
122 return head_rev
123
124
125
64126
65def _reset(self):127def _reset(self):
66 """See `IDatabase`."""128 """See `IDatabase`."""
67 from mailman.database.model import ModelMeta129 # Avoid a circular import at module level.
130 from mailman.database.model import Model
68 self.store.rollback()131 self.store.rollback()
69 self._pre_reset(self.store)132 self._pre_reset(self.store)
70 ModelMeta._reset(self.store)133 Model._reset(self)
71 self._post_reset(self.store)134 self._post_reset(self.store)
72 self.store.commit()135 self.store.commit()
73136
@@ -81,24 +144,8 @@
81 database = call_name(database_class)144 database = call_name(database_class)
82 verifyObject(IDatabase, database)145 verifyObject(IDatabase, database)
83 database.initialize()146 database.initialize()
84 database.load_migrations()147 Model.metadata.create_all(database.engine)
85 database.commit()148 database.commit()
86 # Make _reset() a bound method of the database instance.149 # Make _reset() a bound method of the database instance.
87 database._reset = types.MethodType(_reset, database)150 database._reset = types.MethodType(_reset, database)
88 return database151 return database
89
90
91
92152
93@implementer(IDatabaseFactory)
94class DatabaseTemporaryFactory:
95 """Create a temporary database for some of the migration tests."""
96
97 @staticmethod
98 def create():
99 """See `IDatabaseFactory`."""
100 database_class_name = config.database['class']
101 database = call_name(database_class_name)
102 verifyObject(IDatabase, database)
103 adapted_database = getAdapter(
104 database, ITemporaryDatabase, database.TAG)
105 return adapted_database
106153
=== modified file 'src/mailman/database/model.py'
--- src/mailman/database/model.py 2014-01-01 14:59:42 +0000
+++ src/mailman/database/model.py 2014-10-11 02:14:38 +0000
@@ -25,44 +25,34 @@
25 ]25 ]
2626
2727
28from operator import attrgetter
29
30from storm.properties import PropertyPublisherMeta
31
32
33
3428
35class ModelMeta(PropertyPublisherMeta):29import contextlib
36 """Do more magic on table classes."""30
3731from sqlalchemy.ext.declarative import declarative_base
38 _class_registry = set()32
3933from mailman.config import config
40 def __init__(self, name, bases, dict):34
41 # Before we let the base class do it's thing, force an __storm_table__35
42 # property to enforce our table naming convention.36class ModelMeta:
43 self.__storm_table__ = name.lower()37 """The custom metaclass for all model base classes.
44 super(ModelMeta, self).__init__(name, bases, dict)38
45 # Register the model class so that it can be more easily cleared.39 This is used in the test suite to quickly reset the database after each
46 # This is required by the test framework so that the corresponding40 test. It works by iterating over all the tables, deleting each. The test
47 # table can be reset between tests.41 suite will then recreate the tables before each test.
48 #42 """
49 # The PRESERVE flag indicates whether the table should be reset or
50 # not. We have to handle the actual Model base class explicitly
51 # because it does not correspond to a table in the database.
52 if not getattr(self, 'PRESERVE', False) and name != 'Model':
53 ModelMeta._class_registry.add(self)
54
55 @staticmethod43 @staticmethod
56 def _reset(store):
57 from mailman.config import config
58 config.db._pre_reset(store)
59 # Make sure this is deterministic, by sorting on the storm table name.
60 classes = sorted(ModelMeta._class_registry,
61 key=attrgetter('__storm_table__'))
62 for model_class in classes:
63 store.find(model_class).remove()
64
65
66
6744
68class Model:45 def _reset(db):
69 """Like Storm's `Storm` subclass, but with a bit extra."""46 with contextlib.closing(config.db.engine.connect()) as connection:
70 __metaclass__ = ModelMeta47 transaction = connection.begin()
48 try:
49 # Delete all the tables in reverse foreign key dependency
50 # order. http://tinyurl.com/on8dy6f
51 for table in reversed(Model.metadata.sorted_tables):
52 connection.execute(table.delete())
53 except:
54 transaction.rollback()
55 raise
56 else:
57 transaction.commit()
58
59
60Model = declarative_base(cls=ModelMeta)
7161
=== modified file 'src/mailman/database/postgresql.py'
--- src/mailman/database/postgresql.py 2014-01-01 14:59:42 +0000
+++ src/mailman/database/postgresql.py 2014-10-11 02:14:38 +0000
@@ -22,34 +22,17 @@
22__metaclass__ = type22__metaclass__ = type
23__all__ = [23__all__ = [
24 'PostgreSQLDatabase',24 'PostgreSQLDatabase',
25 'make_temporary',
26 ]25 ]
2726
2827
29import types28from mailman.database.base import SABaseDatabase
3029from mailman.database.model import Model
31from functools import partial
32from operator import attrgetter
33from urlparse import urlsplit, urlunsplit
34
35from mailman.database.base import StormBaseDatabase
36from mailman.testing.helpers import configuration
3730
3831
3932
4033
41class PostgreSQLDatabase(StormBaseDatabase):34class PostgreSQLDatabase(SABaseDatabase):
42 """Database class for PostgreSQL."""35 """Database class for PostgreSQL."""
4336
44 TAG = 'postgres'
45
46 def _database_exists(self, store):
47 """See `BaseDatabase`."""
48 table_query = ('SELECT table_name FROM information_schema.tables '
49 "WHERE table_schema = 'public'")
50 results = store.execute(table_query)
51 table_names = set(item[0] for item in results)
52 return 'version' in table_names
53
54 def _post_reset(self, store):37 def _post_reset(self, store):
55 """PostgreSQL-specific test suite cleanup.38 """PostgreSQL-specific test suite cleanup.
5639
@@ -57,49 +40,13 @@
57 restart from zero for new tests.40 restart from zero for new tests.
58 """41 """
59 super(PostgreSQLDatabase, self)._post_reset(store)42 super(PostgreSQLDatabase, self)._post_reset(store)
60 from mailman.database.model import ModelMeta43 tables = reversed(Model.metadata.sorted_tables)
61 classes = sorted(ModelMeta._class_registry,
62 key=attrgetter('__storm_table__'))
63 # Recipe adapted from44 # Recipe adapted from
64 # http://stackoverflow.com/questions/544791/45 # http://stackoverflow.com/questions/544791/
65 # django-postgresql-how-to-reset-primary-key46 # django-postgresql-how-to-reset-primary-key
66 for model_class in classes:47 for table in tables:
67 store.execute("""\48 store.execute("""\
68 SELECT setval('"{0}_id_seq"', coalesce(max("id"), 1),49 SELECT setval('"{0}_id_seq"', coalesce(max("id"), 1),
69 max("id") IS NOT null)50 max("id") IS NOT null)
70 FROM "{0}";51 FROM "{0}";
71 """.format(model_class.__storm_table__))
72
73
74
7552
76# Test suite adapter for ITemporaryDatabase.53 """.format(table))
77
78def _cleanup(self, store, tempdb_name):
79 from mailman.config import config
80 store.rollback()
81 store.close()
82 # From the original database connection, drop the now unused database.
83 config.db.store.execute('DROP DATABASE {0}'.format(tempdb_name))
84
85
86def make_temporary(database):
87 """Adapts by monkey patching an existing PostgreSQL IDatabase."""
88 from mailman.config import config
89 parts = urlsplit(config.database.url)
90 assert parts.scheme == 'postgres'
91 new_parts = list(parts)
92 new_parts[2] = '/mmtest'
93 url = urlunsplit(new_parts)
94 # Use the existing database connection to create a new testing
95 # database.
96 config.db.store.execute('ABORT;')
97 config.db.store.execute('CREATE DATABASE mmtest;')
98 with configuration('database', url=url):
99 database.initialize()
100 database._cleanup = types.MethodType(
101 partial(_cleanup, store=database.store, tempdb_name='mmtest'),
102 database)
103 # bool column values in PostgreSQL.
104 database.FALSE = 'False'
105 database.TRUE = 'True'
106 return database
10754
=== removed directory 'src/mailman/database/schema'
=== removed file 'src/mailman/database/schema/__init__.py'
=== removed file 'src/mailman/database/schema/helpers.py'
--- src/mailman/database/schema/helpers.py 2014-01-01 14:59:42 +0000
+++ src/mailman/database/schema/helpers.py 1970-01-01 00:00:00 +0000
@@ -1,43 +0,0 @@
1# Copyright (C) 2013-2014 by the Free Software Foundation, Inc.
2#
3# This file is part of GNU Mailman.
4#
5# GNU Mailman is free software: you can redistribute it and/or modify it under
6# the terms of the GNU General Public License as published by the Free
7# Software Foundation, either version 3 of the License, or (at your option)
8# any later version.
9#
10# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13# more details.
14#
15# You should have received a copy of the GNU General Public License along with
16# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
17
18"""Schema migration helpers."""
19
20from __future__ import absolute_import, print_function, unicode_literals
21
22__metaclass__ = type
23__all__ = [
24 'make_listid',
25 ]
26
27
28
290
30def make_listid(fqdn_listname):
31 """Turn a FQDN list name into a List-ID."""
32 list_name, at, mail_host = fqdn_listname.partition('@')
33 if at == '':
34 # If there is no @ sign in the value, assume it already contains the
35 # list-id.
36 return fqdn_listname
37 return '{0}.{1}'.format(list_name, mail_host)
38
39
40
411
42def pivot(store, table_name):
43 """Pivot a backup table into the real table name."""
44 store.execute('DROP TABLE {}'.format(table_name))
45 store.execute('ALTER TABLE {0}_backup RENAME TO {0}'.format(table_name))
462
=== removed file 'src/mailman/database/schema/mm_00000000000000_base.py'
--- src/mailman/database/schema/mm_00000000000000_base.py 2014-01-01 14:59:42 +0000
+++ src/mailman/database/schema/mm_00000000000000_base.py 1970-01-01 00:00:00 +0000
@@ -1,35 +0,0 @@
1# Copyright (C) 2012-2014 by the Free Software Foundation, Inc.
2#
3# This file is part of GNU Mailman.
4#
5# GNU Mailman is free software: you can redistribute it and/or modify it under
6# the terms of the GNU General Public License as published by the Free
7# Software Foundation, either version 3 of the License, or (at your option)
8# any later version.
9#
10# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13# more details.
14#
15# You should have received a copy of the GNU General Public License along with
16# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
17
18"""Load the base schema."""
19
20from __future__ import absolute_import, print_function, unicode_literals
21
22__metaclass__ = type
23__all__ = [
24 'upgrade',
25 ]
26
27
28VERSION = '00000000000000'
29_helper = None
30
31
32
330
34def upgrade(database, store, version, module_path):
35 filename = '{0}.sql'.format(database.TAG)
36 database.load_schema(store, version, filename, module_path)
371
=== removed file 'src/mailman/database/schema/mm_20120407000000.py'
--- src/mailman/database/schema/mm_20120407000000.py 2014-01-01 14:59:42 +0000
+++ src/mailman/database/schema/mm_20120407000000.py 1970-01-01 00:00:00 +0000
@@ -1,212 +0,0 @@
1# Copyright (C) 2012-2014 by the Free Software Foundation, Inc.
2#
3# This file is part of GNU Mailman.
4#
5# GNU Mailman is free software: you can redistribute it and/or modify it under
6# the terms of the GNU General Public License as published by the Free
7# Software Foundation, either version 3 of the License, or (at your option)
8# any later version.
9#
10# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13# more details.
14#
15# You should have received a copy of the GNU General Public License along with
16# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
17
18"""3.0b1 -> 3.0b2 schema migrations.
19
20All column changes are in the `mailinglist` table.
21
22* Renames:
23 - news_prefix_subject_too -> nntp_prefix_subject_too
24 - news_moderation -> newsgroup_moderation
25
26* Collapsing:
27 - archive, archive_private -> archive_policy
28
29* Remove:
30 - archive_volume_frequency
31 - generic_nonmember_action
32 - nntp_host
33
34* Added:
35 - list_id
36
37* Changes:
38 member.mailing_list holds the list_id not the fqdn_listname
39
40See https://bugs.launchpad.net/mailman/+bug/971013 for details.
41"""
42
43from __future__ import absolute_import, print_function, unicode_literals
44
45__metaclass__ = type
46__all__ = [
47 'upgrade',
48 ]
49
50
51from mailman.database.schema.helpers import pivot
52from mailman.interfaces.archiver import ArchivePolicy
53
54
55VERSION = '20120407000000'
56
57
58
590
60def upgrade(database, store, version, module_path):
61 if database.TAG == 'sqlite':
62 upgrade_sqlite(database, store, version, module_path)
63 else:
64 upgrade_postgres(database, store, version, module_path)
65
66
67
681
69def archive_policy(archive, archive_private):
70 """Convert archive and archive_private to archive_policy."""
71 if archive == 0:
72 return ArchivePolicy.never.value
73 elif archive_private == 1:
74 return ArchivePolicy.private.value
75 else:
76 return ArchivePolicy.public.value
77
78
79
802
81def upgrade_sqlite(database, store, version, module_path):
82 # Load the first part of the migration. This creates a temporary table to
83 # hold the new mailinglist table columns. The problem is that some of the
84 # changes must be performed in Python, so after the first part is loaded,
85 # we do the Python changes, drop the old mailing list table, and then
86 # rename the temporary table to its place.
87 database.load_schema(
88 store, version, 'sqlite_{0}_01.sql'.format(version), module_path)
89 results = store.execute("""
90 SELECT id, include_list_post_header,
91 news_prefix_subject_too, news_moderation,
92 archive, archive_private, list_name, mail_host
93 FROM mailinglist;
94 """)
95 for value in results:
96 (id, list_post,
97 news_prefix, news_moderation,
98 archive, archive_private,
99 list_name, mail_host) = value
100 # Figure out what the new archive_policy column value should be.
101 list_id = '{0}.{1}'.format(list_name, mail_host)
102 fqdn_listname = '{0}@{1}'.format(list_name, mail_host)
103 store.execute("""
104 UPDATE mailinglist_backup SET
105 allow_list_posts = {0},
106 newsgroup_moderation = {1},
107 nntp_prefix_subject_too = {2},
108 archive_policy = {3},
109 list_id = '{4}'
110 WHERE id = {5};
111 """.format(
112 list_post,
113 news_moderation,
114 news_prefix,
115 archive_policy(archive, archive_private),
116 list_id,
117 id))
118 # Also update the member.mailing_list column to hold the list_id
119 # instead of the fqdn_listname.
120 store.execute("""
121 UPDATE member SET
122 mailing_list = '{0}'
123 WHERE mailing_list = '{1}';
124 """.format(list_id, fqdn_listname))
125 # Pivot the backup table to the real thing.
126 pivot(store, 'mailinglist')
127 # Now add some indexes that were previously missing.
128 store.execute(
129 'CREATE INDEX ix_mailinglist_list_id ON mailinglist (list_id);')
130 store.execute(
131 'CREATE INDEX ix_mailinglist_fqdn_listname '
132 'ON mailinglist (list_name, mail_host);')
133 # Now, do the member table.
134 results = store.execute('SELECT id, mailing_list FROM member;')
135 for id, mailing_list in results:
136 list_name, at, mail_host = mailing_list.partition('@')
137 if at == '':
138 list_id = mailing_list
139 else:
140 list_id = '{0}.{1}'.format(list_name, mail_host)
141 store.execute("""
142 UPDATE member_backup SET list_id = '{0}'
143 WHERE id = {1};
144 """.format(list_id, id))
145 # Pivot the backup table to the real thing.
146 pivot(store, 'member')
147
148
149
1503
151def upgrade_postgres(database, store, version, module_path):
152 # Get the old values from the mailinglist table.
153 results = store.execute("""
154 SELECT id, archive, archive_private, list_name, mail_host
155 FROM mailinglist;
156 """)
157 # Do the simple renames first.
158 store.execute("""
159 ALTER TABLE mailinglist
160 RENAME COLUMN news_prefix_subject_too TO nntp_prefix_subject_too;
161 """)
162 store.execute("""
163 ALTER TABLE mailinglist
164 RENAME COLUMN news_moderation TO newsgroup_moderation;
165 """)
166 store.execute("""
167 ALTER TABLE mailinglist
168 RENAME COLUMN include_list_post_header TO allow_list_posts;
169 """)
170 # Do the easy column drops next.
171 for column in ('archive_volume_frequency',
172 'generic_nonmember_action',
173 'nntp_host'):
174 store.execute(
175 'ALTER TABLE mailinglist DROP COLUMN {0};'.format(column))
176 # Now do the trickier collapsing of values. Add the new columns.
177 store.execute('ALTER TABLE mailinglist ADD COLUMN archive_policy INTEGER;')
178 store.execute('ALTER TABLE mailinglist ADD COLUMN list_id TEXT;')
179 # Query the database for the old values of archive and archive_private in
180 # each column. Then loop through all the results and update the new
181 # archive_policy from the old values.
182 for value in results:
183 id, archive, archive_private, list_name, mail_host = value
184 list_id = '{0}.{1}'.format(list_name, mail_host)
185 store.execute("""
186 UPDATE mailinglist SET
187 archive_policy = {0},
188 list_id = '{1}'
189 WHERE id = {2};
190 """.format(archive_policy(archive, archive_private), list_id, id))
191 # Now drop the old columns.
192 for column in ('archive', 'archive_private'):
193 store.execute(
194 'ALTER TABLE mailinglist DROP COLUMN {0};'.format(column))
195 # Now add some indexes that were previously missing.
196 store.execute(
197 'CREATE INDEX ix_mailinglist_list_id ON mailinglist (list_id);')
198 store.execute(
199 'CREATE INDEX ix_mailinglist_fqdn_listname '
200 'ON mailinglist (list_name, mail_host);')
201 # Now, do the member table.
202 results = store.execute('SELECT id, mailing_list FROM member;')
203 store.execute('ALTER TABLE member ADD COLUMN list_id TEXT;')
204 for id, mailing_list in results:
205 list_name, at, mail_host = mailing_list.partition('@')
206 if at == '':
207 list_id = mailing_list
208 else:
209 list_id = '{0}.{1}'.format(list_name, mail_host)
210 store.execute("""
211 UPDATE member SET list_id = '{0}'
212 WHERE id = {1};
213 """.format(list_id, id))
214 store.execute('ALTER TABLE member DROP COLUMN mailing_list;')
215 # Record the migration in the version table.
216 database.load_schema(store, version, None, module_path)
2174
=== removed file 'src/mailman/database/schema/mm_20121015000000.py'
--- src/mailman/database/schema/mm_20121015000000.py 2014-01-01 14:59:42 +0000
+++ src/mailman/database/schema/mm_20121015000000.py 1970-01-01 00:00:00 +0000
@@ -1,95 +0,0 @@
1# Copyright (C) 2012-2014 by the Free Software Foundation, Inc.
2#
3# This file is part of GNU Mailman.
4#
5# GNU Mailman is free software: you can redistribute it and/or modify it under
6# the terms of the GNU General Public License as published by the Free
7# Software Foundation, either version 3 of the License, or (at your option)
8# any later version.
9#
10# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13# more details.
14#
15# You should have received a copy of the GNU General Public License along with
16# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
17
18"""3.0b2 -> 3.0b3 schema migrations.
19
20Renamed:
21 * bans.mailing_list -> bans.list_id
22
23Removed:
24 * mailinglist.new_member_options
25 * mailinglist.send_remindersn
26"""
27
28from __future__ import absolute_import, print_function, unicode_literals
29
30__metaclass__ = type
31__all__ = [
32 'upgrade',
33 ]
34
35
36from mailman.database.schema.helpers import make_listid, pivot
37
38
39VERSION = '20121015000000'
40
41
42
430
44def upgrade(database, store, version, module_path):
45 if database.TAG == 'sqlite':
46 upgrade_sqlite(database, store, version, module_path)
47 else:
48 upgrade_postgres(database, store, version, module_path)
49
50
51
521
53def upgrade_sqlite(database, store, version, module_path):
54 database.load_schema(
55 store, version, 'sqlite_{}_01.sql'.format(version), module_path)
56 results = store.execute("""
57 SELECT id, mailing_list
58 FROM ban;
59 """)
60 for id, mailing_list in results:
61 # Skip global bans since there's nothing to update.
62 if mailing_list is None:
63 continue
64 store.execute("""
65 UPDATE ban_backup SET list_id = '{}'
66 WHERE id = {};
67 """.format(make_listid(mailing_list), id))
68 # Pivot the bans backup table to the real thing.
69 pivot(store, 'ban')
70 pivot(store, 'mailinglist')
71
72
73
742
75def upgrade_postgres(database, store, version, module_path):
76 # Get the old values from the ban table.
77 results = store.execute('SELECT id, mailing_list FROM ban;')
78 store.execute('ALTER TABLE ban ADD COLUMN list_id TEXT;')
79 for id, mailing_list in results:
80 # Skip global bans since there's nothing to update.
81 if mailing_list is None:
82 continue
83 store.execute("""
84 UPDATE ban SET list_id = '{0}'
85 WHERE id = {1};
86 """.format(make_listid(mailing_list), id))
87 store.execute('ALTER TABLE ban DROP COLUMN mailing_list;')
88 store.execute('ALTER TABLE mailinglist DROP COLUMN new_member_options;')
89 store.execute('ALTER TABLE mailinglist DROP COLUMN send_reminders;')
90 store.execute('ALTER TABLE mailinglist DROP COLUMN subscribe_policy;')
91 store.execute('ALTER TABLE mailinglist DROP COLUMN unsubscribe_policy;')
92 store.execute(
93 'ALTER TABLE mailinglist DROP COLUMN subscribe_auto_approval;')
94 store.execute('ALTER TABLE mailinglist DROP COLUMN private_roster;')
95 store.execute(
96 'ALTER TABLE mailinglist DROP COLUMN admin_member_chunksize;')
97 # Record the migration in the version table.
98 database.load_schema(store, version, None, module_path)
993
=== removed file 'src/mailman/database/schema/mm_20130406000000.py'
--- src/mailman/database/schema/mm_20130406000000.py 2014-01-01 14:59:42 +0000
+++ src/mailman/database/schema/mm_20130406000000.py 1970-01-01 00:00:00 +0000
@@ -1,65 +0,0 @@
1# Copyright (C) 2013-2014 by the Free Software Foundation, Inc.
2#
3# This file is part of GNU Mailman.
4#
5# GNU Mailman is free software: you can redistribute it and/or modify it under
6# the terms of the GNU General Public License as published by the Free
7# Software Foundation, either version 3 of the License, or (at your option)
8# any later version.
9#
10# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13# more details.
14#
15# You should have received a copy of the GNU General Public License along with
16# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
17
18"""3.0b3 -> 3.0b4 schema migrations.
19
20Renamed:
21 * bounceevent.list_name -> bounceevent.list_id
22"""
23
24
25from __future__ import absolute_import, print_function, unicode_literals
26
27__metaclass__ = type
28__all__ = [
29 'upgrade'
30 ]
31
32
33from mailman.database.schema.helpers import make_listid, pivot
34
35
36VERSION = '20130406000000'
37
38
39
400
41def upgrade(database, store, version, module_path):
42 if database.TAG == 'sqlite':
43 upgrade_sqlite(database, store, version, module_path)
44 else:
45 upgrade_postgres(database, store, version, module_path)
46
47
48
491
50def upgrade_sqlite(database, store, version, module_path):
51 database.load_schema(
52 store, version, 'sqlite_{}_01.sql'.format(version), module_path)
53 results = store.execute("""
54 SELECT id, list_name
55 FROM bounceevent;
56 """)
57 for id, list_name in results:
58 store.execute("""
59 UPDATE bounceevent_backup SET list_id = '{}'
60 WHERE id = {};
61 """.format(make_listid(list_name), id))
62 pivot(store, 'bounceevent')
63
64
65
662
67def upgrade_postgres(database, store, version, module_path):
68 pass
693
=== removed file 'src/mailman/database/schema/postgres.sql'
--- src/mailman/database/schema/postgres.sql 2012-07-23 14:40:53 +0000
+++ src/mailman/database/schema/postgres.sql 1970-01-01 00:00:00 +0000
@@ -1,349 +0,0 @@
1CREATE TABLE mailinglist (
2 id SERIAL NOT NULL,
3 -- List identity
4 list_name TEXT,
5 mail_host TEXT,
6 include_list_post_header BOOLEAN,
7 include_rfc2369_headers BOOLEAN,
8 -- Attributes not directly modifiable via the web u/i
9 created_at TIMESTAMP,
10 admin_member_chunksize INTEGER,
11 next_request_id INTEGER,
12 next_digest_number INTEGER,
13 digest_last_sent_at TIMESTAMP,
14 volume INTEGER,
15 last_post_at TIMESTAMP,
16 accept_these_nonmembers BYTEA,
17 acceptable_aliases_id INTEGER,
18 admin_immed_notify BOOLEAN,
19 admin_notify_mchanges BOOLEAN,
20 administrivia BOOLEAN,
21 advertised BOOLEAN,
22 anonymous_list BOOLEAN,
23 archive BOOLEAN,
24 archive_private BOOLEAN,
25 archive_volume_frequency INTEGER,
26 -- Automatic responses.
27 autorespond_owner INTEGER,
28 autoresponse_owner_text TEXT,
29 autorespond_postings INTEGER,
30 autoresponse_postings_text TEXT,
31 autorespond_requests INTEGER,
32 autoresponse_request_text TEXT,
33 autoresponse_grace_period TEXT,
34 -- Bounces.
35 forward_unrecognized_bounces_to INTEGER,
36 process_bounces BOOLEAN,
37 bounce_info_stale_after TEXT,
38 bounce_matching_headers TEXT,
39 bounce_notify_owner_on_disable BOOLEAN,
40 bounce_notify_owner_on_removal BOOLEAN,
41 bounce_score_threshold INTEGER,
42 bounce_you_are_disabled_warnings INTEGER,
43 bounce_you_are_disabled_warnings_interval TEXT,
44 -- Content filtering.
45 filter_action INTEGER,
46 filter_content BOOLEAN,
47 collapse_alternatives BOOLEAN,
48 convert_html_to_plaintext BOOLEAN,
49 default_member_action INTEGER,
50 default_nonmember_action INTEGER,
51 description TEXT,
52 digest_footer_uri TEXT,
53 digest_header_uri TEXT,
54 digest_is_default BOOLEAN,
55 digest_send_periodic BOOLEAN,
56 digest_size_threshold REAL,
57 digest_volume_frequency INTEGER,
58 digestable BOOLEAN,
59 discard_these_nonmembers BYTEA,
60 emergency BOOLEAN,
61 encode_ascii_prefixes BOOLEAN,
62 first_strip_reply_to BOOLEAN,
63 footer_uri TEXT,
64 forward_auto_discards BOOLEAN,
65 gateway_to_mail BOOLEAN,
66 gateway_to_news BOOLEAN,
67 generic_nonmember_action INTEGER,
68 goodbye_message_uri TEXT,
69 header_matches BYTEA,
70 header_uri TEXT,
71 hold_these_nonmembers BYTEA,
72 info TEXT,
73 linked_newsgroup TEXT,
74 max_days_to_hold INTEGER,
75 max_message_size INTEGER,
76 max_num_recipients INTEGER,
77 member_moderation_notice TEXT,
78 mime_is_default_digest BOOLEAN,
79 moderator_password TEXT,
80 new_member_options INTEGER,
81 news_moderation INTEGER,
82 news_prefix_subject_too BOOLEAN,
83 nntp_host TEXT,
84 nondigestable BOOLEAN,
85 nonmember_rejection_notice TEXT,
86 obscure_addresses BOOLEAN,
87 owner_chain TEXT,
88 owner_pipeline TEXT,
89 personalize INTEGER,
90 post_id INTEGER,
91 posting_chain TEXT,
92 posting_pipeline TEXT,
93 preferred_language TEXT,
94 private_roster BOOLEAN,
95 display_name TEXT,
96 reject_these_nonmembers BYTEA,
97 reply_goes_to_list INTEGER,
98 reply_to_address TEXT,
99 require_explicit_destination BOOLEAN,
100 respond_to_post_requests BOOLEAN,
101 scrub_nondigest BOOLEAN,
102 send_goodbye_message BOOLEAN,
103 send_reminders BOOLEAN,
104 send_welcome_message BOOLEAN,
105 subject_prefix TEXT,
106 subscribe_auto_approval BYTEA,
107 subscribe_policy INTEGER,
108 topics BYTEA,
109 topics_bodylines_limit INTEGER,
110 topics_enabled BOOLEAN,
111 unsubscribe_policy INTEGER,
112 welcome_message_uri TEXT,
113 -- This was accidentally added by the PostgreSQL porter.
114 -- moderation_callback TEXT,
115 PRIMARY KEY (id)
116 );
117
118CREATE TABLE _request (
119 id SERIAL NOT NULL,
120 "key" TEXT,
121 request_type INTEGER,
122 data_hash BYTEA,
123 mailing_list_id INTEGER,
124 PRIMARY KEY (id)
125 -- XXX: config.db_reset() triggers IntegrityError
126 -- ,
127 -- CONSTRAINT _request_mailing_list_id_fk
128 -- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
129 );
130
131CREATE TABLE acceptablealias (
132 id SERIAL NOT NULL,
133 "alias" TEXT NOT NULL,
134 mailing_list_id INTEGER NOT NULL,
135 PRIMARY KEY (id)
136 -- XXX: config.db_reset() triggers IntegrityError
137 -- ,
138 -- CONSTRAINT acceptablealias_mailing_list_id_fk
139 -- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
140 );
141CREATE INDEX ix_acceptablealias_mailing_list_id
142 ON acceptablealias (mailing_list_id);
143CREATE INDEX ix_acceptablealias_alias ON acceptablealias ("alias");
144
145CREATE TABLE preferences (
146 id SERIAL NOT NULL,
147 acknowledge_posts BOOLEAN,
148 hide_address BOOLEAN,
149 preferred_language TEXT,
150 receive_list_copy BOOLEAN,
151 receive_own_postings BOOLEAN,
152 delivery_mode INTEGER,
153 delivery_status INTEGER,
154 PRIMARY KEY (id)
155 );
156
157CREATE TABLE address (
158 id SERIAL NOT NULL,
159 email TEXT,
160 _original TEXT,
161 display_name TEXT,
162 verified_on TIMESTAMP,
163 registered_on TIMESTAMP,
164 user_id INTEGER,
165 preferences_id INTEGER,
166 PRIMARY KEY (id)
167 -- XXX: config.db_reset() triggers IntegrityError
168 -- ,
169 -- CONSTRAINT address_preferences_id_fk
170 -- FOREIGN KEY (preferences_id) REFERENCES preferences (id)
171 );
172
173CREATE TABLE "user" (
174 id SERIAL NOT NULL,
175 display_name TEXT,
176 password BYTEA,
177 _user_id UUID,
178 _created_on TIMESTAMP,
179 _preferred_address_id INTEGER,
180 preferences_id INTEGER,
181 PRIMARY KEY (id)
182 -- XXX: config.db_reset() triggers IntegrityError
183 -- ,
184 -- CONSTRAINT user_preferences_id_fk
185 -- FOREIGN KEY (preferences_id) REFERENCES preferences (id),
186 -- XXX: config.db_reset() triggers IntegrityError
187 -- CONSTRAINT _preferred_address_id_fk
188 -- FOREIGN KEY (_preferred_address_id) REFERENCES address (id)
189 );
190CREATE INDEX ix_user_user_id ON "user" (_user_id);
191
192-- since user and address have circular foreign key refs, the
193-- constraint on the address table has to be added after
194-- the user table is created
195--
196-- XXX: users.rst triggers an IntegrityError
197-- ALTER TABLE address ADD
198-- CONSTRAINT address_user_id_fk
199-- FOREIGN KEY (user_id) REFERENCES "user" (id);
200
201CREATE TABLE autoresponserecord (
202 id SERIAL NOT NULL,
203 address_id INTEGER,
204 mailing_list_id INTEGER,
205 response_type INTEGER,
206 date_sent TIMESTAMP,
207 PRIMARY KEY (id)
208 -- XXX: config.db_reset() triggers IntegrityError
209 -- ,
210 -- CONSTRAINT autoresponserecord_address_id_fk
211 -- FOREIGN KEY (address_id) REFERENCES address (id)
212 -- XXX: config.db_reset() triggers IntegrityError
213 -- ,
214 -- CONSTRAINT autoresponserecord_mailing_list_id
215 -- FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
216 );
217CREATE INDEX ix_autoresponserecord_address_id
218 ON autoresponserecord (address_id);
219CREATE INDEX ix_autoresponserecord_mailing_list_id
220 ON autoresponserecord (mailing_list_id);
221
222CREATE TABLE bounceevent (
223 id SERIAL NOT NULL,
224 list_name TEXT,
225 email TEXT,
226 "timestamp" TIMESTAMP,
227 message_id TEXT,
228 context INTEGER,
229 processed BOOLEAN,
230 PRIMARY KEY (id)
231 );
232
233CREATE TABLE contentfilter (
234 id SERIAL NOT NULL,
235 mailing_list_id INTEGER,
236 filter_pattern TEXT,
237 filter_type INTEGER,
238 PRIMARY KEY (id),
239 CONSTRAINT contentfilter_mailing_list_id
240 FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
241 );
242CREATE INDEX ix_contentfilter_mailing_list_id
243 ON contentfilter (mailing_list_id);
244
245CREATE TABLE domain (
246 id SERIAL NOT NULL,
247 mail_host TEXT,
248 base_url TEXT,
249 description TEXT,
250 contact_address TEXT,
251 PRIMARY KEY (id)
252 );
253
254CREATE TABLE language (
255 id SERIAL NOT NULL,
256 code TEXT,
257 PRIMARY KEY (id)
258 );
259
260CREATE TABLE member (
261 id SERIAL NOT NULL,
262 _member_id UUID,
263 role INTEGER,
264 mailing_list TEXT,
265 moderation_action INTEGER,
266 address_id INTEGER,
267 preferences_id INTEGER,
268 user_id INTEGER,
269 PRIMARY KEY (id)
270 -- XXX: config.db_reset() triggers IntegrityError
271 -- ,
272 -- CONSTRAINT member_address_id_fk
273 -- FOREIGN KEY (address_id) REFERENCES address (id),
274 -- XXX: config.db_reset() triggers IntegrityError
275 -- CONSTRAINT member_preferences_id_fk
276 -- FOREIGN KEY (preferences_id) REFERENCES preferences (id),
277 -- CONSTRAINT member_user_id_fk
278 -- FOREIGN KEY (user_id) REFERENCES "user" (id)
279 );
280CREATE INDEX ix_member__member_id ON member (_member_id);
281CREATE INDEX ix_member_address_id ON member (address_id);
282CREATE INDEX ix_member_preferences_id ON member (preferences_id);
283
284CREATE TABLE message (
285 id SERIAL NOT NULL,
286 message_id_hash BYTEA,
287 path BYTEA,
288 message_id TEXT,
289 PRIMARY KEY (id)
290 );
291
292CREATE TABLE onelastdigest (
293 id SERIAL NOT NULL,
294 mailing_list_id INTEGER,
295 address_id INTEGER,
296 delivery_mode INTEGER,
297 PRIMARY KEY (id),
298 CONSTRAINT onelastdigest_mailing_list_id_fk
299 FOREIGN KEY (mailing_list_id) REFERENCES mailinglist(id),
300 CONSTRAINT onelastdigest_address_id_fk
301 FOREIGN KEY (address_id) REFERENCES address(id)
302 );
303
304CREATE TABLE pended (
305 id SERIAL NOT NULL,
306 token BYTEA,
307 expiration_date TIMESTAMP,
308 PRIMARY KEY (id)
309 );
310
311CREATE TABLE pendedkeyvalue (
312 id SERIAL NOT NULL,
313 "key" TEXT,
314 value TEXT,
315 pended_id INTEGER,
316 PRIMARY KEY (id)
317 -- ,
318 -- XXX: config.db_reset() triggers IntegrityError
319 -- CONSTRAINT pendedkeyvalue_pended_id_fk
320 -- FOREIGN KEY (pended_id) REFERENCES pended (id)
321 );
322
323CREATE TABLE version (
324 id SERIAL NOT NULL,
325 component TEXT,
326 version TEXT,
327 PRIMARY KEY (id)
328 );
329
330CREATE INDEX ix__request_mailing_list_id ON _request (mailing_list_id);
331CREATE INDEX ix_address_preferences_id ON address (preferences_id);
332CREATE INDEX ix_address_user_id ON address (user_id);
333CREATE INDEX ix_pendedkeyvalue_pended_id ON pendedkeyvalue (pended_id);
334CREATE INDEX ix_user_preferences_id ON "user" (preferences_id);
335
336CREATE TABLE ban (
337 id SERIAL NOT NULL,
338 email TEXT,
339 mailing_list TEXT,
340 PRIMARY KEY (id)
341 );
342
343CREATE TABLE uid (
344 -- Keep track of all assigned unique ids to prevent re-use.
345 id SERIAL NOT NULL,
346 uid UUID,
347 PRIMARY KEY (id)
348 );
349CREATE INDEX ix_uid_uid ON uid (uid);
3500
=== removed file 'src/mailman/database/schema/sqlite.sql'
--- src/mailman/database/schema/sqlite.sql 2012-04-08 16:15:29 +0000
+++ src/mailman/database/schema/sqlite.sql 1970-01-01 00:00:00 +0000
@@ -1,327 +0,0 @@
1-- THIS FILE HAS BEEN FROZEN AS OF 3.0b1
2-- SEE THE SCHEMA MIGRATIONS FOR DIFFERENCES.
3
4PRAGMA foreign_keys = ON;
5
6CREATE TABLE _request (
7 id INTEGER NOT NULL,
8 "key" TEXT,
9 request_type INTEGER,
10 data_hash TEXT,
11 mailing_list_id INTEGER,
12 PRIMARY KEY (id),
13 CONSTRAINT _request_mailing_list_id_fk
14 FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
15 );
16
17CREATE TABLE acceptablealias (
18 id INTEGER NOT NULL,
19 "alias" TEXT NOT NULL,
20 mailing_list_id INTEGER NOT NULL,
21 PRIMARY KEY (id),
22 CONSTRAINT acceptablealias_mailing_list_id_fk
23 FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
24 );
25CREATE INDEX ix_acceptablealias_mailing_list_id
26 ON acceptablealias (mailing_list_id);
27CREATE INDEX ix_acceptablealias_alias ON acceptablealias ("alias");
28
29CREATE TABLE address (
30 id INTEGER NOT NULL,
31 email TEXT,
32 _original TEXT,
33 display_name TEXT,
34 verified_on TIMESTAMP,
35 registered_on TIMESTAMP,
36 user_id INTEGER,
37 preferences_id INTEGER,
38 PRIMARY KEY (id),
39 CONSTRAINT address_user_id_fk
40 FOREIGN KEY (user_id) REFERENCES user (id),
41 CONSTRAINT address_preferences_id_fk
42 FOREIGN KEY (preferences_id) REFERENCES preferences (id)
43 );
44
45CREATE TABLE autoresponserecord (
46 id INTEGER NOT NULL,
47 address_id INTEGER,
48 mailing_list_id INTEGER,
49 response_type INTEGER,
50 date_sent TIMESTAMP,
51 PRIMARY KEY (id),
52 CONSTRAINT autoresponserecord_address_id_fk
53 FOREIGN KEY (address_id) REFERENCES address (id),
54 CONSTRAINT autoresponserecord_mailing_list_id
55 FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
56 );
57CREATE INDEX ix_autoresponserecord_address_id
58 ON autoresponserecord (address_id);
59CREATE INDEX ix_autoresponserecord_mailing_list_id
60 ON autoresponserecord (mailing_list_id);
61
62CREATE TABLE bounceevent (
63 id INTEGER NOT NULL,
64 list_name TEXT,
65 email TEXT,
66 'timestamp' TIMESTAMP,
67 message_id TEXT,
68 context INTEGER,
69 processed BOOLEAN,
70 PRIMARY KEY (id)
71 );
72
73CREATE TABLE contentfilter (
74 id INTEGER NOT NULL,
75 mailing_list_id INTEGER,
76 filter_pattern TEXT,
77 filter_type INTEGER,
78 PRIMARY KEY (id),
79 CONSTRAINT contentfilter_mailing_list_id
80 FOREIGN KEY (mailing_list_id) REFERENCES mailinglist (id)
81 );
82CREATE INDEX ix_contentfilter_mailing_list_id
83 ON contentfilter (mailing_list_id);
84
85CREATE TABLE domain (
86 id INTEGER NOT NULL,
87 mail_host TEXT,
88 base_url TEXT,
89 description TEXT,
90 contact_address TEXT,
91 PRIMARY KEY (id)
92 );
93
94CREATE TABLE language (
95 id INTEGER NOT NULL,
96 code TEXT,
97 PRIMARY KEY (id)
98 );
99
100CREATE TABLE mailinglist (
101 id INTEGER NOT NULL,
102 -- List identity
103 list_name TEXT,
104 mail_host TEXT,
105 include_list_post_header BOOLEAN,
106 include_rfc2369_headers BOOLEAN,
107 -- Attributes not directly modifiable via the web u/i
108 created_at TIMESTAMP,
109 admin_member_chunksize INTEGER,
110 next_request_id INTEGER,
111 next_digest_number INTEGER,
112 digest_last_sent_at TIMESTAMP,
113 volume INTEGER,
114 last_post_at TIMESTAMP,
115 accept_these_nonmembers BLOB,
116 acceptable_aliases_id INTEGER,
117 admin_immed_notify BOOLEAN,
118 admin_notify_mchanges BOOLEAN,
119 administrivia BOOLEAN,
120 advertised BOOLEAN,
121 anonymous_list BOOLEAN,
122 archive BOOLEAN,
123 archive_private BOOLEAN,
124 archive_volume_frequency INTEGER,
125 -- Automatic responses.
126 autorespond_owner INTEGER,
127 autoresponse_owner_text TEXT,
128 autorespond_postings INTEGER,
129 autoresponse_postings_text TEXT,
130 autorespond_requests INTEGER,
131 autoresponse_request_text TEXT,
132 autoresponse_grace_period TEXT,
133 -- Bounces.
134 forward_unrecognized_bounces_to INTEGER,
135 process_bounces BOOLEAN,
136 bounce_info_stale_after TEXT,
137 bounce_matching_headers TEXT,
138 bounce_notify_owner_on_disable BOOLEAN,
139 bounce_notify_owner_on_removal BOOLEAN,
140 bounce_score_threshold INTEGER,
141 bounce_you_are_disabled_warnings INTEGER,
142 bounce_you_are_disabled_warnings_interval TEXT,
143 -- Content filtering.
144 filter_action INTEGER,
145 filter_content BOOLEAN,
146 collapse_alternatives BOOLEAN,
147 convert_html_to_plaintext BOOLEAN,
148 default_member_action INTEGER,
149 default_nonmember_action INTEGER,
150 description TEXT,
151 digest_footer_uri TEXT,
152 digest_header_uri TEXT,
153 digest_is_default BOOLEAN,
154 digest_send_periodic BOOLEAN,
155 digest_size_threshold FLOAT,
156 digest_volume_frequency INTEGER,
157 digestable BOOLEAN,
158 discard_these_nonmembers BLOB,
159 emergency BOOLEAN,
160 encode_ascii_prefixes BOOLEAN,
161 first_strip_reply_to BOOLEAN,
162 footer_uri TEXT,
163 forward_auto_discards BOOLEAN,
164 gateway_to_mail BOOLEAN,
165 gateway_to_news BOOLEAN,
166 generic_nonmember_action INTEGER,
167 goodbye_message_uri TEXT,
168 header_matches BLOB,
169 header_uri TEXT,
170 hold_these_nonmembers BLOB,
171 info TEXT,
172 linked_newsgroup TEXT,
173 max_days_to_hold INTEGER,
174 max_message_size INTEGER,
175 max_num_recipients INTEGER,
176 member_moderation_notice TEXT,
177 mime_is_default_digest BOOLEAN,
178 moderator_password TEXT,
179 new_member_options INTEGER,
180 news_moderation INTEGER,
181 news_prefix_subject_too BOOLEAN,
182 nntp_host TEXT,
183 nondigestable BOOLEAN,
184 nonmember_rejection_notice TEXT,
185 obscure_addresses BOOLEAN,
186 owner_chain TEXT,
187 owner_pipeline TEXT,
188 personalize INTEGER,
189 post_id INTEGER,
190 posting_chain TEXT,
191 posting_pipeline TEXT,
192 preferred_language TEXT,
193 private_roster BOOLEAN,
194 display_name TEXT,
195 reject_these_nonmembers BLOB,
196 reply_goes_to_list INTEGER,
197 reply_to_address TEXT,
198 require_explicit_destination BOOLEAN,
199 respond_to_post_requests BOOLEAN,
200 scrub_nondigest BOOLEAN,
201 send_goodbye_message BOOLEAN,
202 send_reminders BOOLEAN,
203 send_welcome_message BOOLEAN,
204 subject_prefix TEXT,
205 subscribe_auto_approval BLOB,
206 subscribe_policy INTEGER,
207 topics BLOB,
208 topics_bodylines_limit INTEGER,
209 topics_enabled BOOLEAN,
210 unsubscribe_policy INTEGER,
211 welcome_message_uri TEXT,
212 PRIMARY KEY (id)
213 );
214
215CREATE TABLE member (
216 id INTEGER NOT NULL,
217 _member_id TEXT,
218 role INTEGER,
219 mailing_list TEXT,
220 moderation_action INTEGER,
221 address_id INTEGER,
222 preferences_id INTEGER,
223 user_id INTEGER,
224 PRIMARY KEY (id),
225 CONSTRAINT member_address_id_fk
226 FOREIGN KEY (address_id) REFERENCES address (id),
227 CONSTRAINT member_preferences_id_fk
228 FOREIGN KEY (preferences_id) REFERENCES preferences (id)
229 CONSTRAINT member_user_id_fk
230 FOREIGN KEY (user_id) REFERENCES user (id)
231 );
232CREATE INDEX ix_member__member_id ON member (_member_id);
233CREATE INDEX ix_member_address_id ON member (address_id);
234CREATE INDEX ix_member_preferences_id ON member (preferences_id);
235
236CREATE TABLE message (
237 id INTEGER NOT NULL,
238 message_id_hash TEXT,
239 path TEXT,
240 message_id TEXT,
241 PRIMARY KEY (id)
242 );
243
244CREATE TABLE onelastdigest (
245 id INTEGER NOT NULL,
246 mailing_list_id INTEGER,
247 address_id INTEGER,
248 delivery_mode INTEGER,
249 PRIMARY KEY (id),
250 CONSTRAINT onelastdigest_mailing_list_id_fk
251 FOREIGN KEY (mailing_list_id) REFERENCES mailinglist(id),
252 CONSTRAINT onelastdigest_address_id_fk
253 FOREIGN KEY (address_id) REFERENCES address(id)
254 );
255
256CREATE TABLE pended (
257 id INTEGER NOT NULL,
258 token TEXT,
259 expiration_date TIMESTAMP,
260 PRIMARY KEY (id)
261 );
262
263CREATE TABLE pendedkeyvalue (
264 id INTEGER NOT NULL,
265 "key" TEXT,
266 value TEXT,
267 pended_id INTEGER,
268 PRIMARY KEY (id),
269 CONSTRAINT pendedkeyvalue_pended_id_fk
270 FOREIGN KEY (pended_id) REFERENCES pended (id)
271 );
272
273CREATE TABLE preferences (
274 id INTEGER NOT NULL,
275 acknowledge_posts BOOLEAN,
276 hide_address BOOLEAN,
277 preferred_language TEXT,
278 receive_list_copy BOOLEAN,
279 receive_own_postings BOOLEAN,
280 delivery_mode INTEGER,
281 delivery_status INTEGER,
282 PRIMARY KEY (id)
283 );
284
285CREATE TABLE user (
286 id INTEGER NOT NULL,
287 display_name TEXT,
288 password BINARY,
289 _user_id TEXT,
290 _created_on TIMESTAMP,
291 _preferred_address_id INTEGER,
292 preferences_id INTEGER,
293 PRIMARY KEY (id),
294 CONSTRAINT user_preferences_id_fk
295 FOREIGN KEY (preferences_id) REFERENCES preferences (id),
296 CONSTRAINT _preferred_address_id_fk
297 FOREIGN KEY (_preferred_address_id) REFERENCES address (id)
298 );
299CREATE INDEX ix_user_user_id ON user (_user_id);
300
301CREATE TABLE version (
302 id INTEGER NOT NULL,
303 component TEXT,
304 version TEXT,
305 PRIMARY KEY (id)
306 );
307
308CREATE INDEX ix__request_mailing_list_id ON _request (mailing_list_id);
309CREATE INDEX ix_address_preferences_id ON address (preferences_id);
310CREATE INDEX ix_address_user_id ON address (user_id);
311CREATE INDEX ix_pendedkeyvalue_pended_id ON pendedkeyvalue (pended_id);
312CREATE INDEX ix_user_preferences_id ON user (preferences_id);
313
314CREATE TABLE ban (
315 id INTEGER NOT NULL,
316 email TEXT,
317 mailing_list TEXT,
318 PRIMARY KEY (id)
319 );
320
321CREATE TABLE uid (
322 -- Keep track of all assigned unique ids to prevent re-use.
323 id INTEGER NOT NULL,
324 uid TEXT,
325 PRIMARY KEY (id)
326 );
327CREATE INDEX ix_uid_uid ON uid (uid);
3280
=== removed file 'src/mailman/database/schema/sqlite_20120407000000_01.sql'
--- src/mailman/database/schema/sqlite_20120407000000_01.sql 2013-09-01 15:15:08 +0000
+++ src/mailman/database/schema/sqlite_20120407000000_01.sql 1970-01-01 00:00:00 +0000
@@ -1,280 +0,0 @@
1-- This file contains the sqlite3 schema migration from
2-- 3.0b1 TO 3.0b2
3--
4-- 3.0b2 has been released thus you MAY NOT edit this file.
5
6-- For SQLite3 migration strategy, see
7-- http://sqlite.org/faq.html#q11
8
9-- REMOVALS from the mailinglist table:
10-- REM archive
11-- REM archive_private
12-- REM archive_volume_frequency
13-- REM include_list_post_header
14-- REM news_moderation
15-- REM news_prefix_subject_too
16-- REM nntp_host
17--
18-- ADDS to the mailing list table:
19-- ADD allow_list_posts
20-- ADD archive_policy
21-- ADD list_id
22-- ADD newsgroup_moderation
23-- ADD nntp_prefix_subject_too
24
25-- LP: #971013
26-- LP: #967238
27
28-- REMOVALS from the member table:
29-- REM mailing_list
30
31-- ADDS to the member table:
32-- ADD list_id
33
34-- LP: #1024509
35
36
37CREATE TABLE mailinglist_backup (
38 id INTEGER NOT NULL,
39 -- List identity
40 list_name TEXT,
41 mail_host TEXT,
42 allow_list_posts BOOLEAN,
43 include_rfc2369_headers BOOLEAN,
44 -- Attributes not directly modifiable via the web u/i
45 created_at TIMESTAMP,
46 admin_member_chunksize INTEGER,
47 next_request_id INTEGER,
48 next_digest_number INTEGER,
49 digest_last_sent_at TIMESTAMP,
50 volume INTEGER,
51 last_post_at TIMESTAMP,
52 accept_these_nonmembers BLOB,
53 acceptable_aliases_id INTEGER,
54 admin_immed_notify BOOLEAN,
55 admin_notify_mchanges BOOLEAN,
56 administrivia BOOLEAN,
57 advertised BOOLEAN,
58 anonymous_list BOOLEAN,
59 -- Automatic responses.
60 autorespond_owner INTEGER,
61 autoresponse_owner_text TEXT,
62 autorespond_postings INTEGER,
63 autoresponse_postings_text TEXT,
64 autorespond_requests INTEGER,
65 autoresponse_request_text TEXT,
66 autoresponse_grace_period TEXT,
67 -- Bounces.
68 forward_unrecognized_bounces_to INTEGER,
69 process_bounces BOOLEAN,
70 bounce_info_stale_after TEXT,
71 bounce_matching_headers TEXT,
72 bounce_notify_owner_on_disable BOOLEAN,
73 bounce_notify_owner_on_removal BOOLEAN,
74 bounce_score_threshold INTEGER,
75 bounce_you_are_disabled_warnings INTEGER,
76 bounce_you_are_disabled_warnings_interval TEXT,
77 -- Content filtering.
78 filter_action INTEGER,
79 filter_content BOOLEAN,
80 collapse_alternatives BOOLEAN,
81 convert_html_to_plaintext BOOLEAN,
82 default_member_action INTEGER,
83 default_nonmember_action INTEGER,
84 description TEXT,
85 digest_footer_uri TEXT,
86 digest_header_uri TEXT,
87 digest_is_default BOOLEAN,
88 digest_send_periodic BOOLEAN,
89 digest_size_threshold FLOAT,
90 digest_volume_frequency INTEGER,
91 digestable BOOLEAN,
92 discard_these_nonmembers BLOB,
93 emergency BOOLEAN,
94 encode_ascii_prefixes BOOLEAN,
95 first_strip_reply_to BOOLEAN,
96 footer_uri TEXT,
97 forward_auto_discards BOOLEAN,
98 gateway_to_mail BOOLEAN,
99 gateway_to_news BOOLEAN,
100 goodbye_message_uri TEXT,
101 header_matches BLOB,
102 header_uri TEXT,
103 hold_these_nonmembers BLOB,
104 info TEXT,
105 linked_newsgroup TEXT,
106 max_days_to_hold INTEGER,
107 max_message_size INTEGER,
108 max_num_recipients INTEGER,
109 member_moderation_notice TEXT,
110 mime_is_default_digest BOOLEAN,
111 moderator_password TEXT,
112 new_member_options INTEGER,
113 nondigestable BOOLEAN,
114 nonmember_rejection_notice TEXT,
115 obscure_addresses BOOLEAN,
116 owner_chain TEXT,
117 owner_pipeline TEXT,
118 personalize INTEGER,
119 post_id INTEGER,
120 posting_chain TEXT,
121 posting_pipeline TEXT,
122 preferred_language TEXT,
123 private_roster BOOLEAN,
124 display_name TEXT,
125 reject_these_nonmembers BLOB,
126 reply_goes_to_list INTEGER,
127 reply_to_address TEXT,
128 require_explicit_destination BOOLEAN,
129 respond_to_post_requests BOOLEAN,
130 scrub_nondigest BOOLEAN,
131 send_goodbye_message BOOLEAN,
132 send_reminders BOOLEAN,
133 send_welcome_message BOOLEAN,
134 subject_prefix TEXT,
135 subscribe_auto_approval BLOB,
136 subscribe_policy INTEGER,
137 topics BLOB,
138 topics_bodylines_limit INTEGER,
139 topics_enabled BOOLEAN,
140 unsubscribe_policy INTEGER,
141 welcome_message_uri TEXT,
142 PRIMARY KEY (id)
143 );
144
145INSERT INTO mailinglist_backup SELECT
146 id,
147 -- List identity
148 list_name,
149 mail_host,
150 include_list_post_header,
151 include_rfc2369_headers,
152 -- Attributes not directly modifiable via the web u/i
153 created_at,
154 admin_member_chunksize,
155 next_request_id,
156 next_digest_number,
157 digest_last_sent_at,
158 volume,
159 last_post_at,
160 accept_these_nonmembers,
161 acceptable_aliases_id,
162 admin_immed_notify,
163 admin_notify_mchanges,
164 administrivia,
165 advertised,
166 anonymous_list,
167 -- Automatic responses.
168 autorespond_owner,
169 autoresponse_owner_text,
170 autorespond_postings,
171 autoresponse_postings_text,
172 autorespond_requests,
173 autoresponse_request_text,
174 autoresponse_grace_period,
175 -- Bounces.
176 forward_unrecognized_bounces_to,
177 process_bounces,
178 bounce_info_stale_after,
179 bounce_matching_headers,
180 bounce_notify_owner_on_disable,
181 bounce_notify_owner_on_removal,
182 bounce_score_threshold,
183 bounce_you_are_disabled_warnings,
184 bounce_you_are_disabled_warnings_interval,
185 -- Content filtering.
186 filter_action,
187 filter_content,
188 collapse_alternatives,
189 convert_html_to_plaintext,
190 default_member_action,
191 default_nonmember_action,
192 description,
193 digest_footer_uri,
194 digest_header_uri,
195 digest_is_default,
196 digest_send_periodic,
197 digest_size_threshold,
198 digest_volume_frequency,
199 digestable,
200 discard_these_nonmembers,
201 emergency,
202 encode_ascii_prefixes,
203 first_strip_reply_to,
204 footer_uri,
205 forward_auto_discards,
206 gateway_to_mail,
207 gateway_to_news,
208 goodbye_message_uri,
209 header_matches,
210 header_uri,
211 hold_these_nonmembers,
212 info,
213 linked_newsgroup,
214 max_days_to_hold,
215 max_message_size,
216 max_num_recipients,
217 member_moderation_notice,
218 mime_is_default_digest,
219 moderator_password,
220 new_member_options,
221 nondigestable,
222 nonmember_rejection_notice,
223 obscure_addresses,
224 owner_chain,
225 owner_pipeline,
226 personalize,
227 post_id,
228 posting_chain,
229 posting_pipeline,
230 preferred_language,
231 private_roster,
232 display_name,
233 reject_these_nonmembers,
234 reply_goes_to_list,
235 reply_to_address,
236 require_explicit_destination,
237 respond_to_post_requests,
238 scrub_nondigest,
239 send_goodbye_message,
240 send_reminders,
241 send_welcome_message,
242 subject_prefix,
243 subscribe_auto_approval,
244 subscribe_policy,
245 topics,
246 topics_bodylines_limit,
247 topics_enabled,
248 unsubscribe_policy,
249 welcome_message_uri
250 FROM mailinglist;
251
252CREATE TABLE member_backup(
253 id INTEGER NOT NULL,
254 _member_id TEXT,
255 role INTEGER,
256 moderation_action INTEGER,
257 address_id INTEGER,
258 preferences_id INTEGER,
259 user_id INTEGER,
260 PRIMARY KEY (id)
261 );
262
263INSERT INTO member_backup SELECT
264 id,
265 _member_id,
266 role,
267 moderation_action,
268 address_id,
269 preferences_id,
270 user_id
271 FROM member;
272
273
274-- Add the new columns. They'll get inserted at the Python layer.
275ALTER TABLE mailinglist_backup ADD COLUMN archive_policy INTEGER;
276ALTER TABLE mailinglist_backup ADD COLUMN list_id TEXT;
277ALTER TABLE mailinglist_backup ADD COLUMN nntp_prefix_subject_too INTEGER;
278ALTER TABLE mailinglist_backup ADD COLUMN newsgroup_moderation INTEGER;
279
280ALTER TABLE member_backup ADD COLUMN list_id TEXT;
2810
=== removed file 'src/mailman/database/schema/sqlite_20121015000000_01.sql'
--- src/mailman/database/schema/sqlite_20121015000000_01.sql 2013-09-01 15:15:08 +0000
+++ src/mailman/database/schema/sqlite_20121015000000_01.sql 1970-01-01 00:00:00 +0000
@@ -1,230 +0,0 @@
1-- This file contains the sqlite3 schema migration from
2-- 3.0b2 TO 3.0b3
3--
4-- 3.0b3 has been released thus you MAY NOT edit this file.
5
6-- REMOVALS from the ban table:
7-- REM mailing_list
8
9-- ADDS to the ban table:
10-- ADD list_id
11
12CREATE TABLE ban_backup (
13 id INTEGER NOT NULL,
14 email TEXT,
15 PRIMARY KEY (id)
16 );
17
18INSERT INTO ban_backup SELECT
19 id, email
20 FROM ban;
21
22ALTER TABLE ban_backup ADD COLUMN list_id TEXT;
23
24-- REMOVALS from the mailinglist table.
25-- REM new_member_options
26-- REM send_reminders
27-- REM subscribe_policy
28-- REM unsubscribe_policy
29-- REM subscribe_auto_approval
30-- REM private_roster
31-- REM admin_member_chunksize
32
33CREATE TABLE mailinglist_backup (
34 id INTEGER NOT NULL,
35 list_name TEXT,
36 mail_host TEXT,
37 allow_list_posts BOOLEAN,
38 include_rfc2369_headers BOOLEAN,
39 created_at TIMESTAMP,
40 next_request_id INTEGER,
41 next_digest_number INTEGER,
42 digest_last_sent_at TIMESTAMP,
43 volume INTEGER,
44 last_post_at TIMESTAMP,
45 accept_these_nonmembers BLOB,
46 acceptable_aliases_id INTEGER,
47 admin_immed_notify BOOLEAN,
48 admin_notify_mchanges BOOLEAN,
49 administrivia BOOLEAN,
50 advertised BOOLEAN,
51 anonymous_list BOOLEAN,
52 autorespond_owner INTEGER,
53 autoresponse_owner_text TEXT,
54 autorespond_postings INTEGER,
55 autoresponse_postings_text TEXT,
56 autorespond_requests INTEGER,
57 autoresponse_request_text TEXT,
58 autoresponse_grace_period TEXT,
59 forward_unrecognized_bounces_to INTEGER,
60 process_bounces BOOLEAN,
61 bounce_info_stale_after TEXT,
62 bounce_matching_headers TEXT,
63 bounce_notify_owner_on_disable BOOLEAN,
64 bounce_notify_owner_on_removal BOOLEAN,
65 bounce_score_threshold INTEGER,
66 bounce_you_are_disabled_warnings INTEGER,
67 bounce_you_are_disabled_warnings_interval TEXT,
68 filter_action INTEGER,
69 filter_content BOOLEAN,
70 collapse_alternatives BOOLEAN,
71 convert_html_to_plaintext BOOLEAN,
72 default_member_action INTEGER,
73 default_nonmember_action INTEGER,
74 description TEXT,
75 digest_footer_uri TEXT,
76 digest_header_uri TEXT,
77 digest_is_default BOOLEAN,
78 digest_send_periodic BOOLEAN,
79 digest_size_threshold FLOAT,
80 digest_volume_frequency INTEGER,
81 digestable BOOLEAN,
82 discard_these_nonmembers BLOB,
83 emergency BOOLEAN,
84 encode_ascii_prefixes BOOLEAN,
85 first_strip_reply_to BOOLEAN,
86 footer_uri TEXT,
87 forward_auto_discards BOOLEAN,
88 gateway_to_mail BOOLEAN,
89 gateway_to_news BOOLEAN,
90 goodbye_message_uri TEXT,
91 header_matches BLOB,
92 header_uri TEXT,
93 hold_these_nonmembers BLOB,
94 info TEXT,
95 linked_newsgroup TEXT,
96 max_days_to_hold INTEGER,
97 max_message_size INTEGER,
98 max_num_recipients INTEGER,
99 member_moderation_notice TEXT,
100 mime_is_default_digest BOOLEAN,
101 moderator_password TEXT,
102 nondigestable BOOLEAN,
103 nonmember_rejection_notice TEXT,
104 obscure_addresses BOOLEAN,
105 owner_chain TEXT,
106 owner_pipeline TEXT,
107 personalize INTEGER,
108 post_id INTEGER,
109 posting_chain TEXT,
110 posting_pipeline TEXT,
111 preferred_language TEXT,
112 display_name TEXT,
113 reject_these_nonmembers BLOB,
114 reply_goes_to_list INTEGER,
115 reply_to_address TEXT,
116 require_explicit_destination BOOLEAN,
117 respond_to_post_requests BOOLEAN,
118 scrub_nondigest BOOLEAN,
119 send_goodbye_message BOOLEAN,
120 send_welcome_message BOOLEAN,
121 subject_prefix TEXT,
122 topics BLOB,
123 topics_bodylines_limit INTEGER,
124 topics_enabled BOOLEAN,
125 welcome_message_uri TEXT,
126 archive_policy INTEGER,
127 list_id TEXT,
128 nntp_prefix_subject_too INTEGER,
129 newsgroup_moderation INTEGER,
130 PRIMARY KEY (id)
131 );
132
133INSERT INTO mailinglist_backup SELECT
134 id,
135 list_name,
136 mail_host,
137 allow_list_posts,
138 include_rfc2369_headers,
139 created_at,
140 next_request_id,
141 next_digest_number,
142 digest_last_sent_at,
143 volume,
144 last_post_at,
145 accept_these_nonmembers,
146 acceptable_aliases_id,
147 admin_immed_notify,
148 admin_notify_mchanges,
149 administrivia,
150 advertised,
151 anonymous_list,
152 autorespond_owner,
153 autoresponse_owner_text,
154 autorespond_postings,
155 autoresponse_postings_text,
156 autorespond_requests,
157 autoresponse_request_text,
158 autoresponse_grace_period,
159 forward_unrecognized_bounces_to,
160 process_bounces,
161 bounce_info_stale_after,
162 bounce_matching_headers,
163 bounce_notify_owner_on_disable,
164 bounce_notify_owner_on_removal,
165 bounce_score_threshold,
166 bounce_you_are_disabled_warnings,
167 bounce_you_are_disabled_warnings_interval,
168 filter_action,
169 filter_content,
170 collapse_alternatives,
171 convert_html_to_plaintext,
172 default_member_action,
173 default_nonmember_action,
174 description,
175 digest_footer_uri,
176 digest_header_uri,
177 digest_is_default,
178 digest_send_periodic,
179 digest_size_threshold,
180 digest_volume_frequency,
181 digestable,
182 discard_these_nonmembers,
183 emergency,
184 encode_ascii_prefixes,
185 first_strip_reply_to,
186 footer_uri,
187 forward_auto_discards,
188 gateway_to_mail,
189 gateway_to_news,
190 goodbye_message_uri,
191 header_matches,
192 header_uri,
193 hold_these_nonmembers,
194 info,
195 linked_newsgroup,
196 max_days_to_hold,
197 max_message_size,
198 max_num_recipients,
199 member_moderation_notice,
200 mime_is_default_digest,
201 moderator_password,
202 nondigestable,
203 nonmember_rejection_notice,
204 obscure_addresses,
205 owner_chain,
206 owner_pipeline,
207 personalize,
208 post_id,
209 posting_chain,
210 posting_pipeline,
211 preferred_language,
212 display_name,
213 reject_these_nonmembers,
214 reply_goes_to_list,
215 reply_to_address,
216 require_explicit_destination,
217 respond_to_post_requests,
218 scrub_nondigest,
219 send_goodbye_message,
220 send_welcome_message,
221 subject_prefix,
222 topics,
223 topics_bodylines_limit,
224 topics_enabled,
225 welcome_message_uri,
226 archive_policy,
227 list_id,
228 nntp_prefix_subject_too,
229 newsgroup_moderation
230 FROM mailinglist;
2310
=== removed file 'src/mailman/database/schema/sqlite_20130406000000_01.sql'
--- src/mailman/database/schema/sqlite_20130406000000_01.sql 2013-11-26 02:26:15 +0000
+++ src/mailman/database/schema/sqlite_20130406000000_01.sql 1970-01-01 00:00:00 +0000
@@ -1,46 +0,0 @@
1-- This file contains the SQLite schema migration from
2-- 3.0b3 to 3.0b4
3--
4-- After 3.0b4 is released you may not edit this file.
5
6-- For SQLite3 migration strategy, see
7-- http://sqlite.org/faq.html#q11
8
9-- ADD listarchiver table.
10
11-- REMOVALs from the bounceevent table:
12-- REM list_name
13
14-- ADDs to the bounceevent table:
15-- ADD list_id
16
17-- ADDs to the mailinglist table:
18-- ADD archiver_id
19
20CREATE TABLE bounceevent_backup (
21 id INTEGER NOT NULL,
22 email TEXT,
23 'timestamp' TIMESTAMP,
24 message_id TEXT,
25 context INTEGER,
26 processed BOOLEAN,
27 PRIMARY KEY (id)
28 );
29
30INSERT INTO bounceevent_backup SELECT
31 id, email, "timestamp", message_id,
32 context, processed
33 FROM bounceevent;
34
35ALTER TABLE bounceevent_backup ADD COLUMN list_id TEXT;
36
37CREATE TABLE listarchiver (
38 id INTEGER NOT NULL,
39 mailing_list_id INTEGER NOT NULL,
40 name TEXT NOT NULL,
41 _is_enabled BOOLEAN,
42 PRIMARY KEY (id)
43 );
44
45CREATE INDEX ix_listarchiver_mailing_list_id
46 ON listarchiver(mailing_list_id);
470
=== modified file 'src/mailman/database/sqlite.py'
--- src/mailman/database/sqlite.py 2014-01-01 14:59:42 +0000
+++ src/mailman/database/sqlite.py 2014-10-11 02:14:38 +0000
@@ -22,63 +22,27 @@
22__metaclass__ = type22__metaclass__ = type
23__all__ = [23__all__ = [
24 'SQLiteDatabase',24 'SQLiteDatabase',
25 'make_temporary',
26 ]25 ]
2726
2827
29import os28import os
30import types
31import shutil
32import tempfile
3329
34from functools import partial30from mailman.database.base import SABaseDatabase
35from urlparse import urlparse31from urlparse import urlparse
3632
37from mailman.database.base import StormBaseDatabase
38from mailman.testing.helpers import configuration
39
4033
4134
4235
43class SQLiteDatabase(StormBaseDatabase):36class SQLiteDatabase(SABaseDatabase):
44 """Database class for SQLite."""37 """Database class for SQLite."""
4538
46 TAG = 'sqlite'
47
48 def _database_exists(self, store):
49 """See `BaseDatabase`."""
50 table_query = 'select tbl_name from sqlite_master;'
51 table_names = set(item[0] for item in
52 store.execute(table_query))
53 return 'version' in table_names
54
55 def _prepare(self, url):39 def _prepare(self, url):
56 parts = urlparse(url)40 parts = urlparse(url)
57 assert parts.scheme == 'sqlite', (41 assert parts.scheme == 'sqlite', (
58 'Database url mismatch (expected sqlite prefix): {0}'.format(url))42 'Database url mismatch (expected sqlite prefix): {0}'.format(url))
43 # Ensure that the SQLite database file has the proper permissions,
44 # since SQLite doesn't play nice with umask.
59 path = os.path.normpath(parts.path)45 path = os.path.normpath(parts.path)
60 fd = os.open(path, os.O_WRONLY | os.O_NONBLOCK | os.O_CREAT, 0666)46 fd = os.open(path, os.O_WRONLY | os.O_NONBLOCK | os.O_CREAT, 0o666)
61 # Ignore errors47 # Ignore errors
62 if fd > 0:48 if fd > 0:
63 os.close(fd)49 os.close(fd)
64
65
66
6750
68# Test suite adapter for ITemporaryDatabase.
69
70def _cleanup(self, tempdir):
71 shutil.rmtree(tempdir)
72
73
74def make_temporary(database):
75 """Adapts by monkey patching an existing SQLite IDatabase."""
76 tempdir = tempfile.mkdtemp()
77 url = 'sqlite:///' + os.path.join(tempdir, 'mailman.db')
78 with configuration('database', url=url):
79 database.initialize()
80 database._cleanup = types.MethodType(
81 partial(_cleanup, tempdir=tempdir),
82 database)
83 # bool column values in SQLite must be integers.
84 database.FALSE = 0
85 database.TRUE = 1
86 return database
8751
=== added directory 'src/mailman/database/tests'
=== removed directory 'src/mailman/database/tests'
=== added file 'src/mailman/database/tests/__init__.py'
=== removed file 'src/mailman/database/tests/__init__.py'
=== removed directory 'src/mailman/database/tests/data'
=== removed file 'src/mailman/database/tests/data/__init__.py'
=== removed file 'src/mailman/database/tests/data/mailman_01.db'
88Binary files src/mailman/database/tests/data/mailman_01.db 2012-04-20 21:32:27 +0000 and src/mailman/database/tests/data/mailman_01.db 1970-01-01 00:00:00 +0000 differ52Binary files src/mailman/database/tests/data/mailman_01.db 2012-04-20 21:32:27 +0000 and src/mailman/database/tests/data/mailman_01.db 1970-01-01 00:00:00 +0000 differ
=== removed file 'src/mailman/database/tests/data/migration_postgres_1.sql'
--- src/mailman/database/tests/data/migration_postgres_1.sql 2012-07-26 04:22:19 +0000
+++ src/mailman/database/tests/data/migration_postgres_1.sql 1970-01-01 00:00:00 +0000
@@ -1,133 +0,0 @@
1INSERT INTO "acceptablealias" VALUES(1,'foo@example.com',1);
2INSERT INTO "acceptablealias" VALUES(2,'bar@example.com',1);
3
4INSERT INTO "address" VALUES(
5 1,'anne@example.com',NULL,'Anne Person',
6 '2012-04-19 00:52:24.826432','2012-04-19 00:49:42.373769',1,2);
7INSERT INTO "address" VALUES(
8 2,'bart@example.com',NULL,'Bart Person',
9 '2012-04-19 00:53:25.878800','2012-04-19 00:49:52.882050',2,4);
10
11INSERT INTO "domain" VALUES(
12 1,'example.com','http://example.com',NULL,'postmaster@example.com');
13
14INSERT INTO "mailinglist" VALUES(
15 -- id,list_name,mail_host,include_list_post_header,include_rfc2369_headers
16 1,'test','example.com',True,True,
17 -- created_at,admin_member_chunksize,next_request_id,next_digest_number
18 '2012-04-19 00:46:13.173844',30,1,1,
19 -- digest_last_sent_at,volume,last_post_at,accept_these_nonmembers
20 NULL,1,NULL,E'\\x80025D71012E',
21 -- acceptable_aliases_id,admin_immed_notify,admin_notify_mchanges
22 NULL,True,False,
23 -- administrivia,advertised,anonymous_list,archive,archive_private
24 True,True,False,True,False,
25 -- archive_volume_frequency
26 1,
27 --autorespond_owner,autoresponse_owner_text
28 0,'',
29 -- autorespond_postings,autoresponse_postings_text
30 0,'',
31 -- autorespond_requests,authoresponse_requests_text
32 0,'',
33 -- autoresponse_grace_period
34 '90 days, 0:00:00',
35 -- forward_unrecognized_bounces_to,process_bounces
36 1,True,
37 -- bounce_info_stale_after,bounce_matching_headers
38 '7 days, 0:00:00','
39# Lines that *start* with a ''#'' are comments.
40to: friend@public.com
41message-id: relay.comanche.denmark.eu
42from: list@listme.com
43from: .*@uplinkpro.com
44',
45 -- bounce_notify_owner_on_disable,bounce_notify_owner_on_removal
46 True,True,
47 -- bounce_score_threshold,bounce_you_are_disabled_warnings
48 5,3,
49 -- bounce_you_are_disabled_warnings_interval
50 '7 days, 0:00:00',
51 -- filter_action,filter_content,collapse_alternatives
52 2,False,True,
53 -- convert_html_to_plaintext,default_member_action,default_nonmember_action
54 False,4,0,
55 -- description
56 '',
57 -- digest_footer_uri
58 'mailman:///$listname/$language/footer-generic.txt',
59 -- digest_header_uri
60 NULL,
61 -- digest_is_default,digest_send_periodic,digest_size_threshold
62 False,True,30.0,
63 -- digest_volume_frequency,digestable,discard_these_nonmembers
64 1,True,E'\\x80025D71012E',
65 -- emergency,encode_ascii_prefixes,first_strip_reply_to
66 False,False,False,
67 -- footer_uri
68 'mailman:///$listname/$language/footer-generic.txt',
69 -- forward_auto_discards,gateway_to_mail,gateway_to_news
70 True,False,FAlse,
71 -- generic_nonmember_action,goodby_message_uri
72 1,'',
73 -- header_matches,header_uri,hold_these_nonmembers,info,linked_newsgroup
74 E'\\x80025D71012E',NULL,E'\\x80025D71012E','','',
75 -- max_days_to_hold,max_message_size,max_num_recipients
76 0,40,10,
77 -- member_moderation_notice,mime_is_default_digest,moderator_password
78 '',False,NULL,
79 -- new_member_options,news_moderation,news_prefix_subject_too
80 256,0,True,
81 -- nntp_host,nondigestable,nonmember_rejection_notice,obscure_addresses
82 '',True,'',True,
83 -- owner_chain,owner_pipeline,personalize,post_id
84 'default-owner-chain','default-owner-pipeline',0,1,
85 -- posting_chain,posting_pipeline,preferred_language,private_roster
86 'default-posting-chain','default-posting-pipeline','en',True,
87 -- display_name,reject_these_nonmembers
88 'Test',E'\\x80025D71012E',
89 -- reply_goes_to_list,reply_to_address
90 0,'',
91 -- require_explicit_destination,respond_to_post_requests
92 True,True,
93 -- scrub_nondigest,send_goodbye_message,send_reminders,send_welcome_message
94 False,True,True,True,
95 -- subject_prefix,subscribe_auto_approval
96 '[Test] ',E'\\x80025D71012E',
97 -- subscribe_policy,topics,topics_bodylines_limit,topics_enabled
98 1,E'\\x80025D71012E',5,False,
99 -- unsubscribe_policy,welcome_message_uri
100 0,'mailman:///welcome.txt');
101
102INSERT INTO "member" VALUES(
103 1,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1',1,'test@example.com',4,NULL,5,1);
104INSERT INTO "member" VALUES(
105 2,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd',2,'test@example.com',3,NULL,6,1);
106INSERT INTO "member" VALUES(
107 3,'479be431-45f2-473d-bc3c-7eac614030ac',3,'test@example.com',3,NULL,7,2);
108INSERT INTO "member" VALUES(
109 4,'e2dc604c-d93a-4b91-b5a8-749e3caade36',1,'test@example.com',4,NULL,8,2);
110
111INSERT INTO "preferences" VALUES(1,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
112INSERT INTO "preferences" VALUES(2,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
113INSERT INTO "preferences" VALUES(3,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
114INSERT INTO "preferences" VALUES(4,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
115INSERT INTO "preferences" VALUES(5,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
116INSERT INTO "preferences" VALUES(6,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
117INSERT INTO "preferences" VALUES(7,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
118INSERT INTO "preferences" VALUES(8,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
119
120INSERT INTO "user" VALUES(
121 1,'Anne Person',NULL,'0adf3caa-6f26-46f8-a11d-5256c8148592',
122 '2012-04-19 00:49:42.370493',1,1);
123INSERT INTO "user" VALUES(
124 2,'Bart Person',NULL,'63f5d1a2-e533-4055-afe4-475dec3b1163',
125 '2012-04-19 00:49:52.868746',2,3);
126
127INSERT INTO "uid" VALUES(1,'8bf9a615-f23e-4980-b7d1-90ac0203c66f');
128INSERT INTO "uid" VALUES(2,'0adf3caa-6f26-46f8-a11d-5256c8148592');
129INSERT INTO "uid" VALUES(3,'63f5d1a2-e533-4055-afe4-475dec3b1163');
130INSERT INTO "uid" VALUES(4,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1');
131INSERT INTO "uid" VALUES(5,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd');
132INSERT INTO "uid" VALUES(6,'479be431-45f2-473d-bc3c-7eac614030ac');
133INSERT INTO "uid" VALUES(7,'e2dc604c-d93a-4b91-b5a8-749e3caade36');
1340
=== removed file 'src/mailman/database/tests/data/migration_sqlite_1.sql'
--- src/mailman/database/tests/data/migration_sqlite_1.sql 2012-07-26 04:22:19 +0000
+++ src/mailman/database/tests/data/migration_sqlite_1.sql 1970-01-01 00:00:00 +0000
@@ -1,133 +0,0 @@
1INSERT INTO "acceptablealias" VALUES(1,'foo@example.com',1);
2INSERT INTO "acceptablealias" VALUES(2,'bar@example.com',1);
3
4INSERT INTO "address" VALUES(
5 1,'anne@example.com',NULL,'Anne Person',
6 '2012-04-19 00:52:24.826432','2012-04-19 00:49:42.373769',1,2);
7INSERT INTO "address" VALUES(
8 2,'bart@example.com',NULL,'Bart Person',
9 '2012-04-19 00:53:25.878800','2012-04-19 00:49:52.882050',2,4);
10
11INSERT INTO "domain" VALUES(
12 1,'example.com','http://example.com',NULL,'postmaster@example.com');
13
14INSERT INTO "mailinglist" VALUES(
15 -- id,list_name,mail_host,include_list_post_header,include_rfc2369_headers
16 1,'test','example.com',1,1,
17 -- created_at,admin_member_chunksize,next_request_id,next_digest_number
18 '2012-04-19 00:46:13.173844',30,1,1,
19 -- digest_last_sent_at,volume,last_post_at,accept_these_nonmembers
20 NULL,1,NULL,X'80025D71012E',
21 -- acceptable_aliases_id,admin_immed_notify,admin_notify_mchanges
22 NULL,1,0,
23 -- administrivia,advertised,anonymous_list,archive,archive_private
24 1,1,0,1,0,
25 -- archive_volume_frequency
26 1,
27 --autorespond_owner,autoresponse_owner_text
28 0,'',
29 -- autorespond_postings,autoresponse_postings_text
30 0,'',
31 -- autorespond_requests,authoresponse_requests_text
32 0,'',
33 -- autoresponse_grace_period
34 '90 days, 0:00:00',
35 -- forward_unrecognized_bounces_to,process_bounces
36 1,1,
37 -- bounce_info_stale_after,bounce_matching_headers
38 '7 days, 0:00:00','
39# Lines that *start* with a ''#'' are comments.
40to: friend@public.com
41message-id: relay.comanche.denmark.eu
42from: list@listme.com
43from: .*@uplinkpro.com
44',
45 -- bounce_notify_owner_on_disable,bounce_notify_owner_on_removal
46 1,1,
47 -- bounce_score_threshold,bounce_you_are_disabled_warnings
48 5,3,
49 -- bounce_you_are_disabled_warnings_interval
50 '7 days, 0:00:00',
51 -- filter_action,filter_content,collapse_alternatives
52 2,0,1,
53 -- convert_html_to_plaintext,default_member_action,default_nonmember_action
54 0,4,0,
55 -- description
56 '',
57 -- digest_footer_uri
58 'mailman:///$listname/$language/footer-generic.txt',
59 -- digest_header_uri
60 NULL,
61 -- digest_is_default,digest_send_periodic,digest_size_threshold
62 0,1,30.0,
63 -- digest_volume_frequency,digestable,discard_these_nonmembers
64 1,1,X'80025D71012E',
65 -- emergency,encode_ascii_prefixes,first_strip_reply_to
66 0,0,0,
67 -- footer_uri
68 'mailman:///$listname/$language/footer-generic.txt',
69 -- forward_auto_discards,gateway_to_mail,gateway_to_news
70 1,0,0,
71 -- generic_nonmember_action,goodby_message_uri
72 1,'',
73 -- header_matches,header_uri,hold_these_nonmembers,info,linked_newsgroup
74 X'80025D71012E',NULL,X'80025D71012E','','',
75 -- max_days_to_hold,max_message_size,max_num_recipients
76 0,40,10,
77 -- member_moderation_notice,mime_is_default_digest,moderator_password
78 '',0,NULL,
79 -- new_member_options,news_moderation,news_prefix_subject_too
80 256,0,1,
81 -- nntp_host,nondigestable,nonmember_rejection_notice,obscure_addresses
82 '',1,'',1,
83 -- owner_chain,owner_pipeline,personalize,post_id
84 'default-owner-chain','default-owner-pipeline',0,1,
85 -- posting_chain,posting_pipeline,preferred_language,private_roster
86 'default-posting-chain','default-posting-pipeline','en',1,
87 -- display_name,reject_these_nonmembers
88 'Test',X'80025D71012E',
89 -- reply_goes_to_list,reply_to_address
90 0,'',
91 -- require_explicit_destination,respond_to_post_requests
92 1,1,
93 -- scrub_nondigest,send_goodbye_message,send_reminders,send_welcome_message
94 0,1,1,1,
95 -- subject_prefix,subscribe_auto_approval
96 '[Test] ',X'80025D71012E',
97 -- subscribe_policy,topics,topics_bodylines_limit,topics_enabled
98 1,X'80025D71012E',5,0,
99 -- unsubscribe_policy,welcome_message_uri
100 0,'mailman:///welcome.txt');
101
102INSERT INTO "member" VALUES(
103 1,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1',1,'test@example.com',4,NULL,5,1);
104INSERT INTO "member" VALUES(
105 2,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd',2,'test@example.com',3,NULL,6,1);
106INSERT INTO "member" VALUES(
107 3,'479be431-45f2-473d-bc3c-7eac614030ac',3,'test@example.com',3,NULL,7,2);
108INSERT INTO "member" VALUES(
109 4,'e2dc604c-d93a-4b91-b5a8-749e3caade36',1,'test@example.com',4,NULL,8,2);
110
111INSERT INTO "preferences" VALUES(1,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
112INSERT INTO "preferences" VALUES(2,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
113INSERT INTO "preferences" VALUES(3,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
114INSERT INTO "preferences" VALUES(4,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
115INSERT INTO "preferences" VALUES(5,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
116INSERT INTO "preferences" VALUES(6,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
117INSERT INTO "preferences" VALUES(7,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
118INSERT INTO "preferences" VALUES(8,NULL,NULL,NULL,NULL,NULL,NULL,NULL);
119
120INSERT INTO "user" VALUES(
121 1,'Anne Person',NULL,'0adf3caa-6f26-46f8-a11d-5256c8148592',
122 '2012-04-19 00:49:42.370493',1,1);
123INSERT INTO "user" VALUES(
124 2,'Bart Person',NULL,'63f5d1a2-e533-4055-afe4-475dec3b1163',
125 '2012-04-19 00:49:52.868746',2,3);
126
127INSERT INTO "uid" VALUES(1,'8bf9a615-f23e-4980-b7d1-90ac0203c66f');
128INSERT INTO "uid" VALUES(2,'0adf3caa-6f26-46f8-a11d-5256c8148592');
129INSERT INTO "uid" VALUES(3,'63f5d1a2-e533-4055-afe4-475dec3b1163');
130INSERT INTO "uid" VALUES(4,'d1243f4d-e604-4f6b-af52-98d0a7bce0f1');
131INSERT INTO "uid" VALUES(5,'dccc3851-fdfb-4afa-90cf-bdcbf80ad0fd');
132INSERT INTO "uid" VALUES(6,'479be431-45f2-473d-bc3c-7eac614030ac');
133INSERT INTO "uid" VALUES(7,'e2dc604c-d93a-4b91-b5a8-749e3caade36');
1340
=== added file 'src/mailman/database/tests/test_factory.py'
--- src/mailman/database/tests/test_factory.py 1970-01-01 00:00:00 +0000
+++ src/mailman/database/tests/test_factory.py 2014-10-11 02:14:38 +0000
@@ -0,0 +1,162 @@
1# Copyright (C) 2013-2014 by the Free Software Foundation, Inc.
2#
3# This file is part of GNU Mailman.
4#
5# GNU Mailman is free software: you can redistribute it and/or modify it under
6# the terms of the GNU General Public License as published by the Free
7# Software Foundation, either version 3 of the License, or (at your option)
8# any later version.
9#
10# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13# more details.
14#
15# You should have received a copy of the GNU General Public License along with
16# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
17
18"""Test database schema migrations"""
19
20from __future__ import absolute_import, print_function, unicode_literals
21
22__metaclass__ = type
23__all__ = [
24 ]
25
26
27import unittest
28import types
29
30import alembic.command
31from mock import Mock
32from sqlalchemy import MetaData, Table, Column, Integer, Unicode
33from sqlalchemy.schema import Index
34from sqlalchemy.exc import ProgrammingError, OperationalError
35
36from mailman.config import config
37from mailman.testing.layers import ConfigLayer
38from mailman.database.factory import SchemaManager, _reset
39from mailman.database.sqlite import SQLiteDatabase
40from mailman.database.alembic import alembic_cfg
41from mailman.database.model import Model
42
43
44
045
46class TestSchemaManager(unittest.TestCase):
47
48 layer = ConfigLayer
49
50 def setUp(self):
51 # Drop the existing database
52 Model.metadata.drop_all(config.db.engine)
53 md = MetaData()
54 md.reflect(bind=config.db.engine)
55 for tablename in ("alembic_version", "version"):
56 if tablename in md.tables:
57 md.tables[tablename].drop(config.db.engine)
58 self.schema_mgr = SchemaManager(config.db)
59
60 def tearDown(self):
61 self._drop_storm_database()
62 # Restore a virgin DB
63 Model.metadata.create_all(config.db.engine)
64
65
66 def _table_exists(self, tablename):
67 md = MetaData()
68 md.reflect(bind=config.db.engine)
69 return tablename in md.tables
70
71 def _create_storm_database(self, revision):
72 version_table = Table("version", Model.metadata,
73 Column("id", Integer, primary_key=True),
74 Column("component", Unicode),
75 Column("version", Unicode),
76 )
77 version_table.create(config.db.engine)
78 config.db.store.execute(version_table.insert().values(
79 component='schema', version=revision))
80 config.db.commit()
81 # Other Storm specific changes, those SQL statements hopefully work on
82 # all DB engines...
83 config.db.engine.execute(
84 "ALTER TABLE mailinglist ADD COLUMN acceptable_aliases_id INT")
85 Index("ix_user__user_id").drop(bind=config.db.engine)
86 # Don't pollute our main metadata object, create a new one
87 md = MetaData()
88 user_table = Model.metadata.tables["user"].tometadata(md)
89 Index("ix_user_user_id", user_table.c._user_id
90 ).create(bind=config.db.engine)
91 config.db.commit()
92
93 def _drop_storm_database(self):
94 """
95 Remove the leftovers from a Storm DB.
96 (you must issue a drop_all() afterwards)
97 """
98 if "version" in Model.metadata.tables:
99 version = Model.metadata.tables["version"]
100 version.drop(config.db.engine, checkfirst=True)
101 Model.metadata.remove(version)
102 try:
103 Index("ix_user_user_id").drop(bind=config.db.engine)
104 except (ProgrammingError, OperationalError) as e:
105 # non-existant (PGSQL raises a ProgrammingError, while SQLite
106 # raises an OperationalError)
107 pass
108 config.db.commit()
109
110
111 def test_current_db(self):
112 """The database is already at the latest version"""
113 alembic.command.stamp(alembic_cfg, "head")
114 self.schema_mgr._create = Mock()
115 self.schema_mgr._upgrade = Mock()
116 self.schema_mgr.setup_db()
117 self.assertFalse(self.schema_mgr._create.called)
118 self.assertFalse(self.schema_mgr._upgrade.called)
119
120 def test_initial(self):
121 """No existing database"""
122 self.assertFalse(self._table_exists("mailinglist"))
123 self.assertFalse(self._table_exists("alembic_version"))
124 self.schema_mgr._upgrade = Mock()
125 self.schema_mgr.setup_db()
126 self.assertFalse(self.schema_mgr._upgrade.called)
127 self.assertTrue(self._table_exists("mailinglist"))
128 self.assertTrue(self._table_exists("alembic_version"))
129
130 def test_storm(self):
131 """Existing Storm database"""
132 Model.metadata.create_all(config.db.engine)
133 self._create_storm_database(
134 self.schema_mgr.LAST_STORM_SCHEMA_VERSION)
135 self.schema_mgr._create = Mock()
136 self.schema_mgr.setup_db()
137 self.assertFalse(self.schema_mgr._create.called)
138 self.assertTrue(self._table_exists("mailinglist")
139 and self._table_exists("alembic_version")
140 and not self._table_exists("version"))
141
142 def test_old_storm(self):
143 """Existing Storm database in an old version"""
144 Model.metadata.create_all(config.db.engine)
145 self._create_storm_database("001")
146 self.schema_mgr._create = Mock()
147 self.assertRaises(RuntimeError, self.schema_mgr.setup_db)
148 self.assertFalse(self.schema_mgr._create.called)
149
150 def test_old_db(self):
151 """The database is in an old revision, must upgrade"""
152 alembic.command.stamp(alembic_cfg, "head")
153 md = MetaData()
154 md.reflect(bind=config.db.engine)
155 config.db.store.execute(md.tables["alembic_version"].delete())
156 config.db.store.execute(md.tables["alembic_version"].insert().values(
157 version_num="dummyrevision"))
158 config.db.commit()
159 self.schema_mgr._create = Mock()
160 self.schema_mgr._upgrade = Mock()
161 self.schema_mgr.setup_db()
162 self.assertFalse(self.schema_mgr._create.called)
163 self.assertTrue(self.schema_mgr._upgrade.called)
1164
=== removed file 'src/mailman/database/tests/test_migrations.py'
--- src/mailman/database/tests/test_migrations.py 2014-01-01 14:59:42 +0000
+++ src/mailman/database/tests/test_migrations.py 1970-01-01 00:00:00 +0000
@@ -1,506 +0,0 @@
1# Copyright (C) 2012-2014 by the Free Software Foundation, Inc.
2#
3# This file is part of GNU Mailman.
4#
5# GNU Mailman is free software: you can redistribute it and/or modify it under
6# the terms of the GNU General Public License as published by the Free
7# Software Foundation, either version 3 of the License, or (at your option)
8# any later version.
9#
10# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
11# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13# more details.
14#
15# You should have received a copy of the GNU General Public License along with
16# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
17
18"""Test schema migrations."""
19
20from __future__ import absolute_import, print_function, unicode_literals
21
22__metaclass__ = type
23__all__ = [
24 'TestMigration20120407MigratedData',
25 'TestMigration20120407Schema',
26 'TestMigration20120407UnchangedData',
27 'TestMigration20121015MigratedData',
28 'TestMigration20121015Schema',
29 'TestMigration20130406MigratedData',
30 'TestMigration20130406Schema',
31 ]
32
33
34import unittest
35
36from datetime import datetime
37from operator import attrgetter
38from pkg_resources import resource_string
39from sqlite3 import OperationalError
40from storm.exceptions import DatabaseError
41from zope.component import getUtility
42
43from mailman.interfaces.database import IDatabaseFactory
44from mailman.interfaces.domain import IDomainManager
45from mailman.interfaces.archiver import ArchivePolicy
46from mailman.interfaces.bounce import BounceContext
47from mailman.interfaces.listmanager import IListManager
48from mailman.interfaces.mailinglist import IAcceptableAliasSet
49from mailman.interfaces.nntp import NewsgroupModeration
50from mailman.interfaces.subscriptions import ISubscriptionService
51from mailman.model.bans import Ban
52from mailman.model.bounce import BounceEvent
53from mailman.testing.helpers import temporary_db
54from mailman.testing.layers import ConfigLayer
55
56
57
580
59class MigrationTestBase(unittest.TestCase):
60 """Test database migrations."""
61
62 layer = ConfigLayer
63
64 def setUp(self):
65 self._database = getUtility(IDatabaseFactory, 'temporary').create()
66
67 def tearDown(self):
68 self._database._cleanup()
69
70 def _table_missing_present(self, migrations, missing, present):
71 """The appropriate migrations leave some tables missing and present.
72
73 :param migrations: Sequence of migrations to load.
74 :param missing: Tables which should be missing.
75 :param present: Tables which should be present.
76 """
77 for migration in migrations:
78 self._database.load_migrations(migration)
79 self._database.store.commit()
80 for table in missing:
81 self.assertRaises(OperationalError,
82 self._database.store.execute,
83 'select * from {};'.format(table))
84 for table in present:
85 self._database.store.execute('select * from {};'.format(table))
86
87 def _missing_present(self, table, migrations, missing, present):
88 """The appropriate migrations leave columns missing and present.
89
90 :param table: The table to test columns from.
91 :param migrations: Sequence of migrations to load.
92 :param missing: Set of columns which should be missing after the
93 migrations are loaded.
94 :param present: Set of columns which should be present after the
95 migrations are loaded.
96 """
97 for migration in migrations:
98 self._database.load_migrations(migration)
99 self._database.store.commit()
100 for column in missing:
101 self.assertRaises(DatabaseError,
102 self._database.store.execute,
103 'select {0} from {1};'.format(column, table))
104 self._database.store.rollback()
105 for column in present:
106 # This should not produce an exception. Is there some better test
107 # that we can perform?
108 self._database.store.execute(
109 'select {0} from {1};'.format(column, table))
110
111
112
1131
114class TestMigration20120407Schema(MigrationTestBase):
115 """Test column migrations."""
116
117 def test_pre_upgrade_columns_migration(self):
118 # Test that before the migration, the old table columns are present
119 # and the new database columns are not.
120 self._missing_present('mailinglist',
121 ['20120406999999'],
122 # New columns are missing.
123 ('allow_list_posts',
124 'archive_policy',
125 'list_id',
126 'nntp_prefix_subject_too'),
127 # Old columns are present.
128 ('archive',
129 'archive_private',
130 'archive_volume_frequency',
131 'generic_nonmember_action',
132 'include_list_post_header',
133 'news_moderation',
134 'news_prefix_subject_too',
135 'nntp_host'))
136 self._missing_present('member',
137 ['20120406999999'],
138 ('list_id',),
139 ('mailing_list',))
140
141 def test_post_upgrade_columns_migration(self):
142 # Test that after the migration, the old table columns are missing
143 # and the new database columns are present.
144 self._missing_present('mailinglist',
145 ['20120406999999',
146 '20120407000000'],
147 # The old columns are missing.
148 ('archive',
149 'archive_private',
150 'archive_volume_frequency',
151 'generic_nonmember_action',
152 'include_list_post_header',
153 'news_moderation',
154 'news_prefix_subject_too',
155 'nntp_host'),
156 # The new columns are present.
157 ('allow_list_posts',
158 'archive_policy',
159 'list_id',
160 'nntp_prefix_subject_too'))
161 self._missing_present('member',
162 ['20120406999999',
163 '20120407000000'],
164 ('mailing_list',),
165 ('list_id',))
166
167
168
1692
170class TestMigration20120407UnchangedData(MigrationTestBase):
171 """Test non-migrated data."""
172
173 def setUp(self):
174 MigrationTestBase.setUp(self)
175 # Load all the migrations to just before the one we're testing.
176 self._database.load_migrations('20120406999999')
177 # Load the previous schema's sample data.
178 sample_data = resource_string(
179 'mailman.database.tests.data',
180 'migration_{0}_1.sql'.format(self._database.TAG))
181 self._database.load_sql(self._database.store, sample_data)
182 # XXX 2012-12-28: We have to load the last migration defined in the
183 # system, otherwise the ORM model will not match the SQL table
184 # definitions and we'll get OperationalErrors from SQLite.
185 self._database.load_migrations('20121015000000')
186
187 def test_migration_domains(self):
188 # Test that the domains table, which isn't touched, doesn't change.
189 with temporary_db(self._database):
190 # Check that the domains survived the migration. This table
191 # was not touched so it should be fine.
192 domains = list(getUtility(IDomainManager))
193 self.assertEqual(len(domains), 1)
194 self.assertEqual(domains[0].mail_host, 'example.com')
195
196 def test_migration_mailing_lists(self):
197 # Test that the mailing lists survive migration.
198 with temporary_db(self._database):
199 # There should be exactly one mailing list defined.
200 mlists = list(getUtility(IListManager).mailing_lists)
201 self.assertEqual(len(mlists), 1)
202 self.assertEqual(mlists[0].fqdn_listname, 'test@example.com')
203
204 def test_migration_acceptable_aliases(self):
205 # Test that the mailing list's acceptable aliases survive migration.
206 # This proves that foreign key references are migrated properly.
207 with temporary_db(self._database):
208 mlist = getUtility(IListManager).get('test@example.com')
209 aliases_set = IAcceptableAliasSet(mlist)
210 self.assertEqual(set(aliases_set.aliases),
211 set(['foo@example.com', 'bar@example.com']))
212
213 def test_migration_members(self):
214 # Test that the members of a mailing list all survive migration.
215 with temporary_db(self._database):
216 mlist = getUtility(IListManager).get('test@example.com')
217 # Test that all the members we expect are still there. Start with
218 # the two list delivery members.
219 addresses = set(address.email
220 for address in mlist.members.addresses)
221 self.assertEqual(addresses,
222 set(['anne@example.com', 'bart@example.com']))
223 # There is one owner.
224 owners = set(address.email for address in mlist.owners.addresses)
225 self.assertEqual(len(owners), 1)
226 self.assertEqual(owners.pop(), 'anne@example.com')
227 # There is one moderator.
228 moderators = set(address.email
229 for address in mlist.moderators.addresses)
230 self.assertEqual(len(moderators), 1)
231 self.assertEqual(moderators.pop(), 'bart@example.com')
232
233
234
2353
236class TestMigration20120407MigratedData(MigrationTestBase):
237 """Test affected migration data."""
238
239 def setUp(self):
240 MigrationTestBase.setUp(self)
241 # Load all the migrations to just before the one we're testing.
242 self._database.load_migrations('20120406999999')
243 # Load the previous schema's sample data.
244 sample_data = resource_string(
245 'mailman.database.tests.data',
246 'migration_{0}_1.sql'.format(self._database.TAG))
247 self._database.load_sql(self._database.store, sample_data)
248
249 def _upgrade(self):
250 # XXX 2012-12-28: We have to load the last migration defined in the
251 # system, otherwise the ORM model will not match the SQL table
252 # definitions and we'll get OperationalErrors from SQLite.
253 self._database.load_migrations('20121015000000')
254
255 def test_migration_archive_policy_never_0(self):
256 # Test that the new archive_policy value is updated correctly. In the
257 # case of old column archive=0, the archive_private column is
258 # ignored. This test sets it to 0 to ensure it's ignored.
259 self._database.store.execute(
260 'UPDATE mailinglist SET archive = {0}, archive_private = {0} '
261 'WHERE id = 1;'.format(self._database.FALSE))
262 # Complete the migration
263 self._upgrade()
264 with temporary_db(self._database):
265 mlist = getUtility(IListManager).get('test@example.com')
266 self.assertEqual(mlist.archive_policy, ArchivePolicy.never)
267
268 def test_migration_archive_policy_never_1(self):
269 # Test that the new archive_policy value is updated correctly. In the
270 # case of old column archive=0, the archive_private column is
271 # ignored. This test sets it to 1 to ensure it's ignored.
272 self._database.store.execute(
273 'UPDATE mailinglist SET archive = {0}, archive_private = {1} '
274 'WHERE id = 1;'.format(self._database.FALSE,
275 self._database.TRUE))
276 # Complete the migration
277 self._upgrade()
278 with temporary_db(self._database):
279 mlist = getUtility(IListManager).get('test@example.com')
280 self.assertEqual(mlist.archive_policy, ArchivePolicy.never)
281
282 def test_archive_policy_private(self):
283 # Test that the new archive_policy value is updated correctly for
284 # private archives.
285 self._database.store.execute(
286 'UPDATE mailinglist SET archive = {0}, archive_private = {0} '
287 'WHERE id = 1;'.format(self._database.TRUE))
288 # Complete the migration
289 self._upgrade()
290 with temporary_db(self._database):
291 mlist = getUtility(IListManager).get('test@example.com')
292 self.assertEqual(mlist.archive_policy, ArchivePolicy.private)
293
294 def test_archive_policy_public(self):
295 # Test that the new archive_policy value is updated correctly for
296 # public archives.
297 self._database.store.execute(
298 'UPDATE mailinglist SET archive = {1}, archive_private = {0} '
299 'WHERE id = 1;'.format(self._database.FALSE,
300 self._database.TRUE))
301 # Complete the migration
302 self._upgrade()
303 with temporary_db(self._database):
304 mlist = getUtility(IListManager).get('test@example.com')
305 self.assertEqual(mlist.archive_policy, ArchivePolicy.public)
306
307 def test_list_id(self):
308 # Test that the mailinglist table gets a list_id column.
309 self._upgrade()
310 with temporary_db(self._database):
311 mlist = getUtility(IListManager).get('test@example.com')
312 self.assertEqual(mlist.list_id, 'test.example.com')
313
314 def test_list_id_member(self):
315 # Test that the member table's mailing_list column becomes list_id.
316 self._upgrade()
317 with temporary_db(self._database):
318 service = getUtility(ISubscriptionService)
319 members = list(service.find_members(list_id='test.example.com'))
320 self.assertEqual(len(members), 4)
321
322 def test_news_moderation_none(self):
323 # Test that news_moderation becomes newsgroup_moderation.
324 self._database.store.execute(
325 'UPDATE mailinglist SET news_moderation = 0 '
326 'WHERE id = 1;')
327 self._upgrade()
328 with temporary_db(self._database):
329 mlist = getUtility(IListManager).get('test@example.com')
330 self.assertEqual(mlist.newsgroup_moderation,
331 NewsgroupModeration.none)
332
333 def test_news_moderation_open_moderated(self):
334 # Test that news_moderation becomes newsgroup_moderation.
335 self._database.store.execute(
336 'UPDATE mailinglist SET news_moderation = 1 '
337 'WHERE id = 1;')
338 self._upgrade()
339 with temporary_db(self._database):
340 mlist = getUtility(IListManager).get('test@example.com')
341 self.assertEqual(mlist.newsgroup_moderation,
342 NewsgroupModeration.open_moderated)
343
344 def test_news_moderation_moderated(self):
345 # Test that news_moderation becomes newsgroup_moderation.
346 self._database.store.execute(
347 'UPDATE mailinglist SET news_moderation = 2 '
348 'WHERE id = 1;')
349 self._upgrade()
350 with temporary_db(self._database):
351 mlist = getUtility(IListManager).get('test@example.com')
352 self.assertEqual(mlist.newsgroup_moderation,
353 NewsgroupModeration.moderated)
354
355 def test_nntp_prefix_subject_too_false(self):
356 # Test that news_prefix_subject_too becomes nntp_prefix_subject_too.
357 self._database.store.execute(
358 'UPDATE mailinglist SET news_prefix_subject_too = {0} '
359 'WHERE id = 1;'.format(self._database.FALSE))
360 self._upgrade()
361 with temporary_db(self._database):
362 mlist = getUtility(IListManager).get('test@example.com')
363 self.assertFalse(mlist.nntp_prefix_subject_too)
364
365 def test_nntp_prefix_subject_too_true(self):
366 # Test that news_prefix_subject_too becomes nntp_prefix_subject_too.
367 self._database.store.execute(
368 'UPDATE mailinglist SET news_prefix_subject_too = {0} '
369 'WHERE id = 1;'.format(self._database.TRUE))
370 self._upgrade()
371 with temporary_db(self._database):
372 mlist = getUtility(IListManager).get('test@example.com')
373 self.assertTrue(mlist.nntp_prefix_subject_too)
374
375 def test_allow_list_posts_false(self):
376 # Test that include_list_post_header -> allow_list_posts.
377 self._database.store.execute(
378 'UPDATE mailinglist SET include_list_post_header = {0} '
379 'WHERE id = 1;'.format(self._database.FALSE))
380 self._upgrade()
381 with temporary_db(self._database):
382 mlist = getUtility(IListManager).get('test@example.com')
383 self.assertFalse(mlist.allow_list_posts)
384
385 def test_allow_list_posts_true(self):
386 # Test that include_list_post_header -> allow_list_posts.
387 self._database.store.execute(
388 'UPDATE mailinglist SET include_list_post_header = {0} '
389 'WHERE id = 1;'.format(self._database.TRUE))
390 self._upgrade()
391 with temporary_db(self._database):
392 mlist = getUtility(IListManager).get('test@example.com')
393 self.assertTrue(mlist.allow_list_posts)
394
395
396
3974
398class TestMigration20121015Schema(MigrationTestBase):
399 """Test column migrations."""
400
401 def test_pre_upgrade_column_migrations(self):
402 self._missing_present('ban',
403 ['20121014999999'],
404 ('list_id',),
405 ('mailing_list',))
406 self._missing_present('mailinglist',
407 ['20121014999999'],
408 (),
409 ('new_member_options', 'send_reminders',
410 'subscribe_policy', 'unsubscribe_policy',
411 'subscribe_auto_approval', 'private_roster',
412 'admin_member_chunksize'),
413 )
414
415 def test_post_upgrade_column_migrations(self):
416 self._missing_present('ban',
417 ['20121014999999',
418 '20121015000000'],
419 ('mailing_list',),
420 ('list_id',))
421 self._missing_present('mailinglist',
422 ['20121014999999',
423 '20121015000000'],
424 ('new_member_options', 'send_reminders',
425 'subscribe_policy', 'unsubscribe_policy',
426 'subscribe_auto_approval', 'private_roster',
427 'admin_member_chunksize'),
428 ())
429
430
431
4325
433class TestMigration20121015MigratedData(MigrationTestBase):
434 """Test non-migrated data."""
435
436 def test_migration_bans(self):
437 # Load all the migrations to just before the one we're testing.
438 self._database.load_migrations('20121014999999')
439 # Insert a list-specific ban.
440 self._database.store.execute("""
441 INSERT INTO ban VALUES (
442 1, 'anne@example.com', 'test@example.com');
443 """)
444 # Insert a global ban.
445 self._database.store.execute("""
446 INSERT INTO ban VALUES (
447 2, 'bart@example.com', NULL);
448 """)
449 # Update to the current migration we're testing.
450 self._database.load_migrations('20121015000000')
451 # Now both the local and global bans should still be present.
452 bans = sorted(self._database.store.find(Ban),
453 key=attrgetter('email'))
454 self.assertEqual(bans[0].email, 'anne@example.com')
455 self.assertEqual(bans[0].list_id, 'test.example.com')
456 self.assertEqual(bans[1].email, 'bart@example.com')
457 self.assertEqual(bans[1].list_id, None)
458
459
460
4616
462class TestMigration20130406Schema(MigrationTestBase):
463 """Test column migrations."""
464
465 def test_pre_upgrade_column_migrations(self):
466 self._missing_present('bounceevent',
467 ['20130405999999'],
468 ('list_id',),
469 ('list_name',))
470
471 def test_post_upgrade_column_migrations(self):
472 self._missing_present('bounceevent',
473 ['20130405999999',
474 '20130406000000'],
475 ('list_name',),
476 ('list_id',))
477
478 def test_pre_listarchiver_table(self):
479 self._table_missing_present(['20130405999999'], ('listarchiver',), ())
480
481 def test_post_listarchiver_table(self):
482 self._table_missing_present(['20130405999999',
483 '20130406000000'],
484 (),
485 ('listarchiver',))
486
487
488
4897
490class TestMigration20130406MigratedData(MigrationTestBase):
491 """Test migrated data."""
492
493 def test_migration_bounceevent(self):
494 # Load all migrations to just before the one we're testing.
495 self._database.load_migrations('20130405999999')
496 # Insert a bounce event.
497 self._database.store.execute("""
498 INSERT INTO bounceevent VALUES (
499 1, 'test@example.com', 'anne@example.com',
500 '2013-04-06 21:12:00', '<abc@example.com>',
501 1, 0);
502 """)
503 # Update to the current migration we're testing
504 self._database.load_migrations('20130406000000')
505 # The bounce event should exist, but with a list-id instead of a fqdn
506 # list name.
507 events = list(self._database.store.find(BounceEvent))
508 self.assertEqual(len(events), 1)
509 self.assertEqual(events[0].list_id, 'test.example.com')
510 self.assertEqual(events[0].email, 'anne@example.com')
511 self.assertEqual(events[0].timestamp, datetime(2013, 4, 6, 21, 12))
512 self.assertEqual(events[0].message_id, '<abc@example.com>')
513 self.assertEqual(events[0].context, BounceContext.normal)
514 self.assertFalse(events[0].processed)
5158
=== modified file 'src/mailman/database/types.py'
--- src/mailman/database/types.py 2014-04-28 15:23:35 +0000
+++ src/mailman/database/types.py 2014-10-11 02:14:38 +0000
@@ -23,43 +23,70 @@
23__metaclass__ = type23__metaclass__ = type
24__all__ = [24__all__ = [
25 'Enum',25 'Enum',
26 'UUID',
26 ]27 ]
2728
29import uuid
2830
29from storm.properties import SimpleProperty31from sqlalchemy import Integer
30from storm.variables import Variable32from sqlalchemy.dialects import postgresql
33from sqlalchemy.types import TypeDecorator, CHAR
3134
3235
3336
3437
35class _EnumVariable(Variable):38class Enum(TypeDecorator):
36 """Storm variable for supporting enum types.39 """Handle Python 3.4 style enums.
3740
38 To use this, make the database column a INTEGER.41 Stores an integer-based Enum as an integer in the database, and
42 converts it on-the-fly.
39 """43 """
4044 impl = Integer
41 def __init__(self, *args, **kws):45
42 self._enum = kws.pop('enum')46 def __init__(self, enum, *args, **kw):
43 super(_EnumVariable, self).__init__(*args, **kws)47 self.enum = enum
4448 super(Enum, self).__init__(*args, **kw)
45 def parse_set(self, value, from_db):49
46 if value is None:50 def process_bind_param(self, value, dialect):
47 return None51 if value is None:
48 if not from_db:52 return None
49 return value
50 return self._enum(value)
51
52 def parse_get(self, value, to_db):
53 if value is None:
54 return None
55 if not to_db:
56 return value
57 return value.value53 return value.value
5854
5955 def process_result_value(self, value, dialect):
60class Enum(SimpleProperty):56 if value is None:
61 """Custom type for Storm supporting enums."""57 return None
6258 return self.enum(value)
63 variable_class = _EnumVariable59
6460
65 def __init__(self, enum=None):61
66 super(Enum, self).__init__(enum=enum)
6762
63class UUID(TypeDecorator):
64 """Platform-independent GUID type.
65
66 Uses Postgresql's UUID type, otherwise uses
67 CHAR(32), storing as stringified hex values.
68
69 """
70 impl = CHAR
71
72 def load_dialect_impl(self, dialect):
73 if dialect.name == 'postgresql':
74 return dialect.type_descriptor(postgresql.UUID())
75 else:
76 return dialect.type_descriptor(CHAR(32))
77
78 def process_bind_param(self, value, dialect):
79 if value is None:
80 return value
81 elif dialect.name == 'postgresql':
82 return str(value)
83 else:
84 if not isinstance(value, uuid.UUID):
85 return "%.32x" % uuid.UUID(value)
86 else:
87 # hexstring
88 return "%.32x" % value
89
90 def process_result_value(self, value, dialect):
91 if value is None:
92 return value
93 else:
94 return uuid.UUID(value)
6895
=== modified file 'src/mailman/interfaces/database.py'
--- src/mailman/interfaces/database.py 2014-01-01 14:59:42 +0000
+++ src/mailman/interfaces/database.py 2014-10-11 02:14:38 +0000
@@ -24,7 +24,6 @@
24 'DatabaseError',24 'DatabaseError',
25 'IDatabase',25 'IDatabase',
26 'IDatabaseFactory',26 'IDatabaseFactory',
27 'ITemporaryDatabase',
28 ]27 ]
2928
3029
@@ -61,12 +60,7 @@
61 """Abort the current transaction."""60 """Abort the current transaction."""
6261
63 store = Attribute(62 store = Attribute(
64 """The underlying Storm store on which you can do queries.""")
65
66
67
6863
69class ITemporaryDatabase(Interface):64 """The underlying database object on which you can do queries.""")
70 """Marker interface for test suite adaptation."""
7165
7266
7367
7468
7569
=== modified file 'src/mailman/interfaces/messages.py'
--- src/mailman/interfaces/messages.py 2014-04-28 15:23:35 +0000
+++ src/mailman/interfaces/messages.py 2014-10-11 02:14:38 +0000
@@ -83,7 +83,7 @@
8383
84 def get_message_by_hash(message_id_hash):84 def get_message_by_hash(message_id_hash):
85 """Return the message with the matching X-Message-ID-Hash.85 """Return the message with the matching X-Message-ID-Hash.
86 86
87 :param message_id_hash: The X-Message-ID-Hash header contents to87 :param message_id_hash: The X-Message-ID-Hash header contents to
88 search for.88 search for.
89 :returns: The message, or None if no matching message was found.89 :returns: The message, or None if no matching message was found.
9090
=== modified file 'src/mailman/model/address.py'
--- src/mailman/model/address.py 2014-04-15 03:00:41 +0000
+++ src/mailman/model/address.py 2014-10-11 02:14:38 +0000
@@ -26,7 +26,9 @@
2626
2727
28from email.utils import formataddr28from email.utils import formataddr
29from storm.locals import DateTime, Int, Reference, Unicode29from sqlalchemy import (
30 Column, DateTime, ForeignKey, Integer, Unicode)
31from sqlalchemy.orm import relationship, backref
30from zope.component import getUtility32from zope.component import getUtility
31from zope.event import notify33from zope.event import notify
32from zope.interface import implementer34from zope.interface import implementer
@@ -42,17 +44,20 @@
42class Address(Model):44class Address(Model):
43 """See `IAddress`."""45 """See `IAddress`."""
4446
45 id = Int(primary=True)47 __tablename__ = 'address'
46 email = Unicode()48
47 _original = Unicode()49 id = Column(Integer, primary_key=True)
48 display_name = Unicode()50 email = Column(Unicode)
49 _verified_on = DateTime(name='verified_on')51 _original = Column(Unicode)
50 registered_on = DateTime()52 display_name = Column(Unicode)
5153 _verified_on = Column('verified_on', DateTime)
52 user_id = Int()54 registered_on = Column(DateTime)
53 user = Reference(user_id, 'User.id')55
54 preferences_id = Int()56 user_id = Column(Integer, ForeignKey('user.id'), index=True)
55 preferences = Reference(preferences_id, 'Preferences.id')57
58 preferences_id = Column(Integer, ForeignKey('preferences.id'), index=True)
59 preferences = relationship(
60 'Preferences', backref=backref('address', uselist=False))
5661
57 def __init__(self, email, display_name):62 def __init__(self, email, display_name):
58 super(Address, self).__init__()63 super(Address, self).__init__()
5964
=== modified file 'src/mailman/model/autorespond.py'
--- src/mailman/model/autorespond.py 2014-01-01 14:59:42 +0000
+++ src/mailman/model/autorespond.py 2014-10-11 02:14:38 +0000
@@ -26,7 +26,9 @@
26 ]26 ]
2727
2828
29from storm.locals import And, Date, Desc, Int, Reference29from sqlalchemy import Column, Date, ForeignKey, Integer
30from sqlalchemy import desc
31from sqlalchemy.orm import relationship
30from zope.interface import implementer32from zope.interface import implementer
3133
32from mailman.database.model import Model34from mailman.database.model import Model
@@ -42,16 +44,18 @@
42class AutoResponseRecord(Model):44class AutoResponseRecord(Model):
43 """See `IAutoResponseRecord`."""45 """See `IAutoResponseRecord`."""
4446
45 id = Int(primary=True)47 __tablename__ = 'autoresponserecord'
4648
47 address_id = Int()49 id = Column(Integer, primary_key=True)
48 address = Reference(address_id, 'Address.id')50
4951 address_id = Column(Integer, ForeignKey('address.id'), index=True)
50 mailing_list_id = Int()52 address = relationship('Address')
51 mailing_list = Reference(mailing_list_id, 'MailingList.id')53
5254 mailing_list_id = Column(Integer, ForeignKey('mailinglist.id'), index=True)
53 response_type = Enum(Response)55 mailing_list = relationship('MailingList')
54 date_sent = Date()56
57 response_type = Column(Enum(Response))
58 date_sent = Column(Date)
5559
56 def __init__(self, mailing_list, address, response_type):60 def __init__(self, mailing_list, address, response_type):
57 self.mailing_list = mailing_list61 self.mailing_list = mailing_list
@@ -71,12 +75,11 @@
71 @dbconnection75 @dbconnection
72 def todays_count(self, store, address, response_type):76 def todays_count(self, store, address, response_type):
73 """See `IAutoResponseSet`."""77 """See `IAutoResponseSet`."""
74 return store.find(78 return store.query(AutoResponseRecord).filter_by(
75 AutoResponseRecord,79 address=address,
76 And(AutoResponseRecord.address == address,80 mailing_list=self._mailing_list,
77 AutoResponseRecord.mailing_list == self._mailing_list,81 response_type=response_type,
78 AutoResponseRecord.response_type == response_type,82 date_sent=today()).count()
79 AutoResponseRecord.date_sent == today())).count()
8083
81 @dbconnection84 @dbconnection
82 def response_sent(self, store, address, response_type):85 def response_sent(self, store, address, response_type):
@@ -88,10 +91,9 @@
88 @dbconnection91 @dbconnection
89 def last_response(self, store, address, response_type):92 def last_response(self, store, address, response_type):
90 """See `IAutoResponseSet`."""93 """See `IAutoResponseSet`."""
91 results = store.find(94 results = store.query(AutoResponseRecord).filter_by(
92 AutoResponseRecord,95 address=address,
93 And(AutoResponseRecord.address == address,96 mailing_list=self._mailing_list,
94 AutoResponseRecord.mailing_list == self._mailing_list,97 response_type=response_type
95 AutoResponseRecord.response_type == response_type)98 ).order_by(desc(AutoResponseRecord.date_sent))
96 ).order_by(Desc(AutoResponseRecord.date_sent))
97 return (None if results.count() == 0 else results.first())99 return (None if results.count() == 0 else results.first())
98100
=== modified file 'src/mailman/model/bans.py'
--- src/mailman/model/bans.py 2014-01-01 14:59:42 +0000
+++ src/mailman/model/bans.py 2014-10-11 02:14:38 +0000
@@ -27,7 +27,7 @@
2727
28import re28import re
2929
30from storm.locals import Int, Unicode30from sqlalchemy import Column, Integer, Unicode
31from zope.interface import implementer31from zope.interface import implementer
3232
33from mailman.database.model import Model33from mailman.database.model import Model
@@ -40,9 +40,11 @@
40class Ban(Model):40class Ban(Model):
41 """See `IBan`."""41 """See `IBan`."""
4242
43 id = Int(primary=True)43 __tablename__ = 'ban'
44 email = Unicode()44
45 list_id = Unicode()45 id = Column(Integer, primary_key=True)
46 email = Column(Unicode)
47 list_id = Column(Unicode)
4648
47 def __init__(self, email, list_id):49 def __init__(self, email, list_id):
48 super(Ban, self).__init__()50 super(Ban, self).__init__()
@@ -62,7 +64,7 @@
62 @dbconnection64 @dbconnection
63 def ban(self, store, email):65 def ban(self, store, email):
64 """See `IBanManager`."""66 """See `IBanManager`."""
65 bans = store.find(Ban, email=email, list_id=self._list_id)67 bans = store.query(Ban).filter_by(email=email, list_id=self._list_id)
66 if bans.count() == 0:68 if bans.count() == 0:
67 ban = Ban(email, self._list_id)69 ban = Ban(email, self._list_id)
68 store.add(ban)70 store.add(ban)
@@ -70,9 +72,10 @@
70 @dbconnection72 @dbconnection
71 def unban(self, store, email):73 def unban(self, store, email):
72 """See `IBanManager`."""74 """See `IBanManager`."""
73 ban = store.find(Ban, email=email, list_id=self._list_id).one()75 ban = store.query(Ban).filter_by(
76 email=email, list_id=self._list_id).first()
74 if ban is not None:77 if ban is not None:
75 store.remove(ban)78 store.delete(ban)
7679
77 @dbconnection80 @dbconnection
78 def is_banned(self, store, email):81 def is_banned(self, store, email):
@@ -81,32 +84,32 @@
81 if list_id is None:84 if list_id is None:
82 # The client is asking for global bans. Look up bans on the85 # The client is asking for global bans. Look up bans on the
83 # specific email address first.86 # specific email address first.
84 bans = store.find(Ban, email=email, list_id=None)87 bans = store.query(Ban).filter_by(email=email, list_id=None)
85 if bans.count() > 0:88 if bans.count() > 0:
86 return True89 return True
87 # And now look for global pattern bans.90 # And now look for global pattern bans.
88 bans = store.find(Ban, list_id=None)91 bans = store.query(Ban).filter_by(list_id=None)
89 for ban in bans:92 for ban in bans:
90 if (ban.email.startswith('^') and93 if (ban.email.startswith('^') and
91 re.match(ban.email, email, re.IGNORECASE) is not None):94 re.match(ban.email, email, re.IGNORECASE) is not None):
92 return True95 return True
93 else:96 else:
94 # This is a list-specific ban.97 # This is a list-specific ban.
95 bans = store.find(Ban, email=email, list_id=list_id)98 bans = store.query(Ban).filter_by(email=email, list_id=list_id)
96 if bans.count() > 0:99 if bans.count() > 0:
97 return True100 return True
98 # Try global bans next.101 # Try global bans next.
99 bans = store.find(Ban, email=email, list_id=None)102 bans = store.query(Ban).filter_by(email=email, list_id=None)
100 if bans.count() > 0:103 if bans.count() > 0:
101 return True104 return True
102 # Now try specific mailing list bans, but with a pattern.105 # Now try specific mailing list bans, but with a pattern.
103 bans = store.find(Ban, list_id=list_id)106 bans = store.query(Ban).filter_by(list_id=list_id)
104 for ban in bans:107 for ban in bans:
105 if (ban.email.startswith('^') and108 if (ban.email.startswith('^') and
106 re.match(ban.email, email, re.IGNORECASE) is not None):109 re.match(ban.email, email, re.IGNORECASE) is not None):
107 return True110 return True
108 # And now try global pattern bans.111 # And now try global pattern bans.
109 bans = store.find(Ban, list_id=None)112 bans = store.query(Ban).filter_by(list_id=None)
110 for ban in bans:113 for ban in bans:
111 if (ban.email.startswith('^') and114 if (ban.email.startswith('^') and
112 re.match(ban.email, email, re.IGNORECASE) is not None):115 re.match(ban.email, email, re.IGNORECASE) is not None):
113116
=== modified file 'src/mailman/model/bounce.py'
--- src/mailman/model/bounce.py 2014-01-01 14:59:42 +0000
+++ src/mailman/model/bounce.py 2014-10-11 02:14:38 +0000
@@ -26,7 +26,8 @@
26 ]26 ]
2727
2828
29from storm.locals import Bool, Int, DateTime, Unicode29
30from sqlalchemy import Boolean, Column, DateTime, Integer, Unicode
30from zope.interface import implementer31from zope.interface import implementer
3132
32from mailman.database.model import Model33from mailman.database.model import Model
@@ -42,13 +43,15 @@
42class BounceEvent(Model):43class BounceEvent(Model):
43 """See `IBounceEvent`."""44 """See `IBounceEvent`."""
4445
45 id = Int(primary=True)46 __tablename__ = 'bounceevent'
46 list_id = Unicode()47
47 email = Unicode()48 id = Column(Integer, primary_key=True)
48 timestamp = DateTime()49 list_id = Column(Unicode)
49 message_id = Unicode()50 email = Column(Unicode)
50 context = Enum(BounceContext)51 timestamp = Column(DateTime)
51 processed = Bool()52 message_id = Column(Unicode)
53 context = Column(Enum(BounceContext))
54 processed = Column(Boolean)
5255
53 def __init__(self, list_id, email, msg, context=None):56 def __init__(self, list_id, email, msg, context=None):
54 self.list_id = list_id57 self.list_id = list_id
@@ -75,12 +78,12 @@
75 @dbconnection78 @dbconnection
76 def events(self, store):79 def events(self, store):
77 """See `IBounceProcessor`."""80 """See `IBounceProcessor`."""
78 for event in store.find(BounceEvent):81 for event in store.query(BounceEvent).all():
79 yield event82 yield event
8083
81 @property84 @property
82 @dbconnection85 @dbconnection
83 def unprocessed(self, store):86 def unprocessed(self, store):
84 """See `IBounceProcessor`."""87 """See `IBounceProcessor`."""
85 for event in store.find(BounceEvent, BounceEvent.processed == False):88 for event in store.query(BounceEvent).filter_by(processed=False):
86 yield event89 yield event
8790
=== modified file 'src/mailman/model/digests.py'
--- src/mailman/model/digests.py 2014-01-01 14:59:42 +0000
+++ src/mailman/model/digests.py 2014-10-11 02:14:38 +0000
@@ -25,7 +25,8 @@
25 ]25 ]
2626
2727
28from storm.locals import Int, Reference28from sqlalchemy import Column, Integer, ForeignKey
29from sqlalchemy.orm import relationship
29from zope.interface import implementer30from zope.interface import implementer
3031
31from mailman.database.model import Model32from mailman.database.model import Model
@@ -39,15 +40,17 @@
39class OneLastDigest(Model):40class OneLastDigest(Model):
40 """See `IOneLastDigest`."""41 """See `IOneLastDigest`."""
4142
42 id = Int(primary=True)43 __tablename__ = 'onelastdigest'
4344
44 mailing_list_id = Int()45 id = Column(Integer, primary_key=True)
45 mailing_list = Reference(mailing_list_id, 'MailingList.id')46
4647 mailing_list_id = Column(Integer, ForeignKey('mailinglist.id'))
47 address_id = Int()48 mailing_list = relationship('MailingList')
48 address = Reference(address_id, 'Address.id')49
4950 address_id = Column(Integer, ForeignKey('address.id'))
50 delivery_mode = Enum(DeliveryMode)51 address = relationship('Address')
52
53 delivery_mode = Column(Enum(DeliveryMode))
5154
52 def __init__(self, mailing_list, address, delivery_mode):55 def __init__(self, mailing_list, address, delivery_mode):
53 self.mailing_list = mailing_list56 self.mailing_list = mailing_list
5457
=== modified file 'src/mailman/model/docs/messagestore.rst'
--- src/mailman/model/docs/messagestore.rst 2014-04-28 15:23:35 +0000
+++ src/mailman/model/docs/messagestore.rst 2014-10-11 02:14:38 +0000
@@ -28,8 +28,9 @@
28However, if the message has a ``Message-ID`` header, it can be stored.28However, if the message has a ``Message-ID`` header, it can be stored.
2929
30 >>> msg['Message-ID'] = '<87myycy5eh.fsf@uwakimon.sk.tsukuba.ac.jp>'30 >>> msg['Message-ID'] = '<87myycy5eh.fsf@uwakimon.sk.tsukuba.ac.jp>'
31 >>> message_store.add(msg)31 >>> x_message_id_hash = message_store.add(msg)
32 'AGDWSNXXKCWEILKKNYTBOHRDQGOX3Y35'32 >>> print(x_message_id_hash)
33 AGDWSNXXKCWEILKKNYTBOHRDQGOX3Y35
33 >>> print(msg.as_string())34 >>> print(msg.as_string())
34 Subject: An important message35 Subject: An important message
35 Message-ID: <87myycy5eh.fsf@uwakimon.sk.tsukuba.ac.jp>36 Message-ID: <87myycy5eh.fsf@uwakimon.sk.tsukuba.ac.jp>
3637
=== modified file 'src/mailman/model/domain.py'
--- src/mailman/model/domain.py 2014-01-01 14:59:42 +0000
+++ src/mailman/model/domain.py 2014-10-11 02:14:38 +0000
@@ -26,8 +26,8 @@
26 ]26 ]
2727
2828
29from sqlalchemy import Column, Integer, Unicode
29from urlparse import urljoin, urlparse30from urlparse import urljoin, urlparse
30from storm.locals import Int, Unicode
31from zope.event import notify31from zope.event import notify
32from zope.interface import implementer32from zope.interface import implementer
3333
@@ -44,12 +44,14 @@
44class Domain(Model):44class Domain(Model):
45 """Domains."""45 """Domains."""
4646
47 id = Int(primary=True)47 __tablename__ = 'domain'
4848
49 mail_host = Unicode()49 id = Column(Integer, primary_key=True)
50 base_url = Unicode()50
51 description = Unicode()51 mail_host = Column(Unicode)
52 contact_address = Unicode()52 base_url = Column(Unicode)
53 description = Column(Unicode)
54 contact_address = Column(Unicode)
5355
54 def __init__(self, mail_host,56 def __init__(self, mail_host,
55 description=None,57 description=None,
@@ -92,8 +94,7 @@
92 @dbconnection94 @dbconnection
93 def mailing_lists(self, store):95 def mailing_lists(self, store):
94 """See `IDomain`."""96 """See `IDomain`."""
95 mailing_lists = store.find(97 mailing_lists = store.query(MailingList).filter(
96 MailingList,
97 MailingList.mail_host == self.mail_host)98 MailingList.mail_host == self.mail_host)
98 for mlist in mailing_lists:99 for mlist in mailing_lists:
99 yield mlist100 yield mlist
@@ -140,14 +141,14 @@
140 def remove(self, store, mail_host):141 def remove(self, store, mail_host):
141 domain = self[mail_host]142 domain = self[mail_host]
142 notify(DomainDeletingEvent(domain))143 notify(DomainDeletingEvent(domain))
143 store.remove(domain)144 store.delete(domain)
144 notify(DomainDeletedEvent(mail_host))145 notify(DomainDeletedEvent(mail_host))
145 return domain146 return domain
146147
147 @dbconnection148 @dbconnection
148 def get(self, store, mail_host, default=None):149 def get(self, store, mail_host, default=None):
149 """See `IDomainManager`."""150 """See `IDomainManager`."""
150 domains = store.find(Domain, mail_host=mail_host)151 domains = store.query(Domain).filter_by(mail_host=mail_host)
151 if domains.count() < 1:152 if domains.count() < 1:
152 return default153 return default
153 assert domains.count() == 1, (154 assert domains.count() == 1, (
@@ -164,15 +165,15 @@
164165
165 @dbconnection166 @dbconnection
166 def __len__(self, store):167 def __len__(self, store):
167 return store.find(Domain).count()168 return store.query(Domain).count()
168169
169 @dbconnection170 @dbconnection
170 def __iter__(self, store):171 def __iter__(self, store):
171 """See `IDomainManager`."""172 """See `IDomainManager`."""
172 for domain in store.find(Domain):173 for domain in store.query(Domain).all():
173 yield domain174 yield domain
174175
175 @dbconnection176 @dbconnection
176 def __contains__(self, store, mail_host):177 def __contains__(self, store, mail_host):
177 """See `IDomainManager`."""178 """See `IDomainManager`."""
178 return store.find(Domain, mail_host=mail_host).count() > 0179 return store.query(Domain).filter_by(mail_host=mail_host).count() > 0
179180
=== modified file 'src/mailman/model/language.py'
--- src/mailman/model/language.py 2014-01-01 14:59:42 +0000
+++ src/mailman/model/language.py 2014-10-11 02:14:38 +0000
@@ -25,11 +25,11 @@
25 ]25 ]
2626
2727
28from storm.locals import Int, Unicode28from sqlalchemy import Column, Integer, Unicode
29from zope.interface import implementer29from zope.interface import implementer
3030
31from mailman.database import Model31from mailman.database.model import Model
32from mailman.interfaces import ILanguage32from mailman.interfaces.languages import ILanguage
3333
3434
3535
3636
@@ -37,5 +37,7 @@
37class Language(Model):37class Language(Model):
38 """See `ILanguage`."""38 """See `ILanguage`."""
3939
40 id = Int(primary=True)40 __tablename__ = 'language'
41 code = Unicode()41
42 id = Column(Integer, primary_key=True)
43 code = Column(Unicode)
4244
=== modified file 'src/mailman/model/listmanager.py'
--- src/mailman/model/listmanager.py 2014-04-14 16:14:13 +0000
+++ src/mailman/model/listmanager.py 2014-10-11 02:14:38 +0000
@@ -52,9 +52,7 @@
52 raise InvalidEmailAddressError(fqdn_listname)52 raise InvalidEmailAddressError(fqdn_listname)
53 list_id = '{0}.{1}'.format(listname, hostname)53 list_id = '{0}.{1}'.format(listname, hostname)
54 notify(ListCreatingEvent(fqdn_listname))54 notify(ListCreatingEvent(fqdn_listname))
55 mlist = store.find(55 mlist = store.query(MailingList).filter_by(_list_id=list_id).first()
56 MailingList,
57 MailingList._list_id == list_id).one()
58 if mlist:56 if mlist:
59 raise ListAlreadyExistsError(fqdn_listname)57 raise ListAlreadyExistsError(fqdn_listname)
60 mlist = MailingList(fqdn_listname)58 mlist = MailingList(fqdn_listname)
@@ -68,40 +66,40 @@
68 """See `IListManager`."""66 """See `IListManager`."""
69 listname, at, hostname = fqdn_listname.partition('@')67 listname, at, hostname = fqdn_listname.partition('@')
70 list_id = '{0}.{1}'.format(listname, hostname)68 list_id = '{0}.{1}'.format(listname, hostname)
71 return store.find(MailingList, MailingList._list_id == list_id).one()69 return store.query(MailingList).filter_by(_list_id=list_id).first()
7270
73 @dbconnection71 @dbconnection
74 def get_by_list_id(self, store, list_id):72 def get_by_list_id(self, store, list_id):
75 """See `IListManager`."""73 """See `IListManager`."""
76 return store.find(MailingList, MailingList._list_id == list_id).one()74 return store.query(MailingList).filter_by(_list_id=list_id).first()
7775
78 @dbconnection76 @dbconnection
79 def delete(self, store, mlist):77 def delete(self, store, mlist):
80 """See `IListManager`."""78 """See `IListManager`."""
81 fqdn_listname = mlist.fqdn_listname79 fqdn_listname = mlist.fqdn_listname
82 notify(ListDeletingEvent(mlist))80 notify(ListDeletingEvent(mlist))
83 store.find(ContentFilter, ContentFilter.mailing_list == mlist).remove()81 store.query(ContentFilter).filter_by(mailing_list=mlist).delete()
84 store.remove(mlist)82 store.delete(mlist)
85 notify(ListDeletedEvent(fqdn_listname))83 notify(ListDeletedEvent(fqdn_listname))
8684
87 @property85 @property
88 @dbconnection86 @dbconnection
89 def mailing_lists(self, store):87 def mailing_lists(self, store):
90 """See `IListManager`."""88 """See `IListManager`."""
91 for mlist in store.find(MailingList):89 for mlist in store.query(MailingList).all():
92 yield mlist90 yield mlist
9391
94 @dbconnection92 @dbconnection
95 def __iter__(self, store):93 def __iter__(self, store):
96 """See `IListManager`."""94 """See `IListManager`."""
97 for mlist in store.find(MailingList):95 for mlist in store.query(MailingList).all():
98 yield mlist96 yield mlist
9997
100 @property98 @property
101 @dbconnection99 @dbconnection
102 def names(self, store):100 def names(self, store):
103 """See `IListManager`."""101 """See `IListManager`."""
104 result_set = store.find(MailingList)102 result_set = store.query(MailingList)
105 for mail_host, list_name in result_set.values(MailingList.mail_host,103 for mail_host, list_name in result_set.values(MailingList.mail_host,
106 MailingList.list_name):104 MailingList.list_name):
107 yield '{0}@{1}'.format(list_name, mail_host)105 yield '{0}@{1}'.format(list_name, mail_host)
@@ -110,15 +108,16 @@
110 @dbconnection108 @dbconnection
111 def list_ids(self, store):109 def list_ids(self, store):
112 """See `IListManager`."""110 """See `IListManager`."""
113 result_set = store.find(MailingList)111 result_set = store.query(MailingList)
114 for list_id in result_set.values(MailingList._list_id):112 for list_id in result_set.values(MailingList._list_id):
115 yield list_id113 assert isinstance(list_id, tuple) and len(list_id) == 1
114 yield list_id[0]
116115
117 @property116 @property
118 @dbconnection117 @dbconnection
119 def name_components(self, store):118 def name_components(self, store):
120 """See `IListManager`."""119 """See `IListManager`."""
121 result_set = store.find(MailingList)120 result_set = store.query(MailingList)
122 for mail_host, list_name in result_set.values(MailingList.mail_host,121 for mail_host, list_name in result_set.values(MailingList.mail_host,
123 MailingList.list_name):122 MailingList.list_name):
124 yield list_name, mail_host123 yield list_name, mail_host
125124
=== modified file 'src/mailman/model/mailinglist.py'
--- src/mailman/model/mailinglist.py 2014-04-14 16:14:13 +0000
+++ src/mailman/model/mailinglist.py 2014-10-11 02:14:38 +0000
@@ -27,9 +27,11 @@
2727
28import os28import os
2929
30from storm.locals import (30from sqlalchemy import (
31 And, Bool, DateTime, Float, Int, Pickle, RawStr, Reference, Store,31 Boolean, Column, DateTime, Float, ForeignKey, Integer, Interval,
32 TimeDelta, Unicode)32 LargeBinary, PickleType, Unicode)
33from sqlalchemy.event import listen
34from sqlalchemy.orm import relationship
33from urlparse import urljoin35from urlparse import urljoin
34from zope.component import getUtility36from zope.component import getUtility
35from zope.event import notify37from zope.event import notify
@@ -37,6 +39,7 @@
3739
38from mailman.config import config40from mailman.config import config
39from mailman.database.model import Model41from mailman.database.model import Model
42from mailman.database.transaction import dbconnection
40from mailman.database.types import Enum43from mailman.database.types import Enum
41from mailman.interfaces.action import Action, FilterAction44from mailman.interfaces.action import Action, FilterAction
42from mailman.interfaces.address import IAddress45from mailman.interfaces.address import IAddress
@@ -73,121 +76,121 @@
73class MailingList(Model):76class MailingList(Model):
74 """See `IMailingList`."""77 """See `IMailingList`."""
7578
76 id = Int(primary=True)79 __tablename__ = 'mailinglist'
80
81 id = Column(Integer, primary_key=True)
7782
78 # XXX denotes attributes that should be part of the public interface but83 # XXX denotes attributes that should be part of the public interface but
79 # are currently missing.84 # are currently missing.
8085
81 # List identity86 # List identity
82 list_name = Unicode()87 list_name = Column(Unicode)
83 mail_host = Unicode()88 mail_host = Column(Unicode)
84 _list_id = Unicode(name='list_id')89 _list_id = Column('list_id', Unicode)
85 allow_list_posts = Bool()90 allow_list_posts = Column(Boolean)
86 include_rfc2369_headers = Bool()91 include_rfc2369_headers = Column(Boolean)
87 advertised = Bool()92 advertised = Column(Boolean)
88 anonymous_list = Bool()93 anonymous_list = Column(Boolean)
89 # Attributes not directly modifiable via the web u/i94 # Attributes not directly modifiable via the web u/i
90 created_at = DateTime()95 created_at = Column(DateTime)
91 # Attributes which are directly modifiable via the web u/i. The more96 # Attributes which are directly modifiable via the web u/i. The more
92 # complicated attributes are currently stored as pickles, though that97 # complicated attributes are currently stored as pickles, though that
93 # will change as the schema and implementation is developed.98 # will change as the schema and implementation is developed.
94 next_request_id = Int()99 next_request_id = Column(Integer)
95 next_digest_number = Int()100 next_digest_number = Column(Integer)
96 digest_last_sent_at = DateTime()101 digest_last_sent_at = Column(DateTime)
97 volume = Int()102 volume = Column(Integer)
98 last_post_at = DateTime()103 last_post_at = Column(DateTime)
99 # Implicit destination.104 # Attributes which are directly modifiable via the web u/i. The more
100 acceptable_aliases_id = Int()105 # complicated attributes are currently stored as pickles, though that
101 acceptable_alias = Reference(acceptable_aliases_id, 'AcceptableAlias.id')106 # will change as the schema and implementation is developed.
102 # Attributes which are directly modifiable via the web u/i. The more107 accept_these_nonmembers = Column(PickleType) # XXX
103 # complicated attributes are currently stored as pickles, though that108 admin_immed_notify = Column(Boolean)
104 # will change as the schema and implementation is developed.109 admin_notify_mchanges = Column(Boolean)
105 accept_these_nonmembers = Pickle() # XXX110 administrivia = Column(Boolean)
106 admin_immed_notify = Bool()111 archive_policy = Column(Enum(ArchivePolicy))
107 admin_notify_mchanges = Bool()
108 administrivia = Bool()
109 archive_policy = Enum(ArchivePolicy)
110 # Automatic responses.112 # Automatic responses.
111 autoresponse_grace_period = TimeDelta()113 autoresponse_grace_period = Column(Interval)
112 autorespond_owner = Enum(ResponseAction)114 autorespond_owner = Column(Enum(ResponseAction))
113 autoresponse_owner_text = Unicode()115 autoresponse_owner_text = Column(Unicode)
114 autorespond_postings = Enum(ResponseAction)116 autorespond_postings = Column(Enum(ResponseAction))
115 autoresponse_postings_text = Unicode()117 autoresponse_postings_text = Column(Unicode)
116 autorespond_requests = Enum(ResponseAction)118 autorespond_requests = Column(Enum(ResponseAction))
117 autoresponse_request_text = Unicode()119 autoresponse_request_text = Column(Unicode)
118 # Content filters.120 # Content filters.
119 filter_action = Enum(FilterAction)121 filter_action = Column(Enum(FilterAction))
120 filter_content = Bool()122 filter_content = Column(Boolean)
121 collapse_alternatives = Bool()123 collapse_alternatives = Column(Boolean)
122 convert_html_to_plaintext = Bool()124 convert_html_to_plaintext = Column(Boolean)
123 # Bounces.125 # Bounces.
124 bounce_info_stale_after = TimeDelta() # XXX126 bounce_info_stale_after = Column(Interval) # XXX
125 bounce_matching_headers = Unicode() # XXX127 bounce_matching_headers = Column(Unicode) # XXX
126 bounce_notify_owner_on_disable = Bool() # XXX128 bounce_notify_owner_on_disable = Column(Boolean) # XXX
127 bounce_notify_owner_on_removal = Bool() # XXX129 bounce_notify_owner_on_removal = Column(Boolean) # XXX
128 bounce_score_threshold = Int() # XXX130 bounce_score_threshold = Column(Integer) # XXX
129 bounce_you_are_disabled_warnings = Int() # XXX131 bounce_you_are_disabled_warnings = Column(Integer) # XXX
130 bounce_you_are_disabled_warnings_interval = TimeDelta() # XXX132 bounce_you_are_disabled_warnings_interval = Column(Interval) # XXX
131 forward_unrecognized_bounces_to = Enum(UnrecognizedBounceDisposition)133 forward_unrecognized_bounces_to = Column(
132 process_bounces = Bool()134 Enum(UnrecognizedBounceDisposition))
135 process_bounces = Column(Boolean)
133 # Miscellaneous136 # Miscellaneous
134 default_member_action = Enum(Action)137 default_member_action = Column(Enum(Action))
135 default_nonmember_action = Enum(Action)138 default_nonmember_action = Column(Enum(Action))
136 description = Unicode()139 description = Column(Unicode)
137 digest_footer_uri = Unicode()140 digest_footer_uri = Column(Unicode)
138 digest_header_uri = Unicode()141 digest_header_uri = Column(Unicode)
139 digest_is_default = Bool()142 digest_is_default = Column(Boolean)
140 digest_send_periodic = Bool()143 digest_send_periodic = Column(Boolean)
141 digest_size_threshold = Float()144 digest_size_threshold = Column(Float)
142 digest_volume_frequency = Enum(DigestFrequency)145 digest_volume_frequency = Column(Enum(DigestFrequency))
143 digestable = Bool()146 digestable = Column(Boolean)
144 discard_these_nonmembers = Pickle()147 discard_these_nonmembers = Column(PickleType)
145 emergency = Bool()148 emergency = Column(Boolean)
146 encode_ascii_prefixes = Bool()149 encode_ascii_prefixes = Column(Boolean)
147 first_strip_reply_to = Bool()150 first_strip_reply_to = Column(Boolean)
148 footer_uri = Unicode()151 footer_uri = Column(Unicode)
149 forward_auto_discards = Bool()152 forward_auto_discards = Column(Boolean)
150 gateway_to_mail = Bool()153 gateway_to_mail = Column(Boolean)
151 gateway_to_news = Bool()154 gateway_to_news = Column(Boolean)
152 goodbye_message_uri = Unicode()155 goodbye_message_uri = Column(Unicode)
153 header_matches = Pickle()156 header_matches = Column(PickleType)
154 header_uri = Unicode()157 header_uri = Column(Unicode)
155 hold_these_nonmembers = Pickle()158 hold_these_nonmembers = Column(PickleType)
156 info = Unicode()159 info = Column(Unicode)
157 linked_newsgroup = Unicode()160 linked_newsgroup = Column(Unicode)
158 max_days_to_hold = Int()161 max_days_to_hold = Column(Integer)
159 max_message_size = Int()162 max_message_size = Column(Integer)
160 max_num_recipients = Int()163 max_num_recipients = Column(Integer)
161 member_moderation_notice = Unicode()164 member_moderation_notice = Column(Unicode)
162 mime_is_default_digest = Bool()165 mime_is_default_digest = Column(Boolean)
163 # FIXME: There should be no moderator_password166 # FIXME: There should be no moderator_password
164 moderator_password = RawStr()167 moderator_password = Column(LargeBinary) # TODO : was RawStr()
165 newsgroup_moderation = Enum(NewsgroupModeration)168 newsgroup_moderation = Column(Enum(NewsgroupModeration))
166 nntp_prefix_subject_too = Bool()169 nntp_prefix_subject_too = Column(Boolean)
167 nondigestable = Bool()170 nondigestable = Column(Boolean)
168 nonmember_rejection_notice = Unicode()171 nonmember_rejection_notice = Column(Unicode)
169 obscure_addresses = Bool()172 obscure_addresses = Column(Boolean)
170 owner_chain = Unicode()173 owner_chain = Column(Unicode)
171 owner_pipeline = Unicode()174 owner_pipeline = Column(Unicode)
172 personalize = Enum(Personalization)175 personalize = Column(Enum(Personalization))
173 post_id = Int()176 post_id = Column(Integer)
174 posting_chain = Unicode()177 posting_chain = Column(Unicode)
175 posting_pipeline = Unicode()178 posting_pipeline = Column(Unicode)
176 _preferred_language = Unicode(name='preferred_language')179 _preferred_language = Column('preferred_language', Unicode)
177 display_name = Unicode()180 display_name = Column(Unicode)
178 reject_these_nonmembers = Pickle()181 reject_these_nonmembers = Column(PickleType)
179 reply_goes_to_list = Enum(ReplyToMunging)182 reply_goes_to_list = Column(Enum(ReplyToMunging))
180 reply_to_address = Unicode()183 reply_to_address = Column(Unicode)
181 require_explicit_destination = Bool()184 require_explicit_destination = Column(Boolean)
182 respond_to_post_requests = Bool()185 respond_to_post_requests = Column(Boolean)
183 scrub_nondigest = Bool()186 scrub_nondigest = Column(Boolean)
184 send_goodbye_message = Bool()187 send_goodbye_message = Column(Boolean)
185 send_welcome_message = Bool()188 send_welcome_message = Column(Boolean)
186 subject_prefix = Unicode()189 subject_prefix = Column(Unicode)
187 topics = Pickle()190 topics = Column(PickleType)
188 topics_bodylines_limit = Int()191 topics_bodylines_limit = Column(Integer)
189 topics_enabled = Bool()192 topics_enabled = Column(Boolean)
190 welcome_message_uri = Unicode()193 welcome_message_uri = Column(Unicode)
191194
192 def __init__(self, fqdn_listname):195 def __init__(self, fqdn_listname):
193 super(MailingList, self).__init__()196 super(MailingList, self).__init__()
@@ -198,14 +201,15 @@
198 self._list_id = '{0}.{1}'.format(listname, hostname)201 self._list_id = '{0}.{1}'.format(listname, hostname)
199 # For the pending database202 # For the pending database
200 self.next_request_id = 1203 self.next_request_id = 1
201 # We need to set up the rosters. Normally, this method will get204 # We need to set up the rosters. Normally, this method will get called
202 # called when the MailingList object is loaded from the database, but205 # when the MailingList object is loaded from the database, but when the
203 # that's not the case when the constructor is called. So, set up the206 # constructor is called, SQLAlchemy's `load` event isn't triggered.
204 # rosters explicitly.207 # Thus we need to set up the rosters explicitly.
205 self.__storm_loaded__()208 self._post_load()
206 makedirs(self.data_path)209 makedirs(self.data_path)
207210
208 def __storm_loaded__(self):211 def _post_load(self, *args):
212 # This hooks up to SQLAlchemy's `load` event.
209 self.owners = roster.OwnerRoster(self)213 self.owners = roster.OwnerRoster(self)
210 self.moderators = roster.ModeratorRoster(self)214 self.moderators = roster.ModeratorRoster(self)
211 self.administrators = roster.AdministratorRoster(self)215 self.administrators = roster.AdministratorRoster(self)
@@ -215,6 +219,13 @@
215 self.subscribers = roster.Subscribers(self)219 self.subscribers = roster.Subscribers(self)
216 self.nonmembers = roster.NonmemberRoster(self)220 self.nonmembers = roster.NonmemberRoster(self)
217221
222 @classmethod
223 def __declare_last__(cls):
224 # SQLAlchemy special directive hook called after mappings are assumed
225 # to be complete. Use this to connect the roster instance creation
226 # method with the SA `load` event.
227 listen(cls, 'load', cls._post_load)
228
218 def __repr__(self):229 def __repr__(self):
219 return '<mailing list "{0}" at {1:#x}>'.format(230 return '<mailing list "{0}" at {1:#x}>'.format(
220 self.fqdn_listname, id(self))231 self.fqdn_listname, id(self))
@@ -323,42 +334,42 @@
323 except AttributeError:334 except AttributeError:
324 self._preferred_language = language335 self._preferred_language = language
325336
326 def send_one_last_digest_to(self, address, delivery_mode):337 @dbconnection
338 def send_one_last_digest_to(self, store, address, delivery_mode):
327 """See `IMailingList`."""339 """See `IMailingList`."""
328 digest = OneLastDigest(self, address, delivery_mode)340 digest = OneLastDigest(self, address, delivery_mode)
329 Store.of(self).add(digest)341 store.add(digest)
330342
331 @property343 @property
332 def last_digest_recipients(self):344 @dbconnection
345 def last_digest_recipients(self, store):
333 """See `IMailingList`."""346 """See `IMailingList`."""
334 results = Store.of(self).find(347 results = store.query(OneLastDigest).filter(
335 OneLastDigest,
336 OneLastDigest.mailing_list == self)348 OneLastDigest.mailing_list == self)
337 recipients = [(digest.address, digest.delivery_mode)349 recipients = [(digest.address, digest.delivery_mode)
338 for digest in results]350 for digest in results]
339 results.remove()351 results.delete()
340 return recipients352 return recipients
341353
342 @property354 @property
343 def filter_types(self):355 @dbconnection
356 def filter_types(self, store):
344 """See `IMailingList`."""357 """See `IMailingList`."""
345 results = Store.of(self).find(358 results = store.query(ContentFilter).filter(
346 ContentFilter,359 ContentFilter.mailing_list == self,
347 And(ContentFilter.mailing_list == self,360 ContentFilter.filter_type == FilterType.filter_mime)
348 ContentFilter.filter_type == FilterType.filter_mime))
349 for content_filter in results:361 for content_filter in results:
350 yield content_filter.filter_pattern362 yield content_filter.filter_pattern
351363
352 @filter_types.setter364 @filter_types.setter
353 def filter_types(self, sequence):
The diff has been truncated for viewing.