Merge lp:~abompard/mailman/pgsql_upgrade into lp:mailman

Proposed by Aurélien Bompard
Status: Rejected
Rejected by: Barry Warsaw
Proposed branch: lp:~abompard/mailman/pgsql_upgrade
Merge into: lp:mailman
Diff against target: 169 lines (+150/-0)
2 files modified
src/mailman/database/schema/mm_20140715000000.py (+84/-0)
src/mailman/database/tests/test_migrations.py (+66/-0)
To merge this branch: bzr merge lp:~abompard/mailman/pgsql_upgrade
Reviewer Review Type Date Requested Status
Mailman Coders Pending
Review via email: mp+227289@code.launchpad.net

Description of the change

As explained on http://www.mail-archive.com/mailman-developers%40python.org/msg14659.html, the schema upgrade of 20130406 is missing the PostgreSQL part. This branch adds it in a new migration.

To post a comment you must log in.
Revision history for this message
Barry Warsaw (barry) wrote :

Any chance you could write some tests for this branch? I'm basically in favor of it, but tests would be good. OTOH, LP: #1170063 kind of sinks us, doesn't it? Have you looked at Gustavo's comments in that bug?

Revision history for this message
Aurélien Bompard (abompard) wrote :

OK I'll try to write some tests for this migration.
There's a workaround for #1170063 in the ticket that works for me, event if it's not perfect.
And with the SQLite lockups I reported long ago (#1091586), we can't afford to have neither SQLite nor PostgreSQL support...

lp:~abompard/mailman/pgsql_upgrade updated
7253. By Aurelien Bompard <email address hidden> on 2014-07-30

Add tests for the 20140715 migration

Revision history for this message
Barry Warsaw (barry) wrote :

Gosh, I'm finally catching up on merge proposals. Aurelien, I think with the switch to SA and Alembic, this one is no longer relevant.

Unmerged revisions

7253. By Aurelien Bompard <email address hidden> on 2014-07-30

Add tests for the 20140715 migration

7252. By Aurélien Bompard on 2014-07-18

Add PostgreSQL migration missing from the 20130406 schema upgrade

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added file 'src/mailman/database/schema/mm_20140715000000.py'
2--- src/mailman/database/schema/mm_20140715000000.py 1970-01-01 00:00:00 +0000
3+++ src/mailman/database/schema/mm_20140715000000.py 2014-07-30 21:05:44 +0000
4@@ -0,0 +1,84 @@
5+# Copyright (C) 2012-2014 by the Free Software Foundation, Inc.
6+#
7+# This file is part of GNU Mailman.
8+#
9+# GNU Mailman is free software: you can redistribute it and/or modify it under
10+# the terms of the GNU General Public License as published by the Free
11+# Software Foundation, either version 3 of the License, or (at your option)
12+# any later version.
13+#
14+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
15+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
16+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17+# more details.
18+#
19+# You should have received a copy of the GNU General Public License along with
20+# GNU Mailman. If not, see <http://www.gnu.org/licenses/>.
21+
22+"""3.0b3 -> 3.0b4 schema migrations for PostgreSQL.
23+
24+* Renames:
25+ - bounceevent.list_name -> bounceevent.list_id
26+
27+* Added:
28+ - listarchiver table
29+"""
30+
31+from __future__ import absolute_import, print_function, unicode_literals
32+
33+__metaclass__ = type
34+__all__ = [
35+ 'upgrade',
36+ ]
37+
38+
39+from mailman.database.schema.helpers import make_listid
40+
41+
42+VERSION = '20140715000000'
43+
44+
45+
46
47+def upgrade(database, store, version, module_path):
48+ if database.TAG == 'sqlite':
49+ upgrade_sqlite(database, store, version, module_path)
50+ else:
51+ upgrade_postgres(database, store, version, module_path)
52+
53+
54+
55
56+def upgrade_sqlite(database, store, version, module_path):
57+ pass # done with 20130406000000
58+
59+
60
61+def upgrade_postgres(database, store, version, module_path):
62+ # Rename bounceevent.list_name to list_id
63+ store.execute("""
64+ ALTER TABLE bounceevent
65+ RENAME COLUMN list_name TO list_id;
66+ """)
67+ results = store.execute("""
68+ SELECT id, list_id
69+ FROM bounceevent;
70+ """)
71+ for id, list_name in results:
72+ store.execute("""
73+ UPDATE bounceevent SET list_id = '{}'
74+ WHERE id = {};
75+ """.format(make_listid(list_name), id))
76+ # Create listarchiver table
77+ store.execute("""
78+ CREATE TABLE listarchiver (
79+ id SERIAL NOT NULL,
80+ mailing_list_id INTEGER NOT NULL,
81+ name TEXT NOT NULL,
82+ _is_enabled BOOLEAN,
83+ PRIMARY KEY (id)
84+ );
85+ """)
86+ store.execute("""
87+ CREATE INDEX ix_listarchiver_mailing_list_id
88+ ON listarchiver(mailing_list_id);
89+ """)
90+ # Record the migration in the version table.
91+ database.load_schema(store, version, None, module_path)
92
93=== modified file 'src/mailman/database/tests/test_migrations.py'
94--- src/mailman/database/tests/test_migrations.py 2014-01-01 14:59:42 +0000
95+++ src/mailman/database/tests/test_migrations.py 2014-07-30 21:05:44 +0000
96@@ -28,6 +28,7 @@
97 'TestMigration20121015Schema',
98 'TestMigration20130406MigratedData',
99 'TestMigration20130406Schema',
100+ 'TestMigration20140715Schema',
101 ]
102
103
104@@ -504,3 +505,68 @@
105 self.assertEqual(events[0].message_id, '<abc@example.com>')
106 self.assertEqual(events[0].context, BounceContext.normal)
107 self.assertFalse(events[0].processed)
108+
109+
110+
111
112+class TestMigration20140715Schema(MigrationTestBase):
113+ """Test column migrations."""
114+
115+ def test_pre_upgrade_column_migrations(self):
116+ if self._database.TAG == 'sqlite':
117+ raise unittest.SkipTest
118+ self._missing_present('bounceevent',
119+ ['20140714999999'],
120+ ('list_id',),
121+ ('list_name',))
122+
123+ def test_post_upgrade_column_migrations(self):
124+ if self._database.TAG == 'sqlite':
125+ raise unittest.SkipTest
126+ self._missing_present('bounceevent',
127+ ['20140714999999',
128+ '20140715000000'],
129+ ('list_name',),
130+ ('list_id',))
131+
132+ def test_pre_listarchiver_table(self):
133+ if self._database.TAG == 'sqlite':
134+ raise unittest.SkipTest
135+ self._table_missing_present(['20140714999999'], ('listarchiver',), ())
136+
137+ def test_post_listarchiver_table(self):
138+ if self._database.TAG == 'sqlite':
139+ raise unittest.SkipTest
140+ self._table_missing_present(['20140714999999',
141+ '20140715000000'],
142+ (),
143+ ('listarchiver',))
144+
145+
146+
147
148+class TestMigration20140715MigratedData(MigrationTestBase):
149+ """Test migrated data."""
150+
151+ def test_migration_bounceevent(self):
152+ if self._database.TAG == 'sqlite':
153+ raise unittest.SkipTest
154+ # Load all migrations to just before the one we're testing.
155+ self._database.load_migrations('20140714999999')
156+ # Insert a bounce event.
157+ self._database.store.execute("""
158+ INSERT INTO bounceevent VALUES (
159+ 1, 'test@example.com', 'anne@example.com',
160+ '2013-04-06 21:12:00', '<abc@example.com>',
161+ 1, 0);
162+ """)
163+ # Update to the current migration we're testing
164+ self._database.load_migrations('20140715000000')
165+ # The bounce event should exist, but with a list-id instead of a fqdn
166+ # list name.
167+ events = list(self._database.store.find(BounceEvent))
168+ self.assertEqual(len(events), 1)
169+ self.assertEqual(events[0].list_id, 'test.example.com')
170+ self.assertEqual(events[0].email, 'anne@example.com')
171+ self.assertEqual(events[0].timestamp, datetime(2013, 4, 6, 21, 12))
172+ self.assertEqual(events[0].message_id, '<abc@example.com>')
173+ self.assertEqual(events[0].context, BounceContext.normal)
174+ self.assertFalse(events[0].processed)