Merge lp:~abompard/mailman/pgsql_upgrade into lp:mailman

[Aurélien Bompard]

As explained on http://www.mail-archive.com/mailman-developers%40python.org/msg14659.html, the schema upgrade of 20130406 is missing the PostgreSQL part. This branch adds it in a new migration.

[Barry Warsaw (barry)]

Any chance you could write some tests for this branch? I'm basically in favor of it, but tests would be good. OTOH, LP: #1170063 kind of sinks us, doesn't it? Have you looked at Gustavo's comments in that bug?

[Aurélien Bompard (abompard)]

OK I'll try to write some tests for this migration.
There's a workaround for #1170063 in the ticket that works for me, event if it's not perfect.
And with the SQLite lockups I reported long ago (#1091586), we can't afford to have neither SQLite nor PostgreSQL support...

[lp:~abompard/mailman/pgsql_upgrade]

7253. By Aurelien Bompard <email address hidden> on 2014-07-30

Add tests for the 20140715 migration

[Barry Warsaw (barry)]

Gosh, I'm finally catching up on merge proposals. Aurelien, I think with the switch to SA and Alembic, this one is no longer relevant.

Unmerged revisions

7253. By Aurelien Bompard <email address hidden> on 2014-07-30

Add tests for the 20140715 migration

7252. By Aurélien Bompard on 2014-07-18

Add PostgreSQL migration missing from the 20130406 schema upgrade

=== added file 'src/mailman/database/schema/mm_20140715000000.py'
--- src/mailman/database/schema/mm_20140715000000.py	1970-01-01 00:00:00 +0000
+++ src/mailman/database/schema/mm_20140715000000.py	2014-07-30 21:05:44 +0000
@@ -0,0 +1,84 @@
+# Copyright (C) 2012-2014 by the Free Software Foundation, Inc.
+#
+# This file is part of GNU Mailman.
+#
+# GNU Mailman is free software: you can redistribute it and/or modify it under
+# the terms of the GNU General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option)
+# any later version.
+#
+# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+# more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# GNU Mailman.  If not, see <http://www.gnu.org/licenses/>.
+
+"""3.0b3 -> 3.0b4 schema migrations for PostgreSQL.
+
+* Renames:
+ - bounceevent.list_name -> bounceevent.list_id
+
+* Added:
+ - listarchiver table
+"""
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+__metaclass__ = type
+__all__ = [
+    'upgrade',
+    ]
+
+
+from mailman.database.schema.helpers import make_listid
+
+
+VERSION = '20140715000000'
+
+
+

+def upgrade(database, store, version, module_path):
+    if database.TAG == 'sqlite':
+        upgrade_sqlite(database, store, version, module_path)
+    else:
+        upgrade_postgres(database, store, version, module_path)
+
+
+

+def upgrade_sqlite(database, store, version, module_path):
+    pass # done with 20130406000000
+
+

+def upgrade_postgres(database, store, version, module_path):
+    # Rename bounceevent.list_name to list_id
+    store.execute("""
+        ALTER TABLE bounceevent
+           RENAME COLUMN list_name TO list_id;
+        """)
+    results = store.execute("""
+        SELECT id, list_id
+        FROM bounceevent;
+        """)
+    for id, list_name in results:
+        store.execute("""
+            UPDATE bounceevent SET list_id = '{}'
+            WHERE id = {};
+            """.format(make_listid(list_name), id))
+    # Create listarchiver table
+    store.execute("""
+        CREATE TABLE listarchiver (
+            id SERIAL NOT NULL,
+            mailing_list_id INTEGER NOT NULL,
+            name TEXT NOT NULL,
+            _is_enabled BOOLEAN,
+            PRIMARY KEY (id)
+            );
+        """)
+    store.execute("""
+        CREATE INDEX ix_listarchiver_mailing_list_id
+            ON listarchiver(mailing_list_id);
+        """)
+    # Record the migration in the version table.
+    database.load_schema(store, version, None, module_path)

=== modified file 'src/mailman/database/tests/test_migrations.py'
--- src/mailman/database/tests/test_migrations.py	2014-01-01 14:59:42 +0000
+++ src/mailman/database/tests/test_migrations.py	2014-07-30 21:05:44 +0000
@@ -28,6 +28,7 @@
     'TestMigration20121015Schema',
     'TestMigration20130406MigratedData',
     'TestMigration20130406Schema',
+    'TestMigration20140715Schema',
     ]
 
 
@@ -504,3 +505,68 @@
         self.assertEqual(events[0].message_id, '<[e-mail]>')
         self.assertEqual(events[0].context, BounceContext.normal)
         self.assertFalse(events[0].processed)
+
+
+

+class TestMigration20140715Schema(MigrationTestBase):
+    """Test column migrations."""
+
+    def test_pre_upgrade_column_migrations(self):
+        if self._database.TAG == 'sqlite':
+            raise unittest.SkipTest
+        self._missing_present('bounceevent',
+                              ['20140714999999'],
+                              ('list_id',),
+                              ('list_name',))
+
+    def test_post_upgrade_column_migrations(self):
+        if self._database.TAG == 'sqlite':
+            raise unittest.SkipTest
+        self._missing_present('bounceevent',
+                              ['20140714999999',
+                               '20140715000000'],
+                              ('list_name',),
+                              ('list_id',))
+
+    def test_pre_listarchiver_table(self):
+        if self._database.TAG == 'sqlite':
+            raise unittest.SkipTest
+        self._table_missing_present(['20140714999999'], ('listarchiver',), ())
+
+    def test_post_listarchiver_table(self):
+        if self._database.TAG == 'sqlite':
+            raise unittest.SkipTest
+        self._table_missing_present(['20140714999999',
+                                     '20140715000000'],
+                                     (),
+                                     ('listarchiver',))
+
+
+

+class TestMigration20140715MigratedData(MigrationTestBase):
+    """Test migrated data."""
+
+    def test_migration_bounceevent(self):
+        if self._database.TAG == 'sqlite':
+            raise unittest.SkipTest
+        # Load all migrations to just before the one we're testing.
+        self._database.load_migrations('20140714999999')
+        # Insert a bounce event.
+        self._database.store.execute("""
+            INSERT INTO bounceevent VALUES (
+                1, '[e-mail]', '[e-mail]',
+                '2013-04-06 21:12:00', '<[e-mail]>',
+                1, 0);
+            """)
+        # Update to the current migration we're testing
+        self._database.load_migrations('20140715000000')
+        # The bounce event should exist, but with a list-id instead of a fqdn
+        # list name.
+        events = list(self._database.store.find(BounceEvent))
+        self.assertEqual(len(events), 1)
+        self.assertEqual(events[0].list_id, 'test.example.com')
+        self.assertEqual(events[0].email, '[e-mail]')
+        self.assertEqual(events[0].timestamp, datetime(2013, 4, 6, 21, 12))
+        self.assertEqual(events[0].message_id, '<[e-mail]>')
+        self.assertEqual(events[0].context, BounceContext.normal)
+        self.assertFalse(events[0].processed)