Merge lp:~jpds/lmirror/fix_542162 into lp:~lmirror/lmirror/trunk

[Jonathan Davies]

Implemented bz2 compressed changesets and test updates.

[Robert Collins (lifeless)]

So, this is a good start, but not quite complete. Folk with existing lmirror repos will get broken by this patch.

What you need to do a little differently is this:
 - only compress the content after the first (signature) line in a changeset
 - add a new signature for bz2 changesets
 - make the parser switch based on the signature between decompressing or not decompressing.

review: Needs Resubmitting

Unmerged revisions

64. By Jonathan Davies on 2010-05-26

Removed print hack attempt to see what was wrong with tests earlier.

63. By Jonathan Davies on 2010-05-26

Updated init command tests for compressed changesets.

62. By Jonathan Davies on 2010-05-26

Updated finish_change command tests for compressed changesets.

61. By Jonathan Davies on 2010-05-26

Updated mirror set tests for compressed changesets.

60. By Jonathan Davies on 2010-05-26

Updated journal tests for compressed changesets.

59. By Jonathan Davies on 2010-05-26

Write and read compresed changeset files with bz2.

=== modified file 'l_mirror/journals.py'
--- l_mirror/journals.py	2010-04-11 13:20:39 +0000
+++ l_mirror/journals.py	2010-05-27 15:06:27 +0000
@@ -45,6 +45,7 @@
     'FilterCombiner', 'ProcessFilter',
     ]
 
+import bz2
 import errno
 import os
 from hashlib import sha1 as sha
@@ -588,7 +589,11 @@
                 output.extend(kind_data[1].as_tokens())
             else:
                 output.extend(kind_data.as_tokens())
-        return 'l-mirror-journal-2\n' + '\0'.join(output)
+
+        compressed_output = bz2.compress(
+            'l-mirror-journal-2\n' + '\0'.join(output))
+
+        return compressed_output
 
     def as_groups(self):
         """Create a series of groups that can be acted on to apply the journal.
@@ -624,13 +629,16 @@
         return  [adds, replaces, deletes]
 
 
-def parse(a_bytestring):
-    """Parse a_bytestring into a journal.
+def parse(compressed_data):
+    """Parse a_bytestring from compressed_data into a journal.
     
     :return: A Journal.
     """
     header1 = 'l-mirror-journal-1\n'
     header2 = 'l-mirror-journal-2\n'
+
+    a_bytestring = bz2.decompress(compressed_data)
+
     if not a_bytestring.startswith(header1):
         if not a_bytestring.startswith(header2):
             raise ValueError('Not a journal: missing header %r' % (

=== modified file 'l_mirror/tests/commands/test_finish_change.py'
--- l_mirror/tests/commands/test_finish_change.py	2010-03-31 10:11:17 +0000
+++ l_mirror/tests/commands/test_finish_change.py	2010-05-27 15:06:27 +0000
@@ -19,6 +19,8 @@
 
 """Tests for the finish-change command."""
 
+import bz2
+
 from doctest import ELLIPSIS
 
 from bzrlib.transport import get_transport
@@ -77,7 +79,18 @@
 updating = False
 
 """, ELLIPSIS))
-        self.assertThat(t.get_bytes('journals/0'), DocTestMatches("""l-mirror-journal-2
-"""))
-        self.assertThat(t.get_bytes('journals/1'), DocTestMatches("""l-mirror-journal-2
-.lmirror\x00new\x00dir\x00.lmirror/sets\x00new\x00dir\x00.lmirror/sets/myname\x00new\x00dir\x00.lmirror/sets/myname/format\x00new\x00file\x00e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e\x002\x000.000000\x00.lmirror/sets/myname/set.conf\x00new\x00file\x00061df21cf828bb333660621c3743cfc3a3b2bd23\x0023\x000.000000\x00abc\x00new\x00file\x0012039d6dd9a7e27622301e935b6eefc78846802e\x0011\x000.000000\x00dir1\x00new\x00dir\x00dir1/def\x00new\x00file\x001f8ac10f23c5b5bc1167bda84b833e5c057a77d2\x006\x000.000000\x00dir2\x00new\x00dir"""))
+
+        journal_data_set0 = "l-mirror-journal-2\n"
+
+        compressed_journal_data_set0 = bz2.compress(journal_data_set0)
+
+        self.assertThat(t.get_bytes('journals/0'),
+                DocTestMatches(compressed_journal_data_set0))
+
+        journal_data_set1 = """l-mirror-journal-2
+.lmirror\x00new\x00dir\x00.lmirror/sets\x00new\x00dir\x00.lmirror/sets/myname\x00new\x00dir\x00.lmirror/sets/myname/format\x00new\x00file\x00e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e\x002\x000.000000\x00.lmirror/sets/myname/set.conf\x00new\x00file\x00061df21cf828bb333660621c3743cfc3a3b2bd23\x0023\x000.000000\x00abc\x00new\x00file\x0012039d6dd9a7e27622301e935b6eefc78846802e\x0011\x000.000000\x00dir1\x00new\x00dir\x00dir1/def\x00new\x00file\x001f8ac10f23c5b5bc1167bda84b833e5c057a77d2\x006\x000.000000\x00dir2\x00new\x00dir"""
+
+        compressed_journal_data_set1 = bz2.compress(journal_data_set1)
+
+        self.assertThat(t.get_bytes('journals/1'),
+                DocTestMatches(compressed_journal_data_set1))

=== modified file 'l_mirror/tests/commands/test_init.py'
--- l_mirror/tests/commands/test_init.py	2010-03-31 10:11:17 +0000
+++ l_mirror/tests/commands/test_init.py	2010-05-27 15:06:27 +0000
@@ -19,6 +19,8 @@
 
 """Tests for the init command."""
 
+import bz2
+
 from doctest import ELLIPSIS
 
 from bzrlib.transport import get_transport
@@ -82,8 +84,12 @@
 timestamp = ...
 updating = True
 """, ELLIPSIS))
-        self.assertThat(t.get_bytes('journals/0'), DocTestMatches("""l-mirror-journal-2
-"""))
+
+        journal_data = "l-mirror-journal-2\n"
+        compressed_journal_data = bz2.compress(journal_data)
+
+        self.assertThat(t.get_bytes('journals/0'),
+                DocTestMatches(compressed_journal_data))
 
     def test_scans_content_root(self):
         base = self.setup_memory()
@@ -106,7 +112,15 @@
 updating = False
 
 """, ELLIPSIS))
-        self.assertThat(t.get_bytes('journals/0'), DocTestMatches("""l-mirror-journal-2
-"""))
-        self.assertThat(t.get_bytes('journals/1'), DocTestMatches("""l-mirror-journal-2
-.lmirror\x00new\x00dir\x00.lmirror/sets\x00new\x00dir\x00.lmirror/sets/myname\x00new\x00dir\x00.lmirror/sets/myname/format\x00new\x00file\x00e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e\x002\x000.000000\x00.lmirror/sets/myname/set.conf\x00new\x00file\x00061df21cf828bb333660621c3743cfc3a3b2bd23\x0023\x000.000000\x00abc\x00new\x00file\x0012039d6dd9a7e27622301e935b6eefc78846802e\x0011\x000.000000\x00dir1\x00new\x00dir\x00dir1/def\x00new\x00file\x001f8ac10f23c5b5bc1167bda84b833e5c057a77d2\x006\x000.000000\x00dir2\x00new\x00dir"""))
+
+        journal_data_set0 = "l-mirror-journal-2\n"
+        compressed_journal_data_set0 = bz2.compress(journal_data_set0)
+        self.assertThat(t.get_bytes('journals/0'),
+                DocTestMatches(compressed_journal_data_set0))
+
+        journal_data_set1 = """l-mirror-journal-2
+.lmirror\x00new\x00dir\x00.lmirror/sets\x00new\x00dir\x00.lmirror/sets/myname\x00new\x00dir\x00.lmirror/sets/myname/format\x00new\x00file\x00e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e\x002\x000.000000\x00.lmirror/sets/myname/set.conf\x00new\x00file\x00061df21cf828bb333660621c3743cfc3a3b2bd23\x0023\x000.000000\x00abc\x00new\x00file\x0012039d6dd9a7e27622301e935b6eefc78846802e\x0011\x000.000000\x00dir1\x00new\x00dir\x00dir1/def\x00new\x00file\x001f8ac10f23c5b5bc1167bda84b833e5c057a77d2\x006\x000.000000\x00dir2\x00new\x00dir"""
+        compressed_journal_data_set1 = bz2.compress(journal_data_set1)
+
+        self.assertThat(t.get_bytes('journals/1'),
+                DocTestMatches(compressed_journal_data_set1))

=== modified file 'l_mirror/tests/test_journals.py'
--- l_mirror/tests/test_journals.py	2010-04-08 11:01:39 +0000
+++ l_mirror/tests/test_journals.py	2010-05-27 15:06:27 +0000
@@ -19,6 +19,7 @@
 
 """Tests for the journals module."""
 
+import bz2
 from doctest import ELLIPSIS
 from StringIO import StringIO
 import time
@@ -324,8 +325,13 @@
         j1.add('1234', 'replace', (
             journals.SymlinkContent('foo bar/baz'),
             journals.FileContent('e935b6eefc78846802e12039d6dd9a7e27622301', 0, 1.5)))
-        self.assertEqual("""l-mirror-journal-2
-1234\0replace\0symlink\0foo bar/baz\0file\0e935b6eefc78846802e12039d6dd9a7e27622301\0000\x001.500000\x00abc\0new\0file\00012039d6dd9a7e27622301e935b6eefc78846802e\00011\x000.000000\x00abc/def\0del\0dir""", j1.as_bytes())
+
+        expected_result = """l-mirror-journal-2
+1234\0replace\0symlink\0foo bar/baz\0file\0e935b6eefc78846802e12039d6dd9a7e27622301\0000\x001.500000\x00abc\0new\0file\00012039d6dd9a7e27622301e935b6eefc78846802e\00011\x000.000000\x00abc/def\0del\0dir"""
+
+        compressed_result = bz2.compress(expected_result)
+
+        self.assertEqual(compressed_result, j1.as_bytes())
 
 
 class TestTransportReplay(ResourcedTestCase):
@@ -387,9 +393,11 @@
 class TestParser(ResourcedTestCase):
 
     def test_parse_v1(self):
-        journal = journals.parse("""l-mirror-journal-1
-1234\0replace\0symlink\0foo bar/baz\0file\0e935b6eefc78846802e12039d6dd9a7e27622301\0000\x00abc\0new\0file\00012039d6dd9a7e27622301e935b6eefc78846802e\00011\0abc/def\0del\0dir""")
+        journal_data =  """l-mirror-journal-1
+1234\0replace\0symlink\0foo bar/baz\0file\0e935b6eefc78846802e12039d6dd9a7e27622301\0000\x00abc\0new\0file\00012039d6dd9a7e27622301e935b6eefc78846802e\00011\0abc/def\0del\0dir"""
+        compressed_journal_data = bz2.compress(journal_data)
         expected = journals.Journal()
+        journal = journals.parse(compressed_journal_data)
         expected.add('abc', 'new',
             journals.FileContent('12039d6dd9a7e27622301e935b6eefc78846802e', 11, None))
         expected.add('abc/def', 'del', journals.DirContent())
@@ -399,9 +407,12 @@
         self.assertEqual(expected.paths, journal.paths)
 
     def test_parse_v2(self):
-        journal = journals.parse("""l-mirror-journal-2
-1234\0replace\0symlink\0foo bar/baz\0file\0e935b6eefc78846802e12039d6dd9a7e27622301\0000\x000\x00abc\0new\0file\00012039d6dd9a7e27622301e935b6eefc78846802e\00011\x002\0abc/def\0del\0dir""")
+        journal_data = """l-mirror-journal-2
+1234\0replace\0symlink\0foo bar/baz\0file\0e935b6eefc78846802e12039d6dd9a7e27622301\0000\x000\x00abc\0new\0file\00012039d6dd9a7e27622301e935b6eefc78846802e\00011\x002\0abc/def\0del\0dir"""
+        compressed_journal_data = bz2.compress(journal_data)
         expected = journals.Journal()
+
+        journal = journals.parse(compressed_journal_data)
         expected.add('abc', 'new',
             journals.FileContent('12039d6dd9a7e27622301e935b6eefc78846802e', 11, 2.0))
         expected.add('abc/def', 'del', journals.DirContent())
@@ -411,12 +422,17 @@
         self.assertEqual(expected.paths, journal.paths)
 
     def test_parse_empty(self):
-        journal = journals.parse('l-mirror-journal-1\n')
+        compressed_data = bz2.compress('l-mirror-journal-1\n')
+
+        journal = journals.parse(compressed_data)
         self.assertEqual({}, journal.paths)
 
     def test_parse_wrong_header(self):
-        self.assertRaises(ValueError, journals.parse, 'l-mirror-journal-1')
-        self.assertRaises(ValueError, journals.parse, 'l-mirror-journal-3\n')
+        compressed_header_v1 = bz2.compress('l-mirror-journal-1')
+        compressed_header_v3 = bz2.compress('l-mirror-journal-3\n')
+
+        self.assertRaises(ValueError, journals.parse, compressed_header_v1)
+        self.assertRaises(ValueError, journals.parse, compressed_header_v3)
 
 
 class TestDiskUpdater(ResourcedTestCase):

=== modified file 'l_mirror/tests/test_mirrorset.py'
--- l_mirror/tests/test_mirrorset.py	2010-04-11 13:20:39 +0000
+++ l_mirror/tests/test_mirrorset.py	2010-05-27 15:06:27 +0000
@@ -19,6 +19,8 @@
 
 """Tests for the mirrorset module."""
 
+import bz2
+
 from doctest import ELLIPSIS
 
 from bzrlib import gpg as bzrgpg
@@ -97,8 +99,11 @@
 timestamp = 0
 updating = True
 """, ELLIPSIS))
-        self.assertThat(t.get_bytes('journals/0'), DocTestMatches("""l-mirror-journal-2
-"""))
+
+        compressed_journal = bz2.compress("l-mirror-journal-2\n")
+
+        self.assertThat(t.get_bytes('journals/0'),
+                DocTestMatches(compressed_journal))
 
     def test_open_set(self):
         basedir = get_transport(self.setup_memory()).clone('path')
@@ -213,8 +218,14 @@
 updating = False
 
 """, ELLIPSIS))
-        self.assertThat(t.get_bytes('journals/1'), DocTestMatches("""l-mirror-journal-2
-.lmirror\x00new\x00dir\x00.lmirror/sets\x00new\x00dir\x00.lmirror/sets/myname\x00new\x00dir\x00.lmirror/sets/myname/format\x00new\x00file\x00e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e\x002\x000.000000\x00.lmirror/sets/myname/set.conf\x00new\x00file\x00061df21cf828bb333660621c3743cfc3a3b2bd23\x0023\x000.000000\x00abc\x00new\x00file\x0012039d6dd9a7e27622301e935b6eefc78846802e\x0011\x000.000000\x00dir1\x00new\x00dir\x00dir1/def\x00new\x00file\x001f8ac10f23c5b5bc1167bda84b833e5c057a77d2\x006\x000.000000\x00dir2\x00new\x00dir"""))
+
+        expected_journal_data = """l-mirror-journal-2
+.lmirror\x00new\x00dir\x00.lmirror/sets\x00new\x00dir\x00.lmirror/sets/myname\x00new\x00dir\x00.lmirror/sets/myname/format\x00new\x00file\x00e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e\x002\x000.000000\x00.lmirror/sets/myname/set.conf\x00new\x00file\x00061df21cf828bb333660621c3743cfc3a3b2bd23\x0023\x000.000000\x00abc\x00new\x00file\x0012039d6dd9a7e27622301e935b6eefc78846802e\x0011\x000.000000\x00dir1\x00new\x00dir\x00dir1/def\x00new\x00file\x001f8ac10f23c5b5bc1167bda84b833e5c057a77d2\x006\x000.000000\x00dir2\x00new\x00dir"""
+
+        compressed_journal_data = bz2.compress(expected_journal_data)
+
+        self.assertThat(t.get_bytes('journals/1'),
+                DocTestMatches(compressed_journal_data))
     
     def test_include_excludes_honoured(self):
         basedir = get_transport(self.setup_memory()).clone('path')
@@ -239,8 +250,14 @@
 updating = False
 
 """, ELLIPSIS))
-        self.assertThat(t.get_bytes('journals/1'), DocTestMatches("""l-mirror-journal-2
-.lmirror\x00new\x00dir\x00.lmirror/sets\x00new\x00dir\x00.lmirror/sets/myname\x00new\x00dir\x00.lmirror/sets/myname/format\x00new\x00file\x00e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e\x002\x000.000000\x00.lmirror/sets/myname/set.conf\x00new\x00file\x00061df21cf828bb333660621c3743cfc3a3b2bd23\x0023\x000.000000\x00abc\x00new\x00file\x0012039d6dd9a7e27622301e935b6eefc78846802e\x0011\x000.000000\x00dir2\x00new\x00dir\x00dir2/included\x00new\x00file\x0012039d6dd9a7e27622301e935b6eefc78846802e\x0011\x000.000000"""))
+
+        journal_data = """l-mirror-journal-2
+.lmirror\x00new\x00dir\x00.lmirror/sets\x00new\x00dir\x00.lmirror/sets/myname\x00new\x00dir\x00.lmirror/sets/myname/format\x00new\x00file\x00e5fa44f2b31c1fb553b6021e7360d07d5d91ff5e\x002\x000.000000\x00.lmirror/sets/myname/set.conf\x00new\x00file\x00061df21cf828bb333660621c3743cfc3a3b2bd23\x0023\x000.000000\x00abc\x00new\x00file\x0012039d6dd9a7e27622301e935b6eefc78846802e\x0011\x000.000000\x00dir2\x00new\x00dir\x00dir2/included\x00new\x00file\x0012039d6dd9a7e27622301e935b6eefc78846802e\x0011\x000.000000"""
+
+        compressed_journal_data = bz2.compress(journal_data)
+
+        self.assertThat(t.get_bytes('journals/1'),
+                DocTestMatches(compressed_journal_data))
     
     def test_signs_when_there_is_a_keyring(self):
         basedir = get_transport(self.setup_memory()).clone('path')