Merge lp:~gz/brz/py3_rio into lp:brz

[Martin Packman]

Make rio work with Python 3

Change code and tests for the rio module to work with Python 3.

For now, the _rio_pyx helper is still 2.7 only.

[Jelmer Vernooij (jelmer)]

review: Approve

=== modified file 'breezy/rio.py'
--- breezy/rio.py	2017-06-10 01:57:00 +0000
+++ breezy/rio.py	2017-06-11 17:04:24 +0000
@@ -174,17 +174,19 @@
             # max() complains if sequence is empty
             return []
         result = []
-        for tag, value in self.items:
-            if value == '':
-                result.append(tag.encode('ascii') + b': \n')
-            elif '\n' in value:
+        for text_tag, text_value in self.items:
+            tag = text_tag.encode('ascii')
+            value = text_value.encode('utf-8')
+            if value == b'':
+                result.append(tag + b': \n')
+            elif b'\n' in value:
                 # don't want splitlines behaviour on empty lines
-                val_lines = value.split('\n')
-                result.append(tag + b': ' + val_lines[0].encode('utf-8') + b'\n')
+                val_lines = value.split(b'\n')
+                result.append(tag + b': ' + val_lines[0] + b'\n')
                 for line in val_lines[1:]:
-                    result.append(b'\t' + line.encode('utf-8') + b'\n')
+                    result.append(b'\t' + line + b'\n')
             else:
-                result.append(tag.encode('ascii') + b': ' + value.encode('utf-8') + b'\n')
+                result.append(tag + b': ' + value + b'\n')
         return result
 
     def to_string(self):
@@ -302,61 +304,61 @@
     max_rio_width = max_width - 4
     lines = []
     for pline in stanza.to_lines():
-        for line in pline.split('\n')[:-1]:
-            line = re.sub('\\\\', '\\\\\\\\', line)
+        for line in pline.split(b'\n')[:-1]:
+            line = re.sub(b'\\\\', b'\\\\\\\\', line)
             while len(line) > 0:
                 partline = line[:max_rio_width]
                 line = line[max_rio_width:]
-                if len(line) > 0 and line[0] != [' ']:
+                if len(line) > 0 and line[:1] != [b' ']:
                     break_index = -1
-                    break_index = partline.rfind(' ', -20)
+                    break_index = partline.rfind(b' ', -20)
                     if break_index < 3:
-                        break_index = partline.rfind('-', -20)
+                        break_index = partline.rfind(b'-', -20)
                         break_index += 1
                     if break_index < 3:
-                        break_index = partline.rfind('/', -20)
+                        break_index = partline.rfind(b'/', -20)
                     if break_index >= 3:
                         line = partline[break_index:] + line
                         partline = partline[:break_index]
                 if len(line) > 0:
-                    line = '  ' + line
-                partline = re.sub('\r', '\\\\r', partline)
+                    line = b'  ' + line
+                partline = re.sub(b'\r', b'\\\\r', partline)
                 blank_line = False
                 if len(line) > 0:
-                    partline += '\\'
-                elif re.search(' $', partline):
-                    partline += '\\'
+                    partline += b'\\'
+                elif re.search(b' $', partline):
+                    partline += b'\\'
                     blank_line = True
-                lines.append('# ' + partline + '\n')
+                lines.append(b'# ' + partline + b'\n')
                 if blank_line:
-                    lines.append('#   \n')
+                    lines.append(b'#   \n')
     return lines
 
 
 def _patch_stanza_iter(line_iter):
-    map = {'\\\\': '\\',
-           '\\r' : '\r',
-           '\\\n': ''}
+    map = {b'\\\\': b'\\',
+           b'\\r' : b'\r',
+           b'\\\n': b''}
     def mapget(match):
         return map[match.group(0)]
 
     last_line = None
     for line in line_iter:
-        if line.startswith('# '):
+        if line.startswith(b'# '):
             line = line[2:]
-        elif line.startswith('#'):
+        elif line.startswith(b'#'):
             line = line[1:]
         else:
             raise ValueError("bad line %r" % (line,))
         if last_line is not None and len(line) > 2:
             line = line[2:]
-        line = re.sub('\r', '', line)
-        line = re.sub('\\\\(.|\n)', mapget, line)
+        line = re.sub(b'\r', b'', line)
+        line = re.sub(b'\\\\(.|\n)', mapget, line)
         if last_line is None:
             last_line = line
         else:
             last_line += line
-        if last_line[-1] == '\n':
+        if last_line[-1:] == b'\n':
             yield last_line
             last_line = None
     if last_line is not None:

=== modified file 'breezy/tests/test__rio.py'
--- breezy/tests/test__rio.py	2017-05-23 14:08:03 +0000
+++ breezy/tests/test__rio.py	2017-06-11 17:04:24 +0000
@@ -16,10 +16,15 @@
 
 """Tests for _rio_*."""
 
-from breezy import (
+from __future__ import absolute_import
+
+from .. import (
     rio,
     tests,
     )
+from ..sixish import (
+    text_type,
+    )
 
 
 def load_tests(loader, standard_tests, pattern):
@@ -43,7 +48,7 @@
 
     def test_no_colon(self):
         self.assertFalse(self.module._valid_tag("foo:bla"))
-    
+
     def test_type_error(self):
         self.assertRaises(TypeError, self.module._valid_tag, 423)
 
@@ -51,7 +56,11 @@
         self.assertFalse(self.module._valid_tag(""))
 
     def test_unicode(self):
-        self.assertRaises(TypeError, self.module._valid_tag, u"foo")
+        if text_type is str:
+            # When str is a unicode type, it is valid for a tag
+            self.assertTrue(self.module._valid_tag(u"foo"))
+        else:
+            self.assertRaises(TypeError, self.module._valid_tag, u"foo")
 
     def test_non_ascii_char(self):
         self.assertFalse(self.module._valid_tag("\xb5"))
@@ -67,7 +76,7 @@
         if s is not None:
             for tag, value in s.iter_pairs():
                 self.assertIsInstance(tag, str)
-                self.assertIsInstance(value, unicode)
+                self.assertIsInstance(value, text_type)
 
     def assertReadStanzaRaises(self, exception, line_iter):
         self.assertRaises(exception, self.module._read_stanza_utf8, line_iter)
@@ -79,34 +88,34 @@
         self.assertReadStanza(None, [])
 
     def test_none(self):
-        self.assertReadStanza(None, [""])
+        self.assertReadStanza(None, [b""])
 
     def test_simple(self):
-        self.assertReadStanza(rio.Stanza(foo="bar"), ["foo: bar\n", ""])
+        self.assertReadStanza(rio.Stanza(foo="bar"), [b"foo: bar\n", b""])
 
     def test_multi_line(self):
-        self.assertReadStanza(rio.Stanza(foo="bar\nbla"), 
-                ["foo: bar\n", "\tbla\n"])
+        self.assertReadStanza(
+            rio.Stanza(foo="bar\nbla"), [b"foo: bar\n", b"\tbla\n"])
 
     def test_repeated(self):
         s = rio.Stanza()
         s.add("foo", "bar")
         s.add("foo", "foo")
-        self.assertReadStanza(s, ["foo: bar\n", "foo: foo\n"])
+        self.assertReadStanza(s, [b"foo: bar\n", b"foo: foo\n"])
 
     def test_invalid_early_colon(self):
-        self.assertReadStanzaRaises(ValueError, ["f:oo: bar\n"])
+        self.assertReadStanzaRaises(ValueError, [b"f:oo: bar\n"])
 
     def test_invalid_tag(self):
-        self.assertReadStanzaRaises(ValueError, ["f%oo: bar\n"])
+        self.assertReadStanzaRaises(ValueError, [b"f%oo: bar\n"])
 
     def test_continuation_too_early(self):
-        self.assertReadStanzaRaises(ValueError, ["\tbar\n"])
+        self.assertReadStanzaRaises(ValueError, [b"\tbar\n"])
 
     def test_large(self):
-        value = "bla" * 9000
+        value = b"bla" * 9000
         self.assertReadStanza(rio.Stanza(foo=value),
-            ["foo: %s\n" % value])
+            [b"foo: %s\n" % value])
 
     def test_non_ascii_char(self):
         self.assertReadStanza(rio.Stanza(foo=u"n\xe5me"),
@@ -123,7 +132,7 @@
         if s is not None:
             for tag, value in s.iter_pairs():
                 self.assertIsInstance(tag, str)
-                self.assertIsInstance(value, unicode)
+                self.assertIsInstance(value, text_type)
 
     def assertReadStanzaRaises(self, exception, line_iter):
         self.assertRaises(exception, self.module._read_stanza_unicode,

=== modified file 'breezy/tests/test_rio.py'
--- breezy/tests/test_rio.py	2017-05-24 19:44:00 +0000
+++ breezy/tests/test_rio.py	2017-06-11 17:04:24 +0000
@@ -51,26 +51,18 @@
         self.assertEqual(s.get('number'), '42')
         self.assertEqual(s.get('name'), 'fred')
 
-    def test_value_checks(self):
-        """rio checks types on construction"""
-        # these aren't enforced at construction time
-        ## self.assertRaises(ValueError,
-        ##        Stanza, complex=42 + 3j)
-        ## self.assertRaises(ValueError,
-        ##        Stanza, several=range(10))
-
     def test_empty_value(self):
         """Serialize stanza with empty field"""
         s = Stanza(empty='')
-        self.assertEqualDiff(s.to_string(),
-                "empty: \n")
+        self.assertEquals(s.to_string(),
+                b"empty: \n")
 
     def test_to_lines(self):
         """Write simple rio stanza to string"""
         s = Stanza(number='42', name='fred')
         self.assertEqual(list(s.to_lines()),
-                ['name: fred\n',
-                 'number: 42\n'])
+                [b'name: fred\n',
+                 b'number: 42\n'])
 
     def test_as_dict(self):
         """Convert rio Stanza to dictionary"""
@@ -84,18 +76,18 @@
         s = Stanza(a_thing='something with "quotes like \\"this\\""', number='42', name='fred')
         s.write(tmpf)
         tmpf.seek(0)
-        self.assertEqualDiff(tmpf.read(), r'''
-a_thing: something with "quotes like \"this\""
+        self.assertEqual(tmpf.read(), b'''\
+a_thing: something with "quotes like \\"this\\""
 name: fred
 number: 42
-'''[1:])
+''')
 
     def test_multiline_string(self):
         tmpf = TemporaryFile()
         s = Stanza(motto="war is peace\nfreedom is slavery\nignorance is strength")
         s.write(tmpf)
         tmpf.seek(0)
-        self.assertEqualDiff(tmpf.read(), '''\
+        self.assertEqual(tmpf.read(), b'''\
 motto: war is peace
 \tfreedom is slavery
 \tignorance is strength
@@ -106,7 +98,7 @@
 
     def test_read_stanza(self):
         """Load stanza from string"""
-        lines = """\
+        lines = b"""\
 revision: [e-mail]-123-abc
 timestamp: 1130653962
 timezone: 36000
@@ -114,7 +106,7 @@
 """.splitlines(True)
         s = read_stanza(lines)
         self.assertTrue('revision' in s)
-        self.assertEqualDiff(s.get('revision'), '[e-mail]-123-abc')
+        self.assertEqual(s.get('revision'), '[e-mail]-123-abc')
         self.assertEqual(list(s.iter_pairs()),
                 [('revision', '[e-mail]-123-abc'),
                  ('timestamp', '1130653962'),
@@ -136,13 +128,13 @@
     def test_backslash(self):
         s = Stanza(q='\\')
         t = s.to_string()
-        self.assertEqualDiff(t, 'q: \\\n')
+        self.assertEqual(t, b'q: \\\n')
         s2 = read_stanza(s.to_lines())
         self.assertEqual(s, s2)
 
     def test_blank_line(self):
         s = Stanza(none='', one='\n', two='\n\n')
-        self.assertEqualDiff(s.to_string(), """\
+        self.assertEqual(s.to_string(), b"""\
 none:\x20
 one:\x20
 \t
@@ -155,7 +147,7 @@
 
     def test_whitespace_value(self):
         s = Stanza(space=' ', tabs='\t\t\t', combo='\n\t\t\n')
-        self.assertEqualDiff(s.to_string(), """\
+        self.assertEqual(s.to_string(), b"""\
 combo:\x20
 \t\t\t
 \t
@@ -192,16 +184,16 @@
 
     def test_read_nul_byte(self):
         """File consisting of a nul byte causes an error."""
-        self.assertRaises(ValueError, read_stanza, ['\0'])
+        self.assertRaises(ValueError, read_stanza, [b'\0'])
 
     def test_read_nul_bytes(self):
         """File consisting of many nul bytes causes an error."""
-        self.assertRaises(ValueError, read_stanza, ['\0' * 100])
+        self.assertRaises(ValueError, read_stanza, [b'\0' * 100])
 
     def test_read_iter(self):
         """Read several stanzas from file"""
         tmpf = TemporaryFile()
-        tmpf.write("""\
+        tmpf.write(b"""\
 version_header: 1
 
 name: foo
@@ -222,7 +214,7 @@
     def test_read_several(self):
         """Read several stanzas from file"""
         tmpf = TemporaryFile()
-        tmpf.write("""\
+        tmpf.write(b"""\
 version_header: 1
 
 name: foo
@@ -242,8 +234,9 @@
         s = read_stanza(tmpf)
         self.assertEqual(s, Stanza(name="foo", val='123'))
         s = read_stanza(tmpf)
-        self.assertEqualDiff(s.get('name'), 'quoted')
-        self.assertEqualDiff(s.get('address'), '  "Willowglen"\n  42 Wallaby Way\n  Sydney')
+        self.assertEqual(s.get('name'), 'quoted')
+        self.assertEqual(
+            s.get('address'), '  "Willowglen"\n  42 Wallaby Way\n  Sydney')
         s = read_stanza(tmpf)
         self.assertEqual(s, Stanza(name="bar", val='129319'))
         s = read_stanza(tmpf)
@@ -266,7 +259,7 @@
 
     def test_tricky_quoted(self):
         tmpf = TemporaryFile()
-        tmpf.write('''\
+        tmpf.write(b'''\
 s: "one"
 
 s:\x20
@@ -316,7 +309,7 @@
             stanza = read_stanza(tmpf)
             self.rio_file_stanzas([stanza])
             self.assertEqual(len(stanza), 1)
-            self.assertEqualDiff(stanza.get('s'), expected)
+            self.assertEqual(stanza.get('s'), expected)
 
     def test_write_empty_stanza(self):
         """Write empty stanza"""
@@ -339,7 +332,7 @@
         self.assertEqual(s.get('foo'), uni_data)
         raw_lines = s.to_lines()
         self.assertEqual(raw_lines,
-                ['foo: ' + uni_data.encode('utf-8') + '\n'])
+                [b'foo: ' + uni_data.encode('utf-8') + b'\n'])
         new_s = read_stanza(raw_lines)
         self.assertEqual(new_s.get('foo'), uni_data)
 
@@ -356,8 +349,8 @@
         s = Stanza(foo=uni_data)
         parent_stanza = Stanza(child=s.to_unicode())
         raw_lines = parent_stanza.to_lines()
-        self.assertEqual(['child: foo: ' + uni_data.encode('utf-8') + '\n',
-                          '\t\n',
+        self.assertEqual([b'child: foo: ' + uni_data.encode('utf-8') + b'\n',
+                          b'\t\n',
                          ], raw_lines)
         new_parent = read_stanza(raw_lines)
         child_text = new_parent.get('child')
@@ -368,9 +361,9 @@
     def mail_munge(self, lines, dos_nl=True):
         new_lines = []
         for line in lines:
-            line = re.sub(' *\n', '\n', line)
+            line = re.sub(b' *\n', b'\n', line)
             if dos_nl:
-                line = re.sub('([^\r])\n', '\\1\r\n', line)
+                line = re.sub(b'([^\r])\n', b'\\1\r\n', line)
             new_lines.append(line)
         return new_lines
 
@@ -378,7 +371,7 @@
         stanza = Stanza(data='#\n\r\\r ', space=' ' * 255, hash='#' * 255)
         lines = rio.to_patch_lines(stanza)
         for line in lines:
-            self.assertContainsRe(line, '^# ')
+            self.assertContainsRe(line, b'^# ')
             self.assertTrue(72 >= len(line))
         for line in rio.to_patch_lines(stanza, max_width=12):
             self.assertTrue(12 >= len(line))
@@ -393,10 +386,10 @@
     def test_patch_rio_linebreaks(self):
         stanza = Stanza(breaktest='linebreak -/'*30)
         self.assertContainsRe(rio.to_patch_lines(stanza, 71)[0],
-                              'linebreak\\\\\n')
+                              b'linebreak\\\\\n')
         stanza = Stanza(breaktest='linebreak-/'*30)
         self.assertContainsRe(rio.to_patch_lines(stanza, 70)[0],
-                              'linebreak-\\\\\n')
+                              b'linebreak-\\\\\n')
         stanza = Stanza(breaktest='linebreak/'*30)
         self.assertContainsRe(rio.to_patch_lines(stanza, 70)[0],
-                              'linebreak\\\\\n')
+                              b'linebreak\\\\\n')