Merge lp:~jelmer/brz/python3-xml into lp:brz

[Jelmer Vernooij]

Port breezy.bzr.xml* to Python3.

Port breezy.bzr.xml* to Python3.

[Martin Packman (gz)]

I'm not sure on some of the bare encode() calls here, but it's probably on the caller if they pass non-ascii values?

review: Approve

[Jelmer Vernooij (jelmer)]

> I'm not sure on some of the bare encode() calls here, but it's probably on the
> caller if they pass non-ascii values?
Yeah; this should be ascii only. At some point perhaps we can all just make them regular strings.

=== modified file 'breezy/bzr/xml8.py'
--- breezy/bzr/xml8.py	2018-05-13 02:18:13 +0000
+++ breezy/bzr/xml8.py	2018-05-22 02:02:43 +0000
@@ -187,17 +187,15 @@
         # them.
         decode_utf8 = cache_utf8.decode
         revision_id = rev.revision_id
-        if isinstance(revision_id, str):
-            revision_id = decode_utf8(revision_id)
         format_num = self.format_num
         if self.revision_format_num is not None:
             format_num = self.revision_format_num
         root = Element('revision',
                        committer = rev.committer,
                        timestamp = '%.3f' % rev.timestamp,
-                       revision_id = revision_id,
+                       revision_id = decode_utf8(revision_id),
                        inventory_sha1 = rev.inventory_sha1,
-                       format=format_num,
+                       format=format_num.decode(),
                        )
         if rev.timezone is not None:
             root.set('timezone', str(rev.timezone))
@@ -212,9 +210,7 @@
                 _mod_revision.check_not_reserved_id(parent_id)
                 p = SubElement(pelts, 'revision_ref')
                 p.tail = '\n'
-                if isinstance(parent_id, str):
-                    parent_id = decode_utf8(parent_id)
-                p.set('revision_id', parent_id)
+                p.set('revision_id', decode_utf8(parent_id))
         if rev.properties:
             self._pack_revision_properties(rev, root)
         return root
@@ -248,7 +244,7 @@
         if self.revision_format_num is not None:
             format_num = self.revision_format_num
         if format is not None:
-            if format != format_num:
+            if format.encode() != format_num:
                 raise BzrError("invalid format version %r on revision"
                                 % format)
         get_cached = get_utf8_or_ascii

=== modified file 'breezy/bzr/xml_serializer.py'
--- breezy/bzr/xml_serializer.py	2018-02-18 15:21:06 +0000
+++ breezy/bzr/xml_serializer.py	2018-05-22 02:02:43 +0000
@@ -100,7 +100,7 @@
         self._write_element(self._pack_revision(rev), f)
 
     def write_revision_to_string(self, rev):
-        return tostring(self._pack_revision(rev)) + '\n'
+        return tostring(self._pack_revision(rev)) + b'\n'
 
     def read_revision(self, f):
         return self._unpack_revision(self._read_element(f))
@@ -110,7 +110,7 @@
 
     def _write_element(self, elt, f):
         ElementTree(elt).write(f, 'utf-8')
-        f.write('\n')
+        f.write(b'\n')
 
     def _read_element(self, f):
         return ElementTree().parse(f)
@@ -195,9 +195,9 @@
     decode back into Unicode, and then use the XML escape code.
     """
     try:
-        return _map[match.group()]
+        return _map[match.group().decode('ascii', 'replace')].encode()
     except KeyError:
-        return ''.join('&#%d;' % ord(uni_chr)
+        return b''.join(b'&#%d;' % ord(uni_chr)
                        for uni_chr in match.group().decode('utf8'))
 
 
@@ -209,7 +209,7 @@
     # to check if None, rather than try/KeyError
     text = _map.get(unicode_or_utf8_str)
     if text is None:
-        if unicode_or_utf8_str.__class__ is text_type:
+        if isinstance(unicode_or_utf8_str, text_type):
             # The alternative policy is to do a regular UTF8 encoding
             # and then escape only XML meta characters.
             # Performance is equivalent once you use cache_utf8. *However*
@@ -217,7 +217,7 @@
             # of bzr. So no net gain. (Perhaps the read code would handle utf8
             # better than entity escapes, but cElementTree seems to do just fine
             # either way)
-            text = bytes(_unicode_re.sub(_unicode_escape_replace, unicode_or_utf8_str)) + b'"'
+            text = _unicode_re.sub(_unicode_escape_replace, unicode_or_utf8_str).encode() + b'"'
         else:
             # Plain strings are considered to already be in utf-8 so we do a
             # slightly different method for escaping.
@@ -312,11 +312,11 @@
                                      parent_id)
         ie.symlink_target = elt_get('symlink_target')
     elif kind == 'tree-reference':
-        file_id = elt.attrib['file_id']
+        file_id = get_utf8_or_ascii(elt.attrib['file_id'])
         name = elt.attrib['name']
-        parent_id = elt.attrib['parent_id']
-        revision = elt.get('revision')
-        reference_revision = elt.get('reference_revision')
+        parent_id = get_utf8_or_ascii(elt.attrib['parent_id'])
+        revision = get_utf8_or_ascii(elt.get('revision'))
+        reference_revision = get_utf8_or_ascii(elt.get('reference_revision'))
         ie = inventory.TreeReference(file_id, name, parent_id, revision,
                                        reference_revision)
     else:
@@ -346,7 +346,7 @@
     if elt.tag != 'inventory':
         raise errors.UnexpectedInventoryFormat('Root tag is %r' % elt.tag)
     format = elt.get('format')
-    if format != format_num:
+    if format.encode() != format_num:
         raise errors.UnexpectedInventoryFormat('Invalid format version %r'
                                                % format)
     revision_id = elt.get('revision_id')
@@ -387,7 +387,7 @@
                     b'text_sha1="%s" text_size="%d" />\n' % (
                     executable, encode_and_escape(ie.file_id),
                     encode_and_escape(ie.name), parent_str, parent_id,
-                    encode_and_escape(ie.revision), ie.text_sha1,
+                    encode_and_escape(ie.revision), ie.text_sha1.encode(),
                     ie.text_size))
             else:
                 append(b'<file%s file_id="%s name="%s%s%s />\n' % (

=== modified file 'breezy/tests/test_xml.py'
--- breezy/tests/test_xml.py	2018-03-25 11:39:36 +0000
+++ breezy/tests/test_xml.py	2018-05-22 02:02:43 +0000
@@ -27,6 +27,7 @@
     )
 from ..sixish import (
     BytesIO,
+    text_type,
     )
 from ..bzr.inventory import Inventory
 from . import TestCase
@@ -208,7 +209,7 @@
         eq(len(rev.parent_ids), 1)
         eq(rev.timezone, 36000)
         eq(rev.parent_ids[0],
-           "[e-mail]-20050905063503-43948f59fa127d92")
+           b"[e-mail]-20050905063503-43948f59fa127d92")
 
     def test_unpack_revision_5_utc(self):
         inp = BytesIO(_revision_v5_utc)
@@ -219,7 +220,7 @@
         eq(len(rev.parent_ids), 1)
         eq(rev.timezone, 0)
         eq(rev.parent_ids[0],
-           "[e-mail]-20050905063503-43948f59fa127d92")
+           b"[e-mail]-20050905063503-43948f59fa127d92")
 
     def test_unpack_inventory_5(self):
         """Unpack canned new-style inventory"""
@@ -227,9 +228,9 @@
         inv = breezy.bzr.xml5.serializer_v5.read_inventory(inp)
         eq = self.assertEqual
         eq(len(inv), 4)
-        ie = inv.get_entry('bar-20050824000535-6bc48cfad47ed134')
+        ie = inv.get_entry(b'bar-20050824000535-6bc48cfad47ed134')
         eq(ie.kind, 'file')
-        eq(ie.revision, 'mbp@foo-00')
+        eq(ie.revision, b'mbp@foo-00')
         eq(ie.name, 'bar')
         eq(inv.get_entry(ie.parent_id).kind, 'directory')
 
@@ -239,17 +240,17 @@
         inv = breezy.bzr.xml5.serializer_v5.read_inventory(inp)
         eq = self.assertEqual
         eq(len(inv), 4)
-        eq(inv.revision_id, '[e-mail]-20050905063503-43948f59fa127d92')
-        ie = inv.get_entry('bar-20050824000535-6bc48cfad47ed134')
+        eq(inv.revision_id, b'[e-mail]-20050905063503-43948f59fa127d92')
+        ie = inv.get_entry(b'bar-20050824000535-6bc48cfad47ed134')
         eq(ie.kind, 'file')
-        eq(ie.revision, 'mbp@foo-00')
+        eq(ie.revision, b'mbp@foo-00')
         eq(ie.name, 'bar')
         eq(inv.get_entry(ie.parent_id).kind, 'directory')
 
     def test_unpack_inventory_5a(self):
         inv = breezy.bzr.xml5.serializer_v5.read_inventory_from_string(
-                _inventory_v5a, revision_id='test-rev-id')
-        self.assertEqual('test-rev-id', inv.root.revision)
+                _inventory_v5a, revision_id=b'test-rev-id')
+        self.assertEqual(b'test-rev-id', inv.root.revision)
 
     def test_unpack_inventory_5a_cache_and_copy(self):
         # Passing an entry_cache should get populated with the objects
@@ -257,7 +258,7 @@
         # False
         entry_cache = fifo_cache.FIFOCache()
         inv = breezy.bzr.xml5.serializer_v5.read_inventory_from_string(
-            _inventory_v5a, revision_id='test-rev-id',
+            _inventory_v5a, revision_id=b'test-rev-id',
             entry_cache=entry_cache, return_from_cache=False)
         for entry in inv.iter_just_entries():
             key = (entry.file_id, entry.revision)
@@ -273,7 +274,7 @@
         # True
         entry_cache = fifo_cache.FIFOCache()
         inv = breezy.bzr.xml5.serializer_v5.read_inventory_from_string(
-            _inventory_v5a, revision_id='test-rev-id',
+            _inventory_v5a, revision_id=b'test-rev-id',
             entry_cache=entry_cache, return_from_cache=True)
         for entry in inv.iter_just_entries():
             key = (entry.file_id, entry.revision)
@@ -285,8 +286,8 @@
 
     def test_unpack_inventory_5b(self):
         inv = breezy.bzr.xml5.serializer_v5.read_inventory_from_string(
-                _inventory_v5b, revision_id='test-rev-id')
-        self.assertEqual('a-rev-id', inv.root.revision)
+                _inventory_v5b, revision_id=b'test-rev-id')
+        self.assertEqual(b'a-rev-id', inv.root.revision)
 
     def test_repack_inventory_5(self):
         inp = BytesIO(_committed_inv_v5)
@@ -336,7 +337,7 @@
         outp = BytesIO()
         breezy.bzr.xml5.serializer_v5.write_revision(rev, outp)
         outfile_contents = outp.getvalue()
-        self.assertEqual(outfile_contents[-1], '\n')
+        self.assertEqual(outfile_contents[-1:], b'\n')
         self.assertEqualDiff(outfile_contents, breezy.bzr.xml5.serializer_v5.write_revision_to_string(rev))
         self.assertEqualDiff(outfile_contents, _expected_rev_v5)
 
@@ -459,13 +460,13 @@
         """Parsed revision_ids should all be utf-8 strings, not unicode."""
         s_v5 = breezy.bzr.xml5.serializer_v5
         rev = s_v5.read_revision_from_string(_revision_utf8_v5)
-        self.assertEqual('erik@b\xc3\xa5gfors-02', rev.revision_id)
-        self.assertIsInstance(rev.revision_id, str)
-        self.assertEqual(['erik@b\xc3\xa5gfors-01'], rev.parent_ids)
+        self.assertEqual(b'erik@b\xc3\xa5gfors-02', rev.revision_id)
+        self.assertIsInstance(rev.revision_id, bytes)
+        self.assertEqual([b'erik@b\xc3\xa5gfors-01'], rev.parent_ids)
         for parent_id in rev.parent_ids:
-            self.assertIsInstance(parent_id, str)
+            self.assertIsInstance(parent_id, bytes)
         self.assertEqual(u'Include \xb5nicode characters\n', rev.message)
-        self.assertIsInstance(rev.message, unicode)
+        self.assertIsInstance(rev.message, text_type)
 
         # ie.revision should either be None or a utf-8 revision id
         inv = s_v5.read_inventory_from_string(_inventory_utf8_v5)
@@ -481,21 +482,21 @@
                     (u's\xb5bdir/b\xe5r', fid_bar2, fid_sub, rev_id_2),
                    ]
         self.assertEqual(rev_id_2, inv.revision_id)
-        self.assertIsInstance(inv.revision_id, str)
+        self.assertIsInstance(inv.revision_id, bytes)
 
         actual = list(inv.iter_entries_by_dir())
         for ((exp_path, exp_file_id, exp_parent_id, exp_rev_id),
              (act_path, act_ie)) in zip(expected, actual):
             self.assertEqual(exp_path, act_path)
-            self.assertIsInstance(act_path, unicode)
+            self.assertIsInstance(act_path, text_type)
             self.assertEqual(exp_file_id, act_ie.file_id)
-            self.assertIsInstance(act_ie.file_id, str)
+            self.assertIsInstance(act_ie.file_id, bytes)
             self.assertEqual(exp_parent_id, act_ie.parent_id)
             if exp_parent_id is not None:
-                self.assertIsInstance(act_ie.parent_id, str)
+                self.assertIsInstance(act_ie.parent_id, bytes)
             self.assertEqual(exp_rev_id, act_ie.revision)
             if exp_rev_id is not None:
-                self.assertIsInstance(act_ie.revision, str)
+                self.assertIsInstance(act_ie.revision, bytes)
 
         self.assertEqual(len(expected), len(actual))
 
@@ -514,24 +515,24 @@
         # are being used in xml attributes, and by returning it now, we have to
         # do fewer string operations later.
         val = breezy.bzr.xml_serializer.encode_and_escape('foo bar')
-        self.assertEqual('foo bar"', val)
+        self.assertEqual(b'foo bar"', val)
         # The second time should be cached
         val2 = breezy.bzr.xml_serializer.encode_and_escape('foo bar')
         self.assertIs(val2, val)
 
     def test_ascii_with_xml(self):
-        self.assertEqual('&amp;&apos;&quot;&lt;&gt;"',
+        self.assertEqual(b'&amp;&apos;&quot;&lt;&gt;"',
                          breezy.bzr.xml_serializer.encode_and_escape('&\'"<>'))
 
     def test_utf8_with_xml(self):
         # u'\xb5\xe5&\u062c'
-        utf8_str = '\xc2\xb5\xc3\xa5&\xd8\xac'
-        self.assertEqual('&#181;&#229;&amp;&#1580;"',
+        utf8_str = b'\xc2\xb5\xc3\xa5&\xd8\xac'
+        self.assertEqual(b'&#181;&#229;&amp;&#1580;"',
                          breezy.bzr.xml_serializer.encode_and_escape(utf8_str))
 
     def test_unicode(self):
         uni_str = u'\xb5\xe5&\u062c'
-        self.assertEqual('&#181;&#229;&amp;&#1580;"',
+        self.assertEqual(b'&#181;&#229;&amp;&#1580;"',
                          breezy.bzr.xml_serializer.encode_and_escape(uni_str))
 
 

=== modified file 'python3.passing'
--- python3.passing	2018-05-19 14:54:22 +0000
+++ python3.passing	2018-05-22 02:02:43 +0000
@@ -56,11 +56,35 @@
 breezy.plugins.upload.tests.test_upload.TestBranchUploadLocations.test_set_push_location(BranchReferenceFormat)
 breezy.plugins.upload.tests.test_upload.TestBranchUploadLocations.test_set_push_location(BzrBranchFormat7)
 breezy.plugins.weave_fmt.test_bzrdir.TestBranchFormat4.test_no_metadir_support
+breezy.plugins.weave_fmt.test_bzrdir.TestBranchFormat4.test_supports_bzrdir_6
+breezy.plugins.weave_fmt.test_bzrdir.TestFormat5.test_can_convert
+breezy.plugins.weave_fmt.test_bzrdir.TestFormat5.test_needs_conversion
+breezy.plugins.weave_fmt.test_bzrdir.TestFormat5.test_same_lockfiles_between_tree_repo_branch
+breezy.plugins.weave_fmt.test_bzrdir.TestFormat6.test_can_convert
+breezy.plugins.weave_fmt.test_bzrdir.TestFormat6.test_needs_conversion
+breezy.plugins.weave_fmt.test_bzrdir.TestFormat6.test_same_lockfiles_between_tree_repo_branch
+breezy.plugins.weave_fmt.test_bzrdir.V4WeaveBundleTester.test_crlf_bundle
+breezy.plugins.weave_fmt.test_bzrdir.V4WeaveBundleTester.test_malformed
+breezy.plugins.weave_fmt.test_bzrdir.V4WeaveBundleTester.test_non_bundle
+breezy.plugins.weave_fmt.test_repository.TestFormat6.test_attribute__fetch_order
+breezy.plugins.weave_fmt.test_repository.TestFormat6.test_attribute__fetch_reconcile
+breezy.plugins.weave_fmt.test_repository.TestFormat6.test_attribute__fetch_uses_deltas
+breezy.plugins.weave_fmt.test_repository.TestFormat6.test_no_ancestry_weave
+breezy.plugins.weave_fmt.test_repository.TestFormat6.test_supports_external_lookups
+breezy.plugins.weave_fmt.test_repository.TestFormat7.test_attribute__fetch_order
+breezy.plugins.weave_fmt.test_repository.TestFormat7.test_attribute__fetch_reconcile
+breezy.plugins.weave_fmt.test_repository.TestFormat7.test_attribute__fetch_uses_deltas
+breezy.plugins.weave_fmt.test_repository.TestFormat7.test_creates_lockdir
+breezy.plugins.weave_fmt.test_repository.TestFormat7.test_supports_external_lookups
+breezy.plugins.weave_fmt.test_repository.TestFormat7.test_uses_lockdir
 breezy.plugins.weave_fmt.test_repository.TestInterWeaveRepo.test_is_compatible_and_registered
+breezy.plugins.weave_fmt.test_repository.TestSerializer.test_canned_inventory
 breezy.plugins.weave_fmt.test_repository.TestSerializer.test_registry
+breezy.plugins.weave_fmt.test_repository.TestSerializer.test_unpack_revision
 breezy.plugins.weave_fmt.test_store.TestCompressedTextStore.test_multiple_add
 breezy.plugins.weave_fmt.test_store.TestCompressedTextStore.test_total_size
 breezy.plugins.weave_fmt.test_store.TestInstrumentedTransportStore.test__add_records
+breezy.plugins.weave_fmt.test_store.TestMemoryStore.test_add_and_retrieve
 breezy.plugins.weave_fmt.test_store.TestMemoryStore.test_adding_fails_when_present
 breezy.plugins.weave_fmt.test_store.TestMemoryStore.test_missing_is_absent
 breezy.plugins.weave_fmt.test_store.TestMemoryStore.test_total_size
@@ -79,6 +103,8 @@
 breezy.plugins.weave_fmt.test_store.TestTransportStore.test__relpath_simple
 breezy.plugins.weave_fmt.test_store.TestTransportStore.test__relpath_simple_suffixed
 breezy.plugins.weave_fmt.test_store.TestTransportStore.test__relpath_unregister_suffixes
+breezy.plugins.weave_fmt.test_store.TestVersionFileStore.test_get_weave_readonly_cant_write
+breezy.plugins.weave_fmt.test_store.TestVersionFileStore.test_get_weave_registers_dirty_in_write
 breezy.pyutils.get_named_object
 breezy.symbol_versioning.deprecated_in
 breezy.tests.commands.test_checkout.TestCheckout.test_checkout
@@ -10513,6 +10539,7 @@
 breezy.tests.test_ui.TestUIFactoryTests.test_test_ui_factory_progress
 breezy.tests.test_ui.UITests.test_progress_construction
 breezy.tests.test_ui.UITests.test_text_ui_non_terminal
+breezy.tests.test_upgrade.TestUpgrade.test_convert_knit_dirstate_content
 breezy.tests.test_upstream_import.TestImport.test_common_directory
 breezy.tests.test_upstream_import.TestImport.test_get_archive_type
 breezy.tests.test_upstream_import.TestImport.test_top_path
@@ -10670,7 +10697,34 @@
 breezy.tests.test_wsgi.TestWSGI.test_relpath_setter
 breezy.tests.test_wsgi.TestWSGI.test_relpath_setter_bad_path_prefix
 breezy.tests.test_wsgi.TestWSGI.test_relpath_setter_bad_path_suffix
+breezy.tests.test_xml.TestEncodeAndEscape.test_ascii_with_xml
+breezy.tests.test_xml.TestEncodeAndEscape.test_simple_ascii
+breezy.tests.test_xml.TestEncodeAndEscape.test_unicode
+breezy.tests.test_xml.TestEncodeAndEscape.test_utf8_with_xml
 breezy.tests.test_xml.TestMisc.test_unescape_xml
+breezy.tests.test_xml.TestSerializer.test_empty_property_value
+breezy.tests.test_xml.TestSerializer.test_inventory_text_v8
+breezy.tests.test_xml.TestSerializer.test_pack_revision_5
+breezy.tests.test_xml.TestSerializer.test_repack_inventory_5
+breezy.tests.test_xml.TestSerializer.test_repack_revision_5
+breezy.tests.test_xml.TestSerializer.test_repack_revision_5_utc
+breezy.tests.test_xml.TestSerializer.test_revision_ids_are_utf8
+breezy.tests.test_xml.TestSerializer.test_revision_text_v6
+breezy.tests.test_xml.TestSerializer.test_revision_text_v7
+breezy.tests.test_xml.TestSerializer.test_revision_text_v8
+breezy.tests.test_xml.TestSerializer.test_roundtrip_inventory_v6
+breezy.tests.test_xml.TestSerializer.test_roundtrip_inventory_v7
+breezy.tests.test_xml.TestSerializer.test_roundtrip_inventory_v8
+breezy.tests.test_xml.TestSerializer.tests_serialize_inventory_v5_with_root
+breezy.tests.test_xml.TestSerializer.test_tree_reference
+breezy.tests.test_xml.TestSerializer.test_unpack_basis_inventory_5
+breezy.tests.test_xml.TestSerializer.test_unpack_inventory_5
+breezy.tests.test_xml.TestSerializer.test_unpack_inventory_5a
+breezy.tests.test_xml.TestSerializer.test_unpack_inventory_5a_cache_and_copy
+breezy.tests.test_xml.TestSerializer.test_unpack_inventory_5a_cache_no_copy
+breezy.tests.test_xml.TestSerializer.test_unpack_inventory_5b
+breezy.tests.test_xml.TestSerializer.test_unpack_revision_5
+breezy.tests.test_xml.TestSerializer.test_unpack_revision_5_utc
 breezy.tests.test_xml.TestSerializer.test_wrong_format_v7
 breezy.timestamp.format_highres_date
 breezy.transport.http.unhtml_roughly