Merge ~racb/git-ubuntu:non-unicode-changelog-notes into git-ubuntu:master

[Robie Basak]

Make Jenkins happy

[Robie Basak (racb)]

Note the linked bug isn't resolved by this - it's a minor bug created as a consequence of this fix that isn't worth working on right now.

[Server Team CI bot (server-team-bot)]

PASSED: Continuous integration, rev:a3cfea75fe552a708ede1a938e6dc119a7699538
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/529/
Executed test runs:
    SUCCESS: VM Setup
    SUCCESS: Build
    SUCCESS: Unit Tests
    IN_PROGRESS: Declarative: Post Actions

Click here to trigger a rebuild:
https://jenkins.ubuntu.com/server/job/git-ubuntu-ci/529//rebuild

review: Approve (continuous-integration)

[Bryce Harrington (bryce)]

Yep, looks good.

I would probably copy in some of the utf8 characters from the broken changelogs to be 100% sure but this proves for at least one unicode character so should be sufficient.

review: Approve

diff --git a/gitubuntu/importer.py b/gitubuntu/importer.py
index 0fe372f..014d6fd 100644
--- a/gitubuntu/importer.py
+++ b/gitubuntu/importer.py
@@ -1530,8 +1530,36 @@ def add_changelog_note_to_commit(
         spec.SYNTHESIZED_COMMITTER_NAME,
         spec.SYNTHESIZED_COMMITTER_EMAIL,
     )
+
+    # pygit2 only accepts a str as the message of the note, but we have
+    # historical changelog entries that used non-unicode encodings from prior
+    # to Debian defining that the file must be UTF-8. Ideally we'd just pass
+    # through the unknown encoding since git doesn't care, but pygit2 doesn't
+    # support this. Instead, for now we can work around by replacing the
+    # un-decodable characters. This does lose fidelity, but since it's in a git
+    # note it does not impact on anything apart from the individual affected
+    # note and such notes can be updated later if required. See
+    # https://github.com/libgit2/pygit2/issues/1021 for the upstream pygit2
+    # issue. The loss in fidelity here is tracked in LP: #1888254.
+    #
+    # Examples of affected changelog entries:
+    #     autoconf2.13 2.13-55
+    #     autogen 1:5.8.3-2
+    #     dbconfig-common 1.8.17
+    #     dpatch 2.0.10
+    #     dput 0.9.2.16ubuntu1
+    #     emacs-goodies-el 24.9-2
+    #     evolution 2.10.1-0ubuntu2
+    #     gmp 2:4.2.2+dfsg-3ubuntu1
+    #     gnome-session 2.17.92-0ubuntu2
+    #     gnupg 1.4.3-2ubuntu1
+    #     iptables 1.3.5.0debian1-1ubuntu1
+    #     jadetex 3.13-2.1ubuntu2
+    #     llvm-toolchain-3.9 1:3.9.1-4ubuntu3~14.04.2
+    changelog_summary_string = changelog_summary.decode(errors='replace')
+
     repo.raw_repo.create_note(
-        changelog_summary.decode(),
+        changelog_summary_string,
         changelog_signature,
         changelog_signature,
         str(commit.id),
diff --git a/gitubuntu/importer_test.py b/gitubuntu/importer_test.py
index 6ba8e52..42b67cb 100644
--- a/gitubuntu/importer_test.py
+++ b/gitubuntu/importer_test.py
@@ -10,6 +10,7 @@ import pytest
 import shutil
 import tempfile
 
+import pkg_resources
 import pygit2
 
 from gitubuntu.patch_state import PatchState
@@ -951,6 +952,54 @@ def test_add_changelog_note_to_commit(repo):
     assert note.message == "    * Test build from source_builder.\n"
 
 
+def test_add_changelog_note_to_commit_utf8(repo):
+    """A changelog file with non-UTF8 should have such characters substituted
+
+    :param repo gitubuntu.git_repository.GitUbuntuRepository: fixture to hold a
+        temporary output repository
+    """
+    test_utf8_error_changelog_path = os.path.join(
+        pkg_resources.resource_filename('gitubuntu', 'changelog_tests'),
+        'test_utf8_error',
+    )
+    with open(test_utf8_error_changelog_path, 'rb') as f:
+        utf8_changelog_blob = f.read()
+
+    # We only need an example child commit with a debian/changelog file since
+    # this is the only file accessed by add_changelog_note_to_commit().
+    # Further, the parent need only exist and can be empty.
+    repo_builder.Repo(
+        commits=[
+            repo_builder.Commit(name='parent'),
+            repo_builder.Commit(
+                tree=repo_builder.Tree({'debian': repo_builder.Tree(
+                    {'changelog': repo_builder.Blob(utf8_changelog_blob)}
+                )}),
+                name='child',
+            )
+        ],
+        tags={'parent': Placeholder('parent'), 'child': Placeholder('child')},
+    ).write(repo.raw_repo)
+
+    parent_commit_ref = repo.raw_repo.lookup_reference('refs/tags/parent')
+    parent_commit = parent_commit_ref.peel(pygit2.Commit)
+
+    child_commit_ref = repo.raw_repo.lookup_reference('refs/tags/child')
+    child_commit = child_commit_ref.peel(pygit2.Commit)
+
+    target.add_changelog_note_to_commit(
+        repo=repo,
+        namespace='importer',
+        commit=child_commit,
+        changelog_parents=[str(parent_commit.id)],
+    )
+    note = repo.raw_repo.lookup_note(
+        str(child_commit.id),
+        'refs/notes/importer/changelog',
+    )
+    assert "Nicol\N{REPLACEMENT CHARACTER}s" in note.message
+
+
 def test_double_changelog_note_add_does_not_fail(repo):
     """add_changelog_note_to_commit shouldn't fail if a note already exists"""
     repo_builder.Repo(