Merge ~rhansen/git-build-recipe:pristine-tar-refs into git-build-recipe:master

[Richard Hansen]

Fetch all references so that the pristine-tar trees are guaranteed to be fetched.

This is an alternative to:
https://code.launchpad.net/~lool/git-build-recipe/+git/git-build-recipe/+merge/443942

Changes from that merge request:
  - Tags are put in refs/tags/<remote>/* instead of refs/remotes/<remote>/* (to avoid potential name collisions).
  - I added some comments explaining why fetching is done the way it is.
  - I added tests.

Unmerged commits

39f83d9... by Richard Hansen on 2023-07-09

fetch everything for pristine-tar

diff --git a/gitbuildrecipe/recipe.py b/gitbuildrecipe/recipe.py
index f932ca7..f8ffd45 100644
--- a/gitbuildrecipe/recipe.py
+++ b/gitbuildrecipe/recipe.py
@@ -339,11 +339,34 @@ def fetch_branches(child_branch):
         logging.info(
             "Failed to fetch HEAD; recipe instructions for this repository "
             "that do not specify a branch name will fail.")
-    # Fetch all remote branches and (implicitly) any tags that reference
-    # commits in those refs. Tags that aren't on a branch won't be fetched.
+    # Fetch all remote branches and (implicitly) any tags in the commit history
+    # of the remote branches.  Tags that aren't on a branch will be fetched in a
+    # separate fetch call below.  Note that if two remotes have an identically
+    # named tag, the tag from the first fetched remote wins; the second fetch
+    # will silently skip the tag (fetch will not return an error).
+    # Note: extract_upstream_tarball looks for the tags upstream/<version> and
+    # upstream-<version> if the <remote>/pristine-tar branch doesn't exist.
     child_branch.git_call(
         "fetch", url,
         "refs/heads/*:refs/remotes/%s/*" % child_branch.remote_name)
+    # All refs must be fetched for pristine-tar to work because pristine-tar
+    # stores Git tree object IDs in blobs, not as tree objects referenced from
+    # the pristine-tar commits.  Git has no way to know that the blobs reference
+    # the tree objects, so it won't include the trees in the fetched packfile
+    # unless another fetched ref brings them along.  The trees are likely to be
+    # in the upstream branch (already fetched above), but they could
+    # theoretically be anywhere.
+    #
+    # The tags are prefixed with the remote name to avoid collision with other
+    # remotes.  The refspec could instead be the usual
+    # "refs/tags/*:refs/tags/*", but then fetch would fail if two remotes have
+    # an identically named tag pointing at different commits.  The refspec could
+    # be prefixed with "+" (or pass "--force"), but that would change the
+    # semantics from first-fetched-wins to last-fetched-wins, which might break
+    # existing recipes.
+    child_branch.git_call(
+        "fetch", url,
+        "refs/tags/*:refs/tags/%s/*" % child_branch.remote_name)
 
 
 @lru_cache(maxsize=1)
diff --git a/gitbuildrecipe/tests/test_functional.py b/gitbuildrecipe/tests/test_functional.py
index 56fd216..33ed43d 100644
--- a/gitbuildrecipe/tests/test_functional.py
+++ b/gitbuildrecipe/tests/test_functional.py
@@ -579,3 +579,45 @@ class FunctionalBuilderTests(GitTestCase):
         out, err = self.run_recipe(
             "--allow-fallback-to-native test.recipe working", retcode=1)
         self.assertIn("Unknown source format 2.0\n", err)
+
+    def test_all_refs_fetched(self):
+        source = self.make_simple_package("source")
+        main_branch = source._git_output(
+            "branch", "--show-current").rstrip("\n")
+        source._git_call("switch", "--orphan", "branch-for-unconnected-tag")
+        source.build_tree(["a"])
+        source.add(["a"])
+        commit = source.commit("unconnected tag")
+        source.tag("unconnected-tag", commit)
+        source._git_call("switch", main_branch)
+        source._git_call("branch", "-D", "branch-for-unconnected-tag")
+        with open("package-0.1.recipe", "w") as recipe:
+            recipe.write(
+                "# git-build-recipe format 0.4 deb-version 0.1-1\n"
+                "source\n")
+        self.run_recipe("--allow-fallback-to-native package-0.1.recipe working")
+        working = GitRepository("working/package-0.1", allow_create=False)
+        working.rev_parse(commit)
+        working.rev_parse("refs/tags/unconnected-tag")
+
+    def test_tag_conflict_first_fetched_wins(self):
+        # If a tag exists in two remote repositories pointing to different
+        # revisions, the first fetched repository "wins" (its tag appears in the
+        # clone), and the losing tag is silently ignored.  This is probably not
+        # the ideal behavior, but changing it would require bumping the format
+        # version number or else it might break existing recipes.
+        source = GitRepository("source")
+        source.build_tree(["a"])
+        source.add(["a"])
+        commit = source.commit("pristine")
+        source.tag("upstream/0.1", commit)
+        debianized = self.make_simple_package("debianized")
+        debianized.tag("upstream/0.1", "HEAD")
+        with open("package-0.1.recipe", "w") as recipe:
+            recipe.write(
+                "# git-build-recipe format 0.4 deb-version 0.1-1\n"
+                "source\n"
+                "nest-part debianized debianized debian\n")
+        self.run_recipe("package-0.1.recipe working")
+        working = GitRepository("working/package-0.1", allow_create=False)
+        self.assertEqual(working.rev_parse("refs/tags/upstream/0.1"), commit)