Merge ~ilasc/launchpad-buildd:gather-dpkg-yaml into launchpad-buildd:master

[Ioana Lasc]

Gather dpkg.yaml for snap builds

[Colin Watson (cjwatson)]

review: Needs Information

[Ioana Lasc (ilasc)]

Indentation fixed and PRs created to get the dpkg.yaml file copied into the top folder for Buildd to be able to gather it:
https://github.com/snapcore/core18/pull/181
https://github.com/snapcore/core20/pull/113
https://github.com/snapcore/core/pull/125

First 2 PRs are already approved and merged.
Purely from an "order of landing" perspective I don't think we need to wait for all those to be merged however to land this as Buildd just won't gather dpkg.yaml if is not in the expected folder.

MP ready for another look.

[Colin Watson (cjwatson)]

review: Approve

[Colin Watson (cjwatson)]

Could you also add an entry to debian/changelog describing this change before landing it? You should be able to use 'dch' to sort out the syntax.

[Ioana Lasc (ilasc)]

Done, thanks Colin!

diff --git a/debian/changelog b/debian/changelog
index df18746..a7ffa63 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -4,6 +4,9 @@ launchpad-buildd (202) UNRELEASED; urgency=medium
   * Add git_shallow_clone option to vcs_fetch() and use it for OCI build
     (LP: #1939392).
 
+  [ Ioana Lasc ]
+  * Gather dpkg.yaml for snap builds.
+
  -- Colin Watson <[e-mail]>  Thu, 09 Sep 2021 12:58:06 +0100
 
 launchpad-buildd (201) bionic; urgency=medium
diff --git a/lpbuildd/snap.py b/lpbuildd/snap.py
index 2a5ffc4..5afead1 100644
--- a/lpbuildd/snap.py
+++ b/lpbuildd/snap.py
@@ -141,7 +141,8 @@ class SnapBuildManager(BuildManagerProxyMixin, DebianBuildManager):
                 path = os.path.join(output_path, entry)
                 if self.backend.islink(path):
                     continue
-                if entry.endswith(".snap") or entry.endswith(".manifest"):
+                if (entry.endswith(".snap") or entry.endswith(".manifest")
+                        or entry.endswith(".dpkg.yaml")):
                     self.addWaitingFileFromBackend(path)
         if self.build_source_tarball:
             source_tarball_path = os.path.join(
diff --git a/lpbuildd/tests/test_snap.py b/lpbuildd/tests/test_snap.py
index b018a53..5aa0671 100644
--- a/lpbuildd/tests/test_snap.py
+++ b/lpbuildd/tests/test_snap.py
@@ -224,6 +224,67 @@ class TestSnapBuildManagerIteration(TestCase):
         self.assertFalse(self.builder.wasCalled("buildFail"))
 
     @defer.inlineCallbacks
+    def test_iterate_with_dpkg_yaml(self):
+        # The build manager iterates a build that uploads dpkg.yaml from
+        # start to finish.
+        args = {
+            "git_repository": "https://git.launchpad.dev/~example/+git/snap",
+            "git_path": "master",
+            }
+        expected_options = [
+            "--git-repository", "https://git.launchpad.dev/~example/+git/snap",
+            "--git-path", "master",
+            ]
+        yield self.startBuild(args, expected_options)
+
+        log_path = os.path.join(self.buildmanager._cachepath, "buildlog")
+        with open(log_path, "w") as log:
+            log.write("I am a build log.")
+
+        self.buildmanager.backend.add_file(
+            "/build/test-snap/test-snap_0_all.snap", b"I am a snap package.")
+        self.buildmanager.backend.add_file(
+            "/build/test-snap/test-snap_0_all.manifest", b"I am a manifest.")
+        self.buildmanager.backend.add_file(
+            "/build/test-snap/test-snap_0_all.dpkg.yaml", b"I am a yaml file.")
+
+        # After building the package, reap processes.
+        yield self.buildmanager.iterate(0)
+        expected_command = [
+            "sharepath/bin/in-target", "in-target", "scan-for-processes",
+            "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid,
+            ]
+        self.assertEqual(SnapBuildState.BUILD_SNAP, self.getState())
+        self.assertEqual(expected_command, self.buildmanager.commands[-1])
+        self.assertNotEqual(
+            self.buildmanager.iterate, self.buildmanager.iterators[-1])
+        self.assertFalse(self.builder.wasCalled("buildFail"))
+        self.assertThat(self.builder, HasWaitingFiles.byEquality({
+            "test-snap_0_all.manifest": b"I am a manifest.",
+            "test-snap_0_all.snap": b"I am a snap package.",
+            "test-snap_0_all.dpkg.yaml": b"I am a yaml file.",
+            }))
+        # Ensure we don't just gather any yaml file but exactly
+        # the dpkg yaml.
+        self.assertNotEqual(self.builder, HasWaitingFiles.byEquality({
+            "test-snap_0_all.manifest": b"I am a manifest.",
+            "test-snap_0_all.snap": b"I am a snap package.",
+            "test-snap_0_all.snapcraft.yaml": b"I am a yaml file.",
+            }))
+
+        # Control returns to the DebianBuildManager in the UMOUNT state.
+        self.buildmanager.iterateReap(self.getState(), 0)
+        expected_command = [
+            "sharepath/bin/in-target", "in-target", "umount-chroot",
+            "--backend=lxd", "--series=xenial", "--arch=i386", self.buildid,
+            ]
+        self.assertEqual(SnapBuildState.UMOUNT, self.getState())
+        self.assertEqual(expected_command, self.buildmanager.commands[-1])
+        self.assertEqual(
+            self.buildmanager.iterate, self.buildmanager.iterators[-1])
+        self.assertFalse(self.builder.wasCalled("buildFail"))
+
+    @defer.inlineCallbacks
     def test_iterate_with_channels(self):
         # The build manager iterates a build that specifies channels from
         # start to finish.