Merge lp:~twom/launchpad-buildd/include-base-in-digests into lp:launchpad-buildd

[Tom Wardill]

Include base OS information in digests.json

For the final push to a registry, we need to tag the image with the base OS that it is built on.
Grab this information from `/etc/os-release` inside the image, parse it and save it to an appropriate place in the digests.json for processing downstream in Launchpad.

This is a breaking change in the format of digests.json, so needs the equivalent LP branch first.

[lp:~twom/launchpad-buildd/include-base-in-digests]

421. By Tom Wardill on 2020-05-07

Comment fix

422. By Tom Wardill on 2020-05-07

Import ordering

[Thiago F. Pappacena (pappacena)]

LGTM. Thanks!

review: Approve

[lp:~twom/launchpad-buildd/include-base-in-digests]

423. By Tom Wardill on 2020-05-07

Tidy up error handling

[Colin Watson (cjwatson)]

review: Approve

[lp:~twom/launchpad-buildd/include-base-in-digests]

424. By Tom Wardill on 2020-05-07

More exception cleanup

[Thiago F. Pappacena (pappacena)]

LGTM

review: Approve

Unmerged revisions

424. By Tom Wardill on 2020-05-07

More exception cleanup

423. By Tom Wardill on 2020-05-07

Tidy up error handling

422. By Tom Wardill on 2020-05-07

Import ordering

421. By Tom Wardill on 2020-05-07

Comment fix

420. By Tom Wardill on 2020-05-07

Add some debug

419. By Tom Wardill on 2020-05-07

Include base in the digests file

=== modified file 'debian/changelog'
--- debian/changelog	2020-04-09 10:42:50 +0000
+++ debian/changelog	2020-05-07 16:12:43 +0000
@@ -1,3 +1,9 @@
+launchpad-buildd (190) UNRELEASED; urgency=medium
+
+  * Include base in OCI digests
+
+ -- Tom Wardill <[e-mail]>  Thu, 07 May 2020 11:21:58 +0100
+
 launchpad-buildd (189) xenial; urgency=medium
 
   * Fix closing tar files in OCI builds

=== modified file 'lpbuildd/oci.py'
--- lpbuildd/oci.py	2020-04-09 07:51:23 +0000
+++ lpbuildd/oci.py	2020-05-07 16:12:43 +0000
@@ -8,6 +8,7 @@
 import hashlib
 import json
 import os
+from subprocess import CalledProcessError
 import tarfile
 import tempfile
 
@@ -21,6 +22,7 @@
     DebianBuildState,
     )
 from lpbuildd.snap import SnapBuildProxyMixin
+from lpbuildd.target.lxd import LXDException
 
 
 RETCODE_SUCCESS = 0
@@ -147,16 +149,36 @@
 
         return digest_diff_map
 
+    def _extractBase(self):
+        """Extract the base from the os-release file"""
+        os_file = tempfile.NamedTemporaryFile(mode='w+')
+        try:
+            self.backend.copy_out(
+                '/home/buildd/os-release',
+                os_file.name)
+        except (CalledProcessError, KeyError, LXDException) as e:
+            print("Error getting os-release from image: {}".format(e))
+            return ""
+        id = ""
+        version_id = ""
+        os_file.seek(0, os.SEEK_SET)
+        contents = [x.strip() for x in os_file.readlines()]
+        for line in contents:
+            key, value = line.split('=', 1)
+            if key == "ID":
+                id = value
+            elif key == "VERSION_ID":
+                version_id = value.replace("\"", "")
+        return "{}{}".format(id, version_id)
+
+
     def gatherResults(self):
         """Gather the results of the build and add them to the file cache."""
         extract_path = tempfile.mkdtemp(prefix=self.name)
         proc = self.backend.run(
             ['docker', 'save', self.name],
             get_output=True, universal_newlines=False, return_process=True)
-        try:
-            tar = tarfile.open(fileobj=proc.stdout, mode="r|")
-        except Exception as e:
-            print(e)
+        tar = tarfile.open(fileobj=proc.stdout, mode="r|")
 
         current_dir = ''
         directory_tar = None
@@ -187,8 +209,6 @@
                 else:
                     # If it's not in a directory, we need that
                     tar.extract(file, extract_path)
-        except Exception as e:
-            print(e)
         finally:
             if directory_tar is not None:
                 directory_tar.close()
@@ -212,14 +232,14 @@
             manifest = json.load(manifest_fp)
 
         digest_maps = []
-        try:
-            for section in manifest:
-                digest_maps.append(
-                    self._gatherManifestSection(section, extract_path,
-                                                sha_directory))
-            digest_map_file = os.path.join(extract_path, 'digests.json')
-            with open(digest_map_file, 'w') as digest_map_fp:
-                json.dump(digest_maps, digest_map_fp)
-            self._builder.addWaitingFile(digest_map_file)
-        except Exception as e:
-            print(e)
+        for section in manifest:
+            digests = self._gatherManifestSection(
+                section, extract_path, sha_directory)
+            base = self._extractBase()
+            digest_maps.append({
+                "digest_mappings": digests,
+                "base": base})
+        digest_map_file = os.path.join(extract_path, 'digests.json')
+        with open(digest_map_file, 'w') as digest_map_fp:
+            json.dump(digest_maps, digest_map_fp)
+        self._builder.addWaitingFile(digest_map_file)

=== modified file 'lpbuildd/target/build_oci.py'
--- lpbuildd/target/build_oci.py	2020-03-31 13:54:56 +0000
+++ lpbuildd/target/build_oci.py	2020-05-07 16:12:43 +0000
@@ -8,10 +8,12 @@
 from collections import OrderedDict
 import logging
 import os.path
+import subprocess
 import sys
 import tempfile
 from textwrap import dedent
 
+from lpbuildd.target.lxd import LXDException
 from lpbuildd.target.operation import Operation
 from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin
 from lpbuildd.target.snapstore import SnapStoreOperationMixin
@@ -125,6 +127,22 @@
         args.append(self.buildd_path)
         self.run_build_command(args)
 
+        # Copy os-release file out, if it's present
+        args = ["docker", "container", "create",
+                "--name", "os-release-extraction", self.args.name]
+        self.run_build_command(args)
+        # docker cp test_lsb_release:/usr/lib/os-release os-release
+        args = ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
+                "/home/buildd/os-release"]
+        try:
+            self.run_build_command(args)
+            logger.info("os-release file extracted")
+        except (LXDException, subprocess.CalledProcessError) as e:
+            # The os-release file may not exist causing this to error,
+            # we don't care.
+            logger.info("os-release file extraction failed: {}".format(e))
+            pass
+
     def run(self):
         try:
             self.install()

=== modified file 'lpbuildd/target/tests/test_build_oci.py'
--- lpbuildd/target/tests/test_build_oci.py	2020-03-31 14:12:59 +0000
+++ lpbuildd/target/tests/test_build_oci.py	2020-05-07 16:12:43 +0000
@@ -304,6 +304,14 @@
                 ["docker", "build", "--no-cache", "--tag", "test-image",
                  "/home/buildd/test-image"],
                 cwd="/home/buildd/test-image"),
+            RanBuildCommand(
+                ["docker", "container", "create", "--name",
+                 "os-release-extraction", "test-image"],
+                cwd="/home/buildd/test-image"),
+            RanBuildCommand(
+                ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
+                "/home/buildd/os-release"],
+                cwd="/home/buildd/test-image"),
             ]))
 
     def test_build_with_file(self):
@@ -322,6 +330,14 @@
                  "--file", "build-aux/Dockerfile",
                  "/home/buildd/test-image"],
                 cwd="/home/buildd/test-image"),
+            RanBuildCommand(
+                ["docker", "container", "create", "--name",
+                 "os-release-extraction", "test-image"],
+                cwd="/home/buildd/test-image"),
+            RanBuildCommand(
+                ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
+                "/home/buildd/os-release"],
+                cwd="/home/buildd/test-image"),
             ]))
 
     def test_build_proxy(self):
@@ -341,6 +357,14 @@
                  "--build-arg", "https_proxy=http://proxy.example:3128/",
                  "--tag", "test-image", "/home/buildd/test-image"],
                 cwd="/home/buildd/test-image"),
+            RanBuildCommand(
+                ["docker", "container", "create", "--name",
+                 "os-release-extraction", "test-image"],
+                cwd="/home/buildd/test-image"),
+            RanBuildCommand(
+                ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
+                "/home/buildd/os-release"],
+                cwd="/home/buildd/test-image"),
             ]))
 
     def test_run_succeeds(self):
@@ -362,6 +386,14 @@
                 ["docker", "build", "--no-cache", "--tag", "test-image",
                  "/home/buildd/test-image"],
                 cwd="/home/buildd/test-image")),
+            AnyMatch(RanBuildCommand(
+                ["docker", "container", "create", "--name",
+                 "os-release-extraction", "test-image"],
+                cwd="/home/buildd/test-image")),
+            AnyMatch(RanBuildCommand(
+                ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
+                "/home/buildd/os-release"],
+                cwd="/home/buildd/test-image")),
             ))
 
     def test_run_install_fails(self):

=== modified file 'lpbuildd/tests/test_oci.py'
--- lpbuildd/tests/test_oci.py	2020-02-26 10:52:29 +0000
+++ lpbuildd/tests/test_oci.py	2020-05-07 16:12:43 +0000
@@ -135,6 +135,13 @@
             'vfs/distribution/v2metadata-by-diffid/sha256/diff1',
             b"""[{"Digest": "test_digest", "SourceRepository": "test"}]"""
         )
+        self.buildmanager.backend.add_file(
+            '/home/buildd/os-release',
+            b'ID=ubuntu\n'
+            b'ID_LIKE=debian\n'
+            b'PRETTY_NAME="Ubuntu 16.04.6 LTS"\n'
+            b'VERSION_ID="16.04"\n'
+        )
 
         # After building the package, reap processes.
         yield self.buildmanager.iterate(0)
@@ -162,16 +169,18 @@
         with open(cache_path, "rb") as f:
             digests_contents = f.read()
         digests_expected = [{
-            "sha256:diff1": {
-                "source": "test",
-                "digest": "test_digest",
-                "layer_id": "layer-1"
-            },
-            "sha256:diff2": {
-                "source": "",
-                "digest": "testsha",
-                "layer_id": "layer-2"
-            }
+            "digest_mappings": {
+                "sha256:diff1": {
+                    "source": "test",
+                    "digest": "test_digest",
+                    "layer_id": "layer-1"
+                },
+                "sha256:diff2": {
+                    "source": "",
+                    "digest": "testsha",
+                    "layer_id": "layer-2"
+                }},
+            "base": "ubuntu16.04"
         }]
         self.assertEqual(digests_contents, json.dumps(digests_expected))
         # Control returns to the DebianBuildManager in the UMOUNT state.
@@ -245,16 +254,18 @@
         with open(cache_path, "rb") as f:
             digests_contents = f.read()
         digests_expected = [{
-            "sha256:diff1": {
-                "source": "test",
-                "digest": "test_digest",
-                "layer_id": "layer-1"
-            },
-            "sha256:diff2": {
-                "source": "",
-                "digest": "testsha",
-                "layer_id": "layer-2"
-            }
+            "digest_mappings": {
+                "sha256:diff1": {
+                    "source": "test",
+                    "digest": "test_digest",
+                    "layer_id": "layer-1"
+                },
+                "sha256:diff2": {
+                    "source": "",
+                    "digest": "testsha",
+                    "layer_id": "layer-2"
+                }},
+            "base": ""
         }]
         self.assertEqual(digests_contents, json.dumps(digests_expected))