Merge lp:~twom/launchpad-buildd/include-base-in-digests into lp:launchpad-buildd

Proposed by Tom Wardill
Status: Needs review
Proposed branch: lp:~twom/launchpad-buildd/include-base-in-digests
Merge into: lp:launchpad-buildd
Diff against target: 288 lines (+124/-37)
5 files modified
debian/changelog (+6/-0)
lpbuildd/oci.py (+37/-17)
lpbuildd/target/build_oci.py (+18/-0)
lpbuildd/target/tests/test_build_oci.py (+32/-0)
lpbuildd/tests/test_oci.py (+31/-20)
To merge this branch: bzr merge lp:~twom/launchpad-buildd/include-base-in-digests
Reviewer Review Type Date Requested Status
Thiago F. Pappacena (community) Approve
Colin Watson Approve
Review via email: mp+383584@code.launchpad.net

Commit message

Include base OS information in digests.json

Description of the change

For the final push to a registry, we need to tag the image with the base OS that it is built on.
Grab this information from `/etc/os-release` inside the image, parse it and save it to an appropriate place in the digests.json for processing downstream in Launchpad.

This is a breaking change in the format of digests.json, so needs the equivalent LP branch first.

To post a comment you must log in.
421. By Tom Wardill on 2020-05-07

Comment fix

422. By Tom Wardill on 2020-05-07

Import ordering

Revision history for this message
Thiago F. Pappacena (pappacena) wrote :

LGTM. Thanks!

review: Approve
Revision history for this message
Colin Watson (cjwatson) :
Revision history for this message
Colin Watson (cjwatson) :
423. By Tom Wardill on 2020-05-07

Tidy up error handling

Revision history for this message
Colin Watson (cjwatson) :
review: Approve
424. By Tom Wardill on 2020-05-07

More exception cleanup

Revision history for this message
Thiago F. Pappacena (pappacena) wrote :

LGTM

review: Approve

Unmerged revisions

424. By Tom Wardill on 2020-05-07

More exception cleanup

423. By Tom Wardill on 2020-05-07

Tidy up error handling

422. By Tom Wardill on 2020-05-07

Import ordering

421. By Tom Wardill on 2020-05-07

Comment fix

420. By Tom Wardill on 2020-05-07

Add some debug

419. By Tom Wardill on 2020-05-07

Include base in the digests file

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'debian/changelog'
2--- debian/changelog 2020-04-09 10:42:50 +0000
3+++ debian/changelog 2020-05-07 16:12:43 +0000
4@@ -1,3 +1,9 @@
5+launchpad-buildd (190) UNRELEASED; urgency=medium
6+
7+ * Include base in OCI digests
8+
9+ -- Tom Wardill <tom.wardill@canonical.com> Thu, 07 May 2020 11:21:58 +0100
10+
11 launchpad-buildd (189) xenial; urgency=medium
12
13 * Fix closing tar files in OCI builds
14
15=== modified file 'lpbuildd/oci.py'
16--- lpbuildd/oci.py 2020-04-09 07:51:23 +0000
17+++ lpbuildd/oci.py 2020-05-07 16:12:43 +0000
18@@ -8,6 +8,7 @@
19 import hashlib
20 import json
21 import os
22+from subprocess import CalledProcessError
23 import tarfile
24 import tempfile
25
26@@ -21,6 +22,7 @@
27 DebianBuildState,
28 )
29 from lpbuildd.snap import SnapBuildProxyMixin
30+from lpbuildd.target.lxd import LXDException
31
32
33 RETCODE_SUCCESS = 0
34@@ -147,16 +149,36 @@
35
36 return digest_diff_map
37
38+ def _extractBase(self):
39+ """Extract the base from the os-release file"""
40+ os_file = tempfile.NamedTemporaryFile(mode='w+')
41+ try:
42+ self.backend.copy_out(
43+ '/home/buildd/os-release',
44+ os_file.name)
45+ except (CalledProcessError, KeyError, LXDException) as e:
46+ print("Error getting os-release from image: {}".format(e))
47+ return ""
48+ id = ""
49+ version_id = ""
50+ os_file.seek(0, os.SEEK_SET)
51+ contents = [x.strip() for x in os_file.readlines()]
52+ for line in contents:
53+ key, value = line.split('=', 1)
54+ if key == "ID":
55+ id = value
56+ elif key == "VERSION_ID":
57+ version_id = value.replace("\"", "")
58+ return "{}{}".format(id, version_id)
59+
60+
61 def gatherResults(self):
62 """Gather the results of the build and add them to the file cache."""
63 extract_path = tempfile.mkdtemp(prefix=self.name)
64 proc = self.backend.run(
65 ['docker', 'save', self.name],
66 get_output=True, universal_newlines=False, return_process=True)
67- try:
68- tar = tarfile.open(fileobj=proc.stdout, mode="r|")
69- except Exception as e:
70- print(e)
71+ tar = tarfile.open(fileobj=proc.stdout, mode="r|")
72
73 current_dir = ''
74 directory_tar = None
75@@ -187,8 +209,6 @@
76 else:
77 # If it's not in a directory, we need that
78 tar.extract(file, extract_path)
79- except Exception as e:
80- print(e)
81 finally:
82 if directory_tar is not None:
83 directory_tar.close()
84@@ -212,14 +232,14 @@
85 manifest = json.load(manifest_fp)
86
87 digest_maps = []
88- try:
89- for section in manifest:
90- digest_maps.append(
91- self._gatherManifestSection(section, extract_path,
92- sha_directory))
93- digest_map_file = os.path.join(extract_path, 'digests.json')
94- with open(digest_map_file, 'w') as digest_map_fp:
95- json.dump(digest_maps, digest_map_fp)
96- self._builder.addWaitingFile(digest_map_file)
97- except Exception as e:
98- print(e)
99+ for section in manifest:
100+ digests = self._gatherManifestSection(
101+ section, extract_path, sha_directory)
102+ base = self._extractBase()
103+ digest_maps.append({
104+ "digest_mappings": digests,
105+ "base": base})
106+ digest_map_file = os.path.join(extract_path, 'digests.json')
107+ with open(digest_map_file, 'w') as digest_map_fp:
108+ json.dump(digest_maps, digest_map_fp)
109+ self._builder.addWaitingFile(digest_map_file)
110
111=== modified file 'lpbuildd/target/build_oci.py'
112--- lpbuildd/target/build_oci.py 2020-03-31 13:54:56 +0000
113+++ lpbuildd/target/build_oci.py 2020-05-07 16:12:43 +0000
114@@ -8,10 +8,12 @@
115 from collections import OrderedDict
116 import logging
117 import os.path
118+import subprocess
119 import sys
120 import tempfile
121 from textwrap import dedent
122
123+from lpbuildd.target.lxd import LXDException
124 from lpbuildd.target.operation import Operation
125 from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin
126 from lpbuildd.target.snapstore import SnapStoreOperationMixin
127@@ -125,6 +127,22 @@
128 args.append(self.buildd_path)
129 self.run_build_command(args)
130
131+ # Copy os-release file out, if it's present
132+ args = ["docker", "container", "create",
133+ "--name", "os-release-extraction", self.args.name]
134+ self.run_build_command(args)
135+ # docker cp test_lsb_release:/usr/lib/os-release os-release
136+ args = ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
137+ "/home/buildd/os-release"]
138+ try:
139+ self.run_build_command(args)
140+ logger.info("os-release file extracted")
141+ except (LXDException, subprocess.CalledProcessError) as e:
142+ # The os-release file may not exist causing this to error,
143+ # we don't care.
144+ logger.info("os-release file extraction failed: {}".format(e))
145+ pass
146+
147 def run(self):
148 try:
149 self.install()
150
151=== modified file 'lpbuildd/target/tests/test_build_oci.py'
152--- lpbuildd/target/tests/test_build_oci.py 2020-03-31 14:12:59 +0000
153+++ lpbuildd/target/tests/test_build_oci.py 2020-05-07 16:12:43 +0000
154@@ -304,6 +304,14 @@
155 ["docker", "build", "--no-cache", "--tag", "test-image",
156 "/home/buildd/test-image"],
157 cwd="/home/buildd/test-image"),
158+ RanBuildCommand(
159+ ["docker", "container", "create", "--name",
160+ "os-release-extraction", "test-image"],
161+ cwd="/home/buildd/test-image"),
162+ RanBuildCommand(
163+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
164+ "/home/buildd/os-release"],
165+ cwd="/home/buildd/test-image"),
166 ]))
167
168 def test_build_with_file(self):
169@@ -322,6 +330,14 @@
170 "--file", "build-aux/Dockerfile",
171 "/home/buildd/test-image"],
172 cwd="/home/buildd/test-image"),
173+ RanBuildCommand(
174+ ["docker", "container", "create", "--name",
175+ "os-release-extraction", "test-image"],
176+ cwd="/home/buildd/test-image"),
177+ RanBuildCommand(
178+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
179+ "/home/buildd/os-release"],
180+ cwd="/home/buildd/test-image"),
181 ]))
182
183 def test_build_proxy(self):
184@@ -341,6 +357,14 @@
185 "--build-arg", "https_proxy=http://proxy.example:3128/",
186 "--tag", "test-image", "/home/buildd/test-image"],
187 cwd="/home/buildd/test-image"),
188+ RanBuildCommand(
189+ ["docker", "container", "create", "--name",
190+ "os-release-extraction", "test-image"],
191+ cwd="/home/buildd/test-image"),
192+ RanBuildCommand(
193+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
194+ "/home/buildd/os-release"],
195+ cwd="/home/buildd/test-image"),
196 ]))
197
198 def test_run_succeeds(self):
199@@ -362,6 +386,14 @@
200 ["docker", "build", "--no-cache", "--tag", "test-image",
201 "/home/buildd/test-image"],
202 cwd="/home/buildd/test-image")),
203+ AnyMatch(RanBuildCommand(
204+ ["docker", "container", "create", "--name",
205+ "os-release-extraction", "test-image"],
206+ cwd="/home/buildd/test-image")),
207+ AnyMatch(RanBuildCommand(
208+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
209+ "/home/buildd/os-release"],
210+ cwd="/home/buildd/test-image")),
211 ))
212
213 def test_run_install_fails(self):
214
215=== modified file 'lpbuildd/tests/test_oci.py'
216--- lpbuildd/tests/test_oci.py 2020-02-26 10:52:29 +0000
217+++ lpbuildd/tests/test_oci.py 2020-05-07 16:12:43 +0000
218@@ -135,6 +135,13 @@
219 'vfs/distribution/v2metadata-by-diffid/sha256/diff1',
220 b"""[{"Digest": "test_digest", "SourceRepository": "test"}]"""
221 )
222+ self.buildmanager.backend.add_file(
223+ '/home/buildd/os-release',
224+ b'ID=ubuntu\n'
225+ b'ID_LIKE=debian\n'
226+ b'PRETTY_NAME="Ubuntu 16.04.6 LTS"\n'
227+ b'VERSION_ID="16.04"\n'
228+ )
229
230 # After building the package, reap processes.
231 yield self.buildmanager.iterate(0)
232@@ -162,16 +169,18 @@
233 with open(cache_path, "rb") as f:
234 digests_contents = f.read()
235 digests_expected = [{
236- "sha256:diff1": {
237- "source": "test",
238- "digest": "test_digest",
239- "layer_id": "layer-1"
240- },
241- "sha256:diff2": {
242- "source": "",
243- "digest": "testsha",
244- "layer_id": "layer-2"
245- }
246+ "digest_mappings": {
247+ "sha256:diff1": {
248+ "source": "test",
249+ "digest": "test_digest",
250+ "layer_id": "layer-1"
251+ },
252+ "sha256:diff2": {
253+ "source": "",
254+ "digest": "testsha",
255+ "layer_id": "layer-2"
256+ }},
257+ "base": "ubuntu16.04"
258 }]
259 self.assertEqual(digests_contents, json.dumps(digests_expected))
260 # Control returns to the DebianBuildManager in the UMOUNT state.
261@@ -245,16 +254,18 @@
262 with open(cache_path, "rb") as f:
263 digests_contents = f.read()
264 digests_expected = [{
265- "sha256:diff1": {
266- "source": "test",
267- "digest": "test_digest",
268- "layer_id": "layer-1"
269- },
270- "sha256:diff2": {
271- "source": "",
272- "digest": "testsha",
273- "layer_id": "layer-2"
274- }
275+ "digest_mappings": {
276+ "sha256:diff1": {
277+ "source": "test",
278+ "digest": "test_digest",
279+ "layer_id": "layer-1"
280+ },
281+ "sha256:diff2": {
282+ "source": "",
283+ "digest": "testsha",
284+ "layer_id": "layer-2"
285+ }},
286+ "base": ""
287 }]
288 self.assertEqual(digests_contents, json.dumps(digests_expected))
289

Subscribers

People subscribed via source and target branches