Merge lp:~twom/launchpad-buildd/include-base-in-digests into lp:launchpad-buildd

Proposed by Tom Wardill
Status: Needs review
Proposed branch: lp:~twom/launchpad-buildd/include-base-in-digests
Merge into: lp:launchpad-buildd
Diff against target: 288 lines (+124/-37)
5 files modified
debian/changelog (+6/-0)
lpbuildd/oci.py (+37/-17)
lpbuildd/target/build_oci.py (+18/-0)
lpbuildd/target/tests/test_build_oci.py (+32/-0)
lpbuildd/tests/test_oci.py (+31/-20)
To merge this branch: bzr merge lp:~twom/launchpad-buildd/include-base-in-digests
Reviewer Review Type Date Requested Status
Thiago F. Pappacena (community) Approve
Colin Watson Approve
Review via email: mp+383584@code.launchpad.net

Commit message

Include base OS information in digests.json

Description of the change

For the final push to a registry, we need to tag the image with the base OS that it is built on.
Grab this information from `/etc/os-release` inside the image, parse it and save it to an appropriate place in the digests.json for processing downstream in Launchpad.

This is a breaking change in the format of digests.json, so needs the equivalent LP branch first.

To post a comment you must log in.
421. By Tom Wardill

Comment fix

422. By Tom Wardill

Import ordering

Revision history for this message
Thiago F. Pappacena (pappacena) wrote :

LGTM. Thanks!

review: Approve
Revision history for this message
Colin Watson (cjwatson) :
Revision history for this message
Colin Watson (cjwatson) :
423. By Tom Wardill

Tidy up error handling

Revision history for this message
Colin Watson (cjwatson) :
review: Approve
424. By Tom Wardill

More exception cleanup

Revision history for this message
Thiago F. Pappacena (pappacena) wrote :

LGTM

review: Approve

Unmerged revisions

424. By Tom Wardill

More exception cleanup

423. By Tom Wardill

Tidy up error handling

422. By Tom Wardill

Import ordering

421. By Tom Wardill

Comment fix

420. By Tom Wardill

Add some debug

419. By Tom Wardill

Include base in the digests file

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'debian/changelog'
2--- debian/changelog 2020-04-09 10:42:50 +0000
3+++ debian/changelog 2020-05-07 16:12:43 +0000
4@@ -1,3 +1,9 @@
5+launchpad-buildd (190) UNRELEASED; urgency=medium
6+
7+ * Include base in OCI digests
8+
9+ -- Tom Wardill <tom.wardill@canonical.com> Thu, 07 May 2020 11:21:58 +0100
10+
11 launchpad-buildd (189) xenial; urgency=medium
12
13 * Fix closing tar files in OCI builds
14
15=== modified file 'lpbuildd/oci.py'
16--- lpbuildd/oci.py 2020-04-09 07:51:23 +0000
17+++ lpbuildd/oci.py 2020-05-07 16:12:43 +0000
18@@ -8,6 +8,7 @@
19 import hashlib
20 import json
21 import os
22+from subprocess import CalledProcessError
23 import tarfile
24 import tempfile
25
26@@ -21,6 +22,7 @@
27 DebianBuildState,
28 )
29 from lpbuildd.snap import SnapBuildProxyMixin
30+from lpbuildd.target.lxd import LXDException
31
32
33 RETCODE_SUCCESS = 0
34@@ -147,16 +149,36 @@
35
36 return digest_diff_map
37
38+ def _extractBase(self):
39+ """Extract the base from the os-release file"""
40+ os_file = tempfile.NamedTemporaryFile(mode='w+')
41+ try:
42+ self.backend.copy_out(
43+ '/home/buildd/os-release',
44+ os_file.name)
45+ except (CalledProcessError, KeyError, LXDException) as e:
46+ print("Error getting os-release from image: {}".format(e))
47+ return ""
48+ id = ""
49+ version_id = ""
50+ os_file.seek(0, os.SEEK_SET)
51+ contents = [x.strip() for x in os_file.readlines()]
52+ for line in contents:
53+ key, value = line.split('=', 1)
54+ if key == "ID":
55+ id = value
56+ elif key == "VERSION_ID":
57+ version_id = value.replace("\"", "")
58+ return "{}{}".format(id, version_id)
59+
60+
61 def gatherResults(self):
62 """Gather the results of the build and add them to the file cache."""
63 extract_path = tempfile.mkdtemp(prefix=self.name)
64 proc = self.backend.run(
65 ['docker', 'save', self.name],
66 get_output=True, universal_newlines=False, return_process=True)
67- try:
68- tar = tarfile.open(fileobj=proc.stdout, mode="r|")
69- except Exception as e:
70- print(e)
71+ tar = tarfile.open(fileobj=proc.stdout, mode="r|")
72
73 current_dir = ''
74 directory_tar = None
75@@ -187,8 +209,6 @@
76 else:
77 # If it's not in a directory, we need that
78 tar.extract(file, extract_path)
79- except Exception as e:
80- print(e)
81 finally:
82 if directory_tar is not None:
83 directory_tar.close()
84@@ -212,14 +232,14 @@
85 manifest = json.load(manifest_fp)
86
87 digest_maps = []
88- try:
89- for section in manifest:
90- digest_maps.append(
91- self._gatherManifestSection(section, extract_path,
92- sha_directory))
93- digest_map_file = os.path.join(extract_path, 'digests.json')
94- with open(digest_map_file, 'w') as digest_map_fp:
95- json.dump(digest_maps, digest_map_fp)
96- self._builder.addWaitingFile(digest_map_file)
97- except Exception as e:
98- print(e)
99+ for section in manifest:
100+ digests = self._gatherManifestSection(
101+ section, extract_path, sha_directory)
102+ base = self._extractBase()
103+ digest_maps.append({
104+ "digest_mappings": digests,
105+ "base": base})
106+ digest_map_file = os.path.join(extract_path, 'digests.json')
107+ with open(digest_map_file, 'w') as digest_map_fp:
108+ json.dump(digest_maps, digest_map_fp)
109+ self._builder.addWaitingFile(digest_map_file)
110
111=== modified file 'lpbuildd/target/build_oci.py'
112--- lpbuildd/target/build_oci.py 2020-03-31 13:54:56 +0000
113+++ lpbuildd/target/build_oci.py 2020-05-07 16:12:43 +0000
114@@ -8,10 +8,12 @@
115 from collections import OrderedDict
116 import logging
117 import os.path
118+import subprocess
119 import sys
120 import tempfile
121 from textwrap import dedent
122
123+from lpbuildd.target.lxd import LXDException
124 from lpbuildd.target.operation import Operation
125 from lpbuildd.target.snapbuildproxy import SnapBuildProxyOperationMixin
126 from lpbuildd.target.snapstore import SnapStoreOperationMixin
127@@ -125,6 +127,22 @@
128 args.append(self.buildd_path)
129 self.run_build_command(args)
130
131+ # Copy os-release file out, if it's present
132+ args = ["docker", "container", "create",
133+ "--name", "os-release-extraction", self.args.name]
134+ self.run_build_command(args)
135+ # docker cp test_lsb_release:/usr/lib/os-release os-release
136+ args = ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
137+ "/home/buildd/os-release"]
138+ try:
139+ self.run_build_command(args)
140+ logger.info("os-release file extracted")
141+ except (LXDException, subprocess.CalledProcessError) as e:
142+ # The os-release file may not exist causing this to error,
143+ # we don't care.
144+ logger.info("os-release file extraction failed: {}".format(e))
145+ pass
146+
147 def run(self):
148 try:
149 self.install()
150
151=== modified file 'lpbuildd/target/tests/test_build_oci.py'
152--- lpbuildd/target/tests/test_build_oci.py 2020-03-31 14:12:59 +0000
153+++ lpbuildd/target/tests/test_build_oci.py 2020-05-07 16:12:43 +0000
154@@ -304,6 +304,14 @@
155 ["docker", "build", "--no-cache", "--tag", "test-image",
156 "/home/buildd/test-image"],
157 cwd="/home/buildd/test-image"),
158+ RanBuildCommand(
159+ ["docker", "container", "create", "--name",
160+ "os-release-extraction", "test-image"],
161+ cwd="/home/buildd/test-image"),
162+ RanBuildCommand(
163+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
164+ "/home/buildd/os-release"],
165+ cwd="/home/buildd/test-image"),
166 ]))
167
168 def test_build_with_file(self):
169@@ -322,6 +330,14 @@
170 "--file", "build-aux/Dockerfile",
171 "/home/buildd/test-image"],
172 cwd="/home/buildd/test-image"),
173+ RanBuildCommand(
174+ ["docker", "container", "create", "--name",
175+ "os-release-extraction", "test-image"],
176+ cwd="/home/buildd/test-image"),
177+ RanBuildCommand(
178+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
179+ "/home/buildd/os-release"],
180+ cwd="/home/buildd/test-image"),
181 ]))
182
183 def test_build_proxy(self):
184@@ -341,6 +357,14 @@
185 "--build-arg", "https_proxy=http://proxy.example:3128/",
186 "--tag", "test-image", "/home/buildd/test-image"],
187 cwd="/home/buildd/test-image"),
188+ RanBuildCommand(
189+ ["docker", "container", "create", "--name",
190+ "os-release-extraction", "test-image"],
191+ cwd="/home/buildd/test-image"),
192+ RanBuildCommand(
193+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
194+ "/home/buildd/os-release"],
195+ cwd="/home/buildd/test-image"),
196 ]))
197
198 def test_run_succeeds(self):
199@@ -362,6 +386,14 @@
200 ["docker", "build", "--no-cache", "--tag", "test-image",
201 "/home/buildd/test-image"],
202 cwd="/home/buildd/test-image")),
203+ AnyMatch(RanBuildCommand(
204+ ["docker", "container", "create", "--name",
205+ "os-release-extraction", "test-image"],
206+ cwd="/home/buildd/test-image")),
207+ AnyMatch(RanBuildCommand(
208+ ["docker", "cp", "-L", "os-release-extraction:/etc/os-release",
209+ "/home/buildd/os-release"],
210+ cwd="/home/buildd/test-image")),
211 ))
212
213 def test_run_install_fails(self):
214
215=== modified file 'lpbuildd/tests/test_oci.py'
216--- lpbuildd/tests/test_oci.py 2020-02-26 10:52:29 +0000
217+++ lpbuildd/tests/test_oci.py 2020-05-07 16:12:43 +0000
218@@ -135,6 +135,13 @@
219 'vfs/distribution/v2metadata-by-diffid/sha256/diff1',
220 b"""[{"Digest": "test_digest", "SourceRepository": "test"}]"""
221 )
222+ self.buildmanager.backend.add_file(
223+ '/home/buildd/os-release',
224+ b'ID=ubuntu\n'
225+ b'ID_LIKE=debian\n'
226+ b'PRETTY_NAME="Ubuntu 16.04.6 LTS"\n'
227+ b'VERSION_ID="16.04"\n'
228+ )
229
230 # After building the package, reap processes.
231 yield self.buildmanager.iterate(0)
232@@ -162,16 +169,18 @@
233 with open(cache_path, "rb") as f:
234 digests_contents = f.read()
235 digests_expected = [{
236- "sha256:diff1": {
237- "source": "test",
238- "digest": "test_digest",
239- "layer_id": "layer-1"
240- },
241- "sha256:diff2": {
242- "source": "",
243- "digest": "testsha",
244- "layer_id": "layer-2"
245- }
246+ "digest_mappings": {
247+ "sha256:diff1": {
248+ "source": "test",
249+ "digest": "test_digest",
250+ "layer_id": "layer-1"
251+ },
252+ "sha256:diff2": {
253+ "source": "",
254+ "digest": "testsha",
255+ "layer_id": "layer-2"
256+ }},
257+ "base": "ubuntu16.04"
258 }]
259 self.assertEqual(digests_contents, json.dumps(digests_expected))
260 # Control returns to the DebianBuildManager in the UMOUNT state.
261@@ -245,16 +254,18 @@
262 with open(cache_path, "rb") as f:
263 digests_contents = f.read()
264 digests_expected = [{
265- "sha256:diff1": {
266- "source": "test",
267- "digest": "test_digest",
268- "layer_id": "layer-1"
269- },
270- "sha256:diff2": {
271- "source": "",
272- "digest": "testsha",
273- "layer_id": "layer-2"
274- }
275+ "digest_mappings": {
276+ "sha256:diff1": {
277+ "source": "test",
278+ "digest": "test_digest",
279+ "layer_id": "layer-1"
280+ },
281+ "sha256:diff2": {
282+ "source": "",
283+ "digest": "testsha",
284+ "layer_id": "layer-2"
285+ }},
286+ "base": ""
287 }]
288 self.assertEqual(digests_contents, json.dumps(digests_expected))
289

Subscribers

People subscribed via source and target branches