Merge ~pappacena/launchpad-buildd:security-manifest-build-metadata into launchpad-buildd:master

Proposed by Thiago F. Pappacena
Status: Merged
Approved by: Thiago F. Pappacena
Approved revision: d62f592d5af42c136d405d2937dfe235a0761bb3
Merge reported by: Otto Co-Pilot
Merged at revision: not available
Proposed branch: ~pappacena/launchpad-buildd:security-manifest-build-metadata
Merge into: launchpad-buildd:master
Prerequisite: ~pappacena/launchpad-buildd:security-manifest
Diff against target: 216 lines (+142/-4)
3 files modified
lpbuildd/oci.py (+6/-0)
lpbuildd/target/build_oci.py (+53/-4)
lpbuildd/target/tests/test_build_oci.py (+83/-0)
Reviewer Review Type Date Requested Status
Colin Watson Approve
Tom Wardill (community) Approve
Review via email: mp+392564@code.launchpad.net

Commit message

Adding build metadata to OCI security manifest file

To post a comment you must log in.
Revision history for this message
Tom Wardill (twom) :
review: Approve
d62f592... by Thiago F. Pappacena

Logging invalid metadata json on OCI build

Revision history for this message
Thiago F. Pappacena (pappacena) wrote :

Pushed the requested change.

Revision history for this message
Thiago F. Pappacena (pappacena) wrote :

twom, added a MP on Launchpad to cover your comment about a team owning an OCI recipe build: https://code.launchpad.net/~pappacena/launchpad/+git/launchpad/+merge/392713

Revision history for this message
Colin Watson (cjwatson) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/lpbuildd/oci.py b/lpbuildd/oci.py
2index 480c5d6..cdcdab3 100644
3--- a/lpbuildd/oci.py
4+++ b/lpbuildd/oci.py
5@@ -54,6 +54,7 @@ class OCIBuildManager(SnapBuildProxyMixin, DebianBuildManager):
6 self.build_path = extra_args.get("build_path")
7 self.proxy_url = extra_args.get("proxy_url")
8 self.revocation_endpoint = extra_args.get("revocation_endpoint")
9+ self.metadata = extra_args.get("metadata")
10 self.proxy_service = None
11
12 super(OCIBuildManager, self).initiate(files, chroot, extra_args)
13@@ -84,6 +85,11 @@ class OCIBuildManager(SnapBuildProxyMixin, DebianBuildManager):
14 args.extend(["--snap-store-proxy-url", snap_store_proxy_url])
15 except (NoSectionError, NoOptionError):
16 pass
17+ if self.metadata is not None:
18+ try:
19+ args.extend(["--metadata", json.dumps(self.metadata)])
20+ except TypeError as e:
21+ print("Could not JSONify metadata: %s" % e)
22 args.append(self.name)
23 self.runTargetSubProcess("build-oci", *args)
24
25diff --git a/lpbuildd/target/build_oci.py b/lpbuildd/target/build_oci.py
26index 3e4f283..dd372b2 100644
27--- a/lpbuildd/target/build_oci.py
28+++ b/lpbuildd/target/build_oci.py
29@@ -48,6 +48,9 @@ class BuildOCI(SnapBuildProxyOperationMixin, VCSOperationMixin,
30 help="A docker build ARG in the format of key=value. "
31 "This option can be repeated many times. For example: "
32 "--build-arg VAR1=A --build-arg VAR2=B")
33+ parser.add_argument(
34+ "--metadata", default=None,
35+ help="Metadata about this build, used to generate manifest file.")
36 parser.add_argument("name", help="name of snap to build")
37
38 def __init__(self, args, parser):
39@@ -124,14 +127,60 @@ class BuildOCI(SnapBuildProxyOperationMixin, VCSOperationMixin,
40 env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
41 self.vcs_fetch(self.args.name, cwd="/home/buildd", env=env)
42
43+ def _getCurrentVCSRevision(self):
44+ if self.args.branch is not None:
45+ revision_cmd = ["bzr", "revno"]
46+ else:
47+ revision_cmd = ["git", "rev-parse", "HEAD"]
48+ return self.backend.run(
49+ revision_cmd, cwd=os.path.join("/home/buildd", self.args.name),
50+ get_output=True).decode("UTF-8", "replace").strip()
51+
52+ def _getSecurityManifestContent(self):
53+ try:
54+ metadata = json.loads(self.args.metadata) or {}
55+ except TypeError:
56+ logger.warning(
57+ "No valid OCI build metadata received. Expecting a valid "
58+ "JSON, but got %s." % self.args.metadata)
59+ metadata = {}
60+ recipe_owner = metadata.get("recipe_owner", {})
61+ build_requester = metadata.get("build_requester", {})
62+ emails = [i.get("email") for i in (recipe_owner, build_requester)
63+ if i.get("email")]
64+
65+ try:
66+ vcs_current_version = self._getCurrentVCSRevision()
67+ except Exception as e:
68+ logger.warning("Failed to get current VCS revision: %s" % e)
69+ vcs_current_version = None
70+
71+ return {
72+ "manifest-version": "1",
73+ "name": self.args.name,
74+ "architectures": metadata.get("architectures") or [self.args.arch],
75+ "publisher-emails": emails,
76+ "image-info": {
77+ "build-request-id": metadata.get("build_request_id"),
78+ "build-request-timestamp": metadata.get(
79+ "build_request_timestamp"),
80+ "build-urls": metadata.get("build_urls") or {}
81+ },
82+ "vcs-info": [{
83+ "source": self.args.git_repository,
84+ "source-branch": self.args.git_path,
85+ "source-commit": vcs_current_version,
86+ "source-subdir": self.args.build_path,
87+ "source-build-file": self.args.build_file,
88+ "source-build-args": self.args.build_arg
89+ }]
90+ }
91+
92 def createSecurityManifest(self):
93 """Generates the security manifest file, returning the tmp file name
94 where it is stored in the backend.
95 """
96- content = {
97- "manifest-version": "0",
98- "name": self.args.name
99- }
100+ content = self._getSecurityManifestContent()
101 local_filename = tempfile.mktemp()
102 destination_path = self.security_manifest_target_path.lstrip(
103 os.path.sep)
104diff --git a/lpbuildd/target/tests/test_build_oci.py b/lpbuildd/target/tests/test_build_oci.py
105index 87b37b2..0572b4e 100644
106--- a/lpbuildd/target/tests/test_build_oci.py
107+++ b/lpbuildd/target/tests/test_build_oci.py
108@@ -3,6 +3,8 @@
109
110 __metaclass__ = type
111
112+import datetime
113+import json
114 import os.path
115 import stat
116 import subprocess
117@@ -73,6 +75,82 @@ class RanBuildCommand(RanCommand):
118 super(RanBuildCommand, self).__init__(args, **kwargs)
119
120
121+class TestBuildOCIManifestGeneration(TestCase):
122+ def test_getSecurityManifestContent(self):
123+ now = datetime.datetime.now().isoformat()
124+ metadata = {
125+ "architectures": ["amd64", "386"],
126+ "recipe_owner": dict(name="pappacena", email="me@foo.com"),
127+ "build_request_id": 123,
128+ "build_request_timestamp": now,
129+ "build_requester": dict(name="someone", email="someone@foo.com"),
130+ "build_urls": {
131+ "amd64": "http://lp.net/build/1",
132+ "386": "http://lp.net/build/2"
133+ },
134+ }
135+ args = [
136+ "build-oci",
137+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
138+ "--git-repository", "lp:git-repo", "--git-path", "refs/heads/main",
139+ "--build-arg", "VAR1=1", "--build-arg", "VAR2=22",
140+ "--build-file", "SomeDockerfile", "--build-path", "docker/builder",
141+ "--metadata", json.dumps(metadata),
142+ "test-image"
143+ ]
144+ build_oci = parse_args(args=args).operation
145+ build_oci.backend.run.result = b"a1b2c3d4e5f5\n"
146+ self.assertEqual(build_oci._getSecurityManifestContent(), {
147+ "manifest-version": "1",
148+ "name": "test-image",
149+ "architectures": ["amd64", "386"],
150+ "publisher-emails": ["me@foo.com", "someone@foo.com"],
151+ "image-info": {
152+ "build-request-id": 123,
153+ "build-request-timestamp": now,
154+ "build-urls": {
155+ "386": "http://lp.net/build/2",
156+ "amd64": "http://lp.net/build/1"}},
157+ "vcs-info": [{
158+ "source": "lp:git-repo",
159+ "source-branch": "refs/heads/main",
160+ "source-build-args": ["VAR1=1", "VAR2=22"],
161+ "source-build-file": "SomeDockerfile",
162+ "source-commit": "a1b2c3d4e5f5",
163+ "source-subdir": "docker/builder"
164+ }]
165+ })
166+
167+ def test_getSecurityManifestContent_without_manifest(self):
168+ """With minimal parameters, the manifest content should give
169+ something back without breaking."""
170+ args = [
171+ "build-oci",
172+ "--backend=fake", "--series=xenial", "--arch=amd64", "1",
173+ "--git-repository", "lp:git-repo", "--git-path", "refs/heads/main",
174+ "test-image"
175+ ]
176+ build_oci = parse_args(args=args).operation
177+ self.assertEqual(build_oci._getSecurityManifestContent(), {
178+ "manifest-version": "1",
179+ "name": "test-image",
180+ "architectures": ["amd64"],
181+ "publisher-emails": [],
182+ "image-info": {
183+ "build-request-id": None,
184+ "build-request-timestamp": None,
185+ "build-urls": {}},
186+ "vcs-info": [{
187+ "source": "lp:git-repo",
188+ "source-branch": "refs/heads/main",
189+ "source-build-args": [],
190+ "source-build-file": None,
191+ "source-commit": None,
192+ "source-subdir": "."
193+ }]
194+ })
195+
196+
197 class TestBuildOCI(TestCase):
198
199 def test_run_build_command_no_env(self):
200@@ -291,11 +369,16 @@ class TestBuildOCI(TestCase):
201 ]))
202
203 def assertRanPostBuildCommands(self, build_oci):
204+ rev_num_args = (
205+ ['bzr', 'revno'] if build_oci.args.branch
206+ else ['git', 'rev-parse', 'HEAD'])
207 self.assertThat(build_oci.backend.run.calls[1:], MatchesListwise([
208 RanBuildCommand(
209 ['docker', 'create', '--name', 'test-image', 'test-image'],
210 cwd="/home/buildd/test-image"),
211 RanCommand(['mkdir', '-p', '/tmp/image-root-dir/.rocks']),
212+ RanCommand(
213+ rev_num_args, cwd="/home/buildd/test-image", get_output=True),
214 RanBuildCommand(
215 ['docker', 'cp', '/tmp/image-root-dir/.', 'test-image:/'],
216 cwd="/home/buildd/test-image"),

Subscribers

People subscribed via source and target branches