Merge ~cjwatson/launchpad-buildd:ci-clamav into launchpad-buildd:master

Proposed by Colin Watson
Status: Merged
Approved by: Colin Watson
Approved revision: 97169251049741e9a5656c2c2ee3374ec461e05e
Merge reported by: Otto Co-Pilot
Merged at revision: not available
Proposed branch: ~cjwatson/launchpad-buildd:ci-clamav
Merge into: launchpad-buildd:master
Diff against target: 245 lines (+132/-1)
5 files modified
debian/changelog (+7/-0)
lpbuildd/ci.py (+5/-0)
lpbuildd/target/run_ci.py (+28/-0)
lpbuildd/target/tests/test_run_ci.py (+88/-0)
lpbuildd/tests/test_ci.py (+4/-1)
Reviewer Review Type Date Requested Status
Ioana Lasc Approve
Review via email: mp+430040@code.launchpad.net

Commit message

Add optional malware scanning at the end of CI build jobs

Description of the change

This is currently implemented using clamav. It's probably not yet amazingly effective, and I expect we'd need to start doing on-access scanning in order to get much better, but it gives us a starting point.

To post a comment you must log in.
Revision history for this message
Ioana Lasc (ilasc) wrote :

Looks good

review: Approve
Revision history for this message
Otto Co-Pilot (otto-copilot) wrote :

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/debian/changelog b/debian/changelog
2index f6978de..f271b43 100644
3--- a/debian/changelog
4+++ b/debian/changelog
5@@ -1,3 +1,10 @@
6+launchpad-buildd (223) UNRELEASED; urgency=medium
7+
8+ * Add optional malware scanning at the end of CI build jobs, currently
9+ implemented using clamav.
10+
11+ -- Colin Watson <cjwatson@ubuntu.com> Fri, 30 Sep 2022 11:37:20 +0100
12+
13 launchpad-buildd (222) focal; urgency=medium
14
15 * Remove use of six.
16diff --git a/lpbuildd/ci.py b/lpbuildd/ci.py
17index c49a24c..6083298 100644
18--- a/lpbuildd/ci.py
19+++ b/lpbuildd/ci.py
20@@ -64,6 +64,7 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
21 self.environment_variables = extra_args.get("environment_variables")
22 self.plugin_settings = extra_args.get("plugin_settings")
23 self.secrets = extra_args.get("secrets")
24+ self.scan_malware = extra_args.get("scan_malware", False)
25
26 super().initiate(files, chroot, extra_args)
27
28@@ -82,6 +83,8 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
29 args.extend(["--git-repository", self.git_repository])
30 if self.git_path is not None:
31 args.extend(["--git-path", self.git_path])
32+ if self.scan_malware:
33+ args.append("--scan-malware")
34 try:
35 snap_store_proxy_url = self._builder._config.get(
36 "proxy", "snapstore")
37@@ -164,6 +167,8 @@ class CIBuildManager(BuildManagerProxyMixin, DebianBuildManager):
38 )
39 args.extend(
40 ["--secrets", "/build/.launchpad-secrets.yaml"])
41+ if self.scan_malware:
42+ args.append("--scan-malware")
43
44 job_name, job_index = self.current_job
45 self.current_job_id = _make_job_id(job_name, job_index)
46diff --git a/lpbuildd/target/run_ci.py b/lpbuildd/target/run_ci.py
47index 8f23b01..491943b 100644
48--- a/lpbuildd/target/run_ci.py
49+++ b/lpbuildd/target/run_ci.py
50@@ -31,6 +31,12 @@ class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin,
51 parser.add_argument(
52 "--channel", action=SnapChannelsAction, metavar="SNAP=CHANNEL",
53 dest="channels", default={}, help="install SNAP from CHANNEL")
54+ parser.add_argument(
55+ "--scan-malware",
56+ action="store_true",
57+ default=False,
58+ help="perform malware scans on output files",
59+ )
60
61 def install(self):
62 logger.info("Running install phase...")
63@@ -43,6 +49,8 @@ class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin,
64 if self.backend.is_package_available(dep):
65 deps.append(dep)
66 deps.extend(self.vcs_deps)
67+ if self.args.scan_malware:
68+ deps.append("clamav")
69 self.backend.run(["apt-get", "-y", "install"] + deps)
70 if self.backend.supports_snapd:
71 self.snap_store_set_proxy()
72@@ -59,6 +67,16 @@ class RunCIPrepare(BuilderProxyOperationMixin, VCSOperationMixin,
73 cmd.append(snap_name)
74 self.backend.run(cmd)
75 self.backend.run(["lxd", "init", "--auto"])
76+ if self.args.scan_malware:
77+ # lpbuildd.target.lxd configures the container not to run most
78+ # services, which is convenient since it allows us to ensure
79+ # that ClamAV's database is up to date before proceeding.
80+ kwargs = {}
81+ env = self.build_proxy_environment(proxy_url=self.args.proxy_url)
82+ if env:
83+ kwargs["env"] = env
84+ logger.info("Downloading malware definitions...")
85+ self.backend.run(["freshclam", "--quiet"], **kwargs)
86
87 def repo(self):
88 """Collect VCS branch."""
89@@ -121,6 +139,12 @@ class RunCI(BuilderProxyOperationMixin, Operation):
90 type=str,
91 help="secrets where the key and the value are separated by =",
92 )
93+ parser.add_argument(
94+ "--scan-malware",
95+ action="store_true",
96+ default=False,
97+ help="perform malware scans on output files",
98+ )
99
100 def run_job(self):
101 logger.info("Running job phase...")
102@@ -172,6 +196,10 @@ class RunCI(BuilderProxyOperationMixin, Operation):
103 ]
104 self.run_build_command(args, env=env)
105
106+ if self.args.scan_malware:
107+ clamscan = ["clamscan", "--recursive", job_output_path]
108+ self.run_build_command(clamscan, env=env)
109+
110 def run(self):
111 try:
112 self.run_job()
113diff --git a/lpbuildd/target/tests/test_run_ci.py b/lpbuildd/target/tests/test_run_ci.py
114index 0b0b77b..ba941ea 100644
115--- a/lpbuildd/target/tests/test_run_ci.py
116+++ b/lpbuildd/target/tests/test_run_ci.py
117@@ -141,6 +141,53 @@ class TestRunCIPrepare(TestCase):
118 RanCommand(["lxd", "init", "--auto"]),
119 ]))
120
121+ def test_install_scan_malware(self):
122+ args = [
123+ "run-ci-prepare",
124+ "--backend=fake", "--series=focal", "--arch=amd64", "1",
125+ "--git-repository", "lp:foo",
126+ "--scan-malware",
127+ ]
128+ run_ci_prepare = parse_args(args=args).operation
129+ run_ci_prepare.install()
130+ self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
131+ RanAptGet("install", "git", "clamav"),
132+ RanSnap("install", "lxd"),
133+ RanSnap("install", "--classic", "lpcraft"),
134+ RanCommand(["lxd", "init", "--auto"]),
135+ RanCommand(["freshclam", "--quiet"]),
136+ ]))
137+
138+ def test_install_scan_malware_proxy(self):
139+ args = [
140+ "run-ci-prepare",
141+ "--backend=fake", "--series=focal", "--arch=amd64", "1",
142+ "--git-repository", "lp:foo",
143+ "--proxy-url", "http://proxy.example:3128/",
144+ "--scan-malware",
145+ ]
146+ run_ci_prepare = parse_args(args=args).operation
147+ run_ci_prepare.bin = "/builderbin"
148+ self.useFixture(FakeFilesystem()).add("/builderbin")
149+ os.mkdir("/builderbin")
150+ with open("/builderbin/lpbuildd-git-proxy", "w") as proxy_script:
151+ proxy_script.write("proxy script\n")
152+ os.fchmod(proxy_script.fileno(), 0o755)
153+ run_ci_prepare.install()
154+ env = {
155+ "http_proxy": "http://proxy.example:3128/",
156+ "https_proxy": "http://proxy.example:3128/",
157+ "GIT_PROXY_COMMAND": "/usr/local/bin/lpbuildd-git-proxy",
158+ "SNAPPY_STORE_NO_CDN": "1",
159+ }
160+ self.assertThat(run_ci_prepare.backend.run.calls, MatchesListwise([
161+ RanAptGet("install", "python3", "socat", "git", "clamav"),
162+ RanSnap("install", "lxd"),
163+ RanSnap("install", "--classic", "lpcraft"),
164+ RanCommand(["lxd", "init", "--auto"]),
165+ RanCommand(["freshclam", "--quiet"], **env),
166+ ]))
167+
168 def test_repo_git(self):
169 args = [
170 "run-ci-prepare",
171@@ -440,6 +487,47 @@ class TestRunCI(TestCase):
172 ], cwd="/build/tree"),
173 ]))
174
175+ def test_run_job_scan_malware_succeeds(self):
176+ args = [
177+ "run-ci",
178+ "--backend=fake", "--series=focal", "--arch=amd64", "1",
179+ "--scan-malware",
180+ "test", "0",
181+ ]
182+ run_ci = parse_args(args=args).operation
183+ run_ci.run_job()
184+ self.assertThat(run_ci.backend.run.calls, MatchesListwise([
185+ RanCommand(["mkdir", "-p", "/build/output/test/0"]),
186+ RanBuildCommand([
187+ "/bin/bash", "-o", "pipefail", "-c",
188+ "lpcraft -v run-one --output-directory /build/output "
189+ "test 0 "
190+ "2>&1 "
191+ "| tee /build/output/test/0/log",
192+ ], cwd="/build/tree"),
193+ RanBuildCommand(
194+ ["clamscan", "--recursive", "/build/output/test/0"],
195+ cwd="/build/tree"),
196+ ]))
197+
198+ def test_run_job_scan_malware_fails(self):
199+ class FailClamscan(FakeMethod):
200+ def __call__(self, run_args, *args, **kwargs):
201+ super().__call__(run_args, *args, **kwargs)
202+ if run_args[0] == "clamscan":
203+ raise subprocess.CalledProcessError(1, run_args)
204+
205+ self.useFixture(FakeLogger())
206+ args = [
207+ "run-ci",
208+ "--backend=fake", "--series=focal", "--arch=amd64", "1",
209+ "--scan-malware",
210+ "test", "0",
211+ ]
212+ run_ci = parse_args(args=args).operation
213+ run_ci.backend.run = FailClamscan()
214+ self.assertRaises(subprocess.CalledProcessError, run_ci.run_job)
215+
216 def test_run_succeeds(self):
217 args = [
218 "run-ci",
219diff --git a/lpbuildd/tests/test_ci.py b/lpbuildd/tests/test_ci.py
220index 88dda19..bf1b468 100644
221--- a/lpbuildd/tests/test_ci.py
222+++ b/lpbuildd/tests/test_ci.py
223@@ -128,11 +128,13 @@ class TestCIBuildManagerIteration(TestCase):
224 },
225 "secrets": {
226 "auth": "user:pass",
227- }
228+ },
229+ "scan_malware": True,
230 }
231 expected_prepare_options = [
232 "--git-repository", "https://git.launchpad.test/~example/+git/ci",
233 "--git-path", "main",
234+ "--scan-malware",
235 ]
236 yield self.startBuild(args, expected_prepare_options)
237
238@@ -145,6 +147,7 @@ class TestCIBuildManagerIteration(TestCase):
239 "--plugin-setting", "miniconda_conda_channel=https://user:pass@canonical.example.com/artifactory/soss-conda-stable-local/", # noqa: E501
240 "--plugin-setting", "foo=bar",
241 "--secrets", "/build/.launchpad-secrets.yaml",
242+ "--scan-malware",
243 ]
244 yield self.expectRunJob("build", "0", options=expected_job_options)
245 self.buildmanager.backend.add_file(

Subscribers

People subscribed via source and target branches