Merge ~jslarraz/review-tools:rock-container into review-tools:master
- Git
- lp:~jslarraz/review-tools
- rock-container
- Merge into master
Proposed by
Jorge Sancho Larraz
Status: | Merged |
---|---|
Merged at revision: | 10f1ead2610427bf86c358437e6ee0ede4ca6b43 |
Proposed branch: | ~jslarraz/review-tools:rock-container |
Merge into: | review-tools:master |
Diff against target: |
771 lines (+339/-333) 7 files modified
reviewtools/available.py (+3/-2) reviewtools/common.py (+0/-139) reviewtools/containers/rock_container.py (+26/-0) reviewtools/containers/tar_container.py (+96/-0) reviewtools/tests/containers/test_rock_container.py (+144/-0) reviewtools/tests/containers/test_tar_container.py (+70/-0) reviewtools/tests/test_common.py (+0/-192) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Alex Murray | Approve | ||
Evan Caville | Approve | ||
Review via email:
|
Commit message
rock function to container classes
Description of the change
To post a comment you must log in.
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Alex Murray (alexmurray) wrote : | # |
LGTM - thanks for the additional TarContainer tests as well!
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/reviewtools/available.py b/reviewtools/available.py |
2 | index 94bfbe1..90f6917 100755 |
3 | --- a/reviewtools/available.py |
4 | +++ b/reviewtools/available.py |
5 | @@ -24,7 +24,6 @@ from reviewtools.common import ( |
6 | read_file_as_json_dict, |
7 | MKDTEMP_PREFIX, |
8 | _add_error, # make a class |
9 | - get_rock_manifest, |
10 | ) |
11 | import reviewtools.email as email |
12 | |
13 | @@ -35,6 +34,7 @@ from reviewtools.store import ( |
14 | get_faked_build_and_stage_packages, |
15 | ) |
16 | from reviewtools.usn import read_usn_db |
17 | +from reviewtools.containers.rock_container import RockContainer |
18 | from reviewtools.containers.snap_container import SnapContainer, ContainerException |
19 | |
20 | email_update_required_text = """A scan of this %s shows that it was built with packages from the Ubuntu |
21 | @@ -536,7 +536,8 @@ def scan_snap(secnot_db_fn, snap_fn, with_cves=False, ignore_pockets=list()): |
22 | def scan_rock(secnot_db_fn, rock_fn, with_cves=False, ignore_pockets=list()): |
23 | """Scan rock for packages with security notices""" |
24 | out = "" |
25 | - man = get_rock_manifest(rock_fn) |
26 | + rock = RockContainer(rock_fn) |
27 | + man = rock.get_rock_manifest() |
28 | # Since ROCKs can be based on non LTS Ubuntu releases, secnot_db should |
29 | # include them (otherwise the USN db is filtered by tracked LTS releases only |
30 | # since that is the requirement for snaps) |
31 | diff --git a/reviewtools/common.py b/reviewtools/common.py |
32 | index 5de33ee..88c6e58 100644 |
33 | --- a/reviewtools/common.py |
34 | +++ b/reviewtools/common.py |
35 | @@ -25,11 +25,9 @@ import os |
36 | from pkg_resources import resource_filename |
37 | import re |
38 | import shutil |
39 | -import stat |
40 | import subprocess |
41 | import sys |
42 | import tarfile |
43 | -import tempfile |
44 | import time |
45 | import types |
46 | import yaml |
47 | @@ -486,16 +484,6 @@ def unsquashfs_lln_parse(lln_out): |
48 | return hdr, entries |
49 | |
50 | |
51 | -def _calculate_rock_untar_uncompressed_size(rock_pkg): |
52 | - """Calculate size of the uncompressed tar""" |
53 | - size = 0 |
54 | - with tarfile.open(rock_pkg) as tar: |
55 | - for tarinfo in tar: |
56 | - size += tarinfo.size |
57 | - |
58 | - return size |
59 | - |
60 | - |
61 | def unsquashfs_supports_ignore_errors(): |
62 | """Detect if unsquashfs supports the -ignore-errors option""" |
63 | (rc, out) = cmd(["unsquashfs", "-help"]) |
64 | @@ -503,112 +491,6 @@ def unsquashfs_supports_ignore_errors(): |
65 | return "-ig[nore-errors]" in out |
66 | |
67 | |
68 | -def is_pkg_uncompressed_size_valid(pkg_max_size, size, pkg): |
69 | - st = os.statvfs(pkg) |
70 | - avail = st.f_bsize * st.f_bavail * 0.9 # 90% of available space |
71 | - if size > pkg_max_size: |
72 | - return ( |
73 | - False, |
74 | - "uncompressed %s is too large (%dM > %dM)" |
75 | - % (pkg, size / 1024 / 1024, pkg_max_size / 1024 / 1024), |
76 | - ) |
77 | - elif size > avail * 0.9: |
78 | - return ( |
79 | - False, |
80 | - "uncompressed %s is too large for available space (%dM > %dM)" |
81 | - % (pkg, size / 1024 / 1024, avail / 1024 / 1024), |
82 | - ) |
83 | - else: |
84 | - return True, "" |
85 | - |
86 | - |
87 | -def _unpack_rock_tar(rock_pkg, dest): |
88 | - """Unpack a tar based rock package to dest""" |
89 | - size = _calculate_rock_untar_uncompressed_size(rock_pkg) |
90 | - |
91 | - # Assuming MAX_UNCOMPRESSED_SIZE for rocks is same as snaps |
92 | - # MAX_UNCOMPRESSED_SIZE |
93 | - rock_max_size = MAX_UNCOMPRESSED_SIZE * 1024 * 1024 * 1024 |
94 | - |
95 | - valid_size, error_msg = is_pkg_uncompressed_size_valid( |
96 | - rock_max_size, size, rock_pkg |
97 | - ) |
98 | - if valid_size: |
99 | - global MKDTEMP_PREFIX |
100 | - global MKDTEMP_DIR |
101 | - d = tempfile.mkdtemp(prefix=MKDTEMP_PREFIX, dir=MKDTEMP_DIR) |
102 | - |
103 | - try: |
104 | - with tarfile.open(rock_pkg) as tar: |
105 | - # Inspecting members before extracting, since it is possible that |
106 | - # files are created outside of path, e.g. members that have |
107 | - # absolute filenames starting with "/" or filenames with two dots |
108 | - # ".." |
109 | - # https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.extractall |
110 | - for name in tar.getnames(): |
111 | - if name.startswith("..") or name.startswith("/"): |
112 | - logger.error( |
113 | - "Bad path %s while extracting archive at %s" |
114 | - % (name, rock_pkg) |
115 | - ) |
116 | - |
117 | - tar.extractall(path=d) |
118 | - except Exception as e: |
119 | - logger.error("Unexpected exception while unpacking rock %s" % e) |
120 | - if os.path.isdir(d): |
121 | - recursive_rm(d) |
122 | - |
123 | - if dest is None: |
124 | - dest = d |
125 | - else: |
126 | - # Recursively move extracted content to original dir, |
127 | - # keeping permissions |
128 | - move_dir_content(d, dest) |
129 | - |
130 | - return dest |
131 | - else: |
132 | - logger.error(error_msg) |
133 | - |
134 | - |
135 | -def check_dir(dest): |
136 | - if dest is not None and os.path.exists(dest): |
137 | - logger.error("'%s' exists. Aborting." % dest) |
138 | - |
139 | - |
140 | -def check_fn(fn): |
141 | - if not os.path.isfile(fn): |
142 | - logger.error("Could not find '%s'" % fn) |
143 | - pkg = fn |
144 | - if not pkg.startswith("/"): |
145 | - pkg = os.path.abspath(pkg) |
146 | - return pkg |
147 | - |
148 | - |
149 | -def check_max_pkg_size(pkg): |
150 | - size = os.stat(pkg)[stat.ST_SIZE] |
151 | - max = MAX_COMPRESSED_SIZE * 1024 * 1024 * 1024 |
152 | - if size > max: |
153 | - logger.error( |
154 | - "compressed file is too large (%dM > %dM)" |
155 | - % (size / 1024 / 1024, max / 1024 / 1024) |
156 | - ) |
157 | - |
158 | - |
159 | -def unpack_rock(fn, dest=None): |
160 | - """Unpack rock""" |
161 | - pkg = check_fn(fn) |
162 | - check_dir(dest) |
163 | - |
164 | - # Limit the maximimum size of the package |
165 | - check_max_pkg_size(pkg) |
166 | - |
167 | - # check if its a tar based rock |
168 | - if tarfile.is_tarfile(pkg): |
169 | - return _unpack_rock_tar(fn, dest) |
170 | - |
171 | - logger.error("Unsupported package format (not tar)") |
172 | - |
173 | - |
174 | def open_file_read(path): |
175 | """Open specified file read-only""" |
176 | try: |
177 | @@ -858,27 +740,6 @@ def read_file_as_json_dict(fn): |
178 | return raw |
179 | |
180 | |
181 | -def get_rock_manifest(fn): |
182 | - if "SNAP_USER_COMMON" in os.environ and os.path.exists( |
183 | - os.environ["SNAP_USER_COMMON"] |
184 | - ): |
185 | - MKDTEMP_DIR = os.environ["SNAP_USER_COMMON"] |
186 | - else: |
187 | - MKDTEMP_DIR = tempfile.gettempdir() |
188 | - unpack_tmp_dir = tempfile.mktemp(prefix=MKDTEMP_PREFIX, dir=MKDTEMP_DIR) |
189 | - |
190 | - man_fn = "/usr/share/rocks/dpkg.query" |
191 | - unpack_rock(fn, unpack_tmp_dir) |
192 | - |
193 | - # since ROCK manifest is not available yet, we build a manifest using the |
194 | - # information available in /usr/share/rocks/dpkg.query present inside the |
195 | - # rock |
196 | - man_yaml = build_manifest_from_rock_tar(man_fn, unpack_tmp_dir) |
197 | - recursive_rm(unpack_tmp_dir) |
198 | - |
199 | - return man_yaml |
200 | - |
201 | - |
202 | def list_dir(path, dirs_only=False): |
203 | """List directory items""" |
204 | results = list(map(lambda x: os.path.join(path, x), os.listdir(path))) |
205 | diff --git a/reviewtools/containers/rock_container.py b/reviewtools/containers/rock_container.py |
206 | new file mode 100644 |
207 | index 0000000..6a6e799 |
208 | --- /dev/null |
209 | +++ b/reviewtools/containers/rock_container.py |
210 | @@ -0,0 +1,26 @@ |
211 | +import yaml |
212 | +from reviewtools.containers.tar_container import ( |
213 | + TarContainer, |
214 | + ContainerException, |
215 | +) |
216 | +from reviewtools.common import build_manifest_from_rock_tar |
217 | + |
218 | + |
219 | +class RockContainer(TarContainer): |
220 | + # TODO: review-tools was not checking for it. We still have some snaps in |
221 | + # "tests/" that doesn't fit this limit and will need to be regenerated before |
222 | + # this check can be enforced. |
223 | + # _min_packed_size = 16 * 1024 |
224 | + _max_packed_size = 5 * 1024 * 1024 * 1024 |
225 | + _max_unpacked_size = 25 * 1024 * 1024 * 1024 |
226 | + |
227 | + |
228 | + def get_rock_manifest(self): |
229 | + |
230 | + man_fn = "/usr/share/rocks/dpkg.query" |
231 | + # since ROCK manifest is not available yet, we build a manifest using the |
232 | + # information available in /usr/share/rocks/dpkg.query present inside the |
233 | + # rock |
234 | + man_yaml = build_manifest_from_rock_tar(man_fn, self.unpack_dir) |
235 | + |
236 | + return man_yaml |
237 | diff --git a/reviewtools/containers/tar_container.py b/reviewtools/containers/tar_container.py |
238 | new file mode 100644 |
239 | index 0000000..674dd1b |
240 | --- /dev/null |
241 | +++ b/reviewtools/containers/tar_container.py |
242 | @@ -0,0 +1,96 @@ |
243 | +import os |
244 | +import shutil |
245 | +import tarfile |
246 | +from reviewtools.common import ( |
247 | + recursive_rm, |
248 | +) |
249 | +from reviewtools.containers.base_container import BaseContainer, ContainerException |
250 | + |
251 | + |
252 | +class TarContainer(BaseContainer): |
253 | + """ |
254 | + A container is a file that encapsulates a file system. This class extends |
255 | + BaseContainer to handle functionality that is specific to Tar containers. |
256 | + """ |
257 | + |
258 | + def check_container_format(self): |
259 | + """ |
260 | + Checks that the container file has the expected format. |
261 | + |
262 | + Raises: |
263 | + ContainerException: This method will raise an exception if the |
264 | + container file is not a tar container. |
265 | + """ |
266 | + if not tarfile.is_tarfile(self.filename): |
267 | + raise ContainerException("Unsupported package format (not tar)") |
268 | + |
269 | + |
270 | + def calculate_unpacked_size(self) -> int: |
271 | + """ |
272 | + Estimates the space that the unpacked directory will use by adding the |
273 | + individual files size without extracting them. |
274 | + |
275 | + Returns: |
276 | + int: Estimated size in bytes of the unpacked directory |
277 | + """ |
278 | + size = 0 |
279 | + with tarfile.open(self.filename) as tar: |
280 | + for tarinfo in tar: |
281 | + size += tarinfo.size |
282 | + return size |
283 | + |
284 | + def unpack(self, items: list = [], force: bool = False) -> str: |
285 | + """ |
286 | + Extracts the content of the container file and returns the absolute path to |
287 | + the newly unpacked directory. |
288 | + |
289 | + Args: |
290 | + force (bool, optional): If ``True``, the unpack_dir will be overwritten |
291 | + if it already exists. Defaults to ``False`` |
292 | + |
293 | + Returns: |
294 | + str: absolute path to the newly created unpack dir |
295 | + |
296 | + Raises: |
297 | + ContainerException: This method will raise an ContainerException if the |
298 | + unpack_dir already exists and this method was called without the |
299 | + force argument set to True |
300 | + RuntimeError: This method will raise a RuntimeError if the extraction |
301 | + fails for an unknown reason. |
302 | + """ |
303 | + if self._unpack_dir is not None and not force: |
304 | + raise ContainerException("'%s' exists. Aborting." % self._unpack_dir) |
305 | + |
306 | + # Check disk space and space available in disk |
307 | + self.check_unpacked_size() |
308 | + self.check_disk_space() |
309 | + |
310 | + # Unpack the container |
311 | + stage_dir = os.path.join(self.tmp_dir, "stage") |
312 | + |
313 | + with tarfile.open(self.filename) as tar: |
314 | + # Inspecting members before extracting, since it is possible that |
315 | + # files are created outside of path, e.g. members that have |
316 | + # absolute filenames starting with "/" or filenames with two dots |
317 | + # ".." |
318 | + # https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.extractall |
319 | + for name in tar.getnames(): |
320 | + if name.startswith("..") or name.startswith("/"): |
321 | + raise ContainerException( |
322 | + "Bad path %s while extracting archive at %s" |
323 | + % (name, self.filename) |
324 | + ) |
325 | + |
326 | + try: |
327 | + tar.extractall(path=stage_dir) |
328 | + except Exception as e: |
329 | + if os.path.isdir(stage_dir): |
330 | + recursive_rm(stage_dir) |
331 | + raise RuntimeError("Unexpected exception while unpacking rock %s" % e) |
332 | + |
333 | + # Update on success |
334 | + if self._unpack_dir is not None and os.path.exists(self._unpack_dir): |
335 | + recursive_rm(self._unpack_dir) |
336 | + self._unpack_dir = ".".join(self.filename.split(".")[:-1]) |
337 | + shutil.move(stage_dir, self._unpack_dir) |
338 | + return self._unpack_dir |
339 | diff --git a/reviewtools/tests/containers/test_rock_container.py b/reviewtools/tests/containers/test_rock_container.py |
340 | new file mode 100644 |
341 | index 0000000..f59b2d9 |
342 | --- /dev/null |
343 | +++ b/reviewtools/tests/containers/test_rock_container.py |
344 | @@ -0,0 +1,144 @@ |
345 | +import unittest |
346 | +from tempfile import mkdtemp |
347 | +from reviewtools.common import ReviewException |
348 | +from reviewtools.containers.rock_container import ( |
349 | + RockContainer, |
350 | + ContainerException, |
351 | +) |
352 | + |
353 | + |
354 | +class TestRockContainer(unittest.TestCase): |
355 | + """Tests for reviewtools.containers.rock_container""" |
356 | + |
357 | + def test_get_rock_manifest(self): |
358 | + """Test get_rock_manifest()""" |
359 | + valid_rock_fn = "./tests/test-rock-redis_5.0-20.04.tar" |
360 | + expected_manifest_yaml = { |
361 | + "manifest-version": "1", |
362 | + "os-release-id": "ubuntu", |
363 | + "os-release-version-id": "20.04", |
364 | + "stage-packages": [ |
365 | + "adduser=3.118ubuntu2,adduser=3.118ubuntu2", |
366 | + "apt=2.0.2ubuntu0.2,apt=2.0.2ubuntu0.2", |
367 | + "base-files=11ubuntu5.2,base-files=11ubuntu5.2", |
368 | + "base-passwd=3.5.47,base-passwd=3.5.47", |
369 | + "bash=5.0-6ubuntu1.1,bash=5.0-6ubuntu1.1", |
370 | + "bsdutils=1:2.34-0.1ubuntu9.1," "util-linux=2.34-0.1ubuntu9.1", |
371 | + "bzip2=1.0.8-2,bzip2=1.0.8-2", |
372 | + "coreutils=8.30-3ubuntu2," "coreutils=8.30-3ubuntu2", |
373 | + "dash=0.5.10.2-6,dash=0.5.10.2-6", |
374 | + "debconf=1.5.73,debconf=1.5.73", |
375 | + "debianutils=4.9.1,debianutils=4.9.1", |
376 | + "diffutils=1:3.7-3,diffutils=1:3.7-3", |
377 | + "dpkg=1.19.7ubuntu3,dpkg=1.19.7ubuntu3", |
378 | + "e2fsprogs=1.45.5-2ubuntu1," "e2fsprogs=1.45.5-2ubuntu1", |
379 | + "fdisk=2.34-0.1ubuntu9.1," "util-linux=2.34-0.1ubuntu9.1", |
380 | + "findutils=4.7.0-1ubuntu1," "findutils=4.7.0-1ubuntu1", |
381 | + "gcc-10-base:amd64=10.2.0-5ubuntu1~20.04," |
382 | + "gcc-10=10.2.0-5ubuntu1~20.04", |
383 | + "gpgv=2.2.19-3ubuntu2,gnupg2=2.2.19-3ubuntu2", |
384 | + "grep=3.4-1,grep=3.4-1", |
385 | + "gzip=1.10-0ubuntu4,gzip=1.10-0ubuntu4", |
386 | + "hostname=3.23,hostname=3.23", |
387 | + "init-system-helpers=1.57," "init-system-helpers=1.57", |
388 | + "libacl1:amd64=2.2.53-6,acl=2.2.53-6", |
389 | + "libapt-pkg6.0:amd64=2.0.2ubuntu0.2," "apt=2.0.2ubuntu0.2", |
390 | + "libatomic1:amd64=10.2.0-5ubuntu1~20.04," |
391 | + "gcc-10=10.2.0-5ubuntu1~20.04", |
392 | + "libattr1:amd64=1:2.4.48-5,attr=1:2.4.48-5", |
393 | + "libaudit-common=1:2.8.5-2ubuntu6," "audit=1:2.8.5-2ubuntu6", |
394 | + "libaudit1:amd64=1:2.8.5-2ubuntu6," "audit=1:2.8.5-2ubuntu6", |
395 | + "libblkid1:amd64=2.34-0.1ubuntu9.1," "util-linux=2.34-0.1ubuntu9.1", |
396 | + "libbz2-1.0:amd64=1.0.8-2,bzip2=1.0.8-2", |
397 | + "libc-bin=2.31-0ubuntu9.1," "glibc=2.31-0ubuntu9.1", |
398 | + "libc6:amd64=2.31-0ubuntu9.1," "glibc=2.31-0ubuntu9.1", |
399 | + "libcap-ng0:amd64=0.7.9-2.1build1," "libcap-ng=0.7.9-2.1build1", |
400 | + "libcom-err2:amd64=1.45.5-2ubuntu1," "e2fsprogs=1.45.5-2ubuntu1", |
401 | + "libcrypt1:amd64=1:4.4.10-10ubuntu4," "libxcrypt=1:4.4.10-10ubuntu4", |
402 | + "libdb5.3:amd64=5.3.28+dfsg1-0.6ubuntu2," |
403 | + "db5.3=5.3.28+dfsg1-0.6ubuntu2", |
404 | + "libdebconfclient0:amd64=0.251ubuntu1," "cdebconf=0.251ubuntu1", |
405 | + "libext2fs2:amd64=1.45.5-2ubuntu1," "e2fsprogs=1.45.5-2ubuntu1", |
406 | + "libfdisk1:amd64=2.34-0.1ubuntu9.1," "util-linux=2.34-0.1ubuntu9.1", |
407 | + "libffi7:amd64=3.3-4,libffi=3.3-4", |
408 | + "libgcc-s1:amd64=10.2.0-5ubuntu1~20.04," "gcc-10=10.2.0-5ubuntu1~20.04", |
409 | + "libgcrypt20:amd64=1.8.5-5ubuntu1," "libgcrypt20=1.8.5-5ubuntu1", |
410 | + "libgmp10:amd64=2:6.2.0+dfsg-4," "gmp=2:6.2.0+dfsg-4", |
411 | + "libgnutls30:amd64=3.6.13-2ubuntu1.3," "gnutls28=3.6.13-2ubuntu1.3", |
412 | + "libgpg-error0:amd64=1.37-1," "libgpg-error=1.37-1", |
413 | + "libhiredis0.14:amd64=0.14.0-6," "hiredis=0.14.0-6", |
414 | + "libhogweed5:amd64=3.5.1+really3.5.1-2," "nettle=3.5.1+really3.5.1-2", |
415 | + "libidn2-0:amd64=2.2.0-2,libidn2=2.2.0-2", |
416 | + "libjemalloc2:amd64=5.2.1-1ubuntu1," "jemalloc=5.2.1-1ubuntu1", |
417 | + "liblua5.1-0:amd64=5.1.5-8.1build4," "lua5.1=5.1.5-8.1build4", |
418 | + "liblz4-1:amd64=1.9.2-2,lz4=1.9.2-2", |
419 | + "liblzma5:amd64=5.2.4-1ubuntu1," "xz-utils=5.2.4-1ubuntu1", |
420 | + "libmount1:amd64=2.34-0.1ubuntu9.1," "util-linux=2.34-0.1ubuntu9.1", |
421 | + "libncurses6:amd64=6.2-0ubuntu2," "ncurses=6.2-0ubuntu2", |
422 | + "libncursesw6:amd64=6.2-0ubuntu2," "ncurses=6.2-0ubuntu2", |
423 | + "libnettle7:amd64=3.5.1+really3.5.1-2," "nettle=3.5.1+really3.5.1-2", |
424 | + "libp11-kit0:amd64=0.23.20-1build1," "p11-kit=0.23.20-1build1", |
425 | + "libpam-modules:amd64=1.3.1-5ubuntu4.1," "pam=1.3.1-5ubuntu4.1", |
426 | + "libpam-modules-bin=1.3.1-5ubuntu4.1," "pam=1.3.1-5ubuntu4.1", |
427 | + "libpam-runtime=1.3.1-5ubuntu4.1," "pam=1.3.1-5ubuntu4.1", |
428 | + "libpam0g:amd64=1.3.1-5ubuntu4.1," "pam=1.3.1-5ubuntu4.1", |
429 | + "libpcre2-8-0:amd64=10.34-7,pcre2=10.34-7", |
430 | + "libpcre3:amd64=2:8.39-12build1," "pcre3=2:8.39-12build1", |
431 | + "libprocps8:amd64=2:3.3.16-1ubuntu2," "procps=2:3.3.16-1ubuntu2", |
432 | + "libseccomp2:amd64=2.4.3-1ubuntu3.20.04.3," |
433 | + "libseccomp=2.4.3-1ubuntu3.20.04.3", |
434 | + "libselinux1:amd64=3.0-1build2,libselinux=3.0-1build2", |
435 | + "libsemanage-common=3.0-1build2,libsemanage=3.0-1build2", |
436 | + "libsemanage1:amd64=3.0-1build2,libsemanage=3.0-1build2", |
437 | + "libsepol1:amd64=3.0-1,libsepol=3.0-1", |
438 | + "libsmartcols1:amd64=2.34-0.1ubuntu9.1,util-linux=2.34-0.1ubuntu9.1", |
439 | + "libss2:amd64=1.45.5-2ubuntu1,e2fsprogs=1.45.5-2ubuntu1", |
440 | + "libstdc++6:amd64=10.2.0-5ubuntu1~20.04,gcc-10=10.2.0-5ubuntu1~20.04", |
441 | + "libsystemd0:amd64=245.4-4ubuntu3.3,systemd=245.4-4ubuntu3.3", |
442 | + "libtasn1-6:amd64=4.16.0-2,libtasn1-6=4.16.0-2", |
443 | + "libtinfo6:amd64=6.2-0ubuntu2,ncurses=6.2-0ubuntu2", |
444 | + "libudev1:amd64=245.4-4ubuntu3.3,systemd=245.4-4ubuntu3.3", |
445 | + "libunistring2:amd64=0.9.10-2,libunistring=0.9.10-2", |
446 | + "libuuid1:amd64=2.34-0.1ubuntu9.1,util-linux=2.34-0.1ubuntu9.1", |
447 | + "libxcursor1=4.0.0-2,tiff=4.0.0-2", |
448 | + "libzstd1:amd64=1.4.4+dfsg-3,libzstd=1.4.4+dfsg-3", |
449 | + "login=1:4.8.1-1ubuntu5.20.04,shadow=1:4.8.1-1ubuntu5.20.04", |
450 | + "logsave=1.45.5-2ubuntu1,e2fsprogs=1.45.5-2ubuntu1", |
451 | + "lsb-base=11.1.0ubuntu2,lsb=11.1.0ubuntu2", |
452 | + "lua-bitop:amd64=1.0.2-5,lua-bitop=1.0.2-5", |
453 | + "lua-cjson:amd64=2.1.0+dfsg-2.1,lua-cjson=2.1.0+dfsg-2.1", |
454 | + "mawk=1.3.4.20200120-2,mawk=1.3.4.20200120-2", |
455 | + "mount=2.34-0.1ubuntu9.1,util-linux=2.34-0.1ubuntu9.1", |
456 | + "ncurses-base=6.2-0ubuntu2,ncurses=6.2-0ubuntu2", |
457 | + "ncurses-bin=6.2-0ubuntu2,ncurses=6.2-0ubuntu2", |
458 | + "passwd=1:4.8.1-1ubuntu5.20.04,shadow=1:4.8.1-1ubuntu5.20.04", |
459 | + "perl-base=5.30.0-9ubuntu0.2,perl=5.30.0-9ubuntu0.2", |
460 | + "procps=2:3.3.16-1ubuntu2,procps=2:3.3.16-1ubuntu2", |
461 | + "pwgen=2.08-2,pwgen=2.08-2", |
462 | + "redis-server=5:5.0.7-2,redis=5:5.0.7-2", |
463 | + "redis-tools=5:5.0.7-2,redis=5:5.0.7-2", |
464 | + "sed=4.7-1,sed=4.7-1", |
465 | + "sensible-utils=0.0.12+nmu1,sensible-utils=0.0.12+nmu1", |
466 | + "sysvinit-utils=2.96-2.1ubuntu1,sysvinit=2.96-2.1ubuntu1", |
467 | + "tar=1.30+dfsg-7,tar=1.30+dfsg-7", |
468 | + "tzdata=2020d-0ubuntu0.20.04,tzdata=2020d-0ubuntu0.20.04", |
469 | + "ubuntu-keyring=2020.02.11.2,ubuntu-keyring=2020.02.11.2", |
470 | + "util-linux=2.34-0.1ubuntu9.1,util-linux=2.34-0.1ubuntu9.1", |
471 | + "zlib1g:amd64=1:1.2.11.dfsg-2ubuntu1.2,zlib=1:1.2.11.dfsg-2ubuntu1.2", |
472 | + ], |
473 | + } |
474 | + valid_rock = RockContainer(valid_rock_fn) |
475 | + self.assertDictEqual( |
476 | + expected_manifest_yaml, valid_rock.get_rock_manifest() |
477 | + ) |
478 | + |
479 | + def test_get_rock_manifest_invalid_rock(self): |
480 | + """Test get_rock_manifest() - invalid rock tar""" |
481 | + invalid_rock_fn = "./tests/invalid_rock_multiple_layer_tar_archives.tar" |
482 | + invalid_rock = RockContainer(invalid_rock_fn) |
483 | + with self.assertRaises(ReviewException) as e: |
484 | + invalid_rock.get_rock_manifest() |
485 | + self.assertEqual( |
486 | + e.exception.value, |
487 | + "Unexpected number of layer tar archives inside layer directory: 2", |
488 | + ) |
489 | diff --git a/reviewtools/tests/containers/test_tar_container.py b/reviewtools/tests/containers/test_tar_container.py |
490 | new file mode 100644 |
491 | index 0000000..153ef3f |
492 | --- /dev/null |
493 | +++ b/reviewtools/tests/containers/test_tar_container.py |
494 | @@ -0,0 +1,70 @@ |
495 | +import os |
496 | +import unittest |
497 | +from tempfile import mkstemp, mkdtemp |
498 | +from reviewtools.containers.tar_container import ( |
499 | + TarContainer, |
500 | + ContainerException, |
501 | +) |
502 | + |
503 | + |
504 | +class TestTarContainer(unittest.TestCase): |
505 | + """Tests for reviewtools.containers.tar_container""" |
506 | + |
507 | + @classmethod |
508 | + def setUpClass(cls): |
509 | + cls.tmp_dir = mkdtemp() |
510 | + cls.fn = "./tests/test-rock-redis_5.0-20.04.tar" |
511 | + |
512 | + # Test initialization |
513 | + def test_check_initialization__happy(self): |
514 | + TarContainer(self.fn) |
515 | + |
516 | + def test_check_initialization__invalid_format(self): |
517 | + fd, plain_file = mkstemp(suffix=".tar") |
518 | + with self.assertRaises(ContainerException) as context: |
519 | + TarContainer(plain_file) |
520 | + self.assertEqual( |
521 | + "Unsupported package format (not tar)", context.exception.value |
522 | + ) |
523 | + os.unlink(plain_file) |
524 | + |
525 | + # Test calculate unpacked size |
526 | + def test_calculate_unpacked_size__happy(self): |
527 | + # unpacked size calculated is a bit smaller than actual size as it does not consider that |
528 | + # folders require at least one complete block |
529 | + container = TarContainer(self.fn) |
530 | + size = container.calculate_unpacked_size() |
531 | + self.assertTrue(isinstance(size, int)) |
532 | + |
533 | + # Test unpack |
534 | + def test_unpack__happy(self): |
535 | + container = TarContainer(self.fn) |
536 | + container.unpack() |
537 | + |
538 | + def test_unpack__force(self): |
539 | + container = TarContainer(self.fn) |
540 | + container.unpack() |
541 | + container.unpack(force=True) |
542 | + |
543 | + def test_unpack__no_force(self): |
544 | + container = TarContainer(self.fn) |
545 | + container.unpack() |
546 | + with self.assertRaises(ContainerException) as context: |
547 | + container.unpack() |
548 | + self.assertTrue(" exists. Aborting." in context.exception.value) |
549 | + |
550 | + def test_check_unpack__slash_file(self): |
551 | + container = TarContainer("./tests/test-rock-invalid-1.tar") |
552 | + with self.assertRaises(ContainerException) as context: |
553 | + container.unpack() |
554 | + self.assertTrue( |
555 | + " while extracting archive at " in context.exception.value |
556 | + ) |
557 | + |
558 | + def test_check_unpack__two_dots_file(self): |
559 | + container = TarContainer("./tests/test-rock-invalid-2.tar") |
560 | + with self.assertRaises(ContainerException) as context: |
561 | + container.unpack() |
562 | + self.assertTrue( |
563 | + " while extracting archive at " in context.exception.value |
564 | + ) |
565 | diff --git a/reviewtools/tests/test_common.py b/reviewtools/tests/test_common.py |
566 | index 85504ca..953d2c0 100644 |
567 | --- a/reviewtools/tests/test_common.py |
568 | +++ b/reviewtools/tests/test_common.py |
569 | @@ -19,7 +19,6 @@ import os |
570 | import shutil |
571 | import tempfile |
572 | |
573 | -from reviewtools.sr_common import ReviewException |
574 | import reviewtools.sr_tests as sr_tests |
575 | import reviewtools.common |
576 | from reviewtools.common import StatLLN |
577 | @@ -502,194 +501,3 @@ drwxr-xr-x 0/0 48 2020-03-24 09:11 squashfs-root/meta |
578 | continue |
579 | |
580 | raise Exception("parsed input should be invalid") |
581 | - |
582 | - def test_check_pkg_uncompressed_size_ok(self): |
583 | - """Test check_pkg_uncompressed_size() - rocks and snaps""" |
584 | - pkgs = [ |
585 | - "./tests/test-rock-redis_5.0-20.04.tar", |
586 | - "./tests/test-snapcraft-manifest-unittest_0_amd64.snap", |
587 | - ] |
588 | - max_size = 3 * 1024 * 1024 |
589 | - size = 1 * 1024 * 1024 |
590 | - for pkg in pkgs: |
591 | - with self.subTest(pkg=pkg): |
592 | - valid_size, _ = reviewtools.common.is_pkg_uncompressed_size_valid( |
593 | - max_size, size, pkg |
594 | - ) |
595 | - self.assertTrue(valid_size) |
596 | - |
597 | - def test_check_pkg_uncompressed_size_max_size_error(self): |
598 | - """Test check_pkg_uncompressed_size() - rocks and snaps""" |
599 | - pkgs = [ |
600 | - "./tests/test-rock-redis_5.0-20.04.tar", |
601 | - "./tests/test-snapcraft-manifest-unittest_0_amd64.snap", |
602 | - ] |
603 | - max_size = 1024 |
604 | - size = 1 * 1024 * 1024 |
605 | - for pkg in pkgs: |
606 | - with self.subTest(pkg=pkg): |
607 | - valid_size, _ = reviewtools.common.is_pkg_uncompressed_size_valid( |
608 | - max_size, size, pkg |
609 | - ) |
610 | - self.assertFalse(valid_size) |
611 | - |
612 | - def test_unpack_rock_invalid_format(self): |
613 | - """Test unpack_rock() - invalid rock format""" |
614 | - invalid_rock = "./tests/test-snapcraft-manifest-unittest_0_amd64.snap" |
615 | - with self.assertRaises(SystemExit) as e: |
616 | - reviewtools.common.unpack_rock(invalid_rock) |
617 | - self.assertEqual(e.exception.code, 1) |
618 | - |
619 | - def test_unpack_rock_valid_format(self): |
620 | - """Test unpack_rock() - valid rock format""" |
621 | - # TODO: add further unit testing for tar unpacking functionality |
622 | - valid_rock = "./tests/test-rock-redis_5.0-20.04.tar" |
623 | - unpack_dir = reviewtools.common.unpack_rock(valid_rock) |
624 | - self.assertIn("review-tools-", unpack_dir) |
625 | - |
626 | - def test_unpack_rock_invalid_format_filename_starting_with_slash(self): |
627 | - """Test unpack_rock() - invalid - filename starting with slash""" |
628 | - # TODO: add further unit testing for tar unpacking functionality |
629 | - invalid_rock = "./tests/test-rock-invalid-1.tar" |
630 | - with self.assertRaises(SystemExit) as e: |
631 | - reviewtools.common.unpack_rock(invalid_rock) |
632 | - self.assertEqual(e.exception.code, 1) |
633 | - |
634 | - def test_unpack_rock_invalid_format_filename_with_two_dots(self): |
635 | - """Test unpack_rock() - invalid - filename with two dots""" |
636 | - # TODO: add further unit testing for tar unpacking functionality |
637 | - invalid_rock = "./tests/test-rock-invalid-2.tar" |
638 | - with self.assertRaises(SystemExit) as e: |
639 | - reviewtools.common.unpack_rock(invalid_rock) |
640 | - self.assertEqual(e.exception.code, 1) |
641 | - |
642 | - def test_get_rock_manifest(self): |
643 | - """Test get_rock_manifest()""" |
644 | - valid_rock_fn = "./tests/test-rock-redis_5.0-20.04.tar" |
645 | - expected_manifest_yaml = { |
646 | - "manifest-version": "1", |
647 | - "os-release-id": "ubuntu", |
648 | - "os-release-version-id": "20.04", |
649 | - "stage-packages": [ |
650 | - "adduser=3.118ubuntu2,adduser=3.118ubuntu2", |
651 | - "apt=2.0.2ubuntu0.2,apt=2.0.2ubuntu0.2", |
652 | - "base-files=11ubuntu5.2,base-files=11ubuntu5.2", |
653 | - "base-passwd=3.5.47,base-passwd=3.5.47", |
654 | - "bash=5.0-6ubuntu1.1,bash=5.0-6ubuntu1.1", |
655 | - "bsdutils=1:2.34-0.1ubuntu9.1," "util-linux=2.34-0.1ubuntu9.1", |
656 | - "bzip2=1.0.8-2,bzip2=1.0.8-2", |
657 | - "coreutils=8.30-3ubuntu2," "coreutils=8.30-3ubuntu2", |
658 | - "dash=0.5.10.2-6,dash=0.5.10.2-6", |
659 | - "debconf=1.5.73,debconf=1.5.73", |
660 | - "debianutils=4.9.1,debianutils=4.9.1", |
661 | - "diffutils=1:3.7-3,diffutils=1:3.7-3", |
662 | - "dpkg=1.19.7ubuntu3,dpkg=1.19.7ubuntu3", |
663 | - "e2fsprogs=1.45.5-2ubuntu1," "e2fsprogs=1.45.5-2ubuntu1", |
664 | - "fdisk=2.34-0.1ubuntu9.1," "util-linux=2.34-0.1ubuntu9.1", |
665 | - "findutils=4.7.0-1ubuntu1," "findutils=4.7.0-1ubuntu1", |
666 | - "gcc-10-base:amd64=10.2.0-5ubuntu1~20.04," |
667 | - "gcc-10=10.2.0-5ubuntu1~20.04", |
668 | - "gpgv=2.2.19-3ubuntu2,gnupg2=2.2.19-3ubuntu2", |
669 | - "grep=3.4-1,grep=3.4-1", |
670 | - "gzip=1.10-0ubuntu4,gzip=1.10-0ubuntu4", |
671 | - "hostname=3.23,hostname=3.23", |
672 | - "init-system-helpers=1.57," "init-system-helpers=1.57", |
673 | - "libacl1:amd64=2.2.53-6,acl=2.2.53-6", |
674 | - "libapt-pkg6.0:amd64=2.0.2ubuntu0.2," "apt=2.0.2ubuntu0.2", |
675 | - "libatomic1:amd64=10.2.0-5ubuntu1~20.04," |
676 | - "gcc-10=10.2.0-5ubuntu1~20.04", |
677 | - "libattr1:amd64=1:2.4.48-5,attr=1:2.4.48-5", |
678 | - "libaudit-common=1:2.8.5-2ubuntu6," "audit=1:2.8.5-2ubuntu6", |
679 | - "libaudit1:amd64=1:2.8.5-2ubuntu6," "audit=1:2.8.5-2ubuntu6", |
680 | - "libblkid1:amd64=2.34-0.1ubuntu9.1," "util-linux=2.34-0.1ubuntu9.1", |
681 | - "libbz2-1.0:amd64=1.0.8-2,bzip2=1.0.8-2", |
682 | - "libc-bin=2.31-0ubuntu9.1," "glibc=2.31-0ubuntu9.1", |
683 | - "libc6:amd64=2.31-0ubuntu9.1," "glibc=2.31-0ubuntu9.1", |
684 | - "libcap-ng0:amd64=0.7.9-2.1build1," "libcap-ng=0.7.9-2.1build1", |
685 | - "libcom-err2:amd64=1.45.5-2ubuntu1," "e2fsprogs=1.45.5-2ubuntu1", |
686 | - "libcrypt1:amd64=1:4.4.10-10ubuntu4," "libxcrypt=1:4.4.10-10ubuntu4", |
687 | - "libdb5.3:amd64=5.3.28+dfsg1-0.6ubuntu2," |
688 | - "db5.3=5.3.28+dfsg1-0.6ubuntu2", |
689 | - "libdebconfclient0:amd64=0.251ubuntu1," "cdebconf=0.251ubuntu1", |
690 | - "libext2fs2:amd64=1.45.5-2ubuntu1," "e2fsprogs=1.45.5-2ubuntu1", |
691 | - "libfdisk1:amd64=2.34-0.1ubuntu9.1," "util-linux=2.34-0.1ubuntu9.1", |
692 | - "libffi7:amd64=3.3-4,libffi=3.3-4", |
693 | - "libgcc-s1:amd64=10.2.0-5ubuntu1~20.04," "gcc-10=10.2.0-5ubuntu1~20.04", |
694 | - "libgcrypt20:amd64=1.8.5-5ubuntu1," "libgcrypt20=1.8.5-5ubuntu1", |
695 | - "libgmp10:amd64=2:6.2.0+dfsg-4," "gmp=2:6.2.0+dfsg-4", |
696 | - "libgnutls30:amd64=3.6.13-2ubuntu1.3," "gnutls28=3.6.13-2ubuntu1.3", |
697 | - "libgpg-error0:amd64=1.37-1," "libgpg-error=1.37-1", |
698 | - "libhiredis0.14:amd64=0.14.0-6," "hiredis=0.14.0-6", |
699 | - "libhogweed5:amd64=3.5.1+really3.5.1-2," "nettle=3.5.1+really3.5.1-2", |
700 | - "libidn2-0:amd64=2.2.0-2,libidn2=2.2.0-2", |
701 | - "libjemalloc2:amd64=5.2.1-1ubuntu1," "jemalloc=5.2.1-1ubuntu1", |
702 | - "liblua5.1-0:amd64=5.1.5-8.1build4," "lua5.1=5.1.5-8.1build4", |
703 | - "liblz4-1:amd64=1.9.2-2,lz4=1.9.2-2", |
704 | - "liblzma5:amd64=5.2.4-1ubuntu1," "xz-utils=5.2.4-1ubuntu1", |
705 | - "libmount1:amd64=2.34-0.1ubuntu9.1," "util-linux=2.34-0.1ubuntu9.1", |
706 | - "libncurses6:amd64=6.2-0ubuntu2," "ncurses=6.2-0ubuntu2", |
707 | - "libncursesw6:amd64=6.2-0ubuntu2," "ncurses=6.2-0ubuntu2", |
708 | - "libnettle7:amd64=3.5.1+really3.5.1-2," "nettle=3.5.1+really3.5.1-2", |
709 | - "libp11-kit0:amd64=0.23.20-1build1," "p11-kit=0.23.20-1build1", |
710 | - "libpam-modules:amd64=1.3.1-5ubuntu4.1," "pam=1.3.1-5ubuntu4.1", |
711 | - "libpam-modules-bin=1.3.1-5ubuntu4.1," "pam=1.3.1-5ubuntu4.1", |
712 | - "libpam-runtime=1.3.1-5ubuntu4.1," "pam=1.3.1-5ubuntu4.1", |
713 | - "libpam0g:amd64=1.3.1-5ubuntu4.1," "pam=1.3.1-5ubuntu4.1", |
714 | - "libpcre2-8-0:amd64=10.34-7,pcre2=10.34-7", |
715 | - "libpcre3:amd64=2:8.39-12build1," "pcre3=2:8.39-12build1", |
716 | - "libprocps8:amd64=2:3.3.16-1ubuntu2," "procps=2:3.3.16-1ubuntu2", |
717 | - "libseccomp2:amd64=2.4.3-1ubuntu3.20.04.3," |
718 | - "libseccomp=2.4.3-1ubuntu3.20.04.3", |
719 | - "libselinux1:amd64=3.0-1build2,libselinux=3.0-1build2", |
720 | - "libsemanage-common=3.0-1build2,libsemanage=3.0-1build2", |
721 | - "libsemanage1:amd64=3.0-1build2,libsemanage=3.0-1build2", |
722 | - "libsepol1:amd64=3.0-1,libsepol=3.0-1", |
723 | - "libsmartcols1:amd64=2.34-0.1ubuntu9.1,util-linux=2.34-0.1ubuntu9.1", |
724 | - "libss2:amd64=1.45.5-2ubuntu1,e2fsprogs=1.45.5-2ubuntu1", |
725 | - "libstdc++6:amd64=10.2.0-5ubuntu1~20.04,gcc-10=10.2.0-5ubuntu1~20.04", |
726 | - "libsystemd0:amd64=245.4-4ubuntu3.3,systemd=245.4-4ubuntu3.3", |
727 | - "libtasn1-6:amd64=4.16.0-2,libtasn1-6=4.16.0-2", |
728 | - "libtinfo6:amd64=6.2-0ubuntu2,ncurses=6.2-0ubuntu2", |
729 | - "libudev1:amd64=245.4-4ubuntu3.3,systemd=245.4-4ubuntu3.3", |
730 | - "libunistring2:amd64=0.9.10-2,libunistring=0.9.10-2", |
731 | - "libuuid1:amd64=2.34-0.1ubuntu9.1,util-linux=2.34-0.1ubuntu9.1", |
732 | - "libxcursor1=4.0.0-2,tiff=4.0.0-2", |
733 | - "libzstd1:amd64=1.4.4+dfsg-3,libzstd=1.4.4+dfsg-3", |
734 | - "login=1:4.8.1-1ubuntu5.20.04,shadow=1:4.8.1-1ubuntu5.20.04", |
735 | - "logsave=1.45.5-2ubuntu1,e2fsprogs=1.45.5-2ubuntu1", |
736 | - "lsb-base=11.1.0ubuntu2,lsb=11.1.0ubuntu2", |
737 | - "lua-bitop:amd64=1.0.2-5,lua-bitop=1.0.2-5", |
738 | - "lua-cjson:amd64=2.1.0+dfsg-2.1,lua-cjson=2.1.0+dfsg-2.1", |
739 | - "mawk=1.3.4.20200120-2,mawk=1.3.4.20200120-2", |
740 | - "mount=2.34-0.1ubuntu9.1,util-linux=2.34-0.1ubuntu9.1", |
741 | - "ncurses-base=6.2-0ubuntu2,ncurses=6.2-0ubuntu2", |
742 | - "ncurses-bin=6.2-0ubuntu2,ncurses=6.2-0ubuntu2", |
743 | - "passwd=1:4.8.1-1ubuntu5.20.04,shadow=1:4.8.1-1ubuntu5.20.04", |
744 | - "perl-base=5.30.0-9ubuntu0.2,perl=5.30.0-9ubuntu0.2", |
745 | - "procps=2:3.3.16-1ubuntu2,procps=2:3.3.16-1ubuntu2", |
746 | - "pwgen=2.08-2,pwgen=2.08-2", |
747 | - "redis-server=5:5.0.7-2,redis=5:5.0.7-2", |
748 | - "redis-tools=5:5.0.7-2,redis=5:5.0.7-2", |
749 | - "sed=4.7-1,sed=4.7-1", |
750 | - "sensible-utils=0.0.12+nmu1,sensible-utils=0.0.12+nmu1", |
751 | - "sysvinit-utils=2.96-2.1ubuntu1,sysvinit=2.96-2.1ubuntu1", |
752 | - "tar=1.30+dfsg-7,tar=1.30+dfsg-7", |
753 | - "tzdata=2020d-0ubuntu0.20.04,tzdata=2020d-0ubuntu0.20.04", |
754 | - "ubuntu-keyring=2020.02.11.2,ubuntu-keyring=2020.02.11.2", |
755 | - "util-linux=2.34-0.1ubuntu9.1,util-linux=2.34-0.1ubuntu9.1", |
756 | - "zlib1g:amd64=1:1.2.11.dfsg-2ubuntu1.2,zlib=1:1.2.11.dfsg-2ubuntu1.2", |
757 | - ], |
758 | - } |
759 | - self.assertDictEqual( |
760 | - expected_manifest_yaml, reviewtools.common.get_rock_manifest(valid_rock_fn) |
761 | - ) |
762 | - |
763 | - def test_get_rock_manifest_invalid_rock(self): |
764 | - """Test get_rock_manifest() - invalid rock tar""" |
765 | - invalid_rock_fn = "./tests/invalid_rock_multiple_layer_tar_archives.tar" |
766 | - with self.assertRaises(ReviewException) as e: |
767 | - reviewtools.common.get_rock_manifest(invalid_rock_fn) |
768 | - self.assertEqual( |
769 | - e.exception.value, |
770 | - "Unexpected number of layer tar archives inside layer directory: 2", |
771 | - ) |
refactor looks good to me! Use of inheritance looks clean and follows similar pattern for both snaps and rocks. Thanks!