Merge ~jslarraz/review-tools:add-doc-about-libmagic-warning into review-tools:master
- Git
- lp:~jslarraz/review-tools
- add-doc-about-libmagic-warning
- Merge into master
Proposed by
Jorge Sancho Larraz
Status: | Merged |
---|---|
Merged at revision: | f0fe85e3345eb4d0bae3657222b8d4974286a357 |
Proposed branch: | ~jslarraz/review-tools:add-doc-about-libmagic-warning |
Merge into: | review-tools:master |
Diff against target: |
947 lines (+492/-169) 9 files modified
reviewtools/available.py (+7/-2) reviewtools/common.py (+0/-142) reviewtools/containers/base_container.py (+160/-13) reviewtools/containers/snap_container.py (+44/-0) reviewtools/containers/squashfs_container.py (+111/-0) reviewtools/sr_common.py (+2/-2) reviewtools/sr_tests.py (+1/-2) reviewtools/tests/containers/test_base_container.py (+94/-8) reviewtools/tests/containers/test_squashfs_container.py (+73/-0) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Alex Murray | Needs Fixing | ||
Review via email:
|
Commit message
rt/c/base_
Description of the change
To post a comment you must log in.
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/reviewtools/available.py b/reviewtools/available.py |
2 | index eafa366..5b0dd3c 100755 |
3 | --- a/reviewtools/available.py |
4 | +++ b/reviewtools/available.py |
5 | @@ -19,6 +19,7 @@ import shutil |
6 | import tempfile |
7 | |
8 | from reviewtools.common import ( |
9 | + error, |
10 | debug, |
11 | warn, |
12 | open_file_write, |
13 | @@ -26,7 +27,6 @@ from reviewtools.common import ( |
14 | MKDTEMP_PREFIX, |
15 | _add_error, # make a class |
16 | get_rock_manifest, |
17 | - get_snap_manifest, |
18 | ) |
19 | import reviewtools.email as email |
20 | |
21 | @@ -37,6 +37,7 @@ from reviewtools.store import ( |
22 | get_faked_build_and_stage_packages, |
23 | ) |
24 | from reviewtools.usn import read_usn_db |
25 | +from reviewtools.containers.snap_container import SnapContainer, ContainerException |
26 | |
27 | email_update_required_text = """A scan of this %s shows that it was built with packages from the Ubuntu |
28 | archive that have since received security updates. """ |
29 | @@ -484,7 +485,11 @@ def scan_store( |
30 | def scan_snap(secnot_db_fn, snap_fn, with_cves=False, ignore_pockets=list()): |
31 | """Scan snap for packages with security notices""" |
32 | out = "" |
33 | - (man, dpkg) = get_snap_manifest(snap_fn) |
34 | + try: |
35 | + snap = SnapContainer(snap_fn) |
36 | + (man, dpkg) = snap.get_snap_manifest() |
37 | + except ContainerException as e: |
38 | + error(str(e)) |
39 | man = get_faked_build_and_stage_packages(man) |
40 | |
41 | # Use dpkg.list when the snap ships it in a spot we honor. This is limited |
42 | diff --git a/reviewtools/common.py b/reviewtools/common.py |
43 | index e2e80ff..d2c15b8 100644 |
44 | --- a/reviewtools/common.py |
45 | +++ b/reviewtools/common.py |
46 | @@ -424,30 +424,6 @@ def cmdIgnoreErrorStrings(command, ignoreErrorStrings): |
47 | return [rc, out] |
48 | |
49 | |
50 | -def _unpack_cmd(cmd_args, d, dest): |
51 | - """Low level unpack helper""" |
52 | - curdir = os.getcwd() |
53 | - os.chdir(d) |
54 | - |
55 | - (rc, out) = cmdIgnoreErrorStrings(cmd_args, UNSQUASHFS_IGNORED_ERRORS) |
56 | - |
57 | - os.chdir(curdir) |
58 | - |
59 | - if rc != 0: |
60 | - if os.path.isdir(d): |
61 | - recursive_rm(d) |
62 | - error("unpacking failed with '%d':\n%s" % (rc, out)) |
63 | - |
64 | - if dest is None: |
65 | - dest = d |
66 | - else: |
67 | - # Recursively move unpacked content to original dir, keeping |
68 | - # permissions |
69 | - move_dir_content(d, dest) |
70 | - |
71 | - return dest |
72 | - |
73 | - |
74 | # The original directory might have restrictive permissions, so save |
75 | # them off, chmod the dir, do the move and reapply |
76 | def move_dir_content(d, dest): |
77 | @@ -621,24 +597,6 @@ def unsquashfs_lln_parse(lln_out): |
78 | return hdr, entries |
79 | |
80 | |
81 | -def _calculate_snap_unsquashfs_uncompressed_size(snap_pkg): |
82 | - """Calculate size of the uncompressed snap""" |
83 | - (rc, out) = cmd(["unsquashfs", "-lln", snap_pkg]) |
84 | - if rc != 0: |
85 | - error("unsquashfs -lln '%s' failed: %s" % (snap_pkg, out)) |
86 | - |
87 | - size = 0 |
88 | - for line in out.splitlines(): |
89 | - if not line.startswith("-"): # skip non-regular files |
90 | - continue |
91 | - try: |
92 | - size += int(line.split()[2]) |
93 | - except ValueError: # skip non-numbers |
94 | - continue |
95 | - |
96 | - return size |
97 | - |
98 | - |
99 | def _calculate_rock_untar_uncompressed_size(rock_pkg): |
100 | """Calculate size of the uncompressed tar""" |
101 | size = 0 |
102 | @@ -656,34 +614,6 @@ def unsquashfs_supports_ignore_errors(): |
103 | return "-ig[nore-errors]" in out |
104 | |
105 | |
106 | -def _unpack_snap_squashfs(snap_pkg, dest, items=[]): |
107 | - """Unpack a squashfs based snap package to dest""" |
108 | - size = _calculate_snap_unsquashfs_uncompressed_size(snap_pkg) |
109 | - |
110 | - snap_max_size = MAX_UNCOMPRESSED_SIZE * 1024 * 1024 * 1024 |
111 | - valid_size, error_msg = is_pkg_uncompressed_size_valid( |
112 | - snap_max_size, size, snap_pkg |
113 | - ) |
114 | - if valid_size: |
115 | - global MKDTEMP_PREFIX |
116 | - global MKDTEMP_DIR |
117 | - d = tempfile.mkdtemp(prefix=MKDTEMP_PREFIX, dir=MKDTEMP_DIR) |
118 | - cmd = ["unsquashfs", "-no-progress", "-f", "-d", d] |
119 | - |
120 | - # If unsquashfs supports it, pass "-ignore-errors -quiet" |
121 | - if unsquashfs_supports_ignore_errors(): |
122 | - cmd.append("-ignore-errors") |
123 | - cmd.append("-quiet") |
124 | - |
125 | - cmd.append(os.path.abspath(snap_pkg)) |
126 | - |
127 | - if len(items) != 0: |
128 | - cmd += items |
129 | - return _unpack_cmd(cmd, d, dest) |
130 | - else: |
131 | - error(error_msg) |
132 | - |
133 | - |
134 | def is_pkg_uncompressed_size_valid(pkg_max_size, size, pkg): |
135 | st = os.statvfs(pkg) |
136 | avail = st.f_bsize * st.f_bavail * 0.9 # 90% of available space |
137 | @@ -751,21 +681,6 @@ def _unpack_rock_tar(rock_pkg, dest): |
138 | error(error_msg) |
139 | |
140 | |
141 | -def unpack_pkg(fn, dest=None, items=[]): |
142 | - """Unpack package""" |
143 | - pkg = check_fn(fn) |
144 | - check_dir(dest) |
145 | - |
146 | - # Limit the maximimum size of the package |
147 | - check_max_pkg_size(pkg) |
148 | - |
149 | - # check if its a squashfs based snap |
150 | - if is_squashfs(pkg): |
151 | - return _unpack_snap_squashfs(fn, dest, items) |
152 | - |
153 | - error("Unsupported package format (not squashfs)") |
154 | - |
155 | - |
156 | def check_dir(dest): |
157 | if dest is not None and os.path.exists(dest): |
158 | error("'%s' exists. Aborting." % dest) |
159 | @@ -790,13 +705,6 @@ def check_max_pkg_size(pkg): |
160 | ) |
161 | |
162 | |
163 | -def is_squashfs(filename): |
164 | - """Return true if the given filename as a squashfs header""" |
165 | - with open(filename, "rb") as f: |
166 | - header = f.read(10) |
167 | - return header.startswith(b"hsqs") |
168 | - |
169 | - |
170 | def unpack_rock(fn, dest=None): |
171 | """Unpack rock""" |
172 | pkg = check_fn(fn) |
173 | @@ -1078,56 +986,6 @@ def read_file_as_json_dict(fn): |
174 | return raw |
175 | |
176 | |
177 | -def get_snap_manifest(fn): |
178 | - if "SNAP_USER_COMMON" in os.environ and os.path.exists( |
179 | - os.environ["SNAP_USER_COMMON"] |
180 | - ): |
181 | - MKDTEMP_DIR = os.environ["SNAP_USER_COMMON"] |
182 | - else: |
183 | - MKDTEMP_DIR = tempfile.gettempdir() |
184 | - |
185 | - man = "snap/manifest.yaml" |
186 | - os_dpkg = "usr/share/snappy/dpkg.list" |
187 | - snap_dpkg = "snap/dpkg.list" |
188 | - # unpack_pkg() fails if this exists, so this is safe |
189 | - dir = tempfile.mktemp(prefix=MKDTEMP_PREFIX, dir=MKDTEMP_DIR) |
190 | - unpack_pkg(fn, dir, [man, os_dpkg, snap_dpkg]) |
191 | - |
192 | - man_fn = os.path.join(dir, man) |
193 | - if not os.path.isfile(man_fn): |
194 | - recursive_rm(dir) |
195 | - error("%s not in %s" % (man, fn)) |
196 | - |
197 | - with open_file_read(man_fn) as fd: |
198 | - try: |
199 | - man_yaml = yaml.safe_load(fd) |
200 | - except Exception: |
201 | - recursive_rm(dir) |
202 | - error("Could not load %s. Is it properly formatted?" % man) |
203 | - |
204 | - os_dpkg_fn = os.path.join(dir, os_dpkg) |
205 | - snap_dpkg_fn = os.path.join(dir, snap_dpkg) |
206 | - dpkg_list = None |
207 | - if os.path.isfile(os_dpkg_fn): |
208 | - with open_file_read(os_dpkg_fn) as fd: |
209 | - try: |
210 | - dpkg_list = fd.readlines() |
211 | - except Exception: |
212 | - recursive_rm(dir) |
213 | - error("Could not load %s. Is it properly formatted?" % os_dpkg) |
214 | - elif os.path.isfile(snap_dpkg_fn): |
215 | - with open_file_read(snap_dpkg_fn) as fd: |
216 | - try: |
217 | - dpkg_list = fd.readlines() |
218 | - except Exception: |
219 | - recursive_rm(dir) |
220 | - error("Could not load %s. Is it properly formatted?" % snap_dpkg) |
221 | - |
222 | - recursive_rm(dir) |
223 | - |
224 | - return (man_yaml, dpkg_list) |
225 | - |
226 | - |
227 | def get_rock_manifest(fn): |
228 | if "SNAP_USER_COMMON" in os.environ and os.path.exists( |
229 | os.environ["SNAP_USER_COMMON"] |
230 | diff --git a/reviewtools/containers/base_container.py b/reviewtools/containers/base_container.py |
231 | index 9e6f104..0d51021 100644 |
232 | --- a/reviewtools/containers/base_container.py |
233 | +++ b/reviewtools/containers/base_container.py |
234 | @@ -7,12 +7,19 @@ from tempfile import mkdtemp |
235 | from reviewtools.common import ( |
236 | MKDTEMP_PREFIX, |
237 | MKDTEMP_DIR, |
238 | - debug, |
239 | - unpack_pkg, |
240 | recursive_rm, |
241 | ) |
242 | |
243 | |
244 | +# Add base_container private attributes to documentation |
245 | +__pdoc__ = { |
246 | + "BaseContainer._min_packed_size": True, |
247 | + "BaseContainer._max_packed_size": True, |
248 | + "BaseContainer._min_unpacked_size": True, |
249 | + "BaseContainer._max_unpacked_size": True, |
250 | +} |
251 | + |
252 | + |
253 | class ContainerException(Exception): |
254 | """This class represents exceptions""" |
255 | |
256 | @@ -32,16 +39,36 @@ class BaseContainer: |
257 | """ |
258 | |
259 | magic_binary_file_descriptions = [ |
260 | - "application/x-executable; charset=binary", |
261 | - "application/x-sharedlib; charset=binary", |
262 | - "application/x-object; charset=binary", |
263 | - # 18.04 and higher doesn't show the charset |
264 | "application/x-executable", |
265 | "application/x-sharedlib", |
266 | "application/x-object", |
267 | "application/x-pie-executable", |
268 | ] |
269 | |
270 | + _min_packed_size = None |
271 | + """ |
272 | + Minimum size of the container file in bytes. It will be used to check the |
273 | + container size during initialization. |
274 | + """ |
275 | + |
276 | + _max_packed_size = None |
277 | + """ |
278 | + Maximum size of the container file in bytes. It will be used to check the |
279 | + container size during initialization. |
280 | + """ |
281 | + |
282 | + _min_unpacked_size = None |
283 | + """ |
284 | + Minimum size of the unpacked directory in bytes. It will be used to check the |
285 | + container size during initialization. |
286 | + """ |
287 | + |
288 | + _max_unpacked_size = None |
289 | + """ |
290 | + Maximum size of the unpacked directory in bytes. It will be used to check the |
291 | + container size during initialization. |
292 | + """ |
293 | + |
294 | _unpack_dir = None |
295 | _files = None |
296 | _bin_files = None |
297 | @@ -59,6 +86,9 @@ class BaseContainer: |
298 | else: |
299 | raise ContainerException("Could not find '%s'" % fn) |
300 | |
301 | + # Check max/min size if defined |
302 | + self.check_container_size() |
303 | + |
304 | @property |
305 | def unpack_dir(self) -> str: |
306 | """ |
307 | @@ -67,8 +97,7 @@ class BaseContainer: |
308 | will be returned. |
309 | """ |
310 | if self._unpack_dir is None: |
311 | - self._unpack_dir = ".".join(self.filename.split(".")[:-1]) |
312 | - unpack_pkg(self.filename, dest=self._unpack_dir) |
313 | + self._unpack_dir = self.unpack() |
314 | return self._unpack_dir |
315 | |
316 | @property |
317 | @@ -104,6 +133,117 @@ class BaseContainer: |
318 | if os.path.exists(self.tmp_dir): |
319 | recursive_rm(self.tmp_dir) |
320 | |
321 | + def check_container_format(self): |
322 | + """ |
323 | + Checks that the container file has the expected format. |
324 | + |
325 | + Raises: |
326 | + NotImplementedError: Calling this method will raise an exception as |
327 | + format is specific to the container type. This method needs to be |
328 | + implemented by the specific subclasses. |
329 | + """ |
330 | + raise NotImplementedError("Must be implemented by subclasses") |
331 | + |
332 | + def get_container_size(self) -> int: |
333 | + """ |
334 | + Uses os.path.getsize() to return the size of the container file. |
335 | + |
336 | + Retruns: |
337 | + int: Size in bytes of the container file |
338 | + """ |
339 | + return os.path.getsize(self.filename) |
340 | + |
341 | + def check_container_size(self): |
342 | + """ |
343 | + Checks that the container size is within the range specified by |
344 | + min_packed_size and max_packed_size. If any of those values is not specified |
345 | + (i.e. it is None), the respective check will be skipped. |
346 | + |
347 | + Raises: |
348 | + ContainerException: If container size is less than minSize or greater |
349 | + than maxSize, a ContainerException will be raised. |
350 | + """ |
351 | + size = self.get_container_size() |
352 | + if self._min_packed_size and (size < self._min_packed_size): |
353 | + raise ContainerException( |
354 | + "container file is too small (%dM < %dM)" |
355 | + % (size / 1024 / 1024, self._min_packed_size / 1024 / 1024) |
356 | + ) |
357 | + if self._max_packed_size and (size > self._max_packed_size): |
358 | + raise ContainerException( |
359 | + "container file is too large (%dM > %dM)" |
360 | + % (size / 1024 / 1024, self._max_packed_size / 1024 / 1024) |
361 | + ) |
362 | + |
363 | + def calculate_unpacked_size(self) -> int: |
364 | + """ |
365 | + Estimates the space that the unpacked directory will use without unpacking. |
366 | + |
367 | + Returns: |
368 | + int: Estimated size in bytes of the unpacked directory |
369 | + |
370 | + Raises: |
371 | + NotImplementedError: Calling this method will raise an exception as the |
372 | + estimation method is specific to the container type. This method |
373 | + needs to be implemented by the specific subclasses. |
374 | + """ |
375 | + raise NotImplementedError("Must be implemented by subclasses") |
376 | + |
377 | + def check_unpacked_size(self): |
378 | + """ |
379 | + Checks that the container size is within the range specified by |
380 | + min_unpacked_size and max_unpacked_size. If any of those values is not |
381 | + specified (i.e. it is None), the respective check will be skipped. |
382 | + |
383 | + Raises: |
384 | + ContainerException: If unpacked size is less than minSize or greater |
385 | + than maxSize, a ContainerException will be raised. |
386 | + """ |
387 | + size = self.calculate_unpacked_size() |
388 | + if self._min_unpacked_size and (size < self._min_unpacked_size): |
389 | + raise ContainerException( |
390 | + "unpacked directory is too small (%dM < %dM)" |
391 | + % (size / 1024 / 1024, self._min_unpacked_size / 1024 / 1024) |
392 | + ) |
393 | + if self._max_unpacked_size and (size > self._max_unpacked_size): |
394 | + raise ContainerException( |
395 | + "unpacked directory is too large (%dM > %dM)" |
396 | + % (size / 1024 / 1024, self._max_unpacked_size / 1024 / 1024) |
397 | + ) |
398 | + |
399 | + def check_disk_space(self): |
400 | + """ |
401 | + Checks that there is sufficient disk space available for the container to be |
402 | + unpacked, i.e. that it will not consume more than 90% of the available disk |
403 | + |
404 | + Raises: |
405 | + ContainerException: If estimated unpacked size exceed the 90% of the |
406 | + available space, an exception will be raised. |
407 | + """ |
408 | + size = self.calculate_unpacked_size() |
409 | + st = os.statvfs(self.tmp_dir) |
410 | + avail = st.f_bsize * st.f_bavail * 0.9 # 90% of available space |
411 | + if size > avail * 0.9: |
412 | + raise ContainerException( |
413 | + "insufficient available disk space (%dM > %dM)" |
414 | + % (size / 1024 / 1024, avail / 1024 / 1024) |
415 | + ) |
416 | + |
417 | + def unpack(self) -> str: |
418 | + """ |
419 | + Extracts the content of the container file and returns the absolute path to |
420 | + the newly unpacked directory. |
421 | + |
422 | + Returns: |
423 | + str: absolute path to the newly created unpack dir |
424 | + |
425 | + Raises: |
426 | + NotImplementedError: Calling this method will raise an exception as the |
427 | + estimation method is specific to the container type. This method |
428 | + needs to be implemented by the specific subclasses. |
429 | + """ |
430 | + raise NotImplementedError("Must be implemented by subclasses") |
431 | + |
432 | def get_file(self, fn: str) -> Union[str, None]: |
433 | """ |
434 | Reads and return the raw content of the specified file. |
435 | @@ -117,7 +257,7 @@ class BaseContainer: |
436 | |
437 | Raises: |
438 | ContainerException: Calling this method will raise an exception if |
439 | - the requested file is not found. |
440 | + the requested file is not found. |
441 | """ |
442 | if not os.path.exists(os.path.join(self.unpack_dir, fn)): |
443 | raise ContainerException("Could not find '%s'" % fn) |
444 | @@ -156,15 +296,22 @@ class BaseContainer: |
445 | Returns: |
446 | list(str): list of files present in the container |
447 | """ |
448 | - mime = magic.open(magic.MAGIC_MIME) |
449 | + mime = magic.compat.open(magic.MAGIC_MIME) |
450 | mime.load() |
451 | bin_files = [] |
452 | for file in self.get_files_list(abs=True): |
453 | try: |
454 | + # There are two different magic libraries for python, python-magic |
455 | + # and python-libmagic. We are installing python-magic (which is the |
456 | + # only one distributed via apt). However, python-magic seems to hang |
457 | + # when it hits a pipe file. |
458 | + # python-magic also bundles the python-libmagic module and exposes |
459 | + # it through the following compatibility API. Using this bundled |
460 | + # python-libmagic module will generate a deprecation warning. See |
461 | + # https://github.com/ahupp/python-magic/blob/master/magic/__init__.py#L458 |
462 | res = mime.file(file) |
463 | - except Exception: # pragma: nocover |
464 | - # workaround for zesty python3-magic |
465 | - debug("could not detemine mime type of '%s'" % file) |
466 | + except OSError: |
467 | + # TODO: file does not exists. Only hit for external symlinks |
468 | continue |
469 | |
470 | if not abs: |
471 | diff --git a/reviewtools/containers/snap_container.py b/reviewtools/containers/snap_container.py |
472 | new file mode 100644 |
473 | index 0000000..ff1ee57 |
474 | --- /dev/null |
475 | +++ b/reviewtools/containers/snap_container.py |
476 | @@ -0,0 +1,44 @@ |
477 | +import yaml |
478 | +from reviewtools.containers.squashfs_container import ( |
479 | + SquashfsContainer, |
480 | + ContainerException, |
481 | +) |
482 | + |
483 | + |
484 | +class SnapContainer(SquashfsContainer): |
485 | + # TODO: review-tools was not checking for it. We still have some snaps in |
486 | + # "tests/" that doesn't fit this limit and will need to be regenerated before |
487 | + # this check can be enforced. |
488 | + # _min_packed_size = 16 * 1024 |
489 | + _max_packed_size = 5 * 1024 * 1024 * 1024 |
490 | + _max_unpacked_size = 25 * 1024 * 1024 * 1024 |
491 | + |
492 | + def get_snap_manifest(self): |
493 | + man = "snap/manifest.yaml" |
494 | + os_dpkg = "usr/share/snappy/dpkg.list" |
495 | + snap_dpkg = "snap/dpkg.list" |
496 | + |
497 | + if self._unpack_dir is None: |
498 | + self.unpack(items=[man, os_dpkg, snap_dpkg]) |
499 | + |
500 | + man_content = self.get_file(man).decode() |
501 | + try: |
502 | + man_yaml = yaml.safe_load(man_content) |
503 | + except Exception: |
504 | + raise ContainerException( |
505 | + "Could not load %s. Is it properly formatted?" % man |
506 | + ) |
507 | + |
508 | + dpkg_list = None |
509 | + try: |
510 | + dpkg_list = self.get_file(os_dpkg).decode().split("\n") |
511 | + return (man_yaml, dpkg_list) |
512 | + except ContainerException: |
513 | + pass |
514 | + |
515 | + try: |
516 | + dpkg_list = self.get_file(snap_dpkg).decode().split("\n") |
517 | + except ContainerException: |
518 | + pass |
519 | + |
520 | + return (man_yaml, dpkg_list) |
521 | diff --git a/reviewtools/containers/squashfs_container.py b/reviewtools/containers/squashfs_container.py |
522 | new file mode 100644 |
523 | index 0000000..631fe76 |
524 | --- /dev/null |
525 | +++ b/reviewtools/containers/squashfs_container.py |
526 | @@ -0,0 +1,111 @@ |
527 | +import os |
528 | +import shutil |
529 | +from reviewtools.common import cmd, cmdIgnoreErrorStrings, UNSQUASHFS_IGNORED_ERRORS |
530 | +from reviewtools.containers.base_container import BaseContainer, ContainerException |
531 | + |
532 | + |
533 | +class SquashfsContainer(BaseContainer): |
534 | + """ |
535 | + A container is a file that encapsulates a file system. This class extends |
536 | + BaseContainer to handle functionality that is specific to Squashfs containers. |
537 | + """ |
538 | + |
539 | + def __init__(self, fn): |
540 | + """ |
541 | + If SquashfsContainer is initialized with a container file, it will also |
542 | + verify that the provided file is actually a squashfs, in addition to the |
543 | + checks performed by the BaseContainer. |
544 | + """ |
545 | + super().__init__(fn) |
546 | + |
547 | + # Check is squash file |
548 | + self.check_container_format() |
549 | + |
550 | + def check_container_format(self): |
551 | + """ |
552 | + Checks that the container file has the expected format. |
553 | + |
554 | + Raises: |
555 | + ContainerException: This method will raise an exception if the |
556 | + container file is not a squashfs container. |
557 | + """ |
558 | + with open(self.filename, "rb") as f: |
559 | + header = f.read(10) |
560 | + if not header.startswith(b"hsqs"): |
561 | + raise ContainerException("Unsupported package format (not squashfs)") |
562 | + |
563 | + def calculate_unpacked_size(self) -> int: |
564 | + """ |
565 | + Estimates the space that the unpacked directory will use by adding the |
566 | + individual files size reported by "unsquashfs -lln" command. It may not |
567 | + be completely accurate, as this command does not consider that each folder |
568 | + requires at least one complete block |
569 | + |
570 | + Returns: |
571 | + int: Estimated size in bytes of the unpacked directory |
572 | + """ |
573 | + (rc, out) = cmd(["unsquashfs", "-lln", self.filename]) |
574 | + if rc != 0: |
575 | + raise RuntimeError("unsquashfs -lln '%s' failed: %s" % (self.filename, out)) |
576 | + |
577 | + size = 0 |
578 | + for line in out.splitlines(): |
579 | + if not line.startswith("-"): # skip non-regular files |
580 | + continue |
581 | + try: |
582 | + size += int(line.split()[2]) |
583 | + except ValueError: # skip non-numbers |
584 | + continue |
585 | + |
586 | + return size |
587 | + |
588 | + def unpack(self, items: list = [], force: bool = False) -> str: |
589 | + """ |
590 | + Extracts the content of the container file and returns the absolute path to |
591 | + the newly unpacked directory. |
592 | + |
593 | + Args: |
594 | + force (bool, optional): If ``True``, the unpack_dir will be overwritten |
595 | + if it already exists. Defaults to ``False`` |
596 | + |
597 | + Returns: |
598 | + str: absolute path to the newly created unpack dir |
599 | + |
600 | + Raises: |
601 | + ContainerException: This method will raise an ContainerException if the |
602 | + unpack_dir already exists and this method was called without the |
603 | + force argument set to True |
604 | + RuntimeError: This method will raise a RuntimeError under the execution |
605 | + of the "unsquashfs" command returns with an error code other than 0. |
606 | + """ |
607 | + if self._unpack_dir is not None and not force: |
608 | + raise ContainerException("'%s' exists. Aborting." % self._unpack_dir) |
609 | + |
610 | + # Check disk space and space available in disk |
611 | + self.check_unpacked_size() |
612 | + self.check_disk_space() |
613 | + |
614 | + # Unpack the container |
615 | + stage_dir = os.path.join(self.tmp_dir, "stage") |
616 | + cmd = [ |
617 | + "unsquashfs", |
618 | + "-no-progress", |
619 | + "-ignore-errors", |
620 | + "-quiet", |
621 | + "-f", |
622 | + "-d", |
623 | + stage_dir, |
624 | + self.filename, |
625 | + ] + items |
626 | + (rc, out) = cmdIgnoreErrorStrings(cmd, UNSQUASHFS_IGNORED_ERRORS) |
627 | + if rc != 0: |
628 | + if os.path.isdir(stage_dir): |
629 | + shutil.rmtree(stage_dir) |
630 | + raise RuntimeError("unpacking failed with '%d':\n%s" % (rc, out)) |
631 | + |
632 | + # Update on success |
633 | + if self._unpack_dir is not None and os.path.exists(self._unpack_dir): |
634 | + shutil.rmtree(self._unpack_dir) |
635 | + self._unpack_dir = ".".join(self.filename.split(".")[:-1]) |
636 | + shutil.move(stage_dir, self._unpack_dir) |
637 | + return self._unpack_dir |
638 | diff --git a/reviewtools/sr_common.py b/reviewtools/sr_common.py |
639 | index 16790cc..9b961e2 100644 |
640 | --- a/reviewtools/sr_common.py |
641 | +++ b/reviewtools/sr_common.py |
642 | @@ -30,7 +30,7 @@ from reviewtools.common import ( |
643 | unsquashfs_lln_parse, |
644 | verify_type, |
645 | ) |
646 | -from reviewtools.containers.base_container import BaseContainer, ContainerException |
647 | +from reviewtools.containers.snap_container import SnapContainer, ContainerException |
648 | from reviewtools.overrides import interfaces_attribs_addons |
649 | |
650 | |
651 | @@ -398,7 +398,7 @@ class SnapReview(ReviewBase): |
652 | ReviewBase.__init__(self, review_type, overrides=overrides) |
653 | |
654 | try: |
655 | - self.snap = BaseContainer(fn) |
656 | + self.snap = SnapContainer(fn) |
657 | except ContainerException as e: |
658 | error(str(e)) |
659 | # TODO: update code to directly use self.snap.xxx instead of assigning here |
660 | diff --git a/reviewtools/sr_tests.py b/reviewtools/sr_tests.py |
661 | index 6caae54..bfbf375 100644 |
662 | --- a/reviewtools/sr_tests.py |
663 | +++ b/reviewtools/sr_tests.py |
664 | @@ -87,7 +87,7 @@ def create_patches(): |
665 | # child. |
666 | patches = [] |
667 | patches.append( |
668 | - patch("reviewtools.sr_common.BaseContainer.__init__", _BaseContainer.__init__) |
669 | + patch("reviewtools.sr_common.SnapContainer.__init__", _BaseContainer.__init__) |
670 | ) |
671 | patches.append( |
672 | patch("reviewtools.sr_common.SnapReview._extract_snap_yaml", _extract_snap_yaml) |
673 | @@ -98,7 +98,6 @@ def create_patches(): |
674 | _extract_snap_manifest_yaml, |
675 | ) |
676 | ) |
677 | - patches.append(patch("reviewtools.common.unpack_pkg", _mock_func)) |
678 | # sr_common |
679 | patches.append( |
680 | patch("reviewtools.sr_common.SnapReview._get_unpack_dir", __get_unpack_dir) |
681 | diff --git a/reviewtools/tests/containers/test_base_container.py b/reviewtools/tests/containers/test_base_container.py |
682 | index 740e942..1b4e022 100644 |
683 | --- a/reviewtools/tests/containers/test_base_container.py |
684 | +++ b/reviewtools/tests/containers/test_base_container.py |
685 | @@ -3,29 +3,43 @@ import shutil |
686 | import unittest |
687 | from tempfile import mkdtemp |
688 | from reviewtools.containers.base_container import BaseContainer, ContainerException |
689 | -from reviewtools.tests.utils import make_snap2 |
690 | |
691 | |
692 | class TestBaseContainer(unittest.TestCase): |
693 | """Tests for reviewtools.containers.base_container.BaseContainer""" |
694 | |
695 | - expected_files = ["meta/snap.yaml", "meta/icon.png", "nonexecstack.bin"] |
696 | - expected_binaries = ["nonexecstack.bin"] |
697 | + expected_files = ["meta/snap.yaml", "meta/icon.png", "bin/ls"] |
698 | + expected_binaries = ["bin/ls"] |
699 | |
700 | @classmethod |
701 | def setUpClass(cls): |
702 | cls.tmp_dir = mkdtemp() |
703 | - cls.fn = make_snap2( |
704 | - output_dir=cls.tmp_dir, |
705 | - yaml="Test", |
706 | - extra_files=["/bin/ls:nonexecstack.bin"], |
707 | - ) |
708 | + cls.fn = cls._create_container(cls) |
709 | + cls.unpack_dir = cls._create_unpackdir(cls) |
710 | |
711 | @classmethod |
712 | def tearDownClass(cls): |
713 | if os.path.exists(cls.tmp_dir): |
714 | shutil.rmtree(cls.tmp_dir) |
715 | |
716 | + def _create_container(self, container_size: int = 10000): |
717 | + fn = os.path.join(self.tmp_dir, "container.test") |
718 | + with open(fn, "w") as f: |
719 | + f.truncate(container_size) |
720 | + return fn |
721 | + |
722 | + def _create_unpackdir(self): |
723 | + unpack_dir = os.path.join(self.tmp_dir, "unpack_dir") |
724 | + for file in self.expected_files: |
725 | + unpacked_file = os.path.join(unpack_dir, file) |
726 | + os.makedirs(os.path.dirname(unpacked_file), exist_ok=True) |
727 | + if file not in self.expected_binaries: |
728 | + with open(unpacked_file, "w+b") as fd: |
729 | + fd.write(b"Test") |
730 | + else: |
731 | + shutil.copyfile(os.path.join("/", file), unpacked_file) |
732 | + return unpack_dir |
733 | + |
734 | # Test class initialization |
735 | def test_initialization_container__happy(self): |
736 | try: |
737 | @@ -43,10 +57,75 @@ class TestBaseContainer(unittest.TestCase): |
738 | in context.exception.value |
739 | ) |
740 | |
741 | + # Test deletion |
742 | + def test_check_cleanup(self): |
743 | + try: |
744 | + container = BaseContainer(self.fn) |
745 | + tmp_dir = container.tmp_dir |
746 | + del container |
747 | + self.assertFalse(os.path.exists(tmp_dir)) |
748 | + except Exception: |
749 | + self.fail("An unexpected error occurred during deletion test") |
750 | + |
751 | + # Test check container size |
752 | + def test_check_container_size__happy(self): |
753 | + container = BaseContainer(self.fn) |
754 | + container._min_packed_size = 0 |
755 | + container._max_packed_size = 10**6 |
756 | + try: |
757 | + container.check_container_size() |
758 | + except ContainerException: |
759 | + self.fail("An unexpected error occurred while checking file size") |
760 | + |
761 | + def test_check_container_size__too_big(self): |
762 | + container = BaseContainer(self.fn) |
763 | + container._max_packed_size = 1 |
764 | + with self.assertRaises(ContainerException) as context: |
765 | + container.check_container_size() |
766 | + self.assertTrue("container file is too large " in context.exception.value) |
767 | + |
768 | + def test_check_container_size__too_small(self): |
769 | + container = BaseContainer(self.fn) |
770 | + container._min_packed_size = 10**6 |
771 | + with self.assertRaises(ContainerException) as context: |
772 | + container.check_container_size() |
773 | + self.assertTrue("container file is too small " in context.exception.value) |
774 | + |
775 | + # Test check unpacked size |
776 | + def _mock_calculate_unpacked_size(self): |
777 | + return 10**3 |
778 | + |
779 | + def test_check_unpacked_size__happy(self): |
780 | + container = BaseContainer(self.fn) |
781 | + container.calculate_unpacked_size = self._mock_calculate_unpacked_size |
782 | + container._min_unpacked_size = 0 |
783 | + container._max_unpacked_size = 10**6 |
784 | + try: |
785 | + container.check_unpacked_size() |
786 | + except ContainerException: |
787 | + self.fail("An unexpected error occurred while checking file size") |
788 | + |
789 | + def test_check_unpacked_size__too_big(self): |
790 | + container = BaseContainer(self.fn) |
791 | + container.calculate_unpacked_size = self._mock_calculate_unpacked_size |
792 | + container._max_unpacked_size = 1 |
793 | + with self.assertRaises(ContainerException) as context: |
794 | + container.check_unpacked_size() |
795 | + self.assertTrue("container file is too large " in context.exception.value) |
796 | + |
797 | + def test_check_unpacked_size__too_small(self): |
798 | + container = BaseContainer(self.fn) |
799 | + container.calculate_unpacked_size = self._mock_calculate_unpacked_size |
800 | + container._min_unpacked_size = 10**6 |
801 | + with self.assertRaises(ContainerException) as context: |
802 | + container.check_unpacked_size() |
803 | + self.assertTrue("container file is too small " in context.exception.value) |
804 | + |
805 | # Test get file |
806 | def test_get_file__happy(self): |
807 | try: |
808 | container = BaseContainer(self.fn) |
809 | + container._unpack_dir = self.unpack_dir |
810 | snap_yaml = container.get_file("meta/snap.yaml") |
811 | self.assertEqual(snap_yaml.decode(), "Test") |
812 | except ContainerException: |
813 | @@ -55,6 +134,7 @@ class TestBaseContainer(unittest.TestCase): |
814 | def test_get_file__missing_file(self): |
815 | with self.assertRaises(ContainerException) as context: |
816 | container = BaseContainer(self.fn) |
817 | + container._unpack_dir = self.unpack_dir |
818 | container.get_file("non/existent/file") |
819 | self.assertTrue("Could not find" in context.exception.value) |
820 | |
821 | @@ -62,6 +142,7 @@ class TestBaseContainer(unittest.TestCase): |
822 | def test_get_files_list__happy(self): |
823 | try: |
824 | container = BaseContainer(self.fn) |
825 | + container._unpack_dir = self.unpack_dir |
826 | self.assertCountEqual( |
827 | container.get_files_list(), |
828 | self.expected_files, |
829 | @@ -72,6 +153,7 @@ class TestBaseContainer(unittest.TestCase): |
830 | def test_get_files_list_abs__happy(self): |
831 | try: |
832 | container = BaseContainer(self.fn) |
833 | + container._unpack_dir = self.unpack_dir |
834 | self.assertCountEqual( |
835 | container.get_files_list(abs=True), |
836 | [ |
837 | @@ -85,6 +167,7 @@ class TestBaseContainer(unittest.TestCase): |
838 | def test_files__happy(self): |
839 | try: |
840 | container = BaseContainer(self.fn) |
841 | + container._unpack_dir = self.unpack_dir |
842 | self.assertCountEqual( |
843 | container.files, |
844 | self.expected_files, |
845 | @@ -96,6 +179,7 @@ class TestBaseContainer(unittest.TestCase): |
846 | def test_get_compiled_binaries_list__happy(self): |
847 | try: |
848 | container = BaseContainer(self.fn) |
849 | + container._unpack_dir = self.unpack_dir |
850 | self.assertCountEqual( |
851 | container.get_compiled_binaries_list(), self.expected_binaries |
852 | ) |
853 | @@ -107,6 +191,7 @@ class TestBaseContainer(unittest.TestCase): |
854 | def test_get_compiled_binaries_list_abs__happy(self): |
855 | try: |
856 | container = BaseContainer(self.fn) |
857 | + container._unpack_dir = self.unpack_dir |
858 | self.assertCountEqual( |
859 | container.get_compiled_binaries_list(abs=True), |
860 | [ |
861 | @@ -122,6 +207,7 @@ class TestBaseContainer(unittest.TestCase): |
862 | def test_bin_files__happy(self): |
863 | try: |
864 | container = BaseContainer(self.fn) |
865 | + container._unpack_dir = self.unpack_dir |
866 | self.assertCountEqual(container.bin_files, self.expected_binaries) |
867 | except ContainerException: |
868 | self.fail( |
869 | diff --git a/reviewtools/tests/containers/test_squashfs_container.py b/reviewtools/tests/containers/test_squashfs_container.py |
870 | new file mode 100644 |
871 | index 0000000..7d19a77 |
872 | --- /dev/null |
873 | +++ b/reviewtools/tests/containers/test_squashfs_container.py |
874 | @@ -0,0 +1,73 @@ |
875 | +import os |
876 | +import unittest |
877 | +from tempfile import mkstemp, mkdtemp |
878 | +from reviewtools.containers.squashfs_container import ( |
879 | + SquashfsContainer, |
880 | + ContainerException, |
881 | +) |
882 | +from reviewtools.tests.utils import make_snap2 |
883 | + |
884 | + |
885 | +class TestSquashfsContainer(unittest.TestCase): |
886 | + """Tests for reviewtools.containers.squashfs_container""" |
887 | + |
888 | + @classmethod |
889 | + def setUpClass(cls): |
890 | + cls.tmp_dir = mkdtemp() |
891 | + cls.fn = make_snap2( |
892 | + output_dir=cls.tmp_dir, |
893 | + yaml="Test", |
894 | + extra_files=["/bin/ls:bin/ls"], |
895 | + ) |
896 | + |
897 | + # Test initialization |
898 | + def test_check_initialization__happy(self): |
899 | + try: |
900 | + SquashfsContainer(self.fn) |
901 | + except Exception: |
902 | + self.fail( |
903 | + "An unexpected error occurred during SquashfsContainer initialization with container file" |
904 | + ) |
905 | + |
906 | + def test_check_initialization__invalid_format(self): |
907 | + with self.assertRaises(ContainerException) as context: |
908 | + fd, plain_file = mkstemp() |
909 | + SquashfsContainer(plain_file) |
910 | + self.assertEqual( |
911 | + "unsupported package format (not squashfs)", context.exception.value |
912 | + ) |
913 | + os.unlink(plain_file) |
914 | + |
915 | + # Test check format |
916 | + def test_calculate_unpacked_size__happy(self): |
917 | + try: |
918 | + # unpacked size calculated is a bit smaller than actual size as it does not consider that |
919 | + # folders require at least one complete block |
920 | + container = SquashfsContainer(self.fn) |
921 | + size = container.calculate_unpacked_size() |
922 | + self.assertTrue(isinstance(size, int)) |
923 | + except Exception: |
924 | + self.fail("An unexpected error occurred during unpacked size calculation") |
925 | + |
926 | + # Test unpack |
927 | + def test_unpack__happy(self): |
928 | + try: |
929 | + container = SquashfsContainer(self.fn) |
930 | + container.unpack() |
931 | + except Exception: |
932 | + self.fail("An unexpected error occurred during unpack") |
933 | + |
934 | + def test_unpack__force(self): |
935 | + try: |
936 | + container = SquashfsContainer(self.fn) |
937 | + container.unpack() |
938 | + container.unpack(force=True) |
939 | + except Exception: |
940 | + self.fail("An unexpected error occurred during unpack") |
941 | + |
942 | + def test_unpack__no_force(self): |
943 | + with self.assertRaises(ContainerException) as context: |
944 | + container = SquashfsContainer(self.fn) |
945 | + container.unpack() |
946 | + container.unpack() |
947 | + self.assertTrue(" exists. Aborting." in context.exception.value) |
LGTM, but can you please update the commit description to also describe how and why you are removing the 3 entries from magic_binary_ file_descriptio ns as well? Thanks. A commit message more like
rt/c/base_ container: use magic compat API
...
would perhaps make more sense here.