Merge ~jslarraz/review-tools:add-doc-about-libmagic-warning into review-tools:master

Proposed by Jorge Sancho Larraz
Status: Merged
Merged at revision: f0fe85e3345eb4d0bae3657222b8d4974286a357
Proposed branch: ~jslarraz/review-tools:add-doc-about-libmagic-warning
Merge into: review-tools:master
Diff against target: 947 lines (+492/-169)
9 files modified
reviewtools/available.py (+7/-2)
reviewtools/common.py (+0/-142)
reviewtools/containers/base_container.py (+160/-13)
reviewtools/containers/snap_container.py (+44/-0)
reviewtools/containers/squashfs_container.py (+111/-0)
reviewtools/sr_common.py (+2/-2)
reviewtools/sr_tests.py (+1/-2)
reviewtools/tests/containers/test_base_container.py (+94/-8)
reviewtools/tests/containers/test_squashfs_container.py (+73/-0)
Reviewer Review Type Date Requested Status
Alex Murray Needs Fixing
Review via email: mp+466670@code.launchpad.net

Commit message

rt/c/base_container: make use of python-libmagic more eviden

To post a comment you must log in.
Revision history for this message
Alex Murray (alexmurray) wrote :

LGTM, but can you please update the commit description to also describe how and why you are removing the 3 entries from magic_binary_file_descriptions as well? Thanks. A commit message more like

rt/c/base_container: use magic compat API

...

would perhaps make more sense here.

review: Needs Fixing

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1diff --git a/reviewtools/available.py b/reviewtools/available.py
2index eafa366..5b0dd3c 100755
3--- a/reviewtools/available.py
4+++ b/reviewtools/available.py
5@@ -19,6 +19,7 @@ import shutil
6 import tempfile
7
8 from reviewtools.common import (
9+ error,
10 debug,
11 warn,
12 open_file_write,
13@@ -26,7 +27,6 @@ from reviewtools.common import (
14 MKDTEMP_PREFIX,
15 _add_error, # make a class
16 get_rock_manifest,
17- get_snap_manifest,
18 )
19 import reviewtools.email as email
20
21@@ -37,6 +37,7 @@ from reviewtools.store import (
22 get_faked_build_and_stage_packages,
23 )
24 from reviewtools.usn import read_usn_db
25+from reviewtools.containers.snap_container import SnapContainer, ContainerException
26
27 email_update_required_text = """A scan of this %s shows that it was built with packages from the Ubuntu
28 archive that have since received security updates. """
29@@ -484,7 +485,11 @@ def scan_store(
30 def scan_snap(secnot_db_fn, snap_fn, with_cves=False, ignore_pockets=list()):
31 """Scan snap for packages with security notices"""
32 out = ""
33- (man, dpkg) = get_snap_manifest(snap_fn)
34+ try:
35+ snap = SnapContainer(snap_fn)
36+ (man, dpkg) = snap.get_snap_manifest()
37+ except ContainerException as e:
38+ error(str(e))
39 man = get_faked_build_and_stage_packages(man)
40
41 # Use dpkg.list when the snap ships it in a spot we honor. This is limited
42diff --git a/reviewtools/common.py b/reviewtools/common.py
43index e2e80ff..d2c15b8 100644
44--- a/reviewtools/common.py
45+++ b/reviewtools/common.py
46@@ -424,30 +424,6 @@ def cmdIgnoreErrorStrings(command, ignoreErrorStrings):
47 return [rc, out]
48
49
50-def _unpack_cmd(cmd_args, d, dest):
51- """Low level unpack helper"""
52- curdir = os.getcwd()
53- os.chdir(d)
54-
55- (rc, out) = cmdIgnoreErrorStrings(cmd_args, UNSQUASHFS_IGNORED_ERRORS)
56-
57- os.chdir(curdir)
58-
59- if rc != 0:
60- if os.path.isdir(d):
61- recursive_rm(d)
62- error("unpacking failed with '%d':\n%s" % (rc, out))
63-
64- if dest is None:
65- dest = d
66- else:
67- # Recursively move unpacked content to original dir, keeping
68- # permissions
69- move_dir_content(d, dest)
70-
71- return dest
72-
73-
74 # The original directory might have restrictive permissions, so save
75 # them off, chmod the dir, do the move and reapply
76 def move_dir_content(d, dest):
77@@ -621,24 +597,6 @@ def unsquashfs_lln_parse(lln_out):
78 return hdr, entries
79
80
81-def _calculate_snap_unsquashfs_uncompressed_size(snap_pkg):
82- """Calculate size of the uncompressed snap"""
83- (rc, out) = cmd(["unsquashfs", "-lln", snap_pkg])
84- if rc != 0:
85- error("unsquashfs -lln '%s' failed: %s" % (snap_pkg, out))
86-
87- size = 0
88- for line in out.splitlines():
89- if not line.startswith("-"): # skip non-regular files
90- continue
91- try:
92- size += int(line.split()[2])
93- except ValueError: # skip non-numbers
94- continue
95-
96- return size
97-
98-
99 def _calculate_rock_untar_uncompressed_size(rock_pkg):
100 """Calculate size of the uncompressed tar"""
101 size = 0
102@@ -656,34 +614,6 @@ def unsquashfs_supports_ignore_errors():
103 return "-ig[nore-errors]" in out
104
105
106-def _unpack_snap_squashfs(snap_pkg, dest, items=[]):
107- """Unpack a squashfs based snap package to dest"""
108- size = _calculate_snap_unsquashfs_uncompressed_size(snap_pkg)
109-
110- snap_max_size = MAX_UNCOMPRESSED_SIZE * 1024 * 1024 * 1024
111- valid_size, error_msg = is_pkg_uncompressed_size_valid(
112- snap_max_size, size, snap_pkg
113- )
114- if valid_size:
115- global MKDTEMP_PREFIX
116- global MKDTEMP_DIR
117- d = tempfile.mkdtemp(prefix=MKDTEMP_PREFIX, dir=MKDTEMP_DIR)
118- cmd = ["unsquashfs", "-no-progress", "-f", "-d", d]
119-
120- # If unsquashfs supports it, pass "-ignore-errors -quiet"
121- if unsquashfs_supports_ignore_errors():
122- cmd.append("-ignore-errors")
123- cmd.append("-quiet")
124-
125- cmd.append(os.path.abspath(snap_pkg))
126-
127- if len(items) != 0:
128- cmd += items
129- return _unpack_cmd(cmd, d, dest)
130- else:
131- error(error_msg)
132-
133-
134 def is_pkg_uncompressed_size_valid(pkg_max_size, size, pkg):
135 st = os.statvfs(pkg)
136 avail = st.f_bsize * st.f_bavail * 0.9 # 90% of available space
137@@ -751,21 +681,6 @@ def _unpack_rock_tar(rock_pkg, dest):
138 error(error_msg)
139
140
141-def unpack_pkg(fn, dest=None, items=[]):
142- """Unpack package"""
143- pkg = check_fn(fn)
144- check_dir(dest)
145-
146- # Limit the maximimum size of the package
147- check_max_pkg_size(pkg)
148-
149- # check if its a squashfs based snap
150- if is_squashfs(pkg):
151- return _unpack_snap_squashfs(fn, dest, items)
152-
153- error("Unsupported package format (not squashfs)")
154-
155-
156 def check_dir(dest):
157 if dest is not None and os.path.exists(dest):
158 error("'%s' exists. Aborting." % dest)
159@@ -790,13 +705,6 @@ def check_max_pkg_size(pkg):
160 )
161
162
163-def is_squashfs(filename):
164- """Return true if the given filename as a squashfs header"""
165- with open(filename, "rb") as f:
166- header = f.read(10)
167- return header.startswith(b"hsqs")
168-
169-
170 def unpack_rock(fn, dest=None):
171 """Unpack rock"""
172 pkg = check_fn(fn)
173@@ -1078,56 +986,6 @@ def read_file_as_json_dict(fn):
174 return raw
175
176
177-def get_snap_manifest(fn):
178- if "SNAP_USER_COMMON" in os.environ and os.path.exists(
179- os.environ["SNAP_USER_COMMON"]
180- ):
181- MKDTEMP_DIR = os.environ["SNAP_USER_COMMON"]
182- else:
183- MKDTEMP_DIR = tempfile.gettempdir()
184-
185- man = "snap/manifest.yaml"
186- os_dpkg = "usr/share/snappy/dpkg.list"
187- snap_dpkg = "snap/dpkg.list"
188- # unpack_pkg() fails if this exists, so this is safe
189- dir = tempfile.mktemp(prefix=MKDTEMP_PREFIX, dir=MKDTEMP_DIR)
190- unpack_pkg(fn, dir, [man, os_dpkg, snap_dpkg])
191-
192- man_fn = os.path.join(dir, man)
193- if not os.path.isfile(man_fn):
194- recursive_rm(dir)
195- error("%s not in %s" % (man, fn))
196-
197- with open_file_read(man_fn) as fd:
198- try:
199- man_yaml = yaml.safe_load(fd)
200- except Exception:
201- recursive_rm(dir)
202- error("Could not load %s. Is it properly formatted?" % man)
203-
204- os_dpkg_fn = os.path.join(dir, os_dpkg)
205- snap_dpkg_fn = os.path.join(dir, snap_dpkg)
206- dpkg_list = None
207- if os.path.isfile(os_dpkg_fn):
208- with open_file_read(os_dpkg_fn) as fd:
209- try:
210- dpkg_list = fd.readlines()
211- except Exception:
212- recursive_rm(dir)
213- error("Could not load %s. Is it properly formatted?" % os_dpkg)
214- elif os.path.isfile(snap_dpkg_fn):
215- with open_file_read(snap_dpkg_fn) as fd:
216- try:
217- dpkg_list = fd.readlines()
218- except Exception:
219- recursive_rm(dir)
220- error("Could not load %s. Is it properly formatted?" % snap_dpkg)
221-
222- recursive_rm(dir)
223-
224- return (man_yaml, dpkg_list)
225-
226-
227 def get_rock_manifest(fn):
228 if "SNAP_USER_COMMON" in os.environ and os.path.exists(
229 os.environ["SNAP_USER_COMMON"]
230diff --git a/reviewtools/containers/base_container.py b/reviewtools/containers/base_container.py
231index 9e6f104..0d51021 100644
232--- a/reviewtools/containers/base_container.py
233+++ b/reviewtools/containers/base_container.py
234@@ -7,12 +7,19 @@ from tempfile import mkdtemp
235 from reviewtools.common import (
236 MKDTEMP_PREFIX,
237 MKDTEMP_DIR,
238- debug,
239- unpack_pkg,
240 recursive_rm,
241 )
242
243
244+# Add base_container private attributes to documentation
245+__pdoc__ = {
246+ "BaseContainer._min_packed_size": True,
247+ "BaseContainer._max_packed_size": True,
248+ "BaseContainer._min_unpacked_size": True,
249+ "BaseContainer._max_unpacked_size": True,
250+}
251+
252+
253 class ContainerException(Exception):
254 """This class represents exceptions"""
255
256@@ -32,16 +39,36 @@ class BaseContainer:
257 """
258
259 magic_binary_file_descriptions = [
260- "application/x-executable; charset=binary",
261- "application/x-sharedlib; charset=binary",
262- "application/x-object; charset=binary",
263- # 18.04 and higher doesn't show the charset
264 "application/x-executable",
265 "application/x-sharedlib",
266 "application/x-object",
267 "application/x-pie-executable",
268 ]
269
270+ _min_packed_size = None
271+ """
272+ Minimum size of the container file in bytes. It will be used to check the
273+ container size during initialization.
274+ """
275+
276+ _max_packed_size = None
277+ """
278+ Maximum size of the container file in bytes. It will be used to check the
279+ container size during initialization.
280+ """
281+
282+ _min_unpacked_size = None
283+ """
284+ Minimum size of the unpacked directory in bytes. It will be used to check the
285+ container size during initialization.
286+ """
287+
288+ _max_unpacked_size = None
289+ """
290+ Maximum size of the unpacked directory in bytes. It will be used to check the
291+ container size during initialization.
292+ """
293+
294 _unpack_dir = None
295 _files = None
296 _bin_files = None
297@@ -59,6 +86,9 @@ class BaseContainer:
298 else:
299 raise ContainerException("Could not find '%s'" % fn)
300
301+ # Check max/min size if defined
302+ self.check_container_size()
303+
304 @property
305 def unpack_dir(self) -> str:
306 """
307@@ -67,8 +97,7 @@ class BaseContainer:
308 will be returned.
309 """
310 if self._unpack_dir is None:
311- self._unpack_dir = ".".join(self.filename.split(".")[:-1])
312- unpack_pkg(self.filename, dest=self._unpack_dir)
313+ self._unpack_dir = self.unpack()
314 return self._unpack_dir
315
316 @property
317@@ -104,6 +133,117 @@ class BaseContainer:
318 if os.path.exists(self.tmp_dir):
319 recursive_rm(self.tmp_dir)
320
321+ def check_container_format(self):
322+ """
323+ Checks that the container file has the expected format.
324+
325+ Raises:
326+ NotImplementedError: Calling this method will raise an exception as
327+ format is specific to the container type. This method needs to be
328+ implemented by the specific subclasses.
329+ """
330+ raise NotImplementedError("Must be implemented by subclasses")
331+
332+ def get_container_size(self) -> int:
333+ """
334+ Uses os.path.getsize() to return the size of the container file.
335+
336+ Retruns:
337+ int: Size in bytes of the container file
338+ """
339+ return os.path.getsize(self.filename)
340+
341+ def check_container_size(self):
342+ """
343+ Checks that the container size is within the range specified by
344+ min_packed_size and max_packed_size. If any of those values is not specified
345+ (i.e. it is None), the respective check will be skipped.
346+
347+ Raises:
348+ ContainerException: If container size is less than minSize or greater
349+ than maxSize, a ContainerException will be raised.
350+ """
351+ size = self.get_container_size()
352+ if self._min_packed_size and (size < self._min_packed_size):
353+ raise ContainerException(
354+ "container file is too small (%dM < %dM)"
355+ % (size / 1024 / 1024, self._min_packed_size / 1024 / 1024)
356+ )
357+ if self._max_packed_size and (size > self._max_packed_size):
358+ raise ContainerException(
359+ "container file is too large (%dM > %dM)"
360+ % (size / 1024 / 1024, self._max_packed_size / 1024 / 1024)
361+ )
362+
363+ def calculate_unpacked_size(self) -> int:
364+ """
365+ Estimates the space that the unpacked directory will use without unpacking.
366+
367+ Returns:
368+ int: Estimated size in bytes of the unpacked directory
369+
370+ Raises:
371+ NotImplementedError: Calling this method will raise an exception as the
372+ estimation method is specific to the container type. This method
373+ needs to be implemented by the specific subclasses.
374+ """
375+ raise NotImplementedError("Must be implemented by subclasses")
376+
377+ def check_unpacked_size(self):
378+ """
379+ Checks that the container size is within the range specified by
380+ min_unpacked_size and max_unpacked_size. If any of those values is not
381+ specified (i.e. it is None), the respective check will be skipped.
382+
383+ Raises:
384+ ContainerException: If unpacked size is less than minSize or greater
385+ than maxSize, a ContainerException will be raised.
386+ """
387+ size = self.calculate_unpacked_size()
388+ if self._min_unpacked_size and (size < self._min_unpacked_size):
389+ raise ContainerException(
390+ "unpacked directory is too small (%dM < %dM)"
391+ % (size / 1024 / 1024, self._min_unpacked_size / 1024 / 1024)
392+ )
393+ if self._max_unpacked_size and (size > self._max_unpacked_size):
394+ raise ContainerException(
395+ "unpacked directory is too large (%dM > %dM)"
396+ % (size / 1024 / 1024, self._max_unpacked_size / 1024 / 1024)
397+ )
398+
399+ def check_disk_space(self):
400+ """
401+ Checks that there is sufficient disk space available for the container to be
402+ unpacked, i.e. that it will not consume more than 90% of the available disk
403+
404+ Raises:
405+ ContainerException: If estimated unpacked size exceed the 90% of the
406+ available space, an exception will be raised.
407+ """
408+ size = self.calculate_unpacked_size()
409+ st = os.statvfs(self.tmp_dir)
410+ avail = st.f_bsize * st.f_bavail * 0.9 # 90% of available space
411+ if size > avail * 0.9:
412+ raise ContainerException(
413+ "insufficient available disk space (%dM > %dM)"
414+ % (size / 1024 / 1024, avail / 1024 / 1024)
415+ )
416+
417+ def unpack(self) -> str:
418+ """
419+ Extracts the content of the container file and returns the absolute path to
420+ the newly unpacked directory.
421+
422+ Returns:
423+ str: absolute path to the newly created unpack dir
424+
425+ Raises:
426+ NotImplementedError: Calling this method will raise an exception as the
427+ estimation method is specific to the container type. This method
428+ needs to be implemented by the specific subclasses.
429+ """
430+ raise NotImplementedError("Must be implemented by subclasses")
431+
432 def get_file(self, fn: str) -> Union[str, None]:
433 """
434 Reads and return the raw content of the specified file.
435@@ -117,7 +257,7 @@ class BaseContainer:
436
437 Raises:
438 ContainerException: Calling this method will raise an exception if
439- the requested file is not found.
440+ the requested file is not found.
441 """
442 if not os.path.exists(os.path.join(self.unpack_dir, fn)):
443 raise ContainerException("Could not find '%s'" % fn)
444@@ -156,15 +296,22 @@ class BaseContainer:
445 Returns:
446 list(str): list of files present in the container
447 """
448- mime = magic.open(magic.MAGIC_MIME)
449+ mime = magic.compat.open(magic.MAGIC_MIME)
450 mime.load()
451 bin_files = []
452 for file in self.get_files_list(abs=True):
453 try:
454+ # There are two different magic libraries for python, python-magic
455+ # and python-libmagic. We are installing python-magic (which is the
456+ # only one distributed via apt). However, python-magic seems to hang
457+ # when it hits a pipe file.
458+ # python-magic also bundles the python-libmagic module and exposes
459+ # it through the following compatibility API. Using this bundled
460+ # python-libmagic module will generate a deprecation warning. See
461+ # https://github.com/ahupp/python-magic/blob/master/magic/__init__.py#L458
462 res = mime.file(file)
463- except Exception: # pragma: nocover
464- # workaround for zesty python3-magic
465- debug("could not detemine mime type of '%s'" % file)
466+ except OSError:
467+ # TODO: file does not exists. Only hit for external symlinks
468 continue
469
470 if not abs:
471diff --git a/reviewtools/containers/snap_container.py b/reviewtools/containers/snap_container.py
472new file mode 100644
473index 0000000..ff1ee57
474--- /dev/null
475+++ b/reviewtools/containers/snap_container.py
476@@ -0,0 +1,44 @@
477+import yaml
478+from reviewtools.containers.squashfs_container import (
479+ SquashfsContainer,
480+ ContainerException,
481+)
482+
483+
484+class SnapContainer(SquashfsContainer):
485+ # TODO: review-tools was not checking for it. We still have some snaps in
486+ # "tests/" that doesn't fit this limit and will need to be regenerated before
487+ # this check can be enforced.
488+ # _min_packed_size = 16 * 1024
489+ _max_packed_size = 5 * 1024 * 1024 * 1024
490+ _max_unpacked_size = 25 * 1024 * 1024 * 1024
491+
492+ def get_snap_manifest(self):
493+ man = "snap/manifest.yaml"
494+ os_dpkg = "usr/share/snappy/dpkg.list"
495+ snap_dpkg = "snap/dpkg.list"
496+
497+ if self._unpack_dir is None:
498+ self.unpack(items=[man, os_dpkg, snap_dpkg])
499+
500+ man_content = self.get_file(man).decode()
501+ try:
502+ man_yaml = yaml.safe_load(man_content)
503+ except Exception:
504+ raise ContainerException(
505+ "Could not load %s. Is it properly formatted?" % man
506+ )
507+
508+ dpkg_list = None
509+ try:
510+ dpkg_list = self.get_file(os_dpkg).decode().split("\n")
511+ return (man_yaml, dpkg_list)
512+ except ContainerException:
513+ pass
514+
515+ try:
516+ dpkg_list = self.get_file(snap_dpkg).decode().split("\n")
517+ except ContainerException:
518+ pass
519+
520+ return (man_yaml, dpkg_list)
521diff --git a/reviewtools/containers/squashfs_container.py b/reviewtools/containers/squashfs_container.py
522new file mode 100644
523index 0000000..631fe76
524--- /dev/null
525+++ b/reviewtools/containers/squashfs_container.py
526@@ -0,0 +1,111 @@
527+import os
528+import shutil
529+from reviewtools.common import cmd, cmdIgnoreErrorStrings, UNSQUASHFS_IGNORED_ERRORS
530+from reviewtools.containers.base_container import BaseContainer, ContainerException
531+
532+
533+class SquashfsContainer(BaseContainer):
534+ """
535+ A container is a file that encapsulates a file system. This class extends
536+ BaseContainer to handle functionality that is specific to Squashfs containers.
537+ """
538+
539+ def __init__(self, fn):
540+ """
541+ If SquashfsContainer is initialized with a container file, it will also
542+ verify that the provided file is actually a squashfs, in addition to the
543+ checks performed by the BaseContainer.
544+ """
545+ super().__init__(fn)
546+
547+ # Check is squash file
548+ self.check_container_format()
549+
550+ def check_container_format(self):
551+ """
552+ Checks that the container file has the expected format.
553+
554+ Raises:
555+ ContainerException: This method will raise an exception if the
556+ container file is not a squashfs container.
557+ """
558+ with open(self.filename, "rb") as f:
559+ header = f.read(10)
560+ if not header.startswith(b"hsqs"):
561+ raise ContainerException("Unsupported package format (not squashfs)")
562+
563+ def calculate_unpacked_size(self) -> int:
564+ """
565+ Estimates the space that the unpacked directory will use by adding the
566+ individual files size reported by "unsquashfs -lln" command. It may not
567+ be completely accurate, as this command does not consider that each folder
568+ requires at least one complete block
569+
570+ Returns:
571+ int: Estimated size in bytes of the unpacked directory
572+ """
573+ (rc, out) = cmd(["unsquashfs", "-lln", self.filename])
574+ if rc != 0:
575+ raise RuntimeError("unsquashfs -lln '%s' failed: %s" % (self.filename, out))
576+
577+ size = 0
578+ for line in out.splitlines():
579+ if not line.startswith("-"): # skip non-regular files
580+ continue
581+ try:
582+ size += int(line.split()[2])
583+ except ValueError: # skip non-numbers
584+ continue
585+
586+ return size
587+
588+ def unpack(self, items: list = [], force: bool = False) -> str:
589+ """
590+ Extracts the content of the container file and returns the absolute path to
591+ the newly unpacked directory.
592+
593+ Args:
594+ force (bool, optional): If ``True``, the unpack_dir will be overwritten
595+ if it already exists. Defaults to ``False``
596+
597+ Returns:
598+ str: absolute path to the newly created unpack dir
599+
600+ Raises:
601+ ContainerException: This method will raise an ContainerException if the
602+ unpack_dir already exists and this method was called without the
603+ force argument set to True
604+ RuntimeError: This method will raise a RuntimeError under the execution
605+ of the "unsquashfs" command returns with an error code other than 0.
606+ """
607+ if self._unpack_dir is not None and not force:
608+ raise ContainerException("'%s' exists. Aborting." % self._unpack_dir)
609+
610+ # Check disk space and space available in disk
611+ self.check_unpacked_size()
612+ self.check_disk_space()
613+
614+ # Unpack the container
615+ stage_dir = os.path.join(self.tmp_dir, "stage")
616+ cmd = [
617+ "unsquashfs",
618+ "-no-progress",
619+ "-ignore-errors",
620+ "-quiet",
621+ "-f",
622+ "-d",
623+ stage_dir,
624+ self.filename,
625+ ] + items
626+ (rc, out) = cmdIgnoreErrorStrings(cmd, UNSQUASHFS_IGNORED_ERRORS)
627+ if rc != 0:
628+ if os.path.isdir(stage_dir):
629+ shutil.rmtree(stage_dir)
630+ raise RuntimeError("unpacking failed with '%d':\n%s" % (rc, out))
631+
632+ # Update on success
633+ if self._unpack_dir is not None and os.path.exists(self._unpack_dir):
634+ shutil.rmtree(self._unpack_dir)
635+ self._unpack_dir = ".".join(self.filename.split(".")[:-1])
636+ shutil.move(stage_dir, self._unpack_dir)
637+ return self._unpack_dir
638diff --git a/reviewtools/sr_common.py b/reviewtools/sr_common.py
639index 16790cc..9b961e2 100644
640--- a/reviewtools/sr_common.py
641+++ b/reviewtools/sr_common.py
642@@ -30,7 +30,7 @@ from reviewtools.common import (
643 unsquashfs_lln_parse,
644 verify_type,
645 )
646-from reviewtools.containers.base_container import BaseContainer, ContainerException
647+from reviewtools.containers.snap_container import SnapContainer, ContainerException
648 from reviewtools.overrides import interfaces_attribs_addons
649
650
651@@ -398,7 +398,7 @@ class SnapReview(ReviewBase):
652 ReviewBase.__init__(self, review_type, overrides=overrides)
653
654 try:
655- self.snap = BaseContainer(fn)
656+ self.snap = SnapContainer(fn)
657 except ContainerException as e:
658 error(str(e))
659 # TODO: update code to directly use self.snap.xxx instead of assigning here
660diff --git a/reviewtools/sr_tests.py b/reviewtools/sr_tests.py
661index 6caae54..bfbf375 100644
662--- a/reviewtools/sr_tests.py
663+++ b/reviewtools/sr_tests.py
664@@ -87,7 +87,7 @@ def create_patches():
665 # child.
666 patches = []
667 patches.append(
668- patch("reviewtools.sr_common.BaseContainer.__init__", _BaseContainer.__init__)
669+ patch("reviewtools.sr_common.SnapContainer.__init__", _BaseContainer.__init__)
670 )
671 patches.append(
672 patch("reviewtools.sr_common.SnapReview._extract_snap_yaml", _extract_snap_yaml)
673@@ -98,7 +98,6 @@ def create_patches():
674 _extract_snap_manifest_yaml,
675 )
676 )
677- patches.append(patch("reviewtools.common.unpack_pkg", _mock_func))
678 # sr_common
679 patches.append(
680 patch("reviewtools.sr_common.SnapReview._get_unpack_dir", __get_unpack_dir)
681diff --git a/reviewtools/tests/containers/test_base_container.py b/reviewtools/tests/containers/test_base_container.py
682index 740e942..1b4e022 100644
683--- a/reviewtools/tests/containers/test_base_container.py
684+++ b/reviewtools/tests/containers/test_base_container.py
685@@ -3,29 +3,43 @@ import shutil
686 import unittest
687 from tempfile import mkdtemp
688 from reviewtools.containers.base_container import BaseContainer, ContainerException
689-from reviewtools.tests.utils import make_snap2
690
691
692 class TestBaseContainer(unittest.TestCase):
693 """Tests for reviewtools.containers.base_container.BaseContainer"""
694
695- expected_files = ["meta/snap.yaml", "meta/icon.png", "nonexecstack.bin"]
696- expected_binaries = ["nonexecstack.bin"]
697+ expected_files = ["meta/snap.yaml", "meta/icon.png", "bin/ls"]
698+ expected_binaries = ["bin/ls"]
699
700 @classmethod
701 def setUpClass(cls):
702 cls.tmp_dir = mkdtemp()
703- cls.fn = make_snap2(
704- output_dir=cls.tmp_dir,
705- yaml="Test",
706- extra_files=["/bin/ls:nonexecstack.bin"],
707- )
708+ cls.fn = cls._create_container(cls)
709+ cls.unpack_dir = cls._create_unpackdir(cls)
710
711 @classmethod
712 def tearDownClass(cls):
713 if os.path.exists(cls.tmp_dir):
714 shutil.rmtree(cls.tmp_dir)
715
716+ def _create_container(self, container_size: int = 10000):
717+ fn = os.path.join(self.tmp_dir, "container.test")
718+ with open(fn, "w") as f:
719+ f.truncate(container_size)
720+ return fn
721+
722+ def _create_unpackdir(self):
723+ unpack_dir = os.path.join(self.tmp_dir, "unpack_dir")
724+ for file in self.expected_files:
725+ unpacked_file = os.path.join(unpack_dir, file)
726+ os.makedirs(os.path.dirname(unpacked_file), exist_ok=True)
727+ if file not in self.expected_binaries:
728+ with open(unpacked_file, "w+b") as fd:
729+ fd.write(b"Test")
730+ else:
731+ shutil.copyfile(os.path.join("/", file), unpacked_file)
732+ return unpack_dir
733+
734 # Test class initialization
735 def test_initialization_container__happy(self):
736 try:
737@@ -43,10 +57,75 @@ class TestBaseContainer(unittest.TestCase):
738 in context.exception.value
739 )
740
741+ # Test deletion
742+ def test_check_cleanup(self):
743+ try:
744+ container = BaseContainer(self.fn)
745+ tmp_dir = container.tmp_dir
746+ del container
747+ self.assertFalse(os.path.exists(tmp_dir))
748+ except Exception:
749+ self.fail("An unexpected error occurred during deletion test")
750+
751+ # Test check container size
752+ def test_check_container_size__happy(self):
753+ container = BaseContainer(self.fn)
754+ container._min_packed_size = 0
755+ container._max_packed_size = 10**6
756+ try:
757+ container.check_container_size()
758+ except ContainerException:
759+ self.fail("An unexpected error occurred while checking file size")
760+
761+ def test_check_container_size__too_big(self):
762+ container = BaseContainer(self.fn)
763+ container._max_packed_size = 1
764+ with self.assertRaises(ContainerException) as context:
765+ container.check_container_size()
766+ self.assertTrue("container file is too large " in context.exception.value)
767+
768+ def test_check_container_size__too_small(self):
769+ container = BaseContainer(self.fn)
770+ container._min_packed_size = 10**6
771+ with self.assertRaises(ContainerException) as context:
772+ container.check_container_size()
773+ self.assertTrue("container file is too small " in context.exception.value)
774+
775+ # Test check unpacked size
776+ def _mock_calculate_unpacked_size(self):
777+ return 10**3
778+
779+ def test_check_unpacked_size__happy(self):
780+ container = BaseContainer(self.fn)
781+ container.calculate_unpacked_size = self._mock_calculate_unpacked_size
782+ container._min_unpacked_size = 0
783+ container._max_unpacked_size = 10**6
784+ try:
785+ container.check_unpacked_size()
786+ except ContainerException:
787+ self.fail("An unexpected error occurred while checking file size")
788+
789+ def test_check_unpacked_size__too_big(self):
790+ container = BaseContainer(self.fn)
791+ container.calculate_unpacked_size = self._mock_calculate_unpacked_size
792+ container._max_unpacked_size = 1
793+ with self.assertRaises(ContainerException) as context:
794+ container.check_unpacked_size()
795+ self.assertTrue("container file is too large " in context.exception.value)
796+
797+ def test_check_unpacked_size__too_small(self):
798+ container = BaseContainer(self.fn)
799+ container.calculate_unpacked_size = self._mock_calculate_unpacked_size
800+ container._min_unpacked_size = 10**6
801+ with self.assertRaises(ContainerException) as context:
802+ container.check_unpacked_size()
803+ self.assertTrue("container file is too small " in context.exception.value)
804+
805 # Test get file
806 def test_get_file__happy(self):
807 try:
808 container = BaseContainer(self.fn)
809+ container._unpack_dir = self.unpack_dir
810 snap_yaml = container.get_file("meta/snap.yaml")
811 self.assertEqual(snap_yaml.decode(), "Test")
812 except ContainerException:
813@@ -55,6 +134,7 @@ class TestBaseContainer(unittest.TestCase):
814 def test_get_file__missing_file(self):
815 with self.assertRaises(ContainerException) as context:
816 container = BaseContainer(self.fn)
817+ container._unpack_dir = self.unpack_dir
818 container.get_file("non/existent/file")
819 self.assertTrue("Could not find" in context.exception.value)
820
821@@ -62,6 +142,7 @@ class TestBaseContainer(unittest.TestCase):
822 def test_get_files_list__happy(self):
823 try:
824 container = BaseContainer(self.fn)
825+ container._unpack_dir = self.unpack_dir
826 self.assertCountEqual(
827 container.get_files_list(),
828 self.expected_files,
829@@ -72,6 +153,7 @@ class TestBaseContainer(unittest.TestCase):
830 def test_get_files_list_abs__happy(self):
831 try:
832 container = BaseContainer(self.fn)
833+ container._unpack_dir = self.unpack_dir
834 self.assertCountEqual(
835 container.get_files_list(abs=True),
836 [
837@@ -85,6 +167,7 @@ class TestBaseContainer(unittest.TestCase):
838 def test_files__happy(self):
839 try:
840 container = BaseContainer(self.fn)
841+ container._unpack_dir = self.unpack_dir
842 self.assertCountEqual(
843 container.files,
844 self.expected_files,
845@@ -96,6 +179,7 @@ class TestBaseContainer(unittest.TestCase):
846 def test_get_compiled_binaries_list__happy(self):
847 try:
848 container = BaseContainer(self.fn)
849+ container._unpack_dir = self.unpack_dir
850 self.assertCountEqual(
851 container.get_compiled_binaries_list(), self.expected_binaries
852 )
853@@ -107,6 +191,7 @@ class TestBaseContainer(unittest.TestCase):
854 def test_get_compiled_binaries_list_abs__happy(self):
855 try:
856 container = BaseContainer(self.fn)
857+ container._unpack_dir = self.unpack_dir
858 self.assertCountEqual(
859 container.get_compiled_binaries_list(abs=True),
860 [
861@@ -122,6 +207,7 @@ class TestBaseContainer(unittest.TestCase):
862 def test_bin_files__happy(self):
863 try:
864 container = BaseContainer(self.fn)
865+ container._unpack_dir = self.unpack_dir
866 self.assertCountEqual(container.bin_files, self.expected_binaries)
867 except ContainerException:
868 self.fail(
869diff --git a/reviewtools/tests/containers/test_squashfs_container.py b/reviewtools/tests/containers/test_squashfs_container.py
870new file mode 100644
871index 0000000..7d19a77
872--- /dev/null
873+++ b/reviewtools/tests/containers/test_squashfs_container.py
874@@ -0,0 +1,73 @@
875+import os
876+import unittest
877+from tempfile import mkstemp, mkdtemp
878+from reviewtools.containers.squashfs_container import (
879+ SquashfsContainer,
880+ ContainerException,
881+)
882+from reviewtools.tests.utils import make_snap2
883+
884+
885+class TestSquashfsContainer(unittest.TestCase):
886+ """Tests for reviewtools.containers.squashfs_container"""
887+
888+ @classmethod
889+ def setUpClass(cls):
890+ cls.tmp_dir = mkdtemp()
891+ cls.fn = make_snap2(
892+ output_dir=cls.tmp_dir,
893+ yaml="Test",
894+ extra_files=["/bin/ls:bin/ls"],
895+ )
896+
897+ # Test initialization
898+ def test_check_initialization__happy(self):
899+ try:
900+ SquashfsContainer(self.fn)
901+ except Exception:
902+ self.fail(
903+ "An unexpected error occurred during SquashfsContainer initialization with container file"
904+ )
905+
906+ def test_check_initialization__invalid_format(self):
907+ with self.assertRaises(ContainerException) as context:
908+ fd, plain_file = mkstemp()
909+ SquashfsContainer(plain_file)
910+ self.assertEqual(
911+ "unsupported package format (not squashfs)", context.exception.value
912+ )
913+ os.unlink(plain_file)
914+
915+ # Test check format
916+ def test_calculate_unpacked_size__happy(self):
917+ try:
918+ # unpacked size calculated is a bit smaller than actual size as it does not consider that
919+ # folders require at least one complete block
920+ container = SquashfsContainer(self.fn)
921+ size = container.calculate_unpacked_size()
922+ self.assertTrue(isinstance(size, int))
923+ except Exception:
924+ self.fail("An unexpected error occurred during unpacked size calculation")
925+
926+ # Test unpack
927+ def test_unpack__happy(self):
928+ try:
929+ container = SquashfsContainer(self.fn)
930+ container.unpack()
931+ except Exception:
932+ self.fail("An unexpected error occurred during unpack")
933+
934+ def test_unpack__force(self):
935+ try:
936+ container = SquashfsContainer(self.fn)
937+ container.unpack()
938+ container.unpack(force=True)
939+ except Exception:
940+ self.fail("An unexpected error occurred during unpack")
941+
942+ def test_unpack__no_force(self):
943+ with self.assertRaises(ContainerException) as context:
944+ container = SquashfsContainer(self.fn)
945+ container.unpack()
946+ container.unpack()
947+ self.assertTrue(" exists. Aborting." in context.exception.value)

Subscribers

People subscribed via source and target branches