Merge ~jslarraz/review-tools:container-size-checks into review-tools:master
- Git
- lp:~jslarraz/review-tools
- container-size-checks
- Merge into master
Status: | Merged |
---|---|
Merged at revision: | 97ce21fc1803c0e84e77f866486944fb8e745ddf |
Proposed branch: | ~jslarraz/review-tools:container-size-checks |
Merge into: | review-tools:master |
Diff against target: |
909 lines (+481/-160) 9 files modified
reviewtools/available.py (+7/-2) reviewtools/common.py (+0/-142) reviewtools/containers/base_container.py (+149/-4) reviewtools/containers/snap_container.py (+44/-0) reviewtools/containers/squashfs_container.py (+111/-0) reviewtools/sr_common.py (+2/-2) reviewtools/sr_tests.py (+1/-2) reviewtools/tests/containers/test_base_container.py (+94/-8) reviewtools/tests/containers/test_squashfs_container.py (+73/-0) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Alex Murray | Approve | ||
Review via email:
|
Commit message
Add size check functions to base_container
Description of the change
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Jorge Sancho Larraz (jslarraz) wrote (last edit ): | # |
So in a general case, we may want to set up the valid size limits for the container file and for the unpacked content. It is already true for snaps.
For snaps we are checking the size via a set of functions in common.py (see inline comments), however this functionality is intrinsic to the container and only makes sense in the context of it. Thus, I think it should be part of the base_container class.
The specific values for {min,max}
There is some interaction among check_container
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Alex Murray (alexmurray) wrote : | # |
Thanks for the pointers but perhaps I am missing something, but I don't see any users of these new methods - as such, perhaps it would help to also include the changes required to port the existing size checking / limits code to use this new interface?
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Jorge Sancho Larraz (jslarraz) wrote : | # |
Updated to move unpack to SquashfsContainer, which uses size checks
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Alex Murray (alexmurray) wrote : | # |
In general this looks really good - just a few minor things to address if you can thanks @jslarraz
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Jorge Sancho Larraz (jslarraz) wrote : | # |
Comments addressed in https:/
Fix for container size not being checked in SnapContainer in https:/
> "I assume all containers would want to check their format - as such, should this be done in the BaseContainer constructor instead?"
I fully agree with you but it will require some non-obvious changes in test_base_
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Alex Murray (alexmurray) wrote : | # |
> I fully agree with you but it will require some non-obvious changes in test_base_
Sounds good - thanks @jslarraz - LGTM!
Preview Diff
1 | diff --git a/reviewtools/available.py b/reviewtools/available.py |
2 | index eafa366..5b0dd3c 100755 |
3 | --- a/reviewtools/available.py |
4 | +++ b/reviewtools/available.py |
5 | @@ -19,6 +19,7 @@ import shutil |
6 | import tempfile |
7 | |
8 | from reviewtools.common import ( |
9 | + error, |
10 | debug, |
11 | warn, |
12 | open_file_write, |
13 | @@ -26,7 +27,6 @@ from reviewtools.common import ( |
14 | MKDTEMP_PREFIX, |
15 | _add_error, # make a class |
16 | get_rock_manifest, |
17 | - get_snap_manifest, |
18 | ) |
19 | import reviewtools.email as email |
20 | |
21 | @@ -37,6 +37,7 @@ from reviewtools.store import ( |
22 | get_faked_build_and_stage_packages, |
23 | ) |
24 | from reviewtools.usn import read_usn_db |
25 | +from reviewtools.containers.snap_container import SnapContainer, ContainerException |
26 | |
27 | email_update_required_text = """A scan of this %s shows that it was built with packages from the Ubuntu |
28 | archive that have since received security updates. """ |
29 | @@ -484,7 +485,11 @@ def scan_store( |
30 | def scan_snap(secnot_db_fn, snap_fn, with_cves=False, ignore_pockets=list()): |
31 | """Scan snap for packages with security notices""" |
32 | out = "" |
33 | - (man, dpkg) = get_snap_manifest(snap_fn) |
34 | + try: |
35 | + snap = SnapContainer(snap_fn) |
36 | + (man, dpkg) = snap.get_snap_manifest() |
37 | + except ContainerException as e: |
38 | + error(str(e)) |
39 | man = get_faked_build_and_stage_packages(man) |
40 | |
41 | # Use dpkg.list when the snap ships it in a spot we honor. This is limited |
42 | diff --git a/reviewtools/common.py b/reviewtools/common.py |
43 | index e2e80ff..d2c15b8 100644 |
44 | --- a/reviewtools/common.py |
45 | +++ b/reviewtools/common.py |
46 | @@ -424,30 +424,6 @@ def cmdIgnoreErrorStrings(command, ignoreErrorStrings): |
47 | return [rc, out] |
48 | |
49 | |
50 | -def _unpack_cmd(cmd_args, d, dest): |
51 | - """Low level unpack helper""" |
52 | - curdir = os.getcwd() |
53 | - os.chdir(d) |
54 | - |
55 | - (rc, out) = cmdIgnoreErrorStrings(cmd_args, UNSQUASHFS_IGNORED_ERRORS) |
56 | - |
57 | - os.chdir(curdir) |
58 | - |
59 | - if rc != 0: |
60 | - if os.path.isdir(d): |
61 | - recursive_rm(d) |
62 | - error("unpacking failed with '%d':\n%s" % (rc, out)) |
63 | - |
64 | - if dest is None: |
65 | - dest = d |
66 | - else: |
67 | - # Recursively move unpacked content to original dir, keeping |
68 | - # permissions |
69 | - move_dir_content(d, dest) |
70 | - |
71 | - return dest |
72 | - |
73 | - |
74 | # The original directory might have restrictive permissions, so save |
75 | # them off, chmod the dir, do the move and reapply |
76 | def move_dir_content(d, dest): |
77 | @@ -621,24 +597,6 @@ def unsquashfs_lln_parse(lln_out): |
78 | return hdr, entries |
79 | |
80 | |
81 | -def _calculate_snap_unsquashfs_uncompressed_size(snap_pkg): |
82 | - """Calculate size of the uncompressed snap""" |
83 | - (rc, out) = cmd(["unsquashfs", "-lln", snap_pkg]) |
84 | - if rc != 0: |
85 | - error("unsquashfs -lln '%s' failed: %s" % (snap_pkg, out)) |
86 | - |
87 | - size = 0 |
88 | - for line in out.splitlines(): |
89 | - if not line.startswith("-"): # skip non-regular files |
90 | - continue |
91 | - try: |
92 | - size += int(line.split()[2]) |
93 | - except ValueError: # skip non-numbers |
94 | - continue |
95 | - |
96 | - return size |
97 | - |
98 | - |
99 | def _calculate_rock_untar_uncompressed_size(rock_pkg): |
100 | """Calculate size of the uncompressed tar""" |
101 | size = 0 |
102 | @@ -656,34 +614,6 @@ def unsquashfs_supports_ignore_errors(): |
103 | return "-ig[nore-errors]" in out |
104 | |
105 | |
106 | -def _unpack_snap_squashfs(snap_pkg, dest, items=[]): |
107 | - """Unpack a squashfs based snap package to dest""" |
108 | - size = _calculate_snap_unsquashfs_uncompressed_size(snap_pkg) |
109 | - |
110 | - snap_max_size = MAX_UNCOMPRESSED_SIZE * 1024 * 1024 * 1024 |
111 | - valid_size, error_msg = is_pkg_uncompressed_size_valid( |
112 | - snap_max_size, size, snap_pkg |
113 | - ) |
114 | - if valid_size: |
115 | - global MKDTEMP_PREFIX |
116 | - global MKDTEMP_DIR |
117 | - d = tempfile.mkdtemp(prefix=MKDTEMP_PREFIX, dir=MKDTEMP_DIR) |
118 | - cmd = ["unsquashfs", "-no-progress", "-f", "-d", d] |
119 | - |
120 | - # If unsquashfs supports it, pass "-ignore-errors -quiet" |
121 | - if unsquashfs_supports_ignore_errors(): |
122 | - cmd.append("-ignore-errors") |
123 | - cmd.append("-quiet") |
124 | - |
125 | - cmd.append(os.path.abspath(snap_pkg)) |
126 | - |
127 | - if len(items) != 0: |
128 | - cmd += items |
129 | - return _unpack_cmd(cmd, d, dest) |
130 | - else: |
131 | - error(error_msg) |
132 | - |
133 | - |
134 | def is_pkg_uncompressed_size_valid(pkg_max_size, size, pkg): |
135 | st = os.statvfs(pkg) |
136 | avail = st.f_bsize * st.f_bavail * 0.9 # 90% of available space |
137 | @@ -751,21 +681,6 @@ def _unpack_rock_tar(rock_pkg, dest): |
138 | error(error_msg) |
139 | |
140 | |
141 | -def unpack_pkg(fn, dest=None, items=[]): |
142 | - """Unpack package""" |
143 | - pkg = check_fn(fn) |
144 | - check_dir(dest) |
145 | - |
146 | - # Limit the maximimum size of the package |
147 | - check_max_pkg_size(pkg) |
148 | - |
149 | - # check if its a squashfs based snap |
150 | - if is_squashfs(pkg): |
151 | - return _unpack_snap_squashfs(fn, dest, items) |
152 | - |
153 | - error("Unsupported package format (not squashfs)") |
154 | - |
155 | - |
156 | def check_dir(dest): |
157 | if dest is not None and os.path.exists(dest): |
158 | error("'%s' exists. Aborting." % dest) |
159 | @@ -790,13 +705,6 @@ def check_max_pkg_size(pkg): |
160 | ) |
161 | |
162 | |
163 | -def is_squashfs(filename): |
164 | - """Return true if the given filename as a squashfs header""" |
165 | - with open(filename, "rb") as f: |
166 | - header = f.read(10) |
167 | - return header.startswith(b"hsqs") |
168 | - |
169 | - |
170 | def unpack_rock(fn, dest=None): |
171 | """Unpack rock""" |
172 | pkg = check_fn(fn) |
173 | @@ -1078,56 +986,6 @@ def read_file_as_json_dict(fn): |
174 | return raw |
175 | |
176 | |
177 | -def get_snap_manifest(fn): |
178 | - if "SNAP_USER_COMMON" in os.environ and os.path.exists( |
179 | - os.environ["SNAP_USER_COMMON"] |
180 | - ): |
181 | - MKDTEMP_DIR = os.environ["SNAP_USER_COMMON"] |
182 | - else: |
183 | - MKDTEMP_DIR = tempfile.gettempdir() |
184 | - |
185 | - man = "snap/manifest.yaml" |
186 | - os_dpkg = "usr/share/snappy/dpkg.list" |
187 | - snap_dpkg = "snap/dpkg.list" |
188 | - # unpack_pkg() fails if this exists, so this is safe |
189 | - dir = tempfile.mktemp(prefix=MKDTEMP_PREFIX, dir=MKDTEMP_DIR) |
190 | - unpack_pkg(fn, dir, [man, os_dpkg, snap_dpkg]) |
191 | - |
192 | - man_fn = os.path.join(dir, man) |
193 | - if not os.path.isfile(man_fn): |
194 | - recursive_rm(dir) |
195 | - error("%s not in %s" % (man, fn)) |
196 | - |
197 | - with open_file_read(man_fn) as fd: |
198 | - try: |
199 | - man_yaml = yaml.safe_load(fd) |
200 | - except Exception: |
201 | - recursive_rm(dir) |
202 | - error("Could not load %s. Is it properly formatted?" % man) |
203 | - |
204 | - os_dpkg_fn = os.path.join(dir, os_dpkg) |
205 | - snap_dpkg_fn = os.path.join(dir, snap_dpkg) |
206 | - dpkg_list = None |
207 | - if os.path.isfile(os_dpkg_fn): |
208 | - with open_file_read(os_dpkg_fn) as fd: |
209 | - try: |
210 | - dpkg_list = fd.readlines() |
211 | - except Exception: |
212 | - recursive_rm(dir) |
213 | - error("Could not load %s. Is it properly formatted?" % os_dpkg) |
214 | - elif os.path.isfile(snap_dpkg_fn): |
215 | - with open_file_read(snap_dpkg_fn) as fd: |
216 | - try: |
217 | - dpkg_list = fd.readlines() |
218 | - except Exception: |
219 | - recursive_rm(dir) |
220 | - error("Could not load %s. Is it properly formatted?" % snap_dpkg) |
221 | - |
222 | - recursive_rm(dir) |
223 | - |
224 | - return (man_yaml, dpkg_list) |
225 | - |
226 | - |
227 | def get_rock_manifest(fn): |
228 | if "SNAP_USER_COMMON" in os.environ and os.path.exists( |
229 | os.environ["SNAP_USER_COMMON"] |
230 | diff --git a/reviewtools/containers/base_container.py b/reviewtools/containers/base_container.py |
231 | index 9e6f104..15d4d7e 100644 |
232 | --- a/reviewtools/containers/base_container.py |
233 | +++ b/reviewtools/containers/base_container.py |
234 | @@ -8,11 +8,19 @@ from reviewtools.common import ( |
235 | MKDTEMP_PREFIX, |
236 | MKDTEMP_DIR, |
237 | debug, |
238 | - unpack_pkg, |
239 | recursive_rm, |
240 | ) |
241 | |
242 | |
243 | +# Add base_container private attributes to documentation |
244 | +__pdoc__ = { |
245 | + "BaseContainer._min_packed_size": True, |
246 | + "BaseContainer._max_packed_size": True, |
247 | + "BaseContainer._min_unpacked_size": True, |
248 | + "BaseContainer._max_unpacked_size": True, |
249 | +} |
250 | + |
251 | + |
252 | class ContainerException(Exception): |
253 | """This class represents exceptions""" |
254 | |
255 | @@ -42,6 +50,30 @@ class BaseContainer: |
256 | "application/x-pie-executable", |
257 | ] |
258 | |
259 | + _min_packed_size = None |
260 | + """ |
261 | + Minimum size of the container file in bytes. It will be used to check the |
262 | + container size during initialization. |
263 | + """ |
264 | + |
265 | + _max_packed_size = None |
266 | + """ |
267 | + Maximum size of the container file in bytes. It will be used to check the |
268 | + container size during initialization. |
269 | + """ |
270 | + |
271 | + _min_unpacked_size = None |
272 | + """ |
273 | + Minimum size of the unpacked directory in bytes. It will be used to check the |
274 | + container size during initialization. |
275 | + """ |
276 | + |
277 | + _max_unpacked_size = None |
278 | + """ |
279 | + Maximum size of the unpacked directory in bytes. It will be used to check the |
280 | + container size during initialization. |
281 | + """ |
282 | + |
283 | _unpack_dir = None |
284 | _files = None |
285 | _bin_files = None |
286 | @@ -59,6 +91,9 @@ class BaseContainer: |
287 | else: |
288 | raise ContainerException("Could not find '%s'" % fn) |
289 | |
290 | + # Check max/min size if defined |
291 | + self.check_container_size() |
292 | + |
293 | @property |
294 | def unpack_dir(self) -> str: |
295 | """ |
296 | @@ -67,8 +102,7 @@ class BaseContainer: |
297 | will be returned. |
298 | """ |
299 | if self._unpack_dir is None: |
300 | - self._unpack_dir = ".".join(self.filename.split(".")[:-1]) |
301 | - unpack_pkg(self.filename, dest=self._unpack_dir) |
302 | + self._unpack_dir = self.unpack() |
303 | return self._unpack_dir |
304 | |
305 | @property |
306 | @@ -104,6 +138,117 @@ class BaseContainer: |
307 | if os.path.exists(self.tmp_dir): |
308 | recursive_rm(self.tmp_dir) |
309 | |
310 | + def check_container_format(self): |
311 | + """ |
312 | + Checks that the container file has the expected format. |
313 | + |
314 | + Raises: |
315 | + NotImplementedError: Calling this method will raise an exception as |
316 | + format is specific to the container type. This method needs to be |
317 | + implemented by the specific subclasses. |
318 | + """ |
319 | + raise NotImplementedError("Must be implemented by subclasses") |
320 | + |
321 | + def get_container_size(self) -> int: |
322 | + """ |
323 | + Uses os.path.getsize() to return the size of the container file. |
324 | + |
325 | + Retruns: |
326 | + int: Size in bytes of the container file |
327 | + """ |
328 | + return os.path.getsize(self.filename) |
329 | + |
330 | + def check_container_size(self): |
331 | + """ |
332 | + Checks that the container size is within the range specified by |
333 | + min_packed_size and max_packed_size. If any of those values is not specified |
334 | + (i.e. it is None), the respective check will be skipped. |
335 | + |
336 | + Raises: |
337 | + ContainerException: If container size is less than minSize or greater |
338 | + than maxSize, a ContainerException will be raised. |
339 | + """ |
340 | + size = self.get_container_size() |
341 | + if self._min_packed_size and (size < self._min_packed_size): |
342 | + raise ContainerException( |
343 | + "container file is too small (%dM < %dM)" |
344 | + % (size / 1024 / 1024, self._min_packed_size / 1024 / 1024) |
345 | + ) |
346 | + if self._max_packed_size and (size > self._max_packed_size): |
347 | + raise ContainerException( |
348 | + "container file is too large (%dM > %dM)" |
349 | + % (size / 1024 / 1024, self._max_packed_size / 1024 / 1024) |
350 | + ) |
351 | + |
352 | + def calculate_unpacked_size(self) -> int: |
353 | + """ |
354 | + Estimates the space that the unpacked directory will use without unpacking. |
355 | + |
356 | + Returns: |
357 | + int: Estimated size in bytes of the unpacked directory |
358 | + |
359 | + Raises: |
360 | + NotImplementedError: Calling this method will raise an exception as the |
361 | + estimation method is specific to the container type. This method |
362 | + needs to be implemented by the specific subclasses. |
363 | + """ |
364 | + raise NotImplementedError("Must be implemented by subclasses") |
365 | + |
366 | + def check_unpacked_size(self): |
367 | + """ |
368 | + Checks that the container size is within the range specified by |
369 | + min_unpacked_size and max_unpacked_size. If any of those values is not |
370 | + specified (i.e. it is None), the respective check will be skipped. |
371 | + |
372 | + Raises: |
373 | + ContainerException: If unpacked size is less than minSize or greater |
374 | + than maxSize, a ContainerException will be raised. |
375 | + """ |
376 | + size = self.calculate_unpacked_size() |
377 | + if self._min_unpacked_size and (size < self._min_unpacked_size): |
378 | + raise ContainerException( |
379 | + "unpacked directory is too small (%dM < %dM)" |
380 | + % (size / 1024 / 1024, self._min_unpacked_size / 1024 / 1024) |
381 | + ) |
382 | + if self._max_unpacked_size and (size > self._max_unpacked_size): |
383 | + raise ContainerException( |
384 | + "unpacked directory is too large (%dM > %dM)" |
385 | + % (size / 1024 / 1024, self._max_unpacked_size / 1024 / 1024) |
386 | + ) |
387 | + |
388 | + def check_disk_space(self): |
389 | + """ |
390 | + Checks that there is sufficient disk space available for the container to be |
391 | + unpacked, i.e. that it will not consume more than 90% of the available disk |
392 | + |
393 | + Raises: |
394 | + ContainerException: If estimated unpacked size exceed the 90% of the |
395 | + available space, an exception will be raised. |
396 | + """ |
397 | + size = self.calculate_unpacked_size() |
398 | + st = os.statvfs(self.tmp_dir) |
399 | + avail = st.f_bsize * st.f_bavail * 0.9 # 90% of available space |
400 | + if size > avail * 0.9: |
401 | + raise ContainerException( |
402 | + "insufficient available disk space (%dM > %dM)" |
403 | + % (size / 1024 / 1024, avail / 1024 / 1024) |
404 | + ) |
405 | + |
406 | + def unpack(self) -> str: |
407 | + """ |
408 | + Extracts the content of the container file and returns the absolute path to |
409 | + the newly unpacked directory. |
410 | + |
411 | + Returns: |
412 | + str: absolute path to the newly created unpack dir |
413 | + |
414 | + Raises: |
415 | + NotImplementedError: Calling this method will raise an exception as the |
416 | + estimation method is specific to the container type. This method |
417 | + needs to be implemented by the specific subclasses. |
418 | + """ |
419 | + raise NotImplementedError("Must be implemented by subclasses") |
420 | + |
421 | def get_file(self, fn: str) -> Union[str, None]: |
422 | """ |
423 | Reads and return the raw content of the specified file. |
424 | @@ -117,7 +262,7 @@ class BaseContainer: |
425 | |
426 | Raises: |
427 | ContainerException: Calling this method will raise an exception if |
428 | - the requested file is not found. |
429 | + the requested file is not found. |
430 | """ |
431 | if not os.path.exists(os.path.join(self.unpack_dir, fn)): |
432 | raise ContainerException("Could not find '%s'" % fn) |
433 | diff --git a/reviewtools/containers/snap_container.py b/reviewtools/containers/snap_container.py |
434 | new file mode 100644 |
435 | index 0000000..ff1ee57 |
436 | --- /dev/null |
437 | +++ b/reviewtools/containers/snap_container.py |
438 | @@ -0,0 +1,44 @@ |
439 | +import yaml |
440 | +from reviewtools.containers.squashfs_container import ( |
441 | + SquashfsContainer, |
442 | + ContainerException, |
443 | +) |
444 | + |
445 | + |
446 | +class SnapContainer(SquashfsContainer): |
447 | + # TODO: review-tools was not checking for it. We still have some snaps in |
448 | + # "tests/" that doesn't fit this limit and will need to be regenerated before |
449 | + # this check can be enforced. |
450 | + # _min_packed_size = 16 * 1024 |
451 | + _max_packed_size = 5 * 1024 * 1024 * 1024 |
452 | + _max_unpacked_size = 25 * 1024 * 1024 * 1024 |
453 | + |
454 | + def get_snap_manifest(self): |
455 | + man = "snap/manifest.yaml" |
456 | + os_dpkg = "usr/share/snappy/dpkg.list" |
457 | + snap_dpkg = "snap/dpkg.list" |
458 | + |
459 | + if self._unpack_dir is None: |
460 | + self.unpack(items=[man, os_dpkg, snap_dpkg]) |
461 | + |
462 | + man_content = self.get_file(man).decode() |
463 | + try: |
464 | + man_yaml = yaml.safe_load(man_content) |
465 | + except Exception: |
466 | + raise ContainerException( |
467 | + "Could not load %s. Is it properly formatted?" % man |
468 | + ) |
469 | + |
470 | + dpkg_list = None |
471 | + try: |
472 | + dpkg_list = self.get_file(os_dpkg).decode().split("\n") |
473 | + return (man_yaml, dpkg_list) |
474 | + except ContainerException: |
475 | + pass |
476 | + |
477 | + try: |
478 | + dpkg_list = self.get_file(snap_dpkg).decode().split("\n") |
479 | + except ContainerException: |
480 | + pass |
481 | + |
482 | + return (man_yaml, dpkg_list) |
483 | diff --git a/reviewtools/containers/squashfs_container.py b/reviewtools/containers/squashfs_container.py |
484 | new file mode 100644 |
485 | index 0000000..631fe76 |
486 | --- /dev/null |
487 | +++ b/reviewtools/containers/squashfs_container.py |
488 | @@ -0,0 +1,111 @@ |
489 | +import os |
490 | +import shutil |
491 | +from reviewtools.common import cmd, cmdIgnoreErrorStrings, UNSQUASHFS_IGNORED_ERRORS |
492 | +from reviewtools.containers.base_container import BaseContainer, ContainerException |
493 | + |
494 | + |
495 | +class SquashfsContainer(BaseContainer): |
496 | + """ |
497 | + A container is a file that encapsulates a file system. This class extends |
498 | + BaseContainer to handle functionality that is specific to Squashfs containers. |
499 | + """ |
500 | + |
501 | + def __init__(self, fn): |
502 | + """ |
503 | + If SquashfsContainer is initialized with a container file, it will also |
504 | + verify that the provided file is actually a squashfs, in addition to the |
505 | + checks performed by the BaseContainer. |
506 | + """ |
507 | + super().__init__(fn) |
508 | + |
509 | + # Check is squash file |
510 | + self.check_container_format() |
511 | + |
512 | + def check_container_format(self): |
513 | + """ |
514 | + Checks that the container file has the expected format. |
515 | + |
516 | + Raises: |
517 | + ContainerException: This method will raise an exception if the |
518 | + container file is not a squashfs container. |
519 | + """ |
520 | + with open(self.filename, "rb") as f: |
521 | + header = f.read(10) |
522 | + if not header.startswith(b"hsqs"): |
523 | + raise ContainerException("Unsupported package format (not squashfs)") |
524 | + |
525 | + def calculate_unpacked_size(self) -> int: |
526 | + """ |
527 | + Estimates the space that the unpacked directory will use by adding the |
528 | + individual files size reported by "unsquashfs -lln" command. It may not |
529 | + be completely accurate, as this command does not consider that each folder |
530 | + requires at least one complete block |
531 | + |
532 | + Returns: |
533 | + int: Estimated size in bytes of the unpacked directory |
534 | + """ |
535 | + (rc, out) = cmd(["unsquashfs", "-lln", self.filename]) |
536 | + if rc != 0: |
537 | + raise RuntimeError("unsquashfs -lln '%s' failed: %s" % (self.filename, out)) |
538 | + |
539 | + size = 0 |
540 | + for line in out.splitlines(): |
541 | + if not line.startswith("-"): # skip non-regular files |
542 | + continue |
543 | + try: |
544 | + size += int(line.split()[2]) |
545 | + except ValueError: # skip non-numbers |
546 | + continue |
547 | + |
548 | + return size |
549 | + |
550 | + def unpack(self, items: list = [], force: bool = False) -> str: |
551 | + """ |
552 | + Extracts the content of the container file and returns the absolute path to |
553 | + the newly unpacked directory. |
554 | + |
555 | + Args: |
556 | + force (bool, optional): If ``True``, the unpack_dir will be overwritten |
557 | + if it already exists. Defaults to ``False`` |
558 | + |
559 | + Returns: |
560 | + str: absolute path to the newly created unpack dir |
561 | + |
562 | + Raises: |
563 | + ContainerException: This method will raise an ContainerException if the |
564 | + unpack_dir already exists and this method was called without the |
565 | + force argument set to True |
566 | + RuntimeError: This method will raise a RuntimeError under the execution |
567 | + of the "unsquashfs" command returns with an error code other than 0. |
568 | + """ |
569 | + if self._unpack_dir is not None and not force: |
570 | + raise ContainerException("'%s' exists. Aborting." % self._unpack_dir) |
571 | + |
572 | + # Check disk space and space available in disk |
573 | + self.check_unpacked_size() |
574 | + self.check_disk_space() |
575 | + |
576 | + # Unpack the container |
577 | + stage_dir = os.path.join(self.tmp_dir, "stage") |
578 | + cmd = [ |
579 | + "unsquashfs", |
580 | + "-no-progress", |
581 | + "-ignore-errors", |
582 | + "-quiet", |
583 | + "-f", |
584 | + "-d", |
585 | + stage_dir, |
586 | + self.filename, |
587 | + ] + items |
588 | + (rc, out) = cmdIgnoreErrorStrings(cmd, UNSQUASHFS_IGNORED_ERRORS) |
589 | + if rc != 0: |
590 | + if os.path.isdir(stage_dir): |
591 | + shutil.rmtree(stage_dir) |
592 | + raise RuntimeError("unpacking failed with '%d':\n%s" % (rc, out)) |
593 | + |
594 | + # Update on success |
595 | + if self._unpack_dir is not None and os.path.exists(self._unpack_dir): |
596 | + shutil.rmtree(self._unpack_dir) |
597 | + self._unpack_dir = ".".join(self.filename.split(".")[:-1]) |
598 | + shutil.move(stage_dir, self._unpack_dir) |
599 | + return self._unpack_dir |
600 | diff --git a/reviewtools/sr_common.py b/reviewtools/sr_common.py |
601 | index 16790cc..9b961e2 100644 |
602 | --- a/reviewtools/sr_common.py |
603 | +++ b/reviewtools/sr_common.py |
604 | @@ -30,7 +30,7 @@ from reviewtools.common import ( |
605 | unsquashfs_lln_parse, |
606 | verify_type, |
607 | ) |
608 | -from reviewtools.containers.base_container import BaseContainer, ContainerException |
609 | +from reviewtools.containers.snap_container import SnapContainer, ContainerException |
610 | from reviewtools.overrides import interfaces_attribs_addons |
611 | |
612 | |
613 | @@ -398,7 +398,7 @@ class SnapReview(ReviewBase): |
614 | ReviewBase.__init__(self, review_type, overrides=overrides) |
615 | |
616 | try: |
617 | - self.snap = BaseContainer(fn) |
618 | + self.snap = SnapContainer(fn) |
619 | except ContainerException as e: |
620 | error(str(e)) |
621 | # TODO: update code to directly use self.snap.xxx instead of assigning here |
622 | diff --git a/reviewtools/sr_tests.py b/reviewtools/sr_tests.py |
623 | index 6caae54..bfbf375 100644 |
624 | --- a/reviewtools/sr_tests.py |
625 | +++ b/reviewtools/sr_tests.py |
626 | @@ -87,7 +87,7 @@ def create_patches(): |
627 | # child. |
628 | patches = [] |
629 | patches.append( |
630 | - patch("reviewtools.sr_common.BaseContainer.__init__", _BaseContainer.__init__) |
631 | + patch("reviewtools.sr_common.SnapContainer.__init__", _BaseContainer.__init__) |
632 | ) |
633 | patches.append( |
634 | patch("reviewtools.sr_common.SnapReview._extract_snap_yaml", _extract_snap_yaml) |
635 | @@ -98,7 +98,6 @@ def create_patches(): |
636 | _extract_snap_manifest_yaml, |
637 | ) |
638 | ) |
639 | - patches.append(patch("reviewtools.common.unpack_pkg", _mock_func)) |
640 | # sr_common |
641 | patches.append( |
642 | patch("reviewtools.sr_common.SnapReview._get_unpack_dir", __get_unpack_dir) |
643 | diff --git a/reviewtools/tests/containers/test_base_container.py b/reviewtools/tests/containers/test_base_container.py |
644 | index 740e942..1b4e022 100644 |
645 | --- a/reviewtools/tests/containers/test_base_container.py |
646 | +++ b/reviewtools/tests/containers/test_base_container.py |
647 | @@ -3,29 +3,43 @@ import shutil |
648 | import unittest |
649 | from tempfile import mkdtemp |
650 | from reviewtools.containers.base_container import BaseContainer, ContainerException |
651 | -from reviewtools.tests.utils import make_snap2 |
652 | |
653 | |
654 | class TestBaseContainer(unittest.TestCase): |
655 | """Tests for reviewtools.containers.base_container.BaseContainer""" |
656 | |
657 | - expected_files = ["meta/snap.yaml", "meta/icon.png", "nonexecstack.bin"] |
658 | - expected_binaries = ["nonexecstack.bin"] |
659 | + expected_files = ["meta/snap.yaml", "meta/icon.png", "bin/ls"] |
660 | + expected_binaries = ["bin/ls"] |
661 | |
662 | @classmethod |
663 | def setUpClass(cls): |
664 | cls.tmp_dir = mkdtemp() |
665 | - cls.fn = make_snap2( |
666 | - output_dir=cls.tmp_dir, |
667 | - yaml="Test", |
668 | - extra_files=["/bin/ls:nonexecstack.bin"], |
669 | - ) |
670 | + cls.fn = cls._create_container(cls) |
671 | + cls.unpack_dir = cls._create_unpackdir(cls) |
672 | |
673 | @classmethod |
674 | def tearDownClass(cls): |
675 | if os.path.exists(cls.tmp_dir): |
676 | shutil.rmtree(cls.tmp_dir) |
677 | |
678 | + def _create_container(self, container_size: int = 10000): |
679 | + fn = os.path.join(self.tmp_dir, "container.test") |
680 | + with open(fn, "w") as f: |
681 | + f.truncate(container_size) |
682 | + return fn |
683 | + |
684 | + def _create_unpackdir(self): |
685 | + unpack_dir = os.path.join(self.tmp_dir, "unpack_dir") |
686 | + for file in self.expected_files: |
687 | + unpacked_file = os.path.join(unpack_dir, file) |
688 | + os.makedirs(os.path.dirname(unpacked_file), exist_ok=True) |
689 | + if file not in self.expected_binaries: |
690 | + with open(unpacked_file, "w+b") as fd: |
691 | + fd.write(b"Test") |
692 | + else: |
693 | + shutil.copyfile(os.path.join("/", file), unpacked_file) |
694 | + return unpack_dir |
695 | + |
696 | # Test class initialization |
697 | def test_initialization_container__happy(self): |
698 | try: |
699 | @@ -43,10 +57,75 @@ class TestBaseContainer(unittest.TestCase): |
700 | in context.exception.value |
701 | ) |
702 | |
703 | + # Test deletion |
704 | + def test_check_cleanup(self): |
705 | + try: |
706 | + container = BaseContainer(self.fn) |
707 | + tmp_dir = container.tmp_dir |
708 | + del container |
709 | + self.assertFalse(os.path.exists(tmp_dir)) |
710 | + except Exception: |
711 | + self.fail("An unexpected error occurred during deletion test") |
712 | + |
713 | + # Test check container size |
714 | + def test_check_container_size__happy(self): |
715 | + container = BaseContainer(self.fn) |
716 | + container._min_packed_size = 0 |
717 | + container._max_packed_size = 10**6 |
718 | + try: |
719 | + container.check_container_size() |
720 | + except ContainerException: |
721 | + self.fail("An unexpected error occurred while checking file size") |
722 | + |
723 | + def test_check_container_size__too_big(self): |
724 | + container = BaseContainer(self.fn) |
725 | + container._max_packed_size = 1 |
726 | + with self.assertRaises(ContainerException) as context: |
727 | + container.check_container_size() |
728 | + self.assertTrue("container file is too large " in context.exception.value) |
729 | + |
730 | + def test_check_container_size__too_small(self): |
731 | + container = BaseContainer(self.fn) |
732 | + container._min_packed_size = 10**6 |
733 | + with self.assertRaises(ContainerException) as context: |
734 | + container.check_container_size() |
735 | + self.assertTrue("container file is too small " in context.exception.value) |
736 | + |
737 | + # Test check unpacked size |
738 | + def _mock_calculate_unpacked_size(self): |
739 | + return 10**3 |
740 | + |
741 | + def test_check_unpacked_size__happy(self): |
742 | + container = BaseContainer(self.fn) |
743 | + container.calculate_unpacked_size = self._mock_calculate_unpacked_size |
744 | + container._min_unpacked_size = 0 |
745 | + container._max_unpacked_size = 10**6 |
746 | + try: |
747 | + container.check_unpacked_size() |
748 | + except ContainerException: |
749 | + self.fail("An unexpected error occurred while checking file size") |
750 | + |
751 | + def test_check_unpacked_size__too_big(self): |
752 | + container = BaseContainer(self.fn) |
753 | + container.calculate_unpacked_size = self._mock_calculate_unpacked_size |
754 | + container._max_unpacked_size = 1 |
755 | + with self.assertRaises(ContainerException) as context: |
756 | + container.check_unpacked_size() |
757 | + self.assertTrue("container file is too large " in context.exception.value) |
758 | + |
759 | + def test_check_unpacked_size__too_small(self): |
760 | + container = BaseContainer(self.fn) |
761 | + container.calculate_unpacked_size = self._mock_calculate_unpacked_size |
762 | + container._min_unpacked_size = 10**6 |
763 | + with self.assertRaises(ContainerException) as context: |
764 | + container.check_unpacked_size() |
765 | + self.assertTrue("container file is too small " in context.exception.value) |
766 | + |
767 | # Test get file |
768 | def test_get_file__happy(self): |
769 | try: |
770 | container = BaseContainer(self.fn) |
771 | + container._unpack_dir = self.unpack_dir |
772 | snap_yaml = container.get_file("meta/snap.yaml") |
773 | self.assertEqual(snap_yaml.decode(), "Test") |
774 | except ContainerException: |
775 | @@ -55,6 +134,7 @@ class TestBaseContainer(unittest.TestCase): |
776 | def test_get_file__missing_file(self): |
777 | with self.assertRaises(ContainerException) as context: |
778 | container = BaseContainer(self.fn) |
779 | + container._unpack_dir = self.unpack_dir |
780 | container.get_file("non/existent/file") |
781 | self.assertTrue("Could not find" in context.exception.value) |
782 | |
783 | @@ -62,6 +142,7 @@ class TestBaseContainer(unittest.TestCase): |
784 | def test_get_files_list__happy(self): |
785 | try: |
786 | container = BaseContainer(self.fn) |
787 | + container._unpack_dir = self.unpack_dir |
788 | self.assertCountEqual( |
789 | container.get_files_list(), |
790 | self.expected_files, |
791 | @@ -72,6 +153,7 @@ class TestBaseContainer(unittest.TestCase): |
792 | def test_get_files_list_abs__happy(self): |
793 | try: |
794 | container = BaseContainer(self.fn) |
795 | + container._unpack_dir = self.unpack_dir |
796 | self.assertCountEqual( |
797 | container.get_files_list(abs=True), |
798 | [ |
799 | @@ -85,6 +167,7 @@ class TestBaseContainer(unittest.TestCase): |
800 | def test_files__happy(self): |
801 | try: |
802 | container = BaseContainer(self.fn) |
803 | + container._unpack_dir = self.unpack_dir |
804 | self.assertCountEqual( |
805 | container.files, |
806 | self.expected_files, |
807 | @@ -96,6 +179,7 @@ class TestBaseContainer(unittest.TestCase): |
808 | def test_get_compiled_binaries_list__happy(self): |
809 | try: |
810 | container = BaseContainer(self.fn) |
811 | + container._unpack_dir = self.unpack_dir |
812 | self.assertCountEqual( |
813 | container.get_compiled_binaries_list(), self.expected_binaries |
814 | ) |
815 | @@ -107,6 +191,7 @@ class TestBaseContainer(unittest.TestCase): |
816 | def test_get_compiled_binaries_list_abs__happy(self): |
817 | try: |
818 | container = BaseContainer(self.fn) |
819 | + container._unpack_dir = self.unpack_dir |
820 | self.assertCountEqual( |
821 | container.get_compiled_binaries_list(abs=True), |
822 | [ |
823 | @@ -122,6 +207,7 @@ class TestBaseContainer(unittest.TestCase): |
824 | def test_bin_files__happy(self): |
825 | try: |
826 | container = BaseContainer(self.fn) |
827 | + container._unpack_dir = self.unpack_dir |
828 | self.assertCountEqual(container.bin_files, self.expected_binaries) |
829 | except ContainerException: |
830 | self.fail( |
831 | diff --git a/reviewtools/tests/containers/test_squashfs_container.py b/reviewtools/tests/containers/test_squashfs_container.py |
832 | new file mode 100644 |
833 | index 0000000..7d19a77 |
834 | --- /dev/null |
835 | +++ b/reviewtools/tests/containers/test_squashfs_container.py |
836 | @@ -0,0 +1,73 @@ |
837 | +import os |
838 | +import unittest |
839 | +from tempfile import mkstemp, mkdtemp |
840 | +from reviewtools.containers.squashfs_container import ( |
841 | + SquashfsContainer, |
842 | + ContainerException, |
843 | +) |
844 | +from reviewtools.tests.utils import make_snap2 |
845 | + |
846 | + |
847 | +class TestSquashfsContainer(unittest.TestCase): |
848 | + """Tests for reviewtools.containers.squashfs_container""" |
849 | + |
850 | + @classmethod |
851 | + def setUpClass(cls): |
852 | + cls.tmp_dir = mkdtemp() |
853 | + cls.fn = make_snap2( |
854 | + output_dir=cls.tmp_dir, |
855 | + yaml="Test", |
856 | + extra_files=["/bin/ls:bin/ls"], |
857 | + ) |
858 | + |
859 | + # Test initialization |
860 | + def test_check_initialization__happy(self): |
861 | + try: |
862 | + SquashfsContainer(self.fn) |
863 | + except Exception: |
864 | + self.fail( |
865 | + "An unexpected error occurred during SquashfsContainer initialization with container file" |
866 | + ) |
867 | + |
868 | + def test_check_initialization__invalid_format(self): |
869 | + with self.assertRaises(ContainerException) as context: |
870 | + fd, plain_file = mkstemp() |
871 | + SquashfsContainer(plain_file) |
872 | + self.assertEqual( |
873 | + "unsupported package format (not squashfs)", context.exception.value |
874 | + ) |
875 | + os.unlink(plain_file) |
876 | + |
877 | + # Test check format |
878 | + def test_calculate_unpacked_size__happy(self): |
879 | + try: |
880 | + # unpacked size calculated is a bit smaller than actual size as it does not consider that |
881 | + # folders require at least one complete block |
882 | + container = SquashfsContainer(self.fn) |
883 | + size = container.calculate_unpacked_size() |
884 | + self.assertTrue(isinstance(size, int)) |
885 | + except Exception: |
886 | + self.fail("An unexpected error occurred during unpacked size calculation") |
887 | + |
888 | + # Test unpack |
889 | + def test_unpack__happy(self): |
890 | + try: |
891 | + container = SquashfsContainer(self.fn) |
892 | + container.unpack() |
893 | + except Exception: |
894 | + self.fail("An unexpected error occurred during unpack") |
895 | + |
896 | + def test_unpack__force(self): |
897 | + try: |
898 | + container = SquashfsContainer(self.fn) |
899 | + container.unpack() |
900 | + container.unpack(force=True) |
901 | + except Exception: |
902 | + self.fail("An unexpected error occurred during unpack") |
903 | + |
904 | + def test_unpack__no_force(self): |
905 | + with self.assertRaises(ContainerException) as context: |
906 | + container = SquashfsContainer(self.fn) |
907 | + container.unpack() |
908 | + container.unpack() |
909 | + self.assertTrue(" exists. Aborting." in context.exception.value) |
Can you provide some more rationale for this change?