Merge ~jfguedez/charm-ubuntu-advantage:bug/1962335 into charm-ubuntu-advantage:master
- Git
- lp:~jfguedez/charm-ubuntu-advantage
- bug/1962335
- Merge into master
Proposed by
Jose Guedez
Status: | Merged |
---|---|
Approved by: | Tom Haddon |
Approved revision: | 08fa05cc05ecd33ea145f6c453ea19a3ace7ce51 |
Merged at revision: | 05a3802c0fa7c349544c5d29fe11175eb694ac65 |
Proposed branch: | ~jfguedez/charm-ubuntu-advantage:bug/1962335 |
Merge into: | charm-ubuntu-advantage:master |
Diff against target: |
1548 lines (+1363/-45) 4 files modified
lib/charms/operator_libs_linux/v0/apt.py (+1329/-0) run_tests (+4/-3) src/charm.py (+3/-13) tests/test_charm.py (+27/-29) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Tom Haddon | Approve | ||
Canonical IS Reviewers | Pending | ||
Review via email: mp+416293@code.launchpad.net |
Commit message
Use the operator apt library to install/remove packages
Description of the change
To post a comment you must log in.
Revision history for this message
🤖 Canonical IS Merge Bot (canonical-is-mergebot) wrote : | # |
Revision history for this message
Jose Guedez (jfguedez) wrote : | # |
Updated just to reflect that the operator apt library fixes [0] have been released (only the library patch version was updated)
[0] https:/
Revision history for this message
🤖 Canonical IS Merge Bot (canonical-is-mergebot) wrote : | # |
Change successfully merged at revision 05a3802c0fa7c34
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/lib/charms/operator_libs_linux/v0/apt.py b/lib/charms/operator_libs_linux/v0/apt.py |
2 | new file mode 100644 |
3 | index 0000000..2b5c8f2 |
4 | --- /dev/null |
5 | +++ b/lib/charms/operator_libs_linux/v0/apt.py |
6 | @@ -0,0 +1,1329 @@ |
7 | +# Copyright 2021 Canonical Ltd. |
8 | +# |
9 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
10 | +# you may not use this file except in compliance with the License. |
11 | +# You may obtain a copy of the License at |
12 | +# |
13 | +# http://www.apache.org/licenses/LICENSE-2.0 |
14 | +# |
15 | +# Unless required by applicable law or agreed to in writing, software |
16 | +# distributed under the License is distributed on an "AS IS" BASIS, |
17 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
18 | +# See the License for the specific language governing permissions and |
19 | +# limitations under the License. |
20 | + |
21 | +"""Abstractions for the system's Debian/Ubuntu package information and repositories. |
22 | + |
23 | +This module contains abstractions and wrappers around Debian/Ubuntu-style repositories and |
24 | +packages, in order to easily provide an idiomatic and Pythonic mechanism for adding packages and/or |
25 | +repositories to systems for use in machine charms. |
26 | + |
27 | +A sane default configuration is attainable through nothing more than instantiation of the |
28 | +appropriate classes. `DebianPackage` objects provide information about the architecture, version, |
29 | +name, and status of a package. |
30 | + |
31 | +`DebianPackage` will try to look up a package either from `dpkg -L` or from `apt-cache` when |
32 | +provided with a string indicating the package name. If it cannot be located, `PackageNotFoundError` |
33 | +will be returned, as `apt` and `dpkg` otherwise return `100` for all errors, and a meaningful error |
34 | +message if the package is not known is desirable. |
35 | + |
36 | +To install packages with convenience methods: |
37 | + |
38 | +```python |
39 | +try: |
40 | + # Run `apt-get update` |
41 | + apt.update() |
42 | + apt.add_package("zsh") |
43 | + apt.add_package(["vim", "htop", "wget"]) |
44 | +except PackageNotFoundError: |
45 | + logger.error("a specified package not found in package cache or on system") |
46 | +except PackageError as e: |
47 | + logger.error("could not install package. Reason: %s", e.message) |
48 | +```` |
49 | + |
50 | +To find details of a specific package: |
51 | + |
52 | +```python |
53 | +try: |
54 | + vim = apt.DebianPackage.from_system("vim") |
55 | + |
56 | + # To find from the apt cache only |
57 | + # apt.DebianPackage.from_apt_cache("vim") |
58 | + |
59 | + # To find from installed packages only |
60 | + # apt.DebianPackage.from_installed_package("vim") |
61 | + |
62 | + vim.ensure(PackageState.Latest) |
63 | + logger.info("updated vim to version: %s", vim.fullversion) |
64 | +except PackageNotFoundError: |
65 | + logger.error("a specified package not found in package cache or on system") |
66 | +except PackageError as e: |
67 | + logger.error("could not install package. Reason: %s", e.message) |
68 | +``` |
69 | + |
70 | + |
71 | +`RepositoryMapping` will return a dict-like object containing enabled system repositories |
72 | +and their properties (available groups, baseuri. gpg key). This class can add, disable, or |
73 | +manipulate repositories. Items can be retrieved as `DebianRepository` objects. |
74 | + |
75 | +In order add a new repository with explicit details for fields, a new `DebianRepository` can |
76 | +be added to `RepositoryMapping` |
77 | + |
78 | +`RepositoryMapping` provides an abstraction around the existing repositories on the system, |
79 | +and can be accessed and iterated over like any `Mapping` object, to retrieve values by key, |
80 | +iterate, or perform other operations. |
81 | + |
82 | +Keys are constructed as `{repo_type}-{}-{release}` in order to uniquely identify a repository. |
83 | + |
84 | +Repositories can be added with explicit values through a Python constructor. |
85 | + |
86 | +Example: |
87 | + |
88 | +```python |
89 | +repositories = apt.RepositoryMapping() |
90 | + |
91 | +if "deb-example.com-focal" not in repositories: |
92 | + repositories.add(DebianRepository(enabled=True, repotype="deb", |
93 | + uri="https://example.com", release="focal", groups=["universe"])) |
94 | +``` |
95 | + |
96 | +Alternatively, any valid `sources.list` line may be used to construct a new |
97 | +`DebianRepository`. |
98 | + |
99 | +Example: |
100 | + |
101 | +```python |
102 | +repositories = apt.RepositoryMapping() |
103 | + |
104 | +if "deb-us.archive.ubuntu.com-xenial" not in repositories: |
105 | + line = "deb http://us.archive.ubuntu.com/ubuntu xenial main restricted" |
106 | + repo = DebianRepository.from_repo_line(line) |
107 | + repositories.add(repo) |
108 | +``` |
109 | +""" |
110 | + |
111 | +import fileinput |
112 | +import glob |
113 | +import logging |
114 | +import os |
115 | +import re |
116 | +import subprocess |
117 | +from collections.abc import Mapping |
118 | +from enum import Enum |
119 | +from subprocess import PIPE, CalledProcessError, check_call, check_output |
120 | +from typing import Iterable, List, Optional, Tuple, Union |
121 | +from urllib.parse import urlparse |
122 | + |
123 | +logger = logging.getLogger(__name__) |
124 | + |
125 | +# The unique Charmhub library identifier, never change it |
126 | +LIBID = "7c3dbc9c2ad44a47bd6fcb25caa270e5" |
127 | + |
128 | +# Increment this major API version when introducing breaking changes |
129 | +LIBAPI = 0 |
130 | + |
131 | +# Increment this PATCH version before using `charmcraft publish-lib` or reset |
132 | +# to 0 if you are raising the major API version |
133 | +LIBPATCH = 7 |
134 | + |
135 | + |
136 | +VALID_SOURCE_TYPES = ("deb", "deb-src") |
137 | +OPTIONS_MATCHER = re.compile(r"\[.*?\]") |
138 | + |
139 | + |
140 | +class Error(Exception): |
141 | + """Base class of most errors raised by this library.""" |
142 | + |
143 | + def __repr__(self): |
144 | + """String representation of Error.""" |
145 | + return "<{}.{} {}>".format(type(self).__module__, type(self).__name__, self.args) |
146 | + |
147 | + @property |
148 | + def name(self): |
149 | + """Return a string representation of the model plus class.""" |
150 | + return "<{}.{}>".format(type(self).__module__, type(self).__name__) |
151 | + |
152 | + @property |
153 | + def message(self): |
154 | + """Return the message passed as an argument.""" |
155 | + return self.args[0] |
156 | + |
157 | + |
158 | +class PackageError(Error): |
159 | + """Raised when there's an error installing or removing a package.""" |
160 | + |
161 | + |
162 | +class PackageNotFoundError(Error): |
163 | + """Raised when a requested package is not known to the system.""" |
164 | + |
165 | + |
166 | +class PackageState(Enum): |
167 | + """A class to represent possible package states.""" |
168 | + |
169 | + Present = "present" |
170 | + Absent = "absent" |
171 | + Latest = "latest" |
172 | + Available = "available" |
173 | + |
174 | + |
175 | +class DebianPackage: |
176 | + """Represents a traditional Debian package and its utility functions. |
177 | + |
178 | + `DebianPackage` wraps information and functionality around a known package, whether installed |
179 | + or available. The version, epoch, name, and architecture can be easily queried and compared |
180 | + against other `DebianPackage` objects to determine the latest version or to install a specific |
181 | + version. |
182 | + |
183 | + The representation of this object as a string mimics the output from `dpkg` for familiarity. |
184 | + |
185 | + Installation and removal of packages is handled through the `state` property or `ensure` |
186 | + method, with the following options: |
187 | + |
188 | + apt.PackageState.Absent |
189 | + apt.PackageState.Available |
190 | + apt.PackageState.Present |
191 | + apt.PackageState.Latest |
192 | + |
193 | + When `DebianPackage` is initialized, the state of a given `DebianPackage` object will be set to |
194 | + `Available`, `Present`, or `Latest`, with `Absent` implemented as a convenience for removal |
195 | + (though it operates essentially the same as `Available`). |
196 | + """ |
197 | + |
198 | + def __init__( |
199 | + self, name: str, version: str, epoch: str, arch: str, state: PackageState |
200 | + ) -> None: |
201 | + self._name = name |
202 | + self._arch = arch |
203 | + self._state = state |
204 | + self._version = Version(version, epoch) |
205 | + |
206 | + def __eq__(self, other) -> bool: |
207 | + """Equality for comparison. |
208 | + |
209 | + Args: |
210 | + other: a `DebianPackage` object for comparison |
211 | + |
212 | + Returns: |
213 | + A boolean reflecting equality |
214 | + """ |
215 | + return isinstance(other, self.__class__) and ( |
216 | + self._name, |
217 | + self._version.number, |
218 | + ) == (other._name, other._version.number) |
219 | + |
220 | + def __hash__(self): |
221 | + """A basic hash so this class can be used in Mappings and dicts.""" |
222 | + return hash((self._name, self._version.number)) |
223 | + |
224 | + def __repr__(self): |
225 | + """A representation of the package.""" |
226 | + return "<{}.{}: {}>".format(self.__module__, self.__class__.__name__, self.__dict__) |
227 | + |
228 | + def __str__(self): |
229 | + """A human-readable representation of the package.""" |
230 | + return "<{}: {}-{}.{} -- {}>".format( |
231 | + self.__class__.__name__, |
232 | + self._name, |
233 | + self._version, |
234 | + self._arch, |
235 | + str(self._state), |
236 | + ) |
237 | + |
238 | + @staticmethod |
239 | + def _apt( |
240 | + command: str, |
241 | + package_names: Union[str, List], |
242 | + optargs: Optional[List[str]] = None, |
243 | + ) -> None: |
244 | + """Wrap package management commands for Debian/Ubuntu systems. |
245 | + |
246 | + Args: |
247 | + command: the command given to `apt-get` |
248 | + package_names: a package name or list of package names to operate on |
249 | + optargs: an (Optional) list of additioanl arguments |
250 | + |
251 | + Raises: |
252 | + PackageError if an error is encountered |
253 | + """ |
254 | + optargs = optargs if optargs is not None else [] |
255 | + if isinstance(package_names, str): |
256 | + package_names = [package_names] |
257 | + _cmd = ["apt-get", "-y", *optargs, command, *package_names] |
258 | + try: |
259 | + check_call(_cmd, stderr=PIPE, stdout=PIPE) |
260 | + except CalledProcessError as e: |
261 | + raise PackageError( |
262 | + "Could not {} package(s) [{}]: {}".format(command, [*package_names], e.output) |
263 | + ) from None |
264 | + |
265 | + def _add(self) -> None: |
266 | + """Add a package to the system.""" |
267 | + self._apt( |
268 | + "install", |
269 | + "{}={}".format(self.name, self.version), |
270 | + optargs=["--option=Dpkg::Options::=--force-confold"], |
271 | + ) |
272 | + |
273 | + def _remove(self) -> None: |
274 | + """Removes a package from the system. Implementation-specific.""" |
275 | + return self._apt("remove", "{}={}".format(self.name, self.version)) |
276 | + |
277 | + @property |
278 | + def name(self) -> str: |
279 | + """Returns the name of the package.""" |
280 | + return self._name |
281 | + |
282 | + def ensure(self, state: PackageState): |
283 | + """Ensures that a package is in a given state. |
284 | + |
285 | + Args: |
286 | + state: a `PackageState` to reconcile the package to |
287 | + |
288 | + Raises: |
289 | + PackageError from the underlying call to apt |
290 | + """ |
291 | + if self._state is not state: |
292 | + if state not in (PackageState.Present, PackageState.Latest): |
293 | + self._remove() |
294 | + else: |
295 | + self._add() |
296 | + self._state = state |
297 | + |
298 | + @property |
299 | + def present(self) -> bool: |
300 | + """Returns whether or not a package is present.""" |
301 | + return self._state in (PackageState.Present, PackageState.Latest) |
302 | + |
303 | + @property |
304 | + def latest(self) -> bool: |
305 | + """Returns whether the package is the most recent version.""" |
306 | + return self._state is PackageState.Latest |
307 | + |
308 | + @property |
309 | + def state(self) -> PackageState: |
310 | + """Returns the current package state.""" |
311 | + return self._state |
312 | + |
313 | + @state.setter |
314 | + def state(self, state: PackageState) -> None: |
315 | + """Sets the package state to a given value. |
316 | + |
317 | + Args: |
318 | + state: a `PackageState` to reconcile the package to |
319 | + |
320 | + Raises: |
321 | + PackageError from the underlying call to apt |
322 | + """ |
323 | + if state in (PackageState.Latest, PackageState.Present): |
324 | + self._add() |
325 | + else: |
326 | + self._remove() |
327 | + self._state = state |
328 | + |
329 | + @property |
330 | + def version(self) -> "Version": |
331 | + """Returns the version for a package.""" |
332 | + return self._version |
333 | + |
334 | + @property |
335 | + def epoch(self) -> str: |
336 | + """Returns the epoch for a package. May be unset.""" |
337 | + return self._version.epoch |
338 | + |
339 | + @property |
340 | + def arch(self) -> str: |
341 | + """Returns the architecture for a package.""" |
342 | + return self._arch |
343 | + |
344 | + @property |
345 | + def fullversion(self) -> str: |
346 | + """Returns the name+epoch for a package.""" |
347 | + return "{}.{}".format(self._version, self._arch) |
348 | + |
349 | + @staticmethod |
350 | + def _get_epoch_from_version(version: str) -> Tuple[str, str]: |
351 | + """Pull the epoch, if any, out of a version string.""" |
352 | + epoch_matcher = re.compile(r"^((?P<epoch>\d+):)?(?P<version>.*)") |
353 | + matches = epoch_matcher.search(version).groupdict() |
354 | + return matches.get("epoch", ""), matches.get("version") |
355 | + |
356 | + @classmethod |
357 | + def from_system( |
358 | + cls, package: str, version: Optional[str] = "", arch: Optional[str] = "" |
359 | + ) -> "DebianPackage": |
360 | + """Locates a package, either on the system or known to apt, and serializes the information. |
361 | + |
362 | + Args: |
363 | + package: a string representing the package |
364 | + version: an optional string if a specific version isr equested |
365 | + arch: an optional architecture, defaulting to `dpkg --print-architecture`. If an |
366 | + architecture is not specified, this will be used for selection. |
367 | + |
368 | + """ |
369 | + try: |
370 | + return DebianPackage.from_installed_package(package, version, arch) |
371 | + except PackageNotFoundError: |
372 | + logger.debug( |
373 | + "package '%s' is not currently installed or has the wrong architecture.", package |
374 | + ) |
375 | + |
376 | + # Ok, try `apt-cache ...` |
377 | + try: |
378 | + return DebianPackage.from_apt_cache(package, version, arch) |
379 | + except (PackageNotFoundError, PackageError): |
380 | + # If we get here, it's not known to the systems. |
381 | + # This seems unnecessary, but virtually all `apt` commands have a return code of `100`, |
382 | + # and providing meaningful error messages without this is ugly. |
383 | + raise PackageNotFoundError( |
384 | + "Package '{}{}' could not be found on the system or in the apt cache!".format( |
385 | + package, ".{}".format(arch) if arch else "" |
386 | + ) |
387 | + ) from None |
388 | + |
389 | + @classmethod |
390 | + def from_installed_package( |
391 | + cls, package: str, version: Optional[str] = "", arch: Optional[str] = "" |
392 | + ) -> "DebianPackage": |
393 | + """Check whether the package is already installed and return an instance. |
394 | + |
395 | + Args: |
396 | + package: a string representing the package |
397 | + version: an optional string if a specific version isr equested |
398 | + arch: an optional architecture, defaulting to `dpkg --print-architecture`. |
399 | + If an architecture is not specified, this will be used for selection. |
400 | + """ |
401 | + system_arch = check_output( |
402 | + ["dpkg", "--print-architecture"], universal_newlines=True |
403 | + ).strip() |
404 | + arch = arch if arch else system_arch |
405 | + |
406 | + # Regexps are a really terrible way to do this. Thanks dpkg |
407 | + output = "" |
408 | + try: |
409 | + output = check_output(["dpkg", "-l", package], stderr=PIPE, universal_newlines=True) |
410 | + except CalledProcessError: |
411 | + raise PackageNotFoundError("Package is not installed: {}".format(package)) from None |
412 | + |
413 | + # Pop off the output from `dpkg -l' because there's no flag to |
414 | + # omit it` |
415 | + lines = str(output).splitlines()[5:] |
416 | + |
417 | + dpkg_matcher = re.compile( |
418 | + r""" |
419 | + ^(?P<package_status>\w+?)\s+ |
420 | + (?P<package_name>.*?)(?P<throwaway_arch>:\w+?)?\s+ |
421 | + (?P<version>.*?)\s+ |
422 | + (?P<arch>\w+?)\s+ |
423 | + (?P<description>.*) |
424 | + """, |
425 | + re.VERBOSE, |
426 | + ) |
427 | + |
428 | + for line in lines: |
429 | + try: |
430 | + matches = dpkg_matcher.search(line).groupdict() |
431 | + package_status = matches["package_status"] |
432 | + |
433 | + if not package_status.endswith("i"): |
434 | + logger.debug( |
435 | + "package '%s' in dpkg output but not installed, status: '%s'", |
436 | + package, |
437 | + package_status, |
438 | + ) |
439 | + break |
440 | + |
441 | + epoch, split_version = DebianPackage._get_epoch_from_version(matches["version"]) |
442 | + pkg = DebianPackage( |
443 | + matches["package_name"], |
444 | + split_version, |
445 | + epoch, |
446 | + matches["arch"], |
447 | + PackageState.Present, |
448 | + ) |
449 | + if (pkg.arch == "all" or pkg.arch == arch) and ( |
450 | + version == "" or str(pkg.version) == version |
451 | + ): |
452 | + return pkg |
453 | + except AttributeError: |
454 | + logger.warning("dpkg matcher could not parse line: %s", line) |
455 | + |
456 | + # If we didn't find it, fail through |
457 | + raise PackageNotFoundError("Package {}.{} is not installed!".format(package, arch)) |
458 | + |
459 | + @classmethod |
460 | + def from_apt_cache( |
461 | + cls, package: str, version: Optional[str] = "", arch: Optional[str] = "" |
462 | + ) -> "DebianPackage": |
463 | + """Check whether the package is already installed and return an instance. |
464 | + |
465 | + Args: |
466 | + package: a string representing the package |
467 | + version: an optional string if a specific version isr equested |
468 | + arch: an optional architecture, defaulting to `dpkg --print-architecture`. |
469 | + If an architecture is not specified, this will be used for selection. |
470 | + """ |
471 | + system_arch = check_output( |
472 | + ["dpkg", "--print-architecture"], universal_newlines=True |
473 | + ).strip() |
474 | + arch = arch if arch else system_arch |
475 | + |
476 | + # Regexps are a really terrible way to do this. Thanks dpkg |
477 | + keys = ("Package", "Architecture", "Version") |
478 | + |
479 | + try: |
480 | + output = check_output( |
481 | + ["apt-cache", "show", package], stderr=PIPE, universal_newlines=True |
482 | + ) |
483 | + except CalledProcessError as e: |
484 | + raise PackageError( |
485 | + "Could not list packages in apt-cache: {}".format(e.output) |
486 | + ) from None |
487 | + |
488 | + pkg_groups = output.strip().split("\n\n") |
489 | + keys = ("Package", "Architecture", "Version") |
490 | + |
491 | + for pkg_raw in pkg_groups: |
492 | + lines = str(pkg_raw).splitlines() |
493 | + vals = {} |
494 | + for line in lines: |
495 | + if line.startswith(keys): |
496 | + items = line.split(":", 1) |
497 | + vals[items[0]] = items[1].strip() |
498 | + else: |
499 | + continue |
500 | + |
501 | + epoch, split_version = DebianPackage._get_epoch_from_version(vals["Version"]) |
502 | + pkg = DebianPackage( |
503 | + vals["Package"], |
504 | + split_version, |
505 | + epoch, |
506 | + vals["Architecture"], |
507 | + PackageState.Available, |
508 | + ) |
509 | + |
510 | + if (pkg.arch == "all" or pkg.arch == arch) and ( |
511 | + version == "" or str(pkg.version) == version |
512 | + ): |
513 | + return pkg |
514 | + |
515 | + # If we didn't find it, fail through |
516 | + raise PackageNotFoundError("Package {}.{} is not in the apt cache!".format(package, arch)) |
517 | + |
518 | + |
519 | +class Version: |
520 | + """An abstraction around package versions. |
521 | + |
522 | + This seems like it should be strictly unnecessary, except that `apt_pkg` is not usable inside a |
523 | + venv, and wedging version comparisions into `DebianPackage` would overcomplicate it. |
524 | + |
525 | + This class implements the algorithm found here: |
526 | + https://www.debian.org/doc/debian-policy/ch-controlfields.html#version |
527 | + """ |
528 | + |
529 | + def __init__(self, version: str, epoch: str): |
530 | + self._version = version |
531 | + self._epoch = epoch or "" |
532 | + |
533 | + def __repr__(self): |
534 | + """A representation of the package.""" |
535 | + return "<{}.{}: {}>".format(self.__module__, self.__class__.__name__, self.__dict__) |
536 | + |
537 | + def __str__(self): |
538 | + """A human-readable representation of the package.""" |
539 | + return "{}{}".format("{}:".format(self._epoch) if self._epoch else "", self._version) |
540 | + |
541 | + @property |
542 | + def epoch(self): |
543 | + """Returns the epoch for a package. May be empty.""" |
544 | + return self._epoch |
545 | + |
546 | + @property |
547 | + def number(self) -> str: |
548 | + """Returns the version number for a package.""" |
549 | + return self._version |
550 | + |
551 | + def _get_parts(self, version: str) -> Tuple[str, str]: |
552 | + """Separate the version into component upstream and Debian pieces.""" |
553 | + try: |
554 | + version.rindex("-") |
555 | + except ValueError: |
556 | + # No hyphens means no Debian version |
557 | + return version, "0" |
558 | + |
559 | + upstream, debian = version.rsplit("-", 1) |
560 | + return upstream, debian |
561 | + |
562 | + def _listify(self, revision: str) -> List[str]: |
563 | + """Split a revision string into a listself. |
564 | + |
565 | + This list is comprised of alternating between strings and numbers, |
566 | + padded on either end to always be "str, int, str, int..." and |
567 | + always be of even length. This allows us to trivially implement the |
568 | + comparison algorithm described. |
569 | + """ |
570 | + result = [] |
571 | + while revision: |
572 | + rev_1, remains = self._get_alphas(revision) |
573 | + rev_2, remains = self._get_digits(remains) |
574 | + result.extend([rev_1, rev_2]) |
575 | + revision = remains |
576 | + return result |
577 | + |
578 | + def _get_alphas(self, revision: str) -> Tuple[str, str]: |
579 | + """Return a tuple of the first non-digit characters of a revision.""" |
580 | + # get the index of the first digit |
581 | + for i, char in enumerate(revision): |
582 | + if char.isdigit(): |
583 | + if i == 0: |
584 | + return "", revision |
585 | + return revision[0:i], revision[i:] |
586 | + # string is entirely alphas |
587 | + return revision, "" |
588 | + |
589 | + def _get_digits(self, revision: str) -> Tuple[int, str]: |
590 | + """Return a tuple of the first integer characters of a revision.""" |
591 | + # If the string is empty, return (0,'') |
592 | + if not revision: |
593 | + return 0, "" |
594 | + # get the index of the first non-digit |
595 | + for i, char in enumerate(revision): |
596 | + if not char.isdigit(): |
597 | + if i == 0: |
598 | + return 0, revision |
599 | + return int(revision[0:i]), revision[i:] |
600 | + # string is entirely digits |
601 | + return int(revision), "" |
602 | + |
603 | + def _dstringcmp(self, a, b): # noqa: C901 |
604 | + """Debian package version string section lexical sort algorithm. |
605 | + |
606 | + The lexical comparison is a comparison of ASCII values modified so |
607 | + that all the letters sort earlier than all the non-letters and so that |
608 | + a tilde sorts before anything, even the end of a part. |
609 | + """ |
610 | + if a == b: |
611 | + return 0 |
612 | + try: |
613 | + for i, char in enumerate(a): |
614 | + if char == b[i]: |
615 | + continue |
616 | + # "a tilde sorts before anything, even the end of a part" |
617 | + # (emptyness) |
618 | + if char == "~": |
619 | + return -1 |
620 | + if b[i] == "~": |
621 | + return 1 |
622 | + # "all the letters sort earlier than all the non-letters" |
623 | + if char.isalpha() and not b[i].isalpha(): |
624 | + return -1 |
625 | + if not char.isalpha() and b[i].isalpha(): |
626 | + return 1 |
627 | + # otherwise lexical sort |
628 | + if ord(char) > ord(b[i]): |
629 | + return 1 |
630 | + if ord(char) < ord(b[i]): |
631 | + return -1 |
632 | + except IndexError: |
633 | + # a is longer than b but otherwise equal, greater unless there are tildes |
634 | + if char == "~": |
635 | + return -1 |
636 | + return 1 |
637 | + # if we get here, a is shorter than b but otherwise equal, so check for tildes... |
638 | + if b[len(a)] == "~": |
639 | + return 1 |
640 | + return -1 |
641 | + |
642 | + def _compare_revision_strings(self, first: str, second: str): # noqa: C901 |
643 | + """Compare two debian revision strings.""" |
644 | + if first == second: |
645 | + return 0 |
646 | + |
647 | + # listify pads results so that we will always be comparing ints to ints |
648 | + # and strings to strings (at least until we fall off the end of a list) |
649 | + first_list = self._listify(first) |
650 | + second_list = self._listify(second) |
651 | + if first_list == second_list: |
652 | + return 0 |
653 | + try: |
654 | + for i, item in enumerate(first_list): |
655 | + # explicitly raise IndexError if we've fallen off the edge of list2 |
656 | + if i >= len(second_list): |
657 | + raise IndexError |
658 | + # if the items are equal, next |
659 | + if item == second_list[i]: |
660 | + continue |
661 | + # numeric comparison |
662 | + if isinstance(item, int): |
663 | + if item > second_list[i]: |
664 | + return 1 |
665 | + if item < second_list[i]: |
666 | + return -1 |
667 | + else: |
668 | + # string comparison |
669 | + return self._dstringcmp(item, second_list[i]) |
670 | + except IndexError: |
671 | + # rev1 is longer than rev2 but otherwise equal, hence greater |
672 | + # ...except for goddamn tildes |
673 | + if first_list[len(second_list)][0][0] == "~": |
674 | + return 1 |
675 | + return 1 |
676 | + # rev1 is shorter than rev2 but otherwise equal, hence lesser |
677 | + # ...except for goddamn tildes |
678 | + if second_list[len(first_list)][0][0] == "~": |
679 | + return -1 |
680 | + return -1 |
681 | + |
682 | + def _compare_version(self, other) -> int: |
683 | + if (self.number, self.epoch) == (other.number, other.epoch): |
684 | + return 0 |
685 | + |
686 | + if self.epoch < other.epoch: |
687 | + return -1 |
688 | + if self.epoch > other.epoch: |
689 | + return 1 |
690 | + |
691 | + # If none of these are true, follow the algorithm |
692 | + upstream_version, debian_version = self._get_parts(self.number) |
693 | + other_upstream_version, other_debian_version = self._get_parts(other.number) |
694 | + |
695 | + upstream_cmp = self._compare_revision_strings(upstream_version, other_upstream_version) |
696 | + if upstream_cmp != 0: |
697 | + return upstream_cmp |
698 | + |
699 | + debian_cmp = self._compare_revision_strings(debian_version, other_debian_version) |
700 | + if debian_cmp != 0: |
701 | + return debian_cmp |
702 | + |
703 | + return 0 |
704 | + |
705 | + def __lt__(self, other) -> bool: |
706 | + """Less than magic method impl.""" |
707 | + return self._compare_version(other) < 0 |
708 | + |
709 | + def __eq__(self, other) -> bool: |
710 | + """Equality magic method impl.""" |
711 | + return self._compare_version(other) == 0 |
712 | + |
713 | + def __gt__(self, other) -> bool: |
714 | + """Greater than magic method impl.""" |
715 | + return self._compare_version(other) > 0 |
716 | + |
717 | + def __le__(self, other) -> bool: |
718 | + """Less than or equal to magic method impl.""" |
719 | + return self.__eq__(other) or self.__lt__(other) |
720 | + |
721 | + def __ge__(self, other) -> bool: |
722 | + """Greater than or equal to magic method impl.""" |
723 | + return self.__gt__(other) or self.__eq__(other) |
724 | + |
725 | + def __ne__(self, other) -> bool: |
726 | + """Not equal to magic method impl.""" |
727 | + return not self.__eq__(other) |
728 | + |
729 | + |
730 | +def add_package( |
731 | + package_names: Union[str, List[str]], |
732 | + version: Optional[str] = "", |
733 | + arch: Optional[str] = "", |
734 | + update_cache: Optional[bool] = False, |
735 | +) -> Union[DebianPackage, List[DebianPackage]]: |
736 | + """Add a package or list of packages to the system. |
737 | + |
738 | + Args: |
739 | + name: the name(s) of the package(s) |
740 | + version: an (Optional) version as a string. Defaults to the latest known |
741 | + arch: an optional architecture for the package |
742 | + update_cache: whether or not to run `apt-get update` prior to operating |
743 | + |
744 | + Raises: |
745 | + PackageNotFoundError if the package is not in the cache. |
746 | + """ |
747 | + cache_refreshed = False |
748 | + if update_cache: |
749 | + update() |
750 | + cache_refreshed = True |
751 | + |
752 | + packages = {"success": [], "retry": [], "failed": []} |
753 | + |
754 | + package_names = [package_names] if type(package_names) is str else package_names |
755 | + if not package_names: |
756 | + raise TypeError("Expected at least one package name to add, received zero!") |
757 | + |
758 | + if len(package_names) != 1 and version: |
759 | + raise TypeError( |
760 | + "Explicit version should not be set if more than one package is being added!" |
761 | + ) |
762 | + |
763 | + for p in package_names: |
764 | + pkg, success = _add(p, version, arch) |
765 | + if success: |
766 | + packages["success"].append(pkg) |
767 | + else: |
768 | + logger.warning("failed to locate and install/update '%s'", pkg) |
769 | + packages["retry"].append(p) |
770 | + |
771 | + if packages["retry"] and not cache_refreshed: |
772 | + logger.info("updating the apt-cache and retrying installation of failed packages.") |
773 | + update() |
774 | + |
775 | + for p in packages["retry"]: |
776 | + pkg, success = _add(p, version, arch) |
777 | + if success: |
778 | + packages["success"].append(pkg) |
779 | + else: |
780 | + packages["failed"].append(p) |
781 | + |
782 | + if packages["failed"]: |
783 | + raise PackageError("Failed to install packages: {}".format(", ".join(packages["failed"]))) |
784 | + |
785 | + return packages["success"] if len(packages["success"]) > 1 else packages["success"][0] |
786 | + |
787 | + |
788 | +def _add( |
789 | + name: str, |
790 | + version: Optional[str] = "", |
791 | + arch: Optional[str] = "", |
792 | +) -> Tuple[Union[DebianPackage, str], bool]: |
793 | + """Adds a package. |
794 | + |
795 | + Args: |
796 | + name: the name(s) of the package(s) |
797 | + version: an (Optional) version as a string. Defaults to the latest known |
798 | + arch: an optional architecture for the package |
799 | + |
800 | + Returns: a tuple of `DebianPackage` if found, or a :str: if it is not, and |
801 | + a boolean indicating success |
802 | + """ |
803 | + try: |
804 | + pkg = DebianPackage.from_system(name, version, arch) |
805 | + pkg.ensure(state=PackageState.Present) |
806 | + return pkg, True |
807 | + except PackageNotFoundError: |
808 | + return name, False |
809 | + |
810 | + |
811 | +def remove_package( |
812 | + package_names: Union[str, List[str]] |
813 | +) -> Union[DebianPackage, List[DebianPackage]]: |
814 | + """Removes a package from the system. |
815 | + |
816 | + Args: |
817 | + package_names: the name of a package |
818 | + |
819 | + Raises: |
820 | + PackageNotFoundError if the package is not found. |
821 | + """ |
822 | + packages = [] |
823 | + |
824 | + package_names = [package_names] if type(package_names) is str else package_names |
825 | + if not package_names: |
826 | + raise TypeError("Expected at least one package name to add, received zero!") |
827 | + |
828 | + for p in package_names: |
829 | + try: |
830 | + pkg = DebianPackage.from_installed_package(p) |
831 | + pkg.ensure(state=PackageState.Absent) |
832 | + packages.append(pkg) |
833 | + except PackageNotFoundError: |
834 | + logger.info("package '%s' was requested for removal, but it was not installed.", p) |
835 | + |
836 | + # the list of packages will be empty when no package is removed |
837 | + logger.debug("packages: '%s'", packages) |
838 | + return packages[0] if len(packages) == 1 else packages |
839 | + |
840 | + |
841 | +def update() -> None: |
842 | + """Updates the apt cache via `apt-get update`.""" |
843 | + check_call(["apt-get", "update"], stderr=PIPE, stdout=PIPE) |
844 | + |
845 | + |
846 | +class InvalidSourceError(Error): |
847 | + """Exceptions for invalid source entries.""" |
848 | + |
849 | + |
850 | +class GPGKeyError(Error): |
851 | + """Exceptions for GPG keys.""" |
852 | + |
853 | + |
854 | +class DebianRepository: |
855 | + """An abstraction to represent a repository.""" |
856 | + |
857 | + def __init__( |
858 | + self, |
859 | + enabled: bool, |
860 | + repotype: str, |
861 | + uri: str, |
862 | + release: str, |
863 | + groups: List[str], |
864 | + filename: Optional[str] = "", |
865 | + gpg_key_filename: Optional[str] = "", |
866 | + options: Optional[dict] = None, |
867 | + ): |
868 | + self._enabled = enabled |
869 | + self._repotype = repotype |
870 | + self._uri = uri |
871 | + self._release = release |
872 | + self._groups = groups |
873 | + self._filename = filename |
874 | + self._gpg_key_filename = gpg_key_filename |
875 | + self._options = options |
876 | + |
877 | + @property |
878 | + def enabled(self): |
879 | + """Return whether or not the repository is enabled.""" |
880 | + return self._enabled |
881 | + |
882 | + @property |
883 | + def repotype(self): |
884 | + """Return whether it is binary or source.""" |
885 | + return self._repotype |
886 | + |
887 | + @property |
888 | + def uri(self): |
889 | + """Return the URI.""" |
890 | + return self._uri |
891 | + |
892 | + @property |
893 | + def release(self): |
894 | + """Return which Debian/Ubuntu releases it is valid for.""" |
895 | + return self._release |
896 | + |
897 | + @property |
898 | + def groups(self): |
899 | + """Return the enabled package groups.""" |
900 | + return self._groups |
901 | + |
902 | + @property |
903 | + def filename(self): |
904 | + """Returns the filename for a repository.""" |
905 | + return self._filename |
906 | + |
907 | + @filename.setter |
908 | + def filename(self, fname: str) -> None: |
909 | + """Sets the filename used when a repo is written back to diskself. |
910 | + |
911 | + Args: |
912 | + fname: a filename to write the repository information to. |
913 | + """ |
914 | + if not fname.endswith(".list"): |
915 | + raise InvalidSourceError("apt source filenames should end in .list!") |
916 | + |
917 | + self._filename = fname |
918 | + |
919 | + @property |
920 | + def gpg_key(self): |
921 | + """Returns the path to the GPG key for this repository.""" |
922 | + return self._gpg_key_filename |
923 | + |
924 | + @property |
925 | + def options(self): |
926 | + """Returns any additional repo options which are set.""" |
927 | + return self._options |
928 | + |
929 | + def make_options_string(self) -> str: |
930 | + """Generate the complete options string for a a repository. |
931 | + |
932 | + Combining `gpg_key`, if set, and the rest of the options to find |
933 | + a complex repo string. |
934 | + """ |
935 | + options = self._options if self._options else {} |
936 | + if self._gpg_key_filename: |
937 | + options["signed-by"] = self._gpg_key_filename |
938 | + |
939 | + return ( |
940 | + "[{}] ".format(" ".join(["{}={}".format(k, v) for k, v in options.items()])) |
941 | + if options |
942 | + else "" |
943 | + ) |
944 | + |
945 | + @staticmethod |
946 | + def prefix_from_uri(uri: str) -> str: |
947 | + """Get a repo list prefix from the uri, depending on whether a path is set.""" |
948 | + uridetails = urlparse(uri) |
949 | + path = ( |
950 | + uridetails.path.lstrip("/").replace("/", "-") if uridetails.path else uridetails.netloc |
951 | + ) |
952 | + return "/etc/apt/sources.list.d/{}".format(path) |
953 | + |
954 | + @staticmethod |
955 | + def from_repo_line(repo_line: str, write_file: Optional[bool] = True) -> "DebianRepository": |
956 | + """Instantiate a new `DebianRepository` a `sources.list` entry line. |
957 | + |
958 | + Args: |
959 | + repo_line: a string representing a repository entry |
960 | + write_file: boolean to enable writing the new repo to disk |
961 | + """ |
962 | + repo = RepositoryMapping._parse(repo_line, "UserInput") |
963 | + fname = "{}-{}.list".format( |
964 | + DebianRepository.prefix_from_uri(repo.uri), repo.release.replace("/", "-") |
965 | + ) |
966 | + repo.filename = fname |
967 | + |
968 | + options = repo.options if repo.options else {} |
969 | + if repo.gpg_key: |
970 | + options["signed-by"] = repo.gpg_key |
971 | + |
972 | + # For Python 3.5 it's required to use sorted in the options dict in order to not have |
973 | + # different results in the order of the options between executions. |
974 | + options_str = ( |
975 | + "[{}] ".format(" ".join(["{}={}".format(k, v) for k, v in sorted(options.items())])) |
976 | + if options |
977 | + else "" |
978 | + ) |
979 | + |
980 | + if write_file: |
981 | + with open(fname, "wb") as f: |
982 | + f.write( |
983 | + ( |
984 | + "{}".format("#" if not repo.enabled else "") |
985 | + + "{} {}{} ".format(repo.repotype, options_str, repo.uri) |
986 | + + "{} {}\n".format(repo.release, " ".join(repo.groups)) |
987 | + ).encode("utf-8") |
988 | + ) |
989 | + |
990 | + return repo |
991 | + |
992 | + def disable(self) -> None: |
993 | + """Remove this repository from consideration. |
994 | + |
995 | + Disable it instead of removing from the repository file. |
996 | + """ |
997 | + searcher = "{} {}{} {}".format( |
998 | + self.repotype, self.make_options_string(), self.uri, self.release |
999 | + ) |
1000 | + for line in fileinput.input(self._filename, inplace=True): |
1001 | + if re.match(r"^{}\s".format(re.escape(searcher)), line): |
1002 | + print("# {}".format(line), end="") |
1003 | + else: |
1004 | + print(line, end="") |
1005 | + |
1006 | + def import_key(self, key: str) -> None: |
1007 | + """Import an ASCII Armor key. |
1008 | + |
1009 | + A Radix64 format keyid is also supported for backwards |
1010 | + compatibility. In this case Ubuntu keyserver will be |
1011 | + queried for a key via HTTPS by its keyid. This method |
1012 | + is less preferrable because https proxy servers may |
1013 | + require traffic decryption which is equivalent to a |
1014 | + man-in-the-middle attack (a proxy server impersonates |
1015 | + keyserver TLS certificates and has to be explicitly |
1016 | + trusted by the system). |
1017 | + |
1018 | + Args: |
1019 | + key: A GPG key in ASCII armor format, |
1020 | + including BEGIN and END markers or a keyid. |
1021 | + |
1022 | + Raises: |
1023 | + GPGKeyError if the key could not be imported |
1024 | + """ |
1025 | + key = key.strip() |
1026 | + if "-" in key or "\n" in key: |
1027 | + # Send everything not obviously a keyid to GPG to import, as |
1028 | + # we trust its validation better than our own. eg. handling |
1029 | + # comments before the key. |
1030 | + logger.debug("PGP key found (looks like ASCII Armor format)") |
1031 | + if ( |
1032 | + "-----BEGIN PGP PUBLIC KEY BLOCK-----" in key |
1033 | + and "-----END PGP PUBLIC KEY BLOCK-----" in key |
1034 | + ): |
1035 | + logger.debug("Writing provided PGP key in the binary format") |
1036 | + key_bytes = key.encode("utf-8") |
1037 | + key_name = self._get_keyid_by_gpg_key(key_bytes) |
1038 | + key_gpg = self._dearmor_gpg_key(key_bytes) |
1039 | + self._gpg_key_filename = "/etc/apt/trusted.gpg.d/{}.gpg".format(key_name) |
1040 | + self._write_apt_gpg_keyfile(key_name=self._gpg_key_filename, key_material=key_gpg) |
1041 | + else: |
1042 | + raise GPGKeyError("ASCII armor markers missing from GPG key") |
1043 | + else: |
1044 | + logger.warning( |
1045 | + "PGP key found (looks like Radix64 format). " |
1046 | + "SECURELY importing PGP key from keyserver; " |
1047 | + "full key not provided." |
1048 | + ) |
1049 | + # as of bionic add-apt-repository uses curl with an HTTPS keyserver URL |
1050 | + # to retrieve GPG keys. `apt-key adv` command is deprecated as is |
1051 | + # apt-key in general as noted in its manpage. See lp:1433761 for more |
1052 | + # history. Instead, /etc/apt/trusted.gpg.d is used directly to drop |
1053 | + # gpg |
1054 | + key_asc = self._get_key_by_keyid(key) |
1055 | + # write the key in GPG format so that apt-key list shows it |
1056 | + key_gpg = self._dearmor_gpg_key(key_asc.encode("utf-8")) |
1057 | + self._gpg_key_filename = "/etc/apt/trusted.gpg.d/{}.gpg".format(key) |
1058 | + self._write_apt_gpg_keyfile(key_name=key, key_material=key_gpg) |
1059 | + |
1060 | + @staticmethod |
1061 | + def _get_keyid_by_gpg_key(key_material: bytes) -> str: |
1062 | + """Get a GPG key fingerprint by GPG key material. |
1063 | + |
1064 | + Gets a GPG key fingerprint (40-digit, 160-bit) by the ASCII armor-encoded |
1065 | + or binary GPG key material. Can be used, for example, to generate file |
1066 | + names for keys passed via charm options. |
1067 | + """ |
1068 | + # Use the same gpg command for both Xenial and Bionic |
1069 | + cmd = ["gpg", "--with-colons", "--with-fingerprint"] |
1070 | + ps = subprocess.run( |
1071 | + cmd, |
1072 | + stdout=PIPE, |
1073 | + stderr=PIPE, |
1074 | + input=key_material, |
1075 | + ) |
1076 | + out, err = ps.stdout.decode(), ps.stderr.decode() |
1077 | + if "gpg: no valid OpenPGP data found." in err: |
1078 | + raise GPGKeyError("Invalid GPG key material provided") |
1079 | + # from gnupg2 docs: fpr :: Fingerprint (fingerprint is in field 10) |
1080 | + return re.search(r"^fpr:{9}([0-9A-F]{40}):$", out, re.MULTILINE).group(1) |
1081 | + |
1082 | + @staticmethod |
1083 | + def _get_key_by_keyid(keyid: str) -> str: |
1084 | + """Get a key via HTTPS from the Ubuntu keyserver. |
1085 | + |
1086 | + Different key ID formats are supported by SKS keyservers (the longer ones |
1087 | + are more secure, see "dead beef attack" and https://evil32.com/). Since |
1088 | + HTTPS is used, if SSLBump-like HTTPS proxies are in place, they will |
1089 | + impersonate keyserver.ubuntu.com and generate a certificate with |
1090 | + keyserver.ubuntu.com in the CN field or in SubjAltName fields of a |
1091 | + certificate. If such proxy behavior is expected it is necessary to add the |
1092 | + CA certificate chain containing the intermediate CA of the SSLBump proxy to |
1093 | + every machine that this code runs on via ca-certs cloud-init directive (via |
1094 | + cloudinit-userdata model-config) or via other means (such as through a |
1095 | + custom charm option). Also note that DNS resolution for the hostname in a |
1096 | + URL is done at a proxy server - not at the client side. |
1097 | + 8-digit (32 bit) key ID |
1098 | + https://keyserver.ubuntu.com/pks/lookup?search=0x4652B4E6 |
1099 | + 16-digit (64 bit) key ID |
1100 | + https://keyserver.ubuntu.com/pks/lookup?search=0x6E85A86E4652B4E6 |
1101 | + 40-digit key ID: |
1102 | + https://keyserver.ubuntu.com/pks/lookup?search=0x35F77D63B5CEC106C577ED856E85A86E4652B4E6 |
1103 | + |
1104 | + Args: |
1105 | + keyid: An 8, 16 or 40 hex digit keyid to find a key for |
1106 | + |
1107 | + Returns: |
1108 | + A string contining key material for the specified GPG key id |
1109 | + |
1110 | + |
1111 | + Raises: |
1112 | + subprocess.CalledProcessError |
1113 | + """ |
1114 | + # options=mr - machine-readable output (disables html wrappers) |
1115 | + keyserver_url = ( |
1116 | + "https://keyserver.ubuntu.com" "/pks/lookup?op=get&options=mr&exact=on&search=0x{}" |
1117 | + ) |
1118 | + curl_cmd = ["curl", keyserver_url.format(keyid)] |
1119 | + # use proxy server settings in order to retrieve the key |
1120 | + return check_output(curl_cmd).decode() |
1121 | + |
1122 | + @staticmethod |
1123 | + def _dearmor_gpg_key(key_asc: bytes) -> bytes: |
1124 | + """Converts a GPG key in the ASCII armor format to the binary format. |
1125 | + |
1126 | + Args: |
1127 | + key_asc: A GPG key in ASCII armor format. |
1128 | + |
1129 | + Returns: |
1130 | + A GPG key in binary format as a string |
1131 | + |
1132 | + Raises: |
1133 | + GPGKeyError |
1134 | + """ |
1135 | + ps = subprocess.run(["gpg", "--dearmor"], stdout=PIPE, stderr=PIPE, input=key_asc) |
1136 | + out, err = ps.stdout, ps.stderr.decode() |
1137 | + if "gpg: no valid OpenPGP data found." in err: |
1138 | + raise GPGKeyError( |
1139 | + "Invalid GPG key material. Check your network setup" |
1140 | + " (MTU, routing, DNS) and/or proxy server settings" |
1141 | + " as well as destination keyserver status." |
1142 | + ) |
1143 | + else: |
1144 | + return out |
1145 | + |
1146 | + @staticmethod |
1147 | + def _write_apt_gpg_keyfile(key_name: str, key_material: bytes) -> None: |
1148 | + """Writes GPG key material into a file at a provided path. |
1149 | + |
1150 | + Args: |
1151 | + key_name: A key name to use for a key file (could be a fingerprint) |
1152 | + key_material: A GPG key material (binary) |
1153 | + """ |
1154 | + with open(key_name, "wb") as keyf: |
1155 | + keyf.write(key_material) |
1156 | + |
1157 | + |
1158 | +class RepositoryMapping(Mapping): |
1159 | + """An representation of known repositories. |
1160 | + |
1161 | + Instantiation of `RepositoryMapping` will iterate through the |
1162 | + filesystem, parse out repository files in `/etc/apt/...`, and create |
1163 | + `DebianRepository` objects in this list. |
1164 | + |
1165 | + Typical usage: |
1166 | + |
1167 | + repositories = apt.RepositoryMapping() |
1168 | + repositories.add(DebianRepository( |
1169 | + enabled=True, repotype="deb", uri="https://example.com", release="focal", |
1170 | + groups=["universe"] |
1171 | + )) |
1172 | + """ |
1173 | + |
1174 | + def __init__(self): |
1175 | + self._repository_map = {} |
1176 | + # Repositories that we're adding -- used to implement mode param |
1177 | + self.default_file = "/etc/apt/sources.list" |
1178 | + |
1179 | + # read sources.list if it exists |
1180 | + if os.path.isfile(self.default_file): |
1181 | + self.load(self.default_file) |
1182 | + |
1183 | + # read sources.list.d |
1184 | + for file in glob.iglob("/etc/apt/sources.list.d/*.list"): |
1185 | + self.load(file) |
1186 | + |
1187 | + def __contains__(self, key: str) -> bool: |
1188 | + """Magic method for checking presence of repo in mapping.""" |
1189 | + return key in self._repository_map |
1190 | + |
1191 | + def __len__(self) -> int: |
1192 | + """Return number of repositories in map.""" |
1193 | + return len(self._repository_map) |
1194 | + |
1195 | + def __iter__(self) -> Iterable[DebianRepository]: |
1196 | + """Iterator magic method for RepositoryMapping.""" |
1197 | + return iter(self._repository_map.values()) |
1198 | + |
1199 | + def __getitem__(self, repository_uri: str) -> DebianRepository: |
1200 | + """Return a given `DebianRepository`.""" |
1201 | + return self._repository_map[repository_uri] |
1202 | + |
1203 | + def __setitem__(self, repository_uri: str, repository: DebianRepository) -> None: |
1204 | + """Add a `DebianRepository` to the cache.""" |
1205 | + self._repository_map[repository_uri] = repository |
1206 | + |
1207 | + def load(self, filename: str): |
1208 | + """Load a repository source file into the cache. |
1209 | + |
1210 | + Args: |
1211 | + filename: the path to the repository file |
1212 | + """ |
1213 | + parsed = [] |
1214 | + skipped = [] |
1215 | + with open(filename, "r") as f: |
1216 | + for n, line in enumerate(f): |
1217 | + try: |
1218 | + repo = self._parse(line, filename) |
1219 | + except InvalidSourceError: |
1220 | + skipped.append(n) |
1221 | + else: |
1222 | + repo_identifier = "{}-{}-{}".format(repo.repotype, repo.uri, repo.release) |
1223 | + self._repository_map[repo_identifier] = repo |
1224 | + parsed.append(n) |
1225 | + logger.debug("parsed repo: '%s'", repo_identifier) |
1226 | + |
1227 | + if skipped: |
1228 | + skip_list = ", ".join(str(s) for s in skipped) |
1229 | + logger.debug("skipped the following lines in file '%s': %s", filename, skip_list) |
1230 | + |
1231 | + if parsed: |
1232 | + logger.info("parsed %d apt package repositories", len(parsed)) |
1233 | + else: |
1234 | + raise InvalidSourceError("all repository lines in '{}' were invalid!".format(filename)) |
1235 | + |
1236 | + @staticmethod |
1237 | + def _parse(line: str, filename: str) -> DebianRepository: |
1238 | + """Parse a line in a sources.list file. |
1239 | + |
1240 | + Args: |
1241 | + line: a single line from `load` to parse |
1242 | + filename: the filename being read |
1243 | + |
1244 | + Raises: |
1245 | + InvalidSourceError if the source type is unknown |
1246 | + """ |
1247 | + enabled = True |
1248 | + repotype = uri = release = gpg_key = "" |
1249 | + options = {} |
1250 | + groups = [] |
1251 | + |
1252 | + line = line.strip() |
1253 | + if line.startswith("#"): |
1254 | + enabled = False |
1255 | + line = line[1:] |
1256 | + |
1257 | + # Check for "#" in the line and treat a part after it as a comment then strip it off. |
1258 | + i = line.find("#") |
1259 | + if i > 0: |
1260 | + line = line[:i] |
1261 | + |
1262 | + # Split a source into substrings to initialize a new repo. |
1263 | + source = line.strip() |
1264 | + if source: |
1265 | + # Match any repo options, and get a dict representation. |
1266 | + for v in re.findall(OPTIONS_MATCHER, source): |
1267 | + opts = dict(o.split("=") for o in v.strip("[]").split()) |
1268 | + # Extract the 'signed-by' option for the gpg_key |
1269 | + gpg_key = opts.pop("signed-by", "") |
1270 | + options = opts |
1271 | + |
1272 | + # Remove any options from the source string and split the string into chunks |
1273 | + source = re.sub(OPTIONS_MATCHER, "", source) |
1274 | + chunks = source.split() |
1275 | + |
1276 | + # Check we've got a valid list of chunks |
1277 | + if len(chunks) < 3 or chunks[0] not in VALID_SOURCE_TYPES: |
1278 | + raise InvalidSourceError("An invalid sources line was found in %s!", filename) |
1279 | + |
1280 | + repotype = chunks[0] |
1281 | + uri = chunks[1] |
1282 | + release = chunks[2] |
1283 | + groups = chunks[3:] |
1284 | + |
1285 | + return DebianRepository( |
1286 | + enabled, repotype, uri, release, groups, filename, gpg_key, options |
1287 | + ) |
1288 | + else: |
1289 | + raise InvalidSourceError("An invalid sources line was found in %s!", filename) |
1290 | + |
1291 | + def add(self, repo: DebianRepository, default_filename: Optional[bool] = False) -> None: |
1292 | + """Add a new repository to the system. |
1293 | + |
1294 | + Args: |
1295 | + repo: a `DebianRepository` object |
1296 | + default_filename: an (Optional) filename if the default is not desirable |
1297 | + """ |
1298 | + new_filename = "{}-{}.list".format( |
1299 | + DebianRepository.prefix_from_uri(repo.uri), repo.release.replace("/", "-") |
1300 | + ) |
1301 | + |
1302 | + fname = repo.filename or new_filename |
1303 | + |
1304 | + options = repo.options if repo.options else {} |
1305 | + if repo.gpg_key: |
1306 | + options["signed-by"] = repo.gpg_key |
1307 | + |
1308 | + with open(fname, "wb") as f: |
1309 | + f.write( |
1310 | + ( |
1311 | + "{}".format("#" if not repo.enabled else "") |
1312 | + + "{} {}{} ".format(repo.repotype, repo.make_options_string(), repo.uri) |
1313 | + + "{} {}\n".format(repo.release, " ".join(repo.groups)) |
1314 | + ).encode("utf-8") |
1315 | + ) |
1316 | + |
1317 | + self._repository_map["{}-{}-{}".format(repo.repotype, repo.uri, repo.release)] = repo |
1318 | + |
1319 | + def disable(self, repo: DebianRepository) -> None: |
1320 | + """Remove a repository. Disable by default. |
1321 | + |
1322 | + Args: |
1323 | + repo: a `DebianRepository` to disable |
1324 | + """ |
1325 | + searcher = "{} {}{} {}".format( |
1326 | + repo.repotype, repo.make_options_string(), repo.uri, repo.release |
1327 | + ) |
1328 | + |
1329 | + for line in fileinput.input(repo.filename, inplace=True): |
1330 | + if re.match(r"^{}\s".format(re.escape(searcher)), line): |
1331 | + print("# {}".format(line), end="") |
1332 | + else: |
1333 | + print(line, end="") |
1334 | + |
1335 | + self._repository_map["{}-{}-{}".format(repo.repotype, repo.uri, repo.release)] = repo |
1336 | diff --git a/run_tests b/run_tests |
1337 | index 362a995..ef9f058 100755 |
1338 | --- a/run_tests |
1339 | +++ b/run_tests |
1340 | @@ -1,17 +1,18 @@ |
1341 | #!/bin/sh -e |
1342 | # Copyright 2021 Canonical Ltd. |
1343 | # See LICENSE file for licensing details. |
1344 | +PYTHONPATH_INCLUDES="src:lib" |
1345 | |
1346 | if [ -z "$VIRTUAL_ENV" -a -d venv/ ]; then |
1347 | . venv/bin/activate |
1348 | fi |
1349 | |
1350 | if [ -z "$PYTHONPATH" ]; then |
1351 | - export PYTHONPATH=src |
1352 | + export PYTHONPATH="$PYTHONPATH_INCLUDES" |
1353 | else |
1354 | - export PYTHONPATH="src:$PYTHONPATH" |
1355 | + export PYTHONPATH="$PYTHONPATH_INCLUDES:$PYTHONPATH" |
1356 | fi |
1357 | |
1358 | -flake8 |
1359 | +flake8 --extend-exclude operator_libs_linux |
1360 | coverage run --source=src -m unittest -v "$@" |
1361 | coverage report -m |
1362 | diff --git a/src/charm.py b/src/charm.py |
1363 | index 34878c0..84362d3 100755 |
1364 | --- a/src/charm.py |
1365 | +++ b/src/charm.py |
1366 | @@ -10,6 +10,7 @@ import os |
1367 | import subprocess |
1368 | import yaml |
1369 | |
1370 | +from charms.operator_libs_linux.v0 import apt |
1371 | from ops.charm import CharmBase |
1372 | from ops.framework import StoredState |
1373 | from ops.main import main |
1374 | @@ -29,17 +30,6 @@ def remove_ppa(ppa, env): |
1375 | subprocess.check_call(["add-apt-repository", "--remove", "--yes", ppa], env=env) |
1376 | |
1377 | |
1378 | -def install_package(package): |
1379 | - """Install specified apt package (after performing an apt update)""" |
1380 | - subprocess.check_call(["apt", "update"]) |
1381 | - subprocess.check_call(["apt", "install", "--yes", "--quiet", package]) |
1382 | - |
1383 | - |
1384 | -def remove_package(package): |
1385 | - """Remove specified apt package""" |
1386 | - subprocess.check_call(["apt", "remove", "--yes", "--quiet", package]) |
1387 | - |
1388 | - |
1389 | def update_configuration(contract_url): |
1390 | """Write the contract_url to the uaclient configuration file""" |
1391 | with open("/etc/ubuntu-advantage/uaclient.conf", "r+") as f: |
1392 | @@ -137,9 +127,9 @@ class UbuntuAdvantageCharm(CharmBase): |
1393 | """Install apt package if necessary""" |
1394 | if self._state.package_needs_installing: |
1395 | logger.info("Removing package ubuntu-advantage-tools") |
1396 | - remove_package("ubuntu-advantage-tools") |
1397 | + apt.remove_package("ubuntu-advantage-tools") |
1398 | logger.info("Installing package ubuntu-advantage-tools") |
1399 | - install_package("ubuntu-advantage-tools") |
1400 | + apt.add_package("ubuntu-advantage-tools", update_cache=True) |
1401 | self._state.package_needs_installing = False |
1402 | |
1403 | def _handle_subscription_state(self): |
1404 | diff --git a/tests/test_charm.py b/tests/test_charm.py |
1405 | index f55d844..37da2a7 100644 |
1406 | --- a/tests/test_charm.py |
1407 | +++ b/tests/test_charm.py |
1408 | @@ -80,7 +80,8 @@ class TestCharm(TestCase): |
1409 | "check_call": patch("subprocess.check_call").start(), |
1410 | "check_output": patch("subprocess.check_output").start(), |
1411 | "open": patch("builtins.open").start(), |
1412 | - "environ": patch.dict("os.environ", clear=True).start() |
1413 | + "environ": patch.dict("os.environ", clear=True).start(), |
1414 | + "apt": patch("charm.apt").start() |
1415 | } |
1416 | self.mocks["check_output"].side_effect = [ |
1417 | STATUS_DETACHED |
1418 | @@ -100,25 +101,21 @@ class TestCharm(TestCase): |
1419 | |
1420 | def test_config_changed_ppa_new(self): |
1421 | self.harness.update_config({"ppa": "ppa:ua-client/stable"}) |
1422 | - self.assertEqual(self.mocks["check_call"].call_count, 6) |
1423 | + self.assertEqual(self.mocks["check_call"].call_count, 3) |
1424 | self.mocks["check_call"].assert_has_calls(self._add_ua_proxy_setup_calls([ |
1425 | call(["add-apt-repository", "--yes", "ppa:ua-client/stable"], env=self.env), |
1426 | - call(["apt", "remove", "--yes", "--quiet", "ubuntu-advantage-tools"]), |
1427 | - call(["apt", "update"]), |
1428 | - call(["apt", "install", "--yes", "--quiet", "ubuntu-advantage-tools"]) |
1429 | ])) |
1430 | + self._assert_apt_calls() |
1431 | self.assertEqual(self.harness.charm._state.ppa, "ppa:ua-client/stable") |
1432 | self.assertFalse(self.harness.charm._state.package_needs_installing) |
1433 | |
1434 | def test_config_changed_ppa_updated(self): |
1435 | self.harness.update_config({"ppa": "ppa:ua-client/stable"}) |
1436 | - self.assertEqual(self.mocks["check_call"].call_count, 6) |
1437 | + self.assertEqual(self.mocks["check_call"].call_count, 3) |
1438 | self.mocks["check_call"].assert_has_calls(self._add_ua_proxy_setup_calls([ |
1439 | call(["add-apt-repository", "--yes", "ppa:ua-client/stable"], env=self.env), |
1440 | - call(["apt", "remove", "--yes", "--quiet", "ubuntu-advantage-tools"]), |
1441 | - call(["apt", "update"]), |
1442 | - call(["apt", "install", "--yes", "--quiet", "ubuntu-advantage-tools"]) |
1443 | ])) |
1444 | + self._assert_apt_calls() |
1445 | self.assertEqual(self.harness.charm._state.ppa, "ppa:ua-client/stable") |
1446 | self.assertFalse(self.harness.charm._state.package_needs_installing) |
1447 | |
1448 | @@ -126,29 +123,26 @@ class TestCharm(TestCase): |
1449 | STATUS_DETACHED |
1450 | ] |
1451 | self.mocks["check_call"].reset_mock() |
1452 | + self.mocks["apt"].reset_mock() |
1453 | self.harness.update_config({"ppa": "ppa:different-client/unstable"}) |
1454 | - self.assertEqual(self.mocks["check_call"].call_count, 7) |
1455 | + self.assertEqual(self.mocks["check_call"].call_count, 4) |
1456 | self.mocks["check_call"].assert_has_calls(self._add_ua_proxy_setup_calls([ |
1457 | call(["add-apt-repository", "--remove", "--yes", "ppa:ua-client/stable"], |
1458 | env=self.env), |
1459 | call(["add-apt-repository", "--yes", "ppa:different-client/unstable"], |
1460 | env=self.env), |
1461 | - call(["apt", "remove", "--yes", "--quiet", "ubuntu-advantage-tools"]), |
1462 | - call(["apt", "update"]), |
1463 | - call(["apt", "install", "--yes", "--quiet", "ubuntu-advantage-tools"]) |
1464 | ])) |
1465 | + self._assert_apt_calls() |
1466 | self.assertEqual(self.harness.charm._state.ppa, "ppa:different-client/unstable") |
1467 | self.assertFalse(self.harness.charm._state.package_needs_installing) |
1468 | |
1469 | def test_config_changed_ppa_unmodified(self): |
1470 | self.harness.update_config({"ppa": "ppa:ua-client/stable"}) |
1471 | - self.assertEqual(self.mocks["check_call"].call_count, 6) |
1472 | + self.assertEqual(self.mocks["check_call"].call_count, 3) |
1473 | self.mocks["check_call"].assert_has_calls(self._add_ua_proxy_setup_calls([ |
1474 | call(["add-apt-repository", "--yes", "ppa:ua-client/stable"], env=self.env), |
1475 | - call(["apt", "remove", "--yes", "--quiet", "ubuntu-advantage-tools"]), |
1476 | - call(["apt", "update"]), |
1477 | - call(["apt", "install", "--yes", "--quiet", "ubuntu-advantage-tools"]) |
1478 | ])) |
1479 | + self._assert_apt_calls() |
1480 | self.assertEqual(self.harness.charm._state.ppa, "ppa:ua-client/stable") |
1481 | self.assertFalse(self.harness.charm._state.package_needs_installing) |
1482 | |
1483 | @@ -165,30 +159,27 @@ class TestCharm(TestCase): |
1484 | |
1485 | def test_config_changed_ppa_unset(self): |
1486 | self.harness.update_config({"ppa": "ppa:ua-client/stable"}) |
1487 | - self.assertEqual(self.mocks["check_call"].call_count, 6) |
1488 | + self.assertEqual(self.mocks["check_call"].call_count, 3) |
1489 | self.mocks["check_call"].assert_has_calls(self._add_ua_proxy_setup_calls([ |
1490 | call(["add-apt-repository", "--yes", "ppa:ua-client/stable"], env=self.env), |
1491 | - call(["apt", "remove", "--yes", "--quiet", "ubuntu-advantage-tools"]), |
1492 | - call(["apt", "update"]), |
1493 | - call(["apt", "install", "--yes", "--quiet", "ubuntu-advantage-tools"]) |
1494 | ])) |
1495 | + self._assert_apt_calls() |
1496 | self.assertEqual(self.harness.charm._state.ppa, "ppa:ua-client/stable") |
1497 | self.assertFalse(self.harness.charm._state.package_needs_installing) |
1498 | |
1499 | self.mocks["check_call"].reset_mock() |
1500 | self.mocks["check_output"].reset_mock() |
1501 | + self.mocks["apt"].reset_mock() |
1502 | self.mocks["check_output"].side_effect = [ |
1503 | STATUS_DETACHED |
1504 | ] |
1505 | self.harness.update_config({"ppa": ""}) |
1506 | - self.assertEqual(self.mocks["check_call"].call_count, 6) |
1507 | + self.assertEqual(self.mocks["check_call"].call_count, 3) |
1508 | self.mocks["check_call"].assert_has_calls(self._add_ua_proxy_setup_calls([ |
1509 | call(["add-apt-repository", "--remove", "--yes", "ppa:ua-client/stable"], |
1510 | env=self.env), |
1511 | - call(["apt", "remove", "--yes", "--quiet", "ubuntu-advantage-tools"]), |
1512 | - call(["apt", "update"]), |
1513 | - call(["apt", "install", "--yes", "--quiet", "ubuntu-advantage-tools"]) |
1514 | ])) |
1515 | + self._assert_apt_calls() |
1516 | self.assertIsNone(self.harness.charm._state.ppa) |
1517 | self.assertFalse(self.harness.charm._state.package_needs_installing) |
1518 | |
1519 | @@ -232,14 +223,12 @@ class TestCharm(TestCase): |
1520 | STATUS_ATTACHED |
1521 | ] |
1522 | self.harness.update_config({"token": "test-token"}) |
1523 | - self.assertEqual(self.mocks["check_call"].call_count, 6) |
1524 | + self.assertEqual(self.mocks["check_call"].call_count, 3) |
1525 | self.mocks["check_call"].assert_has_calls(self._add_ua_proxy_setup_calls([ |
1526 | call(["add-apt-repository", "--yes", "ppa:ua-client/stable"], env=self.env), |
1527 | - call(["apt", "remove", "--yes", "--quiet", "ubuntu-advantage-tools"]), |
1528 | - call(["apt", "update"]), |
1529 | - call(["apt", "install", "--yes", "--quiet", "ubuntu-advantage-tools"]) |
1530 | ])) |
1531 | self.mocks["open"].assert_called_with("/etc/ubuntu-advantage/uaclient.conf", "r+") |
1532 | + self._assert_apt_calls() |
1533 | handle = self.mocks["open"]() |
1534 | expected = dedent("""\ |
1535 | contract_url: https://contracts.canonical.com |
1536 | @@ -505,3 +494,12 @@ class TestCharm(TestCase): |
1537 | ), |
1538 | ] |
1539 | return call_list + proxy_calls if append else proxy_calls + call_list |
1540 | + |
1541 | + def _assert_apt_calls(self): |
1542 | + """Helper to run the assertions for apt install/remove""" |
1543 | + self.mocks["apt"].remove_package.assert_called_once_with( |
1544 | + "ubuntu-advantage-tools" |
1545 | + ) |
1546 | + self.mocks["apt"].add_package.assert_called_once_with( |
1547 | + "ubuntu-advantage-tools", update_cache=True |
1548 | + ) |
This merge proposal is being monitored by mergebot. Change the status to Approved to merge.