Merge ~przemeklal/charm-juju-ro:initial-code-drop into charm-juju-ro:main

Proposed by Przemyslaw Lal
Status: Merged
Merged at revision: bbb2b7495eb36ec7f44e6d8b51933f968d5be742
Proposed branch: ~przemeklal/charm-juju-ro:initial-code-drop
Merge into: charm-juju-ro:main
Diff against target: 1364 lines (+1242/-0)
20 files modified
.gitignore (+11/-0)
.jujuignore (+3/-0)
Makefile (+97/-0)
README.md (+49/-0)
actions.yaml (+12/-0)
charmcraft.yaml (+14/-0)
config.yaml (+19/-0)
lib/charms/juju_ro/v0/lib_juju_addons.py (+535/-0)
metadata.yaml (+13/-0)
rename.sh (+13/-0)
requirements-dev.txt (+3/-0)
requirements.txt (+3/-0)
scripts/templates/auto_jujuro.py (+80/-0)
src/charm.py (+181/-0)
tests/__init__.py (+3/-0)
tests/test_charm.py (+12/-0)
tests/unit/files/juju_bundle.yaml (+70/-0)
tests/unit/requirements.txt (+6/-0)
tests/unit/test_lib_juju_addons.py (+29/-0)
tox.ini (+89/-0)
Reviewer Review Type Date Requested Status
Alvaro Uria (community) Approve
Review via email: mp+423461@code.launchpad.net
To post a comment you must log in.
Revision history for this message
Alvaro Uria (aluria) wrote :

Code LGTM. Lint and unit tests passed.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
diff --git a/.gitignore b/.gitignore
0new file mode 1006440new file mode 100644
index 0000000..c5cb289
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,11 @@
1venv/
2build/
3*.charm
4.build/
5
6.coverage
7__pycache__/
8*.py[cod]
9
10repo-info
11version
diff --git a/.jujuignore b/.jujuignore
0new file mode 10064412new file mode 100644
index 0000000..6ccd559
--- /dev/null
+++ b/.jujuignore
@@ -0,0 +1,3 @@
1/venv
2*.py[cod]
3*.charm
diff --git a/Makefile b/Makefile
0new file mode 1006444new file mode 100644
index 0000000..f335696
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,97 @@
1PYTHON := /usr/bin/python3
2
3PROJECTPATH=$(dir $(realpath $(MAKEFILE_LIST)))
4ifndef CHARM_BUILD_DIR
5 CHARM_BUILD_DIR=${PROJECTPATH}.build
6endif
7ifdef CONTAINER
8 BUILD_ARGS="--destructive-mode"
9endif
10METADATA_FILE="metadata.yaml"
11CHARM_NAME=$(shell cat ${PROJECTPATH}/${METADATA_FILE} | grep -E '^name:' | awk '{print $$2}')
12
13help:
14 @echo "This project supports the following targets"
15 @echo ""
16 @echo " make help - show this text"
17 @echo " make clean - remove unneeded files"
18 @echo " make submodules - make sure that the submodules are up-to-date"
19 @echo " make submodules-update - update submodules to latest changes on remote branch"
20 @echo " make build - build the charm"
21 @echo " make release - run clean, submodules and build targets"
22 @echo " make lint - run flake8 and black --check"
23 @echo " make black - run black and reformat files"
24 @echo " make proof - run charm proof"
25 @echo " make unittests - run the tests defined in the unittest subdirectory"
26 @echo " make functional - run the tests defined in the functional subdirectory"
27 @echo " make test - run lint, proof, unittests and functional targets"
28 @echo ""
29
30clean:
31 @echo "Cleaning files"
32 @git clean -ffXd -e '!.idea'
33 @echo "Cleaning existing build"
34 @rm -rf ${CHARM_BUILD_DIR}/${CHARM_NAME}
35 @charmcraft clean
36 @rm -rf ${PROJECTPATH}/${CHARM_NAME}.charm
37
38submodules:
39 @echo "Cloning submodules"
40 @git submodule update --init --recursive
41
42submodules-update:
43 @echo "Pulling latest updates for submodules"
44 @git submodule update --init --recursive --remote --merge
45
46build: clean submodules-update
47 @echo "Building charm to base directory ${CHARM_BUILD_DIR}/${CHARM_NAME}"
48 @-git rev-parse --abbrev-ref HEAD > ./repo-info
49 @-git describe --always > ./version
50 @charmcraft -v pack ${BUILD_ARGS}
51 @bash -c ./rename.sh
52 @mkdir -p ${CHARM_BUILD_DIR}/${CHARM_NAME}
53 @unzip ${PROJECTPATH}/${CHARM_NAME}.charm -d ${CHARM_BUILD_DIR}/${CHARM_NAME}
54
55release: clean build unpack
56 @echo "Charm is built at ${CHARM_BUILD_DIR}/${CHARM_NAME}"
57
58unpack: build
59 @-rm -rf ${CHARM_BUILD_DIR}/${CHARM_NAME}
60 @mkdir -p ${CHARM_BUILD_DIR}/${CHARM_NAME}
61 @echo "Unpacking built .charm into ${CHARM_BUILD_DIR}/${CHARM_NAME}"
62 @cd ${CHARM_BUILD_DIR}/${CHARM_NAME} && unzip -q ${CHARM_BUILD_DIR}/${CHARM_NAME}.charm
63 # until charmcraft copies READMEs in, we need to publish charms with readmes in them.
64 @cp ${PROJECTPATH}/README.md ${CHARM_BUILD_DIR}/${CHARM_NAME}
65 @cp ${PROJECTPATH}/copyright ${CHARM_BUILD_DIR}/${CHARM_NAME}
66 @cp ${PROJECTPATH}/repo-info ${CHARM_BUILD_DIR}/${CHARM_NAME}
67 @cp ${PROJECTPATH}/version ${CHARM_BUILD_DIR}/${CHARM_NAME}
68
69lint:
70 @echo "Running lint checks"
71 @tox -e lint
72
73black:
74 @echo "Reformat files with black"
75 @tox -e black
76
77proof: unpack
78 @echo "Running charm proof"
79 @charm proof ${CHARM_BUILD_DIR}/${CHARM_NAME}
80
81unittests:
82 @echo "Running unit tests"
83 @tox -e unit
84
85snap:
86 @echo "Downloading snap"
87 snap download juju-lint --basename juju-lint --target-directory tests/functional/tests/resources
88
89functional: build snap
90 @echo "Executing functional tests in ${CHARM_BUILD_DIR}"
91 @CHARM_LOCATION=${PROJECTPATH} tox -e func
92
93test: lint proof unittests functional
94 @echo "Tests completed for charm ${CHARM_NAME}."
95
96# The targets below don't depend on a file
97.PHONY: help submodules submodules-update clean build release lint black proof unittests functional test unpack snap
diff --git a/README.md b/README.md
0new file mode 10064498new file mode 100644
index 0000000..9c8b093
--- /dev/null
+++ b/README.md
@@ -0,0 +1,49 @@
1# Juju RO
2
3## Description
4
5A charm to collect and publish sanitized and up to date Juju status outputs and bundle exports.
6
7## Usage
8
9Deploy:
10
11 $ juju deploy ch:juju-ro
12
13Next, create a new readonly juju user and grant `read` access to any Juju model:
14
15 $ juju add-user <user_name>
16 # capture the registration key
17 $ juju grant <user_name> read <model_name>
18
19Example:
20
21 $ juju add-user jujureadonly
22 User "jujureadonly" added
23 Please send this command to jujureadonly:
24 juju register <registration_key>
25
26 "jujureadonly" has not been granted access to any models. You can use "juju grant" to grant access.
27 $ juju grant jujureadonly read openstack
28
29Run `register-user` action on the juju-ro unit using the registration key from the previous step:
30
31 $ juju run-action juju-ro/0 register-user reg-key="<registration_key>" controller-name="<juju_controller_name>" --wait
32
33Files with the sanitized juju model bundles and juju status can be found in the `dumpdirectory` specified in the config, by default:
34
35 $ juju run -u juju-ro/0 -- ls /var/lib/jujureadonly
36 RO-juju-bundle-admin_openstack.txt
37 RO-juju-status-admin_openstack.txt
38
39Files are created and refreshed by the cron job configured by this charm. Refresh interval can be specified by updating the config, for example:
40
41 $ juju config juju-ro interval=15
42
43## Configuration
44
45The following options are available:
46
47* `interval` - specifies how often juju status and bundle will be generated in minutes.
48* `dumpdirectory` - Location of the juju status output and exported bundles.
49* `user` and `group` - User and group name for the local UNIX account that will be used to log in to Juju controller.
diff --git a/actions.yaml b/actions.yaml
0new file mode 10064450new file mode 100644
index 0000000..e975c54
--- /dev/null
+++ b/actions.yaml
@@ -0,0 +1,12 @@
1register-user:
2 description: |
3 Register the current user with a registration string.
4 params:
5 reg-key:
6 type: string
7 description: |
8 Registration key for logging into a Juju controller.
9 controller-name:
10 type: string
11 description: |
12 Name to give controller when registering.
0\ No newline at end of file13\ No newline at end of file
diff --git a/charmcraft.yaml b/charmcraft.yaml
1new file mode 10064414new file mode 100644
index 0000000..804eb40
--- /dev/null
+++ b/charmcraft.yaml
@@ -0,0 +1,14 @@
1type: "charm"
2parts:
3 charm:
4 charm-python-packages: [setuptools<58, pip>19]
5 build-packages: [libffi-dev, rustc, gcc, build-essential, cargo, libssl-dev]
6 prime:
7 - scripts
8bases:
9 - build-on:
10 - name: "ubuntu"
11 channel: "20.04"
12 run-on:
13 - name: "ubuntu"
14 channel: "20.04"
diff --git a/config.yaml b/config.yaml
0new file mode 10064415new file mode 100644
index 0000000..ff36f8c
--- /dev/null
+++ b/config.yaml
@@ -0,0 +1,19 @@
1options:
2 # Register readonly user to get juju status and juju bundle
3 user:
4 type: string
5 description: "User to run."
6 default: "jujureadonly"
7 group:
8 type: string
9 description: "Group to run the user as."
10 default: "jujureadonly"
11 dumpdirectory:
12 type: string
13 description: "location to store the juju status output and dump the bundles."
14 default: "/var/lib/jujureadonly"
15 interval:
16 type: int
17 default: 30
18 description: |
19 Specifies how often to generate juju status and bundle in minutes.
diff --git a/lib/charms/juju_ro/v0/lib_juju_addons.py b/lib/charms/juju_ro/v0/lib_juju_addons.py
0new file mode 10064420new file mode 100644
index 0000000..6458fae
--- /dev/null
+++ b/lib/charms/juju_ro/v0/lib_juju_addons.py
@@ -0,0 +1,535 @@
1#!/usr/bin/env python3
2# Copyright 2022 Canonical Ltd.
3# See LICENSE file for licensing details.
4
5"""Shared library for charms interacting with Juju controllers."""
6
7import pathlib
8import re
9import subprocess
10from subprocess import PIPE
11import logging
12import json
13import math
14import sys
15
16import yaml
17
18from juju import loop # noqa E402
19from juju.model import Model # noqa E402
20from juju.controller import Controller
21from juju.status import formatted_status
22from charmhelpers.core import host
23
24logger = logging.getLogger(__name__)
25
26# The unique Charmhub library identifier, never change it
27LIBID = "f846c7b4b30347f9816a1230ae805c9f"
28
29# Increment this major API version when introducing breaking changes
30LIBAPI = 0
31
32# Increment this PATCH version before using `charmcraft publish-lib` or reset
33# to 0 if you are raising the major API version
34LIBPATCH = 1
35
36MAX_FRAME_SIZE = 2**25
37MASK_KEYS = (
38 "(.*(ssl-public-key|ssl[_-](ca|cert|key|chain)|secret|password|"
39 "pagerduty_key|license-file|registration-key|token|accesskey|"
40 r"private-ppa|(http|https)://.*:.+\@|os-credentials).*)|key|"
41 "ldaps://.*|livepatch_key|tls-ca-ldap|"
42 "lb-mgmt.*(cacert|cert|private-key|passphrase)"
43)
44EMPTY_KEY_PLACE_HOLDER = "__EMPTY_KEY_PLACE_HOLDER__"
45BASE64_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
46HEX_CHARS = "1234567890abcdefABCDEF"
47
48
49class RegistrationError(Exception):
50 """Juju user registartion error."""
51
52 pass
53
54
55def register_juju_user(local_user, juju_reg_key, juju_controller_name):
56 """Register Juju controller using a local user account."""
57 pw = host.pwgen()
58 # TODO: Sometimes a third password input is required when adding
59 # a controller and one already exists. This should revisited
60 # when multi-controller support is added.
61 cmd = ["sudo", "-u", local_user, "juju", "register", juju_reg_key]
62 output = subprocess.run(
63 cmd,
64 input=f"{pw}\n{pw}\n{juju_controller_name}\n",
65 universal_newlines=True,
66 stderr=subprocess.STDOUT,
67 )
68
69 if output.returncode != 0:
70 msg = "User registration failed, please investigate logs"
71 raise RegistrationError(msg)
72
73 # save the password so we can authenticate with it going forward
74 juju_dir = f"/home/{local_user}/.local/share/juju"
75 accounts_file = f"{juju_dir}/accounts.yaml"
76 with open(accounts_file, "r") as f:
77 accounts = yaml.safe_load(f)
78 accounts["controllers"][juju_controller_name]["password"] = pw
79 # TODO: add better error handling
80 with open(accounts_file, "w") as f:
81 yaml.dump(accounts, f)
82
83 logger.info(f"Password saved for {juju_controller_name}")
84
85
86class Paths:
87 """Namespace for path constants."""
88
89 RO_SCRIPT_NAME = "auto_jujuro.py"
90 CRON_FILE_NAME = "/etc/cron.d/juju_ro"
91 USR_LOCAL = pathlib.Path("/usr/local")
92 SANITIZED_SCRIPT_PATH = USR_LOCAL / f"bin/{RO_SCRIPT_NAME}"
93 SANITIZED_CRONTAB_PATH = pathlib.Path(CRON_FILE_NAME)
94
95
96class Bundle:
97 """Bundle class to provide sanitizer function."""
98
99 def __init__(self, input_bundle, masks): # noqa C901
100 """Init bundle class."""
101 self.secret_regex = re.compile(masks["secrets"])
102 self.input_bundle = input_bundle
103 self.common_bundle = {"applications": {}, "relations": [], "machines": {}}
104 self.secrets_bundle = {"applications": {}}
105 # No secrets in some parts of the bundle
106 # placements get difficult unless machines are copied
107 for key in ["series", "relations", "machines"]:
108 if key in input_bundle.keys():
109 self.common_bundle[key] = input_bundle[key]
110 for app in input_bundle["applications"].keys():
111 app_data = {}
112 secret_data = {}
113 settings = input_bundle["applications"][app]
114 for param in settings.keys():
115 param_value = settings[param]
116 if param in ["annotations"]:
117 continue
118 if param in ["options"]: # This might want extracting
119 for option in param_value.keys():
120 option_value = param_value[option]
121 if "options" not in app_data.keys():
122 app_data["options"] = {}
123 if self.secret_regex.match(option):
124 if "options" not in secret_data.keys():
125 secret_data["options"] = {}
126 secret_data["options"][option] = option_value
127 app_data["options"][option] = "[REDACTED]"
128 else:
129 app_data["options"][option] = option_value
130 else:
131 if param == "bindings":
132 # in bindings, option with empty key like `"": internal-space`
133 # is allowed. however, yaml will consider empty key as
134 # complex mapping key and dump it to `? ''\n: internal-space`
135 # here we replace the empty key to a placeholder to bypass
136 # this issue it will be replaced back after dumped
137 # ref: https://yaml.org/spec/1.2/spec.html#id2760695
138 if "" in param_value:
139 param_value[EMPTY_KEY_PLACE_HOLDER] = param_value[""]
140 del param_value[""]
141 app_data[param] = param_value
142 self.common_bundle["applications"][app] = app_data
143 if len(secret_data.keys()) > 0:
144 self.secrets_bundle["applications"][app] = secret_data
145
146 def get_common_bundle(self):
147 """Return the common info only."""
148 if self.__check_for_entropy_in_dict(self.common_bundle):
149 logger.warning("Some entropy was found in the common bundle")
150 common_bundle = self.common_bundle
151 return common_bundle
152
153 def get_secrets_bundle(self):
154 """Return the bunlde includes secrets."""
155 return self.secrets_bundle
156
157 def __shannon_entropy(self, data, iterator):
158 """
159 Borrowed from.
160
161 http://blog.dkbza.org/2007/05/scanning-data-for-entropy-anomalies.html.
162 """
163 if not data:
164 return 0
165 entropy = 0
166 for x in iterator:
167 p_x = float(data.count(x)) / len(data)
168 if p_x > 0:
169 entropy += -p_x * math.log(p_x, 2)
170 return entropy
171
172 def __get_strings_of_set(self, word, char_set, threshold=8):
173 count = 0
174 letters = ""
175 strings = []
176 for char in word:
177 if char in char_set:
178 letters += char
179 count += 1
180 else:
181 if count > threshold:
182 strings.append(letters)
183 letters = ""
184 count = 0
185 if count > threshold:
186 strings.append(letters)
187 return strings
188
189 def __check_for_entropy_in_dict(self, bundle):
190 lines = yaml.dump(bundle).split("\n")
191 strings_found = []
192 entropy_found = False
193 for line in lines:
194 for word in line.split():
195 base64_strings = self.__get_strings_of_set(word, BASE64_CHARS)
196 hex_strings = self.__get_strings_of_set(word, HEX_CHARS)
197 for string in base64_strings:
198 b64_entropy = self.__shannon_entropy(string, BASE64_CHARS)
199 if b64_entropy > 3.8:
200 strings_found.append(string)
201 entropy_found = True
202 logger.warning(
203 "WARNING! Entropy found in string " '"{}"'.format(string)
204 )
205 for string in hex_strings:
206 hex_entropy = self.__shannon_entropy(string, HEX_CHARS)
207 if hex_entropy > 3:
208 strings_found.append(string)
209 entropy_found = True
210 logger.warning(
211 "WARNING! Entropy found in string " '"{}"'.format(string)
212 )
213
214 if len(strings_found) != 0:
215 logger.error(
216 "ERROR!!! Check output file for entropy warnings, " "may not be secure."
217 )
218
219 return entropy_found
220
221
222def juju_get_models():
223 """Get juju models."""
224 models = get_juju_models()
225 return models
226
227
228def juju_get_status(
229 model_name=None,
230 keep_relations_info=False,
231 jsfy=False,
232 format_status=False,
233 filename=None,
234 add_charm_config=False,
235):
236 """Get juju status in a named model."""
237 juju_status = loop.run(
238 get_juju_status(
239 model_name,
240 keep_relations_info,
241 jsfy,
242 format_status,
243 filename,
244 add_charm_config,
245 )
246 )
247 return juju_status
248
249
250def juju_get_bundle(model_name, filename=None, sanitized=False):
251 """Get juju bundle."""
252 juju_bundle = loop.run(get_juju_bundle(model_name, filename, sanitized))
253 return juju_bundle
254
255
256def juju_get_mode_config(model_name):
257 """Get juju model config."""
258 model_config = loop.run(get_juju_model_config(model_name))
259 return model_config
260
261
262def juju_get_machines(model_name):
263 """Get juju machines in a model."""
264 machines = loop.run(get_juju_machines(model_name))
265 return machines
266
267
268def sanitized_bundle(bundle_file, out_file):
269 """Sanitize bundle to get rid of secret info."""
270 masks = {"secrets": MASK_KEYS} # this is a dict so we can add things later
271 with open(bundle_file, "r") as in_file:
272 bundle = readyaml(in_file)
273 entire_bundle = Bundle(bundle, masks)
274 grab_out = {
275 "common": entire_bundle.get_common_bundle,
276 "secrets": entire_bundle.get_secrets_bundle,
277 }
278 output = yaml.dump(grab_out["common"](), default_flow_style=False)
279 with open(out_file, "w") as f:
280 print(output.replace(EMPTY_KEY_PLACE_HOLDER, '""'), file=f)
281
282
283class JujuModel:
284 """Context manager that connects and disconnects from the currently active model."""
285
286 def __init__(self, model_name=None) -> None:
287 """Init JujuModel."""
288 self._model = None
289 self.model_name = model_name
290
291 async def __aenter__(self):
292 """Set up model connection."""
293 self._model = Model(max_frame_size=MAX_FRAME_SIZE)
294 await self._model.connect(model_name=self.model_name)
295 return self._model
296
297 async def __aexit__(self, exc_type, exc, tb):
298 """Disconnet model."""
299 await self._model.disconnect()
300
301
302class JujuController:
303 """Context manager connects and disconnects from the current controller."""
304
305 def __init__(self) -> None:
306 """Init Juju controller."""
307 self._controller = None
308
309 async def __aenter__(self):
310 """Set up juju controller connection."""
311 self._controller = Controller()
312 await self._controller.connect()
313 return self._controller
314
315 async def __aexit__(self, exc_type, exc, tb):
316 """Disconnect juju controller."""
317 await self._controller.disconnect()
318
319
320def read_json_file(path):
321 """Read json file."""
322 with open(path) as fp:
323 return json.load(fp)
324
325
326# internal functions
327def get_juju_models():
328 """Controller.list_models requires superuser permission to list models.
329
330 there is a bug for it, see following link. Need to update the code to use libjuju
331 to get models once the bug is fixed.
332 https://github.com/juju/python-libjuju/issues/624.
333 """
334 cmd = (
335 r"juju models --quiet | "
336 r"awk '{{print $1}}' | egrep -v '(Controller|Model)' | sed -e 's/\*//'"
337 )
338 logger.info(cmd)
339 output = subprocess.run(cmd, stdout=PIPE, stderr=PIPE, shell=True)
340 if output.returncode != 0:
341 logger.error(f"juju models failed with: {output}")
342 return None
343 return list(output.stdout.decode().split())
344
345
346# async def get_juju_models():
347# """Use this function to get juju model once the following bug was fixed.
348# https://github.com/juju/python-libjuju/issues/624
349# """
350# async with JujuController() as controller:
351# models = await controller.list_models()
352# return models
353
354
355async def get_juju_machines(model_name):
356 """Get juju model machines."""
357 result = []
358 async with JujuModel(model_name) as model:
359 machines = model.machines.values()
360 for m in machines:
361 result.append(m.data)
362 return result
363
364
365async def get_juju_model_config(model_name):
366 """Get juju model config."""
367 async with JujuModel(model_name) as model:
368 model_config = await model.get_config()
369 return model_config
370
371
372async def get_juju_bundle(model_name, filename, sanitized):
373 """Get juju bundle."""
374 async with JujuModel(model_name) as model:
375 juju_bundle = await model.export_bundle(filename)
376 if sanitized:
377 sanitized_bundle(filename, filename)
378 return juju_bundle
379
380
381async def get_juju_status(
382 model_name,
383 keep_relations_info,
384 jsfy,
385 format_status,
386 filename,
387 add_charm_config,
388):
389 """Get juju status."""
390 async with JujuModel(model_name) as model:
391 juju_status = ""
392 try:
393 if format_status:
394 juju_status = await formatted_status(model)
395 else:
396 juju_status = await model.get_status()
397 juju_status = process_juju_status(
398 juju_status, keep_relations_info, jsfy
399 )
400 if add_charm_config:
401 # inject the charm config into the status, so juju-lint
402 # can check the "config" rules
403 juju_status = await add_charm_config_to_juju_status(
404 juju_status, model
405 )
406 except Exception as e:
407 logger.error(
408 "Failed to get juju status for model {}, {}".format(model_name, str(e))
409 )
410 if filename:
411 write_file(filename, juju_status)
412 return juju_status
413
414
415async def add_charm_config_to_juju_status(juju_status, model):
416 """Add the charm configuration to the juju status output."""
417 for app_name, status_app in juju_status["applications"].items():
418 model_app = model.applications[app_name]
419 model_app_config = await model_app.get_config()
420
421 # Collect all manual/default config settings
422 # this similar to juju export-bundle --include-charm-defaults
423 status_app_config = {}
424 for k, v in model_app_config.items():
425 if v["source"] != "unset":
426 try:
427 status_app_config[k] = v["value"]
428 except KeyError as error:
429 logging.error(
430 'missing info for app: "{}", key: "{}", val: "{}"'.format(
431 app_name, k, v
432 )
433 )
434 raise error
435
436 juju_status["applications"][app_name]["options"] = status_app_config
437
438 return juju_status
439
440
441def write_file(path, text, mode="w"):
442 """Write json file."""
443 with open(path, mode) as fp:
444 fp.write(text)
445
446
447def process_juju_status(juju_status, keep_relations_info, jsfy):
448 """Convert an application status structure to the one expected by juju-lint."""
449 juju_status_serial = juju_status.serialize()
450
451 # drop the "relations" key, otherwise juju-lint will not do the status
452 # checks (assumes this is just a bundle)
453 if not keep_relations_info:
454 juju_status_serial.pop("relations", None)
455
456 # convert the output format to jsfy-style as expected by juju-lint
457 if jsfy:
458 convert_libjuju_to_jsfy(juju_status_serial)
459
460 return juju_status_serial
461
462
463def convert_libjuju_to_jsfy(juju_status):
464 """
465 Perform the conversion between the libjuju output and the jsfy output.
466
467 There are some differences between the output of libjuju and
468 and `juju status --format yaml`, see bug reports below for
469 details. This function acts like a "shim" to convert between the
470 two formats. When the bugs are resolved this can be removed
471
472 https://github.com/juju/python-libjuju/issues/500
473 https://bugs.launchpad.net/juju/+bug/1930184
474 """
475 # perform machine conversion
476 if "machines" in juju_status:
477 for machine_name, machine_struct in juju_status["machines"].items():
478 machine_dict = machine_struct.serialize()
479 remap_instance_dict(
480 machine_dict,
481 {"instance-status": "machine-status", "agent-status": "juju-status"},
482 )
483 juju_status["machines"][machine_name] = machine_dict
484
485 # perform application and unit conversion
486 if "applications" in juju_status:
487 for app_name, app_struct in juju_status["applications"].items():
488 app_dict = app_struct.serialize()
489 remap_instance_dict(app_dict, {"status": "application-status"})
490
491 for unit_name, unit_struct in app_dict["units"].items():
492 unit_dict = app_dict["units"][unit_name] = unit_struct.serialize()
493 remap_instance_dict(
494 unit_dict,
495 {
496 "workload-status": "workload-status",
497 "agent-status": "juju-status",
498 },
499 )
500
501 for subord_name, subord_struct in unit_dict["subordinates"].items():
502 subord_dict = unit_dict["subordinates"][
503 subord_name
504 ] = subord_struct.serialize()
505 remap_instance_dict(
506 subord_dict,
507 {
508 "workload-status": "workload-status",
509 "agent-status": "juju-status",
510 },
511 )
512 juju_status["applications"][app_name] = app_dict
513
514 return juju_status
515
516
517def remap_instance_dict(instance_dict, mapping_dict):
518 """Convert an unit status structure to the one expected by juju-lint."""
519 for from_key, to_key in mapping_dict.items():
520 if from_key in instance_dict:
521 instance_status = instance_dict[to_key] = instance_dict.pop(
522 from_key
523 ).serialize()
524 instance_status["message"] = instance_status.pop("info")
525 instance_status["current"] = instance_status.pop("status")
526
527
528def readyaml(stream):
529 """Read yaml file."""
530 try:
531 myyaml = yaml.safe_load(stream.read())
532 except yaml.YAMLError as exc:
533 msg = "Failed to read yaml with {}".format(exc)
534 sys.exit(msg)
535 return myyaml
diff --git a/metadata.yaml b/metadata.yaml
0new file mode 100644536new file mode 100644
index 0000000..dbaa9be
--- /dev/null
+++ b/metadata.yaml
@@ -0,0 +1,13 @@
1name: juju-ro
2display-name: Juju RO
3maintainer: BootStack Charmers <bootstack-charmers@lists.canonical.com>
4summary: |
5 A charm to collect and publish sanitized and up to date Juju status outputs and bundle exports.
6description: |
7 A charm to collect and publish sanitized and up to date Juju status outputs and bundle exports.
8tags:
9 - ops
10series:
11 - focal
12 - bionic
13subordinate: false
diff --git a/rename.sh b/rename.sh
0new file mode 10075514new file mode 100755
index 0000000..956a76b
--- /dev/null
+++ b/rename.sh
@@ -0,0 +1,13 @@
1#!/bin/bash
2charm=$(grep -E "^name:" metadata.yaml | awk '{print $2}')
3echo "renaming ${charm}_*.charm to ${charm}.charm"
4echo -n "pwd: "
5pwd
6ls -al
7echo "Removing previous charm if it exists"
8if [[ -e "${charm}.charm" ]];
9then
10 rm "${charm}.charm"
11fi
12echo "Renaming charm here."
13mv ${charm}_*.charm ${charm}.charm
diff --git a/requirements-dev.txt b/requirements-dev.txt
0new file mode 10064414new file mode 100644
index 0000000..4f2a3f5
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,3 @@
1-r requirements.txt
2coverage
3flake8
diff --git a/requirements.txt b/requirements.txt
0new file mode 1006444new file mode 100644
index 0000000..5bb7c92
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,3 @@
1ops >= 1.4.0
2charmhelpers
3juju
diff --git a/scripts/templates/auto_jujuro.py b/scripts/templates/auto_jujuro.py
0new file mode 1006444new file mode 100644
index 0000000..bfff447
--- /dev/null
+++ b/scripts/templates/auto_jujuro.py
@@ -0,0 +1,80 @@
1#!/usr/bin/env python3
2# Copyright 2022 Canonical
3# See LICENSE file for licensing details.
4
5"""Automatically grab juju status output and juju sanitized bundle."""
6import sys
7import os
8import logging
9import argparse
10from os.path import join
11
12
13# The path below is templated in during charm install
14sys.path.append("REPLACE_CHARMDIR/venv")
15sys.path.append("REPLACE_CHARMDIR/lib")
16
17from charms.juju_ro.v0.lib_juju_addons import ( # noqa E402
18 juju_get_models,
19 juju_get_status,
20 juju_get_bundle,
21)
22
23
24PID_FILENAME = "/tmp/auto_jujuro.pid"
25
26logger = logging.getLogger(__name__)
27
28
29def main():
30 """Call main function."""
31 parser = argparse.ArgumentParser(
32 description="Grab juju status and sanitized bundle output."
33 )
34
35 parser.add_argument(
36 "-d",
37 "--dump-directory",
38 dest="dumpdir",
39 required=True,
40 help="location to store the juju status output and dump the bundles.",
41 )
42
43 args = parser.parse_args()
44
45 # Ensure a single instance via a simple pidfile
46 pid = str(os.getpid())
47
48 if os.path.isfile(PID_FILENAME):
49 sys.exit("{} already exists, exiting".format(PID_FILENAME))
50
51 with open(PID_FILENAME, "w") as f:
52 f.write(pid)
53
54 for model in juju_get_models():
55 try:
56 juju_status_file = join(
57 args.dumpdir, f"RO-juju-status-{model.replace('/','_')}.txt"
58 )
59 juju_get_status(
60 model_name=model, format_status=True, filename=juju_status_file
61 )
62 except Exception as e:
63 logger.error(
64 "failed to get juju status for model {}, {}".format(model, str(e))
65 )
66
67 for model in juju_get_models():
68 try:
69 bundle_file = join(
70 args.dumpdir, f"RO-juju-bundle-{model.replace('/','_')}.txt"
71 )
72 juju_get_bundle(model_name=model, filename=bundle_file)
73 except Exception as e:
74 logger.error("failed to get bundle for model {}, {}".format(model, str(e)))
75
76 os.unlink(PID_FILENAME)
77
78
79if __name__ == "__main__":
80 main()
diff --git a/src/charm.py b/src/charm.py
0new file mode 10075581new file mode 100755
index 0000000..eb213f6
--- /dev/null
+++ b/src/charm.py
@@ -0,0 +1,181 @@
1#!/usr/bin/env python3
2# Copyright 2022 Canonical
3# See LICENSE file for licensing details.
4#
5# Learn more at: https://juju.is/docs/sdk
6
7"""Collect and santize Juju status and bundles.
8
9A charm to collect and publish sanitized and up to date Juju status
10outputs and bundle exports.
11"""
12
13import pathlib
14import os
15import logging
16
17from ops.charm import CharmBase
18from ops.framework import StoredState
19from ops.main import main
20from ops.model import ActiveStatus, BlockedStatus
21from charmhelpers.fetch import snap
22from charmhelpers.core import hookenv, host
23
24from charms.juju_ro.v0.lib_juju_addons import (
25 Paths,
26 register_juju_user,
27 RegistrationError,
28)
29
30logger = logging.getLogger(__name__)
31
32
33class JujuROHelper:
34 """Juju Sanitizer helper object."""
35
36 def __init__(self, model):
37 """Construct the helper."""
38 self.model = model
39 self.charm_config = model.config
40
41 def update_crontab(self, user, frequency, dumpdir):
42 """Update crontab at /etc/cron.d/juju-ro."""
43 cron_job = (
44 "PATH=/usr/bin:/bin:/snap/bin\n"
45 "*/{} * * * * {} {} -d {} > /tmp/juju_ro.log 2>&1\n".format(
46 frequency,
47 user,
48 Paths.SANITIZED_SCRIPT_PATH,
49 dumpdir,
50 )
51 )
52 with Paths.SANITIZED_CRONTAB_PATH.open("w") as fp:
53 fp.write(cron_job)
54
55 def delete_crontab(self):
56 """Delete cron tab."""
57 if os.path.exists(Paths.SANITIZED_CRONTAB_PATH):
58 os.unlink(Paths.SANITIZED_CRONTAB_PATH)
59
60 def install_snap(self):
61 """Install juju snap."""
62 snaps = ["juju"]
63 snap_flags = "--classic"
64 snap.snap_install(snaps, snap_flags)
65
66 def install_scripts(self):
67 """Install scripts."""
68 charm_dir = pathlib.Path(hookenv.charm_dir())
69 scripts_dir = charm_dir / "scripts"
70 templates_dir = scripts_dir / "templates"
71
72 # Create the auto_jujuro.py script from the template with
73 # the right permissions
74 with open(str(templates_dir / Paths.RO_SCRIPT_NAME)) as f:
75 auto_template = f.read()
76
77 sanitizer_juju_script = auto_template.replace(
78 "REPLACE_CHARMDIR", str(charm_dir)
79 )
80
81 with open(str(Paths.SANITIZED_SCRIPT_PATH), "w") as f:
82 f.write(sanitizer_juju_script)
83
84 os.chmod(str(Paths.SANITIZED_SCRIPT_PATH), 0o755)
85
86
87class JujuRoCharm(CharmBase):
88 """Collect and santize Juju status and bundles."""
89
90 state = StoredState()
91
92 def __init__(self, *args):
93 super().__init__(*args)
94 self.framework.observe(self.on.install, self.on_install)
95 self.framework.observe(self.on.upgrade_charm, self.on_upgrade_charm)
96 self.framework.observe(self.on.config_changed, self.on_config_changed)
97 self.framework.observe(self.on.update_status, self.update_status)
98 self.framework.observe(self.on.register_user_action, self.register_user)
99 self.jujuro_helper = JujuROHelper(self.model)
100 self.state.set_default(
101 installed=False,
102 user_registered=False,
103 )
104 self.ctxt = {
105 "user": self.model.config["user"],
106 "group": self.model.config["group"],
107 "dumpdir": self.model.config["dumpdirectory"],
108 "interval": self.model.config["interval"],
109 }
110 self.logger = logging.getLogger(__name__)
111
112 def on_install(self, event):
113 """Install juju snap and cron scripts."""
114 self.jujuro_helper.install_snap()
115 self.jujuro_helper.install_scripts()
116 self.state.installed = True
117 self.update_status(event)
118
119 def on_upgrade_charm(self, event):
120 """Handle upgrade and resource updates."""
121 # Reinstall snaps and scripts
122 logging.info("upgrade-charm: reinstall")
123 self.jujuro_helper.install_snap()
124 self.jujuro_helper.install_scripts()
125 self.update_status(event)
126
127 def config_juju_readonly_user(self):
128 """Need home dir to configure juju read only user."""
129 host.add_group(self.ctxt["group"])
130 host.adduser(
131 self.ctxt["user"],
132 primary_group=self.ctxt["group"],
133 shell="/bin/bash",
134 password=host.pwgen(), # don't save it, not needed
135 home_dir=f"/home/{self.ctxt['user']}",
136 )
137
138 host.mkdir(
139 self.ctxt["dumpdir"],
140 owner=self.ctxt["user"],
141 group=self.ctxt["group"],
142 perms=0o700,
143 force=True,
144 )
145
146 def on_config_changed(self, event):
147 """Reconfigure charm."""
148 self.config_juju_readonly_user()
149 self.jujuro_helper.update_crontab(
150 self.ctxt["user"], self.ctxt["interval"], self.ctxt["dumpdir"]
151 )
152 self.update_status(event)
153
154 def update_status(self, _):
155 """Asses the current status of the unit."""
156 if self.state.user_registered:
157 self.unit.status = ActiveStatus("Ready")
158 elif self.state.installed:
159 self.unit.status = BlockedStatus("Juju read-only user not registered")
160
161 def register_user(self, event):
162 """Register juju user using registration key."""
163 try:
164 register_juju_user(
165 self.ctxt["user"],
166 event.params["reg-key"],
167 event.params["controller-name"],
168 # TODO: validate if controller_name has acceptable input
169 )
170 self.logger.info("juju registration successful")
171 self.state.user_registered = True
172 self.unit.status = ActiveStatus("Ready")
173 except RegistrationError as err:
174 self.logger.error(f"juju registration failed: {str(err)}")
175 msg = "User registration failed, please investigate logs"
176 self.unit.status = BlockedStatus(msg)
177 return
178
179
180if __name__ == "__main__":
181 main(JujuRoCharm)
diff --git a/tests/__init__.py b/tests/__init__.py
0new file mode 100644182new file mode 100644
index 0000000..c5e3156
--- /dev/null
+++ b/tests/__init__.py
@@ -0,0 +1,3 @@
1import ops.testing
2
3ops.testing.SIMULATE_CAN_CONNECT = True
diff --git a/tests/test_charm.py b/tests/test_charm.py
0new file mode 1006444new file mode 100644
index 0000000..0529d5d
--- /dev/null
+++ b/tests/test_charm.py
@@ -0,0 +1,12 @@
1# Copyright 2022 Canonical
2# See LICENSE file for licensing details.
3#
4# Learn more about testing at: https://juju.is/docs/sdk/testing
5
6import unittest
7
8
9class TestCharm(unittest.TestCase):
10 """Test charm."""
11
12 pass
diff --git a/tests/unit/files/juju_bundle.yaml b/tests/unit/files/juju_bundle.yaml
0new file mode 10064413new file mode 100644
index 0000000..2e139e6
--- /dev/null
+++ b/tests/unit/files/juju_bundle.yaml
@@ -0,0 +1,70 @@
1series: focal
2saas:
3 dashboards:
4 url: foundations-maas:admin/lma.dashboards
5 grafana:
6 url: foundations-maas:admin/lma.grafana
7 graylog:
8 url: foundations-maas:admin/lma.graylog-beats
9 landscape-server:
10 url: foundations-maas:admin/lma.landscape-server
11 nagios:
12 url: foundations-maas:admin/lma.nagios-monitors
13 prometheus:
14 url: foundations-maas:admin/lma.prometheus-target
15 prometheus-rules:
16 url: foundations-maas:admin/lma.prometheus-rules
17 prometheus-target:
18 url: foundations-maas:admin/lma.prometheus-target
19applications:
20 aodh:
21 charm: cs:aodh-46
22 num_units: 3
23 to:
24 - lxd:6
25 - lxd:7
26 - lxd:8
27 options:
28 openstack-origin: distro
29 os-admin-hostname: aodh-admin.example.com
30 os-internal-hostname: aodh-internal.example.com
31 os-public-hostname: aodh.example.com
32 region: example
33 ssl_cert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUdUVENDQlRXZ0F3SUJBZ0lRQWp5UUl2MzU1YTdjd1VZWTBYNE1FekFOQmdrcWhraUc5dzBCQVFzRkFEQlAKTVFzd0NRWURWUVFHRXdKVlV6RVZNQk1HQTFVRUNoTU1SR2xuYVVObGNuUWdTVzVqTVNrd0p3WURWUVFERXlCRQphV2RwUTJWeWRDQlVURk1nVWxOQklGTklRVEkxTmlBeU1ESXdJRU5CTVRBZUZ3MHlNREV4TVRjd01EQXdNREJhCkZ3MHlNVEV4TWpFeU16VTVOVGxhTUZveEN6QUpCZ05WQkFZVEFrZENNUTh3RFFZRFZRUUhFd1pNYjI1a2IyNHgKSERBYUJnTlZCQW9URTBOaGJtOXVhV05oYkNCSGNtOTFjQ0JNZEdReEhEQWFCZ05WQkFNTUV5b3VjSE0xTG1OaApibTl1YVdOaGJDNWpiMjB3Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLQW9JQkFRRHJXUWxPCldsUXNpK1VIWG1wTlIvYVBWVjJ6ZWIrcklKY2l0d3ZwcDBsODJraDU2Y
34 use-internal-endpoints: true
35 vip: 1.1.48.20 1.1.52.20 1.1.56.20
36 worker-multiplier: 0.25
37 bindings:
38 "": oam-space
39 admin: admin-space
40 amqp: oam-space
41 certificates: oam-space
42 cluster: oam-space
43 ha: oam-space
44 identity-service: oam-space
45 internal: internal-space
46 mongodb: oam-space
47 public: public-space
48 shared-db: admin-space
49 aodh-mysql-router:
50 charm: cs:mysql-router-6
51 bindings:
52 "": oam-space
53 certificates: oam-space
54 db-router: admin-space
55 juju-info: oam-space
56 shared-db: oam-space
57 barbican:
58 charm: cs:barbican-36
59 num_units: 3
60 to:
61 - lxd:3
62 - lxd:4
63 - lxd:5
64 options:
65 openstack-origin: distro
66 os-admin-hostname: barbican-admin.example.com
67 os-internal-hostname: barbican-internal.example.com
68 os-public-hostname: barbican.example.com
69 region: example
70 ssl_cert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUdUVENDQlRXZ0F3SUJBZ0lRQWp5UUl2MzU1YTdjd1VZWTBYNE1FekFOQmdrcWhraUc5dzBCQVFzRkFEQlAKTVFzd0NRWURWUVFHRXdKVlV6RVZNQk1HQTFVRUNoTU1SR2xuYVVObGNuUWdTVzVqTVNrd0p3WURWUVFERXlCRQphV2RwUTJWeWRDQlVURk1nVWxOQklGTklRVEkxTmlBeU1ESXdJRU5CTVRBZUZ3MHlNREV4TVRjd01EQXdNREJhCkZ3MHlNVEV4TWpFeU16VTVOVGxhTUZveEN6QUpCZ05WQkFZVEFrZENNUTh3RFFZRFZRUUhFd1pNYjI1a2IyNHgKSERBYUJnTlZCQW9URTBOaGJtOXVhV05oYkNCSGNtOTFjQ0JNZEdReEhEQWFCZ05WQkFNTUV5b3VjSE0xT
0\ No newline at end of file71\ No newline at end of file
diff --git a/tests/unit/requirements.txt b/tests/unit/requirements.txt
1new file mode 10064472new file mode 100644
index 0000000..fd430bd
--- /dev/null
+++ b/tests/unit/requirements.txt
@@ -0,0 +1,6 @@
1mock
2pyyaml
3six
4jinja2
5coverage
6juju
diff --git a/tests/unit/test_lib_juju_addons.py b/tests/unit/test_lib_juju_addons.py
0new file mode 1006447new file mode 100644
index 0000000..ca87659
--- /dev/null
+++ b/tests/unit/test_lib_juju_addons.py
@@ -0,0 +1,29 @@
1"""lib_juju_addons.py unittests."""
2import os
3import unittest
4from os.path import join
5
6from charms.juju_ro.v0.lib_juju_addons import (
7 sanitized_bundle,
8)
9
10
11class TestLibJujuAddons(unittest.TestCase):
12 """test class for lib_juju_addons.py."""
13
14 def test_sanitized_bundle(self):
15 """Test that no secret info in the output."""
16 file_path = "tests/unit/files"
17 test_file = join(file_path, "juju_bundle.yaml")
18 out_file = join(file_path, "sanitized_bundle.yaml")
19 sanitized_bundle(test_file, out_file)
20 with open(out_file, "r") as f:
21 line = f.readline()
22 while line:
23 kv = line.split(":")
24 k = kv[0]
25 v = kv[-1]
26 if k == "ssl_cert":
27 assert v == "[REDACTED]"
28 line = f.readline()
29 os.unlink(out_file)
diff --git a/tox.ini b/tox.ini
0new file mode 10064430new file mode 100644
index 0000000..5ca9012
--- /dev/null
+++ b/tox.ini
@@ -0,0 +1,89 @@
1[tox]
2skipsdist=True
3skip_missing_interpreters = True
4envlist = lint, unit, func
5
6[testenv]
7basepython = python3
8setenv =
9 PYTHONPATH = {toxinidir}:{toxinidir}/lib/:{toxinidir}/hooks/
10passenv =
11 HOME
12 PATH
13 CHARM_*
14 OS_*
15 MODEL_SETTINGS
16 HTTP_PROXY
17 HTTPS_PROXY
18 NO_PROXY
19 SNAP_HTTP_PROXY
20 SNAP_HTTPS_PROXY
21
22[testenv:build]
23deps = charmcraft<1.1.0
24commands = charmcraft build
25
26[testenv:lint]
27commands =
28 flake8 {posargs} src tests lib scripts
29 black --check --exclude "/(\.eggs|\.git|\.tox|\.venv|\.build|build|dist|charmhelpers|mod)/" .
30deps =
31 black
32 flake8
33 flake8-docstrings
34 flake8-import-order
35 pep8-naming
36 flake8-colors
37
38
39[testenv:black]
40commands =
41 black --exclude "/(\.eggs|\.git|\.tox|\.venv|\.build|build|dist|charmhelpers|mod)/" .
42deps =
43 black
44
45[testenv:unit]
46commands =
47 coverage run -m unittest discover -s {toxinidir}/tests/unit -v
48 coverage report --omit tests/*,mod/*,.tox/*
49 coverage html --omit tests/*,mod/*,.tox/*
50deps = -r{toxinidir}/tests/unit/requirements.txt
51 -r{toxinidir}/requirements.txt
52
53[testenv:func]
54changedir = {toxinidir}/tests/functional
55deps = -r{toxinidir}/tests/functional/requirements.txt
56commands = functest-run-suite --keep-faulty-model {posargs}
57
58
59[testenv:func-smoke]
60changedir = {toxinidir}/tests/functional
61deps = -r{toxinidir}/tests/functional/requirements.txt
62commands = functest-run-suite --keep-model --smoke
63
64[testenv:func-dev]
65changedir = {toxinidir}/tests/functional
66deps = -r{toxinidir}/tests/functional/requirements.txt
67commands = functest-run-suite --keep-model --dev
68
69[testenv:func-target]
70changedir = {toxinidir}/tests/functional
71deps = -r{toxinidir}/tests/functional/requirements.txt
72commands = functest-run-suite --keep-model --bundle {posargs}
73
74[flake8]
75exclude =
76 .git,
77 __pycache__,
78 .tox,
79 .build,
80 build
81
82ignore = D100, D104, D107
83
84max-line-length = 88
85max-complexity = 10
86
87# flake8-import-order configurations
88import-order-style = pep8
89application-import-names = src.charm,charms.juju_ro.v0.lib_juju_addons

Subscribers

People subscribed via source and target branches

to all changes: