Merge ~hloeung/charm-graylog:master into ~graylog-charmers/charm-graylog:master
- Git
- lp:~hloeung/charm-graylog
- master
- Merge into master
Status: | Superseded |
---|---|
Proposed branch: | ~hloeung/charm-graylog:master |
Merge into: | ~graylog-charmers/charm-graylog:master |
Diff against target: |
367 lines (+247/-15) 5 files modified
TODO (+1/-2) config.yaml (+11/-0) metadata.yaml (+1/-1) reactive/graylog.py (+169/-11) unit_tests/graylog.py (+65/-1) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Tim Kuhlman (community) | Needs Fixing | ||
Review via email: mp+329093@code.launchpad.net |
This proposal has been superseded by a proposal from 2017-08-23.
Commit message
Description of the change
Add log inputs from charm config
This adds new 'log_inputs' charm config option to define a set of log inputs. These inputs are then set up via the Graylog API and ports opened (juju open-port) so that a juju expose sets up the correct secgroup rules.
While this works, I'm not proud of it. I have a task to split the API parts out to it's own library.
Haw Loeung (hloeung) : | # |
- 0846e8f... by Haw Loeung
-
Add log inputs from charm config
This adds new 'log_inputs' charm config option to define a set of log
inputs. These inputs are then set up via the Graylog API and ports
opened (juju open-port) so that a juju expose sets up the correct
secgroup rules.
Tim Kuhlman (timkuhlman) wrote : | # |
I took a brief look at the updates with some comments below. The changes look good but I would still like to see unit and/or amulet tests and at some point a class for the Graylog API.
- 962a4a4... by Haw Loeung
-
Add unit tests for input type map and input exists
Unmerged commits
- 962a4a4... by Haw Loeung
-
Add unit tests for input type map and input exists
- 0846e8f... by Haw Loeung
-
Add log inputs from charm config
This adds new 'log_inputs' charm config option to define a set of log
inputs. These inputs are then set up via the Graylog API and ports
opened (juju open-port) so that a juju expose sets up the correct
secgroup rules.
Preview Diff
1 | diff --git a/TODO b/TODO |
2 | index e0c4f88..2272c2a 100644 |
3 | --- a/TODO |
4 | +++ b/TODO |
5 | @@ -1,8 +1,7 @@ |
6 | Before Production Use: |
7 | - Clustering of multiple graylog instances. |
8 | -- Support for nrpe relation, including service specific nagios check. |
9 | - Support for removing relations. |
10 | -- Configuration of Inputs, specifically beats setup, most likely via the REST API. |
11 | +- Support for logstash relation. |
12 | - When charm-based configuration of beats is in place, support the logstash relation so that |
13 | filebeats can use that relation to setup the proper output when in the same Juju model. |
14 | |
15 | diff --git a/config.yaml b/config.yaml |
16 | index 8db8a8c..9614533 100644 |
17 | --- a/config.yaml |
18 | +++ b/config.yaml |
19 | @@ -47,6 +47,17 @@ options: |
20 | default: 6 |
21 | description: | |
22 | Maximum number of indices to keep before deleting the oldest ones |
23 | + log_inputs: |
24 | + type: string |
25 | + default: | |
26 | + - name: Beats Input |
27 | + type: Beats |
28 | + bind_address: 0.0.0.0 |
29 | + bind_port: 5044 |
30 | + description: | |
31 | + YAML-formatted list of log inputs. First input gets passed |
32 | + through relations. Any input not defined here will be removed unless it |
33 | + is prefixed with "Custom" in the title. |
34 | nagios_context: |
35 | default: "juju" |
36 | type: string |
37 | diff --git a/metadata.yaml b/metadata.yaml |
38 | index b5f3096..8277908 100644 |
39 | --- a/metadata.yaml |
40 | +++ b/metadata.yaml |
41 | @@ -1,6 +1,6 @@ |
42 | name: graylog |
43 | summary: Graylog log management system |
44 | -maintainer: Tim Kuhlman <timothy.kuhlman@canonical.com> |
45 | +maintainer: Graylog Charmers <graylog-charmers@lists.launchpad.net> |
46 | description: > |
47 | Installs the Graylog log management system. Connections to elasticsearch and |
48 | mongodb are required for a fully functioning system. https://www.graylog.org/ |
49 | diff --git a/reactive/graylog.py b/reactive/graylog.py |
50 | index 016f061..a93b5e8 100644 |
51 | --- a/reactive/graylog.py |
52 | +++ b/reactive/graylog.py |
53 | @@ -4,6 +4,7 @@ import os |
54 | import re |
55 | import requests |
56 | import time |
57 | +import yaml |
58 | from urllib.parse import urlparse, urljoin |
59 | |
60 | from charms.reactive import hook, when, when_not, remove_state, set_state |
61 | @@ -30,8 +31,6 @@ def update_config(): |
62 | |
63 | |
64 | def api_request(url, auth, method='GET', data=None): |
65 | - if method not in ('GET', 'POST', 'PUT'): |
66 | - return |
67 | tries = 0 |
68 | while tries < 3: |
69 | tries += 1 |
70 | @@ -40,6 +39,8 @@ def api_request(url, auth, method='GET', data=None): |
71 | headers={'Accept': 'application/json', |
72 | 'Content-Type': 'application/json'}) |
73 | if resp.ok: |
74 | + if method == 'DELETE': |
75 | + return True |
76 | return json.loads(resp.content.decode('utf-8')) |
77 | except Exception: |
78 | pass |
79 | @@ -60,11 +61,6 @@ def api_retrieve_token(api_base_url, auth, name='graylog-charm'): |
80 | return api_retrieve_token(api_base_url, auth, name=name) |
81 | |
82 | |
83 | -def api_retrieve_index_sets(api_base_url, auth): |
84 | - url = urljoin(api_base_url, 'system/indices/index_sets') |
85 | - return api_request(url, auth=auth)['index_sets'] |
86 | - |
87 | - |
88 | rotation_strategies = { |
89 | 'time': { |
90 | 'class': 'org.graylog2.indexer.rotation.strategies.TimeBasedRotationStrategy', |
91 | @@ -111,14 +107,29 @@ def configure_graylog(): |
92 | @when('elasticsearch.available') |
93 | @when_not('graylog.needs_restart') |
94 | @when_not('graylog_api.configured') |
95 | -def configure_graylog_via_api(*discard): |
96 | - conf = hookenv.config() |
97 | +def configure_graylog_api(*discard): |
98 | db = unitdata.kv() |
99 | admin_password = db.get('admin_password') |
100 | api_token = api_retrieve_token(API_URL, auth=('admin', admin_password)) |
101 | if not api_token: |
102 | return |
103 | - index_sets = api_retrieve_index_sets(API_URL, auth=(api_token, 'token')) |
104 | + if db.get('api_token') != api_token: |
105 | + db.set('api_token', api_token) |
106 | + remove_state('graylog_index_sets.configured') |
107 | + remove_state('graylog_inputs.configured') |
108 | + set_state('graylog_api.configured') |
109 | + |
110 | + |
111 | +@when('graylog_api.configured') |
112 | +@when_not('graylog_index_sets.configured') |
113 | +def configure_index_sets(*discard): |
114 | + conf = hookenv.config() |
115 | + db = unitdata.kv() |
116 | + api_token = db.get('api_token') |
117 | + if not api_token: |
118 | + return |
119 | + url = urljoin(API_URL, 'system/indices/index_sets') |
120 | + index_sets = api_request(url, auth=(api_token, 'token'))['index_sets'] |
121 | if not index_sets: |
122 | return |
123 | for iset in index_sets: |
124 | @@ -141,7 +152,154 @@ def configure_graylog_via_api(*discard): |
125 | data=json.dumps(iset, indent=True)): |
126 | return |
127 | |
128 | - set_state('graylog_api.configured') |
129 | + set_state('graylog_index_sets.configured') |
130 | + |
131 | + |
132 | +def _map_input_type(input_types, type): |
133 | + """ |
134 | + Map specified int types to Gray's input type objects. Examples: |
135 | + |
136 | + "Beats" -> org.graylog.plugins.beats.BeatsInput |
137 | + "Raw TCP" -> org.graylog2.inputs.raw.tcp.RawTCPInput |
138 | + "Syslog UDP" -> org.graylog2.inputs.syslog.udp.SyslogUDPInput |
139 | + """ |
140 | + for k, v in input_types.items(): |
141 | + if v.lower().find(type.lower()) != -1: |
142 | + return k |
143 | + # For raw input types, map something like "Raw TCP" and "Raw UDP" |
144 | + # instead of requiring the exact name, "Raw/Plaintext TCP". |
145 | + if k.lower().find('.'.join(type.split()).lower()) != -1: |
146 | + return k |
147 | + return None |
148 | + |
149 | + |
150 | +def _check_input_exists(current_inputs, new): |
151 | + """ |
152 | + Given the current list of configured Graylog inputs, check if new input |
153 | + exists and return an ID as well as "changed" flag to indicate that the |
154 | + new input provided has changed (input type, bind address, and port). |
155 | + """ |
156 | + changed = True |
157 | + cur = None |
158 | + for input in current_inputs: |
159 | + if input['title'] == new['title']: |
160 | + cur = input |
161 | + break |
162 | + if not cur: |
163 | + return (True, None) |
164 | + current = [cur['type'], cur['attributes']['bind_address'], str(cur['attributes']['port'])] |
165 | + new = [new['type'], new['configuration']['bind_address'], str(new['configuration']['port'])] |
166 | + if sorted(new) == sorted(current): |
167 | + changed = False |
168 | + return (changed, cur['id']) |
169 | + |
170 | + |
171 | +def _close_uneeded_ports(current_inputs, new): |
172 | + for cur in current_inputs: |
173 | + title = cur['title'] |
174 | + if title.lower().startswith('custom'): |
175 | + continue |
176 | + type = cur['type'] |
177 | + if 'udp' in type.lower(): |
178 | + proto = 'UDP' |
179 | + else: |
180 | + proto = 'TCP' |
181 | + port = cur['attributes']['port'] |
182 | + if '{}/{}'.format(port, proto) not in new: |
183 | + hookenv.close_port(port, proto) |
184 | + |
185 | + |
186 | +def _remove_old_inputs(inputs, new_inputs): |
187 | + """ |
188 | + Removes any old/previously defined inputs, or any unknown. We'll keep those |
189 | + prefixed with 'Custom' to allow configuring additional inputs via the web |
190 | + UI. |
191 | + """ |
192 | + db = unitdata.kv() |
193 | + api_token = db.get('api_token') |
194 | + success = True |
195 | + for cur in inputs: |
196 | + title = cur['title'] |
197 | + if title.lower().startswith('custom'): |
198 | + continue |
199 | + if title in new_inputs: |
200 | + continue |
201 | + url = urljoin(urljoin(API_URL, 'system/inputs/'), cur['id']) |
202 | + if api_request(url, auth=(api_token, 'token'), method='DELETE'): |
203 | + hookenv.log('Removed old input: {} ({})'.format(cur['title'], cur['id'])) |
204 | + else: |
205 | + success = False |
206 | + return success |
207 | + |
208 | + |
209 | +@when('graylog_api.configured') |
210 | +@when_not('graylog_inputs.configured') |
211 | +def configure_inputs(*discard): |
212 | + """ |
213 | + Configure log inputs in Graylog via the API. |
214 | + """ |
215 | + conf = hookenv.config() |
216 | + db = unitdata.kv() |
217 | + api_token = db.get('api_token') |
218 | + if not api_token: |
219 | + return |
220 | + url = urljoin(API_URL, 'system/inputs') |
221 | + inputs = api_request(url, auth=(api_token, 'token'))['inputs'] |
222 | + url = urljoin(API_URL, 'system/inputs/types') |
223 | + input_types = api_request(url, auth=(api_token, 'token'))['types'] |
224 | + new_opened_ports = [] |
225 | + new_inputs = [] |
226 | + for new in yaml.safe_load(conf['log_inputs']) or {}: |
227 | + type = _map_input_type(input_types, new['type']) |
228 | + if not type: |
229 | + hookenv.log('Input type "{}" not supported'.format(new['type'])) |
230 | + continue |
231 | + d = { |
232 | + 'title': new['name'], |
233 | + 'type': type, |
234 | + 'global': "true", |
235 | + 'configuration': { |
236 | + 'bind_address': new['bind_address'], |
237 | + 'port': new['bind_port'], |
238 | + } |
239 | + } |
240 | + if 'udp' in type.lower(): |
241 | + proto = 'UDP' |
242 | + else: |
243 | + proto = 'TCP' |
244 | + d['configuration']['tcp_keepalive'] = 'true' |
245 | + |
246 | + hookenv.open_port(new['bind_port'], proto) |
247 | + new_opened_ports.append('{}/{}'.format(new['bind_port'], proto)) |
248 | + |
249 | + (changed, input_id) = _check_input_exists(inputs, d) |
250 | + if not changed: |
251 | + new_inputs.append(new['name']) |
252 | + continue |
253 | + if input_id: |
254 | + change_text = 'Updated existing input' |
255 | + method = 'PUT' |
256 | + url = urljoin(urljoin(API_URL, 'system/inputs/'), input_id) |
257 | + else: |
258 | + change_text = 'Adding new input' |
259 | + method = 'POST' |
260 | + url = urljoin(API_URL, 'system/inputs/') |
261 | + ret = api_request(url, auth=(api_token, 'token'), method=method, |
262 | + data=json.dumps(d, indent=True)) |
263 | + if not ret: |
264 | + # Can't add input, let's try again later |
265 | + return |
266 | + hookenv.log('{}: {} ({})'.format(change_text, new['name'], ret['id'])) |
267 | + new_inputs.append(new['name']) |
268 | + |
269 | + # Now remove inputs that should no longer be there as well as close ports |
270 | + _close_uneeded_ports(inputs, new_opened_ports) |
271 | + url = urljoin(API_URL, 'system/inputs') |
272 | + refreshed_inputs = api_request(url, auth=(api_token, 'token'))['inputs'] |
273 | + if not _remove_old_inputs(refreshed_inputs, new_inputs): |
274 | + # Can't delete input, let's try again later |
275 | + return |
276 | + set_state('graylog_inputs.configured') |
277 | |
278 | |
279 | @when('graylog.configured') |
280 | diff --git a/unit_tests/graylog.py b/unit_tests/graylog.py |
281 | index 0ee4134..2b16c78 100644 |
282 | --- a/unit_tests/graylog.py |
283 | +++ b/unit_tests/graylog.py |
284 | @@ -5,7 +5,7 @@ import unittest |
285 | |
286 | sys.path.append(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) |
287 | |
288 | -from reactive.graylog import set_conf |
289 | +from reactive.graylog import set_conf, _map_input_type, _check_input_exists |
290 | |
291 | |
292 | initial_conf = u"""#key1 = value1 |
293 | @@ -38,6 +38,25 @@ key2 = value6 |
294 | ### key3 = value 4 |
295 | key4 = value5""" |
296 | |
297 | +log_input_types = { |
298 | + "org.graylog.plugins.beats.BeatsInput": "Beats", |
299 | + "org.graylog2.inputs.raw.udp.RawUDPInput": "Raw/Plaintext UDP", |
300 | +} |
301 | + |
302 | +log_inputs = [{ |
303 | + "title": "Beats Input", |
304 | + "global": "true", |
305 | + "name": "Beats", |
306 | + "created_at": "2017-08-17T10:19:10.634Z", |
307 | + "type": "org.graylog.plugins.beats.BeatsInput", |
308 | + "creator_user_id": "admin", |
309 | + "attributes": { |
310 | + "bind_address": "0.0.0.0", |
311 | + "port": 11044 |
312 | + }, |
313 | + "id": "5994f3aab0d37f23c38adbda" |
314 | +}] |
315 | + |
316 | |
317 | class TestSetConf(unittest.TestCase): |
318 | |
319 | @@ -81,3 +100,48 @@ class TestSetConf(unittest.TestCase): |
320 | |
321 | def test_changed(self): |
322 | self.assertTrue(set_conf('key2', 'some-new-value', self.conf_file)) |
323 | + |
324 | + def test_input_type_exists(self): |
325 | + self.assertTrue(_map_input_type(log_input_types, 'beats')) |
326 | + |
327 | + def test_input_type_raw(self): |
328 | + self.assertTrue(_map_input_type(log_input_types, 'raw udp')) |
329 | + |
330 | + def test_input_type_invalid(self): |
331 | + self.assertFalse(_map_input_type(log_input_types, 'some-weird-type')) |
332 | + |
333 | + def test_input_exists(self): |
334 | + new_input = { |
335 | + "title": "Beats Input", |
336 | + "type": "org.graylog.plugins.beats.BeatsInput", |
337 | + "configuration": { |
338 | + "bind_address": "0.0.0.0", |
339 | + "port": 11044 |
340 | + } |
341 | + } |
342 | + (changed, input_id) = _check_input_exists(log_inputs, new_input) |
343 | + self.assertTrue(input_id == "5994f3aab0d37f23c38adbda" and changed is False) |
344 | + |
345 | + def test_input_changed(self): |
346 | + new_input = { |
347 | + "title": "Beats Input", |
348 | + "type": "org.graylog.plugins.beats.BeatsInput", |
349 | + "configuration": { |
350 | + "bind_address": "0.0.0.0", |
351 | + "port": 12222 |
352 | + } |
353 | + } |
354 | + (changed, input_id) = _check_input_exists(log_inputs, new_input) |
355 | + self.assertTrue(changed is True) |
356 | + |
357 | + def test_input_new(self): |
358 | + new_input = { |
359 | + "title": "Beats Input 2", |
360 | + "type": "org.graylog.plugins.beats.BeatsInput", |
361 | + "configuration": { |
362 | + "bind_address": "0.0.0.0", |
363 | + "port": 11033 |
364 | + } |
365 | + } |
366 | + (changed, input_id) = _check_input_exists(log_inputs, new_input) |
367 | + self.assertTrue(input_id is None) |
Nice new functionality!
I think the implementation can be improved with some unit tests and by pulling all of the Graylog API code into a library, ideally one that can be shared between the reactive code and the amulet tests.
Also this is pretty significant new functionality, the Amulet tests should be extended to cover setting and verifying log_inputs.