Merge lp:~junaidali/charms/trusty/plumgrid-gateway/liberty into lp:~plumgrid-team/charms/trusty/plumgrid-gateway/trunk

Proposed by Junaid Ali on 2016-04-18
Status: Merged
Merged at revision: 27
Proposed branch: lp:~junaidali/charms/trusty/plumgrid-gateway/liberty
Merge into: lp:~plumgrid-team/charms/trusty/plumgrid-gateway/trunk
Diff against target: 9964 lines (+7315/-601)
61 files modified
bin/charm_helpers_sync.py (+253/-0)
hooks/charmhelpers/contrib/amulet/deployment.py (+4/-2)
hooks/charmhelpers/contrib/amulet/utils.py (+382/-86)
hooks/charmhelpers/contrib/charmsupport/nrpe.py (+52/-14)
hooks/charmhelpers/contrib/hardening/__init__.py (+15/-0)
hooks/charmhelpers/contrib/hardening/apache/__init__.py (+19/-0)
hooks/charmhelpers/contrib/hardening/apache/checks/__init__.py (+31/-0)
hooks/charmhelpers/contrib/hardening/apache/checks/config.py (+100/-0)
hooks/charmhelpers/contrib/hardening/audits/__init__.py (+63/-0)
hooks/charmhelpers/contrib/hardening/audits/apache.py (+100/-0)
hooks/charmhelpers/contrib/hardening/audits/apt.py (+105/-0)
hooks/charmhelpers/contrib/hardening/audits/file.py (+552/-0)
hooks/charmhelpers/contrib/hardening/harden.py (+84/-0)
hooks/charmhelpers/contrib/hardening/host/__init__.py (+19/-0)
hooks/charmhelpers/contrib/hardening/host/checks/__init__.py (+50/-0)
hooks/charmhelpers/contrib/hardening/host/checks/apt.py (+39/-0)
hooks/charmhelpers/contrib/hardening/host/checks/limits.py (+55/-0)
hooks/charmhelpers/contrib/hardening/host/checks/login.py (+67/-0)
hooks/charmhelpers/contrib/hardening/host/checks/minimize_access.py (+52/-0)
hooks/charmhelpers/contrib/hardening/host/checks/pam.py (+134/-0)
hooks/charmhelpers/contrib/hardening/host/checks/profile.py (+45/-0)
hooks/charmhelpers/contrib/hardening/host/checks/securetty.py (+39/-0)
hooks/charmhelpers/contrib/hardening/host/checks/suid_sgid.py (+131/-0)
hooks/charmhelpers/contrib/hardening/host/checks/sysctl.py (+211/-0)
hooks/charmhelpers/contrib/hardening/mysql/__init__.py (+19/-0)
hooks/charmhelpers/contrib/hardening/mysql/checks/__init__.py (+31/-0)
hooks/charmhelpers/contrib/hardening/mysql/checks/config.py (+89/-0)
hooks/charmhelpers/contrib/hardening/ssh/__init__.py (+19/-0)
hooks/charmhelpers/contrib/hardening/ssh/checks/__init__.py (+31/-0)
hooks/charmhelpers/contrib/hardening/ssh/checks/config.py (+394/-0)
hooks/charmhelpers/contrib/hardening/templating.py (+71/-0)
hooks/charmhelpers/contrib/hardening/utils.py (+157/-0)
hooks/charmhelpers/contrib/mellanox/infiniband.py (+151/-0)
hooks/charmhelpers/contrib/network/ip.py (+55/-23)
hooks/charmhelpers/contrib/network/ovs/__init__.py (+6/-2)
hooks/charmhelpers/contrib/network/ufw.py (+5/-6)
hooks/charmhelpers/contrib/openstack/amulet/deployment.py (+135/-14)
hooks/charmhelpers/contrib/openstack/amulet/utils.py (+421/-13)
hooks/charmhelpers/contrib/openstack/context.py (+318/-79)
hooks/charmhelpers/contrib/openstack/ip.py (+35/-7)
hooks/charmhelpers/contrib/openstack/neutron.py (+62/-21)
hooks/charmhelpers/contrib/openstack/templating.py (+30/-2)
hooks/charmhelpers/contrib/openstack/utils.py (+939/-70)
hooks/charmhelpers/contrib/peerstorage/__init__.py (+5/-4)
hooks/charmhelpers/contrib/python/packages.py (+35/-11)
hooks/charmhelpers/contrib/storage/linux/ceph.py (+823/-61)
hooks/charmhelpers/contrib/storage/linux/loopback.py (+10/-0)
hooks/charmhelpers/contrib/storage/linux/utils.py (+8/-7)
hooks/charmhelpers/contrib/templating/jinja.py (+4/-3)
hooks/charmhelpers/core/hookenv.py (+220/-13)
hooks/charmhelpers/core/host.py (+298/-75)
hooks/charmhelpers/core/hugepage.py (+71/-0)
hooks/charmhelpers/core/kernel.py (+68/-0)
hooks/charmhelpers/core/services/helpers.py (+30/-5)
hooks/charmhelpers/core/strutils.py (+30/-0)
hooks/charmhelpers/core/templating.py (+21/-8)
hooks/charmhelpers/core/unitdata.py (+61/-17)
hooks/charmhelpers/fetch/__init__.py (+18/-2)
hooks/charmhelpers/fetch/archiveurl.py (+1/-1)
hooks/charmhelpers/fetch/bzrurl.py (+22/-32)
hooks/charmhelpers/fetch/giturl.py (+20/-23)
To merge this branch: bzr merge lp:~junaidali/charms/trusty/plumgrid-gateway/liberty
Reviewer Review Type Date Requested Status
Bilal Baqar 2016-04-18 Approve on 2016-04-25
Review via email: mp+292149@code.launchpad.net
To post a comment you must log in.
28. By Junaid Ali on 2016-04-22

make sync

Bilal Baqar (bbaqar) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== added directory 'bin'
2=== added file 'bin/charm_helpers_sync.py'
3--- bin/charm_helpers_sync.py 1970-01-01 00:00:00 +0000
4+++ bin/charm_helpers_sync.py 2016-04-22 08:16:14 +0000
5@@ -0,0 +1,253 @@
6+#!/usr/bin/python
7+
8+# Copyright 2014-2015 Canonical Limited.
9+#
10+# This file is part of charm-helpers.
11+#
12+# charm-helpers is free software: you can redistribute it and/or modify
13+# it under the terms of the GNU Lesser General Public License version 3 as
14+# published by the Free Software Foundation.
15+#
16+# charm-helpers is distributed in the hope that it will be useful,
17+# but WITHOUT ANY WARRANTY; without even the implied warranty of
18+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19+# GNU Lesser General Public License for more details.
20+#
21+# You should have received a copy of the GNU Lesser General Public License
22+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
23+
24+# Authors:
25+# Adam Gandelman <adamg@ubuntu.com>
26+
27+import logging
28+import optparse
29+import os
30+import subprocess
31+import shutil
32+import sys
33+import tempfile
34+import yaml
35+from fnmatch import fnmatch
36+
37+import six
38+
39+CHARM_HELPERS_BRANCH = 'lp:charm-helpers'
40+
41+
42+def parse_config(conf_file):
43+ if not os.path.isfile(conf_file):
44+ logging.error('Invalid config file: %s.' % conf_file)
45+ return False
46+ return yaml.load(open(conf_file).read())
47+
48+
49+def clone_helpers(work_dir, branch):
50+ dest = os.path.join(work_dir, 'charm-helpers')
51+ logging.info('Checking out %s to %s.' % (branch, dest))
52+ cmd = ['bzr', 'checkout', '--lightweight', branch, dest]
53+ subprocess.check_call(cmd)
54+ return dest
55+
56+
57+def _module_path(module):
58+ return os.path.join(*module.split('.'))
59+
60+
61+def _src_path(src, module):
62+ return os.path.join(src, 'charmhelpers', _module_path(module))
63+
64+
65+def _dest_path(dest, module):
66+ return os.path.join(dest, _module_path(module))
67+
68+
69+def _is_pyfile(path):
70+ return os.path.isfile(path + '.py')
71+
72+
73+def ensure_init(path):
74+ '''
75+ ensure directories leading up to path are importable, omitting
76+ parent directory, eg path='/hooks/helpers/foo'/:
77+ hooks/
78+ hooks/helpers/__init__.py
79+ hooks/helpers/foo/__init__.py
80+ '''
81+ for d, dirs, files in os.walk(os.path.join(*path.split('/')[:2])):
82+ _i = os.path.join(d, '__init__.py')
83+ if not os.path.exists(_i):
84+ logging.info('Adding missing __init__.py: %s' % _i)
85+ open(_i, 'wb').close()
86+
87+
88+def sync_pyfile(src, dest):
89+ src = src + '.py'
90+ src_dir = os.path.dirname(src)
91+ logging.info('Syncing pyfile: %s -> %s.' % (src, dest))
92+ if not os.path.exists(dest):
93+ os.makedirs(dest)
94+ shutil.copy(src, dest)
95+ if os.path.isfile(os.path.join(src_dir, '__init__.py')):
96+ shutil.copy(os.path.join(src_dir, '__init__.py'),
97+ dest)
98+ ensure_init(dest)
99+
100+
101+def get_filter(opts=None):
102+ opts = opts or []
103+ if 'inc=*' in opts:
104+ # do not filter any files, include everything
105+ return None
106+
107+ def _filter(dir, ls):
108+ incs = [opt.split('=').pop() for opt in opts if 'inc=' in opt]
109+ _filter = []
110+ for f in ls:
111+ _f = os.path.join(dir, f)
112+
113+ if not os.path.isdir(_f) and not _f.endswith('.py') and incs:
114+ if True not in [fnmatch(_f, inc) for inc in incs]:
115+ logging.debug('Not syncing %s, does not match include '
116+ 'filters (%s)' % (_f, incs))
117+ _filter.append(f)
118+ else:
119+ logging.debug('Including file, which matches include '
120+ 'filters (%s): %s' % (incs, _f))
121+ elif (os.path.isfile(_f) and not _f.endswith('.py')):
122+ logging.debug('Not syncing file: %s' % f)
123+ _filter.append(f)
124+ elif (os.path.isdir(_f) and not
125+ os.path.isfile(os.path.join(_f, '__init__.py'))):
126+ logging.debug('Not syncing directory: %s' % f)
127+ _filter.append(f)
128+ return _filter
129+ return _filter
130+
131+
132+def sync_directory(src, dest, opts=None):
133+ if os.path.exists(dest):
134+ logging.debug('Removing existing directory: %s' % dest)
135+ shutil.rmtree(dest)
136+ logging.info('Syncing directory: %s -> %s.' % (src, dest))
137+
138+ shutil.copytree(src, dest, ignore=get_filter(opts))
139+ ensure_init(dest)
140+
141+
142+def sync(src, dest, module, opts=None):
143+
144+ # Sync charmhelpers/__init__.py for bootstrap code.
145+ sync_pyfile(_src_path(src, '__init__'), dest)
146+
147+ # Sync other __init__.py files in the path leading to module.
148+ m = []
149+ steps = module.split('.')[:-1]
150+ while steps:
151+ m.append(steps.pop(0))
152+ init = '.'.join(m + ['__init__'])
153+ sync_pyfile(_src_path(src, init),
154+ os.path.dirname(_dest_path(dest, init)))
155+
156+ # Sync the module, or maybe a .py file.
157+ if os.path.isdir(_src_path(src, module)):
158+ sync_directory(_src_path(src, module), _dest_path(dest, module), opts)
159+ elif _is_pyfile(_src_path(src, module)):
160+ sync_pyfile(_src_path(src, module),
161+ os.path.dirname(_dest_path(dest, module)))
162+ else:
163+ logging.warn('Could not sync: %s. Neither a pyfile or directory, '
164+ 'does it even exist?' % module)
165+
166+
167+def parse_sync_options(options):
168+ if not options:
169+ return []
170+ return options.split(',')
171+
172+
173+def extract_options(inc, global_options=None):
174+ global_options = global_options or []
175+ if global_options and isinstance(global_options, six.string_types):
176+ global_options = [global_options]
177+ if '|' not in inc:
178+ return (inc, global_options)
179+ inc, opts = inc.split('|')
180+ return (inc, parse_sync_options(opts) + global_options)
181+
182+
183+def sync_helpers(include, src, dest, options=None):
184+ if not os.path.isdir(dest):
185+ os.makedirs(dest)
186+
187+ global_options = parse_sync_options(options)
188+
189+ for inc in include:
190+ if isinstance(inc, str):
191+ inc, opts = extract_options(inc, global_options)
192+ sync(src, dest, inc, opts)
193+ elif isinstance(inc, dict):
194+ # could also do nested dicts here.
195+ for k, v in six.iteritems(inc):
196+ if isinstance(v, list):
197+ for m in v:
198+ inc, opts = extract_options(m, global_options)
199+ sync(src, dest, '%s.%s' % (k, inc), opts)
200+
201+if __name__ == '__main__':
202+ parser = optparse.OptionParser()
203+ parser.add_option('-c', '--config', action='store', dest='config',
204+ default=None, help='helper config file')
205+ parser.add_option('-D', '--debug', action='store_true', dest='debug',
206+ default=False, help='debug')
207+ parser.add_option('-b', '--branch', action='store', dest='branch',
208+ help='charm-helpers bzr branch (overrides config)')
209+ parser.add_option('-d', '--destination', action='store', dest='dest_dir',
210+ help='sync destination dir (overrides config)')
211+ (opts, args) = parser.parse_args()
212+
213+ if opts.debug:
214+ logging.basicConfig(level=logging.DEBUG)
215+ else:
216+ logging.basicConfig(level=logging.INFO)
217+
218+ if opts.config:
219+ logging.info('Loading charm helper config from %s.' % opts.config)
220+ config = parse_config(opts.config)
221+ if not config:
222+ logging.error('Could not parse config from %s.' % opts.config)
223+ sys.exit(1)
224+ else:
225+ config = {}
226+
227+ if 'branch' not in config:
228+ config['branch'] = CHARM_HELPERS_BRANCH
229+ if opts.branch:
230+ config['branch'] = opts.branch
231+ if opts.dest_dir:
232+ config['destination'] = opts.dest_dir
233+
234+ if 'destination' not in config:
235+ logging.error('No destination dir. specified as option or config.')
236+ sys.exit(1)
237+
238+ if 'include' not in config:
239+ if not args:
240+ logging.error('No modules to sync specified as option or config.')
241+ sys.exit(1)
242+ config['include'] = []
243+ [config['include'].append(a) for a in args]
244+
245+ sync_options = None
246+ if 'options' in config:
247+ sync_options = config['options']
248+ tmpd = tempfile.mkdtemp()
249+ try:
250+ checkout = clone_helpers(tmpd, config['branch'])
251+ sync_helpers(config['include'], checkout, config['destination'],
252+ options=sync_options)
253+ except Exception as e:
254+ logging.error("Could not sync: %s" % e)
255+ raise e
256+ finally:
257+ logging.debug('Cleaning up %s' % tmpd)
258+ shutil.rmtree(tmpd)
259
260=== modified file 'hooks/charmhelpers/contrib/amulet/deployment.py'
261--- hooks/charmhelpers/contrib/amulet/deployment.py 2015-07-29 18:23:55 +0000
262+++ hooks/charmhelpers/contrib/amulet/deployment.py 2016-04-22 08:16:14 +0000
263@@ -51,7 +51,8 @@
264 if 'units' not in this_service:
265 this_service['units'] = 1
266
267- self.d.add(this_service['name'], units=this_service['units'])
268+ self.d.add(this_service['name'], units=this_service['units'],
269+ constraints=this_service.get('constraints'))
270
271 for svc in other_services:
272 if 'location' in svc:
273@@ -64,7 +65,8 @@
274 if 'units' not in svc:
275 svc['units'] = 1
276
277- self.d.add(svc['name'], charm=branch_location, units=svc['units'])
278+ self.d.add(svc['name'], charm=branch_location, units=svc['units'],
279+ constraints=svc.get('constraints'))
280
281 def _add_relations(self, relations):
282 """Add all of the relations for the services."""
283
284=== modified file 'hooks/charmhelpers/contrib/amulet/utils.py'
285--- hooks/charmhelpers/contrib/amulet/utils.py 2015-07-29 18:23:55 +0000
286+++ hooks/charmhelpers/contrib/amulet/utils.py 2016-04-22 08:16:14 +0000
287@@ -14,17 +14,25 @@
288 # You should have received a copy of the GNU Lesser General Public License
289 # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
290
291-import amulet
292-import ConfigParser
293-import distro_info
294 import io
295+import json
296 import logging
297 import os
298 import re
299-import six
300+import socket
301+import subprocess
302 import sys
303 import time
304-import urlparse
305+import uuid
306+
307+import amulet
308+import distro_info
309+import six
310+from six.moves import configparser
311+if six.PY3:
312+ from urllib import parse as urlparse
313+else:
314+ import urlparse
315
316
317 class AmuletUtils(object):
318@@ -108,7 +116,7 @@
319 # /!\ DEPRECATION WARNING (beisner):
320 # New and existing tests should be rewritten to use
321 # validate_services_by_name() as it is aware of init systems.
322- self.log.warn('/!\\ DEPRECATION WARNING: use '
323+ self.log.warn('DEPRECATION WARNING: use '
324 'validate_services_by_name instead of validate_services '
325 'due to init system differences.')
326
327@@ -142,19 +150,23 @@
328
329 for service_name in services_list:
330 if (self.ubuntu_releases.index(release) >= systemd_switch or
331- service_name == "rabbitmq-server"):
332- # init is systemd
333+ service_name in ['rabbitmq-server', 'apache2']):
334+ # init is systemd (or regular sysv)
335 cmd = 'sudo service {} status'.format(service_name)
336+ output, code = sentry_unit.run(cmd)
337+ service_running = code == 0
338 elif self.ubuntu_releases.index(release) < systemd_switch:
339 # init is upstart
340 cmd = 'sudo status {}'.format(service_name)
341+ output, code = sentry_unit.run(cmd)
342+ service_running = code == 0 and "start/running" in output
343
344- output, code = sentry_unit.run(cmd)
345 self.log.debug('{} `{}` returned '
346 '{}'.format(sentry_unit.info['unit_name'],
347 cmd, code))
348- if code != 0:
349- return "command `{}` returned {}".format(cmd, str(code))
350+ if not service_running:
351+ return u"command `{}` returned {} {}".format(
352+ cmd, output, str(code))
353 return None
354
355 def _get_config(self, unit, filename):
356@@ -164,7 +176,7 @@
357 # NOTE(beisner): by default, ConfigParser does not handle options
358 # with no value, such as the flags used in the mysql my.cnf file.
359 # https://bugs.python.org/issue7005
360- config = ConfigParser.ConfigParser(allow_no_value=True)
361+ config = configparser.ConfigParser(allow_no_value=True)
362 config.readfp(io.StringIO(file_contents))
363 return config
364
365@@ -259,33 +271,52 @@
366 """Get last modification time of directory."""
367 return sentry_unit.directory_stat(directory)['mtime']
368
369- def _get_proc_start_time(self, sentry_unit, service, pgrep_full=False):
370- """Get process' start time.
371-
372- Determine start time of the process based on the last modification
373- time of the /proc/pid directory. If pgrep_full is True, the process
374- name is matched against the full command line.
375- """
376- if pgrep_full:
377- cmd = 'pgrep -o -f {}'.format(service)
378- else:
379- cmd = 'pgrep -o {}'.format(service)
380- cmd = cmd + ' | grep -v pgrep || exit 0'
381- cmd_out = sentry_unit.run(cmd)
382- self.log.debug('CMDout: ' + str(cmd_out))
383- if cmd_out[0]:
384- self.log.debug('Pid for %s %s' % (service, str(cmd_out[0])))
385- proc_dir = '/proc/{}'.format(cmd_out[0].strip())
386- return self._get_dir_mtime(sentry_unit, proc_dir)
387+ def _get_proc_start_time(self, sentry_unit, service, pgrep_full=None):
388+ """Get start time of a process based on the last modification time
389+ of the /proc/pid directory.
390+
391+ :sentry_unit: The sentry unit to check for the service on
392+ :service: service name to look for in process table
393+ :pgrep_full: [Deprecated] Use full command line search mode with pgrep
394+ :returns: epoch time of service process start
395+ :param commands: list of bash commands
396+ :param sentry_units: list of sentry unit pointers
397+ :returns: None if successful; Failure message otherwise
398+ """
399+ if pgrep_full is not None:
400+ # /!\ DEPRECATION WARNING (beisner):
401+ # No longer implemented, as pidof is now used instead of pgrep.
402+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
403+ self.log.warn('DEPRECATION WARNING: pgrep_full bool is no '
404+ 'longer implemented re: lp 1474030.')
405+
406+ pid_list = self.get_process_id_list(sentry_unit, service)
407+ pid = pid_list[0]
408+ proc_dir = '/proc/{}'.format(pid)
409+ self.log.debug('Pid for {} on {}: {}'.format(
410+ service, sentry_unit.info['unit_name'], pid))
411+
412+ return self._get_dir_mtime(sentry_unit, proc_dir)
413
414 def service_restarted(self, sentry_unit, service, filename,
415- pgrep_full=False, sleep_time=20):
416+ pgrep_full=None, sleep_time=20):
417 """Check if service was restarted.
418
419 Compare a service's start time vs a file's last modification time
420 (such as a config file for that service) to determine if the service
421 has been restarted.
422 """
423+ # /!\ DEPRECATION WARNING (beisner):
424+ # This method is prone to races in that no before-time is known.
425+ # Use validate_service_config_changed instead.
426+
427+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
428+ # used instead of pgrep. pgrep_full is still passed through to ensure
429+ # deprecation WARNS. lp1474030
430+ self.log.warn('DEPRECATION WARNING: use '
431+ 'validate_service_config_changed instead of '
432+ 'service_restarted due to known races.')
433+
434 time.sleep(sleep_time)
435 if (self._get_proc_start_time(sentry_unit, service, pgrep_full) >=
436 self._get_file_mtime(sentry_unit, filename)):
437@@ -294,78 +325,122 @@
438 return False
439
440 def service_restarted_since(self, sentry_unit, mtime, service,
441- pgrep_full=False, sleep_time=20,
442- retry_count=2):
443+ pgrep_full=None, sleep_time=20,
444+ retry_count=30, retry_sleep_time=10):
445 """Check if service was been started after a given time.
446
447 Args:
448 sentry_unit (sentry): The sentry unit to check for the service on
449 mtime (float): The epoch time to check against
450 service (string): service name to look for in process table
451- pgrep_full (boolean): Use full command line search mode with pgrep
452- sleep_time (int): Seconds to sleep before looking for process
453- retry_count (int): If service is not found, how many times to retry
454+ pgrep_full: [Deprecated] Use full command line search mode with pgrep
455+ sleep_time (int): Initial sleep time (s) before looking for file
456+ retry_sleep_time (int): Time (s) to sleep between retries
457+ retry_count (int): If file is not found, how many times to retry
458
459 Returns:
460 bool: True if service found and its start time it newer than mtime,
461 False if service is older than mtime or if service was
462 not found.
463 """
464- self.log.debug('Checking %s restarted since %s' % (service, mtime))
465+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
466+ # used instead of pgrep. pgrep_full is still passed through to ensure
467+ # deprecation WARNS. lp1474030
468+
469+ unit_name = sentry_unit.info['unit_name']
470+ self.log.debug('Checking that %s service restarted since %s on '
471+ '%s' % (service, mtime, unit_name))
472 time.sleep(sleep_time)
473- proc_start_time = self._get_proc_start_time(sentry_unit, service,
474- pgrep_full)
475- while retry_count > 0 and not proc_start_time:
476- self.log.debug('No pid file found for service %s, will retry %i '
477- 'more times' % (service, retry_count))
478- time.sleep(30)
479- proc_start_time = self._get_proc_start_time(sentry_unit, service,
480- pgrep_full)
481- retry_count = retry_count - 1
482+ proc_start_time = None
483+ tries = 0
484+ while tries <= retry_count and not proc_start_time:
485+ try:
486+ proc_start_time = self._get_proc_start_time(sentry_unit,
487+ service,
488+ pgrep_full)
489+ self.log.debug('Attempt {} to get {} proc start time on {} '
490+ 'OK'.format(tries, service, unit_name))
491+ except IOError as e:
492+ # NOTE(beisner) - race avoidance, proc may not exist yet.
493+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
494+ self.log.debug('Attempt {} to get {} proc start time on {} '
495+ 'failed\n{}'.format(tries, service,
496+ unit_name, e))
497+ time.sleep(retry_sleep_time)
498+ tries += 1
499
500 if not proc_start_time:
501 self.log.warn('No proc start time found, assuming service did '
502 'not start')
503 return False
504 if proc_start_time >= mtime:
505- self.log.debug('proc start time is newer than provided mtime'
506- '(%s >= %s)' % (proc_start_time, mtime))
507+ self.log.debug('Proc start time is newer than provided mtime'
508+ '(%s >= %s) on %s (OK)' % (proc_start_time,
509+ mtime, unit_name))
510 return True
511 else:
512- self.log.warn('proc start time (%s) is older than provided mtime '
513- '(%s), service did not restart' % (proc_start_time,
514- mtime))
515+ self.log.warn('Proc start time (%s) is older than provided mtime '
516+ '(%s) on %s, service did not '
517+ 'restart' % (proc_start_time, mtime, unit_name))
518 return False
519
520 def config_updated_since(self, sentry_unit, filename, mtime,
521- sleep_time=20):
522+ sleep_time=20, retry_count=30,
523+ retry_sleep_time=10):
524 """Check if file was modified after a given time.
525
526 Args:
527 sentry_unit (sentry): The sentry unit to check the file mtime on
528 filename (string): The file to check mtime of
529 mtime (float): The epoch time to check against
530- sleep_time (int): Seconds to sleep before looking for process
531+ sleep_time (int): Initial sleep time (s) before looking for file
532+ retry_sleep_time (int): Time (s) to sleep between retries
533+ retry_count (int): If file is not found, how many times to retry
534
535 Returns:
536 bool: True if file was modified more recently than mtime, False if
537- file was modified before mtime,
538+ file was modified before mtime, or if file not found.
539 """
540- self.log.debug('Checking %s updated since %s' % (filename, mtime))
541+ unit_name = sentry_unit.info['unit_name']
542+ self.log.debug('Checking that %s updated since %s on '
543+ '%s' % (filename, mtime, unit_name))
544 time.sleep(sleep_time)
545- file_mtime = self._get_file_mtime(sentry_unit, filename)
546+ file_mtime = None
547+ tries = 0
548+ while tries <= retry_count and not file_mtime:
549+ try:
550+ file_mtime = self._get_file_mtime(sentry_unit, filename)
551+ self.log.debug('Attempt {} to get {} file mtime on {} '
552+ 'OK'.format(tries, filename, unit_name))
553+ except IOError as e:
554+ # NOTE(beisner) - race avoidance, file may not exist yet.
555+ # https://bugs.launchpad.net/charm-helpers/+bug/1474030
556+ self.log.debug('Attempt {} to get {} file mtime on {} '
557+ 'failed\n{}'.format(tries, filename,
558+ unit_name, e))
559+ time.sleep(retry_sleep_time)
560+ tries += 1
561+
562+ if not file_mtime:
563+ self.log.warn('Could not determine file mtime, assuming '
564+ 'file does not exist')
565+ return False
566+
567 if file_mtime >= mtime:
568 self.log.debug('File mtime is newer than provided mtime '
569- '(%s >= %s)' % (file_mtime, mtime))
570+ '(%s >= %s) on %s (OK)' % (file_mtime,
571+ mtime, unit_name))
572 return True
573 else:
574- self.log.warn('File mtime %s is older than provided mtime %s'
575- % (file_mtime, mtime))
576+ self.log.warn('File mtime is older than provided mtime'
577+ '(%s < on %s) on %s' % (file_mtime,
578+ mtime, unit_name))
579 return False
580
581 def validate_service_config_changed(self, sentry_unit, mtime, service,
582- filename, pgrep_full=False,
583- sleep_time=20, retry_count=2):
584+ filename, pgrep_full=None,
585+ sleep_time=20, retry_count=30,
586+ retry_sleep_time=10):
587 """Check service and file were updated after mtime
588
589 Args:
590@@ -373,9 +448,10 @@
591 mtime (float): The epoch time to check against
592 service (string): service name to look for in process table
593 filename (string): The file to check mtime of
594- pgrep_full (boolean): Use full command line search mode with pgrep
595- sleep_time (int): Seconds to sleep before looking for process
596+ pgrep_full: [Deprecated] Use full command line search mode with pgrep
597+ sleep_time (int): Initial sleep in seconds to pass to test helpers
598 retry_count (int): If service is not found, how many times to retry
599+ retry_sleep_time (int): Time in seconds to wait between retries
600
601 Typical Usage:
602 u = OpenStackAmuletUtils(ERROR)
603@@ -392,15 +468,27 @@
604 mtime, False if service is older than mtime or if service was
605 not found or if filename was modified before mtime.
606 """
607- self.log.debug('Checking %s restarted since %s' % (service, mtime))
608- time.sleep(sleep_time)
609- service_restart = self.service_restarted_since(sentry_unit, mtime,
610- service,
611- pgrep_full=pgrep_full,
612- sleep_time=0,
613- retry_count=retry_count)
614- config_update = self.config_updated_since(sentry_unit, filename, mtime,
615- sleep_time=0)
616+
617+ # NOTE(beisner) pgrep_full is no longer implemented, as pidof is now
618+ # used instead of pgrep. pgrep_full is still passed through to ensure
619+ # deprecation WARNS. lp1474030
620+
621+ service_restart = self.service_restarted_since(
622+ sentry_unit, mtime,
623+ service,
624+ pgrep_full=pgrep_full,
625+ sleep_time=sleep_time,
626+ retry_count=retry_count,
627+ retry_sleep_time=retry_sleep_time)
628+
629+ config_update = self.config_updated_since(
630+ sentry_unit,
631+ filename,
632+ mtime,
633+ sleep_time=sleep_time,
634+ retry_count=retry_count,
635+ retry_sleep_time=retry_sleep_time)
636+
637 return service_restart and config_update
638
639 def get_sentry_time(self, sentry_unit):
640@@ -418,7 +506,6 @@
641 """Return a list of all Ubuntu releases in order of release."""
642 _d = distro_info.UbuntuDistroInfo()
643 _release_list = _d.all
644- self.log.debug('Ubuntu release list: {}'.format(_release_list))
645 return _release_list
646
647 def file_to_url(self, file_rel_path):
648@@ -450,15 +537,20 @@
649 cmd, code, output))
650 return None
651
652- def get_process_id_list(self, sentry_unit, process_name):
653+ def get_process_id_list(self, sentry_unit, process_name,
654+ expect_success=True):
655 """Get a list of process ID(s) from a single sentry juju unit
656 for a single process name.
657
658- :param sentry_unit: Pointer to amulet sentry instance (juju unit)
659+ :param sentry_unit: Amulet sentry instance (juju unit)
660 :param process_name: Process name
661+ :param expect_success: If False, expect the PID to be missing,
662+ raise if it is present.
663 :returns: List of process IDs
664 """
665- cmd = 'pidof {}'.format(process_name)
666+ cmd = 'pidof -x {}'.format(process_name)
667+ if not expect_success:
668+ cmd += " || exit 0 && exit 1"
669 output, code = sentry_unit.run(cmd)
670 if code != 0:
671 msg = ('{} `{}` returned {} '
672@@ -467,14 +559,23 @@
673 amulet.raise_status(amulet.FAIL, msg=msg)
674 return str(output).split()
675
676- def get_unit_process_ids(self, unit_processes):
677+ def get_unit_process_ids(self, unit_processes, expect_success=True):
678 """Construct a dict containing unit sentries, process names, and
679- process IDs."""
680+ process IDs.
681+
682+ :param unit_processes: A dictionary of Amulet sentry instance
683+ to list of process names.
684+ :param expect_success: if False expect the processes to not be
685+ running, raise if they are.
686+ :returns: Dictionary of Amulet sentry instance to dictionary
687+ of process names to PIDs.
688+ """
689 pid_dict = {}
690- for sentry_unit, process_list in unit_processes.iteritems():
691+ for sentry_unit, process_list in six.iteritems(unit_processes):
692 pid_dict[sentry_unit] = {}
693 for process in process_list:
694- pids = self.get_process_id_list(sentry_unit, process)
695+ pids = self.get_process_id_list(
696+ sentry_unit, process, expect_success=expect_success)
697 pid_dict[sentry_unit].update({process: pids})
698 return pid_dict
699
700@@ -488,7 +589,7 @@
701 return ('Unit count mismatch. expected, actual: {}, '
702 '{} '.format(len(expected), len(actual)))
703
704- for (e_sentry, e_proc_names) in expected.iteritems():
705+ for (e_sentry, e_proc_names) in six.iteritems(expected):
706 e_sentry_name = e_sentry.info['unit_name']
707 if e_sentry in actual.keys():
708 a_proc_names = actual[e_sentry]
709@@ -500,22 +601,40 @@
710 return ('Process name count mismatch. expected, actual: {}, '
711 '{}'.format(len(expected), len(actual)))
712
713- for (e_proc_name, e_pids_length), (a_proc_name, a_pids) in \
714+ for (e_proc_name, e_pids), (a_proc_name, a_pids) in \
715 zip(e_proc_names.items(), a_proc_names.items()):
716 if e_proc_name != a_proc_name:
717 return ('Process name mismatch. expected, actual: {}, '
718 '{}'.format(e_proc_name, a_proc_name))
719
720 a_pids_length = len(a_pids)
721- if e_pids_length != a_pids_length:
722- return ('PID count mismatch. {} ({}) expected, actual: '
723+ fail_msg = ('PID count mismatch. {} ({}) expected, actual: '
724 '{}, {} ({})'.format(e_sentry_name, e_proc_name,
725- e_pids_length, a_pids_length,
726+ e_pids, a_pids_length,
727 a_pids))
728+
729+ # If expected is a list, ensure at least one PID quantity match
730+ if isinstance(e_pids, list) and \
731+ a_pids_length not in e_pids:
732+ return fail_msg
733+ # If expected is not bool and not list,
734+ # ensure PID quantities match
735+ elif not isinstance(e_pids, bool) and \
736+ not isinstance(e_pids, list) and \
737+ a_pids_length != e_pids:
738+ return fail_msg
739+ # If expected is bool True, ensure 1 or more PIDs exist
740+ elif isinstance(e_pids, bool) and \
741+ e_pids is True and a_pids_length < 1:
742+ return fail_msg
743+ # If expected is bool False, ensure 0 PIDs exist
744+ elif isinstance(e_pids, bool) and \
745+ e_pids is False and a_pids_length != 0:
746+ return fail_msg
747 else:
748 self.log.debug('PID check OK: {} {} {}: '
749 '{}'.format(e_sentry_name, e_proc_name,
750- e_pids_length, a_pids))
751+ e_pids, a_pids))
752 return None
753
754 def validate_list_of_identical_dicts(self, list_of_dicts):
755@@ -531,3 +650,180 @@
756 return 'Dicts within list are not identical'
757
758 return None
759+
760+ def validate_sectionless_conf(self, file_contents, expected):
761+ """A crude conf parser. Useful to inspect configuration files which
762+ do not have section headers (as would be necessary in order to use
763+ the configparser). Such as openstack-dashboard or rabbitmq confs."""
764+ for line in file_contents.split('\n'):
765+ if '=' in line:
766+ args = line.split('=')
767+ if len(args) <= 1:
768+ continue
769+ key = args[0].strip()
770+ value = args[1].strip()
771+ if key in expected.keys():
772+ if expected[key] != value:
773+ msg = ('Config mismatch. Expected, actual: {}, '
774+ '{}'.format(expected[key], value))
775+ amulet.raise_status(amulet.FAIL, msg=msg)
776+
777+ def get_unit_hostnames(self, units):
778+ """Return a dict of juju unit names to hostnames."""
779+ host_names = {}
780+ for unit in units:
781+ host_names[unit.info['unit_name']] = \
782+ str(unit.file_contents('/etc/hostname').strip())
783+ self.log.debug('Unit host names: {}'.format(host_names))
784+ return host_names
785+
786+ def run_cmd_unit(self, sentry_unit, cmd):
787+ """Run a command on a unit, return the output and exit code."""
788+ output, code = sentry_unit.run(cmd)
789+ if code == 0:
790+ self.log.debug('{} `{}` command returned {} '
791+ '(OK)'.format(sentry_unit.info['unit_name'],
792+ cmd, code))
793+ else:
794+ msg = ('{} `{}` command returned {} '
795+ '{}'.format(sentry_unit.info['unit_name'],
796+ cmd, code, output))
797+ amulet.raise_status(amulet.FAIL, msg=msg)
798+ return str(output), code
799+
800+ def file_exists_on_unit(self, sentry_unit, file_name):
801+ """Check if a file exists on a unit."""
802+ try:
803+ sentry_unit.file_stat(file_name)
804+ return True
805+ except IOError:
806+ return False
807+ except Exception as e:
808+ msg = 'Error checking file {}: {}'.format(file_name, e)
809+ amulet.raise_status(amulet.FAIL, msg=msg)
810+
811+ def file_contents_safe(self, sentry_unit, file_name,
812+ max_wait=60, fatal=False):
813+ """Get file contents from a sentry unit. Wrap amulet file_contents
814+ with retry logic to address races where a file checks as existing,
815+ but no longer exists by the time file_contents is called.
816+ Return None if file not found. Optionally raise if fatal is True."""
817+ unit_name = sentry_unit.info['unit_name']
818+ file_contents = False
819+ tries = 0
820+ while not file_contents and tries < (max_wait / 4):
821+ try:
822+ file_contents = sentry_unit.file_contents(file_name)
823+ except IOError:
824+ self.log.debug('Attempt {} to open file {} from {} '
825+ 'failed'.format(tries, file_name,
826+ unit_name))
827+ time.sleep(4)
828+ tries += 1
829+
830+ if file_contents:
831+ return file_contents
832+ elif not fatal:
833+ return None
834+ elif fatal:
835+ msg = 'Failed to get file contents from unit.'
836+ amulet.raise_status(amulet.FAIL, msg)
837+
838+ def port_knock_tcp(self, host="localhost", port=22, timeout=15):
839+ """Open a TCP socket to check for a listening sevice on a host.
840+
841+ :param host: host name or IP address, default to localhost
842+ :param port: TCP port number, default to 22
843+ :param timeout: Connect timeout, default to 15 seconds
844+ :returns: True if successful, False if connect failed
845+ """
846+
847+ # Resolve host name if possible
848+ try:
849+ connect_host = socket.gethostbyname(host)
850+ host_human = "{} ({})".format(connect_host, host)
851+ except socket.error as e:
852+ self.log.warn('Unable to resolve address: '
853+ '{} ({}) Trying anyway!'.format(host, e))
854+ connect_host = host
855+ host_human = connect_host
856+
857+ # Attempt socket connection
858+ try:
859+ knock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
860+ knock.settimeout(timeout)
861+ knock.connect((connect_host, port))
862+ knock.close()
863+ self.log.debug('Socket connect OK for host '
864+ '{} on port {}.'.format(host_human, port))
865+ return True
866+ except socket.error as e:
867+ self.log.debug('Socket connect FAIL for'
868+ ' {} port {} ({})'.format(host_human, port, e))
869+ return False
870+
871+ def port_knock_units(self, sentry_units, port=22,
872+ timeout=15, expect_success=True):
873+ """Open a TCP socket to check for a listening sevice on each
874+ listed juju unit.
875+
876+ :param sentry_units: list of sentry unit pointers
877+ :param port: TCP port number, default to 22
878+ :param timeout: Connect timeout, default to 15 seconds
879+ :expect_success: True by default, set False to invert logic
880+ :returns: None if successful, Failure message otherwise
881+ """
882+ for unit in sentry_units:
883+ host = unit.info['public-address']
884+ connected = self.port_knock_tcp(host, port, timeout)
885+ if not connected and expect_success:
886+ return 'Socket connect failed.'
887+ elif connected and not expect_success:
888+ return 'Socket connected unexpectedly.'
889+
890+ def get_uuid_epoch_stamp(self):
891+ """Returns a stamp string based on uuid4 and epoch time. Useful in
892+ generating test messages which need to be unique-ish."""
893+ return '[{}-{}]'.format(uuid.uuid4(), time.time())
894+
895+# amulet juju action helpers:
896+ def run_action(self, unit_sentry, action,
897+ _check_output=subprocess.check_output,
898+ params=None):
899+ """Run the named action on a given unit sentry.
900+
901+ params a dict of parameters to use
902+ _check_output parameter is used for dependency injection.
903+
904+ @return action_id.
905+ """
906+ unit_id = unit_sentry.info["unit_name"]
907+ command = ["juju", "action", "do", "--format=json", unit_id, action]
908+ if params is not None:
909+ for key, value in params.iteritems():
910+ command.append("{}={}".format(key, value))
911+ self.log.info("Running command: %s\n" % " ".join(command))
912+ output = _check_output(command, universal_newlines=True)
913+ data = json.loads(output)
914+ action_id = data[u'Action queued with id']
915+ return action_id
916+
917+ def wait_on_action(self, action_id, _check_output=subprocess.check_output):
918+ """Wait for a given action, returning if it completed or not.
919+
920+ _check_output parameter is used for dependency injection.
921+ """
922+ command = ["juju", "action", "fetch", "--format=json", "--wait=0",
923+ action_id]
924+ output = _check_output(command, universal_newlines=True)
925+ data = json.loads(output)
926+ return data.get(u"status") == "completed"
927+
928+ def status_get(self, unit):
929+ """Return the current service status of this unit."""
930+ raw_status, return_code = unit.run(
931+ "status-get --format=json --include-data")
932+ if return_code != 0:
933+ return ("unknown", "")
934+ status = json.loads(raw_status)
935+ return (status["status"], status["message"])
936
937=== modified file 'hooks/charmhelpers/contrib/charmsupport/nrpe.py'
938--- hooks/charmhelpers/contrib/charmsupport/nrpe.py 2015-07-29 18:23:55 +0000
939+++ hooks/charmhelpers/contrib/charmsupport/nrpe.py 2016-04-22 08:16:14 +0000
940@@ -148,6 +148,13 @@
941 self.description = description
942 self.check_cmd = self._locate_cmd(check_cmd)
943
944+ def _get_check_filename(self):
945+ return os.path.join(NRPE.nrpe_confdir, '{}.cfg'.format(self.command))
946+
947+ def _get_service_filename(self, hostname):
948+ return os.path.join(NRPE.nagios_exportdir,
949+ 'service__{}_{}.cfg'.format(hostname, self.command))
950+
951 def _locate_cmd(self, check_cmd):
952 search_path = (
953 '/usr/lib/nagios/plugins',
954@@ -163,9 +170,21 @@
955 log('Check command not found: {}'.format(parts[0]))
956 return ''
957
958+ def _remove_service_files(self):
959+ if not os.path.exists(NRPE.nagios_exportdir):
960+ return
961+ for f in os.listdir(NRPE.nagios_exportdir):
962+ if f.endswith('_{}.cfg'.format(self.command)):
963+ os.remove(os.path.join(NRPE.nagios_exportdir, f))
964+
965+ def remove(self, hostname):
966+ nrpe_check_file = self._get_check_filename()
967+ if os.path.exists(nrpe_check_file):
968+ os.remove(nrpe_check_file)
969+ self._remove_service_files()
970+
971 def write(self, nagios_context, hostname, nagios_servicegroups):
972- nrpe_check_file = '/etc/nagios/nrpe.d/{}.cfg'.format(
973- self.command)
974+ nrpe_check_file = self._get_check_filename()
975 with open(nrpe_check_file, 'w') as nrpe_check_config:
976 nrpe_check_config.write("# check {}\n".format(self.shortname))
977 nrpe_check_config.write("command[{}]={}\n".format(
978@@ -180,9 +199,7 @@
979
980 def write_service_config(self, nagios_context, hostname,
981 nagios_servicegroups):
982- for f in os.listdir(NRPE.nagios_exportdir):
983- if re.search('.*{}.cfg'.format(self.command), f):
984- os.remove(os.path.join(NRPE.nagios_exportdir, f))
985+ self._remove_service_files()
986
987 templ_vars = {
988 'nagios_hostname': hostname,
989@@ -192,8 +209,7 @@
990 'command': self.command,
991 }
992 nrpe_service_text = Check.service_template.format(**templ_vars)
993- nrpe_service_file = '{}/service__{}_{}.cfg'.format(
994- NRPE.nagios_exportdir, hostname, self.command)
995+ nrpe_service_file = self._get_service_filename(hostname)
996 with open(nrpe_service_file, 'w') as nrpe_service_config:
997 nrpe_service_config.write(str(nrpe_service_text))
998
999@@ -218,12 +234,32 @@
1000 if hostname:
1001 self.hostname = hostname
1002 else:
1003- self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
1004+ nagios_hostname = get_nagios_hostname()
1005+ if nagios_hostname:
1006+ self.hostname = nagios_hostname
1007+ else:
1008+ self.hostname = "{}-{}".format(self.nagios_context, self.unit_name)
1009 self.checks = []
1010
1011 def add_check(self, *args, **kwargs):
1012 self.checks.append(Check(*args, **kwargs))
1013
1014+ def remove_check(self, *args, **kwargs):
1015+ if kwargs.get('shortname') is None:
1016+ raise ValueError('shortname of check must be specified')
1017+
1018+ # Use sensible defaults if they're not specified - these are not
1019+ # actually used during removal, but they're required for constructing
1020+ # the Check object; check_disk is chosen because it's part of the
1021+ # nagios-plugins-basic package.
1022+ if kwargs.get('check_cmd') is None:
1023+ kwargs['check_cmd'] = 'check_disk'
1024+ if kwargs.get('description') is None:
1025+ kwargs['description'] = ''
1026+
1027+ check = Check(*args, **kwargs)
1028+ check.remove(self.hostname)
1029+
1030 def write(self):
1031 try:
1032 nagios_uid = pwd.getpwnam('nagios').pw_uid
1033@@ -260,7 +296,7 @@
1034 :param str relation_name: Name of relation nrpe sub joined to
1035 """
1036 for rel in relations_of_type(relation_name):
1037- if 'nagios_hostname' in rel:
1038+ if 'nagios_host_context' in rel:
1039 return rel['nagios_host_context']
1040
1041
1042@@ -301,11 +337,13 @@
1043 upstart_init = '/etc/init/%s.conf' % svc
1044 sysv_init = '/etc/init.d/%s' % svc
1045 if os.path.exists(upstart_init):
1046- nrpe.add_check(
1047- shortname=svc,
1048- description='process check {%s}' % unit_name,
1049- check_cmd='check_upstart_job %s' % svc
1050- )
1051+ # Don't add a check for these services from neutron-gateway
1052+ if svc not in ['ext-port', 'os-charm-phy-nic-mtu']:
1053+ nrpe.add_check(
1054+ shortname=svc,
1055+ description='process check {%s}' % unit_name,
1056+ check_cmd='check_upstart_job %s' % svc
1057+ )
1058 elif os.path.exists(sysv_init):
1059 cronpath = '/etc/cron.d/nagios-service-check-%s' % svc
1060 cron_file = ('*/5 * * * * root '
1061
1062=== added directory 'hooks/charmhelpers/contrib/hardening'
1063=== added file 'hooks/charmhelpers/contrib/hardening/__init__.py'
1064--- hooks/charmhelpers/contrib/hardening/__init__.py 1970-01-01 00:00:00 +0000
1065+++ hooks/charmhelpers/contrib/hardening/__init__.py 2016-04-22 08:16:14 +0000
1066@@ -0,0 +1,15 @@
1067+# Copyright 2016 Canonical Limited.
1068+#
1069+# This file is part of charm-helpers.
1070+#
1071+# charm-helpers is free software: you can redistribute it and/or modify
1072+# it under the terms of the GNU Lesser General Public License version 3 as
1073+# published by the Free Software Foundation.
1074+#
1075+# charm-helpers is distributed in the hope that it will be useful,
1076+# but WITHOUT ANY WARRANTY; without even the implied warranty of
1077+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1078+# GNU Lesser General Public License for more details.
1079+#
1080+# You should have received a copy of the GNU Lesser General Public License
1081+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1082
1083=== added directory 'hooks/charmhelpers/contrib/hardening/apache'
1084=== added file 'hooks/charmhelpers/contrib/hardening/apache/__init__.py'
1085--- hooks/charmhelpers/contrib/hardening/apache/__init__.py 1970-01-01 00:00:00 +0000
1086+++ hooks/charmhelpers/contrib/hardening/apache/__init__.py 2016-04-22 08:16:14 +0000
1087@@ -0,0 +1,19 @@
1088+# Copyright 2016 Canonical Limited.
1089+#
1090+# This file is part of charm-helpers.
1091+#
1092+# charm-helpers is free software: you can redistribute it and/or modify
1093+# it under the terms of the GNU Lesser General Public License version 3 as
1094+# published by the Free Software Foundation.
1095+#
1096+# charm-helpers is distributed in the hope that it will be useful,
1097+# but WITHOUT ANY WARRANTY; without even the implied warranty of
1098+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1099+# GNU Lesser General Public License for more details.
1100+#
1101+# You should have received a copy of the GNU Lesser General Public License
1102+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1103+
1104+from os import path
1105+
1106+TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates')
1107
1108=== added directory 'hooks/charmhelpers/contrib/hardening/apache/checks'
1109=== added file 'hooks/charmhelpers/contrib/hardening/apache/checks/__init__.py'
1110--- hooks/charmhelpers/contrib/hardening/apache/checks/__init__.py 1970-01-01 00:00:00 +0000
1111+++ hooks/charmhelpers/contrib/hardening/apache/checks/__init__.py 2016-04-22 08:16:14 +0000
1112@@ -0,0 +1,31 @@
1113+# Copyright 2016 Canonical Limited.
1114+#
1115+# This file is part of charm-helpers.
1116+#
1117+# charm-helpers is free software: you can redistribute it and/or modify
1118+# it under the terms of the GNU Lesser General Public License version 3 as
1119+# published by the Free Software Foundation.
1120+#
1121+# charm-helpers is distributed in the hope that it will be useful,
1122+# but WITHOUT ANY WARRANTY; without even the implied warranty of
1123+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1124+# GNU Lesser General Public License for more details.
1125+#
1126+# You should have received a copy of the GNU Lesser General Public License
1127+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1128+
1129+from charmhelpers.core.hookenv import (
1130+ log,
1131+ DEBUG,
1132+)
1133+from charmhelpers.contrib.hardening.apache.checks import config
1134+
1135+
1136+def run_apache_checks():
1137+ log("Starting Apache hardening checks.", level=DEBUG)
1138+ checks = config.get_audits()
1139+ for check in checks:
1140+ log("Running '%s' check" % (check.__class__.__name__), level=DEBUG)
1141+ check.ensure_compliance()
1142+
1143+ log("Apache hardening checks complete.", level=DEBUG)
1144
1145=== added file 'hooks/charmhelpers/contrib/hardening/apache/checks/config.py'
1146--- hooks/charmhelpers/contrib/hardening/apache/checks/config.py 1970-01-01 00:00:00 +0000
1147+++ hooks/charmhelpers/contrib/hardening/apache/checks/config.py 2016-04-22 08:16:14 +0000
1148@@ -0,0 +1,100 @@
1149+# Copyright 2016 Canonical Limited.
1150+#
1151+# This file is part of charm-helpers.
1152+#
1153+# charm-helpers is free software: you can redistribute it and/or modify
1154+# it under the terms of the GNU Lesser General Public License version 3 as
1155+# published by the Free Software Foundation.
1156+#
1157+# charm-helpers is distributed in the hope that it will be useful,
1158+# but WITHOUT ANY WARRANTY; without even the implied warranty of
1159+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1160+# GNU Lesser General Public License for more details.
1161+#
1162+# You should have received a copy of the GNU Lesser General Public License
1163+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1164+
1165+import os
1166+import re
1167+import subprocess
1168+
1169+
1170+from charmhelpers.core.hookenv import (
1171+ log,
1172+ INFO,
1173+)
1174+from charmhelpers.contrib.hardening.audits.file import (
1175+ FilePermissionAudit,
1176+ DirectoryPermissionAudit,
1177+ NoReadWriteForOther,
1178+ TemplatedFile,
1179+)
1180+from charmhelpers.contrib.hardening.audits.apache import DisabledModuleAudit
1181+from charmhelpers.contrib.hardening.apache import TEMPLATES_DIR
1182+from charmhelpers.contrib.hardening import utils
1183+
1184+
1185+def get_audits():
1186+ """Get Apache hardening config audits.
1187+
1188+ :returns: dictionary of audits
1189+ """
1190+ if subprocess.call(['which', 'apache2'], stdout=subprocess.PIPE) != 0:
1191+ log("Apache server does not appear to be installed on this node - "
1192+ "skipping apache hardening", level=INFO)
1193+ return []
1194+
1195+ context = ApacheConfContext()
1196+ settings = utils.get_settings('apache')
1197+ audits = [
1198+ FilePermissionAudit(paths='/etc/apache2/apache2.conf', user='root',
1199+ group='root', mode=0o0640),
1200+
1201+ TemplatedFile(os.path.join(settings['common']['apache_dir'],
1202+ 'mods-available/alias.conf'),
1203+ context,
1204+ TEMPLATES_DIR,
1205+ mode=0o0755,
1206+ user='root',
1207+ service_actions=[{'service': 'apache2',
1208+ 'actions': ['restart']}]),
1209+
1210+ TemplatedFile(os.path.join(settings['common']['apache_dir'],
1211+ 'conf-enabled/hardening.conf'),
1212+ context,
1213+ TEMPLATES_DIR,
1214+ mode=0o0640,
1215+ user='root',
1216+ service_actions=[{'service': 'apache2',
1217+ 'actions': ['restart']}]),
1218+
1219+ DirectoryPermissionAudit(settings['common']['apache_dir'],
1220+ user='root',
1221+ group='root',
1222+ mode=0o640),
1223+
1224+ DisabledModuleAudit(settings['hardening']['modules_to_disable']),
1225+
1226+ NoReadWriteForOther(settings['common']['apache_dir']),
1227+ ]
1228+
1229+ return audits
1230+
1231+
1232+class ApacheConfContext(object):
1233+ """Defines the set of key/value pairs to set in a apache config file.
1234+
1235+ This context, when called, will return a dictionary containing the
1236+ key/value pairs of setting to specify in the
1237+ /etc/apache/conf-enabled/hardening.conf file.
1238+ """
1239+ def __call__(self):
1240+ settings = utils.get_settings('apache')
1241+ ctxt = settings['hardening']
1242+
1243+ out = subprocess.check_output(['apache2', '-v'])
1244+ ctxt['apache_version'] = re.search(r'.+version: Apache/(.+?)\s.+',
1245+ out).group(1)
1246+ ctxt['apache_icondir'] = '/usr/share/apache2/icons/'
1247+ ctxt['traceenable'] = settings['hardening']['traceenable']
1248+ return ctxt
1249
1250=== added directory 'hooks/charmhelpers/contrib/hardening/audits'
1251=== added file 'hooks/charmhelpers/contrib/hardening/audits/__init__.py'
1252--- hooks/charmhelpers/contrib/hardening/audits/__init__.py 1970-01-01 00:00:00 +0000
1253+++ hooks/charmhelpers/contrib/hardening/audits/__init__.py 2016-04-22 08:16:14 +0000
1254@@ -0,0 +1,63 @@
1255+# Copyright 2016 Canonical Limited.
1256+#
1257+# This file is part of charm-helpers.
1258+#
1259+# charm-helpers is free software: you can redistribute it and/or modify
1260+# it under the terms of the GNU Lesser General Public License version 3 as
1261+# published by the Free Software Foundation.
1262+#
1263+# charm-helpers is distributed in the hope that it will be useful,
1264+# but WITHOUT ANY WARRANTY; without even the implied warranty of
1265+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1266+# GNU Lesser General Public License for more details.
1267+#
1268+# You should have received a copy of the GNU Lesser General Public License
1269+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1270+
1271+
1272+class BaseAudit(object): # NO-QA
1273+ """Base class for hardening checks.
1274+
1275+ The lifecycle of a hardening check is to first check to see if the system
1276+ is in compliance for the specified check. If it is not in compliance, the
1277+ check method will return a value which will be supplied to the.
1278+ """
1279+ def __init__(self, *args, **kwargs):
1280+ self.unless = kwargs.get('unless', None)
1281+ super(BaseAudit, self).__init__()
1282+
1283+ def ensure_compliance(self):
1284+ """Checks to see if the current hardening check is in compliance or
1285+ not.
1286+
1287+ If the check that is performed is not in compliance, then an exception
1288+ should be raised.
1289+ """
1290+ pass
1291+
1292+ def _take_action(self):
1293+ """Determines whether to perform the action or not.
1294+
1295+ Checks whether or not an action should be taken. This is determined by
1296+ the truthy value for the unless parameter. If unless is a callback
1297+ method, it will be invoked with no parameters in order to determine
1298+ whether or not the action should be taken. Otherwise, the truthy value
1299+ of the unless attribute will determine if the action should be
1300+ performed.
1301+ """
1302+ # Do the action if there isn't an unless override.
1303+ if self.unless is None:
1304+ return True
1305+
1306+ # Invoke the callback if there is one.
1307+ if hasattr(self.unless, '__call__'):
1308+ results = self.unless()
1309+ if results:
1310+ return False
1311+ else:
1312+ return True
1313+
1314+ if self.unless:
1315+ return False
1316+ else:
1317+ return True
1318
1319=== added file 'hooks/charmhelpers/contrib/hardening/audits/apache.py'
1320--- hooks/charmhelpers/contrib/hardening/audits/apache.py 1970-01-01 00:00:00 +0000
1321+++ hooks/charmhelpers/contrib/hardening/audits/apache.py 2016-04-22 08:16:14 +0000
1322@@ -0,0 +1,100 @@
1323+# Copyright 2016 Canonical Limited.
1324+#
1325+# This file is part of charm-helpers.
1326+#
1327+# charm-helpers is free software: you can redistribute it and/or modify
1328+# it under the terms of the GNU Lesser General Public License version 3 as
1329+# published by the Free Software Foundation.
1330+#
1331+# charm-helpers is distributed in the hope that it will be useful,
1332+# but WITHOUT ANY WARRANTY; without even the implied warranty of
1333+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1334+# GNU Lesser General Public License for more details.
1335+#
1336+# You should have received a copy of the GNU Lesser General Public License
1337+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1338+
1339+import re
1340+import subprocess
1341+
1342+from six import string_types
1343+
1344+from charmhelpers.core.hookenv import (
1345+ log,
1346+ INFO,
1347+ ERROR,
1348+)
1349+
1350+from charmhelpers.contrib.hardening.audits import BaseAudit
1351+
1352+
1353+class DisabledModuleAudit(BaseAudit):
1354+ """Audits Apache2 modules.
1355+
1356+ Determines if the apache2 modules are enabled. If the modules are enabled
1357+ then they are removed in the ensure_compliance.
1358+ """
1359+ def __init__(self, modules):
1360+ if modules is None:
1361+ self.modules = []
1362+ elif isinstance(modules, string_types):
1363+ self.modules = [modules]
1364+ else:
1365+ self.modules = modules
1366+
1367+ def ensure_compliance(self):
1368+ """Ensures that the modules are not loaded."""
1369+ if not self.modules:
1370+ return
1371+
1372+ try:
1373+ loaded_modules = self._get_loaded_modules()
1374+ non_compliant_modules = []
1375+ for module in self.modules:
1376+ if module in loaded_modules:
1377+ log("Module '%s' is enabled but should not be." %
1378+ (module), level=INFO)
1379+ non_compliant_modules.append(module)
1380+
1381+ if len(non_compliant_modules) == 0:
1382+ return
1383+
1384+ for module in non_compliant_modules:
1385+ self._disable_module(module)
1386+ self._restart_apache()
1387+ except subprocess.CalledProcessError as e:
1388+ log('Error occurred auditing apache module compliance. '
1389+ 'This may have been already reported. '
1390+ 'Output is: %s' % e.output, level=ERROR)
1391+
1392+ @staticmethod
1393+ def _get_loaded_modules():
1394+ """Returns the modules which are enabled in Apache."""
1395+ output = subprocess.check_output(['apache2ctl', '-M'])
1396+ modules = []
1397+ for line in output.strip().split():
1398+ # Each line of the enabled module output looks like:
1399+ # module_name (static|shared)
1400+ # Plus a header line at the top of the output which is stripped
1401+ # out by the regex.
1402+ matcher = re.search(r'^ (\S*)', line)
1403+ if matcher:
1404+ modules.append(matcher.group(1))
1405+ return modules
1406+
1407+ @staticmethod
1408+ def _disable_module(module):
1409+ """Disables the specified module in Apache."""
1410+ try:
1411+ subprocess.check_call(['a2dismod', module])
1412+ except subprocess.CalledProcessError as e:
1413+ # Note: catch error here to allow the attempt of disabling
1414+ # multiple modules in one go rather than failing after the
1415+ # first module fails.
1416+ log('Error occurred disabling module %s. '
1417+ 'Output is: %s' % (module, e.output), level=ERROR)
1418+
1419+ @staticmethod
1420+ def _restart_apache():
1421+ """Restarts the apache process"""
1422+ subprocess.check_output(['service', 'apache2', 'restart'])
1423
1424=== added file 'hooks/charmhelpers/contrib/hardening/audits/apt.py'
1425--- hooks/charmhelpers/contrib/hardening/audits/apt.py 1970-01-01 00:00:00 +0000
1426+++ hooks/charmhelpers/contrib/hardening/audits/apt.py 2016-04-22 08:16:14 +0000
1427@@ -0,0 +1,105 @@
1428+# Copyright 2016 Canonical Limited.
1429+#
1430+# This file is part of charm-helpers.
1431+#
1432+# charm-helpers is free software: you can redistribute it and/or modify
1433+# it under the terms of the GNU Lesser General Public License version 3 as
1434+# published by the Free Software Foundation.
1435+#
1436+# charm-helpers is distributed in the hope that it will be useful,
1437+# but WITHOUT ANY WARRANTY; without even the implied warranty of
1438+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1439+# GNU Lesser General Public License for more details.
1440+#
1441+# You should have received a copy of the GNU Lesser General Public License
1442+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1443+
1444+from __future__ import absolute_import # required for external apt import
1445+from apt import apt_pkg
1446+from six import string_types
1447+
1448+from charmhelpers.fetch import (
1449+ apt_cache,
1450+ apt_purge
1451+)
1452+from charmhelpers.core.hookenv import (
1453+ log,
1454+ DEBUG,
1455+ WARNING,
1456+)
1457+from charmhelpers.contrib.hardening.audits import BaseAudit
1458+
1459+
1460+class AptConfig(BaseAudit):
1461+
1462+ def __init__(self, config, **kwargs):
1463+ self.config = config
1464+
1465+ def verify_config(self):
1466+ apt_pkg.init()
1467+ for cfg in self.config:
1468+ value = apt_pkg.config.get(cfg['key'], cfg.get('default', ''))
1469+ if value and value != cfg['expected']:
1470+ log("APT config '%s' has unexpected value '%s' "
1471+ "(expected='%s')" %
1472+ (cfg['key'], value, cfg['expected']), level=WARNING)
1473+
1474+ def ensure_compliance(self):
1475+ self.verify_config()
1476+
1477+
1478+class RestrictedPackages(BaseAudit):
1479+ """Class used to audit restricted packages on the system."""
1480+
1481+ def __init__(self, pkgs, **kwargs):
1482+ super(RestrictedPackages, self).__init__(**kwargs)
1483+ if isinstance(pkgs, string_types) or not hasattr(pkgs, '__iter__'):
1484+ self.pkgs = [pkgs]
1485+ else:
1486+ self.pkgs = pkgs
1487+
1488+ def ensure_compliance(self):
1489+ cache = apt_cache()
1490+
1491+ for p in self.pkgs:
1492+ if p not in cache:
1493+ continue
1494+
1495+ pkg = cache[p]
1496+ if not self.is_virtual_package(pkg):
1497+ if not pkg.current_ver:
1498+ log("Package '%s' is not installed." % pkg.name,
1499+ level=DEBUG)
1500+ continue
1501+ else:
1502+ log("Restricted package '%s' is installed" % pkg.name,
1503+ level=WARNING)
1504+ self.delete_package(cache, pkg)
1505+ else:
1506+ log("Checking restricted virtual package '%s' provides" %
1507+ pkg.name, level=DEBUG)
1508+ self.delete_package(cache, pkg)
1509+
1510+ def delete_package(self, cache, pkg):
1511+ """Deletes the package from the system.
1512+
1513+ Deletes the package form the system, properly handling virtual
1514+ packages.
1515+
1516+ :param cache: the apt cache
1517+ :param pkg: the package to remove
1518+ """
1519+ if self.is_virtual_package(pkg):
1520+ log("Package '%s' appears to be virtual - purging provides" %
1521+ pkg.name, level=DEBUG)
1522+ for _p in pkg.provides_list:
1523+ self.delete_package(cache, _p[2].parent_pkg)
1524+ elif not pkg.current_ver:
1525+ log("Package '%s' not installed" % pkg.name, level=DEBUG)
1526+ return
1527+ else:
1528+ log("Purging package '%s'" % pkg.name, level=DEBUG)
1529+ apt_purge(pkg.name)
1530+
1531+ def is_virtual_package(self, pkg):
1532+ return pkg.has_provides and not pkg.has_versions
1533
1534=== added file 'hooks/charmhelpers/contrib/hardening/audits/file.py'
1535--- hooks/charmhelpers/contrib/hardening/audits/file.py 1970-01-01 00:00:00 +0000
1536+++ hooks/charmhelpers/contrib/hardening/audits/file.py 2016-04-22 08:16:14 +0000
1537@@ -0,0 +1,552 @@
1538+# Copyright 2016 Canonical Limited.
1539+#
1540+# This file is part of charm-helpers.
1541+#
1542+# charm-helpers is free software: you can redistribute it and/or modify
1543+# it under the terms of the GNU Lesser General Public License version 3 as
1544+# published by the Free Software Foundation.
1545+#
1546+# charm-helpers is distributed in the hope that it will be useful,
1547+# but WITHOUT ANY WARRANTY; without even the implied warranty of
1548+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
1549+# GNU Lesser General Public License for more details.
1550+#
1551+# You should have received a copy of the GNU Lesser General Public License
1552+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
1553+
1554+import grp
1555+import os
1556+import pwd
1557+import re
1558+
1559+from subprocess import (
1560+ CalledProcessError,
1561+ check_output,
1562+ check_call,
1563+)
1564+from traceback import format_exc
1565+from six import string_types
1566+from stat import (
1567+ S_ISGID,
1568+ S_ISUID
1569+)
1570+
1571+from charmhelpers.core.hookenv import (
1572+ log,
1573+ DEBUG,
1574+ INFO,
1575+ WARNING,
1576+ ERROR,
1577+)
1578+from charmhelpers.core import unitdata
1579+from charmhelpers.core.host import file_hash
1580+from charmhelpers.contrib.hardening.audits import BaseAudit
1581+from charmhelpers.contrib.hardening.templating import (
1582+ get_template_path,
1583+ render_and_write,
1584+)
1585+from charmhelpers.contrib.hardening import utils
1586+
1587+
1588+class BaseFileAudit(BaseAudit):
1589+ """Base class for file audits.
1590+
1591+ Provides api stubs for compliance check flow that must be used by any class
1592+ that implemented this one.
1593+ """
1594+
1595+ def __init__(self, paths, always_comply=False, *args, **kwargs):
1596+ """
1597+ :param paths: string path of list of paths of files we want to apply
1598+ compliance checks are criteria to.
1599+ :param always_comply: if true compliance criteria is always applied
1600+ else compliance is skipped for non-existent
1601+ paths.
1602+ """
1603+ super(BaseFileAudit, self).__init__(*args, **kwargs)
1604+ self.always_comply = always_comply
1605+ if isinstance(paths, string_types) or not hasattr(paths, '__iter__'):
1606+ self.paths = [paths]
1607+ else:
1608+ self.paths = paths
1609+
1610+ def ensure_compliance(self):
1611+ """Ensure that the all registered files comply to registered criteria.
1612+ """
1613+ for p in self.paths:
1614+ if os.path.exists(p):
1615+ if self.is_compliant(p):
1616+ continue
1617+
1618+ log('File %s is not in compliance.' % p, level=INFO)
1619+ else:
1620+ if not self.always_comply:
1621+ log("Non-existent path '%s' - skipping compliance check"
1622+ % (p), level=INFO)
1623+ continue
1624+
1625+ if self._take_action():
1626+ log("Applying compliance criteria to '%s'" % (p), level=INFO)
1627+ self.comply(p)
1628+
1629+ def is_compliant(self, path):
1630+ """Audits the path to see if it is compliance.
1631+
1632+ :param path: the path to the file that should be checked.
1633+ """
1634+ raise NotImplementedError
1635+
1636+ def comply(self, path):
1637+ """Enforces the compliance of a path.
1638+
1639+ :param path: the path to the file that should be enforced.
1640+ """
1641+ raise NotImplementedError
1642+
1643+ @classmethod
1644+ def _get_stat(cls, path):
1645+ """Returns the Posix st_stat information for the specified file path.
1646+
1647+ :param path: the path to get the st_stat information for.
1648+ :returns: an st_stat object for the path or None if the path doesn't
1649+ exist.
1650+ """
1651+ return os.stat(path)
1652+
1653+
1654+class FilePermissionAudit(BaseFileAudit):
1655+ """Implements an audit for file permissions and ownership for a user.
1656+
1657+ This class implements functionality that ensures that a specific user/group
1658+ will own the file(s) specified and that the permissions specified are
1659+ applied properly to the file.
1660+ """
1661+ def __init__(self, paths, user, group=None, mode=0o600, **kwargs):
1662+ self.user = user
1663+ self.group = group
1664+ self.mode = mode
1665+ super(FilePermissionAudit, self).__init__(paths, user, group, mode,
1666+ **kwargs)
1667+
1668+ @property
1669+ def user(self):
1670+ return self._user
1671+
1672+ @user.setter
1673+ def user(self, name):
1674+ try:
1675+ user = pwd.getpwnam(name)
1676+ except KeyError:
1677+ log('Unknown user %s' % name, level=ERROR)
1678+ user = None
1679+ self._user = user
1680+
1681+ @property
1682+ def group(self):
1683+ return self._group
1684+
1685+ @group.setter
1686+ def group(self, name):
1687+ try:
1688+ group = None
1689+ if name:
1690+ group = grp.getgrnam(name)
1691+ else:
1692+ group = grp.getgrgid(self.user.pw_gid)
1693+ except KeyError:
1694+ log('Unknown group %s' % name, level=ERROR)
1695+ self._group = group
1696+
1697+ def is_compliant(self, path):
1698+ """Checks if the path is in compliance.
1699+
1700+ Used to determine if the path specified meets the necessary
1701+ requirements to be in compliance with the check itself.
1702+
1703+ :param path: the file path to check
1704+ :returns: True if the path is compliant, False otherwise.
1705+ """
1706+ stat = self._get_stat(path)
1707+ user = self.user
1708+ group = self.group
1709+
1710+ compliant = True
1711+ if stat.st_uid != user.pw_uid or stat.st_gid != group.gr_gid:
1712+ log('File %s is not owned by %s:%s.' % (path, user.pw_name,
1713+ group.gr_name),
1714+ level=INFO)
1715+ compliant = False
1716+
1717+ # POSIX refers to the st_mode bits as corresponding to both the
1718+ # file type and file permission bits, where the least significant 12
1719+ # bits (o7777) are the suid (11), sgid (10), sticky bits (9), and the
1720+ # file permission bits (8-0)
1721+ perms = stat.st_mode & 0o7777
1722+ if perms != self.mode:
1723+ log('File %s has incorrect permissions, currently set to %s' %
1724+ (path, oct(stat.st_mode & 0o7777)), level=INFO)
1725+ compliant = False
1726+
1727+ return compliant
1728+
1729+ def comply(self, path):
1730+ """Issues a chown and chmod to the file paths specified."""
1731+ utils.ensure_permissions(path, self.user.pw_name, self.group.gr_name,
1732+ self.mode)
1733+
1734+
1735+class DirectoryPermissionAudit(FilePermissionAudit):
1736+ """Performs a permission check for the specified directory path."""
1737+
1738+ def __init__(self, paths, user, group=None, mode=0o600,
1739+ recursive=True, **kwargs):
1740+ super(DirectoryPermissionAudit, self).__init__(paths, user, group,
1741+ mode, **kwargs)
1742+ self.recursive = recursive
1743+
1744+ def is_compliant(self, path):
1745+ """Checks if the directory is compliant.
1746+
1747+ Used to determine if the path specified and all of its children
1748+ directories are in compliance with the check itself.
1749+
1750+ :param path: the directory path to check
1751+ :returns: True if the directory tree is compliant, otherwise False.
1752+ """
1753+ if not os.path.isdir(path):
1754+ log('Path specified %s is not a directory.' % path, level=ERROR)
1755+ raise ValueError("%s is not a directory." % path)
1756+
1757+ if not self.recursive:
1758+ return super(DirectoryPermissionAudit, self).is_compliant(path)
1759+
1760+ compliant = True
1761+ for root, dirs, _ in os.walk(path):
1762+ if len(dirs) > 0:
1763+ continue
1764+
1765+ if not super(DirectoryPermissionAudit, self).is_compliant(root):
1766+ compliant = False
1767+ continue
1768+
1769+ return compliant
1770+
1771+ def comply(self, path):
1772+ for root, dirs, _ in os.walk(path):
1773+ if len(dirs) > 0:
1774+ super(DirectoryPermissionAudit, self).comply(root)
1775+
1776+
1777+class ReadOnly(BaseFileAudit):
1778+ """Audits that files and folders are read only."""
1779+ def __init__(self, paths, *args, **kwargs):
1780+ super(ReadOnly, self).__init__(paths=paths, *args, **kwargs)
1781+
1782+ def is_compliant(self, path):
1783+ try:
1784+ output = check_output(['find', path, '-perm', '-go+w',
1785+ '-type', 'f']).strip()
1786+
1787+ # The find above will find any files which have permission sets
1788+ # which allow too broad of write access. As such, the path is
1789+ # compliant if there is no output.
1790+ if output:
1791+ return False
1792+
1793+ return True
1794+ except CalledProcessError as e:
1795+ log('Error occurred checking finding writable files for %s. '
1796+ 'Error information is: command %s failed with returncode '
1797+ '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output,
1798+ format_exc(e)), level=ERROR)
1799+ return False
1800+
1801+ def comply(self, path):
1802+ try:
1803+ check_output(['chmod', 'go-w', '-R', path])
1804+ except CalledProcessError as e:
1805+ log('Error occurred removing writeable permissions for %s. '
1806+ 'Error information is: command %s failed with returncode '
1807+ '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output,
1808+ format_exc(e)), level=ERROR)
1809+
1810+
1811+class NoReadWriteForOther(BaseFileAudit):
1812+ """Ensures that the files found under the base path are readable or
1813+ writable by anyone other than the owner or the group.
1814+ """
1815+ def __init__(self, paths):
1816+ super(NoReadWriteForOther, self).__init__(paths)
1817+
1818+ def is_compliant(self, path):
1819+ try:
1820+ cmd = ['find', path, '-perm', '-o+r', '-type', 'f', '-o',
1821+ '-perm', '-o+w', '-type', 'f']
1822+ output = check_output(cmd).strip()
1823+
1824+ # The find above here will find any files which have read or
1825+ # write permissions for other, meaning there is too broad of access
1826+ # to read/write the file. As such, the path is compliant if there's
1827+ # no output.
1828+ if output:
1829+ return False
1830+
1831+ return True
1832+ except CalledProcessError as e:
1833+ log('Error occurred while finding files which are readable or '
1834+ 'writable to the world in %s. '
1835+ 'Command output is: %s.' % (path, e.output), level=ERROR)
1836+
1837+ def comply(self, path):
1838+ try:
1839+ check_output(['chmod', '-R', 'o-rw', path])
1840+ except CalledProcessError as e:
1841+ log('Error occurred attempting to change modes of files under '
1842+ 'path %s. Output of command is: %s' % (path, e.output))
1843+
1844+
1845+class NoSUIDSGIDAudit(BaseFileAudit):
1846+ """Audits that specified files do not have SUID/SGID bits set."""
1847+ def __init__(self, paths, *args, **kwargs):
1848+ super(NoSUIDSGIDAudit, self).__init__(paths=paths, *args, **kwargs)
1849+
1850+ def is_compliant(self, path):
1851+ stat = self._get_stat(path)
1852+ if (stat.st_mode & (S_ISGID | S_ISUID)) != 0:
1853+ return False
1854+
1855+ return True
1856+
1857+ def comply(self, path):
1858+ try:
1859+ log('Removing suid/sgid from %s.' % path, level=DEBUG)
1860+ check_output(['chmod', '-s', path])
1861+ except CalledProcessError as e:
1862+ log('Error occurred removing suid/sgid from %s.'
1863+ 'Error information is: command %s failed with returncode '
1864+ '%d and output %s.\n%s' % (path, e.cmd, e.returncode, e.output,
1865+ format_exc(e)), level=ERROR)
1866+
1867+
1868+class TemplatedFile(BaseFileAudit):
1869+ """The TemplatedFileAudit audits the contents of a templated file.
1870+
1871+ This audit renders a file from a template, sets the appropriate file
1872+ permissions, then generates a hashsum with which to check the content
1873+ changed.
1874+ """
1875+ def __init__(self, path, context, template_dir, mode, user='root',
1876+ group='root', service_actions=None, **kwargs):
1877+ self.context = context
1878+ self.user = user
1879+ self.group = group
1880+ self.mode = mode
1881+ self.template_dir = template_dir
1882+ self.service_actions = service_actions
1883+ super(TemplatedFile, self).__init__(paths=path, always_comply=True,
1884+ **kwargs)
1885+
1886+ def is_compliant(self, path):
1887+ """Determines if the templated file is compliant.
1888+
1889+ A templated file is only compliant if it has not changed (as
1890+ determined by its sha256 hashsum) AND its file permissions are set
1891+ appropriately.
1892+
1893+ :param path: the path to check compliance.
1894+ """
1895+ same_templates = self.templates_match(path)
1896+ same_content = self.contents_match(path)
1897+ same_permissions = self.permissions_match(path)
1898+
1899+ if same_content and same_permissions and same_templates:
1900+ return True
1901+
1902+ return False
1903+
1904+ def run_service_actions(self):
1905+ """Run any actions on services requested."""
1906+ if not self.service_actions:
1907+ return
1908+
1909+ for svc_action in self.service_actions:
1910+ name = svc_action['service']
1911+ actions = svc_action['actions']
1912+ log("Running service '%s' actions '%s'" % (name, actions),
1913+ level=DEBUG)
1914+ for action in actions:
1915+ cmd = ['service', name, action]
1916+ try:
1917+ check_call(cmd)
1918+ except CalledProcessError as exc:
1919+ log("Service name='%s' action='%s' failed - %s" %
1920+ (name, action, exc), level=WARNING)
1921+
1922+ def comply(self, path):
1923+ """Ensures the contents and the permissions of the file.
1924+
1925+ :param path: the path to correct
1926+ """
1927+ dirname = os.path.dirname(path)
1928+ if not os.path.exists(dirname):
1929+ os.makedirs(dirname)
1930+
1931+ self.pre_write()
1932+ render_and_write(self.template_dir, path, self.context())
1933+ utils.ensure_permissions(path, self.user, self.group, self.mode)
1934+ self.run_service_actions()
1935+ self.save_checksum(path)
1936+ self.post_write()
1937+
1938+ def pre_write(self):
1939+ """Invoked prior to writing the template."""
1940+ pass
1941+
1942+ def post_write(self):
1943+ """Invoked after writing the template."""
1944+ pass
1945+
1946+ def templates_match(self, path):
1947+ """Determines if the template files are the same.
1948+
1949+ The template file equality is determined by the hashsum of the
1950+ template files themselves. If there is no hashsum, then the content
1951+ cannot be sure to be the same so treat it as if they changed.
1952+ Otherwise, return whether or not the hashsums are the same.
1953+
1954+ :param path: the path to check
1955+ :returns: boolean
1956+ """
1957+ template_path = get_template_path(self.template_dir, path)
1958+ key = 'hardening:template:%s' % template_path
1959+ template_checksum = file_hash(template_path)
1960+ kv = unitdata.kv()
1961+ stored_tmplt_checksum = kv.get(key)
1962+ if not stored_tmplt_checksum:
1963+ kv.set(key, template_checksum)
1964+ kv.flush()
1965+ log('Saved template checksum for %s.' % template_path,
1966+ level=DEBUG)
1967+ # Since we don't have a template checksum, then assume it doesn't
1968+ # match and return that the template is different.
1969+ return False
1970+ elif stored_tmplt_checksum != template_checksum:
1971+ kv.set(key, template_checksum)
1972+ kv.flush()
1973+ log('Updated template checksum for %s.' % template_path,
1974+ level=DEBUG)
1975+ return False
1976+
1977+ # Here the template hasn't changed based upon the calculated
1978+ # checksum of the template and what was previously stored.
1979+ return True
1980+
1981+ def contents_match(self, path):
1982+ """Determines if the file content is the same.
1983+
1984+ This is determined by comparing hashsum of the file contents and
1985+ the saved hashsum. If there is no hashsum, then the content cannot
1986+ be sure to be the same so treat them as if they are not the same.
1987+ Otherwise, return True if the hashsums are the same, False if they
1988+ are not the same.
1989+
1990+ :param path: the file to check.
1991+ """
1992+ checksum = file_hash(path)
1993+
1994+ kv = unitdata.kv()
1995+ stored_checksum = kv.get('hardening:%s' % path)
1996+ if not stored_checksum:
1997+ # If the checksum hasn't been generated, return False to ensure
1998+ # the file is written and the checksum stored.
1999+ log('Checksum for %s has not been calculated.' % path, level=DEBUG)
2000+ return False
2001+ elif stored_checksum != checksum:
2002+ log('Checksum mismatch for %s.' % path, level=DEBUG)
2003+ return False
2004+
2005+ return True
2006+
2007+ def permissions_match(self, path):
2008+ """Determines if the file owner and permissions match.
2009+
2010+ :param path: the path to check.
2011+ """
2012+ audit = FilePermissionAudit(path, self.user, self.group, self.mode)
2013+ return audit.is_compliant(path)
2014+
2015+ def save_checksum(self, path):
2016+ """Calculates and saves the checksum for the path specified.
2017+
2018+ :param path: the path of the file to save the checksum.
2019+ """
2020+ checksum = file_hash(path)
2021+ kv = unitdata.kv()
2022+ kv.set('hardening:%s' % path, checksum)
2023+ kv.flush()
2024+
2025+
2026+class DeletedFile(BaseFileAudit):
2027+ """Audit to ensure that a file is deleted."""
2028+ def __init__(self, paths):
2029+ super(DeletedFile, self).__init__(paths)
2030+
2031+ def is_compliant(self, path):
2032+ return not os.path.exists(path)
2033+
2034+ def comply(self, path):
2035+ os.remove(path)
2036+
2037+
2038+class FileContentAudit(BaseFileAudit):
2039+ """Audit the contents of a file."""
2040+ def __init__(self, paths, cases, **kwargs):
2041+ # Cases we expect to pass
2042+ self.pass_cases = cases.get('pass', [])
2043+ # Cases we expect to fail
2044+ self.fail_cases = cases.get('fail', [])
2045+ super(FileContentAudit, self).__init__(paths, **kwargs)
2046+
2047+ def is_compliant(self, path):
2048+ """
2049+ Given a set of content matching cases i.e. tuple(regex, bool) where
2050+ bool value denotes whether or not regex is expected to match, check that
2051+ all cases match as expected with the contents of the file. Cases can be
2052+ expected to pass of fail.
2053+
2054+ :param path: Path of file to check.
2055+ :returns: Boolean value representing whether or not all cases are
2056+ found to be compliant.
2057+ """
2058+ log("Auditing contents of file '%s'" % (path), level=DEBUG)
2059+ with open(path, 'r') as fd:
2060+ contents = fd.read()
2061+
2062+ matches = 0
2063+ for pattern in self.pass_cases:
2064+ key = re.compile(pattern, flags=re.MULTILINE)
2065+ results = re.search(key, contents)
2066+ if results:
2067+ matches += 1
2068+ else:
2069+ log("Pattern '%s' was expected to pass but instead it failed"
2070+ % (pattern), level=WARNING)
2071+
2072+ for pattern in self.fail_cases:
2073+ key = re.compile(pattern, flags=re.MULTILINE)
2074+ results = re.search(key, contents)
2075+ if not results:
2076+ matches += 1
2077+ else:
2078+ log("Pattern '%s' was expected to fail but instead it passed"
2079+ % (pattern), level=WARNING)
2080+
2081+ total = len(self.pass_cases) + len(self.fail_cases)
2082+ log("Checked %s cases and %s passed" % (total, matches), level=DEBUG)
2083+ return matches == total
2084+
2085+ def comply(self, *args, **kwargs):
2086+ """NOOP since we just issue warnings. This is to avoid the
2087+ NotImplememtedError.
2088+ """
2089+ log("Not applying any compliance criteria, only checks.", level=INFO)
2090
2091=== added file 'hooks/charmhelpers/contrib/hardening/harden.py'
2092--- hooks/charmhelpers/contrib/hardening/harden.py 1970-01-01 00:00:00 +0000
2093+++ hooks/charmhelpers/contrib/hardening/harden.py 2016-04-22 08:16:14 +0000
2094@@ -0,0 +1,84 @@
2095+# Copyright 2016 Canonical Limited.
2096+#
2097+# This file is part of charm-helpers.
2098+#
2099+# charm-helpers is free software: you can redistribute it and/or modify
2100+# it under the terms of the GNU Lesser General Public License version 3 as
2101+# published by the Free Software Foundation.
2102+#
2103+# charm-helpers is distributed in the hope that it will be useful,
2104+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2105+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2106+# GNU Lesser General Public License for more details.
2107+#
2108+# You should have received a copy of the GNU Lesser General Public License
2109+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2110+
2111+import six
2112+
2113+from collections import OrderedDict
2114+
2115+from charmhelpers.core.hookenv import (
2116+ config,
2117+ log,
2118+ DEBUG,
2119+ WARNING,
2120+)
2121+from charmhelpers.contrib.hardening.host.checks import run_os_checks
2122+from charmhelpers.contrib.hardening.ssh.checks import run_ssh_checks
2123+from charmhelpers.contrib.hardening.mysql.checks import run_mysql_checks
2124+from charmhelpers.contrib.hardening.apache.checks import run_apache_checks
2125+
2126+
2127+def harden(overrides=None):
2128+ """Hardening decorator.
2129+
2130+ This is the main entry point for running the hardening stack. In order to
2131+ run modules of the stack you must add this decorator to charm hook(s) and
2132+ ensure that your charm config.yaml contains the 'harden' option set to
2133+ one or more of the supported modules. Setting these will cause the
2134+ corresponding hardening code to be run when the hook fires.
2135+
2136+ This decorator can and should be applied to more than one hook or function
2137+ such that hardening modules are called multiple times. This is because
2138+ subsequent calls will perform auditing checks that will report any changes
2139+ to resources hardened by the first run (and possibly perform compliance
2140+ actions as a result of any detected infractions).
2141+
2142+ :param overrides: Optional list of stack modules used to override those
2143+ provided with 'harden' config.
2144+ :returns: Returns value returned by decorated function once executed.
2145+ """
2146+ def _harden_inner1(f):
2147+ log("Hardening function '%s'" % (f.__name__), level=DEBUG)
2148+
2149+ def _harden_inner2(*args, **kwargs):
2150+ RUN_CATALOG = OrderedDict([('os', run_os_checks),
2151+ ('ssh', run_ssh_checks),
2152+ ('mysql', run_mysql_checks),
2153+ ('apache', run_apache_checks)])
2154+
2155+ enabled = overrides or (config("harden") or "").split()
2156+ if enabled:
2157+ modules_to_run = []
2158+ # modules will always be performed in the following order
2159+ for module, func in six.iteritems(RUN_CATALOG):
2160+ if module in enabled:
2161+ enabled.remove(module)
2162+ modules_to_run.append(func)
2163+
2164+ if enabled:
2165+ log("Unknown hardening modules '%s' - ignoring" %
2166+ (', '.join(enabled)), level=WARNING)
2167+
2168+ for hardener in modules_to_run:
2169+ log("Executing hardening module '%s'" %
2170+ (hardener.__name__), level=DEBUG)
2171+ hardener()
2172+ else:
2173+ log("No hardening applied to '%s'" % (f.__name__), level=DEBUG)
2174+
2175+ return f(*args, **kwargs)
2176+ return _harden_inner2
2177+
2178+ return _harden_inner1
2179
2180=== added directory 'hooks/charmhelpers/contrib/hardening/host'
2181=== added file 'hooks/charmhelpers/contrib/hardening/host/__init__.py'
2182--- hooks/charmhelpers/contrib/hardening/host/__init__.py 1970-01-01 00:00:00 +0000
2183+++ hooks/charmhelpers/contrib/hardening/host/__init__.py 2016-04-22 08:16:14 +0000
2184@@ -0,0 +1,19 @@
2185+# Copyright 2016 Canonical Limited.
2186+#
2187+# This file is part of charm-helpers.
2188+#
2189+# charm-helpers is free software: you can redistribute it and/or modify
2190+# it under the terms of the GNU Lesser General Public License version 3 as
2191+# published by the Free Software Foundation.
2192+#
2193+# charm-helpers is distributed in the hope that it will be useful,
2194+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2195+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2196+# GNU Lesser General Public License for more details.
2197+#
2198+# You should have received a copy of the GNU Lesser General Public License
2199+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2200+
2201+from os import path
2202+
2203+TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates')
2204
2205=== added directory 'hooks/charmhelpers/contrib/hardening/host/checks'
2206=== added file 'hooks/charmhelpers/contrib/hardening/host/checks/__init__.py'
2207--- hooks/charmhelpers/contrib/hardening/host/checks/__init__.py 1970-01-01 00:00:00 +0000
2208+++ hooks/charmhelpers/contrib/hardening/host/checks/__init__.py 2016-04-22 08:16:14 +0000
2209@@ -0,0 +1,50 @@
2210+# Copyright 2016 Canonical Limited.
2211+#
2212+# This file is part of charm-helpers.
2213+#
2214+# charm-helpers is free software: you can redistribute it and/or modify
2215+# it under the terms of the GNU Lesser General Public License version 3 as
2216+# published by the Free Software Foundation.
2217+#
2218+# charm-helpers is distributed in the hope that it will be useful,
2219+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2220+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2221+# GNU Lesser General Public License for more details.
2222+#
2223+# You should have received a copy of the GNU Lesser General Public License
2224+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2225+
2226+from charmhelpers.core.hookenv import (
2227+ log,
2228+ DEBUG,
2229+)
2230+from charmhelpers.contrib.hardening.host.checks import (
2231+ apt,
2232+ limits,
2233+ login,
2234+ minimize_access,
2235+ pam,
2236+ profile,
2237+ securetty,
2238+ suid_sgid,
2239+ sysctl
2240+)
2241+
2242+
2243+def run_os_checks():
2244+ log("Starting OS hardening checks.", level=DEBUG)
2245+ checks = apt.get_audits()
2246+ checks.extend(limits.get_audits())
2247+ checks.extend(login.get_audits())
2248+ checks.extend(minimize_access.get_audits())
2249+ checks.extend(pam.get_audits())
2250+ checks.extend(profile.get_audits())
2251+ checks.extend(securetty.get_audits())
2252+ checks.extend(suid_sgid.get_audits())
2253+ checks.extend(sysctl.get_audits())
2254+
2255+ for check in checks:
2256+ log("Running '%s' check" % (check.__class__.__name__), level=DEBUG)
2257+ check.ensure_compliance()
2258+
2259+ log("OS hardening checks complete.", level=DEBUG)
2260
2261=== added file 'hooks/charmhelpers/contrib/hardening/host/checks/apt.py'
2262--- hooks/charmhelpers/contrib/hardening/host/checks/apt.py 1970-01-01 00:00:00 +0000
2263+++ hooks/charmhelpers/contrib/hardening/host/checks/apt.py 2016-04-22 08:16:14 +0000
2264@@ -0,0 +1,39 @@
2265+# Copyright 2016 Canonical Limited.
2266+#
2267+# This file is part of charm-helpers.
2268+#
2269+# charm-helpers is free software: you can redistribute it and/or modify
2270+# it under the terms of the GNU Lesser General Public License version 3 as
2271+# published by the Free Software Foundation.
2272+#
2273+# charm-helpers is distributed in the hope that it will be useful,
2274+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2275+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2276+# GNU Lesser General Public License for more details.
2277+#
2278+# You should have received a copy of the GNU Lesser General Public License
2279+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2280+
2281+from charmhelpers.contrib.hardening.utils import get_settings
2282+from charmhelpers.contrib.hardening.audits.apt import (
2283+ AptConfig,
2284+ RestrictedPackages,
2285+)
2286+
2287+
2288+def get_audits():
2289+ """Get OS hardening apt audits.
2290+
2291+ :returns: dictionary of audits
2292+ """
2293+ audits = [AptConfig([{'key': 'APT::Get::AllowUnauthenticated',
2294+ 'expected': 'false'}])]
2295+
2296+ settings = get_settings('os')
2297+ clean_packages = settings['security']['packages_clean']
2298+ if clean_packages:
2299+ security_packages = settings['security']['packages_list']
2300+ if security_packages:
2301+ audits.append(RestrictedPackages(security_packages))
2302+
2303+ return audits
2304
2305=== added file 'hooks/charmhelpers/contrib/hardening/host/checks/limits.py'
2306--- hooks/charmhelpers/contrib/hardening/host/checks/limits.py 1970-01-01 00:00:00 +0000
2307+++ hooks/charmhelpers/contrib/hardening/host/checks/limits.py 2016-04-22 08:16:14 +0000
2308@@ -0,0 +1,55 @@
2309+# Copyright 2016 Canonical Limited.
2310+#
2311+# This file is part of charm-helpers.
2312+#
2313+# charm-helpers is free software: you can redistribute it and/or modify
2314+# it under the terms of the GNU Lesser General Public License version 3 as
2315+# published by the Free Software Foundation.
2316+#
2317+# charm-helpers is distributed in the hope that it will be useful,
2318+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2319+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2320+# GNU Lesser General Public License for more details.
2321+#
2322+# You should have received a copy of the GNU Lesser General Public License
2323+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2324+
2325+from charmhelpers.contrib.hardening.audits.file import (
2326+ DirectoryPermissionAudit,
2327+ TemplatedFile,
2328+)
2329+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
2330+from charmhelpers.contrib.hardening import utils
2331+
2332+
2333+def get_audits():
2334+ """Get OS hardening security limits audits.
2335+
2336+ :returns: dictionary of audits
2337+ """
2338+ audits = []
2339+ settings = utils.get_settings('os')
2340+
2341+ # Ensure that the /etc/security/limits.d directory is only writable
2342+ # by the root user, but others can execute and read.
2343+ audits.append(DirectoryPermissionAudit('/etc/security/limits.d',
2344+ user='root', group='root',
2345+ mode=0o755))
2346+
2347+ # If core dumps are not enabled, then don't allow core dumps to be
2348+ # created as they may contain sensitive information.
2349+ if not settings['security']['kernel_enable_core_dump']:
2350+ audits.append(TemplatedFile('/etc/security/limits.d/10.hardcore.conf',
2351+ SecurityLimitsContext(),
2352+ template_dir=TEMPLATES_DIR,
2353+ user='root', group='root', mode=0o0440))
2354+ return audits
2355+
2356+
2357+class SecurityLimitsContext(object):
2358+
2359+ def __call__(self):
2360+ settings = utils.get_settings('os')
2361+ ctxt = {'disable_core_dump':
2362+ not settings['security']['kernel_enable_core_dump']}
2363+ return ctxt
2364
2365=== added file 'hooks/charmhelpers/contrib/hardening/host/checks/login.py'
2366--- hooks/charmhelpers/contrib/hardening/host/checks/login.py 1970-01-01 00:00:00 +0000
2367+++ hooks/charmhelpers/contrib/hardening/host/checks/login.py 2016-04-22 08:16:14 +0000
2368@@ -0,0 +1,67 @@
2369+# Copyright 2016 Canonical Limited.
2370+#
2371+# This file is part of charm-helpers.
2372+#
2373+# charm-helpers is free software: you can redistribute it and/or modify
2374+# it under the terms of the GNU Lesser General Public License version 3 as
2375+# published by the Free Software Foundation.
2376+#
2377+# charm-helpers is distributed in the hope that it will be useful,
2378+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2379+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2380+# GNU Lesser General Public License for more details.
2381+#
2382+# You should have received a copy of the GNU Lesser General Public License
2383+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2384+
2385+from six import string_types
2386+
2387+from charmhelpers.contrib.hardening.audits.file import TemplatedFile
2388+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
2389+from charmhelpers.contrib.hardening import utils
2390+
2391+
2392+def get_audits():
2393+ """Get OS hardening login.defs audits.
2394+
2395+ :returns: dictionary of audits
2396+ """
2397+ audits = [TemplatedFile('/etc/login.defs', LoginContext(),
2398+ template_dir=TEMPLATES_DIR,
2399+ user='root', group='root', mode=0o0444)]
2400+ return audits
2401+
2402+
2403+class LoginContext(object):
2404+
2405+ def __call__(self):
2406+ settings = utils.get_settings('os')
2407+
2408+ # Octal numbers in yaml end up being turned into decimal,
2409+ # so check if the umask is entered as a string (e.g. '027')
2410+ # or as an octal umask as we know it (e.g. 002). If its not
2411+ # a string assume it to be octal and turn it into an octal
2412+ # string.
2413+ umask = settings['environment']['umask']
2414+ if not isinstance(umask, string_types):
2415+ umask = '%s' % oct(umask)
2416+
2417+ ctxt = {
2418+ 'additional_user_paths':
2419+ settings['environment']['extra_user_paths'],
2420+ 'umask': umask,
2421+ 'pwd_max_age': settings['auth']['pw_max_age'],
2422+ 'pwd_min_age': settings['auth']['pw_min_age'],
2423+ 'uid_min': settings['auth']['uid_min'],
2424+ 'sys_uid_min': settings['auth']['sys_uid_min'],
2425+ 'sys_uid_max': settings['auth']['sys_uid_max'],
2426+ 'gid_min': settings['auth']['gid_min'],
2427+ 'sys_gid_min': settings['auth']['sys_gid_min'],
2428+ 'sys_gid_max': settings['auth']['sys_gid_max'],
2429+ 'login_retries': settings['auth']['retries'],
2430+ 'login_timeout': settings['auth']['timeout'],
2431+ 'chfn_restrict': settings['auth']['chfn_restrict'],
2432+ 'allow_login_without_home': settings['auth']['allow_homeless']
2433+ }
2434+
2435+ return ctxt
2436
2437=== added file 'hooks/charmhelpers/contrib/hardening/host/checks/minimize_access.py'
2438--- hooks/charmhelpers/contrib/hardening/host/checks/minimize_access.py 1970-01-01 00:00:00 +0000
2439+++ hooks/charmhelpers/contrib/hardening/host/checks/minimize_access.py 2016-04-22 08:16:14 +0000
2440@@ -0,0 +1,52 @@
2441+# Copyright 2016 Canonical Limited.
2442+#
2443+# This file is part of charm-helpers.
2444+#
2445+# charm-helpers is free software: you can redistribute it and/or modify
2446+# it under the terms of the GNU Lesser General Public License version 3 as
2447+# published by the Free Software Foundation.
2448+#
2449+# charm-helpers is distributed in the hope that it will be useful,
2450+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2451+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2452+# GNU Lesser General Public License for more details.
2453+#
2454+# You should have received a copy of the GNU Lesser General Public License
2455+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2456+
2457+from charmhelpers.contrib.hardening.audits.file import (
2458+ FilePermissionAudit,
2459+ ReadOnly,
2460+)
2461+from charmhelpers.contrib.hardening import utils
2462+
2463+
2464+def get_audits():
2465+ """Get OS hardening access audits.
2466+
2467+ :returns: dictionary of audits
2468+ """
2469+ audits = []
2470+ settings = utils.get_settings('os')
2471+
2472+ # Remove write permissions from $PATH folders for all regular users.
2473+ # This prevents changing system-wide commands from normal users.
2474+ path_folders = {'/usr/local/sbin',
2475+ '/usr/local/bin',
2476+ '/usr/sbin',
2477+ '/usr/bin',
2478+ '/bin'}
2479+ extra_user_paths = settings['environment']['extra_user_paths']
2480+ path_folders.update(extra_user_paths)
2481+ audits.append(ReadOnly(path_folders))
2482+
2483+ # Only allow the root user to have access to the shadow file.
2484+ audits.append(FilePermissionAudit('/etc/shadow', 'root', 'root', 0o0600))
2485+
2486+ if 'change_user' not in settings['security']['users_allow']:
2487+ # su should only be accessible to user and group root, unless it is
2488+ # expressly defined to allow users to change to root via the
2489+ # security_users_allow config option.
2490+ audits.append(FilePermissionAudit('/bin/su', 'root', 'root', 0o750))
2491+
2492+ return audits
2493
2494=== added file 'hooks/charmhelpers/contrib/hardening/host/checks/pam.py'
2495--- hooks/charmhelpers/contrib/hardening/host/checks/pam.py 1970-01-01 00:00:00 +0000
2496+++ hooks/charmhelpers/contrib/hardening/host/checks/pam.py 2016-04-22 08:16:14 +0000
2497@@ -0,0 +1,134 @@
2498+# Copyright 2016 Canonical Limited.
2499+#
2500+# This file is part of charm-helpers.
2501+#
2502+# charm-helpers is free software: you can redistribute it and/or modify
2503+# it under the terms of the GNU Lesser General Public License version 3 as
2504+# published by the Free Software Foundation.
2505+#
2506+# charm-helpers is distributed in the hope that it will be useful,
2507+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2508+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2509+# GNU Lesser General Public License for more details.
2510+#
2511+# You should have received a copy of the GNU Lesser General Public License
2512+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2513+
2514+from subprocess import (
2515+ check_output,
2516+ CalledProcessError,
2517+)
2518+
2519+from charmhelpers.core.hookenv import (
2520+ log,
2521+ DEBUG,
2522+ ERROR,
2523+)
2524+from charmhelpers.fetch import (
2525+ apt_install,
2526+ apt_purge,
2527+ apt_update,
2528+)
2529+from charmhelpers.contrib.hardening.audits.file import (
2530+ TemplatedFile,
2531+ DeletedFile,
2532+)
2533+from charmhelpers.contrib.hardening import utils
2534+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
2535+
2536+
2537+def get_audits():
2538+ """Get OS hardening PAM authentication audits.
2539+
2540+ :returns: dictionary of audits
2541+ """
2542+ audits = []
2543+
2544+ settings = utils.get_settings('os')
2545+
2546+ if settings['auth']['pam_passwdqc_enable']:
2547+ audits.append(PasswdqcPAM('/etc/passwdqc.conf'))
2548+
2549+ if settings['auth']['retries']:
2550+ audits.append(Tally2PAM('/usr/share/pam-configs/tally2'))
2551+ else:
2552+ audits.append(DeletedFile('/usr/share/pam-configs/tally2'))
2553+
2554+ return audits
2555+
2556+
2557+class PasswdqcPAMContext(object):
2558+
2559+ def __call__(self):
2560+ ctxt = {}
2561+ settings = utils.get_settings('os')
2562+
2563+ ctxt['auth_pam_passwdqc_options'] = \
2564+ settings['auth']['pam_passwdqc_options']
2565+
2566+ return ctxt
2567+
2568+
2569+class PasswdqcPAM(TemplatedFile):
2570+ """The PAM Audit verifies the linux PAM settings."""
2571+ def __init__(self, path):
2572+ super(PasswdqcPAM, self).__init__(path=path,
2573+ template_dir=TEMPLATES_DIR,
2574+ context=PasswdqcPAMContext(),
2575+ user='root',
2576+ group='root',
2577+ mode=0o0640)
2578+
2579+ def pre_write(self):
2580+ # Always remove?
2581+ for pkg in ['libpam-ccreds', 'libpam-cracklib']:
2582+ log("Purging package '%s'" % pkg, level=DEBUG),
2583+ apt_purge(pkg)
2584+
2585+ apt_update(fatal=True)
2586+ for pkg in ['libpam-passwdqc']:
2587+ log("Installing package '%s'" % pkg, level=DEBUG),
2588+ apt_install(pkg)
2589+
2590+ def post_write(self):
2591+ """Updates the PAM configuration after the file has been written"""
2592+ try:
2593+ check_output(['pam-auth-update', '--package'])
2594+ except CalledProcessError as e:
2595+ log('Error calling pam-auth-update: %s' % e, level=ERROR)
2596+
2597+
2598+class Tally2PAMContext(object):
2599+
2600+ def __call__(self):
2601+ ctxt = {}
2602+ settings = utils.get_settings('os')
2603+
2604+ ctxt['auth_lockout_time'] = settings['auth']['lockout_time']
2605+ ctxt['auth_retries'] = settings['auth']['retries']
2606+
2607+ return ctxt
2608+
2609+
2610+class Tally2PAM(TemplatedFile):
2611+ """The PAM Audit verifies the linux PAM settings."""
2612+ def __init__(self, path):
2613+ super(Tally2PAM, self).__init__(path=path,
2614+ template_dir=TEMPLATES_DIR,
2615+ context=Tally2PAMContext(),
2616+ user='root',
2617+ group='root',
2618+ mode=0o0640)
2619+
2620+ def pre_write(self):
2621+ # Always remove?
2622+ apt_purge('libpam-ccreds')
2623+ apt_update(fatal=True)
2624+ apt_install('libpam-modules')
2625+
2626+ def post_write(self):
2627+ """Updates the PAM configuration after the file has been written"""
2628+ try:
2629+ check_output(['pam-auth-update', '--package'])
2630+ except CalledProcessError as e:
2631+ log('Error calling pam-auth-update: %s' % e, level=ERROR)
2632
2633=== added file 'hooks/charmhelpers/contrib/hardening/host/checks/profile.py'
2634--- hooks/charmhelpers/contrib/hardening/host/checks/profile.py 1970-01-01 00:00:00 +0000
2635+++ hooks/charmhelpers/contrib/hardening/host/checks/profile.py 2016-04-22 08:16:14 +0000
2636@@ -0,0 +1,45 @@
2637+# Copyright 2016 Canonical Limited.
2638+#
2639+# This file is part of charm-helpers.
2640+#
2641+# charm-helpers is free software: you can redistribute it and/or modify
2642+# it under the terms of the GNU Lesser General Public License version 3 as
2643+# published by the Free Software Foundation.
2644+#
2645+# charm-helpers is distributed in the hope that it will be useful,
2646+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2647+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2648+# GNU Lesser General Public License for more details.
2649+#
2650+# You should have received a copy of the GNU Lesser General Public License
2651+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2652+
2653+from charmhelpers.contrib.hardening.audits.file import TemplatedFile
2654+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
2655+from charmhelpers.contrib.hardening import utils
2656+
2657+
2658+def get_audits():
2659+ """Get OS hardening profile audits.
2660+
2661+ :returns: dictionary of audits
2662+ """
2663+ audits = []
2664+
2665+ settings = utils.get_settings('os')
2666+
2667+ # If core dumps are not enabled, then don't allow core dumps to be
2668+ # created as they may contain sensitive information.
2669+ if not settings['security']['kernel_enable_core_dump']:
2670+ audits.append(TemplatedFile('/etc/profile.d/pinerolo_profile.sh',
2671+ ProfileContext(),
2672+ template_dir=TEMPLATES_DIR,
2673+ mode=0o0755, user='root', group='root'))
2674+ return audits
2675+
2676+
2677+class ProfileContext(object):
2678+
2679+ def __call__(self):
2680+ ctxt = {}
2681+ return ctxt
2682
2683=== added file 'hooks/charmhelpers/contrib/hardening/host/checks/securetty.py'
2684--- hooks/charmhelpers/contrib/hardening/host/checks/securetty.py 1970-01-01 00:00:00 +0000
2685+++ hooks/charmhelpers/contrib/hardening/host/checks/securetty.py 2016-04-22 08:16:14 +0000
2686@@ -0,0 +1,39 @@
2687+# Copyright 2016 Canonical Limited.
2688+#
2689+# This file is part of charm-helpers.
2690+#
2691+# charm-helpers is free software: you can redistribute it and/or modify
2692+# it under the terms of the GNU Lesser General Public License version 3 as
2693+# published by the Free Software Foundation.
2694+#
2695+# charm-helpers is distributed in the hope that it will be useful,
2696+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2697+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2698+# GNU Lesser General Public License for more details.
2699+#
2700+# You should have received a copy of the GNU Lesser General Public License
2701+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2702+
2703+from charmhelpers.contrib.hardening.audits.file import TemplatedFile
2704+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
2705+from charmhelpers.contrib.hardening import utils
2706+
2707+
2708+def get_audits():
2709+ """Get OS hardening Secure TTY audits.
2710+
2711+ :returns: dictionary of audits
2712+ """
2713+ audits = []
2714+ audits.append(TemplatedFile('/etc/securetty', SecureTTYContext(),
2715+ template_dir=TEMPLATES_DIR,
2716+ mode=0o0400, user='root', group='root'))
2717+ return audits
2718+
2719+
2720+class SecureTTYContext(object):
2721+
2722+ def __call__(self):
2723+ settings = utils.get_settings('os')
2724+ ctxt = {'ttys': settings['auth']['root_ttys']}
2725+ return ctxt
2726
2727=== added file 'hooks/charmhelpers/contrib/hardening/host/checks/suid_sgid.py'
2728--- hooks/charmhelpers/contrib/hardening/host/checks/suid_sgid.py 1970-01-01 00:00:00 +0000
2729+++ hooks/charmhelpers/contrib/hardening/host/checks/suid_sgid.py 2016-04-22 08:16:14 +0000
2730@@ -0,0 +1,131 @@
2731+# Copyright 2016 Canonical Limited.
2732+#
2733+# This file is part of charm-helpers.
2734+#
2735+# charm-helpers is free software: you can redistribute it and/or modify
2736+# it under the terms of the GNU Lesser General Public License version 3 as
2737+# published by the Free Software Foundation.
2738+#
2739+# charm-helpers is distributed in the hope that it will be useful,
2740+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2741+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2742+# GNU Lesser General Public License for more details.
2743+#
2744+# You should have received a copy of the GNU Lesser General Public License
2745+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2746+
2747+import subprocess
2748+
2749+from charmhelpers.core.hookenv import (
2750+ log,
2751+ INFO,
2752+)
2753+from charmhelpers.contrib.hardening.audits.file import NoSUIDSGIDAudit
2754+from charmhelpers.contrib.hardening import utils
2755+
2756+
2757+BLACKLIST = ['/usr/bin/rcp', '/usr/bin/rlogin', '/usr/bin/rsh',
2758+ '/usr/libexec/openssh/ssh-keysign',
2759+ '/usr/lib/openssh/ssh-keysign',
2760+ '/sbin/netreport',
2761+ '/usr/sbin/usernetctl',
2762+ '/usr/sbin/userisdnctl',
2763+ '/usr/sbin/pppd',
2764+ '/usr/bin/lockfile',
2765+ '/usr/bin/mail-lock',
2766+ '/usr/bin/mail-unlock',
2767+ '/usr/bin/mail-touchlock',
2768+ '/usr/bin/dotlockfile',
2769+ '/usr/bin/arping',
2770+ '/usr/sbin/uuidd',
2771+ '/usr/bin/mtr',
2772+ '/usr/lib/evolution/camel-lock-helper-1.2',
2773+ '/usr/lib/pt_chown',
2774+ '/usr/lib/eject/dmcrypt-get-device',
2775+ '/usr/lib/mc/cons.saver']
2776+
2777+WHITELIST = ['/bin/mount', '/bin/ping', '/bin/su', '/bin/umount',
2778+ '/sbin/pam_timestamp_check', '/sbin/unix_chkpwd', '/usr/bin/at',
2779+ '/usr/bin/gpasswd', '/usr/bin/locate', '/usr/bin/newgrp',
2780+ '/usr/bin/passwd', '/usr/bin/ssh-agent',
2781+ '/usr/libexec/utempter/utempter', '/usr/sbin/lockdev',
2782+ '/usr/sbin/sendmail.sendmail', '/usr/bin/expiry',
2783+ '/bin/ping6', '/usr/bin/traceroute6.iputils',
2784+ '/sbin/mount.nfs', '/sbin/umount.nfs',
2785+ '/sbin/mount.nfs4', '/sbin/umount.nfs4',
2786+ '/usr/bin/crontab',
2787+ '/usr/bin/wall', '/usr/bin/write',
2788+ '/usr/bin/screen',
2789+ '/usr/bin/mlocate',
2790+ '/usr/bin/chage', '/usr/bin/chfn', '/usr/bin/chsh',
2791+ '/bin/fusermount',
2792+ '/usr/bin/pkexec',
2793+ '/usr/bin/sudo', '/usr/bin/sudoedit',
2794+ '/usr/sbin/postdrop', '/usr/sbin/postqueue',
2795+ '/usr/sbin/suexec',
2796+ '/usr/lib/squid/ncsa_auth', '/usr/lib/squid/pam_auth',
2797+ '/usr/kerberos/bin/ksu',
2798+ '/usr/sbin/ccreds_validate',
2799+ '/usr/bin/Xorg',
2800+ '/usr/bin/X',
2801+ '/usr/lib/dbus-1.0/dbus-daemon-launch-helper',
2802+ '/usr/lib/vte/gnome-pty-helper',
2803+ '/usr/lib/libvte9/gnome-pty-helper',
2804+ '/usr/lib/libvte-2.90-9/gnome-pty-helper']
2805+
2806+
2807+def get_audits():
2808+ """Get OS hardening suid/sgid audits.
2809+
2810+ :returns: dictionary of audits
2811+ """
2812+ checks = []
2813+ settings = utils.get_settings('os')
2814+ if not settings['security']['suid_sgid_enforce']:
2815+ log("Skipping suid/sgid hardening", level=INFO)
2816+ return checks
2817+
2818+ # Build the blacklist and whitelist of files for suid/sgid checks.
2819+ # There are a total of 4 lists:
2820+ # 1. the system blacklist
2821+ # 2. the system whitelist
2822+ # 3. the user blacklist
2823+ # 4. the user whitelist
2824+ #
2825+ # The blacklist is the set of paths which should NOT have the suid/sgid bit
2826+ # set and the whitelist is the set of paths which MAY have the suid/sgid
2827+ # bit setl. The user whitelist/blacklist effectively override the system
2828+ # whitelist/blacklist.
2829+ u_b = settings['security']['suid_sgid_blacklist']
2830+ u_w = settings['security']['suid_sgid_whitelist']
2831+
2832+ blacklist = set(BLACKLIST) - set(u_w + u_b)
2833+ whitelist = set(WHITELIST) - set(u_b + u_w)
2834+
2835+ checks.append(NoSUIDSGIDAudit(blacklist))
2836+
2837+ dry_run = settings['security']['suid_sgid_dry_run_on_unknown']
2838+
2839+ if settings['security']['suid_sgid_remove_from_unknown'] or dry_run:
2840+ # If the policy is a dry_run (e.g. complain only) or remove unknown
2841+ # suid/sgid bits then find all of the paths which have the suid/sgid
2842+ # bit set and then remove the whitelisted paths.
2843+ root_path = settings['environment']['root_path']
2844+ unknown_paths = find_paths_with_suid_sgid(root_path) - set(whitelist)
2845+ checks.append(NoSUIDSGIDAudit(unknown_paths, unless=dry_run))
2846+
2847+ return checks
2848+
2849+
2850+def find_paths_with_suid_sgid(root_path):
2851+ """Finds all paths/files which have an suid/sgid bit enabled.
2852+
2853+ Starting with the root_path, this will recursively find all paths which
2854+ have an suid or sgid bit set.
2855+ """
2856+ cmd = ['find', root_path, '-perm', '-4000', '-o', '-perm', '-2000',
2857+ '-type', 'f', '!', '-path', '/proc/*', '-print']
2858+
2859+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
2860+ out, _ = p.communicate()
2861+ return set(out.split('\n'))
2862
2863=== added file 'hooks/charmhelpers/contrib/hardening/host/checks/sysctl.py'
2864--- hooks/charmhelpers/contrib/hardening/host/checks/sysctl.py 1970-01-01 00:00:00 +0000
2865+++ hooks/charmhelpers/contrib/hardening/host/checks/sysctl.py 2016-04-22 08:16:14 +0000
2866@@ -0,0 +1,211 @@
2867+# Copyright 2016 Canonical Limited.
2868+#
2869+# This file is part of charm-helpers.
2870+#
2871+# charm-helpers is free software: you can redistribute it and/or modify
2872+# it under the terms of the GNU Lesser General Public License version 3 as
2873+# published by the Free Software Foundation.
2874+#
2875+# charm-helpers is distributed in the hope that it will be useful,
2876+# but WITHOUT ANY WARRANTY; without even the implied warranty of
2877+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
2878+# GNU Lesser General Public License for more details.
2879+#
2880+# You should have received a copy of the GNU Lesser General Public License
2881+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
2882+
2883+import os
2884+import platform
2885+import re
2886+import six
2887+import subprocess
2888+
2889+from charmhelpers.core.hookenv import (
2890+ log,
2891+ INFO,
2892+ WARNING,
2893+)
2894+from charmhelpers.contrib.hardening import utils
2895+from charmhelpers.contrib.hardening.audits.file import (
2896+ FilePermissionAudit,
2897+ TemplatedFile,
2898+)
2899+from charmhelpers.contrib.hardening.host import TEMPLATES_DIR
2900+
2901+
2902+SYSCTL_DEFAULTS = """net.ipv4.ip_forward=%(net_ipv4_ip_forward)s
2903+net.ipv6.conf.all.forwarding=%(net_ipv6_conf_all_forwarding)s
2904+net.ipv4.conf.all.rp_filter=1
2905+net.ipv4.conf.default.rp_filter=1
2906+net.ipv4.icmp_echo_ignore_broadcasts=1
2907+net.ipv4.icmp_ignore_bogus_error_responses=1
2908+net.ipv4.icmp_ratelimit=100
2909+net.ipv4.icmp_ratemask=88089
2910+net.ipv6.conf.all.disable_ipv6=%(net_ipv6_conf_all_disable_ipv6)s
2911+net.ipv4.tcp_timestamps=%(net_ipv4_tcp_timestamps)s
2912+net.ipv4.conf.all.arp_ignore=%(net_ipv4_conf_all_arp_ignore)s
2913+net.ipv4.conf.all.arp_announce=%(net_ipv4_conf_all_arp_announce)s
2914+net.ipv4.tcp_rfc1337=1
2915+net.ipv4.tcp_syncookies=1
2916+net.ipv4.conf.all.shared_media=1
2917+net.ipv4.conf.default.shared_media=1
2918+net.ipv4.conf.all.accept_source_route=0
2919+net.ipv4.conf.default.accept_source_route=0
2920+net.ipv4.conf.all.accept_redirects=0
2921+net.ipv4.conf.default.accept_redirects=0
2922+net.ipv6.conf.all.accept_redirects=0
2923+net.ipv6.conf.default.accept_redirects=0
2924+net.ipv4.conf.all.secure_redirects=0
2925+net.ipv4.conf.default.secure_redirects=0
2926+net.ipv4.conf.all.send_redirects=0
2927+net.ipv4.conf.default.send_redirects=0
2928+net.ipv4.conf.all.log_martians=0
2929+net.ipv6.conf.default.router_solicitations=0
2930+net.ipv6.conf.default.accept_ra_rtr_pref=0
2931+net.ipv6.conf.default.accept_ra_pinfo=0
2932+net.ipv6.conf.default.accept_ra_defrtr=0
2933+net.ipv6.conf.default.autoconf=0
2934+net.ipv6.conf.default.dad_transmits=0
2935+net.ipv6.conf.default.max_addresses=1
2936+net.ipv6.conf.all.accept_ra=0
2937+net.ipv6.conf.default.accept_ra=0
2938+kernel.modules_disabled=%(kernel_modules_disabled)s
2939+kernel.sysrq=%(kernel_sysrq)s
2940+fs.suid_dumpable=%(fs_suid_dumpable)s
2941+kernel.randomize_va_space=2
2942+"""
2943+
2944+
2945+def get_audits():
2946+ """Get OS hardening sysctl audits.
2947+
2948+ :returns: dictionary of audits
2949+ """
2950+ audits = []
2951+ settings = utils.get_settings('os')
2952+
2953+ # Apply the sysctl settings which are configured to be applied.
2954+ audits.append(SysctlConf())
2955+ # Make sure that only root has access to the sysctl.conf file, and
2956+ # that it is read-only.
2957+ audits.append(FilePermissionAudit('/etc/sysctl.conf',
2958+ user='root',
2959+ group='root', mode=0o0440))
2960+ # If module loading is not enabled, then ensure that the modules
2961+ # file has the appropriate permissions and rebuild the initramfs
2962+ if not settings['security']['kernel_enable_module_loading']:
2963+ audits.append(ModulesTemplate())
2964+
2965+ return audits
2966+
2967+
2968+class ModulesContext(object):
2969+
2970+ def __call__(self):
2971+ settings = utils.get_settings('os')
2972+ with open('/proc/cpuinfo', 'r') as fd:
2973+ cpuinfo = fd.readlines()
2974+
2975+ for line in cpuinfo:
2976+ match = re.search(r"^vendor_id\s+:\s+(.+)", line)
2977+ if match:
2978+ vendor = match.group(1)
2979+
2980+ if vendor == "GenuineIntel":
2981+ vendor = "intel"
2982+ elif vendor == "AuthenticAMD":
2983+ vendor = "amd"
2984+
2985+ ctxt = {'arch': platform.processor(),
2986+ 'cpuVendor': vendor,
2987+ 'desktop_enable': settings['general']['desktop_enable']}
2988+
2989+ return ctxt
2990+
2991+
2992+class ModulesTemplate(object):
2993+
2994+ def __init__(self):
2995+ super(ModulesTemplate, self).__init__('/etc/initramfs-tools/modules',
2996+ ModulesContext(),
2997+ templates_dir=TEMPLATES_DIR,
2998+ user='root', group='root',
2999+ mode=0o0440)
3000+
3001+ def post_write(self):
3002+ subprocess.check_call(['update-initramfs', '-u'])
3003+
3004+
3005+class SysCtlHardeningContext(object):
3006+ def __call__(self):
3007+ settings = utils.get_settings('os')
3008+ ctxt = {'sysctl': {}}
3009+
3010+ log("Applying sysctl settings", level=INFO)
3011+ extras = {'net_ipv4_ip_forward': 0,
3012+ 'net_ipv6_conf_all_forwarding': 0,
3013+ 'net_ipv6_conf_all_disable_ipv6': 1,
3014+ 'net_ipv4_tcp_timestamps': 0,
3015+ 'net_ipv4_conf_all_arp_ignore': 0,
3016+ 'net_ipv4_conf_all_arp_announce': 0,
3017+ 'kernel_sysrq': 0,
3018+ 'fs_suid_dumpable': 0,
3019+ 'kernel_modules_disabled': 1}
3020+
3021+ if settings['sysctl']['ipv6_enable']:
3022+ extras['net_ipv6_conf_all_disable_ipv6'] = 0
3023+
3024+ if settings['sysctl']['forwarding']:
3025+ extras['net_ipv4_ip_forward'] = 1
3026+ extras['net_ipv6_conf_all_forwarding'] = 1
3027+
3028+ if settings['sysctl']['arp_restricted']:
3029+ extras['net_ipv4_conf_all_arp_ignore'] = 1
3030+ extras['net_ipv4_conf_all_arp_announce'] = 2
3031+
3032+ if settings['security']['kernel_enable_module_loading']:
3033+ extras['kernel_modules_disabled'] = 0
3034+
3035+ if settings['sysctl']['kernel_enable_sysrq']:
3036+ sysrq_val = settings['sysctl']['kernel_secure_sysrq']
3037+ extras['kernel_sysrq'] = sysrq_val
3038+
3039+ if settings['security']['kernel_enable_core_dump']:
3040+ extras['fs_suid_dumpable'] = 1
3041+
3042+ settings.update(extras)
3043+ for d in (SYSCTL_DEFAULTS % settings).split():
3044+ d = d.strip().partition('=')
3045+ key = d[0].strip()
3046+ path = os.path.join('/proc/sys', key.replace('.', '/'))
3047+ if not os.path.exists(path):
3048+ log("Skipping '%s' since '%s' does not exist" % (key, path),
3049+ level=WARNING)
3050+ continue
3051+
3052+ ctxt['sysctl'][key] = d[2] or None
3053+
3054+ # Translate for python3
3055+ return {'sysctl_settings':
3056+ [(k, v) for k, v in six.iteritems(ctxt['sysctl'])]}
3057+
3058+
3059+class SysctlConf(TemplatedFile):
3060+ """An audit check for sysctl settings."""
3061+ def __init__(self):
3062+ self.conffile = '/etc/sysctl.d/99-juju-hardening.conf'
3063+ super(SysctlConf, self).__init__(self.conffile,
3064+ SysCtlHardeningContext(),
3065+ template_dir=TEMPLATES_DIR,
3066+ user='root', group='root',
3067+ mode=0o0440)
3068+
3069+ def post_write(self):
3070+ try:
3071+ subprocess.check_call(['sysctl', '-p', self.conffile])
3072+ except subprocess.CalledProcessError as e:
3073+ # NOTE: on some systems if sysctl cannot apply all settings it
3074+ # will return non-zero as well.
3075+ log("sysctl command returned an error (maybe some "
3076+ "keys could not be set) - %s" % (e),
3077+ level=WARNING)
3078
3079=== added directory 'hooks/charmhelpers/contrib/hardening/mysql'
3080=== added file 'hooks/charmhelpers/contrib/hardening/mysql/__init__.py'
3081--- hooks/charmhelpers/contrib/hardening/mysql/__init__.py 1970-01-01 00:00:00 +0000
3082+++ hooks/charmhelpers/contrib/hardening/mysql/__init__.py 2016-04-22 08:16:14 +0000
3083@@ -0,0 +1,19 @@
3084+# Copyright 2016 Canonical Limited.
3085+#
3086+# This file is part of charm-helpers.
3087+#
3088+# charm-helpers is free software: you can redistribute it and/or modify
3089+# it under the terms of the GNU Lesser General Public License version 3 as
3090+# published by the Free Software Foundation.
3091+#
3092+# charm-helpers is distributed in the hope that it will be useful,
3093+# but WITHOUT ANY WARRANTY; without even the implied warranty of
3094+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3095+# GNU Lesser General Public License for more details.
3096+#
3097+# You should have received a copy of the GNU Lesser General Public License
3098+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3099+
3100+from os import path
3101+
3102+TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates')
3103
3104=== added directory 'hooks/charmhelpers/contrib/hardening/mysql/checks'
3105=== added file 'hooks/charmhelpers/contrib/hardening/mysql/checks/__init__.py'
3106--- hooks/charmhelpers/contrib/hardening/mysql/checks/__init__.py 1970-01-01 00:00:00 +0000
3107+++ hooks/charmhelpers/contrib/hardening/mysql/checks/__init__.py 2016-04-22 08:16:14 +0000
3108@@ -0,0 +1,31 @@
3109+# Copyright 2016 Canonical Limited.
3110+#
3111+# This file is part of charm-helpers.
3112+#
3113+# charm-helpers is free software: you can redistribute it and/or modify
3114+# it under the terms of the GNU Lesser General Public License version 3 as
3115+# published by the Free Software Foundation.
3116+#
3117+# charm-helpers is distributed in the hope that it will be useful,
3118+# but WITHOUT ANY WARRANTY; without even the implied warranty of
3119+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3120+# GNU Lesser General Public License for more details.
3121+#
3122+# You should have received a copy of the GNU Lesser General Public License
3123+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3124+
3125+from charmhelpers.core.hookenv import (
3126+ log,
3127+ DEBUG,
3128+)
3129+from charmhelpers.contrib.hardening.mysql.checks import config
3130+
3131+
3132+def run_mysql_checks():
3133+ log("Starting MySQL hardening checks.", level=DEBUG)
3134+ checks = config.get_audits()
3135+ for check in checks:
3136+ log("Running '%s' check" % (check.__class__.__name__), level=DEBUG)
3137+ check.ensure_compliance()
3138+
3139+ log("MySQL hardening checks complete.", level=DEBUG)
3140
3141=== added file 'hooks/charmhelpers/contrib/hardening/mysql/checks/config.py'
3142--- hooks/charmhelpers/contrib/hardening/mysql/checks/config.py 1970-01-01 00:00:00 +0000
3143+++ hooks/charmhelpers/contrib/hardening/mysql/checks/config.py 2016-04-22 08:16:14 +0000
3144@@ -0,0 +1,89 @@
3145+# Copyright 2016 Canonical Limited.
3146+#
3147+# This file is part of charm-helpers.
3148+#
3149+# charm-helpers is free software: you can redistribute it and/or modify
3150+# it under the terms of the GNU Lesser General Public License version 3 as
3151+# published by the Free Software Foundation.
3152+#
3153+# charm-helpers is distributed in the hope that it will be useful,
3154+# but WITHOUT ANY WARRANTY; without even the implied warranty of
3155+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3156+# GNU Lesser General Public License for more details.
3157+#
3158+# You should have received a copy of the GNU Lesser General Public License
3159+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3160+
3161+import six
3162+import subprocess
3163+
3164+from charmhelpers.core.hookenv import (
3165+ log,
3166+ WARNING,
3167+)
3168+from charmhelpers.contrib.hardening.audits.file import (
3169+ FilePermissionAudit,
3170+ DirectoryPermissionAudit,
3171+ TemplatedFile,
3172+)
3173+from charmhelpers.contrib.hardening.mysql import TEMPLATES_DIR
3174+from charmhelpers.contrib.hardening import utils
3175+
3176+
3177+def get_audits():
3178+ """Get MySQL hardening config audits.
3179+
3180+ :returns: dictionary of audits
3181+ """
3182+ if subprocess.call(['which', 'mysql'], stdout=subprocess.PIPE) != 0:
3183+ log("MySQL does not appear to be installed on this node - "
3184+ "skipping mysql hardening", level=WARNING)
3185+ return []
3186+
3187+ settings = utils.get_settings('mysql')
3188+ hardening_settings = settings['hardening']
3189+ my_cnf = hardening_settings['mysql-conf']
3190+
3191+ audits = [
3192+ FilePermissionAudit(paths=[my_cnf], user='root',
3193+ group='root', mode=0o0600),
3194+
3195+ TemplatedFile(hardening_settings['hardening-conf'],
3196+ MySQLConfContext(),
3197+ TEMPLATES_DIR,
3198+ mode=0o0750,
3199+ user='mysql',
3200+ group='root',
3201+ service_actions=[{'service': 'mysql',
3202+ 'actions': ['restart']}]),
3203+
3204+ # MySQL and Percona charms do not allow configuration of the
3205+ # data directory, so use the default.
3206+ DirectoryPermissionAudit('/var/lib/mysql',
3207+ user='mysql',
3208+ group='mysql',
3209+ recursive=False,
3210+ mode=0o755),
3211+
3212+ DirectoryPermissionAudit('/etc/mysql',
3213+ user='root',
3214+ group='root',
3215+ recursive=False,
3216+ mode=0o700),
3217+ ]
3218+
3219+ return audits
3220+
3221+
3222+class MySQLConfContext(object):
3223+ """Defines the set of key/value pairs to set in a mysql config file.
3224+
3225+ This context, when called, will return a dictionary containing the
3226+ key/value pairs of setting to specify in the
3227+ /etc/mysql/conf.d/hardening.cnf file.
3228+ """
3229+ def __call__(self):
3230+ settings = utils.get_settings('mysql')
3231+ # Translate for python3
3232+ return {'mysql_settings':
3233+ [(k, v) for k, v in six.iteritems(settings['security'])]}
3234
3235=== added directory 'hooks/charmhelpers/contrib/hardening/ssh'
3236=== added file 'hooks/charmhelpers/contrib/hardening/ssh/__init__.py'
3237--- hooks/charmhelpers/contrib/hardening/ssh/__init__.py 1970-01-01 00:00:00 +0000
3238+++ hooks/charmhelpers/contrib/hardening/ssh/__init__.py 2016-04-22 08:16:14 +0000
3239@@ -0,0 +1,19 @@
3240+# Copyright 2016 Canonical Limited.
3241+#
3242+# This file is part of charm-helpers.
3243+#
3244+# charm-helpers is free software: you can redistribute it and/or modify
3245+# it under the terms of the GNU Lesser General Public License version 3 as
3246+# published by the Free Software Foundation.
3247+#
3248+# charm-helpers is distributed in the hope that it will be useful,
3249+# but WITHOUT ANY WARRANTY; without even the implied warranty of
3250+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3251+# GNU Lesser General Public License for more details.
3252+#
3253+# You should have received a copy of the GNU Lesser General Public License
3254+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3255+
3256+from os import path
3257+
3258+TEMPLATES_DIR = path.join(path.dirname(__file__), 'templates')
3259
3260=== added directory 'hooks/charmhelpers/contrib/hardening/ssh/checks'
3261=== added file 'hooks/charmhelpers/contrib/hardening/ssh/checks/__init__.py'
3262--- hooks/charmhelpers/contrib/hardening/ssh/checks/__init__.py 1970-01-01 00:00:00 +0000
3263+++ hooks/charmhelpers/contrib/hardening/ssh/checks/__init__.py 2016-04-22 08:16:14 +0000
3264@@ -0,0 +1,31 @@
3265+# Copyright 2016 Canonical Limited.
3266+#
3267+# This file is part of charm-helpers.
3268+#
3269+# charm-helpers is free software: you can redistribute it and/or modify
3270+# it under the terms of the GNU Lesser General Public License version 3 as
3271+# published by the Free Software Foundation.
3272+#
3273+# charm-helpers is distributed in the hope that it will be useful,
3274+# but WITHOUT ANY WARRANTY; without even the implied warranty of
3275+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3276+# GNU Lesser General Public License for more details.
3277+#
3278+# You should have received a copy of the GNU Lesser General Public License
3279+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3280+
3281+from charmhelpers.core.hookenv import (
3282+ log,
3283+ DEBUG,
3284+)
3285+from charmhelpers.contrib.hardening.ssh.checks import config
3286+
3287+
3288+def run_ssh_checks():
3289+ log("Starting SSH hardening checks.", level=DEBUG)
3290+ checks = config.get_audits()
3291+ for check in checks:
3292+ log("Running '%s' check" % (check.__class__.__name__), level=DEBUG)
3293+ check.ensure_compliance()
3294+
3295+ log("SSH hardening checks complete.", level=DEBUG)
3296
3297=== added file 'hooks/charmhelpers/contrib/hardening/ssh/checks/config.py'
3298--- hooks/charmhelpers/contrib/hardening/ssh/checks/config.py 1970-01-01 00:00:00 +0000
3299+++ hooks/charmhelpers/contrib/hardening/ssh/checks/config.py 2016-04-22 08:16:14 +0000
3300@@ -0,0 +1,394 @@
3301+# Copyright 2016 Canonical Limited.
3302+#
3303+# This file is part of charm-helpers.
3304+#
3305+# charm-helpers is free software: you can redistribute it and/or modify
3306+# it under the terms of the GNU Lesser General Public License version 3 as
3307+# published by the Free Software Foundation.
3308+#
3309+# charm-helpers is distributed in the hope that it will be useful,
3310+# but WITHOUT ANY WARRANTY; without even the implied warranty of
3311+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3312+# GNU Lesser General Public License for more details.
3313+#
3314+# You should have received a copy of the GNU Lesser General Public License
3315+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3316+
3317+import os
3318+
3319+from charmhelpers.core.hookenv import (
3320+ log,
3321+ DEBUG,
3322+)
3323+from charmhelpers.fetch import (
3324+ apt_install,
3325+ apt_update,
3326+)
3327+from charmhelpers.core.host import lsb_release
3328+from charmhelpers.contrib.hardening.audits.file import (
3329+ TemplatedFile,
3330+ FileContentAudit,
3331+)
3332+from charmhelpers.contrib.hardening.ssh import TEMPLATES_DIR
3333+from charmhelpers.contrib.hardening import utils
3334+
3335+
3336+def get_audits():
3337+ """Get SSH hardening config audits.
3338+
3339+ :returns: dictionary of audits
3340+ """
3341+ audits = [SSHConfig(), SSHDConfig(), SSHConfigFileContentAudit(),
3342+ SSHDConfigFileContentAudit()]
3343+ return audits
3344+
3345+
3346+class SSHConfigContext(object):
3347+
3348+ type = 'client'
3349+
3350+ def get_macs(self, allow_weak_mac):
3351+ if allow_weak_mac:
3352+ weak_macs = 'weak'
3353+ else:
3354+ weak_macs = 'default'
3355+
3356+ default = 'hmac-sha2-512,hmac-sha2-256,hmac-ripemd160'
3357+ macs = {'default': default,
3358+ 'weak': default + ',hmac-sha1'}
3359+
3360+ default = ('hmac-sha2-512-etm@openssh.com,'
3361+ 'hmac-sha2-256-etm@openssh.com,'
3362+ 'hmac-ripemd160-etm@openssh.com,umac-128-etm@openssh.com,'
3363+ 'hmac-sha2-512,hmac-sha2-256,hmac-ripemd160')
3364+ macs_66 = {'default': default,
3365+ 'weak': default + ',hmac-sha1'}
3366+
3367+ # Use newer ciphers on Ubuntu Trusty and above
3368+ if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
3369+ log("Detected Ubuntu 14.04 or newer, using new macs", level=DEBUG)
3370+ macs = macs_66
3371+
3372+ return macs[weak_macs]
3373+
3374+ def get_kexs(self, allow_weak_kex):
3375+ if allow_weak_kex:
3376+ weak_kex = 'weak'
3377+ else:
3378+ weak_kex = 'default'
3379+
3380+ default = 'diffie-hellman-group-exchange-sha256'
3381+ weak = (default + ',diffie-hellman-group14-sha1,'
3382+ 'diffie-hellman-group-exchange-sha1,'
3383+ 'diffie-hellman-group1-sha1')
3384+ kex = {'default': default,
3385+ 'weak': weak}
3386+
3387+ default = ('curve25519-sha256@libssh.org,'
3388+ 'diffie-hellman-group-exchange-sha256')
3389+ weak = (default + ',diffie-hellman-group14-sha1,'
3390+ 'diffie-hellman-group-exchange-sha1,'
3391+ 'diffie-hellman-group1-sha1')
3392+ kex_66 = {'default': default,
3393+ 'weak': weak}
3394+
3395+ # Use newer kex on Ubuntu Trusty and above
3396+ if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
3397+ log('Detected Ubuntu 14.04 or newer, using new key exchange '
3398+ 'algorithms', level=DEBUG)
3399+ kex = kex_66
3400+
3401+ return kex[weak_kex]
3402+
3403+ def get_ciphers(self, cbc_required):
3404+ if cbc_required:
3405+ weak_ciphers = 'weak'
3406+ else:
3407+ weak_ciphers = 'default'
3408+
3409+ default = 'aes256-ctr,aes192-ctr,aes128-ctr'
3410+ cipher = {'default': default,
3411+ 'weak': default + 'aes256-cbc,aes192-cbc,aes128-cbc'}
3412+
3413+ default = ('chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,'
3414+ 'aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr')
3415+ ciphers_66 = {'default': default,
3416+ 'weak': default + ',aes256-cbc,aes192-cbc,aes128-cbc'}
3417+
3418+ # Use newer ciphers on ubuntu Trusty and above
3419+ if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
3420+ log('Detected Ubuntu 14.04 or newer, using new ciphers',
3421+ level=DEBUG)
3422+ cipher = ciphers_66
3423+
3424+ return cipher[weak_ciphers]
3425+
3426+ def __call__(self):
3427+ settings = utils.get_settings('ssh')
3428+ if settings['common']['network_ipv6_enable']:
3429+ addr_family = 'any'
3430+ else:
3431+ addr_family = 'inet'
3432+
3433+ ctxt = {
3434+ 'addr_family': addr_family,
3435+ 'remote_hosts': settings['common']['remote_hosts'],
3436+ 'password_auth_allowed':
3437+ settings['client']['password_authentication'],
3438+ 'ports': settings['common']['ports'],
3439+ 'ciphers': self.get_ciphers(settings['client']['cbc_required']),
3440+ 'macs': self.get_macs(settings['client']['weak_hmac']),
3441+ 'kexs': self.get_kexs(settings['client']['weak_kex']),
3442+ 'roaming': settings['client']['roaming'],
3443+ }
3444+ return ctxt
3445+
3446+
3447+class SSHConfig(TemplatedFile):
3448+ def __init__(self):
3449+ path = '/etc/ssh/ssh_config'
3450+ super(SSHConfig, self).__init__(path=path,
3451+ template_dir=TEMPLATES_DIR,
3452+ context=SSHConfigContext(),
3453+ user='root',
3454+ group='root',
3455+ mode=0o0644)
3456+
3457+ def pre_write(self):
3458+ settings = utils.get_settings('ssh')
3459+ apt_update(fatal=True)
3460+ apt_install(settings['client']['package'])
3461+ if not os.path.exists('/etc/ssh'):
3462+ os.makedir('/etc/ssh')
3463+ # NOTE: don't recurse
3464+ utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755,
3465+ maxdepth=0)
3466+
3467+ def post_write(self):
3468+ # NOTE: don't recurse
3469+ utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755,
3470+ maxdepth=0)
3471+
3472+
3473+class SSHDConfigContext(SSHConfigContext):
3474+
3475+ type = 'server'
3476+
3477+ def __call__(self):
3478+ settings = utils.get_settings('ssh')
3479+ if settings['common']['network_ipv6_enable']:
3480+ addr_family = 'any'
3481+ else:
3482+ addr_family = 'inet'
3483+
3484+ ctxt = {
3485+ 'ssh_ip': settings['server']['listen_to'],
3486+ 'password_auth_allowed':
3487+ settings['server']['password_authentication'],
3488+ 'ports': settings['common']['ports'],
3489+ 'addr_family': addr_family,
3490+ 'ciphers': self.get_ciphers(settings['server']['cbc_required']),
3491+ 'macs': self.get_macs(settings['server']['weak_hmac']),
3492+ 'kexs': self.get_kexs(settings['server']['weak_kex']),
3493+ 'host_key_files': settings['server']['host_key_files'],
3494+ 'allow_root_with_key': settings['server']['allow_root_with_key'],
3495+ 'password_authentication':
3496+ settings['server']['password_authentication'],
3497+ 'use_priv_sep': settings['server']['use_privilege_separation'],
3498+ 'use_pam': settings['server']['use_pam'],
3499+ 'allow_x11_forwarding': settings['server']['allow_x11_forwarding'],
3500+ 'print_motd': settings['server']['print_motd'],
3501+ 'print_last_log': settings['server']['print_last_log'],
3502+ 'client_alive_interval':
3503+ settings['server']['alive_interval'],
3504+ 'client_alive_count': settings['server']['alive_count'],
3505+ 'allow_tcp_forwarding': settings['server']['allow_tcp_forwarding'],
3506+ 'allow_agent_forwarding':
3507+ settings['server']['allow_agent_forwarding'],
3508+ 'deny_users': settings['server']['deny_users'],
3509+ 'allow_users': settings['server']['allow_users'],
3510+ 'deny_groups': settings['server']['deny_groups'],
3511+ 'allow_groups': settings['server']['allow_groups'],
3512+ 'use_dns': settings['server']['use_dns'],
3513+ 'sftp_enable': settings['server']['sftp_enable'],
3514+ 'sftp_group': settings['server']['sftp_group'],
3515+ 'sftp_chroot': settings['server']['sftp_chroot'],
3516+ 'max_auth_tries': settings['server']['max_auth_tries'],
3517+ 'max_sessions': settings['server']['max_sessions'],
3518+ }
3519+ return ctxt
3520+
3521+
3522+class SSHDConfig(TemplatedFile):
3523+ def __init__(self):
3524+ path = '/etc/ssh/sshd_config'
3525+ super(SSHDConfig, self).__init__(path=path,
3526+ template_dir=TEMPLATES_DIR,
3527+ context=SSHDConfigContext(),
3528+ user='root',
3529+ group='root',
3530+ mode=0o0600,
3531+ service_actions=[{'service': 'ssh',
3532+ 'actions':
3533+ ['restart']}])
3534+
3535+ def pre_write(self):
3536+ settings = utils.get_settings('ssh')
3537+ apt_update(fatal=True)
3538+ apt_install(settings['server']['package'])
3539+ if not os.path.exists('/etc/ssh'):
3540+ os.makedir('/etc/ssh')
3541+ # NOTE: don't recurse
3542+ utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755,
3543+ maxdepth=0)
3544+
3545+ def post_write(self):
3546+ # NOTE: don't recurse
3547+ utils.ensure_permissions('/etc/ssh', 'root', 'root', 0o0755,
3548+ maxdepth=0)
3549+
3550+
3551+class SSHConfigFileContentAudit(FileContentAudit):
3552+ def __init__(self):
3553+ self.path = '/etc/ssh/ssh_config'
3554+ super(SSHConfigFileContentAudit, self).__init__(self.path, {})
3555+
3556+ def is_compliant(self, *args, **kwargs):
3557+ self.pass_cases = []
3558+ self.fail_cases = []
3559+ settings = utils.get_settings('ssh')
3560+
3561+ if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
3562+ if not settings['server']['weak_hmac']:
3563+ self.pass_cases.append(r'^MACs.+,hmac-ripemd160$')
3564+ else:
3565+ self.pass_cases.append(r'^MACs.+,hmac-sha1$')
3566+
3567+ if settings['server']['weak_kex']:
3568+ self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa
3569+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa
3570+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa
3571+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa
3572+ else:
3573+ self.pass_cases.append(r'^KexAlgorithms.+,diffie-hellman-group-exchange-sha256$') # noqa
3574+ self.fail_cases.append(r'^KexAlgorithms.*diffie-hellman-group14-sha1[,\s]?') # noqa
3575+
3576+ if settings['server']['cbc_required']:
3577+ self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
3578+ self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
3579+ self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
3580+ self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
3581+ else:
3582+ self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
3583+ self.pass_cases.append(r'^Ciphers\schacha20-poly1305@openssh.com,.+') # noqa
3584+ self.pass_cases.append(r'^Ciphers\s.*aes128-ctr$')
3585+ self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
3586+ self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
3587+ else:
3588+ if not settings['client']['weak_hmac']:
3589+ self.fail_cases.append(r'^MACs.+,hmac-sha1$')
3590+ else:
3591+ self.pass_cases.append(r'^MACs.+,hmac-sha1$')
3592+
3593+ if settings['client']['weak_kex']:
3594+ self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa
3595+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa
3596+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa
3597+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa
3598+ else:
3599+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256$') # noqa
3600+ self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa
3601+ self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa
3602+ self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa
3603+
3604+ if settings['client']['cbc_required']:
3605+ self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
3606+ self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
3607+ self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
3608+ self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
3609+ else:
3610+ self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
3611+ self.pass_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
3612+ self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
3613+ self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
3614+
3615+ if settings['client']['roaming']:
3616+ self.pass_cases.append(r'^UseRoaming yes$')
3617+ else:
3618+ self.fail_cases.append(r'^UseRoaming yes$')
3619+
3620+ return super(SSHConfigFileContentAudit, self).is_compliant(*args,
3621+ **kwargs)
3622+
3623+
3624+class SSHDConfigFileContentAudit(FileContentAudit):
3625+ def __init__(self):
3626+ self.path = '/etc/ssh/sshd_config'
3627+ super(SSHDConfigFileContentAudit, self).__init__(self.path, {})
3628+
3629+ def is_compliant(self, *args, **kwargs):
3630+ self.pass_cases = []
3631+ self.fail_cases = []
3632+ settings = utils.get_settings('ssh')
3633+
3634+ if lsb_release()['DISTRIB_CODENAME'].lower() >= 'trusty':
3635+ if not settings['server']['weak_hmac']:
3636+ self.pass_cases.append(r'^MACs.+,hmac-ripemd160$')
3637+ else:
3638+ self.pass_cases.append(r'^MACs.+,hmac-sha1$')
3639+
3640+ if settings['server']['weak_kex']:
3641+ self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa
3642+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa
3643+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa
3644+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa
3645+ else:
3646+ self.pass_cases.append(r'^KexAlgorithms.+,diffie-hellman-group-exchange-sha256$') # noqa
3647+ self.fail_cases.append(r'^KexAlgorithms.*diffie-hellman-group14-sha1[,\s]?') # noqa
3648+
3649+ if settings['server']['cbc_required']:
3650+ self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
3651+ self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
3652+ self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
3653+ self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
3654+ else:
3655+ self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
3656+ self.pass_cases.append(r'^Ciphers\schacha20-poly1305@openssh.com,.+') # noqa
3657+ self.pass_cases.append(r'^Ciphers\s.*aes128-ctr$')
3658+ self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
3659+ self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
3660+ else:
3661+ if not settings['server']['weak_hmac']:
3662+ self.pass_cases.append(r'^MACs.+,hmac-ripemd160$')
3663+ else:
3664+ self.pass_cases.append(r'^MACs.+,hmac-sha1$')
3665+
3666+ if settings['server']['weak_kex']:
3667+ self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256[,\s]?') # noqa
3668+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa
3669+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa
3670+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa
3671+ else:
3672+ self.pass_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha256$') # noqa
3673+ self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group14-sha1[,\s]?') # noqa
3674+ self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group-exchange-sha1[,\s]?') # noqa
3675+ self.fail_cases.append(r'^KexAlgorithms\sdiffie-hellman-group1-sha1[,\s]?') # noqa
3676+
3677+ if settings['server']['cbc_required']:
3678+ self.pass_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
3679+ self.fail_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
3680+ self.fail_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
3681+ self.fail_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
3682+ else:
3683+ self.fail_cases.append(r'^Ciphers\s.*-cbc[,\s]?')
3684+ self.pass_cases.append(r'^Ciphers\s.*aes128-ctr[,\s]?')
3685+ self.pass_cases.append(r'^Ciphers\s.*aes192-ctr[,\s]?')
3686+ self.pass_cases.append(r'^Ciphers\s.*aes256-ctr[,\s]?')
3687+
3688+ if settings['server']['sftp_enable']:
3689+ self.pass_cases.append(r'^Subsystem\ssftp')
3690+ else:
3691+ self.fail_cases.append(r'^Subsystem\ssftp')
3692+
3693+ return super(SSHDConfigFileContentAudit, self).is_compliant(*args,
3694+ **kwargs)
3695
3696=== added file 'hooks/charmhelpers/contrib/hardening/templating.py'
3697--- hooks/charmhelpers/contrib/hardening/templating.py 1970-01-01 00:00:00 +0000
3698+++ hooks/charmhelpers/contrib/hardening/templating.py 2016-04-22 08:16:14 +0000
3699@@ -0,0 +1,71 @@
3700+# Copyright 2016 Canonical Limited.
3701+#
3702+# This file is part of charm-helpers.
3703+#
3704+# charm-helpers is free software: you can redistribute it and/or modify
3705+# it under the terms of the GNU Lesser General Public License version 3 as
3706+# published by the Free Software Foundation.
3707+#
3708+# charm-helpers is distributed in the hope that it will be useful,
3709+# but WITHOUT ANY WARRANTY; without even the implied warranty of
3710+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3711+# GNU Lesser General Public License for more details.
3712+#
3713+# You should have received a copy of the GNU Lesser General Public License
3714+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3715+
3716+import os
3717+
3718+from charmhelpers.core.hookenv import (
3719+ log,
3720+ DEBUG,
3721+ WARNING,
3722+)
3723+
3724+try:
3725+ from jinja2 import FileSystemLoader, Environment
3726+except ImportError:
3727+ from charmhelpers.fetch import apt_install
3728+ from charmhelpers.fetch import apt_update
3729+ apt_update(fatal=True)
3730+ apt_install('python-jinja2', fatal=True)
3731+ from jinja2 import FileSystemLoader, Environment
3732+
3733+
3734+# NOTE: function separated from main rendering code to facilitate easier
3735+# mocking in unit tests.
3736+def write(path, data):
3737+ with open(path, 'wb') as out:
3738+ out.write(data)
3739+
3740+
3741+def get_template_path(template_dir, path):
3742+ """Returns the template file which would be used to render the path.
3743+
3744+ The path to the template file is returned.
3745+ :param template_dir: the directory the templates are located in
3746+ :param path: the file path to be written to.
3747+ :returns: path to the template file
3748+ """
3749+ return os.path.join(template_dir, os.path.basename(path))
3750+
3751+
3752+def render_and_write(template_dir, path, context):
3753+ """Renders the specified template into the file.
3754+
3755+ :param template_dir: the directory to load the template from
3756+ :param path: the path to write the templated contents to
3757+ :param context: the parameters to pass to the rendering engine
3758+ """
3759+ env = Environment(loader=FileSystemLoader(template_dir))
3760+ template_file = os.path.basename(path)
3761+ template = env.get_template(template_file)
3762+ log('Rendering from template: %s' % template.name, level=DEBUG)
3763+ rendered_content = template.render(context)
3764+ if not rendered_content:
3765+ log("Render returned None - skipping '%s'" % path,
3766+ level=WARNING)
3767+ return
3768+
3769+ write(path, rendered_content.encode('utf-8').strip())
3770+ log('Wrote template %s' % path, level=DEBUG)
3771
3772=== added file 'hooks/charmhelpers/contrib/hardening/utils.py'
3773--- hooks/charmhelpers/contrib/hardening/utils.py 1970-01-01 00:00:00 +0000
3774+++ hooks/charmhelpers/contrib/hardening/utils.py 2016-04-22 08:16:14 +0000
3775@@ -0,0 +1,157 @@
3776+# Copyright 2016 Canonical Limited.
3777+#
3778+# This file is part of charm-helpers.
3779+#
3780+# charm-helpers is free software: you can redistribute it and/or modify
3781+# it under the terms of the GNU Lesser General Public License version 3 as
3782+# published by the Free Software Foundation.
3783+#
3784+# charm-helpers is distributed in the hope that it will be useful,
3785+# but WITHOUT ANY WARRANTY; without even the implied warranty of
3786+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3787+# GNU Lesser General Public License for more details.
3788+#
3789+# You should have received a copy of the GNU Lesser General Public License
3790+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3791+
3792+import glob
3793+import grp
3794+import os
3795+import pwd
3796+import six
3797+import yaml
3798+
3799+from charmhelpers.core.hookenv import (
3800+ log,
3801+ DEBUG,
3802+ INFO,
3803+ WARNING,
3804+ ERROR,
3805+)
3806+
3807+
3808+# Global settings cache. Since each hook fire entails a fresh module import it
3809+# is safe to hold this in memory and not risk missing config changes (since
3810+# they will result in a new hook fire and thus re-import).
3811+__SETTINGS__ = {}
3812+
3813+
3814+def _get_defaults(modules):
3815+ """Load the default config for the provided modules.
3816+
3817+ :param modules: stack modules config defaults to lookup.
3818+ :returns: modules default config dictionary.
3819+ """
3820+ default = os.path.join(os.path.dirname(__file__),
3821+ 'defaults/%s.yaml' % (modules))
3822+ return yaml.safe_load(open(default))
3823+
3824+
3825+def _get_schema(modules):
3826+ """Load the config schema for the provided modules.
3827+
3828+ NOTE: this schema is intended to have 1-1 relationship with they keys in
3829+ the default config and is used a means to verify valid overrides provided
3830+ by the user.
3831+
3832+ :param modules: stack modules config schema to lookup.
3833+ :returns: modules default schema dictionary.
3834+ """
3835+ schema = os.path.join(os.path.dirname(__file__),
3836+ 'defaults/%s.yaml.schema' % (modules))
3837+ return yaml.safe_load(open(schema))
3838+
3839+
3840+def _get_user_provided_overrides(modules):
3841+ """Load user-provided config overrides.
3842+
3843+ :param modules: stack modules to lookup in user overrides yaml file.
3844+ :returns: overrides dictionary.
3845+ """
3846+ overrides = os.path.join(os.environ['JUJU_CHARM_DIR'],
3847+ 'hardening.yaml')
3848+ if os.path.exists(overrides):
3849+ log("Found user-provided config overrides file '%s'" %
3850+ (overrides), level=DEBUG)
3851+ settings = yaml.safe_load(open(overrides))
3852+ if settings and settings.get(modules):
3853+ log("Applying '%s' overrides" % (modules), level=DEBUG)
3854+ return settings.get(modules)
3855+
3856+ log("No overrides found for '%s'" % (modules), level=DEBUG)
3857+ else:
3858+ log("No hardening config overrides file '%s' found in charm "
3859+ "root dir" % (overrides), level=DEBUG)
3860+
3861+ return {}
3862+
3863+
3864+def _apply_overrides(settings, overrides, schema):
3865+ """Get overrides config overlayed onto modules defaults.
3866+
3867+ :param modules: require stack modules config.
3868+ :returns: dictionary of modules config with user overrides applied.
3869+ """
3870+ if overrides:
3871+ for k, v in six.iteritems(overrides):
3872+ if k in schema:
3873+ if schema[k] is None:
3874+ settings[k] = v
3875+ elif type(schema[k]) is dict:
3876+ settings[k] = _apply_overrides(settings[k], overrides[k],
3877+ schema[k])
3878+ else:
3879+ raise Exception("Unexpected type found in schema '%s'" %
3880+ type(schema[k]), level=ERROR)
3881+ else:
3882+ log("Unknown override key '%s' - ignoring" % (k), level=INFO)
3883+
3884+ return settings
3885+
3886+
3887+def get_settings(modules):
3888+ global __SETTINGS__
3889+ if modules in __SETTINGS__:
3890+ return __SETTINGS__[modules]
3891+
3892+ schema = _get_schema(modules)
3893+ settings = _get_defaults(modules)
3894+ overrides = _get_user_provided_overrides(modules)
3895+ __SETTINGS__[modules] = _apply_overrides(settings, overrides, schema)
3896+ return __SETTINGS__[modules]
3897+
3898+
3899+def ensure_permissions(path, user, group, permissions, maxdepth=-1):
3900+ """Ensure permissions for path.
3901+
3902+ If path is a file, apply to file and return. If path is a directory,
3903+ apply recursively (if required) to directory contents and return.
3904+
3905+ :param user: user name
3906+ :param group: group name
3907+ :param permissions: octal permissions
3908+ :param maxdepth: maximum recursion depth. A negative maxdepth allows
3909+ infinite recursion and maxdepth=0 means no recursion.
3910+ :returns: None
3911+ """
3912+ if not os.path.exists(path):
3913+ log("File '%s' does not exist - cannot set permissions" % (path),
3914+ level=WARNING)
3915+ return
3916+
3917+ _user = pwd.getpwnam(user)
3918+ os.chown(path, _user.pw_uid, grp.getgrnam(group).gr_gid)
3919+ os.chmod(path, permissions)
3920+
3921+ if maxdepth == 0:
3922+ log("Max recursion depth reached - skipping further recursion",
3923+ level=DEBUG)
3924+ return
3925+ elif maxdepth > 0:
3926+ maxdepth -= 1
3927+
3928+ if os.path.isdir(path):
3929+ contents = glob.glob("%s/*" % (path))
3930+ for c in contents:
3931+ ensure_permissions(c, user=user, group=group,
3932+ permissions=permissions, maxdepth=maxdepth)
3933
3934=== added directory 'hooks/charmhelpers/contrib/mellanox'
3935=== added file 'hooks/charmhelpers/contrib/mellanox/__init__.py'
3936=== added file 'hooks/charmhelpers/contrib/mellanox/infiniband.py'
3937--- hooks/charmhelpers/contrib/mellanox/infiniband.py 1970-01-01 00:00:00 +0000
3938+++ hooks/charmhelpers/contrib/mellanox/infiniband.py 2016-04-22 08:16:14 +0000
3939@@ -0,0 +1,151 @@
3940+#!/usr/bin/env python
3941+# -*- coding: utf-8 -*-
3942+
3943+# Copyright 2014-2015 Canonical Limited.
3944+#
3945+# This file is part of charm-helpers.
3946+#
3947+# charm-helpers is free software: you can redistribute it and/or modify
3948+# it under the terms of the GNU Lesser General Public License version 3 as
3949+# published by the Free Software Foundation.
3950+#
3951+# charm-helpers is distributed in the hope that it will be useful,
3952+# but WITHOUT ANY WARRANTY; without even the implied warranty of
3953+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3954+# GNU Lesser General Public License for more details.
3955+#
3956+# You should have received a copy of the GNU Lesser General Public License
3957+# along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
3958+
3959+
3960+__author__ = "Jorge Niedbalski <jorge.niedbalski@canonical.com>"
3961+
3962+from charmhelpers.fetch import (
3963+ apt_install,
3964+ apt_update,
3965+)
3966+
3967+from charmhelpers.core.hookenv import (
3968+ log,
3969+ INFO,
3970+)
3971+
3972+try:
3973+ from netifaces import interfaces as network_interfaces
3974+except ImportError:
3975+ apt_install('python-netifaces')
3976+ from netifaces import interfaces as network_interfaces
3977+
3978+import os
3979+import re
3980+import subprocess
3981+
3982+from charmhelpers.core.kernel import modprobe
3983+
3984+REQUIRED_MODULES = (
3985+ "mlx4_ib",
3986+ "mlx4_en",
3987+ "mlx4_core",
3988+ "ib_ipath",
3989+ "ib_mthca",
3990+ "ib_srpt",
3991+ "ib_srp",
3992+ "ib_ucm",
3993+ "ib_isert",
3994+ "ib_iser",
3995+ "ib_ipoib",
3996+ "ib_cm",
3997+ "ib_uverbs"
3998+ "ib_umad",
3999+ "ib_sa",
4000+ "ib_mad",
4001+ "ib_core",
4002+ "ib_addr",
4003+ "rdma_ucm",
4004+)
4005+
4006+REQUIRED_PACKAGES = (
4007+ "ibutils",
4008+ "infiniband-diags",
4009+ "ibverbs-utils",
4010+)
4011+
4012+IPOIB_DRIVERS = (
4013+ "ib_ipoib",
4014+)
4015+
4016+ABI_VERSION_FILE = "/sys/class/infiniband_mad/abi_version"
4017+
4018+
4019+class DeviceInfo(object):
4020+ pass
4021+
4022+
4023+def install_packages():
4024+ apt_update()
4025+ apt_install(REQUIRED_PACKAGES, fatal=True)
4026+
4027+
4028+def load_modules():
4029+ for module in REQUIRED_MODULES:
4030+ modprobe(module, persist=True)
4031+
4032+
4033+def is_enabled():
4034+ """Check if infiniband is loaded on the system"""
4035+ return os.path.exists(ABI_VERSION_FILE)
4036+
4037+
4038+def stat():
4039+ """Return full output of ibstat"""
4040+ return subprocess.check_output(["ibstat"])
4041+
4042+
4043+def devices():
4044+ """Returns a list of IB enabled devices"""
4045+ return subprocess.check_output(['ibstat', '-l']).splitlines()
4046+
4047+
4048+def device_info(device):
4049+ """Returns a DeviceInfo object with the current device settings"""
4050+
4051+ status = subprocess.check_output([
4052+ 'ibstat', device, '-s']).splitlines()
4053+
4054+ regexes = {
4055+ "CA type: (.*)": "device_type",
4056+ "Number of ports: (.*)": "num_ports",
4057+ "Firmware version: (.*)": "fw_ver",
4058+ "Hardware version: (.*)": "hw_ver",
4059+ "Node GUID: (.*)": "node_guid",
4060+ "System image GUID: (.*)": "sys_guid",
4061+ }
4062+
4063+ device = DeviceInfo()
4064+
4065+ for line in status:
4066+ for expression, key in regexes.items():
4067+ matches = re.search(expression, line)
4068+ if matches:
4069+ setattr(device, key, matches.group(1))
4070+
4071+ return device
4072+
4073+
4074+def ipoib_interfaces():
4075+ """Return a list of IPOIB capable ethernet interfaces"""
4076+ interfaces = []
4077+
4078+ for interface in network_interfaces():
4079+ try:
4080+ driver = re.search('^driver: (.+)$', subprocess.check_output([
4081+ 'ethtool', '-i',
4082+ interface]), re.M).group(1)
4083+
4084+ if driver in IPOIB_DRIVERS:
4085+ interfaces.append(interface)
4086+ except:
4087+ log("Skipping interface %s" % interface, level=INFO)
4088+ continue
4089+
4090+ return interfaces
4091
4092=== modified file 'hooks/charmhelpers/contrib/network/ip.py'
4093--- hooks/charmhelpers/contrib/network/ip.py 2015-05-19 21:32:01 +0000
4094+++ hooks/charmhelpers/contrib/network/ip.py 2016-04-22 08:16:14 +0000
4095@@ -23,7 +23,7 @@
4096 from functools import partial
4097
4098 from charmhelpers.core.hookenv import unit_get
4099-from charmhelpers.fetch import apt_install
4100+from charmhelpers.fetch import apt_install, apt_update
4101 from charmhelpers.core.hookenv import (
4102 log,
4103 WARNING,
4104@@ -32,13 +32,15 @@
4105 try:
4106 import netifaces
4107 except ImportError:
4108- apt_install('python-netifaces')
4109+ apt_update(fatal=True)
4110+ apt_install('python-netifaces', fatal=True)
4111 import netifaces
4112
4113 try:
4114 import netaddr
4115 except ImportError:
4116- apt_install('python-netaddr')
4117+ apt_update(fatal=True)
4118+ apt_install('python-netaddr', fatal=True)
4119 import netaddr
4120
4121
4122@@ -51,7 +53,7 @@
4123
4124
4125 def no_ip_found_error_out(network):
4126- errmsg = ("No IP address found in network: %s" % network)
4127+ errmsg = ("No IP address found in network(s): %s" % network)
4128 raise ValueError(errmsg)
4129
4130
4131@@ -59,7 +61,7 @@
4132 """Get an IPv4 or IPv6 address within the network from the host.
4133
4134 :param network (str): CIDR presentation format. For example,
4135- '192.168.1.0/24'.
4136+ '192.168.1.0/24'. Supports multiple networks as a space-delimited list.
4137 :param fallback (str): If no address is found, return fallback.
4138 :param fatal (boolean): If no address is found, fallback is not
4139 set and fatal is True then exit(1).
4140@@ -73,24 +75,26 @@
4141 else:
4142 return None
4143
4144- _validate_cidr(network)
4145- network = netaddr.IPNetwork(network)
4146- for iface in netifaces.interfaces():
4147- addresses = netifaces.ifaddresses(iface)
4148- if network.version == 4 and netifaces.AF_INET in addresses:
4149- addr = addresses[netifaces.AF_INET][0]['addr']
4150- netmask = addresses[netifaces.AF_INET][0]['netmask']
4151- cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
4152- if cidr in network:
4153- return str(cidr.ip)
4154+ networks = network.split() or [network]
4155+ for network in networks:
4156+ _validate_cidr(network)
4157+ network = netaddr.IPNetwork(network)
4158+ for iface in netifaces.interfaces():
4159+ addresses = netifaces.ifaddresses(iface)
4160+ if network.version == 4 and netifaces.AF_INET in addresses:
4161+ addr = addresses[netifaces.AF_INET][0]['addr']
4162+ netmask = addresses[netifaces.AF_INET][0]['netmask']
4163+ cidr = netaddr.IPNetwork("%s/%s" % (addr, netmask))
4164+ if cidr in network:
4165+ return str(cidr.ip)
4166
4167- if network.version == 6 and netifaces.AF_INET6 in addresses:
4168- for addr in addresses[netifaces.AF_INET6]:
4169- if not addr['addr'].startswith('fe80'):
4170- cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
4171- addr['netmask']))
4172- if cidr in network:
4173- return str(cidr.ip)
4174+ if network.version == 6 and netifaces.AF_INET6 in addresses:
4175+ for addr in addresses[netifaces.AF_INET6]:
4176+ if not addr['addr'].startswith('fe80'):
4177+ cidr = netaddr.IPNetwork("%s/%s" % (addr['addr'],
4178+ addr['netmask']))
4179+ if cidr in network:
4180+ return str(cidr.ip)
4181
4182 if fallback is not None:
4183 return fallback
4184@@ -187,6 +191,15 @@
4185 get_netmask_for_address = partial(_get_for_address, key='netmask')
4186
4187
4188+def resolve_network_cidr(ip_address):
4189+ '''
4190+ Resolves the full address cidr of an ip_address based on
4191+ configured network interfaces
4192+ '''
4193+ netmask = get_netmask_for_address(ip_address)
4194+ return str(netaddr.IPNetwork("%s/%s" % (ip_address, netmask)).cidr)
4195+
4196+
4197 def format_ipv6_addr(address):
4198 """If address is IPv6, wrap it in '[]' otherwise return None.
4199
4200@@ -435,8 +448,12 @@
4201
4202 rev = dns.reversename.from_address(address)
4203 result = ns_query(rev)
4204+
4205 if not result:
4206- return None
4207+ try:
4208+ result = socket.gethostbyaddr(address)[0]
4209+ except:
4210+ return None
4211 else:
4212 result = address
4213
4214@@ -448,3 +465,18 @@
4215 return result
4216 else:
4217 return result.split('.')[0]
4218+
4219+
4220+def port_has_listener(address, port):
4221+ """
4222+ Returns True if the address:port is open and being listened to,
4223+ else False.
4224+
4225+ @param address: an IP address or hostname
4226+ @param port: integer port
4227+
4228+ Note calls 'zc' via a subprocess shell
4229+ """
4230+ cmd = ['nc', '-z', address, str(port)]
4231+ result = subprocess.call(cmd)
4232+ return not(bool(result))
4233
4234=== modified file 'hooks/charmhelpers/contrib/network/ovs/__init__.py'
4235--- hooks/charmhelpers/contrib/network/ovs/__init__.py 2015-05-19 21:32:01 +0000
4236+++ hooks/charmhelpers/contrib/network/ovs/__init__.py 2016-04-22 08:16:14 +0000
4237@@ -25,10 +25,14 @@
4238 )
4239
4240
4241-def add_bridge(name):
4242+def add_bridge(name, datapath_type=None):
4243 ''' Add the named bridge to openvswitch '''
4244 log('Creating bridge {}'.format(name))
4245- subprocess.check_call(["ovs-vsctl", "--", "--may-exist", "add-br", name])
4246+ cmd = ["ovs-vsctl", "--", "--may-exist", "add-br", name]
4247+ if datapath_type is not None:
4248+ cmd += ['--', 'set', 'bridge', name,
4249+ 'datapath_type={}'.format(datapath_type)]
4250+ subprocess.check_call(cmd)
4251
4252
4253 def del_bridge(name):
4254
4255=== modified file 'hooks/charmhelpers/contrib/network/ufw.py'
4256--- hooks/charmhelpers/contrib/network/ufw.py 2015-07-29 18:23:55 +0000
4257+++ hooks/charmhelpers/contrib/network/ufw.py 2016-04-22 08:16:14 +0000
4258@@ -40,7 +40,9 @@
4259 import re
4260 import os
4261 import subprocess
4262+
4263 from charmhelpers.core import hookenv
4264+from charmhelpers.core.kernel import modprobe, is_module_loaded
4265
4266 __author__ = "Felipe Reyes <felipe.reyes@canonical.com>"
4267
4268@@ -82,14 +84,11 @@
4269 # do we have IPv6 in the machine?
4270 if os.path.isdir('/proc/sys/net/ipv6'):
4271 # is ip6tables kernel module loaded?
4272- lsmod = subprocess.check_output(['lsmod'], universal_newlines=True)
4273- matches = re.findall('^ip6_tables[ ]+', lsmod, re.M)
4274- if len(matches) == 0:
4275+ if not is_module_loaded('ip6_tables'):
4276 # ip6tables support isn't complete, let's try to load it
4277 try:
4278- subprocess.check_output(['modprobe', 'ip6_tables'],
4279- universal_newlines=True)
4280- # great, we could load the module
4281+ modprobe('ip6_tables')
4282+ # great, we can load the module
4283 return True
4284 except subprocess.CalledProcessError as ex:
4285 hookenv.log("Couldn't load ip6_tables module: %s" % ex.output,
4286
4287=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
4288--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2015-07-29 18:23:55 +0000
4289+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py 2016-04-22 08:16:14 +0000
4290@@ -14,12 +14,18 @@
4291 # You should have received a copy of the GNU Lesser General Public License
4292 # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>.
4293
4294+import logging
4295+import re
4296+import sys
4297 import six
4298 from collections import OrderedDict
4299 from charmhelpers.contrib.amulet.deployment import (
4300 AmuletDeployment
4301 )
4302
4303+DEBUG = logging.DEBUG
4304+ERROR = logging.ERROR
4305+
4306
4307 class OpenStackAmuletDeployment(AmuletDeployment):
4308 """OpenStack amulet deployment.
4309@@ -28,9 +34,12 @@
4310 that is specifically for use by OpenStack charms.
4311 """
4312
4313- def __init__(self, series=None, openstack=None, source=None, stable=True):
4314+ def __init__(self, series=None, openstack=None, source=None,
4315+ stable=True, log_level=DEBUG):
4316 """Initialize the deployment environment."""
4317 super(OpenStackAmuletDeployment, self).__init__(series)
4318+ self.log = self.get_logger(level=log_level)
4319+ self.log.info('OpenStackAmuletDeployment: init')
4320 self.openstack = openstack
4321 self.source = source
4322 self.stable = stable
4323@@ -38,26 +47,55 @@
4324 # out.
4325 self.current_next = "trusty"
4326
4327+ def get_logger(self, name="deployment-logger", level=logging.DEBUG):
4328+ """Get a logger object that will log to stdout."""
4329+ log = logging
4330+ logger = log.getLogger(name)
4331+ fmt = log.Formatter("%(asctime)s %(funcName)s "
4332+ "%(levelname)s: %(message)s")
4333+
4334+ handler = log.StreamHandler(stream=sys.stdout)
4335+ handler.setLevel(level)
4336+ handler.setFormatter(fmt)
4337+
4338+ logger.addHandler(handler)
4339+ logger.setLevel(level)
4340+
4341+ return logger
4342+
4343 def _determine_branch_locations(self, other_services):
4344 """Determine the branch locations for the other services.
4345
4346 Determine if the local branch being tested is derived from its
4347 stable or next (dev) branch, and based on this, use the corresonding
4348 stable or next branches for the other_services."""
4349- base_charms = ['mysql', 'mongodb']
4350+
4351+ self.log.info('OpenStackAmuletDeployment: determine branch locations')
4352+
4353+ # Charms outside the lp:~openstack-charmers namespace
4354+ base_charms = ['mysql', 'mongodb', 'nrpe']
4355+
4356+ # Force these charms to current series even when using an older series.
4357+ # ie. Use trusty/nrpe even when series is precise, as the P charm
4358+ # does not possess the necessary external master config and hooks.
4359+ force_series_current = ['nrpe']
4360
4361 if self.series in ['precise', 'trusty']:
4362 base_series = self.series
4363 else:
4364 base_series = self.current_next
4365
4366- if self.stable:
4367- for svc in other_services:
4368+ for svc in other_services:
4369+ if svc['name'] in force_series_current:
4370+ base_series = self.current_next
4371+ # If a location has been explicitly set, use it
4372+ if svc.get('location'):
4373+ continue
4374+ if self.stable:
4375 temp = 'lp:charms/{}/{}'
4376 svc['location'] = temp.format(base_series,
4377 svc['name'])
4378- else:
4379- for svc in other_services:
4380+ else:
4381 if svc['name'] in base_charms:
4382 temp = 'lp:charms/{}/{}'
4383 svc['location'] = temp.format(base_series,
4384@@ -66,10 +104,13 @@
4385 temp = 'lp:~openstack-charmers/charms/{}/{}/next'
4386 svc['location'] = temp.format(self.current_next,
4387 svc['name'])
4388+
4389 return other_services
4390
4391 def _add_services(self, this_service, other_services):
4392 """Add services to the deployment and set openstack-origin/source."""
4393+ self.log.info('OpenStackAmuletDeployment: adding services')
4394+
4395 other_services = self._determine_branch_locations(other_services)
4396
4397 super(OpenStackAmuletDeployment, self)._add_services(this_service,
4398@@ -77,29 +118,105 @@
4399
4400 services = other_services
4401 services.append(this_service)
4402+
4403+ # Charms which should use the source config option
4404 use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
4405- 'ceph-osd', 'ceph-radosgw']
4406- # Most OpenStack subordinate charms do not expose an origin option
4407- # as that is controlled by the principle.
4408- ignore = ['cinder-ceph', 'hacluster', 'neutron-openvswitch']
4409+ 'ceph-osd', 'ceph-radosgw', 'ceph-mon']
4410+
4411+ # Charms which can not use openstack-origin, ie. many subordinates
4412+ no_origin = ['cinder-ceph', 'hacluster', 'neutron-openvswitch', 'nrpe',
4413+ 'openvswitch-odl', 'neutron-api-odl', 'odl-controller',
4414+ 'cinder-backup', 'nexentaedge-data',
4415+ 'nexentaedge-iscsi-gw', 'nexentaedge-swift-gw',
4416+ 'cinder-nexentaedge', 'nexentaedge-mgmt']
4417
4418 if self.openstack:
4419 for svc in services:
4420- if svc['name'] not in use_source + ignore:
4421+ if svc['name'] not in use_source + no_origin:
4422 config = {'openstack-origin': self.openstack}
4423 self.d.configure(svc['name'], config)
4424
4425 if self.source:
4426 for svc in services:
4427- if svc['name'] in use_source and svc['name'] not in ignore:
4428+ if svc['name'] in use_source and svc['name'] not in no_origin:
4429 config = {'source': self.source}
4430 self.d.configure(svc['name'], config)
4431
4432 def _configure_services(self, configs):
4433 """Configure all of the services."""
4434+ self.log.info('OpenStackAmuletDeployment: configure services')
4435 for service, config in six.iteritems(configs):
4436 self.d.configure(service, config)
4437
4438+ def _auto_wait_for_status(self, message=None, exclude_services=None,
4439+ include_only=None, timeout=1800):
4440+ """Wait for all units to have a specific extended status, except
4441+ for any defined as excluded. Unless specified via message, any
4442+ status containing any case of 'ready' will be considered a match.
4443+
4444+ Examples of message usage:
4445+
4446+ Wait for all unit status to CONTAIN any case of 'ready' or 'ok':
4447+ message = re.compile('.*ready.*|.*ok.*', re.IGNORECASE)
4448+
4449+ Wait for all units to reach this status (exact match):
4450+ message = re.compile('^Unit is ready and clustered$')
4451+
4452+ Wait for all units to reach any one of these (exact match):
4453+ message = re.compile('Unit is ready|OK|Ready')
4454+
4455+ Wait for at least one unit to reach this status (exact match):
4456+ message = {'ready'}
4457+
4458+ See Amulet's sentry.wait_for_messages() for message usage detail.
4459+ https://github.com/juju/amulet/blob/master/amulet/sentry.py
4460+
4461+ :param message: Expected status match
4462+ :param exclude_services: List of juju service names to ignore,
4463+ not to be used in conjuction with include_only.
4464+ :param include_only: List of juju service names to exclusively check,
4465+ not to be used in conjuction with exclude_services.
4466+ :param timeout: Maximum time in seconds to wait for status match
4467+ :returns: None. Raises if timeout is hit.
4468+ """
4469+ self.log.info('Waiting for extended status on units...')
4470+
4471+ all_services = self.d.services.keys()
4472+
4473+ if exclude_services and include_only:
4474+ raise ValueError('exclude_services can not be used '
4475+ 'with include_only')
4476+
4477+ if message:
4478+ if isinstance(message, re._pattern_type):
4479+ match = message.pattern
4480+ else:
4481+ match = message
4482+
4483+ self.log.debug('Custom extended status wait match: '
4484+ '{}'.format(match))
4485+ else:
4486+ self.log.debug('Default extended status wait match: contains '
4487+ 'READY (case-insensitive)')
4488+ message = re.compile('.*ready.*', re.IGNORECASE)
4489+
4490+ if exclude_services:
4491+ self.log.debug('Excluding services from extended status match: '
4492+ '{}'.format(exclude_services))
4493+ else:
4494+ exclude_services = []
4495+
4496+ if include_only:
4497+ services = include_only
4498+ else:
4499+ services = list(set(all_services) - set(exclude_services))
4500+
4501+ self.log.debug('Waiting up to {}s for extended status on services: '
4502+ '{}'.format(timeout, services))
4503+ service_messages = {service: message for service in services}
4504+ self.d.sentry.wait_for_messages(service_messages, timeout=timeout)
4505+ self.log.info('OK')
4506+
4507 def _get_openstack_release(self):
4508 """Get openstack release.
4509
4510@@ -111,7 +228,8 @@
4511 self.precise_havana, self.precise_icehouse,
4512 self.trusty_icehouse, self.trusty_juno, self.utopic_juno,
4513 self.trusty_kilo, self.vivid_kilo, self.trusty_liberty,
4514- self.wily_liberty) = range(12)
4515+ self.wily_liberty, self.trusty_mitaka,
4516+ self.xenial_mitaka) = range(14)
4517
4518 releases = {
4519 ('precise', None): self.precise_essex,
4520@@ -123,9 +241,11 @@
4521 ('trusty', 'cloud:trusty-juno'): self.trusty_juno,
4522 ('trusty', 'cloud:trusty-kilo'): self.trusty_kilo,
4523 ('trusty', 'cloud:trusty-liberty'): self.trusty_liberty,
4524+ ('trusty', 'cloud:trusty-mitaka'): self.trusty_mitaka,
4525 ('utopic', None): self.utopic_juno,
4526 ('vivid', None): self.vivid_kilo,
4527- ('wily', None): self.wily_liberty}
4528+ ('wily', None): self.wily_liberty,
4529+ ('xenial', None): self.xenial_mitaka}
4530 return releases[(self.series, self.openstack)]
4531
4532 def _get_openstack_release_string(self):
4533@@ -142,6 +262,7 @@
4534 ('utopic', 'juno'),
4535 ('vivid', 'kilo'),
4536 ('wily', 'liberty'),
4537+ ('xenial', 'mitaka'),
4538 ])
4539 if self.openstack:
4540 os_origin = self.openstack.split(':')[1]
4541
4542=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/utils.py'
4543--- hooks/charmhelpers/contrib/openstack/amulet/utils.py 2015-07-29 18:23:55 +0000
4544+++ hooks/charmhelpers/contrib/openstack/amulet/utils.py 2016-04-22 08:16:14 +0000
4545@@ -18,6 +18,7 @@
4546 import json
4547 import logging
4548 import os
4549+import re
4550 import six
4551 import time
4552 import urllib
4553@@ -26,7 +27,12 @@
4554 import glanceclient.v1.client as glance_client
4555 import heatclient.v1.client as heat_client
4556 import keystoneclient.v2_0 as keystone_client
4557-import novaclient.v1_1.client as nova_client
4558+from keystoneclient.auth.identity import v3 as keystone_id_v3
4559+from keystoneclient import session as keystone_session
4560+from keystoneclient.v3 import client as keystone_client_v3
4561+
4562+import novaclient.client as nova_client
4563+import pika
4564 import swiftclient
4565
4566 from charmhelpers.contrib.amulet.utils import (
4567@@ -36,6 +42,8 @@
4568 DEBUG = logging.DEBUG
4569 ERROR = logging.ERROR
4570
4571+NOVA_CLIENT_VERSION = "2"
4572+
4573
4574 class OpenStackAmuletUtils(AmuletUtils):
4575 """OpenStack amulet utilities.
4576@@ -137,7 +145,7 @@
4577 return "role {} does not exist".format(e['name'])
4578 return ret
4579
4580- def validate_user_data(self, expected, actual):
4581+ def validate_user_data(self, expected, actual, api_version=None):
4582 """Validate user data.
4583
4584 Validate a list of actual user data vs a list of expected user
4585@@ -148,10 +156,15 @@
4586 for e in expected:
4587 found = False
4588 for act in actual:
4589- a = {'enabled': act.enabled, 'name': act.name,
4590- 'email': act.email, 'tenantId': act.tenantId,
4591- 'id': act.id}
4592- if e['name'] == a['name']:
4593+ if e['name'] == act.name:
4594+ a = {'enabled': act.enabled, 'name': act.name,
4595+ 'email': act.email, 'id': act.id}
4596+ if api_version == 3:
4597+ a['default_project_id'] = getattr(act,
4598+ 'default_project_id',
4599+ 'none')
4600+ else:
4601+ a['tenantId'] = act.tenantId
4602 found = True
4603 ret = self._validate_dict_data(e, a)
4604 if ret:
4605@@ -186,15 +199,30 @@
4606 return cinder_client.Client(username, password, tenant, ept)
4607
4608 def authenticate_keystone_admin(self, keystone_sentry, user, password,
4609- tenant):
4610+ tenant=None, api_version=None,
4611+ keystone_ip=None):
4612 """Authenticates admin user with the keystone admin endpoint."""
4613 self.log.debug('Authenticating keystone admin...')
4614 unit = keystone_sentry
4615- service_ip = unit.relation('shared-db',
4616- 'mysql:shared-db')['private-address']
4617- ep = "http://{}:35357/v2.0".format(service_ip.strip().decode('utf-8'))
4618- return keystone_client.Client(username=user, password=password,
4619- tenant_name=tenant, auth_url=ep)
4620+ if not keystone_ip:
4621+ keystone_ip = unit.relation('shared-db',
4622+ 'mysql:shared-db')['private-address']
4623+ base_ep = "http://{}:35357".format(keystone_ip.strip().decode('utf-8'))
4624+ if not api_version or api_version == 2:
4625+ ep = base_ep + "/v2.0"
4626+ return keystone_client.Client(username=user, password=password,
4627+ tenant_name=tenant, auth_url=ep)
4628+ else:
4629+ ep = base_ep + "/v3"
4630+ auth = keystone_id_v3.Password(
4631+ user_domain_name='admin_domain',
4632+ username=user,
4633+ password=password,
4634+ domain_name='admin_domain',
4635+ auth_url=ep,
4636+ )
4637+ sess = keystone_session.Session(auth=auth)
4638+ return keystone_client_v3.Client(session=sess)
4639
4640 def authenticate_keystone_user(self, keystone, user, password, tenant):
4641 """Authenticates a regular user with the keystone public endpoint."""
4642@@ -223,7 +251,8 @@
4643 self.log.debug('Authenticating nova user ({})...'.format(user))
4644 ep = keystone.service_catalog.url_for(service_type='identity',
4645 endpoint_type='publicURL')
4646- return nova_client.Client(username=user, api_key=password,
4647+ return nova_client.Client(NOVA_CLIENT_VERSION,
4648+ username=user, api_key=password,
4649 project_id=tenant, auth_url=ep)
4650
4651 def authenticate_swift_user(self, keystone, user, password, tenant):
4652@@ -602,3 +631,382 @@
4653 self.log.debug('Ceph {} samples (OK): '
4654 '{}'.format(sample_type, samples))
4655 return None
4656+
4657+ # rabbitmq/amqp specific helpers:
4658+
4659+ def rmq_wait_for_cluster(self, deployment, init_sleep=15, timeout=1200):
4660+ """Wait for rmq units extended status to show cluster readiness,
4661+ after an optional initial sleep period. Initial sleep is likely
4662+ necessary to be effective following a config change, as status
4663+ message may not instantly update to non-ready."""
4664+
4665+ if init_sleep:
4666+ time.sleep(init_sleep)
4667+
4668+ message = re.compile('^Unit is ready and clustered$')
4669+ deployment._auto_wait_for_status(message=message,
4670+ timeout=timeout,
4671+ include_only=['rabbitmq-server'])
4672+
4673+ def add_rmq_test_user(self, sentry_units,
4674+ username="testuser1", password="changeme"):
4675+ """Add a test user via the first rmq juju unit, check connection as
4676+ the new user against all sentry units.
4677+
4678+ :param sentry_units: list of sentry unit pointers
4679+ :param username: amqp user name, default to testuser1
4680+ :param password: amqp user password
4681+ :returns: None if successful. Raise on error.
4682+ """
4683+ self.log.debug('Adding rmq user ({})...'.format(username))
4684+
4685+ # Check that user does not already exist
4686+ cmd_user_list = 'rabbitmqctl list_users'
4687+ output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
4688+ if username in output:
4689+ self.log.warning('User ({}) already exists, returning '
4690+ 'gracefully.'.format(username))
4691+ return
4692+
4693+ perms = '".*" ".*" ".*"'
4694+ cmds = ['rabbitmqctl add_user {} {}'.format(username, password),
4695+ 'rabbitmqctl set_permissions {} {}'.format(username, perms)]
4696+
4697+ # Add user via first unit
4698+ for cmd in cmds:
4699+ output, _ = self.run_cmd_unit(sentry_units[0], cmd)
4700+
4701+ # Check connection against the other sentry_units
4702+ self.log.debug('Checking user connect against units...')
4703+ for sentry_unit in sentry_units:
4704+ connection = self.connect_amqp_by_unit(sentry_unit, ssl=False,
4705+ username=username,
4706+ password=password)
4707+ connection.close()
4708+
4709+ def delete_rmq_test_user(self, sentry_units, username="testuser1"):
4710+ """Delete a rabbitmq user via the first rmq juju unit.
4711+
4712+ :param sentry_units: list of sentry unit pointers
4713+ :param username: amqp user name, default to testuser1
4714+ :param password: amqp user password
4715+ :returns: None if successful or no such user.
4716+ """
4717+ self.log.debug('Deleting rmq user ({})...'.format(username))
4718+
4719+ # Check that the user exists
4720+ cmd_user_list = 'rabbitmqctl list_users'
4721+ output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_list)
4722+
4723+ if username not in output:
4724+ self.log.warning('User ({}) does not exist, returning '
4725+ 'gracefully.'.format(username))
4726+ return
4727+
4728+ # Delete the user
4729+ cmd_user_del = 'rabbitmqctl delete_user {}'.format(username)
4730+ output, _ = self.run_cmd_unit(sentry_units[0], cmd_user_del)
4731+
4732+ def get_rmq_cluster_status(self, sentry_unit):
4733+ """Execute rabbitmq cluster status command on a unit and return
4734+ the full output.
4735+
4736+ :param unit: sentry unit
4737+ :returns: String containing console output of cluster status command
4738+ """
4739+ cmd = 'rabbitmqctl cluster_status'
4740+ output, _ = self.run_cmd_unit(sentry_unit, cmd)
4741+ self.log.debug('{} cluster_status:\n{}'.format(
4742+ sentry_unit.info['unit_name'], output))
4743+ return str(output)
4744+
4745+ def get_rmq_cluster_running_nodes(self, sentry_unit):
4746+ """Parse rabbitmqctl cluster_status output string, return list of
4747+ running rabbitmq cluster nodes.
4748+
4749+ :param unit: sentry unit
4750+ :returns: List containing node names of running nodes
4751+ """
4752+ # NOTE(beisner): rabbitmqctl cluster_status output is not
4753+ # json-parsable, do string chop foo, then json.loads that.
4754+ str_stat = self.get_rmq_cluster_status(sentry_unit)
4755+ if 'running_nodes' in str_stat:
4756+ pos_start = str_stat.find("{running_nodes,") + 15
4757+ pos_end = str_stat.find("]},", pos_start) + 1
4758+ str_run_nodes = str_stat[pos_start:pos_end].replace("'", '"')
4759+ run_nodes = json.loads(str_run_nodes)
4760+ return run_nodes
4761+ else:
4762+ return []
4763+
4764+ def validate_rmq_cluster_running_nodes(self, sentry_units):
4765+ """Check that all rmq unit hostnames are represented in the
4766+ cluster_status output of all units.
4767+
4768+ :param host_names: dict of juju unit names to host names
4769+ :param units: list of sentry unit pointers (all rmq units)
4770+ :returns: None if successful, otherwise return error message
4771+ """
4772+ host_names = self.get_unit_hostnames(sentry_units)
4773+ errors = []
4774+
4775+ # Query every unit for cluster_status running nodes
4776+ for query_unit in sentry_units:
4777+ query_unit_name = query_unit.info['unit_name']
4778+ running_nodes = self.get_rmq_cluster_running_nodes(query_unit)
4779+
4780+ # Confirm that every unit is represented in the queried unit's
4781+ # cluster_status running nodes output.
4782+ for validate_unit in sentry_units:
4783+ val_host_name = host_names[validate_unit.info['unit_name']]
4784+ val_node_name = 'rabbit@{}'.format(val_host_name)
4785+
4786+ if val_node_name not in running_nodes:
4787+ errors.append('Cluster member check failed on {}: {} not '
4788+ 'in {}\n'.format(query_unit_name,
4789+ val_node_name,
4790+ running_nodes))
4791+ if errors:
4792+ return ''.join(errors)
4793+
4794+ def rmq_ssl_is_enabled_on_unit(self, sentry_unit, port=None):
4795+ """Check a single juju rmq unit for ssl and port in the config file."""
4796+ host = sentry_unit.info['public-address']
4797+ unit_name = sentry_unit.info['unit_name']
4798+
4799+ conf_file = '/etc/rabbitmq/rabbitmq.config'
4800+ conf_contents = str(self.file_contents_safe(sentry_unit,
4801+ conf_file, max_wait=16))
4802+ # Checks
4803+ conf_ssl = 'ssl' in conf_contents
4804+ conf_port = str(port) in conf_contents
4805+
4806+ # Port explicitly checked in config
4807+ if port and conf_port and conf_ssl:
4808+ self.log.debug('SSL is enabled @{}:{} '
4809+ '({})'.format(host, port, unit_name))
4810+ return True
4811+ elif port and not conf_port and conf_ssl:
4812+ self.log.debug('SSL is enabled @{} but not on port {} '
4813+ '({})'.format(host, port, unit_name))
4814+ return False
4815+ # Port not checked (useful when checking that ssl is disabled)
4816+ elif not port and conf_ssl:
4817+ self.log.debug('SSL is enabled @{}:{} '
4818+ '({})'.format(host, port, unit_name))
4819+ return True
4820+ elif not conf_ssl:
4821+ self.log.debug('SSL not enabled @{}:{} '
4822+ '({})'.format(host, port, unit_name))
4823+ return False
4824+ else:
4825+ msg = ('Unknown condition when checking SSL status @{}:{} '
4826+ '({})'.format(host, port, unit_name))
4827+ amulet.raise_status(amulet.FAIL, msg)
4828+
4829+ def validate_rmq_ssl_enabled_units(self, sentry_units, port=None):
4830+ """Check that ssl is enabled on rmq juju sentry units.
4831+
4832+ :param sentry_units: list of all rmq sentry units
4833+ :param port: optional ssl port override to validate
4834+ :returns: None if successful, otherwise return error message
4835+ """
4836+ for sentry_unit in sentry_units:
4837+ if not self.rmq_ssl_is_enabled_on_unit(sentry_unit, port=port):
4838+ return ('Unexpected condition: ssl is disabled on unit '
4839+ '({})'.format(sentry_unit.info['unit_name']))
4840+ return None
4841+
4842+ def validate_rmq_ssl_disabled_units(self, sentry_units):
4843+ """Check that ssl is enabled on listed rmq juju sentry units.
4844+
4845+ :param sentry_units: list of all rmq sentry units
4846+ :returns: True if successful. Raise on error.
4847+ """
4848+ for sentry_unit in sentry_units:
4849+ if self.rmq_ssl_is_enabled_on_unit(sentry_unit):
4850+ return ('Unexpected condition: ssl is enabled on unit '
4851+ '({})'.format(sentry_unit.info['unit_name']))
4852+ return None
4853+
4854+ def configure_rmq_ssl_on(self, sentry_units, deployment,
4855+ port=None, max_wait=60):
4856+ """Turn ssl charm config option on, with optional non-default
4857+ ssl port specification. Confirm that it is enabled on every
4858+ unit.
4859+
4860+ :param sentry_units: list of sentry units
4861+ :param deployment: amulet deployment object pointer
4862+ :param port: amqp port, use defaults if None
4863+ :param max_wait: maximum time to wait in seconds to confirm
4864+ :returns: None if successful. Raise on error.
4865+ """
4866+ self.log.debug('Setting ssl charm config option: on')
4867+
4868+ # Enable RMQ SSL
4869+ config = {'ssl': 'on'}
4870+ if port:
4871+ config['ssl_port'] = port
4872+
4873+ deployment.d.configure('rabbitmq-server', config)
4874+
4875+ # Wait for unit status
4876+ self.rmq_wait_for_cluster(deployment)
4877+
4878+ # Confirm
4879+ tries = 0
4880+ ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
4881+ while ret and tries < (max_wait / 4):
4882+ time.sleep(4)
4883+ self.log.debug('Attempt {}: {}'.format(tries, ret))
4884+ ret = self.validate_rmq_ssl_enabled_units(sentry_units, port=port)
4885+ tries += 1
4886+
4887+ if ret:
4888+ amulet.raise_status(amulet.FAIL, ret)
4889+
4890+ def configure_rmq_ssl_off(self, sentry_units, deployment, max_wait=60):
4891+ """Turn ssl charm config option off, confirm that it is disabled
4892+ on every unit.
4893+
4894+ :param sentry_units: list of sentry units
4895+ :param deployment: amulet deployment object pointer
4896+ :param max_wait: maximum time to wait in seconds to confirm
4897+ :returns: None if successful. Raise on error.
4898+ """
4899+ self.log.debug('Setting ssl charm config option: off')
4900+
4901+ # Disable RMQ SSL
4902+ config = {'ssl': 'off'}
4903+ deployment.d.configure('rabbitmq-server', config)
4904+
4905+ # Wait for unit status
4906+ self.rmq_wait_for_cluster(deployment)
4907+
4908+ # Confirm
4909+ tries = 0
4910+ ret = self.validate_rmq_ssl_disabled_units(sentry_units)
4911+ while ret and tries < (max_wait / 4):
4912+ time.sleep(4)
4913+ self.log.debug('Attempt {}: {}'.format(tries, ret))
4914+ ret = self.validate_rmq_ssl_disabled_units(sentry_units)
4915+ tries += 1
4916+
4917+ if ret:
4918+ amulet.raise_status(amulet.FAIL, ret)
4919+
4920+ def connect_amqp_by_unit(self, sentry_unit, ssl=False,
4921+ port=None, fatal=True,
4922+ username="testuser1", password="changeme"):
4923+ """Establish and return a pika amqp connection to the rabbitmq service
4924+ running on a rmq juju unit.
4925+
4926+ :param sentry_unit: sentry unit pointer
4927+ :param ssl: boolean, default to False
4928+ :param port: amqp port, use defaults if None
4929+ :param fatal: boolean, default to True (raises on connect error)
4930+ :param username: amqp user name, default to testuser1
4931+ :param password: amqp user password
4932+ :returns: pika amqp connection pointer or None if failed and non-fatal
4933+ """
4934+ host = sentry_unit.info['public-address']
4935+ unit_name = sentry_unit.info['unit_name']
4936+
4937+ # Default port logic if port is not specified
4938+ if ssl and not port:
4939+ port = 5671
4940+ elif not ssl and not port:
4941+ port = 5672
4942+
4943+ self.log.debug('Connecting to amqp on {}:{} ({}) as '
4944+ '{}...'.format(host, port, unit_name, username))
4945+
4946+ try:
4947+ credentials = pika.PlainCredentials(username, password)
4948+ parameters = pika.ConnectionParameters(host=host, port=port,
4949+ credentials=credentials,
4950+ ssl=ssl,
4951+ connection_attempts=3,
4952+ retry_delay=5,
4953+ socket_timeout=1)
4954+ connection = pika.BlockingConnection(parameters)
4955+ assert connection.server_properties['product'] == 'RabbitMQ'
4956+ self.log.debug('Connect OK')
4957+ return connection
4958+ except Exception as e:
4959+ msg = ('amqp connection failed to {}:{} as '
4960+ '{} ({})'.format(host, port, username, str(e)))
4961+ if fatal:
4962+ amulet.raise_status(amulet.FAIL, msg)
4963+ else:
4964+ self.log.warn(msg)
4965+ return None
4966+
4967+ def publish_amqp_message_by_unit(self, sentry_unit, message,
4968+ queue="test", ssl=False,
4969+ username="testuser1",
4970+ password="changeme",
4971+ port=None):
4972+ """Publish an amqp message to a rmq juju unit.
4973+
4974+ :param sentry_unit: sentry unit pointer
4975+ :param message: amqp message string
4976+ :param queue: message queue, default to test
4977+ :param username: amqp user name, default to testuser1
4978+ :param password: amqp user password
4979+ :param ssl: boolean, default to False
4980+ :param port: amqp port, use defaults if None
4981+ :returns: None. Raises exception if publish failed.
4982+ """
4983+ self.log.debug('Publishing message to {} queue:\n{}'.format(queue,
4984+ message))
4985+ connection = self.connect_amqp_by_unit(sentry_unit, ssl=ssl,
4986+ port=port,
4987+ username=username,
4988+ password=password)
4989+
4990+ # NOTE(beisner): extra debug here re: pika hang potential:
4991+ # https://github.com/pika/pika/issues/297
4992+ # https://groups.google.com/forum/#!topic/rabbitmq-users/Ja0iyfF0Szw
4993+ self.log.debug('Defining channel...')
4994+ channel = connection.channel()
4995+ self.log.debug('Declaring queue...')
4996+ channel.queue_declare(queue=queue, auto_delete=False, durable=True)
4997+ self.log.debug('Publishing message...')
4998+ channel.basic_publish(exchange='', routing_key=queue, body=message)
4999+ self.log.debug('Closing channel...')
5000+ channel.close()
The diff has been truncated for viewing.

Subscribers

People subscribed via source and target branches