Merge lp:~mbruzek/charms/trusty/fail2ban/trunk into lp:~lazypower/charms/trusty/fail2ban/trunk

[Matt Bruzek]

Found some things that needed to be updated in trusty.

jail.local template
metadata.yaml
00-setup
sync charm-tools

Unmerged revisions

5. By Matt Bruzek on 2014-11-24

Added maxretries to the ssh section in template, updated the metadata with the security tag, synchronized charm helpers.

=== modified file 'lib/charmhelpers/core/fstab.py'
--- lib/charmhelpers/core/fstab.py	2014-10-22 04:18:26 +0000
+++ lib/charmhelpers/core/fstab.py	2014-11-24 15:36:56 +0000
@@ -3,10 +3,11 @@
 
 __author__ = 'Jorge Niedbalski R. <[e-mail]>'
 
+import io
 import os
 
 
-class Fstab(file):
+class Fstab(io.FileIO):
     """This class extends file in order to implement a file reader/writer
     for file `/etc/fstab`
     """
@@ -24,8 +25,8 @@
                 options = "defaults"
 
             self.options = options
-            self.d = d
-            self.p = p
+            self.d = int(d)
+            self.p = int(p)
 
         def __eq__(self, o):
             return str(self) == str(o)
@@ -45,7 +46,7 @@
             self._path = path
         else:
             self._path = self.DEFAULT_PATH
-        file.__init__(self, self._path, 'r+')
+        super(Fstab, self).__init__(self._path, 'rb+')
 
     def _hydrate_entry(self, line):
         # NOTE: use split with no arguments to split on any
@@ -58,8 +59,9 @@
     def entries(self):
         self.seek(0)
         for line in self.readlines():
+            line = line.decode('us-ascii')
             try:
-                if not line.startswith("#"):
+                if line.strip() and not line.startswith("#"):
                     yield self._hydrate_entry(line)
             except ValueError:
                 pass
@@ -75,14 +77,14 @@
         if self.get_entry_by_attr('device', entry.device):
             return False
 
-        self.write(str(entry) + '\n')
+        self.write((str(entry) + '\n').encode('us-ascii'))
         self.truncate()
         return entry
 
     def remove_entry(self, entry):
         self.seek(0)
 
-        lines = self.readlines()
+        lines = [l.decode('us-ascii') for l in self.readlines()]
 
         found = False
         for index, line in enumerate(lines):
@@ -97,7 +99,7 @@
         lines.remove(line)
 
         self.seek(0)
-        self.write(''.join(lines))
+        self.write(''.join(lines).encode('us-ascii'))
         self.truncate()
         return True
 

=== modified file 'lib/charmhelpers/core/hookenv.py'
--- lib/charmhelpers/core/hookenv.py	2014-10-22 04:18:26 +0000
+++ lib/charmhelpers/core/hookenv.py	2014-11-24 15:36:56 +0000
@@ -9,9 +9,11 @@
 import yaml
 import subprocess
 import sys
-import UserDict
 from subprocess import CalledProcessError
 
+import six
+from six.moves import UserDict
+
 CRITICAL = "CRITICAL"
 ERROR = "ERROR"
 WARNING = "WARNING"
@@ -67,12 +69,12 @@
     subprocess.call(command)
 
 
-class Serializable(UserDict.IterableUserDict):
+class Serializable(UserDict):
     """Wrapper, an object that can be serialized to yaml or json"""
 
     def __init__(self, obj):
         # wrap the object
-        UserDict.IterableUserDict.__init__(self)
+        UserDict.__init__(self)
         self.data = obj
 
     def __getattr__(self, attr):
@@ -218,7 +220,7 @@
         prev_keys = []
         if self._prev_dict is not None:
             prev_keys = self._prev_dict.keys()
-        return list(set(prev_keys + dict.keys(self)))
+        return list(set(prev_keys + list(dict.keys(self))))
 
     def load_previous(self, path=None):
         """Load previous copy of config from disk.
@@ -269,7 +271,7 @@
 
         """
         if self._prev_dict:
-            for k, v in self._prev_dict.iteritems():
+            for k, v in six.iteritems(self._prev_dict):
                 if k not in self:
                     self[k] = v
         with open(self.path, 'w') as f:
@@ -306,7 +308,7 @@
         return json.loads(subprocess.check_output(_args))
     except ValueError:
         return None
-    except CalledProcessError, e:
+    except CalledProcessError as e:
         if e.returncode == 2:
             return None
         raise
@@ -318,7 +320,7 @@
     relation_cmd_line = ['relation-set']
     if relation_id is not None:
         relation_cmd_line.extend(('-r', relation_id))
-    for k, v in (relation_settings.items() + kwargs.items()):
+    for k, v in (list(relation_settings.items()) + list(kwargs.items())):
         if v is None:
             relation_cmd_line.append('{}='.format(k))
         else:

=== modified file 'lib/charmhelpers/core/host.py'
--- lib/charmhelpers/core/host.py	2014-10-22 04:18:26 +0000
+++ lib/charmhelpers/core/host.py	2014-11-24 15:36:56 +0000
@@ -14,11 +14,12 @@
 import subprocess
 import hashlib
 from contextlib import contextmanager
-
 from collections import OrderedDict
 
-from hookenv import log
-from fstab import Fstab
+import six
+
+from .hookenv import log
+from .fstab import Fstab
 
 
 def service_start(service_name):
@@ -130,7 +131,7 @@
     subprocess.check_call(cmd)
 
 
-def mkdir(path, owner='root', group='root', perms=0555, force=False):
+def mkdir(path, owner='root', group='root', perms=0o555, force=False):
     """Create a directory"""
     log("Making dir {} {}:{} {:o}".format(path, owner, group,
                                           perms))
@@ -146,7 +147,7 @@
     os.chown(realpath, uid, gid)
 
 
-def write_file(path, content, owner='root', group='root', perms=0444):
+def write_file(path, content, owner='root', group='root', perms=0o444):
     """Create or overwrite a file with the contents of a string"""
     log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
     uid = pwd.getpwnam(owner).pw_uid
@@ -177,7 +178,7 @@
     cmd_args.extend([device, mountpoint])
     try:
         subprocess.check_output(cmd_args)
-    except subprocess.CalledProcessError, e:
+    except subprocess.CalledProcessError as e:
         log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
         return False
 
@@ -191,7 +192,7 @@
     cmd_args = ['umount', mountpoint]
     try:
         subprocess.check_output(cmd_args)
-    except subprocess.CalledProcessError, e:
+    except subprocess.CalledProcessError as e:
         log('Error unmounting {}\n{}'.format(mountpoint, e.output))
         return False
 
@@ -218,8 +219,8 @@
     """
     if os.path.exists(path):
         h = getattr(hashlib, hash_type)()
-        with open(path, 'r') as source:
-            h.update(source.read())  # IGNORE:E1101 - it does have update
+        with open(path, 'rb') as source:
+            h.update(source.read())
         return h.hexdigest()
     else:
         return None
@@ -297,7 +298,7 @@
     if length is None:
         length = random.choice(range(35, 45))
     alphanumeric_chars = [
-        l for l in (string.letters + string.digits)
+        l for l in (string.ascii_letters + string.digits)
         if l not in 'l0QD1vAEIOUaeiou']
     random_chars = [
         random.choice(alphanumeric_chars) for _ in range(length)]
@@ -306,7 +307,7 @@
 
 def list_nics(nic_type):
     '''Return a list of nics of given type(s)'''
-    if isinstance(nic_type, basestring):
+    if isinstance(nic_type, six.string_types):
         int_types = [nic_type]
     else:
         int_types = nic_type

=== modified file 'lib/charmhelpers/core/services/helpers.py'
--- lib/charmhelpers/core/services/helpers.py	2014-10-22 04:18:26 +0000
+++ lib/charmhelpers/core/services/helpers.py	2014-11-24 15:36:56 +0000
@@ -196,7 +196,7 @@
         if not os.path.isabs(file_name):
             file_name = os.path.join(hookenv.charm_dir(), file_name)
         with open(file_name, 'w') as file_stream:
-            os.fchmod(file_stream.fileno(), 0600)
+            os.fchmod(file_stream.fileno(), 0o600)
             yaml.dump(config_data, file_stream)
 
     def read_context(self, file_name):
@@ -211,15 +211,19 @@
 
 class TemplateCallback(ManagerCallback):
     """
-    Callback class that will render a Jinja2 template, for use as a ready action.
-
-    :param str source: The template source file, relative to `$CHARM_DIR/templates`
+    Callback class that will render a Jinja2 template, for use as a ready
+    action.
+
+    :param str source: The template source file, relative to
+    `$CHARM_DIR/templates`
+
     :param str target: The target to write the rendered template to
     :param str owner: The owner of the rendered file
     :param str group: The group of the rendered file
     :param int perms: The permissions of the rendered file
     """
-    def __init__(self, source, target, owner='root', group='root', perms=0444):
+    def __init__(self, source, target,
+                 owner='root', group='root', perms=0o444):
         self.source = source
         self.target = target
         self.owner = owner

=== modified file 'lib/charmhelpers/core/templating.py'
--- lib/charmhelpers/core/templating.py	2014-10-22 04:18:26 +0000
+++ lib/charmhelpers/core/templating.py	2014-11-24 15:36:56 +0000
@@ -4,7 +4,8 @@
 from charmhelpers.core import hookenv
 
 
-def render(source, target, context, owner='root', group='root', perms=0444, templates_dir=None):
+def render(source, target, context, owner='root', group='root',
+           perms=0o444, templates_dir=None):
     """
     Render a template.
 

=== modified file 'lib/charmhelpers/fetch/__init__.py'
--- lib/charmhelpers/fetch/__init__.py	2014-11-20 21:55:44 +0000
+++ lib/charmhelpers/fetch/__init__.py	2014-11-24 15:36:56 +0000
@@ -5,10 +5,6 @@
 from charmhelpers.core.host import (
     lsb_release
 )
-from urlparse import (
-    urlparse,
-    urlunparse,
-)
 import subprocess
 from charmhelpers.core.hookenv import (
     config,
@@ -16,6 +12,9 @@
 )
 import os
 
+import six
+from six.moves.urllib.parse import urlparse, urlunparse
+
 
 CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
 deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
@@ -69,11 +68,16 @@
 
 # The order of this list is very important. Handlers should be listed in from
 # least- to most-specific URL matching.
-FETCH_HANDLERS = (
-    'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
-    'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
-    'charmhelpers.fetch.giturl.GitUrlFetchHandler',
-)
+if six.PY2:
+    FETCH_HANDLERS = (
+        'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
+        'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
+        'charmhelpers.fetch.giturl.GitUrlFetchHandler',
+    )
+else:
+    FETCH_HANDLERS = (
+        'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
+    )
 
 APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.
 APT_NO_LOCK_RETRY_DELAY = 10  # Wait 10 seconds between apt lock checks.
@@ -149,7 +153,7 @@
     cmd = ['apt-get', '--assume-yes']
     cmd.extend(options)
     cmd.append('install')
-    if isinstance(packages, basestring):
+    if isinstance(packages, six.string_types):
         cmd.append(packages)
     else:
         cmd.extend(packages)
@@ -182,7 +186,7 @@
 def apt_purge(packages, fatal=False):
     """Purge one or more packages"""
     cmd = ['apt-get', '--assume-yes', 'purge']
-    if isinstance(packages, basestring):
+    if isinstance(packages, six.string_types):
         cmd.append(packages)
     else:
         cmd.extend(packages)
@@ -193,7 +197,7 @@
 def apt_hold(packages, fatal=False):
     """Hold one or more packages"""
     cmd = ['apt-mark', 'hold']
-    if isinstance(packages, basestring):
+    if isinstance(packages, six.string_types):
         cmd.append(packages)
     else:
         cmd.extend(packages)
@@ -260,7 +264,7 @@
 
     if key:
         if '-----BEGIN PGP PUBLIC KEY BLOCK-----' in key:
-            with NamedTemporaryFile() as key_file:
+            with NamedTemporaryFile('w+') as key_file:
                 key_file.write(key)
                 key_file.flush()
                 key_file.seek(0)
@@ -297,14 +301,14 @@
     sources = safe_load((config(sources_var) or '').strip()) or []
     keys = safe_load((config(keys_var) or '').strip()) or None
 
-    if isinstance(sources, basestring):
+    if isinstance(sources, six.string_types):
         sources = [sources]
 
     if keys is None:
         for source in sources:
             add_source(source, None)
     else:
-        if isinstance(keys, basestring):
+        if isinstance(keys, six.string_types):
             keys = [keys]
 
         if len(sources) != len(keys):
@@ -401,7 +405,7 @@
         while result is None or result == APT_NO_LOCK:
             try:
                 result = subprocess.check_call(cmd, env=env)
-            except subprocess.CalledProcessError, e:
+            except subprocess.CalledProcessError as e:
                 retry_count = retry_count + 1
                 if retry_count > APT_NO_LOCK_RETRY_COUNT:
                     raise

=== modified file 'lib/charmhelpers/fetch/archiveurl.py'
--- lib/charmhelpers/fetch/archiveurl.py	2014-10-22 04:18:26 +0000
+++ lib/charmhelpers/fetch/archiveurl.py	2014-11-24 15:36:56 +0000
@@ -1,8 +1,14 @@
 import os
-import urllib2
-from urllib import urlretrieve
-import urlparse
 import hashlib
+import re
+
+import six
+from six.moves.urllib.request import (
+    build_opener, install_opener, urlopen, urlretrieve,
+    HTTPPasswordMgrWithDefaultRealm, HTTPBasicAuthHandler,
+)
+from six.moves.urllib.parse import urlparse, urlunparse, parse_qs
+from six.moves.urllib.error import URLError
 
 from charmhelpers.fetch import (
     BaseFetchHandler,
@@ -15,6 +21,24 @@
 from charmhelpers.core.host import mkdir, check_hash
 
 
+def splituser(host):
+    '''urllib.splituser(), but six's support of this seems broken'''
+    _userprog = re.compile('^(.*)@(.*)$')
+    match = _userprog.match(host)
+    if match:
+        return match.group(1, 2)
+    return None, host
+
+
+def splitpasswd(user):
+    '''urllib.splitpasswd(), but six's support of this is missing'''
+    _passwdprog = re.compile('^([^:]*):(.*)$', re.S)
+    match = _passwdprog.match(user)
+    if match:
+        return match.group(1, 2)
+    return user, None
+
+
 class ArchiveUrlFetchHandler(BaseFetchHandler):
     """
     Handler to download archive files from arbitrary URLs.
@@ -42,20 +66,20 @@
         """
         # propogate all exceptions
         # URLError, OSError, etc
-        proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
+        proto, netloc, path, params, query, fragment = urlparse(source)
         if proto in ('http', 'https'):
-            auth, barehost = urllib2.splituser(netloc)
+            auth, barehost = splituser(netloc)
             if auth is not None:
-                source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
-                username, password = urllib2.splitpasswd(auth)
-                passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+                source = urlunparse((proto, barehost, path, params, query, fragment))
+                username, password = splitpasswd(auth)
+                passman = HTTPPasswordMgrWithDefaultRealm()
                 # Realm is set to None in add_password to force the username and password
                 # to be used whatever the realm
                 passman.add_password(None, source, username, password)
-                authhandler = urllib2.HTTPBasicAuthHandler(passman)
-                opener = urllib2.build_opener(authhandler)
-                urllib2.install_opener(opener)
-        response = urllib2.urlopen(source)
+                authhandler = HTTPBasicAuthHandler(passman)
+                opener = build_opener(authhandler)
+                install_opener(opener)
+        response = urlopen(source)
         try:
             with open(dest, 'w') as dest_file:
                 dest_file.write(response.read())
@@ -91,17 +115,21 @@
         url_parts = self.parse_url(source)
         dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
         if not os.path.exists(dest_dir):
-            mkdir(dest_dir, perms=0755)
+            mkdir(dest_dir, perms=0o755)
         dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
         try:
             self.download(source, dld_file)
-        except urllib2.URLError as e:
+        except URLError as e:
             raise UnhandledSource(e.reason)
         except OSError as e:
             raise UnhandledSource(e.strerror)
-        options = urlparse.parse_qs(url_parts.fragment)
+        options = parse_qs(url_parts.fragment)
         for key, value in options.items():
-            if key in hashlib.algorithms:
+            if six.PY2:
+                algorithms = hashlib.algorithms
+            else:
+                algorithms = hashlib.algorithms_available
+            if key in algorithms:
                 check_hash(dld_file, value, key)
         if checksum:
             check_hash(dld_file, checksum, hash_type)

=== modified file 'lib/charmhelpers/fetch/bzrurl.py'
--- lib/charmhelpers/fetch/bzrurl.py	2014-10-22 04:18:26 +0000
+++ lib/charmhelpers/fetch/bzrurl.py	2014-11-24 15:36:56 +0000
@@ -5,6 +5,10 @@
 )
 from charmhelpers.core.host import mkdir
 
+import six
+if six.PY3:
+    raise ImportError('bzrlib does not support Python3')
+
 try:
     from bzrlib.branch import Branch
 except ImportError:
@@ -42,7 +46,7 @@
         dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
                                 branch_name)
         if not os.path.exists(dest_dir):
-            mkdir(dest_dir, perms=0755)
+            mkdir(dest_dir, perms=0o755)
         try:
             self.branch(source, dest_dir)
         except OSError as e:

=== modified file 'lib/charmhelpers/fetch/giturl.py'
--- lib/charmhelpers/fetch/giturl.py	2014-10-22 04:18:26 +0000
+++ lib/charmhelpers/fetch/giturl.py	2014-11-24 15:36:56 +0000
@@ -5,6 +5,10 @@
 )
 from charmhelpers.core.host import mkdir
 
+import six
+if six.PY3:
+    raise ImportError('GitPython does not support Python 3')
+
 try:
     from git import Repo
 except ImportError:
@@ -17,7 +21,7 @@
     """Handler for git branches via generic and github URLs"""
     def can_handle(self, source):
         url_parts = self.parse_url(source)
-        #TODO (mattyw) no support for ssh git@ yet
+        # TODO (mattyw) no support for ssh git@ yet
         if url_parts.scheme not in ('http', 'https', 'git'):
             return False
         else:
@@ -36,7 +40,7 @@
         dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
                                 branch_name)
         if not os.path.exists(dest_dir):
-            mkdir(dest_dir, perms=0755)
+            mkdir(dest_dir, perms=0o755)
         try:
             self.clone(source, dest_dir, branch)
         except OSError as e:

=== modified file 'metadata.yaml'
--- metadata.yaml	2014-11-21 16:00:12 +0000
+++ metadata.yaml	2014-11-24 15:36:56 +0000
@@ -13,8 +13,8 @@
   file.  All filters and actions are given in the config files, thus
   fail2ban can be adopted to be used with a variety of files and
   firewalls.
-categories:
-  - misc
+tags:
+  - security
 subordinate: true
 requires:
   juju-info:

=== modified file 'templates/jail.local'
--- templates/jail.local	2014-10-22 04:18:26 +0000
+++ templates/jail.local	2014-11-24 15:36:56 +0000
@@ -95,7 +95,7 @@
 port     = ssh
 filter   = sshd
 logpath  = /var/log/auth.log
-maxretry = 6
+maxretry = {{maxretry}}
 
 [dropbear]
 

=== modified file 'tests/00-setup'
--- tests/00-setup	2014-11-20 21:55:44 +0000
+++ tests/00-setup	2014-11-24 15:36:56 +0000
@@ -2,4 +2,4 @@
 
 sudo add-apt-repository ppa:juju/stable -y
 sudo apt-get update -qq
-sudo apt-get install amulet python3-requests -y
+sudo apt-get install amulet -y