Merge lp:~lazypower/charms/trusty/hadoop2-devel/python_rewrite into lp:~asanjar/charms/trusty/hadoop2-devel/trunk

[Charles Butler]

Removes the class component of the code, and leaves it as a straight shot encapsulated procedural python script.

Adds argparse for future modifications of the charm, and defines the callstack using argparse.

[amir sanjar (asanjar)]

please remove a file called old_install

review: Needs Fixing

[lp:~lazypower/charms/trusty/hadoop2-devel/python_rewrite]

10. By Charles Butler on 2014-06-26

Removed the old_install archived hook

[amir sanjar (asanjar)]

review: Approve

=== added file 'Makefile'
--- Makefile	1970-01-01 00:00:00 +0000
+++ Makefile	2014-06-26 20:05:25 +0000
@@ -0,0 +1,19 @@
+#!/usr/bin/make
+PYTHON := /usr/bin/env python
+
+sync-charm-helpers: bin/charm_helpers_sync.py
+	@$(PYTHON) bin/charm_helpers_sync.py -c charm-helpers.yaml
+
+bin/charm_helpers_sync.py:
+	@mkdir -p bin	
+	@bzr cat lp:charm-helpers/tools/charm_helpers_sync/charm_helpers_sync.py > bin/charm_helpers_sync.py
+
+autotest:
+	watchmedo shell-command --patterns="*.py" --recursive --command="make test"
+
+test:
+	@nosetests tests/*.py
+
+clean:
+	@find -name *.pyc | xargs rm
+	@rm .coverage
 
=== added file 'charm-helpers.yaml'
--- charm-helpers.yaml	1970-01-01 00:00:00 +0000
+++ charm-helpers.yaml	2014-06-26 20:05:25 +0000
@@ -0,0 +1,5 @@
+branch: lp:charm-helpers
+destination: hooks/charmhelpers
+include:
+    - core
+    - fetch
 
=== modified file 'config.yaml'
--- config.yaml	2014-05-22 17:00:22 +0000
+++ config.yaml	2014-06-26 20:05:25 +0000
@@ -1,14 +1,5 @@
 options:
-  string-option:
-    type: string
-    default: "Default Value"
-    description: "A short description of the configuration option"
   boolean-option:
     type: boolean
     default: False
     description: "A short description of the configuration option"
-  int-option:
-    type: int
-    default: 9001
-    description: "A short description of the configuration option"
-
 
=== added file 'files/__init__.py'
=== added file 'files/hadoop/hdfs_format.sh'
--- files/hadoop/hdfs_format.sh	1970-01-01 00:00:00 +0000
+++ files/hadoop/hdfs_format.sh	2014-06-26 20:05:25 +0000
@@ -0,0 +1,15 @@
+#!/bin/sh
+HADOOP_VERSION="hadoop-2.2.0"
+HOME_DIR="/home/ubuntu"
+HADOOP_DIR="/home/ubuntu/hadoop"
+HADOOP_TMP_DIR=$HADOOP_DIR/tmp
+export HADOOP_INSTALL=$HADOOP_DIR/$HADOOP_VERSION
+export HADOOP_HOME=$HADOOP_INSTALL
+export HADOOP_COMMON_HOME=$HADOOP_INSTALL
+export HADOOP_HDFS_HOME=$HADOOP_INSTALL
+export HADOOP_MAPRED_HOME=$HADOOP_INSTALL
+export HADOOP_YARN_HOME=$HADOOP_INSTALL
+export PATH=$PATH:$HADOOP_INSTALL/bin:$HADOOP_INSTALL/sbin
+export YARN_HOME=$HADOOP_INSTALL
+export HADOOP_CONF_DIR=$HADOOP_INSTALL/etc/hadoop
+hdfs namenode -format
 
=== added symlink 'files/template'
=== target is u'template.py'
=== added file 'files/template.py'
--- files/template.py	1970-01-01 00:00:00 +0000
+++ files/template.py	2014-06-26 20:05:25 +0000
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+import argparse
+import logging
+import os
+import shutil
+import subprocess
+
+# Provide failout cases to install jinja2 templating and argparse.
+# This will not work in offline environments.
+try:
+    from jinja2 import Template
+except:
+    subprocess.call(['pip', 'install', 'jinja2'])
+    raise("Attempted to install missing dependencies. Try re-running")
+
+logging.basicConfig(level=logging.INFO)
+
+
+class TemplateWiz:
+
+    def __init__(self, args=None):
+        self.values = {}
+        parser = argparse.ArgumentParser()
+        parser.add_argument("-t", "--template", help="Template Path")
+        parser.add_argument("-o", "--output", help="Fullpath to output file")
+        self.paths, data = parser.parse_known_args(args)
+        if data:
+        # Process key = value keys
+            for item in data:
+                kv = item.split('=')
+                self.values[kv[0]] = kv[-1]
+
+    def read_template(self):
+        if not os.path.exists(self.paths.template):
+            raise IOError("Error loading template: %s" % self.paths.template)
+        else:
+            with open(self.paths.template, 'r') as f:
+                self.template = Template(f.read())
+
+    def write_template(self):
+        if os.path.exists(self.paths.output):
+            shutil.copy2(self.paths.output, "%s.bak" % self.paths.output)
+            logging.info("Saving {f} as {f}.bak".format(f=self.paths.output))
+        with open(self.paths.output, 'w') as f:
+            f.write(self.template.render(self.values))
+            logging.info('Rendered %s' % self.paths.output)
+
+    def run(self):
+        self.read_template()
+        self.write_template()
+
+
+if __name__ == "__main__":
+    tw = TemplateWiz()
+    tw.run()
 
=== added directory 'files/templates'
=== added file 'files/templates/defaults.j2'
--- files/templates/defaults.j2	1970-01-01 00:00:00 +0000
+++ files/templates/defaults.j2	2014-06-26 20:05:25 +0000
@@ -0,0 +1,10 @@
+export JAVA_HOME={{java_home}}
+export HADOOP_INSTALL={{hadoop_home}}
+export HADOOP_HOME=$HADOOP_INSTALL
+export HADOOP_COMMON_HOME=$HADOOP_INSTALL
+export HADOOP_HDFS_HOME=$HADOOP_INSTALL
+export HADOOP_MAPRED_HOME=$HADOOP_INSTALL
+export HADOOP_YARN_HOME=$HADOOP_INSTALL
+export PATH=$JAVA_HOME/bin:$PATH:$HADOOP_INSTALL/bin:$HADOOP_INSTALL/sbin
+export YARN_HOME=$HADOOP_INSTALL
+export HADOOP_CONF_DIR={{conf}}
 
=== removed file 'files/upstart/setenv.sh'
--- files/upstart/setenv.sh	2014-05-22 17:00:22 +0000
+++ files/upstart/setenv.sh	1970-01-01 00:00:00 +0000
@@ -1,13 +0,0 @@
-HADOOP_VERSION="hadoop-2.2.0"
-HOME_DIR="/home/ubuntu"
-HADOOP_DIR="/home/ubuntu/hadoop"
-HADOOP_TMP_DIR=$HADOOP_DIR/tmp
-export HADOOP_INSTALL=$HADOOP_DIR/$HADOOP_VERSION
-export HADOOP_HOME=$HADOOP_INSTALL
-export HADOOP_COMMON_HOME=$HADOOP_INSTALL
-export HADOOP_HDFS_HOME=$HADOOP_INSTALL
-export HADOOP_MAPRED_HOME=$HADOOP_INSTALL
-export HADOOP_YARN_HOME=$HADOOP_INSTALL
-export PATH=$JAVA_HOME/bin:$PATH:$HADOOP_INSTALL/bin:$HADOOP_INSTALL/sbin
-export YARN_HOME=$HADOOP_INSTALL
-export HADOOP_CONF_DIR=/etc/hadoop/conf.juju
 
=== added file 'hooks/__init__.py'
=== added directory 'hooks/charmhelpers'
=== added file 'hooks/charmhelpers/__init__.py'
=== added directory 'hooks/charmhelpers/core'
=== added file 'hooks/charmhelpers/core/__init__.py'
=== added file 'hooks/charmhelpers/core/fstab.py'
--- hooks/charmhelpers/core/fstab.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/fstab.py	2014-06-26 20:05:25 +0000
@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+__author__ = 'Jorge Niedbalski R. <[e-mail]>'
+
+import os
+
+
+class Fstab(file):
+    """This class extends file in order to implement a file reader/writer
+    for file `/etc/fstab`
+    """
+
+    class Entry(object):
+        """Entry class represents a non-comment line on the `/etc/fstab` file
+        """
+        def __init__(self, device, mountpoint, filesystem,
+                     options, d=0, p=0):
+            self.device = device
+            self.mountpoint = mountpoint
+            self.filesystem = filesystem
+
+            if not options:
+                options = "defaults"
+
+            self.options = options
+            self.d = d
+            self.p = p
+
+        def __eq__(self, o):
+            return str(self) == str(o)
+
+        def __str__(self):
+            return "{} {} {} {} {} {}".format(self.device,
+                                              self.mountpoint,
+                                              self.filesystem,
+                                              self.options,
+                                              self.d,
+                                              self.p)
+
+    DEFAULT_PATH = os.path.join(os.path.sep, 'etc', 'fstab')
+
+    def __init__(self, path=None):
+        if path:
+            self._path = path
+        else:
+            self._path = self.DEFAULT_PATH
+        file.__init__(self, self._path, 'r+')
+
+    def _hydrate_entry(self, line):
+        return Fstab.Entry(*filter(
+            lambda x: x not in ('', None),
+            line.strip("\n").split(" ")))
+
+    @property
+    def entries(self):
+        self.seek(0)
+        for line in self.readlines():
+            try:
+                if not line.startswith("#"):
+                    yield self._hydrate_entry(line)
+            except ValueError:
+                pass
+
+    def get_entry_by_attr(self, attr, value):
+        for entry in self.entries:
+            e_attr = getattr(entry, attr)
+            if e_attr == value:
+                return entry
+        return None
+
+    def add_entry(self, entry):
+        if self.get_entry_by_attr('device', entry.device):
+            return False
+
+        self.write(str(entry) + '\n')
+        self.truncate()
+        return entry
+
+    def remove_entry(self, entry):
+        self.seek(0)
+
+        lines = self.readlines()
+
+        found = False
+        for index, line in enumerate(lines):
+            if not line.startswith("#"):
+                if self._hydrate_entry(line) == entry:
+                    found = True
+                    break
+
+        if not found:
+            return False
+
+        lines.remove(line)
+
+        self.seek(0)
+        self.write(''.join(lines))
+        self.truncate()
+        return True
+
+    @classmethod
+    def remove_by_mountpoint(cls, mountpoint, path=None):
+        fstab = cls(path=path)
+        entry = fstab.get_entry_by_attr('mountpoint', mountpoint)
+        if entry:
+            return fstab.remove_entry(entry)
+        return False
+
+    @classmethod
+    def add(cls, device, mountpoint, filesystem, options=None, path=None):
+        return cls(path=path).add_entry(Fstab.Entry(device,
+                                                    mountpoint, filesystem,
+                                                    options=options))
 
=== added file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/hookenv.py	2014-06-26 20:05:25 +0000
@@ -0,0 +1,498 @@
+"Interactions with the Juju environment"
+# Copyright 2013 Canonical Ltd.
+#
+# Authors:
+#  Charm Helpers Developers <[e-mail]>
+
+import os
+import json
+import yaml
+import subprocess
+import sys
+import UserDict
+from subprocess import CalledProcessError
+
+CRITICAL = "CRITICAL"
+ERROR = "ERROR"
+WARNING = "WARNING"
+INFO = "INFO"
+DEBUG = "DEBUG"
+MARKER = object()
+
+cache = {}
+
+
+def cached(func):
+    """Cache return values for multiple executions of func + args
+
+    For example:
+
+        @cached
+        def unit_get(attribute):
+            pass
+
+        unit_get('test')
+
+    will cache the result of unit_get + 'test' for future calls.
+    """
+    def wrapper(*args, **kwargs):
+        global cache
+        key = str((func, args, kwargs))
+        try:
+            return cache[key]
+        except KeyError:
+            res = func(*args, **kwargs)
+            cache[key] = res
+            return res
+    return wrapper
+
+
+def flush(key):
+    """Flushes any entries from function cache where the
+    key is found in the function+args """
+    flush_list = []
+    for item in cache:
+        if key in item:
+            flush_list.append(item)
+    for item in flush_list:
+        del cache[item]
+
+
+def log(message, level=None):
+    """Write a message to the juju log"""
+    command = ['juju-log']
+    if level:
+        command += ['-l', level]
+    command += [message]
+    subprocess.call(command)
+
+
+class Serializable(UserDict.IterableUserDict):
+    """Wrapper, an object that can be serialized to yaml or json"""
+
+    def __init__(self, obj):
+        # wrap the object
+        UserDict.IterableUserDict.__init__(self)
+        self.data = obj
+
+    def __getattr__(self, attr):
+        # See if this object has attribute.
+        if attr in ("json", "yaml", "data"):
+            return self.__dict__[attr]
+        # Check for attribute in wrapped object.
+        got = getattr(self.data, attr, MARKER)
+        if got is not MARKER:
+            return got
+        # Proxy to the wrapped object via dict interface.
+        try:
+            return self.data[attr]
+        except KeyError:
+            raise AttributeError(attr)
+
+    def __getstate__(self):
+        # Pickle as a standard dictionary.
+        return self.data
+
+    def __setstate__(self, state):
+        # Unpickle into our wrapper.
+        self.data = state
+
+    def json(self):
+        """Serialize the object to json"""
+        return json.dumps(self.data)
+
+    def yaml(self):
+        """Serialize the object to yaml"""
+        return yaml.dump(self.data)
+
+
+def execution_environment():
+    """A convenient bundling of the current execution context"""
+    context = {}
+    context['conf'] = config()
+    if relation_id():
+        context['reltype'] = relation_type()
+        context['relid'] = relation_id()
+        context['rel'] = relation_get()
+    context['unit'] = local_unit()
+    context['rels'] = relations()
+    context['env'] = os.environ
+    return context
+
+
+def in_relation_hook():
+    """Determine whether we're running in a relation hook"""
+    return 'JUJU_RELATION' in os.environ
+
+
+def relation_type():
+    """The scope for the current relation hook"""
+    return os.environ.get('JUJU_RELATION', None)
+
+
+def relation_id():
+    """The relation ID for the current relation hook"""
+    return os.environ.get('JUJU_RELATION_ID', None)
+
+
+def local_unit():
+    """Local unit ID"""
+    return os.environ['JUJU_UNIT_NAME']
+
+
+def remote_unit():
+    """The remote unit for the current relation hook"""
+    return os.environ['JUJU_REMOTE_UNIT']
+
+
+def service_name():
+    """The name service group this unit belongs to"""
+    return local_unit().split('/')[0]
+
+
+def hook_name():
+    """The name of the currently executing hook"""
+    return os.path.basename(sys.argv[0])
+
+
+class Config(dict):
+    """A Juju charm config dictionary that can write itself to
+    disk (as json) and track which values have changed since
+    the previous hook invocation.
+
+    Do not instantiate this object directly - instead call
+    ``hookenv.config()``
+
+    Example usage::
+
+        >>> # inside a hook
+        >>> from charmhelpers.core import hookenv
+        >>> config = hookenv.config()
+        >>> config['foo']
+        'bar'
+        >>> config['mykey'] = 'myval'
+        >>> config.save()
+
+
+        >>> # user runs `juju set mycharm foo=baz`
+        >>> # now we're inside subsequent config-changed hook
+        >>> config = hookenv.config()
+        >>> config['foo']
+        'baz'
+        >>> # test to see if this val has changed since last hook
+        >>> config.changed('foo')
+        True
+        >>> # what was the previous value?
+        >>> config.previous('foo')
+        'bar'
+        >>> # keys/values that we add are preserved across hooks
+        >>> config['mykey']
+        'myval'
+        >>> # don't forget to save at the end of hook!
+        >>> config.save()
+
+    """
+    CONFIG_FILE_NAME = '.juju-persistent-config'
+
+    def __init__(self, *args, **kw):
+        super(Config, self).__init__(*args, **kw)
+        self._prev_dict = None
+        self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
+        if os.path.exists(self.path):
+            self.load_previous()
+
+    def load_previous(self, path=None):
+        """Load previous copy of config from disk so that current values
+        can be compared to previous values.
+
+        :param path:
+
+            File path from which to load the previous config. If `None`,
+            config is loaded from the default location. If `path` is
+            specified, subsequent `save()` calls will write to the same
+            path.
+
+        """
+        self.path = path or self.path
+        with open(self.path) as f:
+            self._prev_dict = json.load(f)
+
+    def changed(self, key):
+        """Return true if the value for this key has changed since
+        the last save.
+
+        """
+        if self._prev_dict is None:
+            return True
+        return self.previous(key) != self.get(key)
+
+    def previous(self, key):
+        """Return previous value for this key, or None if there
+        is no "previous" value.
+
+        """
+        if self._prev_dict:
+            return self._prev_dict.get(key)
+        return None
+
+    def save(self):
+        """Save this config to disk.
+
+        Preserves items in _prev_dict that do not exist in self.
+
+        """
+        if self._prev_dict:
+            for k, v in self._prev_dict.iteritems():
+                if k not in self:
+                    self[k] = v
+        with open(self.path, 'w') as f:
+            json.dump(self, f)
+
+
+@cached
+def config(scope=None):
+    """Juju charm configuration"""
+    config_cmd_line = ['config-get']
+    if scope is not None:
+        config_cmd_line.append(scope)
+    config_cmd_line.append('--format=json')
+    try:
+        config_data = json.loads(subprocess.check_output(config_cmd_line))
+        if scope is not None:
+            return config_data
+        return Config(config_data)
+    except ValueError:
+        return None
+
+
+@cached
+def relation_get(attribute=None, unit=None, rid=None):
+    """Get relation information"""
+    _args = ['relation-get', '--format=json']
+    if rid:
+        _args.append('-r')
+        _args.append(rid)
+    _args.append(attribute or '-')
+    if unit:
+        _args.append(unit)
+    try:
+        return json.loads(subprocess.check_output(_args))
+    except ValueError:
+        return None
+    except CalledProcessError, e:
+        if e.returncode == 2:
+            return None
+        raise
+
+
+def relation_set(relation_id=None, relation_settings={}, **kwargs):
+    """Set relation information for the current unit"""
+    relation_cmd_line = ['relation-set']
+    if relation_id is not None:
+        relation_cmd_line.extend(('-r', relation_id))
+    for k, v in (relation_settings.items() + kwargs.items()):
+        if v is None:
+            relation_cmd_line.append('{}='.format(k))
+        else:
+            relation_cmd_line.append('{}={}'.format(k, v))
+    subprocess.check_call(relation_cmd_line)
+    # Flush cache of any relation-gets for local unit
+    flush(local_unit())
+
+
+@cached
+def relation_ids(reltype=None):
+    """A list of relation_ids"""
+    reltype = reltype or relation_type()
+    relid_cmd_line = ['relation-ids', '--format=json']
+    if reltype is not None:
+        relid_cmd_line.append(reltype)
+        return json.loads(subprocess.check_output(relid_cmd_line)) or []
+    return []
+
+
+@cached
+def related_units(relid=None):
+    """A list of related units"""
+    relid = relid or relation_id()
+    units_cmd_line = ['relation-list', '--format=json']
+    if relid is not None:
+        units_cmd_line.extend(('-r', relid))
+    return json.loads(subprocess.check_output(units_cmd_line)) or []
+
+
+@cached
+def relation_for_unit(unit=None, rid=None):
+    """Get the json represenation of a unit's relation"""
+    unit = unit or remote_unit()
+    relation = relation_get(unit=unit, rid=rid)
+    for key in relation:
+        if key.endswith('-list'):
+            relation[key] = relation[key].split()
+    relation['__unit__'] = unit
+    return relation
+
+
+@cached
+def relations_for_id(relid=None):
+    """Get relations of a specific relation ID"""
+    relation_data = []
+    relid = relid or relation_ids()
+    for unit in related_units(relid):
+        unit_data = relation_for_unit(unit, relid)
+        unit_data['__relid__'] = relid
+        relation_data.append(unit_data)
+    return relation_data
+
+
+@cached
+def relations_of_type(reltype=None):
+    """Get relations of a specific type"""
+    relation_data = []
+    reltype = reltype or relation_type()
+    for relid in relation_ids(reltype):
+        for relation in relations_for_id(relid):
+            relation['__relid__'] = relid
+            relation_data.append(relation)
+    return relation_data
+
+
+@cached
+def relation_types():
+    """Get a list of relation types supported by this charm"""
+    charmdir = os.environ.get('CHARM_DIR', '')
+    mdf = open(os.path.join(charmdir, 'metadata.yaml'))
+    md = yaml.safe_load(mdf)
+    rel_types = []
+    for key in ('provides', 'requires', 'peers'):
+        section = md.get(key)
+        if section:
+            rel_types.extend(section.keys())
+    mdf.close()
+    return rel_types
+
+
+@cached
+def relations():
+    """Get a nested dictionary of relation data for all related units"""
+    rels = {}
+    for reltype in relation_types():
+        relids = {}
+        for relid in relation_ids(reltype):
+            units = {local_unit(): relation_get(unit=local_unit(), rid=relid)}
+            for unit in related_units(relid):
+                reldata = relation_get(unit=unit, rid=relid)
+                units[unit] = reldata
+            relids[relid] = units
+        rels[reltype] = relids
+    return rels
+
+
+@cached
+def is_relation_made(relation, keys='private-address'):
+    '''
+    Determine whether a relation is established by checking for
+    presence of key(s).  If a list of keys is provided, they
+    must all be present for the relation to be identified as made
+    '''
+    if isinstance(keys, str):
+        keys = [keys]
+    for r_id in relation_ids(relation):
+        for unit in related_units(r_id):
+            context = {}
+            for k in keys:
+                context[k] = relation_get(k, rid=r_id,
+                                          unit=unit)
+            if None not in context.values():
+                return True
+    return False
+
+
+def open_port(port, protocol="TCP"):
+    """Open a service network port"""
+    _args = ['open-port']
+    _args.append('{}/{}'.format(port, protocol))
+    subprocess.check_call(_args)
+
+
+def close_port(port, protocol="TCP"):
+    """Close a service network port"""
+    _args = ['close-port']
+    _args.append('{}/{}'.format(port, protocol))
+    subprocess.check_call(_args)
+
+
+@cached
+def unit_get(attribute):
+    """Get the unit ID for the remote unit"""
+    _args = ['unit-get', '--format=json', attribute]
+    try:
+        return json.loads(subprocess.check_output(_args))
+    except ValueError:
+        return None
+
+
+def unit_private_ip():
+    """Get this unit's private IP address"""
+    return unit_get('private-address')
+
+
+class UnregisteredHookError(Exception):
+    """Raised when an undefined hook is called"""
+    pass
+
+
+class Hooks(object):
+    """A convenient handler for hook functions.
+
+    Example:
+        hooks = Hooks()
+
+        # register a hook, taking its name from the function name
+        @hooks.hook()
+        def install():
+            ...
+
+        # register a hook, providing a custom hook name
+        @hooks.hook("config-changed")
+        def config_changed():
+            ...
+
+        if __name__ == "__main__":
+            # execute a hook based on the name the program is called by
+            hooks.execute(sys.argv)
+    """
+
+    def __init__(self):
+        super(Hooks, self).__init__()
+        self._hooks = {}
+
+    def register(self, name, function):
+        """Register a hook"""
+        self._hooks[name] = function
+
+    def execute(self, args):
+        """Execute a registered hook based on args[0]"""
+        hook_name = os.path.basename(args[0])
+        if hook_name in self._hooks:
+            self._hooks[hook_name]()
+        else:
+            raise UnregisteredHookError(hook_name)
+
+    def hook(self, *hook_names):
+        """Decorator, registering them as hooks"""
+        def wrapper(decorated):
+            for hook_name in hook_names:
+                self.register(hook_name, decorated)
+            else:
+                self.register(decorated.__name__, decorated)
+                if '_' in decorated.__name__:
+                    self.register(
+                        decorated.__name__.replace('_', '-'), decorated)
+            return decorated
+        return wrapper
+
+
+def charm_dir():
+    """Return the root directory of the current charm"""
+    return os.environ.get('CHARM_DIR')
 
=== added file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/host.py	2014-06-26 20:05:25 +0000
@@ -0,0 +1,325 @@
+"""Tools for working with the host system"""
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+#  Nick Moffitt <[e-mail]>
+#  Matthew Wedgwood <[e-mail]>
+
+import os
+import pwd
+import grp
+import random
+import string
+import subprocess
+import hashlib
+import apt_pkg
+
+from collections import OrderedDict
+
+from hookenv import log
+from fstab import Fstab
+
+
+def service_start(service_name):
+    """Start a system service"""
+    return service('start', service_name)
+
+
+def service_stop(service_name):
+    """Stop a system service"""
+    return service('stop', service_name)
+
+
+def service_restart(service_name):
+    """Restart a system service"""
+    return service('restart', service_name)
+
+
+def service_reload(service_name, restart_on_failure=False):
+    """Reload a system service, optionally falling back to restart if
+    reload fails"""
+    service_result = service('reload', service_name)
+    if not service_result and restart_on_failure:
+        service_result = service('restart', service_name)
+    return service_result
+
+
+def service(action, service_name):
+    """Control a system service"""
+    cmd = ['service', service_name, action]
+    return subprocess.call(cmd) == 0
+
+
+def service_running(service):
+    """Determine whether a system service is running"""
+    try:
+        output = subprocess.check_output(['service', service, 'status'])
+    except subprocess.CalledProcessError:
+        return False
+    else:
+        if ("start/running" in output or "is running" in output):
+            return True
+        else:
+            return False
+
+
+def adduser(username, password=None, shell='/bin/bash', system_user=False):
+    """Add a user to the system"""
+    try:
+        user_info = pwd.getpwnam(username)
+        log('user {0} already exists!'.format(username))
+    except KeyError:
+        log('creating user {0}'.format(username))
+        cmd = ['useradd']
+        if system_user or password is None:
+            cmd.append('--system')
+        else:
+            cmd.extend([
+                '--create-home',
+                '--shell', shell,
+                '--password', password,
+            ])
+        cmd.append(username)
+        subprocess.check_call(cmd)
+        user_info = pwd.getpwnam(username)
+    return user_info
+
+
+def add_user_to_group(username, group):
+    """Add a user to a group"""
+    cmd = [
+        'gpasswd', '-a',
+        username,
+        group
+    ]
+    log("Adding user {} to group {}".format(username, group))
+    subprocess.check_call(cmd)
+
+
+def rsync(from_path, to_path, flags='-r', options=None):
+    """Replicate the contents of a path"""
+    options = options or ['--delete', '--executability']
+    cmd = ['/usr/bin/rsync', flags]
+    cmd.extend(options)
+    cmd.append(from_path)
+    cmd.append(to_path)
+    log(" ".join(cmd))
+    return subprocess.check_output(cmd).strip()
+
+
+def symlink(source, destination):
+    """Create a symbolic link"""
+    log("Symlinking {} as {}".format(source, destination))
+    cmd = [
+        'ln',
+        '-sf',
+        source,
+        destination,
+    ]
+    subprocess.check_call(cmd)
+
+
+def mkdir(path, owner='root', group='root', perms=0555, force=False):
+    """Create a directory"""
+    log("Making dir {} {}:{} {:o}".format(path, owner, group,
+                                          perms))
+    uid = pwd.getpwnam(owner).pw_uid
+    gid = grp.getgrnam(group).gr_gid
+    realpath = os.path.abspath(path)
+    if os.path.exists(realpath):
+        if force and not os.path.isdir(realpath):
+            log("Removing non-directory file {} prior to mkdir()".format(path))
+            os.unlink(realpath)
+    else:
+        os.makedirs(realpath, perms)
+    os.chown(realpath, uid, gid)
+
+
+def write_file(path, content, owner='root', group='root', perms=0444):
+    """Create or overwrite a file with the contents of a string"""
+    log("Writing file {} {}:{} {:o}".format(path, owner, group, perms))
+    uid = pwd.getpwnam(owner).pw_uid
+    gid = grp.getgrnam(group).gr_gid
+    with open(path, 'w') as target:
+        os.fchown(target.fileno(), uid, gid)
+        os.fchmod(target.fileno(), perms)
+        target.write(content)
+
+
+def fstab_remove(mp):
+    """Remove the given mountpoint entry from /etc/fstab
+    """
+    return Fstab.remove_by_mountpoint(mp)
+
+
+def fstab_add(dev, mp, fs, options=None):
+    """Adds the given device entry to the /etc/fstab file
+    """
+    return Fstab.add(dev, mp, fs, options=options)
+
+
+def mount(device, mountpoint, options=None, persist=False, filesystem="ext3"):
+    """Mount a filesystem at a particular mountpoint"""
+    cmd_args = ['mount']
+    if options is not None:
+        cmd_args.extend(['-o', options])
+    cmd_args.extend([device, mountpoint])
+    try:
+        subprocess.check_output(cmd_args)
+    except subprocess.CalledProcessError, e:
+        log('Error mounting {} at {}\n{}'.format(device, mountpoint, e.output))
+        return False
+
+    if persist:
+        return fstab_add(device, mountpoint, filesystem, options=options)
+    return True
+
+
+def umount(mountpoint, persist=False):
+    """Unmount a filesystem"""
+    cmd_args = ['umount', mountpoint]
+    try:
+        subprocess.check_output(cmd_args)
+    except subprocess.CalledProcessError, e:
+        log('Error unmounting {}\n{}'.format(mountpoint, e.output))
+        return False
+
+    if persist:
+        return fstab_remove(mountpoint)
+    return True
+
+
+def mounts():
+    """Get a list of all mounted volumes as [[mountpoint,device],[...]]"""
+    with open('/proc/mounts') as f:
+        # [['/mount/point','/dev/path'],[...]]
+        system_mounts = [m[1::-1] for m in [l.strip().split()
+                                            for l in f.readlines()]]
+    return system_mounts
+
+
+def file_hash(path):
+    """Generate a md5 hash of the contents of 'path' or None if not found """
+    if os.path.exists(path):
+        h = hashlib.md5()
+        with open(path, 'r') as source:
+            h.update(source.read())  # IGNORE:E1101 - it does have update
+        return h.hexdigest()
+    else:
+        return None
+
+
+def restart_on_change(restart_map, stopstart=False):
+    """Restart services based on configuration files changing
+
+    This function is used a decorator, for example
+
+        @restart_on_change({
+            '/etc/ceph/ceph.conf': [ 'cinder-api', 'cinder-volume' ]
+            })
+        def ceph_client_changed():
+            ...
+
+    In this example, the cinder-api and cinder-volume services
+    would be restarted if /etc/ceph/ceph.conf is changed by the
+    ceph_client_changed function.
+    """
+    def wrap(f):
+        def wrapped_f(*args):
+            checksums = {}
+            for path in restart_map:
+                checksums[path] = file_hash(path)
+            f(*args)
+            restarts = []
+            for path in restart_map:
+                if checksums[path] != file_hash(path):
+                    restarts += restart_map[path]
+            services_list = list(OrderedDict.fromkeys(restarts))
+            if not stopstart:
+                for service_name in services_list:
+                    service('restart', service_name)
+            else:
+                for action in ['stop', 'start']:
+                    for service_name in services_list:
+                        service(action, service_name)
+        return wrapped_f
+    return wrap
+
+
+def lsb_release():
+    """Return /etc/lsb-release in a dict"""
+    d = {}
+    with open('/etc/lsb-release', 'r') as lsb:
+        for l in lsb:
+            k, v = l.split('=')
+            d[k.strip()] = v.strip()
+    return d
+
+
+def pwgen(length=None):
+    """Generate a random pasword."""
+    if length is None:
+        length = random.choice(range(35, 45))
+    alphanumeric_chars = [
+        l for l in (string.letters + string.digits)
+        if l not in 'l0QD1vAEIOUaeiou']
+    random_chars = [
+        random.choice(alphanumeric_chars) for _ in range(length)]
+    return(''.join(random_chars))
+
+
+def list_nics(nic_type):
+    '''Return a list of nics of given type(s)'''
+    if isinstance(nic_type, basestring):
+        int_types = [nic_type]
+    else:
+        int_types = nic_type
+    interfaces = []
+    for int_type in int_types:
+        cmd = ['ip', 'addr', 'show', 'label', int_type + '*']
+        ip_output = subprocess.check_output(cmd).split('\n')
+        ip_output = (line for line in ip_output if line)
+        for line in ip_output:
+            if line.split()[1].startswith(int_type):
+                interfaces.append(line.split()[1].replace(":", ""))
+    return interfaces
+
+
+def set_nic_mtu(nic, mtu):
+    '''Set MTU on a network interface'''
+    cmd = ['ip', 'link', 'set', nic, 'mtu', mtu]
+    subprocess.check_call(cmd)
+
+
+def get_nic_mtu(nic):
+    cmd = ['ip', 'addr', 'show', nic]
+    ip_output = subprocess.check_output(cmd).split('\n')
+    mtu = ""
+    for line in ip_output:
+        words = line.split()
+        if 'mtu' in words:
+            mtu = words[words.index("mtu") + 1]
+    return mtu
+
+
+def get_nic_hwaddr(nic):
+    cmd = ['ip', '-o', '-0', 'addr', 'show', nic]
+    ip_output = subprocess.check_output(cmd)
+    hwaddr = ""
+    words = ip_output.split()
+    if 'link/ether' in words:
+        hwaddr = words[words.index('link/ether') + 1]
+    return hwaddr
+
+
+def cmp_pkgrevno(package, revno, pkgcache=None):
+    '''Compare supplied revno with the revno of the installed package
+       1 => Installed revno is greater than supplied arg
+       0 => Installed revno is the same as supplied arg
+      -1 => Installed revno is less than supplied arg
+    '''
+    if not pkgcache:
+        apt_pkg.init()
+        pkgcache = apt_pkg.Cache()
+    pkg = pkgcache[package]
+    return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
 
=== added directory 'hooks/charmhelpers/fetch'
=== added file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-06-26 20:05:25 +0000
@@ -0,0 +1,349 @@
+import importlib
+import time
+from yaml import safe_load
+from charmhelpers.core.host import (
+    lsb_release
+)
+from urlparse import (
+    urlparse,
+    urlunparse,
+)
+import subprocess
+from charmhelpers.core.hookenv import (
+    config,
+    log,
+)
+import apt_pkg
+import os
+
+
+CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
+deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
+"""
+PROPOSED_POCKET = """# Proposed
+deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
+"""
+CLOUD_ARCHIVE_POCKETS = {
+    # Folsom
+    'folsom': 'precise-updates/folsom',
+    'precise-folsom': 'precise-updates/folsom',
+    'precise-folsom/updates': 'precise-updates/folsom',
+    'precise-updates/folsom': 'precise-updates/folsom',
+    'folsom/proposed': 'precise-proposed/folsom',
+    'precise-folsom/proposed': 'precise-proposed/folsom',
+    'precise-proposed/folsom': 'precise-proposed/folsom',
+    # Grizzly
+    'grizzly': 'precise-updates/grizzly',
+    'precise-grizzly': 'precise-updates/grizzly',
+    'precise-grizzly/updates': 'precise-updates/grizzly',
+    'precise-updates/grizzly': 'precise-updates/grizzly',
+    'grizzly/proposed': 'precise-proposed/grizzly',
+    'precise-grizzly/proposed': 'precise-proposed/grizzly',
+    'precise-proposed/grizzly': 'precise-proposed/grizzly',
+    # Havana
+    'havana': 'precise-updates/havana',
+    'precise-havana': 'precise-updates/havana',
+    'precise-havana/updates': 'precise-updates/havana',
+    'precise-updates/havana': 'precise-updates/havana',
+    'havana/proposed': 'precise-proposed/havana',
+    'precise-havana/proposed': 'precise-proposed/havana',
+    'precise-proposed/havana': 'precise-proposed/havana',
+    # Icehouse
+    'icehouse': 'precise-updates/icehouse',
+    'precise-icehouse': 'precise-updates/icehouse',
+    'precise-icehouse/updates': 'precise-updates/icehouse',
+    'precise-updates/icehouse': 'precise-updates/icehouse',
+    'icehouse/proposed': 'precise-proposed/icehouse',
+    'precise-icehouse/proposed': 'precise-proposed/icehouse',
+    'precise-proposed/icehouse': 'precise-proposed/icehouse',
+    # Juno
+    'juno': 'trusty-updates/juno',
+    'trusty-juno': 'trusty-updates/juno',
+    'trusty-juno/updates': 'trusty-updates/juno',
+    'trusty-updates/juno': 'trusty-updates/juno',
+    'juno/proposed': 'trusty-proposed/juno',
+    'juno/proposed': 'trusty-proposed/juno',
+    'trusty-juno/proposed': 'trusty-proposed/juno',
+    'trusty-proposed/juno': 'trusty-proposed/juno',
+}
+
+# The order of this list is very important. Handlers should be listed in from
+# least- to most-specific URL matching.
+FETCH_HANDLERS = (
+    'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
+    'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
+)
+
+APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.
+APT_NO_LOCK_RETRY_DELAY = 10  # Wait 10 seconds between apt lock checks.
+APT_NO_LOCK_RETRY_COUNT = 30  # Retry to acquire the lock X times.
+
+
+class SourceConfigError(Exception):
+    pass
+
+
+class UnhandledSource(Exception):
+    pass
+
+
+class AptLockError(Exception):
+    pass
+
+
+class BaseFetchHandler(object):
+
+    """Base class for FetchHandler implementations in fetch plugins"""
+
+    def can_handle(self, source):
+        """Returns True if the source can be handled. Otherwise returns
+        a string explaining why it cannot"""
+        return "Wrong source type"
+
+    def install(self, source):
+        """Try to download and unpack the source. Return the path to the
+        unpacked files or raise UnhandledSource."""
+        raise UnhandledSource("Wrong source type {}".format(source))
+
+    def parse_url(self, url):
+        return urlparse(url)
+
+    def base_url(self, url):
+        """Return url without querystring or fragment"""
+        parts = list(self.parse_url(url))
+        parts[4:] = ['' for i in parts[4:]]
+        return urlunparse(parts)
+
+
+def filter_installed_packages(packages):
+    """Returns a list of packages that require installation"""
+    apt_pkg.init()
+
+    # Tell apt to build an in-memory cache to prevent race conditions (if
+    # another process is already building the cache).
+    apt_pkg.config.set("Dir::Cache::pkgcache", "")
+
+    cache = apt_pkg.Cache()
+    _pkgs = []
+    for package in packages:
+        try:
+            p = cache[package]
+            p.current_ver or _pkgs.append(package)
+        except KeyError:
+            log('Package {} has no installation candidate.'.format(package),
+                level='WARNING')
+            _pkgs.append(package)
+    return _pkgs
+
+
+def apt_install(packages, options=None, fatal=False):
+    """Install one or more packages"""
+    if options is None:
+        options = ['--option=Dpkg::Options::=--force-confold']
+
+    cmd = ['apt-get', '--assume-yes']
+    cmd.extend(options)
+    cmd.append('install')
+    if isinstance(packages, basestring):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Installing {} with options: {}".format(packages,
+                                                options))
+    _run_apt_command(cmd, fatal)
+
+
+def apt_upgrade(options=None, fatal=False, dist=False):
+    """Upgrade all packages"""
+    if options is None:
+        options = ['--option=Dpkg::Options::=--force-confold']
+
+    cmd = ['apt-get', '--assume-yes']
+    cmd.extend(options)
+    if dist:
+        cmd.append('dist-upgrade')
+    else:
+        cmd.append('upgrade')
+    log("Upgrading with options: {}".format(options))
+    _run_apt_command(cmd, fatal)
+
+
+def apt_update(fatal=False):
+    """Update local apt cache"""
+    cmd = ['apt-get', 'update']
+    _run_apt_command(cmd, fatal)
+
+
+def apt_purge(packages, fatal=False):
+    """Purge one or more packages"""
+    cmd = ['apt-get', '--assume-yes', 'purge']
+    if isinstance(packages, basestring):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Purging {}".format(packages))
+    _run_apt_command(cmd, fatal)
+
+
+def apt_hold(packages, fatal=False):
+    """Hold one or more packages"""
+    cmd = ['apt-mark', 'hold']
+    if isinstance(packages, basestring):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Holding {}".format(packages))
+
+    if fatal:
+        subprocess.check_call(cmd)
+    else:
+        subprocess.call(cmd)
+
+
+def add_source(source, key=None):
+    if source is None:
+        log('Source is not present. Skipping')
+        return
+
+    if (source.startswith('ppa:') or
+        source.startswith('http') or
+        source.startswith('deb ') or
+            source.startswith('cloud-archive:')):
+        subprocess.check_call(['add-apt-repository', '--yes', source])
+    elif source.startswith('cloud:'):
+        apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
+                    fatal=True)
+        pocket = source.split(':')[-1]
+        if pocket not in CLOUD_ARCHIVE_POCKETS:
+            raise SourceConfigError(
+                'Unsupported cloud: source option %s' %
+                pocket)
+        actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
+        with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
+            apt.write(CLOUD_ARCHIVE.format(actual_pocket))
+    elif source == 'proposed':
+        release = lsb_release()['DISTRIB_CODENAME']
+        with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
+            apt.write(PROPOSED_POCKET.format(release))
+    if key:
+        subprocess.check_call(['apt-key', 'adv', '--keyserver',
+                               'hkp://keyserver.ubuntu.com:80', '--recv',
+                               key])
+
+
+def configure_sources(update=False,
+                      sources_var='install_sources',
+                      keys_var='install_keys'):
+    """
+    Configure multiple sources from charm configuration
+
+    Example config:
+        install_sources:
+          - "ppa:foo"
+          - "http://example.com/repo precise main"
+        install_keys:
+          - null
+          - "a1b2c3d4"
+
+    Note that 'null' (a.k.a. None) should not be quoted.
+    """
+    sources = safe_load(config(sources_var))
+    keys = config(keys_var)
+    if keys is not None:
+        keys = safe_load(keys)
+    if isinstance(sources, basestring) and (
+            keys is None or isinstance(keys, basestring)):
+        add_source(sources, keys)
+    else:
+        if not len(sources) == len(keys):
+            msg = 'Install sources and keys lists are different lengths'
+            raise SourceConfigError(msg)
+        for src_num in range(len(sources)):
+            add_source(sources[src_num], keys[src_num])
+    if update:
+        apt_update(fatal=True)
+
+
+def install_remote(source):
+    """
+    Install a file tree from a remote source
+
+    The specified source should be a url of the form:
+        scheme://[host]/path[#[option=value][&...]]
+
+    Schemes supported are based on this modules submodules
+    Options supported are submodule-specific"""
+    # We ONLY check for True here because can_handle may return a string
+    # explaining why it can't handle a given source.
+    handlers = [h for h in plugins() if h.can_handle(source) is True]
+    installed_to = None
+    for handler in handlers:
+        try:
+            installed_to = handler.install(source)
+        except UnhandledSource:
+            pass
+    if not installed_to:
+        raise UnhandledSource("No handler found for source {}".format(source))
+    return installed_to
+
+
+def install_from_config(config_var_name):
+    charm_config = config()
+    source = charm_config[config_var_name]
+    return install_remote(source)
+
+
+def plugins(fetch_handlers=None):
+    if not fetch_handlers:
+        fetch_handlers = FETCH_HANDLERS
+    plugin_list = []
+    for handler_name in fetch_handlers:
+        package, classname = handler_name.rsplit('.', 1)
+        try:
+            handler_class = getattr(
+                importlib.import_module(package),
+                classname)
+            plugin_list.append(handler_class())
+        except (ImportError, AttributeError):
+            # Skip missing plugins so that they can be ommitted from
+            # installation if desired
+            log("FetchHandler {} not found, skipping plugin".format(
+                handler_name))
+    return plugin_list
+
+
+def _run_apt_command(cmd, fatal=False):
+    """
+    Run an APT command, checking output and retrying if the fatal flag is set
+    to True.
+
+    :param: cmd: str: The apt command to run.
+    :param: fatal: bool: Whether the command's output should be checked and
+        retried.
+    """
+    env = os.environ.copy()
+
+    if 'DEBIAN_FRONTEND' not in env:
+        env['DEBIAN_FRONTEND'] = 'noninteractive'
+
+    if fatal:
+        retry_count = 0
+        result = None
+
+        # If the command is considered "fatal", we need to retry if the apt
+        # lock was not acquired.
+
+        while result is None or result == APT_NO_LOCK:
+            try:
+                result = subprocess.check_call(cmd, env=env)
+            except subprocess.CalledProcessError, e:
+                retry_count = retry_count + 1
+                if retry_count > APT_NO_LOCK_RETRY_COUNT:
+                    raise
+                result = e.returncode
+                log("Couldn't acquire DPKG lock. Will retry in {} seconds."
+                    "".format(APT_NO_LOCK_RETRY_DELAY))
+                time.sleep(APT_NO_LOCK_RETRY_DELAY)
+
+    else:
+        subprocess.call(cmd, env=env)
 
=== added file 'hooks/charmhelpers/fetch/archiveurl.py'
--- hooks/charmhelpers/fetch/archiveurl.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/fetch/archiveurl.py	2014-06-26 20:05:25 +0000
@@ -0,0 +1,63 @@
+import os
+import urllib2
+import urlparse
+
+from charmhelpers.fetch import (
+    BaseFetchHandler,
+    UnhandledSource
+)
+from charmhelpers.payload.archive import (
+    get_archive_handler,
+    extract,
+)
+from charmhelpers.core.host import mkdir
+
+
+class ArchiveUrlFetchHandler(BaseFetchHandler):
+    """Handler for archives via generic URLs"""
+    def can_handle(self, source):
+        url_parts = self.parse_url(source)
+        if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
+            return "Wrong source type"
+        if get_archive_handler(self.base_url(source)):
+            return True
+        return False
+
+    def download(self, source, dest):
+        # propogate all exceptions
+        # URLError, OSError, etc
+        proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
+        if proto in ('http', 'https'):
+            auth, barehost = urllib2.splituser(netloc)
+            if auth is not None:
+                source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
+                username, password = urllib2.splitpasswd(auth)
+                passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+                # Realm is set to None in add_password to force the username and password
+                # to be used whatever the realm
+                passman.add_password(None, source, username, password)
+                authhandler = urllib2.HTTPBasicAuthHandler(passman)
+                opener = urllib2.build_opener(authhandler)
+                urllib2.install_opener(opener)
+        response = urllib2.urlopen(source)
+        try:
+            with open(dest, 'w') as dest_file:
+                dest_file.write(response.read())
+        except Exception as e:
+            if os.path.isfile(dest):
+                os.unlink(dest)
+            raise e
+
+    def install(self, source):
+        url_parts = self.parse_url(source)
+        dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
+        if not os.path.exists(dest_dir):
+            mkdir(dest_dir, perms=0755)
+        dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
+        try:
+            self.download(source, dld_file)
+        except urllib2.URLError as e:
+            raise UnhandledSource(e.reason)
+        except OSError as e:
+            raise UnhandledSource(e.strerror)
+        return extract(dld_file)
 
=== added file 'hooks/charmhelpers/fetch/bzrurl.py'
--- hooks/charmhelpers/fetch/bzrurl.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/fetch/bzrurl.py	2014-06-26 20:05:25 +0000
@@ -0,0 +1,50 @@
+import os
+from charmhelpers.fetch import (
+    BaseFetchHandler,
+    UnhandledSource
+)
+from charmhelpers.core.host import mkdir
+
+try:
+    from bzrlib.branch import Branch
+except ImportError:
+    from charmhelpers.fetch import apt_install
+    apt_install("python-bzrlib")
+    from bzrlib.branch import Branch
+
+
+class BzrUrlFetchHandler(BaseFetchHandler):
+    """Handler for bazaar branches via generic and lp URLs"""
+    def can_handle(self, source):
+        url_parts = self.parse_url(source)
+        if url_parts.scheme not in ('bzr+ssh', 'lp'):
+            return False
+        else:
+            return True
+
+    def branch(self, source, dest):
+        url_parts = self.parse_url(source)
+        # If we use lp:branchname scheme we need to load plugins
+        if not self.can_handle(source):
+            raise UnhandledSource("Cannot handle {}".format(source))
+        if url_parts.scheme == "lp":
+            from bzrlib.plugin import load_plugins
+            load_plugins()
+        try:
+            remote_branch = Branch.open(source)
+            remote_branch.bzrdir.sprout(dest).open_branch()
+        except Exception as e:
+            raise e
+
+    def install(self, source):
+        url_parts = self.parse_url(source)
+        branch_name = url_parts.path.strip("/").split("/")[-1]
+        dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched",
+                                branch_name)
+        if not os.path.exists(dest_dir):
+            mkdir(dest_dir, perms=0755)
+        try:
+            self.branch(source, dest_dir)
+        except OSError as e:
+            raise UnhandledSource(e.strerror)
+        return dest_dir
 
=== added file 'hooks/charmhelpers/setup.py'
--- hooks/charmhelpers/setup.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/setup.py	2014-06-26 20:05:25 +0000
@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+
+from distutils.core import setup
+
+setup(name='charmhelpers',
+     version='1.0',
+     description='this is dumb',
+     author='nobody',
+     author_email='dummy@amulet',
+     url='http://google.com',
+     packages=[],
+)
 
=== added file 'hooks/common.py'
--- hooks/common.py	1970-01-01 00:00:00 +0000
+++ hooks/common.py	2014-06-26 20:05:25 +0000
@@ -0,0 +1,231 @@
+#!/usr/bin/env python
+
+import grp
+import os
+import pwd
+import subprocess
+import sys
+import tarfile
+import argparse
+import logging
+
+from shutil import rmtree, copyfile
+from charmhelpers.core.hookenv import log
+# Required for unit tests... :(
+try:
+    from charmhelpers.fetch import apt_install, apt_update
+except:
+    pass
+
+FORMAT = '%(asctime)-15s %(user)-8s %(message)s'
+logging.basicConfig(level=logging.INFO)
+
+
+# ##########################################
+# Globals
+# ##########################################
+JAVA_VERSION = "7"
+JAVA_HOME = None
+HADOOP_VERSION = "hadoop-2.2.0"
+PLATFORM_ARCH = "amd64"
+HOME_DIR = os.path.join(os.path.sep, "home", "ubuntu")
+HADOOP_DIR = os.path.join(HOME_DIR, "hadoop")
+HADOOP_TMP_DIR = os.path.join(HADOOP_DIR, "tmp")
+HADOOP_INSTALL = os.path.join(HADOOP_DIR, HADOOP_VERSION)
+HADOOP_CONF_DIR = os.path.join(HADOOP_INSTALL, "etc/hadoop")
+
+
+# ##########################################
+# Utility Methods
+# ##########################################
+
+def install_dev_packages():
+    packages = ['maven',
+                'build-essential',
+                'autoconf',
+                'automake',
+                'libtool',
+                'cmake',
+                'zlib1g-dev',
+                'pkg-config',
+                'libssl-dev',
+                'snappy',
+                'libsnappy-dev',
+                'openjdk-7-jdk',
+                'python-pip',
+                'python-jinja2',
+                ]
+    apt_update()
+    apt_install(packages)
+
+
+def ssh_wizardry():
+    # Set NonStrict Hostkey Checking to .ssh config
+    # this both confuses and angers me!
+    log("Setting NonStrict HostKey Checking for SSH", "INFO")
+
+    nonstrict = "Host *\n\tStrictHostKeyChecking no"
+    with open("{dir}/.ssh/config".format(dir=HOME_DIR), 'w+') as f:
+        f.write(nonstrict)
+
+    keyfile = os.path.join(os.path.sep, 'home', 'ubuntu', '.ssh', 'id_rsa')
+    cmd = 'yes | ssh-keygen -t rsa -N "" -f {d}'.format(d=keyfile)
+    ps = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
+                          stderr=subprocess.STDOUT)
+    output = ps.communicate()[0]
+    log("Output of ssh keygen: {o}".format(o=output), "INFO")
+    with open("{dir}/.ssh/id_rsa.pub".format(dir=HOME_DIR), 'r') as f:
+        hostkey = f.read()
+
+    auth_keys = "{dir}/.ssh/authorized_keys".format(dir=HOME_DIR)
+    with open(auth_keys, 'a') as f:
+        f.write(hostkey)
+    subprocess.call(['chown', '-R', 'ubuntu.ubuntu',
+                     "{dir}/.ssh".format(dir=HOME_DIR)])
+
+
+def set_java_home():
+    jbin = 'java-{v}-openjdk-{a}'.format(v=JAVA_VERSION, a=PLATFORM_ARCH)
+    jpath = subprocess.check_output(['find', '/usr/', '-name', jbin])
+    if not jpath:
+        raise ValueError("Unable to locate Java Path for %s" % jbin)
+    script = '{d}/etc/hadoop/hadoop-env.sh'.format(d=HADOOP_INSTALL)
+
+    with open(script) as f:
+        contents = f.readlines()
+        for l in range(0, len(contents)):
+            if contents[l].startswith('export JAVA_HOME='):
+                contents[l] = "export JAVA_HOME='{jp}'".format(
+                    jp=jpath.strip())
+
+    with open(script, 'w') as f:
+        f.writelines(contents)
+    JAVA_HOME = jpath.strip()
+    log('Set JAVA_HOME in {p} to {v}'.format(p=script, v=jpath), 'INFO')
+    JAVA_HOME = jpath.strip()
+    log("JAVA_HOME set to: {j}".format(j=JAVA_HOME), "INFO")
+
+
+# TODO: this should be configureable at some point and reach out
+# over the wire to install w/ Sha1 Sig Validation
+def extract_hadoop():
+    pa = os.path.join("{d}", "files", "archives", "{hdpver}.tar.gz")
+    tarballpath = os.path.join(pa.format(
+                               d=os.environ['CHARM_DIR'],
+                               hdpver=HADOOP_VERSION))
+
+    if tarfile.is_tarfile(tarballpath):
+        tball = tarfile.open(tarballpath)
+        tball.extractall(HADOOP_DIR)
+    else:
+        log("Unable to extract Hadoop Tarball {tb}".format(tb=tarballpath),
+            "Warning")
+
+
+def add_shim():
+    if os.path.exists("{d}/.hadoop_shim".format(d=HOME_DIR)):
+        log("Found .hadoop_shim, bailing", "INFO")
+        log("To re-execute the template, re-run this hook after removing"
+            "{d}/.hadoop_shim".format(d=HOME_DIR), 'INFO')
+        return
+
+    # TODO: Refactor this nasty shell callout with some implicit python
+    # by requiring the templatewiz class, and passing args.
+    templ_bin_path = os.path.join('files', 'template')
+    template_path = os.path.join(os.environ['CHARM_DIR'], 'files',
+                                 'templates', 'defaults.j2')
+    user_path = os.path.join(os.path.sep, 'home', 'ubuntu', '.hadoop_shim')
+    subprocess.call(['python', templ_bin_path, '-t', template_path, '-o',
+                     user_path, 'java_home={j}'.format(j=JAVA_HOME),
+                     'hadoop_home={h}'.format(h=HADOOP_INSTALL),
+                     'conf={c}'.format(c=HADOOP_CONF_DIR)])
+
+    with open('{d}/.profile'.format(d=HOME_DIR), 'a') as f:
+        f.write('[ -f "$HOME/.hadoop_shim" ] && . "$HOME/.hadoop_shim"')
+
+
+def template(src, dst, own='root', grp='root', mod=0644):
+    copyfile(src, dst)
+    os.chown(dst, own, grp)
+    os.chmod(dst, mod)
+
+
+def copy_templates():
+    # Get UID's
+    hadoop_uid = pwd.getpwnam('ubuntu').pw_uid
+    hadoop_gid = grp.getgrnam('ubuntu').gr_gid
+
+    srcp = os.path.join('files', 'hadoop')
+    dstp = os.path.join(HADOOP_INSTALL, 'etc', 'hadoop')
+    template(os.path.join(srcp, 'core-site.xml'),
+             os.path.join(dstp, 'core-site.xml'), hadoop_uid,
+             hadoop_gid)
+    template(os.path.join(srcp, 'hdfs-site.xml'),
+             os.path.join(dstp, 'hdfs-site.xml'), hadoop_uid,
+             hadoop_gid)
+    template(os.path.join(srcp, 'yarn-site.xml'),
+             os.path.join(dstp, 'yarn-site.xml'), hadoop_uid,
+             hadoop_gid)
+    template(os.path.join(srcp, 'mapreduce-site.xml'),
+             os.path.join(dstp, 'mapreduce-site.xml'), hadoop_uid,
+             hadoop_gid)
+    template(os.path.join(srcp, 'hdfs_format.sh'),
+             os.path.join(HOME_DIR, 'hdfs_format.sh'), hadoop_uid,
+             hadoop_gid, 0777)
+
+
+# ##########################################
+# Charm Hooks
+# ##########################################
+
+def install():
+    # Remove the hadoop directory if it exists
+    if os.path.exists(HADOOP_DIR):
+        rmtree(HADOOP_DIR)
+    ssh_wizardry()
+
+    os.makedirs(HADOOP_DIR)
+    os.makedirs(HADOOP_TMP_DIR)
+    install_dev_packages()
+    extract_hadoop()
+    set_java_home()
+
+    # Create HDFS Dir Tree
+    nndir = os.path.join("{d}".format(d=HADOOP_DIR), "data",
+                         "hdfs", "namenode")
+    if not os.path.exists(nndir):
+        os.makedirs(nndir)
+    dndir = "{d}/data/hdfs/datanode".format(d=HADOOP_DIR)
+    if not os.path.exists(nndir):
+        os.makedirs(dndir)
+
+    copy_templates()
+    add_shim()
+
+    # Call the HDFS Format script post installation
+    cmd = "{d}{s}hdfs_format.sh".format(d=HOME_DIR, s=os.path.sep)
+    log(cmd, "INFO")
+    subprocess.call(cmd)
+
+    # Ensure the Ubuntu user owns everything in the hadoop tree
+    # Stick a fork in it, we're done.
+    subprocess.call(['chown', '-R', 'ubuntu.ubuntu', HADOOP_DIR])
+    log("Install complete.", "INFO")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-H', '--hook_name', dest='hook_name',
+                        help='hook to call')
+
+    args = parser.parse_args()
+
+    if args.hook_name is not None:
+        hook_name = args.hook_name
+    else:
+        hook_name = os.path.basename(sys.argv[0])
+
+    if hook_name == "install":
+        retVal = install()
+    else:
+        pass
 
=== modified file 'hooks/install'
--- hooks/install	2014-05-22 17:00:22 +0000
+++ hooks/install	1970-01-01 00:00:00 +0000
@@ -1,58 +0,0 @@
-#!/bin/bash
-set -ex
-
-source files/upstart/hadoop_env
-
-if [ -d $HADOOP_DIR ]; then
-  rm -rf $HADOOP_DIR
-fi
-
-echo -e "Host *\n\tStrictHostKeyChecking no" > /home/ubuntu/.ssh/config
-
-yes | ssh-keygen -t rsa -N "" -f /home/ubuntu/.ssh/id_rsa 
-cat /home/ubuntu/.ssh/id_rsa.pub >> /home/ubuntu/.ssh/authorized_keys
-chown -R ubuntu.ubuntu /home/ubuntu/.ssh
-# ssh localhost just to make sure
-# ssh -o StrictHostKeyChecking=no localhost
-
-juju-log "Installing JAVA!"
-
-mkdir -p $HADOOP_DIR
-apt-get update -qqy
-apt-get install -y openjdk-7-jdk
-JAVA_HOME_PATH=$(find /usr/ -name java-$JAVA_VERSION-openjdk-$PLATFORM_ARCH)
-
-juju-log "installing other development packages"
-apt-get -qqy install maven build-essential autoconf automake libtool cmake zlib1g-dev pkg-config libssl-dev snappy libsnappy-dev
-juju-log "Now for Hadoop!"
-
-mkdir -p $HADOOP_DIR
-mkdir -p $HADOOP_TMP_DIR
-# cd $HADOOP_DIR
-tar -xvzf files/archives/$HADOOP_VERSION.tar.gz -C $HADOOP_DIR
-cd $HADOOP_DIR
-sed -ir 's|export JAVA_HOME=.*|export JAVA_HOME='$JAVA_HOME_PATH'|' $(find -name hadoop-env.sh)
-
-mkdir -p $HADOOP_DIR/data/hdfs/namenode
-mkdir -p $HADOOP_DIR/data/hdfs/datanode
-cd $CHARM_DIR
-install -o root -g root -m 0644 files/upstart/defaults /etc/default/hadoop
-
-# Do a bunch of fun stuff in XML
-
-install -o ubuntu -g ubuntu -m 0644 files/hadoop/core-site.xml $HADOOP_DIR/$HADOOP_VERSION/etc/hadoop/core-site.xml
-install -o ubuntu -g ubuntu -m 0644 files/hadoop/hdfs-site.xml $HADOOP_DIR/$HADOOP_VERSION/etc/hadoop/hdfs-site.xml
-install -o ubuntu -g ubuntu -m 0644 files/hadoop/yarn-site.xml $HADOOP_DIR/$HADOOP_VERSION/etc/hadoop/yarn-site.xml
-install -o ubuntu -g ubuntu -m 0644 files/hadoop/mapreduce-site.xml $HADOOP_DIR/$HADOOP_VERSION/etc/hadoop/mapred-site.xml
-
-# Upstart
-
-. /etc/default/hadoop $JAVA_HOME_PATH $HADOOP_INSTALL $HOME_DIR
-source $HOME_DIR/.profile
-hdfs namenode -format
-
-chown -R ubuntu.ubuntu ~ubuntu/hadoop
-
-juju-log "Install complete."
-
-
 
=== target is u'common.py'
=== removed file 'hooks/relation-name-relation-broken'
--- hooks/relation-name-relation-broken	2014-05-22 17:00:22 +0000
+++ hooks/relation-name-relation-broken	1970-01-01 00:00:00 +0000
@@ -1,2 +0,0 @@
-#!/bin/sh
-# This hook runs when the full relation is removed (not just a single member)
 
=== removed file 'hooks/relation-name-relation-changed'
--- hooks/relation-name-relation-changed	2014-05-22 17:00:22 +0000
+++ hooks/relation-name-relation-changed	1970-01-01 00:00:00 +0000
@@ -1,9 +0,0 @@
-#!/bin/bash
-# This must be renamed to the name of the relation. The goal here is to
-# affect any change needed by relationships being formed, modified, or broken
-# This script should be idempotent.
-juju-log $JUJU_REMOTE_UNIT modified its settings
-juju-log Relation settings:
-relation-get
-juju-log Relation members:
-relation-list
 
=== removed file 'hooks/relation-name-relation-departed'
--- hooks/relation-name-relation-departed	2014-05-22 17:00:22 +0000
+++ hooks/relation-name-relation-departed	1970-01-01 00:00:00 +0000
@@ -1,5 +0,0 @@
-#!/bin/sh
-# This must be renamed to the name of the relation. The goal here is to
-# affect any change needed by the remote unit leaving the relationship.
-# This script should be idempotent.
-juju-log $JUJU_REMOTE_UNIT departed
 
=== removed file 'hooks/relation-name-relation-joined'
--- hooks/relation-name-relation-joined	2014-05-22 17:00:22 +0000
+++ hooks/relation-name-relation-joined	1970-01-01 00:00:00 +0000
@@ -1,5 +0,0 @@
-#!/bin/sh
-# This must be renamed to the name of the relation. The goal here is to
-# affect any change needed by relationships being formed
-# This script should be idempotent.
-juju-log $JUJU_REMOTE_UNIT joined
 
=== added directory 'tests'
=== added file 'tests/01_test_install_hook.py'
--- tests/01_test_install_hook.py	1970-01-01 00:00:00 +0000
+++ tests/01_test_install_hook.py	2014-06-26 20:05:25 +0000
@@ -0,0 +1,22 @@
+import sys
+import os
+import unittest
+
+from mock import patch, Mock
+sys.path.insert(0, os.path.abspath(os.path.join('..', 'hooks')))
+
+from hooks.common import HadoopDevel
+
+
+class TestCommon(unittest.TestCase):
+
+    @patch('grp.getgrnam')
+    @patch('pwd.getpwnam')
+    def test_init(self, pwm, grpm):
+        hd = HadoopDevel()
+        self.assertEqual(hd.JAVA_VERSION, "7")
+        self.assertEqual(hd.HADOOP_VERSION, "hadoop-2.2.0")
+        self.assertEqual(hd.PLATFORM_ARCH, "amd64")
+        self.assertEqual(hd.HOME_DIR, "/home/ubuntu")
+        self.assertEqual(hd.HADOOP_TMP_DIR, "/home/ubuntu/hadoop/tmp")
+        self.assertEqual(hd.HADOOP_INSTALL, "/home/ubuntu/hadoop/hadoop-2.2.0")
 
=== added file 'tests/tests.yaml'
--- tests/tests.yaml	1970-01-01 00:00:00 +0000
+++ tests/tests.yaml	2014-06-26 20:05:25 +0000
@@ -0,0 +1,10 @@
+bootstrap: false
+reset: false
+setup: script
+teardown: script
+tests: "[0-9]*"
+virtualenv: false
+#sources:
+packages:
+- amulet
+- python-requests