Merge lp:~lomov-as/charms/trusty/cf-loggregator/trunk into lp:~cf-charmers/charms/trusty/cf-loggregator/trunk

[Alex Lomov]

Use new charm helpers.

Redesign to use new charm helpers.

Update charmhelpers, move config file to template, config file is placed to CF directory, create distict file for install hook, update hooks.py to use services helper

[Alex Lomov (lomov-as)]

Please take a look.

[Alexandr Prismakov (prismakov)]

Reviewed, looks good.

review: Approve

[Alex Lomov (lomov-as)]

Do we need to merge it manually, or it is merged after approval automatically ?

[Alexandr Prismakov (prismakov)]

> Do we need to merge it manually, or it is merged after approval automatically
> ?
There is no automatic merge. So please do it manually.

=== modified file 'Makefile'
--- Makefile	2014-03-28 22:37:40 +0000
+++ Makefile	2014-05-12 11:14:56 +0000
@@ -1,8 +1,23 @@
-
-
-sync:
-	charm-helper-sync -c charm-helpers.yaml
-
+CHARM_DIR := $(shell pwd)
+TEST_TIMEOUT := 900
+
+test: lint 
+
+lint:
+	@echo "Lint check (flake8)"
+	@flake8 --exclude=hooks/charmhelpers hooks 
 clean:
 	find . -name '*.pyc' -delete
 	find . -name '*.bak' -delete
+run: lint deploy
+log:
+	tail -f ~/.juju/local/log/unit-loggregator-0.log
+deploy:
+
+ifdef m
+	juju deploy --to $(m) --repository=../../. local:trusty/cf-loggregator loggregator --show-log
+else
+	juju deploy --repository=../../. local:trusty/cf-loggregator loggregator --show-log
+endif
+upgrade:
+	juju upgrade-charm --repository=../../. loggregator --show-log

=== modified file 'charm-helpers.yaml'
--- charm-helpers.yaml	2014-03-28 22:37:40 +0000
+++ charm-helpers.yaml	2014-05-12 11:14:56 +0000
@@ -1,7 +1,10 @@
 destination: hooks/charmhelpers
-branch: lp:~cf-charmers/charm-helpers/cloud-foundry
+branch: lp:~cf-charmers/charm-helpers/cloud-foundry/
 include:
   - core
-
-
-
+  - fetch
+  - payload.execd
+  - contrib.openstack
+  - contrib.hahelpers
+  - contrib.storage
+  - contrib.cloudfoundry
\ No newline at end of file

=== modified file 'config.yaml'
--- config.yaml	2014-05-12 07:19:53 +0000
+++ config.yaml	2014-05-12 11:14:56 +0000
@@ -2,9 +2,7 @@
   max-retained-logs:
     type: int
     default: 20
-    description: |
-      Not implemented yet
-  client-secret:
+  client_secret:
     type: string
     description: |
       The shared-secret for log clients to use when publishing log

=== modified file 'files/upstart/loggregator.conf'
--- files/upstart/loggregator.conf	2014-03-30 18:32:33 +0000
+++ files/upstart/loggregator.conf	2014-05-12 11:14:56 +0000
@@ -6,5 +6,5 @@
 respawn limit 10 5
 setuid vcap
 setgid vcap
-exec /usr/bin/loggregator -config /etc/vcap/loggregator.json -logFile /var/log/vcap/loggregator.log
+exec /usr/bin/loggregator -config /var/lib/cloudfoundry/cfloggregator/config/loggregator.json -logFile /var/log/vcap/loggregator.log
 

=== added directory 'hooks/charmhelpers/contrib'
=== added file 'hooks/charmhelpers/contrib/__init__.py'
=== added directory 'hooks/charmhelpers/contrib/cloudfoundry'
=== added file 'hooks/charmhelpers/contrib/cloudfoundry/__init__.py'
=== added file 'hooks/charmhelpers/contrib/cloudfoundry/common.py'
--- hooks/charmhelpers/contrib/cloudfoundry/common.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/common.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,71 @@
+import sys
+import os
+import pwd
+import grp
+import subprocess
+
+from contextlib import contextmanager
+from charmhelpers.core.hookenv import log, ERROR, DEBUG
+from charmhelpers.core import host
+
+from charmhelpers.fetch import (
+    apt_install, apt_update, add_source, filter_installed_packages
+)
+
+
+def run(command, exit_on_error=True, quiet=False):
+    '''Run a command and return the output.'''
+    if not quiet:
+        log("Running {!r}".format(command), DEBUG)
+    p = subprocess.Popen(
+        command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+        shell=isinstance(command, basestring))
+    p.stdin.close()
+    lines = []
+    for line in p.stdout:
+        if line:
+            if not quiet:
+                print line
+            lines.append(line)
+        elif p.poll() is not None:
+            break
+
+    p.wait()
+
+    if p.returncode == 0:
+        return '\n'.join(lines)
+
+    if p.returncode != 0 and exit_on_error:
+        log("ERROR: {}".format(p.returncode), ERROR)
+        sys.exit(p.returncode)
+
+    raise subprocess.CalledProcessError(
+        p.returncode, command, '\n'.join(lines))
+
+
+def chownr(path, owner, group):
+    uid = pwd.getpwnam(owner).pw_uid
+    gid = grp.getgrnam(group).gr_gid
+    for root, dirs, files in os.walk(path):
+        for momo in dirs:
+            os.chown(os.path.join(root, momo), uid, gid)
+            for momo in files:
+                os.chown(os.path.join(root, momo), uid, gid)
+
+
+@contextmanager
+def chdir(d):
+    cur = os.getcwd()
+    try:
+        yield os.chdir(d)
+    finally:
+        os.chdir(cur)
+
+
+def prepare_cloudfoundry_environment(config_data, packages):
+    if 'source' in config_data:
+        add_source(config_data['source'], config_data.get('key'))
+        apt_update(fatal=True)
+    if packages:
+        apt_install(packages=filter_installed_packages(packages), fatal=True)
+    host.adduser('vcap')

=== added file 'hooks/charmhelpers/contrib/cloudfoundry/config_helper.py'
--- hooks/charmhelpers/contrib/cloudfoundry/config_helper.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/config_helper.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,11 @@
+import jinja2
+
+TEMPLATES_DIR = 'templates'
+
+def render_template(template_name, context, template_dir=TEMPLATES_DIR):
+    templates = jinja2.Environment(
+        loader=jinja2.FileSystemLoader(template_dir))
+    template = templates.get_template(template_name)
+    return template.render(context)
+
+

=== added file 'hooks/charmhelpers/contrib/cloudfoundry/contexts.py'
--- hooks/charmhelpers/contrib/cloudfoundry/contexts.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/contexts.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,83 @@
+import os
+import yaml
+
+from charmhelpers.core import hookenv
+from charmhelpers.contrib.openstack.context import OSContextGenerator
+
+
+class RelationContext(OSContextGenerator):
+    def __call__(self):
+        if not hookenv.relation_ids(self.interface):
+            return {}
+
+        ctx = {}
+        for rid in hookenv.relation_ids(self.interface):
+            for unit in hookenv.related_units(rid):
+                reldata = hookenv.relation_get(rid=rid, unit=unit)
+                required = set(self.required_keys)
+                if set(reldata.keys()).issuperset(required):
+                    ns = ctx.setdefault(self.interface, {})
+                    for k, v in reldata.items():
+                        ns[k] = v
+                    return ctx
+
+        return {}
+
+
+class ConfigContext(OSContextGenerator):
+    def __call__(self):
+        return hookenv.config()
+
+
+class StorableContext(object):
+    
+    def store_context(self, file_name, config_data):
+        with open(file_name, 'w') as file_stream:
+            yaml.dump(config_data, file_stream)
+
+    def read_context(self, file_name):
+        with open(file_name, 'r') as file_stream:
+            data = yaml.load(file_stream)
+            if not data:
+                raise OSError("%s is empty" % file_name)
+            return data
+
+
+# Stores `config_data` hash into yaml file with `file_name` as a name
+# if `file_name` already exists, then it loads data from `file_name`.
+class StoredContext(OSContextGenerator, StorableContext):
+
+    def __init__(self, file_name, config_data):
+        self.data = config_data
+        if os.path.exists(file_name):
+            self.data = self.read_context(file_name)
+        else:
+            self.store_context(file_name, config_data)
+            self.data = config_data
+
+    def __call__(self):
+        return self.data
+
+
+class StaticContext(OSContextGenerator):
+    def __init__(self, data):
+        self.data = data
+
+    def __call__(self):
+        return self.data
+
+
+class NatsContext(RelationContext):
+    interface = 'nats'
+    required_keys = ['nats_port', 'nats_address', 'nats_user', 'nats_password']
+
+
+class RouterContext(RelationContext):
+    interface = 'router'
+    required_keys = ['domain']
+
+
+class LoggregatorContext(RelationContext):
+    interface = 'loggregator'
+    required_keys = ['shared_secret', 'loggregator_address']
+

=== added file 'hooks/charmhelpers/contrib/cloudfoundry/install.py'
--- hooks/charmhelpers/contrib/cloudfoundry/install.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/install.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,35 @@
+import os
+import subprocess
+
+
+def install(src, dest, fileprops=None, sudo=False):
+    """Install a file from src to dest. Dest can be a complete filename
+    or a target directory. fileprops is a dict with 'owner' (username of owner)
+    and mode (octal string) as keys, the defaults are 'ubuntu' and '400'
+
+    When owner is passed or when access requires it sudo can be set to True and
+    sudo will be used to install the file.
+    """
+    if not fileprops:
+        fileprops = {}
+    mode = fileprops.get('mode', '400')
+    owner = fileprops.get('owner')
+    cmd = ['install']
+
+    if not os.path.exists(src):
+        raise OSError(src)
+
+    if not os.path.exists(dest) and not os.path.exists(os.path.dirname(dest)):
+        # create all but the last component as path
+        cmd.append('-D')
+
+    if mode:
+        cmd.extend(['-m', mode])
+
+    if owner:
+        cmd.extend(['-o', owner])
+
+    if sudo:
+        cmd.insert(0, 'sudo')
+    cmd.extend([src, dest])
+    subprocess.check_call(cmd)

=== added file 'hooks/charmhelpers/contrib/cloudfoundry/services.py'
--- hooks/charmhelpers/contrib/cloudfoundry/services.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/services.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,118 @@
+import os
+import tempfile
+from charmhelpers.core import host
+
+from charmhelpers.contrib.cloudfoundry.install import install
+from charmhelpers.core.hookenv import log
+from jinja2 import Environment, FileSystemLoader
+
+SERVICE_CONFIG = []
+TEMPLATE_LOADER = None
+
+
+def render_template(template_name, context):
+    """Render template to a tempfile returning the name"""
+    _, fn = tempfile.mkstemp()
+    template = load_template(template_name)
+    output = template.render(context)
+    with open(fn, "w") as fp:
+        fp.write(output)
+    return fn
+
+
+def collect_contexts(context_providers):
+    ctx = {}
+    for provider in context_providers:
+        c = provider()
+        if not c:
+            return {}
+        ctx.update(c)
+    return ctx
+
+
+def load_template(name):
+    return TEMPLATE_LOADER.get_template(name)
+
+
+def configure_templates(template_dir):
+    global TEMPLATE_LOADER
+    TEMPLATE_LOADER = Environment(loader=FileSystemLoader(template_dir))
+
+
+def register(service_configs, template_dir):
+    """Register a list of service configs.
+
+    Service Configs are dicts in the following formats:
+
+        {
+            "service": <service name>,
+            "templates": [ {
+                'target': <render target of template>,
+                'source': <optional name of template in passed in template_dir>
+                'file_properties': <optional dict taking owner and octal mode>
+                'contexts': [ context generators, see contexts.py ]
+                }
+            ] }
+
+    If 'source' is not provided for a template the template_dir will
+    be consulted for ``basename(target).j2``.
+    """
+    global SERVICE_CONFIG
+    if template_dir:
+        configure_templates(template_dir)
+    SERVICE_CONFIG.extend(service_configs)
+
+
+def reset():
+    global SERVICE_CONFIG
+    SERVICE_CONFIG = []
+
+
+# def service_context(name):
+#     contexts = collect_contexts(template['contexts'])
+
+def reconfigure_service(service_name, restart=True):
+    global SERVICE_CONFIG
+    service = None
+    for service in SERVICE_CONFIG:
+        if service['service'] == service_name:
+            break
+    if not service or service['service'] != service_name:
+        raise KeyError('Service not registered: %s' % service_name)
+
+    templates = service['templates']
+    for template in templates:
+        contexts = collect_contexts(template['contexts'])
+        if contexts:
+            template_target = template['target']
+            default_template = "%s.j2" % os.path.basename(template_target)
+            template_name = template.get('source', default_template)
+            output_file = render_template(template_name, contexts)
+            file_properties = template.get('file_properties')
+            install(output_file, template_target, file_properties)
+            os.unlink(output_file)
+        else:
+            restart = False
+
+    if restart:
+        host.service_restart(service_name)
+
+
+def stop_services():
+    global SERVICE_CONFIG
+    for service in SERVICE_CONFIG:
+        if host.service_running(service['service']):
+            host.service_stop(service['service'])
+
+
+def get_service(service_name):
+    global SERVICE_CONFIG
+    for service in SERVICE_CONFIG:
+        if service_name == service['service']:
+            return service
+    return None
+
+
+def reconfigure_services(restart=True):
+    for service in SERVICE_CONFIG:
+        reconfigure_service(service['service'], restart=restart)

=== added file 'hooks/charmhelpers/contrib/cloudfoundry/upstart_helper.py'
--- hooks/charmhelpers/contrib/cloudfoundry/upstart_helper.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/upstart_helper.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,14 @@
+import os
+import glob
+from charmhelpers.core import hookenv
+from charmhelpers.core.hookenv import charm_dir
+from charmhelpers.contrib.cloudfoundry.install import install
+
+
+def install_upstart_scripts(dirname=os.path.join(hookenv.charm_dir(),
+                                                 'files/upstart'),
+                            pattern='*.conf'):
+    for script in glob.glob("%s/%s" % (dirname, pattern)):
+        filename = os.path.join(dirname, script)
+        hookenv.log('Installing upstart job:' + filename, hookenv.DEBUG)
+        install(filename, '/etc/init')

=== added directory 'hooks/charmhelpers/contrib/hahelpers'
=== added file 'hooks/charmhelpers/contrib/hahelpers/__init__.py'
=== added file 'hooks/charmhelpers/contrib/hahelpers/apache.py'
--- hooks/charmhelpers/contrib/hahelpers/apache.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/hahelpers/apache.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,58 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# This file is sourced from lp:openstack-charm-helpers
+#
+# Authors:
+#  James Page <[e-mail]>
+#  Adam Gandelman <[e-mail]>
+#
+
+import subprocess
+
+from charmhelpers.core.hookenv import (
+    config as config_get,
+    relation_get,
+    relation_ids,
+    related_units as relation_list,
+    log,
+    INFO,
+)
+
+
+def get_cert():
+    cert = config_get('ssl_cert')
+    key = config_get('ssl_key')
+    if not (cert and key):
+        log("Inspecting identity-service relations for SSL certificate.",
+            level=INFO)
+        cert = key = None
+        for r_id in relation_ids('identity-service'):
+            for unit in relation_list(r_id):
+                if not cert:
+                    cert = relation_get('ssl_cert',
+                                        rid=r_id, unit=unit)
+                if not key:
+                    key = relation_get('ssl_key',
+                                       rid=r_id, unit=unit)
+    return (cert, key)
+
+
+def get_ca_cert():
+    ca_cert = None
+    log("Inspecting identity-service relations for CA SSL certificate.",
+        level=INFO)
+    for r_id in relation_ids('identity-service'):
+        for unit in relation_list(r_id):
+            if not ca_cert:
+                ca_cert = relation_get('ca_cert',
+                                       rid=r_id, unit=unit)
+    return ca_cert
+
+
+def install_ca_cert(ca_cert):
+    if ca_cert:
+        with open('/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt',
+                  'w') as crt:
+            crt.write(ca_cert)
+        subprocess.check_call(['update-ca-certificates', '--fresh'])

=== added file 'hooks/charmhelpers/contrib/hahelpers/cluster.py'
--- hooks/charmhelpers/contrib/hahelpers/cluster.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/hahelpers/cluster.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,183 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+#  James Page <[e-mail]>
+#  Adam Gandelman <[e-mail]>
+#
+
+import subprocess
+import os
+
+from socket import gethostname as get_unit_hostname
+
+from charmhelpers.core.hookenv import (
+    log,
+    relation_ids,
+    related_units as relation_list,
+    relation_get,
+    config as config_get,
+    INFO,
+    ERROR,
+    unit_get,
+)
+
+
+class HAIncompleteConfig(Exception):
+    pass
+
+
+def is_clustered():
+    for r_id in (relation_ids('ha') or []):
+        for unit in (relation_list(r_id) or []):
+            clustered = relation_get('clustered',
+                                     rid=r_id,
+                                     unit=unit)
+            if clustered:
+                return True
+    return False
+
+
+def is_leader(resource):
+    cmd = [
+        "crm", "resource",
+        "show", resource
+    ]
+    try:
+        status = subprocess.check_output(cmd)
+    except subprocess.CalledProcessError:
+        return False
+    else:
+        if get_unit_hostname() in status:
+            return True
+        else:
+            return False
+
+
+def peer_units():
+    peers = []
+    for r_id in (relation_ids('cluster') or []):
+        for unit in (relation_list(r_id) or []):
+            peers.append(unit)
+    return peers
+
+
+def oldest_peer(peers):
+    local_unit_no = int(os.getenv('JUJU_UNIT_NAME').split('/')[1])
+    for peer in peers:
+        remote_unit_no = int(peer.split('/')[1])
+        if remote_unit_no < local_unit_no:
+            return False
+    return True
+
+
+def eligible_leader(resource):
+    if is_clustered():
+        if not is_leader(resource):
+            log('Deferring action to CRM leader.', level=INFO)
+            return False
+    else:
+        peers = peer_units()
+        if peers and not oldest_peer(peers):
+            log('Deferring action to oldest service unit.', level=INFO)
+            return False
+    return True
+
+
+def https():
+    '''
+    Determines whether enough data has been provided in configuration
+    or relation data to configure HTTPS
+    .
+    returns: boolean
+    '''
+    if config_get('use-https') == "yes":
+        return True
+    if config_get('ssl_cert') and config_get('ssl_key'):
+        return True
+    for r_id in relation_ids('identity-service'):
+        for unit in relation_list(r_id):
+            rel_state = [
+                relation_get('https_keystone', rid=r_id, unit=unit),
+                relation_get('ssl_cert', rid=r_id, unit=unit),
+                relation_get('ssl_key', rid=r_id, unit=unit),
+                relation_get('ca_cert', rid=r_id, unit=unit),
+            ]
+            # NOTE: works around (LP: #1203241)
+            if (None not in rel_state) and ('' not in rel_state):
+                return True
+    return False
+
+
+def determine_api_port(public_port):
+    '''
+    Determine correct API server listening port based on
+    existence of HTTPS reverse proxy and/or haproxy.
+
+    public_port: int: standard public port for given service
+
+    returns: int: the correct listening port for the API service
+    '''
+    i = 0
+    if len(peer_units()) > 0 or is_clustered():
+        i += 1
+    if https():
+        i += 1
+    return public_port - (i * 10)
+
+
+def determine_apache_port(public_port):
+    '''
+    Description: Determine correct apache listening port based on public IP +
+    state of the cluster.
+
+    public_port: int: standard public port for given service
+
+    returns: int: the correct listening port for the HAProxy service
+    '''
+    i = 0
+    if len(peer_units()) > 0 or is_clustered():
+        i += 1
+    return public_port - (i * 10)
+
+
+def get_hacluster_config():
+    '''
+    Obtains all relevant configuration from charm configuration required
+    for initiating a relation to hacluster:
+
+        ha-bindiface, ha-mcastport, vip, vip_iface, vip_cidr
+
+    returns: dict: A dict containing settings keyed by setting name.
+    raises: HAIncompleteConfig if settings are missing.
+    '''
+    settings = ['ha-bindiface', 'ha-mcastport', 'vip', 'vip_iface', 'vip_cidr']
+    conf = {}
+    for setting in settings:
+        conf[setting] = config_get(setting)
+    missing = []
+    [missing.append(s) for s, v in conf.iteritems() if v is None]
+    if missing:
+        log('Insufficient config data to configure hacluster.', level=ERROR)
+        raise HAIncompleteConfig
+    return conf
+
+
+def canonical_url(configs, vip_setting='vip'):
+    '''
+    Returns the correct HTTP URL to this host given the state of HTTPS
+    configuration and hacluster.
+
+    :configs    : OSTemplateRenderer: A config tempating object to inspect for
+                                      a complete https context.
+    :vip_setting:                str: Setting in charm config that specifies
+                                      VIP address.
+    '''
+    scheme = 'http'
+    if 'https' in configs.complete_contexts():
+        scheme = 'https'
+    if is_clustered():
+        addr = config_get(vip_setting)
+    else:
+        addr = unit_get('private-address')
+    return '%s://%s' % (scheme, addr)

=== added directory 'hooks/charmhelpers/contrib/openstack'
=== added file 'hooks/charmhelpers/contrib/openstack/__init__.py'
=== added file 'hooks/charmhelpers/contrib/openstack/alternatives.py'
--- hooks/charmhelpers/contrib/openstack/alternatives.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/alternatives.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,17 @@
+''' Helper for managing alternatives for file conflict resolution '''
+
+import subprocess
+import shutil
+import os
+
+
+def install_alternative(name, target, source, priority=50):
+    ''' Install alternative configuration '''
+    if (os.path.exists(target) and not os.path.islink(target)):
+        # Move existing file/directory away before installing
+        shutil.move(target, '{}.bak'.format(target))
+    cmd = [
+        'update-alternatives', '--force', '--install',
+        target, name, source, str(priority)
+    ]
+    subprocess.check_call(cmd)

=== added file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,619 @@
+import json
+import os
+
+from base64 import b64decode
+
+from subprocess import (
+    check_call
+)
+
+
+from charmhelpers.fetch import (
+    apt_install,
+    filter_installed_packages,
+)
+
+from charmhelpers.core.hookenv import (
+    config,
+    local_unit,
+    log,
+    relation_get,
+    relation_ids,
+    related_units,
+    unit_get,
+    unit_private_ip,
+    ERROR,
+)
+
+from charmhelpers.contrib.hahelpers.cluster import (
+    determine_apache_port,
+    determine_api_port,
+    https,
+    is_clustered
+)
+
+from charmhelpers.contrib.hahelpers.apache import (
+    get_cert,
+    get_ca_cert,
+)
+
+from charmhelpers.contrib.openstack.neutron import (
+    neutron_plugin_attribute,
+)
+
+CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
+
+
+class OSContextError(Exception):
+    pass
+
+
+def ensure_packages(packages):
+    '''Install but do not upgrade required plugin packages'''
+    required = filter_installed_packages(packages)
+    if required:
+        apt_install(required, fatal=True)
+
+
+def context_complete(ctxt):
+    _missing = []
+    for k, v in ctxt.iteritems():
+        if v is None or v == '':
+            _missing.append(k)
+    if _missing:
+        log('Missing required data: %s' % ' '.join(_missing), level='INFO')
+        return False
+    return True
+
+
+def config_flags_parser(config_flags):
+    if config_flags.find('==') >= 0:
+        log("config_flags is not in expected format (key=value)",
+            level=ERROR)
+        raise OSContextError
+    # strip the following from each value.
+    post_strippers = ' ,'
+    # we strip any leading/trailing '=' or ' ' from the string then
+    # split on '='.
+    split = config_flags.strip(' =').split('=')
+    limit = len(split)
+    flags = {}
+    for i in xrange(0, limit - 1):
+        current = split[i]
+        next = split[i + 1]
+        vindex = next.rfind(',')
+        if (i == limit - 2) or (vindex < 0):
+            value = next
+        else:
+            value = next[:vindex]
+
+        if i == 0:
+            key = current
+        else:
+            # if this not the first entry, expect an embedded key.
+            index = current.rfind(',')
+            if index < 0:
+                log("invalid config value(s) at index %s" % (i),
+                    level=ERROR)
+                raise OSContextError
+            key = current[index + 1:]
+
+        # Add to collection.
+        flags[key.strip(post_strippers)] = value.rstrip(post_strippers)
+    return flags
+
+
+class OSContextGenerator(object):
+    interfaces = []
+
+    def __call__(self):
+        raise NotImplementedError
+
+
+class SharedDBContext(OSContextGenerator):
+    interfaces = ['shared-db']
+
+    def __init__(self, database=None, user=None, relation_prefix=None):
+        '''
+        Allows inspecting relation for settings prefixed with relation_prefix.
+        This is useful for parsing access for multiple databases returned via
+        the shared-db interface (eg, nova_password, quantum_password)
+        '''
+        self.relation_prefix = relation_prefix
+        self.database = database
+        self.user = user
+
+    def __call__(self):
+        self.database = self.database or config('database')
+        self.user = self.user or config('database-user')
+        if None in [self.database, self.user]:
+            log('Could not generate shared_db context. '
+                'Missing required charm config options. '
+                '(database name and user)')
+            raise OSContextError
+        ctxt = {}
+
+        password_setting = 'password'
+        if self.relation_prefix:
+            password_setting = self.relation_prefix + '_password'
+
+        for rid in relation_ids('shared-db'):
+            for unit in related_units(rid):
+                passwd = relation_get(password_setting, rid=rid, unit=unit)
+                ctxt = {
+                    'database_host': relation_get('db_host', rid=rid,
+                                                  unit=unit),
+                    'database': self.database,
+                    'database_user': self.user,
+                    'database_password': passwd,
+                }
+                if context_complete(ctxt):
+                    return ctxt
+        return {}
+
+
+class IdentityServiceContext(OSContextGenerator):
+    interfaces = ['identity-service']
+
+    def __call__(self):
+        log('Generating template context for identity-service')
+        ctxt = {}
+
+        for rid in relation_ids('identity-service'):
+            for unit in related_units(rid):
+                ctxt = {
+                    'service_port': relation_get('service_port', rid=rid,
+                                                 unit=unit),
+                    'service_host': relation_get('service_host', rid=rid,
+                                                 unit=unit),
+                    'auth_host': relation_get('auth_host', rid=rid, unit=unit),
+                    'auth_port': relation_get('auth_port', rid=rid, unit=unit),
+                    'admin_tenant_name': relation_get('service_tenant',
+                                                      rid=rid, unit=unit),
+                    'admin_user': relation_get('service_username', rid=rid,
+                                               unit=unit),
+                    'admin_password': relation_get('service_password', rid=rid,
+                                                   unit=unit),
+                    # XXX: Hard-coded http.
+                    'service_protocol': 'http',
+                    'auth_protocol': 'http',
+                }
+                if context_complete(ctxt):
+                    return ctxt
+        return {}
+
+
+class AMQPContext(OSContextGenerator):
+    interfaces = ['amqp']
+
+    def __call__(self):
+        log('Generating template context for amqp')
+        conf = config()
+        try:
+            username = conf['rabbit-user']
+            vhost = conf['rabbit-vhost']
+        except KeyError as e:
+            log('Could not generate shared_db context. '
+                'Missing required charm config options: %s.' % e)
+            raise OSContextError
+
+        ctxt = {}
+        for rid in relation_ids('amqp'):
+            ha_vip_only = False
+            for unit in related_units(rid):
+                if relation_get('clustered', rid=rid, unit=unit):
+                    ctxt['clustered'] = True
+                    ctxt['rabbitmq_host'] = relation_get('vip', rid=rid,
+                                                         unit=unit)
+                else:
+                    ctxt['rabbitmq_host'] = relation_get('private-address',
+                                                         rid=rid, unit=unit)
+                ctxt.update({
+                    'rabbitmq_user': username,
+                    'rabbitmq_password': relation_get('password', rid=rid,
+                                                      unit=unit),
+                    'rabbitmq_virtual_host': vhost,
+                })
+                if relation_get('ha_queues', rid=rid, unit=unit) is not None:
+                    ctxt['rabbitmq_ha_queues'] = True
+
+                ha_vip_only = relation_get('ha-vip-only',
+                                           rid=rid, unit=unit) is not None
+
+                if context_complete(ctxt):
+                    # Sufficient information found = break out!
+                    break
+            # Used for active/active rabbitmq >= grizzly
+            if ('clustered' not in ctxt or ha_vip_only) \
+                    and len(related_units(rid)) > 1:
+                rabbitmq_hosts = []
+                for unit in related_units(rid):
+                    rabbitmq_hosts.append(relation_get('private-address',
+                                                       rid=rid, unit=unit))
+                ctxt['rabbitmq_hosts'] = ','.join(rabbitmq_hosts)
+        if not context_complete(ctxt):
+            return {}
+        else:
+            return ctxt
+
+
+class CephContext(OSContextGenerator):
+    interfaces = ['ceph']
+
+    def __call__(self):
+        '''This generates context for /etc/ceph/ceph.conf templates'''
+        if not relation_ids('ceph'):
+            return {}
+
+        log('Generating template context for ceph')
+
+        mon_hosts = []
+        auth = None
+        key = None
+        use_syslog = str(config('use-syslog')).lower()
+        for rid in relation_ids('ceph'):
+            for unit in related_units(rid):
+                mon_hosts.append(relation_get('private-address', rid=rid,
+                                              unit=unit))
+                auth = relation_get('auth', rid=rid, unit=unit)
+                key = relation_get('key', rid=rid, unit=unit)
+
+        ctxt = {
+            'mon_hosts': ' '.join(mon_hosts),
+            'auth': auth,
+            'key': key,
+            'use_syslog': use_syslog
+        }
+
+        if not os.path.isdir('/etc/ceph'):
+            os.mkdir('/etc/ceph')
+
+        if not context_complete(ctxt):
+            return {}
+
+        ensure_packages(['ceph-common'])
+
+        return ctxt
+
+
+class HAProxyContext(OSContextGenerator):
+    interfaces = ['cluster']
+
+    def __call__(self):
+        '''
+        Builds half a context for the haproxy template, which describes
+        all peers to be included in the cluster.  Each charm needs to include
+        its own context generator that describes the port mapping.
+        '''
+        if not relation_ids('cluster'):
+            return {}
+
+        cluster_hosts = {}
+        l_unit = local_unit().replace('/', '-')
+        cluster_hosts[l_unit] = unit_get('private-address')
+
+        for rid in relation_ids('cluster'):
+            for unit in related_units(rid):
+                _unit = unit.replace('/', '-')
+                addr = relation_get('private-address', rid=rid, unit=unit)
+                cluster_hosts[_unit] = addr
+
+        ctxt = {
+            'units': cluster_hosts,
+        }
+        if len(cluster_hosts.keys()) > 1:
+            # Enable haproxy when we have enough peers.
+            log('Ensuring haproxy enabled in /etc/default/haproxy.')
+            with open('/etc/default/haproxy', 'w') as out:
+                out.write('ENABLED=1\n')
+            return ctxt
+        log('HAProxy context is incomplete, this unit has no peers.')
+        return {}
+
+
+class ImageServiceContext(OSContextGenerator):
+    interfaces = ['image-service']
+
+    def __call__(self):
+        '''
+        Obtains the glance API server from the image-service relation.  Useful
+        in nova and cinder (currently).
+        '''
+        log('Generating template context for image-service.')
+        rids = relation_ids('image-service')
+        if not rids:
+            return {}
+        for rid in rids:
+            for unit in related_units(rid):
+                api_server = relation_get('glance-api-server',
+                                          rid=rid, unit=unit)
+                if api_server:
+                    return {'glance_api_servers': api_server}
+        log('ImageService context is incomplete. '
+            'Missing required relation data.')
+        return {}
+
+
+class ApacheSSLContext(OSContextGenerator):
+
+    """
+    Generates a context for an apache vhost configuration that configures
+    HTTPS reverse proxying for one or many endpoints.  Generated context
+    looks something like:
+    {
+        'namespace': 'cinder',
+        'private_address': 'iscsi.mycinderhost.com',
+        'endpoints': [(8776, 8766), (8777, 8767)]
+    }
+
+    The endpoints list consists of a tuples mapping external ports
+    to internal ports.
+    """
+    interfaces = ['https']
+
+    # charms should inherit this context and set external ports
+    # and service namespace accordingly.
+    external_ports = []
+    service_namespace = None
+
+    def enable_modules(self):
+        cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http']
+        check_call(cmd)
+
+    def configure_cert(self):
+        if not os.path.isdir('/etc/apache2/ssl'):
+            os.mkdir('/etc/apache2/ssl')
+        ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace)
+        if not os.path.isdir(ssl_dir):
+            os.mkdir(ssl_dir)
+        cert, key = get_cert()
+        with open(os.path.join(ssl_dir, 'cert'), 'w') as cert_out:
+            cert_out.write(b64decode(cert))
+        with open(os.path.join(ssl_dir, 'key'), 'w') as key_out:
+            key_out.write(b64decode(key))
+        ca_cert = get_ca_cert()
+        if ca_cert:
+            with open(CA_CERT_PATH, 'w') as ca_out:
+                ca_out.write(b64decode(ca_cert))
+            check_call(['update-ca-certificates'])
+
+    def __call__(self):
+        if isinstance(self.external_ports, basestring):
+            self.external_ports = [self.external_ports]
+        if (not self.external_ports or not https()):
+            return {}
+
+        self.configure_cert()
+        self.enable_modules()
+
+        ctxt = {
+            'namespace': self.service_namespace,
+            'private_address': unit_get('private-address'),
+            'endpoints': []
+        }
+        for api_port in self.external_ports:
+            ext_port = determine_apache_port(api_port)
+            int_port = determine_api_port(api_port)
+            portmap = (int(ext_port), int(int_port))
+            ctxt['endpoints'].append(portmap)
+        return ctxt
+
+
+class NeutronContext(OSContextGenerator):
+    interfaces = []
+
+    @property
+    def plugin(self):
+        return None
+
+    @property
+    def network_manager(self):
+        return None
+
+    @property
+    def packages(self):
+        return neutron_plugin_attribute(
+            self.plugin, 'packages', self.network_manager)
+
+    @property
+    def neutron_security_groups(self):
+        return None
+
+    def _ensure_packages(self):
+        [ensure_packages(pkgs) for pkgs in self.packages]
+
+    def _save_flag_file(self):
+        if self.network_manager == 'quantum':
+            _file = '/etc/nova/quantum_plugin.conf'
+        else:
+            _file = '/etc/nova/neutron_plugin.conf'
+        with open(_file, 'wb') as out:
+            out.write(self.plugin + '\n')
+
+    def ovs_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        config = neutron_plugin_attribute(self.plugin, 'config',
+                                          self.network_manager)
+        ovs_ctxt = {
+            'core_plugin': driver,
+            'neutron_plugin': 'ovs',
+            'neutron_security_groups': self.neutron_security_groups,
+            'local_ip': unit_private_ip(),
+            'config': config
+        }
+
+        return ovs_ctxt
+
+    def nvp_ctxt(self):
+        driver = neutron_plugin_attribute(self.plugin, 'driver',
+                                          self.network_manager)
+        config = neutron_plugin_attribute(self.plugin, 'config',
+                                          self.network_manager)
+        nvp_ctxt = {
+            'core_plugin': driver,
+            'neutron_plugin': 'nvp',
+            'neutron_security_groups': self.neutron_security_groups,
+            'local_ip': unit_private_ip(),
+            'config': config
+        }
+
+        return nvp_ctxt
+
+    def neutron_ctxt(self):
+        if https():
+            proto = 'https'
+        else:
+            proto = 'http'
+        if is_clustered():
+            host = config('vip')
+        else:
+            host = unit_get('private-address')
+        url = '%s://%s:%s' % (proto, host, '9696')
+        ctxt = {
+            'network_manager': self.network_manager,
+            'neutron_url': url,
+        }
+        return ctxt
+
+    def __call__(self):
+        self._ensure_packages()
+
+        if self.network_manager not in ['quantum', 'neutron']:
+            return {}
+
+        if not self.plugin:
+            return {}
+
+        ctxt = self.neutron_ctxt()
+
+        if self.plugin == 'ovs':
+            ctxt.update(self.ovs_ctxt())
+        elif self.plugin == 'nvp':
+            ctxt.update(self.nvp_ctxt())
+
+        alchemy_flags = config('neutron-alchemy-flags')
+        if alchemy_flags:
+            flags = config_flags_parser(alchemy_flags)
+            ctxt['neutron_alchemy_flags'] = flags
+
+        self._save_flag_file()
+        return ctxt
+
+
+class OSConfigFlagContext(OSContextGenerator):
+
+        """
+        Responsible for adding user-defined config-flags in charm config to a
+        template context.
+
+        NOTE: the value of config-flags may be a comma-separated list of
+              key=value pairs and some Openstack config files support
+              comma-separated lists as values.
+        """
+
+        def __call__(self):
+            config_flags = config('config-flags')
+            if not config_flags:
+                return {}
+
+            flags = config_flags_parser(config_flags)
+            return {'user_config_flags': flags}
+
+
+class SubordinateConfigContext(OSContextGenerator):
+
+    """
+    Responsible for inspecting relations to subordinates that
+    may be exporting required config via a json blob.
+
+    The subordinate interface allows subordinates to export their
+    configuration requirements to the principle for multiple config
+    files and multiple serivces.  Ie, a subordinate that has interfaces
+    to both glance and nova may export to following yaml blob as json:
+
+        glance:
+            /etc/glance/glance-api.conf:
+                sections:
+                    DEFAULT:
+                        - [key1, value1]
+            /etc/glance/glance-registry.conf:
+                    MYSECTION:
+                        - [key2, value2]
+        nova:
+            /etc/nova/nova.conf:
+                sections:
+                    DEFAULT:
+                        - [key3, value3]
+
+
+    It is then up to the principle charms to subscribe this context to
+    the service+config file it is interestd in.  Configuration data will
+    be available in the template context, in glance's case, as:
+        ctxt = {
+            ... other context ...
+            'subordinate_config': {
+                'DEFAULT': {
+                    'key1': 'value1',
+                },
+                'MYSECTION': {
+                    'key2': 'value2',
+                },
+            }
+        }
+
+    """
+
+    def __init__(self, service, config_file, interface):
+        """
+        :param service     : Service name key to query in any subordinate
+                             data found
+        :param config_file : Service's config file to query sections
+        :param interface   : Subordinate interface to inspect
+        """
+        self.service = service
+        self.config_file = config_file
+        self.interface = interface
+
+    def __call__(self):
+        ctxt = {}
+        for rid in relation_ids(self.interface):
+            for unit in related_units(rid):
+                sub_config = relation_get('subordinate_configuration',
+                                          rid=rid, unit=unit)
+                if sub_config and sub_config != '':
+                    try:
+                        sub_config = json.loads(sub_config)
+                    except:
+                        log('Could not parse JSON from subordinate_config '
+                            'setting from %s' % rid, level=ERROR)
+                        continue
+
+                    if self.service not in sub_config:
+                        log('Found subordinate_config on %s but it contained'
+                            'nothing for %s service' % (rid, self.service))
+                        continue
+
+                    sub_config = sub_config[self.service]
+                    if self.config_file not in sub_config:
+                        log('Found subordinate_config on %s but it contained'
+                            'nothing for %s' % (rid, self.config_file))
+                        continue
+
+                    sub_config = sub_config[self.config_file]
+                    for k, v in sub_config.iteritems():
+                        ctxt[k] = v
+
+        if not ctxt:
+            ctxt['sections'] = {}
+
+        return ctxt
+
+
+class SyslogContext(OSContextGenerator):
+
+    def __call__(self):
+        ctxt = {
+            'use_syslog': config('use-syslog')
+        }
+        return ctxt

=== added file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,161 @@
+# Various utilies for dealing with Neutron and the renaming from Quantum.
+
+from subprocess import check_output
+
+from charmhelpers.core.hookenv import (
+    config,
+    log,
+    ERROR,
+)
+
+from charmhelpers.contrib.openstack.utils import os_release
+
+
+def headers_package():
+    """Ensures correct linux-headers for running kernel are installed,
+    for building DKMS package"""
+    kver = check_output(['uname', '-r']).strip()
+    return 'linux-headers-%s' % kver
+
+
+def kernel_version():
+    """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """
+    kver = check_output(['uname', '-r']).strip()
+    kver = kver.split('.')
+    return (int(kver[0]), int(kver[1]))
+
+
+def determine_dkms_package():
+    """ Determine which DKMS package should be used based on kernel version """
+    # NOTE: 3.13 kernels have support for GRE and VXLAN native
+    if kernel_version() >= (3, 13):
+        return []
+    else:
+        return ['openvswitch-datapath-dkms']
+
+
+# legacy
+def quantum_plugins():
+    from charmhelpers.contrib.openstack import context
+    return {
+        'ovs': {
+            'config': '/etc/quantum/plugins/openvswitch/'
+                      'ovs_quantum_plugin.ini',
+            'driver': 'quantum.plugins.openvswitch.ovs_quantum_plugin.'
+                      'OVSQuantumPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron')],
+            'services': ['quantum-plugin-openvswitch-agent'],
+            'packages': [[headers_package()] + determine_dkms_package(),
+                         ['quantum-plugin-openvswitch-agent']],
+            'server_packages': ['quantum-server',
+                                'quantum-plugin-openvswitch'],
+            'server_services': ['quantum-server']
+        },
+        'nvp': {
+            'config': '/etc/quantum/plugins/nicira/nvp.ini',
+            'driver': 'quantum.plugins.nicira.nicira_nvp_plugin.'
+                      'QuantumPlugin.NvpPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron')],
+            'services': [],
+            'packages': [],
+            'server_packages': ['quantum-server',
+                                'quantum-plugin-nicira'],
+            'server_services': ['quantum-server']
+        }
+    }
+
+
+def neutron_plugins():
+    from charmhelpers.contrib.openstack import context
+    release = os_release('nova-common')
+    plugins = {
+        'ovs': {
+            'config': '/etc/neutron/plugins/openvswitch/'
+                      'ovs_neutron_plugin.ini',
+            'driver': 'neutron.plugins.openvswitch.ovs_neutron_plugin.'
+                      'OVSNeutronPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron')],
+            'services': ['neutron-plugin-openvswitch-agent'],
+            'packages': [[headers_package()] + determine_dkms_package(),
+                         ['neutron-plugin-openvswitch-agent']],
+            'server_packages': ['neutron-server',
+                                'neutron-plugin-openvswitch'],
+            'server_services': ['neutron-server']
+        },
+        'nvp': {
+            'config': '/etc/neutron/plugins/nicira/nvp.ini',
+            'driver': 'neutron.plugins.nicira.nicira_nvp_plugin.'
+                      'NeutronPlugin.NvpPluginV2',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron')],
+            'services': [],
+            'packages': [],
+            'server_packages': ['neutron-server',
+                                'neutron-plugin-nicira'],
+            'server_services': ['neutron-server']
+        }
+    }
+    # NOTE: patch in ml2 plugin for icehouse onwards
+    if release >= 'icehouse':
+        plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini'
+        plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin'
+        plugins['ovs']['server_packages'] = ['neutron-server',
+                                             'neutron-plugin-ml2']
+    return plugins
+
+
+def neutron_plugin_attribute(plugin, attr, net_manager=None):
+    manager = net_manager or network_manager()
+    if manager == 'quantum':
+        plugins = quantum_plugins()
+    elif manager == 'neutron':
+        plugins = neutron_plugins()
+    else:
+        log('Error: Network manager does not support plugins.')
+        raise Exception
+
+    try:
+        _plugin = plugins[plugin]
+    except KeyError:
+        log('Unrecognised plugin for %s: %s' % (manager, plugin), level=ERROR)
+        raise Exception
+
+    try:
+        return _plugin[attr]
+    except KeyError:
+        return None
+
+
+def network_manager():
+    '''
+    Deals with the renaming of Quantum to Neutron in H and any situations
+    that require compatability (eg, deploying H with network-manager=quantum,
+    upgrading from G).
+    '''
+    release = os_release('nova-common')
+    manager = config('network-manager').lower()
+
+    if manager not in ['quantum', 'neutron']:
+        return manager
+
+    if release in ['essex']:
+        # E does not support neutron
+        log('Neutron networking not supported in Essex.', level=ERROR)
+        raise Exception
+    elif release in ['folsom', 'grizzly']:
+        # neutron is named quantum in F and G
+        return 'quantum'
+    else:
+        # ensure accurate naming for all releases post-H
+        return 'neutron'

=== added directory 'hooks/charmhelpers/contrib/openstack/templates'
=== added file 'hooks/charmhelpers/contrib/openstack/templates/__init__.py'
--- hooks/charmhelpers/contrib/openstack/templates/__init__.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/__init__.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,2 @@
+# dummy __init__.py to fool syncer into thinking this is a syncable python
+# module

=== added file 'hooks/charmhelpers/contrib/openstack/templating.py'
--- hooks/charmhelpers/contrib/openstack/templating.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/templating.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,280 @@
+import os
+
+from charmhelpers.fetch import apt_install
+
+from charmhelpers.core.hookenv import (
+    log,
+    ERROR,
+    INFO
+)
+
+from charmhelpers.contrib.openstack.utils import OPENSTACK_CODENAMES
+
+try:
+    from jinja2 import FileSystemLoader, ChoiceLoader, Environment, exceptions
+except ImportError:
+    # python-jinja2 may not be installed yet, or we're running unittests.
+    FileSystemLoader = ChoiceLoader = Environment = exceptions = None
+
+
+class OSConfigException(Exception):
+    pass
+
+
+def get_loader(templates_dir, os_release):
+    """
+    Create a jinja2.ChoiceLoader containing template dirs up to
+    and including os_release.  If directory template directory
+    is missing at templates_dir, it will be omitted from the loader.
+    templates_dir is added to the bottom of the search list as a base
+    loading dir.
+
+    A charm may also ship a templates dir with this module
+    and it will be appended to the bottom of the search list, eg:
+    hooks/charmhelpers/contrib/openstack/templates.
+
+    :param templates_dir: str: Base template directory containing release
+                               sub-directories.
+    :param os_release   : str: OpenStack release codename to construct template
+                               loader.
+
+    :returns            : jinja2.ChoiceLoader constructed with a list of
+                          jinja2.FilesystemLoaders, ordered in descending
+                          order by OpenStack release.
+    """
+    tmpl_dirs = [(rel, os.path.join(templates_dir, rel))
+                 for rel in OPENSTACK_CODENAMES.itervalues()]
+
+    if not os.path.isdir(templates_dir):
+        log('Templates directory not found @ %s.' % templates_dir,
+            level=ERROR)
+        raise OSConfigException
+
+    # the bottom contains tempaltes_dir and possibly a common templates dir
+    # shipped with the helper.
+    loaders = [FileSystemLoader(templates_dir)]
+    helper_templates = os.path.join(os.path.dirname(__file__), 'templates')
+    if os.path.isdir(helper_templates):
+        loaders.append(FileSystemLoader(helper_templates))
+
+    for rel, tmpl_dir in tmpl_dirs:
+        if os.path.isdir(tmpl_dir):
+            loaders.insert(0, FileSystemLoader(tmpl_dir))
+        if rel == os_release:
+            break
+    log('Creating choice loader with dirs: %s' %
+        [l.searchpath for l in loaders], level=INFO)
+    return ChoiceLoader(loaders)
+
+
+class OSConfigTemplate(object):
+    """
+    Associates a config file template with a list of context generators.
+    Responsible for constructing a template context based on those generators.
+    """
+    def __init__(self, config_file, contexts):
+        self.config_file = config_file
+
+        if hasattr(contexts, '__call__'):
+            self.contexts = [contexts]
+        else:
+            self.contexts = contexts
+
+        self._complete_contexts = []
+
+    def context(self):
+        ctxt = {}
+        for context in self.contexts:
+            _ctxt = context()
+            if _ctxt:
+                ctxt.update(_ctxt)
+                # track interfaces for every complete context.
+                [self._complete_contexts.append(interface)
+                 for interface in context.interfaces
+                 if interface not in self._complete_contexts]
+        return ctxt
+
+    def complete_contexts(self):
+        '''
+        Return a list of interfaces that have satisfied contexts.
+        '''
+        if self._complete_contexts:
+            return self._complete_contexts
+        self.context()
+        return self._complete_contexts
+
+
+class OSConfigRenderer(object):
+    """
+    This class provides a common templating system to be used by OpenStack
+    charms.  It is intended to help charms share common code and templates,
+    and ease the burden of managing config templates across multiple OpenStack
+    releases.
+
+    Basic usage:
+        # import some common context generates from charmhelpers
+        from charmhelpers.contrib.openstack import context
+
+        # Create a renderer object for a specific OS release.
+        configs = OSConfigRenderer(templates_dir='/tmp/templates',
+                                   openstack_release='folsom')
+        # register some config files with context generators.
+        configs.register(config_file='/etc/nova/nova.conf',
+                         contexts=[context.SharedDBContext(),
+                                   context.AMQPContext()])
+        configs.register(config_file='/etc/nova/api-paste.ini',
+                         contexts=[context.IdentityServiceContext()])
+        configs.register(config_file='/etc/haproxy/haproxy.conf',
+                         contexts=[context.HAProxyContext()])
+        # write out a single config
+        configs.write('/etc/nova/nova.conf')
+        # write out all registered configs
+        configs.write_all()
+
+    Details:
+
+    OpenStack Releases and template loading
+    ---------------------------------------
+    When the object is instantiated, it is associated with a specific OS
+    release.  This dictates how the template loader will be constructed.
+
+    The constructed loader attempts to load the template from several places
+    in the following order:
+        - from the most recent OS release-specific template dir (if one exists)
+        - the base templates_dir
+        - a template directory shipped in the charm with this helper file.
+
+
+    For the example above, '/tmp/templates' contains the following structure:
+        /tmp/templates/nova.conf
+        /tmp/templates/api-paste.ini
+        /tmp/templates/grizzly/api-paste.ini
+        /tmp/templates/havana/api-paste.ini
+
+    Since it was registered with the grizzly release, it first seraches
+    the grizzly directory for nova.conf, then the templates dir.
+
+    When writing api-paste.ini, it will find the template in the grizzly
+    directory.
+
+    If the object were created with folsom, it would fall back to the
+    base templates dir for its api-paste.ini template.
+
+    This system should help manage changes in config files through
+    openstack releases, allowing charms to fall back to the most recently
+    updated config template for a given release
+
+    The haproxy.conf, since it is not shipped in the templates dir, will
+    be loaded from the module directory's template directory, eg
+    $CHARM/hooks/charmhelpers/contrib/openstack/templates.  This allows
+    us to ship common templates (haproxy, apache) with the helpers.
+
+    Context generators
+    ---------------------------------------
+    Context generators are used to generate template contexts during hook
+    execution.  Doing so may require inspecting service relations, charm
+    config, etc.  When registered, a config file is associated with a list
+    of generators.  When a template is rendered and written, all context
+    generates are called in a chain to generate the context dictionary
+    passed to the jinja2 template. See context.py for more info.
+    """
+    def __init__(self, templates_dir, openstack_release):
+        if not os.path.isdir(templates_dir):
+            log('Could not locate templates dir %s' % templates_dir,
+                level=ERROR)
+            raise OSConfigException
+
+        self.templates_dir = templates_dir
+        self.openstack_release = openstack_release
+        self.templates = {}
+        self._tmpl_env = None
+
+        if None in [Environment, ChoiceLoader, FileSystemLoader]:
+            # if this code is running, the object is created pre-install hook.
+            # jinja2 shouldn't get touched until the module is reloaded on next
+            # hook execution, with proper jinja2 bits successfully imported.
+            apt_install('python-jinja2')
+
+    def register(self, config_file, contexts):
+        """
+        Register a config file with a list of context generators to be called
+        during rendering.
+        """
+        self.templates[config_file] = OSConfigTemplate(config_file=config_file,
+                                                       contexts=contexts)
+        log('Registered config file: %s' % config_file, level=INFO)
+
+    def _get_tmpl_env(self):
+        if not self._tmpl_env:
+            loader = get_loader(self.templates_dir, self.openstack_release)
+            self._tmpl_env = Environment(loader=loader)
+
+    def _get_template(self, template):
+        self._get_tmpl_env()
+        template = self._tmpl_env.get_template(template)
+        log('Loaded template from %s' % template.filename, level=INFO)
+        return template
+
+    def render(self, config_file):
+        if config_file not in self.templates:
+            log('Config not registered: %s' % config_file, level=ERROR)
+            raise OSConfigException
+        ctxt = self.templates[config_file].context()
+
+        _tmpl = os.path.basename(config_file)
+        try:
+            template = self._get_template(_tmpl)
+        except exceptions.TemplateNotFound:
+            # if no template is found with basename, try looking for it
+            # using a munged full path, eg:
+            #   /etc/apache2/apache2.conf -> etc_apache2_apache2.conf
+            _tmpl = '_'.join(config_file.split('/')[1:])
+            try:
+                template = self._get_template(_tmpl)
+            except exceptions.TemplateNotFound as e:
+                log('Could not load template from %s by %s or %s.' %
+                    (self.templates_dir, os.path.basename(config_file), _tmpl),
+                    level=ERROR)
+                raise e
+
+        log('Rendering from template: %s' % _tmpl, level=INFO)
+        return template.render(ctxt)
+
+    def write(self, config_file):
+        """
+        Write a single config file, raises if config file is not registered.
+        """
+        if config_file not in self.templates:
+            log('Config not registered: %s' % config_file, level=ERROR)
+            raise OSConfigException
+
+        _out = self.render(config_file)
+
+        with open(config_file, 'wb') as out:
+            out.write(_out)
+
+        log('Wrote template %s.' % config_file, level=INFO)
+
+    def write_all(self):
+        """
+        Write out all registered config files.
+        """
+        [self.write(k) for k in self.templates.iterkeys()]
+
+    def set_release(self, openstack_release):
+        """
+        Resets the template environment and generates a new template loader
+        based on a the new openstack release.
+        """
+        self._tmpl_env = None
+        self.openstack_release = openstack_release
+        self._get_tmpl_env()
+
+    def complete_contexts(self):
+        '''
+        Returns a list of context interfaces that yield a complete context.
+        '''
+        interfaces = []
+        [interfaces.extend(i.complete_contexts())
+         for i in self.templates.itervalues()]
+        return interfaces

=== added file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,447 @@
+#!/usr/bin/python
+
+# Common python helper functions used for OpenStack charms.
+from collections import OrderedDict
+
+import apt_pkg as apt
+import subprocess
+import os
+import socket
+import sys
+
+from charmhelpers.core.hookenv import (
+    config,
+    log as juju_log,
+    charm_dir,
+    ERROR,
+    INFO
+)
+
+from charmhelpers.contrib.storage.linux.lvm import (
+    deactivate_lvm_volume_group,
+    is_lvm_physical_volume,
+    remove_lvm_physical_volume,
+)
+
+from charmhelpers.core.host import lsb_release, mounts, umount
+from charmhelpers.fetch import apt_install
+from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
+from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device
+
+CLOUD_ARCHIVE_URL = "http://ubuntu-cloud.archive.canonical.com/ubuntu"
+CLOUD_ARCHIVE_KEY_ID = '5EDB1B62EC4926EA'
+
+DISTRO_PROPOSED = ('deb http://archive.ubuntu.com/ubuntu/ %s-proposed '
+                   'restricted main multiverse universe')
+
+
+UBUNTU_OPENSTACK_RELEASE = OrderedDict([
+    ('oneiric', 'diablo'),
+    ('precise', 'essex'),
+    ('quantal', 'folsom'),
+    ('raring', 'grizzly'),
+    ('saucy', 'havana'),
+    ('trusty', 'icehouse')
+])
+
+
+OPENSTACK_CODENAMES = OrderedDict([
+    ('2011.2', 'diablo'),
+    ('2012.1', 'essex'),
+    ('2012.2', 'folsom'),
+    ('2013.1', 'grizzly'),
+    ('2013.2', 'havana'),
+    ('2014.1', 'icehouse'),
+])
+
+# The ugly duckling
+SWIFT_CODENAMES = OrderedDict([
+    ('1.4.3', 'diablo'),
+    ('1.4.8', 'essex'),
+    ('1.7.4', 'folsom'),
+    ('1.8.0', 'grizzly'),
+    ('1.7.7', 'grizzly'),
+    ('1.7.6', 'grizzly'),
+    ('1.10.0', 'havana'),
+    ('1.9.1', 'havana'),
+    ('1.9.0', 'havana'),
+    ('1.13.0', 'icehouse'),
+    ('1.12.0', 'icehouse'),
+    ('1.11.0', 'icehouse'),
+])
+
+DEFAULT_LOOPBACK_SIZE = '5G'
+
+
+def error_out(msg):
+    juju_log("FATAL ERROR: %s" % msg, level='ERROR')
+    sys.exit(1)
+
+
+def get_os_codename_install_source(src):
+    '''Derive OpenStack release codename from a given installation source.'''
+    ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
+    rel = ''
+    if src in ['distro', 'distro-proposed']:
+        try:
+            rel = UBUNTU_OPENSTACK_RELEASE[ubuntu_rel]
+        except KeyError:
+            e = 'Could not derive openstack release for '\
+                'this Ubuntu release: %s' % ubuntu_rel
+            error_out(e)
+        return rel
+
+    if src.startswith('cloud:'):
+        ca_rel = src.split(':')[1]
+        ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0]
+        return ca_rel
+
+    # Best guess match based on deb string provided
+    if src.startswith('deb') or src.startswith('ppa'):
+        for k, v in OPENSTACK_CODENAMES.iteritems():
+            if v in src:
+                return v
+
+
+def get_os_version_install_source(src):
+    codename = get_os_codename_install_source(src)
+    return get_os_version_codename(codename)
+
+
+def get_os_codename_version(vers):
+    '''Determine OpenStack codename from version number.'''
+    try:
+        return OPENSTACK_CODENAMES[vers]
+    except KeyError:
+        e = 'Could not determine OpenStack codename for version %s' % vers
+        error_out(e)
+
+
+def get_os_version_codename(codename):
+    '''Determine OpenStack version number from codename.'''
+    for k, v in OPENSTACK_CODENAMES.iteritems():
+        if v == codename:
+            return k
+    e = 'Could not derive OpenStack version for '\
+        'codename: %s' % codename
+    error_out(e)
+
+
+def get_os_codename_package(package, fatal=True):
+    '''Derive OpenStack release codename from an installed package.'''
+    apt.init()
+    cache = apt.Cache()
+
+    try:
+        pkg = cache[package]
+    except:
+        if not fatal:
+            return None
+        # the package is unknown to the current apt cache.
+        e = 'Could not determine version of package with no installation '\
+            'candidate: %s' % package
+        error_out(e)
+
+    if not pkg.current_ver:
+        if not fatal:
+            return None
+        # package is known, but no version is currently installed.
+        e = 'Could not determine version of uninstalled package: %s' % package
+        error_out(e)
+
+    vers = apt.upstream_version(pkg.current_ver.ver_str)
+
+    try:
+        if 'swift' in pkg.name:
+            swift_vers = vers[:5]
+            if swift_vers not in SWIFT_CODENAMES:
+                # Deal with 1.10.0 upward
+                swift_vers = vers[:6]
+            return SWIFT_CODENAMES[swift_vers]
+        else:
+            vers = vers[:6]
+            return OPENSTACK_CODENAMES[vers]
+    except KeyError:
+        e = 'Could not determine OpenStack codename for version %s' % vers
+        error_out(e)
+
+
+def get_os_version_package(pkg, fatal=True):
+    '''Derive OpenStack version number from an installed package.'''
+    codename = get_os_codename_package(pkg, fatal=fatal)
+
+    if not codename:
+        return None
+
+    if 'swift' in pkg:
+        vers_map = SWIFT_CODENAMES
+    else:
+        vers_map = OPENSTACK_CODENAMES
+
+    for version, cname in vers_map.iteritems():
+        if cname == codename:
+            return version
+    #e = "Could not determine OpenStack version for package: %s" % pkg
+    #error_out(e)
+
+
+os_rel = None
+
+
+def os_release(package, base='essex'):
+    '''
+    Returns OpenStack release codename from a cached global.
+    If the codename can not be determined from either an installed package or
+    the installation source, the earliest release supported by the charm should
+    be returned.
+    '''
+    global os_rel
+    if os_rel:
+        return os_rel
+    os_rel = (get_os_codename_package(package, fatal=False) or
+              get_os_codename_install_source(config('openstack-origin')) or
+              base)
+    return os_rel
+
+
+def import_key(keyid):
+    cmd = "apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 " \
+          "--recv-keys %s" % keyid
+    try:
+        subprocess.check_call(cmd.split(' '))
+    except subprocess.CalledProcessError:
+        error_out("Error importing repo key %s" % keyid)
+
+
+def configure_installation_source(rel):
+    '''Configure apt installation source.'''
+    if rel == 'distro':
+        return
+    elif rel == 'distro-proposed':
+        ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
+        with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f:
+            f.write(DISTRO_PROPOSED % ubuntu_rel)
+    elif rel[:4] == "ppa:":
+        src = rel
+        subprocess.check_call(["add-apt-repository", "-y", src])
+    elif rel[:3] == "deb":
+        l = len(rel.split('|'))
+        if l == 2:
+            src, key = rel.split('|')
+            juju_log("Importing PPA key from keyserver for %s" % src)
+            import_key(key)
+        elif l == 1:
+            src = rel
+        with open('/etc/apt/sources.list.d/juju_deb.list', 'w') as f:
+            f.write(src)
+    elif rel[:6] == 'cloud:':
+        ubuntu_rel = lsb_release()['DISTRIB_CODENAME']
+        rel = rel.split(':')[1]
+        u_rel = rel.split('-')[0]
+        ca_rel = rel.split('-')[1]
+
+        if u_rel != ubuntu_rel:
+            e = 'Cannot install from Cloud Archive pocket %s on this Ubuntu '\
+                'version (%s)' % (ca_rel, ubuntu_rel)
+            error_out(e)
+
+        if 'staging' in ca_rel:
+            # staging is just a regular PPA.
+            os_rel = ca_rel.split('/')[0]
+            ppa = 'ppa:ubuntu-cloud-archive/%s-staging' % os_rel
+            cmd = 'add-apt-repository -y %s' % ppa
+            subprocess.check_call(cmd.split(' '))
+            return
+
+        # map charm config options to actual archive pockets.
+        pockets = {
+            'folsom': 'precise-updates/folsom',
+            'folsom/updates': 'precise-updates/folsom',
+            'folsom/proposed': 'precise-proposed/folsom',
+            'grizzly': 'precise-updates/grizzly',
+            'grizzly/updates': 'precise-updates/grizzly',
+            'grizzly/proposed': 'precise-proposed/grizzly',
+            'havana': 'precise-updates/havana',
+            'havana/updates': 'precise-updates/havana',
+            'havana/proposed': 'precise-proposed/havana',
+            'icehouse': 'precise-updates/icehouse',
+            'icehouse/updates': 'precise-updates/icehouse',
+            'icehouse/proposed': 'precise-proposed/icehouse',
+        }
+
+        try:
+            pocket = pockets[ca_rel]
+        except KeyError:
+            e = 'Invalid Cloud Archive release specified: %s' % rel
+            error_out(e)
+
+        src = "deb %s %s main" % (CLOUD_ARCHIVE_URL, pocket)
+        apt_install('ubuntu-cloud-keyring', fatal=True)
+
+        with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as f:
+            f.write(src)
+    else:
+        error_out("Invalid openstack-release specified: %s" % rel)
+
+
+def save_script_rc(script_path="scripts/scriptrc", **env_vars):
+    """
+    Write an rc file in the charm-delivered directory containing
+    exported environment variables provided by env_vars. Any charm scripts run
+    outside the juju hook environment can source this scriptrc to obtain
+    updated config information necessary to perform health checks or
+    service changes.
+    """
+    juju_rc_path = "%s/%s" % (charm_dir(), script_path)
+    if not os.path.exists(os.path.dirname(juju_rc_path)):
+        os.mkdir(os.path.dirname(juju_rc_path))
+    with open(juju_rc_path, 'wb') as rc_script:
+        rc_script.write(
+            "#!/bin/bash\n")
+        [rc_script.write('export %s=%s\n' % (u, p))
+         for u, p in env_vars.iteritems() if u != "script_path"]
+
+
+def openstack_upgrade_available(package):
+    """
+    Determines if an OpenStack upgrade is available from installation
+    source, based on version of installed package.
+
+    :param package: str: Name of installed package.
+
+    :returns: bool:    : Returns True if configured installation source offers
+                         a newer version of package.
+
+    """
+
+    src = config('openstack-origin')
+    cur_vers = get_os_version_package(package)
+    available_vers = get_os_version_install_source(src)
+    apt.init()
+    return apt.version_compare(available_vers, cur_vers) == 1
+
+
+def ensure_block_device(block_device):
+    '''
+    Confirm block_device, create as loopback if necessary.
+
+    :param block_device: str: Full path of block device to ensure.
+
+    :returns: str: Full path of ensured block device.
+    '''
+    _none = ['None', 'none', None]
+    if (block_device in _none):
+        error_out('prepare_storage(): Missing required input: '
+                  'block_device=%s.' % block_device, level=ERROR)
+
+    if block_device.startswith('/dev/'):
+        bdev = block_device
+    elif block_device.startswith('/'):
+        _bd = block_device.split('|')
+        if len(_bd) == 2:
+            bdev, size = _bd
+        else:
+            bdev = block_device
+            size = DEFAULT_LOOPBACK_SIZE
+        bdev = ensure_loopback_device(bdev, size)
+    else:
+        bdev = '/dev/%s' % block_device
+
+    if not is_block_device(bdev):
+        error_out('Failed to locate valid block device at %s' % bdev,
+                  level=ERROR)
+
+    return bdev
+
+
+def clean_storage(block_device):
+    '''
+    Ensures a block device is clean.  That is:
+        - unmounted
+        - any lvm volume groups are deactivated
+        - any lvm physical device signatures removed
+        - partition table wiped
+
+    :param block_device: str: Full path to block device to clean.
+    '''
+    for mp, d in mounts():
+        if d == block_device:
+            juju_log('clean_storage(): %s is mounted @ %s, unmounting.' %
+                     (d, mp), level=INFO)
+            umount(mp, persist=True)
+
+    if is_lvm_physical_volume(block_device):
+        deactivate_lvm_volume_group(block_device)
+        remove_lvm_physical_volume(block_device)
+    else:
+        zap_disk(block_device)
+
+
+def is_ip(address):
+    """
+    Returns True if address is a valid IP address.
+    """
+    try:
+        # Test to see if already an IPv4 address
+        socket.inet_aton(address)
+        return True
+    except socket.error:
+        return False
+
+
+def ns_query(address):
+    try:
+        import dns.resolver
+    except ImportError:
+        apt_install('python-dnspython')
+        import dns.resolver
+
+    if isinstance(address, dns.name.Name):
+        rtype = 'PTR'
+    elif isinstance(address, basestring):
+        rtype = 'A'
+
+    answers = dns.resolver.query(address, rtype)
+    if answers:
+        return str(answers[0])
+    return None
+
+
+def get_host_ip(hostname):
+    """
+    Resolves the IP for a given hostname, or returns
+    the input if it is already an IP.
+    """
+    if is_ip(hostname):
+        return hostname
+
+    return ns_query(hostname)
+
+
+def get_hostname(address, fqdn=True):
+    """
+    Resolves hostname for given IP, or returns the input
+    if it is already a hostname.
+    """
+    if is_ip(address):
+        try:
+            import dns.reversename
+        except ImportError:
+            apt_install('python-dnspython')
+            import dns.reversename
+
+        rev = dns.reversename.from_address(address)
+        result = ns_query(rev)
+        if not result:
+            return None
+    else:
+        result = address
+
+    if fqdn:
+        # strip trailing .
+        if result.endswith('.'):
+            return result[:-1]
+        else:
+            return result
+    else:
+        return result.split('.')[0]

=== added directory 'hooks/charmhelpers/contrib/storage'
=== added file 'hooks/charmhelpers/contrib/storage/__init__.py'
=== added directory 'hooks/charmhelpers/contrib/storage/linux'
=== added file 'hooks/charmhelpers/contrib/storage/linux/__init__.py'
=== added file 'hooks/charmhelpers/contrib/storage/linux/ceph.py'
--- hooks/charmhelpers/contrib/storage/linux/ceph.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/storage/linux/ceph.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,387 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# This file is sourced from lp:openstack-charm-helpers
+#
+# Authors:
+#  James Page <[e-mail]>
+#  Adam Gandelman <[e-mail]>
+#
+
+import os
+import shutil
+import json
+import time
+
+from subprocess import (
+    check_call,
+    check_output,
+    CalledProcessError
+)
+
+from charmhelpers.core.hookenv import (
+    relation_get,
+    relation_ids,
+    related_units,
+    log,
+    INFO,
+    WARNING,
+    ERROR
+)
+
+from charmhelpers.core.host import (
+    mount,
+    mounts,
+    service_start,
+    service_stop,
+    service_running,
+    umount,
+)
+
+from charmhelpers.fetch import (
+    apt_install,
+)
+
+KEYRING = '/etc/ceph/ceph.client.{}.keyring'
+KEYFILE = '/etc/ceph/ceph.client.{}.key'
+
+CEPH_CONF = """[global]
+ auth supported = {auth}
+ keyring = {keyring}
+ mon host = {mon_hosts}
+ log to syslog = {use_syslog}
+ err to syslog = {use_syslog}
+ clog to syslog = {use_syslog}
+"""
+
+
+def install():
+    ''' Basic Ceph client installation '''
+    ceph_dir = "/etc/ceph"
+    if not os.path.exists(ceph_dir):
+        os.mkdir(ceph_dir)
+    apt_install('ceph-common', fatal=True)
+
+
+def rbd_exists(service, pool, rbd_img):
+    ''' Check to see if a RADOS block device exists '''
+    try:
+        out = check_output(['rbd', 'list', '--id', service,
+                            '--pool', pool])
+    except CalledProcessError:
+        return False
+    else:
+        return rbd_img in out
+
+
+def create_rbd_image(service, pool, image, sizemb):
+    ''' Create a new RADOS block device '''
+    cmd = [
+        'rbd',
+        'create',
+        image,
+        '--size',
+        str(sizemb),
+        '--id',
+        service,
+        '--pool',
+        pool
+    ]
+    check_call(cmd)
+
+
+def pool_exists(service, name):
+    ''' Check to see if a RADOS pool already exists '''
+    try:
+        out = check_output(['rados', '--id', service, 'lspools'])
+    except CalledProcessError:
+        return False
+    else:
+        return name in out
+
+
+def get_osds(service):
+    '''
+    Return a list of all Ceph Object Storage Daemons
+    currently in the cluster
+    '''
+    version = ceph_version()
+    if version and version >= '0.56':
+        return json.loads(check_output(['ceph', '--id', service,
+                                        'osd', 'ls', '--format=json']))
+    else:
+        return None
+
+
+def create_pool(service, name, replicas=2):
+    ''' Create a new RADOS pool '''
+    if pool_exists(service, name):
+        log("Ceph pool {} already exists, skipping creation".format(name),
+            level=WARNING)
+        return
+    # Calculate the number of placement groups based
+    # on upstream recommended best practices.
+    osds = get_osds(service)
+    if osds:
+        pgnum = (len(osds) * 100 / replicas)
+    else:
+        # NOTE(james-page): Default to 200 for older ceph versions
+        # which don't support OSD query from cli
+        pgnum = 200
+    cmd = [
+        'ceph', '--id', service,
+        'osd', 'pool', 'create',
+        name, str(pgnum)
+    ]
+    check_call(cmd)
+    cmd = [
+        'ceph', '--id', service,
+        'osd', 'pool', 'set', name,
+        'size', str(replicas)
+    ]
+    check_call(cmd)
+
+
+def delete_pool(service, name):
+    ''' Delete a RADOS pool from ceph '''
+    cmd = [
+        'ceph', '--id', service,
+        'osd', 'pool', 'delete',
+        name, '--yes-i-really-really-mean-it'
+    ]
+    check_call(cmd)
+
+
+def _keyfile_path(service):
+    return KEYFILE.format(service)
+
+
+def _keyring_path(service):
+    return KEYRING.format(service)
+
+
+def create_keyring(service, key):
+    ''' Create a new Ceph keyring containing key'''
+    keyring = _keyring_path(service)
+    if os.path.exists(keyring):
+        log('ceph: Keyring exists at %s.' % keyring, level=WARNING)
+        return
+    cmd = [
+        'ceph-authtool',
+        keyring,
+        '--create-keyring',
+        '--name=client.{}'.format(service),
+        '--add-key={}'.format(key)
+    ]
+    check_call(cmd)
+    log('ceph: Created new ring at %s.' % keyring, level=INFO)
+
+
+def create_key_file(service, key):
+    ''' Create a file containing key '''
+    keyfile = _keyfile_path(service)
+    if os.path.exists(keyfile):
+        log('ceph: Keyfile exists at %s.' % keyfile, level=WARNING)
+        return
+    with open(keyfile, 'w') as fd:
+        fd.write(key)
+    log('ceph: Created new keyfile at %s.' % keyfile, level=INFO)
+
+
+def get_ceph_nodes():
+    ''' Query named relation 'ceph' to detemine current nodes '''
+    hosts = []
+    for r_id in relation_ids('ceph'):
+        for unit in related_units(r_id):
+            hosts.append(relation_get('private-address', unit=unit, rid=r_id))
+    return hosts
+
+
+def configure(service, key, auth, use_syslog):
+    ''' Perform basic configuration of Ceph '''
+    create_keyring(service, key)
+    create_key_file(service, key)
+    hosts = get_ceph_nodes()
+    with open('/etc/ceph/ceph.conf', 'w') as ceph_conf:
+        ceph_conf.write(CEPH_CONF.format(auth=auth,
+                                         keyring=_keyring_path(service),
+                                         mon_hosts=",".join(map(str, hosts)),
+                                         use_syslog=use_syslog))
+    modprobe('rbd')
+
+
+def image_mapped(name):
+    ''' Determine whether a RADOS block device is mapped locally '''
+    try:
+        out = check_output(['rbd', 'showmapped'])
+    except CalledProcessError:
+        return False
+    else:
+        return name in out
+
+
+def map_block_storage(service, pool, image):
+    ''' Map a RADOS block device for local use '''
+    cmd = [
+        'rbd',
+        'map',
+        '{}/{}'.format(pool, image),
+        '--user',
+        service,
+        '--secret',
+        _keyfile_path(service),
+    ]
+    check_call(cmd)
+
+
+def filesystem_mounted(fs):
+    ''' Determine whether a filesytems is already mounted '''
+    return fs in [f for f, m in mounts()]
+
+
+def make_filesystem(blk_device, fstype='ext4', timeout=10):
+    ''' Make a new filesystem on the specified block device '''
+    count = 0
+    e_noent = os.errno.ENOENT
+    while not os.path.exists(blk_device):
+        if count >= timeout:
+            log('ceph: gave up waiting on block device %s' % blk_device,
+                level=ERROR)
+            raise IOError(e_noent, os.strerror(e_noent), blk_device)
+        log('ceph: waiting for block device %s to appear' % blk_device,
+            level=INFO)
+        count += 1
+        time.sleep(1)
+    else:
+        log('ceph: Formatting block device %s as filesystem %s.' %
+            (blk_device, fstype), level=INFO)
+        check_call(['mkfs', '-t', fstype, blk_device])
+
+
+def place_data_on_block_device(blk_device, data_src_dst):
+    ''' Migrate data in data_src_dst to blk_device and then remount '''
+    # mount block device into /mnt
+    mount(blk_device, '/mnt')
+    # copy data to /mnt
+    copy_files(data_src_dst, '/mnt')
+    # umount block device
+    umount('/mnt')
+    # Grab user/group ID's from original source
+    _dir = os.stat(data_src_dst)
+    uid = _dir.st_uid
+    gid = _dir.st_gid
+    # re-mount where the data should originally be
+    # TODO: persist is currently a NO-OP in core.host
+    mount(blk_device, data_src_dst, persist=True)
+    # ensure original ownership of new mount.
+    os.chown(data_src_dst, uid, gid)
+
+
+# TODO: re-use
+def modprobe(module):
+    ''' Load a kernel module and configure for auto-load on reboot '''
+    log('ceph: Loading kernel module', level=INFO)
+    cmd = ['modprobe', module]
+    check_call(cmd)
+    with open('/etc/modules', 'r+') as modules:
+        if module not in modules.read():
+            modules.write(module)
+
+
+def copy_files(src, dst, symlinks=False, ignore=None):
+    ''' Copy files from src to dst '''
+    for item in os.listdir(src):
+        s = os.path.join(src, item)
+        d = os.path.join(dst, item)
+        if os.path.isdir(s):
+            shutil.copytree(s, d, symlinks, ignore)
+        else:
+            shutil.copy2(s, d)
+
+
+def ensure_ceph_storage(service, pool, rbd_img, sizemb, mount_point,
+                        blk_device, fstype, system_services=[]):
+    """
+    NOTE: This function must only be called from a single service unit for
+          the same rbd_img otherwise data loss will occur.
+
+    Ensures given pool and RBD image exists, is mapped to a block device,
+    and the device is formatted and mounted at the given mount_point.
+
+    If formatting a device for the first time, data existing at mount_point
+    will be migrated to the RBD device before being re-mounted.
+
+    All services listed in system_services will be stopped prior to data
+    migration and restarted when complete.
+    """
+    # Ensure pool, RBD image, RBD mappings are in place.
+    if not pool_exists(service, pool):
+        log('ceph: Creating new pool {}.'.format(pool))
+        create_pool(service, pool)
+
+    if not rbd_exists(service, pool, rbd_img):
+        log('ceph: Creating RBD image ({}).'.format(rbd_img))
+        create_rbd_image(service, pool, rbd_img, sizemb)
+
+    if not image_mapped(rbd_img):
+        log('ceph: Mapping RBD Image {} as a Block Device.'.format(rbd_img))
+        map_block_storage(service, pool, rbd_img)
+
+    # make file system
+    # TODO: What happens if for whatever reason this is run again and
+    # the data is already in the rbd device and/or is mounted??
+    # When it is mounted already, it will fail to make the fs
+    # XXX: This is really sketchy!  Need to at least add an fstab entry
+    #      otherwise this hook will blow away existing data if its executed
+    #      after a reboot.
+    if not filesystem_mounted(mount_point):
+        make_filesystem(blk_device, fstype)
+
+        for svc in system_services:
+            if service_running(svc):
+                log('ceph: Stopping services {} prior to migrating data.'
+                    .format(svc))
+                service_stop(svc)
+
+        place_data_on_block_device(blk_device, mount_point)
+
+        for svc in system_services:
+            log('ceph: Starting service {} after migrating data.'
+                .format(svc))
+            service_start(svc)
+
+
+def ensure_ceph_keyring(service, user=None, group=None):
+    '''
+    Ensures a ceph keyring is created for a named service
+    and optionally ensures user and group ownership.
+
+    Returns False if no ceph key is available in relation state.
+    '''
+    key = None
+    for rid in relation_ids('ceph'):
+        for unit in related_units(rid):
+            key = relation_get('key', rid=rid, unit=unit)
+            if key:
+                break
+    if not key:
+        return False
+    create_keyring(service=service, key=key)
+    keyring = _keyring_path(service)
+    if user and group:
+        check_call(['chown', '%s.%s' % (user, group), keyring])
+    return True
+
+
+def ceph_version():
+    ''' Retrieve the local version of ceph '''
+    if os.path.exists('/usr/bin/ceph'):
+        cmd = ['ceph', '-v']
+        output = check_output(cmd)
+        output = output.split()
+        if len(output) > 3:
+            return output[2]
+        else:
+            return None
+    else:
+        return None

=== added file 'hooks/charmhelpers/contrib/storage/linux/loopback.py'
--- hooks/charmhelpers/contrib/storage/linux/loopback.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/storage/linux/loopback.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,62 @@
+
+import os
+import re
+
+from subprocess import (
+    check_call,
+    check_output,
+)
+
+
+##################################################
+# loopback device helpers.
+##################################################
+def loopback_devices():
+    '''
+    Parse through 'losetup -a' output to determine currently mapped
+    loopback devices. Output is expected to look like:
+
+        /dev/loop0: [0807]:961814 (/tmp/my.img)
+
+    :returns: dict: a dict mapping {loopback_dev: backing_file}
+    '''
+    loopbacks = {}
+    cmd = ['losetup', '-a']
+    devs = [d.strip().split(' ') for d in
+            check_output(cmd).splitlines() if d != '']
+    for dev, _, f in devs:
+        loopbacks[dev.replace(':', '')] = re.search('\((\S+)\)', f).groups()[0]
+    return loopbacks
+
+
+def create_loopback(file_path):
+    '''
+    Create a loopback device for a given backing file.
+
+    :returns: str: Full path to new loopback device (eg, /dev/loop0)
+    '''
+    file_path = os.path.abspath(file_path)
+    check_call(['losetup', '--find', file_path])
+    for d, f in loopback_devices().iteritems():
+        if f == file_path:
+            return d
+
+
+def ensure_loopback_device(path, size):
+    '''
+    Ensure a loopback device exists for a given backing file path and size.
+    If it a loopback device is not mapped to file, a new one will be created.
+
+    TODO: Confirm size of found loopback device.
+
+    :returns: str: Full path to the ensured loopback device (eg, /dev/loop0)
+    '''
+    for d, f in loopback_devices().iteritems():
+        if f == path:
+            return d
+
+    if not os.path.exists(path):
+        cmd = ['truncate', '--size', size, path]
+        check_call(cmd)
+
+    return create_loopback(path)

=== added file 'hooks/charmhelpers/contrib/storage/linux/lvm.py'
--- hooks/charmhelpers/contrib/storage/linux/lvm.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/storage/linux/lvm.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,88 @@
+from subprocess import (
+    CalledProcessError,
+    check_call,
+    check_output,
+    Popen,
+    PIPE,
+)
+
+
+##################################################
+# LVM helpers.
+##################################################
+def deactivate_lvm_volume_group(block_device):
+    '''
+    Deactivate any volume gruop associated with an LVM physical volume.
+
+    :param block_device: str: Full path to LVM physical volume
+    '''
+    vg = list_lvm_volume_group(block_device)
+    if vg:
+        cmd = ['vgchange', '-an', vg]
+        check_call(cmd)
+
+
+def is_lvm_physical_volume(block_device):
+    '''
+    Determine whether a block device is initialized as an LVM PV.
+
+    :param block_device: str: Full path of block device to inspect.
+
+    :returns: boolean: True if block device is a PV, False if not.
+    '''
+    try:
+        check_output(['pvdisplay', block_device])
+        return True
+    except CalledProcessError:
+        return False
+
+
+def remove_lvm_physical_volume(block_device):
+    '''
+    Remove LVM PV signatures from a given block device.
+
+    :param block_device: str: Full path of block device to scrub.
+    '''
+    p = Popen(['pvremove', '-ff', block_device],
+              stdin=PIPE)
+    p.communicate(input='y\n')
+
+
+def list_lvm_volume_group(block_device):
+    '''
+    List LVM volume group associated with a given block device.
+
+    Assumes block device is a valid LVM PV.
+
+    :param block_device: str: Full path of block device to inspect.
+
+    :returns: str: Name of volume group associated with block device or None
+    '''
+    vg = None
+    pvd = check_output(['pvdisplay', block_device]).splitlines()
+    for l in pvd:
+        if l.strip().startswith('VG Name'):
+            vg = ' '.join(l.split()).split(' ').pop()
+    return vg
+
+
+def create_lvm_physical_volume(block_device):
+    '''
+    Initialize a block device as an LVM physical volume.
+
+    :param block_device: str: Full path of block device to initialize.
+
+    '''
+    check_call(['pvcreate', block_device])
+
+
+def create_lvm_volume_group(volume_group, block_device):
+    '''
+    Create an LVM volume group backed by a given block device.
+
+    Assumes block device has already been initialized as an LVM PV.
+
+    :param volume_group: str: Name of volume group to create.
+    :block_device: str: Full path of PV-initialized block device.
+    '''
+    check_call(['vgcreate', volume_group, block_device])

=== added file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,26 @@
+from os import stat
+from stat import S_ISBLK
+
+from subprocess import (
+    check_call
+)
+
+
+def is_block_device(path):
+    '''
+    Confirm device at path is a valid block device node.
+
+    :returns: boolean: True if path is a block device, False if not.
+    '''
+    return S_ISBLK(stat(path).st_mode)
+
+
+def zap_disk(block_device):
+    '''
+    Clear a block device of partition table. Relies on sgdisk, which is
+    installed as pat of the 'gdisk' package in Ubuntu.
+
+    :param block_device: str: Full path of block device to clean.
+    '''
+    check_call(['sgdisk', '--zap-all', '--clear',
+                '--mbrtogpt', block_device])

=== added directory 'hooks/charmhelpers/fetch'
=== added file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,308 @@
+import importlib
+from yaml import safe_load
+from charmhelpers.core.host import (
+    lsb_release
+)
+from urlparse import (
+    urlparse,
+    urlunparse,
+)
+import subprocess
+from charmhelpers.core.hookenv import (
+    config,
+    log,
+)
+import apt_pkg
+import os
+
+CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
+deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
+"""
+PROPOSED_POCKET = """# Proposed
+deb http://archive.ubuntu.com/ubuntu {}-proposed main universe multiverse restricted
+"""
+CLOUD_ARCHIVE_POCKETS = {
+    # Folsom
+    'folsom': 'precise-updates/folsom',
+    'precise-folsom': 'precise-updates/folsom',
+    'precise-folsom/updates': 'precise-updates/folsom',
+    'precise-updates/folsom': 'precise-updates/folsom',
+    'folsom/proposed': 'precise-proposed/folsom',
+    'precise-folsom/proposed': 'precise-proposed/folsom',
+    'precise-proposed/folsom': 'precise-proposed/folsom',
+    # Grizzly
+    'grizzly': 'precise-updates/grizzly',
+    'precise-grizzly': 'precise-updates/grizzly',
+    'precise-grizzly/updates': 'precise-updates/grizzly',
+    'precise-updates/grizzly': 'precise-updates/grizzly',
+    'grizzly/proposed': 'precise-proposed/grizzly',
+    'precise-grizzly/proposed': 'precise-proposed/grizzly',
+    'precise-proposed/grizzly': 'precise-proposed/grizzly',
+    # Havana
+    'havana': 'precise-updates/havana',
+    'precise-havana': 'precise-updates/havana',
+    'precise-havana/updates': 'precise-updates/havana',
+    'precise-updates/havana': 'precise-updates/havana',
+    'havana/proposed': 'precise-proposed/havana',
+    'precise-havana/proposed': 'precise-proposed/havana',
+    'precise-proposed/havana': 'precise-proposed/havana',
+    # Icehouse
+    'icehouse': 'precise-updates/icehouse',
+    'precise-icehouse': 'precise-updates/icehouse',
+    'precise-icehouse/updates': 'precise-updates/icehouse',
+    'precise-updates/icehouse': 'precise-updates/icehouse',
+    'icehouse/proposed': 'precise-proposed/icehouse',
+    'precise-icehouse/proposed': 'precise-proposed/icehouse',
+    'precise-proposed/icehouse': 'precise-proposed/icehouse',
+}
+
+
+def filter_installed_packages(packages):
+    """Returns a list of packages that require installation"""
+    apt_pkg.init()
+    cache = apt_pkg.Cache()
+    _pkgs = []
+    for package in packages:
+        try:
+            p = cache[package]
+            p.current_ver or _pkgs.append(package)
+        except KeyError:
+            log('Package {} has no installation candidate.'.format(package),
+                level='WARNING')
+            _pkgs.append(package)
+    return _pkgs
+
+
+def apt_install(packages, options=None, fatal=False):
+    """Install one or more packages"""
+    if options is None:
+        options = ['--option=Dpkg::Options::=--force-confold']
+
+    cmd = ['apt-get', '--assume-yes']
+    cmd.extend(options)
+    cmd.append('install')
+    if isinstance(packages, basestring):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Installing {} with options: {}".format(packages,
+                                                options))
+    env = os.environ.copy()
+    if 'DEBIAN_FRONTEND' not in env:
+        env['DEBIAN_FRONTEND'] = 'noninteractive'
+
+    if fatal:
+        subprocess.check_call(cmd, env=env)
+    else:
+        subprocess.call(cmd, env=env)
+
+
+def apt_upgrade(options=None, fatal=False, dist=False):
+    """Upgrade all packages"""
+    if options is None:
+        options = ['--option=Dpkg::Options::=--force-confold']
+
+    cmd = ['apt-get', '--assume-yes']
+    cmd.extend(options)
+    if dist:
+        cmd.append('dist-upgrade')
+    else:
+        cmd.append('upgrade')
+    log("Upgrading with options: {}".format(options))
+
+    env = os.environ.copy()
+    if 'DEBIAN_FRONTEND' not in env:
+        env['DEBIAN_FRONTEND'] = 'noninteractive'
+
+    if fatal:
+        subprocess.check_call(cmd, env=env)
+    else:
+        subprocess.call(cmd, env=env)
+
+
+def apt_update(fatal=False):
+    """Update local apt cache"""
+    cmd = ['apt-get', 'update']
+    if fatal:
+        subprocess.check_call(cmd)
+    else:
+        subprocess.call(cmd)
+
+
+def apt_purge(packages, fatal=False):
+    """Purge one or more packages"""
+    cmd = ['apt-get', '--assume-yes', 'purge']
+    if isinstance(packages, basestring):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Purging {}".format(packages))
+    if fatal:
+        subprocess.check_call(cmd)
+    else:
+        subprocess.call(cmd)
+
+
+def apt_hold(packages, fatal=False):
+    """Hold one or more packages"""
+    cmd = ['apt-mark', 'hold']
+    if isinstance(packages, basestring):
+        cmd.append(packages)
+    else:
+        cmd.extend(packages)
+    log("Holding {}".format(packages))
+    if fatal:
+        subprocess.check_call(cmd)
+    else:
+        subprocess.call(cmd)
+
+
+def add_source(source, key=None):
+    if source is None:
+        log('Source is not present. Skipping')
+        return
+
+    if (source.startswith('ppa:') or
+        source.startswith('http') or
+        source.startswith('deb ') or
+            source.startswith('cloud-archive:')):
+        subprocess.check_call(['add-apt-repository', '--yes', source])
+    elif source.startswith('cloud:'):
+        apt_install(filter_installed_packages(['ubuntu-cloud-keyring']),
+                    fatal=True)
+        pocket = source.split(':')[-1]
+        if pocket not in CLOUD_ARCHIVE_POCKETS:
+            raise SourceConfigError(
+                'Unsupported cloud: source option %s' %
+                pocket)
+        actual_pocket = CLOUD_ARCHIVE_POCKETS[pocket]
+        with open('/etc/apt/sources.list.d/cloud-archive.list', 'w') as apt:
+            apt.write(CLOUD_ARCHIVE.format(actual_pocket))
+    elif source == 'proposed':
+        release = lsb_release()['DISTRIB_CODENAME']
+        with open('/etc/apt/sources.list.d/proposed.list', 'w') as apt:
+            apt.write(PROPOSED_POCKET.format(release))
+    if key:
+        subprocess.check_call(['apt-key', 'adv', '--keyserver',
+                               'keyserver.ubuntu.com', '--recv',
+                               key])
+
+
+class SourceConfigError(Exception):
+    pass
+
+
+def configure_sources(update=False,
+                      sources_var='install_sources',
+                      keys_var='install_keys'):
+    """
+    Configure multiple sources from charm configuration
+
+    Example config:
+        install_sources:
+          - "ppa:foo"
+          - "http://example.com/repo precise main"
+        install_keys:
+          - null
+          - "a1b2c3d4"
+
+    Note that 'null' (a.k.a. None) should not be quoted.
+    """
+    sources = safe_load(config(sources_var))
+    keys = config(keys_var)
+    if keys is not None:
+        keys = safe_load(keys)
+    if isinstance(sources, basestring) and (
+            keys is None or isinstance(keys, basestring)):
+        add_source(sources, keys)
+    else:
+        if not len(sources) == len(keys):
+            msg = 'Install sources and keys lists are different lengths'
+            raise SourceConfigError(msg)
+        for src_num in range(len(sources)):
+            add_source(sources[src_num], keys[src_num])
+    if update:
+        apt_update(fatal=True)
+
+# The order of this list is very important. Handlers should be listed in from
+# least- to most-specific URL matching.
+FETCH_HANDLERS = (
+    'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
+    'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
+)
+
+
+class UnhandledSource(Exception):
+    pass
+
+
+def install_remote(source):
+    """
+    Install a file tree from a remote source
+
+    The specified source should be a url of the form:
+        scheme://[host]/path[#[option=value][&...]]
+
+    Schemes supported are based on this modules submodules
+    Options supported are submodule-specific"""
+    # We ONLY check for True here because can_handle may return a string
+    # explaining why it can't handle a given source.
+    handlers = [h for h in plugins() if h.can_handle(source) is True]
+    installed_to = None
+    for handler in handlers:
+        try:
+            installed_to = handler.install(source)
+        except UnhandledSource:
+            pass
+    if not installed_to:
+        raise UnhandledSource("No handler found for source {}".format(source))
+    return installed_to
+
+
+def install_from_config(config_var_name):
+    charm_config = config()
+    source = charm_config[config_var_name]
+    return install_remote(source)
+
+
+class BaseFetchHandler(object):
+
+    """Base class for FetchHandler implementations in fetch plugins"""
+
+    def can_handle(self, source):
+        """Returns True if the source can be handled. Otherwise returns
+        a string explaining why it cannot"""
+        return "Wrong source type"
+
+    def install(self, source):
+        """Try to download and unpack the source. Return the path to the
+        unpacked files or raise UnhandledSource."""
+        raise UnhandledSource("Wrong source type {}".format(source))
+
+    def parse_url(self, url):
+        return urlparse(url)
+
+    def base_url(self, url):
+        """Return url without querystring or fragment"""
+        parts = list(self.parse_url(url))
+        parts[4:] = ['' for i in parts[4:]]
+        return urlunparse(parts)
+
+
+def plugins(fetch_handlers=None):
+    if not fetch_handlers:
+        fetch_handlers = FETCH_HANDLERS
+    plugin_list = []
+    for handler_name in fetch_handlers:
+        package, classname = handler_name.rsplit('.', 1)
+        try:
+            handler_class = getattr(
+                importlib.import_module(package),
+                classname)
+            plugin_list.append(handler_class())
+        except (ImportError, AttributeError):
+            # Skip missing plugins so that they can be ommitted from
+            # installation if desired
+            log("FetchHandler {} not found, skipping plugin".format(
+                handler_name))
+    return plugin_list

=== added file 'hooks/charmhelpers/fetch/archiveurl.py'
--- hooks/charmhelpers/fetch/archiveurl.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/fetch/archiveurl.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,63 @@
+import os
+import urllib2
+import urlparse
+
+from charmhelpers.fetch import (
+    BaseFetchHandler,
+    UnhandledSource
+)
+from charmhelpers.payload.archive import (
+    get_archive_handler,
+    extract,
+)
+from charmhelpers.core.host import mkdir
+
+
+class ArchiveUrlFetchHandler(BaseFetchHandler):
+    """Handler for archives via generic URLs"""
+    def can_handle(self, source):
+        url_parts = self.parse_url(source)
+        if url_parts.scheme not in ('http', 'https', 'ftp', 'file'):
+            return "Wrong source type"
+        if get_archive_handler(self.base_url(source)):
+            return True
+        return False
+
+    def download(self, source, dest):
+        # propogate all exceptions
+        # URLError, OSError, etc
+        proto, netloc, path, params, query, fragment = urlparse.urlparse(source)
+        if proto in ('http', 'https'):
+            auth, barehost = urllib2.splituser(netloc)
+            if auth is not None:
+                source = urlparse.urlunparse((proto, barehost, path, params, query, fragment))
+                username, password = urllib2.splitpasswd(auth)
+                passman = urllib2.HTTPPasswordMgrWithDefaultRealm()
+                # Realm is set to None in add_password to force the username and password
+                # to be used whatever the realm
+                passman.add_password(None, source, username, password)
+                authhandler = urllib2.HTTPBasicAuthHandler(passman)
+                opener = urllib2.build_opener(authhandler)
+                urllib2.install_opener(opener)
+        response = urllib2.urlopen(source)
+        try:
+            with open(dest, 'w') as dest_file:
+                dest_file.write(response.read())
+        except Exception as e:
+            if os.path.isfile(dest):
+                os.unlink(dest)
+            raise e
+
+    def install(self, source):
+        url_parts = self.parse_url(source)
+        dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')
+        if not os.path.exists(dest_dir):
+            mkdir(dest_dir, perms=0755)
+        dld_file = os.path.join(dest_dir, os.path.basename(url_parts.path))
+        try:
+            self.download(source, dld_file)
+        except urllib2.URLError as e:
+            raise UnhandledSource(e.reason)
+        except OSError as e:
+            raise UnhandledSource(e.strerror)
+        return extract(dld_file)

=== added file 'hooks/charmhelpers/fetch/bzrurl.py'
--- hooks/charmhelpers/fetch/bzrurl.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/fetch/bzrurl.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,43 @@
+import os
+from charmhelpers.fetch import (
+    BaseFetchHandler,
+    UnhandledSource
+)
+from charmhelpers.core.host import mkdir
+from bzrlib.branch import Branch
+
+
+class BzrUrlFetchHandler(BaseFetchHandler):
+    """Handler for bazaar branches via generic and lp URLs"""
+    def can_handle(self, source):
+        url_parts = self.parse_url(source)
+        if url_parts.scheme not in ('bzr+ssh', 'lp'):
+            return False
+        else:
+            return True
+
+    def branch(self, source, dest):
+        url_parts = self.parse_url(source)
+        # If we use lp:branchname scheme we need to load plugins
+        if not self.can_handle(source):
+            raise UnhandledSource("Cannot handle {}".format(source))
+        if url_parts.scheme == "lp":
+            from bzrlib.plugin import load_plugins
+            load_plugins()
+        try:
+            remote_branch = Branch.open(source)
+            remote_branch.bzrdir.sprout(dest).open_branch()
+        except Exception as e:
+            raise e
+
+    def install(self, source):
+        url_parts = self.parse_url(source)
+        branch_name = url_parts.path.strip("/").split("/")[-1]
+        dest_dir = os.path.join(os.environ.get('CHARM_DIR'), "fetched", branch_name)
+        if not os.path.exists(dest_dir):
+            mkdir(dest_dir, perms=0755)
+        try:
+            self.branch(source, dest_dir)
+        except OSError as e:
+            raise UnhandledSource(e.strerror)
+        return dest_dir

=== added directory 'hooks/charmhelpers/payload'
=== added file 'hooks/charmhelpers/payload/__init__.py'
--- hooks/charmhelpers/payload/__init__.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/payload/__init__.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,1 @@
+"Tools for working with files injected into a charm just before deployment."

=== added file 'hooks/charmhelpers/payload/execd.py'
--- hooks/charmhelpers/payload/execd.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/payload/execd.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import subprocess
+from charmhelpers.core import hookenv
+
+
+def default_execd_dir():
+    return os.path.join(os.environ['CHARM_DIR'], 'exec.d')
+
+
+def execd_module_paths(execd_dir=None):
+    """Generate a list of full paths to modules within execd_dir."""
+    if not execd_dir:
+        execd_dir = default_execd_dir()
+
+    if not os.path.exists(execd_dir):
+        return
+
+    for subpath in os.listdir(execd_dir):
+        module = os.path.join(execd_dir, subpath)
+        if os.path.isdir(module):
+            yield module
+
+
+def execd_submodule_paths(command, execd_dir=None):
+    """Generate a list of full paths to the specified command within exec_dir.
+    """
+    for module_path in execd_module_paths(execd_dir):
+        path = os.path.join(module_path, command)
+        if os.access(path, os.X_OK) and os.path.isfile(path):
+            yield path
+
+
+def execd_run(command, execd_dir=None, die_on_error=False, stderr=None):
+    """Run command for each module within execd_dir which defines it."""
+    for submodule_path in execd_submodule_paths(command, execd_dir):
+        try:
+            subprocess.check_call(submodule_path, shell=True, stderr=stderr)
+        except subprocess.CalledProcessError as e:
+            hookenv.log("Error ({}) running  {}. Output: {}".format(
+                e.returncode, e.cmd, e.output))
+            if die_on_error:
+                sys.exit(e.returncode)
+
+
+def execd_preinstall(execd_dir=None):
+    """Run charm-pre-install for each module within execd_dir."""
+    execd_run('charm-pre-install', execd_dir=execd_dir)

=== added file 'hooks/config.py'
--- hooks/config.py	1970-01-01 00:00:00 +0000
+++ hooks/config.py	2014-05-12 11:14:56 +0000
@@ -0,0 +1,13 @@
+import os
+
+__all__ = ['CF_DIR', 'LOGGREGATOR_JOB_NAME', 'LOGGREGATOR_CONFIG_PATH',
+           'LOGGREGATOR_DIR', 'LOGGREGATOR_CONFIG_DIR', 'LOGGREGATOR_PACKAGES']
+
+LOGGREGATOR_PACKAGES = ['python-jinja2']
+
+LOGGREGATOR_JOB_NAME = "loggregator"
+CF_DIR = '/var/lib/cloudfoundry'
+LOGGREGATOR_DIR = os.path.join(CF_DIR, 'cfloggregator')
+LOGGREGATOR_CONFIG_DIR = os.path.join(LOGGREGATOR_DIR, 'config')
+LOGGREGATOR_CONFIG_PATH = os.path.join(LOGGREGATOR_CONFIG_DIR,
+                                       'loggregator.json')

=== modified file 'hooks/hooks.py'
--- hooks/hooks.py	2014-04-17 12:27:45 +0000
+++ hooks/hooks.py	2014-05-12 11:14:56 +0000
@@ -1,88 +1,75 @@
 #!/usr/bin/env python
 
 import os
-import subprocess
+
 import sys
-
-from charmhelpers.core import hookenv, host
-from utils import update_log_config
+from charmhelpers.core.hookenv import log
+from charmhelpers.contrib.cloudfoundry import contexts
+from charmhelpers.contrib.cloudfoundry import services
+from charmhelpers.core import hookenv
 from charmhelpers.core.hookenv import unit_get
-
-CHARM_DIR = os.environ['CHARM_DIR']
+from config import *
+
+
 LOGSVC = "loggregator"
 
 hooks = hookenv.Hooks()
 
-loggregator_address = unit_get('private-address').encode('utf-8')
-
-def install_files():
-    subprocess.check_call(
-        ["gunzip", "-k", "loggregator.gz"],
-        cwd=os.path.join(CHARM_DIR, "files"))
-    subprocess.check_call(
-        ["/usr/bin/install", "-m", "0755",
-         os.path.join(CHARM_DIR, "files", "loggregator"),
-         "/usr/bin/loggregator"])
-    subprocess.check_call(
-        ["/usr/bin/install", "-m", "0644",
-         os.path.join(CHARM_DIR, "files", "upstart", "loggregator.conf"),
-         "/etc/init/%s.conf" % LOGSVC])
-
-
[e-mail]()
-def install():
-    host.adduser('vcap')
-    host.mkdir('/var/log/vcap', owner='vcap', group='vcap', perms=0755)
-    host.mkdir('/etc/vcap')
-    install_files()
+service_name, _ = os.environ['JUJU_UNIT_NAME'].split('/')
+fileproperties = {'owner': 'vcap'}
+services.register([
+    {
+        'service': LOGGREGATOR_JOB_NAME,
+        'templates': [{
+            'source': 'loggregator.json',
+            'target': LOGGREGATOR_CONFIG_PATH,
+            'file_properties': fileproperties,
+            'contexts': [
+                contexts.StaticContext({'service_name': service_name}),
+                contexts.ConfigContext(),
+                contexts.NatsContext()
+            ]
+        }]
+    }], os.path.join(hookenv.charm_dir(), 'templates'))
 
 
 @hooks.hook()
 def upgrade_charm():
-    install_files()
+    pass
 
 
 @hooks.hook()
 def start():
-    """We start post nats relation."""
+    pass
 
 
 @hooks.hook()
 def stop():
-    if host.service_running(LOGSVC):
-        host.service_stop(LOGSVC)
-
-
[e-mail]()
+    services.stop_services()
+
+
[e-mail]('config-changed')
 def config_changed():
-    config = hookenv.config()
-    if update_log_config(
-            max_retained_logs=config['max-retained-logs'],
-            shared_secret=config['client-secret']):
-        host.service_restart(LOGSVC)
-
-    # If the secret is updated, propogate it to log emitters.
-    for rel_id in hookenv.relation_ids('logs'):
-        hookenv.relation_set(
-            rel_id, {'shared-secret': config['client-secret'], 'private-address': loggregator_address})
-
-
[e-mail]('logs-relation-changed')
-def logs_relation_joined():
-    config = hookenv.config()
-    hookenv.relation_set(
-        relation_settings={
-            'shared-secret': config['client-secret']})
+    pass
+
+
[e-mail]('loggregator-relation-changed')
+def loggregator_relation_joined():
+    config = hookenv.config()
+    loggregator_address = hookenv.unit_get('private-address').encode('utf-8')
+    hookenv.relation_set(None, {
+        'shared_secret': config['client_secret'],
+        'loggregator_address': loggregator_address})
 
 
 @hooks.hook('nats-relation-changed')
 def nats_relation_changed():
-    nats = hookenv.relation_get()
-    if 'nats_address' not in nats:
-        return
-    if update_log_config(**nats):
-        host.service_restart(LOGSVC)
+    services.reconfigure_services()
 
 
 if __name__ == '__main__':
+    log("Running {} hook".format(sys.argv[0]))
+    if hookenv.relation_id():
+        log("Relation {} with {}".format(
+            hookenv.relation_id(), hookenv.remote_unit()))
     hooks.execute(sys.argv)

=== added file 'hooks/install'
--- hooks/install	1970-01-01 00:00:00 +0000
+++ hooks/install	2014-05-12 11:14:56 +0000
@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+# vim: et ai ts=4 sw=4:
+import os
+import subprocess
+from config import *
+from charmhelpers.core import hookenv, host
+from charmhelpers.contrib.cloudfoundry.common import (
+    prepare_cloudfoundry_environment
+)
+from charmhelpers.contrib.cloudfoundry.upstart_helper import (
+    install_upstart_scripts
+)
+from charmhelpers.contrib.cloudfoundry.install import install
+
+
+def install_charm():
+    prepare_cloudfoundry_environment(hookenv.config(), LOGGREGATOR_PACKAGES)
+    install_upstart_scripts()
+    dirs = [CF_DIR, LOGGREGATOR_DIR, LOGGREGATOR_CONFIG_DIR, '/var/log/vcap']
+
+    for item in dirs:
+        host.mkdir(item, owner='vcap', group='vcap', perms=0775)
+
+    files_dir = os.path.join(hookenv.charm_dir(), "files")
+
+    subprocess.check_call(["gunzip", "-k", "loggregator.gz"], cwd=files_dir)
+
+    install(os.path.join(files_dir, 'loggregator'),
+            "/usr/bin/loggregator",
+            fileprops={'mode': '0755', 'owner': 'vcap'})
+
+
+if __name__ == '__main__':
+    install_charm()

=== removed symlink 'hooks/install'
=== target was u'hooks.py'
=== added symlink 'hooks/loggregator-relation-changed'
=== target is u'hooks.py'
=== removed symlink 'hooks/logs-relation-joined'
=== target was u'hooks.py'
=== removed file 'hooks/utils.py'
--- hooks/utils.py	2014-03-30 18:32:33 +0000
+++ hooks/utils.py	1970-01-01 00:00:00 +0000
@@ -1,53 +0,0 @@
-import json
-import os
-
-CONF_PATH = "/etc/vcap/loggregator.json"
-
-
-def update_log_config(**kw):
-    service, unit_seq = os.environ['JUJU_UNIT_NAME'].split('/')
-    data = {
-        "IncomingPort": 3456,
-        "OutgoingPort": 8080,
-        "MaxRetainedLogMessages": 10,
-        "Syslog": "",
-        "NatsHost": None,
-        "NatsPort": None,
-        "NatsUser": None,
-        "NatsPass": None,
-        "SharedSecret": None,
-
-        # All of the following need documentation.
-        "SkipCertVerify": False,  # For connections to the api server..
-        "Index": int(unit_seq),  # Bosh thingy for leader + for metrics
-        # Future stuff for metrics relation
-        "VarzUser": service,
-        "VarzPass": service,
-        "VarzPort": 8888}
-
-    if os.path.exists(CONF_PATH):
-        with open(CONF_PATH) as fh:
-            data.update(json.loads(fh.read()))
-
-    previous = dict(data)
-
-    # Relation changes
-    if 'nats_address' in kw:
-        data['NatsHost'] = kw['nats_address']
-        data['NatsPort'] = int(kw['nats_port'])
-        data['NatsUser'] = kw['nats_user']
-        data['NatsPass'] = kw['nats_password']
-
-    # Config changes
-    if 'max_retained_logs' in kw:
-        data['MaxRetainedLogMessages'] = kw['max_retained_logs']
-    if 'shared_secret' in kw:
-        data['SharedSecret'] = kw['shared_secret']
-
-    if data != previous:
-        with open(CONF_PATH, 'w') as fh:
-            fh.write(json.dumps(data, indent=2))
-        if data['NatsHost']:
-            return True
-
-    return False

=== modified file 'metadata.yaml'
--- metadata.yaml	2014-05-12 07:19:53 +0000
+++ metadata.yaml	2014-05-12 11:14:56 +0000
@@ -7,9 +7,10 @@
 categories:
   - misc
 provides:
-  logs:
+  loggregator:
     interface: loggregator
 requires:
   nats:
     interface: nats
-
+  # logrouter:
+  #   interface: logrouter
\ No newline at end of file

=== added directory 'templates'
=== added file 'templates/loggregator.json'
--- templates/loggregator.json	1970-01-01 00:00:00 +0000
+++ templates/loggregator.json	2014-05-12 11:14:56 +0000
@@ -0,0 +1,16 @@
+{
+  "Index": 0,
+  "NatsHost": "{{ nats['nats_address'] }}",
+  "VarzPort": 8888,
+  "SkipCertVerify": false,
+  "VarzUser": "{{ service_name }}",
+  "MaxRetainedLogMessages": 20,
+  "OutgoingPort": 8080,
+  "Syslog": "",
+  "VarzPass": "{{ service_name }}",
+  "NatsUser": "{{ nats['nats_user'] }}",
+  "NatsPass": "{{ nats['nats_password'] }}",
+  "IncomingPort": 3456,
+  "NatsPort": {{ nats['nats_port'] }},
+  "SharedSecret": "{{ client_secret }}"
+}
\ No newline at end of file