Merge lp:~johnsca/charms/trusty/cf-loggregator/refactor into lp:~cf-charmers/charms/trusty/cf-loggregator/trunk

[Cory Johns]

Refactored to use refactored charm-helpers

https://codereview.appspot.com/91510047/

[Cory Johns (johnsca)]

Reviewers: mp+219915_code.launchpad.net,

Message:
Please take a look.

Description:
Refactored to use refactored charm-helpers

https://code.launchpad.net/~johnsca/charms/trusty/cf-loggregator/refactor/+merge/219915

(do not edit description out of merge proposal)

Please review this at https://codereview.appspot.com/91510047/

Affected files (+485, -361 lines):
   A [revision details]
   M config.yaml
   M hooks/charmhelpers/contrib/cloudfoundry/common.py
   D hooks/charmhelpers/contrib/cloudfoundry/config_helper.py
   M hooks/charmhelpers/contrib/cloudfoundry/contexts.py
   D hooks/charmhelpers/contrib/cloudfoundry/install.py
   D hooks/charmhelpers/contrib/cloudfoundry/services.py
   D hooks/charmhelpers/contrib/cloudfoundry/upstart_helper.py
   M hooks/charmhelpers/contrib/hahelpers/apache.py
   M hooks/charmhelpers/contrib/openstack/context.py
   M hooks/charmhelpers/contrib/openstack/neutron.py
   M hooks/charmhelpers/contrib/openstack/utils.py
   M hooks/charmhelpers/contrib/storage/linux/utils.py
   M hooks/charmhelpers/core/host.py
   A hooks/charmhelpers/core/services.py
   A hooks/charmhelpers/core/templating.py
   M hooks/charmhelpers/fetch/__init__.py
   M hooks/hooks.py
   M hooks/install
   M templates/loggregator.conf

[Benjamin Saller (bcsaller)]

LGTM +1

Thanks again

https://codereview.appspot.com/91510047/diff/1/hooks/hooks.py
File hooks/hooks.py (right):

https://codereview.appspot.com/91510047/diff/1/hooks/hooks.py#newcode11
hooks/hooks.py:11: import config as lg_config
just config? or we could rename to common as in some of the others now.

https://codereview.appspot.com/91510047/

[Cory Johns (johnsca)]

https://codereview.appspot.com/91510047/diff/1/hooks/hooks.py
File hooks/hooks.py (right):

https://codereview.appspot.com/91510047/diff/1/hooks/hooks.py#newcode11
hooks/hooks.py:11: import config as lg_config
On 2014/05/19 18:58:39, benjamin.saller wrote:
> just config? or we could rename to common as in some of the others
now.

I was using xx_config to prevent potential conflicts with
hookenv.config, but those are all referenced indirectly anyway, so it's
probably not worth bothering with the prefix.

https://codereview.appspot.com/91510047/

[lp:~johnsca/charms/trusty/cf-loggregator/refactor]

32. By Cory Johns on 2014-05-20

Merged :parent

33. By Cory Johns on 2014-05-20

Resynced charm-helpers

34. By Cory Johns on 2014-05-20

Removed unnecessary prefix from config var

35. By Cory Johns on 2014-05-20

Slight addendum to previous commit

[Cory Johns (johnsca)]

*** Submitted:

Refactored to use refactored charm-helpers

R=benjamin.saller, cory.johns
CC=
https://codereview.appspot.com/91510047

https://codereview.appspot.com/91510047/

=== modified file 'config.yaml'
--- config.yaml	2014-05-12 11:12:06 +0000
+++ config.yaml	2014-05-20 21:57:20 +0000
@@ -8,3 +8,22 @@
       The shared-secret for log clients to use when publishing log
       messages.
     default: 17b6cc14-eea8-433e-b8b7-a65ff8941e3b
+  source:
+    type: string
+    default: 'ppa:cf-charm/ppa'
+    description: |
+      Optional configuration to support use of additional sources such as:
+      .
+        - ppa:myteam/ppa
+        - cloud:precise-proposed/folsom
+        - http://my.archive.com/ubuntu main
+      .
+      The last option should be used in conjunction with the key configuration
+      option.
+      .
+  key:
+    type: string
+    default: '4C430C3C2828E07D'
+    description: |
+      Key ID to import to the apt keyring to support use with arbitary source
+      configuration from outside of Launchpad archives or PPA's.

=== removed directory 'files/upstart'
=== modified file 'hooks/charmhelpers/contrib/cloudfoundry/common.py'
--- hooks/charmhelpers/contrib/cloudfoundry/common.py	2014-05-07 16:31:54 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/common.py	2014-05-20 21:57:20 +0000
@@ -1,11 +1,3 @@
-import sys
-import os
-import pwd
-import grp
-import subprocess
-
-from contextlib import contextmanager
-from charmhelpers.core.hookenv import log, ERROR, DEBUG
 from charmhelpers.core import host
 
 from charmhelpers.fetch import (
@@ -13,59 +5,8 @@
 )
 
 
-def run(command, exit_on_error=True, quiet=False):
-    '''Run a command and return the output.'''
-    if not quiet:
-        log("Running {!r}".format(command), DEBUG)
-    p = subprocess.Popen(
-        command, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
-        shell=isinstance(command, basestring))
-    p.stdin.close()
-    lines = []
-    for line in p.stdout:
-        if line:
-            if not quiet:
-                print line
-            lines.append(line)
-        elif p.poll() is not None:
-            break
-
-    p.wait()
-
-    if p.returncode == 0:
-        return '\n'.join(lines)
-
-    if p.returncode != 0 and exit_on_error:
-        log("ERROR: {}".format(p.returncode), ERROR)
-        sys.exit(p.returncode)
-
-    raise subprocess.CalledProcessError(
-        p.returncode, command, '\n'.join(lines))
-
-
-def chownr(path, owner, group):
-    uid = pwd.getpwnam(owner).pw_uid
-    gid = grp.getgrnam(group).gr_gid
-    for root, dirs, files in os.walk(path):
-        for momo in dirs:
-            os.chown(os.path.join(root, momo), uid, gid)
-            for momo in files:
-                os.chown(os.path.join(root, momo), uid, gid)
-
-
-@contextmanager
-def chdir(d):
-    cur = os.getcwd()
-    try:
-        yield os.chdir(d)
-    finally:
-        os.chdir(cur)
-
-
 def prepare_cloudfoundry_environment(config_data, packages):
-    if 'source' in config_data:
-        add_source(config_data['source'], config_data.get('key'))
-        apt_update(fatal=True)
-    if packages:
-        apt_install(packages=filter_installed_packages(packages), fatal=True)
+    add_source(config_data['source'], config_data.get('key'))
+    apt_update(fatal=True)
+    apt_install(packages=filter_installed_packages(packages), fatal=True)
     host.adduser('vcap')

=== removed file 'hooks/charmhelpers/contrib/cloudfoundry/config_helper.py'
--- hooks/charmhelpers/contrib/cloudfoundry/config_helper.py	2014-04-25 12:49:48 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/config_helper.py	1970-01-01 00:00:00 +0000
@@ -1,11 +0,0 @@
-import jinja2
-
-TEMPLATES_DIR = 'templates'
-
-def render_template(template_name, context, template_dir=TEMPLATES_DIR):
-    templates = jinja2.Environment(
-        loader=jinja2.FileSystemLoader(template_dir))
-    template = templates.get_template(template_name)
-    return template.render(context)
-
-

=== modified file 'hooks/charmhelpers/contrib/cloudfoundry/contexts.py'
--- hooks/charmhelpers/contrib/cloudfoundry/contexts.py	2014-05-07 16:31:54 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/contexts.py	2014-05-20 21:57:20 +0000
@@ -1,54 +1,16 @@
 import os
-import yaml
-
-from charmhelpers.core import hookenv
-from charmhelpers.contrib.openstack.context import OSContextGenerator
-
-
-class RelationContext(OSContextGenerator):
-    def __call__(self):
-        if not hookenv.relation_ids(self.interface):
-            return {}
-
-        ctx = {}
-        for rid in hookenv.relation_ids(self.interface):
-            for unit in hookenv.related_units(rid):
-                reldata = hookenv.relation_get(rid=rid, unit=unit)
-                required = set(self.required_keys)
-                if set(reldata.keys()).issuperset(required):
-                    ns = ctx.setdefault(self.interface, {})
-                    for k, v in reldata.items():
-                        ns[k] = v
-                    return ctx
-
-        return {}
-
-
-class ConfigContext(OSContextGenerator):
-    def __call__(self):
-        return hookenv.config()
-
-
-class StorableContext(object):
-    
-    def store_context(self, file_name, config_data):
-        with open(file_name, 'w') as file_stream:
-            yaml.dump(config_data, file_stream)
-
-    def read_context(self, file_name):
-        with open(file_name, 'r') as file_stream:
-            data = yaml.load(file_stream)
-            if not data:
-                raise OSError("%s is empty" % file_name)
-            return data
+
+from charmhelpers.core.templating import (
+    ContextGenerator,
+    RelationContext,
+    StorableContext,
+)
 
 
 # Stores `config_data` hash into yaml file with `file_name` as a name
 # if `file_name` already exists, then it loads data from `file_name`.
-class StoredContext(OSContextGenerator, StorableContext):
-
+class StoredContext(ContextGenerator, StorableContext):
     def __init__(self, file_name, config_data):
-        self.data = config_data
         if os.path.exists(file_name):
             self.data = self.read_context(file_name)
         else:
@@ -59,25 +21,35 @@
         return self.data
 
 
-class StaticContext(OSContextGenerator):
-    def __init__(self, data):
-        self.data = data
-
-    def __call__(self):
-        return self.data
-
-
 class NatsContext(RelationContext):
     interface = 'nats'
     required_keys = ['nats_port', 'nats_address', 'nats_user', 'nats_password']
 
 
+class MysqlDSNContext(RelationContext):
+    interface = 'db'
+    required_keys = ['user', 'password', 'host', 'database']
+    dsn_template = "mysql2://{user}:{password}@{host}:{port}/{database}"
+
+    def __call__(self):
+        ctx = RelationContext.__call__(self)
+        if ctx:
+            if 'port' not in ctx:
+                ctx['db']['port'] = '3306'
+            ctx['db']['dsn'] = self.dsn_template.format(**ctx['db'])
+        return ctx
+
+
 class RouterContext(RelationContext):
     interface = 'router'
     required_keys = ['domain']
 
 
+class LogRouterContext(RelationContext):
+    interface = 'logrouter'
+    required_keys = ['shared-secret', 'logrouter-address']
+
+
 class LoggregatorContext(RelationContext):
     interface = 'loggregator'
     required_keys = ['shared_secret', 'loggregator_address']
-

=== removed file 'hooks/charmhelpers/contrib/cloudfoundry/install.py'
--- hooks/charmhelpers/contrib/cloudfoundry/install.py	2014-04-25 12:49:48 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/install.py	1970-01-01 00:00:00 +0000
@@ -1,35 +0,0 @@
-import os
-import subprocess
-
-
-def install(src, dest, fileprops=None, sudo=False):
-    """Install a file from src to dest. Dest can be a complete filename
-    or a target directory. fileprops is a dict with 'owner' (username of owner)
-    and mode (octal string) as keys, the defaults are 'ubuntu' and '400'
-
-    When owner is passed or when access requires it sudo can be set to True and
-    sudo will be used to install the file.
-    """
-    if not fileprops:
-        fileprops = {}
-    mode = fileprops.get('mode', '400')
-    owner = fileprops.get('owner')
-    cmd = ['install']
-
-    if not os.path.exists(src):
-        raise OSError(src)
-
-    if not os.path.exists(dest) and not os.path.exists(os.path.dirname(dest)):
-        # create all but the last component as path
-        cmd.append('-D')
-
-    if mode:
-        cmd.extend(['-m', mode])
-
-    if owner:
-        cmd.extend(['-o', owner])
-
-    if sudo:
-        cmd.insert(0, 'sudo')
-    cmd.extend([src, dest])
-    subprocess.check_call(cmd)

=== removed file 'hooks/charmhelpers/contrib/cloudfoundry/services.py'
--- hooks/charmhelpers/contrib/cloudfoundry/services.py	2014-04-25 12:49:48 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/services.py	1970-01-01 00:00:00 +0000
@@ -1,118 +0,0 @@
-import os
-import tempfile
-from charmhelpers.core import host
-
-from charmhelpers.contrib.cloudfoundry.install import install
-from charmhelpers.core.hookenv import log
-from jinja2 import Environment, FileSystemLoader
-
-SERVICE_CONFIG = []
-TEMPLATE_LOADER = None
-
-
-def render_template(template_name, context):
-    """Render template to a tempfile returning the name"""
-    _, fn = tempfile.mkstemp()
-    template = load_template(template_name)
-    output = template.render(context)
-    with open(fn, "w") as fp:
-        fp.write(output)
-    return fn
-
-
-def collect_contexts(context_providers):
-    ctx = {}
-    for provider in context_providers:
-        c = provider()
-        if not c:
-            return {}
-        ctx.update(c)
-    return ctx
-
-
-def load_template(name):
-    return TEMPLATE_LOADER.get_template(name)
-
-
-def configure_templates(template_dir):
-    global TEMPLATE_LOADER
-    TEMPLATE_LOADER = Environment(loader=FileSystemLoader(template_dir))
-
-
-def register(service_configs, template_dir):
-    """Register a list of service configs.
-
-    Service Configs are dicts in the following formats:
-
-        {
-            "service": <service name>,
-            "templates": [ {
-                'target': <render target of template>,
-                'source': <optional name of template in passed in template_dir>
-                'file_properties': <optional dict taking owner and octal mode>
-                'contexts': [ context generators, see contexts.py ]
-                }
-            ] }
-
-    If 'source' is not provided for a template the template_dir will
-    be consulted for ``basename(target).j2``.
-    """
-    global SERVICE_CONFIG
-    if template_dir:
-        configure_templates(template_dir)
-    SERVICE_CONFIG.extend(service_configs)
-
-
-def reset():
-    global SERVICE_CONFIG
-    SERVICE_CONFIG = []
-
-
-# def service_context(name):
-#     contexts = collect_contexts(template['contexts'])
-
-def reconfigure_service(service_name, restart=True):
-    global SERVICE_CONFIG
-    service = None
-    for service in SERVICE_CONFIG:
-        if service['service'] == service_name:
-            break
-    if not service or service['service'] != service_name:
-        raise KeyError('Service not registered: %s' % service_name)
-
-    templates = service['templates']
-    for template in templates:
-        contexts = collect_contexts(template['contexts'])
-        if contexts:
-            template_target = template['target']
-            default_template = "%s.j2" % os.path.basename(template_target)
-            template_name = template.get('source', default_template)
-            output_file = render_template(template_name, contexts)
-            file_properties = template.get('file_properties')
-            install(output_file, template_target, file_properties)
-            os.unlink(output_file)
-        else:
-            restart = False
-
-    if restart:
-        host.service_restart(service_name)
-
-
-def stop_services():
-    global SERVICE_CONFIG
-    for service in SERVICE_CONFIG:
-        if host.service_running(service['service']):
-            host.service_stop(service['service'])
-
-
-def get_service(service_name):
-    global SERVICE_CONFIG
-    for service in SERVICE_CONFIG:
-        if service_name == service['service']:
-            return service
-    return None
-
-
-def reconfigure_services(restart=True):
-    for service in SERVICE_CONFIG:
-        reconfigure_service(service['service'], restart=restart)

=== removed file 'hooks/charmhelpers/contrib/cloudfoundry/upstart_helper.py'
--- hooks/charmhelpers/contrib/cloudfoundry/upstart_helper.py	2014-04-25 12:49:48 +0000
+++ hooks/charmhelpers/contrib/cloudfoundry/upstart_helper.py	1970-01-01 00:00:00 +0000
@@ -1,14 +0,0 @@
-import os
-import glob
-from charmhelpers.core import hookenv
-from charmhelpers.core.hookenv import charm_dir
-from charmhelpers.contrib.cloudfoundry.install import install
-
-
-def install_upstart_scripts(dirname=os.path.join(hookenv.charm_dir(),
-                                                 'files/upstart'),
-                            pattern='*.conf'):
-    for script in glob.glob("%s/%s" % (dirname, pattern)):
-        filename = os.path.join(dirname, script)
-        hookenv.log('Installing upstart job:' + filename, hookenv.DEBUG)
-        install(filename, '/etc/init')

=== modified file 'hooks/charmhelpers/contrib/hahelpers/apache.py'
--- hooks/charmhelpers/contrib/hahelpers/apache.py	2014-04-25 12:49:48 +0000
+++ hooks/charmhelpers/contrib/hahelpers/apache.py	2014-05-20 21:57:20 +0000
@@ -39,14 +39,15 @@
 
 
 def get_ca_cert():
-    ca_cert = None
-    log("Inspecting identity-service relations for CA SSL certificate.",
-        level=INFO)
-    for r_id in relation_ids('identity-service'):
-        for unit in relation_list(r_id):
-            if not ca_cert:
-                ca_cert = relation_get('ca_cert',
-                                       rid=r_id, unit=unit)
+    ca_cert = config_get('ssl_ca')
+    if ca_cert is None:
+        log("Inspecting identity-service relations for CA SSL certificate.",
+            level=INFO)
+        for r_id in relation_ids('identity-service'):
+            for unit in relation_list(r_id):
+                if ca_cert is None:
+                    ca_cert = relation_get('ca_cert',
+                                           rid=r_id, unit=unit)
     return ca_cert
 
 

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2014-04-25 12:49:48 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2014-05-20 21:57:20 +0000
@@ -1,5 +1,6 @@
 import json
 import os
+import time
 
 from base64 import b64decode
 
@@ -113,7 +114,8 @@
 class SharedDBContext(OSContextGenerator):
     interfaces = ['shared-db']
 
-    def __init__(self, database=None, user=None, relation_prefix=None):
+    def __init__(self,
+                 database=None, user=None, relation_prefix=None, ssl_dir=None):
         '''
         Allows inspecting relation for settings prefixed with relation_prefix.
         This is useful for parsing access for multiple databases returned via
@@ -122,6 +124,7 @@
         self.relation_prefix = relation_prefix
         self.database = database
         self.user = user
+        self.ssl_dir = ssl_dir
 
     def __call__(self):
         self.database = self.database or config('database')
@@ -139,17 +142,72 @@
 
         for rid in relation_ids('shared-db'):
             for unit in related_units(rid):
-                passwd = relation_get(password_setting, rid=rid, unit=unit)
+                rdata = relation_get(rid=rid, unit=unit)
                 ctxt = {
-                    'database_host': relation_get('db_host', rid=rid,
-                                                  unit=unit),
+                    'database_host': rdata.get('db_host'),
                     'database': self.database,
                     'database_user': self.user,
-                    'database_password': passwd,
-                }
-                if context_complete(ctxt):
-                    return ctxt
-        return {}
+                    'database_password': rdata.get(password_setting),
+                    'database_type': 'mysql'
+                }
+                if context_complete(ctxt):
+                    db_ssl(rdata, ctxt, self.ssl_dir)
+                    return ctxt
+        return {}
+
+
+class PostgresqlDBContext(OSContextGenerator):
+    interfaces = ['pgsql-db']
+
+    def __init__(self, database=None):
+        self.database = database
+
+    def __call__(self):
+        self.database = self.database or config('database')
+        if self.database is None:
+            log('Could not generate postgresql_db context. '
+                'Missing required charm config options. '
+                '(database name)')
+            raise OSContextError
+        ctxt = {}
+
+        for rid in relation_ids(self.interfaces[0]):
+            for unit in related_units(rid):
+                ctxt = {
+                    'database_host': relation_get('host', rid=rid, unit=unit),
+                    'database': self.database,
+                    'database_user': relation_get('user', rid=rid, unit=unit),
+                    'database_password': relation_get('password', rid=rid, unit=unit),
+                    'database_type': 'postgresql',
+                }
+                if context_complete(ctxt):
+                    return ctxt
+        return {}
+
+
+def db_ssl(rdata, ctxt, ssl_dir):
+    if 'ssl_ca' in rdata and ssl_dir:
+        ca_path = os.path.join(ssl_dir, 'db-client.ca')
+        with open(ca_path, 'w') as fh:
+            fh.write(b64decode(rdata['ssl_ca']))
+        ctxt['database_ssl_ca'] = ca_path
+    elif 'ssl_ca' in rdata:
+        log("Charm not setup for ssl support but ssl ca found")
+        return ctxt
+    if 'ssl_cert' in rdata:
+        cert_path = os.path.join(
+            ssl_dir, 'db-client.cert')
+        if not os.path.exists(cert_path):
+            log("Waiting 1m for ssl client cert validity")
+            time.sleep(60)
+        with open(cert_path, 'w') as fh:
+            fh.write(b64decode(rdata['ssl_cert']))
+        ctxt['database_ssl_cert'] = cert_path
+        key_path = os.path.join(ssl_dir, 'db-client.key')
+        with open(key_path, 'w') as fh:
+            fh.write(b64decode(rdata['ssl_key']))
+        ctxt['database_ssl_key'] = key_path
+    return ctxt
 
 
 class IdentityServiceContext(OSContextGenerator):
@@ -161,24 +219,25 @@
 
         for rid in relation_ids('identity-service'):
             for unit in related_units(rid):
+                rdata = relation_get(rid=rid, unit=unit)
                 ctxt = {
-                    'service_port': relation_get('service_port', rid=rid,
-                                                 unit=unit),
-                    'service_host': relation_get('service_host', rid=rid,
-                                                 unit=unit),
-                    'auth_host': relation_get('auth_host', rid=rid, unit=unit),
-                    'auth_port': relation_get('auth_port', rid=rid, unit=unit),
-                    'admin_tenant_name': relation_get('service_tenant',
-                                                      rid=rid, unit=unit),
-                    'admin_user': relation_get('service_username', rid=rid,
-                                               unit=unit),
-                    'admin_password': relation_get('service_password', rid=rid,
-                                                   unit=unit),
-                    # XXX: Hard-coded http.
-                    'service_protocol': 'http',
-                    'auth_protocol': 'http',
+                    'service_port': rdata.get('service_port'),
+                    'service_host': rdata.get('service_host'),
+                    'auth_host': rdata.get('auth_host'),
+                    'auth_port': rdata.get('auth_port'),
+                    'admin_tenant_name': rdata.get('service_tenant'),
+                    'admin_user': rdata.get('service_username'),
+                    'admin_password': rdata.get('service_password'),
+                    'service_protocol':
+                    rdata.get('service_protocol') or 'http',
+                    'auth_protocol':
+                    rdata.get('auth_protocol') or 'http',
                 }
                 if context_complete(ctxt):
+                    # NOTE(jamespage) this is required for >= icehouse
+                    # so a missing value just indicates keystone needs
+                    # upgrading
+                    ctxt['admin_tenant_id'] = rdata.get('service_tenant_id')
                     return ctxt
         return {}
 
@@ -186,6 +245,9 @@
 class AMQPContext(OSContextGenerator):
     interfaces = ['amqp']
 
+    def __init__(self, ssl_dir=None):
+        self.ssl_dir = ssl_dir
+
     def __call__(self):
         log('Generating template context for amqp')
         conf = config()
@@ -196,7 +258,6 @@
             log('Could not generate shared_db context. '
                 'Missing required charm config options: %s.' % e)
             raise OSContextError
-
         ctxt = {}
         for rid in relation_ids('amqp'):
             ha_vip_only = False
@@ -214,6 +275,14 @@
                                                       unit=unit),
                     'rabbitmq_virtual_host': vhost,
                 })
+
+                ssl_port = relation_get('ssl_port', rid=rid, unit=unit)
+                if ssl_port:
+                    ctxt['rabbit_ssl_port'] = ssl_port
+                ssl_ca = relation_get('ssl_ca', rid=rid, unit=unit)
+                if ssl_ca:
+                    ctxt['rabbit_ssl_ca'] = ssl_ca
+
                 if relation_get('ha_queues', rid=rid, unit=unit) is not None:
                     ctxt['rabbitmq_ha_queues'] = True
 
@@ -221,6 +290,16 @@
                                            rid=rid, unit=unit) is not None
 
                 if context_complete(ctxt):
+                    if 'rabbit_ssl_ca' in ctxt:
+                        if not self.ssl_dir:
+                            log(("Charm not setup for ssl support "
+                                 "but ssl ca found"))
+                            break
+                        ca_path = os.path.join(
+                            self.ssl_dir, 'rabbit-client-ca.pem')
+                        with open(ca_path, 'w') as fh:
+                            fh.write(b64decode(ctxt['rabbit_ssl_ca']))
+                            ctxt['rabbit_ssl_ca'] = ca_path
                     # Sufficient information found = break out!
                     break
             # Used for active/active rabbitmq >= grizzly
@@ -391,6 +470,8 @@
             'private_address': unit_get('private-address'),
             'endpoints': []
         }
+        if is_clustered():
+            ctxt['private_address'] = config('vip')
         for api_port in self.external_ports:
             ext_port = determine_apache_port(api_port)
             int_port = determine_api_port(api_port)
@@ -489,7 +570,7 @@
 
         if self.plugin == 'ovs':
             ctxt.update(self.ovs_ctxt())
-        elif self.plugin == 'nvp':
+        elif self.plugin in ['nvp', 'nsx']:
             ctxt.update(self.nvp_ctxt())
 
         alchemy_flags = config('neutron-alchemy-flags')

=== modified file 'hooks/charmhelpers/contrib/openstack/neutron.py'
--- hooks/charmhelpers/contrib/openstack/neutron.py	2014-04-25 12:49:48 +0000
+++ hooks/charmhelpers/contrib/openstack/neutron.py	2014-05-20 21:57:20 +0000
@@ -17,6 +17,8 @@
     kver = check_output(['uname', '-r']).strip()
     return 'linux-headers-%s' % kver
 
+QUANTUM_CONF_DIR = '/etc/quantum'
+
 
 def kernel_version():
     """ Retrieve the current major kernel version as a tuple e.g. (3, 13) """
@@ -35,6 +37,8 @@
 
 
 # legacy
+
+
 def quantum_plugins():
     from charmhelpers.contrib.openstack import context
     return {
@@ -46,7 +50,8 @@
             'contexts': [
                 context.SharedDBContext(user=config('neutron-database-user'),
                                         database=config('neutron-database'),
-                                        relation_prefix='neutron')],
+                                        relation_prefix='neutron',
+                                        ssl_dir=QUANTUM_CONF_DIR)],
             'services': ['quantum-plugin-openvswitch-agent'],
             'packages': [[headers_package()] + determine_dkms_package(),
                          ['quantum-plugin-openvswitch-agent']],
@@ -61,7 +66,8 @@
             'contexts': [
                 context.SharedDBContext(user=config('neutron-database-user'),
                                         database=config('neutron-database'),
-                                        relation_prefix='neutron')],
+                                        relation_prefix='neutron',
+                                        ssl_dir=QUANTUM_CONF_DIR)],
             'services': [],
             'packages': [],
             'server_packages': ['quantum-server',
@@ -70,6 +76,8 @@
         }
     }
 
+NEUTRON_CONF_DIR = '/etc/neutron'
+
 
 def neutron_plugins():
     from charmhelpers.contrib.openstack import context
@@ -83,7 +91,8 @@
             'contexts': [
                 context.SharedDBContext(user=config('neutron-database-user'),
                                         database=config('neutron-database'),
-                                        relation_prefix='neutron')],
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
             'services': ['neutron-plugin-openvswitch-agent'],
             'packages': [[headers_package()] + determine_dkms_package(),
                          ['neutron-plugin-openvswitch-agent']],
@@ -98,20 +107,37 @@
             'contexts': [
                 context.SharedDBContext(user=config('neutron-database-user'),
                                         database=config('neutron-database'),
-                                        relation_prefix='neutron')],
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
             'services': [],
             'packages': [],
             'server_packages': ['neutron-server',
                                 'neutron-plugin-nicira'],
             'server_services': ['neutron-server']
+        },
+        'nsx': {
+            'config': '/etc/neutron/plugins/vmware/nsx.ini',
+            'driver': 'vmware',
+            'contexts': [
+                context.SharedDBContext(user=config('neutron-database-user'),
+                                        database=config('neutron-database'),
+                                        relation_prefix='neutron',
+                                        ssl_dir=NEUTRON_CONF_DIR)],
+            'services': [],
+            'packages': [],
+            'server_packages': ['neutron-server',
+                                'neutron-plugin-vmware'],
+            'server_services': ['neutron-server']
         }
     }
-    # NOTE: patch in ml2 plugin for icehouse onwards
     if release >= 'icehouse':
+        # NOTE: patch in ml2 plugin for icehouse onwards
         plugins['ovs']['config'] = '/etc/neutron/plugins/ml2/ml2_conf.ini'
         plugins['ovs']['driver'] = 'neutron.plugins.ml2.plugin.Ml2Plugin'
         plugins['ovs']['server_packages'] = ['neutron-server',
                                              'neutron-plugin-ml2']
+        # NOTE: patch in vmware renames nvp->nsx for icehouse onwards
+        plugins['nvp'] = plugins['nsx']
     return plugins
 
 

=== modified file 'hooks/charmhelpers/contrib/openstack/utils.py'
--- hooks/charmhelpers/contrib/openstack/utils.py	2014-04-25 12:49:48 +0000
+++ hooks/charmhelpers/contrib/openstack/utils.py	2014-05-20 21:57:20 +0000
@@ -65,6 +65,7 @@
     ('1.10.0', 'havana'),
     ('1.9.1', 'havana'),
     ('1.9.0', 'havana'),
+    ('1.13.1', 'icehouse'),
     ('1.13.0', 'icehouse'),
     ('1.12.0', 'icehouse'),
     ('1.11.0', 'icehouse'),
@@ -130,6 +131,11 @@
 def get_os_codename_package(package, fatal=True):
     '''Derive OpenStack release codename from an installed package.'''
     apt.init()
+
+    # Tell apt to build an in-memory cache to prevent race conditions (if
+    # another process is already building the cache).
+    apt.config.set("Dir::Cache::pkgcache", "")
+
     cache = apt.Cache()
 
     try:
@@ -182,7 +188,7 @@
         if cname == codename:
             return version
     #e = "Could not determine OpenStack version for package: %s" % pkg
-    #error_out(e)
+    # error_out(e)
 
 
 os_rel = None
@@ -400,6 +406,8 @@
         rtype = 'PTR'
     elif isinstance(address, basestring):
         rtype = 'A'
+    else:
+        return None
 
     answers = dns.resolver.query(address, rtype)
     if answers:

=== modified file 'hooks/charmhelpers/contrib/storage/linux/lvm.py'
--- hooks/charmhelpers/contrib/storage/linux/lvm.py	2014-04-25 12:49:48 +0000
+++ hooks/charmhelpers/contrib/storage/linux/lvm.py	2014-05-20 21:57:20 +0000
@@ -62,7 +62,7 @@
     pvd = check_output(['pvdisplay', block_device]).splitlines()
     for l in pvd:
         if l.strip().startswith('VG Name'):
-            vg = ' '.join(l.split()).split(' ').pop()
+            vg = ' '.join(l.strip().split()[2:])
     return vg
 
 

=== modified file 'hooks/charmhelpers/contrib/storage/linux/utils.py'
--- hooks/charmhelpers/contrib/storage/linux/utils.py	2014-04-25 12:49:48 +0000
+++ hooks/charmhelpers/contrib/storage/linux/utils.py	2014-05-20 21:57:20 +0000
@@ -1,8 +1,11 @@
-from os import stat
+import os
+import re
 from stat import S_ISBLK
 
 from subprocess import (
-    check_call
+    check_call,
+    check_output,
+    call
 )
 
 
@@ -12,7 +15,9 @@
 
     :returns: boolean: True if path is a block device, False if not.
     '''
-    return S_ISBLK(stat(path).st_mode)
+    if not os.path.exists(path):
+        return False
+    return S_ISBLK(os.stat(path).st_mode)
 
 
 def zap_disk(block_device):
@@ -22,5 +27,23 @@
 
     :param block_device: str: Full path of block device to clean.
     '''
-    check_call(['sgdisk', '--zap-all', '--clear',
-                '--mbrtogpt', block_device])
+    # sometimes sgdisk exits non-zero; this is OK, dd will clean up
+    call(['sgdisk', '--zap-all', '--mbrtogpt',
+          '--clear', block_device])
+    dev_end = check_output(['blockdev', '--getsz', block_device])
+    gpt_end = int(dev_end.split()[0]) - 100
+    check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
+                'bs=1M', 'count=1'])
+    check_call(['dd', 'if=/dev/zero', 'of=%s' % (block_device),
+                'bs=512', 'count=100', 'seek=%s' % (gpt_end)])
+
+def is_device_mounted(device):
+    '''Given a device path, return True if that device is mounted, and False
+    if it isn't.
+
+    :param device: str: Full path of the device to check.
+    :returns: boolean: True if the path represents a mounted device, False if
+        it doesn't.
+    '''
+    out = check_output(['mount'])
+    return bool(re.search(device + r"[0-9]+\b", out))

=== modified file 'hooks/charmhelpers/core/hookenv.py'
--- hooks/charmhelpers/core/hookenv.py	2014-03-28 22:37:40 +0000
+++ hooks/charmhelpers/core/hookenv.py	2014-05-20 21:57:20 +0000
@@ -155,6 +155,100 @@
     return os.path.basename(sys.argv[0])
 
 
+class Config(dict):
+    """A Juju charm config dictionary that can write itself to
+    disk (as json) and track which values have changed since
+    the previous hook invocation.
+
+    Do not instantiate this object directly - instead call
+    ``hookenv.config()``
+
+    Example usage::
+
+        >>> # inside a hook
+        >>> from charmhelpers.core import hookenv
+        >>> config = hookenv.config()
+        >>> config['foo']
+        'bar'
+        >>> config['mykey'] = 'myval'
+        >>> config.save()
+
+
+        >>> # user runs `juju set mycharm foo=baz`
+        >>> # now we're inside subsequent config-changed hook
+        >>> config = hookenv.config()
+        >>> config['foo']
+        'baz'
+        >>> # test to see if this val has changed since last hook
+        >>> config.changed('foo')
+        True
+        >>> # what was the previous value?
+        >>> config.previous('foo')
+        'bar'
+        >>> # keys/values that we add are preserved across hooks
+        >>> config['mykey']
+        'myval'
+        >>> # don't forget to save at the end of hook!
+        >>> config.save()
+
+    """
+    CONFIG_FILE_NAME = '.juju-persistent-config'
+
+    def __init__(self, *args, **kw):
+        super(Config, self).__init__(*args, **kw)
+        self._prev_dict = None
+        self.path = os.path.join(charm_dir(), Config.CONFIG_FILE_NAME)
+        if os.path.exists(self.path):
+            self.load_previous()
+
+    def load_previous(self, path=None):
+        """Load previous copy of config from disk so that current values
+        can be compared to previous values.
+
+        :param path:
+
+            File path from which to load the previous config. If `None`,
+            config is loaded from the default location. If `path` is
+            specified, subsequent `save()` calls will write to the same
+            path.
+
+        """
+        self.path = path or self.path
+        with open(self.path) as f:
+            self._prev_dict = json.load(f)
+
+    def changed(self, key):
+        """Return true if the value for this key has changed since
+        the last save.
+
+        """
+        if self._prev_dict is None:
+            return True
+        return self.previous(key) != self.get(key)
+
+    def previous(self, key):
+        """Return previous value for this key, or None if there
+        is no "previous" value.
+
+        """
+        if self._prev_dict:
+            return self._prev_dict.get(key)
+        return None
+
+    def save(self):
+        """Save this config to disk.
+
+        Preserves items in _prev_dict that do not exist in self.
+
+        """
+        if self._prev_dict:
+            for k, v in self._prev_dict.iteritems():
+                if k not in self:
+                    self[k] = v
+        with open(self.path, 'w') as f:
+            json.dump(self, f)
+
+
 @cached
 def config(scope=None):
     """Juju charm configuration"""
@@ -163,7 +257,10 @@
         config_cmd_line.append(scope)
     config_cmd_line.append('--format=json')
     try:
-        return json.loads(subprocess.check_output(config_cmd_line))
+        config_data = json.loads(subprocess.check_output(config_cmd_line))
+        if scope is not None:
+            return config_data
+        return Config(config_data)
     except ValueError:
         return None
 

=== modified file 'hooks/charmhelpers/core/host.py'
--- hooks/charmhelpers/core/host.py	2014-03-28 22:37:40 +0000
+++ hooks/charmhelpers/core/host.py	2014-05-20 21:57:20 +0000
@@ -12,6 +12,9 @@
 import string
 import subprocess
 import hashlib
+import shutil
+import apt_pkg
+from contextlib import contextmanager
 
 from collections import OrderedDict
 
@@ -143,6 +146,16 @@
         target.write(content)
 
 
+def copy_file(src, dst, owner='root', group='root', perms=0444):
+    """Create or overwrite a file with the contents of another file"""
+    log("Writing file {} {}:{} {:o} from {}".format(dst, owner, group, perms, src))
+    uid = pwd.getpwnam(owner).pw_uid
+    gid = grp.getgrnam(group).gr_gid
+    shutil.copyfile(src, dst)
+    os.chown(dst, uid, gid)
+    os.chmod(dst, perms)
+
+
 def mount(device, mountpoint, options=None, persist=False):
     """Mount a filesystem at a particular mountpoint"""
     cmd_args = ['mount']
@@ -295,3 +308,37 @@
     if 'link/ether' in words:
         hwaddr = words[words.index('link/ether') + 1]
     return hwaddr
+
+
+def cmp_pkgrevno(package, revno, pkgcache=None):
+    '''Compare supplied revno with the revno of the installed package
+       1 => Installed revno is greater than supplied arg
+       0 => Installed revno is the same as supplied arg
+      -1 => Installed revno is less than supplied arg
+    '''
+    if not pkgcache:
+        apt_pkg.init()
+        pkgcache = apt_pkg.Cache()
+    pkg = pkgcache[package]
+    return apt_pkg.version_compare(pkg.current_ver.ver_str, revno)
+
+
+@contextmanager
+def chdir(d):
+    cur = os.getcwd()
+    try:
+        yield os.chdir(d)
+    finally:
+        os.chdir(cur)
+
+
+def chownr(path, owner, group):
+    uid = pwd.getpwnam(owner).pw_uid
+    gid = grp.getgrnam(group).gr_gid
+
+    for root, dirs, files in os.walk(path):
+        for name in dirs + files:
+            full = os.path.join(root, name)
+            broken_symlink = os.path.lexists(full) and not os.path.exists(full)
+            if not broken_symlink:
+                os.chown(full, uid, gid)

=== added file 'hooks/charmhelpers/core/services.py'
--- hooks/charmhelpers/core/services.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/services.py	2014-05-20 21:57:20 +0000
@@ -0,0 +1,84 @@
+from charmhelpers.core import templating
+from charmhelpers.core import host
+
+
+SERVICES = {}
+
+
+def register(services, templates_dir=None):
+    """
+    Register a list of service configs.
+
+    Service Configs are dicts in the following formats:
+
+        {
+            "service": <service name>,
+            "templates": [ {
+                'target': <render target of template>,
+                'source': <optional name of template in passed in templates_dir>
+                'file_properties': <optional dict taking owner and octal mode>
+                'contexts': [ context generators, see contexts.py ]
+                }
+            ] }
+
+    Either `source` or `target` must be provided.
+
+    If 'source' is not provided for a template the templates_dir will
+    be consulted for ``basename(target).j2``.
+
+    If `target` is not provided, it will be assumed to be
+    ``/etc/init/<service name>.conf``.
+    """
+    for service in services:
+        service.setdefault('templates_dir', templates_dir)
+        SERVICES[service['service']] = service
+
+
+def reconfigure_services(restart=True):
+    """
+    Update all files for all services and optionally restart them, if ready.
+    """
+    for service_name in SERVICES.keys():
+        reconfigure_service(service_name, restart=restart)
+
+
+def reconfigure_service(service_name, restart=True):
+    """
+    Update all files for a single service and optionally restart it, if ready.
+    """
+    service = SERVICES.get(service_name)
+    if not service or service['service'] != service_name:
+        raise KeyError('Service not registered: %s' % service_name)
+
+    manager_type = service.get('type', UpstartService)
+    manager_type(service).reconfigure(restart)
+
+
+def stop_services():
+    for service_name in SERVICES.keys():
+        if host.service_running(service_name):
+            host.service_stop(service_name)
+
+
+class ServiceTypeManager(object):
+    def __init__(self, service_definition):
+        self.service_name = service_definition['service']
+        self.templates = service_definition['templates']
+        self.templates_dir = service_definition['templates_dir']
+
+    def reconfigure(self, restart=True):
+        raise NotImplementedError()
+
+
+class UpstartService(ServiceTypeManager):
+    def __init__(self, service_definition):
+        super(UpstartService, self).__init__(service_definition)
+        for tmpl in self.templates:
+            if 'target' not in tmpl:
+                tmpl['target'] = '/etc/init/%s.conf' % self.service_name
+
+    def reconfigure(self, restart):
+        complete = templating.render(self.templates, self.templates_dir)
+
+        if restart and complete:
+            host.service_restart(self.service_name)

=== added file 'hooks/charmhelpers/core/templating.py'
--- hooks/charmhelpers/core/templating.py	1970-01-01 00:00:00 +0000
+++ hooks/charmhelpers/core/templating.py	2014-05-20 21:57:20 +0000
@@ -0,0 +1,158 @@
+import os
+import yaml
+
+from charmhelpers.core import host
+from charmhelpers.core import hookenv
+
+
+class ContextGenerator(object):
+    """
+    Base interface for template context container generators.
+
+    A template context is a dictionary that contains data needed to populate
+    the template.  The generator instance should produce the context when
+    called (without arguments) by collecting information from juju (config-get,
+    relation-get, etc), the system, or whatever other sources are appropriate.
+
+    A context generator should only return any values if it has enough information
+    to provide all of its values.  Any context that is missing data is considered
+    incomplete and will cause that template to not render until it has all of its
+    necessary data.
+
+    The template may receive several contexts, which will be merged together,
+    so care should be taken in the key names.
+    """
+    def __call__(self):
+        raise NotImplementedError
+
+
+class StorableContext(object):
+    """
+    A mixin for persisting a context to disk.
+    """
+    def store_context(self, file_name, config_data):
+        with open(file_name, 'w') as file_stream:
+            yaml.dump(config_data, file_stream)
+
+    def read_context(self, file_name):
+        with open(file_name, 'r') as file_stream:
+            data = yaml.load(file_stream)
+            if not data:
+                raise OSError("%s is empty" % file_name)
+            return data
+
+
+class ConfigContext(ContextGenerator):
+    """
+    A context generator that generates a context containing all of the
+    juju config values.
+    """
+    def __call__(self):
+        return hookenv.config()
+
+
+class RelationContext(ContextGenerator):
+    """
+    Base class for a context generator that gets relation data from juju.
+
+    Subclasses must provide `interface`, which is the interface type of interest,
+    and `required_keys`, which is the set of keys required for the relation to
+    be considered complete.  The first relation for the interface that is complete
+    will be used to populate the data for template.
+
+    The generated context will be namespaced under the interface type, to prevent
+    potential naming conflicts.
+    """
+    interface = None
+    required_keys = []
+
+    def __call__(self):
+        if not hookenv.relation_ids(self.interface):
+            return {}
+
+        ctx = {}
+        for rid in hookenv.relation_ids(self.interface):
+            for unit in hookenv.related_units(rid):
+                reldata = hookenv.relation_get(rid=rid, unit=unit)
+                required = set(self.required_keys)
+                if set(reldata.keys()).issuperset(required):
+                    ns = ctx.setdefault(self.interface, {})
+                    for k, v in reldata.items():
+                        ns[k] = v
+                    return ctx
+
+        return {}
+
+
+class StaticContext(ContextGenerator):
+    def __init__(self, data):
+        self.data = data
+
+    def __call__(self):
+        return self.data
+
+
+def _collect_contexts(context_providers):
+    """
+    Helper function to collect and merge contexts from a list of providers.
+
+    If any of the contexts are incomplete (i.e., they return an empty dict),
+    the template is considered incomplete and will not render.
+    """
+    ctx = {}
+    for provider in context_providers:
+        c = provider()
+        if not c:
+            return False
+        ctx.update(c)
+    return ctx
+
+
+def render(template_definitions, templates_dir=None):
+    """
+    Render one or more templates, given a list of template definitions.
+
+    The template definitions should be dicts with the keys: `source`, `target`,
+    `file_properties`, and `contexts`.
+
+    The `source` path, if not absolute, is relative to the `templates_dir`
+    given when the rendered was created.  If `source` is not provided
+    for a template the `template_dir` will be consulted for
+    ``basename(target).j2``.
+
+    The `target` path should be absolute.
+
+    The `file_properties` should be a dict optionally containing
+    `owner`, `group`, or `perms` options, to be passed to `write_file`.
+
+    The `contexts` should be a list containing zero or more ContextGenerators.
+
+    The `template_dir` defaults to `$CHARM_DIR/templates`
+
+    Returns True if all of the templates were "complete" (i.e., the context
+    generators were able to collect the information needed to render the
+    template) and were rendered.
+    """
+    # lazy import jinja2 in case templating is needed in install hook
+    from jinja2 import FileSystemLoader, Environment, exceptions
+    all_complete = True
+    if templates_dir is None:
+        templates_dir = os.path.join(hookenv.charm_dir(), 'templates')
+    loader = Environment(loader=FileSystemLoader(templates_dir))
+    for tmpl in template_definitions:
+        ctx = _collect_contexts(tmpl.get('contexts', []))
+        if ctx is False:
+            all_complete = False
+            continue
+        try:
+            source = tmpl.get('source', os.path.basename(tmpl['target'])+'.j2')
+            template = loader.get_template(source)
+        except exceptions.TemplateNotFound as e:
+            hookenv.log('Could not load template %s from %s.' %
+                        (tmpl['source'], templates_dir),
+                        level=hookenv.ERROR)
+            raise e
+        content = template.render(ctx)
+        host.mkdir(os.path.dirname(tmpl['target']))
+        host.write_file(tmpl['target'], content, **tmpl.get('file_properties', {}))
+    return all_complete

=== modified file 'hooks/charmhelpers/fetch/__init__.py'
--- hooks/charmhelpers/fetch/__init__.py	2014-04-25 12:49:48 +0000
+++ hooks/charmhelpers/fetch/__init__.py	2014-05-20 21:57:20 +0000
@@ -1,4 +1,5 @@
 import importlib
+import time
 from yaml import safe_load
 from charmhelpers.core.host import (
     lsb_release
@@ -15,6 +16,7 @@
 import apt_pkg
 import os
 
+
 CLOUD_ARCHIVE = """# Ubuntu Cloud Archive
 deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
 """
@@ -56,10 +58,62 @@
     'precise-proposed/icehouse': 'precise-proposed/icehouse',
 }
 
+# The order of this list is very important. Handlers should be listed in from
+# least- to most-specific URL matching.
+FETCH_HANDLERS = (
+    'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
+    'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
+)
+
+APT_NO_LOCK = 100  # The return code for "couldn't acquire lock" in APT.
+APT_NO_LOCK_RETRY_DELAY = 10  # Wait 10 seconds between apt lock checks.
+APT_NO_LOCK_RETRY_COUNT = 30  # Retry to acquire the lock X times.
+
+
+class SourceConfigError(Exception):
+    pass
+
+
+class UnhandledSource(Exception):
+    pass
+
+
+class AptLockError(Exception):
+    pass
+
+
+class BaseFetchHandler(object):
+
+    """Base class for FetchHandler implementations in fetch plugins"""
+
+    def can_handle(self, source):
+        """Returns True if the source can be handled. Otherwise returns
+        a string explaining why it cannot"""
+        return "Wrong source type"
+
+    def install(self, source):
+        """Try to download and unpack the source. Return the path to the
+        unpacked files or raise UnhandledSource."""
+        raise UnhandledSource("Wrong source type {}".format(source))
+
+    def parse_url(self, url):
+        return urlparse(url)
+
+    def base_url(self, url):
+        """Return url without querystring or fragment"""
+        parts = list(self.parse_url(url))
+        parts[4:] = ['' for i in parts[4:]]
+        return urlunparse(parts)
+
 
 def filter_installed_packages(packages):
     """Returns a list of packages that require installation"""
     apt_pkg.init()
+
+    # Tell apt to build an in-memory cache to prevent race conditions (if
+    # another process is already building the cache).
+    apt_pkg.config.set("Dir::Cache::pkgcache", "")
+
     cache = apt_pkg.Cache()
     _pkgs = []
     for package in packages:
@@ -87,14 +141,7 @@
         cmd.extend(packages)
     log("Installing {} with options: {}".format(packages,
                                                 options))
-    env = os.environ.copy()
-    if 'DEBIAN_FRONTEND' not in env:
-        env['DEBIAN_FRONTEND'] = 'noninteractive'
-
-    if fatal:
-        subprocess.check_call(cmd, env=env)
-    else:
-        subprocess.call(cmd, env=env)
+    _run_apt_command(cmd, fatal)
 
 
 def apt_upgrade(options=None, fatal=False, dist=False):
@@ -109,24 +156,13 @@
     else:
         cmd.append('upgrade')
     log("Upgrading with options: {}".format(options))
-
-    env = os.environ.copy()
-    if 'DEBIAN_FRONTEND' not in env:
-        env['DEBIAN_FRONTEND'] = 'noninteractive'
-
-    if fatal:
-        subprocess.check_call(cmd, env=env)
-    else:
-        subprocess.call(cmd, env=env)
+    _run_apt_command(cmd, fatal)
 
 
 def apt_update(fatal=False):
     """Update local apt cache"""
     cmd = ['apt-get', 'update']
-    if fatal:
-        subprocess.check_call(cmd)
-    else:
-        subprocess.call(cmd)
+    _run_apt_command(cmd, fatal)
 
 
 def apt_purge(packages, fatal=False):
@@ -137,10 +173,7 @@
     else:
         cmd.extend(packages)
     log("Purging {}".format(packages))
-    if fatal:
-        subprocess.check_call(cmd)
-    else:
-        subprocess.call(cmd)
+    _run_apt_command(cmd, fatal)
 
 
 def apt_hold(packages, fatal=False):
@@ -151,6 +184,7 @@
     else:
         cmd.extend(packages)
     log("Holding {}".format(packages))
+
     if fatal:
         subprocess.check_call(cmd)
     else:
@@ -184,14 +218,10 @@
             apt.write(PROPOSED_POCKET.format(release))
     if key:
         subprocess.check_call(['apt-key', 'adv', '--keyserver',
-                               'keyserver.ubuntu.com', '--recv',
+                               'hkp://keyserver.ubuntu.com:80', '--recv',
                                key])
 
 
-class SourceConfigError(Exception):
-    pass
-
-
 def configure_sources(update=False,
                       sources_var='install_sources',
                       keys_var='install_keys'):
@@ -224,17 +254,6 @@
     if update:
         apt_update(fatal=True)
 
-# The order of this list is very important. Handlers should be listed in from
-# least- to most-specific URL matching.
-FETCH_HANDLERS = (
-    'charmhelpers.fetch.archiveurl.ArchiveUrlFetchHandler',
-    'charmhelpers.fetch.bzrurl.BzrUrlFetchHandler',
-)
-
-
-class UnhandledSource(Exception):
-    pass
-
 
 def install_remote(source):
     """
@@ -265,30 +284,6 @@
     return install_remote(source)
 
 
-class BaseFetchHandler(object):
-
-    """Base class for FetchHandler implementations in fetch plugins"""
-
-    def can_handle(self, source):
-        """Returns True if the source can be handled. Otherwise returns
-        a string explaining why it cannot"""
-        return "Wrong source type"
-
-    def install(self, source):
-        """Try to download and unpack the source. Return the path to the
-        unpacked files or raise UnhandledSource."""
-        raise UnhandledSource("Wrong source type {}".format(source))
-
-    def parse_url(self, url):
-        return urlparse(url)
-
-    def base_url(self, url):
-        """Return url without querystring or fragment"""
-        parts = list(self.parse_url(url))
-        parts[4:] = ['' for i in parts[4:]]
-        return urlunparse(parts)
-
-
 def plugins(fetch_handlers=None):
     if not fetch_handlers:
         fetch_handlers = FETCH_HANDLERS
@@ -306,3 +301,40 @@
             log("FetchHandler {} not found, skipping plugin".format(
                 handler_name))
     return plugin_list
+
+
+def _run_apt_command(cmd, fatal=False):
+    """
+    Run an APT command, checking output and retrying if the fatal flag is set
+    to True.
+
+    :param: cmd: str: The apt command to run.
+    :param: fatal: bool: Whether the command's output should be checked and
+        retried.
+    """
+    env = os.environ.copy()
+
+    if 'DEBIAN_FRONTEND' not in env:
+        env['DEBIAN_FRONTEND'] = 'noninteractive'
+
+    if fatal:
+        retry_count = 0
+        result = None
+
+        # If the command is considered "fatal", we need to retry if the apt
+        # lock was not acquired.
+
+        while result is None or result == APT_NO_LOCK:
+            try:
+                result = subprocess.check_call(cmd, env=env)
+            except subprocess.CalledProcessError, e:
+                retry_count = retry_count + 1
+                if retry_count > APT_NO_LOCK_RETRY_COUNT:
+                    raise
+                result = e.returncode
+                log("Couldn't acquire DPKG lock. Will retry in {} seconds."
+                    "".format(APT_NO_LOCK_RETRY_DELAY))
+                time.sleep(APT_NO_LOCK_RETRY_DELAY)
+
+    else:
+        subprocess.call(cmd, env=env)

=== modified file 'hooks/hooks.py'
--- hooks/hooks.py	2014-05-08 11:54:54 +0000
+++ hooks/hooks.py	2014-05-20 21:57:20 +0000
@@ -5,10 +5,10 @@
 import sys
 from charmhelpers.core.hookenv import log
 from charmhelpers.contrib.cloudfoundry import contexts
-from charmhelpers.contrib.cloudfoundry import services
+from charmhelpers.core import services
+from charmhelpers.core import templating
 from charmhelpers.core import hookenv
-from charmhelpers.core.hookenv import unit_get
-from config import *
+import config
 
 
 LOGSVC = "loggregator"
@@ -19,18 +19,19 @@
 fileproperties = {'owner': 'vcap'}
 services.register([
     {
-        'service': LOGGREGATOR_JOB_NAME,
-        'templates': [{
-            'source': 'loggregator.json',
-            'target': LOGGREGATOR_CONFIG_PATH,
-            'file_properties': fileproperties,
-            'contexts': [
-                contexts.StaticContext({'service_name': service_name}),
-                contexts.ConfigContext(),
-                contexts.NatsContext()
-            ]
-        }]
-    }], os.path.join(hookenv.charm_dir(), 'templates'))
+        'service': config.LOGGREGATOR_JOB_NAME,
+        'templates': [
+            {'source': 'loggregator.conf'},
+            {'source': 'loggregator.json',
+             'target': config.LOGGREGATOR_CONFIG_PATH,
+             'file_properties': fileproperties,
+             'contexts': [
+                 templating.StaticContext({'service_name': service_name}),
+                 templating.ConfigContext(),
+                 contexts.NatsContext()
+             ]},
+        ]
+    }])
 
 
 @hooks.hook()

=== modified file 'hooks/install' (properties changed: -x to +x)
--- hooks/install	2014-05-08 11:47:19 +0000
+++ hooks/install	2014-05-20 21:57:20 +0000
@@ -2,21 +2,20 @@
 # vim: et ai ts=4 sw=4:
 import os
 import subprocess
-from config import *
 from charmhelpers.core import hookenv, host
 from charmhelpers.contrib.cloudfoundry.common import (
     prepare_cloudfoundry_environment
 )
-from charmhelpers.contrib.cloudfoundry.upstart_helper import (
-    install_upstart_scripts
-)
-from charmhelpers.contrib.cloudfoundry.install import install
+
+import config
 
 
 def install_charm():
-    prepare_cloudfoundry_environment(hookenv.config(), LOGGREGATOR_PACKAGES)
-    install_upstart_scripts()
-    dirs = [CF_DIR, LOGGREGATOR_DIR, LOGGREGATOR_CONFIG_DIR, '/var/log/vcap']
+    prepare_cloudfoundry_environment(hookenv.config(), config.LOGGREGATOR_PACKAGES)
+    dirs = [config.CF_DIR,
+            config.LOGGREGATOR_DIR,
+            config.LOGGREGATOR_CONFIG_DIR,
+            '/var/log/vcap']
 
     for item in dirs:
         host.mkdir(item, owner='vcap', group='vcap', perms=0775)
@@ -24,10 +23,9 @@
     files_dir = os.path.join(hookenv.charm_dir(), "files")
 
     subprocess.check_call(["gunzip", "-k", "loggregator.gz"], cwd=files_dir)
-
-    install(os.path.join(files_dir, 'loggregator'),
-            "/usr/bin/loggregator",
-            fileprops={'mode': '0755', 'owner': 'vcap'})
+    host.copy_file(os.path.join(files_dir, 'loggregator'),
+                   '/usr/bin/loggregator',
+                   owner='vcap', perms=0755)
 
 
 if __name__ == '__main__':

=== renamed file 'files/upstart/loggregator.conf' => 'templates/loggregator.conf'