Merge lp:~aluria/charms/trusty/collectd/charm-collectd-allinone into lp:~canonical-bootstack/charms/trusty/collectd/pre-layer

[Alvaro Uria]

=== modified file 'config.yaml'
--- config.yaml	2015-12-10 15:09:45 +0000
+++ config.yaml	2015-12-29 16:06:20 +0000
@@ -42,30 +42,6 @@
     type: string
     default: '/metrics'
     description: "Path under which to expose Prometheus metrics"
-  openstack-enable-keystone:
-    type: boolean
-    default: true
-    description: "load keystone plugin"
-  openstack-keystone-vip:
-    type: string
-    default: '172.20.168.103'
-    description: "keystone entrypoint"
-  openstack-keystone-port:
-    type: int
-    default: 5000
-    description: "port where keystone is listening"
-  openstack-username:
-    type: string
-    default: 'admin'
-    description: "keystone admin user"
-  openstack-password:
-    type: string
-    default: 'xe5iej2AiSaed5Tu'
-    description: "keystone admin password"
-  openstack-tenant-name:
-    type: string
-    default: 'admin'
-    description: "keystone admin tenant"
   openstack-keystone-verbose:
     type: string
     default: 'False'
 
=== modified file 'hooks/collectd_contexts.py'
--- hooks/collectd_contexts.py	2015-12-10 17:25:19 +0000
+++ hooks/collectd_contexts.py	2015-12-29 16:06:20 +0000
@@ -4,18 +4,110 @@
 #from charmhelpers.fetch import apt_install, filter_installed_packages
 from charmhelpers.core.hookenv import (
     config,
+    log,
+    related_units,
+    relation_get,
+    relation_ids,
 )
 
 from socket import gethostname
 
 
 class CollectdContext(object):
-    interfaces = ['target']
 
     def __call__(self):
+        conf = config()
         ctxt = {
             'hostname': gethostname(),
-            'interval': config('interval'),
+            'interval': conf['interval'],
+        }
+
+        return ctxt
+
+
+class CollectdExporterContext(object):
+    interfaces = ['target']
+
+    def __call__(self):
+        conf = config()
+        ctxt = {
+            'host': conf['host'],
+            'port': conf['port'],
+            'push_path': conf['push-path'],
+            'format': conf['format'],
+            'store_rates': conf['store-rates']
+        }
+
+        return ctxt
+
+
+class CollectdExporterDefaultContext(object):
+    interfaces = ['target']
+
+    def __call__(self):
+        conf = config()
+        ctxt = {
+            'host': conf['host'],
+            'port': conf['port'],
+            'push_path': conf['push-path'],
+            'telemetry_path': conf['telemetry-path']
+        }
+
+        return ctxt
+
+
+class CollectdOpenstackContext(object):
+    interfaces = ['identity-admin']
+
+    def keystone_values(self):
+        """{u'service_region': u'RegionOne', u'service_password': u'XXXXXXXX',
+          u'service_port': u'5000', u'private-address': u'10.XX.XX.XXX',
+          u'service_hostname': u'10.XX.XX.XXX', u'service_username': u'admin',
+          u'service_tenant_name': u'Admin'}
+        """
+        r_id = relation_ids('identity-admin')
+        if len(r_id) != 1:
+            log('keystone:identity-admin changed but not rel-id found')
+            return None
+
+        r_unit = related_units(r_id[0])
+        if r_unit:
+            settings = relation_get(unit=r_unit[0], rid=r_id[0])
+            log('settings(rid={})=> {}'.format(str(r_id[0]), settings))
+            if 'service_password' in settings:
+                hostname = settings.get('service_hostname', None)
+                port = settings.get('service_port', None)
+                username = settings.get('service_username', None)
+                passwd = settings.get('service_password', None)
+                tenantname = settings.get('service_tenant_name', None)
+            else:
+                return None
+
+        if hostname and port and username and passwd and tenantname:
+            return (hostname, port, username, passwd, tenantname)
+        else:
+            return None
+
+    def __call__(self):
+        conf = config()
+        ks_vals = self.keystone_values()
+        if ks_vals and len(filter(lambda x: x, ks_vals)) == len(ks_vals):
+            enable_ks = True
+        else:
+            enable_ks = False
+            ks_vals = (None, None, None, None, None)
+
+        ctxt = {
+            'enable_keystone': enable_ks,
+            'keystone_vip': ks_vals[0],
+            'keystone_port': ks_vals[1],
+            'username': ks_vals[2],
+            'password': ks_vals[3],
+            'tenant_name': ks_vals[4],
+            'keystone_verbose': conf['openstack-keystone-verbose'],
+            'debug': conf['openstack-debug'],
+            'allocation_ratio_cores': conf['openstack-allocation-ratio-cores'],
+            'allocation_ratio_ram': conf['openstack-allocation-ratio-ram']
         }
 
         return ctxt
 
=== modified file 'hooks/collectd_hooks.py'
--- hooks/collectd_hooks.py	2015-12-11 17:29:48 +0000
+++ hooks/collectd_hooks.py	2015-12-29 16:06:20 +0000
@@ -9,9 +9,9 @@
     #INFO,
     #ERROR,
     #local_unit,
-    relation_ids,
-    relation_get,
-    related_units,
+    #relation_ids,
+    #relation_get,
+    #related_units,
     relation_set,
     #service_name,
     unit_get,
@@ -26,7 +26,7 @@
 )
 
 from charmhelpers.fetch import (
-    #add_source,
+    add_source,
     apt_install,
     #apt_purge,
     apt_update,
@@ -38,31 +38,33 @@
 
 from collectd_utils import (
     determine_packages,
+    determine_exporter_packages,
+    determine_openstack_packages,
     juju_log,
     register_configs,
     restart_map,
 )
 
-#from socket import gethostname
-
 hooks = Hooks()
-#CONFIGS = register_configs()
 
 
 @hooks.hook()
 def install():
     execd_preinstall()
+    if config('source'):
+        for s in config('source').split(';'):
+            add_source(s)
     apt_update()
     apt_install(determine_packages(), fatal=True)
 
 
 @hooks.hook('start')
 def start():
+    if is_relation_made('target'):
+        service_start('collectd-exporter-prometheus')
+
     service_start('collectd')
 
-    if is_relation_made('target'):
-        service_start('collectd-exporter-prometheus')
-
 
 @hooks.hook('stop')
 def stop():
@@ -72,7 +74,7 @@
         service_stop('collectd-exporter-prometheus')
 
 
[e-mail]('config-changed')
[e-mail]('config-changed', 'upgrade-charm')
 @restart_on_change(restart_map())
 def config_changed():
     register_configs()
@@ -83,24 +85,36 @@
     if is_relation_made('target'):
         juju_log('config-changed for collectd-exporter-prometheus')
 
-    if is_relation_made('identity-service'):
+    if is_relation_made('identity-admin'):
         juju_log('config-changed for collectd-openstack')
 
 
[e-mail]('upgrade-charm')
-def upgrade_charm():
-    pass
-
-
 # prometheus <-> collectd
 # collectd-exporter IP:port
[e-mail]('target-relation-changed', 'target-relation-joined')
[e-mail]('target-relation-changed')
 def target_relation():
     if is_relation_made('target'):
         host = unit_get('public-address')
         conf = config()
         if conf['exporter-host'] != '127.0.0.1':
             relation_set(host=str(host), port=conf['exporter-port'])
+        register_configs()
+
+
[e-mail]('target-relation-joined')
+def target_joined():
+    if is_relation_made('target'):
+        apt_install(determine_exporter_packages(), fatal=True)
+        target_relation()
+
+
[e-mail]('identity-admin-relation-joined')
+def keystone_joined():
+    if is_relation_made('identity-admin'):
+        apt_install(determine_openstack_packages(), fatal=True)
+        register_configs()
+        juju_log('collectd-openstack module should be loaded with the '
+                 'info provided by keystone:identity-admin')
 
 
 # collectd <-> keystone
@@ -108,33 +122,9 @@
 @hooks.hook('identity-admin-relation-changed')
 @restart_on_change(restart_map())
 def identity_admin_relation_changed():
-    #CONFIGS.write_all()
     keystone_joined()
 
 
[e-mail]('identity-admin-relation-joined')
-def keystone_joined():
-    """
-    service_hostname
-    service_port
-    service_username
-    service_tenant_name
-    service_region
-    service_password
-    """
-    r_id = relation_ids('identity-admin')
-    if len(r_id) != 1:
-        juju_log('keystone:identity-admin changed by not rel-id found')
-        return
-
-    for r_unit in related_units(r_id[0]):
-        settings = relation_get(unit=r_unit, rid=r_id[0])
-        juju_log('settings(rid={})=> {}'.format(str(r_id[0]), settings))
-
-    juju_log('collectd-openstack module should be loaded with the '
-             'info provided by keystone:identity-admin')
-
-
 def main():
     try:
         hooks.execute(sys.argv)
 
=== modified file 'hooks/collectd_utils.py'
--- hooks/collectd_utils.py	2015-12-10 17:25:19 +0000
+++ hooks/collectd_utils.py	2015-12-29 16:06:20 +0000
@@ -24,6 +24,7 @@
 from charmhelpers.core.hookenv import (
     #charm_dir,
     #config,
+    is_relation_made,
     log,
     #related_units,
     #relation_ids,
@@ -37,6 +38,9 @@
 
 from collectd_contexts import (
     CollectdContext,
+    CollectdExporterContext,
+    CollectdExporterDefaultContext,
+    CollectdOpenstackContext,
 )
 
 TEMPLATES = 'templates/'
@@ -45,9 +49,23 @@
     'collectd-core',
 ]
 
+COLLECTD_EXPORTER_PACKAGES = [
+    'canonical-bootstack-collectd-exporter',
+]
+
+COLLECTD_OPENSTACK_PACKAGES = [
+    'canonical-bootstack-collectd-openstack',
+]
+
 COLLECTD_CONF_DIR = "/etc/collectd"
 COLLECTD_CONF = '%s/collectd.conf' % COLLECTD_CONF_DIR
-
+COLLECTD_CONF_INC = '%s/collectd.d' % COLLECTD_CONF_DIR
+
+
+COLLECTD_EXPORTER_CONF = '%s/write_http.conf' % COLLECTD_CONF_INC
+COLLECTD_EXPORTER_DEFAULT = '/etc/default/collectd-exporter-prometheus'
+
+COLLECTD_OPENSTACK_CONF = '%s/openstack.conf' % COLLECTD_CONF_INC
 
 BASE_RESOURCE_MAP = {
     COLLECTD_CONF: {
@@ -56,6 +74,26 @@
     },
 }
 
+EXPORTER_RESOURCE_MAP = {
+    COLLECTD_EXPORTER_CONF: {
+        'services': ['collectd'],
+        'templates': ['write_http.conf.tmpl'],
+        'contexts': [CollectdExporterContext()],
+    },
+    COLLECTD_EXPORTER_DEFAULT: {
+        'services': ['collectd-exporter-prometheus'],
+        'templates': ['default.tmpl'],
+        'contexts': [CollectdExporterDefaultContext()],
+    },
+}
+
+OPENSTACK_RESOURCE_MAP = {
+    COLLECTD_OPENSTACK_CONF: {
+        'services': ['collectd'],
+        'contexts': [CollectdOpenstackContext()],
+    },
+}
+
 
 def juju_log(msg):
     log('[collectd] {}'.format(msg))
@@ -66,7 +104,16 @@
     Dynamically generate a map of resources that will be managed for a single
     hook execution.
     '''
-    resource_map = deepcopy(BASE_RESOURCE_MAP)
+    d_resource = {}
+    d_resource.update(BASE_RESOURCE_MAP)
+
+    if is_relation_made('target'):
+        d_resource.update(EXPORTER_RESOURCE_MAP)
+
+    if is_relation_made('identity-admin'):
+        d_resource.update(OPENSTACK_RESOURCE_MAP)
+
+    resource_map = deepcopy(d_resource)
 
     return resource_map
 
@@ -103,3 +150,13 @@
     packages = [] + BASE_PACKAGES
 
     return packages
+
+
+def determine_exporter_packages():
+    packages = [] + COLLECTD_EXPORTER_PACKAGES
+    return packages
+
+
+def determine_openstack_packages():
+    packages = [] + COLLECTD_OPENSTACK_PACKAGES
+    return packages
 
=== added file 'templates/default.tmpl'
--- templates/default.tmpl	1970-01-01 00:00:00 +0000
+++ templates/default.tmpl	2015-12-29 16:06:20 +0000
@@ -0,0 +1,10 @@
+# Defaults for canonical-bootstack-collectd-exporter initscript
+#
+
+# Path under which to accept POST requests from collectd.
+PUSH_PATH='{{ push_path }}'
+# Address on which to expose metrics and web interface.
+LISTEN_ADDR='{{ host }}:{{ port }}'
+# Path under which to expose Prometheus metrics.
+TELEMETRY_PATH='{{ telemetry_path }}'
+
 
=== added file 'templates/openstack.conf.tmpl'
--- templates/openstack.conf.tmpl	1970-01-01 00:00:00 +0000
+++ templates/openstack.conf.tmpl	2015-12-29 16:06:20 +0000
@@ -0,0 +1,62 @@
+TypesDB "/usr/share/collectd/types.db"
+TypesDB "/usr/share/collectd/ostypes.db"
+<LoadPlugin "python">
+    Globals true
+</LoadPlugin>
+
+<Plugin "python">
+    ModulePath "/usr/lib/collectd/plugins/openstack"
+
+    Import "keystone_plugin"
+    <Module "keystone_plugin">
+        Username "{{ username }}"
+        Password "{{ password }}"
+        TenantName "{{ tenant_name }}"
+        AuthURL "http://{{ keystone_vip }}:{{ keystone_port }}/v2.0"
+        Verbose "{{ keystone_verbose }}"
+        Debug "{{ debug }}"
+    </Module>
+    Import "cinder_plugin"
+    <Module "cinder_plugin">
+        Username "{{ username }}"
+        Password "{{ password }}"
+        TenantName "{{ tenant_name }}"
+        AuthURL "http://{{ keystone_vip }}:{{ keystone_port }}/v2.0"
+        Verbose "{{ keystone_verbose }}"
+        Debug "{{ debug }}"
+    </Module>
+    Import "glance_plugin"
+    <Module "glance_plugin">
+        Username "{{ username }}"
+        Password "{{ password }}"
+        TenantName "{{ tenant_name }}"
+        AuthURL "http://{{ keystone_vip }}:{{ keystone_port }}/v2.0"
+        Verbose "{{ keystone_verbose }}"
+        Debug "{{ debug }}"
+    </Module>
+    Import "neutron_plugin"
+    <Module "neutron_plugin">
+        Username "{{ username }}"
+        Password "{{ password }}"
+        TenantName "{{ tenant_name }}"
+        AuthURL "http://{{ keystone_vip }}:{{ keystone_port }}/v2.0"
+        Verbose "{{ keystone_verbose }}"
+        Debug "{{ debug }}"
+    </Module>
+    Import "nova_plugin"
+    <Module "nova_plugin">
+        Username "{{ username }}"
+        Password "{{ password }}"
+        TenantName "{{ tenant_name }}"
+        AuthURL "http://{{ keystone_vip }}:{{ keystone_port }}/v2.0"
+        Verbose "{{ keystone_verbose }}"
+        Debug "{{ debug }}"
+        AllocationRatioCores "{{ allocation_ratio_cores }}"
+        AllocationRatioRam "{{ allocation_ratio_ram }}"
+        ReservedNodeCores "0"
+        ReservedNodeRamMB "0"
+        ReservedCores "0"
+        ReservedRamMB "0"
+    </Module>
+</Plugin>
+#
 
=== added file 'templates/write_http.conf.tmpl'
--- templates/write_http.conf.tmpl	1970-01-01 00:00:00 +0000
+++ templates/write_http.conf.tmpl	2015-12-29 16:06:20 +0000
@@ -0,0 +1,8 @@
+LoadPlugin write_http
+<Plugin write_http>
+  <URL "http://127.0.0.1:{{ port }}{{ push_path }}">
+    Format "{{ format }}"
+    StoreRates {{ store_rates }}
+  </URL>
+</Plugin>
+#
 
=== modified file 'unit_tests/test_collectd_contexts.py'
--- unit_tests/test_collectd_contexts.py	2015-12-10 17:25:19 +0000
+++ unit_tests/test_collectd_contexts.py	2015-12-29 16:06:20 +0000
@@ -1,5 +1,6 @@
 import collectd_contexts as contexts
 import os
+from collections import defaultdict
 
 os.environ['JUJU_UNIT_NAME'] = 'collectd'
 #import collectd_utils as utils
@@ -13,6 +14,9 @@
 TO_PATCH = [
     'gethostname',
     'config',
+    'related_units',
+    'relation_get',
+    'relation_ids',
 ]
 
 
@@ -22,11 +26,64 @@
         super(TestCollectdContext, self).setUp(contexts, TO_PATCH)
 
     def test_collectd_context(self):
-        #self.relation_ids.return_value = ['target:0']
         self.gethostname.return_value = '1'
-        self.config.return_value = '1'
-        #service = 'mycollectd'
-        #self.service_name.return_value = service
+        self.config.return_value = defaultdict(lambda: '1')
         self.assertEquals(contexts.CollectdContext()(),
                           {'hostname': '1',
                            'interval': '1'})
+
+    def test_collectd_exporter_context(self):
+        self.config.return_value = defaultdict(lambda: '1')
+        self.assertEquals(contexts.CollectdExporterContext()(),
+                          {'host': '1',
+                           'port': '1',
+                           'push_path': '1',
+                           'format': '1',
+                           'store_rates': '1'})
+
+    def test_collectd_exporter_default_context(self):
+        self.gethostname.return_value = '1'
+        self.config.return_value = defaultdict(lambda: '1')
+        self.assertEquals(contexts.CollectdExporterDefaultContext()(),
+                          {'host': '1',
+                           'port': '1',
+                           'push_path': '1',
+                           'telemetry_path': '1'})
+
+    def test_collectd_openstack_context(self):
+        self.relation_ids.return_value = ['identity-admin:0']
+        self.related_units.return_value = ['collectd/0']
+        self.relation_get.return_value = {'service_hostname': '1',
+                                          'service_port': '1',
+                                          'service_username': '1',
+                                          'service_password': '1',
+                                          'service_tenant_name': '1'}
+        self.config.return_value = defaultdict(lambda: '1')
+        self.assertEquals(contexts.CollectdOpenstackContext()(),
+                          {'enable_keystone': True,
+                           'keystone_vip': '1',
+                           'keystone_port': '1',
+                           'username': '1',
+                           'password': '1',
+                           'tenant_name': '1',
+                           'keystone_verbose': '1',
+                           'debug': '1',
+                           'allocation_ratio_cores': '1',
+                           'allocation_ratio_ram': '1'})
+
+    def test_collectd_openstack_context_nodata(self):
+        self.relation_ids.return_value = ['identity-admin:0']
+        self.related_units.return_value = ['collectd/0']
+        self.relation_get.return_value = {}
+        self.config.return_value = defaultdict(lambda: '1')
+        self.assertEquals(contexts.CollectdOpenstackContext()(),
+                          {'enable_keystone': False,
+                           'keystone_vip': None,
+                           'keystone_port': None,
+                           'username': None,
+                           'password': None,
+                           'tenant_name': None,
+                           'keystone_verbose': '1',
+                           'debug': '1',
+                           'allocation_ratio_cores': '1',
+                           'allocation_ratio_ram': '1'})
 
=== modified file 'unit_tests/test_collectd_hooks.py'
--- unit_tests/test_collectd_hooks.py	2015-12-10 17:25:19 +0000
+++ unit_tests/test_collectd_hooks.py	2015-12-29 16:06:20 +0000
@@ -34,12 +34,12 @@
     'register_configs',
     'restart_map',
     # charmhelpers.core.hookenv
-    #'config',
+    'config',
     'is_relation_made',
-    'relation_get',
-    'relation_ids',
+    #'relation_get',
+    #'relation_ids',
     'relation_set',
-    'related_units',
+    #'related_units',
     'unit_get',
     # charmhelpers.core.host
     'apt_install',
@@ -57,5 +57,6 @@
     def test_correct_install_packages(self):
         'It installs the correct packages based on what is determined'
         self.determine_packages.return_value = ['collectd']
+        self.config.return_value = ''
         hooks.hooks.execute(['hooks/install'])
         self.apt_install.assert_called_with(['collectd'], fatal=True)
 
=== modified file 'unit_tests/test_collectd_utils.py'
--- unit_tests/test_collectd_utils.py	2015-12-10 17:25:19 +0000
+++ unit_tests/test_collectd_utils.py	2015-12-29 16:06:20 +0000
@@ -14,6 +14,7 @@
     # helpers.core.hookenv
     #'config',
     'log',
+    'is_relation_made',
     # fetch
     #'add_source',
     #'apt_update',
@@ -38,12 +39,18 @@
 
     def test_creates_restart_map_all_enabled(self):
         'It creates correct restart map when all services enabled'
+        self.is_relation_made.return_value = True
         ex_map = OrderedDict([
             ('/etc/collectd/collectd.conf', ['collectd']),
+            ('/etc/collectd/collectd.d/write_http.conf', ['collectd']),
+            ('/etc/default/collectd-exporter-prometheus',
+             ['collectd-exporter-prometheus']),
+            ('/etc/collectd/collectd.d/openstack.conf', ['collectd']),
         ])
         self.assertEquals(collectd_utils.restart_map(), ex_map)
 
     def test_list_services(self):
         'It returns a list of services associate with this charm'
-        ex_list = ['collectd']
-        self.assertEquals(collectd_utils.services(), ex_list)
+        self.is_relation_made.return_value = True
+        ex_list = ['collectd', 'collectd-exporter-prometheus']
+        self.assertEquals(sorted(collectd_utils.services()), sorted(ex_list))