Merge lp:~james-page/charms/trusty/heat/liberty-fixes into lp:~openstack-charmers-archive/charms/trusty/heat/next

[James Page]

Lots of fixes for kilo and liberty support:

1) new template for liberty heat

2) generation of openrc file with credentials and setup for keystone v3

3) new action to configure required domains and accounts for heat stacks.

This change requires leader election - the charm will error out on older juju versions.

[James Page (james-page)]

Re-testing on kilo as well

[James Page (james-page)]

Tested OK on Kilo; needs a different template for heat...

[Liam Young (gnuoy)]

lgtm

review: Approve

[Ryan Beisner (1chb1n)]

FYI - this broke the amulet tests @ Kilo.

=== modified file 'README.md'
--- README.md	2013-11-29 15:36:45 +0000
+++ README.md	2016-02-10 14:39:12 +0000
@@ -1,8 +1,8 @@
 Overview
 ========
 
-Heat is the main project in the OpenStack Orchestration program. It implements 
-an orchestration engine to launch multiple composite cloud applications based 
+Heat is the main project in the OpenStack Orchestration program. It implements
+an orchestration engine to launch multiple composite cloud applications based
 on templates in the form of text files that can be treated like code.
 
 This charm deploys the Heat infrastructure.
@@ -15,6 +15,13 @@
 nova-cloud-controller. The following assumes these services have already
 been deployed.
 
+After deployment of the cloud, the domain-setup action must be run to configure
+required domains, roles and users in the cloud for Heat stacks:
+
+    juju action do heat/0 domain-setup
+
+This is only required for >= OpenStack Kilo.
+
 Contact Information
 ===================
 

=== modified file 'actions.yaml'
--- actions.yaml	2015-09-21 19:35:41 +0000
+++ actions.yaml	2016-02-10 14:39:12 +0000
@@ -2,3 +2,6 @@
   description:
     Perform openstack upgrades. Config option action-managed-upgrade must be
     set to True.
+domain-setup:
+  description:
+    Setup the keystone domains, roles and user required for Heat to operate. Only required for OpenStack >= Kilo.

=== added file 'actions/domain-setup'
--- actions/domain-setup	1970-01-01 00:00:00 +0000
+++ actions/domain-setup	2016-02-10 14:39:12 +0000
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+. /root/admin-openrc-v3
+
+dpkg -l | grep python-openstackclient || apt-get install -y python-openstackclient
+
+openstack domain show heat || {
+    openstack domain create --description "Stack projects and users" heat
+}
+
+openstack user show heat_domain_admin || {
+    openstack user create --domain heat --password `leader-get heat-domain-admin-passwd` heat_domain_admin
+}
+
+openstack role add --domain heat --user heat_domain_admin admin || :
+
+openstack role show heat_stack_user || {
+    openstack role create heat_stack_user
+}

=== modified file 'hooks/heat_context.py'
--- hooks/heat_context.py	2015-05-06 14:03:04 +0000
+++ hooks/heat_context.py	2016-02-10 14:39:12 +0000
@@ -1,7 +1,7 @@
 import os
 
 from charmhelpers.contrib.openstack import context
-from charmhelpers.core.hookenv import config
+from charmhelpers.core.hookenv import config, leader_get
 from charmhelpers.core.host import pwgen
 from charmhelpers.contrib.hahelpers.cluster import (
     determine_apache_port,
@@ -32,6 +32,7 @@
                                          ctxt['service_host'],
                                          ctxt['service_port'])
         ctxt['keystone_ec2_url'] = ec2_tokens
+        ctxt['region'] = config('region')
         return ctxt
 
 
@@ -53,14 +54,15 @@
     return encryption
 
 
-class EncryptionContext(context.OSContextGenerator):
+class HeatSecurityContext(context.OSContextGenerator):
 
     def __call__(self):
         ctxt = {}
-
         # check if we have stored encryption key
         encryption = get_encryption_key()
         ctxt['encryption_key'] = encryption
+        ctxt['heat_domain_admin_passwd'] = \
+            leader_get('heat-domain-admin-passwd')
         return ctxt
 
 

=== modified file 'hooks/heat_relations.py'
--- hooks/heat_relations.py	2015-10-30 11:22:56 +0000
+++ hooks/heat_relations.py	2016-02-10 14:39:12 +0000
@@ -23,11 +23,15 @@
     open_port,
     unit_get,
     status_set,
+    leader_get,
+    leader_set,
+    is_leader,
 )
 
 from charmhelpers.core.host import (
     restart_on_change,
     service_reload,
+    pwgen,
 )
 
 from charmhelpers.fetch import (
@@ -109,6 +113,11 @@
     configure_https()
 
 
[e-mail]('upgrade-charm')
+def upgrade_charm():
+    leader_elected()
+
+
 @hooks.hook('amqp-relation-joined')
 def amqp_joined(relation_id=None):
     relation_set(relation_id=relation_id,
@@ -216,6 +225,12 @@
     CONFIGS.write_all()
 
 
[e-mail]('leader-elected')
+def leader_elected():
+    if is_leader() and not leader_get('heat-domain-admin-passwd'):
+        leader_set({'heat-domain-admin-passwd': pwgen(32)})
+
+
 def main():
     try:
         hooks.execute(sys.argv)

=== modified file 'hooks/heat_utils.py'
--- hooks/heat_utils.py	2015-12-02 15:50:33 +0000
+++ hooks/heat_utils.py	2016-02-10 14:39:12 +0000
@@ -37,7 +37,7 @@
 from heat_context import (
     API_PORTS,
     HeatIdentityServiceContext,
-    EncryptionContext,
+    HeatSecurityContext,
     InstanceUserContext,
     HeatApacheSSLContext,
     HeatHAProxyContext,
@@ -76,6 +76,7 @@
 HTTPS_APACHE_CONF = '/etc/apache2/sites-available/openstack_https_frontend'
 HTTPS_APACHE_24_CONF = os.path.join('/etc/apache2/sites-available',
                                     'openstack_https_frontend.conf')
+ADMIN_OPENRC = '/root/admin-openrc-v3'
 
 CONFIG_FILES = OrderedDict([
     (HEAT_CONF, {
@@ -86,7 +87,7 @@
                      context.OSConfigFlagContext(),
                      HeatIdentityServiceContext(service=SVC, service_user=SVC),
                      HeatHAProxyContext(),
-                     EncryptionContext(),
+                     HeatSecurityContext(),
                      InstanceUserContext(),
                      context.SyslogContext(),
                      context.LogLevelContext(),
@@ -108,7 +109,12 @@
     (HTTPS_APACHE_24_CONF, {
         'contexts': [HeatApacheSSLContext()],
         'services': ['apache2'],
-    })
+    }),
+    (ADMIN_OPENRC, {
+        'contexts': [HeatIdentityServiceContext(service=SVC,
+                                                service_user=SVC)],
+        'services': []
+    }),
 ])
 
 
@@ -117,7 +123,7 @@
     configs = templating.OSConfigRenderer(templates_dir=TEMPLATES,
                                           openstack_release=release)
 
-    confs = [HEAT_CONF, HEAT_API_PASTE, HAPROXY_CONF]
+    confs = [HEAT_CONF, HEAT_API_PASTE, HAPROXY_CONF, ADMIN_OPENRC]
     for conf in confs:
         configs.register(conf, CONFIG_FILES[conf]['contexts'])
 

=== added symlink 'hooks/leader-elected'
=== target is u'heat_relations.py'
=== added file 'templates/admin-openrc-v3'
--- templates/admin-openrc-v3	1970-01-01 00:00:00 +0000
+++ templates/admin-openrc-v3	2016-02-10 14:39:12 +0000
@@ -0,0 +1,10 @@
+# Service credentials for Keystone v3
+export OS_REGION_NAME={{ region }}
+export OS_PASSWORD={{ admin_password }}
+export OS_AUTH_URL={{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}/v3
+export OS_USERNAME={{ admin_user }}
+export OS_TENANT_NAME={{ admin_tenant_name }}
+export OS_PROJECT_DOMAIN_ID=default
+export OS_USER_DOMAIN_ID=default
+export OS_PROJECT_NAME={{ admin_tenant_name }}
+export OS_IDENTITY_API_VERSION=3

=== modified file 'templates/kilo/heat.conf'
--- templates/kilo/heat.conf	2015-12-02 16:06:55 +0000
+++ templates/kilo/heat.conf	2016-02-10 14:39:12 +0000
@@ -1,64 +1,37 @@
 [DEFAULT]
 use_syslog = {{ use_syslog }}
-debug = False
-verbose = False
+debug = {{ debug }}
+verbose = {{ verbose }}
 log_dir = /var/log/heat
-instance_user={{ instance_user }}
-instance_driver=heat.engine.nova
-plugin_dirs=/usr/lib64/heat,/usr/lib/heat
-environment_dir=/etc/heat/environment.d
-deferred_auth_method=password
-host=heat
-auth_encryption_key={{ encryption_key }}
-
-{% if database_host -%}
-# < Icehouse db config
-sql_connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %}
-{% endif %}
-
-{% if rabbitmq_host or rabbitmq_hosts -%}
-rabbit_userid = {{ rabbitmq_user }}
-rabbit_virtual_host = {{ rabbitmq_virtual_host }}
-rabbit_password = {{ rabbitmq_password }}
-{% if rabbitmq_hosts -%}
-rabbit_hosts = {{ rabbitmq_hosts }}
-{% if rabbitmq_ha_queues -%}
-rabbit_ha_queues = True
-rabbit_durable_queues = False
-{% endif -%}
-{% else -%}
-rabbit_host = {{ rabbitmq_host }}
-{% endif -%}
-{% if rabbit_ssl_port -%}
-rabbit_use_ssl = True
-rabbit_port = {{ rabbit_ssl_port }}
-{% if rabbit_ssl_ca -%}
-kombu_ssl_ca_certs = {{ rabbit_ssl_ca }}
-{% endif -%}
-{% endif -%}
-{% endif %}
+instance_user = {{ instance_user }}
+instance_driver = heat.engine.nova
+plugin_dirs = /usr/lib64/heat,/usr/lib/heat
+environment_dir = /etc/heat/environment.d
+host = heat
+auth_encryption_key = {{ encryption_key }}
+deferred_auth_method = trusts
+stack_domain_admin = heat_domain_admin
+stack_domain_admin_password = {{ heat_domain_admin_passwd }}
+stack_user_domain_name = heat
 
 {% if auth_host -%}
 [keystone_authtoken]
-identity_uri = {{ service_protocol }}://{{ service_host }}:{{ auth_port }}
+identity_uri = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
 auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}/v2.0
-auth_host = {{ auth_host }}
-auth_port = {{ auth_port }}
-auth_protocol = {{ auth_protocol }}
 admin_tenant_name = {{ admin_tenant_name }}
 admin_user = {{ admin_user }}
 admin_password = {{ admin_password }}
 signing_dir = {{ signing_dir }}
-{% endif %}
 
 [ec2_authtoken]
 auth_uri = {{service_protocol }}://{{ service_host }}:{{ service_port }}/v2.0
 keystone_ec2_uri = {{service_protocol }}://{{ service_host }}:{{ service_port }}/v2.0/ec2tokens
+{% endif %}
 
 {% if database_host -%}
 [database]
 connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %}
-{% endif -%}
+{% endif %}
 
 [paste_deploy]
 api_paste_config=/etc/heat/api-paste.ini
@@ -78,3 +51,6 @@
 {% else -%}
 bind_port=8000
 {% endif %}
+
+{% include "section-rabbitmq-oslo" %}
+

=== added directory 'templates/liberty'
=== added file 'templates/liberty/heat.conf'
--- templates/liberty/heat.conf	1970-01-01 00:00:00 +0000
+++ templates/liberty/heat.conf	2016-02-10 14:39:12 +0000
@@ -0,0 +1,69 @@
+[DEFAULT]
+use_syslog = {{ use_syslog }}
+debug = {{ debug }}
+verbose = {{ verbose }}
+log_dir = /var/log/heat
+instance_user = {{ instance_user }}
+instance_driver = heat.engine.nova
+plugin_dirs = /usr/lib64/heat,/usr/lib/heat
+environment_dir = /etc/heat/environment.d
+host = heat
+auth_encryption_key = {{ encryption_key }}
+deferred_auth_method = trusts
+stack_domain_admin = heat_domain_admin
+stack_domain_admin_password = {{ heat_domain_admin_passwd }}
+stack_user_domain_name = heat
+
+{% if auth_host -%}
+[keystone_authtoken]
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
+auth_plugin = password
+project_domain_id = default
+user_domain_id = default
+project_name = {{ admin_tenant_name }}
+username = {{ admin_user }}
+password = {{ admin_password }}
+signing_dir = {{ signing_dir }}
+
+[trustee]
+auth_plugin = password
+auth_url = {{ auth_protocol }}://{{ auth_host }}:{{ auth_port }}
+username = {{ admin_user }}
+password = {{ admin_password }}
+user_domain_id = default
+
+[clients_keystone]
+auth_uri = {{ service_protocol }}://{{ service_host }}:{{ service_port }}
+
+[ec2_authtoken]
+auth_uri = {{service_protocol }}://{{ service_host }}:{{ service_port }}
+keystone_ec2_uri = {{service_protocol }}://{{ service_host }}:{{ service_port }}/v2.0/ec2tokens
+{% endif %}
+
+{% if database_host -%}
+[database]
+connection = {{ database_type }}://{{ database_user }}:{{ database_password }}@{{ database_host }}/{{ database }}{% if database_ssl_ca %}?ssl_ca={{ database_ssl_ca }}{% if database_ssl_cert %}&ssl_cert={{ database_ssl_cert }}&ssl_key={{ database_ssl_key }}{% endif %}{% endif %}
+{% endif %}
+
+[paste_deploy]
+api_paste_config=/etc/heat/api-paste.ini
+
+[heat_api]
+bind_host = {{ bind_host }}
+{% if api_listen_port -%} 
+bind_port={{ api_listen_port }}
+{% else -%}
+bind_port=8004
+{% endif %}
+
+[heat_api_cfn]
+bind_host = {{ bind_host }}
+{% if api_cfn_listen_port -%}
+bind_port={{ api_cfn_listen_port }}
+{% else -%}
+bind_port=8000
+{% endif %}
+
+{% include "section-rabbitmq-oslo" %}
+

=== modified file 'unit_tests/test_heat_context.py'
--- unit_tests/test_heat_context.py	2015-05-06 14:03:04 +0000
+++ unit_tests/test_heat_context.py	2016-02-10 14:39:12 +0000
@@ -5,7 +5,8 @@
 TO_PATCH = [
     'get_encryption_key',
     'generate_ec2_tokens',
-    'config'
+    'config',
+    'leader_get',
 ]
 
 
@@ -16,9 +17,12 @@
 
     def test_encryption_configuration(self):
         self.get_encryption_key.return_value = 'key'
+        self.leader_get.return_value = 'password'
         self.assertEquals(
-            heat_context.EncryptionContext()(),
-            {'encryption_key': 'key'})
+            heat_context.HeatSecurityContext()(),
+            {'encryption_key': 'key',
+             'heat_domain_admin_passwd': 'password'})
+        self.leader_get.assert_called_with('heat-domain-admin-passwd')
 
     def test_instance_user_empty_configuration(self):
         self.config.return_value = None

=== modified file 'unit_tests/test_heat_relations.py'
--- unit_tests/test_heat_relations.py	2015-11-16 09:14:33 +0000
+++ unit_tests/test_heat_relations.py	2016-02-10 14:39:12 +0000
@@ -240,5 +240,5 @@
         self.sync_db_with_multi_ipv6_addresses.return_value = MagicMock()
         self.test_config.set('prefer-ipv6', True)
         relations.db_joined()
-        self.sync_db_with_multi_ipv6_addresses.assert_called_with_once(
-            'heat', 'heat')
+        self.sync_db_with_multi_ipv6_addresses.assert_called_with(
+            'heat', 'heat', relation_prefix='heat')