Merge lp:~james-page/charms/trusty/swift-proxy/network-splits-https into lp:~openstack-charmers-archive/charms/trusty/swift-proxy/next

[James Page]

Add support for HTTPS+HA with multiple network endpoints

[lp:~james-page/charms/trusty/swift-proxy/network-splits-https]

70. By James Page on 2014-10-02

Resync

71. By James Page on 2014-10-02

Drop use-https and associate configuration options

[Liam Young (gnuoy)]

hooks/swift_context.py:10:1: F401 'CA_CERT_PATH' imported but unused
hooks/swift_context.py:31:1: F401 'get_ca_cert' imported but unused
hooks/swift_context.py:31:1: F401 'get_cert' imported but unused
hooks/swift_context.py:36:1: F401 'b64decode' imported but unused
Makefile:5: recipe for target 'lint' failed
make: *** [lint] Error 1

+1 once ^ are fixed

review: Needs Fixing

=== modified file 'config.yaml'
--- config.yaml	2014-09-25 16:43:34 +0000
+++ config.yaml	2014-10-02 20:31:34 +0000
@@ -63,28 +63,6 @@
     type: string
     description: |
       Base64 encoded SSL key to use with certificate specified as ssl_cert.
-  # Locally generated CA Cert info (only use without keystone)
-  # These options are deprecated and will be removed sometime
-  use-https:
-    default: "yes"
-    type: string
-    description: Whether to listen on HTTPS
-  country:
-    default: US
-    type: string
-    description: Country
-  state:
-    default: ST
-    type: string
-    description: State
-  locale:
-    default: City
-    type: string
-    description: Locale
-  common-name:
-    default: CN
-    type: string
-    description: Common Name
   # General Swift Proxy configuration
   bind-port:
     default: 8080

=== modified file 'hooks/charmhelpers/contrib/network/ip.py'
--- hooks/charmhelpers/contrib/network/ip.py	2014-09-30 12:57:13 +0000
+++ hooks/charmhelpers/contrib/network/ip.py	2014-10-02 20:31:34 +0000
@@ -57,6 +57,8 @@
         else:
             if fatal:
                 not_found_error_out()
+            else:
+                return None
 
     _validate_cidr(network)
     network = netaddr.IPNetwork(network)

=== modified file 'hooks/charmhelpers/contrib/openstack/amulet/deployment.py'
--- hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2014-09-26 12:41:28 +0000
+++ hooks/charmhelpers/contrib/openstack/amulet/deployment.py	2014-10-02 20:31:34 +0000
@@ -1,6 +1,3 @@
-from bzrlib.branch import Branch
-import os
-import re
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -13,62 +10,62 @@
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None):
+    def __init__(self, series=None, openstack=None, source=None, stable=True):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
         self.openstack = openstack
         self.source = source
-
-    def _is_dev_branch(self):
-        """Determine if branch being tested is a dev (i.e. next) branch."""
-        branch = Branch.open(os.getcwd())
-        parent = branch.get_parent()
-        pattern = re.compile("^.*/next/$")
-        if (pattern.match(parent)):
-            return True
-        else:
-            return False
+        self.stable = stable
+        # Note(coreycb): this needs to be changed when new next branches come
+        # out.
+        self.current_next = "trusty"
 
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
 
-           If the branch being tested is a dev branch, then determine the
-           development branch locations for the other services. Otherwise,
-           the default charm store branches will be used."""
-        name = 0
-        if self._is_dev_branch():
-            updated_services = []
-            for svc in other_services:
-                if svc[name] in ['mysql', 'mongodb', 'rabbitmq-server']:
-                    location = 'lp:charms/{}'.format(svc[name])
+           Determine if the local branch being tested is derived from its
+           stable or next (dev) branch, and based on this, use the corresonding
+           stable or next branches for the other_services."""
+        base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
+
+        if self.stable:
+            for svc in other_services:
+                temp = 'lp:charms/{}'
+                svc['location'] = temp.format(svc['name'])
+        else:
+            for svc in other_services:
+                if svc['name'] in base_charms:
+                    temp = 'lp:charms/{}'
+                    svc['location'] = temp.format(svc['name'])
                 else:
-                    temp = 'lp:~openstack-charmers/charms/trusty/{}/next'
-                    location = temp.format(svc[name])
-                updated_services.append(svc + (location,))
-            other_services = updated_services
+                    temp = 'lp:~openstack-charmers/charms/{}/{}/next'
+                    svc['location'] = temp.format(self.current_next,
+                                                  svc['name'])
         return other_services
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment and set openstack-origin/source."""
-        name = 0
         other_services = self._determine_branch_locations(other_services)
+
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
                                                              other_services)
+
         services = other_services
         services.append(this_service)
-        use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph']
+        use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
+                      'ceph-osd', 'ceph-radosgw']
 
         if self.openstack:
             for svc in services:
-                if svc[name] not in use_source:
+                if svc['name'] not in use_source:
                     config = {'openstack-origin': self.openstack}
-                    self.d.configure(svc[name], config)
+                    self.d.configure(svc['name'], config)
 
         if self.source:
             for svc in services:
-                if svc[name] in use_source:
+                if svc['name'] in use_source:
                     config = {'source': self.source}
-                    self.d.configure(svc[name], config)
+                    self.d.configure(svc['name'], config)
 
     def _configure_services(self, configs):
         """Configure all of the services."""

=== modified file 'hooks/charmhelpers/contrib/openstack/context.py'
--- hooks/charmhelpers/contrib/openstack/context.py	2014-09-26 14:54:16 +0000
+++ hooks/charmhelpers/contrib/openstack/context.py	2014-10-02 20:31:34 +0000
@@ -52,6 +52,7 @@
 from charmhelpers.contrib.network.ip import (
     get_address_in_network,
     get_ipv6_addr,
+    get_netmask_for_address,
     format_ipv6_addr,
     is_address_in_network
 )
@@ -408,6 +409,9 @@
         return ctxt
 
 
+ADDRESS_TYPES = ['admin', 'internal', 'public']
+
+
 class HAProxyContext(OSContextGenerator):
     interfaces = ['cluster']
 
@@ -420,7 +424,6 @@
         if not relation_ids('cluster'):
             return {}
 
-        cluster_hosts = {}
         l_unit = local_unit().replace('/', '-')
 
         if config('prefer-ipv6'):
@@ -428,17 +431,49 @@
         else:
             addr = unit_get('private-address')
 
-        cluster_hosts[l_unit] = get_address_in_network(config('os-internal-network'),
-                                                       addr)
-
-        for rid in relation_ids('cluster'):
-            for unit in related_units(rid):
-                _unit = unit.replace('/', '-')
-                addr = relation_get('private-address', rid=rid, unit=unit)
-                cluster_hosts[_unit] = addr
+        cluster_hosts = {}
+
+        # NOTE(jamespage): build out map of configured network endpoints
+        # and associated backends
+        for addr_type in ADDRESS_TYPES:
+            laddr = get_address_in_network(
+                config('os-{}-network'.format(addr_type)))
+            if laddr:
+                cluster_hosts[laddr] = {}
+                cluster_hosts[laddr]['network'] = "{}/{}".format(
+                    laddr,
+                    get_netmask_for_address(laddr)
+                )
+                cluster_hosts[laddr]['backends'] = {}
+                cluster_hosts[laddr]['backends'][l_unit] = laddr
+                for rid in relation_ids('cluster'):
+                    for unit in related_units(rid):
+                        _unit = unit.replace('/', '-')
+                        _laddr = relation_get('{}-address'.format(addr_type),
+                                              rid=rid, unit=unit)
+                        if _laddr:
+                            cluster_hosts[laddr]['backends'][_unit] = _laddr
+
+        # NOTE(jamespage) no split configurations found, just use
+        # private addresses
+        if not cluster_hosts:
+            cluster_hosts[addr] = {}
+            cluster_hosts[addr]['network'] = "{}/{}".format(
+                addr,
+                get_netmask_for_address(addr)
+            )
+            cluster_hosts[addr]['backends'] = {}
+            cluster_hosts[addr]['backends'][l_unit] = addr
+            for rid in relation_ids('cluster'):
+                for unit in related_units(rid):
+                    _unit = unit.replace('/', '-')
+                    _laddr = relation_get('private-address',
+                                          rid=rid, unit=unit)
+                    if _laddr:
+                        cluster_hosts[addr]['backends'][_unit] = _laddr
 
         ctxt = {
-            'units': cluster_hosts,
+            'frontends': cluster_hosts,
         }
 
         if config('haproxy-server-timeout'):
@@ -455,12 +490,13 @@
             ctxt['haproxy_host'] = '0.0.0.0'
             ctxt['stat_port'] = ':8888'
 
-        if len(cluster_hosts.keys()) > 1:
-            # Enable haproxy when we have enough peers.
-            log('Ensuring haproxy enabled in /etc/default/haproxy.')
-            with open('/etc/default/haproxy', 'w') as out:
-                out.write('ENABLED=1\n')
-            return ctxt
+        for frontend in cluster_hosts:
+            if len(cluster_hosts[frontend]['backends']) > 1:
+                # Enable haproxy when we have enough peers.
+                log('Ensuring haproxy enabled in /etc/default/haproxy.')
+                with open('/etc/default/haproxy', 'w') as out:
+                    out.write('ENABLED=1\n')
+                return ctxt
         log('HAProxy context is incomplete, this unit has no peers.')
         return {}
 
@@ -722,22 +758,22 @@
 
 class OSConfigFlagContext(OSContextGenerator):
 
-        """
-        Responsible for adding user-defined config-flags in charm config to a
-        template context.
-
-        NOTE: the value of config-flags may be a comma-separated list of
-              key=value pairs and some Openstack config files support
-              comma-separated lists as values.
-        """
-
-        def __call__(self):
-            config_flags = config('config-flags')
-            if not config_flags:
-                return {}
-
-            flags = config_flags_parser(config_flags)
-            return {'user_config_flags': flags}
+    """
+    Responsible for adding user-defined config-flags in charm config to a
+    template context.
+
+    NOTE: the value of config-flags may be a comma-separated list of
+          key=value pairs and some Openstack config files support
+          comma-separated lists as values.
+    """
+
+    def __call__(self):
+        config_flags = config('config-flags')
+        if not config_flags:
+            return {}
+
+        flags = config_flags_parser(config_flags)
+        return {'user_config_flags': flags}
 
 
 class SubordinateConfigContext(OSContextGenerator):

=== modified file 'hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg'
--- hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2014-09-26 14:54:16 +0000
+++ hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg	2014-10-02 20:31:34 +0000
@@ -34,17 +34,21 @@
     stats uri /
     stats auth admin:password
 
-{% if units -%}
+{% if frontends -%}
 {% for service, ports in service_ports.iteritems() -%}
-listen {{ service }}_ipv4 0.0.0.0:{{ ports[0] }}
-    balance roundrobin
-    {% for unit, address in units.iteritems() -%}
-    server {{ unit }} {{ address }}:{{ ports[1] }} check
-    {% endfor %}
-listen {{ service }}_ipv6 :::{{ ports[0] }}
-    balance roundrobin
-    {% for unit, address in units.iteritems() -%}
-    server {{ unit }} {{ address }}:{{ ports[1] }} check
-    {% endfor %}
+frontend tcp-in_{{ service }}
+    bind *:{{ ports[0] }}
+    bind :::{{ ports[0] }}
+    {% for frontend in frontends -%}
+    acl net_{{ frontend }} dst {{ frontends[frontend]['network'] }}
+    use_backend {{ service }}_{{ frontend }} if net_{{ frontend }}
+    {% endfor %}
+{% for frontend in frontends -%}
+backend {{ service }}_{{ frontend }}
+    balance leastconn
+    {% for unit, address in frontends[frontend]['backends'].iteritems() -%}
+    server {{ unit }} {{ address }}:{{ ports[1] }} check
+    {% endfor %}
+{% endfor -%}
 {% endfor -%}
 {% endif -%}

=== added symlink 'hooks/cluster-relation-departed'
=== target is u'swift_hooks.py'
=== modified file 'hooks/swift_context.py'
--- hooks/swift_context.py	2014-09-25 06:20:53 +0000
+++ hooks/swift_context.py	2014-10-02 20:31:34 +0000
@@ -85,33 +85,9 @@
 
 class ApacheSSLContext(SSLContext):
     interfaces = ['https']
-    external_ports = [determine_apache_port(config('bind-port'))]
+    external_ports = [config('bind-port')]
     service_namespace = 'swift'
 
-    def configure_cert(self):
-        if not os.path.isdir('/etc/apache2/ssl'):
-            os.mkdir('/etc/apache2/ssl')
-        ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace)
-        if not os.path.isdir(ssl_dir):
-            os.mkdir(ssl_dir)
-        cert, key = get_cert()
-        # Swift specific - generate a cert by default if not using
-        # a) user supplied cert or b) keystone signed cert
-        if None in [cert, key]:
-            cert, key = generate_cert()
-        with open(os.path.join(ssl_dir, 'cert'), 'w') as cert_out:
-            cert_out.write(b64decode(cert))
-        with open(os.path.join(ssl_dir, 'key'), 'w') as key_out:
-            key_out.write(b64decode(key))
-        ca_cert = get_ca_cert()
-        if ca_cert:
-            with open(CA_CERT_PATH, 'w') as ca_out:
-                ca_out.write(b64decode(ca_cert))
-            subprocess.check_call(['update-ca-certificates'])
-
-    def __call__(self):
-        return super(ApacheSSLContext, self).__call__()
-
 
 class SwiftRingContext(OSContextGenerator):
 

=== modified file 'hooks/swift_hooks.py'
--- hooks/swift_hooks.py	2014-09-30 07:48:30 +0000
+++ hooks/swift_hooks.py	2014-10-02 20:31:34 +0000
@@ -62,6 +62,8 @@
     is_ipv6
 )
 
+from charmhelpers.contrib.openstack.context import ADDRESS_TYPES
+
 extra_pkgs = [
     "haproxy",
     "python-jinja2"
@@ -223,22 +225,28 @@
         do_openstack_upgrade(CONFIGS)
     for r_id in relation_ids('identity-service'):
         keystone_joined(relid=r_id)
+    [cluster_joined(rid) for rid in relation_ids('cluster')]
 
 
 @hooks.hook('cluster-relation-joined')
 def cluster_joined(relation_id=None):
+    for addr_type in ADDRESS_TYPES:
+        address = get_address_in_network(
+            config('os-{}-network'.format(addr_type))
+        )
+        if address:
+            relation_set(
+                relation_id=relation_id,
+                relation_settings={'{}-address'.format(addr_type): address}
+            )
     if config('prefer-ipv6'):
         private_addr = get_ipv6_addr(exc_list=[config('vip')])[0]
-    else:
-        private_addr = unit_get('private-address')
-
-    address = get_address_in_network(config('os-internal-network'),
-                                     private_addr)
-    relation_set(relation_id=relation_id,
-                 relation_settings={'private-address': address})
-
-
[e-mail]('cluster-relation-changed')
+        relation_set(relation_id=relation_id,
+                     relation_settings={'private-address': private_addr})
+
+
[e-mail]('cluster-relation-changed',
+            'cluster-relation-departed')
 @restart_on_change(restart_map())
 def cluster_changed():
     CONFIGS.write_all()

=== modified file 'tests/basic_deployment.py'
--- tests/basic_deployment.py	2014-07-11 16:57:37 +0000
+++ tests/basic_deployment.py	2014-10-02 20:31:34 +0000
@@ -58,8 +58,7 @@
                            'admin-token': 'ubuntutesting'}
         swift_proxy_config = {'zone-assignment': 'manual',
                            'replicas': '1',
-                           'swift-hash': 'fdfef9d4-8b06-11e2-8ac0-531c923c8fae',
-                           'use-https': 'no'}
+                           'swift-hash': 'fdfef9d4-8b06-11e2-8ac0-531c923c8fae'}
         swift_storage_config = {'zone': '1',
                                 'block-device': 'vdb',
                                 'overwrite': 'true'}

=== modified file 'tests/charmhelpers/contrib/amulet/deployment.py'
--- tests/charmhelpers/contrib/amulet/deployment.py	2014-09-26 12:41:28 +0000
+++ tests/charmhelpers/contrib/amulet/deployment.py	2014-10-02 20:31:34 +0000
@@ -25,25 +25,30 @@
 
            Add services to the deployment where this_service is the local charm
            that we're testing and other_services are the other services that
-           are being used in the amulet tests.
+           are being used in the local amulet tests.
            """
-        name, units, location = range(3)
-
-        if this_service[name] != os.path.basename(os.getcwd()):
-            s = this_service[name]
+        if this_service['name'] != os.path.basename(os.getcwd()):
+            s = this_service['name']
             msg = "The charm's root directory name needs to be {}".format(s)
             amulet.raise_status(amulet.FAIL, msg=msg)
 
-        self.d.add(this_service[name], units=this_service[units])
+        if 'units' not in this_service:
+            this_service['units'] = 1
+
+        self.d.add(this_service['name'], units=this_service['units'])
 
         for svc in other_services:
-            if len(svc) > 2:
-                branch_location = svc[location]
+            if 'location' in svc:
+                branch_location = svc['location']
             elif self.series:
-                branch_location = 'cs:{}/{}'.format(self.series, svc[name]),
+                branch_location = 'cs:{}/{}'.format(self.series, svc['name']),
             else:
                 branch_location = None
-            self.d.add(svc[name], charm=branch_location, units=svc[units])
+
+            if 'units' not in svc:
+                svc['units'] = 1
+
+            self.d.add(svc['name'], charm=branch_location, units=svc['units'])
 
     def _add_relations(self, relations):
         """Add all of the relations for the services."""

=== modified file 'tests/charmhelpers/contrib/openstack/amulet/deployment.py'
--- tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-09-26 12:41:28 +0000
+++ tests/charmhelpers/contrib/openstack/amulet/deployment.py	2014-10-02 20:31:34 +0000
@@ -1,6 +1,3 @@
-from bzrlib.branch import Branch
-import os
-import re
 from charmhelpers.contrib.amulet.deployment import (
     AmuletDeployment
 )
@@ -13,62 +10,62 @@
        that is specifically for use by OpenStack charms.
        """
 
-    def __init__(self, series=None, openstack=None, source=None):
+    def __init__(self, series=None, openstack=None, source=None, stable=True):
         """Initialize the deployment environment."""
         super(OpenStackAmuletDeployment, self).__init__(series)
         self.openstack = openstack
         self.source = source
-
-    def _is_dev_branch(self):
-        """Determine if branch being tested is a dev (i.e. next) branch."""
-        branch = Branch.open(os.getcwd())
-        parent = branch.get_parent()
-        pattern = re.compile("^.*/next/$")
-        if (pattern.match(parent)):
-            return True
-        else:
-            return False
+        self.stable = stable
+        # Note(coreycb): this needs to be changed when new next branches come
+        # out.
+        self.current_next = "trusty"
 
     def _determine_branch_locations(self, other_services):
         """Determine the branch locations for the other services.
 
-           If the branch being tested is a dev branch, then determine the
-           development branch locations for the other services. Otherwise,
-           the default charm store branches will be used."""
-        name = 0
-        if self._is_dev_branch():
-            updated_services = []
-            for svc in other_services:
-                if svc[name] in ['mysql', 'mongodb', 'rabbitmq-server']:
-                    location = 'lp:charms/{}'.format(svc[name])
+           Determine if the local branch being tested is derived from its
+           stable or next (dev) branch, and based on this, use the corresonding
+           stable or next branches for the other_services."""
+        base_charms = ['mysql', 'mongodb', 'rabbitmq-server']
+
+        if self.stable:
+            for svc in other_services:
+                temp = 'lp:charms/{}'
+                svc['location'] = temp.format(svc['name'])
+        else:
+            for svc in other_services:
+                if svc['name'] in base_charms:
+                    temp = 'lp:charms/{}'
+                    svc['location'] = temp.format(svc['name'])
                 else:
-                    temp = 'lp:~openstack-charmers/charms/trusty/{}/next'
-                    location = temp.format(svc[name])
-                updated_services.append(svc + (location,))
-            other_services = updated_services
+                    temp = 'lp:~openstack-charmers/charms/{}/{}/next'
+                    svc['location'] = temp.format(self.current_next,
+                                                  svc['name'])
         return other_services
 
     def _add_services(self, this_service, other_services):
         """Add services to the deployment and set openstack-origin/source."""
-        name = 0
         other_services = self._determine_branch_locations(other_services)
+
         super(OpenStackAmuletDeployment, self)._add_services(this_service,
                                                              other_services)
+
         services = other_services
         services.append(this_service)
-        use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph']
+        use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph',
+                      'ceph-osd', 'ceph-radosgw']
 
         if self.openstack:
             for svc in services:
-                if svc[name] not in use_source:
+                if svc['name'] not in use_source:
                     config = {'openstack-origin': self.openstack}
-                    self.d.configure(svc[name], config)
+                    self.d.configure(svc['name'], config)
 
         if self.source:
             for svc in services:
-                if svc[name] in use_source:
+                if svc['name'] in use_source:
                     config = {'source': self.source}
-                    self.d.configure(svc[name], config)
+                    self.d.configure(svc['name'], config)
 
     def _configure_services(self, configs):
         """Configure all of the services."""