Merge lp:~johnsca/charms/trusty/cloudfoundry/catchup into lp:~cf-charmers/charms/trusty/cloudfoundry/trunk

[Cory Johns]

Changes to support 190. I haven't run CATS on it, but pushing an app and logging works.

[lp:~johnsca/charms/trusty/cloudfoundry/catchup]

171. By Cory Johns on 2014-12-09

Fixes for CATs test to match cfdeploy changes

172. By Cory Johns on 2014-12-09

Fixes for running CATs

173. By Cory Johns on 2014-12-10

Test arg needs to be optional

[Benjamin Saller (bcsaller)]

These changes LGTM. Its nice that you found the dangling reference issue with placement and the clean up on interface types, like http->login.

I'd like to hold off landing this until we've identified the remaining issue with CATs, even if we can't fix that one remaining test run today I'd like some clarity as to why. If it works on a fresh run this morning its most likely a GC issue and we'll need to address that separately.

Thanks

review: Approve

=== modified file '.bzrignore'
--- .bzrignore	2014-11-13 16:22:27 +0000
+++ .bzrignore	2014-12-10 15:53:53 +0000
@@ -47,3 +47,7 @@
 # Generated test charms
 cloudfoundry-r*
 output-v*.yaml
+
+# Generated when running CATs
+src/
+pkg/
 
=== modified file 'charmgen/generator.py'
--- charmgen/generator.py	2014-11-07 21:50:48 +0000
+++ charmgen/generator.py	2014-12-10 15:53:53 +0000
@@ -227,6 +227,10 @@
             if zone:
                 driver = zone[0]
                 for follower in zone[1:]:
+                    if follower not in services:
+                        # allow dangling references in placement to handle
+                        # removed services
+                        continue
                     with_references.add(driver)
                     if 'to' not in services[follower]:
                         services[follower]['to'] = [driver]
 
=== modified file 'charmgen/placements.yaml'
--- charmgen/placements.yaml	2014-11-11 03:52:11 +0000
+++ charmgen/placements.yaml	2014-12-10 15:53:53 +0000
@@ -5,7 +5,7 @@
         b: [router, nats, nats-sf]
         c: [login, uaa, mysql]
         d: [dea, hm, etcd]
-        e: [loggregator, loggregator-trafficcontrol]
+        e: [loggregator-trafficcontrol, loggregator, doppler]
         f: [haproxy, collector]
 local:
     __default__:
 
=== modified file 'cloudfoundry/contexts.py'
--- cloudfoundry/contexts.py	2014-11-11 04:02:53 +0000
+++ cloudfoundry/contexts.py	2014-12-10 15:53:53 +0000
@@ -226,7 +226,7 @@
 
 class LoginRelation(RelationContext):
     name = 'login'
-    interface = 'http'
+    interface = 'login'
     required_keys = []
     port = 8080
 
@@ -239,6 +239,8 @@
         data = self[self.name][0]
         return {
             'login.port': data['port'],
+            'login.saml.socket.connectionManagerTimeout': 10000,
+            'login.saml.socket.soTimeout': 10000,
         }
 
 
@@ -250,6 +252,11 @@
     def erb_mapping(self):
         return {
             'dea_next.directory_server_protocol': 'http',
+            # these are the default values, but since they're used in hm and
+            # cc and those specs don't give the default value (only dea_next),
+            # we have to manually specify them here
+            'dea_next.advertise_interval_in_seconds': 5,
+            'dea_next.heartbeat_interval_in_seconds': 10,
         }
 
 
@@ -296,8 +303,6 @@
             'traffic_controller.host': data[0]['host'],
             'traffic_controller.incoming_port': data[0]['port'],
             'traffic_controller.outgoing_port': data[0]['outgoing_port'],
-            'logger_endpoint.use_ssl': False,  # TODO: support SSL option
-            'logger_endpoint.port': 80,  # default is 443
         }
 
 
@@ -328,6 +333,48 @@
         }
 
 
+class DopplerRelation(RelationContext):
+    name = 'doppler'
+    interface = 'doppler'
+    required_keys = ['address', 'incoming_port', 'outgoing_port',
+                     'dropsonde_port', 'doppler_endpoint_secret']
+    incoming_port = 3457
+    outgoing_port = 8083
+    dropsonde_port = 3460
+
+    def get_shared_secrets(self):
+        secret_context = StoredContext('doppler-secrets.yml', {
+            'doppler_endpoint_secret': host.pwgen(20),
+        })
+        return secret_context
+
+    def provide_data(self):
+        secrets = self.get_shared_secrets()
+        return {
+            'address': hookenv.unit_get('private-address').encode('utf-8'),
+            'incoming_port': self.incoming_port,
+            'outgoing_port': self.outgoing_port,
+            'dropsonde_port': self.dropsonde_port,
+            'doppler_endpoint_secret': secrets['doppler_endpoint_secret']
+        }
+
+    def erb_mapping(self):
+        data = self[self.name]
+        return {
+            'doppler.zone': 'z1',
+            'doppler.blacklisted_syslog_ranges': [],
+            'doppler.incoming_port': data[0]['incoming_port'],
+            'doppler.outgoing_port': data[0]['outgoing_port'],
+            'doppler.dropsonde_incoming_port': data[0]['dropsonde_port'],
+            'doppler_endpoint.shared_secret': data[0]['doppler_endpoint_secret'],
+            'loggregator.doppler_port': data[0]['outgoing_port'],
+            'loggregator.incoming_port': data[0]['incoming_port'],
+            'loggregator.dropsonde_incoming_port': data[0]['dropsonde_port'],
+            'loggregator.dropsonde_outgoing_port': data[0]['outgoing_port'],
+            'loggregator_endpoint.shared_secret': data[0]['doppler_endpoint_secret'],
+        }
+
+
 class EtcdRelation(RelationContext):
     name = 'etcd'
     interface = 'etcd'
@@ -343,13 +390,15 @@
 class CloudControllerRelation(RelationContext):
     name = 'cc'
     interface = 'controller'
-    required_keys = ['hostname', 'port', 'user', 'password', 'db_encryption_key']
+    required_keys = ['hostname', 'port', 'user', 'password',
+                     'internal_api_password', 'db_encryption_key']
 
     def get_credentials(self):
         return StoredContext('api_credentials.yml', {
             'user': host.pwgen(7),
             'password': host.pwgen(7),
             'db_encryption_key': host.pwgen(7),
+            'internal_api_password': host.pwgen(7),
         })
 
     def provide_data(self):
@@ -357,6 +406,7 @@
         return {
             'user': creds['user'],
             'password': creds['password'],
+            'internal_api_password': creds['internal_api_password'],
             'db_encryption_key': creds['db_encryption_key'],
             'hostname': hookenv.unit_get('private-address').encode('utf-8'),
             'port': 9022,
@@ -369,6 +419,7 @@
             'cc.srv_api_uri': 'http://{}:{}'.format(data['hostname'], data['port']),
             'cc.bulk_api_user': data['user'],
             'cc.bulk_api_password': data['password'],
+            'cc.internal_api_password': data['internal_api_password'],
             'cc.staging_upload_user': 'ignored',  # FIXME: We need a staging cache set up
             'cc.staging_upload_password': 'ignored',
             'cc.db_encryption_key': data['db_encryption_key'],
@@ -439,7 +490,7 @@
 
 class RouterRelation(RelationContext):
     name = 'router'
-    interface = 'http'
+    interface = 'router'
     required_keys = ['address']
     port = 8000
     varz_port = 8084  # not currently used
@@ -545,6 +596,8 @@
             'app_domains': [d['domain'] for d in self[self.name]],
             'system_domain': domain,  # TODO: These should probably be config options
             'system_domain_organization': 'juju-org',
+            'logger_endpoint.use_ssl': False,  # TODO: support SSL option
+            'logger_endpoint.port': 80,  # default is 443
         }
 
 
 
=== modified file 'cloudfoundry/releases.py'
--- cloudfoundry/releases.py	2014-11-05 18:22:07 +0000
+++ cloudfoundry/releases.py	2014-12-10 15:53:53 +0000
@@ -8,8 +8,6 @@
     ('uaa-v1', 'uaa'),
     ('login-v1', 'login'),
     ('nats-stream-forwarder-v1', 'nats-sf'),
-    ('loggregator-v1', 'loggregator'),
-    ('hm9000-v1', 'hm'),
     ('haproxy-v1', 'haproxy'),
     ('collector-v1', 'collector'),
 
@@ -25,7 +23,6 @@
     ('mysql:db', 'cc:db'),
     ('mysql:db', 'uaa:db'),
     ('etcd:client', 'hm:etcd'),
-    ('etcd:client', 'loggregator:etcd'),
     ('etcd:client', 'loggregator-trafficcontrol:etcd'),
 ]
 
@@ -34,6 +31,38 @@
 
 RELEASES = [
     {
+        "releases": (190, 190),
+        "topology": {
+            "services": COMMON_SERVICES + [
+                ('router-v3', 'router'),
+                ('cloud-controller-v3', 'cc'),
+                ('cloud-controller-clock-v3', 'cc-clock'),
+                ('cloud-controller-worker-v3', 'cc-worker'),
+                ('dea-v3', 'dea'),
+                ('loggregator-trafficcontroller-v3', 'loggregator-trafficcontrol'),
+                ('hm9000-v2', 'hm'),
+                ('doppler-v1', 'doppler'),
+            ],
+            "relations": COMMON_RELATIONS + [
+                ('etcd:client', 'cc:etcd'),
+                ('etcd:client', 'cc-worker:etcd'),
+                ('etcd:client', 'cc-clock:etcd'),
+                ('etcd:client', 'router:etcd'),
+                ('etcd:client', 'dea:etcd'),
+                ('etcd:client', 'doppler:etcd'),
+            ],
+            "expose": ['haproxy'],
+            "constraints": {
+                "__default__": "arch=amd64 instance-type=m3.medium",
+                "cc": "arch=amd64 root-disk=12G mem=12G",
+                "cc-worker": "arch=amd64 root-disk=10G",
+                "cc-clock": "arch=amd64 root-disk=10G",
+                "dea": "arch=amd64 mem=5G",
+            },
+        },
+        "upgrades": COMMON_UPGRADES
+    },
+    {
         "releases": (177, 180),
         "topology": {
             "services": COMMON_SERVICES + [
@@ -42,7 +71,9 @@
                 ('cloud-controller-clock-v2', 'cc-clock'),
                 ('cloud-controller-worker-v2', 'cc-worker'),
                 ('dea-v2', 'dea'),
+                ('loggregator-v1', 'loggregator'),
                 ('loggregator-trafficcontroller-v2', 'loggregator-trafficcontrol'),
+                ('hm9000-v1', 'hm'),
             ],
             "relations": COMMON_RELATIONS + [
                 ('etcd:client', 'cc:etcd'),
@@ -50,6 +81,7 @@
                 ('etcd:client', 'cc-clock:etcd'),
                 ('etcd:client', 'router:etcd'),
                 ('etcd:client', 'dea:etcd'),
+                ('etcd:client', 'loggregator:etcd'),
             ],
             "expose": ['haproxy'],
             "constraints": {
@@ -71,9 +103,13 @@
                 ('cloud-controller-clock-v1', 'cc-clock'),
                 ('cloud-controller-worker-v1', 'cc-worker'),
                 ('dea-v1', 'dea'),
+                ('loggregator-v1', 'loggregator'),
                 ('loggregator-trafficcontroller-v1', 'loggregator-trafficcontrol'),
-            ],
-            "relations": COMMON_RELATIONS,
+                ('hm9000-v1', 'hm'),
+            ],
+            "relations": COMMON_RELATIONS + [
+                ('etcd:client', 'loggregator:etcd'),
+            ],
             "expose": ['haproxy'],
             "constraints": {
                 "__default__": "arch=amd64",
 
=== modified file 'cloudfoundry/services.py'
--- cloudfoundry/services.py	2014-11-07 21:18:42 +0000
+++ cloudfoundry/services.py	2014-12-10 15:53:53 +0000
@@ -49,6 +49,27 @@
 
     },
 
+    'cloud-controller-clock-v3': {
+        'summary': "A shared clock",
+        'description': '',
+        'jobs': [
+            {'job_name': 'cloud_controller_clock',
+             'mapping': {'ccdb': mapper.ccdb},
+             'provided_data': [],
+             'required_data': [contexts.NatsRelation,
+                               contexts.CloudControllerRelation,
+                               contexts.UAARelation,
+                               contexts.CloudControllerDBRelation],
+             },
+            {'job_name': 'metron_agent',
+             'required_data': [contexts.LTCRelation,
+                               contexts.NatsRelation,
+                               contexts.DopplerRelation,
+                               contexts.EtcdRelation]},
+            ],
+
+    },
+
     'cloud-controller-v1': {
         'summary': 'CF Cloud Controller, the brains of the operation',
         'description': '',
@@ -89,6 +110,29 @@
         ]
     },
 
+    'cloud-controller-v3': {
+        'summary': 'CF Cloud Controller, the brains of the operation',
+        'description': '',
+        'jobs': [
+            {'job_name': 'cloud_controller_ng',
+             'mapping': {'db': mapper.ccdb},
+             'provided_data': [contexts.CloudControllerRelation,
+                               contexts.CloudControllerDBRelation],
+             'required_data': [contexts.NatsRelation,
+                               contexts.MysqlRelation,
+                               contexts.UAARelation,
+                               contexts.DEARelation,
+                               contexts.CloudControllerRelation.remote_view,
+                               ],
+             },
+            {'job_name': 'metron_agent',
+             'required_data': [contexts.LTCRelation,
+                               contexts.NatsRelation,
+                               contexts.DopplerRelation,
+                               contexts.EtcdRelation]},
+        ]
+    },
+
     'cloud-controller-worker-v1': {
         'summary': "Worker for cc",
         'description': '',
@@ -128,6 +172,27 @@
             ]
     },
 
+    'cloud-controller-worker-v3': {
+        'summary': "Worker for cc",
+        'description': '',
+        'jobs': [
+            {'job_name': 'cloud_controller_worker',
+             'mapping': {'ccdb': mapper.ccdb},
+             'provided_data': [],
+             'required_data': [contexts.NatsRelation,
+                               contexts.UAARelation,
+                               contexts.CloudControllerRelation,
+                               contexts.CloudControllerDBRelation,
+                               ],
+             },
+            {'job_name': 'metron_agent',
+             'required_data': [contexts.LTCRelation,
+                               contexts.NatsRelation,
+                               contexts.DopplerRelation,
+                               contexts.EtcdRelation]},
+            ]
+    },
+
     'dea-v1': {
         'summary': 'DEA runs CF apps in containers',
         'description': '',
@@ -199,6 +264,47 @@
 
     },
 
+
+    'dea-v3': {
+        'summary': 'DEA runs CF apps in containers',
+        'description': '',
+        'jobs': [
+            {
+                'job_name': 'dea_next',
+                'mapping': {},
+                'install': [
+                    utils.install_linux_image_extra,
+                    utils.apt_install(['quota']),
+                    utils.modprobe(['quota_v1', 'quota_v2'])
+                ],
+                'provided_data': [contexts.DEARelation],
+                'required_data': [
+                    contexts.NatsRelation,
+                    contexts.LTCRelation,
+                    contexts.DEARelation.remote_view,
+                    contexts.RouterRelation,
+                ],
+                'data_ready': [
+                    #tasks.enable_swapaccounting
+                    tasks.patch_dea
+                ]
+            },
+            {
+                'job_name': 'dea_logging_agent',
+                'mapping': {},
+                'required_data': [contexts.NatsRelation,
+                                  contexts.LTCRelation,
+                                  contexts.EtcdRelation]
+            },
+            {'job_name': 'metron_agent',
+             'required_data': [contexts.LTCRelation,
+                               contexts.NatsRelation,
+                               contexts.DopplerRelation,
+                               contexts.EtcdRelation]},
+        ]
+
+    },
+
     'nats-v1': {
         'service': 'nats',
         'summary': 'NATS message bus for CF',
@@ -259,6 +365,25 @@
 
     },
 
+    'router-v3': {
+        'service': 'router',
+        'summary': 'CF Router',
+        'jobs': [
+            {'job_name': 'gorouter',
+             'ports': [contexts.RouterRelation.port],
+             'mapping': {},
+             'provided_data': [contexts.RouterRelation],
+             'required_data': [contexts.NatsRelation,
+                               contexts.RouterRelation.remote_view]},
+            {'job_name': 'metron_agent',
+             'required_data': [contexts.LTCRelation,
+                               contexts.NatsRelation,
+                               contexts.DopplerRelation,
+                               contexts.EtcdRelation]},
+        ],
+
+    },
+
     'uaa-v1': {
         'service': 'uaa',
         'summary': 'CF Oauth2 for identity management service',
@@ -303,6 +428,21 @@
             }]
         },
 
+    'doppler-v1': {
+        'service': 'doppler',
+        'summary': 'successor of loggregator',
+        'description': 'Successor of loggregator.',
+        'jobs': [{
+            'job_name': 'doppler',
+            'mapping': {},
+            'provided_data': [contexts.DopplerRelation],
+            'required_data': [contexts.NatsRelation,
+                              contexts.EtcdRelation,
+                              contexts.LTCRelation,
+                              contexts.DopplerRelation.remote_view]
+            }]
+        },
+
     'loggregator-trafficcontroller-v1': {
         'service': 'loggregator-trafficcontroller',
         'summary': 'loggregator-trafficcontroller',
@@ -337,6 +477,23 @@
             ]
         },
 
+    'loggregator-trafficcontroller-v3': {
+        'service': 'loggregator-trafficcontroller',
+        'summary': 'loggregator-trafficcontroller',
+        'description': '',
+        'jobs': [
+            {'job_name': 'loggregator_trafficcontroller',
+             'ports': [contexts.LTCRelation.outgoing_port],
+             'mapping': {},
+             'provided_data': [contexts.LTCRelation],
+             'required_data': [contexts.DopplerRelation,
+                               contexts.LTCRelation.remote_view,
+                               contexts.NatsRelation,
+                               contexts.CloudControllerRelation,
+                               contexts.EtcdRelation]},
+            ]
+        },
+
     'hm9000-v1': {
         'service': 'hm9000',
         'summary': 'health monitor',
@@ -351,6 +508,21 @@
             }]
         },
 
+    'hm9000-v2': {
+        'service': 'hm9000',
+        'summary': 'health monitor',
+        'description': '',
+        'jobs': [{
+            'job_name': 'hm9000',
+            'mapping': {},
+            'provided_data': [],
+            'required_data': [contexts.NatsRelation,
+                              contexts.CloudControllerRelation,
+                              contexts.EtcdRelation,
+                              contexts.DEARelation]
+            }]
+        },
+
     'haproxy-v1': {
         'service': 'haproxy',
         'summary': 'loadbalance the routers',
 
=== modified file 'cloudfoundry/utils.py'
--- cloudfoundry/utils.py	2014-12-03 23:14:23 +0000
+++ cloudfoundry/utils.py	2014-12-10 15:53:53 +0000
@@ -23,8 +23,10 @@
 
 
 @contextmanager
-def cd(directory):
+def cd(directory, make=False):
     cwd = os.getcwd()
+    if not os.path.exists(directory) and make:
+        os.makedirs(directory)
     os.chdir(directory)
     try:
         yield
 
=== modified file 'reconciler/app.py'
--- reconciler/app.py	2014-11-20 16:09:27 +0000
+++ reconciler/app.py	2014-12-10 15:53:53 +0000
@@ -135,6 +135,8 @@
         service['health'] = 'warn'
     else:
         service['health'] = 'pass'
+    if service_name == 'cloudfoundry' and db.error:  # XXX don't hard-code
+        service['health'] = 'fail'
 
 
 def get_current_state():
 
=== modified file 'reconciler/strategy.py'
--- reconciler/strategy.py	2014-11-11 18:22:53 +0000
+++ reconciler/strategy.py	2014-12-10 15:53:53 +0000
@@ -28,6 +28,7 @@
         self.strategy = Strategy(self.env)
         self.history = []
         self.exec_lock = threading.Lock()
+        self.error = False
 
     def reset(self):
         self.expected = {}
@@ -106,8 +107,12 @@
             return []
 
         # Service Deltas
-        self.strategy.extend(self.build_services())
-        self.strategy.extend(self.build_relations())
+        try:
+            self.strategy.extend(self.build_services())
+            self.strategy.extend(self.build_relations())
+        except Exception as e:
+            self.error = True
+            logging.error('Error building strategy: %s', e)
 
     def _changed(self):
         previous = hashlib.md5(json.dumps(self.previous or {}, sort_keys=True))
@@ -205,6 +210,10 @@
                     self.execute_strategy)
             else:
                 self._reset_strategy()
+            self.error = False
+        except Exception as e:
+            self.error = True
+            logging.error('Error executing strategy: %s', e)
         finally:
             self.exec_lock.release()
 
 
=== modified file 'tests/02-cats'
--- tests/02-cats	2014-10-01 18:50:57 +0000
+++ tests/02-cats	2014-12-10 15:53:53 +0000
@@ -1,3 +1,3 @@
 #!/bin/bash
 
-python tests/cats.py
+python tests/cats.py "$@"
 
=== modified file 'tests/cats.py'
--- tests/cats.py	2014-10-16 15:01:08 +0000
+++ tests/cats.py	2014-12-10 15:53:53 +0000
@@ -6,6 +6,7 @@
 import subprocess
 import sys
 import tempfile
+from functools import partial
 
 from cloudfoundry.releases import RELEASES
 from cloudfoundry.utils import (api,
@@ -26,6 +27,7 @@
     global options
     parser = argparse.ArgumentParser()
     parser.add_argument('-v', '--version', default="latest")
+    parser.add_argument('admin_password', default="password", nargs='?')
     options = parser.parse_args()
     if options.version == 'latest':
         options.version = RELEASES[0]['releases'][1]
@@ -50,10 +52,11 @@
 
 def get_cats(version):
     logging.info("Getting CATs from github")
-    if not os.path.exists('cf-acceptance-tests'):
-        sh.git('clone',
-               'https://github.com/cloudfoundry/cf-acceptance-tests.git')
-    with cd('cf-acceptance-tests'):
+    if not os.path.exists('src/github.com/cloudfoundry/cf-acceptance-tests'):
+        with cd('src/github.com/cloudfoundry', make=True):
+            sh.git('clone',
+                   'https://github.com/cloudfoundry/cf-acceptance-tests.git')
+    with cd('src/github.com/cloudfoundry/cf-acceptance-tests'):
         sh.git('fetch')
         sh.check('./bin/compile')
         # Switch to the branch of the revision in question
@@ -66,12 +69,13 @@
         sha = acceptance[0]['sha']
         print "Switching CATs to {} to test version {}".format(
             sha[:6], version)
-        sh.git('reset', '--hard', sha)
-
-
-def run_cats():
+        sh.git('checkout', sha)
+
+
+def run_cats(options):
     ep = endpoint()
-    with cd('cf-acceptance-tests'):
+    gopath = os.getcwd()
+    with cd('src/github.com/cloudfoundry/cf-acceptance-tests'):
         fd, fn = tempfile.mkstemp(suffix=".json")
         os.close(fd)
         with open(fn, 'w') as fp:
@@ -81,8 +85,8 @@
                 "api": api(),
                 "syslog_drain_port": 8082,
                 "syslog_ip_address": ep,
-                "password": "admin",
-                "admin_password": "admin",
+                "password": options.admin_password,
+                "admin_password": options.admin_password,
                 "user": "admin",
                 "admin_user": "admin",
                 "org": "juju-org",
@@ -94,7 +98,10 @@
             json.dump(defaults, fp, indent=2, sort_keys=True)
         logging.info("Running CATs %s", defaults)
         env = os.environ.copy()
-        env.update({'CONFIG': os.path.abspath(fn)})
+        env.update({
+            'CONFIG': os.path.abspath(fn),
+            'GOPATH': gopath,
+        })
         subprocess.call('./bin/test', shell=True, env=env)
 
 
@@ -103,9 +110,11 @@
     logging.basicConfig(level=logging.INFO)
     get_cats(options.version)
     bootstrap()
-    deploy()
-    wait_for(60 * 40, 30, cf_service, endpoint, login)
-    run_cats()
+    deploy(generate_dependents=True,
+           admin_password=options.admin_password)
+    wait_for(60 * 40, 30,
+             cf_service, endpoint, partial(login, options.admin_password))
+    run_cats(options)
     sys.exit(0)