Merge lp:~whitmo/charms/trusty/cloudfoundry/cf-disco-check into lp:~cf-charmers/charms/trusty/cloudfoundry/trunk

[Whit Morriss]

Basic implementation of health check using healthz & varz endpoint. Only applied to DEA node currently.

[lp:~whitmo/charms/trusty/cloudfoundry/cf-disco-check]

184. By Whit Morriss on 2014-12-12

fix typo

185. By Whit Morriss on 2014-12-12

fix tests for chmod

186. By Whit Morriss on 2014-12-12

fix path for disco install

187. By Whit Morriss on 2014-12-17

merge

Unmerged revisions

187. By Whit Morriss on 2014-12-17

merge

186. By Whit Morriss on 2014-12-12

fix path for disco install

185. By Whit Morriss on 2014-12-12

fix tests for chmod

184. By Whit Morriss on 2014-12-12

fix typo

183. By Whit Morriss on 2014-12-12

make disco executable

182. By Whit Morriss on 2014-12-12

move disco install to data_ready

181. By Whit Morriss on 2014-12-12

clean up chaining

180. By Whit Morriss on 2014-12-11

refactor config to not include component type. other tweaks

179. By Whit Morriss on 2014-12-11

initial declaration of healthz for DEA

178. By Whit Morriss on 2014-12-11

extend to use job level health check definitions (untested)

=== modified file 'cfdeploy'
--- cfdeploy	2014-12-03 23:14:23 +0000
+++ cfdeploy	2014-12-17 16:51:23 +0000
@@ -19,7 +19,7 @@
 
 
 def install_python_deps():
-    print "Setting up virutalenv."
+    print "Setting up virtualenv."
     subprocess.check_output(['pip', 'install', 'wheel'])
     for dep in ['progress', 'requests', 'jujuclient']:
         subprocess.check_output(['pip', 'install',
@@ -77,6 +77,7 @@
     parser.add_argument('-l', '--log', action='store_true',
                         help='Write debug log to cfdeploy.log')
     parser.add_argument('-c', '--constraints')
+    parser.add_argument('--to')
     parser.add_argument('-g', '--generate', action='store_true')
     parser.add_argument('admin_password')
 
@@ -152,6 +153,7 @@
     bar.next(message='Deploying Orchestrator')
     wait_for(30, 5, partial(deploy,
              constraints=options.constraints,
+             to=options.to,
              generate_dependents=options.generate,
              admin_password=options.admin_password))
     until(lambda: socket_open(reconciler_endpoint(), 8888),

=== modified file 'cloudfoundry/health_checks.py'
--- cloudfoundry/health_checks.py	2014-11-20 15:50:36 +0000
+++ cloudfoundry/health_checks.py	2014-12-17 16:51:23 +0000
@@ -36,3 +36,14 @@
         return result
     else:
         return dict(result, health='warn', message='Working (%s)' % status['status'])
+
+
+def healthz_check(ctype, service, healthz=tasks.healthz, varz=tasks.varz):
+    health = healthz(ctype) and 'pass' or 'fail'
+    result = {
+        'name': 'healthz',
+        'health': health,
+        'message': None,
+        'data': varz(ctype),
+    }
+    return result

=== modified file 'cloudfoundry/jobs.py'
--- cloudfoundry/jobs.py	2014-11-20 15:50:36 +0000
+++ cloudfoundry/jobs.py	2014-12-17 16:51:23 +0000
@@ -2,6 +2,7 @@
 from functools import partial
 from .path import path
 import yaml
+import itertools
 
 from charmhelpers.core import hookenv
 from charmhelpers.core import services
@@ -68,7 +69,13 @@
     service_def = service_data[charm_name]
     results = []
     health = 'pass'
-    checks = service_def.get('health', []) + [health_checks.status]
+    job_health = [job['health'] for job in service_def['jobs'] \
+                  if 'health' in job]
+    base_checks = service_def.get('health', [])
+    checks = itertools.chain(base_checks,
+                             itertools.chain(*job_health),
+                             [health_checks.status])
+
     for health_check in checks:
         result = health_check(service_def)
         if result['health'] == 'fail':

=== modified file 'cloudfoundry/releases.py'
--- cloudfoundry/releases.py	2014-12-08 19:48:50 +0000
+++ cloudfoundry/releases.py	2014-12-17 16:51:23 +0000
@@ -26,9 +26,12 @@
     ('etcd:client', 'loggregator-trafficcontrol:etcd'),
 ]
 
+UTILITIES = {
+    "cf-disco": "cf-disco-v0.0.1-538768711c7545d87e60ad79ec3568d8c77c7db6"
+}
+
 COMMON_UPGRADES = []
 
-
 RELEASES = [
     {
         "releases": (190, 190),

=== modified file 'cloudfoundry/services.py'
--- cloudfoundry/services.py	2014-12-08 23:20:58 +0000
+++ cloudfoundry/services.py	2014-12-17 16:51:23 +0000
@@ -3,6 +3,10 @@
 import tasks
 import utils
 
+from functools import partial
+from .health_checks import healthz_check
+
+
 __all__ = ['SERVICES']
 
 
@@ -264,7 +268,6 @@
 
     },
 
-
     'dea-v3': {
         'summary': 'DEA runs CF apps in containers',
         'description': '',
@@ -285,9 +288,12 @@
                     contexts.RouterRelation,
                 ],
                 'data_ready': [
-                    #tasks.enable_swapaccounting
+                    tasks.install_cfdisco,
+                    tasks.config_cfdisco,
+                    # tasks.enable_swapaccounting,
                     tasks.patch_dea
-                ]
+                ],
+                'health': [partial(healthz_check, "DEA")]
             },
             {
                 'job_name': 'dea_logging_agent',
@@ -297,10 +303,13 @@
                                   contexts.EtcdRelation]
             },
             {'job_name': 'metron_agent',
+             'data_ready': [tasks.install_cfdisco,
+                            tasks.config_cfdisco],
              'required_data': [contexts.LTCRelation,
                                contexts.NatsRelation,
                                contexts.DopplerRelation,
-                               contexts.EtcdRelation]},
+                               contexts.EtcdRelation],
+             'health': [partial(healthz_check, "MetronAgent")]},
         ]
 
     },

=== modified file 'cloudfoundry/tasks.py'
--- cloudfoundry/tasks.py	2014-10-03 15:47:17 +0000
+++ cloudfoundry/tasks.py	2014-12-17 16:51:23 +0000
@@ -1,12 +1,14 @@
+import json
+import logging
 import os
 import re
+import shlex
 import shutil
+import stat
 import subprocess
-import yaml
-import stat
 import tempfile
 import textwrap
-import logging
+import yaml
 from urlparse import urlparse
 from functools import partial
 from charmhelpers.core import host
@@ -16,6 +18,7 @@
 from cloudfoundry import contexts
 from cloudfoundry import templating
 from cloudfoundry import utils
+from .releases import UTILITIES
 from .path import path
 
 logger = logging.getLogger(__name__)
@@ -41,7 +44,8 @@
     enable_monit_http_interface()
     subprocess.check_call(['gem', 'install', '--no-ri', '--no-rdoc', gem_file])
     subprocess.check_call([
-        'pip', 'install', '--use-wheel', '-f', './wheelhouse', '--pre', 'raindance'])
+        'pip', 'install', '--use-wheel', '-f', './wheelhouse', '--pre',
+        'raindance'])
 
 
 def install(service_def):
@@ -99,7 +103,8 @@
         return
     from raindance.package import PackageArchive
     pa = PackageArchive(url)
-    mirror = pa.build_mirror_section(ARTIFACTS_DIR, SOFTWARE, [(version, ARCH)], [job_name])
+    mirror = pa.build_mirror_section(ARTIFACTS_DIR,
+                                     SOFTWARE, [(version, ARCH)], [job_name])
     for filename in mirror:
         pass  # just need to iterate to force the (lazy) download
 
@@ -203,6 +208,76 @@
         subprocess.check_call(['patch', '-s', '-F4'], stdin=patch)
         os.unlink(fn)
 
+JUJU_VCAP_BIN = path("/var/vcap/juju/bin")
+
+
+def install_cfdisco(job_name, jujubin=JUJU_VCAP_BIN,
+                    discov=UTILITIES['cf-disco']):
+    orch = contexts.OrchestratorRelation()
+
+    urlbase = path(orch.get_first('artifacts_url'))
+    url = urlbase / ('cf/utilities/%s' % discov)
+    sha1 = discov.rsplit("-", 1)[1]
+
+    outfile = jujubin / 'cf-disco'
+
+    from raindance.package import PackageArchive
+    verify = PackageArchive.verify_file
+
+    if not (outfile.exists() and verify(outfile, sha1)):
+        jujubin.makedirs_p()
+        PackageArchive.wget(url, outfile)
+        assert PackageArchive.verify_file(outfile, sha1), "Bad sha1 for %s" % outfile
+        outfile.chmod(755)
+
+
+JUJU_VCAP_ETC = path("/var/vcap/juju/etc")
+
+
+def config_cfdisco(job_name, jujuetc=JUJU_VCAP_ETC):
+    disco_conf = jujuetc / 'disco.json'
+    if not disco_conf.exists():
+        nats = contexts.NatsRelation()
+        nats_info = nats.erb_mapping()
+        nats_info = dict((k.replace('nats.', ''), v) for k, v in nats_info.items())
+        nats_info['addr'] = nats.get_first('address')
+
+        # @@ fix to use multiple nats servers
+        nats_uri_tmpt = "nats://{user}:{password}@{addr}:{port}"
+        nats_uri = nats_uri_tmpt.format(**nats_info)
+        privip = hookenv.unit_private_ip()
+
+        jujuetc.makedirs_p()
+        disco_conf.write_text(json.dumps(dict(nats_uri=nats_uri,
+                                              ip=privip)))
+
+
+def healthz(ctype, exe=JUJU_VCAP_BIN / 'cf-disco',
+            config=JUJU_VCAP_ETC / 'disco.json'):
+    conf = json.loads(config.text())
+    conf['type'] = ctype
+    tmplt = "{exe} {type} {ip} {nats_uri}"
+    conf['exe'] = exe
+    args = shlex.split(tmplt.format(**conf))
+    try:
+        subprocess.check_output(args)
+        return True
+    except subprocess.CalledProcessError as e:
+        if e.returncode == 1:
+            return False
+        raise
+
+
+def varz(ctype, exe=JUJU_VCAP_BIN / 'cf-disco',
+         config=JUJU_VCAP_ETC / 'disco.json'):
+    conf = json.loads(config.text())
+    conf['type'] = ctype
+    tmplt = "{exe} --check='varz' {type} {ip} {nats_uri}"
+    conf['exe'] = exe
+    args = shlex.split(tmplt.format(**conf))
+    out = subprocess.check_output(args)
+    return json.loads(out)
+
 
 class JobTemplates(services.ManagerCallback):
     template_base_dir = TEMPLATES_BASE_DIR

=== modified file 'cloudfoundry/utils.py'
--- cloudfoundry/utils.py	2014-12-09 22:56:46 +0000
+++ cloudfoundry/utils.py	2014-12-17 16:51:23 +0000
@@ -171,6 +171,7 @@
 def command(*base_args, **kwargs):
     check = kwargs.pop('check', False)
     throw = kwargs.pop('throw', True)
+    combine_output = kwargs.pop('combine_output', False)
 
     def callable_command(*args, **kws):
         kwargs.update(kws)
@@ -182,17 +183,18 @@
 
         p = subprocess.Popen(all_args,
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.STDOUT,
+                             stderr=subprocess.PIPE,
                              **kwargs)
-        output, _ = p.communicate()
+        output, outerr = p.communicate()
         ret_code = p.poll()
-        logging.debug('output: %s', output)
-        if check is True:
-            if ret_code != 0 and throw:
-                raise subprocess.CalledProcessError(ret_code, all_args, output=output)
+        if check and throw and ret_code != 0:
+            raise subprocess.CalledProcessError(ret_code, all_args, output=output)
+        elif check and not throw:
             return ret_code
+        elif combine_output:
+            return (output + outerr).strip()
         else:
-            return output.strip()
+            return map(str.strip, (output, outerr))
     return callable_command
 
 
@@ -407,6 +409,7 @@
 def deploy(**config):
     status = get_client().status()
     constraints = config.pop('constraints', None)
+    to = config.pop('to', None)
     if 'cloudfoundry' in status['Services']:
         return True
     # create an up to date config
@@ -427,6 +430,10 @@
             '--repository=%s' % repo_path]
     if constraints:
         args.append('--constraints=%s' % constraints)
+
+    if to:
+        args.append('--to=%s' % to)
+
     args.append('local:trusty/cloudfoundry')
     juju = sh.check('juju', throw=False)
     if juju(*args) != 0:

=== modified file 'reconciler/app.py'
--- reconciler/app.py	2014-12-08 19:38:24 +0000
+++ reconciler/app.py	2014-12-17 16:51:23 +0000
@@ -102,7 +102,7 @@
     try:
         unit_dash = unit_name.replace('/', '-')
         key_file = path(__file__).dirname() / '..' / 'orchestrator-key'
-        output = subprocess.check_output([
+        output, outerr = utils.command(
             'ssh', 'root@{}'.format(unit_addr),
             '-i', key_file,
             '-o', 'UserKnownHostsFile=/dev/null',
@@ -112,12 +112,15 @@
                 'cd $CHARM_DIR',
                 'hooks/health',
             ]),
-        ], stderr=subprocess.STDOUT)
+            combine_output=False, check=True)()
         try:
             unit.update(yaml.safe_load(output))
         except (yaml.error.YAMLError, ValueError):
             unit['health'] = 'fail'
-            unit['state'] = {'message': 'Unable to parse health: {}'.format(output)}
+            unit['state'] = {
+                'message': 'Unable to parse health: '
+                           'stdout: {}'
+                           'stderr: {}'.format(output, outerr)}
     except subprocess.CalledProcessError as e:
         unit['state'] = {'message': 'Unable to retrieve health: {}'.format(e.output)}
         if unit_state == 'started':

=== modified file 'tests/test_tasks.py'
--- tests/test_tasks.py	2014-10-03 15:47:17 +0000
+++ tests/test_tasks.py	2014-12-17 16:51:23 +0000
@@ -1,11 +1,20 @@
 import unittest
 import mock
+import contextlib
+import tempfile
+import json
 
+from subprocess import CalledProcessError
 from charmhelpers.core import services
 from cloudfoundry.path import path
 from cloudfoundry import tasks
 
 
+def patch_set(*methods):
+    patches = [mock.patch(x) for x in methods]
+    return contextlib.nested(*patches)
+
+
 class TestTasks(unittest.TestCase):
     def setUp(self):
         self.charm_dir_patch = mock.patch(
@@ -161,3 +170,85 @@
         output = tasks._enable_swapaccounting(sample)
         self.assertIn("arg1 cgroup_enable=memory swapaccount=1", output)
         self.assertIn("recovery arg2 cgroup_enable=memory swapaccount=1", output)
+
+    def test_cfdisco_install(self):
+        patches = ['cloudfoundry.contexts.OrchestratorRelation',
+                   'raindance.package.PackageArchive.wget',
+                   'raindance.package.PackageArchive.verify_file',
+                   'cloudfoundry.path.path.chmod']
+
+        bin_d = path(tempfile.mkdtemp(prefix='cf-tests-'))
+        with patch_set(*patches) as (om, wm, vm, chm):
+            om().get_first.return_value = 'http://pkg/url'
+            vm.return_value = True
+            tasks.install_cfdisco("some_job",
+                                  jujubin=bin_d,
+                                  discov="bin-version")
+            assert wm.called
+            expect = mock.call(path(u'http://pkg/url/utilities/bin-version'),
+                               bin_d / 'cf-disco')
+            assert wm.call_args == expect
+            assert vm.called
+
+    def test_config_cfdisco(self):
+        patches = ('cloudfoundry.contexts.NatsRelation',
+                   'charmhelpers.core.hookenv.unit_private_ip')
+        with patch_set(*patches) as (cxt, ipm):
+            cxt().get_first.return_value = "10.0.0.1"
+            cxt().erb_mapping.return_value = {
+                'nats.user': 'user',
+                'nats.password': 'pw',
+                'nats.port': "1234"
+                }
+            local_ip = ipm.return_value = "10.0.0.10"
+            etc = path(tempfile.mkdtemp(prefix='cf-tests-'))
+            result = etc / 'disco.json'
+            tasks.config_cfdisco("JOB", etc)
+            assert result.exists()
+            outval = json.loads(result.text())
+            assert outval['ip'] == local_ip
+            assert outval['nats_uri'] == 'nats://user:pw@10.0.0.1:1234'
+
+    def test_check_healthz_ok(self):
+        config = path(tempfile.mkstemp()[1])
+        conf_txt = json.dumps(dict(type='COMP',
+                                   ip='10.0.0.1',
+                                   nats_uri='NATS_URI'))
+        config.write_text(conf_txt)
+        with mock.patch('subprocess.check_call') as cm:
+            cm.return_value = 0
+            assert tasks.healthz('CF-THANG', exe='cd', config=config) is True
+
+    def test_check_healthz_not_ok(self):
+        config = path(tempfile.mkstemp()[1])
+        conf_txt = json.dumps(dict(type='COMP',
+                                   ip='10.0.0.1',
+                                   nats_uri='NATS_URI'))
+
+        config.write_text(conf_txt)
+        with mock.patch('subprocess.check_call') as cm:
+            cm.side_effect = CalledProcessError(1, "CMD")
+            assert tasks.healthz('CF-THANG', exe='cd', config=config) is False
+
+    def test_healthz_raises(self):
+        config = path(tempfile.mkstemp()[1])
+        conf_txt = json.dumps(dict(type='COMP',
+                                   ip='10.0.0.1',
+                                   nats_uri='NATS_URI'))
+        config.write_text(conf_txt)
+        with mock.patch('subprocess.check_call') as cm:
+            cm.side_effect = CalledProcessError(2, "CMD")
+            with self.assertRaises(CalledProcessError):
+                tasks.healthz('CF-THANG', exe='cd', config=config)
+
+    def test_check_varz(self):
+        config = path(tempfile.mkstemp()[1])
+        conf_txt = json.dumps(dict(type='COMP',
+                                   ip='10.0.0.1',
+                                   nats_uri='NATS_URI'))
+
+        config.write_text(conf_txt)
+        with mock.patch('subprocess.check_output') as co:
+            co.return_value = "{}"
+            out = tasks.varz('CF-THANG', "exe", config)
+            assert out == {}