Merge lp:~johnsca/charms/trusty/cloudfoundry/cfdeploy-errors into lp:~cf-charmers/charms/trusty/cloudfoundry/trunk

[Cory Johns]

Fixed cfdeploy handling of errors during bootstrap and deploy
Improved -l option to log to a file and assume DEBUG

[Benjamin Saller (bcsaller)]

LGTM +1

review: Approve

[lp:~johnsca/charms/trusty/cloudfoundry/cfdeploy-errors]

164. By Cory Johns on 2014-12-04

Improvements to reconciler websocket handling to hopefully prevent errors in log and long delays on initial load

=== modified file 'cfdeploy'
--- cfdeploy	2014-11-13 16:22:27 +0000
+++ cfdeploy	2014-12-04 19:30:36 +0000
@@ -61,6 +61,8 @@
                                 webadmin_endpoint,
                                 sh,
                                 socket_open,
+                                retry,
+                                wait_for,
                                 until,
                                 which
                                 )
@@ -72,7 +74,8 @@
     parser = argparse.ArgumentParser()
     parser.add_argument('-e', '--env', default=current_env())
     parser.add_argument('-v', '--version', default="latest")
-    parser.add_argument('-l', '--log-level', default=logging.INFO)
+    parser.add_argument('-l', '--log', action='store_true',
+                        help='Write debug log to cfdeploy.log')
     parser.add_argument('-c', '--constraints')
     parser.add_argument('-g', '--generate', action='store_true')
     parser.add_argument('admin_password')
@@ -136,18 +139,21 @@
 
 def main():
     options = setup()
-    logging.basicConfig(level=options.log_level)
+    if options.log:
+        root_logger = logging.getLogger()
+        root_logger.handlers = [logging.FileHandler('cfdeploy.log', 'w')]
+        root_logger.setLevel(logging.DEBUG)
     install_deps()
 
     bar = ProgressBar('Deploying CloudFoundry', max=10)
     bar.start()
-    bar.next(message='Bootstrapping')
-    bootstrap()
+    retry(3, bootstrap, bar=bar, message='Bootstrapping')
     until(juju_state_server, bar=bar, message="Waiting for State Server")
     bar.next(message='Deploying Orchestrator')
-    deploy(constraints=options.constraints,
-           generate_dependents=options.generate,
-           admin_password=options.admin_password)
+    wait_for(30, 5, partial(deploy,
+             constraints=options.constraints,
+             generate_dependents=options.generate,
+             admin_password=options.admin_password))
     until(lambda: socket_open(reconciler_endpoint(), 8888),
           bar=bar, message="Waiting on Reconciler")
     bar.next(message='Showing Reconciler')

=== modified file 'cloudfoundry/utils.py'
--- cloudfoundry/utils.py	2014-11-12 18:24:39 +0000
+++ cloudfoundry/utils.py	2014-12-04 19:30:36 +0000
@@ -184,7 +184,7 @@
                              **kwargs)
         output, _ = p.communicate()
         ret_code = p.poll()
-        logging.debug('result: %s', output)
+        logging.debug('output: %s', output)
         if check is True:
             if ret_code != 0 and throw:
                 raise subprocess.CalledProcessError(ret_code, all_args, output=output)
@@ -265,6 +265,23 @@
     return wait_for(0, 20, *callbacks, **kwargs)
 
 
+def retry(attempts, *callbacks, **kwargs):
+    """
+    Repeatedly try callbacks a fixed number of times or until all return True
+    """
+    for attempt in xrange(attempts):
+        if 'bar' in kwargs:
+            kwargs['bar'].next(attempt == 0, message=kwargs.get('message'))
+        for callback in callbacks:
+            if not callback():
+                break
+        else:
+            break
+    else:
+        raise OSError("Retry attempts exceeded")
+    return True
+
+
 def juju_state_server():
     if api_endpoints() != 0:
         return False
@@ -356,7 +373,9 @@
 
 def bootstrap():
     if not os.path.exists(get_jenv()) or api_endpoints() != 0:
-        sh.juju('bootstrap')
+        juju = sh.check('juju', throw=False)
+        return juju('bootstrap') != 0
+    return True
 
 
 def get_admin_password():
@@ -386,30 +405,35 @@
 def deploy(**config):
     status = get_client().status()
     constraints = config.pop('constraints', None)
-    if 'cloudfoundry' not in status['Services']:
-        # create an up to date config
-        config = dict({
-            'admin_secret': get_admin_password(),
-            'cf_version': 'latest',
-            'placement': 'dense',
-        }, **config)
-
-        fd, fn = tempfile.mkstemp(suffix='.yaml')
-        os.close(fd)
-        with open(fn, 'w') as fp:
-            yaml.dump({'cloudfoundry': config}, fp)
-
-        repo_path = get_repo_path()
-
-        args = ['deploy', '--config=%s' % fn,
-                '--repository=%s' % repo_path]
-        if constraints:
-            args.append('--constraints=%s' % constraints)
-        args.append('local:trusty/cloudfoundry')
-        sh.juju(*args)
-        time.sleep(5)
-        sh.juju('expose', 'cloudfoundry')
-        os.unlink(fn)
+    if 'cloudfoundry' in status['Services']:
+        return True
+    # create an up to date config
+    config = dict({
+        'admin_secret': get_admin_password(),
+        'cf_version': 'latest',
+        'placement': 'dense',
+    }, **config)
+
+    fd, fn = tempfile.mkstemp(suffix='.yaml')
+    os.close(fd)
+    with open(fn, 'w') as fp:
+        yaml.dump({'cloudfoundry': config}, fp)
+
+    repo_path = get_repo_path()
+
+    args = ['deploy', '--config=%s' % fn,
+            '--repository=%s' % repo_path]
+    if constraints:
+        args.append('--constraints=%s' % constraints)
+    args.append('local:trusty/cloudfoundry')
+    juju = sh.check('juju', throw=False)
+    if juju(*args) != 0:
+        return False
+    time.sleep(5)
+    if juju('expose', 'cloudfoundry') != 0:
+        return False
+    os.unlink(fn)
+    return True
 
 
 def login(password):

=== modified file 'reconciler/ui/app.py'
--- reconciler/ui/app.py	2014-10-17 22:34:42 +0000
+++ reconciler/ui/app.py	2014-12-04 19:30:36 +0000
@@ -1,4 +1,5 @@
 import json
+import logging
 from path import path
 from tornado.websocket import WebSocketHandler
 import tornado.ioloop
@@ -29,7 +30,7 @@
     listeners = {}
 
     def open(self):
-        self.connections.add(self)
+        DashboardIO.connections.add(self)
 
     def on_message(self, message):
         """
@@ -43,7 +44,7 @@
             listener(data)
 
     def on_close(self):
-        self.connections.remove(self)
+        DashboardIO.connections.discard(self)
 
     @classmethod
     def send(cls, message):
@@ -51,7 +52,10 @@
         Send a message out over all connected WebSockets.
         """
         for connection in cls.connections:
-            connection.write_message(message)
+            try:
+                connection.write_message(message)
+            except:
+                logging.error('Error sending message', exc_info=True)
 
     @classmethod
     def add_listener(cls, message_type, listener):