Merge lp:~bcsaller/charms/trusty/cloudfoundry/progressbar into lp:~cf-charmers/charms/trusty/cloudfoundry/trunk

[Benjamin Saller]

[lp:~bcsaller/charms/trusty/cloudfoundry/progressbar]

178. By Benjamin Saller on 2014-11-12

attempt to install deps from the wheelhouse, needs verification

[Cory Johns (johnsca)]

See inline comments.

[lp:~bcsaller/charms/trusty/cloudfoundry/progressbar]

179. By Benjamin Saller on 2014-11-12

change func name

180. By Benjamin Saller on 2014-11-13

build out a virtualenv with deps prior to kickoff

[Benjamin Saller (bcsaller)]

I re-pushed this with a venv.

[Cory Johns (johnsca)]

Typo in python-virtualenv package, noted inline, below.

review: Needs Fixing

[Cory Johns (johnsca)]

Actually, I can just fix this while merging.

[Cory Johns (johnsca)]

I propose this change to make it less chatty: http://pastebin.ubuntu.com/8988136/

=== modified file 'README.rst'
--- README.rst	2014-11-06 23:37:55 +0000
+++ README.rst	2014-11-13 00:05:21 +0000
@@ -24,11 +24,6 @@
 We provide a set of helper scripts (in bash) which can be used to assist in the deployment and
 management of CF.
 
-You will need the cf command line client. This can easily be installed using the following:
-
-  wget http://go-cli.s3-website-us-east-1.amazonaws.com/releases/latest/cf-cli_amd64.deb
-  dpkg -i cf-cli_amd64.deb
-
 
 Deployment
 ----------
@@ -51,7 +46,10 @@
 will block until the deployment is fully up and running. It also will create a
 CF "admin" user with the given password, leaving you logged in and ready to push
 a CF app. Note that while this can take quite some time, the whole process is
-automated.
+automated. 
+
+If you don't have the cf command line client the latest one will be installed
+from a .deb package to your local system as part of cfdeploy.
 
 Once the deploy completes, you can push apps. A sample application we recommend
 to become familiar with the system is GitHub High Score website. This can be
 
=== modified file 'cfdeploy'
--- cfdeploy	2014-11-07 00:06:24 +0000
+++ cfdeploy	2014-11-13 00:05:21 +0000
@@ -2,9 +2,48 @@
 # modeline vim:set syntax=python
 import argparse
 import logging
-import webbrowser
+import os
+import subprocess
+import sys
+from contextlib import contextmanager
 from functools import partial
 
+
+def verify_dep(name):
+    try:
+        __import__(name)
+        return True
+    except ImportError:
+        pass
+    return False
+
+
+def install_python_deps():
+    print "Setting up virutalenv."
+    subprocess.check_output(['pip', 'install', 'wheel'])
+    for dep in ['progress', 'requests', 'jujuclient']:
+        subprocess.check_output(['pip', 'install',
+                                 '--use-wheel', '--find-links',
+                                 os.path.abspath('wheelhouse'),
+                                 dep])
+
+
+def prepare_runtime():
+    activate = "bin/activate_this.py"
+    if not verify_dep('virtualenv'):
+        print "(sudo) installing python-virtualenv package."
+        subprocess.check_call(
+            ['sudo', 'apt-get', 'install', '-yq', 'python-virutalenv'])
+    import virtualenv
+    venv = os.path.join(os.getcwd(), '.venv')
+    if not os.path.exists(venv):
+        virtualenv.create_environment(venv, site_packages=False)
+    venv_launcher = os.path.join(venv, activate)
+    execfile(venv_launcher, dict(__file__=venv_launcher))
+    install_python_deps()
+
+prepare_runtime()
+
 from cloudfoundry.releases import RELEASES
 from cloudfoundry.utils import (bootstrap,
                                 cf_service,
@@ -15,9 +54,13 @@
                                 login,
                                 reconciler_endpoint,
                                 webadmin_endpoint,
+                                sh,
                                 socket_open,
                                 until,
-                                wait_for)
+                                which
+                                )
+
+from progress.bar import Bar
 
 
 def setup():
@@ -35,31 +78,91 @@
     return options
 
 
+# This is done because the default webbrowser invocation will output
+# on stderr and muck with the progress bar. By hiding the output
+# with devnull it proceeds as expected.
+show = sh.check('xdg-open')
+
+
+@contextmanager
+def devnull():
+    _save = sys.stderr
+    fp = open('/dev/null', 'w')
+    sys.stderr = fp
+    yield
+    sys.stderr = _save
+
+
 def show_reconciler():
-    uri = "http://%s:8888/" % reconciler_endpoint()
-    webbrowser.open_new_tab(uri)
+    with devnull():
+        uri = "http://%s:8888/" % reconciler_endpoint()
+        show(uri)
 
 
 def show_webadmin():
-    uri = "http://%s:8070/" % webadmin_endpoint()
-    webbrowser.open_new_tab(uri)
+    with devnull():
+        uri = "http://%s:8070/" % webadmin_endpoint()
+        show(uri)
+
+
+class ProgressBar(Bar):
+    message = "Deploying CloudFoundry"
+    fill = "."
+    suffix = "%(percent).1f%% %(elapsed_td)s"
+
+    def next(self, i=1, message=None):
+        if message:
+            self.message = message
+        super(ProgressBar, self).next(i)
+
+
+def install_deps():
+    if not which('cf'):
+        from platform import machine
+        if machine() != "x86_64":
+            print "Unable to install CF CLI for your architecture. "
+            "Deploy will not work as expected."
+            return
+        sh.wget('http://go-cli.s3-website-us-east-1.amazonaws.com/'
+                'releases/latest/cf-cli_amd64.deb')
+        print "Installing CF CLI (this requires sudo access)"
+        sh.sudo('dpkg', '-i', 'cf-cli_amd64.deb')
 
 
 def main():
     options = setup()
     logging.basicConfig(level=options.log_level)
+    install_deps()
+
+    bar = ProgressBar('Deploying CloudFoundry', max=10)
+    bar.start()
+    bar.next(message='Bootstrapping')
     bootstrap()
-    until(juju_state_server)
+    until(juju_state_server, bar=bar, message="Waiting for State Server")
+    bar.next(message='Deploying Orchestrator')
     deploy(constraints=options.constraints,
            generate_dependents=options.generate,
            admin_password=options.admin_password)
-    until(lambda: socket_open(reconciler_endpoint(), 8888))
+    until(lambda: socket_open(reconciler_endpoint(), 8888),
+          bar=bar, message="Waiting on Reconciler")
+    bar.next(message='Showing Reconciler')
     show_reconciler()
+
     # Wait forever, its in the reconciler's hands now.
-    wait_for(0, 30, cf_service, endpoint, partial(login, options.admin_password))
-    until(lambda: socket_open(webadmin_endpoint(), 8070))
+    until(cf_service, bar=bar, message="Waiting for Orchestrator Charm")
+    until(endpoint, bar=bar, message='Waiting for CloudFoundry API endpoint')
+    until(partial(login, options.admin_password),
+          bar=bar, message='Waiting to login to CloudFoundry (long)')
+    until(lambda: socket_open(webadmin_endpoint(), 8070),
+          bar=bar, message="Waiting on webadmin.")
+    bar.next(message="Opening Admin Console")
     show_webadmin()
-    print "You should now be logged into a running CF deployment"
+    bar.finish()
+
+    print "You should now be logged into a running CF deployment."
+    if which('cf'):
+        print "The 'cf' command line client is installed and available."
+
 
 if __name__ == "__main__":
     main()
 
=== modified file 'cloudfoundry/utils.py'
--- cloudfoundry/utils.py	2014-11-07 00:06:24 +0000
+++ cloudfoundry/utils.py	2014-11-13 00:05:21 +0000
@@ -168,6 +168,7 @@
 
 def command(*base_args, **kwargs):
     check = kwargs.pop('check', False)
+    throw = kwargs.pop('throw', True)
 
     def callable_command(*args, **kws):
         kwargs.update(kws)
@@ -176,10 +177,20 @@
         if 'env' not in kwargs:
             kwargs['env'] = os.environ
         logging.debug("invoke: %s", all_args)
+
+        p = subprocess.Popen(all_args,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.STDOUT,
+                             **kwargs)
+        output, _ = p.communicate()
+        ret_code = p.poll()
+        logging.debug('result: %s', output)
         if check is True:
-            return subprocess.check_call(all_args, **kwargs)
+            if ret_code != 0 and throw:
+                raise subprocess.CalledProcessError(ret_code, all_args, output=output)
+            return ret_code
         else:
-            return subprocess.check_output(all_args, **kwargs).strip()
+            return output.strip()
     return callable_command
 
 
@@ -196,7 +207,7 @@
 
 sh = Commander()
 dig = command('dig', '+short')
-api_endpoints = sh.check('juju', 'api-endpoints')
+api_endpoints = sh.check('juju', 'api-endpoints', throw=False)
 
 
 def current_env():
@@ -210,7 +221,7 @@
                         'environments/%s.jenv' % cenv)
 
 
-def wait_for(timeout, interval, *callbacks):
+def wait_for(timeout, interval, *callbacks, **kwargs):
     """
     Repeatedly try callbacks until all return True
 
@@ -223,25 +234,35 @@
     hardware fails, or the heat death of the universe.
     """
     start = time.time()
+    if timeout:
+        end = start + timeout
+    else:
+        end = 0
+
+    bar = kwargs.get('bar', None)
+    message = kwargs.get('message', None)
+    once = 1
     while True:
         passes = True
-        for callback in callbacks:
-            result = callback()
-            passes = passes & bool(result)
-            if passes is False:
-                break
-        if passes is True:
-            break
-        current = time.time()
-        if timeout != 0 and (
-            current - start >= timeout or
-                (current - start) + interval > timeout):
+        if end > 0 and time.time() > end:
             raise OSError("Timeout exceeded in wait_for")
-        time.sleep(interval)
-
-
-def until(*callbacks):
-    return wait_for(0, 20, *callbacks)
+        if bar:
+            bar.next(once, message=message)
+            if once == 1:
+                once = 0
+        if int(time.time()) % interval == 0:
+            for callback in callbacks:
+                result = callback()
+                passes = passes & bool(result)
+                if passes is False:
+                    break
+            if passes is True:
+                break
+        time.sleep(1)
+
+
+def until(*callbacks, **kwargs):
+    return wait_for(0, 20, *callbacks, **kwargs)
 
 
 def juju_state_server():
@@ -249,7 +270,6 @@
         return False
     endpoints = json.loads(sh.juju('api-endpoints', '--format=json'))
     for ep in endpoints:
-        print "Attempt to connect to juju state server:", ep
         host, port = ep.split(':', 1)
         result = socket_open(host, int(port))
         if result is True:
@@ -266,7 +286,6 @@
             # .jenv with the state server
             sh.juju('status')
             data = yaml.load(open(get_jenv()))
-            print "Waiting for state server(s)", data['state-servers']
             return data.get('state-servers', []) != []
         current = current_env()
         wait_for(1000, 20, env_connection)
@@ -387,7 +406,6 @@
         if constraints:
             args.append('--constraints=%s' % constraints)
         args.append('local:trusty/cloudfoundry')
-        print "Deploying:", ' '.join(args)
         sh.juju(*args)
         time.sleep(5)
         sh.juju('expose', 'cloudfoundry')
 
=== modified file 'reconciler/app.py'
--- reconciler/app.py	2014-11-03 16:13:57 +0000
+++ reconciler/app.py	2014-11-13 00:05:21 +0000
@@ -49,7 +49,6 @@
 import tornado.web
 
 from tornado.options import define, options
-from cloudfoundry import config
 from cloudfoundry import utils
 from cloudfoundry.path import path
 from reconciler import strategy
@@ -57,6 +56,8 @@
 from reconciler.ui.app import DashboardIO
 from reconciler.ui.app import static_resources
 
+import config
+
 application = None
 env_name = None
 server = None
@@ -283,7 +284,6 @@
     loop = tornado.ioloop.IOLoop.instance()
     # call poll_health ASAP, to populate the initial health data, and
     # also schedule poll_health for every 60s to refresh health data
-
     loop.add_callback(poll_health)
     tornado.ioloop.PeriodicCallback(poll_health, 60000, io_loop=loop).start()
     tornado.ioloop.PeriodicCallback(poll_current_state, 60000,
@@ -294,6 +294,5 @@
                                         io_loop=loop).start()
     loop.start()
 
-
 if __name__ == "__main__":
     main()
 
=== renamed file 'cloudfoundry/config.py' => 'reconciler/config.py'
=== modified file 'reconciler/strategy.py'
--- reconciler/strategy.py	2014-11-05 23:40:05 +0000
+++ reconciler/strategy.py	2014-11-13 00:05:21 +0000
@@ -7,7 +7,7 @@
 from tornado import gen
 
 from reconciler import tactics
-from cloudfoundry.config import (PENDING, COMPLETE, FAILED, RUNNING)
+from reconciler.config import (PENDING, COMPLETE, FAILED, RUNNING)
 from cloudfoundry import utils
 from cloudfoundry import tsort
 
@@ -160,7 +160,7 @@
             elif expose_intent is False and exposed is True:
                 result.append(tactics.UnexposeTactic(service=service))
 
-        logging.debug("Build New %s", result)
+        logging.debug("Build Strategy %s", result)
         return result
 
     def build_relations(self):
 
=== modified file 'reconciler/tactics.py'
--- reconciler/tactics.py	2014-11-05 23:40:05 +0000
+++ reconciler/tactics.py	2014-11-13 00:05:21 +0000
@@ -3,7 +3,7 @@
 import os
 import shutil
 
-from cloudfoundry.config import (
+from reconciler.config import (
     PENDING, COMPLETE, RUNNING, FAILED, STATES
     )
 
@@ -169,7 +169,7 @@
 class UnexposeTactic(Tactic):
     name = "Unexpose Service"
 
-    def run(self, env, **kwargs):
+    def _run(self, env, **kwargs):
         s = kwargs['service']
         env.unexpose(s['service_name'])
 
 
=== modified file 'tests/test_strategy.py'
--- tests/test_strategy.py	2014-10-30 18:34:30 +0000
+++ tests/test_strategy.py	2014-11-13 00:05:21 +0000
@@ -2,7 +2,7 @@
 import unittest
 import mock
 
-from cloudfoundry import config
+from reconciler import config
 from reconciler import tactics
 from reconciler import strategy
 
 
=== modified file 'tests/test_utils.py'
--- tests/test_utils.py	2014-10-27 19:48:56 +0000
+++ tests/test_utils.py	2014-11-13 00:05:21 +0000
@@ -7,10 +7,10 @@
 
 
 class TestUtils(unittest.TestCase):
-    @mock.patch('subprocess.check_output')
+    @mock.patch('cloudfoundry.utils.sh.juju')
     def test_current_env(self, check_output):
         utils.current_env()
-        check_output.assert_called_once_with(('juju', 'switch'), env=mock.ANY)
+        check_output.assert_called_once_with('switch')
 
     def test_flatten_relations(self):
         r = utils.flatten_relations([
 
=== modified file 'tox.ini'
--- tox.ini	2014-09-30 21:15:05 +0000
+++ tox.ini	2014-11-13 00:05:21 +0000
@@ -30,3 +30,4 @@
     clint
     path.py
     subparse
+    progress
 
=== added file 'wheelhouse/progress-1.2-py2-none-any.whl'
 Binary files wheelhouse/progress-1.2-py2-none-any.whl	1970-01-01 00:00:00 +0000 and wheelhouse/progress-1.2-py2-none-any.whl	2014-11-13 00:05:21 +0000 differ