Merge lp:~gnuoy/openstack-mojo-specs/juju-when-are-you-done into lp:openstack-mojo-specs

[Liam Young]

[Liam Young (gnuoy)]

How mojo detects when juju has finished can now be influenced by two environment variables.

1) JUJU_RUN_TIMEOUT When the system being deployed on is busy sometimes the default 5mins is not enough so this can be set to a higher value to make juju run wait longer for the queued run action to complete
2) USE_WORKLOAD_STATUS By default this is true and will use the charms workload status to decide when the charms are finished. Setting this to 'false' will cause mojo to flip back to the old behaviour of relying on juju run.

[uosci-testing-bot (uosci-testing-bot)]

charm_lint_check #11894 mojo-openstack-specs for gnuoy mp274390
    LINT OK: passed

Build: http://10.245.162.77:8080/job/charm_lint_check/11894/

Unmerged revisions

229. By Liam Young on 2015-10-14

Allow JUJU_RUN_TIMEOUT to be set to specify juju run timeout

228. By Liam Young on 2015-10-14

Use workload status to check deploy has finished unless disabled by USE_WORKLOAD_STATUS

=== modified file 'helper/tests/check_juju.py'
--- helper/tests/check_juju.py	2015-01-13 13:12:47 +0000
+++ helper/tests/check_juju.py	2015-10-14 13:11:05 +0000
@@ -1,5 +1,4 @@
 #!/usr/bin/python
 import utils.mojo_utils as mojo_utils
 
-mojo_utils.juju_check_hooks_complete()
-mojo_utils.juju_status_check_and_wait()
+mojo_utils.juju_wait_finished()
 
=== modified file 'helper/utils/mojo_utils.py'
--- helper/utils/mojo_utils.py	2015-10-09 13:47:36 +0000
+++ helper/utils/mojo_utils.py	2015-10-14 13:11:05 +0000
@@ -9,11 +9,13 @@
 import yaml
 from collections import Counter
 
+# XXX 'blocked' can be a transitional or bad state depending on whether the
+#     deployment has finished
 JUJU_STATUSES = {
-    'good': ['ACTIVE', 'started'],
+    'good': ['ACTIVE', 'started', 'active'],
     'bad': ['error'],
     'transitional': ['pending', 'pending', 'down', 'installed', 'stopped',
-                     'allocating'],
+                     'allocating', 'maintenance', 'blocked'],
 }
 
 
@@ -83,6 +85,10 @@
 
 def remote_shell_check(unit, timeout=None):
     cmd = ['juju', 'run']
+    env_timeout = os.environ.get('JUJU_RUN_TIMEOUT')
+    # Environment specified timeout overrides internally set timeout
+    if env_timeout:
+        timeout = env_timeout
     if timeout:
         cmd.extend(['--timeout', str(timeout)])
     cmd.extend(['--unit', unit, 'uname -a'])
@@ -94,6 +100,10 @@
     if fatal is None:
         fatal = True
     cmd = ['juju', 'run', '--unit', unit]
+    env_timeout = os.environ.get('JUJU_RUN_TIMEOUT')
+    # Environment specified timeout overrides internally set timeout
+    if env_timeout:
+        timeout = env_timeout
     if timeout:
         cmd.extend(['--timeout', str(timeout)])
     if remote_cmd:
@@ -379,6 +389,22 @@
     return get_machine_state(juju_status, 'instance-state')
 
 
+def get_workload_states(juju_status):
+    service_state = Counter()
+    for service in juju_status['services']:
+        if 'units' in juju_status['services'][service]:
+            for unit in juju_status['services'][service]['units']:
+                unit_info = juju_status['services'][service]['units'][unit]
+                service_state[unit_info['workload-status']['current']] += 1
+                if 'subordinates' in unit_info:
+                    subs = unit_info['subordinates']
+                    for sub_unit in subs:
+                        sub_sstate = \
+                            subs[sub_unit]['workload-status']['current']
+                        service_state[sub_sstate] += 1
+    return service_state
+
+
 def get_service_agent_states(juju_status):
     service_state = Counter()
     for service in juju_status['services']:
@@ -419,7 +445,7 @@
     return True
 
 
-def juju_status_check_and_wait():
+def juju_status_check_and_wait(use_workload_status=True):
     checks = {
         'Machines': [
             {
@@ -438,6 +464,11 @@
             }
         ]
     }
+    if use_workload_status:
+        checks['Services'].append({
+            'Heading': 'Workload State',
+            'check_func': get_workload_states
+        })
     stable_state = [False]
     while False in stable_state:
         juju_status = get_juju_status()
@@ -470,12 +501,19 @@
 
 
 def juju_wait_finished():
-    # Wait till all statuses are green
-    juju_status_check_and_wait()
-    # juju status may report all has finished but hooks are still firing.
-    juju_check_hooks_complete()
-    # Check nothing has subsequently gone bad
-    juju_status_check_and_wait()
+    use_wls = os.environ.get('USE_WORKLOAD_STATUS')
+    print use_wls
+    if use_wls and use_wls.lower() == 'false':
+        logging.debug("Ignoring workload status, falling back to juju run")
+        # Wait till all statuses are green
+        juju_status_check_and_wait(use_workload_status=False)
+        # juju status may report all has finished but hooks are still firing.
+        juju_check_hooks_complete()
+        # Check nothing has subsequently gone bad
+        juju_status_check_and_wait(use_workload_status=False)
+    else:
+        logging.debug("Using workload status to check deployment complete")
+        juju_status_check_and_wait()
 
 
 def dict_to_yaml(dict_data):