Merge lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/status into lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk

[Cory Johns]

Added extended status support.

Example status (tabular) outputs:

http://pastebin.ubuntu.com/11769979/ (in progress)
http://pastebin.ubuntu.com/11770196/ (complete)

Example status-history outputs:

http://pastebin.ubuntu.com/11770192/ (hdfs-master)
http://pastebin.ubuntu.com/11770195/ (compute-slave)

[Cory Johns (johnsca)]

I also bundled charmhelpers and its dependencies, since there is almost no churn on it now and it made the status handling code cleaner as I could depend on hookenv.status_set() always being there.

[Kevin W Monroe (kwmonroe)]

Looks really good Cory! I deployed to AWS and all the status changes looked legit (i like the inclusion of '/etc/hosts registration' in addition to the normal boring enhanced status).

I have a beef, though. This feels heavy:

+ archives = glob('resources/*')
+ subprocess.check_call(['pip', 'install'] + archives)

What if we want to shove things in ./resources that aren't pip'able? We can either cross that bridge when we get to it, or shove pip stuffs in ./resources/pip. Thoughts?

[Cory Johns (johnsca)]

> Looks really good Cory! I deployed to AWS and all the status changes looked
> legit (i like the inclusion of '/etc/hosts registration' in addition to the
> normal boring enhanced status).

What /etc/hosts registration are you referring to?

> I have a beef, though. This feels heavy:
>
> + archives = glob('resources/*')
> + subprocess.check_call(['pip', 'install'] + archives)
>
> What if we want to shove things in ./resources that aren't pip'able? We can
> either cross that bridge when we get to it, or shove pip stuffs in
> ./resources/pip. Thoughts?

Yeah. Honestly, I just got tired of adding things there that I had missed. Could do a python subdirectory, though.

[lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/status]

86. By Cory Johns on 2015-06-25

Nested python resources to avoid attempting to pip install future non-python resources

[Kevin W Monroe (kwmonroe)]

+1 Cory! LGTM, merged into r85.

review: Approve

[amir sanjar (asanjar)]

review: Approve

=== modified file 'hooks/callbacks.py'
--- hooks/callbacks.py	2015-05-12 21:56:13 +0000
+++ hooks/callbacks.py	2015-06-25 13:44:06 +0000
@@ -15,3 +15,46 @@
 Add any additional tasks / setup here.  If a callback is used by mutliple
 charms, consider refactoring it up to the jujubigdata library.
 """
+
+from charmhelpers.core import hookenv
+from charmhelpers.core import unitdata
+from jujubigdata.relations import NameNodeMaster, ResourceManagerMaster
+
+
+def update_blocked_status():
+    if unitdata.kv().get('charm.active', False):
+        return
+    rels = (
+        ('Yarn', 'ResourceManager', ResourceManagerMaster()),
+        ('HDFS', 'NameNode', NameNodeMaster()),
+    )
+    missing_rel = [rel for rel, res, impl in rels if not impl.connected_units()]
+    missing_hosts = [rel for rel, res, impl in rels if not impl.am_i_registered()]
+    not_ready = [(rel, res) for rel, res, impl in rels if not impl.is_ready()]
+    if missing_rel:
+        hookenv.status_set('blocked', 'Waiting for relation to %s master%s' % (
+            ' and '.join(missing_rel),
+            's' if len(missing_rel) > 1 else '',
+        )),
+    elif missing_hosts:
+        hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % (
+            ' and '.join(missing_hosts),
+        ))
+    elif not_ready:
+        unready_rels, unready_ress = zip(*not_ready)
+        hookenv.status_set('waiting', 'Waiting for %s to provide %s' % (
+            ' and '.join(unready_rels),
+            ' and '.join(unready_ress),
+        ))
+
+
+def update_working_status():
+    if unitdata.kv().get('charm.active', False):
+        hookenv.status_set('maintenance', 'Updating configuration')
+        return
+    hookenv.status_set('maintenance', 'Setting up NodeManager and DataNode')
+
+
+def update_active_status():
+    unitdata.kv().set('charm.active', True)
+    hookenv.status_set('active', 'Ready')

=== modified file 'hooks/common.py'
--- hooks/common.py	2015-06-19 17:24:59 +0000
+++ hooks/common.py	2015-06-25 13:44:06 +0000
@@ -15,29 +15,39 @@
 """
 
 import jujuresources
+from charmhelpers.core import hookenv
+from charmhelpers.core import unitdata
+from charmhelpers.core import charmframework
 
 
 def bootstrap_resources():
     """
     Install required resources defined in resources.yaml
     """
+    if unitdata.kv().get('charm.bootstrapped', False):
+        return True
+    hookenv.status_set('maintenance', 'Installing base resources')
     mirror_url = jujuresources.config_get('resources_mirror')
     if not jujuresources.fetch(mirror_url=mirror_url):
-        jujuresources.juju_log('Resources unavailable; manual intervention required', 'ERROR')
+        missing = jujuresources.invalid()
+        hookenv.status_set('blocked', 'Unable to fetch required resource%s: %s' % (
+            's' if len(missing) > 1 else '',
+            ', '.join(missing),
+        ))
         return False
-    jujuresources.install(['pathlib', 'pyaml', 'six', 'charmhelpers', 'jujubigdata'])
+    jujuresources.install(['pathlib', 'jujubigdata'])
+    unitdata.kv().set('charm.bootstrapped', True)
     return True
 
 
 def manage():
     if not bootstrap_resources():
-        # defer until resources are available, since charmhelpers, and thus
-        # the framework, (will be) managed by jujuresources
+        # defer until resources are available, since jujubigdata, and thus the
+        # classes needed for the requires blocks, (will be) managed by jujuresources
         return
 
-    from charmhelpers.core import charmframework
     import jujubigdata
-    import callbacks  # noqa  (ignore when linting)
+    import callbacks
 
     # list of keys required to be in the dist.yaml
     slave_reqs = ['vendor', 'hadoop_version', 'packages', 'groups', 'users',
@@ -57,51 +67,43 @@
             ],
             'callbacks': [
                 hadoop.install,
+                callbacks.update_blocked_status,
             ],
         },
         {
-            'name': 'compute-slave-hdfs',
+            'name': 'compute-slave',
             'provides': [
                 jujubigdata.relations.DataNode(),
+                jujubigdata.relations.NodeManager(),
             ],
             'requires': [
                 hadoop.is_installed,
                 hdfs_relation,
+                yarn_relation,
                 hdfs_relation.am_i_registered,
+                yarn_relation.am_i_registered,
             ],
             'callbacks': [
+                callbacks.update_working_status,
                 hdfs_relation.register_provided_hosts,
+                yarn_relation.register_provided_hosts,
                 jujubigdata.utils.manage_etc_hosts,
                 hdfs_relation.install_ssh_keys,
+                yarn_relation.install_ssh_keys,
                 hdfs.configure_datanode,
+                yarn.configure_nodemanager,
                 hdfs.start_datanode,
-                charmframework.helpers.open_ports(dist_config.exposed_ports('compute-slave-hdfs')),
+                yarn.start_nodemanager,
+                charmframework.helpers.open_ports(
+                    dist_config.exposed_ports('compute-slave-hdfs') +
+                    dist_config.exposed_ports('compute-slave-yarn')),
+                callbacks.update_active_status,
             ],
             'cleanup': [
-                charmframework.helpers.close_ports(dist_config.exposed_ports('compute-slave-hdfs')),
+                charmframework.helpers.close_ports(
+                    dist_config.exposed_ports('compute-slave-hdfs') +
+                    dist_config.exposed_ports('compute-slave-yarn')),
                 hdfs.stop_datanode,
-            ],
-        },
-        {
-            'name': 'compute-slave-yarn',
-            'provides': [
-                jujubigdata.relations.NodeManager(),
-            ],
-            'requires': [
-                hadoop.is_installed,
-                yarn_relation,
-                yarn_relation.am_i_registered,
-            ],
-            'callbacks': [
-                yarn_relation.register_provided_hosts,
-                jujubigdata.utils.manage_etc_hosts,
-                yarn_relation.install_ssh_keys,
-                yarn.configure_nodemanager,
-                yarn.start_nodemanager,
-                charmframework.helpers.open_ports(dist_config.exposed_ports('compute-slave-yarn')),
-            ],
-            'cleanup': [
-                charmframework.helpers.close_ports(dist_config.exposed_ports('compute-slave-yarn')),
                 yarn.stop_nodemanager,
             ],
         },

=== modified file 'hooks/setup.py'
--- hooks/setup.py	2015-03-03 22:38:32 +0000
+++ hooks/setup.py	2015-06-25 13:44:06 +0000
@@ -12,24 +12,22 @@
 import subprocess
 from glob import glob
 
+
 def pre_install():
     """
     Do any setup required before the install hook.
     """
     install_pip()
-    install_jujuresources()
+    install_bundled_resources()
 
 
 def install_pip():
     subprocess.check_call(['apt-get', 'install', '-yq', 'python-pip', 'bzr'])
 
 
-def install_jujuresources():
-    """
-    Install the bundled jujuresources library, if not present.
-    """
-    try:
-        import jujuresources  # noqa
-    except ImportError:
-        jr_archive = glob('resources/jujuresources-*.tar.gz')[0]
-        subprocess.check_call(['pip', 'install', jr_archive])
+def install_bundled_resources():
+    """
+    Install the bundled resources libraries.
+    """
+    archives = glob('resources/python/*')
+    subprocess.check_call(['pip', 'install'] + archives)

=== modified file 'resources.yaml'
--- resources.yaml	2015-06-19 17:24:59 +0000
+++ resources.yaml	2015-06-25 13:44:06 +0000
@@ -3,12 +3,8 @@
 resources:
   pathlib:
     pypi: path.py>=7.0
-  pyaml:
-    pypi: pyaml
-  six:
-    pypi: six
   jujubigdata:
-    pypi: jujubigdata>=2.0.0,<3.0.0
+    pypi: jujubigdata>=2.0.2,<3.0.0
   charmhelpers:
     pypi: http://bazaar.launchpad.net/~bigdata-dev/bigdata-data/trunk/download/kevin.monroe%40canonical.com-20150610215108-gq4ud2rriy1sp1h6/charmhelpers0.3.1.ta-20150610215050-vxrqhizdsrqwnmcm-1/charmhelpers-0.3.1.tar.gz
     hash: 061fe7204289b96fab5d3ca02883040ea3026526ebf0ad38e21457527a18d139

=== removed file 'resources/jujuresources-0.2.5.tar.gz'
Binary files resources/jujuresources-0.2.5.tar.gz	2015-03-03 22:38:32 +0000 and resources/jujuresources-0.2.5.tar.gz	1970-01-01 00:00:00 +0000 differ
=== added directory 'resources/python'
=== added file 'resources/python/PyYAML-3.11.tar.gz'
Binary files resources/python/PyYAML-3.11.tar.gz	1970-01-01 00:00:00 +0000 and resources/python/PyYAML-3.11.tar.gz	2015-06-25 13:44:06 +0000 differ
=== added file 'resources/python/charmhelpers-0.3.1.tar.gz'
Binary files resources/python/charmhelpers-0.3.1.tar.gz	1970-01-01 00:00:00 +0000 and resources/python/charmhelpers-0.3.1.tar.gz	2015-06-25 13:44:06 +0000 differ
=== added file 'resources/python/jujuresources-0.2.8.tar.gz'
Binary files resources/python/jujuresources-0.2.8.tar.gz	1970-01-01 00:00:00 +0000 and resources/python/jujuresources-0.2.8.tar.gz	2015-06-25 13:44:06 +0000 differ
=== added file 'resources/python/pyaml-15.5.7.tar.gz'
Binary files resources/python/pyaml-15.5.7.tar.gz	1970-01-01 00:00:00 +0000 and resources/python/pyaml-15.5.7.tar.gz	2015-06-25 13:44:06 +0000 differ
=== added file 'resources/python/six-1.9.0-py2.py3-none-any.whl'
Binary files resources/python/six-1.9.0-py2.py3-none-any.whl	1970-01-01 00:00:00 +0000 and resources/python/six-1.9.0-py2.py3-none-any.whl	2015-06-25 13:44:06 +0000 differ