Merge lp:~bigdata-dev/charms/trusty/apache-hadoop-yarn-master/trunk into lp:charms/trusty/apache-hadoop-yarn-master

[amir sanjar]

adding YARN start, stop, and restart Juju actions to the charm

[lp:~bigdata-dev/charms/trusty/apache-hadoop-yarn-master/trunk]

89. By Cory Johns on 2015-06-25

Added extended status support

90. By Cory Johns on 2015-06-25

Nested python resources to avoid attempting to pip install future non-python resources

91. By Cory Johns on 2015-06-25

Register clients as well as slaves to ensure all nodes can communicate

92. By Kevin W Monroe on 2015-06-25

update to latest bigdata-data java-installer.sh

93. By Kevin W Monroe on 2015-06-26

updated DEV-README to correct historyserver-port description; pass jobhistory port instead of jh_webapp_http for the historyserver-port since related units want to do ipc, not browse. duh.

[Kevin W Monroe (kwmonroe)]

actions + status + bundled resources + /etc/hosts refactor + java fix + jobhistory fix = LGTM, +1.

Merged into to -dev and pushed to -charmers.

review: Approve

=== modified file 'DEV-README.md'
--- DEV-README.md	2015-06-18 17:13:00 +0000
+++ DEV-README.md	2015-06-26 22:25:03 +0000
@@ -18,7 +18,7 @@
   * `private-address`: Address of this unit, to provide the ResourceManager
   * `has_slave`: Flag indicating if YARN has at least one NodeManager
   * `port`: Port where the ResourceManager is listening for YARN operations (IPC)
-  * `historyserver-port`: Port for the JobHistory web interface
+  * `historyserver-port`: JobHistory port (IPC)
 
 * Received from the plugin:
 
@@ -68,7 +68,7 @@
   * `private-address`: Address of the YARN master unit, to provide the ResourceManager
   * `has_slave`: Flag indicating if YARN has at least one NodeManager
   * `port`: Port where the ResourceManager is listening for YARN operations (IPC)
-  * `historyserver-port`: Port for the JobHistory web interface
+  * `historyserver-port`: JobHistory port (IPC)
 
 * Received from compute-slave:
 

=== added directory 'actions'
=== added file 'actions.yaml'
--- actions.yaml	1970-01-01 00:00:00 +0000
+++ actions.yaml	2015-06-26 22:25:03 +0000
@@ -0,0 +1,6 @@
+start-yarn:
+    description: All of the YARN processes can be started with this Juju action.
+stop-yarn:
+    description: All of the YARN processes can be stopped with this Juju action.
+restart-yarn:
+    description: All of the YARN processes can be restarted with this Juju action.

=== added file 'actions/restart-yarn'
--- actions/restart-yarn	1970-01-01 00:00:00 +0000
+++ actions/restart-yarn	2015-06-26 22:25:03 +0000
@@ -0,0 +1,5 @@
+#!/bin/bash
+. /etc/environment
+cd ${HADOOP_HOME}/sbin
+su yarn -c "./stop-yarn.sh"
+su yarn -c "./start-yarn.sh"

=== added file 'actions/start-yarn'
--- actions/start-yarn	1970-01-01 00:00:00 +0000
+++ actions/start-yarn	2015-06-26 22:25:03 +0000
@@ -0,0 +1,4 @@
+#!/bin/bash
+. /etc/environment
+cd ${HADOOP_HOME}/sbin
+su yarn -c "./start-yarn.sh"

=== added file 'actions/stop-yarn'
--- actions/stop-yarn	1970-01-01 00:00:00 +0000
+++ actions/stop-yarn	2015-06-26 22:25:03 +0000
@@ -0,0 +1,4 @@
+#!/bin/bash
+. /etc/environment
+cd ${HADOOP_HOME}/sbin
+su yarn -c "./stop-yarn.sh"

=== modified file 'hooks/callbacks.py'
--- hooks/callbacks.py	2015-05-12 21:51:52 +0000
+++ hooks/callbacks.py	2015-06-26 22:25:03 +0000
@@ -15,3 +15,34 @@
 Add any additional tasks / setup here.  If a callback is used by mutliple
 charms, consider refactoring it up to the jujubigdata library.
 """
+
+from charmhelpers.core import hookenv
+from charmhelpers.core import unitdata
+from jujubigdata.relations import NameNode, NodeManager
+
+
+def update_blocked_status():
+    if unitdata.kv().get('charm.active', False):
+        return
+    if NameNode().connected_units():
+        hookenv.status_set('waiting', 'Waiting for HDFS master to provide NameNode'),
+    else:
+        hookenv.status_set('blocked', 'Waiting for relation to HDFS master'),
+
+
+def update_working_status():
+    if unitdata.kv().get('charm.active', False):
+        hookenv.status_set('maintenance', 'Updating configuration')
+        return
+    hookenv.status_set('maintenance', 'Setting up Yarn master')
+
+
+def update_active_status():
+    nodemanager = NodeManager()
+    if nodemanager.is_ready():
+        hookenv.status_set('active', 'Ready (%s NodeManagers)' % len(nodemanager.filtered_data()))
+        unitdata.kv().set('charm.active', True)
+    elif nodemanager.connected_units():
+        hookenv.status_set('waiting', 'Waiting for compute slaves to provide NodeManagers')
+    else:
+        hookenv.status_set('blocked', 'Waiting for relation to compute slaves')

=== modified file 'hooks/common.py'
--- hooks/common.py	2015-05-13 16:48:39 +0000
+++ hooks/common.py	2015-06-26 22:25:03 +0000
@@ -15,29 +15,39 @@
 """
 
 import jujuresources
+from charmhelpers.core import hookenv
+from charmhelpers.core import unitdata
+from charmhelpers.core import charmframework
 
 
 def bootstrap_resources():
     """
     Attempt to load and install resources defined in resources.yaml
     """
+    if unitdata.kv().get('charm.bootstrapped', False):
+        return True
+    hookenv.status_set('maintenance', 'Installing base resources')
     mirror_url = jujuresources.config_get('resources_mirror')
     if not jujuresources.fetch(mirror_url=mirror_url):
-        jujuresources.juju_log('Required resources unavailable; changing resources_mirror option is required', 'ERROR')
+        missing = jujuresources.invalid()
+        hookenv.status_set('blocked', 'Unable to fetch required resource%s: %s' % (
+            's' if len(missing) > 1 else '',
+            ', '.join(missing),
+        ))
         return False
-    jujuresources.install(['pathlib', 'pyaml', 'six', 'charmhelpers', 'jujubigdata'])
+    jujuresources.install(['pathlib', 'jujubigdata'])
+    unitdata.kv().set('charm.bootstrapped', True)
     return True
 
 
 def manage():
     if not bootstrap_resources():
-        # defer until resources are available, since charmhelpers, and thus
-        # the framework, are required (will require manual intervention)
+        # defer until resources are available, since jujubigdata, and thus the
+        # classes needed for the requires blocks, (will be) managed by jujuresources
         return
 
-    from charmhelpers.core import charmframework
     import jujubigdata
-    import callbacks  # noqa  (ignore when linting)
+    import callbacks
 
     # list of keys required to be in the dist.yaml
     yarn_reqs = ['vendor', 'hadoop_version', 'packages', 'groups',
@@ -48,9 +58,10 @@
     yarn = jujubigdata.handlers.YARN(hadoop)
     hdfs = jujubigdata.handlers.HDFS(hadoop)
     port = dist_config.port('resourcemanager')
-    hs_port = dist_config.port('jh_webapp_http')
+    hs_port = dist_config.port('jobhistory')
     nodemanagers = jujubigdata.relations.ResourceManagerMaster(spec=hadoop.spec, port=port, historyserver_port=hs_port)
     clients = jujubigdata.relations.ResourceManager(spec=hadoop.spec, port=port, historyserver_port=hs_port)
+    namenode = jujubigdata.relations.NameNode(spec=hadoop.client_spec)
     manager = charmframework.Manager([
         {
             'name': 'hadoop-base',
@@ -59,6 +70,7 @@
             ],
             'callbacks': [
                 hadoop.install,
+                callbacks.update_blocked_status,
             ],
         },
         {
@@ -69,12 +81,19 @@
             ],
             'requires': [
                 hadoop.is_installed,
-                jujubigdata.relations.NameNode(spec=hadoop.client_spec),
+                namenode,
                 jujubigdata.relations.NodeManager(optional=True),
             ],
             'callbacks': [
-                nodemanagers.register_host_ips,
-                jujubigdata.utils.update_etc_hosts_from_kv,
+                # These callbacks will be executed once the Hadoop base packages
+                # are installed and HDFS is available.  New items can be added
+                # to the end of this list and to hooks/callbacks.py to extend
+                # the functionality of this charm.
+                callbacks.update_working_status,
+                nodemanagers.register_connected_hosts,
+                clients.register_connected_hosts,
+                namenode.register_provided_hosts,
+                jujubigdata.utils.manage_etc_hosts,
                 hdfs.configure_client,
                 yarn.configure_resourcemanager,
                 yarn.configure_jobhistory,
@@ -82,6 +101,7 @@
                 yarn.start_resourcemanager,
                 yarn.start_jobhistory,
                 charmframework.helpers.open_ports(dist_config.exposed_ports('yarn-master')),
+                callbacks.update_active_status,
             ],
             'cleanup': [
                 charmframework.helpers.close_ports(dist_config.exposed_ports('yarn-master')),

=== modified file 'hooks/config-changed'
--- hooks/config-changed	2015-02-09 18:17:08 +0000
+++ hooks/config-changed	2015-06-26 22:25:03 +0000
@@ -11,5 +11,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful.  As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py.  New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
 import common
 common.manage()

=== modified file 'hooks/install'
--- hooks/install	2015-02-09 18:17:08 +0000
+++ hooks/install	2015-06-26 22:25:03 +0000
@@ -13,5 +13,16 @@
 import setup
 setup.pre_install()
 
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful.  As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py.  New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
 import common
 common.manage()

=== modified file 'hooks/namenode-relation-changed'
--- hooks/namenode-relation-changed	2015-05-12 21:51:52 +0000
+++ hooks/namenode-relation-changed	2015-06-26 22:25:03 +0000
@@ -11,5 +11,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful.  As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py.  New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
 import common
 common.manage()

=== modified file 'hooks/nodemanager-relation-changed'
--- hooks/nodemanager-relation-changed	2015-05-12 21:51:52 +0000
+++ hooks/nodemanager-relation-changed	2015-06-26 22:25:03 +0000
@@ -11,5 +11,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful.  As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py.  New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
 import common
 common.manage()

=== modified file 'hooks/resourcemanager-relation-changed'
--- hooks/resourcemanager-relation-changed	2015-05-19 16:28:14 +0000
+++ hooks/resourcemanager-relation-changed	2015-06-26 22:25:03 +0000
@@ -11,5 +11,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful.  As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py.  New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
 import common
 common.manage()

=== modified file 'hooks/setup.py'
--- hooks/setup.py	2015-03-03 21:31:05 +0000
+++ hooks/setup.py	2015-06-26 22:25:03 +0000
@@ -12,24 +12,22 @@
 import subprocess
 from glob import glob
 
+
 def pre_install():
     """
     Do any setup required before the install hook.
     """
     install_pip()
-    install_jujuresources()
+    install_bundled_resources()
 
 
 def install_pip():
     subprocess.check_call(['apt-get', 'install', '-yq', 'python-pip', 'bzr'])
 
 
-def install_jujuresources():
-    """
-    Install the bundled jujuresources library, if not present.
-    """
-    try:
-        import jujuresources  # noqa
-    except ImportError:
-        jr_archive = glob('resources/jujuresources-*.tar.gz')[0]
-        subprocess.check_call(['pip', 'install', jr_archive])
+def install_bundled_resources():
+    """
+    Install the bundled resources libraries.
+    """
+    archives = glob('resources/python/*')
+    subprocess.check_call(['pip', 'install'] + archives)

=== modified file 'hooks/start'
--- hooks/start	2015-02-09 18:17:08 +0000
+++ hooks/start	2015-06-26 22:25:03 +0000
@@ -11,5 +11,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful.  As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py.  New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
 import common
 common.manage()

=== modified file 'hooks/stop'
--- hooks/stop	2015-02-09 18:17:08 +0000
+++ hooks/stop	2015-06-26 22:25:03 +0000
@@ -11,5 +11,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""
+All hooks in this charm are managed by the Charm Framework.
+The framework helps manage dependencies and preconditions to ensure that
+steps are only executed when they can be successful.  As such, no additional
+code should be added to this hook; instead, please integrate new functionality
+into the 'callbacks' list in hooks/common.py.  New callbacks can be placed
+in hooks/callbacks.py, if necessary.
+
+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
+for more information.
+"""
 import common
 common.manage()

=== modified file 'resources.yaml'
--- resources.yaml	2015-05-28 18:17:21 +0000
+++ resources.yaml	2015-06-26 22:25:03 +0000
@@ -3,23 +3,15 @@
 resources:
   pathlib:
     pypi: path.py>=7.0
-  pyaml:
-    pypi: pyaml
-  six:
-    pypi: six
   jujubigdata:
-    pypi: jujubigdata>=1.2.14,<2.0.0
-  charmhelpers:
-    pypi: http://bazaar.launchpad.net/~bigdata-dev/bigdata-data/trunk/download/kevin.monroe%40canonical.com-20150511173636-5rblzf5r2o1zcv2p/charmhelpers0.2.3.ta-20150417221203-zg62z8c220egc3ch-1/charmhelpers-0.2.3.tar.gz
-    hash: 44340a6fd6f192bcc9d390c0d9c3901d4fc190166485b107047bc1c6ba102a2f
-    hash_type: sha256
+    pypi: jujubigdata>=2.0.2,<3.0.0
   java-installer:
     # This points to a script which manages installing Java.
     # If replaced with an alternate implementation, it must output *only* two
     # lines containing the JAVA_HOME path, and the Java version, respectively,
     # on stdout.  Upon error, it must exit with a non-zero exit code.
-    url: http://bazaar.launchpad.net/~bigdata-dev/bigdata-data/trunk/download/cory.johns%40canonical.com-20150312205309-2ji1etk44gep01w1/javainstaller.sh-20150311213053-4vq7369jhlvc6qy8-1/java-installer.sh
-    hash: 130984f1dc3bc624d4245234d0fca22f529d234d0eaa1241c5e9f701319bdea9
+    url: http://bazaar.launchpad.net/~bigdata-dev/bigdata-data/trunk/download/kevin.monroe%40canonical.com-20150625222410-qfck64q46ubx5i6p/javainstaller.sh-20150311213053-4vq7369jhlvc6qy8-1/java-installer.sh
+    hash: 8fdff60270ea4be7bbef1e013e503057fe6efc2f4b5761edebc206a54f303023
     hash_type: sha256
 optional_resources:
   hadoop-aarch64:

=== removed file 'resources/jujuresources-0.2.5.tar.gz'
Binary files resources/jujuresources-0.2.5.tar.gz	2015-03-03 19:58:27 +0000 and resources/jujuresources-0.2.5.tar.gz	1970-01-01 00:00:00 +0000 differ
=== added directory 'resources/python'
=== added file 'resources/python/PyYAML-3.11.tar.gz'
Binary files resources/python/PyYAML-3.11.tar.gz	1970-01-01 00:00:00 +0000 and resources/python/PyYAML-3.11.tar.gz	2015-06-26 22:25:03 +0000 differ
=== added file 'resources/python/charmhelpers-0.3.1.tar.gz'
Binary files resources/python/charmhelpers-0.3.1.tar.gz	1970-01-01 00:00:00 +0000 and resources/python/charmhelpers-0.3.1.tar.gz	2015-06-26 22:25:03 +0000 differ
=== added file 'resources/python/jujuresources-0.2.8.tar.gz'
Binary files resources/python/jujuresources-0.2.8.tar.gz	1970-01-01 00:00:00 +0000 and resources/python/jujuresources-0.2.8.tar.gz	2015-06-26 22:25:03 +0000 differ
=== added file 'resources/python/pyaml-15.5.7.tar.gz'
Binary files resources/python/pyaml-15.5.7.tar.gz	1970-01-01 00:00:00 +0000 and resources/python/pyaml-15.5.7.tar.gz	2015-06-26 22:25:03 +0000 differ
=== added file 'resources/python/six-1.9.0-py2.py3-none-any.whl'
Binary files resources/python/six-1.9.0-py2.py3-none-any.whl	1970-01-01 00:00:00 +0000 and resources/python/six-1.9.0-py2.py3-none-any.whl	2015-06-26 22:25:03 +0000 differ