Merge lp:~bigdata-dev/charms/trusty/hdp-hadoop/trunk into lp:charms/trusty/hdp-hadoop

[Charles Butler]

a cleanup and reimplementation of amir's branch:

adding following features:
1) Openstack enablement for multi-node compute nodes
2) Enable external big data applications query hadoop compute-nodes ip&hostname
for task discovery.
3) clean-up
4) update amulet testcase

[lp:~bigdata-dev/charms/trusty/hdp-hadoop/trunk]

25. By Charles Butler on 2014-11-21

Re-add env to shebang

[David Britton (dpb)]

[0] Please convert all non-standard function comment headers to python docstrings.

[David Britton (dpb)]

A couple diff comments, I'll update more as I find them. I'm submitting as I go since this was time-sensitive.

[David Britton (dpb)]

Another round of diff comments. added

[David Britton (dpb)]

Please move the following code:

#################################### Global Data ################################

hdp_hellper_script = os.path.join(os.path.sep, os.environ['CHARM_DIR'],'files', 'scripts', "HDPHelperScripts.sh")
if not os.path.isfile(hdp_hellper_script):
    log ("Erro ==> {} not found".format(hdp_hellper_script), "ERROR")

createDadoopConfDir = os.path.join(os.path.sep, os.environ['CHARM_DIR'],'files', 'scripts', "createHadoopConfDir.sh")
hdpScript = "hdp_scripts"
hdpScriptPath = os.path.join(os.path.sep,home, hdpScript,'scripts')
usersAndGroupsScript = os.path.join(os.path.sep, hdpScriptPath, "usersAndGroups.sh")
directoriesScript = os.path.join(os.path.sep, hdpScriptPath, "directories.sh")
tarfilename="hdp_manual_install_rpm_helper_files-2.1.1.385.tar.gz"
tarfilenamePre="hdp_manual_install_rpm_helper_files-2.1.1.385"
bashrc = os.path.join(os.path.sep, home, '.bashrc')
hadoopMemoryOptimizationData = os.path.join(os.path.sep, hdpScriptPath, "hdpMemOpt.txt");
hosts_path = os.path.join(os.path.sep, 'etc', 'hosts')
resourceManagerReady = False
##########################################################################################

# Start the Hook Logic Block
hooks = Hooks()

to right above:

if __name__ == "__main__":
    hooks.execute(sys.argv)

So all the non method definition code will be together?

[David Britton (dpb)]

Last round, marking needs fixing

review: Needs Fixing

[amir sanjar (asanjar)]

David, I understand your concerns regarding abbreviation (i.e. dn, nd,..) but these selections were made due to same use of abbreviation, internal and externally, by hortonworks itself. For example Hortonworks uses:
 "/grid/hadoop/hdfs/nn"
    description: "Space separated list of directories where NameNode will store file system image."
or
    default: "/grid/hadoop/hdfs/dn"
    description: "Space separated list of directories where DataNode will store file system image."
or
    default: "/grid/hadoop/hdfs/snn"

Hortonworks users/developers would see these as a problem but a common way of abbreviation.

[amir sanjar (asanjar)]

BTW david, anyone that who is going to touch this charm has to understand big data and how hortonworks stack installs/configure. This charm is not for common developers. it is targeted for Big Data developers.

=== removed file 'hadoop_cluster.yaml'
--- hadoop_cluster.yaml	2014-08-25 16:33:37 +0000
+++ hadoop_cluster.yaml	1970-01-01 00:00:00 +0000
@@ -1,20 +0,0 @@
-hdp-hadoop-cluster: 
-  services: 
-    "compute-node": 
-      charm: "cs:~asanjar/trusty/hdp-hadoop"
-      num_units: 1
-      annotations: 
-        "gui-x": "525.7681500094167"
-        "gui-y": "608.4847070634866"
-    "yarn-hdfs-master": 
-      charm: "cs:~asanjar/trusty/hdp-hadoop"
-      num_units: 1
-      annotations: 
-        "gui-x": "532"
-        "gui-y": "236.51529293651342"
-  relations: 
-    - - "yarn-hdfs-master:namenode"
-      - "compute-node:datanode"
-    - - "yarn-hdfs-master:resourcemanager"
-      - "compute-node:nodemanager"
-  series: trusty

=== added symlink 'hooks/compute-nodes-relation-changed'
=== target is u'hdp-hadoop-common.py'
=== added symlink 'hooks/compute-nodes-relation-joined'
=== target is u'hdp-hadoop-common.py'
=== added symlink 'hooks/compute-relation-joined'
=== target is u'hdp-hadoop-common.py'
=== added symlink 'hooks/hadoop-nodes-relation-joined'
=== target is u'hdp-hadoop-common.py'
=== modified file 'hooks/hdp-hadoop-common.py'
--- hooks/hdp-hadoop-common.py	2014-11-03 17:56:47 +0000
+++ hooks/hdp-hadoop-common.py	2014-11-21 15:37:12 +0000
@@ -6,6 +6,7 @@
 import shlex
 import shutil
 import inspect
+import time
 
 from hdputils import install_base_pkg, updateHDPDirectoryScript, config_all_nodes, \
                     setHadoopEnvVar, home, hdpScript, configureJAVA, config_all_nodes
@@ -13,7 +14,7 @@
 
 from charmhelpers.lib.utils import config_get, get_unit_hostname
 from shutil import rmtree, copyfile
-from charmhelpers.core.hookenv import log, Hooks, relation_get, relation_set, unit_get, open_port
+from charmhelpers.core.hookenv import log, Hooks, relation_get, relation_set, unit_get, open_port, local_unit, related_units
 from charmhelpers.core.host import service_start, service_stop, add_user_to_group
 from time import sleep
 
@@ -88,7 +89,6 @@
     subprocess.call(createDadoopConfDir)
 
 
-
 def uninstall_base_pkg():
     log("==> {}".format(inspect.stack()[0][3]),"INFO")
     packages = ['ntp',
@@ -144,9 +144,11 @@
 def callHDFS_fs(command):
     cmd = shlex.split("su hdfs -c '/usr/lib/hadoop/bin/hadoop fs {}'".format(command))
     subprocess.call(cmd)
-
-def startJobHistory():
-    log("==> startJobHistory")
+###########################
+# Start Job History server
+###########################
+def start_jobhistory():
+    log("==> start_jh")
     path = os.path.join(os.path.sep, 'usr', 'lib', 'hadoop-yarn', 'bin', 'container-executor')
     chownRecursive(path, 'root', 'hadoop')
     os.chmod(path, 650)
@@ -164,15 +166,28 @@
                       format(os.environ['MAPRED_USER'], hadoopConfDir))
     subprocess.call(cmd)
 
-def stopJobHistory():
+###########################
+# Stop Job History server
+###########################
+def stop_jobhistory():
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     os.environ["HADOOP_LIBEXEC_DIR"]="/usr/lib/hadoop/libexec"
-    cmd = shlex.split("su {} -c '/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config {} stop resourcemanager'".\
+    cmd = shlex.split("su {} -c '/usr/lib/hadoop-mapreduce/sbin/mr-jobhistory-daemon.sh --config {} stop historyserver'".\
                       format(os.environ['MAPRED_USER'], hadoopConfDir))
     subprocess.call(cmd)
 
-# candidate for BD charm helper
+############################################################################################
+# restart Job History server - Must be done everytime yarn-site.xml and
+# mpared-site.xml are modfied
+############################################################################################
+def restart_jobhistory():
+    if is_jvm_service_active("JobHistoryServer"):
+       stop_jobhistory()
+       start_jobhistory()
 
+##########################
+# Start NameNode server
+###########################
 def start_namenode(hdfsUser):
     log("==> start namenode for user={}".format(hdfsUser), "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
@@ -180,28 +195,40 @@
                       format(hdfsUser, hadoopConfDir))
     subprocess.check_call(cmd)
 
+###########################
+# Stop Name Node server
+###########################
 def stop_namenode(hdfsUser):
-    log("==> start namenode for user={}".format(hdfsUser), "INFO")
+    log("==> stop namenode for user={}".format(hdfsUser), "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     cmd = shlex.split("su {} -c '/usr/lib/hadoop/sbin/hadoop-daemon.sh --config {} stop namenode'".\
                       format(hdfsUser, hadoopConfDir))
     subprocess.call(cmd)
-# candidate for BD charm helper
 
+###########################
+# Start Data Node
+###########################
 def start_datanode(hdfsUser):
-    log("==> start namenode for user={}".format(hdfsUser), "INFO")
+    log("==> start datanode for user={}".format(hdfsUser), "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     cmd = shlex.split("su {} -c '/usr/lib/hadoop/sbin/hadoop-daemon.sh --config {} start datanode'".\
                       format(hdfsUser, hadoopConfDir))
     subprocess.check_call(cmd)
 
+###########################
+# Stop Data Node
+###########################
 def stop_datanode(hdfsUser):
-    log("==> start namenode for user={}".format(hdfsUser), "INFO")
+    log("==> stop datanode for user={}".format(hdfsUser), "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     cmd = shlex.split("su {} -c '/usr/lib/hadoop/sbin/hadoop-daemon.sh --config {} stop datanode'".\
                       format(hdfsUser, hadoopConfDir))
     subprocess.call(cmd)
 # candidate for BD charm helper
+
+#############################################
+# Configure YARN-SITE.XML and MAPRED-SITE.XML
+#############################################
 def configureYarn(RMhostname):
     yarnConfPath = os.path.join(os.path.sep, os.environ['HADOOP_CONF_DIR'],"yarn-site.xml")
     mapConfDir = os.path.join(os.path.sep, os.environ['HADOOP_CONF_DIR'],"mapred-site.xml")
@@ -220,8 +247,10 @@
     setHadoopConfigXML(mapConfDir, "mapreduce.jobhistory.webapp.address", RMhostname+":19888")
     setHadoopConfigXML(mapConfDir, "mapreduce.jobhistory.address",  RMhostname+":10020")
 
-# candidate for BD charm helper
-def start_RM(yarnUser):
+################################
+# Start Resource Manager server
+################################
+def start_resourcemanager(yarnUser):
     log("==> start resourcemanager", "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     os.environ["HADOOP_LIBEXEC_DIR"]="/usr/lib/hadoop/libexec"
@@ -229,23 +258,32 @@
                       format(yarnUser, hadoopConfDir))
     subprocess.call(cmd)
 
-def stop_RM(yarnUser):
+################################
+# Stop Resource Manager server
+################################
+def stop_resourcemanager(yarnUser):
     log("==> stop resourcemanager", "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     os.environ["HADOOP_LIBEXEC_DIR"]="/usr/lib/hadoop/libexec"
     cmd = shlex.split("su {} -c '/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config {} stop resourcemanager'".\
                       format(yarnUser, hadoopConfDir))
     subprocess.check_call(cmd)
-# candidate for BD charm helper
-def start_NM(yarnUser):
+
+################################################
+# Start Node Manager daemon on each compute node
+################################################
+def start_nodemanager(yarnUser):
     log("==> start nodemanager", "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     os.environ["HADOOP_LIBEXEC_DIR"]="/usr/lib/hadoop/libexec"
     cmd = shlex.split("su {} -c '/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config {} start nodemanager'".\
                       format(yarnUser, hadoopConfDir))
-    subprocess.call(cmd)
+    subprocess.check_call(cmd)
 
-def stop_NM(yarnUser):
+################################################
+# Stop Node Manager daemon on each compute node
+################################################
+def stop_nodemanager(yarnUser):
     log("==> stop nodemanager", "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     os.environ["HADOOP_LIBEXEC_DIR"]="/usr/lib/hadoop/libexec"
@@ -253,29 +291,52 @@
                       format(yarnUser, hadoopConfDir))
     subprocess.call(cmd)
 
+################################################
+# Stop all running hadoop services
+# NOTE: Order is important - DO NOT CHANGE
+################################################
 def stop_hadoop_services():
     if is_jvm_service_active("ResourceManager"):
-        stop_RM(os.environ['YARN_USER'])
+        stop_resourcemanager(os.environ['YARN_USER'])
     if is_jvm_service_active("NodeManager"):
-        stop_NM(os.environ['YARN_USER'])
+        stop_nodemanager(os.environ['YARN_USER'])
     if is_jvm_service_active("NameNode"):
         stop_namenode(os.environ['HDFS_USER'])
     if is_jvm_service_active("DataNode"):
         stop_datanode(os.environ['HDFS_USER'])
+    if is_jvm_service_active("JobHistoryServer"):
+        stop_jobhistory()
 
+################################################
+# restart all running hadoop services
+# NOTE: Order is important - DO NOT CHENAGE
+################################################
 def restart_hadoop_services():
-    if is_jvm_service_active("ResourceManager"):
-        stop_RM(os.environ['YARN_USER'])
-        start_RM(os.environ['YARN_USER'])
-    if is_jvm_service_active("NodeManager"):
-        stop_NM(os.environ['YARN_USER'])
-        start_NM(os.environ['YARN_USER'])
     if is_jvm_service_active("NameNode"):
         stop_namenode(os.environ['HDFS_USER'])
         start_namenode(os.environ['HDFS_USER'])
+    if is_jvm_service_active("ResourceManager"):
+        stop_resourcemanager(os.environ['YARN_USER'])
+        start_resourcemanager(os.environ['YARN_USER'])
+    if is_jvm_service_active("NodeManager"):
+        stop_nodemanager(os.environ['YARN_USER'])
+        start_nodemanager(os.environ['YARN_USER'])
     if is_jvm_service_active("DataNode"):
         stop_datanode(os.environ['HDFS_USER'])
         start_datanode(os.environ['HDFS_USER'])
+    restart_jobhistory()
+
+def wait_for_hadoop_service(service):
+    ticks = time.time()
+    while True:
+        if (time.time() - ticks) > 200:
+            log("Error ==> Reached timeout value for hadoop service {}..".format(service), "ERROR")
+            sys.exit(1)
+        if not is_jvm_service_active(service):
+            time.sleep(2)
+            log("Waiting..  ==> {} not ready..".format(service),"INFO")
+            continue
+        break
 
 def configureHDFS(hostname):
     hdfsConfPath = os.path.join(os.path.sep, os.environ['HADOOP_CONF_DIR'],'hdfs-site.xml')
@@ -301,7 +362,6 @@
 bashrc = os.path.join(os.path.sep, home, '.bashrc')
 hadoopMemoryOptimizationData = os.path.join(os.path.sep, hdpScriptPath, "hdpMemOpt.txt");
 hosts_path = os.path.join(os.path.sep, 'etc', 'hosts')
-nameNodeReady = False
 resourceManagerReady = False
 ##########################################################################################
 
@@ -332,6 +392,7 @@
     config_hadoop_nodes()
     fileSetKV(hosts_path, unit_get('private-address')+' ', get_unit_hostname())
 
+
 @hooks.hook('resourcemanager-relation-joined')
 def resourcemanager_relation_joined():
     log ("==> resourcemanager-relation-joined","INFO")
@@ -339,12 +400,14 @@
         relation_set(resourceManagerReady=True)
         relation_set(resourceManager_hostname=get_unit_hostname())
         return
+    if not is_jvm_service_active("NameNode"):
+        sys.exit(0)
     setHadoopEnvVar()
     relation_set(resourceManager_ip=unit_get('private-address'))
     relation_set(resourceManager_hostname=get_unit_hostname())
     configureYarn(unit_get('private-address'))
-    start_RM(os.environ["YARN_USER"])
-    startJobHistory()
+    start_resourcemanager(os.environ["YARN_USER"])
+    start_jobhistory()
     open_port(8025)
     open_port(8030)
     open_port(8050)
@@ -365,8 +428,10 @@
     rm_ip = relation_get('private-address')
     configureYarn(rm_ip)
     fileSetKV(hosts_path, rm_ip+' ', relation_get('resourceManager_hostname'))
-    start_NM(os.environ["YARN_USER"])
-
+    # nodemanager requires data node daemon
+    if not is_jvm_service_active("DataNode"):
+        start_datanode(os.environ['HDFS_USER'])
+    start_nodemanager(os.environ["YARN_USER"])
     open_port(8025)
     open_port(8030)
     open_port(8050)
@@ -376,6 +441,7 @@
     open_port(10020)
     relation_set(nodemanager_hostname=get_unit_hostname())
 
+
 @hooks.hook('resourcemanager-relation-broken')
 def resourcemanager_relation_broken():
     log ("Configuring resourcemanager - broken phase","INFO")
@@ -404,13 +470,16 @@
     configureHDFS(unit_get('private-address'))
     format_namenode(os.environ["HDFS_USER"])
     start_namenode(os.environ["HDFS_USER"])
-
+    start_jobhistory()
     open_port(8020)
     open_port(8010)
     open_port(50070)
     open_port(50075)
     open_port(8480)
     open_port(50470)
+    if not is_jvm_service_active("NameNode"):
+        log("error ==> NameNode failed to start")
+        sys.exit(1)
     sleep(5)
     relation_set(nameNodeReady=True)
 
@@ -423,13 +492,15 @@
     if not nameNodeReady:
         sys.exit(0)
     setHadoopEnvVar()
-    nodeType="namenode"
-    namenode_hostname = relation_get("namenode_hostname")
-    namenode_ip = relation_get('private-address')
-    fileSetKV(hosts_path, namenode_ip+' ', namenode_hostname)
-    configureHDFS(namenode_hostname)
+    nn_hostname = relation_get("namenode_hostname")
+    nn_ip = relation_get('private-address')
+    fileSetKV(hosts_path, nn_ip+' ', nn_hostname)
+    configureHDFS(nn_ip)
     setDirPermission(os.environ['DFS_DATA_DIR'], os.environ['HDFS_USER'], os.environ['HADOOP_GROUP'], 0750)
     start_datanode(os.environ["HDFS_USER"])
+    if not is_jvm_service_active("DataNode"):
+        log("error ==> DataNode failed to start")
+        sys.exit(1)
     open_port(8010)
     open_port(8480)
     open_port(50010)
@@ -448,7 +519,7 @@
 
 @hooks.hook('namenode-relation-changed')
 def namenode_relation_changed():
-    dn_host = relation_get('dn-hostname')
+    dn_host = relation_get('dn_hostname')
     if dn_host == None:
         sys.exit(0)
     log("Configuring namenode - changed phase", "INFO")
@@ -464,6 +535,30 @@
     setHadoopEnvVar()
     stop_hadoop_services()
 
[e-mail]('compute-nodes-relation-changed')
+def compute_nodes_relation_changed():
+    log("==> Configuring compute-nodes, local ID = {} - Changed phase".format(local_unit()),"INFO")
+    nodeList = related_units()
+    for n in nodeList:
+        dn_ip = relation_get('private-address', unit=n)
+        hostname = relation_get('hostname', unit=n)
+        log("==> Configuring compute-nodes {}={}".format(dn_ip, hostname),"INFO")
+        if hostname != None:
+            fileSetKV(hosts_path, dn_ip+' ', hostname)
+        else:
+            sys.exit(0)
+
[e-mail]('compute-nodes-relation-joined')
+def compute_nodes_relation_joined():
+    log("==> compute_nodes_relation_joined {}".format(get_unit_hostname()),"INFO")
+    relation_set(hostname=get_unit_hostname())
+
+
[e-mail]('hadoop-nodes-relation-joined')
+def hadoop_nodes_relation_joined():
+    log("==> hadoop_nodes_relation_joined {}".format(get_unit_hostname()),"INFO")
+    relation_set(hostname=get_unit_hostname())
+
 
 
 

=== modified file 'hooks/hdputils.py'
--- hooks/hdputils.py	2014-08-11 03:35:08 +0000
+++ hooks/hdputils.py	2014-11-21 15:37:12 +0000
@@ -22,6 +22,8 @@
     subprocess.call(cmd)   
     apt_update()
     apt_install(packages)
+    if not os.path.isdir(os.path.join(os.path.sep,'usr','lib', 'hadoop')):
+        log("Error, apt-get install Hadoop failed", "ERROR")
     os.chdir(home);
     wgetPkg("http://public-repo-1.hortonworks.com/HDP/tools/2.1.1.0/hdp_manual_install_rpm_helper_files-2.1.1.385.tar.gz","")
     if tarfile.is_tarfile(tarfilename):

=== modified file 'metadata.yaml'
--- metadata.yaml	2014-07-23 12:16:51 +0000
+++ metadata.yaml	2014-11-21 15:37:12 +0000
@@ -4,22 +4,19 @@
 description: |
   Hadoop is a software platform that lets one easily write and
   run applications that process vast amounts of data.
-categories: ["applications"]
+tags: ["applications"]
 provides:
   namenode:
     interface: dfs
   resourcemanager:
     interface: mapred
-  ganglia:
-    interface: monitor
+  hadoop-nodes:
+    interface: mapred
 requires:
   datanode:
     interface: dfs
-  secondarynamenode:
-    interface: dfs
   nodemanager:
     interface: mapred
-  mapred-namenode:
-    interface: dfs
-  elasticsearch:
-    interface: elasticsearch
+peers:
+  compute-nodes:
+    interface: mapred

=== modified file 'tests/01-hadoop-cluster-deployment-1.py'
--- tests/01-hadoop-cluster-deployment-1.py	2014-09-17 15:27:28 +0000
+++ tests/01-hadoop-cluster-deployment-1.py	2014-11-21 15:37:12 +0000
@@ -6,12 +6,12 @@
 class TestDeployment(object):
     def __init__(self):
         self.d =  amulet.Deployment(series='trusty')
-        bpath = os.path.join(os.path.dirname( __file__), "../hadoop_cluster.yaml")
+        bpath = os.path.join(os.path.dirname( __file__), "hadoop_cluster.yaml")
         f = open(bpath)
         bun = f.read()
         self.d.load(yaml.safe_load(bun))
         try:
-            self.d.setup(timeout=9000)
+            self.d.setup(timeout=900000)
             self.d.sentry.wait()
         except amulet.helpers.TimeoutError:
             amulet.raise_status(amulet.SKIP, msg="Environment wasn't stood up in time")

=== added file 'tests/hadoop_cluster.yaml'
--- tests/hadoop_cluster.yaml	1970-01-01 00:00:00 +0000
+++ tests/hadoop_cluster.yaml	2014-11-21 15:37:12 +0000
@@ -0,0 +1,20 @@
+hdp-hadoop-cluster: 
+  services: 
+    "compute-node": 
+      charm: "cs:~asanjar/trusty/hdp-hadoop"
+      num_units: 4
+      annotations: 
+        "gui-x": "525.7681500094167"
+        "gui-y": "608.4847070634866"
+    "yarn-hdfs-master": 
+      charm: "cs:~asanjar/trusty/hdp-hadoop"
+      num_units: 1
+      annotations: 
+        "gui-x": "532"
+        "gui-y": "236.51529293651342"
+  relations: 
+    - - "yarn-hdfs-master:namenode"
+      - "compute-node:datanode"
+    - - "yarn-hdfs-master:resourcemanager"
+      - "compute-node:nodemanager"
+  series: trusty