Merge lp:~bigdata-dev/charms/trusty/hdp-pig/trunk into lp:charms/trusty/hdp-pig

[Charles Butler]

Adds amir's openstack workarounds, and updates testcase

[amir sanjar (asanjar)]

review: Approve

=== added symlink 'hooks/hadoop-nodes-relation-changed'
=== target is u'hooks.py'
=== modified file 'hooks/hooks.py'
--- hooks/hooks.py	2014-08-07 16:47:38 +0000
+++ hooks/hooks.py	2014-12-09 17:14:57 +0000
@@ -3,10 +3,11 @@
 import os
 import sys
 from charmhelpers.core import hookenv, host
-from bdutils import chmodRecursive
+from bdutils import chmodRecursive, fileSetKV
 from hdputils import  install_base_pkg, config_all_nodes, home, hdpScript, setHadoopConfigXML, configureYarnClient, setHadoopEnvVar
 from shutil import copyfile, copy
-from charmhelpers.core.hookenv import log, relation_get
+from charmhelpers.core.hookenv import log, relation_get, unit_get
+from charmhelpers.lib.utils import  get_unit_hostname
 
 sys.path.insert(0, os.path.join(os.environ['CHARM_DIR'], 'lib'))
 
@@ -36,6 +37,7 @@
         
 pigHDPConfPath = os.path.join(os.path.sep,home, hdpScript, "configuration_files", "pig")
 hadoopHDPConfPath = os.path.join(os.path.sep,home, hdpScript, "configuration_files", "core_hadoop","hadoop-env.sh")
+hosts_path = os.path.join(os.path.sep, 'etc', 'hosts')
 
 hooks = hookenv.Hooks()
 log = hookenv.log
@@ -49,6 +51,9 @@
     packages =['ntp', 'openjdk-7-jdk',"pig"]
     install_base_pkg(packages)
     config_pig_node()
+    fileSetKV(hosts_path, unit_get('private-address')+' ', get_unit_hostname())
+    host.add_user_to_group("ubuntu", "hadoop")
+
 
 
 @hooks.hook('config-changed')
@@ -85,9 +90,12 @@
     if not resourceManagerReady:
         log ("PIG ==> resourcemanager not ready","INFO")
         sys.exit(0)
-    log("PIG ==> resourcemanager IP={}".format(relation_get('private-address')),"INFO")
+    rm_ip = relation_get('private-address')
+    log("PIG ==> resourcemanager IP={}".format(rm_ip),"INFO")
     setHadoopEnvVar()
-    configureYarnClient(relation_get('private-address'))
+    configureYarnClient(rm_ip)
+    rm_hostname = relation_get('resourceManager_hostname')
+    fileSetKV(hosts_path, rm_ip+' ', rm_hostname)
     
 @hooks.hook('namenode-relation-changed')
 def namenode_relation_changed():
@@ -97,10 +105,21 @@
     if not nameNodeReady:
         log("PIG ==> namenode not ready- Changed phase","INFO")
         sys.exit(0)
-    log("PIG ==> namenode_IP={}".format(relation_get('private-address')),"INFO")
+    nn_ip = relation_get('private-address')
+    log("PIG ==> namenode_IP={}".format(nn_ip),"INFO")
     setHadoopEnvVar()
-    setHadoopConfigXML("core-site.xml", "fs.defaultFS", "hdfs://"+relation_get('private-address')+":8020")
+    setHadoopConfigXML("core-site.xml", "fs.defaultFS", "hdfs://"+nn_ip+":8020")
+    nn_hostname = relation_get("namenode_hostname")
+    fileSetKV(hosts_path, nn_ip+' ', nn_hostname)
     
[e-mail]('hadoop-nodes-relation-changed')
+def hadoop_nodes_relation_changed():
+    compute_ip = relation_get('private-address')
+    hostname = relation_get('hostname')
+    log("==> Connect to hadoop compute node {}={}".format(compute_ip, hostname),"INFO")
+    if hostname != None:
+       fileSetKV(hosts_path, compute_ip+' ', hostname) 
+ 
 if __name__ == "__main__":
     # execute a hook based on the name the program is called by
     hooks.execute(sys.argv)
 
=== modified file 'metadata.yaml'
--- metadata.yaml	2014-09-10 18:24:08 +0000
+++ metadata.yaml	2014-12-09 17:14:57 +0000
@@ -18,3 +18,5 @@
     interface: mapred
   namenode:
     interface: dfs
+  hadoop-nodes:
+    interface: mapred
 
=== modified file 'tests/10-deploy-hadoop-pig'
--- tests/10-deploy-hadoop-pig	2014-09-19 15:13:12 +0000
+++ tests/10-deploy-hadoop-pig	2014-12-09 17:14:57 +0000
@@ -12,6 +12,17 @@
         f = open(os.path.join(os.path.dirname(__file__), 'hadoop-pig-cluster.yaml'))
         bun = f.read()
         self.d.load(yaml.safe_load(bun))
+#        self.d.add('yarn-hdfs-master', charm='/home/sanjar/development/charms/trusty/hdp-hadoop',constraints={'cpu-cores':4, 'mem':8192})
+#        self.d.add('compute-node', charm='/home/sanjar/development/charms/trusty/hdp-hadoop', constraints={'cpu-cores':4, 'mem':8192})
+ #       self.d.relate("yarn-hdfs-master:namenode","compute-node:datanode" ) 
+ #       self.d.relate("yarn-hdfs-master:resourcemanager","compute-node:nodemanager" ) 
+ #       self.d.add_unit("compute-node", units=1)
+ #       self.d.add('hdp-pig', charm='/home/sanjar/development/charms/trusty/hdp-pig', constraints={'cpu-cores':4, 'mem':8192})
+  #      self.d.relate("hdp-pig:namenode","yarn-hdfs-master:namenode" ) 
+  #      self.d.relate("hdp-pig:resourcemanager","yarn-hdfs-master:resourcemanager" ) 
+  #      self.d.relate("compute-node:hadoop-nodes","hdp-pig:hadoop-nodes") 
+        
+        
         try:
             self.d.setup(timeout=9000000)
             self.d.sentry.wait()
@@ -31,7 +42,7 @@
 
     def test_remote_hdfs_status(self):
        o,c= self.pig_unit.run("sudo su hdfs -c ' hdfs dfsadmin -report'")
-       if o.find('Datanodes available: 1') == -1:
+       if o.find('Datanodes available: 2') == -1:
           amulet.raise_status(amulet.FAIL, msg="Remote HDFS not available")
 
     def test_remote_yarn_status(self):
@@ -40,22 +51,25 @@
             amulet.raise_status(amulet.FAIL, msg="Remote YARN not available")
 ##        
     
-    def test_hdfs_mkdir(self):
-        o,c= self.pig_unit.run("su hdfs -c 'hdfs dfs -mkdir -p /user/hduser'")
+    def test_hdfs(self):
+        o,c= self.pig_unit.run("su hdfs -c 'hdfs dfs -mkdir -p /tmp'")
+        o,c= self.pig_unit.run("su hdfs -c 'hdfs dfs -mkdir -p /user/ubuntu'")
         if c != 0:
            amulet.raise_status(amulet.FAIL, msg=" Created a user directory on hdfs FAILED")
+        o,c= self.pig_unit.run("su hdfs -c 'hdfs dfs -chown -R ubuntu:hdfs /tmp'")
+        o,c= self.pig_unit.run("su hdfs -c 'hdfs dfs -chown -R ubuntu:hdfs /user'")
+        if c != 0:
+           amulet.raise_status(amulet.FAIL, msg=" Assign hdfs owner FAILED")
 
   #  def test_hdfs_copyfile(self):
   #      if c != 0:
   #          amulet.raise_status(amulet.FAIL, msg=" Created a user directory on hdfs FAILED")  
 
     def test_run_pig_mapreduce(self):
-        o,c= self.pig_unit.run("sudo su hdfs -c 'hdfs dfs -copyFromLocal /etc/passwd /user/hduser/passwd'")
-        o,c= self.pig_unit.run("echo \"A = load \'/user/hduser/passwd\' using PigStorage(\':\');\">/tmp/id.pig ")
+        o,c= self.pig_unit.run("sudo su hdfs -c 'hdfs dfs -copyFromLocal /etc/passwd /user/ubuntu/passwd'")
+        o,c= self.pig_unit.run("echo \"A = load \'/user/ubuntu/passwd\' using PigStorage(\':\');\">/tmp/id.pig ")
         o,c= self.pig_unit.run("echo \"B = foreach A generate \\$0 as id; store B into \'/tmp/id.out\';\">>/tmp/id.pig ")  
         o,c= self.pig_unit.run("sudo su hdfs -c 'pig -l /tmp/pig.log /tmp/id.pig'") 
-        if c != 0:
-           amulet.raise_status(amulet.FAIL, msg=" Pig script-> mapreduce FAILED")
         o,c= self.pig_unit.run("hadoop fs -ls /tmp/id.out/_SUCCESS")
         if c != 0:
            amulet.raise_status(amulet.FAIL, msg=" Pig script-> mapreduce passes but mapreduce failed during execution")
 
=== modified file 'tests/hadoop-pig-cluster.yaml'
--- tests/hadoop-pig-cluster.yaml	2014-09-17 19:36:42 +0000
+++ tests/hadoop-pig-cluster.yaml	2014-12-09 17:14:57 +0000
@@ -1,20 +1,23 @@
 hortonworks-pig-solution: 
   services: 
     "compute-node": 
-      charm: "cs:~asanjar/trusty/hdp-hadoop"
-      num_units: 1
+      charm: "cs:trusty/hdp-hadoop"
+      num_units: 2
+      constraints: "cpu-cores=4 mem=8192"
       annotations: 
         "gui-x": "656.6386478556842"
         "gui-y": "997.2758247589375"
     "yarn-hdfs-master": 
-      charm: "cs:~asanjar/trusty/hdp-hadoop"
+      charm: "cs:trusty/hdp-hadoop"
       num_units: 1
+      constraints: "cpu-cores=4 mem=8192"
       annotations: 
         "gui-x": "657.3613521443158"
         "gui-y": "654.7241752410625"
     "hdp-pig": 
-      charm: "cs:~asanjar/trusty/hdp-pig"
+      charm: "hdp-pig"
       num_units: 1
+      constraints: "cpu-cores=4 mem=8192"
       annotations: 
         "gui-x": "675.9159435670526"
         "gui-y": "279.7241752410624"
@@ -27,4 +30,6 @@
       - "compute-node:nodemanager"
     - - "hdp-pig:resourcemanager"
       - "yarn-hdfs-master:resourcemanager"
+    - - "compute-node:hadoop-nodes"
+      - "hdp-pig:hadoop-nodes"
   series: trusty