Merge lp:~bigdata-dev/charms/trusty/hdp-hive/hive-fixes into lp:charms/trusty/hdp-hive

[amir sanjar]

adding elasticsearch and Openstack support to hive

[Charles Butler (lazypower)]

Thanks for the proposed fixes Amir, I have however run into an issue with the MP - it appears there is an issue with the amulet tests:

The hive tests are failing during testing:
2014-12-08 13:46:45 Adding relation hdp-hive:db <-> mysql:db
2014-12-08 13:47:51 Deployment complete in 3444.33 seconds
 Hive Create table FAILED

Upon further investigation I see that hive is indeed connected to hadoop - and i was able to get an HDFS listing. So it appears the issue is with the test and not with the deployment, as the command works when being run locally on the node:

hive> create table test(col1 int, col2 string);
OK
Time taken: 2.465 seconds

Thanks for the submission, keep up the great work

review: Needs Fixing

Unmerged revisions

15. By amir sanjar on 2014-12-01

Merge elasticserach + Openstak with trunk

=== modified file 'README.md'
--- README.md	2014-09-10 20:48:42 +0000
+++ README.md	2014-12-01 23:23:55 +0000
@@ -63,6 +63,7 @@
     juju add-relation hdphive:namenode yarn-hdfs-master:namenode
     # associate Hive with resourcemanager
     juju add-relation hdphive:resourcemanager yarn-hdfs-master:resourcemanager
+    juju add-relation compute-node:hadoop-nodes hdphive:hadoop-nodes
     
 # Smoke Test
 ## Usage
@@ -72,7 +73,7 @@
        hadoop file system - Below steps verifies/demos HDFS functionality  
        a) sudo su $HDFS_USER
        b) hdfs dfs -mkdir -p /user/ubuntu
-       c) hdfs dfs -chown ubuntu:ubuntu /user/ubuntu
+       c) hdfs dfs -chown -R ubuntu:hdfs /user
        d) hdfs dfs -chmod -R 755 /user/ubuntu
        e) exit
 

=== added directory 'files/data'
=== added file 'files/data/elasticsearch-hadoop-2.0.0.zip'
Binary files files/data/elasticsearch-hadoop-2.0.0.zip	1970-01-01 00:00:00 +0000 and files/data/elasticsearch-hadoop-2.0.0.zip	2014-12-01 23:23:55 +0000 differ
=== added symlink 'hooks/elasticsearch-relation-joined'
=== target is u'hdp-hive-common.py'
=== removed symlink 'hooks/elk-relation-joined'
=== target was u'hdp-hive-common.py'
=== added symlink 'hooks/hadoop-nodes-relation-changed'
=== target is u'hdp-hive-common.py'
=== modified file 'hooks/hdp-hive-common.py'
--- hooks/hdp-hive-common.py	2014-08-22 10:54:53 +0000
+++ hooks/hdp-hive-common.py	2014-12-01 23:23:55 +0000
@@ -15,6 +15,8 @@
 from charmhelpers.core.hookenv import log, Hooks, relation_get, relation_set, unit_get, open_port
 from charmhelpers.core.host import service_start, service_stop, add_user_to_group
 from time import sleep
+from charmhelpers.lib.utils import get_unit_hostname
+
 
 
 # Required for unit tests... :(
@@ -79,6 +81,9 @@
     cmd = shlex.split("su hdfs -c '{} fs {}'".format(hadoopPath, command))
     subprocess.call(cmd)
 
+def callHDFS(command):
+    cmd = shlex.split("su {u} -c 'hdfs {c}'".format(u="hdfs", c=command))
+    return subprocess.check_output(cmd)
 
 def configurMySQLClient():
     host = relation_get('host')
@@ -120,7 +125,20 @@
     sleep(5)
     start_hive_metastore()
     start_hive_server2()
-    
+
+def waitForHDFSReady(namenode_ip):
+    import time
+    ticks = time.time()
+    while True:
+        if (time.time() - ticks) > 4000:
+            log("spark ==> Reached datanode timeout value..", "ERROR")
+            sys.exit(1)
+        o = callHDFS("dfsadmin -D fs.defaultFS={} -report".format("hdfs://"+namenode_ip+":8020"))
+        if o.find("Live datanodes:") == -1:
+            time.sleep(2)
+            log("spark ==> damenode not ready","INFO")
+            continue
+        break    
     
 #################################### Global Data ################################     
 
@@ -131,6 +149,7 @@
 hdpScript = "hdp_scripts"
 hdpScriptPath = os.path.join(os.path.sep,home, hdpScript,'scripts')
 bashrc = os.path.join(os.path.sep, home, '.bashrc')
+hosts_path = os.path.join(os.path.sep, 'etc', 'hosts')
 # hadoopMemoryOptimizationData = os.path.join(os.path.sep, hdpScriptPath, "hdpMemOpt.txt");
 
 ##########################################################################################
@@ -147,6 +166,7 @@
                 'openjdk-7-jdk',
                 'hive',
                 'hive-hcatalog',
+                'unzip',
                 'mysql-connector-java*']
     install_base_pkg(packages)
     config_hive_nodes()
@@ -154,6 +174,9 @@
     init_derby_Metastore_database_schema()
     start_hive_metastore()
     start_hive_server2()
+    fileSetKV(hosts_path, unit_get('private-address')+' ', get_unit_hostname())
+    add_user_to_group("ubuntu", "hadoop")
+    add_user_to_group(os.environ['HIVE_USER'], "hadoop")
     
 
 @hooks.hook('resourcemanager-relation-changed')
@@ -184,15 +207,19 @@
     if not nameNodeReady:
         log("HIVE ==> namenode not ready- Changed phase","INFO")
         sys.exit(0)
-    log("HIVE ==> namenode_IP={}".format(relation_get('private-address')),"INFO")
+    namenode_ip = relation_get('private-address')
+    log("HIVE ==> namenode_IP={}".format(namenode_ip),"INFO")
     setHadoopEnvVar()
+    waitForHDFSReady(namenode_ip)
     hdfsConfPath = os.path.join(os.path.sep, os.environ['HADOOP_CONF_DIR'],"core-site.xml")
-    setHadoopConfigXML(hdfsConfPath, "fs.defaultFS", "hdfs://"+relation_get('private-address')+":8020")
+    setHadoopConfigXML(hdfsConfPath, "fs.defaultFS", "hdfs://"+namenode_ip+":8020")
+    namenode_hostname = relation_get("namenode_hostname")
+    fileSetKV(hosts_path, namenode_ip+' ', namenode_hostname)
     hive_user = os.environ['HIVE_USER']
     hdfs_user = os.environ['HDFS_USER']
     #Create the Hive user home directory on HDFS.
     callHDFS_fs("-mkdir -p /user/{}".format(hive_user))
-    callHDFS_fs('-chown {h}:{d} /user/{h}'.format(h=hive_user, d=hdfs_user))
+    callHDFS_fs('-chown -R {h}:{d} /user'.format(h=hive_user, d=hdfs_user))
     # Create the warehouse directory on HDFS.
     callHDFS_fs('-mkdir -p /apps/hive/warehouse')
     callHDFS_fs('-chown -R {h}:{d} /apps/hive'.format(h=hive_user, d=hdfs_user))
@@ -203,6 +230,13 @@
     callHDFS_fs("-chmod -R 777 /tmp/scratch ")
     restart_hive_server()
     
[e-mail]('hadoop-nodes-relation-changed')
+def hadoop_nodes_relation_changed():
+    compute_ip = relation_get('private-address')
+    hostname = relation_get('hostname')
+    log("==> Connect to hadoop compute node {}={}".format(compute_ip, hostname),"INFO")
+    if hostname != None:
+       fileSetKV(hosts_path, compute_ip+' ', hostname) 
         
 @hooks.hook('config-changed')
 def config_changed():     
@@ -234,6 +268,23 @@
     init_MYSQL_Metastore_database_schema()
     start_hive_metastore()
     start_hive_server2()  
+    
[e-mail]('elasticsearch-relation-joined')   
+def elasticsearch_relation_joined():
+    elsZipfile = os.path.join(os.path.sep, os.environ['CHARM_DIR'],'files', 'data',
+                               "elasticsearch-hadoop-2.0.0.zip")
+    hive_lib = os.path.join(os.path.sep, 'usr', 'lib', 'hive', 'lib')
+    os.chdir(home)
+    cmd = "unzip {}".format(elsZipfile)
+    subprocess.call(cmd.split())
+    srcpath = os.path.join(os.path.sep, home, "elasticsearch-hadoop-2.0.0", 'dist')
+    source = os.listdir(srcpath)
+    for files in source:
+        srcFile = os.path.join(os.path.sep, srcpath, files)
+        desFile = os.path.join(os.path.sep, hive_lib, files)
+        copyfile(srcFile, desFile)
+    
+    
 
 if __name__ == "__main__":
     hooks.execute(sys.argv)

=== modified file 'metadata.yaml'
--- metadata.yaml	2014-08-18 10:05:16 +0000
+++ metadata.yaml	2014-12-01 23:23:55 +0000
@@ -28,5 +28,7 @@
     interface: hive
   db:
     interface: mysql
-  elk:
+  elasticsearch:
     interface: elasticsearch
+  hadoop-nodes:
+    interface: mapred

=== modified file 'tests/01-deploy-hive-cluster'
--- tests/01-deploy-hive-cluster	2014-09-19 18:26:01 +0000
+++ tests/01-deploy-hive-cluster	2014-12-01 23:23:55 +0000
@@ -30,7 +30,7 @@
 
     def test_remote_hdfs_status(self):
        o,c= self.hive_unit.run("sudo su hdfs -c ' hdfs dfsadmin -report'")
-       if o.find('Datanodes available: 1') == -1:
+       if o.find('Datanodes available: 2') == -1:
           amulet.raise_status(amulet.FAIL, msg="Remote HDFS not available")
 
     def test_remote_yarn_status(self):
@@ -43,8 +43,17 @@
         o,c= self.hive_unit.run("su hdfs -c 'hdfs dfs -mkdir -p /user/hduser'")
         if c != 0:
            amulet.raise_status(amulet.FAIL, msg=" Created a user directory on hdfs FAILED")
-
- 
+           
+    def test_hive(self):
+        o,c= self.hive_unit.run("su hive -c 'hive -e \"create table test(col1 int, col2 string);\"'")
+        if o.find("OK") == -1:
+           amulet.raise_status(amulet.FAIL, msg=" Hive Create table FAILED")
+           print(o)
+        o,c= self.hive_unit.run("su hive -c 'hive -e \"show tables;\"'")
+        if o.find("test") == -1:
+           amulet.raise_status(amulet.FAIL, msg=" Hive show table FAILED")
+           print(o)
+     
 
 if __name__ == '__main__':
     runner = TestDeployment()

=== modified file 'tests/hive-hadoop-sql-cluster.yaml'
--- tests/hive-hadoop-sql-cluster.yaml	2014-09-18 01:41:58 +0000
+++ tests/hive-hadoop-sql-cluster.yaml	2014-12-01 23:23:55 +0000
@@ -1,8 +1,8 @@
 hdp-hadoop-hive-mysql: 
   services: 
     "compute-node": 
-      charm: "cs:~asanjar/trusty/hdp-hadoop"
-      num_units: 1
+      charm: "hdp-hadoop"
+      num_units: 2
       annotations: 
         "gui-x": "768"
         "gui-y": "591.0585428804295"
@@ -15,13 +15,13 @@
         "gui-x": "1102.9983551210835"
         "gui-y": "591.0585428804295"
     hdphive: 
-      charm: "cs:~asanjar/trusty/hdp-hive"
+      charm: "hdp-hive"
       num_units: 1
       annotations: 
         "gui-x": "1105.4991775605417"
         "gui-y": "284.9414571195705"
     "yarn-hdfs-master": 
-      charm: "cs:~asanjar/trusty/hdp-hadoop"
+      charm: "hdp-hadoop"
       num_units: 1
       annotations: 
         "gui-x": "769.0016448789165"
@@ -35,6 +35,8 @@
       - "yarn-hdfs-master:resourcemanager"
     - - "yarn-hdfs-master:resourcemanager"
       - "compute-node:nodemanager"
+    - - "compute-node:hadoop-nodes"
+      - "hdphive:hadoop-nodes"
     - - "hdphive:db"
       - "mysql:db"
   series: trusty