Merge lp:~asanjar/charms/trusty/hdp-hadoop/trunk into lp:charms/trusty/hdp-hadoop

[amir sanjar]

adding following features:
1) Openstack enablement for multi-node compute nodes
2) Enable external big data applications query hadoop compute-nodes ip&hostname
for task discovery.
3) clean-up
4) update amulet testcase

[lp:~asanjar/charms/trusty/hdp-hadoop/trunk]

24. By amir sanjar on 2014-11-07

updating hortonworks ubuntu repository

25. By amir sanjar on 2014-11-08

update hortonworks helper files

26. By amir sanjar on 2014-11-08

undo upgrade to the latest hortonworks branch

[Charles Butler (lazypower)]

Amir - this merge has some great content however its not going to cleanly apply over the existing charm store branch.

I've got a fixup that took your changes and applied them over the charm store series - minus the whitespace changes and pushed it to bigdata-dev and will propose for bigdata-charmers.

https://code.launchpad.net/~bigdata-dev/charms/trusty/hdp-hadoop/trunk

Make sure you backport these changes into your personal namespace when doing future dev, otherwise you will have this problem again.

Thanks for the contribution!

review: Needs Fixing

=== removed file 'hadoop_cluster.yaml'
--- hadoop_cluster.yaml	2014-08-25 16:33:37 +0000
+++ hadoop_cluster.yaml	1970-01-01 00:00:00 +0000
@@ -1,20 +0,0 @@
-hdp-hadoop-cluster: 
-  services: 
-    "compute-node": 
-      charm: "cs:~asanjar/trusty/hdp-hadoop"
-      num_units: 1
-      annotations: 
-        "gui-x": "525.7681500094167"
-        "gui-y": "608.4847070634866"
-    "yarn-hdfs-master": 
-      charm: "cs:~asanjar/trusty/hdp-hadoop"
-      num_units: 1
-      annotations: 
-        "gui-x": "532"
-        "gui-y": "236.51529293651342"
-  relations: 
-    - - "yarn-hdfs-master:namenode"
-      - "compute-node:datanode"
-    - - "yarn-hdfs-master:resourcemanager"
-      - "compute-node:nodemanager"
-  series: trusty

=== added symlink 'hooks/compute-nodes-relation-changed'
=== target is u'hdp-hadoop-common.py'
=== added symlink 'hooks/compute-nodes-relation-joined'
=== target is u'hdp-hadoop-common.py'
=== added symlink 'hooks/compute-relation-joined'
=== target is u'hdp-hadoop-common.py'
=== added symlink 'hooks/hadoop-nodes-relation-joined'
=== target is u'hdp-hadoop-common.py'
=== modified file 'hooks/hdp-hadoop-common.py'
--- hooks/hdp-hadoop-common.py	2014-11-03 17:56:47 +0000
+++ hooks/hdp-hadoop-common.py	2014-11-08 05:32:58 +0000
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/python
 import os
 import subprocess
 import sys
@@ -6,6 +6,7 @@
 import shlex
 import shutil
 import inspect
+import time
 
 from hdputils import install_base_pkg, updateHDPDirectoryScript, config_all_nodes, \
                     setHadoopEnvVar, home, hdpScript, configureJAVA, config_all_nodes
@@ -13,7 +14,12 @@
 
 from charmhelpers.lib.utils import config_get, get_unit_hostname
 from shutil import rmtree, copyfile
+<<<<<<< TREE
 from charmhelpers.core.hookenv import log, Hooks, relation_get, relation_set, unit_get, open_port
+=======
+from socket import gethostname
+from charmhelpers.core.hookenv import log, Hooks, relation_get, relation_set, unit_get, open_port, local_unit, related_units
+>>>>>>> MERGE-SOURCE
 from charmhelpers.core.host import service_start, service_stop, add_user_to_group
 from time import sleep
 
@@ -144,9 +150,17 @@
 def callHDFS_fs(command):
     cmd = shlex.split("su hdfs -c '/usr/lib/hadoop/bin/hadoop fs {}'".format(command))
     subprocess.call(cmd)
+<<<<<<< TREE
 
 def startJobHistory():
     log("==> startJobHistory")
+=======
+###########################
+# Start Job History server
+###########################    
+def start_jh():
+    log("==> start_jh")
+>>>>>>> MERGE-SOURCE
     path = os.path.join(os.path.sep, 'usr', 'lib', 'hadoop-yarn', 'bin', 'container-executor')
     chownRecursive(path, 'root', 'hadoop')
     os.chmod(path, 650)
@@ -163,45 +177,96 @@
     cmd = shlex.split("su {} -c '/usr/lib/hadoop-mapreduce/sbin/mr-jobhistory-daemon.sh --config {} start historyserver'".\
                       format(os.environ['MAPRED_USER'], hadoopConfDir))
     subprocess.call(cmd)
+<<<<<<< TREE
 
 def stopJobHistory():
+=======
+    
+###########################
+# Stop Job History server
+###########################   
+def stop_jh():
+>>>>>>> MERGE-SOURCE
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     os.environ["HADOOP_LIBEXEC_DIR"]="/usr/lib/hadoop/libexec"
-    cmd = shlex.split("su {} -c '/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config {} stop resourcemanager'".\
+    cmd = shlex.split("su {} -c '/usr/lib/hadoop-mapreduce/sbin/mr-jobhistory-daemon.sh --config {} stop historyserver'".\
                       format(os.environ['MAPRED_USER'], hadoopConfDir))
     subprocess.call(cmd)
+<<<<<<< TREE
 
 # candidate for BD charm helper
 
 def start_namenode(hdfsUser):
+=======
+    
+############################################################################################
+# restart Job History server - Must be done everytime yarn-site.xml and
+# mpared-site.xml are modfied
+############################################################################################
+def restart_jh():
+    if is_jvm_service_active("JobHistoryServer"):
+       stop_jh()
+       start_jh()
+          
+##########################
+# Start NameNode server
+###########################   
+def start_nn(hdfsUser):
+>>>>>>> MERGE-SOURCE
     log("==> start namenode for user={}".format(hdfsUser), "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     cmd = shlex.split("su {} -c '/usr/lib/hadoop/sbin/hadoop-daemon.sh --config {} start namenode'".\
                       format(hdfsUser, hadoopConfDir))
+<<<<<<< TREE
     subprocess.check_call(cmd)
 
 def stop_namenode(hdfsUser):
     log("==> start namenode for user={}".format(hdfsUser), "INFO")
+=======
+    subprocess.check_call(cmd)
+
+###########################
+# Stop Name Node server
+###########################   
+def stop_nn(hdfsUser):
+    log("==> stop namenode for user={}".format(hdfsUser), "INFO")
+>>>>>>> MERGE-SOURCE
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     cmd = shlex.split("su {} -c '/usr/lib/hadoop/sbin/hadoop-daemon.sh --config {} stop namenode'".\
                       format(hdfsUser, hadoopConfDir))
     subprocess.call(cmd)
-# candidate for BD charm helper
 
-def start_datanode(hdfsUser):
-    log("==> start namenode for user={}".format(hdfsUser), "INFO")
+###########################
+# Start Data Node 
+###########################   
+def start_dn(hdfsUser):
+    log("==> start datanode for user={}".format(hdfsUser), "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     cmd = shlex.split("su {} -c '/usr/lib/hadoop/sbin/hadoop-daemon.sh --config {} start datanode'".\
                       format(hdfsUser, hadoopConfDir))
+<<<<<<< TREE
     subprocess.check_call(cmd)
 
 def stop_datanode(hdfsUser):
     log("==> start namenode for user={}".format(hdfsUser), "INFO")
+=======
+    subprocess.check_call(cmd)
+    
+###########################
+# Stop Data Node 
+###########################       
+def stop_dn(hdfsUser):
+    log("==> stop datanode for user={}".format(hdfsUser), "INFO")
+>>>>>>> MERGE-SOURCE
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     cmd = shlex.split("su {} -c '/usr/lib/hadoop/sbin/hadoop-daemon.sh --config {} stop datanode'".\
                       format(hdfsUser, hadoopConfDir))
     subprocess.call(cmd)
 # candidate for BD charm helper
+
+#############################################
+# Configure YARN-SITE.XML and MAPRED-SITE.XML
+#############################################
 def configureYarn(RMhostname):
     yarnConfPath = os.path.join(os.path.sep, os.environ['HADOOP_CONF_DIR'],"yarn-site.xml")
     mapConfDir = os.path.join(os.path.sep, os.environ['HADOOP_CONF_DIR'],"mapred-site.xml")
@@ -219,64 +284,136 @@
     #jobhistory server
     setHadoopConfigXML(mapConfDir, "mapreduce.jobhistory.webapp.address", RMhostname+":19888")
     setHadoopConfigXML(mapConfDir, "mapreduce.jobhistory.address",  RMhostname+":10020")
+<<<<<<< TREE
 
 # candidate for BD charm helper
 def start_RM(yarnUser):
+=======
+    
+################################
+# Start Resource Manager server 
+################################
+def start_rm(yarnUser):
+>>>>>>> MERGE-SOURCE
     log("==> start resourcemanager", "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     os.environ["HADOOP_LIBEXEC_DIR"]="/usr/lib/hadoop/libexec"
     cmd = shlex.split("su {} -c '/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config {} start resourcemanager'".\
                       format(yarnUser, hadoopConfDir))
     subprocess.call(cmd)
+<<<<<<< TREE
 
 def stop_RM(yarnUser):
+=======
+    
+################################
+# Stop Resource Manager server 
+################################   
+def stop_rm(yarnUser):
+>>>>>>> MERGE-SOURCE
     log("==> stop resourcemanager", "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     os.environ["HADOOP_LIBEXEC_DIR"]="/usr/lib/hadoop/libexec"
     cmd = shlex.split("su {} -c '/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config {} stop resourcemanager'".\
                       format(yarnUser, hadoopConfDir))
+<<<<<<< TREE
     subprocess.check_call(cmd)
 # candidate for BD charm helper
 def start_NM(yarnUser):
+=======
+    subprocess.check_call(cmd)
+    
+################################################
+# Start Node Manager daemon on each compute node 
+################################################
+def start_nm(yarnUser):
+>>>>>>> MERGE-SOURCE
     log("==> start nodemanager", "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     os.environ["HADOOP_LIBEXEC_DIR"]="/usr/lib/hadoop/libexec"
     cmd = shlex.split("su {} -c '/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config {} start nodemanager'".\
                       format(yarnUser, hadoopConfDir))
+<<<<<<< TREE
     subprocess.call(cmd)
 
 def stop_NM(yarnUser):
+=======
+    subprocess.check_call(cmd)
+
+################################################
+# Stop Node Manager daemon on each compute node 
+################################################   
+def stop_nm(yarnUser):
+>>>>>>> MERGE-SOURCE
     log("==> stop nodemanager", "INFO")
     hadoopConfDir = os.environ["HADOOP_CONF_DIR"]
     os.environ["HADOOP_LIBEXEC_DIR"]="/usr/lib/hadoop/libexec"
     cmd = shlex.split("su {} -c '/usr/lib/hadoop-yarn/sbin/yarn-daemon.sh --config {} stop nodemanager'".\
                       format(yarnUser, hadoopConfDir))
     subprocess.call(cmd)
-
+<<<<<<< TREE
+
+=======
+
+################################################
+# Stop all running hadoop services
+# NOTE: Order is important - DO NOT CHENAGE 
+################################################  
+>>>>>>> MERGE-SOURCE
 def stop_hadoop_services():
     if is_jvm_service_active("ResourceManager"):
-        stop_RM(os.environ['YARN_USER'])
+        stop_rm(os.environ['YARN_USER'])
     if is_jvm_service_active("NodeManager"):
-        stop_NM(os.environ['YARN_USER'])
+        stop_nm(os.environ['YARN_USER'])
     if is_jvm_service_active("NameNode"):
-        stop_namenode(os.environ['HDFS_USER'])
+        stop_nn(os.environ['HDFS_USER'])
     if is_jvm_service_active("DataNode"):
+<<<<<<< TREE
         stop_datanode(os.environ['HDFS_USER'])
 
+=======
+        stop_dn(os.environ['HDFS_USER'])
+    if is_jvm_service_active("JobHistoryServer"):
+        stop_jh()
+             
+################################################
+# restart all running hadoop services
+# NOTE: Order is important - DO NOT CHENAGE 
+################################################ 
+>>>>>>> MERGE-SOURCE
 def restart_hadoop_services():
+    if is_jvm_service_active("NameNode"):
+        stop_nn(os.environ['HDFS_USER'])
+        start_nn(os.environ['HDFS_USER'])
     if is_jvm_service_active("ResourceManager"):
-        stop_RM(os.environ['YARN_USER'])
-        start_RM(os.environ['YARN_USER'])
+        stop_rm(os.environ['YARN_USER'])
+        start_rm(os.environ['YARN_USER'])
     if is_jvm_service_active("NodeManager"):
-        stop_NM(os.environ['YARN_USER'])
-        start_NM(os.environ['YARN_USER'])
-    if is_jvm_service_active("NameNode"):
-        stop_namenode(os.environ['HDFS_USER'])
-        start_namenode(os.environ['HDFS_USER'])
+        stop_nm(os.environ['YARN_USER'])
+        start_nm(os.environ['YARN_USER'])
     if is_jvm_service_active("DataNode"):
+<<<<<<< TREE
         stop_datanode(os.environ['HDFS_USER'])
         start_datanode(os.environ['HDFS_USER'])
 
+=======
+        stop_dn(os.environ['HDFS_USER'])
+        start_dn(os.environ['HDFS_USER'])
+    restart_jh()
+    
+def wait_for_hadoop_service(service):
+    ticks = time.time()
+    while True:
+        if (time.time() - ticks) > 200:
+            log("Error ==> Reached timeout value for hadoop service {}..".format(service), "ERROR")
+            sys.exit(1)
+        if not is_jvm_service_active(service):
+            time.sleep(2)
+            log("Waiting..  ==> {} not ready..".format(service),"INFO")
+            continue
+        break  
+    
+>>>>>>> MERGE-SOURCE
 def configureHDFS(hostname):
     hdfsConfPath = os.path.join(os.path.sep, os.environ['HADOOP_CONF_DIR'],'hdfs-site.xml')
     coreConfPath = os.path.join(os.path.sep, os.environ['HADOOP_CONF_DIR'],'core-site.xml')
@@ -300,8 +437,12 @@
 tarfilenamePre="hdp_manual_install_rpm_helper_files-2.1.1.385"
 bashrc = os.path.join(os.path.sep, home, '.bashrc')
 hadoopMemoryOptimizationData = os.path.join(os.path.sep, hdpScriptPath, "hdpMemOpt.txt");
+<<<<<<< TREE
 hosts_path = os.path.join(os.path.sep, 'etc', 'hosts')
 nameNodeReady = False
+=======
+hosts_path = os.path.join(os.path.sep, 'etc', 'hosts')
+>>>>>>> MERGE-SOURCE
 resourceManagerReady = False
 ##########################################################################################
 
@@ -330,8 +471,14 @@
                 'hadoop-lzo']
     install_base_pkg(packages)
     config_hadoop_nodes()
+<<<<<<< TREE
     fileSetKV(hosts_path, unit_get('private-address')+' ', get_unit_hostname())
 
+=======
+    fileSetKV(hosts_path, unit_get('private-address')+' ', get_unit_hostname())
+        
+    
+>>>>>>> MERGE-SOURCE
 @hooks.hook('resourcemanager-relation-joined')
 def resourcemanager_relation_joined():
     log ("==> resourcemanager-relation-joined","INFO")
@@ -339,12 +486,22 @@
         relation_set(resourceManagerReady=True)
         relation_set(resourceManager_hostname=get_unit_hostname())
         return
+    if not is_jvm_service_active("NameNode"):
+        sys.exit(0)
     setHadoopEnvVar()
+<<<<<<< TREE
     relation_set(resourceManager_ip=unit_get('private-address'))
     relation_set(resourceManager_hostname=get_unit_hostname())
     configureYarn(unit_get('private-address'))
     start_RM(os.environ["YARN_USER"])
     startJobHistory()
+=======
+    relation_set(resourceManager_ip=unit_get('private-address'))
+    relation_set(resourceManager_hostname=get_unit_hostname())
+    configureYarn(unit_get('private-address'))
+    start_rm(os.environ["YARN_USER"])
+    start_jh()
+>>>>>>> MERGE-SOURCE
     open_port(8025)
     open_port(8030)
     open_port(8050)
@@ -362,11 +519,21 @@
     if not resourceManagerReady:
         sys.exit(0)
     setHadoopEnvVar()
+<<<<<<< TREE
     rm_ip = relation_get('private-address')
     configureYarn(rm_ip)
     fileSetKV(hosts_path, rm_ip+' ', relation_get('resourceManager_hostname'))
     start_NM(os.environ["YARN_USER"])
 
+=======
+    rm_ip = relation_get('private-address')
+    configureYarn(rm_ip)
+    fileSetKV(hosts_path, rm_ip+' ', relation_get('resourceManager_hostname'))
+    # nodemanager requires data node daemon
+    if not is_jvm_service_active("DataNode"):
+        start_dn(os.environ['HDFS_USER'])
+    start_nm(os.environ["YARN_USER"])
+>>>>>>> MERGE-SOURCE
     open_port(8025)
     open_port(8030)
     open_port(8050)
@@ -374,8 +541,14 @@
     open_port(19888)
     open_port(8088)
     open_port(10020)
+<<<<<<< TREE
     relation_set(nodemanager_hostname=get_unit_hostname())
 
+=======
+    relation_set(nodemanager_hostname=get_unit_hostname())
+    
+          
+>>>>>>> MERGE-SOURCE
 @hooks.hook('resourcemanager-relation-broken')
 def resourcemanager_relation_broken():
     log ("Configuring resourcemanager - broken phase","INFO")
@@ -384,11 +557,19 @@
 @hooks.hook('resourcemanager-relation-changed')
 def resourcemanager_relation_changed():
     log ("Configuring resourcemanager - changed phase","INFO")
+<<<<<<< TREE
     nm_host = relation_get('nodemanager_hostname')
     if nm_host == None:
         sys.exit(0)
     setHadoopEnvVar()
     fileSetKV(hosts_path, relation_get('private-address')+' ', nm_host)
+=======
+    nm_host = relation_get('nodemanager_hostname')
+    if nm_host == None:
+        sys.exit(0)
+    setHadoopEnvVar() 
+    fileSetKV(hosts_path, relation_get('private-address')+' ', nm_host)     
+>>>>>>> MERGE-SOURCE
 
 @hooks.hook('namenode-relation-joined')
 def namenode_relation_joined():
@@ -403,14 +584,22 @@
     relation_set(namenode_hostname=get_unit_hostname())
     configureHDFS(unit_get('private-address'))
     format_namenode(os.environ["HDFS_USER"])
+<<<<<<< TREE
     start_namenode(os.environ["HDFS_USER"])
 
+=======
+    start_nn(os.environ["HDFS_USER"])
+    start_jh()
+>>>>>>> MERGE-SOURCE
     open_port(8020)
     open_port(8010)
     open_port(50070)
     open_port(50075)
     open_port(8480)
     open_port(50470)
+    if not is_jvm_service_active("NameNode"):
+        log("error ==> NameNode failed to start")
+        sys.exit(1)
     sleep(5)
     relation_set(nameNodeReady=True)
 
@@ -423,20 +612,36 @@
     if not nameNodeReady:
         sys.exit(0)
     setHadoopEnvVar()
+<<<<<<< TREE
     nodeType="namenode"
     namenode_hostname = relation_get("namenode_hostname")
     namenode_ip = relation_get('private-address')
     fileSetKV(hosts_path, namenode_ip+' ', namenode_hostname)
     configureHDFS(namenode_hostname)
+=======
+    nn_hostname = relation_get("namenode_hostname")
+    nn_ip = relation_get('private-address')
+    fileSetKV(hosts_path, nn_ip+' ', nn_hostname)
+    configureHDFS(nn_ip)
+>>>>>>> MERGE-SOURCE
     setDirPermission(os.environ['DFS_DATA_DIR'], os.environ['HDFS_USER'], os.environ['HADOOP_GROUP'], 0750)
-    start_datanode(os.environ["HDFS_USER"])
+    start_dn(os.environ["HDFS_USER"])
+    if not is_jvm_service_active("DataNode"):
+        log("error ==> DataNode failed to start")
+        sys.exit(1)
     open_port(8010)
     open_port(8480)
     open_port(50010)
     open_port(50075)
-    relation_set(dn_hostname = get_unit_hostname())
-
-
+<<<<<<< TREE
+    relation_set(dn_hostname = get_unit_hostname())
+
+
+=======
+    relation_set(dn_hostname = get_unit_hostname())
+    
+        
+>>>>>>> MERGE-SOURCE
 @hooks.hook('config-changed')
 def config_changed():
     log( "config-changed called", "INFO")
@@ -448,11 +653,19 @@
 
 @hooks.hook('namenode-relation-changed')
 def namenode_relation_changed():
+<<<<<<< TREE
     dn_host = relation_get('dn-hostname')
     if dn_host == None:
         sys.exit(0)
     log("Configuring namenode - changed phase", "INFO")
     fileSetKV(hosts_path, relation_get('private-address')+' ', dn_host)
+=======
+    dn_host = relation_get('dn_hostname')
+    if dn_host == None:
+        sys.exit(0)
+    log("Configuring namenode - changed phase", "INFO") 
+    fileSetKV(hosts_path, relation_get('private-address')+' ', dn_host) 
+>>>>>>> MERGE-SOURCE
 
 @hooks.hook('start')
 def start():
@@ -463,9 +676,39 @@
     log( "stop called", "INFO")
     setHadoopEnvVar()
     stop_hadoop_services()
-
-
-
+<<<<<<< TREE
+
+
+
+=======
+
[e-mail]('compute-nodes-relation-changed')
+def compute_nodes_relation_changed():
+    log("==> Configuring compute-nodes, local ID = {} - Changed phase".format(local_unit()),"INFO")
+    nodeList = related_units()
+    for n in nodeList:
+        dn_ip = relation_get('private-address', unit=n)
+        hostname = relation_get('hostname', unit=n)
+        log("==> Configuring compute-nodes {}={}".format(dn_ip, hostname),"INFO")
+        if hostname != None:
+            fileSetKV(hosts_path, dn_ip+' ', hostname) 
+        else:
+            sys.exit(0)
+
[e-mail]('compute-nodes-relation-joined')
+def compute_nodes_relation_joined():
+    log("==> compute_nodes_relation_joined {}".format(get_unit_hostname()),"INFO")
+    relation_set(hostname=get_unit_hostname())
+
+
[e-mail]('hadoop-nodes-relation-joined')
+def hadoop_nodes_relation_joined():
+    log("==> hadoop_nodes_relation_joined {}".format(get_unit_hostname()),"INFO")
+    relation_set(hostname=get_unit_hostname())
+        
+    
+    
+>>>>>>> MERGE-SOURCE
 
 if __name__ == "__main__":
     hooks.execute(sys.argv)

=== modified file 'hooks/hdputils.py'
--- hooks/hdputils.py	2014-08-11 03:35:08 +0000
+++ hooks/hdputils.py	2014-11-08 05:32:58 +0000
@@ -22,6 +22,8 @@
     subprocess.call(cmd)   
     apt_update()
     apt_install(packages)
+    if not os.path.isdir(os.path.join(os.path.sep,'usr','lib', 'hadoop')):
+        log("Error, apt-get install Hadoop failed", "ERROR")
     os.chdir(home);
     wgetPkg("http://public-repo-1.hortonworks.com/HDP/tools/2.1.1.0/hdp_manual_install_rpm_helper_files-2.1.1.385.tar.gz","")
     if tarfile.is_tarfile(tarfilename):

=== modified file 'metadata.yaml'
--- metadata.yaml	2014-07-23 12:16:51 +0000
+++ metadata.yaml	2014-11-08 05:32:58 +0000
@@ -4,22 +4,19 @@
 description: |
   Hadoop is a software platform that lets one easily write and
   run applications that process vast amounts of data.
-categories: ["applications"]
+tags: ["applications"]
 provides:
   namenode:
     interface: dfs
   resourcemanager:
     interface: mapred
-  ganglia:
-    interface: monitor
+  hadoop-nodes:
+    interface: mapred
 requires:
   datanode:
     interface: dfs
-  secondarynamenode:
-    interface: dfs
   nodemanager:
     interface: mapred
-  mapred-namenode:
-    interface: dfs
-  elasticsearch:
-    interface: elasticsearch
+peers:
+  compute-nodes:
+    interface: mapred

=== modified file 'tests/01-hadoop-cluster-deployment-1.py'
--- tests/01-hadoop-cluster-deployment-1.py	2014-09-17 15:27:28 +0000
+++ tests/01-hadoop-cluster-deployment-1.py	2014-11-08 05:32:58 +0000
@@ -6,12 +6,12 @@
 class TestDeployment(object):
     def __init__(self):
         self.d =  amulet.Deployment(series='trusty')
-        bpath = os.path.join(os.path.dirname( __file__), "../hadoop_cluster.yaml")
+        bpath = os.path.join(os.path.dirname( __file__), "hadoop_cluster.yaml")
         f = open(bpath)
         bun = f.read()
         self.d.load(yaml.safe_load(bun))
         try:
-            self.d.setup(timeout=9000)
+            self.d.setup(timeout=900000)
             self.d.sentry.wait()
         except amulet.helpers.TimeoutError:
             amulet.raise_status(amulet.SKIP, msg="Environment wasn't stood up in time")

=== added file 'tests/hadoop_cluster.yaml'
--- tests/hadoop_cluster.yaml	1970-01-01 00:00:00 +0000
+++ tests/hadoop_cluster.yaml	2014-11-08 05:32:58 +0000
@@ -0,0 +1,20 @@
+hdp-hadoop-cluster: 
+  services: 
+    "compute-node": 
+      charm: "cs:~asanjar/trusty/hdp-hadoop"
+      num_units: 4
+      annotations: 
+        "gui-x": "525.7681500094167"
+        "gui-y": "608.4847070634866"
+    "yarn-hdfs-master": 
+      charm: "cs:~asanjar/trusty/hdp-hadoop"
+      num_units: 1
+      annotations: 
+        "gui-x": "532"
+        "gui-y": "236.51529293651342"
+  relations: 
+    - - "yarn-hdfs-master:namenode"
+      - "compute-node:datanode"
+    - - "yarn-hdfs-master:resourcemanager"
+      - "compute-node:nodemanager"
+  series: trusty