Merge lp:~bigdata-dev/charms/bundles/apache-analytics-pig/trunk into lp:~charmers/charms/bundles/apache-analytics-pig/bundle

[Cory Johns]

Tests for CWR

[lp:~bigdata-dev/charms/bundles/apache-analytics-pig/trunk]

15. By Cory Johns on 2015-10-07

Added tests.yaml and removed no longer needed 00-setup

16. By Cory Johns on 2015-10-07

Added -dev and -local bundle files

17. By Cory Johns on 2015-10-07

Fixed test perms

[Kevin W Monroe (kwmonroe)]

LGTM, +1

review: Approve

[lp:~bigdata-dev/charms/bundles/apache-analytics-pig/trunk]

18. By Kevin W Monroe on 2015-10-07

update charm revnos with latest promulgated versions

=== modified file 'README.md'
--- README.md	2015-05-29 21:03:16 +0000
+++ README.md	2015-10-07 21:27:49 +0000
@@ -14,7 +14,7 @@
 ## Usage
 Deploy this bundle using juju-quickstart:
 
-    juju quickstart u/bigdata-dev/apache-analytics-pig
+    juju quickstart apache-analytics-pig
 
 See `juju quickstart --help` for deployment options, including machine 
 constraints and how to deploy a locally modified version of the

=== added file 'bundle-dev.yaml'
--- bundle-dev.yaml	1970-01-01 00:00:00 +0000
+++ bundle-dev.yaml	2015-10-07 21:27:49 +0000
@@ -0,0 +1,50 @@
+services:
+  compute-slave:
+    charm: cs:~bigdata-dev/trusty/apache-hadoop-compute-slave
+    num_units: 3
+    annotations:
+      gui-x: "300"
+      gui-y: "200"
+    constraints: mem=3G
+  hdfs-master:
+    charm: cs:~bigdata-dev/trusty/apache-hadoop-hdfs-master
+    num_units: 1
+    annotations:
+      gui-x: "600"
+      gui-y: "350"
+    constraints: mem=7G
+  pig:
+    charm: cs:~bigdata-dev/trusty/apache-pig
+    num_units: 1
+    annotations:
+      gui-x: "1200"
+      gui-y: "200"
+    constraints: mem=3G
+  plugin:
+    charm: cs:~bigdata-dev/trusty/apache-hadoop-plugin
+    annotations:
+      gui-x: "900"
+      gui-y: "200"
+  secondary-namenode:
+    charm: cs:~bigdata-dev/trusty/apache-hadoop-hdfs-secondary
+    num_units: 1
+    annotations:
+      gui-x: "600"
+      gui-y: "600"
+    constraints: mem=7G
+  yarn-master:
+    charm: cs:~bigdata-dev/trusty/apache-hadoop-yarn-master
+    num_units: 1
+    annotations:
+      gui-x: "600"
+      gui-y: "100"
+    constraints: mem=7G
+series: trusty
+relations:
+  - [yarn-master, hdfs-master]
+  - [secondary-namenode, hdfs-master]
+  - [compute-slave, yarn-master]
+  - [compute-slave, hdfs-master]
+  - [plugin, yarn-master]
+  - [plugin, hdfs-master]
+  - [pig, plugin]

=== added file 'bundle-local.yaml'
--- bundle-local.yaml	1970-01-01 00:00:00 +0000
+++ bundle-local.yaml	2015-10-07 21:27:49 +0000
@@ -0,0 +1,50 @@
+services:
+  compute-slave:
+    charm: apache-hadoop-compute-slave
+    num_units: 3
+    annotations:
+      gui-x: "300"
+      gui-y: "200"
+    constraints: mem=3G
+  hdfs-master:
+    charm: apache-hadoop-hdfs-master
+    num_units: 1
+    annotations:
+      gui-x: "600"
+      gui-y: "350"
+    constraints: mem=7G
+  pig:
+    charm: apache-pig
+    num_units: 1
+    annotations:
+      gui-x: "1200"
+      gui-y: "200"
+    constraints: mem=3G
+  plugin:
+    charm: apache-hadoop-plugin
+    annotations:
+      gui-x: "900"
+      gui-y: "200"
+  secondary-namenode:
+    charm: apache-hadoop-hdfs-secondary
+    num_units: 1
+    annotations:
+      gui-x: "600"
+      gui-y: "600"
+    constraints: mem=7G
+  yarn-master:
+    charm: apache-hadoop-yarn-master
+    num_units: 1
+    annotations:
+      gui-x: "600"
+      gui-y: "100"
+    constraints: mem=7G
+series: trusty
+relations:
+  - [yarn-master, hdfs-master]
+  - [secondary-namenode, hdfs-master]
+  - [compute-slave, yarn-master]
+  - [compute-slave, hdfs-master]
+  - [plugin, yarn-master]
+  - [plugin, hdfs-master]
+  - [pig, plugin]

=== modified file 'bundle.yaml'
--- bundle.yaml	2015-09-16 22:07:20 +0000
+++ bundle.yaml	2015-10-07 21:27:49 +0000
@@ -1,39 +1,39 @@
 services:
   compute-slave:
-    charm: cs:trusty/apache-hadoop-compute-slave-8
+    charm: cs:trusty/apache-hadoop-compute-slave-9
     num_units: 3
     annotations:
       gui-x: "300"
       gui-y: "200"
     constraints: mem=3G
   hdfs-master:
-    charm: cs:trusty/apache-hadoop-hdfs-master-8
+    charm: cs:trusty/apache-hadoop-hdfs-master-9
     num_units: 1
     annotations:
       gui-x: "600"
       gui-y: "350"
     constraints: mem=7G
   pig:
-    charm: cs:trusty/apache-pig-6
+    charm: cs:trusty/apache-pig-7
     num_units: 1
     annotations:
       gui-x: "1200"
       gui-y: "200"
     constraints: mem=3G
   plugin:
-    charm: cs:trusty/apache-hadoop-plugin-7
+    charm: cs:trusty/apache-hadoop-plugin-8
     annotations:
       gui-x: "900"
       gui-y: "200"
   secondary-namenode:
-    charm: cs:trusty/apache-hadoop-hdfs-secondary-6
+    charm: cs:trusty/apache-hadoop-hdfs-secondary-7
     num_units: 1
     annotations:
       gui-x: "600"
       gui-y: "600"
     constraints: mem=7G
   yarn-master:
-    charm: cs:trusty/apache-hadoop-yarn-master-6
+    charm: cs:trusty/apache-hadoop-yarn-master-7
     num_units: 1
     annotations:
       gui-x: "600"

=== added directory 'tests'
=== added file 'tests/01-bundle.py'
--- tests/01-bundle.py	1970-01-01 00:00:00 +0000
+++ tests/01-bundle.py	2015-10-07 21:27:49 +0000
@@ -0,0 +1,124 @@
+#!/usr/bin/env python3
+
+import os
+import unittest
+
+import yaml
+import amulet
+
+
+class TestBundle(unittest.TestCase):
+    bundle_file = os.path.join(os.path.dirname(__file__), '..', 'bundle.yaml')
+
+    @classmethod
+    def setUpClass(cls):
+        cls.d = amulet.Deployment(series='trusty')
+        with open(cls.bundle_file) as f:
+            bun = f.read()
+        bundle = yaml.safe_load(bun)
+        cls.d.load(bundle)
+        cls.d.setup(timeout=1800)
+        cls.d.sentry.wait_for_messages({'pig': 'Ready'}, timeout=1800)
+        cls.hdfs = cls.d.sentry['hdfs-master'][0]
+        cls.yarn = cls.d.sentry['yarn-master'][0]
+        cls.slave = cls.d.sentry['compute-slave'][0]
+        cls.secondary = cls.d.sentry['secondary-namenode'][0]
+        cls.pig = cls.d.sentry['pig'][0]
+
+    def test_components(self):
+        """
+        Confirm that all of the required components are up and running.
+        """
+        hdfs, retcode = self.hdfs.run("pgrep -a java")
+        yarn, retcode = self.yarn.run("pgrep -a java")
+        slave, retcode = self.slave.run("pgrep -a java")
+        secondary, retcode = self.secondary.run("pgrep -a java")
+        pig, retcode = self.pig.run("pgrep -a java")
+
+        # .NameNode needs the . to differentiate it from SecondaryNameNode
+        assert '.NameNode' in hdfs, "NameNode not started"
+        assert '.NameNode' not in yarn, "NameNode should not be running on yarn-master"
+        assert '.NameNode' not in slave, "NameNode should not be running on compute-slave"
+        assert '.NameNode' not in secondary, "NameNode should not be running on secondary-namenode"
+        assert '.NameNode' not in pig, "NameNode should not be running on pig"
+
+        assert 'ResourceManager' in yarn, "ResourceManager not started"
+        assert 'ResourceManager' not in hdfs, "ResourceManager should not be running on hdfs-master"
+        assert 'ResourceManager' not in slave, "ResourceManager should not be running on compute-slave"
+        assert 'ResourceManager' not in secondary, "ResourceManager should not be running on secondary-namenode"
+        assert 'ResourceManager' not in pig, "ResourceManager should not be running on pig"
+
+        assert 'JobHistoryServer' in yarn, "JobHistoryServer not started"
+        assert 'JobHistoryServer' not in hdfs, "JobHistoryServer should not be running on hdfs-master"
+        assert 'JobHistoryServer' not in slave, "JobHistoryServer should not be running on compute-slave"
+        assert 'JobHistoryServer' not in secondary, "JobHistoryServer should not be running on secondary-namenode"
+        assert 'JobHistoryServer' not in pig, "JobHistoryServer should not be running on pig"
+
+        assert 'NodeManager' in slave, "NodeManager not started"
+        assert 'NodeManager' not in yarn, "NodeManager should not be running on yarn-master"
+        assert 'NodeManager' not in hdfs, "NodeManager should not be running on hdfs-master"
+        assert 'NodeManager' not in secondary, "NodeManager should not be running on secondary-namenode"
+        assert 'NodeManager' not in pig, "NodeManager should not be running on pig"
+
+        assert 'DataNode' in slave, "DataServer not started"
+        assert 'DataNode' not in yarn, "DataNode should not be running on yarn-master"
+        assert 'DataNode' not in hdfs, "DataNode should not be running on hdfs-master"
+        assert 'DataNode' not in secondary, "DataNode should not be running on secondary-namenode"
+        assert 'DataNode' not in pig, "DataNode should not be running on pig"
+
+        assert 'SecondaryNameNode' in secondary, "SecondaryNameNode not started"
+        assert 'SecondaryNameNode' not in yarn, "SecondaryNameNode should not be running on yarn-master"
+        assert 'SecondaryNameNode' not in hdfs, "SecondaryNameNode should not be running on hdfs-master"
+        assert 'SecondaryNameNode' not in slave, "SecondaryNameNode should not be running on compute-slave"
+        assert 'SecondaryNameNode' not in pig, "SecondaryNameNode should not be running on pig"
+
+    def test_hdfs_dir(self):
+        """
+        Validate admin few hadoop activities on HDFS cluster.
+            1) This test validates mkdir on hdfs cluster
+            2) This test validates change hdfs dir owner on the cluster
+            3) This test validates setting hdfs directory access permission on the cluster
+
+        NB: These are order-dependent, so must be done as part of a single test case.
+        """
+        output, retcode = self.pig.run("su hdfs -c 'hdfs dfs -mkdir -p /user/ubuntu'")
+        assert retcode == 0, "Created a user directory on hdfs FAILED:\n{}".format(output)
+        output, retcode = self.pig.run("su hdfs -c 'hdfs dfs -chown ubuntu:ubuntu /user/ubuntu'")
+        assert retcode == 0, "Assigning an owner to hdfs directory FAILED:\n{}".format(output)
+        output, retcode = self.pig.run("su hdfs -c 'hdfs dfs -chmod -R 755 /user/ubuntu'")
+        assert retcode == 0, "seting directory permission on hdfs FAILED:\n{}".format(output)
+
+    def test_yarn_mapreduce_exe(self):
+        """
+        Validate yarn mapreduce operations:
+            1) validate mapreduce execution - writing to hdfs
+            2) validate successful mapreduce operation after the execution
+            3) validate mapreduce execution - reading and writing to hdfs
+            4) validate successful mapreduce operation after the execution
+            5) validate successful deletion of mapreduce operation result from hdfs
+
+        NB: These are order-dependent, so must be done as part of a single test case.
+        """
+        jar_file = '/usr/lib/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar'
+        test_steps = [
+            ('teragen',      "su ubuntu -c 'hadoop jar {} teragen  10000 /user/ubuntu/teragenout'".format(jar_file)),
+            ('mapreduce #1', "su hdfs -c 'hdfs dfs -ls /user/ubuntu/teragenout/_SUCCESS'"),
+            ('terasort',     "su ubuntu -c 'hadoop jar {} terasort /user/ubuntu/teragenout /user/ubuntu/terasortout'".
+                format(jar_file)),
+            ('mapreduce #2', "su hdfs -c 'hdfs dfs -ls /user/ubuntu/terasortout/_SUCCESS'"),
+            ('cleanup',      "su hdfs -c 'hdfs dfs -rm -r /user/ubuntu/teragenout'"),
+        ]
+        for name, step in test_steps:
+            output, retcode = self.pig.run(step)
+            assert retcode == 0, "{} FAILED:\n{}".format(name, output)
+
+    def test_pig(self):
+        o, c = self.pig.run("su ubuntu -c 'pig -i -x local'")
+        assert "Apache" in o, "Pig local mode failed: %s" % o
+
+        o, c = self.pig.run("su ubuntu -c 'pig -i'")
+        assert "Apache" in o, "Pig MapReduce mode failed: %s" % o
+
+
+if __name__ == '__main__':
+    unittest.main()

=== added file 'tests/tests.yaml'
--- tests/tests.yaml	1970-01-01 00:00:00 +0000
+++ tests/tests.yaml	2015-10-07 21:27:49 +0000
@@ -0,0 +1,3 @@
+reset: false
+packages:
+  - amulet