Merge lp:~bigdata-dev/charms/bundles/apache-analytics-sql/trunk into lp:~charmers/charms/bundles/apache-analytics-sql/bundle

[Cory Johns]

Tests for CWR

[lp:~bigdata-dev/charms/bundles/apache-analytics-sql/trunk]

18. By Cory Johns on 2015-10-07

Added tests.yaml and removed no longer needed 00-setup

19. By Cory Johns on 2015-10-07

Fixed reference to mysql in dev bundle

[Kevin W Monroe (kwmonroe)]

LGTM, +1

review: Approve

[lp:~bigdata-dev/charms/bundles/apache-analytics-sql/trunk]

20. By Cory Johns on 2015-10-07

Updated to latest charm revisions

=== modified file 'bundle-dev.yaml'
--- bundle-dev.yaml	2015-09-17 14:37:10 +0000
+++ bundle-dev.yaml	2015-10-07 21:29:42 +0000
@@ -21,7 +21,7 @@
       gui-y: "200"
     constraints: mem=3G
   mysql:
-    charm: cs:~bigdata-dev/trusty/mysql
+    charm: cs:trusty/mysql
     num_units: 1
     options:
       binlog-format: ROW

=== modified file 'bundle.yaml'
--- bundle.yaml	2015-09-28 14:07:17 +0000
+++ bundle.yaml	2015-10-07 21:29:42 +0000
@@ -1,20 +1,20 @@
 services:
   compute-slave:
-    charm: cs:trusty/apache-hadoop-compute-slave-8
+    charm: cs:trusty/apache-hadoop-compute-slave-9
     num_units: 3
     annotations:
       gui-x: "300"
       gui-y: "200"
     constraints: mem=3G
   hdfs-master:
-    charm: cs:trusty/apache-hadoop-hdfs-master-8
+    charm: cs:trusty/apache-hadoop-hdfs-master-9
     num_units: 1
     annotations:
       gui-x: "600"
       gui-y: "350"
     constraints: mem=7G
   hive:
-    charm: cs:trusty/apache-hive-9
+    charm: cs:trusty/apache-hive-10
     num_units: 1
     annotations:
       gui-x: "1200"
@@ -29,19 +29,19 @@
       gui-x: "1200"
       gui-y: "450"
   plugin:
-    charm: cs:trusty/apache-hadoop-plugin-7
+    charm: cs:trusty/apache-hadoop-plugin-8
     annotations:
       gui-x: "900"
       gui-y: "200"
   secondary-namenode:
-    charm: cs:trusty/apache-hadoop-hdfs-secondary-6
+    charm: cs:trusty/apache-hadoop-hdfs-secondary-7
     num_units: 1
     annotations:
       gui-x: "600"
       gui-y: "600"
     constraints: mem=7G
   yarn-master:
-    charm: cs:trusty/apache-hadoop-yarn-master-6
+    charm: cs:trusty/apache-hadoop-yarn-master-7
     num_units: 1
     annotations:
       gui-x: "600"

=== added directory 'tests'
=== added file 'tests/01-bundle.py'
--- tests/01-bundle.py	1970-01-01 00:00:00 +0000
+++ tests/01-bundle.py	2015-10-07 21:29:42 +0000
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+
+import os
+import unittest
+
+import yaml
+import amulet
+
+
+class TestBundle(unittest.TestCase):
+    bundle_file = os.path.join(os.path.dirname(__file__), '..', 'bundle.yaml')
+
+    @classmethod
+    def setUpClass(cls):
+        cls.d = amulet.Deployment(series='trusty')
+        with open(cls.bundle_file) as f:
+            bun = f.read()
+        bundle = yaml.safe_load(bun)
+        cls.d.load(bundle)
+        cls.d.setup(timeout=1800)
+        cls.d.sentry.wait_for_messages({'hive': 'Ready'}, timeout=1800)
+        cls.hdfs = cls.d.sentry['hdfs-master'][0]
+        cls.yarn = cls.d.sentry['yarn-master'][0]
+        cls.slave = cls.d.sentry['compute-slave'][0]
+        cls.secondary = cls.d.sentry['secondary-namenode'][0]
+        cls.hive = cls.d.sentry['hive'][0]
+
+    def test_components(self):
+        """
+        Confirm that all of the required components are up and running.
+        """
+        hdfs, retcode = self.hdfs.run("pgrep -a java")
+        yarn, retcode = self.yarn.run("pgrep -a java")
+        slave, retcode = self.slave.run("pgrep -a java")
+        secondary, retcode = self.secondary.run("pgrep -a java")
+        hive, retcode = self.hive.run("pgrep -a java")
+
+        # .NameNode needs the . to differentiate it from SecondaryNameNode
+        assert '.NameNode' in hdfs, "NameNode not started"
+        assert '.NameNode' not in yarn, "NameNode should not be running on yarn-master"
+        assert '.NameNode' not in slave, "NameNode should not be running on compute-slave"
+        assert '.NameNode' not in secondary, "NameNode should not be running on secondary-namenode"
+        assert '.NameNode' not in hive, "NameNode should not be running on hive"
+
+        assert 'ResourceManager' in yarn, "ResourceManager not started"
+        assert 'ResourceManager' not in hdfs, "ResourceManager should not be running on hdfs-master"
+        assert 'ResourceManager' not in slave, "ResourceManager should not be running on compute-slave"
+        assert 'ResourceManager' not in secondary, "ResourceManager should not be running on secondary-namenode"
+        assert 'ResourceManager' not in hive, "ResourceManager should not be running on hive"
+
+        assert 'JobHistoryServer' in yarn, "JobHistoryServer not started"
+        assert 'JobHistoryServer' not in hdfs, "JobHistoryServer should not be running on hdfs-master"
+        assert 'JobHistoryServer' not in slave, "JobHistoryServer should not be running on compute-slave"
+        assert 'JobHistoryServer' not in secondary, "JobHistoryServer should not be running on secondary-namenode"
+        assert 'JobHistoryServer' not in hive, "JobHistoryServer should not be running on hive"
+
+        assert 'NodeManager' in slave, "NodeManager not started"
+        assert 'NodeManager' not in yarn, "NodeManager should not be running on yarn-master"
+        assert 'NodeManager' not in hdfs, "NodeManager should not be running on hdfs-master"
+        assert 'NodeManager' not in secondary, "NodeManager should not be running on secondary-namenode"
+        assert 'NodeManager' not in hive, "NodeManager should not be running on hive"
+
+        assert 'DataNode' in slave, "DataServer not started"
+        assert 'DataNode' not in yarn, "DataNode should not be running on yarn-master"
+        assert 'DataNode' not in hdfs, "DataNode should not be running on hdfs-master"
+        assert 'DataNode' not in secondary, "DataNode should not be running on secondary-namenode"
+        assert 'DataNode' not in hive, "DataNode should not be running on hive"
+
+        assert 'SecondaryNameNode' in secondary, "SecondaryNameNode not started"
+        assert 'SecondaryNameNode' not in yarn, "SecondaryNameNode should not be running on yarn-master"
+        assert 'SecondaryNameNode' not in hdfs, "SecondaryNameNode should not be running on hdfs-master"
+        assert 'SecondaryNameNode' not in slave, "SecondaryNameNode should not be running on compute-slave"
+        assert 'SecondaryNameNode' not in hive, "SecondaryNameNode should not be running on hive"
+
+        assert 'HiveServer2' in hive, 'Hive should be running on hive'
+
+    def test_hdfs_dir(self):
+        """
+        Validate admin few hadoop activities on HDFS cluster.
+            1) This test validates mkdir on hdfs cluster
+            2) This test validates change hdfs dir owner on the cluster
+            3) This test validates setting hdfs directory access permission on the cluster
+
+        NB: These are order-dependent, so must be done as part of a single test case.
+        """
+        output, retcode = self.hive.run("su hdfs -c 'hdfs dfs -mkdir -p /user/ubuntu'")
+        assert retcode == 0, "Created a user directory on hdfs FAILED:\n{}".format(output)
+        output, retcode = self.hive.run("su hdfs -c 'hdfs dfs -chown ubuntu:ubuntu /user/ubuntu'")
+        assert retcode == 0, "Assigning an owner to hdfs directory FAILED:\n{}".format(output)
+        output, retcode = self.hive.run("su hdfs -c 'hdfs dfs -chmod -R 755 /user/ubuntu'")
+        assert retcode == 0, "seting directory permission on hdfs FAILED:\n{}".format(output)
+
+    def test_yarn_mapreduce_exe(self):
+        """
+        Validate yarn mapreduce operations:
+            1) validate mapreduce execution - writing to hdfs
+            2) validate successful mapreduce operation after the execution
+            3) validate mapreduce execution - reading and writing to hdfs
+            4) validate successful mapreduce operation after the execution
+            5) validate successful deletion of mapreduce operation result from hdfs
+
+        NB: These are order-dependent, so must be done as part of a single test case.
+        """
+        jar_file = '/usr/lib/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar'
+        test_steps = [
+            ('teragen',      "su ubuntu -c 'hadoop jar {} teragen  10000 /user/ubuntu/teragenout'".format(jar_file)),
+            ('mapreduce #1', "su hdfs -c 'hdfs dfs -ls /user/ubuntu/teragenout/_SUCCESS'"),
+            ('terasort',     "su ubuntu -c 'hadoop jar {} terasort /user/ubuntu/teragenout /user/ubuntu/terasortout'".
+                format(jar_file)),
+            ('mapreduce #2', "su hdfs -c 'hdfs dfs -ls /user/ubuntu/terasortout/_SUCCESS'"),
+            ('cleanup',      "su hdfs -c 'hdfs dfs -rm -r /user/ubuntu/teragenout'"),
+        ]
+        for name, step in test_steps:
+            output, retcode = self.hive.run(step)
+            assert retcode == 0, "{} FAILED:\n{}".format(name, output)
+
+    def test_hive(self):
+        self.hive.run("echo 'create table test(col1 int, col2 string); show tables;' > test.sql")
+        output, retcode = self.hive.run("sudo su hive -c 'hive -f test.sql'")
+
+        assert retcode == 0, 'Hive command failed (%s): %s' % (retcode, output)
+        self.assertEqual('test', output)
+
+
+if __name__ == '__main__':
+    unittest.main()

=== added file 'tests/tests.yaml'
--- tests/tests.yaml	1970-01-01 00:00:00 +0000
+++ tests/tests.yaml	2015-10-07 21:29:42 +0000
@@ -0,0 +1,3 @@
+reset: false
+packages:
+  - amulet