Merge lp:~bigdata-dev/charms/bundles/realtime-syslog-analytics/bundle into lp:~charmers/charms/bundles/realtime-syslog-analytics/bundle

Proposed by Cory Johns on 2015-10-02
Status: Merged
Merged at revision: 7
Proposed branch: lp:~bigdata-dev/charms/bundles/realtime-syslog-analytics/bundle
Merge into: lp:~charmers/charms/bundles/realtime-syslog-analytics/bundle
Diff against target: 185 lines (+163/-1)
3 files modified
bundle.yaml (+1/-1)
tests/00-setup (+8/-0)
tests/01-bundle.py (+154/-0)
To merge this branch: bzr merge lp:~bigdata-dev/charms/bundles/realtime-syslog-analytics/bundle
Reviewer Review Type Date Requested Status
Tim Van Steenburgh 2015-10-02 Approve on 2015-10-03
Review via email: mp+273290@code.launchpad.net

Description of the change

Test fixes for the CWR

To post a comment you must log in.
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'bundle.yaml'
2--- bundle.yaml 2015-09-22 03:52:32 +0000
3+++ bundle.yaml 2015-10-02 19:59:25 +0000
4@@ -31,7 +31,7 @@
5 gui-x: "900"
6 gui-y: "200"
7 rsyslog-forwarder:
8- charm: cs:trusty/rsyslog-forwarder-ha-4
9+ charm: cs:trusty/rsyslog-forwarder-ha-5
10 annotations:
11 gui-x: "900"
12 gui-y: "600"
13
14=== added directory 'tests'
15=== added file 'tests/00-setup'
16--- tests/00-setup 1970-01-01 00:00:00 +0000
17+++ tests/00-setup 2015-10-02 19:59:25 +0000
18@@ -0,0 +1,8 @@
19+#!/bin/bash
20+
21+if ! python3 -c 'import amulet' &> /dev/null; then
22+ echo Installing Amulet...
23+ sudo add-apt-repository -y ppa:juju/stable
24+ sudo apt-get update
25+ sudo apt-get -y install amulet
26+fi
27
28=== added file 'tests/01-bundle.py'
29--- tests/01-bundle.py 1970-01-01 00:00:00 +0000
30+++ tests/01-bundle.py 2015-10-02 19:59:25 +0000
31@@ -0,0 +1,154 @@
32+#!/usr/bin/env python3
33+
34+import re
35+import os
36+import unittest
37+import textwrap
38+
39+import yaml
40+import amulet
41+
42+
43+class TestBundle(unittest.TestCase):
44+ bundle_file = os.path.join(os.path.dirname(__file__), '..', 'bundle.yaml')
45+
46+ @classmethod
47+ def setUpClass(cls):
48+ cls.d = amulet.Deployment(series='trusty')
49+ with open(cls.bundle_file) as f:
50+ bun = f.read()
51+ bundle = yaml.safe_load(bun)
52+ cls.d.load(bundle)
53+ cls.d.setup(timeout=1800)
54+ cls.d.sentry.wait_for_messages({
55+ 'compute-slave': 'Ready',
56+ 'plugin': 'Ready',
57+ 'zeppelin': 'Ready',
58+ 'flume-syslog': 'Ready',
59+ }, timeout=1800)
60+ cls.hdfs = cls.d.sentry['hdfs-master'][0]
61+ cls.yarn = cls.d.sentry['yarn-master'][0]
62+ cls.slave = cls.d.sentry['compute-slave'][0]
63+ cls.secondary = cls.d.sentry['secondary-namenode'][0]
64+ cls.spark = cls.d.sentry['spark'][0]
65+ cls.zeppelin = cls.d.sentry['zeppelin'][0]
66+
67+ def test_components(self):
68+ """
69+ Confirm that all of the required components are up and running.
70+ """
71+ hdfs, retcode = self.hdfs.run("pgrep -a java")
72+ yarn, retcode = self.yarn.run("pgrep -a java")
73+ slave, retcode = self.slave.run("pgrep -a java")
74+ secondary, retcode = self.secondary.run("pgrep -a java")
75+ spark, retcode = self.spark.run("pgrep -a java")
76+
77+ # .NameNode needs the . to differentiate it from SecondaryNameNode
78+ assert '.NameNode' in hdfs, "NameNode not started"
79+ assert '.NameNode' not in yarn, "NameNode should not be running on yarn-master"
80+ assert '.NameNode' not in slave, "NameNode should not be running on compute-slave"
81+ assert '.NameNode' not in secondary, "NameNode should not be running on secondary-namenode"
82+ assert '.NameNode' not in spark, "NameNode should not be running on spark"
83+
84+ assert 'ResourceManager' in yarn, "ResourceManager not started"
85+ assert 'ResourceManager' not in hdfs, "ResourceManager should not be running on hdfs-master"
86+ assert 'ResourceManager' not in slave, "ResourceManager should not be running on compute-slave"
87+ assert 'ResourceManager' not in secondary, "ResourceManager should not be running on secondary-namenode"
88+ assert 'ResourceManager' not in spark, "ResourceManager should not be running on spark"
89+
90+ assert 'JobHistoryServer' in yarn, "JobHistoryServer not started"
91+ assert 'JobHistoryServer' not in hdfs, "JobHistoryServer should not be running on hdfs-master"
92+ assert 'JobHistoryServer' not in slave, "JobHistoryServer should not be running on compute-slave"
93+ assert 'JobHistoryServer' not in secondary, "JobHistoryServer should not be running on secondary-namenode"
94+ assert 'JobHistoryServer' not in spark, "JobHistoryServer should not be running on spark"
95+
96+ assert 'NodeManager' in slave, "NodeManager not started"
97+ assert 'NodeManager' not in yarn, "NodeManager should not be running on yarn-master"
98+ assert 'NodeManager' not in hdfs, "NodeManager should not be running on hdfs-master"
99+ assert 'NodeManager' not in secondary, "NodeManager should not be running on secondary-namenode"
100+ assert 'NodeManager' not in spark, "NodeManager should not be running on spark"
101+
102+ assert 'DataNode' in slave, "DataServer not started"
103+ assert 'DataNode' not in yarn, "DataNode should not be running on yarn-master"
104+ assert 'DataNode' not in hdfs, "DataNode should not be running on hdfs-master"
105+ assert 'DataNode' not in secondary, "DataNode should not be running on secondary-namenode"
106+ assert 'DataNode' not in spark, "DataNode should not be running on spark"
107+
108+ assert 'SecondaryNameNode' in secondary, "SecondaryNameNode not started"
109+ assert 'SecondaryNameNode' not in yarn, "SecondaryNameNode should not be running on yarn-master"
110+ assert 'SecondaryNameNode' not in hdfs, "SecondaryNameNode should not be running on hdfs-master"
111+ assert 'SecondaryNameNode' not in slave, "SecondaryNameNode should not be running on compute-slave"
112+ assert 'SecondaryNameNode' not in spark, "SecondaryNameNode should not be running on spark"
113+
114+ assert 'spark' in spark, 'Spark should be running on spark'
115+ assert 'zeppelin' in spark, 'Zeppelin should be running on spark'
116+
117+ def test_hdfs_dir(self):
118+ """
119+ Validate admin few hadoop activities on HDFS cluster.
120+ 1) This test validates mkdir on hdfs cluster
121+ 2) This test validates change hdfs dir owner on the cluster
122+ 3) This test validates setting hdfs directory access permission on the cluster
123+
124+ NB: These are order-dependent, so must be done as part of a single test case.
125+ """
126+ output, retcode = self.spark.run("su hdfs -c 'hdfs dfs -mkdir -p /user/ubuntu'")
127+ assert retcode == 0, "Created a user directory on hdfs FAILED:\n{}".format(output)
128+ output, retcode = self.spark.run("su hdfs -c 'hdfs dfs -chown ubuntu:ubuntu /user/ubuntu'")
129+ assert retcode == 0, "Assigning an owner to hdfs directory FAILED:\n{}".format(output)
130+ output, retcode = self.spark.run("su hdfs -c 'hdfs dfs -chmod -R 755 /user/ubuntu'")
131+ assert retcode == 0, "seting directory permission on hdfs FAILED:\n{}".format(output)
132+ output, retcode = self.spark.run("su hdfs -c 'hdfs dfs -rmdir /user/ubuntu'")
133+
134+ def test_yarn_mapreduce_exe(self):
135+ """
136+ Validate yarn mapreduce operations:
137+ 1) validate mapreduce execution - writing to hdfs
138+ 2) validate successful mapreduce operation after the execution
139+ 3) validate mapreduce execution - reading and writing to hdfs
140+ 4) validate successful mapreduce operation after the execution
141+ 5) validate successful deletion of mapreduce operation result from hdfs
142+
143+ NB: These are order-dependent, so must be done as part of a single test case.
144+ """
145+ jar_file = '/usr/lib/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar'
146+ test_steps = [
147+ ('teragen', "su ubuntu -c 'hadoop jar {} teragen 10000 /user/ubuntu/teragenout'".format(jar_file)),
148+ ('mapreduce #1', "su hdfs -c 'hdfs dfs -ls /user/ubuntu/teragenout/_SUCCESS'"),
149+ ('terasort', "su ubuntu -c 'hadoop jar {} terasort /user/ubuntu/teragenout /user/ubuntu/terasortout'".
150+ format(jar_file)),
151+ ('mapreduce #2', "su hdfs -c 'hdfs dfs -ls /user/ubuntu/terasortout/_SUCCESS'"),
152+ ('cleanup', "su hdfs -c 'hdfs dfs -rm -r /user/ubuntu/teragenout'"),
153+ ]
154+ for name, step in test_steps:
155+ output, retcode = self.spark.run(step)
156+ assert retcode == 0, "{} FAILED:\n{}".format(name, output)
157+
158+ def test_spark(self):
159+ output, retcode = self.spark.run("su ubuntu -c 'bash -lc /home/ubuntu/sparkpi.sh 2>&1'")
160+ assert 'Pi is roughly' in output, 'SparkPI test failed: %s' % output
161+
162+ def test_ingest(self):
163+ self.spark.ssh('ls /home/ubuntu') # ensure at least one pure ssh session for the logs
164+ for i in amulet.helpers.timeout_gen(60 * 5): # wait for the log messages to be ingested
165+ output, retcode = self.spark.run("su hdfs -c 'hdfs dfs -ls /user/flume/flume-syslog/*/*'")
166+ if retcode == 0 and 'FlumeData' in output:
167+ break
168+
169+ ssh_count = textwrap.dedent("""
170+ from pyspark import SparkContext
171+ sc = SparkContext(appName="ssh-count")
172+ count = sc.textFile("/user/flume/flume-syslog/*/*").filter(lambda line: "sshd" in line).count()
173+ print "SSH Logins: %s" % count
174+ """)
175+ output, retcode = self.spark.run("cat << EOP > /home/ubuntu/ssh-count.py\n{}\nEOP".format(ssh_count))
176+ assert retcode == 0
177+ output, retcode = self.spark.run("su ubuntu -c 'spark-submit --master yarn-client /home/ubuntu/ssh-count.py'")
178+ assert re.search(r'SSH Logins: [1-9][0-9]*', output), 'ssh-count.py failed: %s' % output
179+
180+ def test_zeppelin(self):
181+ pass # requires javascript; how to test?
182+
183+
184+if __name__ == '__main__':
185+ unittest.main()

Subscribers

People subscribed via source and target branches