Merge lp:~bigdata-dev/charms/bundles/realtime-syslog-analytics/bundle into lp:~charmers/charms/bundles/realtime-syslog-analytics/bundle

Proposed by Cory Johns on 2015-10-02
Status: Merged
Merged at revision: 7
Proposed branch: lp:~bigdata-dev/charms/bundles/realtime-syslog-analytics/bundle
Merge into: lp:~charmers/charms/bundles/realtime-syslog-analytics/bundle
Diff against target: 185 lines (+163/-1)
3 files modified
bundle.yaml (+1/-1)
tests/00-setup (+8/-0)
tests/01-bundle.py (+154/-0)
To merge this branch: bzr merge lp:~bigdata-dev/charms/bundles/realtime-syslog-analytics/bundle
Reviewer Review Type Date Requested Status
Tim Van Steenburgh 2015-10-02 Approve on 2015-10-03
Review via email: mp+273290@code.launchpad.net

Description of the change

Test fixes for the CWR

To post a comment you must log in.
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'bundle.yaml'
--- bundle.yaml 2015-09-22 03:52:32 +0000
+++ bundle.yaml 2015-10-02 19:59:25 +0000
@@ -31,7 +31,7 @@
31 gui-x: "900"31 gui-x: "900"
32 gui-y: "200"32 gui-y: "200"
33 rsyslog-forwarder:33 rsyslog-forwarder:
34 charm: cs:trusty/rsyslog-forwarder-ha-434 charm: cs:trusty/rsyslog-forwarder-ha-5
35 annotations:35 annotations:
36 gui-x: "900"36 gui-x: "900"
37 gui-y: "600"37 gui-y: "600"
3838
=== added directory 'tests'
=== added file 'tests/00-setup'
--- tests/00-setup 1970-01-01 00:00:00 +0000
+++ tests/00-setup 2015-10-02 19:59:25 +0000
@@ -0,0 +1,8 @@
1#!/bin/bash
2
3if ! python3 -c 'import amulet' &> /dev/null; then
4 echo Installing Amulet...
5 sudo add-apt-repository -y ppa:juju/stable
6 sudo apt-get update
7 sudo apt-get -y install amulet
8fi
09
=== added file 'tests/01-bundle.py'
--- tests/01-bundle.py 1970-01-01 00:00:00 +0000
+++ tests/01-bundle.py 2015-10-02 19:59:25 +0000
@@ -0,0 +1,154 @@
1#!/usr/bin/env python3
2
3import re
4import os
5import unittest
6import textwrap
7
8import yaml
9import amulet
10
11
12class TestBundle(unittest.TestCase):
13 bundle_file = os.path.join(os.path.dirname(__file__), '..', 'bundle.yaml')
14
15 @classmethod
16 def setUpClass(cls):
17 cls.d = amulet.Deployment(series='trusty')
18 with open(cls.bundle_file) as f:
19 bun = f.read()
20 bundle = yaml.safe_load(bun)
21 cls.d.load(bundle)
22 cls.d.setup(timeout=1800)
23 cls.d.sentry.wait_for_messages({
24 'compute-slave': 'Ready',
25 'plugin': 'Ready',
26 'zeppelin': 'Ready',
27 'flume-syslog': 'Ready',
28 }, timeout=1800)
29 cls.hdfs = cls.d.sentry['hdfs-master'][0]
30 cls.yarn = cls.d.sentry['yarn-master'][0]
31 cls.slave = cls.d.sentry['compute-slave'][0]
32 cls.secondary = cls.d.sentry['secondary-namenode'][0]
33 cls.spark = cls.d.sentry['spark'][0]
34 cls.zeppelin = cls.d.sentry['zeppelin'][0]
35
36 def test_components(self):
37 """
38 Confirm that all of the required components are up and running.
39 """
40 hdfs, retcode = self.hdfs.run("pgrep -a java")
41 yarn, retcode = self.yarn.run("pgrep -a java")
42 slave, retcode = self.slave.run("pgrep -a java")
43 secondary, retcode = self.secondary.run("pgrep -a java")
44 spark, retcode = self.spark.run("pgrep -a java")
45
46 # .NameNode needs the . to differentiate it from SecondaryNameNode
47 assert '.NameNode' in hdfs, "NameNode not started"
48 assert '.NameNode' not in yarn, "NameNode should not be running on yarn-master"
49 assert '.NameNode' not in slave, "NameNode should not be running on compute-slave"
50 assert '.NameNode' not in secondary, "NameNode should not be running on secondary-namenode"
51 assert '.NameNode' not in spark, "NameNode should not be running on spark"
52
53 assert 'ResourceManager' in yarn, "ResourceManager not started"
54 assert 'ResourceManager' not in hdfs, "ResourceManager should not be running on hdfs-master"
55 assert 'ResourceManager' not in slave, "ResourceManager should not be running on compute-slave"
56 assert 'ResourceManager' not in secondary, "ResourceManager should not be running on secondary-namenode"
57 assert 'ResourceManager' not in spark, "ResourceManager should not be running on spark"
58
59 assert 'JobHistoryServer' in yarn, "JobHistoryServer not started"
60 assert 'JobHistoryServer' not in hdfs, "JobHistoryServer should not be running on hdfs-master"
61 assert 'JobHistoryServer' not in slave, "JobHistoryServer should not be running on compute-slave"
62 assert 'JobHistoryServer' not in secondary, "JobHistoryServer should not be running on secondary-namenode"
63 assert 'JobHistoryServer' not in spark, "JobHistoryServer should not be running on spark"
64
65 assert 'NodeManager' in slave, "NodeManager not started"
66 assert 'NodeManager' not in yarn, "NodeManager should not be running on yarn-master"
67 assert 'NodeManager' not in hdfs, "NodeManager should not be running on hdfs-master"
68 assert 'NodeManager' not in secondary, "NodeManager should not be running on secondary-namenode"
69 assert 'NodeManager' not in spark, "NodeManager should not be running on spark"
70
71 assert 'DataNode' in slave, "DataServer not started"
72 assert 'DataNode' not in yarn, "DataNode should not be running on yarn-master"
73 assert 'DataNode' not in hdfs, "DataNode should not be running on hdfs-master"
74 assert 'DataNode' not in secondary, "DataNode should not be running on secondary-namenode"
75 assert 'DataNode' not in spark, "DataNode should not be running on spark"
76
77 assert 'SecondaryNameNode' in secondary, "SecondaryNameNode not started"
78 assert 'SecondaryNameNode' not in yarn, "SecondaryNameNode should not be running on yarn-master"
79 assert 'SecondaryNameNode' not in hdfs, "SecondaryNameNode should not be running on hdfs-master"
80 assert 'SecondaryNameNode' not in slave, "SecondaryNameNode should not be running on compute-slave"
81 assert 'SecondaryNameNode' not in spark, "SecondaryNameNode should not be running on spark"
82
83 assert 'spark' in spark, 'Spark should be running on spark'
84 assert 'zeppelin' in spark, 'Zeppelin should be running on spark'
85
86 def test_hdfs_dir(self):
87 """
88 Validate admin few hadoop activities on HDFS cluster.
89 1) This test validates mkdir on hdfs cluster
90 2) This test validates change hdfs dir owner on the cluster
91 3) This test validates setting hdfs directory access permission on the cluster
92
93 NB: These are order-dependent, so must be done as part of a single test case.
94 """
95 output, retcode = self.spark.run("su hdfs -c 'hdfs dfs -mkdir -p /user/ubuntu'")
96 assert retcode == 0, "Created a user directory on hdfs FAILED:\n{}".format(output)
97 output, retcode = self.spark.run("su hdfs -c 'hdfs dfs -chown ubuntu:ubuntu /user/ubuntu'")
98 assert retcode == 0, "Assigning an owner to hdfs directory FAILED:\n{}".format(output)
99 output, retcode = self.spark.run("su hdfs -c 'hdfs dfs -chmod -R 755 /user/ubuntu'")
100 assert retcode == 0, "seting directory permission on hdfs FAILED:\n{}".format(output)
101 output, retcode = self.spark.run("su hdfs -c 'hdfs dfs -rmdir /user/ubuntu'")
102
103 def test_yarn_mapreduce_exe(self):
104 """
105 Validate yarn mapreduce operations:
106 1) validate mapreduce execution - writing to hdfs
107 2) validate successful mapreduce operation after the execution
108 3) validate mapreduce execution - reading and writing to hdfs
109 4) validate successful mapreduce operation after the execution
110 5) validate successful deletion of mapreduce operation result from hdfs
111
112 NB: These are order-dependent, so must be done as part of a single test case.
113 """
114 jar_file = '/usr/lib/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar'
115 test_steps = [
116 ('teragen', "su ubuntu -c 'hadoop jar {} teragen 10000 /user/ubuntu/teragenout'".format(jar_file)),
117 ('mapreduce #1', "su hdfs -c 'hdfs dfs -ls /user/ubuntu/teragenout/_SUCCESS'"),
118 ('terasort', "su ubuntu -c 'hadoop jar {} terasort /user/ubuntu/teragenout /user/ubuntu/terasortout'".
119 format(jar_file)),
120 ('mapreduce #2', "su hdfs -c 'hdfs dfs -ls /user/ubuntu/terasortout/_SUCCESS'"),
121 ('cleanup', "su hdfs -c 'hdfs dfs -rm -r /user/ubuntu/teragenout'"),
122 ]
123 for name, step in test_steps:
124 output, retcode = self.spark.run(step)
125 assert retcode == 0, "{} FAILED:\n{}".format(name, output)
126
127 def test_spark(self):
128 output, retcode = self.spark.run("su ubuntu -c 'bash -lc /home/ubuntu/sparkpi.sh 2>&1'")
129 assert 'Pi is roughly' in output, 'SparkPI test failed: %s' % output
130
131 def test_ingest(self):
132 self.spark.ssh('ls /home/ubuntu') # ensure at least one pure ssh session for the logs
133 for i in amulet.helpers.timeout_gen(60 * 5): # wait for the log messages to be ingested
134 output, retcode = self.spark.run("su hdfs -c 'hdfs dfs -ls /user/flume/flume-syslog/*/*'")
135 if retcode == 0 and 'FlumeData' in output:
136 break
137
138 ssh_count = textwrap.dedent("""
139 from pyspark import SparkContext
140 sc = SparkContext(appName="ssh-count")
141 count = sc.textFile("/user/flume/flume-syslog/*/*").filter(lambda line: "sshd" in line).count()
142 print "SSH Logins: %s" % count
143 """)
144 output, retcode = self.spark.run("cat << EOP > /home/ubuntu/ssh-count.py\n{}\nEOP".format(ssh_count))
145 assert retcode == 0
146 output, retcode = self.spark.run("su ubuntu -c 'spark-submit --master yarn-client /home/ubuntu/ssh-count.py'")
147 assert re.search(r'SSH Logins: [1-9][0-9]*', output), 'ssh-count.py failed: %s' % output
148
149 def test_zeppelin(self):
150 pass # requires javascript; how to test?
151
152
153if __name__ == '__main__':
154 unittest.main()

Subscribers

People subscribed via source and target branches