Merge lp:~samuel-cozannet/charms/trusty/hdp-storm/config into lp:~bigdata-dev/charms/trusty/hdp-storm/trunk

[Samuel Cozannet]

Updated the storm.yaml file provided with the default Apache configuration file.
Commented everything but the required lines, and added netty settings to make them smoother for AWS

[lp:~samuel-cozannet/charms/trusty/hdp-storm/config]

20. By Samuel Cozannet <email address hidden> on 2015-01-05

replacing default http java libs by 4.3.2 versions

21. By Samuel Cozannet <email address hidden> on 2015-01-08

moving java lib installation to a later process for idempotence

22. By Samuel Cozannet <email address hidden> on 2015-02-26

merging changes for jps

Unmerged revisions

22. By Samuel Cozannet <email address hidden> on 2015-02-26

merging changes for jps

21. By Samuel Cozannet <email address hidden> on 2015-01-08

moving java lib installation to a later process for idempotence

20. By Samuel Cozannet <email address hidden> on 2015-01-05

replacing default http java libs by 4.3.2 versions

19. By Samuel Cozannet <email address hidden> on 2014-12-10

Reusing previous commit (update to default configuration file)

18. By Samuel Cozannet <email address hidden> on 2014-12-10

improving the default configuration file provided with the charm

=== modified file 'files/scripts/storm.yaml'
--- files/scripts/storm.yaml	2014-09-03 04:10:49 +0000
+++ files/scripts/storm.yaml	2015-02-26 13:19:33 +0000
@@ -1,5 +1,202 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+########### These all have default values as shown
+########### Additional configuration goes into storm.yaml
+
+# java.library.path: "/usr/local/lib:/opt/local/lib:/usr/lib"
+
+### storm.* configs are general configurations
+# the local dir is where jars are kept
+storm.local.dir: $STORM_LOCAL_DIR
 storm.zookeeper.servers:
+#storm.zookeeper.port: 2181
+#storm.zookeeper.root: "/storm"
+#storm.zookeeper.session.timeout: 20000
+#storm.zookeeper.connection.timeout: 15000
+#storm.zookeeper.retry.times: 5
+#storm.zookeeper.retry.interval: 1000
+#storm.zookeeper.retry.intervalceiling.millis: 30000
+#storm.zookeeper.auth.user: null
+#storm.zookeeper.auth.password: null
+#storm.cluster.mode: "distributed" # can be distributed or local
+#storm.local.mode.zmq: false
+#storm.thrift.transport: "backtype.storm.security.auth.SimpleTransportPlugin"
+#storm.principal.tolocal: "backtype.storm.security.auth.DefaultPrincipalToLocal"
+#storm.group.mapping.service: "backtype.storm.security.auth.ShellBasedGroupsMapping"
+#storm.messaging.transport: "backtype.storm.messaging.netty.Context"
+#storm.nimbus.retry.times: 5
+#storm.nimbus.retry.interval.millis: 2000
+#storm.nimbus.retry.intervalceiling.millis: 60000
+#storm.auth.simple-white-list.users: []
+#storm.auth.simple-acl.users: []
+#storm.auth.simple-acl.users.commands: []
+#storm.auth.simple-acl.admins: []
+#storm.meta.serialization.delegate: "backtype.storm.serialization.DefaultSerializationDelegate"
+
+### nimbus.* configs are for the master
 nimbus.host:
+#nimbus.thrift.port: 6627
+#nimbus.thrift.threads: 64
+#nimbus.thrift.max_buffer_size: 1048576
+#nimbus.childopts: "-Xmx1024m"
+#nimbus.task.timeout.secs: 30
+#nimbus.supervisor.timeout.secs: 60
+#nimbus.monitor.freq.secs: 10
+#nimbus.cleanup.inbox.freq.secs: 600
+#nimbus.inbox.jar.expiration.secs: 3600
+#nimbus.task.launch.secs: 120
+#nimbus.reassign: true
+#nimbus.file.copy.expiration.secs: 600
+#nimbus.topology.validator: "backtype.storm.nimbus.DefaultTopologyValidator"
+#nimbus.credential.renewers.freq.secs: 600
+
+### ui.* configs are for the master
+#ui.port: 8080
+#ui.childopts: "-Xmx768m"
+#ui.actions.enabled: true
+#ui.filter: null
+#ui.filter.params: null
+#ui.users: null
+#ui.header.buffer.bytes: 4096
+#ui.http.creds.plugin: backtype.storm.security.auth.DefaultHttpCredentialsPlugin
+
+logviewer.port: 8081
+#logviewer.childopts: "-Xmx128m"
+#logviewer.cleanup.age.mins: 10080
+#logviewer.appender.name: "A1"
+
+#logs.users: null
+
 drpc.servers:
-storm.local.dir: $STORM_LOCAL_DIR
-logviewer.port: 8081
+#drpc.port: 3772
+#drpc.worker.threads: 64
+#drpc.max_buffer_size: 1048576
+#drpc.queue.size: 128
+#drpc.invocations.port: 3773
+#drpc.invocations.threads: 64
+#drpc.request.timeout.secs: 600
+#drpc.childopts: "-Xmx768m"
+#drpc.http.port: 3774
+#drpc.https.port: -1
+#drpc.https.keystore.password: ""
+#drpc.https.keystore.type: "JKS"
+#drpc.http.creds.plugin: backtype.storm.security.auth.DefaultHttpCredentialsPlugin
+#drpc.authorizer.acl.filename: "drpc-auth-acl.yaml"
+#drpc.authorizer.acl.strict: false
+
+#transactional.zookeeper.root: "/transactional"
+#transactional.zookeeper.servers: null
+#transactional.zookeeper.port: null
+
+### supervisor.* configs are for node supervisors
+# Define the amount of workers that can be run on this machine. Each worker is assigned a port to use for communication
+#supervisor.slots.ports:
+#    - 6700
+#    - 6701
+#    - 6702
+#    - 6703
+#supervisor.childopts: "-Xmx256m"
+#supervisor.run.worker.as.user: false
+#how long supervisor will wait to ensure that a worker process is started
+#supervisor.worker.start.timeout.secs: 120
+#how long between heartbeats until supervisor considers that worker dead and tries to restart it
+#supervisor.worker.timeout.secs: 30
+#how frequently the supervisor checks on the status of the processes it's monitoring and restarts if necessary
+#supervisor.monitor.frequency.secs: 3
+#how frequently the supervisor heartbeats to the cluster state (for nimbus)
+#supervisor.heartbeat.frequency.secs: 5
+#supervisor.enable: true
+#supervisor.supervisors: []
+#supervisor.supervisors.commands: []
+
+
+### worker.* configs are for task workers
+#worker.childopts: "-Xmx768m"
+#worker.gc.childopts: ""
+#worker.heartbeat.frequency.secs: 1
+
+# control how many worker receiver threads we need per worker
+#topology.worker.receiver.thread.count: 1
+
+#task.heartbeat.frequency.secs: 3
+#task.refresh.poll.secs: 10
+#task.credentials.poll.secs: 30
+
+#zmq.threads: 1
+#zmq.linger.millis: 5000
+#zmq.hwm: 0
+
+
+#storm.messaging.netty.server_worker_threads: 1
+#storm.messaging.netty.client_worker_threads: 1
+#storm.messaging.netty.buffer_size: 5242880 #5MB buffer
+# Since nimbus.task.launch.secs and supervisor.worker.start.timeout.secs are 120, other workers should also wait at least that long before giving up on connecting to the other worker.
+storm.messaging.netty.max_retries: 300
+storm.messaging.netty.max_wait_ms: 2000
+storm.messaging.netty.min_wait_ms: 100
+
+# If the Netty messaging layer is busy(netty internal buffer not writable), the Netty client will try to batch message as more as possible up to the size of storm.messaging.netty.transfer.batch.size bytes, otherwise it will try to flush message as soon as possible to reduce latency.
+#storm.messaging.netty.transfer.batch.size: 262144
+
+# We check with this interval that whether the Netty channel is writable and try to write pending messages if it is.
+#storm.messaging.netty.flush.check.interval.ms: 10
+
+# By default, the Netty SASL authentication is set to false.  Users can override and set it true for a specific topology.
+#storm.messaging.netty.authentication: false
+
+# default number of seconds group mapping service will cache user group
+#storm.group.mapping.service.cache.duration.secs: 120
+
+### topology.* configs are for specific executing storms
+#topology.enable.message.timeouts: true
+#topology.debug: false
+#topology.workers: 1
+#topology.acker.executors: null
+#topology.tasks: null
+# maximum amount of time a message has to complete before it's considered failed
+#topology.message.timeout.secs: 30
+#topology.multilang.serializer: "backtype.storm.multilang.JsonSerializer"
+#topology.skip.missing.kryo.registrations: false
+#topology.max.task.parallelism: null
+#topology.max.spout.pending: null
+#topology.state.synchronization.timeout.secs: 60
+#topology.stats.sample.rate: 0.05
+#topology.builtin.metrics.bucket.size.secs: 60
+#topology.fall.back.on.java.serialization: true
+#topology.worker.childopts: null
+#topology.executor.receive.buffer.size: 1024 #batched
+#topology.executor.send.buffer.size: 1024 #individual messages
+#topology.receiver.buffer.size: 8 # setting it too high causes a lot of problems (heartbeat thread gets starved, throughput plummets)
+#topology.transfer.buffer.size: 1024 # batched
+#topology.tick.tuple.freq.secs: null
+#topology.worker.shared.thread.pool.size: 4
+#topology.disruptor.wait.strategy: "com.lmax.disruptor.BlockingWaitStrategy"
+#topology.spout.wait.strategy: "backtype.storm.spout.SleepSpoutWaitStrategy"
+#topology.sleep.spout.wait.strategy.time.ms: 1
+#topology.error.throttle.interval.secs: 10
+#topology.max.error.report.per.interval: 5
+#topology.kryo.factory: "backtype.storm.serialization.DefaultKryoFactory"
+#topology.tuple.serializer: "backtype.storm.serialization.types.ListDelegateSerializer"
+#topology.trident.batch.emit.interval.millis: 500
+#topology.testing.always.try.serialize: false
+#topology.classpath: null
+#topology.environment: null
+#topology.bolts.outgoing.overflow.buffer.enable: false
+
+#dev.zookeeper.path: "/tmp/dev-storm-zookeeper"
+

=== added directory 'files/usr'
=== added directory 'files/usr/lib'
=== added directory 'files/usr/lib/storm'
=== added directory 'files/usr/lib/storm/lib'
=== added file 'files/usr/lib/storm/lib/httpclient-4.3.2.jar'
Binary files files/usr/lib/storm/lib/httpclient-4.3.2.jar	1970-01-01 00:00:00 +0000 and files/usr/lib/storm/lib/httpclient-4.3.2.jar	2015-02-26 13:19:33 +0000 differ
=== added file 'files/usr/lib/storm/lib/httpcore-4.3.2.jar'
Binary files files/usr/lib/storm/lib/httpcore-4.3.2.jar	1970-01-01 00:00:00 +0000 and files/usr/lib/storm/lib/httpcore-4.3.2.jar	2015-02-26 13:19:33 +0000 differ
=== added directory 'files/usr/lib/zookeeper'
=== added directory 'files/usr/lib/zookeeper/lib'
=== added file 'files/usr/lib/zookeeper/lib/httpclient-4.3.2.jar'
Binary files files/usr/lib/zookeeper/lib/httpclient-4.3.2.jar	1970-01-01 00:00:00 +0000 and files/usr/lib/zookeeper/lib/httpclient-4.3.2.jar	2015-02-26 13:19:33 +0000 differ
=== added file 'files/usr/lib/zookeeper/lib/httpcore-4.3.2.jar'
Binary files files/usr/lib/zookeeper/lib/httpcore-4.3.2.jar	1970-01-01 00:00:00 +0000 and files/usr/lib/zookeeper/lib/httpcore-4.3.2.jar	2015-02-26 13:19:33 +0000 differ
=== modified file 'hooks/bdutils.py'
--- hooks/bdutils.py	2014-09-03 04:10:49 +0000
+++ hooks/bdutils.py	2015-02-26 13:19:33 +0000
@@ -1,4 +1,5 @@
 #!/usr/bin/python
+import re
 import os
 import pwd
 import grp
@@ -133,28 +134,30 @@
                 m = ll[0]+" = "+ll[1].strip().strip(';').strip("\"").strip()
                 #log ("==> {} ".format("\""+m+"\""))
                 os.environ[ll[0]] = ll[1].strip().strip(';').strip("\"").strip()
-                
+
+
+def jps(name):
+    """
+    Get PIDs for named Java processes, for any user.
+    """
+    pat = re.sub(r'^(.)', r'^[^ ]*java .*[\1]', name)
+    try:
+        output = subprocess.check_output(['sudo', 'pgrep', '-f', pat])
+    except subprocess.CalledProcessError:
+        return []
+    return map(int, filter(None, map(str.strip, output.split('\n'))))
+
+
 def is_jvm_service_active(processname):
-    cmd=["jps"]
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-    out, err = p.communicate()
-    if err == None and str(out).find(processname) != -1:
-        return True
-    else:
-        return False
+    return jps(processname) != []
+
 
 def kill_java_process(process):
-    cmd=["jps"]
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE)
-    out, err = p.communicate()
-    cmd = out.split()
-    for i in range(0, len(cmd)):
-        if cmd[i] == process:
-            pid = int(cmd[i-1])
-            os.kill(pid, signal.SIGTERM)
+    for pid in jps(process):
+        os.kill(pid, signal.SIGTERM)
     return 0
 
 def mergeFiles(file1, file2):
     with open(file1, 'a') as f1:
         with open(file2) as f2:
-            f1.write(f2.read())
\ No newline at end of file
+            f1.write(f2.read())

=== modified file 'hooks/common.py'
--- hooks/common.py	2014-11-12 15:51:11 +0000
+++ hooks/common.py	2015-02-26 13:19:33 +0000
@@ -9,3 +9,7 @@
 storm_local_dir = os.path.join(os.path.sep, 'tmp', 'storm', 'local')
 storm_data_dir = os.path.join(os.path.sep, 'home', 'storm')
 hosts_path = os.path.join(os.path.sep, 'etc', 'hosts')
+
+java_libs = [ [ 'httpcore', '4.3.2' ], [ 'httpclient', '4.3.2' ] ]
+java_apps = [ 'storm', 'zookeeper' ]
+

=== modified file 'hooks/install'
--- hooks/install	2014-11-12 15:51:11 +0000
+++ hooks/install	2015-02-26 13:19:33 +0000
@@ -22,7 +22,12 @@
     setDirPermission(com.storm_data_dir, "storm", "storm", 0755)
     copy(com.storm_yaml, com.storm_conf_path)
     fileSetKV(com.storm_yaml_path, "storm.local.dir: ", com.storm_local_dir)
-        
+    for app in com.java_apps:
+        for lib in com.java_libs:
+            target_path = os.path.join('usr', 'lib', app, 'lib')
+            source_path = os.path.join(os.environ['CHARM_DIR'], 'files', target_path, lib[0] + '-' + lib[1] + '.jar' )
+            os.system('rm -f '+ '/' + target_path + '/' + lib[0] + '*.jar')
+            copy(source_path, '/' + target_path)        
     
 hooks = hookenv.Hooks()
 log = hookenv.log

=== modified file 'tests/10-deploy'
--- tests/10-deploy	2014-09-19 22:54:54 +0000
+++ tests/10-deploy	2015-02-26 13:19:33 +0000
@@ -28,7 +28,7 @@
                 getattr(self, test)()
 
     def test_zookeeper_service_status(self):
-        o,c= self.zk_unit.run("jps | awk '{print $2}'")
+        o,c= self.zk_unit.run("pgrep -a java")
         if o.find('QuorumPeerMain') == -1:
             amulet.raise_status(amulet.FAIL, msg="Zookeeper QuorumPeerMain not started")
     
@@ -51,13 +51,13 @@
             if (time.time() - ticks) > 300:
                 amulet.raise_status(amulet.FAIL, msg="nimbus-server is not started")
                 break
-            o,c= self.st_unit.run("jps | awk '{print $2}'")
+            o,c= self.st_unit.run("pgrep -a java")
             if o.find('nimbus') != -1:
                 break
             time.sleep(2)
                      
     def test_storm_slave_service_status(self):
-        o,c= self.st_slave_unit.run("jps | awk '{print $2}'")
+        o,c= self.st_slave_unit.run("pgrep -a java")
         if o.find('supervisor') == -1:
             amulet.raise_status(amulet.FAIL, msg="supervisor is not started")