Merge lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk into lp:charms/trusty/apache-hadoop-compute-slave

Proposed by Kevin W Monroe on 2015-09-15
Status: Merged
Merged at revision: 88
Proposed branch: lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk
Merge into: lp:charms/trusty/apache-hadoop-compute-slave
Diff against target: 412 lines (+289/-35) (has conflicts)
9 files modified
README.md (+25/-0)
config.yaml (+10/-0)
hooks/callbacks.py (+106/-30)
hooks/common.py (+23/-3)
hooks/ganglia-relation-broken (+26/-0)
hooks/ganglia-relation-changed (+26/-0)
metadata.yaml (+2/-0)
templates/hadoop-metrics2.properties.j2 (+69/-0)
tests/01-basic-deployment.py (+2/-2)
Text conflict in hooks/callbacks.py
Text conflict in hooks/common.py
To merge this branch: bzr merge lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk
Reviewer Review Type Date Requested Status
Kevin W Monroe Approve on 2015-09-15
Review via email: mp+271162@code.launchpad.net
To post a comment you must log in.
Kevin W Monroe (kwmonroe) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'README.md'
2--- README.md 2015-08-24 23:09:31 +0000
3+++ README.md 2015-09-15 17:34:38 +0000
4@@ -37,6 +37,26 @@
5 juju add-unit compute-slave -n 3
6
7
8+## Monitoring
9+
10+This charm supports monitoring via Ganglia. To enable monitoring, you must
11+do **both** of the following (the order does not matter):
12+
13+ * Add a relation to the [Ganglia charm][] via the `:master` relation
14+ * Enable the `ganglia_metrics` config option
15+
16+You must **also** enable metrics on [yarn-master][] and / or [hdfs-master][]
17+to initiate the restart of the NodeManager and / or DataNode components for
18+them to begin collecting metrics.
19+
20+For example:
21+
22+ juju add-relation compute-slave ganglia:master
23+ juju add-relation yarn-master ganglia:master
24+ juju set compute-slave ganglia_metrics=true
25+ juju set yarn-master ganglia_metrics=true
26+
27+
28 ## Deploying in Network-Restricted Environments
29
30 The Apache Hadoop charms can be deployed in environments with limited network
31@@ -85,3 +105,8 @@
32 - [Apache Hadoop bug trackers](http://hadoop.apache.org/issue_tracking.html)
33 - [Apache Hadoop mailing lists](http://hadoop.apache.org/mailing_lists.html)
34 - [Apache Hadoop Juju Charm](http://jujucharms.com/?text=hadoop)
35+
36+
37+[Ganglia charm]: http://jujucharms.com/ganglia/
38+[yarn-master]: http://jujucharms.com/apache-hadoop-yarn-master/
39+[hdfs-master]: http://jujucharms.com/apache-hadoop-hdfs-master/
40
41=== modified file 'config.yaml'
42--- config.yaml 2015-04-03 16:49:16 +0000
43+++ config.yaml 2015-09-15 17:34:38 +0000
44@@ -4,3 +4,13 @@
45 default: ''
46 description: |
47 URL from which to fetch resources (e.g., Hadoop binaries) instead of Launchpad.
48+ ganglia_metrics:
49+ type: boolean
50+ default: false
51+ description: |
52+ Enable metrics using Ganglia. Note that enabling this option will
53+ have no effect if the service is not related to a ganglia service
54+ via the ganglia:master relation. Enabling this option also will
55+ *not* restart the DataNode nor NodeManager components, so it will
56+ also be necessary to enable metrics on one or more of the hdfs-master
57+ or yarn-master services. See the README for more information.
58
59=== modified file 'hooks/callbacks.py'
60--- hooks/callbacks.py 2015-08-13 21:19:50 +0000
61+++ hooks/callbacks.py 2015-09-15 17:34:38 +0000
62@@ -18,7 +18,10 @@
63
64 from charmhelpers.core import hookenv
65 from charmhelpers.core import unitdata
66-from jujubigdata.relations import NameNodeMaster, ResourceManagerMaster
67+from jujubigdata.relations import NameNodeMaster, ResourceManagerMaster, Ganglia
68+from charmhelpers.core.templating import render
69+from functools import partial
70+from subprocess import check_call
71
72
73 def update_blocked_status():
74@@ -42,35 +45,108 @@
75 ' and '.join(unready_rels),
76 ' and '.join(unready_ress),
77 ))
78- elif missing_hosts:
79- hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % (
80- ' and '.join(missing_hosts),
81- ))
82-
83-
84-def update_working_status():
85- if unitdata.kv().get('charm.active', False):
86- hookenv.status_set('maintenance', 'Updating configuration')
87- return
88- yarn_connected = ResourceManagerMaster().connected_units()
89- hookenv.status_set('maintenance', 'Setting up DataNode%s' % (
90- ' and NodeManager' if yarn_connected else '',
91- ))
92+<<<<<<< TREE
93+ elif missing_hosts:
94+ hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % (
95+ ' and '.join(missing_hosts),
96+ ))
97+
98+
99+def update_working_status():
100+ if unitdata.kv().get('charm.active', False):
101+ hookenv.status_set('maintenance', 'Updating configuration')
102+ return
103+ yarn_connected = ResourceManagerMaster().connected_units()
104+ hookenv.status_set('maintenance', 'Setting up DataNode%s' % (
105+ ' and NodeManager' if yarn_connected else '',
106+ ))
107+=======
108+ elif missing_hosts:
109+ hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % (
110+ ' and '.join(missing_hosts),
111+ ))
112+
113+
114+def update_working_status():
115+ if unitdata.kv().get('charm.active', False):
116+ hookenv.status_set('maintenance', 'Updating configuration')
117+ return
118+ yarn_connected = ResourceManagerMaster().connected_units()
119+ hookenv.status_set('maintenance', 'Setting up DataNode%s' % (
120+ ' and NodeManager' if yarn_connected else '',
121+ ))
122+
123+
124+def update_working_status():
125+ if unitdata.kv().get('charm.active', False):
126+ hookenv.status_set('maintenance', 'Updating configuration')
127+ return
128+ yarn_connected = ResourceManagerMaster().connected_units()
129+ hookenv.status_set('maintenance', 'Setting up DataNode%s' % (
130+ ' and NodeManager' if yarn_connected else '',
131+ ))
132+>>>>>>> MERGE-SOURCE
133
134
135 def update_active_status():
136- hdfs_ready = NameNodeMaster().is_ready()
137- yarn_connected = ResourceManagerMaster().connected_units()
138- yarn_ready = ResourceManagerMaster().is_ready()
139- if hdfs_ready and (not yarn_connected or yarn_ready):
140- unitdata.kv().set('charm.active', True)
141- hookenv.status_set('active', 'Ready%s' % (
142- '' if yarn_ready else ' (HDFS only)'
143- ))
144- else:
145- clear_active_flag()
146- update_blocked_status()
147-
148-
149-def clear_active_flag():
150- unitdata.kv().set('charm.active', False)
151+<<<<<<< TREE
152+ hdfs_ready = NameNodeMaster().is_ready()
153+ yarn_connected = ResourceManagerMaster().connected_units()
154+ yarn_ready = ResourceManagerMaster().is_ready()
155+ if hdfs_ready and (not yarn_connected or yarn_ready):
156+ unitdata.kv().set('charm.active', True)
157+ hookenv.status_set('active', 'Ready%s' % (
158+ '' if yarn_ready else ' (HDFS only)'
159+ ))
160+ else:
161+ clear_active_flag()
162+ update_blocked_status()
163+
164+
165+def clear_active_flag():
166+ unitdata.kv().set('charm.active', False)
167+=======
168+ hdfs_ready = NameNodeMaster().is_ready()
169+ yarn_connected = ResourceManagerMaster().connected_units()
170+ yarn_ready = ResourceManagerMaster().is_ready()
171+ if hdfs_ready and (not yarn_connected or yarn_ready):
172+ unitdata.kv().set('charm.active', True)
173+ hookenv.status_set('active', 'Ready%s' % (
174+ '' if yarn_ready else ' (HDFS only)'
175+ ))
176+ else:
177+ clear_active_flag()
178+ update_blocked_status()
179+
180+
181+def clear_active_flag():
182+ unitdata.kv().set('charm.active', False)
183+
184+
185+def conf_ganglia_metrics(purgeConf=False):
186+ """
187+ Send hadoop specific metrics to a ganglia server
188+ """
189+ config = hookenv.config()
190+ ganglia_metrics = config['ganglia_metrics'] and not purgeConf
191+ ganglia_metrics_changed = ganglia_metrics != unitdata.kv().get('ganglia_metrics', False)
192+ unitdata.kv().set('ganglia_metrics', ganglia_metrics)
193+ comment = '#' if not ganglia_metrics else ''
194+ ganglia_host = 'UNSET_BY_JUJU' if not ganglia_metrics else Ganglia().host()
195+ ganglia_sink_str = comment + '*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31'
196+ hookenv.log("Configuring ganglia sink in /etc/hadoop/conf/hadoop-metrics2.properties", level=None)
197+ render(
198+ source='hadoop-metrics2.properties.j2',
199+ target='/etc/hadoop/conf/hadoop-metrics2.properties',
200+ context={
201+ 'ganglia_host': ganglia_host,
202+ 'ganglia_sink_str': ganglia_sink_str,
203+ },
204+ ),
205+ if ganglia_metrics_changed:
206+ #check_call(['actions/restart-hdfs'])
207+ # IMPLEMENT RESTART COMPUTE SLAVE?
208+ hookenv.log("please manually restart compute slave hadoop components", level=None)
209+
210+purge_ganglia_metrics = partial(conf_ganglia_metrics, purgeConf=True)
211+>>>>>>> MERGE-SOURCE
212
213=== modified file 'hooks/common.py' (properties changed: +x to -x)
214--- hooks/common.py 2015-08-18 15:18:07 +0000
215+++ hooks/common.py 2015-09-15 17:34:38 +0000
216@@ -125,9 +125,29 @@
217 charmframework.helpers.close_ports(
218 dist_config.exposed_ports('compute-slave-yarn')),
219 yarn.stop_nodemanager,
220- callbacks.update_active_status, # might still be active if HDFS-only
221- ],
222- },
223+<<<<<<< TREE
224+ callbacks.update_active_status, # might still be active if HDFS-only
225+ ],
226+ },
227+=======
228+ callbacks.update_active_status, # might still be active if HDFS-only
229+ ],
230+ },
231+ {
232+ 'name': 'ganglia',
233+ 'requires': [
234+ hadoop.is_installed,
235+ jujubigdata.relations.Ganglia,
236+ ],
237+ 'callbacks': [
238+ callbacks.conf_ganglia_metrics,
239+ ],
240+ 'cleanup': [
241+ callbacks.purge_ganglia_metrics
242+ ],
243+ },
244+
245+>>>>>>> MERGE-SOURCE
246 ])
247 manager.manage()
248
249
250=== added file 'hooks/ganglia-relation-broken'
251--- hooks/ganglia-relation-broken 1970-01-01 00:00:00 +0000
252+++ hooks/ganglia-relation-broken 2015-09-15 17:34:38 +0000
253@@ -0,0 +1,26 @@
254+#!/usr/bin/env python
255+# Licensed under the Apache License, Version 2.0 (the "License");
256+# you may not use this file except in compliance with the License.
257+# You may obtain a copy of the License at
258+#
259+# http://www.apache.org/licenses/LICENSE-2.0
260+#
261+# Unless required by applicable law or agreed to in writing, software
262+# distributed under the License is distributed on an "AS IS" BASIS,
263+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
264+# See the License for the specific language governing permissions and
265+# limitations under the License.
266+
267+"""
268+All hooks in this charm are managed by the Charm Framework.
269+The framework helps manage dependencies and preconditions to ensure that
270+steps are only executed when they can be successful. As such, no additional
271+code should be added to this hook; instead, please integrate new functionality
272+into the 'callbacks' list in hooks/common.py. New callbacks can be placed
273+in hooks/callbacks.py, if necessary.
274+
275+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
276+for more information.
277+"""
278+import common
279+common.manage()
280
281=== added file 'hooks/ganglia-relation-changed'
282--- hooks/ganglia-relation-changed 1970-01-01 00:00:00 +0000
283+++ hooks/ganglia-relation-changed 2015-09-15 17:34:38 +0000
284@@ -0,0 +1,26 @@
285+#!/usr/bin/env python
286+# Licensed under the Apache License, Version 2.0 (the "License");
287+# you may not use this file except in compliance with the License.
288+# You may obtain a copy of the License at
289+#
290+# http://www.apache.org/licenses/LICENSE-2.0
291+#
292+# Unless required by applicable law or agreed to in writing, software
293+# distributed under the License is distributed on an "AS IS" BASIS,
294+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
295+# See the License for the specific language governing permissions and
296+# limitations under the License.
297+
298+"""
299+All hooks in this charm are managed by the Charm Framework.
300+The framework helps manage dependencies and preconditions to ensure that
301+steps are only executed when they can be successful. As such, no additional
302+code should be added to this hook; instead, please integrate new functionality
303+into the 'callbacks' list in hooks/common.py. New callbacks can be placed
304+in hooks/callbacks.py, if necessary.
305+
306+See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html
307+for more information.
308+"""
309+import common
310+common.manage()
311
312=== modified file 'metadata.yaml'
313--- metadata.yaml 2015-03-06 22:28:48 +0000
314+++ metadata.yaml 2015-09-15 17:34:38 +0000
315@@ -12,3 +12,5 @@
316 interface: dfs-slave
317 nodemanager:
318 interface: mapred-slave
319+ ganglia:
320+ interface: monitor
321
322=== added file 'resources/python/jujuresources-0.2.11.tar.gz'
323Binary files resources/python/jujuresources-0.2.11.tar.gz 1970-01-01 00:00:00 +0000 and resources/python/jujuresources-0.2.11.tar.gz 2015-09-15 17:34:38 +0000 differ
324=== added directory 'templates'
325=== added file 'templates/hadoop-metrics2.properties.j2'
326--- templates/hadoop-metrics2.properties.j2 1970-01-01 00:00:00 +0000
327+++ templates/hadoop-metrics2.properties.j2 2015-09-15 17:34:38 +0000
328@@ -0,0 +1,69 @@
329+#
330+# Licensed to the Apache Software Foundation (ASF) under one or more
331+# contributor license agreements. See the NOTICE file distributed with
332+# this work for additional information regarding copyright ownership.
333+# The ASF licenses this file to You under the Apache License, Version 2.0
334+# (the "License"); you may not use this file except in compliance with
335+# the License. You may obtain a copy of the License at
336+#
337+# http://www.apache.org/licenses/LICENSE-2.0
338+#
339+# Unless required by applicable law or agreed to in writing, software
340+# distributed under the License is distributed on an "AS IS" BASIS,
341+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
342+# See the License for the specific language governing permissions and
343+# limitations under the License.
344+#
345+
346+# syntax: [prefix].[source|sink].[instance].[options]
347+# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details
348+
349+*.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink
350+# default sampling period, in seconds
351+*.period=10
352+
353+# Defining sink for Ganglia 3.1
354+{{ ganglia_sink_str }}
355+
356+# Default polling period for GangliaSink
357+*.sink.ganglia.period=10
358+
359+# default for supportsparse is false
360+*.sink.ganglia.supportsparse=true
361+
362+# Directing output to ganglia servers
363+
364+*.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both
365+*.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40
366+
367+namenode.sink.ganglia.servers={{ ganglia_host }}:8649
368+datanode.sink.ganglia.servers={{ ganglia_host }}:8649
369+jobtracker.sink.ganglia.servers={{ ganglia_host }}:8649
370+tasktracker.sink.ganglia.servers={{ ganglia_host }}:8649
371+maptask.sink.ganglia.servers={{ ganglia_host }}:8649
372+reducetask.sink.ganglia.servers={{ ganglia_host }}:8649
373+resourcemanager.sink.ganglia.servers={{ ganglia_host }}:8649
374+nodemanager.sink.ganglia.servers={{ ganglia_host }}:8649
375+historyserver.sink.ganglia.servers={{ ganglia_host }}:8649
376+journalnode.sink.ganglia.servers={{ ganglia_host }}:8649
377+resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue
378+
379+# The namen de-metrics. ut will contain metrics from all context
380+#namenode.sink.file.filename=namenode-metrics.out
381+# Specifying a special sampling period for namenode:
382+#namenode.sink.*.period=8
383+
384+#datanode.sink.file.filename=datanode-metrics.out
385+
386+# the following example split metrics of different
387+# context to different sinks (in this case files)
388+#jobtracker.sink.file_jvm.context=jvm
389+#jobtracker.sink.file_jvm.filename=jobtracker-jvm-metrics.out
390+#jobtracker.sink.file_mapred.context=mapred
391+#jobtracker.sink.file_mapred.filename=jobtracker-mapred-metrics.out
392+
393+#tasktracker.sink.file.filename=tasktracker-metrics.out
394+
395+#maptask.sink.file.filename=maptask-metrics.out
396+
397+#reducetask.sink.file.filename=reducetask-metrics.out
398
399=== modified file 'tests/01-basic-deployment.py'
400--- tests/01-basic-deployment.py 2015-03-04 00:56:45 +0000
401+++ tests/01-basic-deployment.py 2015-09-15 17:34:38 +0000
402@@ -16,8 +16,8 @@
403 def setUpClass(cls):
404 cls.d = amulet.Deployment(series='trusty')
405 cls.d.add('apache-hadoop-compute-slave')
406- cls.d.setup(timeout=9000)
407- cls.d.sentry.wait()
408+ cls.d.setup(timeout=900)
409+ cls.d.sentry.wait(timeout=1800)
410 cls.unit = cls.d.sentry.unit['apache-hadoop-compute-slave/0']
411
412 def test_deploy(self):

Subscribers

People subscribed via source and target branches