Merge lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk into lp:charms/trusty/apache-hadoop-compute-slave
- Trusty Tahr (14.04)
- trunk
- Merge into trunk
Proposed by
Kevin W Monroe
Status: | Merged |
---|---|
Merged at revision: | 88 |
Proposed branch: | lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk |
Merge into: | lp:charms/trusty/apache-hadoop-compute-slave |
Diff against target: |
412 lines (+289/-35) (has conflicts) 9 files modified
README.md (+25/-0) config.yaml (+10/-0) hooks/callbacks.py (+106/-30) hooks/common.py (+23/-3) hooks/ganglia-relation-broken (+26/-0) hooks/ganglia-relation-changed (+26/-0) metadata.yaml (+2/-0) templates/hadoop-metrics2.properties.j2 (+69/-0) tests/01-basic-deployment.py (+2/-2) Text conflict in hooks/callbacks.py Text conflict in hooks/common.py |
To merge this branch: | bzr merge lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Kevin W Monroe | Approve | ||
Review via email: mp+271162@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
Revision history for this message
Kevin W Monroe (kwmonroe) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'README.md' |
2 | --- README.md 2015-08-24 23:09:31 +0000 |
3 | +++ README.md 2015-09-15 17:34:38 +0000 |
4 | @@ -37,6 +37,26 @@ |
5 | juju add-unit compute-slave -n 3 |
6 | |
7 | |
8 | +## Monitoring |
9 | + |
10 | +This charm supports monitoring via Ganglia. To enable monitoring, you must |
11 | +do **both** of the following (the order does not matter): |
12 | + |
13 | + * Add a relation to the [Ganglia charm][] via the `:master` relation |
14 | + * Enable the `ganglia_metrics` config option |
15 | + |
16 | +You must **also** enable metrics on [yarn-master][] and / or [hdfs-master][] |
17 | +to initiate the restart of the NodeManager and / or DataNode components for |
18 | +them to begin collecting metrics. |
19 | + |
20 | +For example: |
21 | + |
22 | + juju add-relation compute-slave ganglia:master |
23 | + juju add-relation yarn-master ganglia:master |
24 | + juju set compute-slave ganglia_metrics=true |
25 | + juju set yarn-master ganglia_metrics=true |
26 | + |
27 | + |
28 | ## Deploying in Network-Restricted Environments |
29 | |
30 | The Apache Hadoop charms can be deployed in environments with limited network |
31 | @@ -85,3 +105,8 @@ |
32 | - [Apache Hadoop bug trackers](http://hadoop.apache.org/issue_tracking.html) |
33 | - [Apache Hadoop mailing lists](http://hadoop.apache.org/mailing_lists.html) |
34 | - [Apache Hadoop Juju Charm](http://jujucharms.com/?text=hadoop) |
35 | + |
36 | + |
37 | +[Ganglia charm]: http://jujucharms.com/ganglia/ |
38 | +[yarn-master]: http://jujucharms.com/apache-hadoop-yarn-master/ |
39 | +[hdfs-master]: http://jujucharms.com/apache-hadoop-hdfs-master/ |
40 | |
41 | === modified file 'config.yaml' |
42 | --- config.yaml 2015-04-03 16:49:16 +0000 |
43 | +++ config.yaml 2015-09-15 17:34:38 +0000 |
44 | @@ -4,3 +4,13 @@ |
45 | default: '' |
46 | description: | |
47 | URL from which to fetch resources (e.g., Hadoop binaries) instead of Launchpad. |
48 | + ganglia_metrics: |
49 | + type: boolean |
50 | + default: false |
51 | + description: | |
52 | + Enable metrics using Ganglia. Note that enabling this option will |
53 | + have no effect if the service is not related to a ganglia service |
54 | + via the ganglia:master relation. Enabling this option also will |
55 | + *not* restart the DataNode nor NodeManager components, so it will |
56 | + also be necessary to enable metrics on one or more of the hdfs-master |
57 | + or yarn-master services. See the README for more information. |
58 | |
59 | === modified file 'hooks/callbacks.py' |
60 | --- hooks/callbacks.py 2015-08-13 21:19:50 +0000 |
61 | +++ hooks/callbacks.py 2015-09-15 17:34:38 +0000 |
62 | @@ -18,7 +18,10 @@ |
63 | |
64 | from charmhelpers.core import hookenv |
65 | from charmhelpers.core import unitdata |
66 | -from jujubigdata.relations import NameNodeMaster, ResourceManagerMaster |
67 | +from jujubigdata.relations import NameNodeMaster, ResourceManagerMaster, Ganglia |
68 | +from charmhelpers.core.templating import render |
69 | +from functools import partial |
70 | +from subprocess import check_call |
71 | |
72 | |
73 | def update_blocked_status(): |
74 | @@ -42,35 +45,108 @@ |
75 | ' and '.join(unready_rels), |
76 | ' and '.join(unready_ress), |
77 | )) |
78 | - elif missing_hosts: |
79 | - hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % ( |
80 | - ' and '.join(missing_hosts), |
81 | - )) |
82 | - |
83 | - |
84 | -def update_working_status(): |
85 | - if unitdata.kv().get('charm.active', False): |
86 | - hookenv.status_set('maintenance', 'Updating configuration') |
87 | - return |
88 | - yarn_connected = ResourceManagerMaster().connected_units() |
89 | - hookenv.status_set('maintenance', 'Setting up DataNode%s' % ( |
90 | - ' and NodeManager' if yarn_connected else '', |
91 | - )) |
92 | +<<<<<<< TREE |
93 | + elif missing_hosts: |
94 | + hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % ( |
95 | + ' and '.join(missing_hosts), |
96 | + )) |
97 | + |
98 | + |
99 | +def update_working_status(): |
100 | + if unitdata.kv().get('charm.active', False): |
101 | + hookenv.status_set('maintenance', 'Updating configuration') |
102 | + return |
103 | + yarn_connected = ResourceManagerMaster().connected_units() |
104 | + hookenv.status_set('maintenance', 'Setting up DataNode%s' % ( |
105 | + ' and NodeManager' if yarn_connected else '', |
106 | + )) |
107 | +======= |
108 | + elif missing_hosts: |
109 | + hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % ( |
110 | + ' and '.join(missing_hosts), |
111 | + )) |
112 | + |
113 | + |
114 | +def update_working_status(): |
115 | + if unitdata.kv().get('charm.active', False): |
116 | + hookenv.status_set('maintenance', 'Updating configuration') |
117 | + return |
118 | + yarn_connected = ResourceManagerMaster().connected_units() |
119 | + hookenv.status_set('maintenance', 'Setting up DataNode%s' % ( |
120 | + ' and NodeManager' if yarn_connected else '', |
121 | + )) |
122 | + |
123 | + |
124 | +def update_working_status(): |
125 | + if unitdata.kv().get('charm.active', False): |
126 | + hookenv.status_set('maintenance', 'Updating configuration') |
127 | + return |
128 | + yarn_connected = ResourceManagerMaster().connected_units() |
129 | + hookenv.status_set('maintenance', 'Setting up DataNode%s' % ( |
130 | + ' and NodeManager' if yarn_connected else '', |
131 | + )) |
132 | +>>>>>>> MERGE-SOURCE |
133 | |
134 | |
135 | def update_active_status(): |
136 | - hdfs_ready = NameNodeMaster().is_ready() |
137 | - yarn_connected = ResourceManagerMaster().connected_units() |
138 | - yarn_ready = ResourceManagerMaster().is_ready() |
139 | - if hdfs_ready and (not yarn_connected or yarn_ready): |
140 | - unitdata.kv().set('charm.active', True) |
141 | - hookenv.status_set('active', 'Ready%s' % ( |
142 | - '' if yarn_ready else ' (HDFS only)' |
143 | - )) |
144 | - else: |
145 | - clear_active_flag() |
146 | - update_blocked_status() |
147 | - |
148 | - |
149 | -def clear_active_flag(): |
150 | - unitdata.kv().set('charm.active', False) |
151 | +<<<<<<< TREE |
152 | + hdfs_ready = NameNodeMaster().is_ready() |
153 | + yarn_connected = ResourceManagerMaster().connected_units() |
154 | + yarn_ready = ResourceManagerMaster().is_ready() |
155 | + if hdfs_ready and (not yarn_connected or yarn_ready): |
156 | + unitdata.kv().set('charm.active', True) |
157 | + hookenv.status_set('active', 'Ready%s' % ( |
158 | + '' if yarn_ready else ' (HDFS only)' |
159 | + )) |
160 | + else: |
161 | + clear_active_flag() |
162 | + update_blocked_status() |
163 | + |
164 | + |
165 | +def clear_active_flag(): |
166 | + unitdata.kv().set('charm.active', False) |
167 | +======= |
168 | + hdfs_ready = NameNodeMaster().is_ready() |
169 | + yarn_connected = ResourceManagerMaster().connected_units() |
170 | + yarn_ready = ResourceManagerMaster().is_ready() |
171 | + if hdfs_ready and (not yarn_connected or yarn_ready): |
172 | + unitdata.kv().set('charm.active', True) |
173 | + hookenv.status_set('active', 'Ready%s' % ( |
174 | + '' if yarn_ready else ' (HDFS only)' |
175 | + )) |
176 | + else: |
177 | + clear_active_flag() |
178 | + update_blocked_status() |
179 | + |
180 | + |
181 | +def clear_active_flag(): |
182 | + unitdata.kv().set('charm.active', False) |
183 | + |
184 | + |
185 | +def conf_ganglia_metrics(purgeConf=False): |
186 | + """ |
187 | + Send hadoop specific metrics to a ganglia server |
188 | + """ |
189 | + config = hookenv.config() |
190 | + ganglia_metrics = config['ganglia_metrics'] and not purgeConf |
191 | + ganglia_metrics_changed = ganglia_metrics != unitdata.kv().get('ganglia_metrics', False) |
192 | + unitdata.kv().set('ganglia_metrics', ganglia_metrics) |
193 | + comment = '#' if not ganglia_metrics else '' |
194 | + ganglia_host = 'UNSET_BY_JUJU' if not ganglia_metrics else Ganglia().host() |
195 | + ganglia_sink_str = comment + '*.sink.ganglia.class=org.apache.hadoop.metrics2.sink.ganglia.GangliaSink31' |
196 | + hookenv.log("Configuring ganglia sink in /etc/hadoop/conf/hadoop-metrics2.properties", level=None) |
197 | + render( |
198 | + source='hadoop-metrics2.properties.j2', |
199 | + target='/etc/hadoop/conf/hadoop-metrics2.properties', |
200 | + context={ |
201 | + 'ganglia_host': ganglia_host, |
202 | + 'ganglia_sink_str': ganglia_sink_str, |
203 | + }, |
204 | + ), |
205 | + if ganglia_metrics_changed: |
206 | + #check_call(['actions/restart-hdfs']) |
207 | + # IMPLEMENT RESTART COMPUTE SLAVE? |
208 | + hookenv.log("please manually restart compute slave hadoop components", level=None) |
209 | + |
210 | +purge_ganglia_metrics = partial(conf_ganglia_metrics, purgeConf=True) |
211 | +>>>>>>> MERGE-SOURCE |
212 | |
213 | === modified file 'hooks/common.py' (properties changed: +x to -x) |
214 | --- hooks/common.py 2015-08-18 15:18:07 +0000 |
215 | +++ hooks/common.py 2015-09-15 17:34:38 +0000 |
216 | @@ -125,9 +125,29 @@ |
217 | charmframework.helpers.close_ports( |
218 | dist_config.exposed_ports('compute-slave-yarn')), |
219 | yarn.stop_nodemanager, |
220 | - callbacks.update_active_status, # might still be active if HDFS-only |
221 | - ], |
222 | - }, |
223 | +<<<<<<< TREE |
224 | + callbacks.update_active_status, # might still be active if HDFS-only |
225 | + ], |
226 | + }, |
227 | +======= |
228 | + callbacks.update_active_status, # might still be active if HDFS-only |
229 | + ], |
230 | + }, |
231 | + { |
232 | + 'name': 'ganglia', |
233 | + 'requires': [ |
234 | + hadoop.is_installed, |
235 | + jujubigdata.relations.Ganglia, |
236 | + ], |
237 | + 'callbacks': [ |
238 | + callbacks.conf_ganglia_metrics, |
239 | + ], |
240 | + 'cleanup': [ |
241 | + callbacks.purge_ganglia_metrics |
242 | + ], |
243 | + }, |
244 | + |
245 | +>>>>>>> MERGE-SOURCE |
246 | ]) |
247 | manager.manage() |
248 | |
249 | |
250 | === added file 'hooks/ganglia-relation-broken' |
251 | --- hooks/ganglia-relation-broken 1970-01-01 00:00:00 +0000 |
252 | +++ hooks/ganglia-relation-broken 2015-09-15 17:34:38 +0000 |
253 | @@ -0,0 +1,26 @@ |
254 | +#!/usr/bin/env python |
255 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
256 | +# you may not use this file except in compliance with the License. |
257 | +# You may obtain a copy of the License at |
258 | +# |
259 | +# http://www.apache.org/licenses/LICENSE-2.0 |
260 | +# |
261 | +# Unless required by applicable law or agreed to in writing, software |
262 | +# distributed under the License is distributed on an "AS IS" BASIS, |
263 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
264 | +# See the License for the specific language governing permissions and |
265 | +# limitations under the License. |
266 | + |
267 | +""" |
268 | +All hooks in this charm are managed by the Charm Framework. |
269 | +The framework helps manage dependencies and preconditions to ensure that |
270 | +steps are only executed when they can be successful. As such, no additional |
271 | +code should be added to this hook; instead, please integrate new functionality |
272 | +into the 'callbacks' list in hooks/common.py. New callbacks can be placed |
273 | +in hooks/callbacks.py, if necessary. |
274 | + |
275 | +See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html |
276 | +for more information. |
277 | +""" |
278 | +import common |
279 | +common.manage() |
280 | |
281 | === added file 'hooks/ganglia-relation-changed' |
282 | --- hooks/ganglia-relation-changed 1970-01-01 00:00:00 +0000 |
283 | +++ hooks/ganglia-relation-changed 2015-09-15 17:34:38 +0000 |
284 | @@ -0,0 +1,26 @@ |
285 | +#!/usr/bin/env python |
286 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
287 | +# you may not use this file except in compliance with the License. |
288 | +# You may obtain a copy of the License at |
289 | +# |
290 | +# http://www.apache.org/licenses/LICENSE-2.0 |
291 | +# |
292 | +# Unless required by applicable law or agreed to in writing, software |
293 | +# distributed under the License is distributed on an "AS IS" BASIS, |
294 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
295 | +# See the License for the specific language governing permissions and |
296 | +# limitations under the License. |
297 | + |
298 | +""" |
299 | +All hooks in this charm are managed by the Charm Framework. |
300 | +The framework helps manage dependencies and preconditions to ensure that |
301 | +steps are only executed when they can be successful. As such, no additional |
302 | +code should be added to this hook; instead, please integrate new functionality |
303 | +into the 'callbacks' list in hooks/common.py. New callbacks can be placed |
304 | +in hooks/callbacks.py, if necessary. |
305 | + |
306 | +See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html |
307 | +for more information. |
308 | +""" |
309 | +import common |
310 | +common.manage() |
311 | |
312 | === modified file 'metadata.yaml' |
313 | --- metadata.yaml 2015-03-06 22:28:48 +0000 |
314 | +++ metadata.yaml 2015-09-15 17:34:38 +0000 |
315 | @@ -12,3 +12,5 @@ |
316 | interface: dfs-slave |
317 | nodemanager: |
318 | interface: mapred-slave |
319 | + ganglia: |
320 | + interface: monitor |
321 | |
322 | === added file 'resources/python/jujuresources-0.2.11.tar.gz' |
323 | Binary files resources/python/jujuresources-0.2.11.tar.gz 1970-01-01 00:00:00 +0000 and resources/python/jujuresources-0.2.11.tar.gz 2015-09-15 17:34:38 +0000 differ |
324 | === added directory 'templates' |
325 | === added file 'templates/hadoop-metrics2.properties.j2' |
326 | --- templates/hadoop-metrics2.properties.j2 1970-01-01 00:00:00 +0000 |
327 | +++ templates/hadoop-metrics2.properties.j2 2015-09-15 17:34:38 +0000 |
328 | @@ -0,0 +1,69 @@ |
329 | +# |
330 | +# Licensed to the Apache Software Foundation (ASF) under one or more |
331 | +# contributor license agreements. See the NOTICE file distributed with |
332 | +# this work for additional information regarding copyright ownership. |
333 | +# The ASF licenses this file to You under the Apache License, Version 2.0 |
334 | +# (the "License"); you may not use this file except in compliance with |
335 | +# the License. You may obtain a copy of the License at |
336 | +# |
337 | +# http://www.apache.org/licenses/LICENSE-2.0 |
338 | +# |
339 | +# Unless required by applicable law or agreed to in writing, software |
340 | +# distributed under the License is distributed on an "AS IS" BASIS, |
341 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
342 | +# See the License for the specific language governing permissions and |
343 | +# limitations under the License. |
344 | +# |
345 | + |
346 | +# syntax: [prefix].[source|sink].[instance].[options] |
347 | +# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details |
348 | + |
349 | +*.sink.file.class=org.apache.hadoop.metrics2.sink.FileSink |
350 | +# default sampling period, in seconds |
351 | +*.period=10 |
352 | + |
353 | +# Defining sink for Ganglia 3.1 |
354 | +{{ ganglia_sink_str }} |
355 | + |
356 | +# Default polling period for GangliaSink |
357 | +*.sink.ganglia.period=10 |
358 | + |
359 | +# default for supportsparse is false |
360 | +*.sink.ganglia.supportsparse=true |
361 | + |
362 | +# Directing output to ganglia servers |
363 | + |
364 | +*.sink.ganglia.slope=jvm.metrics.gcCount=zero,jvm.metrics.memHeapUsedM=both |
365 | +*.sink.ganglia.dmax=jvm.metrics.threadsBlocked=70,jvm.metrics.memHeapUsedM=40 |
366 | + |
367 | +namenode.sink.ganglia.servers={{ ganglia_host }}:8649 |
368 | +datanode.sink.ganglia.servers={{ ganglia_host }}:8649 |
369 | +jobtracker.sink.ganglia.servers={{ ganglia_host }}:8649 |
370 | +tasktracker.sink.ganglia.servers={{ ganglia_host }}:8649 |
371 | +maptask.sink.ganglia.servers={{ ganglia_host }}:8649 |
372 | +reducetask.sink.ganglia.servers={{ ganglia_host }}:8649 |
373 | +resourcemanager.sink.ganglia.servers={{ ganglia_host }}:8649 |
374 | +nodemanager.sink.ganglia.servers={{ ganglia_host }}:8649 |
375 | +historyserver.sink.ganglia.servers={{ ganglia_host }}:8649 |
376 | +journalnode.sink.ganglia.servers={{ ganglia_host }}:8649 |
377 | +resourcemanager.sink.ganglia.tagsForPrefix.yarn=Queue |
378 | + |
379 | +# The namen de-metrics. ut will contain metrics from all context |
380 | +#namenode.sink.file.filename=namenode-metrics.out |
381 | +# Specifying a special sampling period for namenode: |
382 | +#namenode.sink.*.period=8 |
383 | + |
384 | +#datanode.sink.file.filename=datanode-metrics.out |
385 | + |
386 | +# the following example split metrics of different |
387 | +# context to different sinks (in this case files) |
388 | +#jobtracker.sink.file_jvm.context=jvm |
389 | +#jobtracker.sink.file_jvm.filename=jobtracker-jvm-metrics.out |
390 | +#jobtracker.sink.file_mapred.context=mapred |
391 | +#jobtracker.sink.file_mapred.filename=jobtracker-mapred-metrics.out |
392 | + |
393 | +#tasktracker.sink.file.filename=tasktracker-metrics.out |
394 | + |
395 | +#maptask.sink.file.filename=maptask-metrics.out |
396 | + |
397 | +#reducetask.sink.file.filename=reducetask-metrics.out |
398 | |
399 | === modified file 'tests/01-basic-deployment.py' |
400 | --- tests/01-basic-deployment.py 2015-03-04 00:56:45 +0000 |
401 | +++ tests/01-basic-deployment.py 2015-09-15 17:34:38 +0000 |
402 | @@ -16,8 +16,8 @@ |
403 | def setUpClass(cls): |
404 | cls.d = amulet.Deployment(series='trusty') |
405 | cls.d.add('apache-hadoop-compute-slave') |
406 | - cls.d.setup(timeout=9000) |
407 | - cls.d.sentry.wait() |
408 | + cls.d.setup(timeout=900) |
409 | + cls.d.sentry.wait(timeout=1800) |
410 | cls.unit = cls.d.sentry.unit['apache-hadoop-compute-slave/0'] |
411 | |
412 | def test_deploy(self): |