Merge lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/datanode-blocking into lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk

Proposed by Cory Johns on 2015-08-10
Status: Merged
Merged at revision: 97
Proposed branch: lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/datanode-blocking
Merge into: lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/trunk
Diff against target: 187 lines (+91/-22)
4 files modified
hooks/callbacks.py (+28/-12)
hooks/common.py (+31/-10)
hooks/datanode-relation-departed (+16/-0)
hooks/nodemanager-relation-departed (+16/-0)
To merge this branch: bzr merge lp:~bigdata-dev/charms/trusty/apache-hadoop-compute-slave/datanode-blocking
Reviewer Review Type Date Requested Status
Kevin W Monroe 2015-08-10 Approve on 2015-08-12
Review via email: mp+267597@code.launchpad.net

Description of the change

Fixed datanode component being blocked by nodemanager component, and thus blocking hdfs-master on yarn-master
Fixed status reporting not being accurate when relations were removed

To post a comment you must log in.
Kevin W Monroe (kwmonroe) wrote :

My head hurts with your status wizardry. I'm pretty sure it LGTM :)

review: Approve
98. By Cory Johns on 2015-08-13

Improved status reporting for HDFS-only configuration

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
=== modified file 'hooks/callbacks.py'
--- hooks/callbacks.py 2015-06-24 22:12:57 +0000
+++ hooks/callbacks.py 2015-08-13 21:20:20 +0000
@@ -24,37 +24,53 @@
24def update_blocked_status():24def update_blocked_status():
25 if unitdata.kv().get('charm.active', False):25 if unitdata.kv().get('charm.active', False):
26 return26 return
27 rels = (27 rels = [
28 ('Yarn', 'ResourceManager', ResourceManagerMaster()),
29 ('HDFS', 'NameNode', NameNodeMaster()),28 ('HDFS', 'NameNode', NameNodeMaster()),
30 )29 ]
31 missing_rel = [rel for rel, res, impl in rels if not impl.connected_units()]30 missing_rel = [rel for rel, res, impl in rels if not impl.connected_units()]
32 missing_hosts = [rel for rel, res, impl in rels if not impl.am_i_registered()]31 rels.append(('Yarn', 'ResourceManager', ResourceManagerMaster()))
33 not_ready = [(rel, res) for rel, res, impl in rels if not impl.is_ready()]32 not_ready = [(rel, res) for rel, res, impl in rels if impl.connected_units() and not impl.is_ready()]
33 missing_hosts = [rel for rel, res, impl in rels if impl.connected_units() and not impl.am_i_registered()]
34 if missing_rel:34 if missing_rel:
35 hookenv.status_set('blocked', 'Waiting for relation to %s master%s' % (35 hookenv.status_set('blocked', 'Waiting for relation to %s master%s' % (
36 ' and '.join(missing_rel),36 ' and '.join(missing_rel),
37 's' if len(missing_rel) > 1 else '',37 's' if len(missing_rel) > 1 else '',
38 )),38 )),
39 elif missing_hosts:
40 hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % (
41 ' and '.join(missing_hosts),
42 ))
43 elif not_ready:39 elif not_ready:
44 unready_rels, unready_ress = zip(*not_ready)40 unready_rels, unready_ress = zip(*not_ready)
45 hookenv.status_set('waiting', 'Waiting for %s to provide %s' % (41 hookenv.status_set('waiting', 'Waiting for %s to provide %s' % (
46 ' and '.join(unready_rels),42 ' and '.join(unready_rels),
47 ' and '.join(unready_ress),43 ' and '.join(unready_ress),
48 ))44 ))
45 elif missing_hosts:
46 hookenv.status_set('waiting', 'Waiting for /etc/hosts registration on %s' % (
47 ' and '.join(missing_hosts),
48 ))
4949
5050
51def update_working_status():51def update_working_status():
52 if unitdata.kv().get('charm.active', False):52 if unitdata.kv().get('charm.active', False):
53 hookenv.status_set('maintenance', 'Updating configuration')53 hookenv.status_set('maintenance', 'Updating configuration')
54 return54 return
55 hookenv.status_set('maintenance', 'Setting up NodeManager and DataNode')55 yarn_connected = ResourceManagerMaster().connected_units()
56 hookenv.status_set('maintenance', 'Setting up DataNode%s' % (
57 ' and NodeManager' if yarn_connected else '',
58 ))
5659
5760
58def update_active_status():61def update_active_status():
59 unitdata.kv().set('charm.active', True)62 hdfs_ready = NameNodeMaster().is_ready()
60 hookenv.status_set('active', 'Ready')63 yarn_connected = ResourceManagerMaster().connected_units()
64 yarn_ready = ResourceManagerMaster().is_ready()
65 if hdfs_ready and (not yarn_connected or yarn_ready):
66 unitdata.kv().set('charm.active', True)
67 hookenv.status_set('active', 'Ready%s' % (
68 '' if yarn_ready else ' (HDFS only)'
69 ))
70 else:
71 clear_active_flag()
72 update_blocked_status()
73
74
75def clear_active_flag():
76 unitdata.kv().set('charm.active', False)
6177
=== modified file 'hooks/common.py'
--- hooks/common.py 2015-06-24 22:12:57 +0000
+++ hooks/common.py 2015-08-13 21:20:20 +0000
@@ -71,40 +71,61 @@
71 ],71 ],
72 },72 },
73 {73 {
74 'name': 'compute-slave',74 'name': 'datanode',
75 'provides': [75 'provides': [
76 jujubigdata.relations.DataNode(),76 jujubigdata.relations.DataNode(),
77 ],
78 'requires': [
79 hadoop.is_installed,
80 hdfs_relation,
81 hdfs_relation.am_i_registered,
82 ],
83 'callbacks': [
84 callbacks.update_working_status,
85 hdfs_relation.register_provided_hosts,
86 jujubigdata.utils.manage_etc_hosts,
87 hdfs_relation.install_ssh_keys,
88 hdfs.configure_datanode,
89 hdfs.start_datanode,
90 charmframework.helpers.open_ports(
91 dist_config.exposed_ports('compute-slave-hdfs')),
92 callbacks.update_active_status,
93 ],
94 'cleanup': [
95 callbacks.clear_active_flag,
96 charmframework.helpers.close_ports(
97 dist_config.exposed_ports('compute-slave-hdfs')),
98 hdfs.stop_datanode,
99 callbacks.update_blocked_status,
100 ],
101 },
102 {
103 'name': 'nodemanager',
104 'provides': [
77 jujubigdata.relations.NodeManager(),105 jujubigdata.relations.NodeManager(),
78 ],106 ],
79 'requires': [107 'requires': [
80 hadoop.is_installed,108 hadoop.is_installed,
81 hdfs_relation,
82 yarn_relation,109 yarn_relation,
83 hdfs_relation.am_i_registered,
84 yarn_relation.am_i_registered,110 yarn_relation.am_i_registered,
85 ],111 ],
86 'callbacks': [112 'callbacks': [
87 callbacks.update_working_status,113 callbacks.update_working_status,
88 hdfs_relation.register_provided_hosts,
89 yarn_relation.register_provided_hosts,114 yarn_relation.register_provided_hosts,
90 jujubigdata.utils.manage_etc_hosts,115 jujubigdata.utils.manage_etc_hosts,
91 hdfs_relation.install_ssh_keys,
92 yarn_relation.install_ssh_keys,116 yarn_relation.install_ssh_keys,
93 hdfs.configure_datanode,
94 yarn.configure_nodemanager,117 yarn.configure_nodemanager,
95 hdfs.start_datanode,
96 yarn.start_nodemanager,118 yarn.start_nodemanager,
97 charmframework.helpers.open_ports(119 charmframework.helpers.open_ports(
98 dist_config.exposed_ports('compute-slave-hdfs') +
99 dist_config.exposed_ports('compute-slave-yarn')),120 dist_config.exposed_ports('compute-slave-yarn')),
100 callbacks.update_active_status,121 callbacks.update_active_status,
101 ],122 ],
102 'cleanup': [123 'cleanup': [
124 callbacks.clear_active_flag,
103 charmframework.helpers.close_ports(125 charmframework.helpers.close_ports(
104 dist_config.exposed_ports('compute-slave-hdfs') +
105 dist_config.exposed_ports('compute-slave-yarn')),126 dist_config.exposed_ports('compute-slave-yarn')),
106 hdfs.stop_datanode,
107 yarn.stop_nodemanager,127 yarn.stop_nodemanager,
128 callbacks.update_active_status, # might still be active w/ for HDFS-only
108 ],129 ],
109 },130 },
110 ])131 ])
111132
=== added file 'hooks/datanode-relation-departed'
--- hooks/datanode-relation-departed 1970-01-01 00:00:00 +0000
+++ hooks/datanode-relation-departed 2015-08-13 21:20:20 +0000
@@ -0,0 +1,16 @@
1#!/usr/bin/env python
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7#
8# Unless required by applicable law or agreed to in writing, software
9# distributed under the License is distributed on an "AS IS" BASIS,
10# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11# See the License for the specific language governing permissions and
12# limitations under the License.
13
14import common
15
16common.manage()
017
=== added file 'hooks/nodemanager-relation-departed'
--- hooks/nodemanager-relation-departed 1970-01-01 00:00:00 +0000
+++ hooks/nodemanager-relation-departed 2015-08-13 21:20:20 +0000
@@ -0,0 +1,16 @@
1#!/usr/bin/env python
2# Licensed under the Apache License, Version 2.0 (the "License");
3# you may not use this file except in compliance with the License.
4# You may obtain a copy of the License at
5#
6# http://www.apache.org/licenses/LICENSE-2.0
7#
8# Unless required by applicable law or agreed to in writing, software
9# distributed under the License is distributed on an "AS IS" BASIS,
10# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11# See the License for the specific language governing permissions and
12# limitations under the License.
13
14import common
15
16common.manage()

Subscribers

People subscribed via source and target branches

to all changes: