Merge lp:~bigdata-dev/charms/trusty/apache-hadoop-hdfs-master/trunk into lp:charms/trusty/apache-hadoop-hdfs-master
- Trusty Tahr (14.04)
- trunk
- Merge into trunk
Proposed by
amir sanjar
Status: | Merged |
---|---|
Merged at revision: | 85 |
Proposed branch: | lp:~bigdata-dev/charms/trusty/apache-hadoop-hdfs-master/trunk |
Merge into: | lp:charms/trusty/apache-hadoop-hdfs-master |
Diff against target: |
383 lines (+155/-38) 15 files modified
actions.yaml (+6/-0) actions/restart-hdfs (+5/-0) actions/start-hdfs (+4/-0) actions/stop-hdfs (+4/-0) hooks/callbacks.py (+22/-0) hooks/common.py (+26/-17) hooks/config-changed (+11/-0) hooks/datanode-relation-changed (+11/-0) hooks/install (+11/-0) hooks/namenode-relation-changed (+11/-0) hooks/secondary-relation-changed (+11/-0) hooks/setup.py (+8/-10) hooks/start (+11/-0) hooks/stop (+11/-0) resources.yaml (+3/-11) |
To merge this branch: | bzr merge lp:~bigdata-dev/charms/trusty/apache-hadoop-hdfs-master/trunk |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Kevin W Monroe | Approve | ||
Review via email: mp+262764@code.launchpad.net |
Commit message
Description of the change
adding hdfs start, stop, and restart Juju actions to the charm
To post a comment you must log in.
- 87. By Cory Johns
-
Added extended status support
- 88. By Cory Johns
-
Register clients as well as slaves to ensure all nodes can communicate
- 89. By Kevin W Monroe
-
update to latest bigdata-data java-installer.sh
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'README.md' |
2 | === added directory 'actions' |
3 | === added file 'actions.yaml' |
4 | --- actions.yaml 1970-01-01 00:00:00 +0000 |
5 | +++ actions.yaml 2015-06-25 22:42:45 +0000 |
6 | @@ -0,0 +1,6 @@ |
7 | +start-hdfs: |
8 | + description: All of the HDFS processes can be started with this Juju action. |
9 | +stop-hdfs: |
10 | + description: All of the HDFS processes can be stopped with this Juju action. |
11 | +restart-hdfs: |
12 | + description: All of the HDFS processes can be restarted with this Juju action. |
13 | |
14 | === added file 'actions/restart-hdfs' |
15 | --- actions/restart-hdfs 1970-01-01 00:00:00 +0000 |
16 | +++ actions/restart-hdfs 2015-06-25 22:42:45 +0000 |
17 | @@ -0,0 +1,5 @@ |
18 | +#!/bin/bash |
19 | +. /etc/environment |
20 | +cd ${HADOOP_HOME}/sbin |
21 | +su hdfs -c "./stop-dfs.sh" |
22 | +su hdfs -c "./start-dfs.sh" |
23 | |
24 | === added file 'actions/start-hdfs' |
25 | --- actions/start-hdfs 1970-01-01 00:00:00 +0000 |
26 | +++ actions/start-hdfs 2015-06-25 22:42:45 +0000 |
27 | @@ -0,0 +1,4 @@ |
28 | +#!/bin/bash |
29 | +. /etc/environment |
30 | +cd ${HADOOP_HOME}/sbin |
31 | +su hdfs -c "./start-dfs.sh" |
32 | |
33 | === added file 'actions/stop-hdfs' |
34 | --- actions/stop-hdfs 1970-01-01 00:00:00 +0000 |
35 | +++ actions/stop-hdfs 2015-06-25 22:42:45 +0000 |
36 | @@ -0,0 +1,4 @@ |
37 | +#!/bin/bash |
38 | +. /etc/environment |
39 | +cd ${HADOOP_HOME}/sbin |
40 | +su hdfs -c "./stop-dfs.sh" |
41 | |
42 | === modified file 'hooks/callbacks.py' |
43 | --- hooks/callbacks.py 2015-05-12 21:56:03 +0000 |
44 | +++ hooks/callbacks.py 2015-06-25 22:42:45 +0000 |
45 | @@ -15,3 +15,25 @@ |
46 | Add any additional tasks / setup here. If a callback is used by mutliple |
47 | charms, consider refactoring it up to the jujubigdata library. |
48 | """ |
49 | + |
50 | +from charmhelpers.core import hookenv |
51 | +from charmhelpers.core import unitdata |
52 | +from jujubigdata.relations import DataNode |
53 | + |
54 | + |
55 | +def update_working_status(): |
56 | + if unitdata.kv().get('charm.active', False): |
57 | + hookenv.status_set('maintenance', 'Updating configuration') |
58 | + return |
59 | + hookenv.status_set('maintenance', 'Setting up HDFS master') |
60 | + |
61 | + |
62 | +def update_active_status(): |
63 | + datanode = DataNode() |
64 | + if datanode.is_ready(): |
65 | + hookenv.status_set('active', 'Ready (%s DataNodes)' % len(datanode.filtered_data())) |
66 | + unitdata.kv().set('charm.active', True) |
67 | + elif datanode.connected_units(): |
68 | + hookenv.status_set('waiting', 'Waiting for compute slaves to provide DataNodes') |
69 | + else: |
70 | + hookenv.status_set('blocked', 'Waiting for relation to compute slaves') |
71 | |
72 | === modified file 'hooks/common.py' |
73 | --- hooks/common.py 2015-05-18 20:50:53 +0000 |
74 | +++ hooks/common.py 2015-06-25 22:42:45 +0000 |
75 | @@ -12,37 +12,39 @@ |
76 | # limitations under the License. |
77 | |
78 | import jujuresources |
79 | - |
80 | - |
81 | -def status_set(status, message): |
82 | - import subprocess |
83 | - try: |
84 | - subprocess.check_call(['status-set', status, message]) |
85 | - except (subprocess.CalledProcessError, OSError): |
86 | - jujuresources.juju_log('status: {}: {}'.format(status, message), 'ERROR') |
87 | +from charmhelpers.core import hookenv |
88 | +from charmhelpers.core import unitdata |
89 | +from charmhelpers.core import charmframework |
90 | |
91 | |
92 | def bootstrap_resources(): |
93 | """ |
94 | Install required resources defined in resources.yaml |
95 | """ |
96 | + if unitdata.kv().get('charm.bootstrapped', False): |
97 | + return True |
98 | + hookenv.status_set('maintenance', 'Installing base resources') |
99 | mirror_url = jujuresources.config_get('resources_mirror') |
100 | if not jujuresources.fetch(mirror_url=mirror_url): |
101 | - status_set('blocked', 'Resources unavailable; manual intervention required') |
102 | + missing = jujuresources.invalid() |
103 | + hookenv.status_set('blocked', 'Unable to fetch required resource%s: %s' % ( |
104 | + 's' if len(missing) > 1 else '', |
105 | + ', '.join(missing), |
106 | + )) |
107 | return False |
108 | - jujuresources.install(['pathlib', 'pyaml', 'six', 'charmhelpers', 'jujubigdata']) |
109 | + jujuresources.install(['pathlib', 'jujubigdata']) |
110 | + unitdata.kv().set('charm.bootstrapped', True) |
111 | return True |
112 | |
113 | |
114 | def manage(): |
115 | if not bootstrap_resources(): |
116 | - # defer until resources are available, since charmhelpers, and thus |
117 | - # the framework, (will be) managed by jujuresources |
118 | + # defer until resources are available, since jujubigdata, and thus the |
119 | + # classes needed for the requires blocks, (will be) managed by jujuresources |
120 | return |
121 | |
122 | - from charmhelpers.core import charmframework |
123 | import jujubigdata |
124 | - import callbacks # noqa (ignore when linting) |
125 | + import callbacks |
126 | |
127 | # list of keys required to be in the dist.yaml |
128 | master_reqs = ['vendor', 'hadoop_version', 'packages', 'groups', 'users', |
129 | @@ -80,15 +82,22 @@ |
130 | jujubigdata.relations.SecondaryNameNode(optional=True), |
131 | ], |
132 | 'callbacks': [ |
133 | - datanodes.register_host_ips, |
134 | - secondary.register_host_ips, |
135 | - jujubigdata.utils.update_etc_hosts_from_kv, |
136 | + # These callbacks will be executed once the Hadoop base packages |
137 | + # are installed. New items can be added to the end of this list |
138 | + # and to hooks/callbacks.py to extend the functionality of this |
139 | + # charm. |
140 | + callbacks.update_working_status, |
141 | + datanodes.register_connected_hosts, |
142 | + secondary.register_connected_hosts, |
143 | + clients.register_connected_hosts, |
144 | + jujubigdata.utils.manage_etc_hosts, |
145 | hdfs.configure_namenode, |
146 | hdfs.register_slaves, |
147 | hdfs.format_namenode, |
148 | hdfs.start_namenode, |
149 | hdfs.create_hdfs_dirs, |
150 | charmframework.helpers.open_ports(dist_config.exposed_ports('hdfs-master')), |
151 | + callbacks.update_active_status, |
152 | ], |
153 | 'cleanup': [ |
154 | charmframework.helpers.close_ports(dist_config.exposed_ports('hdfs-master')), |
155 | |
156 | === modified file 'hooks/config-changed' |
157 | --- hooks/config-changed 2015-02-09 18:19:59 +0000 |
158 | +++ hooks/config-changed 2015-06-25 22:42:45 +0000 |
159 | @@ -11,5 +11,16 @@ |
160 | # See the License for the specific language governing permissions and |
161 | # limitations under the License. |
162 | |
163 | +""" |
164 | +All hooks in this charm are managed by the Charm Framework. |
165 | +The framework helps manage dependencies and preconditions to ensure that |
166 | +steps are only executed when they can be successful. As such, no additional |
167 | +code should be added to this hook; instead, please integrate new functionality |
168 | +into the 'callbacks' list in hooks/common.py. New callbacks can be placed |
169 | +in hooks/callbacks.py, if necessary. |
170 | + |
171 | +See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html |
172 | +for more information. |
173 | +""" |
174 | import common |
175 | common.manage() |
176 | |
177 | === modified file 'hooks/datanode-relation-changed' |
178 | --- hooks/datanode-relation-changed 2015-05-12 21:56:03 +0000 |
179 | +++ hooks/datanode-relation-changed 2015-06-25 22:42:45 +0000 |
180 | @@ -11,5 +11,16 @@ |
181 | # See the License for the specific language governing permissions and |
182 | # limitations under the License. |
183 | |
184 | +""" |
185 | +All hooks in this charm are managed by the Charm Framework. |
186 | +The framework helps manage dependencies and preconditions to ensure that |
187 | +steps are only executed when they can be successful. As such, no additional |
188 | +code should be added to this hook; instead, please integrate new functionality |
189 | +into the 'callbacks' list in hooks/common.py. New callbacks can be placed |
190 | +in hooks/callbacks.py, if necessary. |
191 | + |
192 | +See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html |
193 | +for more information. |
194 | +""" |
195 | import common |
196 | common.manage() |
197 | |
198 | === modified file 'hooks/install' |
199 | --- hooks/install 2015-02-09 18:19:59 +0000 |
200 | +++ hooks/install 2015-06-25 22:42:45 +0000 |
201 | @@ -13,5 +13,16 @@ |
202 | import setup |
203 | setup.pre_install() |
204 | |
205 | +""" |
206 | +All hooks in this charm are managed by the Charm Framework. |
207 | +The framework helps manage dependencies and preconditions to ensure that |
208 | +steps are only executed when they can be successful. As such, no additional |
209 | +code should be added to this hook; instead, please integrate new functionality |
210 | +into the 'callbacks' list in hooks/common.py. New callbacks can be placed |
211 | +in hooks/callbacks.py, if necessary. |
212 | + |
213 | +See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html |
214 | +for more information. |
215 | +""" |
216 | import common |
217 | common.manage() |
218 | |
219 | === modified file 'hooks/namenode-relation-changed' |
220 | --- hooks/namenode-relation-changed 2015-05-12 21:56:03 +0000 |
221 | +++ hooks/namenode-relation-changed 2015-06-25 22:42:45 +0000 |
222 | @@ -11,5 +11,16 @@ |
223 | # See the License for the specific language governing permissions and |
224 | # limitations under the License. |
225 | |
226 | +""" |
227 | +All hooks in this charm are managed by the Charm Framework. |
228 | +The framework helps manage dependencies and preconditions to ensure that |
229 | +steps are only executed when they can be successful. As such, no additional |
230 | +code should be added to this hook; instead, please integrate new functionality |
231 | +into the 'callbacks' list in hooks/common.py. New callbacks can be placed |
232 | +in hooks/callbacks.py, if necessary. |
233 | + |
234 | +See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html |
235 | +for more information. |
236 | +""" |
237 | import common |
238 | common.manage() |
239 | |
240 | === modified file 'hooks/secondary-relation-changed' |
241 | --- hooks/secondary-relation-changed 2015-05-12 21:56:03 +0000 |
242 | +++ hooks/secondary-relation-changed 2015-06-25 22:42:45 +0000 |
243 | @@ -11,5 +11,16 @@ |
244 | # See the License for the specific language governing permissions and |
245 | # limitations under the License. |
246 | |
247 | +""" |
248 | +All hooks in this charm are managed by the Charm Framework. |
249 | +The framework helps manage dependencies and preconditions to ensure that |
250 | +steps are only executed when they can be successful. As such, no additional |
251 | +code should be added to this hook; instead, please integrate new functionality |
252 | +into the 'callbacks' list in hooks/common.py. New callbacks can be placed |
253 | +in hooks/callbacks.py, if necessary. |
254 | + |
255 | +See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html |
256 | +for more information. |
257 | +""" |
258 | import common |
259 | common.manage() |
260 | |
261 | === modified file 'hooks/setup.py' |
262 | --- hooks/setup.py 2015-03-03 21:30:47 +0000 |
263 | +++ hooks/setup.py 2015-06-25 22:42:45 +0000 |
264 | @@ -12,24 +12,22 @@ |
265 | import subprocess |
266 | from glob import glob |
267 | |
268 | + |
269 | def pre_install(): |
270 | """ |
271 | Do any setup required before the install hook. |
272 | """ |
273 | install_pip() |
274 | - install_jujuresources() |
275 | + install_bundled_resources() |
276 | |
277 | |
278 | def install_pip(): |
279 | subprocess.check_call(['apt-get', 'install', '-yq', 'python-pip', 'bzr']) |
280 | |
281 | |
282 | -def install_jujuresources(): |
283 | - """ |
284 | - Install the bundled jujuresources library, if not present. |
285 | - """ |
286 | - try: |
287 | - import jujuresources # noqa |
288 | - except ImportError: |
289 | - jr_archive = glob('resources/jujuresources-*.tar.gz')[0] |
290 | - subprocess.check_call(['pip', 'install', jr_archive]) |
291 | +def install_bundled_resources(): |
292 | + """ |
293 | + Install the bundled resources libraries. |
294 | + """ |
295 | + archives = glob('resources/python/*') |
296 | + subprocess.check_call(['pip', 'install'] + archives) |
297 | |
298 | === modified file 'hooks/start' |
299 | --- hooks/start 2015-02-09 18:19:59 +0000 |
300 | +++ hooks/start 2015-06-25 22:42:45 +0000 |
301 | @@ -11,5 +11,16 @@ |
302 | # See the License for the specific language governing permissions and |
303 | # limitations under the License. |
304 | |
305 | +""" |
306 | +All hooks in this charm are managed by the Charm Framework. |
307 | +The framework helps manage dependencies and preconditions to ensure that |
308 | +steps are only executed when they can be successful. As such, no additional |
309 | +code should be added to this hook; instead, please integrate new functionality |
310 | +into the 'callbacks' list in hooks/common.py. New callbacks can be placed |
311 | +in hooks/callbacks.py, if necessary. |
312 | + |
313 | +See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html |
314 | +for more information. |
315 | +""" |
316 | import common |
317 | common.manage() |
318 | |
319 | === modified file 'hooks/stop' |
320 | --- hooks/stop 2015-02-09 18:19:59 +0000 |
321 | +++ hooks/stop 2015-06-25 22:42:45 +0000 |
322 | @@ -11,5 +11,16 @@ |
323 | # See the License for the specific language governing permissions and |
324 | # limitations under the License. |
325 | |
326 | +""" |
327 | +All hooks in this charm are managed by the Charm Framework. |
328 | +The framework helps manage dependencies and preconditions to ensure that |
329 | +steps are only executed when they can be successful. As such, no additional |
330 | +code should be added to this hook; instead, please integrate new functionality |
331 | +into the 'callbacks' list in hooks/common.py. New callbacks can be placed |
332 | +in hooks/callbacks.py, if necessary. |
333 | + |
334 | +See http://big-data-charm-helpers.readthedocs.org/en/latest/examples/framework.html |
335 | +for more information. |
336 | +""" |
337 | import common |
338 | common.manage() |
339 | |
340 | === modified file 'resources.yaml' |
341 | --- resources.yaml 2015-05-28 18:17:01 +0000 |
342 | +++ resources.yaml 2015-06-25 22:42:45 +0000 |
343 | @@ -3,23 +3,15 @@ |
344 | resources: |
345 | pathlib: |
346 | pypi: path.py>=7.0 |
347 | - pyaml: |
348 | - pypi: pyaml |
349 | - six: |
350 | - pypi: six |
351 | jujubigdata: |
352 | - pypi: jujubigdata>=1.2.14,<2.0.0 |
353 | - charmhelpers: |
354 | - pypi: http://bazaar.launchpad.net/~bigdata-dev/bigdata-data/trunk/download/kevin.monroe%40canonical.com-20150511173636-5rblzf5r2o1zcv2p/charmhelpers0.2.3.ta-20150417221203-zg62z8c220egc3ch-1/charmhelpers-0.2.3.tar.gz |
355 | - hash: 44340a6fd6f192bcc9d390c0d9c3901d4fc190166485b107047bc1c6ba102a2f |
356 | - hash_type: sha256 |
357 | + pypi: jujubigdata>=2.0.2,<3.0.0 |
358 | java-installer: |
359 | # This points to a script which manages installing Java. |
360 | # If replaced with an alternate implementation, it must output *only* two |
361 | # lines containing the JAVA_HOME path, and the Java version, respectively, |
362 | # on stdout. Upon error, it must exit with a non-zero exit code. |
363 | - url: http://bazaar.launchpad.net/~bigdata-dev/bigdata-data/trunk/download/cory.johns%40canonical.com-20150312205309-2ji1etk44gep01w1/javainstaller.sh-20150311213053-4vq7369jhlvc6qy8-1/java-installer.sh |
364 | - hash: 130984f1dc3bc624d4245234d0fca22f529d234d0eaa1241c5e9f701319bdea9 |
365 | + url: http://bazaar.launchpad.net/~bigdata-dev/bigdata-data/trunk/download/kevin.monroe%40canonical.com-20150625222410-qfck64q46ubx5i6p/javainstaller.sh-20150311213053-4vq7369jhlvc6qy8-1/java-installer.sh |
366 | + hash: 8fdff60270ea4be7bbef1e013e503057fe6efc2f4b5761edebc206a54f303023 |
367 | hash_type: sha256 |
368 | optional_resources: |
369 | hadoop-aarch64: |
370 | |
371 | === removed file 'resources/jujuresources-0.2.5.tar.gz' |
372 | Binary files resources/jujuresources-0.2.5.tar.gz 2015-03-03 19:57:43 +0000 and resources/jujuresources-0.2.5.tar.gz 1970-01-01 00:00:00 +0000 differ |
373 | === added directory 'resources/python' |
374 | === added file 'resources/python/PyYAML-3.11.tar.gz' |
375 | Binary files resources/python/PyYAML-3.11.tar.gz 1970-01-01 00:00:00 +0000 and resources/python/PyYAML-3.11.tar.gz 2015-06-25 22:42:45 +0000 differ |
376 | === added file 'resources/python/charmhelpers-0.3.1.tar.gz' |
377 | Binary files resources/python/charmhelpers-0.3.1.tar.gz 1970-01-01 00:00:00 +0000 and resources/python/charmhelpers-0.3.1.tar.gz 2015-06-25 22:42:45 +0000 differ |
378 | === added file 'resources/python/jujuresources-0.2.8.tar.gz' |
379 | Binary files resources/python/jujuresources-0.2.8.tar.gz 1970-01-01 00:00:00 +0000 and resources/python/jujuresources-0.2.8.tar.gz 2015-06-25 22:42:45 +0000 differ |
380 | === added file 'resources/python/pyaml-15.5.7.tar.gz' |
381 | Binary files resources/python/pyaml-15.5.7.tar.gz 1970-01-01 00:00:00 +0000 and resources/python/pyaml-15.5.7.tar.gz 2015-06-25 22:42:45 +0000 differ |
382 | === added file 'resources/python/six-1.9.0-py2.py3-none-any.whl' |
383 | Binary files resources/python/six-1.9.0-py2.py3-none-any.whl 1970-01-01 00:00:00 +0000 and resources/python/six-1.9.0-py2.py3-none-any.whl 2015-06-25 22:42:45 +0000 differ |
actions + status + /etc/hosts refactor + java fix = LGTM.
Merged into to -dev and pushed to -charmers.