Merge lp:~bcsaller/charms/trusty/cloudfoundry/progressbar into lp:~cf-charmers/charms/trusty/cloudfoundry/trunk
- Trusty Tahr (14.04)
- progressbar
- Merge into trunk
Proposed by
Benjamin Saller
Status: | Merged |
---|---|
Merged at revision: | 160 |
Proposed branch: | lp:~bcsaller/charms/trusty/cloudfoundry/progressbar |
Merge into: | lp:~cf-charmers/charms/trusty/cloudfoundry/trunk |
Diff against target: |
449 lines (+169/-50) 9 files modified
README.rst (+4/-6) cfdeploy (+114/-11) cloudfoundry/utils.py (+41/-23) reconciler/app.py (+2/-3) reconciler/strategy.py (+2/-2) reconciler/tactics.py (+2/-2) tests/test_strategy.py (+1/-1) tests/test_utils.py (+2/-2) tox.ini (+1/-0) |
To merge this branch: | bzr merge lp:~bcsaller/charms/trusty/cloudfoundry/progressbar |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Cory Johns (community) | Needs Fixing | ||
Review via email: mp+241618@code.launchpad.net |
Commit message
Description of the change
To post a comment you must log in.
- 178. By Benjamin Saller
-
attempt to install deps from the wheelhouse, needs verification
Revision history for this message
Cory Johns (johnsca) wrote : | # |
- 179. By Benjamin Saller
-
change func name
- 180. By Benjamin Saller
-
build out a virtualenv with deps prior to kickoff
Revision history for this message
Benjamin Saller (bcsaller) wrote : | # |
I re-pushed this with a venv.
Revision history for this message
Cory Johns (johnsca) wrote : | # |
Typo in python-virtualenv package, noted inline, below.
review:
Needs Fixing
Revision history for this message
Cory Johns (johnsca) wrote : | # |
Actually, I can just fix this while merging.
Revision history for this message
Cory Johns (johnsca) wrote : | # |
I propose this change to make it less chatty: http://
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'README.rst' |
2 | --- README.rst 2014-11-06 23:37:55 +0000 |
3 | +++ README.rst 2014-11-13 00:05:21 +0000 |
4 | @@ -24,11 +24,6 @@ |
5 | We provide a set of helper scripts (in bash) which can be used to assist in the deployment and |
6 | management of CF. |
7 | |
8 | -You will need the cf command line client. This can easily be installed using the following: |
9 | - |
10 | - wget http://go-cli.s3-website-us-east-1.amazonaws.com/releases/latest/cf-cli_amd64.deb |
11 | - dpkg -i cf-cli_amd64.deb |
12 | - |
13 | |
14 | Deployment |
15 | ---------- |
16 | @@ -51,7 +46,10 @@ |
17 | will block until the deployment is fully up and running. It also will create a |
18 | CF "admin" user with the given password, leaving you logged in and ready to push |
19 | a CF app. Note that while this can take quite some time, the whole process is |
20 | -automated. |
21 | +automated. |
22 | + |
23 | +If you don't have the cf command line client the latest one will be installed |
24 | +from a .deb package to your local system as part of cfdeploy. |
25 | |
26 | Once the deploy completes, you can push apps. A sample application we recommend |
27 | to become familiar with the system is GitHub High Score website. This can be |
28 | |
29 | === modified file 'cfdeploy' |
30 | --- cfdeploy 2014-11-07 00:06:24 +0000 |
31 | +++ cfdeploy 2014-11-13 00:05:21 +0000 |
32 | @@ -2,9 +2,48 @@ |
33 | # modeline vim:set syntax=python |
34 | import argparse |
35 | import logging |
36 | -import webbrowser |
37 | +import os |
38 | +import subprocess |
39 | +import sys |
40 | +from contextlib import contextmanager |
41 | from functools import partial |
42 | |
43 | + |
44 | +def verify_dep(name): |
45 | + try: |
46 | + __import__(name) |
47 | + return True |
48 | + except ImportError: |
49 | + pass |
50 | + return False |
51 | + |
52 | + |
53 | +def install_python_deps(): |
54 | + print "Setting up virutalenv." |
55 | + subprocess.check_output(['pip', 'install', 'wheel']) |
56 | + for dep in ['progress', 'requests', 'jujuclient']: |
57 | + subprocess.check_output(['pip', 'install', |
58 | + '--use-wheel', '--find-links', |
59 | + os.path.abspath('wheelhouse'), |
60 | + dep]) |
61 | + |
62 | + |
63 | +def prepare_runtime(): |
64 | + activate = "bin/activate_this.py" |
65 | + if not verify_dep('virtualenv'): |
66 | + print "(sudo) installing python-virtualenv package." |
67 | + subprocess.check_call( |
68 | + ['sudo', 'apt-get', 'install', '-yq', 'python-virutalenv']) |
69 | + import virtualenv |
70 | + venv = os.path.join(os.getcwd(), '.venv') |
71 | + if not os.path.exists(venv): |
72 | + virtualenv.create_environment(venv, site_packages=False) |
73 | + venv_launcher = os.path.join(venv, activate) |
74 | + execfile(venv_launcher, dict(__file__=venv_launcher)) |
75 | + install_python_deps() |
76 | + |
77 | +prepare_runtime() |
78 | + |
79 | from cloudfoundry.releases import RELEASES |
80 | from cloudfoundry.utils import (bootstrap, |
81 | cf_service, |
82 | @@ -15,9 +54,13 @@ |
83 | login, |
84 | reconciler_endpoint, |
85 | webadmin_endpoint, |
86 | + sh, |
87 | socket_open, |
88 | until, |
89 | - wait_for) |
90 | + which |
91 | + ) |
92 | + |
93 | +from progress.bar import Bar |
94 | |
95 | |
96 | def setup(): |
97 | @@ -35,31 +78,91 @@ |
98 | return options |
99 | |
100 | |
101 | +# This is done because the default webbrowser invocation will output |
102 | +# on stderr and muck with the progress bar. By hiding the output |
103 | +# with devnull it proceeds as expected. |
104 | +show = sh.check('xdg-open') |
105 | + |
106 | + |
107 | +@contextmanager |
108 | +def devnull(): |
109 | + _save = sys.stderr |
110 | + fp = open('/dev/null', 'w') |
111 | + sys.stderr = fp |
112 | + yield |
113 | + sys.stderr = _save |
114 | + |
115 | + |
116 | def show_reconciler(): |
117 | - uri = "http://%s:8888/" % reconciler_endpoint() |
118 | - webbrowser.open_new_tab(uri) |
119 | + with devnull(): |
120 | + uri = "http://%s:8888/" % reconciler_endpoint() |
121 | + show(uri) |
122 | |
123 | |
124 | def show_webadmin(): |
125 | - uri = "http://%s:8070/" % webadmin_endpoint() |
126 | - webbrowser.open_new_tab(uri) |
127 | + with devnull(): |
128 | + uri = "http://%s:8070/" % webadmin_endpoint() |
129 | + show(uri) |
130 | + |
131 | + |
132 | +class ProgressBar(Bar): |
133 | + message = "Deploying CloudFoundry" |
134 | + fill = "." |
135 | + suffix = "%(percent).1f%% %(elapsed_td)s" |
136 | + |
137 | + def next(self, i=1, message=None): |
138 | + if message: |
139 | + self.message = message |
140 | + super(ProgressBar, self).next(i) |
141 | + |
142 | + |
143 | +def install_deps(): |
144 | + if not which('cf'): |
145 | + from platform import machine |
146 | + if machine() != "x86_64": |
147 | + print "Unable to install CF CLI for your architecture. " |
148 | + "Deploy will not work as expected." |
149 | + return |
150 | + sh.wget('http://go-cli.s3-website-us-east-1.amazonaws.com/' |
151 | + 'releases/latest/cf-cli_amd64.deb') |
152 | + print "Installing CF CLI (this requires sudo access)" |
153 | + sh.sudo('dpkg', '-i', 'cf-cli_amd64.deb') |
154 | |
155 | |
156 | def main(): |
157 | options = setup() |
158 | logging.basicConfig(level=options.log_level) |
159 | + install_deps() |
160 | + |
161 | + bar = ProgressBar('Deploying CloudFoundry', max=10) |
162 | + bar.start() |
163 | + bar.next(message='Bootstrapping') |
164 | bootstrap() |
165 | - until(juju_state_server) |
166 | + until(juju_state_server, bar=bar, message="Waiting for State Server") |
167 | + bar.next(message='Deploying Orchestrator') |
168 | deploy(constraints=options.constraints, |
169 | generate_dependents=options.generate, |
170 | admin_password=options.admin_password) |
171 | - until(lambda: socket_open(reconciler_endpoint(), 8888)) |
172 | + until(lambda: socket_open(reconciler_endpoint(), 8888), |
173 | + bar=bar, message="Waiting on Reconciler") |
174 | + bar.next(message='Showing Reconciler') |
175 | show_reconciler() |
176 | + |
177 | # Wait forever, its in the reconciler's hands now. |
178 | - wait_for(0, 30, cf_service, endpoint, partial(login, options.admin_password)) |
179 | - until(lambda: socket_open(webadmin_endpoint(), 8070)) |
180 | + until(cf_service, bar=bar, message="Waiting for Orchestrator Charm") |
181 | + until(endpoint, bar=bar, message='Waiting for CloudFoundry API endpoint') |
182 | + until(partial(login, options.admin_password), |
183 | + bar=bar, message='Waiting to login to CloudFoundry (long)') |
184 | + until(lambda: socket_open(webadmin_endpoint(), 8070), |
185 | + bar=bar, message="Waiting on webadmin.") |
186 | + bar.next(message="Opening Admin Console") |
187 | show_webadmin() |
188 | - print "You should now be logged into a running CF deployment" |
189 | + bar.finish() |
190 | + |
191 | + print "You should now be logged into a running CF deployment." |
192 | + if which('cf'): |
193 | + print "The 'cf' command line client is installed and available." |
194 | + |
195 | |
196 | if __name__ == "__main__": |
197 | main() |
198 | |
199 | === modified file 'cloudfoundry/utils.py' |
200 | --- cloudfoundry/utils.py 2014-11-07 00:06:24 +0000 |
201 | +++ cloudfoundry/utils.py 2014-11-13 00:05:21 +0000 |
202 | @@ -168,6 +168,7 @@ |
203 | |
204 | def command(*base_args, **kwargs): |
205 | check = kwargs.pop('check', False) |
206 | + throw = kwargs.pop('throw', True) |
207 | |
208 | def callable_command(*args, **kws): |
209 | kwargs.update(kws) |
210 | @@ -176,10 +177,20 @@ |
211 | if 'env' not in kwargs: |
212 | kwargs['env'] = os.environ |
213 | logging.debug("invoke: %s", all_args) |
214 | + |
215 | + p = subprocess.Popen(all_args, |
216 | + stdout=subprocess.PIPE, |
217 | + stderr=subprocess.STDOUT, |
218 | + **kwargs) |
219 | + output, _ = p.communicate() |
220 | + ret_code = p.poll() |
221 | + logging.debug('result: %s', output) |
222 | if check is True: |
223 | - return subprocess.check_call(all_args, **kwargs) |
224 | + if ret_code != 0 and throw: |
225 | + raise subprocess.CalledProcessError(ret_code, all_args, output=output) |
226 | + return ret_code |
227 | else: |
228 | - return subprocess.check_output(all_args, **kwargs).strip() |
229 | + return output.strip() |
230 | return callable_command |
231 | |
232 | |
233 | @@ -196,7 +207,7 @@ |
234 | |
235 | sh = Commander() |
236 | dig = command('dig', '+short') |
237 | -api_endpoints = sh.check('juju', 'api-endpoints') |
238 | +api_endpoints = sh.check('juju', 'api-endpoints', throw=False) |
239 | |
240 | |
241 | def current_env(): |
242 | @@ -210,7 +221,7 @@ |
243 | 'environments/%s.jenv' % cenv) |
244 | |
245 | |
246 | -def wait_for(timeout, interval, *callbacks): |
247 | +def wait_for(timeout, interval, *callbacks, **kwargs): |
248 | """ |
249 | Repeatedly try callbacks until all return True |
250 | |
251 | @@ -223,25 +234,35 @@ |
252 | hardware fails, or the heat death of the universe. |
253 | """ |
254 | start = time.time() |
255 | + if timeout: |
256 | + end = start + timeout |
257 | + else: |
258 | + end = 0 |
259 | + |
260 | + bar = kwargs.get('bar', None) |
261 | + message = kwargs.get('message', None) |
262 | + once = 1 |
263 | while True: |
264 | passes = True |
265 | - for callback in callbacks: |
266 | - result = callback() |
267 | - passes = passes & bool(result) |
268 | - if passes is False: |
269 | - break |
270 | - if passes is True: |
271 | - break |
272 | - current = time.time() |
273 | - if timeout != 0 and ( |
274 | - current - start >= timeout or |
275 | - (current - start) + interval > timeout): |
276 | + if end > 0 and time.time() > end: |
277 | raise OSError("Timeout exceeded in wait_for") |
278 | - time.sleep(interval) |
279 | - |
280 | - |
281 | -def until(*callbacks): |
282 | - return wait_for(0, 20, *callbacks) |
283 | + if bar: |
284 | + bar.next(once, message=message) |
285 | + if once == 1: |
286 | + once = 0 |
287 | + if int(time.time()) % interval == 0: |
288 | + for callback in callbacks: |
289 | + result = callback() |
290 | + passes = passes & bool(result) |
291 | + if passes is False: |
292 | + break |
293 | + if passes is True: |
294 | + break |
295 | + time.sleep(1) |
296 | + |
297 | + |
298 | +def until(*callbacks, **kwargs): |
299 | + return wait_for(0, 20, *callbacks, **kwargs) |
300 | |
301 | |
302 | def juju_state_server(): |
303 | @@ -249,7 +270,6 @@ |
304 | return False |
305 | endpoints = json.loads(sh.juju('api-endpoints', '--format=json')) |
306 | for ep in endpoints: |
307 | - print "Attempt to connect to juju state server:", ep |
308 | host, port = ep.split(':', 1) |
309 | result = socket_open(host, int(port)) |
310 | if result is True: |
311 | @@ -266,7 +286,6 @@ |
312 | # .jenv with the state server |
313 | sh.juju('status') |
314 | data = yaml.load(open(get_jenv())) |
315 | - print "Waiting for state server(s)", data['state-servers'] |
316 | return data.get('state-servers', []) != [] |
317 | current = current_env() |
318 | wait_for(1000, 20, env_connection) |
319 | @@ -387,7 +406,6 @@ |
320 | if constraints: |
321 | args.append('--constraints=%s' % constraints) |
322 | args.append('local:trusty/cloudfoundry') |
323 | - print "Deploying:", ' '.join(args) |
324 | sh.juju(*args) |
325 | time.sleep(5) |
326 | sh.juju('expose', 'cloudfoundry') |
327 | |
328 | === modified file 'reconciler/app.py' |
329 | --- reconciler/app.py 2014-11-03 16:13:57 +0000 |
330 | +++ reconciler/app.py 2014-11-13 00:05:21 +0000 |
331 | @@ -49,7 +49,6 @@ |
332 | import tornado.web |
333 | |
334 | from tornado.options import define, options |
335 | -from cloudfoundry import config |
336 | from cloudfoundry import utils |
337 | from cloudfoundry.path import path |
338 | from reconciler import strategy |
339 | @@ -57,6 +56,8 @@ |
340 | from reconciler.ui.app import DashboardIO |
341 | from reconciler.ui.app import static_resources |
342 | |
343 | +import config |
344 | + |
345 | application = None |
346 | env_name = None |
347 | server = None |
348 | @@ -283,7 +284,6 @@ |
349 | loop = tornado.ioloop.IOLoop.instance() |
350 | # call poll_health ASAP, to populate the initial health data, and |
351 | # also schedule poll_health for every 60s to refresh health data |
352 | - |
353 | loop.add_callback(poll_health) |
354 | tornado.ioloop.PeriodicCallback(poll_health, 60000, io_loop=loop).start() |
355 | tornado.ioloop.PeriodicCallback(poll_current_state, 60000, |
356 | @@ -294,6 +294,5 @@ |
357 | io_loop=loop).start() |
358 | loop.start() |
359 | |
360 | - |
361 | if __name__ == "__main__": |
362 | main() |
363 | |
364 | === renamed file 'cloudfoundry/config.py' => 'reconciler/config.py' |
365 | === modified file 'reconciler/strategy.py' |
366 | --- reconciler/strategy.py 2014-11-05 23:40:05 +0000 |
367 | +++ reconciler/strategy.py 2014-11-13 00:05:21 +0000 |
368 | @@ -7,7 +7,7 @@ |
369 | from tornado import gen |
370 | |
371 | from reconciler import tactics |
372 | -from cloudfoundry.config import (PENDING, COMPLETE, FAILED, RUNNING) |
373 | +from reconciler.config import (PENDING, COMPLETE, FAILED, RUNNING) |
374 | from cloudfoundry import utils |
375 | from cloudfoundry import tsort |
376 | |
377 | @@ -160,7 +160,7 @@ |
378 | elif expose_intent is False and exposed is True: |
379 | result.append(tactics.UnexposeTactic(service=service)) |
380 | |
381 | - logging.debug("Build New %s", result) |
382 | + logging.debug("Build Strategy %s", result) |
383 | return result |
384 | |
385 | def build_relations(self): |
386 | |
387 | === modified file 'reconciler/tactics.py' |
388 | --- reconciler/tactics.py 2014-11-05 23:40:05 +0000 |
389 | +++ reconciler/tactics.py 2014-11-13 00:05:21 +0000 |
390 | @@ -3,7 +3,7 @@ |
391 | import os |
392 | import shutil |
393 | |
394 | -from cloudfoundry.config import ( |
395 | +from reconciler.config import ( |
396 | PENDING, COMPLETE, RUNNING, FAILED, STATES |
397 | ) |
398 | |
399 | @@ -169,7 +169,7 @@ |
400 | class UnexposeTactic(Tactic): |
401 | name = "Unexpose Service" |
402 | |
403 | - def run(self, env, **kwargs): |
404 | + def _run(self, env, **kwargs): |
405 | s = kwargs['service'] |
406 | env.unexpose(s['service_name']) |
407 | |
408 | |
409 | === modified file 'tests/test_strategy.py' |
410 | --- tests/test_strategy.py 2014-10-30 18:34:30 +0000 |
411 | +++ tests/test_strategy.py 2014-11-13 00:05:21 +0000 |
412 | @@ -2,7 +2,7 @@ |
413 | import unittest |
414 | import mock |
415 | |
416 | -from cloudfoundry import config |
417 | +from reconciler import config |
418 | from reconciler import tactics |
419 | from reconciler import strategy |
420 | |
421 | |
422 | === modified file 'tests/test_utils.py' |
423 | --- tests/test_utils.py 2014-10-27 19:48:56 +0000 |
424 | +++ tests/test_utils.py 2014-11-13 00:05:21 +0000 |
425 | @@ -7,10 +7,10 @@ |
426 | |
427 | |
428 | class TestUtils(unittest.TestCase): |
429 | - @mock.patch('subprocess.check_output') |
430 | + @mock.patch('cloudfoundry.utils.sh.juju') |
431 | def test_current_env(self, check_output): |
432 | utils.current_env() |
433 | - check_output.assert_called_once_with(('juju', 'switch'), env=mock.ANY) |
434 | + check_output.assert_called_once_with('switch') |
435 | |
436 | def test_flatten_relations(self): |
437 | r = utils.flatten_relations([ |
438 | |
439 | === modified file 'tox.ini' |
440 | --- tox.ini 2014-09-30 21:15:05 +0000 |
441 | +++ tox.ini 2014-11-13 00:05:21 +0000 |
442 | @@ -30,3 +30,4 @@ |
443 | clint |
444 | path.py |
445 | subparse |
446 | + progress |
447 | |
448 | === added file 'wheelhouse/progress-1.2-py2-none-any.whl' |
449 | Binary files wheelhouse/progress-1.2-py2-none-any.whl 1970-01-01 00:00:00 +0000 and wheelhouse/progress-1.2-py2-none-any.whl 2014-11-13 00:05:21 +0000 differ |
See inline comments.