Merge lp:~benji/charms/precise/juju-gui/use-npm-cache into lp:~juju-gui/charms/precise/juju-gui/trunk
- Precise Pangolin (12.04)
- use-npm-cache
- Merge into trunk
Status: | Merged |
---|---|
Merged at revision: | 51 |
Proposed branch: | lp:~benji/charms/precise/juju-gui/use-npm-cache |
Merge into: | lp:~juju-gui/charms/precise/juju-gui/trunk |
Diff against target: |
1233 lines (+1076/-22) (has conflicts) 6 files modified
hooks/backend.py (+38/-17) hooks/utils.py (+27/-1) revision (+4/-0) tests/backend.py (+291/-0) tests/test_utils.py (+87/-4) tests/utils.py (+629/-0) Text conflict in revision |
To merge this branch: | bzr merge lp:~benji/charms/precise/juju-gui/use-npm-cache |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Gary Poster (community) | Approve | ||
charmers | Pending | ||
Review via email: mp+161477@code.launchpad.net |
Commit message
Description of the change
Use the npm cache to speed installs from a branch
Creating the cache is handled by the GUI Makefile
Benji York (benji) wrote : | # |
Gary Poster (gary) wrote : | # |
Thanks for the docs and comments! They are great to have.
29 + # If the given installable thing ("backend") requires one or more debs
30 + # that are not yet installed, install them.
31 missing = backend.
Yeah, check_packages might be better named install_
39 + # Inject NPM packages into the cache for faster building.
40 + #prime_
Uh? I hope this wasn't commented out for your tests. :-)
44 + # XXX Why do we only set up the GUI if the "juju-gui-source"
45 + # configuration is non-default?
It's not non-default: it is not-what-
124 + if e.errno != 17: # File exists.
Maybe use errno.EEXIST instead?
126 + uncompress = command('tar', '-x', '-z', '-C', npm_cache_dir, '-f')
127 + cmd_log(
That reads very nicely. Good idea.
164 + get_npm_
You look like you were on your way to a test? That would have been nice. :-) If not, remove the import. But adding an import of prime_npm_cache and testing them both would be nicer. :-)
Thanks!
Gary
Benji York (benji) wrote : | # |
On Mon, Apr 29, 2013 at 3:31 PM, Gary Poster <email address hidden> wrote:
> Review: Approve
>
> Thanks for the docs and comments! They are great to have.
>
> 29 + # If the given installable thing ("backend") requires one or more debs
> 30 + # that are not yet installed, install them.
> 31 missing = backend.
>
> Yeah, check_packages might be better named install_
> something like that.
Yep (I was only commenting things at the time, but this looks like a
local-enough change that I'll go ahead and do it).
> 39 + # Inject NPM packages into the cache for faster building.
> 40 + #prime_
>
> Uh? I hope this wasn't commented out for your tests. :-)
Heh. Yeah, I was doing some non-cache timings and forgot to put it
back. Fixed.
> 44 + # XXX Why do we only set up the GUI if the "juju-gui-source"
> 45 + # configuration is non-default?
>
> It's not non-default: it is not-what-
> the intent. I think that's the behavior too, or we'd be in trouble.
> :-) At the start, what-it-was-before is some empty value, I believe.
Ah! I think you're right. Maybe I can come up with variable/method
names that communicate better; or a comment at least.
> 124 + if e.errno != 17: # File exists.
>
> Maybe use errno.EEXIST instead?
Good call. Will do.
> 126 + uncompress = command('tar', '-x', '-z', '-C', npm_cache_dir, '-f')
> 127 + cmd_log(
>
> That reads very nicely. Good idea.
I thought so too. That's why I copied it from another part of the code. ;)
> 164 + get_npm_
>
> You look like you were on your way to a test? That would have been
> nice. :-)
Indeed. I thought I had added it. I'll do so.
--
Benji York
- 49. By Benji York
-
review fixes
Benji York (benji) wrote : | # |
Please take a look.
Benji York (benji) wrote : | # |
*** Submitted:
Use the npm cache to speed installs from a branch
Creating the cache is handled by the GUI Makefile
Preview Diff
1 | === modified file 'hooks/backend.py' | |||
2 | --- hooks/backend.py 2013-04-30 13:53:20 +0000 | |||
3 | +++ hooks/backend.py 2013-04-30 14:57:26 +0000 | |||
4 | @@ -29,17 +29,19 @@ | |||
5 | 29 | IMPROV, | 29 | IMPROV, |
6 | 30 | JUJU_DIR, | 30 | JUJU_DIR, |
7 | 31 | chain, | 31 | chain, |
9 | 32 | check_packages, | 32 | find_missing_packages, |
10 | 33 | cmd_log, | 33 | cmd_log, |
11 | 34 | fetch_api, | 34 | fetch_api, |
12 | 35 | fetch_gui, | 35 | fetch_gui, |
13 | 36 | get_config, | 36 | get_config, |
14 | 37 | get_npm_cache_archive_url, | ||
15 | 37 | legacy_juju, | 38 | legacy_juju, |
16 | 38 | merge, | 39 | merge, |
17 | 39 | overrideable, | 40 | overrideable, |
18 | 41 | prime_npm_cache, | ||
19 | 40 | save_or_create_certificates, | 42 | save_or_create_certificates, |
20 | 43 | setup_apache, | ||
21 | 41 | setup_gui, | 44 | setup_gui, |
22 | 42 | setup_apache, | ||
23 | 43 | start_agent, | 45 | start_agent, |
24 | 44 | start_gui, | 46 | start_gui, |
25 | 45 | start_improv, | 47 | start_improv, |
26 | @@ -53,20 +55,40 @@ | |||
27 | 53 | 55 | ||
28 | 54 | 56 | ||
29 | 55 | class InstallMixin(object): | 57 | class InstallMixin(object): |
30 | 58 | """Provide for the GUI and its dependencies to be installed.""" | ||
31 | 59 | |||
32 | 56 | def install(self, backend): | 60 | def install(self, backend): |
33 | 61 | """Install the GUI and dependencies.""" | ||
34 | 57 | config = backend.config | 62 | config = backend.config |
36 | 58 | missing = backend.check_packages(*backend.debs) | 63 | # If the given installable thing ("backend") requires one or more debs |
37 | 64 | # that are not yet installed, install them. | ||
38 | 65 | missing = backend.find_missing_packages(*backend.debs) | ||
39 | 59 | if missing: | 66 | if missing: |
40 | 60 | cmd_log(backend.install_extra_repositories(*backend.repositories)) | 67 | cmd_log(backend.install_extra_repositories(*backend.repositories)) |
41 | 61 | cmd_log(apt_get_install(*backend.debs)) | 68 | cmd_log(apt_get_install(*backend.debs)) |
42 | 62 | 69 | ||
43 | 70 | # If we are not using a pre-built release of the GUI (i.e., we are | ||
44 | 71 | # using a branch) then we need to build a release archive to use. | ||
45 | 63 | if backend.different('juju-gui-source'): | 72 | if backend.different('juju-gui-source'): |
49 | 64 | release_tarball = fetch_gui( | 73 | # Inject NPM packages into the cache for faster building. |
50 | 65 | config['juju-gui-source'], config['command-log-file']) | 74 | self._prime_npm_cache() |
51 | 66 | setup_gui(release_tarball) | 75 | # Build a release from the branch. |
52 | 76 | self._build_and_install_from_branch() | ||
53 | 77 | |||
54 | 78 | def _prime_npm_cache(self): | ||
55 | 79 | # This is a separate method so it can be easily overridden for testing. | ||
56 | 80 | prime_npm_cache(get_npm_cache_archive_url()) | ||
57 | 81 | |||
58 | 82 | def _build_and_install_from_branch(self): | ||
59 | 83 | # This is a separate method so it can be easily overridden for testing. | ||
60 | 84 | release_tarball = fetch_gui( | ||
61 | 85 | config['juju-gui-source'], config['command-log-file']) | ||
62 | 86 | setup_gui(release_tarball) | ||
63 | 67 | 87 | ||
64 | 68 | 88 | ||
65 | 69 | class UpstartMixin(object): | 89 | class UpstartMixin(object): |
66 | 90 | """Manage (install, start, stop, etc.) some service via Upstart.""" | ||
67 | 91 | |||
68 | 70 | upstart_scripts = ('haproxy.conf', ) | 92 | upstart_scripts = ('haproxy.conf', ) |
69 | 71 | debs = ('curl', 'openssl', 'haproxy', 'apache2') | 93 | debs = ('curl', 'openssl', 'haproxy', 'apache2') |
70 | 72 | 94 | ||
71 | @@ -152,16 +174,14 @@ | |||
72 | 152 | 174 | ||
73 | 153 | 175 | ||
74 | 154 | class Backend(object): | 176 | class Backend(object): |
78 | 155 | """Compose methods and policy needed to interact | 177 | """Compose methods and policy needed to interact with a Juju backend. |
79 | 156 | with a Juju backend. Given a config dict (which typically | 178 | |
80 | 157 | comes from the JSON de-serialization of config.json in JujuGUI). | 179 | "config" is a config dict (which typically comes from the JSON |
81 | 180 | de-serialization of config.json in JujuGUI). | ||
82 | 158 | """ | 181 | """ |
83 | 159 | 182 | ||
84 | 160 | def __init__(self, config=None, prev_config=None, **overrides): | 183 | def __init__(self, config=None, prev_config=None, **overrides): |
89 | 161 | """ | 184 | """Generate a selection of strategy classes that implement the backend. |
86 | 162 | Backends function through composition. __init__ becomes the | ||
87 | 163 | factory method to generate a selection of strategy classes | ||
88 | 164 | to use together to implement the backend proper. | ||
90 | 165 | """ | 185 | """ |
91 | 166 | # Ingest the config and build out the ordered list of | 186 | # Ingest the config and build out the ordered list of |
92 | 167 | # backend elements to include | 187 | # backend elements to include |
93 | @@ -171,7 +191,7 @@ | |||
94 | 171 | self.prev_config = prev_config | 191 | self.prev_config = prev_config |
95 | 172 | self.overrides = overrides | 192 | self.overrides = overrides |
96 | 173 | 193 | ||
98 | 174 | # We always use upstart. | 194 | # We always install the GUI. |
99 | 175 | backends = [InstallMixin, ] | 195 | backends = [InstallMixin, ] |
100 | 176 | 196 | ||
101 | 177 | api = "python" if legacy_juju() else "go" | 197 | api = "python" if legacy_juju() else "go" |
102 | @@ -194,7 +214,8 @@ | |||
103 | 194 | "Unable to use sandbox with {} backend".format(api)) | 214 | "Unable to use sandbox with {} backend".format(api)) |
104 | 195 | backends.append(GoBackend) | 215 | backends.append(GoBackend) |
105 | 196 | 216 | ||
107 | 197 | # All backends can manage the gui. | 217 | # All backends need to install, start, and stop the services that |
108 | 218 | # provide the GUI. | ||
109 | 198 | backends.append(GuiMixin) | 219 | backends.append(GuiMixin) |
110 | 199 | backends.append(UpstartMixin) | 220 | backends.append(UpstartMixin) |
111 | 200 | 221 | ||
112 | @@ -213,8 +234,8 @@ | |||
113 | 213 | raise | 234 | raise |
114 | 214 | 235 | ||
115 | 215 | @overrideable | 236 | @overrideable |
118 | 216 | def check_packages(self, *packages): | 237 | def find_missing_packages(self, *packages): |
119 | 217 | return check_packages(*packages) | 238 | return find_missing_packages(*packages) |
120 | 218 | 239 | ||
121 | 219 | @overrideable | 240 | @overrideable |
122 | 220 | def service_control(self, service, action): | 241 | def service_control(self, service, action): |
123 | 221 | 242 | ||
124 | === modified file 'hooks/utils.py' | |||
125 | --- hooks/utils.py 2013-04-30 13:53:20 +0000 | |||
126 | +++ hooks/utils.py 2013-04-30 14:57:26 +0000 | |||
127 | @@ -404,6 +404,32 @@ | |||
128 | 404 | render_to_file('apache-site.template', context, JUJU_GUI_SITE) | 404 | render_to_file('apache-site.template', context, JUJU_GUI_SITE) |
129 | 405 | 405 | ||
130 | 406 | 406 | ||
131 | 407 | def get_npm_cache_archive_url(Launchpad=Launchpad): | ||
132 | 408 | """Figure out the URL of the most recent NPM cache archive on Launchpad.""" | ||
133 | 409 | launchpad = Launchpad.login_anonymously('Juju GUI charm', 'production') | ||
134 | 410 | project = launchpad.projects['juju-gui'] | ||
135 | 411 | # Find the URL of the most recently created NPM cache archive. | ||
136 | 412 | npm_cache_url = get_release_file_url(project, 'npm-cache', None) | ||
137 | 413 | return npm_cache_url | ||
138 | 414 | |||
139 | 415 | |||
140 | 416 | def prime_npm_cache(npm_cache_url): | ||
141 | 417 | """Download NPM cache archive and prime the NPM cache with it.""" | ||
142 | 418 | # Download the cache archive and then uncompress it into the NPM cache. | ||
143 | 419 | npm_cache_archive = os.path.join(CURRENT_DIR, 'npm-cache.tgz') | ||
144 | 420 | cmd_log(run('curl', '-L', '-o', npm_cache_archive, npm_cache_url)) | ||
145 | 421 | npm_cache_dir = os.path.expanduser('~/.npm') | ||
146 | 422 | # The NPM cache directory probably does not exist, so make it if not. | ||
147 | 423 | try: | ||
148 | 424 | os.mkdir(npm_cache_dir) | ||
149 | 425 | except OSError, e: | ||
150 | 426 | # If the directory already exists then ignore the error. | ||
151 | 427 | if e.errno != errno.EEXIST: # File exists. | ||
152 | 428 | raise | ||
153 | 429 | uncompress = command('tar', '-x', '-z', '-C', npm_cache_dir, '-f') | ||
154 | 430 | cmd_log(uncompress(npm_cache_archive)) | ||
155 | 431 | |||
156 | 432 | |||
157 | 407 | def fetch_gui(juju_gui_source, logpath): | 433 | def fetch_gui(juju_gui_source, logpath): |
158 | 408 | """Retrieve the Juju GUI release/branch.""" | 434 | """Retrieve the Juju GUI release/branch.""" |
159 | 409 | # Retrieve a Juju GUI release. | 435 | # Retrieve a Juju GUI release. |
160 | @@ -520,7 +546,7 @@ | |||
161 | 520 | shutil.copyfileobj(open(crt_path), pem_file) | 546 | shutil.copyfileobj(open(crt_path), pem_file) |
162 | 521 | 547 | ||
163 | 522 | 548 | ||
165 | 523 | def check_packages(*packages): | 549 | def find_missing_packages(*packages): |
166 | 524 | """Given a list of packages, return the packages which are not installed. | 550 | """Given a list of packages, return the packages which are not installed. |
167 | 525 | """ | 551 | """ |
168 | 526 | cache = apt.Cache() | 552 | cache = apt.Cache() |
169 | 527 | 553 | ||
170 | === modified file 'revision' | |||
171 | --- revision 2013-04-30 13:53:20 +0000 | |||
172 | +++ revision 2013-04-30 14:57:26 +0000 | |||
173 | @@ -1,1 +1,5 @@ | |||
174 | 1 | <<<<<<< TREE | ||
175 | 1 | 41 | 2 | 41 |
176 | 3 | ======= | ||
177 | 4 | 44 | ||
178 | 5 | >>>>>>> MERGE-SOURCE | ||
179 | 2 | 6 | ||
180 | === modified symlink 'tests/backend.py' | |||
181 | === target was u'../hooks/backend.py' | |||
182 | --- tests/backend.py 1970-01-01 00:00:00 +0000 | |||
183 | +++ tests/backend.py 2013-04-30 14:57:26 +0000 | |||
184 | @@ -0,0 +1,291 @@ | |||
185 | 1 | """ | ||
186 | 2 | A composition system for creating backend object. | ||
187 | 3 | |||
188 | 4 | Backends implement start(), stop() and install() methods. A backend is composed | ||
189 | 5 | of many mixins and each mixin will implement any/all of those methods and all | ||
190 | 6 | will be called. Backends additionally provide for collecting property values | ||
191 | 7 | from each mixin into a single final property on the backend. There is also a | ||
192 | 8 | feature for determining if configuration values have changed between old and | ||
193 | 9 | new configurations so we can selectively take action. | ||
194 | 10 | """ | ||
195 | 11 | |||
196 | 12 | from charmhelpers import ( | ||
197 | 13 | RESTART, | ||
198 | 14 | STOP, | ||
199 | 15 | log, | ||
200 | 16 | open_port, | ||
201 | 17 | service_control, | ||
202 | 18 | ) | ||
203 | 19 | from shelltoolbox import ( | ||
204 | 20 | apt_get_install, | ||
205 | 21 | command, | ||
206 | 22 | install_extra_repositories, | ||
207 | 23 | su, | ||
208 | 24 | ) | ||
209 | 25 | from utils import ( | ||
210 | 26 | AGENT, | ||
211 | 27 | APACHE, | ||
212 | 28 | HAPROXY, | ||
213 | 29 | IMPROV, | ||
214 | 30 | JUJU_DIR, | ||
215 | 31 | chain, | ||
216 | 32 | find_missing_packages, | ||
217 | 33 | cmd_log, | ||
218 | 34 | fetch_api, | ||
219 | 35 | fetch_gui, | ||
220 | 36 | get_config, | ||
221 | 37 | get_npm_cache_archive_url, | ||
222 | 38 | legacy_juju, | ||
223 | 39 | merge, | ||
224 | 40 | overrideable, | ||
225 | 41 | prime_npm_cache, | ||
226 | 42 | save_or_create_certificates, | ||
227 | 43 | setup_apache, | ||
228 | 44 | setup_gui, | ||
229 | 45 | start_agent, | ||
230 | 46 | start_gui, | ||
231 | 47 | start_improv, | ||
232 | 48 | ) | ||
233 | 49 | |||
234 | 50 | import os | ||
235 | 51 | import shutil | ||
236 | 52 | |||
237 | 53 | |||
238 | 54 | apt_get = command('apt-get') | ||
239 | 55 | |||
240 | 56 | |||
241 | 57 | class InstallMixin(object): | ||
242 | 58 | """Provide for the GUI and its dependencies to be installed.""" | ||
243 | 59 | |||
244 | 60 | def install(self, backend): | ||
245 | 61 | """Install the GUI and dependencies.""" | ||
246 | 62 | config = backend.config | ||
247 | 63 | # If the given installable thing ("backend") requires one or more debs | ||
248 | 64 | # that are not yet installed, install them. | ||
249 | 65 | missing = backend.find_missing_packages(*backend.debs) | ||
250 | 66 | if missing: | ||
251 | 67 | cmd_log(backend.install_extra_repositories(*backend.repositories)) | ||
252 | 68 | cmd_log(apt_get_install(*backend.debs)) | ||
253 | 69 | |||
254 | 70 | # If we are not using a pre-built release of the GUI (i.e., we are | ||
255 | 71 | # using a branch) then we need to build a release archive to use. | ||
256 | 72 | if backend.different('juju-gui-source'): | ||
257 | 73 | # Inject NPM packages into the cache for faster building. | ||
258 | 74 | self._prime_npm_cache() | ||
259 | 75 | # Build a release from the branch. | ||
260 | 76 | self._build_and_install_from_branch() | ||
261 | 77 | |||
262 | 78 | def _prime_npm_cache(self): | ||
263 | 79 | # This is a separate method so it can be easily overridden for testing. | ||
264 | 80 | prime_npm_cache(get_npm_cache_archive_url()) | ||
265 | 81 | |||
266 | 82 | def _build_and_install_from_branch(self): | ||
267 | 83 | # This is a separate method so it can be easily overridden for testing. | ||
268 | 84 | release_tarball = fetch_gui( | ||
269 | 85 | config['juju-gui-source'], config['command-log-file']) | ||
270 | 86 | setup_gui(release_tarball) | ||
271 | 87 | |||
272 | 88 | |||
273 | 89 | class UpstartMixin(object): | ||
274 | 90 | """Manage (install, start, stop, etc.) some service via Upstart.""" | ||
275 | 91 | |||
276 | 92 | upstart_scripts = ('haproxy.conf', ) | ||
277 | 93 | debs = ('curl', 'openssl', 'haproxy', 'apache2') | ||
278 | 94 | |||
279 | 95 | def install(self, backend): | ||
280 | 96 | """Set up haproxy and nginx upstart configuration files.""" | ||
281 | 97 | setup_apache() | ||
282 | 98 | backend.log('Setting up haproxy and nginx start up scripts.') | ||
283 | 99 | config = backend.config | ||
284 | 100 | if backend.different( | ||
285 | 101 | 'ssl-cert-path', 'ssl-cert-contents', 'ssl-key-contents'): | ||
286 | 102 | save_or_create_certificates( | ||
287 | 103 | config['ssl-cert-path'], config.get('ssl-cert-contents'), | ||
288 | 104 | config.get('ssl-key-contents')) | ||
289 | 105 | |||
290 | 106 | source_dir = os.path.join(os.path.dirname(__file__), '..', 'config') | ||
291 | 107 | for config_file in backend.upstart_scripts: | ||
292 | 108 | shutil.copy(os.path.join(source_dir, config_file), '/etc/init/') | ||
293 | 109 | |||
294 | 110 | def start(self, backend): | ||
295 | 111 | with su('root'): | ||
296 | 112 | backend.service_control(APACHE, RESTART) | ||
297 | 113 | backend.service_control(HAPROXY, RESTART) | ||
298 | 114 | |||
299 | 115 | def stop(self, backend): | ||
300 | 116 | with su('root'): | ||
301 | 117 | backend.service_control(HAPROXY, STOP) | ||
302 | 118 | backend.service_control(APACHE, STOP) | ||
303 | 119 | |||
304 | 120 | |||
305 | 121 | class GuiMixin(object): | ||
306 | 122 | gui_properties = set([ | ||
307 | 123 | 'juju-gui-console-enabled', 'login-help', 'read-only', | ||
308 | 124 | 'serve-tests', 'secure']) | ||
309 | 125 | |||
310 | 126 | repositories = ('ppa:juju-gui/ppa',) | ||
311 | 127 | |||
312 | 128 | def start(self, config): | ||
313 | 129 | start_gui( | ||
314 | 130 | config['juju-gui-console-enabled'], config['login-help'], | ||
315 | 131 | config['read-only'], config['staging'], config['ssl-cert-path'], | ||
316 | 132 | config['charmworld-url'], config['serve-tests'], | ||
317 | 133 | secure=config['secure'], sandbox=config['sandbox']) | ||
318 | 134 | open_port(80) | ||
319 | 135 | open_port(443) | ||
320 | 136 | |||
321 | 137 | |||
322 | 138 | class SandboxBackend(object): | ||
323 | 139 | pass | ||
324 | 140 | |||
325 | 141 | |||
326 | 142 | class PythonBackend(object): | ||
327 | 143 | |||
328 | 144 | def install(self, config): | ||
329 | 145 | if (not os.path.exists(JUJU_DIR) or | ||
330 | 146 | config.different('staging', 'juju-api-branch')): | ||
331 | 147 | fetch_api(config['juju-api-branch']) | ||
332 | 148 | |||
333 | 149 | def start(self, backend): | ||
334 | 150 | backend.start_agent(backend['ssl-cert-path']) | ||
335 | 151 | |||
336 | 152 | def stop(self, backend): | ||
337 | 153 | backend.service_control(AGENT, STOP) | ||
338 | 154 | |||
339 | 155 | |||
340 | 156 | class ImprovBackend(object): | ||
341 | 157 | debs = ('zookeeper', ) | ||
342 | 158 | |||
343 | 159 | def install(self, config): | ||
344 | 160 | if (not os.path.exists(JUJU_DIR) or | ||
345 | 161 | config.different('staging', 'juju-api-branch')): | ||
346 | 162 | fetch_api(config['juju-api-branch']) | ||
347 | 163 | |||
348 | 164 | def start(self, backend): | ||
349 | 165 | backend.start_improv( | ||
350 | 166 | backend['staging-environment'], backend['ssl-cert-path']) | ||
351 | 167 | |||
352 | 168 | def stop(self, backend): | ||
353 | 169 | backend.service_control(IMPROV, STOP) | ||
354 | 170 | |||
355 | 171 | |||
356 | 172 | class GoBackend(object): | ||
357 | 173 | debs = ('python-yaml', ) | ||
358 | 174 | |||
359 | 175 | |||
360 | 176 | class Backend(object): | ||
361 | 177 | """Compose methods and policy needed to interact with a Juju backend. | ||
362 | 178 | |||
363 | 179 | "config" is a config dict (which typically comes from the JSON | ||
364 | 180 | de-serialization of config.json in JujuGUI). | ||
365 | 181 | """ | ||
366 | 182 | |||
367 | 183 | def __init__(self, config=None, prev_config=None, **overrides): | ||
368 | 184 | """Generate a selection of strategy classes that implement the backend. | ||
369 | 185 | """ | ||
370 | 186 | # Ingest the config and build out the ordered list of | ||
371 | 187 | # backend elements to include | ||
372 | 188 | if config is None: | ||
373 | 189 | config = get_config() | ||
374 | 190 | self.config = config | ||
375 | 191 | self.prev_config = prev_config | ||
376 | 192 | self.overrides = overrides | ||
377 | 193 | |||
378 | 194 | # We always install the GUI. | ||
379 | 195 | backends = [InstallMixin, ] | ||
380 | 196 | |||
381 | 197 | api = "python" if legacy_juju() else "go" | ||
382 | 198 | sandbox = config.get('sandbox', False) | ||
383 | 199 | staging = config.get('staging', False) | ||
384 | 200 | |||
385 | 201 | if api == 'python': | ||
386 | 202 | if staging: | ||
387 | 203 | backends.append(ImprovBackend) | ||
388 | 204 | elif sandbox: | ||
389 | 205 | backends.append(SandboxBackend) | ||
390 | 206 | else: | ||
391 | 207 | backends.append(PythonBackend) | ||
392 | 208 | else: | ||
393 | 209 | if staging: | ||
394 | 210 | raise ValueError( | ||
395 | 211 | "Unable to use staging with {} backend".format(api)) | ||
396 | 212 | if sandbox: | ||
397 | 213 | raise ValueError( | ||
398 | 214 | "Unable to use sandbox with {} backend".format(api)) | ||
399 | 215 | backends.append(GoBackend) | ||
400 | 216 | |||
401 | 217 | # All backends need to install, start, and stop the services that | ||
402 | 218 | # provide the GUI. | ||
403 | 219 | backends.append(GuiMixin) | ||
404 | 220 | backends.append(UpstartMixin) | ||
405 | 221 | |||
406 | 222 | # record our choice mapping classes to instances | ||
407 | 223 | for i, b in enumerate(backends): | ||
408 | 224 | if callable(b): | ||
409 | 225 | backends[i] = b() | ||
410 | 226 | self.backends = backends | ||
411 | 227 | |||
412 | 228 | def __getitem__(self, key): | ||
413 | 229 | try: | ||
414 | 230 | return self.config[key] | ||
415 | 231 | except KeyError: | ||
416 | 232 | print("Unable to extract config key '%s' from %s" % | ||
417 | 233 | (key, self.config)) | ||
418 | 234 | raise | ||
419 | 235 | |||
420 | 236 | @overrideable | ||
421 | 237 | def find_missing_packages(self, *packages): | ||
422 | 238 | return find_missing_packages(*packages) | ||
423 | 239 | |||
424 | 240 | @overrideable | ||
425 | 241 | def service_control(self, service, action): | ||
426 | 242 | service_control(service, action) | ||
427 | 243 | |||
428 | 244 | @overrideable | ||
429 | 245 | def start_agent(self, cert_path): | ||
430 | 246 | start_agent(cert_path) | ||
431 | 247 | |||
432 | 248 | @overrideable | ||
433 | 249 | def start_improv(self, stage_env, cert_path): | ||
434 | 250 | start_improv(stage_env, cert_path) | ||
435 | 251 | |||
436 | 252 | @overrideable | ||
437 | 253 | def log(self, msg, *args): | ||
438 | 254 | log(msg, *args) | ||
439 | 255 | |||
440 | 256 | @overrideable | ||
441 | 257 | def install_extra_repositories(self, *packages): | ||
442 | 258 | if self.config.get('allow-additional-deb-repositories', True): | ||
443 | 259 | install_extra_repositories(*packages) | ||
444 | 260 | else: | ||
445 | 261 | apt_get('update') | ||
446 | 262 | |||
447 | 263 | def different(self, *keys): | ||
448 | 264 | """Return a boolean indicating if the current config | ||
449 | 265 | value differs from the config value passed in prev_config | ||
450 | 266 | with respect to any of the passed in string keys. | ||
451 | 267 | """ | ||
452 | 268 | if self.prev_config is None: | ||
453 | 269 | return True | ||
454 | 270 | |||
455 | 271 | for key in keys: | ||
456 | 272 | current = self.config.get(key) | ||
457 | 273 | prev = self.prev_config.get(key) | ||
458 | 274 | r = current != prev | ||
459 | 275 | if r: | ||
460 | 276 | return True | ||
461 | 277 | return False | ||
462 | 278 | |||
463 | 279 | ## Composed Methods | ||
464 | 280 | install = chain('install') | ||
465 | 281 | start = chain('start') | ||
466 | 282 | stop = chain('stop') | ||
467 | 283 | |||
468 | 284 | ## Merged Properties | ||
469 | 285 | dependencies = merge('dependencies') | ||
470 | 286 | build_dependencies = merge('build_dependencies') | ||
471 | 287 | staging_dependencies = merge('staging_dependencies') | ||
472 | 288 | |||
473 | 289 | repositories = merge('repositories') | ||
474 | 290 | debs = merge('debs') | ||
475 | 291 | upstart_scripts = merge('upstart_scripts') | ||
476 | 0 | 292 | ||
477 | === modified file 'tests/test_utils.py' | |||
478 | --- tests/test_utils.py 2013-04-28 00:59:29 +0000 | |||
479 | +++ tests/test_utils.py 2013-04-30 14:57:26 +0000 | |||
480 | @@ -12,25 +12,27 @@ | |||
481 | 12 | import charmhelpers | 12 | import charmhelpers |
482 | 13 | import yaml | 13 | import yaml |
483 | 14 | 14 | ||
484 | 15 | from backend import InstallMixin | ||
485 | 15 | from utils import ( | 16 | from utils import ( |
486 | 17 | API_PORT, | ||
487 | 18 | JUJU_GUI_DIR, | ||
488 | 19 | JUJU_PEM, | ||
489 | 20 | WEB_PORT, | ||
490 | 16 | _get_by_attr, | 21 | _get_by_attr, |
491 | 17 | API_PORT, | ||
492 | 18 | cmd_log, | 22 | cmd_log, |
493 | 19 | first_path_in_dir, | 23 | first_path_in_dir, |
494 | 20 | get_api_address, | 24 | get_api_address, |
495 | 21 | get_release_file_url, | 25 | get_release_file_url, |
496 | 22 | get_zookeeper_address, | 26 | get_zookeeper_address, |
497 | 23 | JUJU_GUI_DIR, | ||
498 | 24 | JUJU_PEM, | ||
499 | 25 | legacy_juju, | 27 | legacy_juju, |
500 | 26 | log_hook, | 28 | log_hook, |
501 | 27 | parse_source, | 29 | parse_source, |
502 | 30 | get_npm_cache_archive_url, | ||
503 | 28 | render_to_file, | 31 | render_to_file, |
504 | 29 | save_or_create_certificates, | 32 | save_or_create_certificates, |
505 | 30 | start_agent, | 33 | start_agent, |
506 | 31 | start_gui, | 34 | start_gui, |
507 | 32 | start_improv, | 35 | start_improv, |
508 | 33 | WEB_PORT, | ||
509 | 34 | ) | 36 | ) |
510 | 35 | # Import the whole utils package for monkey patching. | 37 | # Import the whole utils package for monkey patching. |
511 | 36 | import utils | 38 | import utils |
512 | @@ -603,5 +605,86 @@ | |||
513 | 603 | self.assertNotIn('redirect scheme https', haproxy_conf) | 605 | self.assertNotIn('redirect scheme https', haproxy_conf) |
514 | 604 | 606 | ||
515 | 605 | 607 | ||
516 | 608 | class TestNpmCache(unittest.TestCase): | ||
517 | 609 | """To speed building from a branch we prepopulate the NPM cache.""" | ||
518 | 610 | |||
519 | 611 | def test_retrieving_cache_url(self): | ||
520 | 612 | # The URL for the latest cache file can be retrieved from Launchpad. | ||
521 | 613 | class FauxLaunchpadFactory(object): | ||
522 | 614 | @staticmethod | ||
523 | 615 | def login_anonymously(agent, site): | ||
524 | 616 | # We download the cache from the production site. | ||
525 | 617 | self.assertEqual(site, 'production') | ||
526 | 618 | return FauxLaunchpad | ||
527 | 619 | |||
528 | 620 | class CacheFile(object): | ||
529 | 621 | file_link = 'http://launchpad.example/path/to/cache/file' | ||
530 | 622 | |||
531 | 623 | def __str__(self): | ||
532 | 624 | return 'cache-file-123.tgz' | ||
533 | 625 | |||
534 | 626 | class NpmRelease(object): | ||
535 | 627 | files = [CacheFile()] | ||
536 | 628 | |||
537 | 629 | class NpmSeries(object): | ||
538 | 630 | name = 'npm-cache' | ||
539 | 631 | releases = [NpmRelease] | ||
540 | 632 | |||
541 | 633 | class FauxProject(object): | ||
542 | 634 | series = [NpmSeries] | ||
543 | 635 | |||
544 | 636 | class FauxLaunchpad(object): | ||
545 | 637 | projects = {'juju-gui': FauxProject()} | ||
546 | 638 | |||
547 | 639 | url = get_npm_cache_archive_url(Launchpad=FauxLaunchpadFactory()) | ||
548 | 640 | self.assertEqual(url, 'http://launchpad.example/path/to/cache/file') | ||
549 | 641 | |||
550 | 642 | |||
551 | 643 | def test_InstallMixin_primes_npm_cache(self): | ||
552 | 644 | # The InstallMixin.install() method primes the NPM cache before | ||
553 | 645 | # building (and installing) the GUI from a branch. | ||
554 | 646 | assertFalse = self.assertFalse | ||
555 | 647 | assertTrue = self.assertTrue | ||
556 | 648 | |||
557 | 649 | class TestableInstallMixin(InstallMixin): | ||
558 | 650 | """An InstallMixin that records actions instead of taking them.""" | ||
559 | 651 | cache_primed = False | ||
560 | 652 | branch_installed = False | ||
561 | 653 | |||
562 | 654 | def _prime_npm_cache(self): | ||
563 | 655 | # The cache is primed before the branch is installed. | ||
564 | 656 | assertFalse(self.branch_installed) | ||
565 | 657 | self.cache_primed = True | ||
566 | 658 | |||
567 | 659 | def _build_and_install_from_branch(self): | ||
568 | 660 | # The cache is primed before the branch is installed. | ||
569 | 661 | assertTrue(self.cache_primed) | ||
570 | 662 | self.branch_installed = True | ||
571 | 663 | |||
572 | 664 | class FauxBackend(object): | ||
573 | 665 | """A test backend.""" | ||
574 | 666 | config = None | ||
575 | 667 | debs = ('DEBS',) | ||
576 | 668 | |||
577 | 669 | @classmethod | ||
578 | 670 | def find_missing_packages(cls, *debs): | ||
579 | 671 | self.assertEqual(cls.debs, debs) | ||
580 | 672 | return False | ||
581 | 673 | |||
582 | 674 | @classmethod | ||
583 | 675 | def different(cls, key): | ||
584 | 676 | self.assertEqual(key, 'juju-gui-source') | ||
585 | 677 | return True | ||
586 | 678 | |||
587 | 679 | mixin = TestableInstallMixin() | ||
588 | 680 | # Prior to "install" the NPM cache has not been primed. | ||
589 | 681 | self.assertFalse(mixin.cache_primed) | ||
590 | 682 | self.assertFalse(mixin.branch_installed) | ||
591 | 683 | mixin.install(FauxBackend()) | ||
592 | 684 | # After "install" the NPM cache has been primed. | ||
593 | 685 | self.assertTrue(mixin.cache_primed) | ||
594 | 686 | self.assertTrue(mixin.branch_installed) | ||
595 | 687 | |||
596 | 688 | |||
597 | 606 | if __name__ == '__main__': | 689 | if __name__ == '__main__': |
598 | 607 | unittest.main(verbosity=2) | 690 | unittest.main(verbosity=2) |
599 | 608 | 691 | ||
600 | === modified symlink 'tests/utils.py' | |||
601 | === target was u'../hooks/utils.py' | |||
602 | --- tests/utils.py 1970-01-01 00:00:00 +0000 | |||
603 | +++ tests/utils.py 2013-04-30 14:57:26 +0000 | |||
604 | @@ -0,0 +1,629 @@ | |||
605 | 1 | """Juju GUI charm utilities.""" | ||
606 | 2 | |||
607 | 3 | __all__ = [ | ||
608 | 4 | 'AGENT', | ||
609 | 5 | 'APACHE', | ||
610 | 6 | 'API_PORT', | ||
611 | 7 | 'CURRENT_DIR', | ||
612 | 8 | 'HAPROXY', | ||
613 | 9 | 'IMPROV', | ||
614 | 10 | 'JUJU_DIR', | ||
615 | 11 | 'JUJU_GUI_DIR', | ||
616 | 12 | 'JUJU_GUI_SITE', | ||
617 | 13 | 'JUJU_PEM', | ||
618 | 14 | 'StopChain', | ||
619 | 15 | 'WEB_PORT', | ||
620 | 16 | 'bzr_checkout', | ||
621 | 17 | 'chain', | ||
622 | 18 | 'cmd_log', | ||
623 | 19 | 'fetch_api', | ||
624 | 20 | 'fetch_gui', | ||
625 | 21 | 'first_path_in_dir', | ||
626 | 22 | 'get_api_address', | ||
627 | 23 | 'get_release_file_url', | ||
628 | 24 | 'get_staging_dependencies', | ||
629 | 25 | 'get_zookeeper_address', | ||
630 | 26 | 'legacy_juju', | ||
631 | 27 | 'log_hook', | ||
632 | 28 | 'merge', | ||
633 | 29 | 'overrideable', | ||
634 | 30 | 'parse_source', | ||
635 | 31 | 'render_to_file', | ||
636 | 32 | 'save_or_create_certificates', | ||
637 | 33 | 'setup_apache', | ||
638 | 34 | 'setup_gui', | ||
639 | 35 | 'start_agent', | ||
640 | 36 | 'start_gui', | ||
641 | 37 | 'start_improv', | ||
642 | 38 | 'write_apache_config', | ||
643 | 39 | ] | ||
644 | 40 | |||
645 | 41 | from contextlib import contextmanager | ||
646 | 42 | import json | ||
647 | 43 | import os | ||
648 | 44 | import logging | ||
649 | 45 | import shutil | ||
650 | 46 | from subprocess import CalledProcessError | ||
651 | 47 | import tempfile | ||
652 | 48 | from urlparse import urlparse | ||
653 | 49 | |||
654 | 50 | from launchpadlib.launchpad import Launchpad | ||
655 | 51 | from shelltoolbox import ( | ||
656 | 52 | Serializer, | ||
657 | 53 | apt_get_install, | ||
658 | 54 | command, | ||
659 | 55 | environ, | ||
660 | 56 | install_extra_repositories, | ||
661 | 57 | run, | ||
662 | 58 | script_name, | ||
663 | 59 | search_file, | ||
664 | 60 | su, | ||
665 | 61 | ) | ||
666 | 62 | from charmhelpers import ( | ||
667 | 63 | START, | ||
668 | 64 | get_config, | ||
669 | 65 | log, | ||
670 | 66 | service_control, | ||
671 | 67 | unit_get, | ||
672 | 68 | ) | ||
673 | 69 | |||
674 | 70 | import apt | ||
675 | 71 | import tempita | ||
676 | 72 | |||
677 | 73 | |||
678 | 74 | AGENT = 'juju-api-agent' | ||
679 | 75 | APACHE = 'apache2' | ||
680 | 76 | IMPROV = 'juju-api-improv' | ||
681 | 77 | HAPROXY = 'haproxy' | ||
682 | 78 | |||
683 | 79 | API_PORT = 8080 | ||
684 | 80 | WEB_PORT = 8000 | ||
685 | 81 | |||
686 | 82 | CURRENT_DIR = os.getcwd() | ||
687 | 83 | JUJU_DIR = os.path.join(CURRENT_DIR, 'juju') | ||
688 | 84 | JUJU_GUI_DIR = os.path.join(CURRENT_DIR, 'juju-gui') | ||
689 | 85 | JUJU_GUI_SITE = '/etc/apache2/sites-available/juju-gui' | ||
690 | 86 | JUJU_GUI_PORTS = '/etc/apache2/ports.conf' | ||
691 | 87 | JUJU_PEM = 'juju.includes-private-key.pem' | ||
692 | 88 | BUILD_REPOSITORIES = ('ppa:chris-lea/node.js-legacy',) | ||
693 | 89 | DEB_BUILD_DEPENDENCIES = ( | ||
694 | 90 | 'bzr', 'imagemagick', 'make', 'nodejs', 'npm', | ||
695 | 91 | ) | ||
696 | 92 | DEB_STAGE_DEPENDENCIES = ( | ||
697 | 93 | 'zookeeper', | ||
698 | 94 | ) | ||
699 | 95 | |||
700 | 96 | |||
701 | 97 | # Store the configuration from on invocation to the next. | ||
702 | 98 | config_json = Serializer('/tmp/config.json') | ||
703 | 99 | # Bazaar checkout command. | ||
704 | 100 | bzr_checkout = command('bzr', 'co', '--lightweight') | ||
705 | 101 | # Whether or not the charm is deployed using juju-core. | ||
706 | 102 | # If juju-core has been used to deploy the charm, an agent.conf file must | ||
707 | 103 | # be present in the charm parent directory. | ||
708 | 104 | legacy_juju = lambda: not os.path.exists( | ||
709 | 105 | os.path.join(CURRENT_DIR, '..', 'agent.conf')) | ||
710 | 106 | |||
711 | 107 | |||
712 | 108 | def _get_build_dependencies(): | ||
713 | 109 | """Install deb dependencies for building.""" | ||
714 | 110 | log('Installing build dependencies.') | ||
715 | 111 | cmd_log(install_extra_repositories(*BUILD_REPOSITORIES)) | ||
716 | 112 | cmd_log(apt_get_install(*DEB_BUILD_DEPENDENCIES)) | ||
717 | 113 | |||
718 | 114 | |||
719 | 115 | def get_api_address(unit_dir): | ||
720 | 116 | """Return the Juju API address stored in the uniter agent.conf file.""" | ||
721 | 117 | import yaml # python-yaml is only installed if juju-core is used. | ||
722 | 118 | # XXX 2013-03-27 frankban bug=1161443: | ||
723 | 119 | # currently the uniter agent.conf file does not include the API | ||
724 | 120 | # address. For now retrieve it from the machine agent file. | ||
725 | 121 | base_dir = os.path.abspath(os.path.join(unit_dir, '..')) | ||
726 | 122 | for dirname in os.listdir(base_dir): | ||
727 | 123 | if dirname.startswith('machine-'): | ||
728 | 124 | agent_conf = os.path.join(base_dir, dirname, 'agent.conf') | ||
729 | 125 | break | ||
730 | 126 | else: | ||
731 | 127 | raise IOError('Juju agent configuration file not found.') | ||
732 | 128 | contents = yaml.load(open(agent_conf)) | ||
733 | 129 | return contents['apiinfo']['addrs'][0] | ||
734 | 130 | |||
735 | 131 | |||
736 | 132 | def get_staging_dependencies(): | ||
737 | 133 | """Install deb dependencies for the stage (improv) environment.""" | ||
738 | 134 | log('Installing stage dependencies.') | ||
739 | 135 | cmd_log(apt_get_install(*DEB_STAGE_DEPENDENCIES)) | ||
740 | 136 | |||
741 | 137 | |||
742 | 138 | def first_path_in_dir(directory): | ||
743 | 139 | """Return the full path of the first file/dir in *directory*.""" | ||
744 | 140 | return os.path.join(directory, os.listdir(directory)[0]) | ||
745 | 141 | |||
746 | 142 | |||
747 | 143 | def _get_by_attr(collection, attr, value): | ||
748 | 144 | """Return the first item in collection having attr == value. | ||
749 | 145 | |||
750 | 146 | Return None if the item is not found. | ||
751 | 147 | """ | ||
752 | 148 | for item in collection: | ||
753 | 149 | if getattr(item, attr) == value: | ||
754 | 150 | return item | ||
755 | 151 | |||
756 | 152 | |||
757 | 153 | def get_release_file_url(project, series_name, release_version): | ||
758 | 154 | """Return the URL of the release file hosted in Launchpad. | ||
759 | 155 | |||
760 | 156 | The returned URL points to a release file for the given project, series | ||
761 | 157 | name and release version. | ||
762 | 158 | The argument *project* is a project object as returned by launchpadlib. | ||
763 | 159 | The arguments *series_name* and *release_version* are strings. If | ||
764 | 160 | *release_version* is None, the URL of the latest release will be returned. | ||
765 | 161 | """ | ||
766 | 162 | series = _get_by_attr(project.series, 'name', series_name) | ||
767 | 163 | if series is None: | ||
768 | 164 | raise ValueError('%r: series not found' % series_name) | ||
769 | 165 | # Releases are returned by Launchpad in reverse date order. | ||
770 | 166 | releases = list(series.releases) | ||
771 | 167 | if not releases: | ||
772 | 168 | raise ValueError('%r: series does not contain releases' % series_name) | ||
773 | 169 | if release_version is not None: | ||
774 | 170 | release = _get_by_attr(releases, 'version', release_version) | ||
775 | 171 | if release is None: | ||
776 | 172 | raise ValueError('%r: release not found' % release_version) | ||
777 | 173 | releases = [release] | ||
778 | 174 | for release in releases: | ||
779 | 175 | for file_ in release.files: | ||
780 | 176 | if str(file_).endswith('.tgz'): | ||
781 | 177 | return file_.file_link | ||
782 | 178 | raise ValueError('%r: file not found' % release_version) | ||
783 | 179 | |||
784 | 180 | |||
785 | 181 | def get_zookeeper_address(agent_file_path): | ||
786 | 182 | """Retrieve the Zookeeper address contained in the given *agent_file_path*. | ||
787 | 183 | |||
788 | 184 | The *agent_file_path* is a path to a file containing a line similar to the | ||
789 | 185 | following:: | ||
790 | 186 | |||
791 | 187 | env JUJU_ZOOKEEPER="address" | ||
792 | 188 | """ | ||
793 | 189 | line = search_file('JUJU_ZOOKEEPER', agent_file_path).strip() | ||
794 | 190 | return line.split('=')[1].strip('"') | ||
795 | 191 | |||
796 | 192 | |||
797 | 193 | @contextmanager | ||
798 | 194 | def log_hook(): | ||
799 | 195 | """Log when a hook starts and stops its execution. | ||
800 | 196 | |||
801 | 197 | Also log to stdout possible CalledProcessError exceptions raised executing | ||
802 | 198 | the hook. | ||
803 | 199 | """ | ||
804 | 200 | script = script_name() | ||
805 | 201 | log(">>> Entering {}".format(script)) | ||
806 | 202 | try: | ||
807 | 203 | yield | ||
808 | 204 | except CalledProcessError as err: | ||
809 | 205 | log('Exception caught:') | ||
810 | 206 | log(err.output) | ||
811 | 207 | raise | ||
812 | 208 | finally: | ||
813 | 209 | log("<<< Exiting {}".format(script)) | ||
814 | 210 | |||
815 | 211 | |||
816 | 212 | def parse_source(source): | ||
817 | 213 | """Parse the ``juju-gui-source`` option. | ||
818 | 214 | |||
819 | 215 | Return a tuple of two elements representing info on how to deploy Juju GUI. | ||
820 | 216 | Examples: | ||
821 | 217 | - ('stable', None): latest stable release; | ||
822 | 218 | - ('stable', '0.1.0'): stable release v0.1.0; | ||
823 | 219 | - ('trunk', None): latest trunk release; | ||
824 | 220 | - ('trunk', '0.1.0+build.1'): trunk release v0.1.0 bzr revision 1; | ||
825 | 221 | - ('branch', 'lp:juju-gui'): release is made from a branch; | ||
826 | 222 | - ('url', 'http://example.com/gui'): release from a downloaded file. | ||
827 | 223 | """ | ||
828 | 224 | if source.startswith('url:'): | ||
829 | 225 | source = source[4:] | ||
830 | 226 | # Support file paths, including relative paths. | ||
831 | 227 | if urlparse(source).scheme == '': | ||
832 | 228 | if not source.startswith('/'): | ||
833 | 229 | source = os.path.join(os.path.abspath(CURRENT_DIR), source) | ||
834 | 230 | source = "file://%s" % source | ||
835 | 231 | return 'url', source | ||
836 | 232 | if source in ('stable', 'trunk'): | ||
837 | 233 | return source, None | ||
838 | 234 | if source.startswith('lp:') or source.startswith('http://'): | ||
839 | 235 | return 'branch', source | ||
840 | 236 | if 'build' in source: | ||
841 | 237 | return 'trunk', source | ||
842 | 238 | return 'stable', source | ||
843 | 239 | |||
844 | 240 | |||
845 | 241 | def render_to_file(template_name, context, destination): | ||
846 | 242 | """Render the given *template_name* into *destination* using *context*. | ||
847 | 243 | |||
848 | 244 | The tempita template language is used to render contents | ||
849 | 245 | (see http://pythonpaste.org/tempita/). | ||
850 | 246 | The argument *template_name* is the name or path of the template file: | ||
851 | 247 | it may be either a path relative to ``../config`` or an absolute path. | ||
852 | 248 | The argument *destination* is a file path. | ||
853 | 249 | The argument *context* is a dict-like object. | ||
854 | 250 | """ | ||
855 | 251 | template_path = os.path.join( | ||
856 | 252 | os.path.dirname(__file__), '..', 'config', template_name) | ||
857 | 253 | template = tempita.Template.from_filename(template_path) | ||
858 | 254 | with open(destination, 'w') as stream: | ||
859 | 255 | stream.write(template.substitute(context)) | ||
860 | 256 | |||
861 | 257 | |||
862 | 258 | results_log = None | ||
863 | 259 | |||
864 | 260 | |||
865 | 261 | def _setupLogging(): | ||
866 | 262 | global results_log | ||
867 | 263 | if results_log is not None: | ||
868 | 264 | return | ||
869 | 265 | config = get_config() | ||
870 | 266 | logging.basicConfig( | ||
871 | 267 | filename=config['command-log-file'], | ||
872 | 268 | level=logging.INFO, | ||
873 | 269 | format="%(asctime)s: %(name)s@%(levelname)s %(message)s") | ||
874 | 270 | results_log = logging.getLogger('juju-gui') | ||
875 | 271 | |||
876 | 272 | |||
877 | 273 | def cmd_log(results): | ||
878 | 274 | global results_log | ||
879 | 275 | if not results: | ||
880 | 276 | return | ||
881 | 277 | if results_log is None: | ||
882 | 278 | _setupLogging() | ||
883 | 279 | # Since 'results' may be multi-line output, start it on a separate line | ||
884 | 280 | # from the logger timestamp, etc. | ||
885 | 281 | results_log.info('\n' + results) | ||
886 | 282 | |||
887 | 283 | |||
888 | 284 | def start_improv(staging_env, ssl_cert_path, | ||
889 | 285 | config_path='/etc/init/juju-api-improv.conf'): | ||
890 | 286 | """Start a simulated juju environment using ``improv.py``.""" | ||
891 | 287 | log('Setting up staging start up script.') | ||
892 | 288 | context = { | ||
893 | 289 | 'juju_dir': JUJU_DIR, | ||
894 | 290 | 'keys': ssl_cert_path, | ||
895 | 291 | 'port': API_PORT, | ||
896 | 292 | 'staging_env': staging_env, | ||
897 | 293 | } | ||
898 | 294 | render_to_file('juju-api-improv.conf.template', context, config_path) | ||
899 | 295 | log('Starting the staging backend.') | ||
900 | 296 | with su('root'): | ||
901 | 297 | service_control(IMPROV, START) | ||
902 | 298 | |||
903 | 299 | |||
904 | 300 | def start_agent(ssl_cert_path, config_path='/etc/init/juju-api-agent.conf'): | ||
905 | 301 | """Start the Juju agent and connect to the current environment.""" | ||
906 | 302 | # Retrieve the Zookeeper address from the start up script. | ||
907 | 303 | unit_dir = os.path.realpath(os.path.join(CURRENT_DIR, '..')) | ||
908 | 304 | agent_file = '/etc/init/juju-{0}.conf'.format(os.path.basename(unit_dir)) | ||
909 | 305 | zookeeper = get_zookeeper_address(agent_file) | ||
910 | 306 | log('Setting up API agent start up script.') | ||
911 | 307 | context = { | ||
912 | 308 | 'juju_dir': JUJU_DIR, | ||
913 | 309 | 'keys': ssl_cert_path, | ||
914 | 310 | 'port': API_PORT, | ||
915 | 311 | 'zookeeper': zookeeper, | ||
916 | 312 | } | ||
917 | 313 | render_to_file('juju-api-agent.conf.template', context, config_path) | ||
918 | 314 | log('Starting API agent.') | ||
919 | 315 | with su('root'): | ||
920 | 316 | service_control(AGENT, START) | ||
921 | 317 | |||
922 | 318 | |||
923 | 319 | def start_gui( | ||
924 | 320 | console_enabled, login_help, readonly, in_staging, ssl_cert_path, | ||
925 | 321 | charmworld_url, serve_tests, haproxy_path='/etc/haproxy/haproxy.cfg', | ||
926 | 322 | config_js_path=None, secure=True, sandbox=False): | ||
927 | 323 | """Set up and start the Juju GUI server.""" | ||
928 | 324 | with su('root'): | ||
929 | 325 | run('chown', '-R', 'ubuntu:', JUJU_GUI_DIR) | ||
930 | 326 | # XXX 2013-02-05 frankban bug=1116320: | ||
931 | 327 | # External insecure resources are still loaded when testing in the | ||
932 | 328 | # debug environment. For now, switch to the production environment if | ||
933 | 329 | # the charm is configured to serve tests. | ||
934 | 330 | if in_staging and not serve_tests: | ||
935 | 331 | build_dirname = 'build-debug' | ||
936 | 332 | else: | ||
937 | 333 | build_dirname = 'build-prod' | ||
938 | 334 | build_dir = os.path.join(JUJU_GUI_DIR, build_dirname) | ||
939 | 335 | log('Generating the Juju GUI configuration file.') | ||
940 | 336 | is_legacy_juju = legacy_juju() | ||
941 | 337 | user, password = None, None | ||
942 | 338 | if is_legacy_juju and in_staging: | ||
943 | 339 | user, password = 'admin', 'admin' | ||
944 | 340 | else: | ||
945 | 341 | user, password = None, None | ||
946 | 342 | |||
947 | 343 | api_backend = 'python' if is_legacy_juju else 'go' | ||
948 | 344 | if secure: | ||
949 | 345 | protocol = 'wss' | ||
950 | 346 | else: | ||
951 | 347 | log('Running in insecure mode! Port 80 will serve unencrypted.') | ||
952 | 348 | protocol = 'ws' | ||
953 | 349 | |||
954 | 350 | context = { | ||
955 | 351 | 'raw_protocol': protocol, | ||
956 | 352 | 'address': unit_get('public-address'), | ||
957 | 353 | 'console_enabled': json.dumps(console_enabled), | ||
958 | 354 | 'login_help': json.dumps(login_help), | ||
959 | 355 | 'password': json.dumps(password), | ||
960 | 356 | 'api_backend': json.dumps(api_backend), | ||
961 | 357 | 'readonly': json.dumps(readonly), | ||
962 | 358 | 'user': json.dumps(user), | ||
963 | 359 | 'protocol': json.dumps(protocol), | ||
964 | 360 | 'sandbox': json.dumps(sandbox), | ||
965 | 361 | 'charmworld_url': json.dumps(charmworld_url), | ||
966 | 362 | } | ||
967 | 363 | if config_js_path is None: | ||
968 | 364 | config_js_path = os.path.join( | ||
969 | 365 | build_dir, 'juju-ui', 'assets', 'config.js') | ||
970 | 366 | render_to_file('config.js.template', context, config_js_path) | ||
971 | 367 | |||
972 | 368 | write_apache_config(build_dir, serve_tests) | ||
973 | 369 | |||
974 | 370 | log('Generating haproxy configuration file.') | ||
975 | 371 | if is_legacy_juju: | ||
976 | 372 | # The PyJuju API agent is listening on localhost. | ||
977 | 373 | api_address = '127.0.0.1:{0}'.format(API_PORT) | ||
978 | 374 | else: | ||
979 | 375 | # Retrieve the juju-core API server address. | ||
980 | 376 | api_address = get_api_address(os.path.join(CURRENT_DIR, '..')) | ||
981 | 377 | context = { | ||
982 | 378 | 'api_address': api_address, | ||
983 | 379 | 'api_pem': JUJU_PEM, | ||
984 | 380 | 'legacy_juju': is_legacy_juju, | ||
985 | 381 | 'ssl_cert_path': ssl_cert_path, | ||
986 | 382 | # In PyJuju environments, use the same certificate for both HTTPS and | ||
987 | 383 | # WebSocket connections. In juju-core the system already has the proper | ||
988 | 384 | # certificate installed. | ||
989 | 385 | 'web_pem': JUJU_PEM, | ||
990 | 386 | 'web_port': WEB_PORT, | ||
991 | 387 | 'secure': secure | ||
992 | 388 | } | ||
993 | 389 | render_to_file('haproxy.cfg.template', context, haproxy_path) | ||
994 | 390 | log('Starting Juju GUI.') | ||
995 | 391 | |||
996 | 392 | def write_apache_config(build_dir, serve_tests=False): | ||
997 | 393 | log('Generating the apache site configuration file.') | ||
998 | 394 | context = { | ||
999 | 395 | 'port': WEB_PORT, | ||
1000 | 396 | 'serve_tests': serve_tests, | ||
1001 | 397 | 'server_root': build_dir, | ||
1002 | 398 | 'tests_root': os.path.join(JUJU_GUI_DIR, 'test', ''), | ||
1003 | 399 | } | ||
1004 | 400 | render_to_file('apache-ports.template', context, JUJU_GUI_PORTS) | ||
1005 | 401 | render_to_file('apache-site.template', context, JUJU_GUI_SITE) | ||
1006 | 402 | |||
1007 | 403 | |||
1008 | 404 | def get_npm_cache_archive_url(Launchpad=Launchpad): | ||
1009 | 405 | """Figure out the URL of the most recent NPM cache archive on Launchpad.""" | ||
1010 | 406 | launchpad = Launchpad.login_anonymously('Juju GUI charm', 'production') | ||
1011 | 407 | project = launchpad.projects['juju-gui'] | ||
1012 | 408 | # Find the URL of the most recently created NPM cache archive. | ||
1013 | 409 | npm_cache_url = get_release_file_url(project, 'npm-cache', None) | ||
1014 | 410 | return npm_cache_url | ||
1015 | 411 | |||
1016 | 412 | |||
1017 | 413 | def prime_npm_cache(npm_cache_url): | ||
1018 | 414 | """Download NPM cache archive and prime the NPM cache with it.""" | ||
1019 | 415 | # Download the cache archive and then uncompress it into the NPM cache. | ||
1020 | 416 | npm_cache_archive = os.path.join(CURRENT_DIR, 'npm-cache.tgz') | ||
1021 | 417 | cmd_log(run('curl', '-L', '-o', npm_cache_archive, npm_cache_url)) | ||
1022 | 418 | npm_cache_dir = os.path.expanduser('~/.npm') | ||
1023 | 419 | # The NPM cache directory probably does not exist, so make it if not. | ||
1024 | 420 | try: | ||
1025 | 421 | os.mkdir(npm_cache_dir) | ||
1026 | 422 | except OSError, e: | ||
1027 | 423 | # If the directory already exists then ignore the error. | ||
1028 | 424 | if e.errno != 17: # File exists. | ||
1029 | 425 | raise | ||
1030 | 426 | uncompress = command('tar', '-x', '-z', '-C', npm_cache_dir, '-f') | ||
1031 | 427 | cmd_log(uncompress(npm_cache_archive)) | ||
1032 | 428 | |||
1033 | 429 | |||
1034 | 430 | def fetch_gui(juju_gui_source, logpath): | ||
1035 | 431 | """Retrieve the Juju GUI release/branch.""" | ||
1036 | 432 | # Retrieve a Juju GUI release. | ||
1037 | 433 | origin, version_or_branch = parse_source(juju_gui_source) | ||
1038 | 434 | if origin == 'branch': | ||
1039 | 435 | # Make sure we have the dependencies necessary for us to actually make | ||
1040 | 436 | # a build. | ||
1041 | 437 | _get_build_dependencies() | ||
1042 | 438 | # Create a release starting from a branch. | ||
1043 | 439 | juju_gui_source_dir = os.path.join(CURRENT_DIR, 'juju-gui-source') | ||
1044 | 440 | log('Retrieving Juju GUI source checkout from %s.' % version_or_branch) | ||
1045 | 441 | cmd_log(run('rm', '-rf', juju_gui_source_dir)) | ||
1046 | 442 | cmd_log(bzr_checkout(version_or_branch, juju_gui_source_dir)) | ||
1047 | 443 | log('Preparing a Juju GUI release.') | ||
1048 | 444 | logdir = os.path.dirname(logpath) | ||
1049 | 445 | fd, name = tempfile.mkstemp(prefix='make-distfile-', dir=logdir) | ||
1050 | 446 | log('Output from "make distfile" sent to %s' % name) | ||
1051 | 447 | with environ(NO_BZR='1'): | ||
1052 | 448 | run('make', '-C', juju_gui_source_dir, 'distfile', | ||
1053 | 449 | stdout=fd, stderr=fd) | ||
1054 | 450 | release_tarball = first_path_in_dir( | ||
1055 | 451 | os.path.join(juju_gui_source_dir, 'releases')) | ||
1056 | 452 | else: | ||
1057 | 453 | log('Retrieving Juju GUI release.') | ||
1058 | 454 | if origin == 'url': | ||
1059 | 455 | file_url = version_or_branch | ||
1060 | 456 | else: | ||
1061 | 457 | # Retrieve a release from Launchpad. | ||
1062 | 458 | launchpad = Launchpad.login_anonymously( | ||
1063 | 459 | 'Juju GUI charm', 'production') | ||
1064 | 460 | project = launchpad.projects['juju-gui'] | ||
1065 | 461 | file_url = get_release_file_url(project, origin, version_or_branch) | ||
1066 | 462 | log('Downloading release file from %s.' % file_url) | ||
1067 | 463 | release_tarball = os.path.join(CURRENT_DIR, 'release.tgz') | ||
1068 | 464 | cmd_log(run('curl', '-L', '-o', release_tarball, file_url)) | ||
1069 | 465 | return release_tarball | ||
1070 | 466 | |||
1071 | 467 | |||
1072 | 468 | def fetch_api(juju_api_branch): | ||
1073 | 469 | """Retrieve the Juju branch.""" | ||
1074 | 470 | # Retrieve Juju API source checkout. | ||
1075 | 471 | log('Retrieving Juju API source checkout.') | ||
1076 | 472 | cmd_log(run('rm', '-rf', JUJU_DIR)) | ||
1077 | 473 | cmd_log(bzr_checkout(juju_api_branch, JUJU_DIR)) | ||
1078 | 474 | |||
1079 | 475 | |||
1080 | 476 | def setup_gui(release_tarball): | ||
1081 | 477 | """Set up Juju GUI.""" | ||
1082 | 478 | # Uncompress the release tarball. | ||
1083 | 479 | log('Installing Juju GUI.') | ||
1084 | 480 | release_dir = os.path.join(CURRENT_DIR, 'release') | ||
1085 | 481 | cmd_log(run('rm', '-rf', release_dir)) | ||
1086 | 482 | os.mkdir(release_dir) | ||
1087 | 483 | uncompress = command('tar', '-x', '-z', '-C', release_dir, '-f') | ||
1088 | 484 | cmd_log(uncompress(release_tarball)) | ||
1089 | 485 | # Link the Juju GUI dir to the contents of the release tarball. | ||
1090 | 486 | cmd_log(run('ln', '-sf', first_path_in_dir(release_dir), JUJU_GUI_DIR)) | ||
1091 | 487 | |||
1092 | 488 | |||
1093 | 489 | def setup_apache(): | ||
1094 | 490 | """Set up apache.""" | ||
1095 | 491 | log('Setting up apache.') | ||
1096 | 492 | if not os.path.exists(JUJU_GUI_SITE): | ||
1097 | 493 | cmd_log(run('touch', JUJU_GUI_SITE)) | ||
1098 | 494 | cmd_log(run('chown', 'ubuntu:', JUJU_GUI_SITE)) | ||
1099 | 495 | cmd_log( | ||
1100 | 496 | run('ln', '-s', JUJU_GUI_SITE, | ||
1101 | 497 | '/etc/apache2/sites-enabled/juju-gui')) | ||
1102 | 498 | |||
1103 | 499 | if not os.path.exists(JUJU_GUI_PORTS): | ||
1104 | 500 | cmd_log(run('touch', JUJU_GUI_PORTS)) | ||
1105 | 501 | cmd_log(run('chown', 'ubuntu:', JUJU_GUI_PORTS)) | ||
1106 | 502 | |||
1107 | 503 | with su('root'): | ||
1108 | 504 | run('a2dissite', 'default') | ||
1109 | 505 | run('a2ensite', 'juju-gui') | ||
1110 | 506 | |||
1111 | 507 | |||
1112 | 508 | def save_or_create_certificates( | ||
1113 | 509 | ssl_cert_path, ssl_cert_contents, ssl_key_contents): | ||
1114 | 510 | """Generate the SSL certificates. | ||
1115 | 511 | |||
1116 | 512 | If both *ssl_cert_contents* and *ssl_key_contents* are provided, use them | ||
1117 | 513 | as certificates; otherwise, generate them. | ||
1118 | 514 | |||
1119 | 515 | Also create a pem file, suitable for use in the haproxy configuration, | ||
1120 | 516 | concatenating the key and the certificate files. | ||
1121 | 517 | """ | ||
1122 | 518 | crt_path = os.path.join(ssl_cert_path, 'juju.crt') | ||
1123 | 519 | key_path = os.path.join(ssl_cert_path, 'juju.key') | ||
1124 | 520 | if not os.path.exists(ssl_cert_path): | ||
1125 | 521 | os.makedirs(ssl_cert_path) | ||
1126 | 522 | if ssl_cert_contents and ssl_key_contents: | ||
1127 | 523 | # Save the provided certificates. | ||
1128 | 524 | with open(crt_path, 'w') as cert_file: | ||
1129 | 525 | cert_file.write(ssl_cert_contents) | ||
1130 | 526 | with open(key_path, 'w') as key_file: | ||
1131 | 527 | key_file.write(ssl_key_contents) | ||
1132 | 528 | else: | ||
1133 | 529 | # Generate certificates. | ||
1134 | 530 | # See http://superuser.com/questions/226192/openssl-without-prompt | ||
1135 | 531 | cmd_log(run( | ||
1136 | 532 | 'openssl', 'req', '-new', '-newkey', 'rsa:4096', | ||
1137 | 533 | '-days', '365', '-nodes', '-x509', '-subj', | ||
1138 | 534 | # These are arbitrary test values for the certificate. | ||
1139 | 535 | '/C=GB/ST=Juju/L=GUI/O=Ubuntu/CN=juju.ubuntu.com', | ||
1140 | 536 | '-keyout', key_path, '-out', crt_path)) | ||
1141 | 537 | # Generate the pem file. | ||
1142 | 538 | pem_path = os.path.join(ssl_cert_path, JUJU_PEM) | ||
1143 | 539 | if os.path.exists(pem_path): | ||
1144 | 540 | os.remove(pem_path) | ||
1145 | 541 | with open(pem_path, 'w') as pem_file: | ||
1146 | 542 | shutil.copyfileobj(open(key_path), pem_file) | ||
1147 | 543 | shutil.copyfileobj(open(crt_path), pem_file) | ||
1148 | 544 | |||
1149 | 545 | |||
1150 | 546 | def find_missing_packages(*packages): | ||
1151 | 547 | """Given a list of packages, return the packages which are not installed. | ||
1152 | 548 | """ | ||
1153 | 549 | cache = apt.Cache() | ||
1154 | 550 | missing = set() | ||
1155 | 551 | for pkg_name in packages: | ||
1156 | 552 | try: | ||
1157 | 553 | pkg = cache[pkg_name] | ||
1158 | 554 | except KeyError: | ||
1159 | 555 | missing.add(pkg_name) | ||
1160 | 556 | continue | ||
1161 | 557 | if pkg.is_installed: | ||
1162 | 558 | continue | ||
1163 | 559 | missing.add(pkg_name) | ||
1164 | 560 | return missing | ||
1165 | 561 | |||
1166 | 562 | |||
1167 | 563 | ## Backend support decorators | ||
1168 | 564 | class StopChain(Exception): | ||
1169 | 565 | """Stop Processing a chain command without raising | ||
1170 | 566 | another error. | ||
1171 | 567 | """ | ||
1172 | 568 | |||
1173 | 569 | |||
1174 | 570 | def chain(name, reverse=False): | ||
1175 | 571 | """Helper method to compose a set of strategy objects into | ||
1176 | 572 | a callable. | ||
1177 | 573 | |||
1178 | 574 | Each method is called in the context of its strategy | ||
1179 | 575 | instance (normal OOP) and its argument is the Backend | ||
1180 | 576 | instance. | ||
1181 | 577 | """ | ||
1182 | 578 | # chain method calls through all implementing mixins | ||
1183 | 579 | def method(self): | ||
1184 | 580 | workingset = self.backends | ||
1185 | 581 | if reverse: | ||
1186 | 582 | workingset = reversed(workingset) | ||
1187 | 583 | for backend in workingset: | ||
1188 | 584 | call = backend.__class__.__dict__.get(name) | ||
1189 | 585 | if call: | ||
1190 | 586 | try: | ||
1191 | 587 | call(backend, self) | ||
1192 | 588 | except StopChain: | ||
1193 | 589 | break | ||
1194 | 590 | |||
1195 | 591 | method.__name__ = name | ||
1196 | 592 | return method | ||
1197 | 593 | |||
1198 | 594 | |||
1199 | 595 | def overrideable(f): | ||
1200 | 596 | """Helper to support very limited overrides for use in testing. | ||
1201 | 597 | |||
1202 | 598 | def foo(): | ||
1203 | 599 | return True | ||
1204 | 600 | b = Backend(foo=foo) | ||
1205 | 601 | assert b.foo() is True | ||
1206 | 602 | """ | ||
1207 | 603 | name = f.__name__ | ||
1208 | 604 | |||
1209 | 605 | def overridden(self, *args, **kwargs): | ||
1210 | 606 | if name in self.overrides: | ||
1211 | 607 | return self.overrides[name](*args, **kwargs) | ||
1212 | 608 | else: | ||
1213 | 609 | return f(self, *args, **kwargs) | ||
1214 | 610 | |||
1215 | 611 | overridden.__name__ = name | ||
1216 | 612 | return overridden | ||
1217 | 613 | |||
1218 | 614 | |||
1219 | 615 | def merge(name): | ||
1220 | 616 | """Helper to merge a property from a set of strategy objects | ||
1221 | 617 | into a unified set. | ||
1222 | 618 | """ | ||
1223 | 619 | # return merged property from every providing backend as a set | ||
1224 | 620 | @property | ||
1225 | 621 | def method(self): | ||
1226 | 622 | result = set() | ||
1227 | 623 | for backend in self.backends: | ||
1228 | 624 | segment = backend.__class__.__dict__.get(name) | ||
1229 | 625 | if segment and isinstance(segment, (list, tuple, set)): | ||
1230 | 626 | result |= set(segment) | ||
1231 | 627 | |||
1232 | 628 | return result | ||
1233 | 629 | return method |
Please take a look.