Merge lp:~corey.bryant/charms/trusty/neutron-openvswitch/git-ods into lp:~openstack-charmers-archive/charms/trusty/neutron-openvswitch/next
- Trusty Tahr (14.04)
- git-ods
- Merge into next
Status: | Merged |
---|---|
Merged at revision: | 68 |
Proposed branch: | lp:~corey.bryant/charms/trusty/neutron-openvswitch/git-ods |
Merge into: | lp:~openstack-charmers-archive/charms/trusty/neutron-openvswitch/next |
Diff against target: |
410 lines (+130/-34) 8 files modified
hooks/charmhelpers/contrib/openstack/utils.py (+65/-18) hooks/charmhelpers/contrib/python/packages.py (+28/-5) hooks/charmhelpers/fetch/giturl.py (+7/-5) hooks/neutron_ovs_utils.py (+17/-0) templates/git/upstart/neutron-ovs-cleanup.upstart (+1/-1) templates/git/upstart/neutron-plugin-openvswitch-agent.upstart (+1/-1) tests/basic_deployment.py (+2/-2) unit_tests/test_neutron_ovs_utils.py (+9/-2) |
To merge this branch: | bzr merge lp:~corey.bryant/charms/trusty/neutron-openvswitch/git-ods |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
OpenStack Charmers | Pending | ||
Review via email: mp+257190@code.launchpad.net |
Commit message
Adds support to:
* Use a Python virtualenv when deploying from source
* Default to shallow clone for git (depth=1)
* Pass http_proxy to pip installs if specified
Description of the change
uosci-testing-bot (uosci-testing-bot) wrote : | # |
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #3610 neutron-
UNIT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #3609 neutron-
AMULET OK: passed
Build: http://
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #3867 neutron-
LINT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #3654 neutron-
UNIT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #3876 neutron-
LINT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #3663 neutron-
UNIT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #3658 neutron-
AMULET OK: passed
Build: http://
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #4260 neutron-
LINT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #3986 neutron-
UNIT OK: passed
- 88. By Corey Bryant
-
Sync charm-helpers
- 89. By Corey Bryant
-
Add comment to fix bin symlinks
- 90. By Corey Bryant
-
Sync charm-helpers
- 91. By Corey Bryant
-
Sync charm-helpers
- 92. By Corey Bryant
-
Clone from github in deploy from source amulet tests
- 93. By Corey Bryant
-
Add libyaml-dev as base git package
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #4466 neutron-
LINT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #4191 neutron-
UNIT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #4089 neutron-
AMULET OK: passed
Build: http://
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #4528 neutron-
LINT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #4253 neutron-
UNIT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #4107 neutron-
AMULET OK: passed
Build: http://
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #4536 neutron-
LINT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #4261 neutron-
UNIT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #4115 neutron-
AMULET OK: passed
Build: http://
- 94. By Corey Bryant
-
Sync charm-helpers
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_lint_check #4840 neutron-
LINT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_unit_test #4520 neutron-
UNIT OK: passed
uosci-testing-bot (uosci-testing-bot) wrote : | # |
charm_amulet_test #4331 neutron-
AMULET OK: passed
Build: http://
Preview Diff
1 | === modified file 'hooks/charmhelpers/contrib/openstack/utils.py' |
2 | --- hooks/charmhelpers/contrib/openstack/utils.py 2015-04-20 20:40:25 +0000 |
3 | +++ hooks/charmhelpers/contrib/openstack/utils.py 2015-05-27 13:07:08 +0000 |
4 | @@ -53,9 +53,13 @@ |
5 | get_ipv6_addr |
6 | ) |
7 | |
8 | +from charmhelpers.contrib.python.packages import ( |
9 | + pip_create_virtualenv, |
10 | + pip_install, |
11 | +) |
12 | + |
13 | from charmhelpers.core.host import lsb_release, mounts, umount |
14 | from charmhelpers.fetch import apt_install, apt_cache, install_remote |
15 | -from charmhelpers.contrib.python.packages import pip_install |
16 | from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk |
17 | from charmhelpers.contrib.storage.linux.loopback import ensure_loopback_device |
18 | |
19 | @@ -497,7 +501,17 @@ |
20 | requirements_dir = None |
21 | |
22 | |
23 | -def git_clone_and_install(projects_yaml, core_project): |
24 | +def _git_yaml_load(projects_yaml): |
25 | + """ |
26 | + Load the specified yaml into a dictionary. |
27 | + """ |
28 | + if not projects_yaml: |
29 | + return None |
30 | + |
31 | + return yaml.load(projects_yaml) |
32 | + |
33 | + |
34 | +def git_clone_and_install(projects_yaml, core_project, depth=1): |
35 | """ |
36 | Clone/install all specified OpenStack repositories. |
37 | |
38 | @@ -510,23 +524,22 @@ |
39 | repository: 'git://git.openstack.org/openstack/requirements.git', |
40 | branch: 'stable/icehouse'} |
41 | directory: /mnt/openstack-git |
42 | - http_proxy: http://squid.internal:3128 |
43 | - https_proxy: https://squid.internal:3128 |
44 | + http_proxy: squid-proxy-url |
45 | + https_proxy: squid-proxy-url |
46 | |
47 | The directory, http_proxy, and https_proxy keys are optional. |
48 | """ |
49 | global requirements_dir |
50 | parent_dir = '/mnt/openstack-git' |
51 | - |
52 | - if not projects_yaml: |
53 | - return |
54 | - |
55 | - projects = yaml.load(projects_yaml) |
56 | + http_proxy = None |
57 | + |
58 | + projects = _git_yaml_load(projects_yaml) |
59 | _git_validate_projects_yaml(projects, core_project) |
60 | |
61 | old_environ = dict(os.environ) |
62 | |
63 | if 'http_proxy' in projects.keys(): |
64 | + http_proxy = projects['http_proxy'] |
65 | os.environ['http_proxy'] = projects['http_proxy'] |
66 | if 'https_proxy' in projects.keys(): |
67 | os.environ['https_proxy'] = projects['https_proxy'] |
68 | @@ -534,15 +547,19 @@ |
69 | if 'directory' in projects.keys(): |
70 | parent_dir = projects['directory'] |
71 | |
72 | + pip_create_virtualenv(os.path.join(parent_dir, 'venv')) |
73 | + |
74 | for p in projects['repositories']: |
75 | repo = p['repository'] |
76 | branch = p['branch'] |
77 | if p['name'] == 'requirements': |
78 | - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, |
79 | + repo_dir = _git_clone_and_install_single(repo, branch, depth, |
80 | + parent_dir, http_proxy, |
81 | update_requirements=False) |
82 | requirements_dir = repo_dir |
83 | else: |
84 | - repo_dir = _git_clone_and_install_single(repo, branch, parent_dir, |
85 | + repo_dir = _git_clone_and_install_single(repo, branch, depth, |
86 | + parent_dir, http_proxy, |
87 | update_requirements=True) |
88 | |
89 | os.environ = old_environ |
90 | @@ -574,7 +591,8 @@ |
91 | error_out('openstack-origin-git key \'{}\' is missing'.format(key)) |
92 | |
93 | |
94 | -def _git_clone_and_install_single(repo, branch, parent_dir, update_requirements): |
95 | +def _git_clone_and_install_single(repo, branch, depth, parent_dir, http_proxy, |
96 | + update_requirements): |
97 | """ |
98 | Clone and install a single git repository. |
99 | """ |
100 | @@ -587,7 +605,8 @@ |
101 | |
102 | if not os.path.exists(dest_dir): |
103 | juju_log('Cloning git repo: {}, branch: {}'.format(repo, branch)) |
104 | - repo_dir = install_remote(repo, dest=parent_dir, branch=branch) |
105 | + repo_dir = install_remote(repo, dest=parent_dir, branch=branch, |
106 | + depth=depth) |
107 | else: |
108 | repo_dir = dest_dir |
109 | |
110 | @@ -598,7 +617,12 @@ |
111 | _git_update_requirements(repo_dir, requirements_dir) |
112 | |
113 | juju_log('Installing git repo from dir: {}'.format(repo_dir)) |
114 | - pip_install(repo_dir) |
115 | + if http_proxy: |
116 | + pip_install(repo_dir, proxy=http_proxy, |
117 | + venv=os.path.join(parent_dir, 'venv')) |
118 | + else: |
119 | + pip_install(repo_dir, |
120 | + venv=os.path.join(parent_dir, 'venv')) |
121 | |
122 | return repo_dir |
123 | |
124 | @@ -621,16 +645,27 @@ |
125 | os.chdir(orig_dir) |
126 | |
127 | |
128 | +def git_pip_venv_dir(projects_yaml): |
129 | + """ |
130 | + Return the pip virtualenv path. |
131 | + """ |
132 | + parent_dir = '/mnt/openstack-git' |
133 | + |
134 | + projects = _git_yaml_load(projects_yaml) |
135 | + |
136 | + if 'directory' in projects.keys(): |
137 | + parent_dir = projects['directory'] |
138 | + |
139 | + return os.path.join(parent_dir, 'venv') |
140 | + |
141 | + |
142 | def git_src_dir(projects_yaml, project): |
143 | """ |
144 | Return the directory where the specified project's source is located. |
145 | """ |
146 | parent_dir = '/mnt/openstack-git' |
147 | |
148 | - if not projects_yaml: |
149 | - return |
150 | - |
151 | - projects = yaml.load(projects_yaml) |
152 | + projects = _git_yaml_load(projects_yaml) |
153 | |
154 | if 'directory' in projects.keys(): |
155 | parent_dir = projects['directory'] |
156 | @@ -640,3 +675,15 @@ |
157 | return os.path.join(parent_dir, os.path.basename(p['repository'])) |
158 | |
159 | return None |
160 | + |
161 | + |
162 | +def git_yaml_value(projects_yaml, key): |
163 | + """ |
164 | + Return the value in projects_yaml for the specified key. |
165 | + """ |
166 | + projects = _git_yaml_load(projects_yaml) |
167 | + |
168 | + if key in projects.keys(): |
169 | + return projects[key] |
170 | + |
171 | + return None |
172 | |
173 | === modified file 'hooks/charmhelpers/contrib/python/packages.py' |
174 | --- hooks/charmhelpers/contrib/python/packages.py 2015-02-16 11:22:05 +0000 |
175 | +++ hooks/charmhelpers/contrib/python/packages.py 2015-05-27 13:07:08 +0000 |
176 | @@ -17,8 +17,11 @@ |
177 | # You should have received a copy of the GNU Lesser General Public License |
178 | # along with charm-helpers. If not, see <http://www.gnu.org/licenses/>. |
179 | |
180 | +import os |
181 | +import subprocess |
182 | + |
183 | from charmhelpers.fetch import apt_install, apt_update |
184 | -from charmhelpers.core.hookenv import log |
185 | +from charmhelpers.core.hookenv import charm_dir, log |
186 | |
187 | try: |
188 | from pip import main as pip_execute |
189 | @@ -51,11 +54,15 @@ |
190 | pip_execute(command) |
191 | |
192 | |
193 | -def pip_install(package, fatal=False, upgrade=False, **options): |
194 | +def pip_install(package, fatal=False, upgrade=False, venv=None, **options): |
195 | """Install a python package""" |
196 | - command = ["install"] |
197 | + if venv: |
198 | + venv_python = os.path.join(venv, 'bin/pip') |
199 | + command = [venv_python, "install"] |
200 | + else: |
201 | + command = ["install"] |
202 | |
203 | - available_options = ('proxy', 'src', 'log', "index-url", ) |
204 | + available_options = ('proxy', 'src', 'log', 'index-url', ) |
205 | for option in parse_options(options, available_options): |
206 | command.append(option) |
207 | |
208 | @@ -69,7 +76,10 @@ |
209 | |
210 | log("Installing {} package with options: {}".format(package, |
211 | command)) |
212 | - pip_execute(command) |
213 | + if venv: |
214 | + subprocess.check_call(command) |
215 | + else: |
216 | + pip_execute(command) |
217 | |
218 | |
219 | def pip_uninstall(package, **options): |
220 | @@ -94,3 +104,16 @@ |
221 | """Returns the list of current python installed packages |
222 | """ |
223 | return pip_execute(["list"]) |
224 | + |
225 | + |
226 | +def pip_create_virtualenv(path=None): |
227 | + """Create an isolated Python environment.""" |
228 | + apt_install('python-virtualenv') |
229 | + |
230 | + if path: |
231 | + venv_path = path |
232 | + else: |
233 | + venv_path = os.path.join(charm_dir(), 'venv') |
234 | + |
235 | + if not os.path.exists(venv_path): |
236 | + subprocess.check_call(['virtualenv', venv_path]) |
237 | |
238 | === modified file 'hooks/charmhelpers/fetch/giturl.py' |
239 | --- hooks/charmhelpers/fetch/giturl.py 2015-02-16 11:22:05 +0000 |
240 | +++ hooks/charmhelpers/fetch/giturl.py 2015-05-27 13:07:08 +0000 |
241 | @@ -45,14 +45,16 @@ |
242 | else: |
243 | return True |
244 | |
245 | - def clone(self, source, dest, branch): |
246 | + def clone(self, source, dest, branch, depth=None): |
247 | if not self.can_handle(source): |
248 | raise UnhandledSource("Cannot handle {}".format(source)) |
249 | |
250 | - repo = Repo.clone_from(source, dest) |
251 | - repo.git.checkout(branch) |
252 | + if depth: |
253 | + Repo.clone_from(source, dest, branch=branch, depth=depth) |
254 | + else: |
255 | + Repo.clone_from(source, dest, branch=branch) |
256 | |
257 | - def install(self, source, branch="master", dest=None): |
258 | + def install(self, source, branch="master", dest=None, depth=None): |
259 | url_parts = self.parse_url(source) |
260 | branch_name = url_parts.path.strip("/").split("/")[-1] |
261 | if dest: |
262 | @@ -63,7 +65,7 @@ |
263 | if not os.path.exists(dest_dir): |
264 | mkdir(dest_dir, perms=0o755) |
265 | try: |
266 | - self.clone(source, dest_dir, branch) |
267 | + self.clone(source, dest_dir, branch, depth) |
268 | except GitCommandError as e: |
269 | raise UnhandledSource(e.message) |
270 | except OSError as e: |
271 | |
272 | === modified file 'hooks/neutron_ovs_utils.py' |
273 | --- hooks/neutron_ovs_utils.py 2015-04-17 12:13:07 +0000 |
274 | +++ hooks/neutron_ovs_utils.py 2015-05-27 13:07:08 +0000 |
275 | @@ -9,6 +9,7 @@ |
276 | git_install_requested, |
277 | git_clone_and_install, |
278 | git_src_dir, |
279 | + git_pip_venv_dir, |
280 | ) |
281 | from collections import OrderedDict |
282 | from charmhelpers.contrib.openstack.utils import ( |
283 | @@ -45,6 +46,7 @@ |
284 | BASE_GIT_PACKAGES = [ |
285 | 'libxml2-dev', |
286 | 'libxslt1-dev', |
287 | + 'libyaml-dev', |
288 | 'openvswitch-switch', |
289 | 'python-dev', |
290 | 'python-pip', |
291 | @@ -262,13 +264,27 @@ |
292 | shutil.rmtree(c['dest']) |
293 | shutil.copytree(c['src'], c['dest']) |
294 | |
295 | + # NOTE(coreycb): Need to find better solution than bin symlinks. |
296 | + symlinks = [ |
297 | + {'src': os.path.join(git_pip_venv_dir(projects_yaml), |
298 | + 'bin/neutron-rootwrap'), |
299 | + 'link': '/usr/local/bin/neutron-rootwrap'}, |
300 | + ] |
301 | + |
302 | + for s in symlinks: |
303 | + if os.path.lexists(s['link']): |
304 | + os.remove(s['link']) |
305 | + os.symlink(s['src'], s['link']) |
306 | + |
307 | render('git/neutron_sudoers', '/etc/sudoers.d/neutron_sudoers', {}, |
308 | perms=0o440) |
309 | |
310 | + bin_dir = os.path.join(git_pip_venv_dir(projects_yaml), 'bin') |
311 | neutron_ovs_agent_context = { |
312 | 'service_description': 'Neutron OpenvSwitch Plugin Agent', |
313 | 'charm_name': 'neutron-openvswitch', |
314 | 'process_name': 'neutron-openvswitch-agent', |
315 | + 'executable_name': os.path.join(bin_dir, 'neutron-openvswitch-agent'), |
316 | 'cleanup_process_name': 'neutron-ovs-cleanup', |
317 | 'plugin_config': '/etc/neutron/plugins/ml2/ml2_conf.ini', |
318 | 'log_file': '/var/log/neutron/openvswitch-agent.log', |
319 | @@ -278,6 +294,7 @@ |
320 | 'service_description': 'Neutron OpenvSwitch Cleanup', |
321 | 'charm_name': 'neutron-openvswitch', |
322 | 'process_name': 'neutron-ovs-cleanup', |
323 | + 'executable_name': os.path.join(bin_dir, 'neutron-ovs-cleanup'), |
324 | 'log_file': '/var/log/neutron/ovs-cleanup.log', |
325 | } |
326 | |
327 | |
328 | === modified file 'templates/git/upstart/neutron-ovs-cleanup.upstart' |
329 | --- templates/git/upstart/neutron-ovs-cleanup.upstart 2015-04-13 14:08:56 +0000 |
330 | +++ templates/git/upstart/neutron-ovs-cleanup.upstart 2015-05-27 13:07:08 +0000 |
331 | @@ -11,7 +11,7 @@ |
332 | |
333 | pre-start script |
334 | [ ! -x /usr/bin/{{ process_name }} ] && exit 0 |
335 | - start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \ |
336 | + start-stop-daemon --start --chuid neutron --exec {{ executable_name }} -- \ |
337 | --log-file /var/log/neutron/{{ log_file }} \ |
338 | --config-file /etc/neutron/neutron.conf --verbose |
339 | end script |
340 | |
341 | === modified file 'templates/git/upstart/neutron-plugin-openvswitch-agent.upstart' |
342 | --- templates/git/upstart/neutron-plugin-openvswitch-agent.upstart 2015-04-13 14:08:56 +0000 |
343 | +++ templates/git/upstart/neutron-plugin-openvswitch-agent.upstart 2015-05-27 13:07:08 +0000 |
344 | @@ -13,6 +13,6 @@ |
345 | chown neutron:root /var/run/neutron |
346 | end script |
347 | |
348 | -exec start-stop-daemon --start --chuid neutron --exec /usr/local/bin/{{ process_name }} -- \ |
349 | +exec start-stop-daemon --start --chuid neutron --exec {{ executable_name }} -- \ |
350 | --config-file=/etc/neutron/neutron.conf --config-file={{ plugin_config }} \ |
351 | --log-file={{ log_file }} |
352 | |
353 | === modified file 'tests/basic_deployment.py' |
354 | --- tests/basic_deployment.py 2015-04-13 19:17:58 +0000 |
355 | +++ tests/basic_deployment.py 2015-05-27 13:07:08 +0000 |
356 | @@ -72,10 +72,10 @@ |
357 | openstack_origin_git = { |
358 | 'repositories': [ |
359 | {'name': 'requirements', |
360 | - 'repository': 'git://git.openstack.org/openstack/requirements', |
361 | + 'repository': 'git://github.com/openstack/requirements', |
362 | 'branch': branch}, |
363 | {'name': 'neutron', |
364 | - 'repository': 'git://git.openstack.org/openstack/neutron', |
365 | + 'repository': 'git://github.com/openstack/neutron', |
366 | 'branch': branch}, |
367 | ], |
368 | 'directory': '/mnt/openstack-git', |
369 | |
370 | === modified file 'unit_tests/test_neutron_ovs_utils.py' |
371 | --- unit_tests/test_neutron_ovs_utils.py 2015-04-17 12:13:07 +0000 |
372 | +++ unit_tests/test_neutron_ovs_utils.py 2015-05-27 13:07:08 +0000 |
373 | @@ -265,10 +265,11 @@ |
374 | @patch.object(nutils, 'render') |
375 | @patch('os.path.join') |
376 | @patch('os.path.exists') |
377 | + @patch('os.symlink') |
378 | @patch('shutil.copytree') |
379 | @patch('shutil.rmtree') |
380 | - def test_git_post_install(self, rmtree, copytree, exists, join, render, |
381 | - service_restart, git_src_dir): |
382 | + def test_git_post_install(self, rmtree, copytree, symlink, exists, join, |
383 | + render, service_restart, git_src_dir): |
384 | projects_yaml = openstack_origin_git |
385 | join.return_value = 'joined-string' |
386 | nutils.git_post_install(projects_yaml) |
387 | @@ -278,10 +279,15 @@ |
388 | call('joined-string', '/etc/neutron/rootwrap.d'), |
389 | ] |
390 | copytree.assert_has_calls(expected) |
391 | + expected = [ |
392 | + call('joined-string', '/usr/local/bin/neutron-rootwrap'), |
393 | + ] |
394 | + symlink.assert_has_calls(expected, any_order=True) |
395 | neutron_ovs_agent_context = { |
396 | 'service_description': 'Neutron OpenvSwitch Plugin Agent', |
397 | 'charm_name': 'neutron-openvswitch', |
398 | 'process_name': 'neutron-openvswitch-agent', |
399 | + 'executable_name': 'joined-string', |
400 | 'cleanup_process_name': 'neutron-ovs-cleanup', |
401 | 'plugin_config': '/etc/neutron/plugins/ml2/ml2_conf.ini', |
402 | 'log_file': '/var/log/neutron/openvswitch-agent.log', |
403 | @@ -290,6 +296,7 @@ |
404 | 'service_description': 'Neutron OpenvSwitch Cleanup', |
405 | 'charm_name': 'neutron-openvswitch', |
406 | 'process_name': 'neutron-ovs-cleanup', |
407 | + 'executable_name': 'joined-string', |
408 | 'log_file': '/var/log/neutron/ovs-cleanup.log', |
409 | } |
410 | expected = [ |
charm_lint_check #3823 neutron- openvswitch- next for corey.bryant mp257190
LINT OK: passed
Build: http:// 10.245. 162.77: 8080/job/ charm_lint_ check/3823/