Merge ~chad.smith/cloud-init:pyver-fix into cloud-init:master
- Git
- lp:~chad.smith/cloud-init
- pyver-fix
- Merge into master
Proposed by
Chad Smith
Status: | Merged |
---|---|
Approved by: | Scott Moser |
Approved revision: | a4f5cf8dfc587b59662af3754146d85798263292 |
Merged at revision: | e7c95208451422cfd3da7edfbd67dd271e7aa337 |
Proposed branch: | ~chad.smith/cloud-init:pyver-fix |
Merge into: | cloud-init:master |
Diff against target: |
485 lines (+209/-123) 8 files modified
Makefile (+8/-1) packages/bddeb (+11/-32) packages/brpm (+12/-28) packages/debian/control.in (+2/-9) packages/pkg-deps.json (+1/-0) packages/redhat/cloud-init.spec.in (+6/-15) packages/suse/cloud-init.spec.in (+3/-12) tools/read-dependencies (+166/-26) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Scott Moser | Approve | ||
Server Team CI bot | continuous-integration | Needs Fixing | |
Review via email: mp+325283@code.launchpad.net |
Commit message
Description of the change
makefile: fix python 2/3 detection in the Makefile
Fix detection of python2 in a non-python3 environment.
The previous version ahead ended up using python 3.
To post a comment you must log in.
Revision history for this message
Server Team CI bot (server-team-bot) wrote : | # |
review:
Needs Fixing
(continuous-integration)
Revision history for this message
Scott Moser (smoser) wrote : | # |
I'm going to grab this.
your request included a bunch of unrelated stuff, so I'll just cherry pick the Makefile commit.
thanks.
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | diff --git a/Makefile b/Makefile | |||
2 | index 09cd147..1cec9ae 100644 | |||
3 | --- a/Makefile | |||
4 | +++ b/Makefile | |||
5 | @@ -1,6 +1,6 @@ | |||
6 | 1 | CWD=$(shell pwd) | 1 | CWD=$(shell pwd) |
7 | 2 | PYVER ?= $(shell for p in python3 python2; do \ | 2 | PYVER ?= $(shell for p in python3 python2; do \ |
9 | 3 | out=$(which $$p 2>&1) && echo $$p && exit; done; \ | 3 | out=$$(command -v $$p 2>&1) && echo $$p && exit; done; \ |
10 | 4 | exit 1) | 4 | exit 1) |
11 | 5 | noseopts ?= -v | 5 | noseopts ?= -v |
12 | 6 | 6 | ||
13 | @@ -53,6 +53,12 @@ unittest: clean_pyc | |||
14 | 53 | unittest3: clean_pyc | 53 | unittest3: clean_pyc |
15 | 54 | nosetests3 $(noseopts) tests/unittests | 54 | nosetests3 $(noseopts) tests/unittests |
16 | 55 | 55 | ||
17 | 56 | ci-deps-ubuntu: | ||
18 | 57 | @$(PYVER) $(CWD)/tools/read-dependencies --distro ubuntu --install | ||
19 | 58 | |||
20 | 59 | ci-deps-centos: | ||
21 | 60 | @$(PYVER) $(CWD)/tools/read-dependencies --distro centos --install | ||
22 | 61 | |||
23 | 56 | pip-requirements: | 62 | pip-requirements: |
24 | 57 | @echo "Installing cloud-init dependencies..." | 63 | @echo "Installing cloud-init dependencies..." |
25 | 58 | $(PIP_INSTALL) -r "$@.txt" -q | 64 | $(PIP_INSTALL) -r "$@.txt" -q |
26 | @@ -75,6 +81,7 @@ clean_pyc: | |||
27 | 75 | clean: clean_pyc | 81 | clean: clean_pyc |
28 | 76 | rm -rf /var/log/cloud-init.log /var/lib/cloud/ | 82 | rm -rf /var/log/cloud-init.log /var/lib/cloud/ |
29 | 77 | 83 | ||
30 | 84 | |||
31 | 78 | yaml: | 85 | yaml: |
32 | 79 | @$(PYVER) $(CWD)/tools/validate-yaml.py $(YAML_FILES) | 86 | @$(PYVER) $(CWD)/tools/validate-yaml.py $(YAML_FILES) |
33 | 80 | 87 | ||
34 | diff --git a/packages/bddeb b/packages/bddeb | |||
35 | index f415209..e45af6e 100755 | |||
36 | --- a/packages/bddeb | |||
37 | +++ b/packages/bddeb | |||
38 | @@ -24,19 +24,6 @@ if "avoid-pep8-E402-import-not-top-of-file": | |||
39 | 24 | from cloudinit import templater | 24 | from cloudinit import templater |
40 | 25 | from cloudinit import util | 25 | from cloudinit import util |
41 | 26 | 26 | ||
42 | 27 | # Package names that will showup in requires which have unique package names. | ||
43 | 28 | # Format is '<pypi-name>': {'<python_major_version>': <pkg_name_or_none>, ...}. | ||
44 | 29 | NONSTD_NAMED_PACKAGES = { | ||
45 | 30 | 'argparse': {'2': 'python-argparse', '3': None}, | ||
46 | 31 | 'contextlib2': {'2': 'python-contextlib2', '3': None}, | ||
47 | 32 | 'cheetah': {'2': 'python-cheetah', '3': None}, | ||
48 | 33 | 'pyserial': {'2': 'python-serial', '3': 'python3-serial'}, | ||
49 | 34 | 'pyyaml': {'2': 'python-yaml', '3': 'python3-yaml'}, | ||
50 | 35 | 'six': {'2': 'python-six', '3': 'python3-six'}, | ||
51 | 36 | 'pep8': {'2': 'pep8', '3': 'python3-pep8'}, | ||
52 | 37 | 'pyflakes': {'2': 'pyflakes', '3': 'pyflakes'}, | ||
53 | 38 | } | ||
54 | 39 | |||
55 | 40 | DEBUILD_ARGS = ["-S", "-d"] | 27 | DEBUILD_ARGS = ["-S", "-d"] |
56 | 41 | 28 | ||
57 | 42 | 29 | ||
58 | @@ -59,7 +46,6 @@ def write_debian_folder(root, templ_data, is_python2, cloud_util_deps): | |||
59 | 59 | else: | 46 | else: |
60 | 60 | pyver = "3" | 47 | pyver = "3" |
61 | 61 | python = "python3" | 48 | python = "python3" |
62 | 62 | pkgfmt = "{}-{}" | ||
63 | 63 | 49 | ||
64 | 64 | deb_dir = util.abs_join(root, 'debian') | 50 | deb_dir = util.abs_join(root, 'debian') |
65 | 65 | 51 | ||
66 | @@ -74,30 +60,23 @@ def write_debian_folder(root, templ_data, is_python2, cloud_util_deps): | |||
67 | 74 | params=templ_data) | 60 | params=templ_data) |
68 | 75 | 61 | ||
69 | 76 | # Write out the control file template | 62 | # Write out the control file template |
71 | 77 | reqs = run_helper('read-dependencies').splitlines() | 63 | reqs_output = run_helper( |
72 | 64 | 'read-dependencies', | ||
73 | 65 | args=['--distro', 'debian', '--python-version', pyver]) | ||
74 | 66 | reqs = reqs_output.splitlines() | ||
75 | 78 | test_reqs = run_helper( | 67 | test_reqs = run_helper( |
80 | 79 | 'read-dependencies', ['test-requirements.txt']).splitlines() | 68 | 'read-dependencies', |
81 | 80 | 69 | ['--requirements-file', 'test-requirements.txt', | |
82 | 81 | pypi_pkgs = [p.lower().strip() for p in reqs] | 70 | '--system-pkg-names', '--python-version', pyver]).splitlines() |
79 | 82 | pypi_test_pkgs = [p.lower().strip() for p in test_reqs] | ||
83 | 83 | 71 | ||
84 | 84 | # Map to known packages | ||
85 | 85 | requires = ['cloud-utils | cloud-guest-utils'] if cloud_util_deps else [] | 72 | requires = ['cloud-utils | cloud-guest-utils'] if cloud_util_deps else [] |
96 | 86 | test_requires = [] | 73 | # We consolidate all deps as Build-Depends as our package build runs all |
97 | 87 | lists = ((pypi_pkgs, requires), (pypi_test_pkgs, test_requires)) | 74 | # tests so we need all runtime dependencies anyway. |
98 | 88 | for pypilist, target in lists: | 75 | requires.extend(reqs + test_reqs + [python]) |
89 | 89 | for p in pypilist: | ||
90 | 90 | if p in NONSTD_NAMED_PACKAGES: | ||
91 | 91 | if NONSTD_NAMED_PACKAGES[p][pyver]: | ||
92 | 92 | target.append(NONSTD_NAMED_PACKAGES[p][pyver]) | ||
93 | 93 | else: # Then standard package prefix | ||
94 | 94 | target.append(pkgfmt.format(python, p)) | ||
95 | 95 | |||
99 | 96 | templater.render_to_file(util.abs_join(find_root(), | 76 | templater.render_to_file(util.abs_join(find_root(), |
100 | 97 | 'packages', 'debian', 'control.in'), | 77 | 'packages', 'debian', 'control.in'), |
101 | 98 | util.abs_join(deb_dir, 'control'), | 78 | util.abs_join(deb_dir, 'control'), |
104 | 99 | params={'requires': ','.join(requires), | 79 | params={'build_depends': ','.join(requires), |
103 | 100 | 'test_requires': ','.join(test_requires), | ||
105 | 101 | 'python': python}) | 80 | 'python': python}) |
106 | 102 | 81 | ||
107 | 103 | templater.render_to_file(util.abs_join(find_root(), | 82 | templater.render_to_file(util.abs_join(find_root(), |
108 | diff --git a/packages/brpm b/packages/brpm | |||
109 | index 89696ab..b8e7c72 100755 | |||
110 | --- a/packages/brpm | |||
111 | +++ b/packages/brpm | |||
112 | @@ -27,17 +27,6 @@ if "avoid-pep8-E402-import-not-top-of-file": | |||
113 | 27 | from cloudinit import templater | 27 | from cloudinit import templater |
114 | 28 | from cloudinit import util | 28 | from cloudinit import util |
115 | 29 | 29 | ||
116 | 30 | # Map python requirements to package names. If a match isn't found | ||
117 | 31 | # here, we assume 'python-<pypi_name>'. | ||
118 | 32 | PACKAGE_MAP = { | ||
119 | 33 | 'redhat': { | ||
120 | 34 | 'pyserial': 'pyserial', | ||
121 | 35 | 'pyyaml': 'PyYAML', | ||
122 | 36 | }, | ||
123 | 37 | 'suse': { | ||
124 | 38 | 'pyyaml': 'python-yaml', | ||
125 | 39 | } | ||
126 | 40 | } | ||
127 | 41 | 30 | ||
128 | 42 | # Subdirectories of the ~/rpmbuild dir | 31 | # Subdirectories of the ~/rpmbuild dir |
129 | 43 | RPM_BUILD_SUBDIRS = ['BUILD', 'RPMS', 'SOURCES', 'SPECS', 'SRPMS'] | 32 | RPM_BUILD_SUBDIRS = ['BUILD', 'RPMS', 'SOURCES', 'SPECS', 'SRPMS'] |
130 | @@ -57,19 +46,15 @@ def read_dependencies(): | |||
131 | 57 | '''Returns the Python depedencies from requirements.txt. This explicitly | 46 | '''Returns the Python depedencies from requirements.txt. This explicitly |
132 | 58 | removes 'argparse' from the list of requirements for python >= 2.7, | 47 | removes 'argparse' from the list of requirements for python >= 2.7, |
133 | 59 | because with 2.7 argparse became part of the standard library.''' | 48 | because with 2.7 argparse became part of the standard library.''' |
147 | 60 | stdout = run_helper('read-dependencies') | 49 | pkg_deps = run_helper( |
148 | 61 | return [p.lower().strip() for p in stdout.splitlines() | 50 | 'read-dependencies', args=['--distro', 'redhat']).splitlines() |
149 | 62 | if p != 'argparse' or (p == 'argparse' and | 51 | test_deps = run_helper( |
150 | 63 | sys.version_info[0:2] < (2, 7))] | 52 | 'read-dependencies', args=[ |
151 | 64 | 53 | '--requirements-file', 'test-requirements.txt', | |
152 | 65 | 54 | '--system-pkg-names']).splitlines() | |
153 | 66 | def translate_dependencies(deps, distro): | 55 | return [pkg_dep for pkg_dep in pkg_deps + test_deps |
154 | 67 | '''Maps python requirements into package names. We assume | 56 | if p != 'python-argparse' or (p == 'python-argparse' and |
155 | 68 | python-<pypi_name> for packages not listed explicitly in | 57 | sys.version_info[0:2] < (2, 7))] |
143 | 69 | PACKAGE_MAP.''' | ||
144 | 70 | return [PACKAGE_MAP[distro][req] | ||
145 | 71 | if req in PACKAGE_MAP[distro] else 'python-%s' % req | ||
146 | 72 | for req in deps] | ||
156 | 73 | 58 | ||
157 | 74 | 59 | ||
158 | 75 | def read_version(): | 60 | def read_version(): |
159 | @@ -99,10 +84,9 @@ def generate_spec_contents(args, version_data, tmpl_fn, top_dir, arc_fn): | |||
160 | 99 | rpm_upstream_version = version_data['version'] | 84 | rpm_upstream_version = version_data['version'] |
161 | 100 | subs['rpm_upstream_version'] = rpm_upstream_version | 85 | subs['rpm_upstream_version'] = rpm_upstream_version |
162 | 101 | 86 | ||
167 | 102 | # Map to known packages | 87 | deps = read_dependencies() |
168 | 103 | python_deps = read_dependencies() | 88 | subs['buildrequires'] = deps |
169 | 104 | package_deps = translate_dependencies(python_deps, args.distro) | 89 | subs['requires'] = deps |
166 | 105 | subs['requires'] = package_deps | ||
170 | 106 | 90 | ||
171 | 107 | if args.boot == 'sysvinit': | 91 | if args.boot == 'sysvinit': |
172 | 108 | subs['sysvinit'] = True | 92 | subs['sysvinit'] = True |
173 | diff --git a/packages/debian/control.in b/packages/debian/control.in | |||
174 | index 6c39d53..265b261 100644 | |||
175 | --- a/packages/debian/control.in | |||
176 | +++ b/packages/debian/control.in | |||
177 | @@ -3,20 +3,13 @@ Source: cloud-init | |||
178 | 3 | Section: admin | 3 | Section: admin |
179 | 4 | Priority: optional | 4 | Priority: optional |
180 | 5 | Maintainer: Scott Moser <smoser@ubuntu.com> | 5 | Maintainer: Scott Moser <smoser@ubuntu.com> |
187 | 6 | Build-Depends: debhelper (>= 9), | 6 | Build-Depends: ${build_depends} |
182 | 7 | dh-python, | ||
183 | 8 | dh-systemd, | ||
184 | 9 | ${python}, | ||
185 | 10 | ${test_requires}, | ||
186 | 11 | ${requires} | ||
188 | 12 | XS-Python-Version: all | 7 | XS-Python-Version: all |
189 | 13 | Standards-Version: 3.9.6 | 8 | Standards-Version: 3.9.6 |
190 | 14 | 9 | ||
191 | 15 | Package: cloud-init | 10 | Package: cloud-init |
192 | 16 | Architecture: all | 11 | Architecture: all |
196 | 17 | Depends: procps, | 12 | Depends: ${misc:Depends}, |
194 | 18 | ${python}, | ||
195 | 19 | ${misc:Depends}, | ||
197 | 20 | ${${python}:Depends} | 13 | ${${python}:Depends} |
198 | 21 | Recommends: eatmydata, sudo, software-properties-common, gdisk | 14 | Recommends: eatmydata, sudo, software-properties-common, gdisk |
199 | 22 | XB-Python-Version: ${python:Versions} | 15 | XB-Python-Version: ${python:Versions} |
200 | diff --git a/packages/pkg-deps.json b/packages/pkg-deps.json | |||
201 | 23 | new file mode 100644 | 16 | new file mode 100644 |
202 | index 0000000..042cde7 | |||
203 | --- /dev/null | |||
204 | +++ b/packages/pkg-deps.json | |||
205 | @@ -0,0 +1 @@ | |||
206 | 1 | {"debian": {"build-requires": ["debhelper", "dh-python", "dh-systemd"], "requires": ["procps"], "renames": {"argparse": {"2": "python-argparse", "3": null},"contextlib2": {"2": "python-contextlib2", "3": null}, "cheetah": {"2": "python-cheetah", "3": null}, "pyserial": {"2": "python-serial", "3": "python3-serial"}, "pyyaml": {"2": "python-yaml", "3": "python3-yaml"}, "six": {"2": "python-six", "3": "python3-six"}, "pep8": {"2": "pep8", "3": "python3-pep8"}, "pyflakes": {"2": "pyflakes", "3": "pyflakes"}}}, "redhat": {"requires": ["shadow-utils", "rsyslog", "iproute", "e2fsprogs", "net-tools", "procps", "shadow-utils", "sudo >= 1.7.2p2-3"], "build-requires": ["python-devel", "python-setuptools", "python-cheetah"], "renames": {"pyserial": {"2": "pyserial", "3": null}, "pyyaml": {"2": "PyYAML", "3": null}}}, "suse": {"build-requires": ["fdupes", "filesystem", "python-devel", "python-setuptools", "python-cheetah"], "requires": ["iproute2", "e2fsprogs", "net-tools", "procps", "sudo"], "renames": {"pyyaml": {"2": "python-yaml", "3": null}}}} | ||
207 | diff --git a/packages/redhat/cloud-init.spec.in b/packages/redhat/cloud-init.spec.in | |||
208 | index fd3cf93..a62d860 100644 | |||
209 | --- a/packages/redhat/cloud-init.spec.in | |||
210 | +++ b/packages/redhat/cloud-init.spec.in | |||
211 | @@ -18,21 +18,12 @@ Source0: ${archive_name} | |||
212 | 18 | BuildArch: noarch | 18 | BuildArch: noarch |
213 | 19 | BuildRoot: %{_tmppath} | 19 | BuildRoot: %{_tmppath} |
214 | 20 | 20 | ||
230 | 21 | BuildRequires: python-devel | 21 | # Install 'dynamic' build reqs from *requirements.txt and pkg-deps.json |
231 | 22 | BuildRequires: python-setuptools | 22 | #for $r in $buildrequires |
232 | 23 | BuildRequires: python-cheetah | 23 | BuildRequires: ${r} |
233 | 24 | 24 | #end for | |
234 | 25 | # System util packages needed | 25 | |
235 | 26 | Requires: shadow-utils | 26 | # Install 'dynamic' runtime reqs from *requirements.txt and pkg-deps.json |
221 | 27 | Requires: rsyslog | ||
222 | 28 | Requires: iproute | ||
223 | 29 | Requires: e2fsprogs | ||
224 | 30 | Requires: net-tools | ||
225 | 31 | Requires: procps | ||
226 | 32 | Requires: shadow-utils | ||
227 | 33 | Requires: sudo >= 1.7.2p2-3 | ||
228 | 34 | |||
229 | 35 | # Install pypi 'dynamic' requirements | ||
236 | 36 | #for $r in $requires | 27 | #for $r in $requires |
237 | 37 | Requires: ${r} | 28 | Requires: ${r} |
238 | 38 | #end for | 29 | #end for |
239 | diff --git a/packages/suse/cloud-init.spec.in b/packages/suse/cloud-init.spec.in | |||
240 | index 6ce0be8..444401c 100644 | |||
241 | --- a/packages/suse/cloud-init.spec.in | |||
242 | +++ b/packages/suse/cloud-init.spec.in | |||
243 | @@ -22,11 +22,9 @@ BuildRoot: %{_tmppath}/%{name}-%{version}-build | |||
244 | 22 | BuildArch: noarch | 22 | BuildArch: noarch |
245 | 23 | %endif | 23 | %endif |
246 | 24 | 24 | ||
252 | 25 | BuildRequires: fdupes | 25 | #for $r in $buildrequires |
253 | 26 | BuildRequires: filesystem | 26 | BuildRequires: ${r} |
254 | 27 | BuildRequires: python-devel | 27 | #end for |
250 | 28 | BuildRequires: python-setuptools | ||
251 | 29 | BuildRequires: python-cheetah | ||
255 | 30 | 28 | ||
256 | 31 | %if 0%{?suse_version} && 0%{?suse_version} <= 1210 | 29 | %if 0%{?suse_version} && 0%{?suse_version} <= 1210 |
257 | 32 | %define initsys sysvinit | 30 | %define initsys sysvinit |
258 | @@ -34,13 +32,6 @@ BuildRequires: python-cheetah | |||
259 | 34 | %define initsys systemd | 32 | %define initsys systemd |
260 | 35 | %endif | 33 | %endif |
261 | 36 | 34 | ||
262 | 37 | # System util packages needed | ||
263 | 38 | Requires: iproute2 | ||
264 | 39 | Requires: e2fsprogs | ||
265 | 40 | Requires: net-tools | ||
266 | 41 | Requires: procps | ||
267 | 42 | Requires: sudo | ||
268 | 43 | |||
269 | 44 | # Install pypi 'dynamic' requirements | 35 | # Install pypi 'dynamic' requirements |
270 | 45 | #for $r in $requires | 36 | #for $r in $requires |
271 | 46 | Requires: ${r} | 37 | Requires: ${r} |
272 | diff --git a/tools/read-dependencies b/tools/read-dependencies | |||
273 | index f434905..e01e718 100755 | |||
274 | --- a/tools/read-dependencies | |||
275 | +++ b/tools/read-dependencies | |||
276 | @@ -1,43 +1,183 @@ | |||
277 | 1 | #!/usr/bin/env python | 1 | #!/usr/bin/env python |
278 | 2 | """List pip dependencies or system package dependencies for cloud-init.""" | ||
279 | 2 | 3 | ||
280 | 3 | # You might be tempted to rewrite this as a shell script, but you | 4 | # You might be tempted to rewrite this as a shell script, but you |
281 | 4 | # would be surprised to discover that things like 'egrep' or 'sed' may | 5 | # would be surprised to discover that things like 'egrep' or 'sed' may |
282 | 5 | # differ between Linux and *BSD. | 6 | # differ between Linux and *BSD. |
283 | 6 | 7 | ||
284 | 8 | try: | ||
285 | 9 | from argparse import ArgumentParser | ||
286 | 10 | except ImportError: | ||
287 | 11 | raise RuntimeError( | ||
288 | 12 | 'Could not import python-argparse. Please install python-argparse ' | ||
289 | 13 | 'package to continue') | ||
290 | 14 | |||
291 | 15 | import json | ||
292 | 7 | import os | 16 | import os |
293 | 8 | import re | 17 | import re |
294 | 9 | import sys | ||
295 | 10 | import subprocess | 18 | import subprocess |
296 | 19 | import sys | ||
297 | 11 | 20 | ||
298 | 12 | if 'CLOUD_INIT_TOP_D' in os.environ: | ||
299 | 13 | topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D')) | ||
300 | 14 | else: | ||
301 | 15 | topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) | ||
302 | 16 | 21 | ||
318 | 17 | for fname in ("setup.py", "requirements.txt"): | 22 | # Map the appropriate package dir needed for each distro choice |
319 | 18 | if not os.path.isfile(os.path.join(topd, fname)): | 23 | DISTRO_PKG_TYPE_MAP = { |
320 | 19 | sys.stderr.write("Unable to locate '%s' file that should " | 24 | 'centos': 'redhat', |
321 | 20 | "exist in cloud-init root directory." % fname) | 25 | 'redhat': 'redhat', |
322 | 21 | sys.exit(1) | 26 | 'debian': 'debian', |
323 | 22 | 27 | 'ubuntu': 'debian', | |
324 | 23 | if len(sys.argv) > 1: | 28 | 'opensuse': 'suse', |
325 | 24 | reqfile = sys.argv[1] | 29 | 'suse': 'suse' |
326 | 25 | else: | 30 | } |
327 | 26 | reqfile = "requirements.txt" | 31 | |
328 | 27 | 32 | DISTRO_INSTALL_PKG_CMD = { | |
329 | 28 | with open(os.path.join(topd, reqfile), "r") as fp: | 33 | 'cent': ['yum', 'install'], |
330 | 29 | for line in fp: | 34 | 'redhat': ['yum', 'install'], |
331 | 30 | line = line.strip() | 35 | 'debian': ['apt', 'install'], |
332 | 31 | if not line or line.startswith("#"): | 36 | 'ubuntu': ['apt', 'install'], |
333 | 37 | 'opensuse': ['zypper', 'install'], | ||
334 | 38 | 'suse': ['zypper', 'install'] | ||
335 | 39 | } | ||
336 | 40 | |||
337 | 41 | PY_26_ONLY = ['argparse'] | ||
338 | 42 | |||
339 | 43 | |||
340 | 44 | # JSON definition of distro-specific package dependencies | ||
341 | 45 | DISTRO_PKG_DEPS_PATH = "packages/pkg-deps.json" | ||
342 | 46 | |||
343 | 47 | |||
344 | 48 | def get_parser(): | ||
345 | 49 | """Return an argument parser for this command.""" | ||
346 | 50 | parser = ArgumentParser(description=__doc__) | ||
347 | 51 | parser.add_argument( | ||
348 | 52 | '-r', '--requirements-file', type=str, dest='req_file', | ||
349 | 53 | default='requirements.txt', help='The pip-style requirements file') | ||
350 | 54 | parser.add_argument( | ||
351 | 55 | '-d', '--distro', type=str, choices=DISTRO_PKG_TYPE_MAP.keys(), | ||
352 | 56 | help='The name of the distro to generate package deps for.') | ||
353 | 57 | parser.add_argument( | ||
354 | 58 | '-s', '--system-pkg-names', action='store_true', default=False, | ||
355 | 59 | dest='system_pkg_names', | ||
356 | 60 | help='The name of the distro to generate package deps for.') | ||
357 | 61 | parser.add_argument( | ||
358 | 62 | '-i', '--install', action='store_true', default=False, | ||
359 | 63 | dest='install', | ||
360 | 64 | help='When specified, install the required system packages.') | ||
361 | 65 | parser.add_argument( | ||
362 | 66 | '-v', '--python-version', type=str, dest='python_version', default="2", | ||
363 | 67 | choices=["2", "3"], | ||
364 | 68 | help='The version of python we want to generate system package ' | ||
365 | 69 | 'dependencies for.') | ||
366 | 70 | return parser | ||
367 | 71 | |||
368 | 72 | |||
369 | 73 | def get_package_deps_from_json(topdir, distro): | ||
370 | 74 | """Get a dict of build and runtime package requirements for a distro. | ||
371 | 75 | |||
372 | 76 | @param topdir: The root directory in which to search for the | ||
373 | 77 | DISTRO_PKG_DEPS_PATH json blob of package requirements information. | ||
374 | 78 | @param distro: The specific distribution shortname to pull dependencies | ||
375 | 79 | for. | ||
376 | 80 | @return: Dict containing "requires", "build-requires" and "rename" lists | ||
377 | 81 | for a given distribution. | ||
378 | 82 | """ | ||
379 | 83 | with open(os.path.join(topdir, DISTRO_PKG_DEPS_PATH), 'r') as stream: | ||
380 | 84 | deps = json.loads(stream.read()) | ||
381 | 85 | if distro is None: | ||
382 | 86 | return {} | ||
383 | 87 | return deps[DISTRO_PKG_TYPE_MAP[distro]] | ||
384 | 88 | |||
385 | 89 | |||
386 | 90 | def parse_pip_requirements(requirements_path): | ||
387 | 91 | """Return the pip requirement names from pip-style requirements_path.""" | ||
388 | 92 | dep_names = [] | ||
389 | 93 | with open(requirements_path, "r") as fp: | ||
390 | 94 | for line in fp: | ||
391 | 95 | line = line.strip() | ||
392 | 96 | if not line or line.startswith("#"): | ||
393 | 97 | continue | ||
394 | 98 | |||
395 | 99 | # remove pip-style markers | ||
396 | 100 | dep = line.split(';')[0] | ||
397 | 101 | |||
398 | 102 | # remove version requirements | ||
399 | 103 | if re.search('[>=.<]+', dep): | ||
400 | 104 | dep_names.append(re.split("[>=.<]*", dep)[0].strip()) | ||
401 | 105 | else: | ||
402 | 106 | dep_names.append(dep) | ||
403 | 107 | return dep_names | ||
404 | 108 | |||
405 | 109 | |||
406 | 110 | def translate_pip_to_system_pkg(pip_requires, renames, python_ver="2"): | ||
407 | 111 | """Translate pip package names to distro-specific package names. | ||
408 | 112 | |||
409 | 113 | @param pip_requires: List of versionless pip package names to translate. | ||
410 | 114 | @param renames: Dict containg special case renames from pip name to system | ||
411 | 115 | package name for the distro. | ||
412 | 116 | """ | ||
413 | 117 | if python_ver == "2": | ||
414 | 118 | prefix = "python-" | ||
415 | 119 | else: | ||
416 | 120 | prefix = "python3-" | ||
417 | 121 | standard_pkg_name = "{0}{1}" | ||
418 | 122 | translated_names = [] | ||
419 | 123 | for pip_name in pip_requires: | ||
420 | 124 | pip_name = pip_name.lower() | ||
421 | 125 | if pip_name in PY_26_ONLY and sys.version_info[0:2] > (2, 6): | ||
422 | 126 | # Skip PY26 pkg deps unless we are actually on python 2.6 | ||
423 | 32 | continue | 127 | continue |
424 | 128 | # Find a rename if present for the distro package and python version | ||
425 | 129 | rename = renames.get(pip_name, {}).get(python_ver, None) | ||
426 | 130 | if rename: | ||
427 | 131 | translated_names.append(rename) | ||
428 | 132 | else: | ||
429 | 133 | translated_names.append( | ||
430 | 134 | standard_pkg_name.format(prefix, pip_name)) | ||
431 | 135 | return translated_names | ||
432 | 136 | |||
433 | 137 | |||
434 | 138 | def main(distro): | ||
435 | 139 | parser = get_parser() | ||
436 | 140 | args = parser.parse_args() | ||
437 | 141 | if 'CLOUD_INIT_TOP_D' in os.environ: | ||
438 | 142 | topd = os.path.realpath(os.environ.get('CLOUD_INIT_TOP_D')) | ||
439 | 143 | else: | ||
440 | 144 | topd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) | ||
441 | 145 | req_path = os.path.join(topd, args.req_file) | ||
442 | 146 | if not os.path.isfile(req_path): | ||
443 | 147 | sys.stderr.write("Unable to locate '%s' file that should " | ||
444 | 148 | "exist in cloud-init root directory." % req_path) | ||
445 | 149 | return 1 | ||
446 | 150 | pip_pkg_names = parse_pip_requirements(req_path) | ||
447 | 151 | deps_from_json = get_package_deps_from_json(topd, args.distro) | ||
448 | 152 | renames = deps_from_json.get('renames', {}) | ||
449 | 153 | translated_pip_names = translate_pip_to_system_pkg( | ||
450 | 154 | pip_pkg_names, renames, args.python_version) | ||
451 | 155 | all_deps = [] | ||
452 | 156 | if args.distro: | ||
453 | 157 | all_deps.extend( | ||
454 | 158 | translated_pip_names + deps_from_json['requires'] + | ||
455 | 159 | deps_from_json['build-requires']) | ||
456 | 160 | else: | ||
457 | 161 | if args.system_pkg_names: | ||
458 | 162 | all_deps = translated_pip_names | ||
459 | 163 | else: | ||
460 | 164 | all_deps = pip_pkg_names | ||
461 | 165 | if args.install: | ||
462 | 166 | pkg_install(all_deps, args.distro) | ||
463 | 167 | else: | ||
464 | 168 | print('\n'.join(all_deps)) | ||
465 | 169 | |||
466 | 33 | 170 | ||
469 | 34 | # remove pip-style markers | 171 | def pkg_install(pkg_list, distro): |
470 | 35 | dep = line.split(';')[0] | 172 | """Install a list of packages using the DISTRO_INSTALL_PKG_CMD""" |
471 | 173 | print("Installing deps:", ' '.join(pkg_list)) | ||
472 | 174 | cmd = ['sudo'] + DISTRO_INSTALL_PKG_CMD[distro] + pkg_list | ||
473 | 175 | subprocess.check_call(cmd) | ||
474 | 36 | 176 | ||
475 | 37 | # remove version requirements | ||
476 | 38 | dep = re.split("[>=.<]*", dep)[0].strip() | ||
477 | 39 | print(dep) | ||
478 | 40 | 177 | ||
480 | 41 | sys.exit(0) | 178 | if __name__ == "__main__": |
481 | 179 | parser = get_parser() | ||
482 | 180 | args = parser.parse_args() | ||
483 | 181 | sys.exit(main(args.distro)) | ||
484 | 42 | 182 | ||
485 | 43 | # vi: ts=4 expandtab | 183 | # vi: ts=4 expandtab |
FAILED: Continuous integration, rev:a4f5cf8dfc5 87b59662af37541 46d85798263292 /jenkins. ubuntu. com/server/ job/cloud- init-ci/ 478/ /jenkins. ubuntu. com/server/ job/cloud- init-ci/ nodes=metal- amd64/478/ console /jenkins. ubuntu. com/server/ job/cloud- init-ci/ nodes=metal- arm64/478/ console /jenkins. ubuntu. com/server/ job/cloud- init-ci/ nodes=metal- ppc64el/ 478/console /jenkins. ubuntu. com/server/ job/cloud- init-ci/ nodes=metal- s390x/478/ console /jenkins. ubuntu. com/server/ job/cloud- init-ci/ nodes=vm- i386/478/ console
https:/
Executed test runs:
FAILURE: https:/
FAILURE: https:/
FAILURE: https:/
FAILURE: https:/
FAILURE: https:/
Click here to trigger a rebuild: /jenkins. ubuntu. com/server/ job/cloud- init-ci/ 478/rebuild
https:/