Merge lp:~andreserl/maas/revert_rev5602_lp1686169_2.1 into lp:maas/2.1
- revert_rev5602_lp1686169_2.1
- Merge into 2.1
Proposed by
Andres Rodriguez
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | Andres Rodriguez | ||||
Approved revision: | no longer in the source branch. | ||||
Merged at revision: | 5603 | ||||
Proposed branch: | lp:~andreserl/maas/revert_rev5602_lp1686169_2.1 | ||||
Merge into: | lp:maas/2.1 | ||||
Diff against target: |
637 lines (+160/-251) 4 files modified
src/maasserver/bootresources.py (+63/-2) src/maasserver/tests/test_bootresources.py (+86/-59) src/provisioningserver/import_images/download_descriptions.py (+0/-72) src/provisioningserver/import_images/tests/test_download_descriptions.py (+11/-118) |
||||
To merge this branch: | bzr merge lp:~andreserl/maas/revert_rev5602_lp1686169_2.1 | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Andres Rodriguez (community) | Approve | ||
Review via email: mp+323264@code.launchpad.net |
This proposal supersedes a proposal from 2017-04-26.
Commit message
Revert rev5602, which fixed LP: #1686169. This causes rack controllers to fail importing images
Description of the change
To post a comment you must log in.
Revision history for this message
Andres Rodriguez (andreserl) wrote : | # |
Revision history for this message
Andres Rodriguez (andreserl) : | # |
review:
Approve
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'src/maasserver/bootresources.py' | |||
2 | --- src/maasserver/bootresources.py 2017-04-26 00:27:23 +0000 | |||
3 | +++ src/maasserver/bootresources.py 2017-04-26 19:06:22 +0000 | |||
4 | @@ -18,6 +18,7 @@ | |||
5 | 18 | from datetime import timedelta | 18 | from datetime import timedelta |
6 | 19 | from operator import itemgetter | 19 | from operator import itemgetter |
7 | 20 | import os | 20 | import os |
8 | 21 | import re | ||
9 | 21 | from subprocess import CalledProcessError | 22 | from subprocess import CalledProcessError |
10 | 22 | from textwrap import dedent | 23 | from textwrap import dedent |
11 | 23 | import threading | 24 | import threading |
12 | @@ -80,7 +81,6 @@ | |||
13 | 80 | from provisioningserver.import_images.download_descriptions import ( | 81 | from provisioningserver.import_images.download_descriptions import ( |
14 | 81 | download_all_image_descriptions, | 82 | download_all_image_descriptions, |
15 | 82 | image_passes_filter, | 83 | image_passes_filter, |
16 | 83 | validate_product, | ||
17 | 84 | ) | 84 | ) |
18 | 85 | from provisioningserver.import_images.helpers import ( | 85 | from provisioningserver.import_images.helpers import ( |
19 | 86 | get_os_from_product, | 86 | get_os_from_product, |
20 | @@ -1019,6 +1019,12 @@ | |||
21 | 1019 | # will be downloaded from simplestreams. | 1019 | # will be downloaded from simplestreams. |
22 | 1020 | 'max_items': 1, | 1020 | 'max_items': 1, |
23 | 1021 | }) | 1021 | }) |
24 | 1022 | # Compile a regex to validate bootloader product names. This only | ||
25 | 1023 | # allows V1 bootloaders. | ||
26 | 1024 | self.bootloader_regex = re.compile('.*:1:.*', re.IGNORECASE) | ||
27 | 1025 | # Compile a regex to validate Ubuntu product names. This only allows | ||
28 | 1026 | # V2 and V3 Ubuntu products. | ||
29 | 1027 | self.ubuntu_regex = re.compile('.*:v[23]:.*', re.IGNORECASE) | ||
30 | 1022 | 1028 | ||
31 | 1023 | def load_products(self, path=None, content_id=None): | 1029 | def load_products(self, path=None, content_id=None): |
32 | 1024 | """Overridable from `BasicMirrorWriter`.""" | 1030 | """Overridable from `BasicMirrorWriter`.""" |
33 | @@ -1032,6 +1038,59 @@ | |||
34 | 1032 | return self.product_mapping.contains( | 1038 | return self.product_mapping.contains( |
35 | 1033 | sutil.products_exdata(src, pedigree)) | 1039 | sutil.products_exdata(src, pedigree)) |
36 | 1034 | 1040 | ||
37 | 1041 | def _validate_ubuntu(self, data, product_name): | ||
38 | 1042 | osystem = data.get('os') | ||
39 | 1043 | if 'ubuntu' not in osystem.lower(): | ||
40 | 1044 | # It's not an Ubuntu product, nothing to validate. | ||
41 | 1045 | return True | ||
42 | 1046 | elif self.ubuntu_regex.search(product_name) is None: | ||
43 | 1047 | # Only insert v2 or v3 Ubuntu products. | ||
44 | 1048 | return False | ||
45 | 1049 | else: | ||
46 | 1050 | return True | ||
47 | 1051 | |||
48 | 1052 | def _validate_bootloader(self, data, product_name): | ||
49 | 1053 | bootloader_type = data.get('bootloader-type') | ||
50 | 1054 | if bootloader_type is None: | ||
51 | 1055 | # It's not a bootloader, nothing to validate | ||
52 | 1056 | return True | ||
53 | 1057 | if self.bootloader_regex.search(product_name) is None: | ||
54 | 1058 | # Only insert V1 bootloaders from the stream | ||
55 | 1059 | return False | ||
56 | 1060 | # Validate MAAS supports the specific bootloader_type, os, arch | ||
57 | 1061 | # combination. | ||
58 | 1062 | SUPPORTED_BOOTLOADERS = { | ||
59 | 1063 | 'pxe': [ | ||
60 | 1064 | { | ||
61 | 1065 | 'os': 'pxelinux', | ||
62 | 1066 | 'arch': 'i386', | ||
63 | 1067 | } | ||
64 | 1068 | ], | ||
65 | 1069 | 'uefi': [ | ||
66 | 1070 | { | ||
67 | 1071 | 'os': 'grub-efi-signed', | ||
68 | 1072 | 'arch': 'amd64', | ||
69 | 1073 | }, | ||
70 | 1074 | { | ||
71 | 1075 | 'os': 'grub-efi', | ||
72 | 1076 | 'arch': 'arm64', | ||
73 | 1077 | } | ||
74 | 1078 | ], | ||
75 | 1079 | 'open-firmware': [ | ||
76 | 1080 | { | ||
77 | 1081 | 'os': 'grub-ieee1275', | ||
78 | 1082 | 'arch': 'ppc64el', | ||
79 | 1083 | } | ||
80 | 1084 | ], | ||
81 | 1085 | } | ||
82 | 1086 | for bootloader in SUPPORTED_BOOTLOADERS.get(bootloader_type, []): | ||
83 | 1087 | if ( | ||
84 | 1088 | data.get('os') == bootloader['os'] and | ||
85 | 1089 | data.get('arch') == bootloader['arch']): | ||
86 | 1090 | return True | ||
87 | 1091 | # Bootloader not supported, ignore | ||
88 | 1092 | return False | ||
89 | 1093 | |||
90 | 1035 | def insert_item(self, data, src, target, pedigree, contentsource): | 1094 | def insert_item(self, data, src, target, pedigree, contentsource): |
91 | 1036 | """Overridable from `BasicMirrorWriter`.""" | 1095 | """Overridable from `BasicMirrorWriter`.""" |
92 | 1037 | item = sutil.products_exdata(src, pedigree) | 1096 | item = sutil.products_exdata(src, pedigree) |
93 | @@ -1048,7 +1107,9 @@ | |||
94 | 1048 | elif item['ftype'] not in dict(BOOT_RESOURCE_FILE_TYPE_CHOICES).keys(): | 1107 | elif item['ftype'] not in dict(BOOT_RESOURCE_FILE_TYPE_CHOICES).keys(): |
95 | 1049 | # Skip filetypes that we don't know about. | 1108 | # Skip filetypes that we don't know about. |
96 | 1050 | return | 1109 | return |
98 | 1051 | elif not validate_product(item, product_name): | 1110 | elif not self._validate_ubuntu(item, product_name): |
99 | 1111 | return | ||
100 | 1112 | elif not self._validate_bootloader(item, product_name): | ||
101 | 1052 | return | 1113 | return |
102 | 1053 | else: | 1114 | else: |
103 | 1054 | self.store.insert(item, contentsource) | 1115 | self.store.insert(item, contentsource) |
104 | 1055 | 1116 | ||
105 | === modified file 'src/maasserver/tests/test_bootresources.py' | |||
106 | --- src/maasserver/tests/test_bootresources.py 2017-04-26 00:27:23 +0000 | |||
107 | +++ src/maasserver/tests/test_bootresources.py 2017-04-26 19:06:22 +0000 | |||
108 | @@ -2103,7 +2103,7 @@ | |||
109 | 2103 | class TestBootResourceRepoWriter(MAASServerTestCase): | 2103 | class TestBootResourceRepoWriter(MAASServerTestCase): |
110 | 2104 | """Tests for `BootResourceRepoWriter`.""" | 2104 | """Tests for `BootResourceRepoWriter`.""" |
111 | 2105 | 2105 | ||
113 | 2106 | def create_ubuntu_simplestream(self, ftypes, stream_version=None): | 2106 | def create_simplestream(self, ftypes, stream_version=None): |
114 | 2107 | version = '16.04' | 2107 | version = '16.04' |
115 | 2108 | arch = 'amd64' | 2108 | arch = 'amd64' |
116 | 2109 | subarch = 'hwe-x' | 2109 | subarch = 'hwe-x' |
117 | @@ -2145,47 +2145,10 @@ | |||
118 | 2145 | } | 2145 | } |
119 | 2146 | return src, product, version | 2146 | return src, product, version |
120 | 2147 | 2147 | ||
121 | 2148 | def create_bootloader_simplestream(self, stream_version=None): | ||
122 | 2149 | if stream_version is None: | ||
123 | 2150 | stream_version = '1' | ||
124 | 2151 | product = ( | ||
125 | 2152 | 'com.ubuntu.maas:daily:%s:bootloader-download' % stream_version) | ||
126 | 2153 | version = datetime.now().date().strftime('%Y%m%d.0') | ||
127 | 2154 | versions = { | ||
128 | 2155 | version: { | ||
129 | 2156 | 'items': { | ||
130 | 2157 | BOOT_RESOURCE_FILE_TYPE.BOOTLOADER: { | ||
131 | 2158 | 'sha256': factory.make_name('sha256'), | ||
132 | 2159 | 'path': factory.make_name('path'), | ||
133 | 2160 | 'ftype': BOOT_RESOURCE_FILE_TYPE.BOOTLOADER, | ||
134 | 2161 | 'size': random.randint(0, 2**64), | ||
135 | 2162 | } | ||
136 | 2163 | } | ||
137 | 2164 | } | ||
138 | 2165 | } | ||
139 | 2166 | products = { | ||
140 | 2167 | product: { | ||
141 | 2168 | 'label': 'daily', | ||
142 | 2169 | 'os': 'grub-efi-signed', | ||
143 | 2170 | 'arch': 'amd64', | ||
144 | 2171 | 'bootloader-type': 'uefi', | ||
145 | 2172 | 'version': version, | ||
146 | 2173 | 'versions': versions, | ||
147 | 2174 | } | ||
148 | 2175 | } | ||
149 | 2176 | src = { | ||
150 | 2177 | 'datatype': 'image-downloads', | ||
151 | 2178 | 'format': 'products:1.0', | ||
152 | 2179 | 'updated': format_datetime(datetime.now()), | ||
153 | 2180 | 'products': products, | ||
154 | 2181 | 'content_id': 'com.ubuntu.maas:daily:1:bootloader-download' | ||
155 | 2182 | } | ||
156 | 2183 | return src, product, version | ||
157 | 2184 | |||
158 | 2185 | def test_insert_prefers_squashfs_over_root_image(self): | 2148 | def test_insert_prefers_squashfs_over_root_image(self): |
159 | 2186 | boot_resource_repo_writer = BootResourceRepoWriter( | 2149 | boot_resource_repo_writer = BootResourceRepoWriter( |
160 | 2187 | BootResourceStore(), None) | 2150 | BootResourceStore(), None) |
162 | 2188 | src, product, version = self.create_ubuntu_simplestream([ | 2151 | src, product, version = self.create_simplestream([ |
163 | 2189 | BOOT_RESOURCE_FILE_TYPE.ROOT_IMAGE, | 2152 | BOOT_RESOURCE_FILE_TYPE.ROOT_IMAGE, |
164 | 2190 | BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE, | 2153 | BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE, |
165 | 2191 | ]) | 2154 | ]) |
166 | @@ -2199,7 +2162,7 @@ | |||
167 | 2199 | def test_insert_allows_squashfs(self): | 2162 | def test_insert_allows_squashfs(self): |
168 | 2200 | boot_resource_repo_writer = BootResourceRepoWriter( | 2163 | boot_resource_repo_writer = BootResourceRepoWriter( |
169 | 2201 | BootResourceStore(), None) | 2164 | BootResourceStore(), None) |
171 | 2202 | src, product, version = self.create_ubuntu_simplestream([ | 2165 | src, product, version = self.create_simplestream([ |
172 | 2203 | BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE, | 2166 | BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE, |
173 | 2204 | ]) | 2167 | ]) |
174 | 2205 | data = src['products'][product]['versions'][version]['items'][ | 2168 | data = src['products'][product]['versions'][version]['items'][ |
175 | @@ -2212,7 +2175,7 @@ | |||
176 | 2212 | def test_insert_allows_root_image(self): | 2175 | def test_insert_allows_root_image(self): |
177 | 2213 | boot_resource_repo_writer = BootResourceRepoWriter( | 2176 | boot_resource_repo_writer = BootResourceRepoWriter( |
178 | 2214 | BootResourceStore(), None) | 2177 | BootResourceStore(), None) |
180 | 2215 | src, product, version = self.create_ubuntu_simplestream([ | 2178 | src, product, version = self.create_simplestream([ |
181 | 2216 | BOOT_RESOURCE_FILE_TYPE.ROOT_IMAGE, | 2179 | BOOT_RESOURCE_FILE_TYPE.ROOT_IMAGE, |
182 | 2217 | ]) | 2180 | ]) |
183 | 2218 | data = src['products'][product]['versions'][version]['items'][ | 2181 | data = src['products'][product]['versions'][version]['items'][ |
184 | @@ -2225,7 +2188,9 @@ | |||
185 | 2225 | def test_insert_allows_bootloader(self): | 2188 | def test_insert_allows_bootloader(self): |
186 | 2226 | boot_resource_repo_writer = BootResourceRepoWriter( | 2189 | boot_resource_repo_writer = BootResourceRepoWriter( |
187 | 2227 | BootResourceStore(), None) | 2190 | BootResourceStore(), None) |
189 | 2228 | src, product, version = self.create_bootloader_simplestream() | 2191 | src, product, version = self.create_simplestream([ |
190 | 2192 | BOOT_RESOURCE_FILE_TYPE.BOOTLOADER, | ||
191 | 2193 | ]) | ||
192 | 2229 | data = src['products'][product]['versions'][version]['items'][ | 2194 | data = src['products'][product]['versions'][version]['items'][ |
193 | 2230 | BOOT_RESOURCE_FILE_TYPE.BOOTLOADER] | 2195 | BOOT_RESOURCE_FILE_TYPE.BOOTLOADER] |
194 | 2231 | pedigree = (product, version, BOOT_RESOURCE_FILE_TYPE.BOOTLOADER) | 2196 | pedigree = (product, version, BOOT_RESOURCE_FILE_TYPE.BOOTLOADER) |
195 | @@ -2236,7 +2201,7 @@ | |||
196 | 2236 | def test_insert_allows_archive_tar_xz(self): | 2201 | def test_insert_allows_archive_tar_xz(self): |
197 | 2237 | boot_resource_repo_writer = BootResourceRepoWriter( | 2202 | boot_resource_repo_writer = BootResourceRepoWriter( |
198 | 2238 | BootResourceStore(), None) | 2203 | BootResourceStore(), None) |
200 | 2239 | src, product, version = self.create_ubuntu_simplestream([ | 2204 | src, product, version = self.create_simplestream([ |
201 | 2240 | BOOT_RESOURCE_FILE_TYPE.ARCHIVE_TAR_XZ, | 2205 | BOOT_RESOURCE_FILE_TYPE.ARCHIVE_TAR_XZ, |
202 | 2241 | ]) | 2206 | ]) |
203 | 2242 | data = src['products'][product]['versions'][version]['items'][ | 2207 | data = src['products'][product]['versions'][version]['items'][ |
204 | @@ -2250,7 +2215,7 @@ | |||
205 | 2250 | boot_resource_repo_writer = BootResourceRepoWriter( | 2215 | boot_resource_repo_writer = BootResourceRepoWriter( |
206 | 2251 | BootResourceStore(), None) | 2216 | BootResourceStore(), None) |
207 | 2252 | unknown_ftype = factory.make_name('ftype') | 2217 | unknown_ftype = factory.make_name('ftype') |
209 | 2253 | src, product, version = self.create_ubuntu_simplestream([ | 2218 | src, product, version = self.create_simplestream([ |
210 | 2254 | unknown_ftype, | 2219 | unknown_ftype, |
211 | 2255 | ]) | 2220 | ]) |
212 | 2256 | data = src['products'][product]['versions'][version]['items'][ | 2221 | data = src['products'][product]['versions'][version]['items'][ |
213 | @@ -2263,43 +2228,105 @@ | |||
214 | 2263 | def test_insert_validates_bootloader(self): | 2228 | def test_insert_validates_bootloader(self): |
215 | 2264 | boot_resource_repo_writer = BootResourceRepoWriter( | 2229 | boot_resource_repo_writer = BootResourceRepoWriter( |
216 | 2265 | BootResourceStore(), None) | 2230 | BootResourceStore(), None) |
218 | 2266 | src, product, version = self.create_bootloader_simplestream() | 2231 | src, product, version = self.create_simplestream([ |
219 | 2232 | BOOT_RESOURCE_FILE_TYPE.BOOTLOADER, | ||
220 | 2233 | ]) | ||
221 | 2267 | data = src['products'][product]['versions'][version]['items'][ | 2234 | data = src['products'][product]['versions'][version]['items'][ |
222 | 2268 | BOOT_RESOURCE_FILE_TYPE.BOOTLOADER] | 2235 | BOOT_RESOURCE_FILE_TYPE.BOOTLOADER] |
223 | 2269 | pedigree = (product, version, BOOT_RESOURCE_FILE_TYPE.BOOTLOADER) | 2236 | pedigree = (product, version, BOOT_RESOURCE_FILE_TYPE.BOOTLOADER) |
224 | 2270 | mock_insert = self.patch(boot_resource_repo_writer.store, 'insert') | 2237 | mock_insert = self.patch(boot_resource_repo_writer.store, 'insert') |
225 | 2238 | mock_validate_bootloader = self.patch( | ||
226 | 2239 | boot_resource_repo_writer, '_validate_bootloader') | ||
227 | 2271 | boot_resource_repo_writer.insert_item(data, src, None, pedigree, None) | 2240 | boot_resource_repo_writer.insert_item(data, src, None, pedigree, None) |
228 | 2272 | self.assertThat(mock_insert, MockCalledOnce()) | 2241 | self.assertThat(mock_insert, MockCalledOnce()) |
241 | 2273 | 2242 | self.assertThat(mock_validate_bootloader, MockCalledOnce()) | |
242 | 2274 | def test_insert_validates_rejects_unknown_version(self): | 2243 | |
243 | 2275 | boot_resource_repo_writer = BootResourceRepoWriter( | 2244 | def test_validate_bootloader_ignores_non_bootloaders(self): |
244 | 2276 | BootResourceStore(), None) | 2245 | boot_resource_repo_writer = BootResourceRepoWriter( |
245 | 2277 | src, product, version = self.create_bootloader_simplestream( | 2246 | BootResourceStore(), None) |
246 | 2278 | factory.make_name('stream_version')) | 2247 | self.assertTrue( |
247 | 2279 | data = src['products'][product]['versions'][version]['items'][ | 2248 | boot_resource_repo_writer._validate_bootloader( |
248 | 2280 | BOOT_RESOURCE_FILE_TYPE.BOOTLOADER] | 2249 | {}, 'com.ubuntu.maas.daily:v3:boot:16.04:amd64:hwe-16.04')) |
249 | 2281 | pedigree = (product, version, BOOT_RESOURCE_FILE_TYPE.BOOTLOADER) | 2250 | |
250 | 2282 | mock_insert = self.patch(boot_resource_repo_writer.store, 'insert') | 2251 | def test_validate_bootloader_checks_version(self): |
251 | 2283 | boot_resource_repo_writer.insert_item(data, src, None, pedigree, None) | 2252 | boot_resource_repo_writer = BootResourceRepoWriter( |
252 | 2284 | self.assertThat(mock_insert, MockNotCalled()) | 2253 | BootResourceStore(), None) |
253 | 2254 | version = random.randint(2, 100) | ||
254 | 2255 | product_name = "com.ubuntu.maas.daily:%d:pxelinux:pxe:i386" % version | ||
255 | 2256 | self.assertFalse( | ||
256 | 2257 | boot_resource_repo_writer._validate_bootloader( | ||
257 | 2258 | {'bootloader-type': factory.make_name('bootloader-type')}, | ||
258 | 2259 | product_name)) | ||
259 | 2260 | |||
260 | 2261 | def test_validate_bootloader_allows_acceptable_bootloaders(self): | ||
261 | 2262 | acceptable_bootloaders = [ | ||
262 | 2263 | { | ||
263 | 2264 | 'os': 'pxelinux', | ||
264 | 2265 | 'arch': 'i386', | ||
265 | 2266 | 'bootloader-type': 'pxe', | ||
266 | 2267 | }, | ||
267 | 2268 | { | ||
268 | 2269 | 'os': 'grub-efi-signed', | ||
269 | 2270 | 'arch': 'amd64', | ||
270 | 2271 | 'bootloader-type': 'uefi', | ||
271 | 2272 | }, | ||
272 | 2273 | { | ||
273 | 2274 | 'os': 'grub-efi', | ||
274 | 2275 | 'arch': 'arm64', | ||
275 | 2276 | 'bootloader-type': 'uefi', | ||
276 | 2277 | }, | ||
277 | 2278 | { | ||
278 | 2279 | 'os': 'grub-ieee1275', | ||
279 | 2280 | 'arch': 'ppc64el', | ||
280 | 2281 | 'bootloader-type': 'open-firmware', | ||
281 | 2282 | }, | ||
282 | 2283 | ] | ||
283 | 2284 | boot_resource_repo_writer = BootResourceRepoWriter( | ||
284 | 2285 | BootResourceStore(), None) | ||
285 | 2286 | for bootloader in acceptable_bootloaders: | ||
286 | 2287 | product_name = "com.ubuntu.maas.daily:1:%s:%s:%s" % ( | ||
287 | 2288 | bootloader['os'], bootloader['bootloader-type'], | ||
288 | 2289 | bootloader['arch']) | ||
289 | 2290 | self.assertTrue( | ||
290 | 2291 | boot_resource_repo_writer._validate_bootloader( | ||
291 | 2292 | bootloader, product_name), | ||
292 | 2293 | "Failed to validate %s" % product_name) | ||
293 | 2294 | |||
294 | 2295 | def test_validate_bootloader_denies_unacceptable_bootloader(self): | ||
295 | 2296 | bootloader = { | ||
296 | 2297 | 'os': factory.make_name('os'), | ||
297 | 2298 | 'arch': factory.make_name('arch'), | ||
298 | 2299 | 'bootloader-type': factory.make_name('bootloader_type'), | ||
299 | 2300 | } | ||
300 | 2301 | product_name = "com.ubuntu.maas.daily:1:%s:%s:%s" % ( | ||
301 | 2302 | bootloader['os'], bootloader['bootloader-type'], | ||
302 | 2303 | bootloader['arch']) | ||
303 | 2304 | boot_resource_repo_writer = BootResourceRepoWriter( | ||
304 | 2305 | BootResourceStore(), None) | ||
305 | 2306 | self.assertFalse( | ||
306 | 2307 | boot_resource_repo_writer._validate_bootloader( | ||
307 | 2308 | bootloader, product_name)) | ||
308 | 2285 | 2309 | ||
309 | 2286 | def test_insert_validates_ubuntu(self): | 2310 | def test_insert_validates_ubuntu(self): |
310 | 2287 | boot_resource_repo_writer = BootResourceRepoWriter( | 2311 | boot_resource_repo_writer = BootResourceRepoWriter( |
311 | 2288 | BootResourceStore(), None) | 2312 | BootResourceStore(), None) |
313 | 2289 | src, product, version = self.create_ubuntu_simplestream([ | 2313 | src, product, version = self.create_simplestream([ |
314 | 2290 | BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE, | 2314 | BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE, |
315 | 2291 | ]) | 2315 | ]) |
316 | 2292 | data = src['products'][product]['versions'][version]['items'][ | 2316 | data = src['products'][product]['versions'][version]['items'][ |
317 | 2293 | BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE] | 2317 | BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE] |
318 | 2294 | pedigree = (product, version, BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE) | 2318 | pedigree = (product, version, BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE) |
319 | 2295 | mock_insert = self.patch(boot_resource_repo_writer.store, 'insert') | 2319 | mock_insert = self.patch(boot_resource_repo_writer.store, 'insert') |
320 | 2320 | mock_validate_bootloader = self.patch( | ||
321 | 2321 | boot_resource_repo_writer, '_validate_ubuntu') | ||
322 | 2296 | boot_resource_repo_writer.insert_item(data, src, None, pedigree, None) | 2322 | boot_resource_repo_writer.insert_item(data, src, None, pedigree, None) |
323 | 2297 | self.assertThat(mock_insert, MockCalledOnce()) | 2323 | self.assertThat(mock_insert, MockCalledOnce()) |
324 | 2324 | self.assertThat(mock_validate_bootloader, MockCalledOnce()) | ||
325 | 2298 | 2325 | ||
326 | 2299 | def test_validate_ubuntu_rejects_unknown_version(self): | 2326 | def test_validate_ubuntu_rejects_unknown_version(self): |
327 | 2300 | boot_resource_repo_writer = BootResourceRepoWriter( | 2327 | boot_resource_repo_writer = BootResourceRepoWriter( |
328 | 2301 | BootResourceStore(), None) | 2328 | BootResourceStore(), None) |
330 | 2302 | src, product, version = self.create_ubuntu_simplestream( | 2329 | src, product, version = self.create_simplestream( |
331 | 2303 | [BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE], | 2330 | [BOOT_RESOURCE_FILE_TYPE.SQUASHFS_IMAGE], |
332 | 2304 | factory.make_name('stream_version')) | 2331 | factory.make_name('stream_version')) |
333 | 2305 | data = src['products'][product]['versions'][version]['items'][ | 2332 | data = src['products'][product]['versions'][version]['items'][ |
334 | 2306 | 2333 | ||
335 | === modified file 'src/provisioningserver/import_images/download_descriptions.py' | |||
336 | --- src/provisioningserver/import_images/download_descriptions.py 2017-04-26 00:27:23 +0000 | |||
337 | +++ src/provisioningserver/import_images/download_descriptions.py 2017-04-26 19:06:22 +0000 | |||
338 | @@ -10,10 +10,8 @@ | |||
339 | 10 | 10 | ||
340 | 11 | __all__ = [ | 11 | __all__ = [ |
341 | 12 | 'download_all_image_descriptions', | 12 | 'download_all_image_descriptions', |
342 | 13 | 'validate_product', | ||
343 | 14 | ] | 13 | ] |
344 | 15 | 14 | ||
345 | 16 | import re | ||
346 | 17 | 15 | ||
347 | 18 | from provisioningserver.import_images.boot_image_mapping import ( | 16 | from provisioningserver.import_images.boot_image_mapping import ( |
348 | 19 | BootImageMapping, | 17 | BootImageMapping, |
349 | @@ -33,13 +31,6 @@ | |||
350 | 33 | products_exdata, | 31 | products_exdata, |
351 | 34 | ) | 32 | ) |
352 | 35 | 33 | ||
353 | 36 | # Compile a regex to validate Ubuntu product names. This only allows V2 and V3 | ||
354 | 37 | # Ubuntu images. | ||
355 | 38 | UBUNTU_REGEX = re.compile('.*:v[23]:.*', re.IGNORECASE) | ||
356 | 39 | # Compile a regex to validate bootloader product names. This only allows V1 | ||
357 | 40 | # bootloaders. | ||
358 | 41 | BOOTLOADER_REGEX = re.compile('.*:1:.*', re.IGNORECASE) | ||
359 | 42 | |||
360 | 43 | 34 | ||
361 | 44 | def clean_up_repo_item(item): | 35 | def clean_up_repo_item(item): |
362 | 45 | """Return a subset of dict `item` for storing in a boot images dict.""" | 36 | """Return a subset of dict `item` for storing in a boot images dict.""" |
363 | @@ -55,67 +46,6 @@ | |||
364 | 55 | return compact_item | 46 | return compact_item |
365 | 56 | 47 | ||
366 | 57 | 48 | ||
367 | 58 | def validate_ubuntu(data, product_name): | ||
368 | 59 | osystem = data.get('os', '') | ||
369 | 60 | if 'ubuntu' not in osystem.lower(): | ||
370 | 61 | # It's not an Ubuntu product, nothing to validate. | ||
371 | 62 | return True | ||
372 | 63 | elif UBUNTU_REGEX.search(product_name) is None: | ||
373 | 64 | # Only insert v2 or v3 Ubuntu products. | ||
374 | 65 | return False | ||
375 | 66 | else: | ||
376 | 67 | return True | ||
377 | 68 | |||
378 | 69 | |||
379 | 70 | def validate_bootloader(data, product_name): | ||
380 | 71 | bootloader_type = data.get('bootloader-type') | ||
381 | 72 | if bootloader_type is None: | ||
382 | 73 | # It's not a bootloader, nothing to validate | ||
383 | 74 | return True | ||
384 | 75 | if BOOTLOADER_REGEX.search(product_name) is None: | ||
385 | 76 | # Only insert V1 bootloaders from the stream | ||
386 | 77 | return False | ||
387 | 78 | # Validate MAAS supports the specific bootloader_type, os, arch | ||
388 | 79 | # combination. | ||
389 | 80 | SUPPORTED_BOOTLOADERS = { | ||
390 | 81 | 'pxe': [ | ||
391 | 82 | { | ||
392 | 83 | 'os': 'pxelinux', | ||
393 | 84 | 'arch': 'i386', | ||
394 | 85 | } | ||
395 | 86 | ], | ||
396 | 87 | 'uefi': [ | ||
397 | 88 | { | ||
398 | 89 | 'os': 'grub-efi-signed', | ||
399 | 90 | 'arch': 'amd64', | ||
400 | 91 | }, | ||
401 | 92 | { | ||
402 | 93 | 'os': 'grub-efi', | ||
403 | 94 | 'arch': 'arm64', | ||
404 | 95 | } | ||
405 | 96 | ], | ||
406 | 97 | 'open-firmware': [ | ||
407 | 98 | { | ||
408 | 99 | 'os': 'grub-ieee1275', | ||
409 | 100 | 'arch': 'ppc64el', | ||
410 | 101 | } | ||
411 | 102 | ], | ||
412 | 103 | } | ||
413 | 104 | for bootloader in SUPPORTED_BOOTLOADERS.get(bootloader_type, []): | ||
414 | 105 | if ( | ||
415 | 106 | data.get('os') == bootloader['os'] and | ||
416 | 107 | data.get('arch') == bootloader['arch']): | ||
417 | 108 | return True | ||
418 | 109 | |||
419 | 110 | # Bootloader not supported, ignore | ||
420 | 111 | return False | ||
421 | 112 | |||
422 | 113 | |||
423 | 114 | def validate_product(data, product_name): | ||
424 | 115 | return (validate_ubuntu(data, product_name) and | ||
425 | 116 | validate_bootloader(data, product_name)) | ||
426 | 117 | |||
427 | 118 | |||
428 | 119 | class RepoDumper(BasicMirrorWriter): | 49 | class RepoDumper(BasicMirrorWriter): |
429 | 120 | """Gather metadata about boot images available in a Simplestreams repo. | 50 | """Gather metadata about boot images available in a Simplestreams repo. |
430 | 121 | 51 | ||
431 | @@ -149,8 +79,6 @@ | |||
432 | 149 | def insert_item(self, data, src, target, pedigree, contentsource): | 79 | def insert_item(self, data, src, target, pedigree, contentsource): |
433 | 150 | """Overridable from `BasicMirrorWriter`.""" | 80 | """Overridable from `BasicMirrorWriter`.""" |
434 | 151 | item = products_exdata(src, pedigree) | 81 | item = products_exdata(src, pedigree) |
435 | 152 | if not validate_product(item, pedigree[0]): | ||
436 | 153 | return | ||
437 | 154 | os = get_os_from_product(item) | 82 | os = get_os_from_product(item) |
438 | 155 | arch = item['arch'] | 83 | arch = item['arch'] |
439 | 156 | subarches = item.get('subarches', 'generic') | 84 | subarches = item.get('subarches', 'generic') |
440 | 157 | 85 | ||
441 | === modified file 'src/provisioningserver/import_images/tests/test_download_descriptions.py' | |||
442 | --- src/provisioningserver/import_images/tests/test_download_descriptions.py 2017-04-26 00:27:23 +0000 | |||
443 | +++ src/provisioningserver/import_images/tests/test_download_descriptions.py 2017-04-26 19:06:22 +0000 | |||
444 | @@ -6,7 +6,6 @@ | |||
445 | 6 | __all__ = [] | 6 | __all__ = [] |
446 | 7 | 7 | ||
447 | 8 | import logging | 8 | import logging |
448 | 9 | import random | ||
449 | 10 | from unittest.mock import ( | 9 | from unittest.mock import ( |
450 | 11 | ANY, | 10 | ANY, |
451 | 12 | call, | 11 | call, |
452 | @@ -28,7 +27,6 @@ | |||
453 | 28 | from provisioningserver.import_images.download_descriptions import ( | 27 | from provisioningserver.import_images.download_descriptions import ( |
454 | 29 | clean_up_repo_item, | 28 | clean_up_repo_item, |
455 | 30 | RepoDumper, | 29 | RepoDumper, |
456 | 31 | validate_product, | ||
457 | 32 | ) | 30 | ) |
458 | 33 | from provisioningserver.import_images.testing.factory import ( | 31 | from provisioningserver.import_images.testing.factory import ( |
459 | 34 | make_image_spec, | 32 | make_image_spec, |
460 | @@ -36,78 +34,6 @@ | |||
461 | 36 | ) | 34 | ) |
462 | 37 | 35 | ||
463 | 38 | 36 | ||
464 | 39 | class TestValidateProduct(MAASTestCase): | ||
465 | 40 | """Tests for `validate_product`.""" | ||
466 | 41 | |||
467 | 42 | def test__ignores_random(self): | ||
468 | 43 | self.assertTrue( | ||
469 | 44 | validate_product({}, factory.make_name('product_name'))) | ||
470 | 45 | |||
471 | 46 | def test__validates_ubuntu(self): | ||
472 | 47 | self.assertTrue(validate_product( | ||
473 | 48 | {'os': 'ubuntu'}, | ||
474 | 49 | 'com.ubuntu.maas.daily:v%d:boot:%s:%s:%s' % ( | ||
475 | 50 | random.choice([2, 3]), factory.make_name('version'), | ||
476 | 51 | factory.make_name('arch'), factory.make_name('sub_arch')))) | ||
477 | 52 | |||
478 | 53 | def test__rejects_unknown_ubuntu_version(self): | ||
479 | 54 | self.assertFalse(validate_product( | ||
480 | 55 | {'os': 'ubuntu'}, | ||
481 | 56 | 'com.ubuntu.maas.daily:v%d:boot:%s:%s:%s' % ( | ||
482 | 57 | random.randint(4, 100), factory.make_name('version'), | ||
483 | 58 | factory.make_name('arch'), factory.make_name('sub_arch')))) | ||
484 | 59 | |||
485 | 60 | def test__validates_bootloaders(self): | ||
486 | 61 | acceptable_bootloaders = [ | ||
487 | 62 | { | ||
488 | 63 | 'os': 'pxelinux', | ||
489 | 64 | 'arch': 'i386', | ||
490 | 65 | 'bootloader-type': 'pxe', | ||
491 | 66 | }, | ||
492 | 67 | { | ||
493 | 68 | 'os': 'grub-efi-signed', | ||
494 | 69 | 'arch': 'amd64', | ||
495 | 70 | 'bootloader-type': 'uefi', | ||
496 | 71 | }, | ||
497 | 72 | { | ||
498 | 73 | 'os': 'grub-efi', | ||
499 | 74 | 'arch': 'arm64', | ||
500 | 75 | 'bootloader-type': 'uefi', | ||
501 | 76 | }, | ||
502 | 77 | { | ||
503 | 78 | 'os': 'grub-ieee1275', | ||
504 | 79 | 'arch': 'ppc64el', | ||
505 | 80 | 'bootloader-type': 'open-firmware', | ||
506 | 81 | }, | ||
507 | 82 | ] | ||
508 | 83 | for bootloader in acceptable_bootloaders: | ||
509 | 84 | product_name = "com.ubuntu.maas.daily:1:%s:%s:%s" % ( | ||
510 | 85 | bootloader['os'], bootloader['bootloader-type'], | ||
511 | 86 | bootloader['arch']) | ||
512 | 87 | self.assertTrue( | ||
513 | 88 | validate_product(bootloader, product_name), | ||
514 | 89 | "Failed to validate %s" % product_name) | ||
515 | 90 | |||
516 | 91 | def test__rejects_unknown_bootloader_version(self): | ||
517 | 92 | version = random.randint(2, 100) | ||
518 | 93 | product_name = "com.ubuntu.maas.daily:%d:pxelinux:pxe:i386" % version | ||
519 | 94 | self.assertFalse( | ||
520 | 95 | validate_product( | ||
521 | 96 | {'bootloader-type': factory.make_name('bootloader-type')}, | ||
522 | 97 | product_name)) | ||
523 | 98 | |||
524 | 99 | def test__rejects_unknown_bootloader(self): | ||
525 | 100 | bootloader = { | ||
526 | 101 | 'os': factory.make_name('os'), | ||
527 | 102 | 'arch': factory.make_name('arch'), | ||
528 | 103 | 'bootloader-type': factory.make_name('bootloader_type'), | ||
529 | 104 | } | ||
530 | 105 | product_name = "com.ubuntu.maas.daily:1:%s:%s:%s" % ( | ||
531 | 106 | bootloader['os'], bootloader['bootloader-type'], | ||
532 | 107 | bootloader['arch']) | ||
533 | 108 | self.assertFalse(validate_product(bootloader, product_name)) | ||
534 | 109 | |||
535 | 110 | |||
536 | 111 | class TestValuePassesFilterList(MAASTestCase): | 37 | class TestValuePassesFilterList(MAASTestCase): |
537 | 112 | """Tests for `value_passes_filter_list`.""" | 38 | """Tests for `value_passes_filter_list`.""" |
538 | 113 | 39 | ||
539 | @@ -354,10 +280,7 @@ | |||
540 | 354 | download_descriptions, 'products_exdata').return_value = item | 280 | download_descriptions, 'products_exdata').return_value = item |
541 | 355 | dumper.insert_item( | 281 | dumper.insert_item( |
542 | 356 | sentinel.data, sentinel.src, sentinel.target, | 282 | sentinel.data, sentinel.src, sentinel.target, |
547 | 357 | ( | 283 | sentinel.pedigree, sentinel.contentsource) |
544 | 358 | factory.make_name('product_name'), | ||
545 | 359 | factory.make_name('product_version') | ||
546 | 360 | ), sentinel.contentsource) | ||
548 | 361 | image_specs = [ | 284 | image_specs = [ |
549 | 362 | make_image_spec( | 285 | make_image_spec( |
550 | 363 | os=item['os'], release=item['release'], | 286 | os=item['os'], release=item['release'], |
551 | @@ -383,10 +306,7 @@ | |||
552 | 383 | for _ in range(2): | 306 | for _ in range(2): |
553 | 384 | dumper.insert_item( | 307 | dumper.insert_item( |
554 | 385 | sentinel.data, sentinel.src, sentinel.target, | 308 | sentinel.data, sentinel.src, sentinel.target, |
559 | 386 | ( | 309 | sentinel.pedigree, sentinel.contentsource) |
556 | 387 | factory.make_name('product_name'), | ||
557 | 388 | factory.make_name('product_version') | ||
558 | 389 | ), sentinel.contentsource) | ||
560 | 390 | image_spec = make_image_spec( | 310 | image_spec = make_image_spec( |
561 | 391 | os=item['os'], release=item['release'], | 311 | os=item['os'], release=item['release'], |
562 | 392 | arch=item['arch'], subarch=compat_subarch, | 312 | arch=item['arch'], subarch=compat_subarch, |
563 | @@ -417,11 +337,8 @@ | |||
564 | 417 | 'products_exdata').side_effect = [hwep_item, hwes_item] | 337 | 'products_exdata').side_effect = [hwep_item, hwes_item] |
565 | 418 | for _ in range(2): | 338 | for _ in range(2): |
566 | 419 | dumper.insert_item( | 339 | dumper.insert_item( |
572 | 420 | {'os': 'ubuntu'}, sentinel.src, sentinel.target, | 340 | sentinel.data, sentinel.src, sentinel.target, |
573 | 421 | ( | 341 | sentinel.pedigree, sentinel.contentsource) |
569 | 422 | 'com.ubuntu.maas.daily:v3:boot:12.04:amd64:hwe-p', | ||
570 | 423 | factory.make_name('product_version'), | ||
571 | 424 | ), sentinel.contentsource) | ||
574 | 425 | image_spec = make_image_spec( | 342 | image_spec = make_image_spec( |
575 | 426 | os=os, release=release, arch=arch, subarch='generic', | 343 | os=os, release=release, arch=arch, subarch='generic', |
576 | 427 | label=label) | 344 | label=label) |
577 | @@ -451,11 +368,8 @@ | |||
578 | 451 | 'products_exdata').side_effect = [hwep_item, hwes_item] | 368 | 'products_exdata').side_effect = [hwep_item, hwes_item] |
579 | 452 | for _ in range(2): | 369 | for _ in range(2): |
580 | 453 | dumper.insert_item( | 370 | dumper.insert_item( |
586 | 454 | {'os': 'ubuntu'}, sentinel.src, sentinel.target, | 371 | sentinel.data, sentinel.src, sentinel.target, |
587 | 455 | ( | 372 | sentinel.pedigree, sentinel.contentsource) |
583 | 456 | 'com.ubuntu.maas.daily:v3:boot:12.04:amd64:hwe-p', | ||
584 | 457 | factory.make_name('product_version'), | ||
585 | 458 | ), sentinel.contentsource) | ||
588 | 459 | image_spec = make_image_spec( | 373 | image_spec = make_image_spec( |
589 | 460 | os=os, release=release, arch=arch, subarch='generic', | 374 | os=os, release=release, arch=arch, subarch='generic', |
590 | 461 | label=label) | 375 | label=label) |
591 | @@ -464,42 +378,21 @@ | |||
592 | 464 | def test_insert_item_sets_release_to_bootloader_type(self): | 378 | def test_insert_item_sets_release_to_bootloader_type(self): |
593 | 465 | boot_images_dict = BootImageMapping() | 379 | boot_images_dict = BootImageMapping() |
594 | 466 | dumper = RepoDumper(boot_images_dict) | 380 | dumper = RepoDumper(boot_images_dict) |
597 | 467 | item, _ = self.make_item( | 381 | bootloader_type = factory.make_name('bootloader-type') |
598 | 468 | arch='amd64', bootloader_type='uefi', os='grub-efi-signed') | 382 | item, _ = self.make_item(bootloader_type=bootloader_type) |
599 | 469 | self.patch( | 383 | self.patch( |
600 | 470 | download_descriptions, 'products_exdata').return_value = item | 384 | download_descriptions, 'products_exdata').return_value = item |
601 | 471 | dumper.insert_item( | 385 | dumper.insert_item( |
611 | 472 | { | 386 | sentinel.data, sentinel.src, sentinel.target, |
612 | 473 | 'bootloader_type': 'uefi', | 387 | sentinel.pedigree, sentinel.contentsource) |
604 | 474 | 'os': 'grub-efi-signed', | ||
605 | 475 | 'arch': 'amd64', | ||
606 | 476 | }, sentinel.src, sentinel.target, | ||
607 | 477 | ( | ||
608 | 478 | 'com.ubuntu.maas.daily:1:grub-efi-signed:uefi:amd64', | ||
609 | 479 | factory.make_name('product_version'), | ||
610 | 480 | ), sentinel.contentsource) | ||
613 | 481 | image_specs = [ | 388 | image_specs = [ |
614 | 482 | make_image_spec( | 389 | make_image_spec( |
616 | 483 | os=item['os'], release='uefi', arch=item['arch'], | 390 | os=item['os'], release=bootloader_type, arch=item['arch'], |
617 | 484 | subarch=subarch, kflavor='bootloader', label=item['label']) | 391 | subarch=subarch, kflavor='bootloader', label=item['label']) |
618 | 485 | for subarch in item['subarches'].split(',') | 392 | for subarch in item['subarches'].split(',') |
619 | 486 | ] | 393 | ] |
620 | 487 | self.assertItemsEqual(image_specs, list(boot_images_dict.mapping)) | 394 | self.assertItemsEqual(image_specs, list(boot_images_dict.mapping)) |
621 | 488 | 395 | ||
622 | 489 | def test_insert_item_validates(self): | ||
623 | 490 | boot_images_dict = BootImageMapping() | ||
624 | 491 | dumper = RepoDumper(boot_images_dict) | ||
625 | 492 | item, _ = self.make_item(os='ubuntu') | ||
626 | 493 | self.patch( | ||
627 | 494 | download_descriptions, 'products_exdata').return_value = item | ||
628 | 495 | dumper.insert_item( | ||
629 | 496 | {'os': 'ubuntu'}, sentinel.src, sentinel.target, | ||
630 | 497 | ( | ||
631 | 498 | factory.make_name('product_name'), | ||
632 | 499 | factory.make_name('product_version'), | ||
633 | 500 | ), sentinel.contentsource) | ||
634 | 501 | self.assertItemsEqual([], list(boot_images_dict.mapping)) | ||
635 | 502 | |||
636 | 503 | def test_sync_does_propagate_ioerror(self): | 396 | def test_sync_does_propagate_ioerror(self): |
637 | 504 | io_error = factory.make_exception_type(bases=(IOError,)) | 397 | io_error = factory.make_exception_type(bases=(IOError,)) |
638 | 505 | 398 |
selfie!