Status: | Merged |
---|---|
Merged at revision: | 17338 |
Proposed branch: | lp:~wgrant/launchpad/xbia |
Merge into: | lp:launchpad |
Prerequisite: | lp:~wgrant/launchpad/xbia-refactor |
Diff against target: |
398 lines (+184/-80) 6 files modified
lib/lp/soyuz/adapters/buildarch.py (+54/-21) lib/lp/soyuz/adapters/tests/test_buildarch.py (+82/-21) lib/lp/soyuz/interfaces/sourcepackagerelease.py (+3/-0) lib/lp/soyuz/model/binarypackagebuild.py (+26/-38) lib/lp/soyuz/model/sourcepackagerelease.py (+6/-0) lib/lp/soyuz/tests/test_build_set.py (+13/-0) |
To merge this branch: | bzr merge lp:~wgrant/launchpad/xbia |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Colin Watson (community) | Approve | ||
Review via email: mp+249296@code.launchpad.net |
Commit message
Source packages can select their arch-indep build architecture using a new Build-Indep-
"all" in a hint list now forces a nominatedarchindep build only if it is the sole term, rather than if it was the only term we could resolve.
Description of the change
Source packages can select their arch-indep build architecture using a new Build-Indep-
"all" in a hint list now forces a nominatedarchindep build only if it is the sole term, rather than if it was the only term we could resolve.
To post a comment you must log in.
Revision history for this message
Colin Watson (cjwatson) : | # |
review:
Approve
Revision history for this message
William Grant (wgrant) : | # |
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lib/lp/soyuz/adapters/buildarch.py' |
2 | --- lib/lp/soyuz/adapters/buildarch.py 2015-02-11 14:44:42 +0000 |
3 | +++ lib/lp/soyuz/adapters/buildarch.py 2015-02-11 14:44:42 +0000 |
4 | @@ -36,29 +36,62 @@ |
5 | dpkg_architecture = DpkgArchitectureCache() |
6 | |
7 | |
8 | -def determine_architectures_to_build(hintlist, valid_archs, |
9 | +def resolve_arch_spec(hintlist, valid_archs): |
10 | + hint_archs = set(hintlist.split()) |
11 | + # 'all' is only used if it's a purely arch-indep package. |
12 | + if hint_archs == set(["all"]): |
13 | + return set(), True |
14 | + return ( |
15 | + set(dpkg_architecture.findAllMatches(valid_archs, hint_archs)), False) |
16 | + |
17 | + |
18 | +def determine_architectures_to_build(hint_list, indep_hint_list, need_archs, |
19 | nominated_arch_indep, need_arch_indep): |
20 | """Return a set of architectures to build. |
21 | |
22 | - :param hintlist: A string of the architectures this source package |
23 | + :param hint_list: a string of the architectures this source package |
24 | specifies it builds for. |
25 | - :param valid_archs: a list of all architecture tags that we can |
26 | - create builds for. |
27 | - :param nominated_arch_indep: a preferred architecture tag for |
28 | - architecture-independent builds. May be None. |
29 | - :return: a set of architecture tags for which the source publication in |
30 | - question should be built. |
31 | + :param indep_hint_list: a string of the architectures this source package |
32 | + specifies it can build architecture-independent packages on. |
33 | + :param need_archs: an ordered list of all architecture tags that we can |
34 | + create builds for. the first usable one gets the arch-indep flag. |
35 | + :param nominated_arch_indep: the default architecture tag for |
36 | + arch-indep-only packages. may be None. |
37 | + :param need_arch_indep: should an arch-indep build be created if possible? |
38 | + :return: a map of architecture tag to arch-indep flag for each build |
39 | + that should be created. |
40 | """ |
41 | - hint_archs = set(hintlist.split()) |
42 | - build_archs = set( |
43 | - dpkg_architecture.findAllMatches(valid_archs, hint_archs)) |
44 | - |
45 | - # 'all' is only used as a last resort, to create an arch-indep build |
46 | - # where no builds would otherwise exist. |
47 | - if need_arch_indep and len(build_archs) == 0 and 'all' in hint_archs: |
48 | - if nominated_arch_indep in valid_archs: |
49 | - return set([nominated_arch_indep]) |
50 | - else: |
51 | - return set() |
52 | - |
53 | - return build_archs |
54 | + build_archs, indep_only = resolve_arch_spec(hint_list, need_archs) |
55 | + |
56 | + # Use the indep hint list if it's set, otherwise fall back to the |
57 | + # main architecture list. If that's not set either (ie. it's just |
58 | + # "all"), default to nominatedarchindep. |
59 | + if indep_hint_list: |
60 | + indep_archs, _ = resolve_arch_spec(indep_hint_list, need_archs) |
61 | + elif not indep_only: |
62 | + indep_archs = set(build_archs) |
63 | + elif nominated_arch_indep in need_archs: |
64 | + indep_archs = set([nominated_arch_indep]) |
65 | + else: |
66 | + indep_archs = set() |
67 | + |
68 | + indep_arch = None |
69 | + if need_arch_indep: |
70 | + # Try to avoid adding a new build if an existing one would work. |
71 | + both_archs = set(build_archs) & set(indep_archs) |
72 | + if both_archs: |
73 | + indep_archs = both_archs |
74 | + |
75 | + # The ideal arch_indep build is nominatedarchindep. But if we're |
76 | + # not creating a build for it, use the first candidate DAS that |
77 | + # made it this far. |
78 | + for arch in [nominated_arch_indep] + need_archs: |
79 | + if arch in indep_archs: |
80 | + indep_arch = arch |
81 | + break |
82 | + |
83 | + # Ensure that we build the indep arch. |
84 | + if indep_arch is not None and indep_arch not in build_archs: |
85 | + build_archs.add(indep_arch) |
86 | + |
87 | + return {arch: arch == indep_arch for arch in build_archs} |
88 | |
89 | === modified file 'lib/lp/soyuz/adapters/tests/test_buildarch.py' |
90 | --- lib/lp/soyuz/adapters/tests/test_buildarch.py 2015-02-11 14:44:42 +0000 |
91 | +++ lib/lp/soyuz/adapters/tests/test_buildarch.py 2015-02-11 14:44:42 +0000 |
92 | @@ -43,82 +43,143 @@ |
93 | """ |
94 | |
95 | def assertArchsForHint(self, hint_string, expected_arch_tags, |
96 | - allowed_arch_tags=None): |
97 | + allowed_arch_tags=None, indep_hint_list=None, |
98 | + need_arch_indep=True): |
99 | if allowed_arch_tags is None: |
100 | allowed_arch_tags = ['armel', 'hppa', 'i386'] |
101 | arch_tags = determine_architectures_to_build( |
102 | - hint_string, allowed_arch_tags, 'i386', True) |
103 | - self.assertContentEqual(expected_arch_tags, arch_tags) |
104 | + hint_string, indep_hint_list, allowed_arch_tags, 'i386', |
105 | + need_arch_indep) |
106 | + self.assertContentEqual(expected_arch_tags.items(), arch_tags.items()) |
107 | |
108 | def test_single_architecture(self): |
109 | # A hint string with a single arch resolves to just that arch. |
110 | - self.assertArchsForHint('hppa', ['hppa']) |
111 | + self.assertArchsForHint('hppa', {'hppa': True}) |
112 | |
113 | def test_three_architectures(self): |
114 | # A hint string with multiple archs resolves to just those |
115 | # archs. |
116 | - self.assertArchsForHint('amd64 i386 hppa', ['hppa', 'i386']) |
117 | + self.assertArchsForHint( |
118 | + 'amd64 i386 hppa', {'hppa': False, 'i386': True}) |
119 | |
120 | def test_independent(self): |
121 | # 'all' is special, meaning just a single build. The |
122 | # nominatedarchindep architecture is used -- in this case i386. |
123 | - self.assertArchsForHint('all', ['i386']) |
124 | + self.assertArchsForHint('all', {'i386': True}) |
125 | |
126 | def test_one_and_independent(self): |
127 | # 'all' is redundant if we have another build anyway. |
128 | - self.assertArchsForHint('hppa all', ['hppa']) |
129 | + self.assertArchsForHint('hppa all', {'hppa': True}) |
130 | |
131 | def test_fictional_and_independent(self): |
132 | - # But 'all' is useful if present with an arch that wouldn't |
133 | - # generate a build. |
134 | - self.assertArchsForHint('foo all', ['i386']) |
135 | + # 'all' doesn't make an unbuildable string buildable. |
136 | + self.assertArchsForHint('fiction all', {}) |
137 | |
138 | def test_wildcard(self): |
139 | # 'any' is a wildcard that matches all available archs. |
140 | - self.assertArchsForHint('any', ['armel', 'hppa', 'i386']) |
141 | + self.assertArchsForHint( |
142 | + 'any', {'armel': False, 'hppa': False, 'i386': True}) |
143 | |
144 | def test_kernel_specific_architecture(self): |
145 | # Since we only support Linux-based architectures, 'linux-foo' |
146 | # is treated the same as 'foo'. |
147 | - self.assertArchsForHint('linux-hppa', ['hppa']) |
148 | + self.assertArchsForHint('linux-hppa', {'hppa': True}) |
149 | |
150 | def test_unknown_kernel_specific_architecture(self): |
151 | # Non-Linux architectures aren't supported. |
152 | - self.assertArchsForHint('kfreebsd-hppa', []) |
153 | + self.assertArchsForHint('kfreebsd-hppa', {}) |
154 | |
155 | def test_kernel_wildcard_architecture(self): |
156 | # Wildcards work for kernels: 'any-foo' is treated like 'foo'. |
157 | - self.assertArchsForHint('any-hppa', ['hppa']) |
158 | + self.assertArchsForHint('any-hppa', {'hppa': True}) |
159 | |
160 | def test_kernel_wildcard_architecture_arm(self): |
161 | # The second part of a wildcard matches the canonical CPU name, not |
162 | # on the Debian architecture, so 'any-arm' matches 'armel'. |
163 | - self.assertArchsForHint('any-arm', ['armel']) |
164 | + self.assertArchsForHint('any-arm', {'armel': True}) |
165 | |
166 | def test_kernel_specific_architecture_wildcard(self): |
167 | # Wildcards work for archs too: 'linux-any' is treated like 'any'. |
168 | - self.assertArchsForHint('linux-any', ['armel', 'hppa', 'i386']) |
169 | + self.assertArchsForHint( |
170 | + 'linux-any', {'armel': False, 'hppa': False, 'i386': True}) |
171 | |
172 | def test_unknown_kernel_specific_architecture_wildcard(self): |
173 | # But unknown kernels continue to result in nothing. |
174 | - self.assertArchsForHint('kfreebsd-any', []) |
175 | + self.assertArchsForHint('kfreebsd-any', {}) |
176 | |
177 | def test_wildcard_and_independent(self): |
178 | # 'all' continues to be ignored alongside a valid wildcard. |
179 | - self.assertArchsForHint('all linux-any', ['armel', 'hppa', 'i386']) |
180 | + self.assertArchsForHint( |
181 | + 'all linux-any', {'armel': False, 'hppa': False, 'i386': True}) |
182 | |
183 | def test_kernel_independent_is_invalid(self): |
184 | # 'linux-all' isn't supported. |
185 | - self.assertArchsForHint('linux-all', []) |
186 | + self.assertArchsForHint('linux-all', {}) |
187 | |
188 | def test_double_wildcard_is_same_as_single(self): |
189 | # 'any-any' is redundant with 'any', but dpkg-architecture supports |
190 | # it anyway. |
191 | - self.assertArchsForHint('any-any', ['armel', 'hppa', 'i386']) |
192 | + self.assertArchsForHint( |
193 | + 'any-any', {'armel': False, 'hppa': False, 'i386': True}) |
194 | |
195 | def test_no_all_builds_when_nominatedarchindep_not_permitted(self): |
196 | # Some archives (eg. armel rebuilds) don't want arch-indep |
197 | # builds. If the nominatedarchindep architecture (normally |
198 | # i386) is omitted, no builds will be created for arch-indep |
199 | # sources. |
200 | - self.assertArchsForHint('all', [], allowed_arch_tags=['hppa']) |
201 | + self.assertArchsForHint('all', {}, allowed_arch_tags=['hppa']) |
202 | + |
203 | + def test_indep_hint_only(self): |
204 | + # Some packages need to build arch-indep builds on a specific |
205 | + # architecture, declared using XS-Build-Indep-Architecture. |
206 | + self.assertArchsForHint('all', {'hppa': True}, indep_hint_list='hppa') |
207 | + |
208 | + def test_indep_hint_only_multiple(self): |
209 | + # The earliest available architecture in the available list (not |
210 | + # the hint list) is chosen. |
211 | + self.assertArchsForHint( |
212 | + 'all', {'armel': True}, indep_hint_list='armel hppa') |
213 | + self.assertArchsForHint( |
214 | + 'all', {'hppa': True}, indep_hint_list='armel hppa', |
215 | + allowed_arch_tags=['hppa', 'armel', 'i386']) |
216 | + |
217 | + def test_indep_hint_only_unsatisfiable(self): |
218 | + # An indep hint list that matches nothing results in no builds |
219 | + self.assertArchsForHint('all', {}, indep_hint_list='fiction') |
220 | + |
221 | + def test_indep_hint(self): |
222 | + # Unlike nominatedarchindep, a hinted indep will cause an |
223 | + # additional build to be created if necessary. |
224 | + self.assertArchsForHint( |
225 | + 'armel all', {'armel': False, 'hppa': True}, |
226 | + indep_hint_list='hppa') |
227 | + |
228 | + def test_indep_hint_wildcard(self): |
229 | + # An indep hint list can include wildcards. |
230 | + self.assertArchsForHint( |
231 | + 'armel all', {'armel': False, 'hppa': True}, |
232 | + indep_hint_list='any-hppa') |
233 | + |
234 | + def test_indep_hint_coalesces(self): |
235 | + # An indep hint list that matches an existing build will avoid |
236 | + # creating another. |
237 | + self.assertArchsForHint( |
238 | + 'hppa all', {'hppa': True}, indep_hint_list='linux-any') |
239 | + |
240 | + def test_indep_hint_unsatisfiable(self): |
241 | + # An indep hint list that matches nothing results in no |
242 | + # additional builds |
243 | + self.assertArchsForHint( |
244 | + 'armel all', {'armel': False}, indep_hint_list='fiction') |
245 | + |
246 | + def test_no_need_arch_indep(self): |
247 | + self.assertArchsForHint( |
248 | + 'armel all', {'armel': False}, need_arch_indep=False) |
249 | + |
250 | + def test_no_need_arch_indep_hint(self): |
251 | + self.assertArchsForHint( |
252 | + 'armel all', {'armel': False}, indep_hint_list='hppa', |
253 | + need_arch_indep=False) |
254 | + |
255 | + def test_no_need_arch_indep_only(self): |
256 | + self.assertArchsForHint('all', {}, need_arch_indep=False) |
257 | |
258 | === modified file 'lib/lp/soyuz/interfaces/sourcepackagerelease.py' |
259 | --- lib/lp/soyuz/interfaces/sourcepackagerelease.py 2014-11-09 23:28:27 +0000 |
260 | +++ lib/lp/soyuz/interfaces/sourcepackagerelease.py 2015-02-11 14:44:42 +0000 |
261 | @@ -151,6 +151,9 @@ |
262 | title=_("Source package recipe build"), |
263 | required=False, readonly=True) |
264 | |
265 | + def getUserDefinedField(name): |
266 | + """Case-insensitively get a user-defined field.""" |
267 | + |
268 | def addFile(file, filetype=None): |
269 | """Add the provided library file alias (file) to the list of files |
270 | in this package. |
271 | |
272 | === modified file 'lib/lp/soyuz/model/binarypackagebuild.py' |
273 | --- lib/lp/soyuz/model/binarypackagebuild.py 2015-02-11 14:44:42 +0000 |
274 | +++ lib/lp/soyuz/model/binarypackagebuild.py 2015-02-11 14:44:42 +0000 |
275 | @@ -1423,56 +1423,44 @@ |
276 | # this series already has it set. |
277 | need_arch_indep = not any(bpb.arch_indep for bpb in relevant_builds) |
278 | |
279 | - # Find the architectures for which the source should end up with |
280 | - # binaries, parsing architecturehintlist as you'd expect. |
281 | - # For an architecturehintlist of just 'all', this will |
282 | - # be the current nominatedarchindep if need_arch_indep, |
283 | - # otherwise nothing. |
284 | - valid_archs = list( |
285 | - architectures_available or distroseries.buildable_architectures) |
286 | - valid_archs = self._getAllowedArchitectures(archive, valid_archs) |
287 | + # Find the architectures for which the source chould end up with |
288 | + # new binaries. Exclude architectures not allowed in this |
289 | + # archive and architectures that have already built. Order by |
290 | + # Processor.id so determine_architectures_to_build is |
291 | + # deterministic. |
292 | + # XXX wgrant 2014-11-06: The fact that production's |
293 | + # Processor 1 is i386, a good arch-indep candidate, is a |
294 | + # total coincidence and this isn't a hack. I promise. |
295 | + need_archs = sorted( |
296 | + [das for das in |
297 | + self._getAllowedArchitectures( |
298 | + archive, |
299 | + architectures_available |
300 | + or distroseries.buildable_architectures) |
301 | + if das.architecturetag not in skip_archtags], |
302 | + key=attrgetter('processor.id')) |
303 | nominated_arch_indep_tag = ( |
304 | distroseries.nominatedarchindep.architecturetag |
305 | if distroseries.nominatedarchindep else None) |
306 | |
307 | - # Filter the valid archs against the hint list. |
308 | - candidate_arch_tags = determine_architectures_to_build( |
309 | + # Filter the valid archs against the hint list and work out |
310 | + # their arch-indepness. |
311 | + create_tag_map = determine_architectures_to_build( |
312 | sourcepackagerelease.architecturehintlist, |
313 | - [das.architecturetag for das in valid_archs], |
314 | + sourcepackagerelease.getUserDefinedField( |
315 | + 'Build-Indep-Architecture'), |
316 | + [das.architecturetag for das in need_archs], |
317 | nominated_arch_indep_tag, need_arch_indep) |
318 | - candidate_architectures = set( |
319 | - das for das in valid_archs |
320 | - if das.architecturetag in candidate_arch_tags) |
321 | - |
322 | - # Filter out any architectures for which we earlier found sufficient |
323 | - # builds. |
324 | - needed_architectures = [ |
325 | - das for das in candidate_architectures |
326 | - if das.architecturetag not in skip_archtags] |
327 | - if not needed_architectures: |
328 | - return [] |
329 | - |
330 | - arch_indep_das = None |
331 | - if need_arch_indep: |
332 | - # The ideal arch_indep build is nominatedarchindep. But if |
333 | - # that isn't a build we would otherwise create, use the DAS |
334 | - # with the lowest Processor.id. |
335 | - # XXX wgrant 2014-11-06: The fact that production's |
336 | - # Processor 1 is i386, a good arch-indep candidate, is a |
337 | - # total coincidence and this isn't a hack. I promise. |
338 | - if distroseries.nominatedarchindep in needed_architectures: |
339 | - arch_indep_das = distroseries.nominatedarchindep |
340 | - else: |
341 | - arch_indep_das = sorted( |
342 | - needed_architectures, key=attrgetter('processor.id'))[0] |
343 | |
344 | # Create builds for the remaining architectures. |
345 | new_builds = [] |
346 | - for das in needed_architectures: |
347 | + for das in sorted(need_archs, key=attrgetter('architecturetag')): |
348 | + if das.architecturetag not in create_tag_map: |
349 | + continue |
350 | build = self.new( |
351 | source_package_release=sourcepackagerelease, |
352 | distro_arch_series=das, archive=archive, pocket=pocket, |
353 | - arch_indep=das == arch_indep_das) |
354 | + arch_indep=create_tag_map[das.architecturetag]) |
355 | new_builds.append(build) |
356 | # Create the builds in suspended mode for disabled archives. |
357 | build_queue = build.queueBuild(suspended=not archive.enabled) |
358 | |
359 | === modified file 'lib/lp/soyuz/model/sourcepackagerelease.py' |
360 | --- lib/lp/soyuz/model/sourcepackagerelease.py 2014-11-09 23:28:27 +0000 |
361 | +++ lib/lp/soyuz/model/sourcepackagerelease.py 2015-02-11 14:44:42 +0000 |
362 | @@ -170,6 +170,12 @@ |
363 | return [] |
364 | return simplejson.loads(self._user_defined_fields) |
365 | |
366 | + def getUserDefinedField(self, name): |
367 | + if self.user_defined_fields: |
368 | + for k, v in self.user_defined_fields: |
369 | + if k.lower() == name.lower(): |
370 | + return v |
371 | + |
372 | @cachedproperty |
373 | def package_diffs(self): |
374 | return list(Store.of(self).find( |
375 | |
376 | === modified file 'lib/lp/soyuz/tests/test_build_set.py' |
377 | --- lib/lp/soyuz/tests/test_build_set.py 2015-02-11 14:44:42 +0000 |
378 | +++ lib/lp/soyuz/tests/test_build_set.py 2015-02-11 14:44:42 +0000 |
379 | @@ -491,6 +491,19 @@ |
380 | new_builds = self.createBuilds(spr, self.distroseries) |
381 | self.assertBuildsMatch({}, new_builds) |
382 | |
383 | + def test_createForSource_build_indep_architecture(self): |
384 | + # A user defined field of Build-Indep-Architecture provides a |
385 | + # custom hint list to override Architecture and |
386 | + # nominatedarchindep for arch-indep purposes. |
387 | + spr = self.factory.makeSourcePackageRelease( |
388 | + architecturehintlist='sparc all', |
389 | + user_defined_fields=[('build-indep-architecture', 'avr')]) |
390 | + builds = self.createBuilds(spr, self.distroseries2) |
391 | + self.assertBuildsMatch({'sparc': False, 'avr': True}, builds) |
392 | + self.completeBuilds(builds, {'sparc': True, 'avr': True}) |
393 | + new_builds = self.createBuilds(spr, self.distroseries) |
394 | + self.assertBuildsMatch({}, new_builds) |
395 | + |
396 | |
397 | class TestFindBuiltOrPublishedBySourceAndArchive(TestCaseWithFactory): |
398 | """Tests for findBuiltOrPublishedBySourceAndArchive().""" |