Merge lp:~michael.nelson/launchpad/487009-db-more-soyuz-extraction into lp:launchpad/db-devel
- 487009-db-more-soyuz-extraction
- Merge into db-devel
Proposed by
Michael Nelson
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | Henning Eggers | ||||
Approved revision: | not available | ||||
Merged at revision: | not available | ||||
Proposed branch: | lp:~michael.nelson/launchpad/487009-db-more-soyuz-extraction | ||||
Merge into: | lp:launchpad/db-devel | ||||
Prerequisite: | lp:~michael.nelson/launchpad/487009-db-generalise-ibuilder-1b | ||||
Diff against target: |
565 lines (+246/-191) 5 files modified
lib/lp/buildmaster/interfaces/buildfarmjobbehavior.py (+19/-4) lib/lp/buildmaster/model/buildfarmjobbehavior.py (+14/-0) lib/lp/soyuz/interfaces/builder.py (+0/-13) lib/lp/soyuz/model/binarypackagebuildbehavior.py (+206/-0) lib/lp/soyuz/model/builder.py (+7/-174) |
||||
To merge this branch: | bzr merge lp:~michael.nelson/launchpad/487009-db-more-soyuz-extraction | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Henning Eggers (community) | code | Approve | |
Review via email:
|
Commit message
Description of the change
To post a comment you must log in.
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Michael Nelson (michael.nelson) wrote : | # |
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
Henning Eggers (henninge) wrote : | # |
Thank you for doing this important groundwork for our bridging-the-gap strategy. It will benefit all of us and most importantly Ubuntu.
review:
Approve
(code)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lib/lp/buildmaster/interfaces/buildfarmjobbehavior.py' | |||
2 | --- lib/lp/buildmaster/interfaces/buildfarmjobbehavior.py 2009-12-02 08:50:30 +0000 | |||
3 | +++ lib/lp/buildmaster/interfaces/buildfarmjobbehavior.py 2009-12-02 08:50:31 +0000 | |||
4 | @@ -11,7 +11,7 @@ | |||
5 | 11 | 'IBuildFarmJobBehavior', | 11 | 'IBuildFarmJobBehavior', |
6 | 12 | ] | 12 | ] |
7 | 13 | 13 | ||
9 | 14 | from zope.interface import Interface | 14 | from zope.interface import Attribute, Interface |
10 | 15 | 15 | ||
11 | 16 | 16 | ||
12 | 17 | class BuildBehaviorMismatch(Exception): | 17 | class BuildBehaviorMismatch(Exception): |
13 | @@ -23,6 +23,9 @@ | |||
14 | 23 | 23 | ||
15 | 24 | class IBuildFarmJobBehavior(Interface): | 24 | class IBuildFarmJobBehavior(Interface): |
16 | 25 | 25 | ||
17 | 26 | status = Attribute( | ||
18 | 27 | "Generated status information for this particular job.") | ||
19 | 28 | |||
20 | 26 | def setBuilder(builder): | 29 | def setBuilder(builder): |
21 | 27 | """Sets the associated builder reference for this instance.""" | 30 | """Sets the associated builder reference for this instance.""" |
22 | 28 | 31 | ||
23 | @@ -33,6 +36,18 @@ | |||
24 | 33 | :param build_queue_item: A BuildQueueItem to build. | 36 | :param build_queue_item: A BuildQueueItem to build. |
25 | 34 | :param logger: A logger to be used to log diagnostic information. | 37 | :param logger: A logger to be used to log diagnostic information. |
26 | 35 | """ | 38 | """ |
30 | 36 | # A number of other methods to go here that can be customised for each of | 39 | |
31 | 37 | # the different build types (branch build, recipe build, translation | 40 | def dispatchBuildToSlave(build_queue_item, logger): |
32 | 38 | # build etc.) | 41 | """Dispatch a specific build to the slave. |
33 | 42 | |||
34 | 43 | :param build_queue_item: The `BuildQueueItem` that will be built. | ||
35 | 44 | :logger: A logger to be used to log diagnostic information. | ||
36 | 45 | """ | ||
37 | 46 | |||
38 | 47 | def verifyBuildRequest(build_queue_item, logger): | ||
39 | 48 | """Carry out any pre-build checks. | ||
40 | 49 | |||
41 | 50 | :param build_queue_item: The `BuildQueueItem` that is to be built. | ||
42 | 51 | :logger: A logger to be used to log diagnostic information. | ||
43 | 52 | """ | ||
44 | 53 | |||
45 | 39 | 54 | ||
46 | === modified file 'lib/lp/buildmaster/model/buildfarmjobbehavior.py' | |||
47 | --- lib/lp/buildmaster/model/buildfarmjobbehavior.py 2009-12-02 08:50:30 +0000 | |||
48 | +++ lib/lp/buildmaster/model/buildfarmjobbehavior.py 2009-12-02 08:50:31 +0000 | |||
49 | @@ -33,6 +33,10 @@ | |||
50 | 33 | """The builder should be set once and not changed.""" | 33 | """The builder should be set once and not changed.""" |
51 | 34 | self._builder = builder | 34 | self._builder = builder |
52 | 35 | 35 | ||
53 | 36 | def verifyBuildRequest(self, build_queue_item, logger): | ||
54 | 37 | """The default behavior is a no-op.""" | ||
55 | 38 | pass | ||
56 | 39 | |||
57 | 36 | 40 | ||
58 | 37 | class IdleBuildBehavior(BuildFarmJobBehaviorBase): | 41 | class IdleBuildBehavior(BuildFarmJobBehaviorBase): |
59 | 38 | 42 | ||
60 | @@ -48,3 +52,13 @@ | |||
61 | 48 | """See `IBuildFarmJobBehavior`.""" | 52 | """See `IBuildFarmJobBehavior`.""" |
62 | 49 | raise BuildBehaviorMismatch( | 53 | raise BuildBehaviorMismatch( |
63 | 50 | "Builder was idle when asked to log the start of a build.") | 54 | "Builder was idle when asked to log the start of a build.") |
64 | 55 | |||
65 | 56 | def dispatchBuildToSlave(self, build_queue_item, logger): | ||
66 | 57 | """See `IBuildFarmJobBehavior`.""" | ||
67 | 58 | raise BuildBehaviorMismatch( | ||
68 | 59 | "Builder was idle when asked to dispatch a build to the slave.") | ||
69 | 60 | |||
70 | 61 | @property | ||
71 | 62 | def status(self): | ||
72 | 63 | """See `IBuildFarmJobBehavior`.""" | ||
73 | 64 | return "Idle" | ||
74 | 51 | 65 | ||
75 | === modified file 'lib/lp/soyuz/interfaces/builder.py' | |||
76 | --- lib/lp/soyuz/interfaces/builder.py 2009-12-02 08:50:30 +0000 | |||
77 | +++ lib/lp/soyuz/interfaces/builder.py 2009-12-02 08:50:31 +0000 | |||
78 | @@ -154,19 +154,6 @@ | |||
79 | 154 | file. | 154 | file. |
80 | 155 | """ | 155 | """ |
81 | 156 | 156 | ||
82 | 157 | def cachePrivateSourceOnSlave(logger, build_queue_item): | ||
83 | 158 | """Ask the slave to download source files for a private build. | ||
84 | 159 | |||
85 | 160 | The slave will cache the files for the source in build_queue_item | ||
86 | 161 | to its local disk in preparation for a private build. Private builds | ||
87 | 162 | will always take the source files from the archive rather than the | ||
88 | 163 | librarian since the archive has more granular access to each | ||
89 | 164 | archive's files. | ||
90 | 165 | |||
91 | 166 | :param logger: A logger used for providing debug information. | ||
92 | 167 | :param build_queue_item: The `IBuildQueue` being built. | ||
93 | 168 | """ | ||
94 | 169 | |||
95 | 170 | def checkCanBuildForDistroArchSeries(distro_arch_series): | 157 | def checkCanBuildForDistroArchSeries(distro_arch_series): |
96 | 171 | """Check that the slave can compile for the given distro_arch_release. | 158 | """Check that the slave can compile for the given distro_arch_release. |
97 | 172 | 159 | ||
98 | 173 | 160 | ||
99 | === modified file 'lib/lp/soyuz/model/binarypackagebuildbehavior.py' | |||
100 | --- lib/lp/soyuz/model/binarypackagebuildbehavior.py 2009-12-02 08:50:30 +0000 | |||
101 | +++ lib/lp/soyuz/model/binarypackagebuildbehavior.py 2009-12-02 08:50:31 +0000 | |||
102 | @@ -11,11 +11,20 @@ | |||
103 | 11 | 'BinaryPackageBuildBehavior', | 11 | 'BinaryPackageBuildBehavior', |
104 | 12 | ] | 12 | ] |
105 | 13 | 13 | ||
106 | 14 | import socket | ||
107 | 15 | import xmlrpclib | ||
108 | 16 | |||
109 | 17 | from canonical.launchpad.webapp import urlappend | ||
110 | 14 | from lp.buildmaster.interfaces.buildfarmjobbehavior import ( | 18 | from lp.buildmaster.interfaces.buildfarmjobbehavior import ( |
111 | 15 | IBuildFarmJobBehavior) | 19 | IBuildFarmJobBehavior) |
112 | 16 | from lp.buildmaster.model.buildfarmjobbehavior import ( | 20 | from lp.buildmaster.model.buildfarmjobbehavior import ( |
113 | 17 | BuildFarmJobBehaviorBase) | 21 | BuildFarmJobBehaviorBase) |
114 | 22 | from lp.registry.interfaces.pocket import PackagePublishingPocket | ||
115 | 23 | from lp.soyuz.adapters.archivedependencies import ( | ||
116 | 24 | get_primary_current_component, get_sources_list_for_building) | ||
117 | 25 | from lp.soyuz.interfaces.archive import ArchivePurpose | ||
118 | 18 | from lp.soyuz.interfaces.build import IBuildSet | 26 | from lp.soyuz.interfaces.build import IBuildSet |
119 | 27 | from lp.soyuz.interfaces.builder import BuildSlaveFailure, CannotBuild | ||
120 | 19 | 28 | ||
121 | 20 | from zope.component import getUtility | 29 | from zope.component import getUtility |
122 | 21 | from zope.interface import implements | 30 | from zope.interface import implements |
123 | @@ -33,3 +42,200 @@ | |||
124 | 33 | 42 | ||
125 | 34 | logger.info("startBuild(%s, %s, %s, %s)", self._builder.url, | 43 | logger.info("startBuild(%s, %s, %s, %s)", self._builder.url, |
126 | 35 | spr.name, spr.version, build.pocket.title) | 44 | spr.name, spr.version, build.pocket.title) |
127 | 45 | |||
128 | 46 | @property | ||
129 | 47 | def status(self): | ||
130 | 48 | """See `IBuildFarmJobBehavior`.""" | ||
131 | 49 | build = getUtility(IBuildSet).getByQueueEntry( | ||
132 | 50 | self._builder.currentjob) | ||
133 | 51 | msg = 'Building %s' % build.title | ||
134 | 52 | archive = build.archive | ||
135 | 53 | if not archive.owner.private and (archive.is_ppa or archive.is_copy): | ||
136 | 54 | return '%s [%s/%s]' % (msg, archive.owner.name, archive.name) | ||
137 | 55 | else: | ||
138 | 56 | return msg | ||
139 | 57 | |||
140 | 58 | def dispatchBuildToSlave(self, build_queue_item, logger): | ||
141 | 59 | """See `IBuildFarmJobBehavior`.""" | ||
142 | 60 | |||
143 | 61 | # Start the binary package build on the slave builder. First | ||
144 | 62 | # we send the chroot. | ||
145 | 63 | build = getUtility(IBuildSet).getByQueueEntry(build_queue_item) | ||
146 | 64 | chroot = build.distroarchseries.getChroot() | ||
147 | 65 | self._builder.cacheFileOnSlave(logger, chroot) | ||
148 | 66 | |||
149 | 67 | # Build filemap structure with the files required in this build | ||
150 | 68 | # and send them to the slave. | ||
151 | 69 | # If the build is private we tell the slave to get the files from the | ||
152 | 70 | # archive instead of the librarian because the slaves cannot | ||
153 | 71 | # access the restricted librarian. | ||
154 | 72 | private = build.archive.private | ||
155 | 73 | if private: | ||
156 | 74 | self._cachePrivateSourceOnSlave(build_queue_item, logger) | ||
157 | 75 | filemap = {} | ||
158 | 76 | for source_file in build.sourcepackagerelease.files: | ||
159 | 77 | lfa = source_file.libraryfile | ||
160 | 78 | filemap[lfa.filename] = lfa.content.sha1 | ||
161 | 79 | if not private: | ||
162 | 80 | self._builder.cacheFileOnSlave( | ||
163 | 81 | logger, source_file.libraryfile) | ||
164 | 82 | |||
165 | 83 | # Generate a string which can be used to cross-check when obtaining | ||
166 | 84 | # results so we know we are referring to the right database object in | ||
167 | 85 | # subsequent runs. | ||
168 | 86 | buildid = "%s-%s" % (build.id, build_queue_item.id) | ||
169 | 87 | chroot_sha1 = chroot.content.sha1 | ||
170 | 88 | logger.debug( | ||
171 | 89 | "Initiating build %s on %s" % (buildid, self._builder.url)) | ||
172 | 90 | |||
173 | 91 | try: | ||
174 | 92 | args = self._extraBuildArgs(build) | ||
175 | 93 | status, info = self._builder.slave.build( | ||
176 | 94 | buildid, "debian", chroot_sha1, filemap, args) | ||
177 | 95 | message = """%s (%s): | ||
178 | 96 | ***** RESULT ***** | ||
179 | 97 | %s | ||
180 | 98 | %s | ||
181 | 99 | %s: %s | ||
182 | 100 | ****************** | ||
183 | 101 | """ % ( | ||
184 | 102 | self._builder.name, | ||
185 | 103 | self._builder.url, | ||
186 | 104 | filemap, | ||
187 | 105 | args, | ||
188 | 106 | status, | ||
189 | 107 | info, | ||
190 | 108 | ) | ||
191 | 109 | logger.info(message) | ||
192 | 110 | except xmlrpclib.Fault, info: | ||
193 | 111 | # Mark builder as 'failed'. | ||
194 | 112 | logger.debug( | ||
195 | 113 | "Disabling builder: %s" % self._builder.url, exc_info=1) | ||
196 | 114 | self._builder.failbuilder( | ||
197 | 115 | "Exception (%s) when setting up to new job" % info) | ||
198 | 116 | raise BuildSlaveFailure | ||
199 | 117 | except socket.error, info: | ||
200 | 118 | error_message = "Exception (%s) when setting up new job" % info | ||
201 | 119 | self._builder.handleTimeout(logger, error_message) | ||
202 | 120 | raise BuildSlaveFailure | ||
203 | 121 | |||
204 | 122 | def verifyBuildRequest(self, build_queue_item, logger): | ||
205 | 123 | """Assert some pre-build checks. | ||
206 | 124 | |||
207 | 125 | The build request is checked: | ||
208 | 126 | * Virtualized builds can't build on a non-virtual builder | ||
209 | 127 | * Ensure that we have a chroot | ||
210 | 128 | * Ensure that the build pocket allows builds for the current | ||
211 | 129 | distroseries state. | ||
212 | 130 | """ | ||
213 | 131 | build = getUtility(IBuildSet).getByQueueEntry(build_queue_item) | ||
214 | 132 | assert not (not self._builder.virtualized and build.is_virtualized), ( | ||
215 | 133 | "Attempt to build non-virtual item on a virtual builder.") | ||
216 | 134 | |||
217 | 135 | # Assert that we are not silently building SECURITY jobs. | ||
218 | 136 | # See findBuildCandidates. Once we start building SECURITY | ||
219 | 137 | # correctly from EMBARGOED archive this assertion can be removed. | ||
220 | 138 | # XXX Julian 2007-12-18 spec=security-in-soyuz: This is being | ||
221 | 139 | # addressed in the work on the blueprint: | ||
222 | 140 | # https://blueprints.launchpad.net/soyuz/+spec/security-in-soyuz | ||
223 | 141 | target_pocket = build.pocket | ||
224 | 142 | assert target_pocket != PackagePublishingPocket.SECURITY, ( | ||
225 | 143 | "Soyuz is not yet capable of building SECURITY uploads.") | ||
226 | 144 | |||
227 | 145 | # Ensure build has the needed chroot | ||
228 | 146 | chroot = build.distroarchseries.getChroot() | ||
229 | 147 | if chroot is None: | ||
230 | 148 | raise CannotBuild( | ||
231 | 149 | "Missing CHROOT for %s/%s/%s" % ( | ||
232 | 150 | build.distroseries.distribution.name, | ||
233 | 151 | build.distroseries.name, | ||
234 | 152 | build.distroarchseries.architecturetag) | ||
235 | 153 | ) | ||
236 | 154 | |||
237 | 155 | # The main distribution has policies to prevent uploads to some | ||
238 | 156 | # pockets (e.g. security) during different parts of the distribution | ||
239 | 157 | # series lifecycle. These do not apply to PPA builds nor any archive | ||
240 | 158 | # that allows release pocket updates. | ||
241 | 159 | if (build.archive.purpose != ArchivePurpose.PPA and | ||
242 | 160 | not build.archive.allowUpdatesToReleasePocket()): | ||
243 | 161 | # XXX Robert Collins 2007-05-26: not an explicit CannotBuild | ||
244 | 162 | # exception yet because the callers have not been audited | ||
245 | 163 | assert build.distroseries.canUploadToPocket(build.pocket), ( | ||
246 | 164 | "%s (%s) can not be built for pocket %s: invalid pocket due " | ||
247 | 165 | "to the series status of %s." | ||
248 | 166 | % (build.title, build.id, build.pocket.name, | ||
249 | 167 | build.distroseries.name)) | ||
250 | 168 | |||
251 | 169 | def _cachePrivateSourceOnSlave(self, build_queue_item, logger): | ||
252 | 170 | """Ask the slave to download source files for a private build. | ||
253 | 171 | |||
254 | 172 | The slave will cache the files for the source in build_queue_item | ||
255 | 173 | to its local disk in preparation for a private build. Private builds | ||
256 | 174 | will always take the source files from the archive rather than the | ||
257 | 175 | librarian since the archive has more granular access to each | ||
258 | 176 | archive's files. | ||
259 | 177 | |||
260 | 178 | :param build_queue_item: The `IBuildQueue` being built. | ||
261 | 179 | :param logger: A logger used for providing debug information. | ||
262 | 180 | """ | ||
263 | 181 | # The URL to the file in the archive consists of these parts: | ||
264 | 182 | # archive_url / makePoolPath() / filename | ||
265 | 183 | # Once this is constructed we add the http basic auth info. | ||
266 | 184 | |||
267 | 185 | # Avoid circular imports. | ||
268 | 186 | from lp.soyuz.model.publishing import makePoolPath | ||
269 | 187 | |||
270 | 188 | build = getUtility(IBuildSet).getByQueueEntry(build_queue_item) | ||
271 | 189 | archive = build.archive | ||
272 | 190 | archive_url = archive.archive_url | ||
273 | 191 | component_name = build.current_component.name | ||
274 | 192 | for source_file in build.sourcepackagerelease.files: | ||
275 | 193 | file_name = source_file.libraryfile.filename | ||
276 | 194 | sha1 = source_file.libraryfile.content.sha1 | ||
277 | 195 | source_name = build.sourcepackagerelease.sourcepackagename.name | ||
278 | 196 | poolpath = makePoolPath(source_name, component_name) | ||
279 | 197 | url = urlappend(archive_url, poolpath) | ||
280 | 198 | url = urlappend(url, file_name) | ||
281 | 199 | logger.debug("Asking builder on %s to ensure it has file %s " | ||
282 | 200 | "(%s, %s)" % ( | ||
283 | 201 | self._builder.url, file_name, url, sha1)) | ||
284 | 202 | self._builder._sendFileToSlave( | ||
285 | 203 | url, sha1, "buildd", archive.buildd_secret) | ||
286 | 204 | |||
287 | 205 | def _extraBuildArgs(self, build): | ||
288 | 206 | """ | ||
289 | 207 | Return the extra arguments required by the slave for the given build. | ||
290 | 208 | """ | ||
291 | 209 | # Build extra arguments. | ||
292 | 210 | args = {} | ||
293 | 211 | # turn 'arch_indep' ON only if build is archindep or if | ||
294 | 212 | # the specific architecture is the nominatedarchindep for | ||
295 | 213 | # this distroseries (in case it requires any archindep source) | ||
296 | 214 | args['arch_indep'] = build.distroarchseries.isNominatedArchIndep | ||
297 | 215 | |||
298 | 216 | suite = build.distroarchseries.distroseries.name | ||
299 | 217 | if build.pocket != PackagePublishingPocket.RELEASE: | ||
300 | 218 | suite += "-%s" % (build.pocket.name.lower()) | ||
301 | 219 | args['suite'] = suite | ||
302 | 220 | |||
303 | 221 | archive_purpose = build.archive.purpose | ||
304 | 222 | if (archive_purpose == ArchivePurpose.PPA and | ||
305 | 223 | not build.archive.require_virtualized): | ||
306 | 224 | # If we're building a non-virtual PPA, override the purpose | ||
307 | 225 | # to PRIMARY and use the primary component override. | ||
308 | 226 | # This ensures that the package mangling tools will run over | ||
309 | 227 | # the built packages. | ||
310 | 228 | args['archive_purpose'] = ArchivePurpose.PRIMARY.name | ||
311 | 229 | args["ogrecomponent"] = ( | ||
312 | 230 | get_primary_current_component(build)) | ||
313 | 231 | else: | ||
314 | 232 | args['archive_purpose'] = archive_purpose.name | ||
315 | 233 | args["ogrecomponent"] = ( | ||
316 | 234 | build.current_component.name) | ||
317 | 235 | |||
318 | 236 | args['archives'] = get_sources_list_for_building(build) | ||
319 | 237 | |||
320 | 238 | # Let the build slave know whether this is a build in a private | ||
321 | 239 | # archive. | ||
322 | 240 | args['archive_private'] = build.archive.private | ||
323 | 241 | return args | ||
324 | 36 | 242 | ||
325 | === modified file 'lib/lp/soyuz/model/builder.py' | |||
326 | --- lib/lp/soyuz/model/builder.py 2009-12-02 08:50:30 +0000 | |||
327 | +++ lib/lp/soyuz/model/builder.py 2009-12-02 08:50:31 +0000 | |||
328 | @@ -38,8 +38,6 @@ | |||
329 | 38 | from lp.buildmaster.master import BuilddMaster | 38 | from lp.buildmaster.master import BuilddMaster |
330 | 39 | from lp.buildmaster.model.buildfarmjobbehavior import IdleBuildBehavior | 39 | from lp.buildmaster.model.buildfarmjobbehavior import IdleBuildBehavior |
331 | 40 | from canonical.database.sqlbase import SQLBase, sqlvalues | 40 | from canonical.database.sqlbase import SQLBase, sqlvalues |
332 | 41 | from lp.soyuz.adapters.archivedependencies import ( | ||
333 | 42 | get_primary_current_component, get_sources_list_for_building) | ||
334 | 43 | from lp.soyuz.model.buildqueue import BuildQueue | 41 | from lp.soyuz.model.buildqueue import BuildQueue |
335 | 44 | from lp.registry.interfaces.person import validate_public_person | 42 | from lp.registry.interfaces.person import validate_public_person |
336 | 45 | from lp.registry.interfaces.pocket import PackagePublishingPocket | 43 | from lp.registry.interfaces.pocket import PackagePublishingPocket |
337 | @@ -117,6 +115,7 @@ | |||
338 | 117 | 115 | ||
339 | 118 | return (stdout, stderr, resume_process.returncode) | 116 | return (stdout, stderr, resume_process.returncode) |
340 | 119 | 117 | ||
341 | 118 | |||
342 | 120 | class Builder(SQLBase): | 119 | class Builder(SQLBase): |
343 | 121 | 120 | ||
344 | 122 | implements(IBuilder, IHasBuildRecords) | 121 | implements(IBuilder, IHasBuildRecords) |
345 | @@ -195,30 +194,6 @@ | |||
346 | 195 | url, libraryfilealias.content.sha1)) | 194 | url, libraryfilealias.content.sha1)) |
347 | 196 | self._sendFileToSlave(url, libraryfilealias.content.sha1) | 195 | self._sendFileToSlave(url, libraryfilealias.content.sha1) |
348 | 197 | 196 | ||
349 | 198 | def cachePrivateSourceOnSlave(self, logger, build_queue_item): | ||
350 | 199 | """See `IBuilder`.""" | ||
351 | 200 | # The URL to the file in the archive consists of these parts: | ||
352 | 201 | # archive_url / makePoolPath() / filename | ||
353 | 202 | # Once this is constructed we add the http basic auth info. | ||
354 | 203 | |||
355 | 204 | # Avoid circular imports. | ||
356 | 205 | from lp.soyuz.model.publishing import makePoolPath | ||
357 | 206 | |||
358 | 207 | build = getUtility(IBuildSet).getByQueueEntry(build_queue_item) | ||
359 | 208 | archive = build.archive | ||
360 | 209 | archive_url = archive.archive_url | ||
361 | 210 | component_name = build.current_component.name | ||
362 | 211 | for source_file in build.sourcepackagerelease.files: | ||
363 | 212 | file_name = source_file.libraryfile.filename | ||
364 | 213 | sha1 = source_file.libraryfile.content.sha1 | ||
365 | 214 | source_name = build.sourcepackagerelease.sourcepackagename.name | ||
366 | 215 | poolpath = makePoolPath(source_name, component_name) | ||
367 | 216 | url = urlappend(archive_url, poolpath) | ||
368 | 217 | url = urlappend(url, file_name) | ||
369 | 218 | logger.debug("Asking builder on %s to ensure it has file %s " | ||
370 | 219 | "(%s, %s)" % (self.url, file_name, url, sha1)) | ||
371 | 220 | self._sendFileToSlave(url, sha1, "buildd", archive.buildd_secret) | ||
372 | 221 | |||
373 | 222 | def _sendFileToSlave(self, url, sha1, username="", password=""): | 197 | def _sendFileToSlave(self, url, sha1, username="", password=""): |
374 | 223 | """Helper to send the file at 'url' with 'sha1' to this builder.""" | 198 | """Helper to send the file at 'url' with 'sha1' to this builder.""" |
375 | 224 | if not self.builderok: | 199 | if not self.builderok: |
376 | @@ -308,99 +283,6 @@ | |||
377 | 308 | """See IBuilder.""" | 283 | """See IBuilder.""" |
378 | 309 | self.slave = proxy | 284 | self.slave = proxy |
379 | 310 | 285 | ||
380 | 311 | def _verifyBuildRequest(self, build_queue_item, logger): | ||
381 | 312 | """Assert some pre-build checks. | ||
382 | 313 | |||
383 | 314 | The build request is checked: | ||
384 | 315 | * Virtualized builds can't build on a non-virtual builder | ||
385 | 316 | * Ensure that we have a chroot | ||
386 | 317 | * Ensure that the build pocket allows builds for the current | ||
387 | 318 | distroseries state. | ||
388 | 319 | """ | ||
389 | 320 | build = getUtility(IBuildSet).getByQueueEntry(build_queue_item) | ||
390 | 321 | assert not (not self.virtualized and build.is_virtualized), ( | ||
391 | 322 | "Attempt to build non-virtual item on a virtual builder.") | ||
392 | 323 | |||
393 | 324 | # Assert that we are not silently building SECURITY jobs. | ||
394 | 325 | # See findBuildCandidates. Once we start building SECURITY | ||
395 | 326 | # correctly from EMBARGOED archive this assertion can be removed. | ||
396 | 327 | # XXX Julian 2007-12-18 spec=security-in-soyuz: This is being | ||
397 | 328 | # addressed in the work on the blueprint: | ||
398 | 329 | # https://blueprints.launchpad.net/soyuz/+spec/security-in-soyuz | ||
399 | 330 | target_pocket = build.pocket | ||
400 | 331 | assert target_pocket != PackagePublishingPocket.SECURITY, ( | ||
401 | 332 | "Soyuz is not yet capable of building SECURITY uploads.") | ||
402 | 333 | |||
403 | 334 | # Ensure build has the needed chroot | ||
404 | 335 | build = getUtility(IBuildSet).getByQueueEntry(build_queue_item) | ||
405 | 336 | chroot = build.distroarchseries.getChroot() | ||
406 | 337 | if chroot is None: | ||
407 | 338 | raise CannotBuild( | ||
408 | 339 | "Missing CHROOT for %s/%s/%s" % ( | ||
409 | 340 | build.distroseries.distribution.name, | ||
410 | 341 | build.distroseries.name, | ||
411 | 342 | build.distroarchseries.architecturetag) | ||
412 | 343 | ) | ||
413 | 344 | |||
414 | 345 | # The main distribution has policies to prevent uploads to some | ||
415 | 346 | # pockets (e.g. security) during different parts of the distribution | ||
416 | 347 | # series lifecycle. These do not apply to PPA builds nor any archive | ||
417 | 348 | # that allows release pocket updates. | ||
418 | 349 | if (build.archive.purpose != ArchivePurpose.PPA and | ||
419 | 350 | not build.archive.allowUpdatesToReleasePocket()): | ||
420 | 351 | # XXX Robert Collins 2007-05-26: not an explicit CannotBuild | ||
421 | 352 | # exception yet because the callers have not been audited | ||
422 | 353 | assert build.distroseries.canUploadToPocket(build.pocket), ( | ||
423 | 354 | "%s (%s) can not be built for pocket %s: invalid pocket due " | ||
424 | 355 | "to the series status of %s." | ||
425 | 356 | % (build.title, build.id, build.pocket.name, | ||
426 | 357 | build.distroseries.name)) | ||
427 | 358 | |||
428 | 359 | def _dispatchBuildToSlave(self, build_queue_item, args, buildid, logger): | ||
429 | 360 | """Start the build on the slave builder.""" | ||
430 | 361 | # Send chroot. | ||
431 | 362 | build = getUtility(IBuildSet).getByQueueEntry(build_queue_item) | ||
432 | 363 | chroot = build.distroarchseries.getChroot() | ||
433 | 364 | self.cacheFileOnSlave(logger, chroot) | ||
434 | 365 | |||
435 | 366 | # Build filemap structure with the files required in this build | ||
436 | 367 | # and send them to the slave. | ||
437 | 368 | # If the build is private we tell the slave to get the files from the | ||
438 | 369 | # archive instead of the librarian because the slaves cannot | ||
439 | 370 | # access the restricted librarian. | ||
440 | 371 | private = build.archive.private | ||
441 | 372 | if private: | ||
442 | 373 | self.cachePrivateSourceOnSlave(logger, build_queue_item) | ||
443 | 374 | filemap = {} | ||
444 | 375 | for source_file in build.sourcepackagerelease.files: | ||
445 | 376 | lfa = source_file.libraryfile | ||
446 | 377 | filemap[lfa.filename] = lfa.content.sha1 | ||
447 | 378 | if not private: | ||
448 | 379 | self.cacheFileOnSlave(logger, source_file.libraryfile) | ||
449 | 380 | |||
450 | 381 | chroot_sha1 = chroot.content.sha1 | ||
451 | 382 | try: | ||
452 | 383 | status, info = self.slave.build( | ||
453 | 384 | buildid, "debian", chroot_sha1, filemap, args) | ||
454 | 385 | message = """%s (%s): | ||
455 | 386 | ***** RESULT ***** | ||
456 | 387 | %s | ||
457 | 388 | %s | ||
458 | 389 | %s: %s | ||
459 | 390 | ****************** | ||
460 | 391 | """ % (self.name, self.url, filemap, args, status, info) | ||
461 | 392 | logger.info(message) | ||
462 | 393 | except xmlrpclib.Fault, info: | ||
463 | 394 | # Mark builder as 'failed'. | ||
464 | 395 | logger.debug("Disabling builder: %s" % self.url, exc_info=1) | ||
465 | 396 | self.failbuilder( | ||
466 | 397 | "Exception (%s) when setting up to new job" % info) | ||
467 | 398 | raise BuildSlaveFailure | ||
468 | 399 | except socket.error, info: | ||
469 | 400 | error_message = "Exception (%s) when setting up new job" % info | ||
470 | 401 | self.handleTimeout(logger, error_message) | ||
471 | 402 | raise BuildSlaveFailure | ||
472 | 403 | |||
473 | 404 | def startBuild(self, build_queue_item, logger): | 286 | def startBuild(self, build_queue_item, logger): |
474 | 405 | """See IBuilder.""" | 287 | """See IBuilder.""" |
475 | 406 | # Set the build behavior depending on the provided build queue item. | 288 | # Set the build behavior depending on the provided build queue item. |
476 | @@ -408,55 +290,15 @@ | |||
477 | 408 | self.logStartBuild(build_queue_item, logger) | 290 | self.logStartBuild(build_queue_item, logger) |
478 | 409 | 291 | ||
479 | 410 | # Make sure the request is valid; an exception is raised if it's not. | 292 | # Make sure the request is valid; an exception is raised if it's not. |
481 | 411 | self._verifyBuildRequest(build_queue_item, logger) | 293 | self.verifyBuildRequest(build_queue_item, logger) |
482 | 412 | 294 | ||
483 | 413 | # If we are building a virtual build, resume the virtual machine. | 295 | # If we are building a virtual build, resume the virtual machine. |
484 | 414 | if self.virtualized: | 296 | if self.virtualized: |
485 | 415 | self.resumeSlaveHost() | 297 | self.resumeSlaveHost() |
486 | 416 | 298 | ||
487 | 417 | # Build extra arguments. | ||
488 | 418 | args = {} | ||
489 | 419 | # turn 'arch_indep' ON only if build is archindep or if | ||
490 | 420 | # the specific architecture is the nominatedarchindep for | ||
491 | 421 | # this distroseries (in case it requires any archindep source) | ||
492 | 422 | build = getUtility(IBuildSet).getByQueueEntry(build_queue_item) | ||
493 | 423 | args['arch_indep'] = build.distroarchseries.isNominatedArchIndep | ||
494 | 424 | |||
495 | 425 | suite = build.distroarchseries.distroseries.name | ||
496 | 426 | if build.pocket != PackagePublishingPocket.RELEASE: | ||
497 | 427 | suite += "-%s" % (build.pocket.name.lower()) | ||
498 | 428 | args['suite'] = suite | ||
499 | 429 | |||
500 | 430 | archive_purpose = build.archive.purpose | ||
501 | 431 | if (archive_purpose == ArchivePurpose.PPA and | ||
502 | 432 | not build.archive.require_virtualized): | ||
503 | 433 | # If we're building a non-virtual PPA, override the purpose | ||
504 | 434 | # to PRIMARY and use the primary component override. | ||
505 | 435 | # This ensures that the package mangling tools will run over | ||
506 | 436 | # the built packages. | ||
507 | 437 | args['archive_purpose'] = ArchivePurpose.PRIMARY.name | ||
508 | 438 | args["ogrecomponent"] = ( | ||
509 | 439 | get_primary_current_component(build)) | ||
510 | 440 | else: | ||
511 | 441 | args['archive_purpose'] = archive_purpose.name | ||
512 | 442 | args["ogrecomponent"] = ( | ||
513 | 443 | build.current_component.name) | ||
514 | 444 | |||
515 | 445 | args['archives'] = get_sources_list_for_building(build) | ||
516 | 446 | |||
517 | 447 | # Let the build slave know whether this is a build in a private | ||
518 | 448 | # archive. | ||
519 | 449 | args['archive_private'] = build.archive.private | ||
520 | 450 | |||
521 | 451 | # Generate a string which can be used to cross-check when obtaining | ||
522 | 452 | # results so we know we are referring to the right database object in | ||
523 | 453 | # subsequent runs. | ||
524 | 454 | buildid = "%s-%s" % (build.id, build_queue_item.id) | ||
525 | 455 | logger.debug("Initiating build %s on %s" % (buildid, self.url)) | ||
526 | 456 | |||
527 | 457 | # Do it. | 299 | # Do it. |
528 | 458 | build_queue_item.markAsBuilding(self) | 300 | build_queue_item.markAsBuilding(self) |
530 | 459 | self._dispatchBuildToSlave(build_queue_item, args, buildid, logger) | 301 | self.dispatchBuildToSlave(build_queue_item, logger) |
531 | 460 | 302 | ||
532 | 461 | # XXX cprov 2009-06-24: This code does not belong to the content | 303 | # XXX cprov 2009-06-24: This code does not belong to the content |
533 | 462 | # class domain. Here we cannot make sensible decisions about what | 304 | # class domain. Here we cannot make sensible decisions about what |
534 | @@ -469,19 +311,10 @@ | |||
535 | 469 | if self.failnotes is not None: | 311 | if self.failnotes is not None: |
536 | 470 | return self.failnotes | 312 | return self.failnotes |
537 | 471 | return 'Disabled' | 313 | return 'Disabled' |
538 | 472 | # Cache the 'currentjob', so we don't have to hit the database | ||
539 | 473 | # more than once. | ||
540 | 474 | currentjob = self.currentjob | ||
541 | 475 | if currentjob is None: | ||
542 | 476 | return 'Idle' | ||
543 | 477 | 314 | ||
551 | 478 | build = getUtility(IBuildSet).getByQueueEntry(currentjob) | 315 | # If the builder is OK then we delegate the status |
552 | 479 | msg = 'Building %s' % build.title | 316 | # to our current behavior. |
553 | 480 | archive = build.archive | 317 | return self.current_build_behavior.status |
547 | 481 | if not archive.owner.private and (archive.is_ppa or archive.is_copy): | ||
548 | 482 | return '%s [%s/%s]' % (msg, archive.owner.name, archive.name) | ||
549 | 483 | else: | ||
550 | 484 | return msg | ||
554 | 485 | 318 | ||
555 | 486 | def failbuilder(self, reason): | 319 | def failbuilder(self, reason): |
556 | 487 | """See IBuilder""" | 320 | """See IBuilder""" |
557 | @@ -722,7 +555,7 @@ | |||
558 | 722 | logger = self._getSlaveScannerLogger() | 555 | logger = self._getSlaveScannerLogger() |
559 | 723 | try: | 556 | try: |
560 | 724 | self.startBuild(candidate, logger) | 557 | self.startBuild(candidate, logger) |
562 | 725 | except (BuildSlaveFailure, CannotBuild), err: | 558 | except (BuildSlaveFailure, CannotBuild, BuildBehaviorMismatch), err: |
563 | 726 | logger.warn('Could not build: %s' % err) | 559 | logger.warn('Could not build: %s' % err) |
564 | 727 | 560 | ||
565 | 728 | def handleTimeout(self, logger, error_message): | 561 | def handleTimeout(self, logger, error_message): |
= Summary =
This branch continues on from the prerequisite at:
https:/ /code.edge. launchpad. net/~michael. nelson/ launchpad/ 487009- db-generalise- ibuilder- 1b/+merge/ 15477
and extracts most of the remaining soyuz-specific binary build logic from IBuilder and into the custom BinaryPackageBu ildBehavior to which IBuilder delegates for binary builds, in particular leaving the IBuilder. startBuild( ) method looking like:
http:// pastebin. ubuntu. com/332947/
with verifyBuildRequ est() and dispatchBuildTo Slave() now being delegated to the current build behavior.
I will have one further branch that: getBuildRecords () slaveStatus( ) havior with examples for adding further behaviors.
* extracts IBuilder.
* extracts IBuilder.
* adds documentation for the IBuildFarmJobBe
== Pre-implementation notes ==
See the pre-implementation notes on bug 487009.
== Implementation details ==
== Tests ==
bin/test -vv -t doc/builder.txt -t doc/buildd- dispatching. txt
== Demo and Q/A ==
We will need to Q/A this on dogfood.
= Launchpad lint =
lib/lp/ soyuz/model/ builder. py
23: [F0401] Unable to import 'lazr.delegates' (No module named delegates)