Launchpad itself

Merge lp:~wgrant/launchpad/flatten-bfj-4-app-eliminate into lp:launchpad

  1. flatten-bfj-4-app-eliminate
  2. Merge into devel
Proposed by William Grant
Status: Merged
Approved by: William Grant
Approved revision: no longer in the source branch.
Merged at revision: 16479
Proposed branch: lp:~wgrant/launchpad/flatten-bfj-4-app-eliminate
Merge into: lp:launchpad
Prerequisite: lp:~wgrant/launchpad/flatten-bfj-3.5-more-query
Diff against target: 365 lines (+32/-91)
9 files modified
lib/lp/buildmaster/model/buildfarmjob.py (+9/-31)
lib/lp/buildmaster/model/packagebuild.py (+3/-4)
lib/lp/buildmaster/tests/test_buildfarmjob.py (+0/-7)
lib/lp/buildmaster/tests/test_manager.py (+0/-1)
lib/lp/buildmaster/tests/test_packagebuild.py (+1/-2)
lib/lp/code/model/sourcepackagerecipebuild.py (+8/-19)
lib/lp/code/model/tests/test_sourcepackagerecipebuild.py (+2/-8)
lib/lp/soyuz/model/binarypackagebuild.py (+8/-18)
lib/lp/translations/model/translationtemplatesbuild.py (+1/-1)
To merge this branch: bzr merge lp:~wgrant/launchpad/flatten-bfj-4-app-eliminate
Reviewer Review Type Date Requested Status
Steve Kowalik (community) code Approve
Review via email: mp+145820@code.launchpad.net

Commit message

Remove the deprecated copies of the denormalised BuildFarmJob columns, and stop creating PackageBuilds.

Description of the change

The build farm job schema is being reworked to improve performance. Columns from PackageBuild and BuildFarmJob are being merged into tables that previously delegated to them. The PackageBuild table will end up dying entirely, but BuildFarmJob will remain, a shadow of its former self, to answer questions about Archive:+builds and Builder:+history. Additionally, BinaryPackageBuild is growing new distribution, distroseries, sourcepackagename and is_distro_archive columns to make searches even faster.

This branch removes the writes to deprecated columns. PackageBuild records are no longer created for BPB/SPRBs, and most of BuildFarmJob's columns have been removed.

To post a comment you must log in.
Revision history for this message
Steve Kowalik (stevenk) :
review: Approve (code)

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'lib/lp/buildmaster/model/buildfarmjob.py'
2--- lib/lp/buildmaster/model/buildfarmjob.py 2013-02-05 00:28:25 +0000
3+++ lib/lp/buildmaster/model/buildfarmjob.py 2013-02-06 10:45:34 +0000
4@@ -18,7 +18,6 @@
5 Or,
6 )
7 from storm.locals import (
8- Bool,
9 DateTime,
10 Int,
11 Reference,
12@@ -157,56 +156,37 @@
13
14 id = Int(primary=True)
15
16- processor_id = Int(name='processor', allow_none=True)
17- processor = Reference(processor_id, 'Processor.id')
18-
19- virtualized = Bool()
20-
21 date_created = DateTime(
22 name='date_created', allow_none=False, tzinfo=pytz.UTC)
23
24- date_started = DateTime(
25- name='date_started', allow_none=True, tzinfo=pytz.UTC)
26-
27 date_finished = DateTime(
28 name='date_finished', allow_none=True, tzinfo=pytz.UTC)
29
30- date_first_dispatched = DateTime(
31- name='date_first_dispatched', allow_none=True, tzinfo=pytz.UTC)
32-
33 builder_id = Int(name='builder', allow_none=True)
34 builder = Reference(builder_id, 'Builder.id')
35
36 status = DBEnum(name='status', allow_none=False, enum=BuildStatus)
37
38- log_id = Int(name='log', allow_none=True)
39- log = Reference(log_id, 'LibraryFileAlias.id')
40-
41 job_type = DBEnum(
42 name='job_type', allow_none=False, enum=BuildFarmJobType)
43
44- failure_count = Int(name='failure_count', allow_none=False)
45-
46 archive_id = Int(name='archive')
47 archive = Reference(archive_id, 'Archive.id')
48
49 def __init__(self, job_type, status=BuildStatus.NEEDSBUILD,
50- processor=None, virtualized=None, date_created=None,
51- builder=None, archive=None):
52+ date_created=None, builder=None, archive=None):
53 super(BuildFarmJob, self).__init__()
54- (self.job_type, self.status, self.processor, self.virtualized,
55- self.builder, self.archive) = (
56- job_type, status, processor, virtualized, builder, archive)
57+ (self.job_type, self.status, self.builder, self.archive) = (
58+ job_type, status, builder, archive)
59 if date_created is not None:
60 self.date_created = date_created
61
62 @classmethod
63- def new(cls, job_type, status=BuildStatus.NEEDSBUILD, processor=None,
64- virtualized=None, date_created=None, builder=None, archive=None):
65+ def new(cls, job_type, status=BuildStatus.NEEDSBUILD, date_created=None,
66+ builder=None, archive=None):
67 """See `IBuildFarmJobSource`."""
68 build_farm_job = BuildFarmJob(
69- job_type, status, processor, virtualized, date_created, builder,
70- archive)
71+ job_type, status, date_created, builder, archive)
72 store = IMasterStore(BuildFarmJob)
73 store.add(build_farm_job)
74 return build_farm_job
75@@ -309,7 +289,7 @@
76
77 def setLog(self, log):
78 """See `IBuildFarmJob`."""
79- self.build_farm_job.log = self._new_log = log
80+ self._new_log = log
81
82 def updateStatus(self, status, builder=None, slave_status=None,
83 date_started=None, date_finished=None):
84@@ -328,10 +308,9 @@
85 # If we're starting to build, set date_started and
86 # date_first_dispatched if required.
87 if self.date_started is None and status == BuildStatus.BUILDING:
88- self.build_farm_job.date_started = self._new_date_started = (
89+ self._new_date_started = (
90 date_started or datetime.datetime.now(pytz.UTC))
91 if self.date_first_dispatched is None:
92- self.build_farm_job.date_first_dispatched = self.date_started
93 self._new_date_first_dispatched = self.date_started
94
95 # If we're in a final build state (or UPLOADING, which sort of
96@@ -348,8 +327,7 @@
97
98 def gotFailure(self):
99 """See `IBuildFarmJob`."""
100- self.build_farm_job.failure_count += 1
101- self._new_failure_count = self.build_farm_job.failure_count
102+ self._new_failure_count += 1
103
104
105 class BuildFarmJobSet:
106
107=== modified file 'lib/lp/buildmaster/model/packagebuild.py'
108--- lib/lp/buildmaster/model/packagebuild.py 2013-01-31 23:09:35 +0000
109+++ lib/lp/buildmaster/model/packagebuild.py 2013-02-06 10:45:34 +0000
110@@ -158,10 +158,9 @@
111
112 if (status == BuildStatus.MANUALDEPWAIT and slave_status is not None
113 and slave_status.get('dependencies') is not None):
114- self.package_build.dependencies = self._new_dependencies = (
115- unicode(slave_status.get('dependencies')))
116+ self._new_dependencies = unicode(slave_status.get('dependencies'))
117 else:
118- self.package_build.dependencies = self._new_dependencies = None
119+ self._new_dependencies = None
120
121 def verifySuccessfulUpload(self):
122 """See `IPackageBuild`."""
123@@ -197,7 +196,7 @@
124 """See `IPackageBuild`."""
125 filename = "upload_%s_log.txt" % self.id
126 library_file = self.createUploadLog(content, filename=filename)
127- self.package_build.upload_log = self._new_upload_log = library_file
128+ self._new_upload_log = library_file
129
130 def notify(self, extra_info=None):
131 """See `IPackageBuild`."""
132
133=== modified file 'lib/lp/buildmaster/tests/test_buildfarmjob.py'
134--- lib/lp/buildmaster/tests/test_buildfarmjob.py 2013-01-31 23:09:35 +0000
135+++ lib/lp/buildmaster/tests/test_buildfarmjob.py 2013-02-06 10:45:34 +0000
136@@ -92,16 +92,9 @@
137 # The job type is required to create a build farm job.
138 self.assertEqual(
139 BuildFarmJobType.PACKAGEBUILD, bfj.job_type)
140- # Failure count defaults to zero.
141- self.assertEqual(0, bfj.failure_count)
142 # Other attributes are unset by default.
143- self.assertEqual(None, bfj.processor)
144- self.assertEqual(None, bfj.virtualized)
145- self.assertEqual(None, bfj.date_started)
146 self.assertEqual(None, bfj.date_finished)
147- self.assertEqual(None, bfj.date_first_dispatched)
148 self.assertEqual(None, bfj.builder)
149- self.assertEqual(None, bfj.log)
150
151 def test_date_created(self):
152 # date_created can be passed optionally when creating a
153
154=== modified file 'lib/lp/buildmaster/tests/test_manager.py'
155--- lib/lp/buildmaster/tests/test_manager.py 2013-01-30 08:02:59 +0000
156+++ lib/lp/buildmaster/tests/test_manager.py 2013-02-06 10:45:34 +0000
157@@ -322,7 +322,6 @@
158
159 builder.failure_count = builder_count
160 naked_job = removeSecurityProxy(builder.currentjob.specific_job)
161- naked_job.build.build_farm_job.failure_count = job_count
162 naked_job.build._new_failure_count = job_count
163 # The _scanFailed() calls abort, so make sure our existing
164 # failure counts are persisted.
165
166=== modified file 'lib/lp/buildmaster/tests/test_packagebuild.py'
167--- lib/lp/buildmaster/tests/test_packagebuild.py 2013-01-31 23:09:35 +0000
168+++ lib/lp/buildmaster/tests/test_packagebuild.py 2013-02-06 10:45:34 +0000
169@@ -46,8 +46,7 @@
170 if archive is None:
171 archive = self.factory.makeArchive()
172
173- bfj = getUtility(IBuildFarmJobSource).new(
174- job_type, virtualized=True, status=status)
175+ bfj = getUtility(IBuildFarmJobSource).new(job_type, status=status)
176 return getUtility(IPackageBuildSource).new(bfj, archive, pocket)
177
178
179
180=== modified file 'lib/lp/code/model/sourcepackagerecipebuild.py'
181--- lib/lp/code/model/sourcepackagerecipebuild.py 2013-01-31 10:55:59 +0000
182+++ lib/lp/code/model/sourcepackagerecipebuild.py 2013-02-06 10:45:34 +0000
183@@ -40,16 +40,12 @@
184 BuildStatus,
185 )
186 from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobSource
187-from lp.buildmaster.interfaces.packagebuild import IPackageBuildSource
188 from lp.buildmaster.model.buildfarmjob import (
189 BuildFarmJob,
190 BuildFarmJobOld,
191 )
192 from lp.buildmaster.model.buildqueue import BuildQueue
193-from lp.buildmaster.model.packagebuild import (
194- PackageBuild,
195- PackageBuildMixin,
196- )
197+from lp.buildmaster.model.packagebuild import PackageBuildMixin
198 from lp.code.errors import (
199 BuildAlreadyPending,
200 BuildNotAllowedForDistro,
201@@ -89,9 +85,6 @@
202 implements(ISourcePackageRecipeBuild)
203 classProvides(ISourcePackageRecipeBuildSource)
204
205- package_build_id = Int(name='package_build', allow_none=False)
206- package_build = Reference(package_build_id, 'PackageBuild.id')
207-
208 build_farm_job_type = BuildFarmJobType.RECIPEBRANCHBUILD
209 job_type = build_farm_job_type
210
211@@ -213,12 +206,11 @@
212 branch_name = self.recipe.base_branch.unique_name
213 return '%s recipe build' % branch_name
214
215- def __init__(self, build_farm_job, package_build, distroseries, recipe,
216- requester, archive, pocket, date_created):
217+ def __init__(self, build_farm_job, distroseries, recipe, requester,
218+ archive, pocket, date_created):
219 """Construct a SourcePackageRecipeBuild."""
220 super(SourcePackageRecipeBuild, self).__init__()
221 self.build_farm_job = build_farm_job
222- self.package_build = package_build
223 self.distroseries = distroseries
224 self.recipe = recipe
225 self.requester = requester
226@@ -239,13 +231,11 @@
227 if date_created is None:
228 date_created = UTC_NOW
229 build_farm_job = getUtility(IBuildFarmJobSource).new(
230- cls.build_farm_job_type, BuildStatus.NEEDSBUILD, None, True,
231- date_created, None, archive)
232- packagebuild = getUtility(IPackageBuildSource).new(
233- build_farm_job, archive, pocket)
234+ cls.build_farm_job_type, BuildStatus.NEEDSBUILD, date_created,
235+ None, archive)
236 spbuild = cls(
237- build_farm_job, packagebuild, distroseries, recipe, requester,
238- archive, pocket, date_created)
239+ build_farm_job, distroseries, recipe, requester, archive, pocket,
240+ date_created)
241 store.add(spbuild)
242 return spbuild
243
244@@ -315,9 +305,8 @@
245 SourcePackageRelease.source_package_recipe_build == self.id)
246 for release in releases:
247 release.source_package_recipe_build = None
248- package_build = self.package_build
249 store.remove(self)
250- package_build.destroySelf()
251+ store.remove(self.build_farm_job)
252
253 @classmethod
254 def getByID(cls, build_id):
255
256=== modified file 'lib/lp/code/model/tests/test_sourcepackagerecipebuild.py'
257--- lib/lp/code/model/tests/test_sourcepackagerecipebuild.py 2013-01-23 10:16:18 +0000
258+++ lib/lp/code/model/tests/test_sourcepackagerecipebuild.py 2013-02-06 10:45:34 +0000
259@@ -22,7 +22,6 @@
260 from lp.buildmaster.enums import BuildStatus
261 from lp.buildmaster.interfaces.buildqueue import IBuildQueue
262 from lp.buildmaster.model.buildfarmjob import BuildFarmJob
263-from lp.buildmaster.model.packagebuild import PackageBuild
264 from lp.code.interfaces.sourcepackagerecipebuild import (
265 ISourcePackageRecipeBuild,
266 ISourcePackageRecipeBuildJob,
267@@ -479,14 +478,9 @@
268 naked_build = removeSecurityProxy(build)
269 # Ensure database ids are set.
270 store.flush()
271- package_build_id = naked_build.package_build_id
272- build_farm_job_id = naked_build.package_build.build_farm_job_id
273+ build_farm_job_id = naked_build.build_farm_job_id
274 build.destroySelf()
275- result = store.find(PackageBuild, PackageBuild.id == package_build_id)
276- self.assertIs(None, result.one())
277- result = store.find(
278- BuildFarmJob, BuildFarmJob.id == build_farm_job_id)
279- self.assertIs(None, result.one())
280+ self.assertIs(None, store.get(BuildFarmJob, build_farm_job_id))
281
282 def test_cancelBuild(self):
283 # ISourcePackageRecipeBuild should make sure to remove jobs and build
284
285=== modified file 'lib/lp/soyuz/model/binarypackagebuild.py'
286--- lib/lp/soyuz/model/binarypackagebuild.py 2013-02-05 06:11:57 +0000
287+++ lib/lp/soyuz/model/binarypackagebuild.py 2013-02-06 10:45:34 +0000
288@@ -43,7 +43,6 @@
289 BuildStatus,
290 )
291 from lp.buildmaster.interfaces.buildfarmjob import IBuildFarmJobSource
292-from lp.buildmaster.interfaces.packagebuild import IPackageBuildSource
293 from lp.buildmaster.model.builder import Builder
294 from lp.buildmaster.model.buildfarmjob import BuildFarmJob
295 from lp.buildmaster.model.buildqueue import BuildQueue
296@@ -104,9 +103,6 @@
297 build_farm_job_type = BuildFarmJobType.PACKAGEBUILD
298 job_type = build_farm_job_type
299
300- package_build_id = Int(name='package_build', allow_none=False)
301- package_build = Reference(package_build_id, 'PackageBuild.id')
302-
303 distro_arch_series_id = Int(name='distro_arch_series', allow_none=False)
304 distro_arch_series = Reference(
305 distro_arch_series_id, 'DistroArchSeries.id')
306@@ -415,11 +411,11 @@
307 assert self.can_be_retried, "Build %s cannot be retried" % self.id
308 self.build_farm_job.status = self._new_status = BuildStatus.NEEDSBUILD
309 self.build_farm_job.date_finished = self._new_date_finished = None
310- self.build_farm_job.date_started = self._new_date_started = None
311+ self._new_date_started = None
312 self.build_farm_job.builder = self._new_builder = None
313- self.build_farm_job.log = self._new_log = None
314- self.package_build.upload_log = self._new_upload_log = None
315- self.package_build.dependencies = self._new_dependencies = None
316+ self._new_log = None
317+ self._new_upload_log = None
318+ self._new_dependencies = None
319 self.queueBuild()
320
321 def rescore(self, score):
322@@ -569,8 +565,7 @@
323 if not self._isDependencySatisfied(token)]
324
325 # Update dependencies line
326- self.package_build.dependencies = self._new_dependencies = (
327- u", ".join(remaining_deps))
328+ self._new_dependencies = u", ".join(remaining_deps)
329
330 def __getitem__(self, name):
331 return self.getBinaryPackageRelease(name)
332@@ -866,17 +861,12 @@
333 archive, pocket, status=BuildStatus.NEEDSBUILD,
334 date_created=None, builder=None):
335 """See `IBinaryPackageBuildSet`."""
336- # Create the BuildFarmJob and PackageBuild to which the new
337- # BinaryPackageBuild will delegate.
338+ # Create the BuildFarmJob for the new BinaryPackageBuild.
339 build_farm_job = getUtility(IBuildFarmJobSource).new(
340- BinaryPackageBuild.build_farm_job_type, status, processor,
341- archive.require_virtualized, date_created, builder, archive)
342- package_build = getUtility(IPackageBuildSource).new(
343- build_farm_job, archive, pocket)
344-
345+ BinaryPackageBuild.build_farm_job_type, status, date_created,
346+ builder, archive)
347 binary_package_build = BinaryPackageBuild(
348 build_farm_job=build_farm_job,
349- package_build=package_build,
350 distro_arch_series=distro_arch_series,
351 source_package_release=source_package_release,
352 _new_archive=archive, _new_pocket=pocket,
353
354=== modified file 'lib/lp/translations/model/translationtemplatesbuild.py'
355--- lib/lp/translations/model/translationtemplatesbuild.py 2013-01-31 10:39:46 +0000
356+++ lib/lp/translations/model/translationtemplatesbuild.py 2013-02-06 10:45:34 +0000
357@@ -138,7 +138,7 @@
358 """See `ITranslationTemplatesBuildSource`."""
359 processor = cls._getBuildArch()
360 build_farm_job = getUtility(IBuildFarmJobSource).new(
361- BuildFarmJobType.TRANSLATIONTEMPLATESBUILD, processor=processor)
362+ BuildFarmJobType.TRANSLATIONTEMPLATESBUILD)
363 build = TranslationTemplatesBuild(build_farm_job, branch, processor)
364 store = cls._getStore()
365 store.add(build)