Merge ~pappacena/launchpad:pkg-upload-log-api into launchpad:master
- Git
- lp:~pappacena/launchpad
- pkg-upload-log-api
- Merge into master
Status: | Rejected |
---|---|
Rejected by: | Thiago F. Pappacena |
Proposed branch: | ~pappacena/launchpad:pkg-upload-log-api |
Merge into: | launchpad:master |
Diff against target: |
800 lines (+330/-40) 12 files modified
lib/lp/_schema_circular_imports.py (+3/-1) lib/lp/security.py (+11/-1) lib/lp/soyuz/browser/configure.zcml (+6/-1) lib/lp/soyuz/browser/queue.py (+5/-10) lib/lp/soyuz/browser/tests/test_queue.py (+32/-4) lib/lp/soyuz/configure.zcml (+8/-1) lib/lp/soyuz/interfaces/queue.py (+67/-10) lib/lp/soyuz/interfaces/webservice.py (+6/-2) lib/lp/soyuz/model/queue.py (+98/-7) lib/lp/soyuz/stories/soyuz/xx-queue-pages.txt (+25/-0) lib/lp/soyuz/templates/distroseries-queue.pt (+10/-0) lib/lp/soyuz/tests/test_packageupload.py (+59/-3) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Colin Watson (community) | Needs Resubmitting | ||
Review via email: mp+377897@code.launchpad.net |
Commit message
API for Package Upload logs
Description of the change
I'm adding some changes here the API part to fetch package upload logs, including pre-fetching related objects.
This branch should only be merged after https:/
Thiago F. Pappacena (pappacena) wrote : | # |
Thiago F. Pappacena (pappacena) wrote : | # |
QueueItemsView is caching some extra attributes that are not beign cached on PackageUploadSet, but there was room for a small improvement anyway.
This should be ok to be reviewed, but should only be merged once the other MP is approved.
Colin Watson (cjwatson) wrote : | # |
It would be best to resubmit this MP specifying "Prerequisite repository: ~pappacena/
Colin Watson (cjwatson) wrote : | # |
Just a few things I noticed while trying to filter mentally for just what's added by this branch. It'll be easier to review this once it's resubmitted with the right prerequisite, so I've probably missed some things.
- 4c4c95c... by Thiago F. Pappacena
-
exporting the package upload reference on pkg upload log api
- 3858ddd... by Thiago F. Pappacena
-
do not snapshot pkg upload logs
- 3aa046c... by Thiago F. Pappacena
-
fixing typo and writing better comment
Thiago F. Pappacena (pappacena) wrote : | # |
I did some of the changes, but I'm going to open a new MP removing from the diff the changes not related to the API code.
Thiago F. Pappacena (pappacena) wrote : | # |
Reopening the MP here: https:/
Unmerged commits
- 3aa046c... by Thiago F. Pappacena
-
fixing typo and writing better comment
- 3858ddd... by Thiago F. Pappacena
-
do not snapshot pkg upload logs
- 4c4c95c... by Thiago F. Pappacena
-
exporting the package upload reference on pkg upload log api
- 88e3582... by Thiago F. Pappacena
-
Merge branch 'archive-
queue-audit- trail' into pkg-upload-log-api - 6aed200... by Thiago F. Pappacena
-
avoiding to pre-fetch same thing twice when building CompletePackage
Upload - 59b6065... by Thiago F. Pappacena
-
code formatting and comment
- 176ce89... by Thiago F. Pappacena
-
formatting import
- f557d63... by Thiago F. Pappacena
-
fix typo and remove duplicated entry in __all__
- b9845f2... by Thiago F. Pappacena
-
import style
- e85aaed... by Thiago F. Pappacena
-
moving log preloading PackageUploadSet, to reuse it both on browser and api
Preview Diff
1 | diff --git a/lib/lp/_schema_circular_imports.py b/lib/lp/_schema_circular_imports.py |
2 | index 20db323..49b37ec 100644 |
3 | --- a/lib/lp/_schema_circular_imports.py |
4 | +++ b/lib/lp/_schema_circular_imports.py |
5 | @@ -206,7 +206,7 @@ from lp.soyuz.interfaces.publishing import ( |
6 | ISourcePackagePublishingHistoryEdit, |
7 | ISourcePackagePublishingHistoryPublic, |
8 | ) |
9 | -from lp.soyuz.interfaces.queue import IPackageUpload |
10 | +from lp.soyuz.interfaces.queue import IPackageUpload, IPackageUploadLog |
11 | from lp.soyuz.interfaces.sourcepackagerelease import ISourcePackageRelease |
12 | from lp.translations.interfaces.hastranslationimports import ( |
13 | IHasTranslationImports, |
14 | @@ -363,6 +363,8 @@ patch_reference_property( |
15 | ISourcePackagePublishingHistory) |
16 | patch_reference_property( |
17 | ISourcePackagePublishingHistory, 'packageupload', IPackageUpload) |
18 | +patch_reference_property( |
19 | + IPackageUploadLog, 'package_upload', IPackageUpload) |
20 | patch_entry_return_type( |
21 | ISourcePackagePublishingHistoryEdit, 'changeOverride', |
22 | ISourcePackagePublishingHistory) |
23 | diff --git a/lib/lp/security.py b/lib/lp/security.py |
24 | index 66c7f73..c2d45ac 100644 |
25 | --- a/lib/lp/security.py |
26 | +++ b/lib/lp/security.py |
27 | @@ -1,4 +1,4 @@ |
28 | -# Copyright 2009-2019 Canonical Ltd. This software is licensed under the |
29 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
30 | # GNU Affero General Public License version 3 (see the file LICENSE). |
31 | |
32 | """Security policies for using content objects.""" |
33 | @@ -246,6 +246,7 @@ from lp.soyuz.interfaces.publishing import ( |
34 | ) |
35 | from lp.soyuz.interfaces.queue import ( |
36 | IPackageUpload, |
37 | + IPackageUploadLog, |
38 | IPackageUploadQueue, |
39 | ) |
40 | from lp.soyuz.interfaces.sourcepackagerelease import ISourcePackageRelease |
41 | @@ -1918,6 +1919,15 @@ class ViewPackageUpload(AuthorizationBase): |
42 | methodcaller("checkUnauthenticated"), self.iter_adapters())) |
43 | |
44 | |
45 | +class ViewPackageUploadLog(DelegatedAuthorization): |
46 | + """Anyone who can view a package upload can view its logs.""" |
47 | + permission = 'launchpad.View' |
48 | + usedfor = IPackageUploadLog |
49 | + |
50 | + def __init__(self, obj): |
51 | + super(ViewPackageUploadLog, self).__init__(obj, obj.package_upload) |
52 | + |
53 | + |
54 | class EditPackageUpload(AdminByAdminsTeam): |
55 | permission = 'launchpad.Edit' |
56 | usedfor = IPackageUpload |
57 | diff --git a/lib/lp/soyuz/browser/configure.zcml b/lib/lp/soyuz/browser/configure.zcml |
58 | index 1ce6b53..4ab2e41 100644 |
59 | --- a/lib/lp/soyuz/browser/configure.zcml |
60 | +++ b/lib/lp/soyuz/browser/configure.zcml |
61 | @@ -1,4 +1,4 @@ |
62 | -<!-- Copyright 2009-2019 Canonical Ltd. This software is licensed under the |
63 | +<!-- Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
64 | GNU Affero General Public License version 3 (see the file LICENSE). |
65 | --> |
66 | |
67 | @@ -661,6 +661,11 @@ |
68 | path_expression="string:+upload/${id}" |
69 | attribute_to_parent="distroseries" |
70 | /> |
71 | + <browser:url |
72 | + for="lp.soyuz.interfaces.queue.IPackageUploadLog" |
73 | + path_expression="string:+log/${id}" |
74 | + attribute_to_parent="package_upload" |
75 | + /> |
76 | <browser:navigation |
77 | module="lp.soyuz.browser.queue" |
78 | classes="PackageUploadNavigation" |
79 | diff --git a/lib/lp/soyuz/browser/queue.py b/lib/lp/soyuz/browser/queue.py |
80 | index a476e1d..02d5f54 100644 |
81 | --- a/lib/lp/soyuz/browser/queue.py |
82 | +++ b/lib/lp/soyuz/browser/queue.py |
83 | @@ -1,4 +1,4 @@ |
84 | -# Copyright 2009-2017 Canonical Ltd. This software is licensed under the |
85 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
86 | # GNU Affero General Public License version 3 (see the file LICENSE). |
87 | |
88 | """Browser views for package queue.""" |
89 | @@ -10,6 +10,7 @@ __all__ = [ |
90 | 'QueueItemsView', |
91 | ] |
92 | |
93 | +from collections import defaultdict |
94 | from operator import attrgetter |
95 | |
96 | from lazr.delegates import delegate_to |
97 | @@ -207,7 +208,7 @@ class QueueItemsView(LaunchpadView): |
98 | jobs = load_related(Job, package_copy_jobs, ['job_id']) |
99 | person_ids.extend(map(attrgetter('requester_id'), jobs)) |
100 | list(getUtility(IPersonSet).getPrecachedPersonsFromIDs( |
101 | - person_ids, need_validity=True, need_icon=True)) |
102 | + person_ids, need_validity=True)) |
103 | |
104 | def decoratedQueueBatch(self): |
105 | """Return the current batch, converted to decorated objects. |
106 | @@ -222,10 +223,8 @@ class QueueItemsView(LaunchpadView): |
107 | return None |
108 | |
109 | upload_ids = [upload.id for upload in uploads] |
110 | - puses = load_referencing( |
111 | - PackageUploadSource, uploads, ['packageuploadID']) |
112 | - pubs = load_referencing( |
113 | - PackageUploadBuild, uploads, ['packageuploadID']) |
114 | + puses = sum([u.sources for u in uploads], []) |
115 | + pubs = sum([u.builds for u in uploads], []) |
116 | |
117 | source_sprs = load_related( |
118 | SourcePackageRelease, puses, ['sourcepackagereleaseID']) |
119 | @@ -491,8 +490,6 @@ class CompletePackageUpload: |
120 | # (i.e. no proxying of __set__). |
121 | pocket = None |
122 | date_created = None |
123 | - sources = None |
124 | - builds = None |
125 | customfiles = None |
126 | contains_source = None |
127 | contains_build = None |
128 | @@ -503,9 +500,7 @@ class CompletePackageUpload: |
129 | self.pocket = packageupload.pocket |
130 | self.date_created = packageupload.date_created |
131 | self.context = packageupload |
132 | - self.sources = list(packageupload.sources) |
133 | self.contains_source = len(self.sources) > 0 |
134 | - self.builds = list(packageupload.builds) |
135 | self.contains_build = len(self.builds) > 0 |
136 | self.customfiles = list(packageupload.customfiles) |
137 | |
138 | diff --git a/lib/lp/soyuz/browser/tests/test_queue.py b/lib/lp/soyuz/browser/tests/test_queue.py |
139 | index c2d15f2..0e08739 100644 |
140 | --- a/lib/lp/soyuz/browser/tests/test_queue.py |
141 | +++ b/lib/lp/soyuz/browser/tests/test_queue.py |
142 | @@ -1,4 +1,4 @@ |
143 | -# Copyright 2010-2018 Canonical Ltd. This software is licensed under the |
144 | +# Copyright 2010-2020 Canonical Ltd. This software is licensed under the |
145 | # GNU Affero General Public License version 3 (see the file LICENSE). |
146 | |
147 | """Unit tests for QueueItemsView.""" |
148 | @@ -33,6 +33,7 @@ from lp.testing import ( |
149 | login_person, |
150 | logout, |
151 | person_logged_in, |
152 | + record_two_runs, |
153 | StormStatementRecorder, |
154 | TestCaseWithFactory, |
155 | ) |
156 | @@ -342,10 +343,10 @@ class TestQueueItemsView(TestCaseWithFactory): |
157 | |
158 | layer = LaunchpadFunctionalLayer |
159 | |
160 | - def makeView(self, distroseries, user): |
161 | + def makeView(self, distroseries, user, form=None): |
162 | """Create a queue view.""" |
163 | return create_initialized_view( |
164 | - distroseries, name='+queue', principal=user) |
165 | + distroseries, name='+queue', principal=user, form=form) |
166 | |
167 | def test_view_renders_source_upload(self): |
168 | login(ADMIN_EMAIL) |
169 | @@ -437,7 +438,34 @@ class TestQueueItemsView(TestCaseWithFactory): |
170 | with StormStatementRecorder() as recorder: |
171 | view = self.makeView(distroseries, queue_admin) |
172 | view() |
173 | - self.assertThat(recorder, HasQueryCount(Equals(56))) |
174 | + self.assertThat(recorder, HasQueryCount(Equals(55))) |
175 | + |
176 | + def test_package_upload_with_logs_query_count(self): |
177 | + login(ADMIN_EMAIL) |
178 | + uploads = [] |
179 | + distroseries = self.factory.makeDistroSeries() |
180 | + |
181 | + for i in range(11): |
182 | + uploads.append(self.factory.makeSourcePackageUpload(distroseries)) |
183 | + queue_admin = self.factory.makeArchiveAdmin(distroseries.main_archive) |
184 | + |
185 | + def reject_some_package(): |
186 | + for upload in uploads: |
187 | + if len(upload.logs) == 0: |
188 | + person = self.factory.makePerson() |
189 | + upload.rejectFromQueue(person) |
190 | + break |
191 | + |
192 | + def run_view(): |
193 | + with person_logged_in(queue_admin): |
194 | + url = ("%s/+queue?queue_state=%s" % ( |
195 | + canonical_url(distroseries), |
196 | + PackageUploadStatus.REJECTED.value)) |
197 | + self.getUserBrowser(url, queue_admin) |
198 | + |
199 | + recorder1, recorder2 = record_two_runs( |
200 | + run_view, reject_some_package, 1, 10) |
201 | + self.assertThat(recorder2, HasQueryCount.byEquality(recorder1)) |
202 | |
203 | |
204 | class TestCompletePackageUpload(TestCaseWithFactory): |
205 | diff --git a/lib/lp/soyuz/configure.zcml b/lib/lp/soyuz/configure.zcml |
206 | index 0e35ccb..9b50239 100644 |
207 | --- a/lib/lp/soyuz/configure.zcml |
208 | +++ b/lib/lp/soyuz/configure.zcml |
209 | @@ -1,4 +1,4 @@ |
210 | -<!-- Copyright 2009-2019 Canonical Ltd. This software is licensed under the |
211 | +<!-- Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
212 | GNU Affero General Public License version 3 (see the file LICENSE). |
213 | --> |
214 | |
215 | @@ -151,6 +151,7 @@ |
216 | displayarchs |
217 | displayversion |
218 | isPPA |
219 | + logs |
220 | package_copy_job |
221 | package_copy_job_id |
222 | package_name |
223 | @@ -183,6 +184,12 @@ |
224 | set_attributes="status distroseries pocket changesfile archive"/> |
225 | </class> |
226 | <class |
227 | + class="lp.soyuz.model.queue.PackageUploadLog"> |
228 | + <require |
229 | + permission="launchpad.View" |
230 | + interface="lp.soyuz.interfaces.queue.IPackageUploadLog"/> |
231 | + </class> |
232 | + <class |
233 | class="lp.soyuz.model.queue.PackageUploadSource"> |
234 | <allow |
235 | interface="lp.soyuz.interfaces.queue.IPackageUploadSource"/> |
236 | diff --git a/lib/lp/soyuz/interfaces/queue.py b/lib/lp/soyuz/interfaces/queue.py |
237 | index c57da73..3a02689 100644 |
238 | --- a/lib/lp/soyuz/interfaces/queue.py |
239 | +++ b/lib/lp/soyuz/interfaces/queue.py |
240 | @@ -1,4 +1,4 @@ |
241 | -# Copyright 2009-2016 Canonical Ltd. This software is licensed under the |
242 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
243 | # GNU Affero General Public License version 3 (see the file LICENSE). |
244 | |
245 | """Queue interfaces.""" |
246 | @@ -9,22 +9,24 @@ __all__ = [ |
247 | 'CustomUploadError', |
248 | 'ICustomUploadHandler', |
249 | 'IHasQueueItems', |
250 | - 'IPackageUploadQueue', |
251 | 'IPackageUpload', |
252 | 'IPackageUploadBuild', |
253 | - 'IPackageUploadSource', |
254 | 'IPackageUploadCustom', |
255 | + 'IPackageUploadLog', |
256 | + 'IPackageUploadQueue', |
257 | 'IPackageUploadSet', |
258 | + 'IPackageUploadSource', |
259 | 'NonBuildableSourceUploadError', |
260 | 'QueueAdminUnauthorizedError', |
261 | 'QueueBuildAcceptError', |
262 | 'QueueInconsistentStateError', |
263 | 'QueueSourceAcceptError', |
264 | - 'QueueStateWriteProtectedError', |
265 | + 'QueueStateWriteProtectedError' |
266 | ] |
267 | |
268 | import httplib |
269 | |
270 | +from lazr.lifecycle.snapshot import doNotSnapshot |
271 | from lazr.restful.declarations import ( |
272 | call_with, |
273 | error_status, |
274 | @@ -36,7 +38,10 @@ from lazr.restful.declarations import ( |
275 | operation_parameters, |
276 | REQUEST_USER, |
277 | ) |
278 | -from lazr.restful.fields import Reference |
279 | +from lazr.restful.fields import ( |
280 | + CollectionField, |
281 | + Reference, |
282 | + ) |
283 | from zope.interface import ( |
284 | Attribute, |
285 | Interface, |
286 | @@ -53,6 +58,7 @@ from zope.schema import ( |
287 | from zope.security.interfaces import Unauthorized |
288 | |
289 | from lp import _ |
290 | +from lp.registry.interfaces.person import IPerson |
291 | from lp.registry.interfaces.pocket import PackagePublishingPocket |
292 | from lp.soyuz.enums import PackageUploadStatus |
293 | from lp.soyuz.interfaces.packagecopyjob import IPackageCopyJob |
294 | @@ -111,6 +117,44 @@ class IPackageUploadQueue(Interface): |
295 | """ |
296 | |
297 | |
298 | +class IPackageUploadLog(Interface): |
299 | + """A log entry recording a change in a package upload's status.""" |
300 | + |
301 | + export_as_webservice_entry(publish_web_link=True, as_of="devel") |
302 | + |
303 | + id = Int(title=_('ID'), required=True, readonly=True) |
304 | + |
305 | + package_upload = exported( |
306 | + Reference( |
307 | + Interface, title=_("The package upload that generated this log"), |
308 | + required=True, readonly=True)) |
309 | + |
310 | + date_created = exported( |
311 | + Datetime( |
312 | + title=_("When this action happened."), required=True, |
313 | + readonly=True)) |
314 | + |
315 | + reviewer = exported( |
316 | + Reference( |
317 | + IPerson, title=_("Who did this action."), |
318 | + required=True, readonly=True)) |
319 | + |
320 | + old_status = exported( |
321 | + Choice( |
322 | + vocabulary=PackageUploadStatus, description=_("Old status."), |
323 | + required=True, readonly=True)) |
324 | + |
325 | + new_status = exported( |
326 | + Choice( |
327 | + vocabulary=PackageUploadStatus, description=_("New status."), |
328 | + required=True, readonly=True)) |
329 | + |
330 | + comment = exported( |
331 | + TextLine( |
332 | + title=_("User's comment about this change."), |
333 | + required=False, readonly=True)) |
334 | + |
335 | + |
336 | class IPackageUpload(Interface): |
337 | """A Queue item for the archive uploader.""" |
338 | |
339 | @@ -181,6 +225,14 @@ class IPackageUpload(Interface): |
340 | sources = Attribute("The queue sources associated with this queue item") |
341 | builds = Attribute("The queue builds associated with the queue item") |
342 | |
343 | + logs = exported( |
344 | + doNotSnapshot( |
345 | + CollectionField( |
346 | + title=_("The package upload logs"), |
347 | + value_type=Reference(schema=IPackageUploadLog), |
348 | + readonly=True)), |
349 | + as_of="devel") |
350 | + |
351 | customfiles = Attribute("Custom upload files associated with this " |
352 | "queue item") |
353 | custom_file_urls = exported( |
354 | @@ -503,9 +555,11 @@ class IPackageUploadBuild(Interface): |
355 | readonly=False, |
356 | ) |
357 | |
358 | - build = Int( |
359 | - title=_("The related build"), required=True, readonly=False, |
360 | - ) |
361 | + build = Int(title=_("The related build"), required=True, readonly=False) |
362 | + |
363 | + buildID = Int( |
364 | + title=_("The Related build ID"), required=True, |
365 | + readonly=True) |
366 | |
367 | def binaries(): |
368 | """Returns the properties of the binaries in this build. |
369 | @@ -548,8 +602,11 @@ class IPackageUploadSource(Interface): |
370 | |
371 | sourcepackagerelease = Int( |
372 | title=_("The related source package release"), required=True, |
373 | - readonly=False, |
374 | - ) |
375 | + readonly=False) |
376 | + |
377 | + sourcepackagereleaseID = Int( |
378 | + title=_("The related source package release ID"), required=True, |
379 | + readonly=True) |
380 | |
381 | def getSourceAncestryForDiffs(): |
382 | """Return a suitable ancestry publication for this context. |
383 | diff --git a/lib/lp/soyuz/interfaces/webservice.py b/lib/lp/soyuz/interfaces/webservice.py |
384 | index 3620e81..90ef0e1 100644 |
385 | --- a/lib/lp/soyuz/interfaces/webservice.py |
386 | +++ b/lib/lp/soyuz/interfaces/webservice.py |
387 | @@ -1,4 +1,4 @@ |
388 | -# Copyright 2010-2019 Canonical Ltd. This software is licensed under the |
389 | +# Copyright 2010-2020 Canonical Ltd. This software is licensed under the |
390 | # GNU Affero General Public License version 3 (see the file LICENSE). |
391 | |
392 | """All the interfaces that are exposed through the webservice. |
393 | @@ -35,6 +35,7 @@ __all__ = [ |
394 | 'ILiveFSBuild', |
395 | 'ILiveFSSet', |
396 | 'IPackageUpload', |
397 | + 'IPackageUploadLog', |
398 | 'IPackageset', |
399 | 'IPackagesetSet', |
400 | 'ISourcePackagePublishingHistory', |
401 | @@ -104,7 +105,10 @@ from lp.soyuz.interfaces.publishing import ( |
402 | IBinaryPackagePublishingHistory, |
403 | ISourcePackagePublishingHistory, |
404 | ) |
405 | -from lp.soyuz.interfaces.queue import IPackageUpload |
406 | +from lp.soyuz.interfaces.queue import ( |
407 | + IPackageUpload, |
408 | + IPackageUploadLog, |
409 | + ) |
410 | |
411 | |
412 | _schema_circular_imports |
413 | diff --git a/lib/lp/soyuz/model/queue.py b/lib/lp/soyuz/model/queue.py |
414 | index 0fd1799..91e92cd 100644 |
415 | --- a/lib/lp/soyuz/model/queue.py |
416 | +++ b/lib/lp/soyuz/model/queue.py |
417 | @@ -1,18 +1,22 @@ |
418 | -# Copyright 2009-2018 Canonical Ltd. This software is licensed under the |
419 | +# Copyright 2009-2020 Canonical Ltd. This software is licensed under the |
420 | # GNU Affero General Public License version 3 (see the file LICENSE). |
421 | |
422 | __metaclass__ = type |
423 | __all__ = [ |
424 | - 'PackageUploadQueue', |
425 | 'PackageUpload', |
426 | 'PackageUploadBuild', |
427 | - 'PackageUploadSource', |
428 | 'PackageUploadCustom', |
429 | + 'PackageUploadLog', |
430 | + 'PackageUploadQueue', |
431 | 'PackageUploadSet', |
432 | + 'PackageUploadSource', |
433 | ] |
434 | |
435 | +from collections import defaultdict |
436 | from itertools import chain |
437 | +from operator import attrgetter |
438 | |
439 | +import pytz |
440 | from sqlobject import ( |
441 | ForeignKey, |
442 | SQLMultipleJoin, |
443 | @@ -29,6 +33,7 @@ from storm.locals import ( |
444 | SQL, |
445 | Unicode, |
446 | ) |
447 | +from storm.properties import DateTime |
448 | from storm.store import ( |
449 | EmptyResultSet, |
450 | Store, |
451 | @@ -39,6 +44,7 @@ from zope.interface import implementer |
452 | from lp.app.errors import NotFoundError |
453 | from lp.archiveuploader.tagfiles import parse_tagfile_content |
454 | from lp.registry.interfaces.gpg import IGPGKeySet |
455 | +from lp.registry.interfaces.person import IPersonSet |
456 | from lp.registry.interfaces.pocket import PackagePublishingPocket |
457 | from lp.registry.model.sourcepackagename import SourcePackageName |
458 | from lp.services.auditor.client import AuditorClient |
459 | @@ -46,10 +52,16 @@ from lp.services.database.bulk import ( |
460 | load_referencing, |
461 | load_related, |
462 | ) |
463 | -from lp.services.database.constants import UTC_NOW |
464 | +from lp.services.database.constants import ( |
465 | + DEFAULT, |
466 | + UTC_NOW, |
467 | + ) |
468 | from lp.services.database.datetimecol import UtcDateTimeCol |
469 | from lp.services.database.decoratedresultset import DecoratedResultSet |
470 | -from lp.services.database.enumcol import EnumCol |
471 | +from lp.services.database.enumcol import ( |
472 | + DBEnum, |
473 | + EnumCol, |
474 | + ) |
475 | from lp.services.database.interfaces import ( |
476 | IMasterStore, |
477 | IStore, |
478 | @@ -58,6 +70,7 @@ from lp.services.database.sqlbase import ( |
479 | SQLBase, |
480 | sqlvalues, |
481 | ) |
482 | +from lp.services.database.stormbase import StormBase |
483 | from lp.services.database.stormexpr import ( |
484 | Array, |
485 | ArrayContains, |
486 | @@ -97,6 +110,7 @@ from lp.soyuz.interfaces.queue import ( |
487 | IPackageUpload, |
488 | IPackageUploadBuild, |
489 | IPackageUploadCustom, |
490 | + IPackageUploadLog, |
491 | IPackageUploadQueue, |
492 | IPackageUploadSet, |
493 | IPackageUploadSource, |
494 | @@ -205,6 +219,23 @@ class PackageUpload(SQLBase): |
495 | self.addSearchableVersions([self.package_copy_job.package_version]) |
496 | |
497 | @cachedproperty |
498 | + def logs(self): |
499 | + logs = Store.of(self).find( |
500 | + PackageUploadLog, |
501 | + PackageUploadLog.package_upload == self) |
502 | + return list(logs.order_by(Desc('date_created'))) |
503 | + |
504 | + def _addLog(self, reviewer, new_status, comment=None): |
505 | + del get_property_cache(self).logs # clean cache |
506 | + return Store.of(self).add(PackageUploadLog( |
507 | + package_upload=self, |
508 | + reviewer=reviewer, |
509 | + old_status=self.status, |
510 | + new_status=new_status, |
511 | + comment=comment |
512 | + )) |
513 | + |
514 | + @cachedproperty |
515 | def sources(self): |
516 | return list(self._sources) |
517 | |
518 | @@ -571,6 +602,10 @@ class PackageUpload(SQLBase): |
519 | |
520 | def acceptFromQueue(self, user=None): |
521 | """See `IPackageUpload`.""" |
522 | + # XXX: Only tests are not passing user here. We should adjust the |
523 | + # tests and always create the log entries after |
524 | + if user is not None: |
525 | + self._addLog(user, PackageUploadStatus.ACCEPTED, None) |
526 | if self.package_copy_job is None: |
527 | self._acceptNonSyncFromQueue() |
528 | else: |
529 | @@ -581,6 +616,7 @@ class PackageUpload(SQLBase): |
530 | |
531 | def rejectFromQueue(self, user, comment=None): |
532 | """See `IPackageUpload`.""" |
533 | + self._addLog(user, PackageUploadStatus.REJECTED, comment) |
534 | self.setRejected() |
535 | if self.package_copy_job is not None: |
536 | # Circular imports :( |
537 | @@ -1153,6 +1189,45 @@ def get_properties_for_binary(bpr): |
538 | } |
539 | |
540 | |
541 | +@implementer(IPackageUploadLog) |
542 | +class PackageUploadLog(StormBase): |
543 | + """Tracking of status changes for a given package upload""" |
544 | + |
545 | + __storm_table__ = "PackageUploadLog" |
546 | + |
547 | + id = Int(primary=True) |
548 | + |
549 | + package_upload_id = Int(name='package_upload') |
550 | + package_upload = Reference(package_upload_id, PackageUpload.id) |
551 | + |
552 | + date_created = DateTime(tzinfo=pytz.UTC, allow_none=False, |
553 | + default=UTC_NOW) |
554 | + |
555 | + reviewer_id = Int(name='reviewer', allow_none=False) |
556 | + reviewer = Reference(reviewer_id, 'Person.id') |
557 | + |
558 | + old_status = DBEnum(enum=PackageUploadStatus, allow_none=False) |
559 | + |
560 | + new_status = DBEnum(enum=PackageUploadStatus, allow_none=False) |
561 | + |
562 | + comment = Unicode(allow_none=True) |
563 | + |
564 | + def __init__(self, package_upload, reviewer, old_status, new_status, |
565 | + comment=None, date_created=DEFAULT): |
566 | + self.package_upload = package_upload |
567 | + self.reviewer = reviewer |
568 | + self.old_status = old_status |
569 | + self.new_status = new_status |
570 | + self.comment = comment |
571 | + self.date_created = date_created |
572 | + |
573 | + def __repr__(self): |
574 | + return ( |
575 | + "<{self.__class__.__name__} ~{self.reviewer.name} " |
576 | + "changed {self.package_upload} to {self.new_status} " |
577 | + "on {self.date_created}>").format(self=self) |
578 | + |
579 | + |
580 | @implementer(IPackageUploadBuild) |
581 | class PackageUploadBuild(SQLBase): |
582 | """A Queue item's related builds.""" |
583 | @@ -1528,8 +1603,10 @@ class PackageUploadSet: |
584 | PackageUploadBuild, rows, ["packageuploadID"]) |
585 | pucs = load_referencing( |
586 | PackageUploadCustom, rows, ["packageuploadID"]) |
587 | + logs = load_referencing( |
588 | + PackageUploadLog, rows, ["package_upload_id"]) |
589 | |
590 | - prefill_packageupload_caches(rows, puses, pubs, pucs) |
591 | + prefill_packageupload_caches(rows, puses, pubs, pucs, logs) |
592 | |
593 | return DecoratedResultSet(query, pre_iter_hook=preload_hook) |
594 | |
595 | @@ -1550,18 +1627,32 @@ class PackageUploadSet: |
596 | PackageUpload.package_copy_job_id.is_in(pcj_ids)) |
597 | |
598 | |
599 | -def prefill_packageupload_caches(uploads, puses, pubs, pucs): |
600 | +def prefill_packageupload_caches(uploads, puses, pubs, pucs, logs): |
601 | # Circular imports. |
602 | from lp.soyuz.model.archive import Archive |
603 | from lp.soyuz.model.binarypackagebuild import BinaryPackageBuild |
604 | from lp.soyuz.model.publishing import SourcePackagePublishingHistory |
605 | from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease |
606 | |
607 | + logs_per_pu = defaultdict(list) |
608 | + reviewer_ids = set() |
609 | + for log in logs: |
610 | + reviewer_ids.add(log.reviewer_id) |
611 | + logs_per_pu[log.package_upload_id].append(log) |
612 | + |
613 | + # Preload reviewers of the logs. |
614 | + # We are not using `need_icon` here because reviewers are persons, |
615 | + # and icons are only available for teams. |
616 | + list(getUtility(IPersonSet).getPrecachedPersonsFromIDs( |
617 | + reviewer_ids, need_validity=True)) |
618 | + |
619 | for pu in uploads: |
620 | cache = get_property_cache(pu) |
621 | cache.sources = [] |
622 | cache.builds = [] |
623 | cache.customfiles = [] |
624 | + cache.logs = sorted( |
625 | + logs_per_pu[pu.id], key=attrgetter("date_created"), reverse=True) |
626 | |
627 | for pus in puses: |
628 | get_property_cache(pus.packageupload).sources.append(pus) |
629 | diff --git a/lib/lp/soyuz/stories/soyuz/xx-queue-pages.txt b/lib/lp/soyuz/stories/soyuz/xx-queue-pages.txt |
630 | index c5f4741..b2bc537 100644 |
631 | --- a/lib/lp/soyuz/stories/soyuz/xx-queue-pages.txt |
632 | +++ b/lib/lp/soyuz/stories/soyuz/xx-queue-pages.txt |
633 | @@ -512,6 +512,7 @@ values: |
634 | >>> for row in filelist: |
635 | ... print(extract_text(row)) |
636 | pmount_1.0-1_all.deb (18 bytes) NEW 0.1-1 restricted admin extra |
637 | + Accepted a moment ago by Sample Person |
638 | |
639 | 'netapplet' has gone straight to the 'done' queue because it's a single |
640 | source upload, and we can see its overridden values there: |
641 | @@ -547,6 +548,15 @@ Rejecting 'alsa-utils' source: |
642 | netapplet...ddtp... - Release 2006-... |
643 | netapplet...dist... - Release 2006-... |
644 | |
645 | + >>> upload_manager_browser.getControl( |
646 | + ... name="queue_state", index=0).displayValue=['Rejected'] |
647 | + >>> upload_manager_browser.getControl("Update").click() |
648 | + >>> logs = find_tags_by_class( |
649 | + ... upload_manager_browser.contents, "log-content") |
650 | + >>> for log in logs: |
651 | + ... print(extract_text(log)) |
652 | + Rejected...a moment ago...by Sample Person...Foo |
653 | + |
654 | One rejection email is generated: |
655 | |
656 | >>> run_package_upload_notification_jobs() |
657 | @@ -600,6 +610,21 @@ button will be disabled. |
658 | >>> upload_manager_browser.getControl(name="Reject").disabled |
659 | True |
660 | |
661 | +Accepting alsa again, and check that the package upload log has more rows |
662 | + |
663 | + >>> upload_manager_browser.getControl(name="QUEUE_ID").value = ['4'] |
664 | + >>> upload_manager_browser.getControl(name="Accept").click() |
665 | + >>> upload_manager_browser.getControl( |
666 | + ... name="queue_state", index=0).displayValue=['Accepted'] |
667 | + >>> upload_manager_browser.getControl("Update").click() |
668 | + >>> pkg_content = first_tag_by_class(upload_manager_browser.contents, |
669 | + ... "queue-4") |
670 | + >>> logs = find_tags_by_class(str(pkg_content), "log-content") |
671 | + >>> for log in logs: |
672 | + ... print(extract_text(log)) |
673 | + Accepted...a moment ago...by Sample Person... |
674 | + Rejected...a moment ago...by Sample Person...Foo |
675 | + |
676 | |
677 | Clean up |
678 | ======== |
679 | diff --git a/lib/lp/soyuz/templates/distroseries-queue.pt b/lib/lp/soyuz/templates/distroseries-queue.pt |
680 | index 59b77d5..39629ce 100644 |
681 | --- a/lib/lp/soyuz/templates/distroseries-queue.pt |
682 | +++ b/lib/lp/soyuz/templates/distroseries-queue.pt |
683 | @@ -149,6 +149,16 @@ |
684 | </tbody> |
685 | <tbody tal:attributes="class string:${filelist_class}"> |
686 | <metal:filelist use-macro="template/macros/package-filelist"/> |
687 | + <tr class="log-content" tal:repeat="log packageupload/logs"> |
688 | + <td colspan="2" style="border: 0"></td> |
689 | + <td colspan="8" style="border: 0"> |
690 | + <span tal:content="log/new_status"></span> |
691 | + <span tal:attributes="title log/date_created/fmt:datetime" |
692 | + tal:content="log/date_created/fmt:displaydate" /> |
693 | + by <span tal:content="structure log/reviewer/fmt:link" /> |
694 | + <p tal:condition="log/comment" tal:content="log/comment" /> |
695 | + </td> |
696 | + </tr> |
697 | </tbody> |
698 | </tal:block> |
699 | </tal:batch> |
700 | diff --git a/lib/lp/soyuz/tests/test_packageupload.py b/lib/lp/soyuz/tests/test_packageupload.py |
701 | index cb4d038..1e7e655 100644 |
702 | --- a/lib/lp/soyuz/tests/test_packageupload.py |
703 | +++ b/lib/lp/soyuz/tests/test_packageupload.py |
704 | @@ -16,7 +16,10 @@ from lazr.restfulclient.errors import ( |
705 | BadRequest, |
706 | Unauthorized, |
707 | ) |
708 | -from testtools.matchers import Equals |
709 | +from testtools.matchers import ( |
710 | + Equals, |
711 | + MatchesStructure, |
712 | + ) |
713 | import transaction |
714 | from zope.component import ( |
715 | getUtility, |
716 | @@ -91,6 +94,27 @@ class PackageUploadTestCase(TestCaseWithFactory): |
717 | if os.path.exists(config.personalpackagearchive.root): |
718 | shutil.rmtree(config.personalpackagearchive.root) |
719 | |
720 | + def test_add_log_entry(self): |
721 | + upload = self.factory.makePackageUpload( |
722 | + status=PackageUploadStatus.UNAPPROVED) |
723 | + upload = removeSecurityProxy(upload) |
724 | + self.assertEqual(0, len(upload.logs)) |
725 | + |
726 | + person = self.factory.makePerson(name='lpusername') |
727 | + |
728 | + upload._addLog(person, PackageUploadStatus.REJECTED, 'just because') |
729 | + |
730 | + log = upload.logs[0] |
731 | + self.assertThat(log, MatchesStructure.byEquality( |
732 | + reviewer=person, old_status=PackageUploadStatus.UNAPPROVED, |
733 | + new_status=PackageUploadStatus.REJECTED, comment='just because')) |
734 | + |
735 | + expected_repr = ( |
736 | + "<PackageUploadLog ~lpusername " |
737 | + "changed {self.package_upload} to Rejected " |
738 | + "on {self.date_created}>").format(self=log) |
739 | + self.assertEqual(str(expected_repr), repr(log)) |
740 | + |
741 | def test_realiseUpload_for_overridden_component_archive(self): |
742 | # If the component of an upload is overridden to 'Partner' for |
743 | # example, then the new publishing record should be for the |
744 | @@ -358,8 +382,14 @@ class PackageUploadTestCase(TestCaseWithFactory): |
745 | |
746 | # Accepting one of them works. (Since it's a single source upload, |
747 | # it goes straight to DONE.) |
748 | - upload_one.acceptFromQueue() |
749 | + person = self.factory.makePerson() |
750 | + upload_one.acceptFromQueue(person) |
751 | self.assertEqual("DONE", upload_one.status.name) |
752 | + |
753 | + log = upload_one.logs[0] |
754 | + self.assertThat(log, MatchesStructure.byEquality( |
755 | + reviewer=person, old_status=PackageUploadStatus.UNAPPROVED, |
756 | + new_status=PackageUploadStatus.ACCEPTED, comment=None)) |
757 | transaction.commit() |
758 | |
759 | # Trying to accept the second fails. |
760 | @@ -368,9 +398,15 @@ class PackageUploadTestCase(TestCaseWithFactory): |
761 | self.assertEqual("UNAPPROVED", upload_two.status.name) |
762 | |
763 | # Rejecting the second upload works. |
764 | - upload_two.rejectFromQueue(self.factory.makePerson()) |
765 | + upload_two.rejectFromQueue(person, 'Because yes') |
766 | self.assertEqual("REJECTED", upload_two.status.name) |
767 | |
768 | + self.assertEqual(1, len(upload_two.logs)) |
769 | + log = upload_two.logs[0] |
770 | + self.assertThat(log, MatchesStructure.byEquality( |
771 | + reviewer=person, old_status=PackageUploadStatus.UNAPPROVED, |
772 | + new_status=PackageUploadStatus.REJECTED, comment='Because yes')) |
773 | + |
774 | def test_rejectFromQueue_source_sends_email(self): |
775 | # Rejecting a source package sends an email to the uploader. |
776 | self.test_publisher.prepareBreezyAutotest() |
777 | @@ -1447,3 +1483,23 @@ class TestPackageUploadWebservice(TestCaseWithFactory): |
778 | person, component=self.universe), |
779 | 5) |
780 | self.assertThat(recorder2, HasQueryCount.byEquality(recorder1)) |
781 | + |
782 | + def test_api_package_upload_log(self): |
783 | + # API clients can see upload logs of a source uploads. |
784 | + admin = self.makeQueueAdmin([self.universe]) |
785 | + upload, ws_upload = self.makeSourcePackageUpload( |
786 | + admin, sourcepackagename="hello", component=self.universe) |
787 | + with person_logged_in(admin): |
788 | + upload.rejectFromQueue(admin, 'not a good change') |
789 | + upload.acceptFromQueue(admin) |
790 | + |
791 | + logs = removeSecurityProxy(upload).logs |
792 | + ws_logs = ws_upload.logs |
793 | + self.assertEqual(len(ws_logs), len(logs)) |
794 | + for log, ws_log in zip(logs, ws_logs): |
795 | + self.assertEqual(log.comment, ws_log.comment) |
796 | + self.assertEqual(log.date_created, ws_log.date_created) |
797 | + self.assertEqual(log.new_status.title, ws_log.new_status) |
798 | + self.assertEqual(log.old_status.title, ws_log.old_status) |
799 | + self.assertEqual(log.reviewer.name, ws_log.reviewer.name) |
800 | + self.assertEqual(ws_log.package_upload.id, ws_upload.id) |
This is still a work in progress, since I want to check if it would be possible to refactor some of the pre-fetching to happen only on PackageUploadSet, instead of beign kind of duplicated in the browser. queue.QueueItem sView.