Merge lp:~cjwatson/launchpad/archive-index-by-hash into lp:launchpad
- archive-index-by-hash
- Merge into devel
Proposed by
Colin Watson
Status: | Merged | ||||
---|---|---|---|---|---|
Merged at revision: | 17975 | ||||
Proposed branch: | lp:~cjwatson/launchpad/archive-index-by-hash | ||||
Merge into: | lp:launchpad | ||||
Prerequisite: | lp:~cjwatson/launchpad/ds-publish-by-hash | ||||
Diff against target: |
1502 lines (+1079/-53) 10 files modified
lib/lp/archivepublisher/model/ftparchive.py (+6/-2) lib/lp/archivepublisher/publishing.py (+282/-19) lib/lp/archivepublisher/tests/test_publisher.py (+599/-1) lib/lp/registry/model/distribution.py (+14/-2) lib/lp/services/helpers.py (+31/-12) lib/lp/services/librarian/interfaces/__init__.py (+1/-1) lib/lp/services/librarian/model.py (+4/-2) lib/lp/soyuz/interfaces/archivefile.py (+25/-1) lib/lp/soyuz/model/archivefile.py (+63/-11) lib/lp/soyuz/tests/test_archivefile.py (+54/-2) |
||||
To merge this branch: | bzr merge lp:~cjwatson/launchpad/archive-index-by-hash | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
William Grant (community) | code | Approve | |
Review via email: mp+289379@code.launchpad.net |
Commit message
Add files indexed by Release to the librarian and to ArchiveFile. Publish them in by-hash directories, keeping old versions for a day.
Description of the change
Add files indexed by Release to the librarian and to ArchiveFile. Publish them in by-hash directories, keeping old versions for a day.
DistroSeries.
To post a comment you must log in.
Revision history for this message
William Grant (wgrant) : | # |
review:
Needs Fixing
(code)
Revision history for this message
Colin Watson (cjwatson) : | # |
Revision history for this message
William Grant (wgrant) : | # |
Revision history for this message
William Grant (wgrant) : | # |
review:
Needs Fixing
(code)
Revision history for this message
William Grant (wgrant) : | # |
Revision history for this message
Colin Watson (cjwatson) wrote : | # |
Revision history for this message
William Grant (wgrant) : | # |
review:
Approve
(code)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'lib/lp/archivepublisher/model/ftparchive.py' | |||
2 | --- lib/lp/archivepublisher/model/ftparchive.py 2016-02-09 15:51:19 +0000 | |||
3 | +++ lib/lp/archivepublisher/model/ftparchive.py 2016-04-02 00:45:52 +0000 | |||
4 | @@ -54,10 +54,14 @@ | |||
5 | 54 | """Ensure that the path exists and is an empty directory.""" | 54 | """Ensure that the path exists and is an empty directory.""" |
6 | 55 | if os.path.isdir(path): | 55 | if os.path.isdir(path): |
7 | 56 | for name in os.listdir(path): | 56 | for name in os.listdir(path): |
8 | 57 | if name == "by-hash": | ||
9 | 58 | # Ignore existing by-hash directories; they will be cleaned | ||
10 | 59 | # up to match the rest of the directory tree later. | ||
11 | 60 | continue | ||
12 | 57 | child_path = os.path.join(path, name) | 61 | child_path = os.path.join(path, name) |
13 | 58 | # Directories containing index files should never have | 62 | # Directories containing index files should never have |
16 | 59 | # subdirectories. Guard against expensive mistakes by not | 63 | # subdirectories other than by-hash. Guard against expensive |
17 | 60 | # recursing here. | 64 | # mistakes by not recursing here. |
18 | 61 | os.unlink(child_path) | 65 | os.unlink(child_path) |
19 | 62 | else: | 66 | else: |
20 | 63 | os.makedirs(path, 0o755) | 67 | os.makedirs(path, 0o755) |
21 | 64 | 68 | ||
22 | === modified file 'lib/lp/archivepublisher/publishing.py' | |||
23 | --- lib/lp/archivepublisher/publishing.py 2016-03-30 09:17:31 +0000 | |||
24 | +++ lib/lp/archivepublisher/publishing.py 2016-04-02 00:45:52 +0000 | |||
25 | @@ -12,7 +12,11 @@ | |||
26 | 12 | __metaclass__ = type | 12 | __metaclass__ = type |
27 | 13 | 13 | ||
28 | 14 | import bz2 | 14 | import bz2 |
30 | 15 | from datetime import datetime | 15 | from collections import defaultdict |
31 | 16 | from datetime import ( | ||
32 | 17 | datetime, | ||
33 | 18 | timedelta, | ||
34 | 19 | ) | ||
35 | 16 | import errno | 20 | import errno |
36 | 17 | import gzip | 21 | import gzip |
37 | 18 | import hashlib | 22 | import hashlib |
38 | @@ -31,6 +35,11 @@ | |||
39 | 31 | ) | 35 | ) |
40 | 32 | from storm.expr import Desc | 36 | from storm.expr import Desc |
41 | 33 | from zope.component import getUtility | 37 | from zope.component import getUtility |
42 | 38 | from zope.interface import ( | ||
43 | 39 | Attribute, | ||
44 | 40 | implementer, | ||
45 | 41 | Interface, | ||
46 | 42 | ) | ||
47 | 34 | 43 | ||
48 | 35 | from lp.app.interfaces.launchpad import ILaunchpadCelebrities | 44 | from lp.app.interfaces.launchpad import ILaunchpadCelebrities |
49 | 36 | from lp.archivepublisher import HARDCODED_COMPONENT_ORDER | 45 | from lp.archivepublisher import HARDCODED_COMPONENT_ORDER |
50 | @@ -64,8 +73,12 @@ | |||
51 | 64 | from lp.services.database.constants import UTC_NOW | 73 | from lp.services.database.constants import UTC_NOW |
52 | 65 | from lp.services.database.interfaces import IStore | 74 | from lp.services.database.interfaces import IStore |
53 | 66 | from lp.services.features import getFeatureFlag | 75 | from lp.services.features import getFeatureFlag |
54 | 76 | from lp.services.helpers import filenameToContentType | ||
55 | 67 | from lp.services.librarian.client import LibrarianClient | 77 | from lp.services.librarian.client import LibrarianClient |
57 | 68 | from lp.services.osutils import open_for_writing | 78 | from lp.services.osutils import ( |
58 | 79 | ensure_directory_exists, | ||
59 | 80 | open_for_writing, | ||
60 | 81 | ) | ||
61 | 69 | from lp.services.utils import file_exists | 82 | from lp.services.utils import file_exists |
62 | 70 | from lp.soyuz.enums import ( | 83 | from lp.soyuz.enums import ( |
63 | 71 | ArchivePurpose, | 84 | ArchivePurpose, |
64 | @@ -74,6 +87,7 @@ | |||
65 | 74 | PackagePublishingStatus, | 87 | PackagePublishingStatus, |
66 | 75 | ) | 88 | ) |
67 | 76 | from lp.soyuz.interfaces.archive import NoSuchPPA | 89 | from lp.soyuz.interfaces.archive import NoSuchPPA |
68 | 90 | from lp.soyuz.interfaces.archivefile import IArchiveFileSet | ||
69 | 77 | from lp.soyuz.interfaces.publishing import ( | 91 | from lp.soyuz.interfaces.publishing import ( |
70 | 78 | active_publishing_status, | 92 | active_publishing_status, |
71 | 79 | IPublishingSet, | 93 | IPublishingSet, |
72 | @@ -96,6 +110,10 @@ | |||
73 | 96 | } | 110 | } |
74 | 97 | 111 | ||
75 | 98 | 112 | ||
76 | 113 | # Number of days before unreferenced files are removed from by-hash. | ||
77 | 114 | BY_HASH_STAY_OF_EXECUTION = 1 | ||
78 | 115 | |||
79 | 116 | |||
80 | 99 | def reorder_components(components): | 117 | def reorder_components(components): |
81 | 100 | """Return a list of the components provided. | 118 | """Return a list of the components provided. |
82 | 101 | 119 | ||
83 | @@ -232,6 +250,152 @@ | |||
84 | 232 | return max(len(str(item['size'])) for item in self[key]) | 250 | return max(len(str(item['size'])) for item in self[key]) |
85 | 233 | 251 | ||
86 | 234 | 252 | ||
87 | 253 | class IArchiveHash(Interface): | ||
88 | 254 | """Represents a hash algorithm used for index files.""" | ||
89 | 255 | |||
90 | 256 | hash_factory = Attribute("A hashlib class suitable for this algorithm.") | ||
91 | 257 | deb822_name = Attribute( | ||
92 | 258 | "Algorithm name expected by debian.deb822.Release.") | ||
93 | 259 | apt_name = Attribute( | ||
94 | 260 | "Algorithm name used by apt in Release files and by-hash " | ||
95 | 261 | "subdirectories.") | ||
96 | 262 | lfc_name = Attribute( | ||
97 | 263 | "LibraryFileContent attribute name corresponding to this algorithm.") | ||
98 | 264 | |||
99 | 265 | |||
100 | 266 | @implementer(IArchiveHash) | ||
101 | 267 | class MD5ArchiveHash: | ||
102 | 268 | hash_factory = hashlib.md5 | ||
103 | 269 | deb822_name = "md5sum" | ||
104 | 270 | apt_name = "MD5Sum" | ||
105 | 271 | lfc_name = "md5" | ||
106 | 272 | |||
107 | 273 | |||
108 | 274 | @implementer(IArchiveHash) | ||
109 | 275 | class SHA1ArchiveHash: | ||
110 | 276 | hash_factory = hashlib.sha1 | ||
111 | 277 | deb822_name = "sha1" | ||
112 | 278 | apt_name = "SHA1" | ||
113 | 279 | lfc_name = "sha1" | ||
114 | 280 | |||
115 | 281 | |||
116 | 282 | @implementer(IArchiveHash) | ||
117 | 283 | class SHA256ArchiveHash: | ||
118 | 284 | hash_factory = hashlib.sha256 | ||
119 | 285 | deb822_name = "sha256" | ||
120 | 286 | apt_name = "SHA256" | ||
121 | 287 | lfc_name = "sha256" | ||
122 | 288 | |||
123 | 289 | |||
124 | 290 | archive_hashes = [ | ||
125 | 291 | MD5ArchiveHash(), | ||
126 | 292 | SHA1ArchiveHash(), | ||
127 | 293 | SHA256ArchiveHash(), | ||
128 | 294 | ] | ||
129 | 295 | |||
130 | 296 | |||
131 | 297 | class ByHash: | ||
132 | 298 | """Represents a single by-hash directory tree.""" | ||
133 | 299 | |||
134 | 300 | def __init__(self, root, key, log): | ||
135 | 301 | self.root = root | ||
136 | 302 | self.path = os.path.join(root, key, "by-hash") | ||
137 | 303 | self.log = log | ||
138 | 304 | self.known_digests = defaultdict(lambda: defaultdict(set)) | ||
139 | 305 | |||
140 | 306 | def add(self, name, lfa, copy_from_path=None): | ||
141 | 307 | """Ensure that by-hash entries for a single file exist. | ||
142 | 308 | |||
143 | 309 | :param name: The name of the file under this directory tree. | ||
144 | 310 | :param lfa: The `ILibraryFileAlias` to add. | ||
145 | 311 | :param copy_from_path: If not None, copy file content from here | ||
146 | 312 | rather than fetching it from the librarian. This can be used | ||
147 | 313 | for newly-added files to avoid needing to commit the transaction | ||
148 | 314 | before calling this method. | ||
149 | 315 | """ | ||
150 | 316 | for archive_hash in archive_hashes: | ||
151 | 317 | digest = getattr(lfa.content, archive_hash.lfc_name) | ||
152 | 318 | digest_path = os.path.join( | ||
153 | 319 | self.path, archive_hash.apt_name, digest) | ||
154 | 320 | self.known_digests[archive_hash.apt_name][digest].add(name) | ||
155 | 321 | if not os.path.exists(digest_path): | ||
156 | 322 | self.log.debug( | ||
157 | 323 | "by-hash: Creating %s for %s" % (digest_path, name)) | ||
158 | 324 | ensure_directory_exists(os.path.dirname(digest_path)) | ||
159 | 325 | if copy_from_path is not None: | ||
160 | 326 | os.link( | ||
161 | 327 | os.path.join(self.root, copy_from_path), digest_path) | ||
162 | 328 | else: | ||
163 | 329 | with open(digest_path, "wb") as outfile: | ||
164 | 330 | lfa.open() | ||
165 | 331 | try: | ||
166 | 332 | shutil.copyfileobj(lfa, outfile, 4 * 1024 * 1024) | ||
167 | 333 | finally: | ||
168 | 334 | lfa.close() | ||
169 | 335 | |||
170 | 336 | def known(self, name, hashname, digest): | ||
171 | 337 | """Do we know about a file with this name and digest?""" | ||
172 | 338 | names = self.known_digests[hashname].get(digest) | ||
173 | 339 | return names is not None and name in names | ||
174 | 340 | |||
175 | 341 | def prune(self): | ||
176 | 342 | """Remove all by-hash entries that we have not been told to add. | ||
177 | 343 | |||
178 | 344 | This also removes the by-hash directory itself if no entries remain. | ||
179 | 345 | """ | ||
180 | 346 | prune_directory = True | ||
181 | 347 | for archive_hash in archive_hashes: | ||
182 | 348 | hash_path = os.path.join(self.path, archive_hash.apt_name) | ||
183 | 349 | if os.path.exists(hash_path): | ||
184 | 350 | prune_hash_directory = True | ||
185 | 351 | for digest in list(os.listdir(hash_path)): | ||
186 | 352 | if digest not in self.known_digests[archive_hash.apt_name]: | ||
187 | 353 | digest_path = os.path.join(hash_path, digest) | ||
188 | 354 | self.log.debug( | ||
189 | 355 | "by-hash: Deleting unreferenced %s" % digest_path) | ||
190 | 356 | os.unlink(digest_path) | ||
191 | 357 | else: | ||
192 | 358 | prune_hash_directory = False | ||
193 | 359 | if prune_hash_directory: | ||
194 | 360 | os.rmdir(hash_path) | ||
195 | 361 | else: | ||
196 | 362 | prune_directory = False | ||
197 | 363 | if prune_directory and os.path.exists(self.path): | ||
198 | 364 | os.rmdir(self.path) | ||
199 | 365 | |||
200 | 366 | |||
201 | 367 | class ByHashes: | ||
202 | 368 | """Represents all by-hash directory trees in an archive.""" | ||
203 | 369 | |||
204 | 370 | def __init__(self, root, log): | ||
205 | 371 | self.root = root | ||
206 | 372 | self.log = log | ||
207 | 373 | self.children = {} | ||
208 | 374 | |||
209 | 375 | def registerChild(self, dirpath): | ||
210 | 376 | """Register a single by-hash directory. | ||
211 | 377 | |||
212 | 378 | Only directories that have been registered here will be pruned by | ||
213 | 379 | the `prune` method. | ||
214 | 380 | """ | ||
215 | 381 | if dirpath not in self.children: | ||
216 | 382 | self.children[dirpath] = ByHash(self.root, dirpath, self.log) | ||
217 | 383 | return self.children[dirpath] | ||
218 | 384 | |||
219 | 385 | def add(self, path, lfa, copy_from_path=None): | ||
220 | 386 | dirpath, name = os.path.split(path) | ||
221 | 387 | self.registerChild(dirpath).add( | ||
222 | 388 | name, lfa, copy_from_path=copy_from_path) | ||
223 | 389 | |||
224 | 390 | def known(self, path, hashname, digest): | ||
225 | 391 | dirpath, name = os.path.split(path) | ||
226 | 392 | return self.registerChild(dirpath).known(name, hashname, digest) | ||
227 | 393 | |||
228 | 394 | def prune(self): | ||
229 | 395 | for child in self.children.values(): | ||
230 | 396 | child.prune() | ||
231 | 397 | |||
232 | 398 | |||
233 | 235 | class Publisher(object): | 399 | class Publisher(object): |
234 | 236 | """Publisher is the class used to provide the facility to publish | 400 | """Publisher is the class used to provide the facility to publish |
235 | 237 | files in the pool of a Distribution. The publisher objects will be | 401 | files in the pool of a Distribution. The publisher objects will be |
236 | @@ -567,10 +731,20 @@ | |||
237 | 567 | Otherwise we include only pockets flagged as true in dirty_pockets. | 731 | Otherwise we include only pockets flagged as true in dirty_pockets. |
238 | 568 | """ | 732 | """ |
239 | 569 | self.log.debug("* Step D: Generating Release files.") | 733 | self.log.debug("* Step D: Generating Release files.") |
240 | 734 | |||
241 | 735 | archive_file_suites = set() | ||
242 | 736 | for container in getUtility(IArchiveFileSet).getContainersToReap( | ||
243 | 737 | self.archive, container_prefix=u"release:"): | ||
244 | 738 | distroseries, pocket = self.distro.getDistroSeriesAndPocket( | ||
245 | 739 | container[len(u"release:"):]) | ||
246 | 740 | archive_file_suites.add((distroseries, pocket)) | ||
247 | 741 | self.release_files_needed.update(archive_file_suites) | ||
248 | 742 | |||
249 | 570 | for distroseries in self.distro: | 743 | for distroseries in self.distro: |
250 | 571 | for pocket in self.archive.getPockets(): | 744 | for pocket in self.archive.getPockets(): |
251 | 572 | if not is_careful: | 745 | if not is_careful: |
253 | 573 | if not self.isDirty(distroseries, pocket): | 746 | if (not self.isDirty(distroseries, pocket) and |
254 | 747 | (distroseries, pocket) not in archive_file_suites): | ||
255 | 574 | self.log.debug("Skipping release files for %s/%s" % | 748 | self.log.debug("Skipping release files for %s/%s" % |
256 | 575 | (distroseries.name, pocket.name)) | 749 | (distroseries.name, pocket.name)) |
257 | 576 | continue | 750 | continue |
258 | @@ -811,6 +985,95 @@ | |||
259 | 811 | return self.distro.displayname | 985 | return self.distro.displayname |
260 | 812 | return "LP-PPA-%s" % get_ppa_reference(self.archive) | 986 | return "LP-PPA-%s" % get_ppa_reference(self.archive) |
261 | 813 | 987 | ||
262 | 988 | def _updateByHash(self, suite, release_data): | ||
263 | 989 | """Update by-hash files for a suite. | ||
264 | 990 | |||
265 | 991 | This takes Release file data which references a set of on-disk | ||
266 | 992 | files, injects any newly-modified files from that set into the | ||
267 | 993 | librarian and the ArchiveFile table, and updates the on-disk by-hash | ||
268 | 994 | directories to be in sync with ArchiveFile. Any on-disk by-hash | ||
269 | 995 | entries that ceased to be current sufficiently long ago are removed. | ||
270 | 996 | """ | ||
271 | 997 | archive_file_set = getUtility(IArchiveFileSet) | ||
272 | 998 | by_hashes = ByHashes(self._config.archiveroot, self.log) | ||
273 | 999 | suite_dir = os.path.relpath( | ||
274 | 1000 | os.path.join(self._config.distsroot, suite), | ||
275 | 1001 | self._config.archiveroot) | ||
276 | 1002 | container = "release:%s" % suite | ||
277 | 1003 | |||
278 | 1004 | # Gather information on entries in the current Release file, and | ||
279 | 1005 | # make sure nothing there is condemned. | ||
280 | 1006 | current_files = {} | ||
281 | 1007 | current_sha256_checksums = set() | ||
282 | 1008 | for current_entry in release_data["SHA256"]: | ||
283 | 1009 | path = os.path.join(suite_dir, current_entry["name"]) | ||
284 | 1010 | current_files[path] = ( | ||
285 | 1011 | current_entry["size"], current_entry["sha256"]) | ||
286 | 1012 | current_sha256_checksums.add(current_entry["sha256"]) | ||
287 | 1013 | for container, path, sha256 in archive_file_set.unscheduleDeletion( | ||
288 | 1014 | self.archive, container=container, | ||
289 | 1015 | sha256_checksums=current_sha256_checksums): | ||
290 | 1016 | self.log.debug( | ||
291 | 1017 | "by-hash: Unscheduled %s for %s in %s for deletion" % ( | ||
292 | 1018 | sha256, path, container)) | ||
293 | 1019 | |||
294 | 1020 | # Remove any condemned files from the database whose stay of | ||
295 | 1021 | # execution has elapsed. We ensure that we know about all the | ||
296 | 1022 | # relevant by-hash directory trees before doing any removals so that | ||
297 | 1023 | # we can prune them properly later. | ||
298 | 1024 | for db_file in archive_file_set.getByArchive( | ||
299 | 1025 | self.archive, container=container): | ||
300 | 1026 | by_hashes.registerChild(os.path.dirname(db_file.path)) | ||
301 | 1027 | for container, path, sha256 in archive_file_set.reap( | ||
302 | 1028 | self.archive, container=container): | ||
303 | 1029 | self.log.debug( | ||
304 | 1030 | "by-hash: Deleted %s for %s in %s" % (sha256, path, container)) | ||
305 | 1031 | |||
306 | 1032 | # Ensure that all files recorded in the database are in by-hash. | ||
307 | 1033 | db_files = archive_file_set.getByArchive( | ||
308 | 1034 | self.archive, container=container, eager_load=True) | ||
309 | 1035 | for db_file in db_files: | ||
310 | 1036 | by_hashes.add(db_file.path, db_file.library_file) | ||
311 | 1037 | |||
312 | 1038 | # Condemn any database records that do not correspond to current | ||
313 | 1039 | # index files. | ||
314 | 1040 | condemned_files = set() | ||
315 | 1041 | for db_file in db_files: | ||
316 | 1042 | if db_file.scheduled_deletion_date is None: | ||
317 | 1043 | path = db_file.path | ||
318 | 1044 | if path in current_files: | ||
319 | 1045 | current_sha256 = current_files[path][1] | ||
320 | 1046 | else: | ||
321 | 1047 | current_sha256 = None | ||
322 | 1048 | if db_file.library_file.content.sha256 != current_sha256: | ||
323 | 1049 | condemned_files.add(db_file) | ||
324 | 1050 | if condemned_files: | ||
325 | 1051 | for container, path, sha256 in archive_file_set.scheduleDeletion( | ||
326 | 1052 | condemned_files, | ||
327 | 1053 | timedelta(days=BY_HASH_STAY_OF_EXECUTION)): | ||
328 | 1054 | self.log.debug( | ||
329 | 1055 | "by-hash: Scheduled %s for %s in %s for deletion" % ( | ||
330 | 1056 | sha256, path, container)) | ||
331 | 1057 | |||
332 | 1058 | # Ensure that all the current index files are in by-hash and have | ||
333 | 1059 | # corresponding database entries. | ||
334 | 1060 | # XXX cjwatson 2016-03-15: This should possibly use bulk creation, | ||
335 | 1061 | # although we can only avoid about a third of the queries since the | ||
336 | 1062 | # librarian client has no bulk upload methods. | ||
337 | 1063 | for path, (size, sha256) in current_files.items(): | ||
338 | 1064 | full_path = os.path.join(self._config.archiveroot, path) | ||
339 | 1065 | if (os.path.exists(full_path) and | ||
340 | 1066 | not by_hashes.known(path, "SHA256", sha256)): | ||
341 | 1067 | with open(full_path, "rb") as fileobj: | ||
342 | 1068 | db_file = archive_file_set.newFromFile( | ||
343 | 1069 | self.archive, container, path, fileobj, | ||
344 | 1070 | size, filenameToContentType(path)) | ||
345 | 1071 | by_hashes.add(path, db_file.library_file, copy_from_path=path) | ||
346 | 1072 | |||
347 | 1073 | # Finally, remove any files from disk that aren't recorded in the | ||
348 | 1074 | # database and aren't active. | ||
349 | 1075 | by_hashes.prune() | ||
350 | 1076 | |||
351 | 814 | def _writeReleaseFile(self, suite, release_data): | 1077 | def _writeReleaseFile(self, suite, release_data): |
352 | 815 | """Write a Release file to the archive. | 1078 | """Write a Release file to the archive. |
353 | 816 | 1079 | ||
354 | @@ -919,9 +1182,14 @@ | |||
355 | 919 | hashes = self._readIndexFileHashes(suite, filename) | 1182 | hashes = self._readIndexFileHashes(suite, filename) |
356 | 920 | if hashes is None: | 1183 | if hashes is None: |
357 | 921 | continue | 1184 | continue |
361 | 922 | release_file.setdefault("MD5Sum", []).append(hashes["md5sum"]) | 1185 | for archive_hash in archive_hashes: |
362 | 923 | release_file.setdefault("SHA1", []).append(hashes["sha1"]) | 1186 | release_file.setdefault(archive_hash.apt_name, []).append( |
363 | 924 | release_file.setdefault("SHA256", []).append(hashes["sha256"]) | 1187 | hashes[archive_hash.deb822_name]) |
364 | 1188 | |||
365 | 1189 | if distroseries.publish_by_hash: | ||
366 | 1190 | self._updateByHash(suite, release_file) | ||
367 | 1191 | if distroseries.advertise_by_hash: | ||
368 | 1192 | release_file["Acquire-By-Hash"] = "yes" | ||
369 | 925 | 1193 | ||
370 | 926 | self._writeReleaseFile(suite, release_file) | 1194 | self._writeReleaseFile(suite, release_file) |
371 | 927 | core_files.add("Release") | 1195 | core_files.add("Release") |
372 | @@ -1041,16 +1309,14 @@ | |||
373 | 1041 | # Schedule this for inclusion in the Release file. | 1309 | # Schedule this for inclusion in the Release file. |
374 | 1042 | all_series_files.add(os.path.join(component, "i18n", "Index")) | 1310 | all_series_files.add(os.path.join(component, "i18n", "Index")) |
375 | 1043 | 1311 | ||
378 | 1044 | def _readIndexFileHashes(self, distroseries_name, file_name, | 1312 | def _readIndexFileHashes(self, suite, file_name, subpath=None): |
377 | 1045 | subpath=None): | ||
379 | 1046 | """Read an index file and return its hashes. | 1313 | """Read an index file and return its hashes. |
380 | 1047 | 1314 | ||
382 | 1048 | :param distroseries_name: Distro series name | 1315 | :param suite: Suite name. |
383 | 1049 | :param file_name: Filename relative to the parent container directory. | 1316 | :param file_name: Filename relative to the parent container directory. |
388 | 1050 | :param subpath: Optional subpath within the distroseries root. | 1317 | :param subpath: Optional subpath within the suite root. Generated |
389 | 1051 | Generated indexes will not include this path. If omitted, | 1318 | indexes will not include this path. If omitted, filenames are |
390 | 1052 | filenames are assumed to be relative to the distroseries | 1319 | assumed to be relative to the suite root. |
387 | 1053 | root. | ||
391 | 1054 | :return: A dictionary mapping hash field names to dictionaries of | 1320 | :return: A dictionary mapping hash field names to dictionaries of |
392 | 1055 | their components as defined by debian.deb822.Release (e.g. | 1321 | their components as defined by debian.deb822.Release (e.g. |
393 | 1056 | {"md5sum": {"md5sum": ..., "size": ..., "name": ...}}), or None | 1322 | {"md5sum": {"md5sum": ..., "size": ..., "name": ...}}), or None |
394 | @@ -1058,8 +1324,7 @@ | |||
395 | 1058 | """ | 1324 | """ |
396 | 1059 | open_func = open | 1325 | open_func = open |
397 | 1060 | full_name = os.path.join( | 1326 | full_name = os.path.join( |
400 | 1061 | self._config.distsroot, distroseries_name, subpath or '.', | 1327 | self._config.distsroot, suite, subpath or '.', file_name) |
399 | 1062 | file_name) | ||
401 | 1063 | if not os.path.exists(full_name): | 1328 | if not os.path.exists(full_name): |
402 | 1064 | if os.path.exists(full_name + '.gz'): | 1329 | if os.path.exists(full_name + '.gz'): |
403 | 1065 | open_func = gzip.open | 1330 | open_func = gzip.open |
404 | @@ -1075,10 +1340,8 @@ | |||
405 | 1075 | return None | 1340 | return None |
406 | 1076 | 1341 | ||
407 | 1077 | hashes = { | 1342 | hashes = { |
412 | 1078 | "md5sum": hashlib.md5(), | 1343 | archive_hash.deb822_name: archive_hash.hash_factory() |
413 | 1079 | "sha1": hashlib.sha1(), | 1344 | for archive_hash in archive_hashes} |
410 | 1080 | "sha256": hashlib.sha256(), | ||
411 | 1081 | } | ||
414 | 1082 | size = 0 | 1345 | size = 0 |
415 | 1083 | with open_func(full_name) as in_file: | 1346 | with open_func(full_name) as in_file: |
416 | 1084 | for chunk in iter(lambda: in_file.read(256 * 1024), ""): | 1347 | for chunk in iter(lambda: in_file.read(256 * 1024), ""): |
417 | 1085 | 1348 | ||
418 | === modified file 'lib/lp/archivepublisher/tests/test_publisher.py' | |||
419 | --- lib/lp/archivepublisher/tests/test_publisher.py 2016-03-30 09:17:31 +0000 | |||
420 | +++ lib/lp/archivepublisher/tests/test_publisher.py 2016-04-02 00:45:52 +0000 | |||
421 | @@ -7,9 +7,14 @@ | |||
422 | 7 | 7 | ||
423 | 8 | import bz2 | 8 | import bz2 |
424 | 9 | import crypt | 9 | import crypt |
425 | 10 | from datetime import ( | ||
426 | 11 | datetime, | ||
427 | 12 | timedelta, | ||
428 | 13 | ) | ||
429 | 10 | from functools import partial | 14 | from functools import partial |
430 | 11 | import gzip | 15 | import gzip |
431 | 12 | import hashlib | 16 | import hashlib |
432 | 17 | from operator import attrgetter | ||
433 | 13 | import os | 18 | import os |
434 | 14 | import shutil | 19 | import shutil |
435 | 15 | import stat | 20 | import stat |
436 | @@ -22,9 +27,20 @@ | |||
437 | 22 | import lzma | 27 | import lzma |
438 | 23 | except ImportError: | 28 | except ImportError: |
439 | 24 | from backports import lzma | 29 | from backports import lzma |
440 | 30 | import pytz | ||
441 | 25 | from testtools.matchers import ( | 31 | from testtools.matchers import ( |
442 | 26 | ContainsAll, | 32 | ContainsAll, |
443 | 33 | DirContains, | ||
444 | 34 | Equals, | ||
445 | 35 | FileContains, | ||
446 | 36 | Is, | ||
447 | 27 | LessThan, | 37 | LessThan, |
448 | 38 | Matcher, | ||
449 | 39 | MatchesListwise, | ||
450 | 40 | MatchesSetwise, | ||
451 | 41 | MatchesStructure, | ||
452 | 42 | Not, | ||
453 | 43 | PathExists, | ||
454 | 28 | ) | 44 | ) |
455 | 29 | import transaction | 45 | import transaction |
456 | 30 | from zope.component import getUtility | 46 | from zope.component import getUtility |
457 | @@ -36,6 +52,8 @@ | |||
458 | 36 | IArchiveSigningKey, | 52 | IArchiveSigningKey, |
459 | 37 | ) | 53 | ) |
460 | 38 | from lp.archivepublisher.publishing import ( | 54 | from lp.archivepublisher.publishing import ( |
461 | 55 | ByHash, | ||
462 | 56 | ByHashes, | ||
463 | 39 | getPublisher, | 57 | getPublisher, |
464 | 40 | I18nIndex, | 58 | I18nIndex, |
465 | 41 | Publisher, | 59 | Publisher, |
466 | @@ -51,6 +69,7 @@ | |||
467 | 51 | from lp.registry.interfaces.series import SeriesStatus | 69 | from lp.registry.interfaces.series import SeriesStatus |
468 | 52 | from lp.services.config import config | 70 | from lp.services.config import config |
469 | 53 | from lp.services.database.constants import UTC_NOW | 71 | from lp.services.database.constants import UTC_NOW |
470 | 72 | from lp.services.database.sqlbase import flush_database_caches | ||
471 | 54 | from lp.services.features import getFeatureFlag | 73 | from lp.services.features import getFeatureFlag |
472 | 55 | from lp.services.features.testing import FeatureFixture | 74 | from lp.services.features.testing import FeatureFixture |
473 | 56 | from lp.services.gpg.interfaces import IGPGHandler | 75 | from lp.services.gpg.interfaces import IGPGHandler |
474 | @@ -69,12 +88,16 @@ | |||
475 | 69 | PackageUploadStatus, | 88 | PackageUploadStatus, |
476 | 70 | ) | 89 | ) |
477 | 71 | from lp.soyuz.interfaces.archive import IArchiveSet | 90 | from lp.soyuz.interfaces.archive import IArchiveSet |
478 | 91 | from lp.soyuz.interfaces.archivefile import IArchiveFileSet | ||
479 | 72 | from lp.soyuz.tests.test_publishing import TestNativePublishingBase | 92 | from lp.soyuz.tests.test_publishing import TestNativePublishingBase |
480 | 73 | from lp.testing import TestCaseWithFactory | 93 | from lp.testing import TestCaseWithFactory |
481 | 74 | from lp.testing.fakemethod import FakeMethod | 94 | from lp.testing.fakemethod import FakeMethod |
482 | 75 | from lp.testing.gpgkeys import gpgkeysdir | 95 | from lp.testing.gpgkeys import gpgkeysdir |
483 | 76 | from lp.testing.keyserver import KeyServerTac | 96 | from lp.testing.keyserver import KeyServerTac |
485 | 77 | from lp.testing.layers import ZopelessDatabaseLayer | 97 | from lp.testing.layers import ( |
486 | 98 | LaunchpadZopelessLayer, | ||
487 | 99 | ZopelessDatabaseLayer, | ||
488 | 100 | ) | ||
489 | 78 | 101 | ||
490 | 79 | 102 | ||
491 | 80 | RELEASE = PackagePublishingPocket.RELEASE | 103 | RELEASE = PackagePublishingPocket.RELEASE |
492 | @@ -424,6 +447,226 @@ | |||
493 | 424 | 'i386', publications[0].distroarchseries.architecturetag) | 447 | 'i386', publications[0].distroarchseries.architecturetag) |
494 | 425 | 448 | ||
495 | 426 | 449 | ||
496 | 450 | class ByHashHasContents(Matcher): | ||
497 | 451 | """Matches if a by-hash directory has exactly the specified contents.""" | ||
498 | 452 | |||
499 | 453 | def __init__(self, contents): | ||
500 | 454 | self.contents = contents | ||
501 | 455 | |||
502 | 456 | def match(self, by_hash_path): | ||
503 | 457 | mismatch = DirContains(["MD5Sum", "SHA1", "SHA256"]).match( | ||
504 | 458 | by_hash_path) | ||
505 | 459 | if mismatch is not None: | ||
506 | 460 | return mismatch | ||
507 | 461 | for hashname, hashattr in ( | ||
508 | 462 | ("MD5Sum", "md5"), ("SHA1", "sha1"), ("SHA256", "sha256")): | ||
509 | 463 | digests = { | ||
510 | 464 | getattr(hashlib, hashattr)(content).hexdigest(): content | ||
511 | 465 | for content in self.contents} | ||
512 | 466 | path = os.path.join(by_hash_path, hashname) | ||
513 | 467 | mismatch = DirContains(digests.keys()).match(path) | ||
514 | 468 | if mismatch is not None: | ||
515 | 469 | return mismatch | ||
516 | 470 | for digest, content in digests.items(): | ||
517 | 471 | mismatch = FileContains(content).match( | ||
518 | 472 | os.path.join(path, digest)) | ||
519 | 473 | if mismatch is not None: | ||
520 | 474 | return mismatch | ||
521 | 475 | |||
522 | 476 | |||
523 | 477 | class ByHashesHaveContents(Matcher): | ||
524 | 478 | """Matches if only these by-hash directories exist with proper contents.""" | ||
525 | 479 | |||
526 | 480 | def __init__(self, path_contents): | ||
527 | 481 | self.path_contents = path_contents | ||
528 | 482 | |||
529 | 483 | def match(self, root): | ||
530 | 484 | children = set() | ||
531 | 485 | for dirpath, dirnames, _ in os.walk(root): | ||
532 | 486 | if "by-hash" in dirnames: | ||
533 | 487 | children.add(os.path.relpath(dirpath, root)) | ||
534 | 488 | mismatch = MatchesSetwise( | ||
535 | 489 | *(Equals(path) for path in self.path_contents)).match(children) | ||
536 | 490 | if mismatch is not None: | ||
537 | 491 | return mismatch | ||
538 | 492 | for path, contents in self.path_contents.items(): | ||
539 | 493 | by_hash_path = os.path.join(root, path, "by-hash") | ||
540 | 494 | mismatch = ByHashHasContents(contents).match(by_hash_path) | ||
541 | 495 | if mismatch is not None: | ||
542 | 496 | return mismatch | ||
543 | 497 | |||
544 | 498 | |||
545 | 499 | class TestByHash(TestCaseWithFactory): | ||
546 | 500 | """Unit tests for details of handling a single by-hash directory tree.""" | ||
547 | 501 | |||
548 | 502 | layer = LaunchpadZopelessLayer | ||
549 | 503 | |||
550 | 504 | def test_add(self): | ||
551 | 505 | root = self.makeTemporaryDirectory() | ||
552 | 506 | contents = ["abc\n", "def\n"] | ||
553 | 507 | lfas = [ | ||
554 | 508 | self.factory.makeLibraryFileAlias(content=content) | ||
555 | 509 | for content in contents] | ||
556 | 510 | transaction.commit() | ||
557 | 511 | by_hash = ByHash(root, "dists/foo/main/source", DevNullLogger()) | ||
558 | 512 | for lfa in lfas: | ||
559 | 513 | by_hash.add("Sources", lfa) | ||
560 | 514 | by_hash_path = os.path.join(root, "dists/foo/main/source/by-hash") | ||
561 | 515 | self.assertThat(by_hash_path, ByHashHasContents(contents)) | ||
562 | 516 | |||
563 | 517 | def test_add_copy_from_path(self): | ||
564 | 518 | root = self.makeTemporaryDirectory() | ||
565 | 519 | content = "abc\n" | ||
566 | 520 | sources_path = "dists/foo/main/source/Sources" | ||
567 | 521 | with open_for_writing( | ||
568 | 522 | os.path.join(root, sources_path), "w") as sources: | ||
569 | 523 | sources.write(content) | ||
570 | 524 | lfa = self.factory.makeLibraryFileAlias(content=content, db_only=True) | ||
571 | 525 | by_hash = ByHash(root, "dists/foo/main/source", DevNullLogger()) | ||
572 | 526 | by_hash.add("Sources", lfa, copy_from_path=sources_path) | ||
573 | 527 | by_hash_path = os.path.join(root, "dists/foo/main/source/by-hash") | ||
574 | 528 | self.assertThat(by_hash_path, ByHashHasContents([content])) | ||
575 | 529 | |||
576 | 530 | def test_add_existing(self): | ||
577 | 531 | root = self.makeTemporaryDirectory() | ||
578 | 532 | content = "abc\n" | ||
579 | 533 | lfa = self.factory.makeLibraryFileAlias(content=content) | ||
580 | 534 | by_hash_path = os.path.join(root, "dists/foo/main/source/by-hash") | ||
581 | 535 | for hashname, hashattr in ( | ||
582 | 536 | ("MD5Sum", "md5"), ("SHA1", "sha1"), ("SHA256", "sha256")): | ||
583 | 537 | digest = getattr(hashlib, hashattr)(content).hexdigest() | ||
584 | 538 | with open_for_writing( | ||
585 | 539 | os.path.join(by_hash_path, hashname, digest), "w") as f: | ||
586 | 540 | f.write(content) | ||
587 | 541 | by_hash = ByHash(root, "dists/foo/main/source", DevNullLogger()) | ||
588 | 542 | self.assertThat(by_hash_path, ByHashHasContents([content])) | ||
589 | 543 | by_hash.add("Sources", lfa) | ||
590 | 544 | self.assertThat(by_hash_path, ByHashHasContents([content])) | ||
591 | 545 | |||
592 | 546 | def test_known(self): | ||
593 | 547 | root = self.makeTemporaryDirectory() | ||
594 | 548 | content = "abc\n" | ||
595 | 549 | with open_for_writing(os.path.join(root, "abc"), "w") as f: | ||
596 | 550 | f.write(content) | ||
597 | 551 | lfa = self.factory.makeLibraryFileAlias(content=content, db_only=True) | ||
598 | 552 | by_hash = ByHash(root, "", DevNullLogger()) | ||
599 | 553 | md5 = hashlib.md5(content).hexdigest() | ||
600 | 554 | sha1 = hashlib.sha1(content).hexdigest() | ||
601 | 555 | sha256 = hashlib.sha256(content).hexdigest() | ||
602 | 556 | self.assertFalse(by_hash.known("abc", "MD5Sum", md5)) | ||
603 | 557 | self.assertFalse(by_hash.known("abc", "SHA1", sha1)) | ||
604 | 558 | self.assertFalse(by_hash.known("abc", "SHA256", sha256)) | ||
605 | 559 | by_hash.add("abc", lfa, copy_from_path="abc") | ||
606 | 560 | self.assertTrue(by_hash.known("abc", "MD5Sum", md5)) | ||
607 | 561 | self.assertTrue(by_hash.known("abc", "SHA1", sha1)) | ||
608 | 562 | self.assertTrue(by_hash.known("abc", "SHA256", sha256)) | ||
609 | 563 | self.assertFalse(by_hash.known("def", "SHA256", sha256)) | ||
610 | 564 | by_hash.add("def", lfa, copy_from_path="abc") | ||
611 | 565 | self.assertTrue(by_hash.known("def", "SHA256", sha256)) | ||
612 | 566 | |||
613 | 567 | def test_prune(self): | ||
614 | 568 | root = self.makeTemporaryDirectory() | ||
615 | 569 | content = "abc\n" | ||
616 | 570 | sources_path = "dists/foo/main/source/Sources" | ||
617 | 571 | with open_for_writing(os.path.join(root, sources_path), "w") as f: | ||
618 | 572 | f.write(content) | ||
619 | 573 | lfa = self.factory.makeLibraryFileAlias(content=content, db_only=True) | ||
620 | 574 | by_hash = ByHash(root, "dists/foo/main/source", DevNullLogger()) | ||
621 | 575 | by_hash.add("Sources", lfa, copy_from_path=sources_path) | ||
622 | 576 | by_hash_path = os.path.join(root, "dists/foo/main/source/by-hash") | ||
623 | 577 | with open_for_writing(os.path.join(by_hash_path, "MD5Sum/0"), "w"): | ||
624 | 578 | pass | ||
625 | 579 | self.assertThat(by_hash_path, Not(ByHashHasContents([content]))) | ||
626 | 580 | by_hash.prune() | ||
627 | 581 | self.assertThat(by_hash_path, ByHashHasContents([content])) | ||
628 | 582 | |||
629 | 583 | def test_prune_empty(self): | ||
630 | 584 | root = self.makeTemporaryDirectory() | ||
631 | 585 | by_hash = ByHash(root, "dists/foo/main/source", DevNullLogger()) | ||
632 | 586 | by_hash_path = os.path.join(root, "dists/foo/main/source/by-hash") | ||
633 | 587 | with open_for_writing(os.path.join(by_hash_path, "MD5Sum/0"), "w"): | ||
634 | 588 | pass | ||
635 | 589 | self.assertThat(by_hash_path, PathExists()) | ||
636 | 590 | by_hash.prune() | ||
637 | 591 | self.assertThat(by_hash_path, Not(PathExists())) | ||
638 | 592 | |||
639 | 593 | |||
640 | 594 | class TestByHashes(TestCaseWithFactory): | ||
641 | 595 | """Unit tests for details of handling a set of by-hash directory trees.""" | ||
642 | 596 | |||
643 | 597 | layer = LaunchpadZopelessLayer | ||
644 | 598 | |||
645 | 599 | def test_add(self): | ||
646 | 600 | root = self.makeTemporaryDirectory() | ||
647 | 601 | self.assertThat(root, ByHashesHaveContents({})) | ||
648 | 602 | path_contents = { | ||
649 | 603 | "dists/foo/main/source": {"Sources": "abc\n"}, | ||
650 | 604 | "dists/foo/main/binary-amd64": { | ||
651 | 605 | "Packages.gz": "def\n", "Packages.xz": "ghi\n"}, | ||
652 | 606 | } | ||
653 | 607 | by_hashes = ByHashes(root, DevNullLogger()) | ||
654 | 608 | for dirpath, contents in path_contents.items(): | ||
655 | 609 | for name, content in contents.items(): | ||
656 | 610 | path = os.path.join(dirpath, name) | ||
657 | 611 | with open_for_writing(os.path.join(root, path), "w") as f: | ||
658 | 612 | f.write(content) | ||
659 | 613 | lfa = self.factory.makeLibraryFileAlias( | ||
660 | 614 | content=content, db_only=True) | ||
661 | 615 | by_hashes.add(path, lfa, copy_from_path=path) | ||
662 | 616 | self.assertThat(root, ByHashesHaveContents({ | ||
663 | 617 | path: contents.values() | ||
664 | 618 | for path, contents in path_contents.items()})) | ||
665 | 619 | |||
666 | 620 | def test_known(self): | ||
667 | 621 | root = self.makeTemporaryDirectory() | ||
668 | 622 | content = "abc\n" | ||
669 | 623 | sources_path = "dists/foo/main/source/Sources" | ||
670 | 624 | with open_for_writing(os.path.join(root, sources_path), "w") as f: | ||
671 | 625 | f.write(content) | ||
672 | 626 | lfa = self.factory.makeLibraryFileAlias(content=content, db_only=True) | ||
673 | 627 | by_hashes = ByHashes(root, DevNullLogger()) | ||
674 | 628 | md5 = hashlib.md5(content).hexdigest() | ||
675 | 629 | sha1 = hashlib.sha1(content).hexdigest() | ||
676 | 630 | sha256 = hashlib.sha256(content).hexdigest() | ||
677 | 631 | self.assertFalse(by_hashes.known(sources_path, "MD5Sum", md5)) | ||
678 | 632 | self.assertFalse(by_hashes.known(sources_path, "SHA1", sha1)) | ||
679 | 633 | self.assertFalse(by_hashes.known(sources_path, "SHA256", sha256)) | ||
680 | 634 | by_hashes.add(sources_path, lfa, copy_from_path=sources_path) | ||
681 | 635 | self.assertTrue(by_hashes.known(sources_path, "MD5Sum", md5)) | ||
682 | 636 | self.assertTrue(by_hashes.known(sources_path, "SHA1", sha1)) | ||
683 | 637 | self.assertTrue(by_hashes.known(sources_path, "SHA256", sha256)) | ||
684 | 638 | |||
685 | 639 | def test_prune(self): | ||
686 | 640 | root = self.makeTemporaryDirectory() | ||
687 | 641 | path_contents = { | ||
688 | 642 | "dists/foo/main/source": {"Sources": "abc\n"}, | ||
689 | 643 | "dists/foo/main/binary-amd64": { | ||
690 | 644 | "Packages.gz": "def\n", "Packages.xz": "ghi\n"}, | ||
691 | 645 | } | ||
692 | 646 | by_hashes = ByHashes(root, DevNullLogger()) | ||
693 | 647 | for dirpath, contents in path_contents.items(): | ||
694 | 648 | for name, content in contents.items(): | ||
695 | 649 | path = os.path.join(dirpath, name) | ||
696 | 650 | with open_for_writing(os.path.join(root, path), "w") as f: | ||
697 | 651 | f.write(content) | ||
698 | 652 | lfa = self.factory.makeLibraryFileAlias( | ||
699 | 653 | content=content, db_only=True) | ||
700 | 654 | by_hashes.add(path, lfa, copy_from_path=path) | ||
701 | 655 | strays = [ | ||
702 | 656 | "dists/foo/main/source/by-hash/MD5Sum/0", | ||
703 | 657 | "dists/foo/main/binary-amd64/by-hash/MD5Sum/0", | ||
704 | 658 | ] | ||
705 | 659 | for stray in strays: | ||
706 | 660 | with open_for_writing(os.path.join(root, stray), "w"): | ||
707 | 661 | pass | ||
708 | 662 | matcher = ByHashesHaveContents({ | ||
709 | 663 | path: contents.values() | ||
710 | 664 | for path, contents in path_contents.items()}) | ||
711 | 665 | self.assertThat(root, Not(matcher)) | ||
712 | 666 | by_hashes.prune() | ||
713 | 667 | self.assertThat(root, matcher) | ||
714 | 668 | |||
715 | 669 | |||
716 | 427 | class TestPublisher(TestPublisherBase): | 670 | class TestPublisher(TestPublisherBase): |
717 | 428 | """Testing `Publisher` behaviour.""" | 671 | """Testing `Publisher` behaviour.""" |
718 | 429 | 672 | ||
719 | @@ -1018,6 +1261,22 @@ | |||
720 | 1018 | self.assertEqual( | 1261 | self.assertEqual( |
721 | 1019 | 1 + old_num_pending_archives, new_num_pending_archives) | 1262 | 1 + old_num_pending_archives, new_num_pending_archives) |
722 | 1020 | 1263 | ||
723 | 1264 | def testPendingArchiveWithReapableFiles(self): | ||
724 | 1265 | # getPendingPublicationPPAs returns archives that have reapable | ||
725 | 1266 | # ArchiveFiles. | ||
726 | 1267 | ubuntu = getUtility(IDistributionSet)['ubuntu'] | ||
727 | 1268 | archive = self.factory.makeArchive() | ||
728 | 1269 | self.assertNotIn(archive, ubuntu.getPendingPublicationPPAs()) | ||
729 | 1270 | archive_file = self.factory.makeArchiveFile(archive=archive) | ||
730 | 1271 | self.assertNotIn(archive, ubuntu.getPendingPublicationPPAs()) | ||
731 | 1272 | now = datetime.now(pytz.UTC) | ||
732 | 1273 | removeSecurityProxy(archive_file).scheduled_deletion_date = ( | ||
733 | 1274 | now + timedelta(hours=12)) | ||
734 | 1275 | self.assertNotIn(archive, ubuntu.getPendingPublicationPPAs()) | ||
735 | 1276 | removeSecurityProxy(archive_file).scheduled_deletion_date = ( | ||
736 | 1277 | now - timedelta(hours=12)) | ||
737 | 1278 | self.assertIn(archive, ubuntu.getPendingPublicationPPAs()) | ||
738 | 1279 | |||
739 | 1021 | def _checkCompressedFiles(self, archive_publisher, base_file_path, | 1280 | def _checkCompressedFiles(self, archive_publisher, base_file_path, |
740 | 1022 | suffixes): | 1281 | suffixes): |
741 | 1023 | """Assert that the various compressed versions of a file are equal. | 1282 | """Assert that the various compressed versions of a file are equal. |
742 | @@ -1930,6 +2189,345 @@ | |||
743 | 1930 | 'Release') | 2189 | 'Release') |
744 | 1931 | self.assertTrue(file_exists(source_release)) | 2190 | self.assertTrue(file_exists(source_release)) |
745 | 1932 | 2191 | ||
746 | 2192 | def testUpdateByHashDisabled(self): | ||
747 | 2193 | # The publisher does not create by-hash directories if it is | ||
748 | 2194 | # disabled in the series configuration. | ||
749 | 2195 | self.assertFalse(self.breezy_autotest.publish_by_hash) | ||
750 | 2196 | self.assertFalse(self.breezy_autotest.advertise_by_hash) | ||
751 | 2197 | publisher = Publisher( | ||
752 | 2198 | self.logger, self.config, self.disk_pool, | ||
753 | 2199 | self.ubuntutest.main_archive) | ||
754 | 2200 | |||
755 | 2201 | self.getPubSource(filecontent='Source: foo\n') | ||
756 | 2202 | |||
757 | 2203 | publisher.A_publish(False) | ||
758 | 2204 | publisher.C_doFTPArchive(False) | ||
759 | 2205 | publisher.D_writeReleaseFiles(False) | ||
760 | 2206 | |||
761 | 2207 | suite_path = partial( | ||
762 | 2208 | os.path.join, self.config.distsroot, 'breezy-autotest') | ||
763 | 2209 | self.assertThat( | ||
764 | 2210 | suite_path('main', 'source', 'by-hash'), Not(PathExists())) | ||
765 | 2211 | release = self.parseRelease(suite_path('Release')) | ||
766 | 2212 | self.assertNotIn('Acquire-By-Hash', release) | ||
767 | 2213 | |||
768 | 2214 | def testUpdateByHashUnadvertised(self): | ||
769 | 2215 | # If the series configuration sets publish_by_hash but not | ||
770 | 2216 | # advertise_by_hash, then by-hash directories are created but not | ||
771 | 2217 | # advertised in Release. This is useful for testing. | ||
772 | 2218 | self.breezy_autotest.publish_by_hash = True | ||
773 | 2219 | self.assertFalse(self.breezy_autotest.advertise_by_hash) | ||
774 | 2220 | publisher = Publisher( | ||
775 | 2221 | self.logger, self.config, self.disk_pool, | ||
776 | 2222 | self.ubuntutest.main_archive) | ||
777 | 2223 | |||
778 | 2224 | self.getPubSource(filecontent='Source: foo\n') | ||
779 | 2225 | |||
780 | 2226 | publisher.A_publish(False) | ||
781 | 2227 | publisher.C_doFTPArchive(False) | ||
782 | 2228 | publisher.D_writeReleaseFiles(False) | ||
783 | 2229 | |||
784 | 2230 | suite_path = partial( | ||
785 | 2231 | os.path.join, self.config.distsroot, 'breezy-autotest') | ||
786 | 2232 | self.assertThat(suite_path('main', 'source', 'by-hash'), PathExists()) | ||
787 | 2233 | release = self.parseRelease(suite_path('Release')) | ||
788 | 2234 | self.assertNotIn('Acquire-By-Hash', release) | ||
789 | 2235 | |||
790 | 2236 | def testUpdateByHashInitial(self): | ||
791 | 2237 | # An initial publisher run populates by-hash directories and leaves | ||
792 | 2238 | # no archive files scheduled for deletion. | ||
793 | 2239 | self.breezy_autotest.publish_by_hash = True | ||
794 | 2240 | self.breezy_autotest.advertise_by_hash = True | ||
795 | 2241 | publisher = Publisher( | ||
796 | 2242 | self.logger, self.config, self.disk_pool, | ||
797 | 2243 | self.ubuntutest.main_archive) | ||
798 | 2244 | |||
799 | 2245 | self.getPubSource(filecontent='Source: foo\n') | ||
800 | 2246 | |||
801 | 2247 | publisher.A_publish(False) | ||
802 | 2248 | publisher.C_doFTPArchive(False) | ||
803 | 2249 | publisher.D_writeReleaseFiles(False) | ||
804 | 2250 | flush_database_caches() | ||
805 | 2251 | |||
806 | 2252 | suite_path = partial( | ||
807 | 2253 | os.path.join, self.config.distsroot, 'breezy-autotest') | ||
808 | 2254 | contents = set() | ||
809 | 2255 | for name in ('Release', 'Sources.gz', 'Sources.bz2'): | ||
810 | 2256 | with open(suite_path('main', 'source', name), 'rb') as f: | ||
811 | 2257 | contents.add(f.read()) | ||
812 | 2258 | |||
813 | 2259 | self.assertThat( | ||
814 | 2260 | suite_path('main', 'source', 'by-hash'), | ||
815 | 2261 | ByHashHasContents(contents)) | ||
816 | 2262 | |||
817 | 2263 | archive_files = getUtility(IArchiveFileSet).getByArchive( | ||
818 | 2264 | self.ubuntutest.main_archive) | ||
819 | 2265 | self.assertNotEqual([], archive_files) | ||
820 | 2266 | self.assertEqual([], [ | ||
821 | 2267 | archive_file for archive_file in archive_files | ||
822 | 2268 | if archive_file.scheduled_deletion_date is not None]) | ||
823 | 2269 | |||
824 | 2270 | def testUpdateByHashSubsequent(self): | ||
825 | 2271 | # A subsequent publisher run updates by-hash directories where | ||
826 | 2272 | # necessary, and marks inactive index files for later deletion. | ||
827 | 2273 | self.breezy_autotest.publish_by_hash = True | ||
828 | 2274 | self.breezy_autotest.advertise_by_hash = True | ||
829 | 2275 | publisher = Publisher( | ||
830 | 2276 | self.logger, self.config, self.disk_pool, | ||
831 | 2277 | self.ubuntutest.main_archive) | ||
832 | 2278 | |||
833 | 2279 | self.getPubSource(filecontent='Source: foo\n') | ||
834 | 2280 | |||
835 | 2281 | publisher.A_publish(False) | ||
836 | 2282 | publisher.C_doFTPArchive(False) | ||
837 | 2283 | publisher.D_writeReleaseFiles(False) | ||
838 | 2284 | |||
839 | 2285 | suite_path = partial( | ||
840 | 2286 | os.path.join, self.config.distsroot, 'breezy-autotest') | ||
841 | 2287 | main_contents = set() | ||
842 | 2288 | universe_contents = set() | ||
843 | 2289 | for name in ('Release', 'Sources.gz', 'Sources.bz2'): | ||
844 | 2290 | with open(suite_path('main', 'source', name), 'rb') as f: | ||
845 | 2291 | main_contents.add(f.read()) | ||
846 | 2292 | with open(suite_path('universe', 'source', name), 'rb') as f: | ||
847 | 2293 | universe_contents.add(f.read()) | ||
848 | 2294 | |||
849 | 2295 | self.getPubSource(sourcename='baz', filecontent='Source: baz\n') | ||
850 | 2296 | |||
851 | 2297 | publisher.A_publish(False) | ||
852 | 2298 | publisher.C_doFTPArchive(False) | ||
853 | 2299 | publisher.D_writeReleaseFiles(False) | ||
854 | 2300 | flush_database_caches() | ||
855 | 2301 | |||
856 | 2302 | for name in ('Release', 'Sources.gz', 'Sources.bz2'): | ||
857 | 2303 | with open(suite_path('main', 'source', name), 'rb') as f: | ||
858 | 2304 | main_contents.add(f.read()) | ||
859 | 2305 | |||
860 | 2306 | self.assertThat( | ||
861 | 2307 | suite_path('main', 'source', 'by-hash'), | ||
862 | 2308 | ByHashHasContents(main_contents)) | ||
863 | 2309 | self.assertThat( | ||
864 | 2310 | suite_path('universe', 'source', 'by-hash'), | ||
865 | 2311 | ByHashHasContents(universe_contents)) | ||
866 | 2312 | |||
867 | 2313 | archive_files = getUtility(IArchiveFileSet).getByArchive( | ||
868 | 2314 | self.ubuntutest.main_archive) | ||
869 | 2315 | self.assertContentEqual( | ||
870 | 2316 | ['dists/breezy-autotest/main/source/Sources.bz2', | ||
871 | 2317 | 'dists/breezy-autotest/main/source/Sources.gz'], | ||
872 | 2318 | [archive_file.path for archive_file in archive_files | ||
873 | 2319 | if archive_file.scheduled_deletion_date is not None]) | ||
874 | 2320 | |||
875 | 2321 | def testUpdateByHashIdenticalFiles(self): | ||
876 | 2322 | # Multiple identical files in the same directory receive multiple | ||
877 | 2323 | # ArchiveFile rows, even though they share a by-hash entry. | ||
878 | 2324 | self.breezy_autotest.publish_by_hash = True | ||
879 | 2325 | publisher = Publisher( | ||
880 | 2326 | self.logger, self.config, self.disk_pool, | ||
881 | 2327 | self.ubuntutest.main_archive) | ||
882 | 2328 | suite_path = partial( | ||
883 | 2329 | os.path.join, self.config.distsroot, 'breezy-autotest') | ||
884 | 2330 | get_contents_files = lambda: [ | ||
885 | 2331 | archive_file | ||
886 | 2332 | for archive_file in getUtility(IArchiveFileSet).getByArchive( | ||
887 | 2333 | self.ubuntutest.main_archive) | ||
888 | 2334 | if archive_file.path.startswith('dists/breezy-autotest/Contents-')] | ||
889 | 2335 | |||
890 | 2336 | # Create the first file. | ||
891 | 2337 | with open_for_writing(suite_path('Contents-i386'), 'w') as f: | ||
892 | 2338 | f.write('A Contents file\n') | ||
893 | 2339 | publisher.markPocketDirty( | ||
894 | 2340 | self.breezy_autotest, PackagePublishingPocket.RELEASE) | ||
895 | 2341 | publisher.A_publish(False) | ||
896 | 2342 | publisher.C_doFTPArchive(False) | ||
897 | 2343 | publisher.D_writeReleaseFiles(False) | ||
898 | 2344 | flush_database_caches() | ||
899 | 2345 | matchers = [ | ||
900 | 2346 | MatchesStructure( | ||
901 | 2347 | path=Equals('dists/breezy-autotest/Contents-i386'), | ||
902 | 2348 | scheduled_deletion_date=Is(None))] | ||
903 | 2349 | self.assertThat(get_contents_files(), MatchesSetwise(*matchers)) | ||
904 | 2350 | self.assertThat( | ||
905 | 2351 | suite_path('by-hash'), ByHashHasContents(['A Contents file\n'])) | ||
906 | 2352 | |||
907 | 2353 | # Add a second identical file. | ||
908 | 2354 | with open_for_writing(suite_path('Contents-hppa'), 'w') as f: | ||
909 | 2355 | f.write('A Contents file\n') | ||
910 | 2356 | publisher.D_writeReleaseFiles(False) | ||
911 | 2357 | flush_database_caches() | ||
912 | 2358 | matchers.append( | ||
913 | 2359 | MatchesStructure( | ||
914 | 2360 | path=Equals('dists/breezy-autotest/Contents-hppa'), | ||
915 | 2361 | scheduled_deletion_date=Is(None))) | ||
916 | 2362 | self.assertThat(get_contents_files(), MatchesSetwise(*matchers)) | ||
917 | 2363 | self.assertThat( | ||
918 | 2364 | suite_path('by-hash'), ByHashHasContents(['A Contents file\n'])) | ||
919 | 2365 | |||
920 | 2366 | # Delete the first file, but allow it its stay of execution. | ||
921 | 2367 | os.unlink(suite_path('Contents-i386')) | ||
922 | 2368 | publisher.D_writeReleaseFiles(False) | ||
923 | 2369 | flush_database_caches() | ||
924 | 2370 | matchers[0] = matchers[0].update(scheduled_deletion_date=Not(Is(None))) | ||
925 | 2371 | self.assertThat(get_contents_files(), MatchesSetwise(*matchers)) | ||
926 | 2372 | self.assertThat( | ||
927 | 2373 | suite_path('by-hash'), ByHashHasContents(['A Contents file\n'])) | ||
928 | 2374 | |||
929 | 2375 | # Arrange for the first file to be pruned, and delete the second | ||
930 | 2376 | # file. | ||
931 | 2377 | now = datetime.now(pytz.UTC) | ||
932 | 2378 | i386_file = getUtility(IArchiveFileSet).getByArchive( | ||
933 | 2379 | self.ubuntutest.main_archive, | ||
934 | 2380 | path=u'dists/breezy-autotest/Contents-i386').one() | ||
935 | 2381 | removeSecurityProxy(i386_file).scheduled_deletion_date = ( | ||
936 | 2382 | now - timedelta(hours=1)) | ||
937 | 2383 | os.unlink(suite_path('Contents-hppa')) | ||
938 | 2384 | publisher.D_writeReleaseFiles(False) | ||
939 | 2385 | flush_database_caches() | ||
940 | 2386 | matchers = [matchers[1].update(scheduled_deletion_date=Not(Is(None)))] | ||
941 | 2387 | self.assertThat(get_contents_files(), MatchesSetwise(*matchers)) | ||
942 | 2388 | self.assertThat( | ||
943 | 2389 | suite_path('by-hash'), ByHashHasContents(['A Contents file\n'])) | ||
944 | 2390 | |||
945 | 2391 | # Arrange for the second file to be pruned. | ||
946 | 2392 | hppa_file = getUtility(IArchiveFileSet).getByArchive( | ||
947 | 2393 | self.ubuntutest.main_archive, | ||
948 | 2394 | path=u'dists/breezy-autotest/Contents-hppa').one() | ||
949 | 2395 | removeSecurityProxy(hppa_file).scheduled_deletion_date = ( | ||
950 | 2396 | now - timedelta(hours=1)) | ||
951 | 2397 | publisher.D_writeReleaseFiles(False) | ||
952 | 2398 | flush_database_caches() | ||
953 | 2399 | self.assertContentEqual([], get_contents_files()) | ||
954 | 2400 | self.assertThat(suite_path('by-hash'), Not(PathExists())) | ||
955 | 2401 | |||
956 | 2402 | def testUpdateByHashReprieve(self): | ||
957 | 2403 | # If a newly-modified index file is identical to a | ||
958 | 2404 | # previously-condemned one, then it is reprieved and not pruned. | ||
959 | 2405 | self.breezy_autotest.publish_by_hash = True | ||
960 | 2406 | # Enable uncompressed index files to avoid relying on stable output | ||
961 | 2407 | # from compressors in this test. | ||
962 | 2408 | self.breezy_autotest.index_compressors = [ | ||
963 | 2409 | IndexCompressionType.UNCOMPRESSED] | ||
964 | 2410 | publisher = Publisher( | ||
965 | 2411 | self.logger, self.config, self.disk_pool, | ||
966 | 2412 | self.ubuntutest.main_archive) | ||
967 | 2413 | |||
968 | 2414 | # Publish empty index files. | ||
969 | 2415 | publisher.markPocketDirty( | ||
970 | 2416 | self.breezy_autotest, PackagePublishingPocket.RELEASE) | ||
971 | 2417 | publisher.A_publish(False) | ||
972 | 2418 | publisher.C_doFTPArchive(False) | ||
973 | 2419 | publisher.D_writeReleaseFiles(False) | ||
974 | 2420 | suite_path = partial( | ||
975 | 2421 | os.path.join, self.config.distsroot, 'breezy-autotest') | ||
976 | 2422 | main_contents = set() | ||
977 | 2423 | for name in ('Release', 'Sources'): | ||
978 | 2424 | with open(suite_path('main', 'source', name), 'rb') as f: | ||
979 | 2425 | main_contents.add(f.read()) | ||
980 | 2426 | |||
981 | 2427 | # Add a source package so that Sources is non-empty. | ||
982 | 2428 | pub_source = self.getPubSource(filecontent='Source: foo\n') | ||
983 | 2429 | publisher.A_publish(False) | ||
984 | 2430 | publisher.C_doFTPArchive(False) | ||
985 | 2431 | publisher.D_writeReleaseFiles(False) | ||
986 | 2432 | transaction.commit() | ||
987 | 2433 | with open(suite_path('main', 'source', 'Sources'), 'rb') as f: | ||
988 | 2434 | main_contents.add(f.read()) | ||
989 | 2435 | self.assertEqual(3, len(main_contents)) | ||
990 | 2436 | self.assertThat( | ||
991 | 2437 | suite_path('main', 'source', 'by-hash'), | ||
992 | 2438 | ByHashHasContents(main_contents)) | ||
993 | 2439 | |||
994 | 2440 | # Make the empty Sources file ready to prune. | ||
995 | 2441 | old_archive_files = [] | ||
996 | 2442 | for archive_file in getUtility(IArchiveFileSet).getByArchive( | ||
997 | 2443 | self.ubuntutest.main_archive): | ||
998 | 2444 | if ('main/source' in archive_file.path and | ||
999 | 2445 | archive_file.scheduled_deletion_date is not None): | ||
1000 | 2446 | old_archive_files.append(archive_file) | ||
1001 | 2447 | self.assertEqual(1, len(old_archive_files)) | ||
1002 | 2448 | removeSecurityProxy(old_archive_files[0]).scheduled_deletion_date = ( | ||
1003 | 2449 | datetime.now(pytz.UTC) - timedelta(hours=1)) | ||
1004 | 2450 | |||
1005 | 2451 | # Delete the source package so that Sources is empty again. The | ||
1006 | 2452 | # empty file is reprieved and the non-empty one is condemned. | ||
1007 | 2453 | pub_source.requestDeletion(self.ubuntutest.owner) | ||
1008 | 2454 | publisher.A_publish(False) | ||
1009 | 2455 | publisher.C_doFTPArchive(False) | ||
1010 | 2456 | publisher.D_writeReleaseFiles(False) | ||
1011 | 2457 | transaction.commit() | ||
1012 | 2458 | self.assertThat( | ||
1013 | 2459 | suite_path('main', 'source', 'by-hash'), | ||
1014 | 2460 | ByHashHasContents(main_contents)) | ||
1015 | 2461 | archive_files = getUtility(IArchiveFileSet).getByArchive( | ||
1016 | 2462 | self.ubuntutest.main_archive, | ||
1017 | 2463 | path=u'dists/breezy-autotest/main/source/Sources') | ||
1018 | 2464 | self.assertThat( | ||
1019 | 2465 | sorted(archive_files, key=attrgetter('id')), | ||
1020 | 2466 | MatchesListwise([ | ||
1021 | 2467 | MatchesStructure(scheduled_deletion_date=Is(None)), | ||
1022 | 2468 | MatchesStructure(scheduled_deletion_date=Not(Is(None))), | ||
1023 | 2469 | ])) | ||
1024 | 2470 | |||
1025 | 2471 | def testUpdateByHashPrune(self): | ||
1026 | 2472 | # The publisher prunes files from by-hash that were condemned more | ||
1027 | 2473 | # than a day ago. | ||
1028 | 2474 | self.breezy_autotest.publish_by_hash = True | ||
1029 | 2475 | self.breezy_autotest.advertise_by_hash = True | ||
1030 | 2476 | publisher = Publisher( | ||
1031 | 2477 | self.logger, self.config, self.disk_pool, | ||
1032 | 2478 | self.ubuntutest.main_archive) | ||
1033 | 2479 | |||
1034 | 2480 | suite_path = partial( | ||
1035 | 2481 | os.path.join, self.config.distsroot, 'breezy-autotest') | ||
1036 | 2482 | main_contents = set() | ||
1037 | 2483 | for sourcename in ('foo', 'bar'): | ||
1038 | 2484 | self.getPubSource( | ||
1039 | 2485 | sourcename=sourcename, filecontent='Source: %s\n' % sourcename) | ||
1040 | 2486 | publisher.A_publish(False) | ||
1041 | 2487 | publisher.C_doFTPArchive(False) | ||
1042 | 2488 | publisher.D_writeReleaseFiles(False) | ||
1043 | 2489 | for name in ('Release', 'Sources.gz', 'Sources.bz2'): | ||
1044 | 2490 | with open(suite_path('main', 'source', name), 'rb') as f: | ||
1045 | 2491 | main_contents.add(f.read()) | ||
1046 | 2492 | transaction.commit() | ||
1047 | 2493 | # Undo any previous determination that breezy-autotest is dirty, so | ||
1048 | 2494 | # that we can use that to check that future runs don't force index | ||
1049 | 2495 | # regeneration. | ||
1050 | 2496 | publisher.dirty_pockets = set() | ||
1051 | 2497 | |||
1052 | 2498 | self.assertThat( | ||
1053 | 2499 | suite_path('main', 'source', 'by-hash'), | ||
1054 | 2500 | ByHashHasContents(main_contents)) | ||
1055 | 2501 | old_archive_files = [] | ||
1056 | 2502 | for archive_file in getUtility(IArchiveFileSet).getByArchive( | ||
1057 | 2503 | self.ubuntutest.main_archive): | ||
1058 | 2504 | if ('main/source' in archive_file.path and | ||
1059 | 2505 | archive_file.scheduled_deletion_date is not None): | ||
1060 | 2506 | old_archive_files.append(archive_file) | ||
1061 | 2507 | self.assertEqual(2, len(old_archive_files)) | ||
1062 | 2508 | |||
1063 | 2509 | now = datetime.now(pytz.UTC) | ||
1064 | 2510 | removeSecurityProxy(old_archive_files[0]).scheduled_deletion_date = ( | ||
1065 | 2511 | now + timedelta(hours=12)) | ||
1066 | 2512 | removeSecurityProxy(old_archive_files[1]).scheduled_deletion_date = ( | ||
1067 | 2513 | now - timedelta(hours=12)) | ||
1068 | 2514 | old_archive_files[1].library_file.open() | ||
1069 | 2515 | try: | ||
1070 | 2516 | main_contents.remove(old_archive_files[1].library_file.read()) | ||
1071 | 2517 | finally: | ||
1072 | 2518 | old_archive_files[1].library_file.close() | ||
1073 | 2519 | self.assertThat( | ||
1074 | 2520 | suite_path('main', 'source', 'by-hash'), | ||
1075 | 2521 | Not(ByHashHasContents(main_contents))) | ||
1076 | 2522 | |||
1077 | 2523 | publisher.A2_markPocketsWithDeletionsDirty() | ||
1078 | 2524 | publisher.C_doFTPArchive(False) | ||
1079 | 2525 | publisher.D_writeReleaseFiles(False) | ||
1080 | 2526 | self.assertEqual(set(), publisher.dirty_pockets) | ||
1081 | 2527 | self.assertThat( | ||
1082 | 2528 | suite_path('main', 'source', 'by-hash'), | ||
1083 | 2529 | ByHashHasContents(main_contents)) | ||
1084 | 2530 | |||
1085 | 1933 | def testCreateSeriesAliasesNoAlias(self): | 2531 | def testCreateSeriesAliasesNoAlias(self): |
1086 | 1934 | """createSeriesAliases has nothing to do by default.""" | 2532 | """createSeriesAliases has nothing to do by default.""" |
1087 | 1935 | publisher = Publisher( | 2533 | publisher = Publisher( |
1088 | 1936 | 2534 | ||
1089 | === modified file 'lib/lp/registry/model/distribution.py' | |||
1090 | --- lib/lp/registry/model/distribution.py 2015-10-13 13:22:08 +0000 | |||
1091 | +++ lib/lp/registry/model/distribution.py 2016-04-02 00:45:52 +0000 | |||
1092 | @@ -1,4 +1,4 @@ | |||
1094 | 1 | # Copyright 2009-2015 Canonical Ltd. This software is licensed under the | 1 | # Copyright 2009-2016 Canonical Ltd. This software is licensed under the |
1095 | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). | 2 | # GNU Affero General Public License version 3 (see the file LICENSE). |
1096 | 3 | 3 | ||
1097 | 4 | """Database classes for implementing distribution items.""" | 4 | """Database classes for implementing distribution items.""" |
1098 | @@ -1283,10 +1283,22 @@ | |||
1099 | 1283 | bin_query, clauseTables=['BinaryPackagePublishingHistory'], | 1283 | bin_query, clauseTables=['BinaryPackagePublishingHistory'], |
1100 | 1284 | orderBy=['archive.id'], distinct=True) | 1284 | orderBy=['archive.id'], distinct=True) |
1101 | 1285 | 1285 | ||
1102 | 1286 | reapable_af_query = """ | ||
1103 | 1287 | Archive.purpose = %s AND | ||
1104 | 1288 | Archive.distribution = %s AND | ||
1105 | 1289 | ArchiveFile.archive = archive.id AND | ||
1106 | 1290 | ArchiveFile.scheduled_deletion_date < %s | ||
1107 | 1291 | """ % sqlvalues(ArchivePurpose.PPA, self, UTC_NOW) | ||
1108 | 1292 | |||
1109 | 1293 | reapable_af_archives = Archive.select( | ||
1110 | 1294 | reapable_af_query, clauseTables=['ArchiveFile'], | ||
1111 | 1295 | orderBy=['archive.id'], distinct=True) | ||
1112 | 1296 | |||
1113 | 1286 | deleting_archives = Archive.selectBy( | 1297 | deleting_archives = Archive.selectBy( |
1114 | 1287 | status=ArchiveStatus.DELETING).orderBy(['archive.id']) | 1298 | status=ArchiveStatus.DELETING).orderBy(['archive.id']) |
1115 | 1288 | 1299 | ||
1117 | 1289 | return src_archives.union(bin_archives).union(deleting_archives) | 1300 | return src_archives.union(bin_archives).union( |
1118 | 1301 | reapable_af_archives).union(deleting_archives) | ||
1119 | 1290 | 1302 | ||
1120 | 1291 | def getArchiveByComponent(self, component_name): | 1303 | def getArchiveByComponent(self, component_name): |
1121 | 1292 | """See `IDistribution`.""" | 1304 | """See `IDistribution`.""" |
1122 | 1293 | 1305 | ||
1123 | === modified file 'lib/lp/services/helpers.py' | |||
1124 | --- lib/lp/services/helpers.py 2014-05-07 15:28:50 +0000 | |||
1125 | +++ lib/lp/services/helpers.py 2016-04-02 00:45:52 +0000 | |||
1126 | @@ -10,6 +10,7 @@ | |||
1127 | 10 | 10 | ||
1128 | 11 | __metaclass__ = type | 11 | __metaclass__ = type |
1129 | 12 | 12 | ||
1130 | 13 | from collections import OrderedDict | ||
1131 | 13 | from difflib import unified_diff | 14 | from difflib import unified_diff |
1132 | 14 | import re | 15 | import re |
1133 | 15 | from StringIO import StringIO | 16 | from StringIO import StringIO |
1134 | @@ -224,19 +225,37 @@ | |||
1135 | 224 | 225 | ||
1136 | 225 | >>> filenameToContentType('test.tgz') | 226 | >>> filenameToContentType('test.tgz') |
1137 | 226 | 'application/octet-stream' | 227 | 'application/octet-stream' |
1138 | 228 | |||
1139 | 229 | Build logs | ||
1140 | 230 | >>> filenameToContentType('buildlog.txt.gz') | ||
1141 | 231 | 'text/plain' | ||
1142 | 232 | |||
1143 | 233 | Various compressed files | ||
1144 | 234 | |||
1145 | 235 | >>> filenameToContentType('Packages.gz') | ||
1146 | 236 | 'application/x-gzip' | ||
1147 | 237 | >>> filenameToContentType('Packages.bz2') | ||
1148 | 238 | 'application/x-bzip2' | ||
1149 | 239 | >>> filenameToContentType('Packages.xz') | ||
1150 | 240 | 'application/x-xz' | ||
1151 | 227 | """ | 241 | """ |
1164 | 228 | ftmap = {".dsc": "text/plain", | 242 | ftmap = OrderedDict([ |
1165 | 229 | ".changes": "text/plain", | 243 | (".dsc", "text/plain"), |
1166 | 230 | ".deb": "application/x-debian-package", | 244 | (".changes", "text/plain"), |
1167 | 231 | ".udeb": "application/x-debian-package", | 245 | (".deb", "application/x-debian-package"), |
1168 | 232 | ".txt": "text/plain", | 246 | (".udeb", "application/x-debian-package"), |
1169 | 233 | # For the build master logs | 247 | (".txt", "text/plain"), |
1170 | 234 | ".txt.gz": "text/plain", | 248 | # For the build master logs |
1171 | 235 | # For live filesystem builds | 249 | (".txt.gz", "text/plain"), |
1172 | 236 | ".manifest": "text/plain", | 250 | # For live filesystem builds |
1173 | 237 | ".manifest-remove": "text/plain", | 251 | (".manifest", "text/plain"), |
1174 | 238 | ".size": "text/plain", | 252 | (".manifest-remove", "text/plain"), |
1175 | 239 | } | 253 | (".size", "text/plain"), |
1176 | 254 | # Compressed files | ||
1177 | 255 | (".gz", "application/x-gzip"), | ||
1178 | 256 | (".bz2", "application/x-bzip2"), | ||
1179 | 257 | (".xz", "application/x-xz"), | ||
1180 | 258 | ]) | ||
1181 | 240 | for ending in ftmap: | 259 | for ending in ftmap: |
1182 | 241 | if fname.endswith(ending): | 260 | if fname.endswith(ending): |
1183 | 242 | return ftmap[ending] | 261 | return ftmap[ending] |
1184 | 243 | 262 | ||
1185 | === modified file 'lib/lp/services/librarian/interfaces/__init__.py' | |||
1186 | --- lib/lp/services/librarian/interfaces/__init__.py 2016-03-14 16:28:19 +0000 | |||
1187 | +++ lib/lp/services/librarian/interfaces/__init__.py 2016-04-02 00:45:52 +0000 | |||
1188 | @@ -155,7 +155,7 @@ | |||
1189 | 155 | class ILibraryFileAliasSet(Interface): | 155 | class ILibraryFileAliasSet(Interface): |
1190 | 156 | 156 | ||
1191 | 157 | def create(name, size, file, contentType, expires=None, debugID=None, | 157 | def create(name, size, file, contentType, expires=None, debugID=None, |
1193 | 158 | restricted=False): | 158 | restricted=False, allow_zero_length=False): |
1194 | 159 | """Create a file in the Librarian, returning the new alias. | 159 | """Create a file in the Librarian, returning the new alias. |
1195 | 160 | 160 | ||
1196 | 161 | An expiry time of None means the file will never expire until it | 161 | An expiry time of None means the file will never expire until it |
1197 | 162 | 162 | ||
1198 | === modified file 'lib/lp/services/librarian/model.py' | |||
1199 | --- lib/lp/services/librarian/model.py 2016-03-14 16:28:19 +0000 | |||
1200 | +++ lib/lp/services/librarian/model.py 2016-04-02 00:45:52 +0000 | |||
1201 | @@ -244,7 +244,7 @@ | |||
1202 | 244 | """Create and find LibraryFileAliases.""" | 244 | """Create and find LibraryFileAliases.""" |
1203 | 245 | 245 | ||
1204 | 246 | def create(self, name, size, file, contentType, expires=None, | 246 | def create(self, name, size, file, contentType, expires=None, |
1206 | 247 | debugID=None, restricted=False): | 247 | debugID=None, restricted=False, allow_zero_length=False): |
1207 | 248 | """See `ILibraryFileAliasSet`""" | 248 | """See `ILibraryFileAliasSet`""" |
1208 | 249 | if restricted: | 249 | if restricted: |
1209 | 250 | client = getUtility(IRestrictedLibrarianClient) | 250 | client = getUtility(IRestrictedLibrarianClient) |
1210 | @@ -252,7 +252,9 @@ | |||
1211 | 252 | client = getUtility(ILibrarianClient) | 252 | client = getUtility(ILibrarianClient) |
1212 | 253 | if '/' in name: | 253 | if '/' in name: |
1213 | 254 | raise InvalidFilename("Filename cannot contain slashes.") | 254 | raise InvalidFilename("Filename cannot contain slashes.") |
1215 | 255 | fid = client.addFile(name, size, file, contentType, expires, debugID) | 255 | fid = client.addFile( |
1216 | 256 | name, size, file, contentType, expires=expires, debugID=debugID, | ||
1217 | 257 | allow_zero_length=allow_zero_length) | ||
1218 | 256 | lfa = IMasterStore(LibraryFileAlias).find( | 258 | lfa = IMasterStore(LibraryFileAlias).find( |
1219 | 257 | LibraryFileAlias, LibraryFileAlias.id == fid).one() | 259 | LibraryFileAlias, LibraryFileAlias.id == fid).one() |
1220 | 258 | assert lfa is not None, "client.addFile didn't!" | 260 | assert lfa is not None, "client.addFile didn't!" |
1221 | 259 | 261 | ||
1222 | === modified file 'lib/lp/soyuz/interfaces/archivefile.py' | |||
1223 | --- lib/lp/soyuz/interfaces/archivefile.py 2016-03-18 15:09:37 +0000 | |||
1224 | +++ lib/lp/soyuz/interfaces/archivefile.py 2016-04-02 00:45:52 +0000 | |||
1225 | @@ -79,13 +79,15 @@ | |||
1226 | 79 | :param content_type: The MIME type of the file. | 79 | :param content_type: The MIME type of the file. |
1227 | 80 | """ | 80 | """ |
1228 | 81 | 81 | ||
1230 | 82 | def getByArchive(archive, container=None, eager_load=False): | 82 | def getByArchive(archive, container=None, path=None, eager_load=False): |
1231 | 83 | """Get files in an archive. | 83 | """Get files in an archive. |
1232 | 84 | 84 | ||
1233 | 85 | :param archive: Return files in this `IArchive`. | 85 | :param archive: Return files in this `IArchive`. |
1234 | 86 | :param container: Return only files with this container. | 86 | :param container: Return only files with this container. |
1235 | 87 | :param path: Return only files with this path. | ||
1236 | 87 | :param eager_load: If True, preload related `LibraryFileAlias` and | 88 | :param eager_load: If True, preload related `LibraryFileAlias` and |
1237 | 88 | `LibraryFileContent` rows. | 89 | `LibraryFileContent` rows. |
1238 | 90 | :return: An iterable of matched files. | ||
1239 | 89 | """ | 91 | """ |
1240 | 90 | 92 | ||
1241 | 91 | def scheduleDeletion(archive_files, stay_of_execution): | 93 | def scheduleDeletion(archive_files, stay_of_execution): |
1242 | @@ -94,6 +96,25 @@ | |||
1243 | 94 | :param archive_files: The `IArchiveFile`s to schedule for deletion. | 96 | :param archive_files: The `IArchiveFile`s to schedule for deletion. |
1244 | 95 | :param stay_of_execution: A `timedelta`; schedule files for deletion | 97 | :param stay_of_execution: A `timedelta`; schedule files for deletion |
1245 | 96 | this amount of time in the future. | 98 | this amount of time in the future. |
1246 | 99 | :return: An iterable of (container, path, sha256) for files that | ||
1247 | 100 | were scheduled for deletion. | ||
1248 | 101 | """ | ||
1249 | 102 | |||
1250 | 103 | def unscheduleDeletion(archive, container=None, sha256_checksums=set()): | ||
1251 | 104 | """Unschedule these archive files for deletion. | ||
1252 | 105 | |||
1253 | 106 | This is useful in the case when the new content of a file is | ||
1254 | 107 | identical to a version that was previously condemned. This method's | ||
1255 | 108 | signature does not match that of `scheduleDeletion`; this is more | ||
1256 | 109 | convenient because in such cases we normally do not yet have | ||
1257 | 110 | `ArchiveFile` rows in hand. | ||
1258 | 111 | |||
1259 | 112 | :param archive: Operate on files in this `IArchive`. | ||
1260 | 113 | :param container: Operate only on files with this container. | ||
1261 | 114 | :param sha256_checksums: Operate only on files with any of these | ||
1262 | 115 | checksums. | ||
1263 | 116 | :return: An iterable of (container, path, sha256) for files that | ||
1264 | 117 | were unscheduled for deletion. | ||
1265 | 97 | """ | 118 | """ |
1266 | 98 | 119 | ||
1267 | 99 | def getContainersToReap(archive, container_prefix=None): | 120 | def getContainersToReap(archive, container_prefix=None): |
1268 | @@ -102,6 +123,7 @@ | |||
1269 | 102 | :param archive: Return containers in this `IArchive`. | 123 | :param archive: Return containers in this `IArchive`. |
1270 | 103 | :param container_prefix: Return only containers that start with this | 124 | :param container_prefix: Return only containers that start with this |
1271 | 104 | prefix. | 125 | prefix. |
1272 | 126 | :return: An iterable of matched container names. | ||
1273 | 105 | """ | 127 | """ |
1274 | 106 | 128 | ||
1275 | 107 | def reap(archive, container=None): | 129 | def reap(archive, container=None): |
1276 | @@ -109,4 +131,6 @@ | |||
1277 | 109 | 131 | ||
1278 | 110 | :param archive: Delete files from this `IArchive`. | 132 | :param archive: Delete files from this `IArchive`. |
1279 | 111 | :param container: Delete only files with this container. | 133 | :param container: Delete only files with this container. |
1280 | 134 | :return: An iterable of (container, path, sha256) for files that | ||
1281 | 135 | were deleted. | ||
1282 | 112 | """ | 136 | """ |
1283 | 113 | 137 | ||
1284 | === modified file 'lib/lp/soyuz/model/archivefile.py' | |||
1285 | --- lib/lp/soyuz/model/archivefile.py 2016-03-18 15:09:37 +0000 | |||
1286 | +++ lib/lp/soyuz/model/archivefile.py 2016-04-02 00:45:52 +0000 | |||
1287 | @@ -14,7 +14,9 @@ | |||
1288 | 14 | import os.path | 14 | import os.path |
1289 | 15 | 15 | ||
1290 | 16 | import pytz | 16 | import pytz |
1291 | 17 | from storm.databases.postgres import Returning | ||
1292 | 17 | from storm.locals import ( | 18 | from storm.locals import ( |
1293 | 19 | And, | ||
1294 | 18 | DateTime, | 20 | DateTime, |
1295 | 19 | Int, | 21 | Int, |
1296 | 20 | Reference, | 22 | Reference, |
1297 | @@ -31,6 +33,7 @@ | |||
1298 | 31 | IMasterStore, | 33 | IMasterStore, |
1299 | 32 | IStore, | 34 | IStore, |
1300 | 33 | ) | 35 | ) |
1301 | 36 | from lp.services.database.stormexpr import BulkUpdate | ||
1302 | 34 | from lp.services.librarian.interfaces import ILibraryFileAliasSet | 37 | from lp.services.librarian.interfaces import ILibraryFileAliasSet |
1303 | 35 | from lp.services.librarian.model import ( | 38 | from lp.services.librarian.model import ( |
1304 | 36 | LibraryFileAlias, | 39 | LibraryFileAlias, |
1305 | @@ -89,17 +92,19 @@ | |||
1306 | 89 | content_type): | 92 | content_type): |
1307 | 90 | library_file = getUtility(ILibraryFileAliasSet).create( | 93 | library_file = getUtility(ILibraryFileAliasSet).create( |
1308 | 91 | os.path.basename(path), size, fileobj, content_type, | 94 | os.path.basename(path), size, fileobj, content_type, |
1310 | 92 | restricted=archive.private) | 95 | restricted=archive.private, allow_zero_length=True) |
1311 | 93 | return cls.new(archive, container, path, library_file) | 96 | return cls.new(archive, container, path, library_file) |
1312 | 94 | 97 | ||
1313 | 95 | @staticmethod | 98 | @staticmethod |
1315 | 96 | def getByArchive(archive, container=None, eager_load=False): | 99 | def getByArchive(archive, container=None, path=None, eager_load=False): |
1316 | 97 | """See `IArchiveFileSet`.""" | 100 | """See `IArchiveFileSet`.""" |
1317 | 98 | clauses = [ArchiveFile.archive == archive] | 101 | clauses = [ArchiveFile.archive == archive] |
1318 | 99 | # XXX cjwatson 2016-03-15: We'll need some more sophisticated way to | 102 | # XXX cjwatson 2016-03-15: We'll need some more sophisticated way to |
1319 | 100 | # match containers once we're using them for custom uploads. | 103 | # match containers once we're using them for custom uploads. |
1320 | 101 | if container is not None: | 104 | if container is not None: |
1321 | 102 | clauses.append(ArchiveFile.container == container) | 105 | clauses.append(ArchiveFile.container == container) |
1322 | 106 | if path is not None: | ||
1323 | 107 | clauses.append(ArchiveFile.path == path) | ||
1324 | 103 | archive_files = IStore(ArchiveFile).find(ArchiveFile, *clauses) | 108 | archive_files = IStore(ArchiveFile).find(ArchiveFile, *clauses) |
1325 | 104 | 109 | ||
1326 | 105 | def eager_load(rows): | 110 | def eager_load(rows): |
1327 | @@ -114,11 +119,43 @@ | |||
1328 | 114 | @staticmethod | 119 | @staticmethod |
1329 | 115 | def scheduleDeletion(archive_files, stay_of_execution): | 120 | def scheduleDeletion(archive_files, stay_of_execution): |
1330 | 116 | """See `IArchiveFileSet`.""" | 121 | """See `IArchiveFileSet`.""" |
1336 | 117 | archive_file_ids = set( | 122 | clauses = [ |
1337 | 118 | archive_file.id for archive_file in archive_files) | 123 | ArchiveFile.id.is_in( |
1338 | 119 | rows = IMasterStore(ArchiveFile).find( | 124 | set(archive_file.id for archive_file in archive_files)), |
1339 | 120 | ArchiveFile, ArchiveFile.id.is_in(archive_file_ids)) | 125 | ArchiveFile.library_file == LibraryFileAlias.id, |
1340 | 121 | rows.set(scheduled_deletion_date=UTC_NOW + stay_of_execution) | 126 | LibraryFileAlias.content == LibraryFileContent.id, |
1341 | 127 | ] | ||
1342 | 128 | new_date = UTC_NOW + stay_of_execution | ||
1343 | 129 | return_columns = [ | ||
1344 | 130 | ArchiveFile.container, ArchiveFile.path, LibraryFileContent.sha256] | ||
1345 | 131 | return list(IMasterStore(ArchiveFile).execute(Returning( | ||
1346 | 132 | BulkUpdate( | ||
1347 | 133 | {ArchiveFile.scheduled_deletion_date: new_date}, | ||
1348 | 134 | table=ArchiveFile, | ||
1349 | 135 | values=[LibraryFileAlias, LibraryFileContent], | ||
1350 | 136 | where=And(*clauses)), | ||
1351 | 137 | columns=return_columns))) | ||
1352 | 138 | |||
1353 | 139 | @staticmethod | ||
1354 | 140 | def unscheduleDeletion(archive, container=None, sha256_checksums=set()): | ||
1355 | 141 | """See `IArchiveFileSet`.""" | ||
1356 | 142 | clauses = [ | ||
1357 | 143 | ArchiveFile.archive == archive, | ||
1358 | 144 | ArchiveFile.library_file == LibraryFileAlias.id, | ||
1359 | 145 | LibraryFileAlias.content == LibraryFileContent.id, | ||
1360 | 146 | LibraryFileContent.sha256.is_in(sha256_checksums), | ||
1361 | 147 | ] | ||
1362 | 148 | if container is not None: | ||
1363 | 149 | clauses.append(ArchiveFile.container == container) | ||
1364 | 150 | return_columns = [ | ||
1365 | 151 | ArchiveFile.container, ArchiveFile.path, LibraryFileContent.sha256] | ||
1366 | 152 | return list(IMasterStore(ArchiveFile).execute(Returning( | ||
1367 | 153 | BulkUpdate( | ||
1368 | 154 | {ArchiveFile.scheduled_deletion_date: None}, | ||
1369 | 155 | table=ArchiveFile, | ||
1370 | 156 | values=[LibraryFileAlias, LibraryFileContent], | ||
1371 | 157 | where=And(*clauses)), | ||
1372 | 158 | columns=return_columns))) | ||
1373 | 122 | 159 | ||
1374 | 123 | @staticmethod | 160 | @staticmethod |
1375 | 124 | def getContainersToReap(archive, container_prefix=None): | 161 | def getContainersToReap(archive, container_prefix=None): |
1376 | @@ -134,10 +171,25 @@ | |||
1377 | 134 | @staticmethod | 171 | @staticmethod |
1378 | 135 | def reap(archive, container=None): | 172 | def reap(archive, container=None): |
1379 | 136 | """See `IArchiveFileSet`.""" | 173 | """See `IArchiveFileSet`.""" |
1380 | 174 | # XXX cjwatson 2016-03-30 bug=322972: Requires manual SQL due to | ||
1381 | 175 | # lack of support for DELETE FROM ... USING ... in Storm. | ||
1382 | 137 | clauses = [ | 176 | clauses = [ |
1385 | 138 | ArchiveFile.archive == archive, | 177 | "ArchiveFile.archive = ?", |
1386 | 139 | ArchiveFile.scheduled_deletion_date < UTC_NOW, | 178 | "ArchiveFile.scheduled_deletion_date < " |
1387 | 179 | "CURRENT_TIMESTAMP AT TIME ZONE 'UTC'", | ||
1388 | 180 | "ArchiveFile.library_file = LibraryFileAlias.id", | ||
1389 | 181 | "LibraryFileAlias.content = LibraryFileContent.id", | ||
1390 | 140 | ] | 182 | ] |
1391 | 183 | values = [archive.id] | ||
1392 | 141 | if container is not None: | 184 | if container is not None: |
1395 | 142 | clauses.append(ArchiveFile.container == container) | 185 | clauses.append("ArchiveFile.container = ?") |
1396 | 143 | IMasterStore(ArchiveFile).find(ArchiveFile, *clauses).remove() | 186 | values.append(container) |
1397 | 187 | return list(IMasterStore(ArchiveFile).execute(""" | ||
1398 | 188 | DELETE FROM ArchiveFile | ||
1399 | 189 | USING LibraryFileAlias, LibraryFileContent | ||
1400 | 190 | WHERE """ + " AND ".join(clauses) + """ | ||
1401 | 191 | RETURNING | ||
1402 | 192 | ArchiveFile.container, | ||
1403 | 193 | ArchiveFile.path, | ||
1404 | 194 | LibraryFileContent.sha256 | ||
1405 | 195 | """, values)) | ||
1406 | 144 | 196 | ||
1407 | === modified file 'lib/lp/soyuz/tests/test_archivefile.py' | |||
1408 | --- lib/lp/soyuz/tests/test_archivefile.py 2016-03-18 15:09:37 +0000 | |||
1409 | +++ lib/lp/soyuz/tests/test_archivefile.py 2016-04-02 00:45:52 +0000 | |||
1410 | @@ -19,6 +19,7 @@ | |||
1411 | 19 | from zope.component import getUtility | 19 | from zope.component import getUtility |
1412 | 20 | from zope.security.proxy import removeSecurityProxy | 20 | from zope.security.proxy import removeSecurityProxy |
1413 | 21 | 21 | ||
1414 | 22 | from lp.services.database.sqlbase import flush_database_caches | ||
1415 | 22 | from lp.services.osutils import open_for_writing | 23 | from lp.services.osutils import open_for_writing |
1416 | 23 | from lp.soyuz.interfaces.archivefile import IArchiveFileSet | 24 | from lp.soyuz.interfaces.archivefile import IArchiveFileSet |
1417 | 24 | from lp.testing import TestCaseWithFactory | 25 | from lp.testing import TestCaseWithFactory |
1418 | @@ -75,17 +76,35 @@ | |||
1419 | 75 | self.assertContentEqual( | 76 | self.assertContentEqual( |
1420 | 76 | [], archive_file_set.getByArchive(archives[0], container="bar")) | 77 | [], archive_file_set.getByArchive(archives[0], container="bar")) |
1421 | 77 | self.assertContentEqual( | 78 | self.assertContentEqual( |
1422 | 79 | [archive_files[1]], | ||
1423 | 80 | archive_file_set.getByArchive( | ||
1424 | 81 | archives[0], path=archive_files[1].path)) | ||
1425 | 82 | self.assertContentEqual( | ||
1426 | 83 | [], archive_file_set.getByArchive(archives[0], path="other")) | ||
1427 | 84 | self.assertContentEqual( | ||
1428 | 78 | archive_files[2:], archive_file_set.getByArchive(archives[1])) | 85 | archive_files[2:], archive_file_set.getByArchive(archives[1])) |
1429 | 79 | self.assertContentEqual( | 86 | self.assertContentEqual( |
1430 | 80 | [archive_files[3]], | 87 | [archive_files[3]], |
1431 | 81 | archive_file_set.getByArchive(archives[1], container="foo")) | 88 | archive_file_set.getByArchive(archives[1], container="foo")) |
1432 | 82 | self.assertContentEqual( | 89 | self.assertContentEqual( |
1433 | 83 | [], archive_file_set.getByArchive(archives[1], container="bar")) | 90 | [], archive_file_set.getByArchive(archives[1], container="bar")) |
1434 | 91 | self.assertContentEqual( | ||
1435 | 92 | [archive_files[3]], | ||
1436 | 93 | archive_file_set.getByArchive( | ||
1437 | 94 | archives[1], path=archive_files[3].path)) | ||
1438 | 95 | self.assertContentEqual( | ||
1439 | 96 | [], archive_file_set.getByArchive(archives[1], path="other")) | ||
1440 | 84 | 97 | ||
1441 | 85 | def test_scheduleDeletion(self): | 98 | def test_scheduleDeletion(self): |
1442 | 86 | archive_files = [self.factory.makeArchiveFile() for _ in range(3)] | 99 | archive_files = [self.factory.makeArchiveFile() for _ in range(3)] |
1444 | 87 | getUtility(IArchiveFileSet).scheduleDeletion( | 100 | expected_rows = [ |
1445 | 101 | (archive_file.container, archive_file.path, | ||
1446 | 102 | archive_file.library_file.content.sha256) | ||
1447 | 103 | for archive_file in archive_files[:2]] | ||
1448 | 104 | rows = getUtility(IArchiveFileSet).scheduleDeletion( | ||
1449 | 88 | archive_files[:2], timedelta(days=1)) | 105 | archive_files[:2], timedelta(days=1)) |
1450 | 106 | self.assertContentEqual(expected_rows, rows) | ||
1451 | 107 | flush_database_caches() | ||
1452 | 89 | tomorrow = datetime.now(pytz.UTC) + timedelta(days=1) | 108 | tomorrow = datetime.now(pytz.UTC) + timedelta(days=1) |
1453 | 90 | # Allow a bit of timing slack for slow tests. | 109 | # Allow a bit of timing slack for slow tests. |
1454 | 91 | self.assertThat( | 110 | self.assertThat( |
1455 | @@ -96,6 +115,34 @@ | |||
1456 | 96 | LessThan(timedelta(minutes=5))) | 115 | LessThan(timedelta(minutes=5))) |
1457 | 97 | self.assertIsNone(archive_files[2].scheduled_deletion_date) | 116 | self.assertIsNone(archive_files[2].scheduled_deletion_date) |
1458 | 98 | 117 | ||
1459 | 118 | def test_unscheduleDeletion(self): | ||
1460 | 119 | archives = [self.factory.makeArchive() for _ in range(2)] | ||
1461 | 120 | lfas = [ | ||
1462 | 121 | self.factory.makeLibraryFileAlias(db_only=True) for _ in range(3)] | ||
1463 | 122 | archive_files = [] | ||
1464 | 123 | for archive in archives: | ||
1465 | 124 | for container in ("foo", "bar"): | ||
1466 | 125 | archive_files.extend([ | ||
1467 | 126 | self.factory.makeArchiveFile( | ||
1468 | 127 | archive=archive, container=container, library_file=lfa) | ||
1469 | 128 | for lfa in lfas]) | ||
1470 | 129 | now = datetime.now(pytz.UTC) | ||
1471 | 130 | for archive_file in archive_files: | ||
1472 | 131 | removeSecurityProxy(archive_file).scheduled_deletion_date = now | ||
1473 | 132 | expected_rows = [ | ||
1474 | 133 | ("foo", archive_files[0].path, lfas[0].content.sha256), | ||
1475 | 134 | ("foo", archive_files[1].path, lfas[1].content.sha256), | ||
1476 | 135 | ] | ||
1477 | 136 | rows = getUtility(IArchiveFileSet).unscheduleDeletion( | ||
1478 | 137 | archive=archives[0], container="foo", | ||
1479 | 138 | sha256_checksums=[lfas[0].content.sha256, lfas[1].content.sha256]) | ||
1480 | 139 | self.assertContentEqual(expected_rows, rows) | ||
1481 | 140 | flush_database_caches() | ||
1482 | 141 | self.assertContentEqual( | ||
1483 | 142 | [archive_files[0], archive_files[1]], | ||
1484 | 143 | [archive_file for archive_file in archive_files | ||
1485 | 144 | if archive_file.scheduled_deletion_date is None]) | ||
1486 | 145 | |||
1487 | 99 | def test_getContainersToReap(self): | 146 | def test_getContainersToReap(self): |
1488 | 100 | archive = self.factory.makeArchive() | 147 | archive = self.factory.makeArchive() |
1489 | 101 | archive_files = [] | 148 | archive_files = [] |
1490 | @@ -149,6 +196,11 @@ | |||
1491 | 149 | removeSecurityProxy(archive_files[4]).scheduled_deletion_date = ( | 196 | removeSecurityProxy(archive_files[4]).scheduled_deletion_date = ( |
1492 | 150 | now - timedelta(days=1)) | 197 | now - timedelta(days=1)) |
1493 | 151 | archive_file_set = getUtility(IArchiveFileSet) | 198 | archive_file_set = getUtility(IArchiveFileSet) |
1495 | 152 | archive_file_set.reap(archive, container="foo") | 199 | expected_rows = [ |
1496 | 200 | ("foo", archive_files[0].path, | ||
1497 | 201 | archive_files[0].library_file.content.sha256), | ||
1498 | 202 | ] | ||
1499 | 203 | rows = archive_file_set.reap(archive, container="foo") | ||
1500 | 204 | self.assertContentEqual(expected_rows, rows) | ||
1501 | 153 | self.assertContentEqual( | 205 | self.assertContentEqual( |
1502 | 154 | archive_files[1:4], archive_file_set.getByArchive(archive)) | 206 | archive_files[1:4], archive_file_set.getByArchive(archive)) |
Should be worth another look now.