Merge ~cjwatson/launchpad:diskpool-pathlib into launchpad:master
- Git
- lp:~cjwatson/launchpad
- diskpool-pathlib
- Merge into master
Status: | Merged |
---|---|
Approved by: | Colin Watson |
Approved revision: | 46fffe9983e56732d08c20b6f51e1bb29ccea038 |
Merge reported by: | Otto Co-Pilot |
Merged at revision: | not available |
Proposed branch: | ~cjwatson/launchpad:diskpool-pathlib |
Merge into: | launchpad:master |
Diff against target: |
712 lines (+128/-132) 13 files modified
lib/lp/archivepublisher/deathrow.py (+3/-4) lib/lp/archivepublisher/diskpool.py (+68/-72) lib/lp/archivepublisher/model/ftparchive.py (+2/-2) lib/lp/archivepublisher/tests/deathrow.txt (+4/-5) lib/lp/archivepublisher/tests/test_config.py (+9/-4) lib/lp/archivepublisher/tests/test_deathrow.py (+15/-15) lib/lp/archivepublisher/tests/test_ftparchive.py (+6/-10) lib/lp/archivepublisher/tests/test_pool.py (+11/-11) lib/lp/registry/model/distributionmirror.py (+1/-1) lib/lp/soyuz/model/publishing.py (+3/-3) lib/lp/soyuz/scripts/gina/handlers.py (+3/-2) lib/lp/soyuz/scripts/gina/packages.py (+2/-2) lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py (+1/-1) |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
Jürgen Gmach | Approve | ||
Review via email: mp+416755@code.launchpad.net |
Commit message
Refactor lp.archivepubli
Description of the change
The Artifactory bindings are based on `pathlib`, and so it makes it a little easier for my forthcoming Artifactory publication work to have a more `pathlib`-style implementation of `DiskPool` and `DiskPoolEntry`.
I don't think it makes sense to go all-in on `pathlib` while we're stuck on Python 3.5, because Python 3.6 improved much of the standard library to accept path-like objects as paths rather than just (byte) strings, and without that we have to add some awkward stringifications in several places. However, at this scale it doesn't work out too badly.
I also added a few type annotations, mainly to serve as documentation for the types I was changing. These are the first type annotations in Launchpad, and I haven't even tried to run `mypy` to check them yet. Similarly to the above, I don't think it makes sense to go all-in on type annotations until we're on at least Python 3.6 and can use variable annotations, but I saw no reason not to at least add these.
Colin Watson (cjwatson) wrote : | # |
Type annotations are going to be a bit imperfect here, partly because `FileAddActionEnum` is a slightly weird type (I'd like to turn that into a proper enum at some point so that its items can be type-checked as such rather than just being `str`), and partly because we don't yet have a way to check them so I may well have made mistakes. However, I've gone over this again and added what I can, including all the cases you pointed out.
Preview Diff
1 | diff --git a/lib/lp/archivepublisher/deathrow.py b/lib/lp/archivepublisher/deathrow.py |
2 | index b540c80..06e430a 100644 |
3 | --- a/lib/lp/archivepublisher/deathrow.py |
4 | +++ b/lib/lp/archivepublisher/deathrow.py |
5 | @@ -6,7 +6,6 @@ Processes removals of packages that are scheduled for deletion. |
6 | """ |
7 | |
8 | import datetime |
9 | -import os |
10 | |
11 | import pytz |
12 | from storm.expr import Exists |
13 | @@ -92,9 +91,9 @@ class DeathRow: |
14 | self.logger.debug("(Not really!) removing %s %s/%s" % |
15 | (cn, sn, fn)) |
16 | fullpath = self.diskpool.pathFor(cn, sn, fn) |
17 | - if not os.path.exists(fullpath): |
18 | + if not fullpath.exists(): |
19 | raise NotInPool |
20 | - return os.lstat(fullpath).st_size |
21 | + return fullpath.lstat().st_size |
22 | self._removeFile = _mockRemoveFile |
23 | |
24 | source_files, binary_files = self._collectCondemned() |
25 | @@ -232,7 +231,7 @@ class DeathRow: |
26 | pub_record.source_package_name, |
27 | pub_record.component_name, |
28 | ) |
29 | - file_path = self.diskpool.pathFor(*pub_file_details) |
30 | + file_path = str(self.diskpool.pathFor(*pub_file_details)) |
31 | |
32 | # Check if the LibraryFileAlias in question was already |
33 | # verified. If the verification was already made and the |
34 | diff --git a/lib/lp/archivepublisher/diskpool.py b/lib/lp/archivepublisher/diskpool.py |
35 | index 5232497..fe21aec 100644 |
36 | --- a/lib/lp/archivepublisher/diskpool.py |
37 | +++ b/lib/lp/archivepublisher/diskpool.py |
38 | @@ -3,8 +3,15 @@ |
39 | |
40 | __all__ = ['DiskPoolEntry', 'DiskPool', 'poolify', 'unpoolify'] |
41 | |
42 | +import logging |
43 | import os |
44 | +from pathlib import Path |
45 | import tempfile |
46 | +from typing import ( |
47 | + Optional, |
48 | + Tuple, |
49 | + Union, |
50 | + ) |
51 | |
52 | from lp.archivepublisher import HARDCODED_COMPONENT_ORDER |
53 | from lp.services.librarian.utils import ( |
54 | @@ -20,20 +27,20 @@ from lp.soyuz.interfaces.publishing import ( |
55 | ) |
56 | |
57 | |
58 | -def poolify(source, component): |
59 | +def poolify(source: str, component: str) -> Path: |
60 | """Poolify a given source and component name.""" |
61 | if source.startswith("lib"): |
62 | - return os.path.join(component, source[:4], source) |
63 | + return Path(component) / source[:4] / source |
64 | else: |
65 | - return os.path.join(component, source[:1], source) |
66 | + return Path(component) / source[:1] / source |
67 | |
68 | |
69 | -def unpoolify(self, path): |
70 | +def unpoolify(self, path: Path) -> Tuple[str, str, Optional[str]]: |
71 | """Take a path and unpoolify it. |
72 | |
73 | Return a tuple of component, source, filename |
74 | """ |
75 | - p = path.split("/") |
76 | + p = path.parts |
77 | if len(p) < 3 or len(p) > 4: |
78 | raise ValueError("Path %s is not in a valid pool form" % path) |
79 | if len(p) == 4: |
80 | @@ -41,22 +48,16 @@ def unpoolify(self, path): |
81 | return p[0], p[2], None |
82 | |
83 | |
84 | -def relative_symlink(src_path, dst_path): |
85 | - """os.symlink replacement that creates relative symbolic links.""" |
86 | - path_sep = os.path.sep |
87 | - src_path = os.path.normpath(src_path) |
88 | - dst_path = os.path.normpath(dst_path) |
89 | - src_path_elems = src_path.split(path_sep) |
90 | - dst_path_elems = dst_path.split(path_sep) |
91 | - if os.path.isabs(src_path): |
92 | - if not os.path.isabs(dst_path): |
93 | - dst_path = os.path.abspath(dst_path) |
94 | - common_prefix = os.path.commonprefix([src_path_elems, dst_path_elems]) |
95 | - backward_elems = ['..'] * ( |
96 | - len(dst_path_elems) - len(common_prefix) - 1) |
97 | - forward_elems = src_path_elems[len(common_prefix):] |
98 | - src_path = path_sep.join(backward_elems + forward_elems) |
99 | - os.symlink(src_path, dst_path) |
100 | +def relative_symlink(src_path: Path, dst_path: Path) -> None: |
101 | + """Path.symlink_to replacement that creates relative symbolic links.""" |
102 | + src_path = Path(os.path.normpath(str(src_path))) |
103 | + dst_path = Path(os.path.normpath(str(dst_path))) |
104 | + common_prefix = Path(os.path.commonpath([str(src_path), str(dst_path)])) |
105 | + backward_elems = [os.path.pardir] * ( |
106 | + len(dst_path.parts) - len(common_prefix.parts) - 1) |
107 | + forward_elems = src_path.parts[len(common_prefix.parts):] |
108 | + src_path = Path(*backward_elems, *forward_elems) |
109 | + dst_path.symlink_to(src_path) |
110 | |
111 | |
112 | class FileAddActionEnum: |
113 | @@ -87,7 +88,8 @@ class _diskpool_atomicfile: |
114 | the filename is present in the pool, it is definitely complete. |
115 | """ |
116 | |
117 | - def __init__(self, targetfilename, mode, rootpath="/tmp"): |
118 | + def __init__(self, targetfilename: Path, mode: str, |
119 | + rootpath: Union[str, Path] = "/tmp") -> None: |
120 | # atomicfile implements the file object interface, but it is only |
121 | # really used (or useful) for writing binary files, which is why we |
122 | # keep the mode constructor argument but assert it's sane below. |
123 | @@ -95,21 +97,21 @@ class _diskpool_atomicfile: |
124 | mode = "wb" |
125 | assert mode == "wb" |
126 | |
127 | - assert not os.path.exists(targetfilename) |
128 | + assert not targetfilename.exists() |
129 | |
130 | self.targetfilename = targetfilename |
131 | - fd, name = tempfile.mkstemp(prefix="temp-download.", dir=rootpath) |
132 | + fd, name = tempfile.mkstemp(prefix="temp-download.", dir=str(rootpath)) |
133 | self.fd = os.fdopen(fd, mode) |
134 | - self.tempname = name |
135 | + self.tempname = Path(name) |
136 | self.write = self.fd.write |
137 | |
138 | - def close(self): |
139 | + def close(self) -> None: |
140 | """Make the atomic move into place having closed the temp file.""" |
141 | self.fd.close() |
142 | - os.chmod(self.tempname, 0o644) |
143 | + self.tempname.chmod(0o644) |
144 | # Note that this will fail if the target and the temp dirs are on |
145 | # different filesystems. |
146 | - os.rename(self.tempname, self.targetfilename) |
147 | + self.tempname.rename(self.targetfilename) |
148 | |
149 | |
150 | class DiskPoolEntry: |
151 | @@ -126,7 +128,8 @@ class DiskPoolEntry: |
152 | Remaining files in the 'temppath' indicated installation failures and |
153 | require manual removal after further investigation. |
154 | """ |
155 | - def __init__(self, rootpath, temppath, source, filename, logger): |
156 | + def __init__(self, rootpath: Path, temppath: Path, source: str, |
157 | + filename: str, logger: logging.Logger) -> None: |
158 | self.rootpath = rootpath |
159 | self.temppath = temppath |
160 | self.source = source |
161 | @@ -138,24 +141,23 @@ class DiskPoolEntry: |
162 | |
163 | for component in HARDCODED_COMPONENT_ORDER: |
164 | path = self.pathFor(component) |
165 | - if os.path.islink(path): |
166 | + if path.is_symlink(): |
167 | self.symlink_components.add(component) |
168 | - elif os.path.isfile(path): |
169 | + elif path.is_file(): |
170 | assert not self.file_component |
171 | self.file_component = component |
172 | if self.symlink_components: |
173 | assert self.file_component |
174 | |
175 | - def debug(self, *args, **kwargs): |
176 | + def debug(self, *args, **kwargs) -> None: |
177 | self.logger.debug(*args, **kwargs) |
178 | |
179 | - def pathFor(self, component): |
180 | + def pathFor(self, component: str) -> Path: |
181 | """Return the path for this file in the given component.""" |
182 | - return os.path.join(self.rootpath, |
183 | - poolify(self.source, component), |
184 | - self.filename) |
185 | + return self.rootpath / poolify(self.source, component) / self.filename |
186 | |
187 | - def preferredComponent(self, add=None, remove=None): |
188 | + def preferredComponent(self, add: Optional[str] = None, |
189 | + remove: Optional[str] = None) -> Optional[str]: |
190 | """Return the appropriate component for the real file. |
191 | |
192 | If add is passed, add it to the list before calculating. |
193 | @@ -177,18 +179,17 @@ class DiskPoolEntry: |
194 | return component |
195 | |
196 | @cachedproperty |
197 | - def file_hash(self): |
198 | + def file_hash(self) -> str: |
199 | """Return the SHA1 sum of this file.""" |
200 | targetpath = self.pathFor(self.file_component) |
201 | - return sha1_from_path(targetpath) |
202 | + return sha1_from_path(str(targetpath)) |
203 | |
204 | - def addFile(self, component, pub_file: IPackageReleaseFile): |
205 | + def addFile(self, component: str, pub_file: IPackageReleaseFile): |
206 | """See DiskPool.addFile.""" |
207 | assert component in HARDCODED_COMPONENT_ORDER |
208 | |
209 | targetpath = self.pathFor(component) |
210 | - if not os.path.exists(os.path.dirname(targetpath)): |
211 | - os.makedirs(os.path.dirname(targetpath)) |
212 | + targetpath.parent.mkdir(parents=True, exist_ok=True) |
213 | lfa = pub_file.libraryfile |
214 | |
215 | if self.file_component: |
216 | @@ -215,7 +216,7 @@ class DiskPoolEntry: |
217 | return FileAddActionEnum.SYMLINK_ADDED |
218 | |
219 | # If we get to here, we want to write the file. |
220 | - assert not os.path.exists(targetpath) |
221 | + assert not targetpath.exists() |
222 | |
223 | self.debug("Making new file in %s for %s/%s" % |
224 | (component, self.source, self.filename)) |
225 | @@ -227,7 +228,7 @@ class DiskPoolEntry: |
226 | self.file_component = component |
227 | return FileAddActionEnum.FILE_ADDED |
228 | |
229 | - def removeFile(self, component): |
230 | + def removeFile(self, component: str) -> int: |
231 | """Remove a file from a given component; return bytes freed. |
232 | |
233 | This method handles three situations: |
234 | @@ -251,7 +252,7 @@ class DiskPoolEntry: |
235 | # ensure we are removing a symbolic link and |
236 | # it is published in one or more components |
237 | link_path = self.pathFor(component) |
238 | - assert os.path.islink(link_path) |
239 | + assert link_path.is_symlink() |
240 | return self._reallyRemove(component) |
241 | |
242 | if component != self.file_component: |
243 | @@ -274,14 +275,14 @@ class DiskPoolEntry: |
244 | |
245 | return self._reallyRemove(component) |
246 | |
247 | - def _reallyRemove(self, component): |
248 | + def _reallyRemove(self, component: str) -> int: |
249 | """Remove file and return file size. |
250 | |
251 | Remove the file from the filesystem and from our data |
252 | structures. |
253 | """ |
254 | fullpath = self.pathFor(component) |
255 | - assert os.path.exists(fullpath) |
256 | + assert fullpath.exists() |
257 | |
258 | if component == self.file_component: |
259 | # Deleting the master file is only allowed if there |
260 | @@ -291,11 +292,11 @@ class DiskPoolEntry: |
261 | elif component in self.symlink_components: |
262 | self.symlink_components.remove(component) |
263 | |
264 | - size = os.lstat(fullpath).st_size |
265 | - os.remove(fullpath) |
266 | + size = fullpath.lstat().st_size |
267 | + fullpath.unlink() |
268 | return size |
269 | |
270 | - def _shufflesymlinks(self, targetcomponent): |
271 | + def _shufflesymlinks(self, targetcomponent: str) -> None: |
272 | """Shuffle the symlinks for filename so that targetcomponent contains |
273 | the real file and the rest are symlinks to the right place...""" |
274 | if targetcomponent == self.file_component: |
275 | @@ -312,7 +313,7 @@ class DiskPoolEntry: |
276 | |
277 | # Okay, so first up, we unlink the targetcomponent symlink. |
278 | targetpath = self.pathFor(targetcomponent) |
279 | - os.remove(targetpath) |
280 | + targetpath.unlink() |
281 | |
282 | # Now we rename the source file into the target component. |
283 | sourcepath = self.pathFor(self.file_component) |
284 | @@ -326,9 +327,9 @@ class DiskPoolEntry: |
285 | |
286 | # ensure targetpath doesn't exists and the sourcepath exists |
287 | # before rename them. |
288 | - assert not os.path.exists(targetpath) |
289 | - assert os.path.exists(sourcepath) |
290 | - os.rename(sourcepath, targetpath) |
291 | + assert not targetpath.exists() |
292 | + assert sourcepath.exists() |
293 | + sourcepath.rename(targetpath) |
294 | |
295 | # XXX cprov 2006-06-12: it may cause problems to the database, since |
296 | # ZTM isn't handled properly in scripts/publish-distro.py. Things are |
297 | @@ -343,13 +344,13 @@ class DiskPoolEntry: |
298 | for comp in self.symlink_components: |
299 | newpath = self.pathFor(comp) |
300 | try: |
301 | - os.remove(newpath) |
302 | + newpath.unlink() |
303 | except OSError: |
304 | # Do nothing because it's almost certainly a not found. |
305 | pass |
306 | relative_symlink(targetpath, newpath) |
307 | |
308 | - def _sanitiseLinks(self): |
309 | + def _sanitiseLinks(self) -> None: |
310 | """Ensure the real file is in the most preferred component. |
311 | |
312 | If this file is in more than one component, ensure the real |
313 | @@ -378,24 +379,19 @@ class DiskPool: |
314 | """ |
315 | results = FileAddActionEnum |
316 | |
317 | - def __init__(self, rootpath, temppath, logger): |
318 | - self.rootpath = rootpath |
319 | - if not rootpath.endswith("/"): |
320 | - self.rootpath += "/" |
321 | - |
322 | - self.temppath = temppath |
323 | - if not temppath.endswith("/"): |
324 | - self.temppath += "/" |
325 | - |
326 | + def __init__(self, rootpath, temppath, logger: logging.Logger) -> None: |
327 | + self.rootpath = Path(rootpath) |
328 | + self.temppath = Path(temppath) |
329 | self.entries = {} |
330 | self.logger = logger |
331 | |
332 | - def _getEntry(self, sourcename, file): |
333 | + def _getEntry(self, sourcename: str, file: str) -> DiskPoolEntry: |
334 | """Return a new DiskPoolEntry for the given sourcename and file.""" |
335 | return DiskPoolEntry( |
336 | self.rootpath, self.temppath, sourcename, file, self.logger) |
337 | |
338 | - def pathFor(self, comp, source, file=None): |
339 | + def pathFor(self, comp: str, source: str, |
340 | + file: Optional[str] = None) -> Path: |
341 | """Return the path for the given pool folder or file. |
342 | |
343 | If file is none, the path to the folder containing all packages |
344 | @@ -404,13 +400,12 @@ class DiskPool: |
345 | If file is specified, the path to the specific package file will |
346 | be returned. |
347 | """ |
348 | - path = os.path.join( |
349 | - self.rootpath, poolify(source, comp)) |
350 | + path = self.rootpath / poolify(source, comp) |
351 | if file: |
352 | - return os.path.join(path, file) |
353 | + path = path / file |
354 | return path |
355 | |
356 | - def addFile(self, component, sourcename, filename, |
357 | + def addFile(self, component: str, sourcename: str, filename: str, |
358 | pub_file: IPackageReleaseFile): |
359 | """Add a file with the given contents to the pool. |
360 | |
361 | @@ -444,7 +439,8 @@ class DiskPool: |
362 | entry = self._getEntry(sourcename, filename) |
363 | return entry.addFile(component, pub_file) |
364 | |
365 | - def removeFile(self, component, sourcename, filename): |
366 | + def removeFile(self, component: str, sourcename: str, |
367 | + filename: str) -> int: |
368 | """Remove the specified file from the pool. |
369 | |
370 | There are three possible outcomes: |
371 | diff --git a/lib/lp/archivepublisher/model/ftparchive.py b/lib/lp/archivepublisher/model/ftparchive.py |
372 | index f5bfe02..1a5d8af 100644 |
373 | --- a/lib/lp/archivepublisher/model/ftparchive.py |
374 | +++ b/lib/lp/archivepublisher/model/ftparchive.py |
375 | @@ -693,8 +693,8 @@ class FTPArchiveHandler: |
376 | |
377 | def updateFileList(sourcepackagename, filename, component, |
378 | architecturetag=None): |
379 | - ondiskname = self._diskpool.pathFor( |
380 | - component, sourcepackagename, filename) |
381 | + ondiskname = str( |
382 | + self._diskpool.pathFor(component, sourcepackagename, filename)) |
383 | if architecturetag is None: |
384 | architecturetag = "source" |
385 | filelist[component][architecturetag].append(ondiskname) |
386 | diff --git a/lib/lp/archivepublisher/tests/deathrow.txt b/lib/lp/archivepublisher/tests/deathrow.txt |
387 | index 4d77654..c46474d 100644 |
388 | --- a/lib/lp/archivepublisher/tests/deathrow.txt |
389 | +++ b/lib/lp/archivepublisher/tests/deathrow.txt |
390 | @@ -211,18 +211,17 @@ Publish files on disk and build a list of all created file paths |
391 | ... unique_file_paths.add(file_path) |
392 | ... pub.publish(quiet_disk_pool, BufferLogger()) |
393 | |
394 | - >>> all_test_file_paths = sorted(unique_file_paths) |
395 | + >>> all_test_file_paths = sorted(unique_file_paths, key=str) |
396 | |
397 | Create a helper function to check if the publication files exist in |
398 | the temporary repository. |
399 | |
400 | - >>> import os |
401 | >>> def check_pool_files(): |
402 | ... for file_path in all_test_file_paths: |
403 | - ... if os.path.exists(file_path): |
404 | - ... print('%s: OK' % os.path.basename(file_path)) |
405 | + ... if file_path.exists(): |
406 | + ... print('%s: OK' % file_path.name) |
407 | ... else: |
408 | - ... print('%s: REMOVED' % os.path.basename(file_path)) |
409 | + ... print('%s: REMOVED' % file_path.name) |
410 | |
411 | >>> check_pool_files() |
412 | deleted-bin_666_i386.deb: OK |
413 | diff --git a/lib/lp/archivepublisher/tests/test_config.py b/lib/lp/archivepublisher/tests/test_config.py |
414 | index ec11bfa..915b4e7 100644 |
415 | --- a/lib/lp/archivepublisher/tests/test_config.py |
416 | +++ b/lib/lp/archivepublisher/tests/test_config.py |
417 | @@ -8,6 +8,7 @@ Publisher configuration provides archive-dependent filesystem paths. |
418 | |
419 | import logging |
420 | import os |
421 | +from pathlib import Path |
422 | |
423 | from zope.component import getUtility |
424 | |
425 | @@ -111,16 +112,20 @@ class TestGetPubConfig(TestCaseWithFactory): |
426 | def test_getDiskPool(self): |
427 | primary_config = getPubConfig(self.ubuntutest.main_archive) |
428 | disk_pool = primary_config.getDiskPool(BufferLogger()) |
429 | - self.assertEqual(self.root + "/ubuntutest/pool/", disk_pool.rootpath) |
430 | - self.assertEqual(self.root + "/ubuntutest-temp/", disk_pool.temppath) |
431 | + self.assertEqual( |
432 | + Path(self.root + "/ubuntutest/pool/"), disk_pool.rootpath) |
433 | + self.assertEqual( |
434 | + Path(self.root + "/ubuntutest-temp/"), disk_pool.temppath) |
435 | self.assertEqual(logging.INFO, disk_pool.logger.level) |
436 | |
437 | def test_getDiskPool_pool_root_override(self): |
438 | primary_config = getPubConfig(self.ubuntutest.main_archive) |
439 | disk_pool = primary_config.getDiskPool( |
440 | BufferLogger(), pool_root_override="/path/to/pool") |
441 | - self.assertEqual("/path/to/pool/", disk_pool.rootpath) |
442 | - self.assertEqual(self.root + "/ubuntutest-temp/", disk_pool.temppath) |
443 | + self.assertEqual( |
444 | + Path("/path/to/pool/"), disk_pool.rootpath) |
445 | + self.assertEqual( |
446 | + Path(self.root + "/ubuntutest-temp/"), disk_pool.temppath) |
447 | self.assertEqual(logging.INFO, disk_pool.logger.level) |
448 | |
449 | |
450 | diff --git a/lib/lp/archivepublisher/tests/test_deathrow.py b/lib/lp/archivepublisher/tests/test_deathrow.py |
451 | index c493805..81bf3dd 100644 |
452 | --- a/lib/lp/archivepublisher/tests/test_deathrow.py |
453 | +++ b/lib/lp/archivepublisher/tests/test_deathrow.py |
454 | @@ -3,7 +3,7 @@ |
455 | |
456 | """Tests for deathrow class.""" |
457 | |
458 | -import os |
459 | +from pathlib import Path |
460 | import shutil |
461 | import tempfile |
462 | |
463 | @@ -55,29 +55,29 @@ class TestDeathRow(TestCase): |
464 | pub.source_package_name, |
465 | pub_file.libraryfile.filename) |
466 | |
467 | - def assertIsFile(self, path): |
468 | + def assertIsFile(self, path: Path) -> None: |
469 | """Assert the path exists and is a regular file.""" |
470 | self.assertTrue( |
471 | - os.path.exists(path), |
472 | - "File %s does not exist" % os.path.basename(path)) |
473 | + path.exists(), |
474 | + "File %s does not exist" % path.name) |
475 | self.assertFalse( |
476 | - os.path.islink(path), |
477 | - "File %s is a symbolic link" % os.path.basename(path)) |
478 | + path.is_symlink(), |
479 | + "File %s is a symbolic link" % path.name) |
480 | |
481 | - def assertIsLink(self, path): |
482 | + def assertIsLink(self, path: Path) -> None: |
483 | """Assert the path exists and is a symbolic link.""" |
484 | self.assertTrue( |
485 | - os.path.exists(path), |
486 | - "File %s does not exist" % os.path.basename(path)) |
487 | + path.exists(), |
488 | + "File %s does not exist" % path.name) |
489 | self.assertTrue( |
490 | - os.path.islink(path), |
491 | - "File %s is a not symbolic link" % os.path.basename(path)) |
492 | + path.is_symlink(), |
493 | + "File %s is a not symbolic link" % path.name) |
494 | |
495 | - def assertDoesNotExist(self, path): |
496 | + def assertDoesNotExist(self, path: Path) -> None: |
497 | """Assert the path does not exit.""" |
498 | self.assertFalse( |
499 | - os.path.exists(path), |
500 | - "File %s exists" % os.path.basename(path)) |
501 | + path.exists(), |
502 | + "File %s exists" % path.name) |
503 | |
504 | def test_MissingSymLinkInPool(self): |
505 | # When a publication is promoted from 'universe' to 'main' and |
506 | @@ -118,7 +118,7 @@ class TestDeathRow(TestCase): |
507 | self.assertIsLink(universe_dsc_path) |
508 | |
509 | # Remove the symbolic link to emulate MissingSymlinkInPool scenario. |
510 | - os.remove(universe_dsc_path) |
511 | + universe_dsc_path.unlink() |
512 | |
513 | # Remove the testing publications. |
514 | for pub in test_publications: |
515 | diff --git a/lib/lp/archivepublisher/tests/test_ftparchive.py b/lib/lp/archivepublisher/tests/test_ftparchive.py |
516 | index 5a1219b..efe3e63 100755 |
517 | --- a/lib/lp/archivepublisher/tests/test_ftparchive.py |
518 | +++ b/lib/lp/archivepublisher/tests/test_ftparchive.py |
519 | @@ -7,6 +7,7 @@ import difflib |
520 | from functools import partial |
521 | import gzip |
522 | import os |
523 | +from pathlib import Path |
524 | import re |
525 | import shutil |
526 | from textwrap import dedent |
527 | @@ -149,16 +150,11 @@ class TestFTPArchive(TestCaseWithFactory): |
528 | samplename=None): |
529 | """Create a repository file.""" |
530 | fullpath = self._dp.pathFor(component, sourcename, leafname) |
531 | - dirname = os.path.dirname(fullpath) |
532 | - if not os.path.exists(dirname): |
533 | - os.makedirs(dirname) |
534 | + fullpath.parent.mkdir(parents=True, exist_ok=True) |
535 | if samplename is None: |
536 | samplename = leafname |
537 | - leaf = os.path.join(self._sampledir, samplename) |
538 | - with open(leaf, "rb") as leaf_file: |
539 | - leafcontent = leaf_file.read() |
540 | - with open(fullpath, "wb") as f: |
541 | - f.write(leafcontent) |
542 | + leaf = Path(self._sampledir) / samplename |
543 | + fullpath.write_bytes(leaf.read_bytes()) |
544 | |
545 | def _setUpFTPArchiveHandler(self): |
546 | return FTPArchiveHandler( |
547 | @@ -792,8 +788,8 @@ class TestFTPArchive(TestCaseWithFactory): |
548 | # Remove most of this repository's files so that cleanCaches has |
549 | # something to do. |
550 | for i in range(49): |
551 | - os.unlink( |
552 | - self._dp.pathFor("main", "bin%d" % i, "bin%d_1_i386.deb" % i)) |
553 | + self._dp.pathFor( |
554 | + "main", "bin%d" % i, "bin%d_1_i386.deb" % i).unlink() |
555 | |
556 | cache_path = os.path.join(self._config.cacheroot, "packages-i386.db") |
557 | old_cache_size = os.stat(cache_path).st_size |
558 | diff --git a/lib/lp/archivepublisher/tests/test_pool.py b/lib/lp/archivepublisher/tests/test_pool.py |
559 | index e6bc01c..ae65460 100644 |
560 | --- a/lib/lp/archivepublisher/tests/test_pool.py |
561 | +++ b/lib/lp/archivepublisher/tests/test_pool.py |
562 | @@ -4,7 +4,7 @@ |
563 | """Tests for pool.py.""" |
564 | |
565 | import hashlib |
566 | -import os |
567 | +from pathlib import Path |
568 | import shutil |
569 | from tempfile import mkdtemp |
570 | import unittest |
571 | @@ -55,23 +55,23 @@ class PoolTestingFile: |
572 | self.filename = filename |
573 | self.contents = sourcename.encode("UTF-8") |
574 | |
575 | - def addToPool(self, component): |
576 | + def addToPool(self, component: str): |
577 | return self.pool.addFile( |
578 | component, self.sourcename, self.filename, |
579 | FakePackageReleaseFile(self.contents)) |
580 | |
581 | - def removeFromPool(self, component): |
582 | + def removeFromPool(self, component: str) -> int: |
583 | return self.pool.removeFile(component, self.sourcename, self.filename) |
584 | |
585 | - def checkExists(self, component): |
586 | + def checkExists(self, component: str) -> bool: |
587 | path = self.pool.pathFor(component, self.sourcename, self.filename) |
588 | - return os.path.exists(path) |
589 | + return path.exists() |
590 | |
591 | - def checkIsLink(self, component): |
592 | + def checkIsLink(self, component: str) -> bool: |
593 | path = self.pool.pathFor(component, self.sourcename, self.filename) |
594 | - return os.path.islink(path) |
595 | + return path.is_symlink() |
596 | |
597 | - def checkIsFile(self, component): |
598 | + def checkIsFile(self, component: str) -> bool: |
599 | return self.checkExists(component) and not self.checkIsLink(component) |
600 | |
601 | |
602 | @@ -80,9 +80,9 @@ class TestPoolification(unittest.TestCase): |
603 | def testPoolificationOkay(self): |
604 | """poolify should poolify properly""" |
605 | cases = ( |
606 | - ("foo", "main", "main/f/foo"), |
607 | - ("foo", "universe", "universe/f/foo"), |
608 | - ("libfoo", "main", "main/libf/libfoo"), |
609 | + ("foo", "main", Path("main/f/foo")), |
610 | + ("foo", "universe", Path("universe/f/foo")), |
611 | + ("libfoo", "main", Path("main/libf/libfoo")), |
612 | ) |
613 | for case in cases: |
614 | self.assertEqual(case[2], poolify(case[0], case[1])) |
615 | diff --git a/lib/lp/registry/model/distributionmirror.py b/lib/lp/registry/model/distributionmirror.py |
616 | index 2454187..37b912b 100644 |
617 | --- a/lib/lp/registry/model/distributionmirror.py |
618 | +++ b/lib/lp/registry/model/distributionmirror.py |
619 | @@ -876,7 +876,7 @@ class MirrorDistroArchSeries(SQLBase, _MirrorSeriesMixIn): |
620 | """ |
621 | bpr = publishing_record.binarypackagerelease |
622 | base_url = self.distribution_mirror.base_url |
623 | - path = poolify(bpr.sourcepackagename, self.component.name) |
624 | + path = poolify(bpr.sourcepackagename, self.component.name).as_posix() |
625 | file = BinaryPackageFile.selectOneBy( |
626 | binarypackagerelease=bpr, filetype=BinaryPackageFileType.DEB) |
627 | full_path = 'pool/%s/%s' % (path, file.libraryfile.filename) |
628 | diff --git a/lib/lp/soyuz/model/publishing.py b/lib/lp/soyuz/model/publishing.py |
629 | index 810475d..500711d 100644 |
630 | --- a/lib/lp/soyuz/model/publishing.py |
631 | +++ b/lib/lp/soyuz/model/publishing.py |
632 | @@ -16,7 +16,7 @@ from operator import ( |
633 | attrgetter, |
634 | itemgetter, |
635 | ) |
636 | -import os |
637 | +from pathlib import Path |
638 | import sys |
639 | |
640 | import pytz |
641 | @@ -117,11 +117,11 @@ from lp.soyuz.model.section import Section |
642 | from lp.soyuz.model.sourcepackagerelease import SourcePackageRelease |
643 | |
644 | |
645 | -def makePoolPath(source_name, component_name): |
646 | +def makePoolPath(source_name: str, component_name: str) -> str: |
647 | # XXX cprov 2006-08-18: move it away, perhaps archivepublisher/pool.py |
648 | """Return the pool path for a given source name and component name.""" |
649 | from lp.archivepublisher.diskpool import poolify |
650 | - return os.path.join('pool', poolify(source_name, component_name)) |
651 | + return str(Path('pool') / poolify(source_name, component_name)) |
652 | |
653 | |
654 | def get_component(archive, distroseries, component): |
655 | diff --git a/lib/lp/soyuz/scripts/gina/handlers.py b/lib/lp/soyuz/scripts/gina/handlers.py |
656 | index 4a17e60..380637b 100644 |
657 | --- a/lib/lp/soyuz/scripts/gina/handlers.py |
658 | +++ b/lib/lp/soyuz/scripts/gina/handlers.py |
659 | @@ -20,6 +20,7 @@ __all__ = [ |
660 | |
661 | import io |
662 | import os |
663 | +from pathlib import Path |
664 | import re |
665 | |
666 | from storm.exceptions import NotOneError |
667 | @@ -507,8 +508,8 @@ class SourcePackageHandler: |
668 | |
669 | # Since the dsc doesn't know, we add in the directory, package |
670 | # component and section |
671 | - dsc_contents['directory'] = os.path.join("pool", |
672 | - poolify(sp_name, sp_component)).encode("ASCII") |
673 | + dsc_contents['directory'] = bytes( |
674 | + Path('pool') / poolify(sp_name, sp_component)) |
675 | dsc_contents['package'] = sp_name.encode("ASCII") |
676 | dsc_contents['component'] = sp_component.encode("ASCII") |
677 | dsc_contents['section'] = sp_section.encode("ASCII") |
678 | diff --git a/lib/lp/soyuz/scripts/gina/packages.py b/lib/lp/soyuz/scripts/gina/packages.py |
679 | index 15b4a45..9bda8ae 100644 |
680 | --- a/lib/lp/soyuz/scripts/gina/packages.py |
681 | +++ b/lib/lp/soyuz/scripts/gina/packages.py |
682 | @@ -82,7 +82,7 @@ def get_dsc_path(name, version, component, archive_root): |
683 | # We do a first attempt using the obvious directory name, composed |
684 | # with the component. However, this may fail if a binary is being |
685 | # published in another component. |
686 | - pool_dir = poolify(name, component) |
687 | + pool_dir = str(poolify(name, component)) |
688 | fullpath = os.path.join(pool_root, pool_dir, filename) |
689 | if os.path.exists(fullpath): |
690 | return filename, fullpath, component |
691 | @@ -91,7 +91,7 @@ def get_dsc_path(name, version, component, archive_root): |
692 | for alt_component_entry in os.scandir(pool_root): |
693 | if not alt_component_entry.is_dir(): |
694 | continue |
695 | - pool_dir = poolify(name, alt_component_entry.name) |
696 | + pool_dir = str(poolify(name, alt_component_entry.name)) |
697 | fullpath = os.path.join(pool_root, pool_dir, filename) |
698 | if os.path.exists(fullpath): |
699 | return filename, fullpath, alt_component_entry.name |
700 | diff --git a/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py b/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py |
701 | index d687a9d..867d495 100644 |
702 | --- a/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py |
703 | +++ b/lib/lp/soyuz/tests/test_binarypackagebuildbehaviour.py |
704 | @@ -279,7 +279,7 @@ class TestBinaryBuildPackageBehaviour(StatsMixin, TestCaseWithFactory): |
705 | % (archive.owner.name, archive.name, |
706 | poolify( |
707 | build.source_package_release.sourcepackagename.name, |
708 | - 'main'), |
709 | + 'main').as_posix(), |
710 | sprf.libraryfile.filename)) |
711 | lf = self.factory.makeLibraryFileAlias(db_only=True) |
712 | build.distro_arch_series.addOrUpdateChroot(lf) |
Looks good!
Even on 3.5 Pathlib really looks like a massive improvement in terms of readability.
I suggested to add full type annotations to all methods/functions where either the function signature or the return value was in some way touched to prevent one certain class for error when we will start linting with `mypy`.
Feel free to ignore them if you cannot find the time to add them.