Merge lp:~cjwatson/txpkgupload/drop-distro-files into lp:~lazr-developers/txpkgupload/trunk
- drop-distro-files
- Merge into trunk
Proposed by
Colin Watson
Status: | Merged | ||||||||
---|---|---|---|---|---|---|---|---|---|
Merged at revision: | 19 | ||||||||
Proposed branch: | lp:~cjwatson/txpkgupload/drop-distro-files | ||||||||
Merge into: | lp:~lazr-developers/txpkgupload/trunk | ||||||||
Diff against target: |
826 lines (+95/-462) 8 files modified
etc/txpkgupload.yaml (+5/-1) src/txpkgupload/glock.py (+0/-316) src/txpkgupload/hooks.py (+23/-106) src/txpkgupload/plugin.py (+22/-6) src/txpkgupload/tests/test_plugin.py (+23/-12) src/txpkgupload/tests/test_twistedsftp.py (+8/-6) src/txpkgupload/twistedftp.py (+11/-11) src/txpkgupload/twistedsftp.py (+3/-4) |
||||||||
To merge this branch: | bzr merge lp:~cjwatson/txpkgupload/drop-distro-files | ||||||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
William Grant | code | Approve | |
Review via email:
|
Commit message
Institute a temporary directory on the same filesystem as the upload root; stop creating obsolete .distro files; remove locking, since we now move directories into place atomically.
Description of the change
Institute a temporary directory on the same filesystem as the upload root; stop creating obsolete .distro files; remove locking, since we now move directories into place atomically.
Before deploying this, we need to make sure that the tmp-incoming directory is created with appropriate permissions, and it wouldn't hurt to land an lp-production-
To post a comment you must log in.
Revision history for this message
![](/+icing/build/overlay/assets/skins/sam/images/close.gif)
William Grant (wgrant) : | # |
review:
Approve
(code)
Preview Diff
[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1 | === modified file 'etc/txpkgupload.yaml' |
2 | --- etc/txpkgupload.yaml 2015-01-12 13:14:55 +0000 |
3 | +++ etc/txpkgupload.yaml 2015-01-21 13:10:45 +0000 |
4 | @@ -41,7 +41,11 @@ |
5 | # idle_timeout: 3600 |
6 | |
7 | ## Where on the filesystem do uploads live? |
8 | -# fsroot: "/var/tmp/txpkgupload" |
9 | +# fsroot: "/var/tmp/txpkgupload/incoming" |
10 | + |
11 | +## Where do we write temporary files during uploads? This must be on the |
12 | +## same filesystem as fsroot. |
13 | +# temp_dir: "/var/tmp/txpkgupload/tmp-incoming" |
14 | |
15 | # If true, enable additional debug logging. |
16 | # debug: false |
17 | |
18 | === removed file 'src/txpkgupload/glock.py' |
19 | --- src/txpkgupload/glock.py 2015-01-07 14:20:13 +0000 |
20 | +++ src/txpkgupload/glock.py 1970-01-01 00:00:00 +0000 |
21 | @@ -1,316 +0,0 @@ |
22 | -#!/usr/bin/env python |
23 | -# -*- coding: latin1 -*- |
24 | -#---------------------------------------------------------------------------- |
25 | -# glock.py: Global mutex |
26 | -# |
27 | -# See __doc__ string below. |
28 | -# |
29 | -# Requires: |
30 | -# - Python 1.5.2 or newer (www.python.org) |
31 | -# - On windows: win32 extensions installed |
32 | -# (http://www.python.org/windows/win32all/win32all.exe) |
33 | -# - OS: Unix, Windows. |
34 | -# |
35 | -# $Id: //depot/rgutils/rgutils/glock.py#5 $ |
36 | -#---------------------------------------------------------------------------- |
37 | -''' |
38 | -This module defines the class GlobalLock that implements a global |
39 | -(inter-process) mutex on Windows and Unix, using file-locking on |
40 | -Unix. |
41 | - |
42 | -@see: class L{GlobalLock} for more details. |
43 | -''' |
44 | -__version__ = '0.2.' + '$Revision: #5 $'[12:-2] |
45 | -__author__ = 'Richard Gruet', 'rjgruet@yahoo.com' |
46 | -__date__ = '$Date: 2005/06/19 $'[7:-2], '$Author: rgruet $'[9:-2] |
47 | -__since__ = '2000-01-22' |
48 | -__doc__ += '\n@author: %s (U{%s})\n@version: %s' % (__author__[0], |
49 | - __author__[1], __version__) |
50 | -__all__ = ['GlobalLock', 'GlobalLockError', 'LockAlreadyAcquired', 'NotOwner'] |
51 | - |
52 | -# Imports: |
53 | -import sys, string, os, errno, re |
54 | - |
55 | -# System-dependent imports for locking implementation: |
56 | -_windows = (sys.platform == 'win32') |
57 | - |
58 | -if _windows: |
59 | - try: |
60 | - import win32event, win32api, pywintypes |
61 | - except ImportError: |
62 | - sys.stderr.write('The win32 extensions need to be installed!') |
63 | - try: |
64 | - import ctypes |
65 | - except ImportError: |
66 | - ctypes = None |
67 | -else: # assume Unix |
68 | - try: |
69 | - import fcntl |
70 | - except ImportError: |
71 | - sys.stderr.write("On what kind of OS am I ? (Mac?) I should be on " |
72 | - "Unix but can't import fcntl.\n") |
73 | - raise |
74 | - import threading |
75 | - |
76 | -# Exceptions : |
77 | -# ---------- |
78 | -class GlobalLockError(Exception): |
79 | - ''' Error raised by the glock module. |
80 | - ''' |
81 | - pass |
82 | - |
83 | -class NotOwner(GlobalLockError): |
84 | - ''' Attempt to release somebody else's lock. |
85 | - ''' |
86 | - pass |
87 | - |
88 | -class LockAlreadyAcquired(GlobalLockError): |
89 | - ''' Non-blocking acquire but lock already seized. |
90 | - ''' |
91 | - pass |
92 | - |
93 | - |
94 | -# Constants |
95 | -# ---------: |
96 | -if sys.version[:3] < '2.2': |
97 | - True, False = 1, 0 # built-in in Python 2.2+ |
98 | - |
99 | -#---------------------------------------------------------------------------- |
100 | -class GlobalLock: |
101 | -#---------------------------------------------------------------------------- |
102 | - ''' A global mutex. |
103 | - |
104 | - B{Specification} |
105 | - |
106 | - - The lock must act as a global mutex, ie block between different |
107 | - candidate processus, but ALSO between different candidate |
108 | - threads of the same process. |
109 | - |
110 | - - It must NOT block in case of reentrant lock request issued by |
111 | - the SAME thread. |
112 | - - Extraneous unlocks should be ideally harmless. |
113 | - |
114 | - B{Implementation} |
115 | - |
116 | - In Python there is no portable global lock AFAIK. There is only a |
117 | - LOCAL/ in-process Lock mechanism (threading.RLock), so we have to |
118 | - implement our own solution: |
119 | - |
120 | - - Unix: use fcntl.flock(). Recursive calls OK. Different process OK. |
121 | - But <> threads, same process don't block so we have to use an extra |
122 | - threading.RLock to fix that point. |
123 | - - Windows: We use WIN32 mutex from Python Win32 extensions. Can't use |
124 | - std module msvcrt.locking(), because global lock is OK, but |
125 | - blocks also for 2 calls from the same thread! |
126 | - ''' |
127 | - RE_ERROR_MSG = re.compile ("^\[Errno ([0-9]+)\]") |
128 | - |
129 | - def __init__(self, fpath, lockInitially=False, logger=None): |
130 | - ''' Creates (or opens) a global lock. |
131 | - |
132 | - @param fpath: Path of the file used as lock target. This is also |
133 | - the global id of the lock. The file will be created |
134 | - if non existent. |
135 | - @param lockInitially: if True locks initially. |
136 | - @param logger: an optional logger object. |
137 | - ''' |
138 | - self.logger = logger |
139 | - self.fpath = fpath |
140 | - if os.path.exists(fpath): |
141 | - self.previous_lockfile_present = True |
142 | - else: |
143 | - self.previous_lockfile_present = False |
144 | - if _windows: |
145 | - self.name = string.replace(fpath, '\\', '_') |
146 | - self.mutex = win32event.CreateMutex(None, lockInitially, self.name) |
147 | - else: # Unix |
148 | - self.name = fpath |
149 | - self.flock = open(fpath, 'w') |
150 | - self.fdlock = self.flock.fileno() |
151 | - self.threadLock = threading.RLock() |
152 | - if lockInitially: |
153 | - self.acquire() |
154 | - |
155 | - def __del__(self): |
156 | - #print '__del__ called' ## |
157 | - try: self.release() |
158 | - except: pass |
159 | - if _windows: |
160 | - win32api.CloseHandle(self.mutex) |
161 | - else: |
162 | - try: self.flock.close() |
163 | - except: pass |
164 | - |
165 | - def __repr__(self): |
166 | - return '<Global lock @ %s>' % self.name |
167 | - |
168 | - def acquire(self, blocking=False): |
169 | - """ Locks. Attemps to acquire a lock. |
170 | - |
171 | - @param blocking: If True, suspends caller until done. Otherwise, |
172 | - LockAlreadyAcquired is raised if the lock cannot be acquired immediately. |
173 | - |
174 | - On windows an IOError is always raised after ~10 sec if the lock |
175 | - can't be acquired. |
176 | - @exception GlobalLockError: if lock can't be acquired (timeout) |
177 | - @exception LockAlreadyAcquired: someone already has the lock and |
178 | - the caller decided not to block. |
179 | - """ |
180 | - if self.logger: |
181 | - self.logger.info('Creating lockfile: %s', self.fpath) |
182 | - if _windows: |
183 | - if blocking: |
184 | - timeout = win32event.INFINITE |
185 | - else: |
186 | - timeout = 0 |
187 | - r = win32event.WaitForSingleObject(self.mutex, timeout) |
188 | - if r == win32event.WAIT_FAILED: |
189 | - raise GlobalLockError("Can't acquire mutex: error") |
190 | - if not blocking and r == win32event.WAIT_TIMEOUT: |
191 | - raise LockAlreadyAcquired('Lock %s already acquired by ' |
192 | - 'someone else' % self.name) |
193 | - else: |
194 | - # First, acquire the global (inter-process) lock: |
195 | - if blocking: |
196 | - options = fcntl.LOCK_EX |
197 | - else: |
198 | - options = fcntl.LOCK_EX|fcntl.LOCK_NB |
199 | - try: |
200 | - fcntl.flock(self.fdlock, options) |
201 | - except IOError as message: #(errno 13: perm. denied, |
202 | - # 36: Resource deadlock avoided) |
203 | - if not blocking and self._errnoOf (message) == errno.EWOULDBLOCK: |
204 | - raise LockAlreadyAcquired('Lock %s already acquired by ' |
205 | - 'someone else' % self.name) |
206 | - else: |
207 | - raise GlobalLockError('Cannot acquire lock on "file" ' |
208 | - '%s: %s\n' % (self.name, message)) |
209 | - #print 'got file lock.' ## |
210 | - |
211 | - # Then acquire the local (inter-thread) lock: |
212 | - if not self.threadLock.acquire(blocking): |
213 | - fcntl.flock(self.fdlock, fcntl.LOCK_UN) # release global lock |
214 | - raise LockAlreadyAcquired('Lock %s already acquired by ' |
215 | - 'someone else' % self.name) |
216 | - if self.previous_lockfile_present and self.logger: |
217 | - self.logger.warn("Stale lockfile detected and claimed.") |
218 | - #print 'got thread lock.' ## |
219 | - |
220 | - self.is_locked = True |
221 | - |
222 | - def release(self, skip_delete=False): |
223 | - ''' Unlocks. (caller must own the lock!) |
224 | - |
225 | - @param skip_delete: don't try to delete the file. This can |
226 | - be used when the original filename has changed; for |
227 | - instance, if the lockfile is erased out-of-band, or if |
228 | - the directory it contains has been renamed. |
229 | - |
230 | - @return: The lock count. |
231 | - @exception IOError: if file lock can't be released |
232 | - @exception NotOwner: Attempt to release somebody else's lock. |
233 | - ''' |
234 | - if not self.is_locked: |
235 | - return |
236 | - if not skip_delete: |
237 | - if self.logger: |
238 | - self.logger.debug('Removing lock file: %s', self.fpath) |
239 | - os.unlink(self.fpath) |
240 | - elif self.logger: |
241 | - # At certain times the lockfile will have been removed or |
242 | - # moved away before we call release(); log a message because |
243 | - # this is unusual and could be an error. |
244 | - self.logger.debug('Oops, my lock file disappeared: %s', self.fpath) |
245 | - if _windows: |
246 | - if ctypes: |
247 | - result = ctypes.windll.kernel32.ReleaseMutex(self.mutex.handle) |
248 | - if not result: |
249 | - raise NotOwner("Attempt to release somebody else's lock") |
250 | - else: |
251 | - try: |
252 | - win32event.ReleaseMutex(self.mutex) |
253 | - #print "released mutex" |
254 | - except pywintypes.error as e: |
255 | - errCode, fctName, errMsg = e.args |
256 | - if errCode == 288: |
257 | - raise NotOwner("Attempt to release somebody else's lock") |
258 | - else: |
259 | - raise GlobalLockError('%s: err#%d: %s' % (fctName, errCode, |
260 | - errMsg)) |
261 | - else: |
262 | - # First release the local (inter-thread) lock: |
263 | - try: |
264 | - self.threadLock.release() |
265 | - except AssertionError: |
266 | - raise NotOwner("Attempt to release somebody else's lock") |
267 | - |
268 | - # Then release the global (inter-process) lock: |
269 | - try: |
270 | - fcntl.flock(self.fdlock, fcntl.LOCK_UN) |
271 | - except IOError: # (errno 13: permission denied) |
272 | - raise GlobalLockError('Unlock of file "%s" failed\n' % |
273 | - self.name) |
274 | - self.is_locked = False |
275 | - |
276 | - def _errnoOf (self, message): |
277 | - match = self.RE_ERROR_MSG.search(str(message)) |
278 | - if match: |
279 | - return int(match.group(1)) |
280 | - else: |
281 | - raise Exception ('Malformed error message "%s"' % message) |
282 | - |
283 | -#---------------------------------------------------------------------------- |
284 | -def test(): |
285 | -#---------------------------------------------------------------------------- |
286 | - ##TODO: a more serious test with distinct processes ! |
287 | - |
288 | - print 'Testing glock.py...' |
289 | - |
290 | - # unfortunately can't test inter-process lock here! |
291 | - lockName = 'myFirstLock' |
292 | - l = GlobalLock(lockName) |
293 | - if not _windows: |
294 | - assert os.path.exists(lockName) |
295 | - l.acquire() |
296 | - l.acquire() # reentrant lock, must not block |
297 | - l.release() |
298 | - l.release() |
299 | - |
300 | - try: l.release() |
301 | - except NotOwner: pass |
302 | - else: raise Exception('should have raised a NotOwner exception') |
303 | - |
304 | - # Check that <> threads of same process do block: |
305 | - import threading, time |
306 | - thread = threading.Thread(target=threadMain, args=(l,)) |
307 | - print 'main: locking...', |
308 | - l.acquire() |
309 | - print ' done.' |
310 | - thread.start() |
311 | - time.sleep(3) |
312 | - print '\nmain: unlocking...', |
313 | - l.release() |
314 | - print ' done.' |
315 | - time.sleep(0.1) |
316 | - |
317 | - print '=> Test of glock.py passed.' |
318 | - return l |
319 | - |
320 | -def threadMain(lock): |
321 | - print 'thread started(%s).' % lock |
322 | - try: lock.acquire(blocking=False) |
323 | - except LockAlreadyAcquired: pass |
324 | - else: raise Exception('should have raised LockAlreadyAcquired') |
325 | - print 'thread: locking (should stay blocked for ~ 3 sec)...', |
326 | - lock.acquire() |
327 | - print 'thread: locking done.' |
328 | - print 'thread: unlocking...', |
329 | - lock.release() |
330 | - print ' done.' |
331 | - print 'thread ended.' |
332 | - |
333 | -#---------------------------------------------------------------------------- |
334 | -# M A I N |
335 | -#---------------------------------------------------------------------------- |
336 | -if __name__ == "__main__": |
337 | - l = test() |
338 | |
339 | === modified file 'src/txpkgupload/hooks.py' |
340 | --- src/txpkgupload/hooks.py 2015-01-09 15:55:08 +0000 |
341 | +++ src/txpkgupload/hooks.py 2015-01-21 13:10:45 +0000 |
342 | @@ -10,14 +10,10 @@ |
343 | |
344 | |
345 | import os |
346 | -import shutil |
347 | -import stat |
348 | import time |
349 | |
350 | from twisted.python import log |
351 | |
352 | -from txpkgupload.glock import GlobalLock |
353 | - |
354 | |
355 | class InterfaceFailure(Exception): |
356 | pass |
357 | @@ -29,11 +25,8 @@ |
358 | LOG_MAGIC = "Post-processing finished" |
359 | _targetcount = 0 |
360 | |
361 | - def __init__(self, targetpath, allow_user, cmd=None, |
362 | - targetstart=0, perms=None, prefix=''): |
363 | + def __init__(self, targetpath, targetstart=0, perms=None, prefix=''): |
364 | self.targetpath = targetpath |
365 | - self.cmd = cmd |
366 | - self.allow_user = allow_user |
367 | self.perms = perms |
368 | self.prefix = prefix |
369 | |
370 | @@ -53,9 +46,7 @@ |
371 | log.msg("Session from %s:%s" % (host, port), debug=True) |
372 | |
373 | def client_done_hook(self, fsroot, host, port): |
374 | - """A client has completed. If it authenticated then it stands a chance |
375 | - of having uploaded a file to the set. If not; then it is simply an |
376 | - aborted transaction and we remove the fsroot.""" |
377 | + """A client has completed.""" |
378 | |
379 | if fsroot not in self.clients: |
380 | raise InterfaceFailure("Unable to find fsroot in client set") |
381 | @@ -63,103 +54,29 @@ |
382 | log.msg("Processing session complete in %s" % fsroot, debug=True) |
383 | |
384 | client = self.clients[fsroot] |
385 | - if "distro" not in client: |
386 | - # Login username defines the distribution context of the upload. |
387 | - # So abort unauthenticated sessions by removing its contents |
388 | - shutil.rmtree(fsroot) |
389 | - return |
390 | - |
391 | - # Protect from race condition between creating the directory |
392 | - # and creating the distro file, and also in cases where the |
393 | - # temporary directory and the upload directory are not in the |
394 | - # same filesystem (non-atomic "rename"). |
395 | - lockfile_path = os.path.join(self.targetpath, ".lock") |
396 | - self.lock = GlobalLock(lockfile_path) |
397 | - |
398 | - # XXX cprov 20071024 bug=156795: We try to acquire the lock as soon |
399 | - # as possible after creating the lockfile but are still open to |
400 | - # a race. |
401 | - self.lock.acquire(blocking=True) |
402 | - mode = stat.S_IMODE(os.stat(lockfile_path).st_mode) |
403 | - |
404 | - # XXX cprov 20081024 bug=185731: The lockfile permission can only be |
405 | - # changed by its owner. Since we can't predict which process will |
406 | - # create it in production systems we simply ignore errors when trying |
407 | - # to grant the right permission. At least, one of the process will |
408 | - # be able to do so. |
409 | - try: |
410 | - os.chmod(lockfile_path, mode | stat.S_IWGRP) |
411 | - except OSError: |
412 | - pass |
413 | - |
414 | - try: |
415 | - timestamp = time.strftime("%Y%m%d-%H%M%S") |
416 | - path = "upload%s-%s-%06d" % ( |
417 | - self.prefix, timestamp, self.targetcount) |
418 | - target_fsroot = os.path.join(self.targetpath, path) |
419 | - |
420 | - # Create file to store the distro used. |
421 | - log.msg( |
422 | - "Upload was targetted at %s" % client["distro"], debug=True) |
423 | - distro_filename = target_fsroot + ".distro" |
424 | - distro_file = open(distro_filename, "w") |
425 | - distro_file.write(client["distro"]) |
426 | - distro_file.close() |
427 | - |
428 | - # Move the session directory to the target directory. |
429 | - if os.path.exists(target_fsroot): |
430 | - log.msg("Targeted upload already present: %s" % path) |
431 | - log.msg("System clock skewed ?") |
432 | - else: |
433 | - try: |
434 | - shutil.move(fsroot, target_fsroot) |
435 | - except (OSError, IOError): |
436 | - if not os.path.exists(target_fsroot): |
437 | - raise |
438 | - |
439 | - # XXX cprov 20071024: We should replace os.system call by os.chmod |
440 | - # and fix the default permission value accordingly in txpkgupload |
441 | - if self.perms is not None: |
442 | - os.system("chmod %s -R %s" % (self.perms, target_fsroot)) |
443 | - |
444 | - # Invoke processing script, if provided. |
445 | - if self.cmd: |
446 | - cmd = self.cmd |
447 | - cmd = cmd.replace("@fsroot@", target_fsroot) |
448 | - cmd = cmd.replace("@distro@", client["distro"]) |
449 | - log.msg("Running upload handler: %s" % cmd, debug=True) |
450 | - os.system(cmd) |
451 | - finally: |
452 | - # We never delete the lockfile, this way the inode will be |
453 | - # constant while the machine is up. See comment on 'acquire' |
454 | - self.lock.release(skip_delete=True) |
455 | + |
456 | + timestamp = time.strftime("%Y%m%d-%H%M%S") |
457 | + path = "upload%s-%s-%06d" % ( |
458 | + self.prefix, timestamp, self.targetcount) |
459 | + target_fsroot = os.path.join(self.targetpath, path) |
460 | + |
461 | + # Move the session directory to the target directory. |
462 | + if os.path.exists(target_fsroot): |
463 | + log.msg("Targeted upload already present: %s" % path) |
464 | + log.msg("System clock skewed?") |
465 | + else: |
466 | + try: |
467 | + os.rename(fsroot, target_fsroot) |
468 | + except (OSError, IOError): |
469 | + if not os.path.exists(target_fsroot): |
470 | + raise |
471 | + |
472 | + # XXX cprov 20071024: We should replace os.system call by os.chmod |
473 | + # and fix the default permission value accordingly in txpkgupload |
474 | + if self.perms is not None: |
475 | + os.system("chmod %s -R %s" % (self.perms, target_fsroot)) |
476 | |
477 | self.clients.pop(fsroot) |
478 | # This is mainly done so that tests know when the |
479 | # post-processing hook has finished. |
480 | log.msg(self.LOG_MAGIC) |
481 | - |
482 | - def auth_verify_hook(self, fsroot, user, password): |
483 | - """Verify that the username matches a distribution we care about. |
484 | - |
485 | - The password is irrelevant to auth, as is the fsroot""" |
486 | - if fsroot not in self.clients: |
487 | - raise InterfaceFailure("Unable to find fsroot in client set") |
488 | - |
489 | - # local authentication |
490 | - self.clients[fsroot]["distro"] = self.allow_user |
491 | - return True |
492 | - |
493 | - # When we get on with the txpkgupload path stuff, the below may be |
494 | - # useful and is thus left in rather than being removed. |
495 | - |
496 | - #try: |
497 | - # d = Distribution.byName(user) |
498 | - # if d: |
499 | - # log.msg("Accepting login for %s" % user, debug=True) |
500 | - # self.clients[fsroot]["distro"] = user |
501 | - # return True |
502 | - #except object as e: |
503 | - # print e |
504 | - #return False |
505 | - |
506 | |
507 | === modified file 'src/txpkgupload/plugin.py' |
508 | --- src/txpkgupload/plugin.py 2015-01-12 18:50:14 +0000 |
509 | +++ src/txpkgupload/plugin.py 2015-01-21 13:10:45 +0000 |
510 | @@ -7,6 +7,7 @@ |
511 | ] |
512 | |
513 | from functools import partial |
514 | +import os |
515 | |
516 | from formencode import Schema |
517 | from formencode.api import set_stdtranslation |
518 | @@ -119,6 +120,10 @@ |
519 | # Where on the filesystem do uploads live? |
520 | fsroot = String(if_missing=None) |
521 | |
522 | + # Where do we write temporary files during uploads? This must be on the |
523 | + # same filesystem as fsroot. |
524 | + temp_dir = String(if_missing=None) |
525 | + |
526 | @classmethod |
527 | def parse(cls, stream): |
528 | """Load a YAML configuration from `stream` and validate.""" |
529 | @@ -143,19 +148,22 @@ |
530 | """An SSH avatar specific to txpkgupload. |
531 | |
532 | :ivar fs_root: The file system root for this session. |
533 | + :ivar temp_dir: The temporary directory for this session. |
534 | """ |
535 | |
536 | - def __init__(self, user_dict, fs_root): |
537 | + def __init__(self, user_dict, fs_root, temp_dir): |
538 | LaunchpadAvatar.__init__(self, user_dict) |
539 | self.fs_root = fs_root |
540 | + self.temp_dir = temp_dir |
541 | |
542 | |
543 | class Realm: |
544 | implements(IRealm) |
545 | |
546 | - def __init__(self, authentication_proxy, fs_root): |
547 | + def __init__(self, authentication_proxy, fs_root, temp_dir): |
548 | self.authentication_proxy = authentication_proxy |
549 | self.fs_root = fs_root |
550 | + self.temp_dir = temp_dir |
551 | |
552 | def requestAvatar(self, avatar_id, mind, *interfaces): |
553 | # Fetch the user's details from the authserver |
554 | @@ -164,20 +172,20 @@ |
555 | |
556 | # Once all those details are retrieved, we can construct the avatar. |
557 | def got_user_dict(user_dict): |
558 | - avatar = PkgUploadAvatar(user_dict, self.fs_root) |
559 | + avatar = PkgUploadAvatar(user_dict, self.fs_root, self.temp_dir) |
560 | return interfaces[0], avatar, avatar.logout |
561 | |
562 | return deferred.addCallback(got_user_dict) |
563 | |
564 | |
565 | -def make_portal(authentication_endpoint, fs_root): |
566 | +def make_portal(authentication_endpoint, fs_root, temp_dir): |
567 | """Create and return a `Portal` for the SSH service. |
568 | |
569 | This portal accepts SSH credentials and returns our customized SSH |
570 | avatars (see `LaunchpadAvatar`). |
571 | """ |
572 | authentication_proxy = Proxy(authentication_endpoint) |
573 | - portal = Portal(Realm(authentication_proxy, fs_root)) |
574 | + portal = Portal(Realm(authentication_proxy, fs_root, temp_dir)) |
575 | portal.registerChecker( |
576 | PublicKeyFromLaunchpadChecker(authentication_proxy)) |
577 | return portal |
578 | @@ -212,18 +220,26 @@ |
579 | oops_dir, oops_reporter, server_argv=server_argv) |
580 | |
581 | root = get_txpkgupload_root(config["fsroot"]) |
582 | + temp_dir = config["temp_dir"] |
583 | + if temp_dir is None: |
584 | + temp_dir = os.path.abspath(os.path.join( |
585 | + root, os.pardir, "tmp-incoming")) |
586 | + if not os.path.exists(temp_dir): |
587 | + os.makedirs(temp_dir, 0775) |
588 | |
589 | ftp_config = config["ftp"] |
590 | ftp_service = FTPServiceFactory.makeFTPService( |
591 | port=ftp_config["port"], |
592 | root=root, |
593 | + temp_dir=temp_dir, |
594 | idle_timeout=config["idle_timeout"]) |
595 | ftp_service.name = "ftp" |
596 | ftp_service.setServiceParent(services) |
597 | |
598 | sftp_config = config["sftp"] |
599 | sftp_service = SSHService( |
600 | - portal=make_portal(sftp_config["authentication_endpoint"], root), |
601 | + portal=make_portal( |
602 | + sftp_config["authentication_endpoint"], root, temp_dir), |
603 | private_key_path=sftp_config["host_key_private"], |
604 | public_key_path=sftp_config["host_key_public"], |
605 | main_log='txpkgupload', |
606 | |
607 | === modified file 'src/txpkgupload/tests/test_plugin.py' |
608 | --- src/txpkgupload/tests/test_plugin.py 2015-01-12 13:14:55 +0000 |
609 | +++ src/txpkgupload/tests/test_plugin.py 2015-01-21 13:10:45 +0000 |
610 | @@ -115,6 +115,7 @@ |
611 | "host_key_public": None, |
612 | "port": "tcp:5022", |
613 | }, |
614 | + "temp_dir": None, |
615 | } |
616 | observed = Config.to_python({}) |
617 | self.assertEqual(expected, observed) |
618 | @@ -262,6 +263,8 @@ |
619 | |
620 | def __init__(self, root_dir): |
621 | self.root_dir = root_dir |
622 | + self.fsroot = os.path.join(self.root_dir, "incoming") |
623 | + self.temp_dir = os.path.join(self.root_dir, "tmp-incoming") |
624 | top = os.path.join( |
625 | os.path.dirname(__file__), os.pardir, os.pardir, os.pardir) |
626 | with open(os.path.join(top, "etc", "txpkgupload.yaml")) as stream: |
627 | @@ -270,8 +273,11 @@ |
628 | |
629 | def setUp(self): |
630 | super(FTPServer, self).setUp() |
631 | - self.pkgupload = self.useFixture(PkgUploadFixture( |
632 | - "fsroot: %s" % self.root_dir)) |
633 | + os.mkdir(self.fsroot) |
634 | + os.mkdir(self.temp_dir) |
635 | + self.pkgupload = self.useFixture(PkgUploadFixture(dedent("""\ |
636 | + fsroot: %s |
637 | + temp_dir: %s""") % (self.fsroot, self.temp_dir))) |
638 | |
639 | def getAnonClient(self): |
640 | creator = ClientCreator( |
641 | @@ -368,6 +374,8 @@ |
642 | |
643 | def __init__(self, root_dir): |
644 | self.root_dir = root_dir |
645 | + self.fsroot = os.path.join(self.root_dir, "incoming") |
646 | + self.temp_dir = os.path.join(self.root_dir, "tmp-incoming") |
647 | #self._factory = factory |
648 | top = os.path.join( |
649 | os.path.dirname(__file__), os.pardir, os.pardir, os.pardir) |
650 | @@ -401,10 +409,14 @@ |
651 | self.authserver_url = b"http://localhost:%d/" % self.authserver_port |
652 | self.addCleanup(self.authserver_listener.stopListening) |
653 | self.setUpUser('joe') |
654 | + os.mkdir(self.fsroot) |
655 | + os.mkdir(self.temp_dir) |
656 | self.pkgupload = self.useFixture(PkgUploadFixture(dedent("""\ |
657 | sftp: |
658 | authentication_endpoint: %s |
659 | - fsroot: %s""") % (self.authserver_url, self.root_dir))) |
660 | + fsroot: %s |
661 | + temp_dir: %s""") % |
662 | + (self.authserver_url, self.fsroot, self.temp_dir))) |
663 | |
664 | @defer.inlineCallbacks |
665 | def getClient(self): |
666 | @@ -460,8 +472,8 @@ |
667 | def setUp(self): |
668 | """Set up txpkgupload in a temp dir.""" |
669 | super(TestPkgUploadServiceMakerMixin, self).setUp() |
670 | - self.root_dir = self.useFixture(TempDir()).path |
671 | - self.server = self.server_factory(self.root_dir) |
672 | + root_dir = self.useFixture(TempDir()).path |
673 | + self.server = self.server_factory(root_dir) |
674 | self.useFixture(self.server) |
675 | |
676 | def test_init(self): |
677 | @@ -484,9 +496,9 @@ |
678 | |
679 | Only works for a single upload (txpkgupload transaction). |
680 | """ |
681 | - contents = sorted(os.listdir(self.root_dir)) |
682 | - upload_dir = contents[1] |
683 | - return os.path.join(self.root_dir, upload_dir, path) |
684 | + contents = sorted(os.listdir(self.server.fsroot)) |
685 | + upload_dir = contents[0] |
686 | + return os.path.join(self.server.fsroot, upload_dir, path) |
687 | |
688 | @defer.inlineCallbacks |
689 | def test_mkdir(self): |
690 | @@ -611,16 +623,15 @@ |
691 | yield self.server.waitForClose(4) |
692 | |
693 | # Build a list of directories representing the 4 sessions. |
694 | - upload_dirs = [leaf for leaf in sorted(os.listdir(self.root_dir)) |
695 | - if not leaf.startswith(".") and |
696 | - not leaf.endswith(".distro")] |
697 | + upload_dirs = [leaf for leaf in sorted(os.listdir(self.server.fsroot)) |
698 | + if not leaf.startswith(".")] |
699 | self.assertEqual(len(upload_dirs), 4) |
700 | |
701 | # Check the contents of files on each session. |
702 | expected_contents = ['ONE', 'TWO', 'THREE', 'FOUR'] |
703 | for index in range(4): |
704 | content = open(os.path.join( |
705 | - self.root_dir, upload_dirs[index], "test")).read() |
706 | + self.server.fsroot, upload_dirs[index], "test")).read() |
707 | self.assertEqual(content, expected_contents[index]) |
708 | |
709 | |
710 | |
711 | === modified file 'src/txpkgupload/tests/test_twistedsftp.py' |
712 | --- src/txpkgupload/tests/test_twistedsftp.py 2015-01-21 12:20:52 +0000 |
713 | +++ src/txpkgupload/tests/test_twistedsftp.py 2015-01-21 13:10:45 +0000 |
714 | @@ -16,18 +16,20 @@ |
715 | |
716 | class MockAvatar: |
717 | |
718 | - def __init__(self, fs_root): |
719 | + def __init__(self, fs_root, temp_dir): |
720 | self.fs_root = fs_root |
721 | + self.temp_dir = temp_dir |
722 | |
723 | |
724 | class TestSFTPServer(testtools.TestCase): |
725 | |
726 | def setUp(self): |
727 | - tempdir = fixtures.TempDir() |
728 | - self.useFixture(tempdir) |
729 | - self.useFixture(fixtures.MonkeyPatch("tempfile.tempdir", tempdir.path)) |
730 | - self.fs_root = self.useFixture(fixtures.TempDir()).path |
731 | - self.sftp_server = SFTPServer(MockAvatar(self.fs_root)) |
732 | + temp_dir = self.useFixture(fixtures.TempDir()).path |
733 | + fs_root = os.path.join(temp_dir, "incoming") |
734 | + temp_dir = os.path.join(temp_dir, "tmp-incoming") |
735 | + os.mkdir(fs_root) |
736 | + os.mkdir(temp_dir) |
737 | + self.sftp_server = SFTPServer(MockAvatar(fs_root, temp_dir)) |
738 | super(TestSFTPServer, self).setUp() |
739 | |
740 | def assertPermissions(self, expected, file_name): |
741 | |
742 | === modified file 'src/txpkgupload/twistedftp.py' |
743 | --- src/txpkgupload/twistedftp.py 2015-01-09 15:55:08 +0000 |
744 | +++ src/txpkgupload/twistedftp.py 2015-01-21 13:10:45 +0000 |
745 | @@ -52,15 +52,14 @@ |
746 | Roughly equivalent to the SFTPServer in the sftp side of things. |
747 | """ |
748 | |
749 | - def __init__(self, fsroot): |
750 | + def __init__(self, fsroot, temp_dir): |
751 | self._fs_root = fsroot |
752 | - self.uploadfilesystem = UploadFileSystem(tempfile.mkdtemp()) |
753 | + self.uploadfilesystem = UploadFileSystem( |
754 | + tempfile.mkdtemp(dir=temp_dir)) |
755 | self._current_upload = self.uploadfilesystem.rootpath |
756 | os.chmod(self._current_upload, 0770) |
757 | - self.hook = Hooks( |
758 | - self._fs_root, "ubuntu", perms='g+rws', prefix='-ftp') |
759 | + self.hook = Hooks(self._fs_root, perms='g+rws', prefix='-ftp') |
760 | self.hook.new_client_hook(self._current_upload, 0, 0) |
761 | - self.hook.auth_verify_hook(self._current_upload, None, None) |
762 | super(AnonymousShell, self).__init__( |
763 | filepath.FilePath(self._current_upload)) |
764 | |
765 | @@ -113,8 +112,9 @@ |
766 | """FTP Realm that lets anyone in.""" |
767 | implements(IRealm) |
768 | |
769 | - def __init__(self, root): |
770 | + def __init__(self, root, temp_dir): |
771 | self.root = root |
772 | + self.temp_dir = temp_dir |
773 | |
774 | def requestAvatar(self, avatarId, mind, *interfaces): |
775 | """Return a txpkgupload avatar - that is, an "authorisation". |
776 | @@ -124,7 +124,7 @@ |
777 | """ |
778 | for iface in interfaces: |
779 | if iface is ftp.IFTPShell: |
780 | - avatar = AnonymousShell(self.root) |
781 | + avatar = AnonymousShell(self.root, self.temp_dir) |
782 | return ftp.IFTPShell, avatar, getattr( |
783 | avatar, 'logout', lambda: None) |
784 | raise NotImplementedError( |
785 | @@ -134,8 +134,8 @@ |
786 | class FTPServiceFactory(service.Service): |
787 | """A factory that makes an `FTPService`""" |
788 | |
789 | - def __init__(self, port, root, idle_timeout): |
790 | - realm = FTPRealm(root) |
791 | + def __init__(self, port, root, temp_dir, idle_timeout): |
792 | + realm = FTPRealm(root, temp_dir) |
793 | portal = Portal(realm) |
794 | portal.registerChecker(AccessCheck()) |
795 | factory = ftp.FTPFactory(portal) |
796 | @@ -149,7 +149,7 @@ |
797 | self.portno = port |
798 | |
799 | @staticmethod |
800 | - def makeFTPService(port, root, idle_timeout): |
801 | + def makeFTPService(port, root, temp_dir, idle_timeout): |
802 | strport = "tcp:%s" % port |
803 | - factory = FTPServiceFactory(port, root, idle_timeout) |
804 | + factory = FTPServiceFactory(port, root, temp_dir, idle_timeout) |
805 | return strports.service(strport, factory.ftpfactory) |
806 | |
807 | === modified file 'src/txpkgupload/twistedsftp.py' |
808 | --- src/txpkgupload/twistedsftp.py 2015-01-09 15:55:08 +0000 |
809 | +++ src/txpkgupload/twistedsftp.py 2015-01-21 13:10:45 +0000 |
810 | @@ -38,13 +38,12 @@ |
811 | provideHandler(self.connectionClosed) |
812 | self._avatar = avatar |
813 | self._fs_root = avatar.fs_root |
814 | - self.uploadfilesystem = UploadFileSystem(tempfile.mkdtemp()) |
815 | + self.uploadfilesystem = UploadFileSystem( |
816 | + tempfile.mkdtemp(dir=avatar.temp_dir)) |
817 | self._current_upload = self.uploadfilesystem.rootpath |
818 | os.chmod(self._current_upload, 0770) |
819 | - self.hook = Hooks( |
820 | - self._fs_root, "ubuntu", perms='g+rws', prefix='-sftp') |
821 | + self.hook = Hooks(self._fs_root, perms='g+rws', prefix='-sftp') |
822 | self.hook.new_client_hook(self._current_upload, 0, 0) |
823 | - self.hook.auth_verify_hook(self._current_upload, None, None) |
824 | |
825 | def gotVersion(self, other_version, ext_data): |
826 | return {} |