Ubuntu One Client

Merge lp:~facundo/ubuntuone-client/fix-eq-pushing into lp:ubuntuone-client

  1. fix-eq-pushing
  2. Merge into trunk
Proposed by Facundo Batista
Status: Merged
Approved by: Facundo Batista
Approved revision: 843
Merged at revision: 843
Proposed branch: lp:~facundo/ubuntuone-client/fix-eq-pushing
Merge into: lp:ubuntuone-client
Diff against target: 464 lines (+91/-92)
4 files modified
tests/syncdaemon/test_hashqueue.py (+81/-84)
tests/syncdaemon/test_localrescan.py (+4/-4)
ubuntuone/syncdaemon/hash_queue.py (+5/-3)
ubuntuone/syncdaemon/local_rescan.py (+1/-1)
To merge this branch: bzr merge lp:~facundo/ubuntuone-client/fix-eq-pushing
Reviewer Review Type Date Requested Status
Facundo Batista (community) Approve
Natalia Bidart (community) Approve
Review via email: mp+48673@code.launchpad.net

Commit message

Fixed LR and HQ pushing. (LP: #713142)

Description of the change

Fixed LR and HQ pushing.

Tests included.

To post a comment you must log in.
Revision history for this message
Natalia Bidart (nataliabidart) wrote :

Very good!

review: Approve
Revision history for this message
Facundo Batista (facundo) wrote :

Approving with only one real review

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'tests/syncdaemon/test_hashqueue.py'
2--- tests/syncdaemon/test_hashqueue.py 2011-01-20 21:27:24 +0000
3+++ tests/syncdaemon/test_hashqueue.py 2011-02-04 22:28:56 +0000
4@@ -57,9 +57,9 @@
5 mark = object()
6 queue = hash_queue.UniqueQueue()
7 class Helper(object):
8- """helper class"""
9- def push(self, *args):
10- """callback"""
11+ """Helper class."""
12+ def push(self, event, **kwargs):
13+ """Callback."""
14 receiver = Helper()
15 hasher = hash_queue._Hasher(queue, mark, receiver)
16 hasher.start()
17@@ -77,16 +77,16 @@
18
19 @defer.inlineCallbacks
20 def test_called_back_log_ok(self):
21- """Tests that the hasher produces correct info."""
22+ """Test that the hasher produces correct info."""
23 # create the hasher
24 mark = object()
25 queue = hash_queue.UniqueQueue()
26 d = defer.Deferred()
27 class Helper(object):
28 """Helper class."""
29- def push(self, *args):
30+ def push(self, event, **kwargs):
31 """Callback."""
32- d.callback(args)
33+ d.callback(kwargs)
34 receiver = Helper()
35 hasher = hash_queue._Hasher(queue, mark, receiver)
36 hasher.start()
37@@ -120,16 +120,16 @@
38 hasher.logger.removeHandler(handler)
39
40 def test_called_back_ok(self):
41- """Tests that the hasher produces correct info."""
42+ """Test that the hasher produces correct info."""
43 # create the hasher
44 mark = object()
45 queue = hash_queue.UniqueQueue()
46 d = defer.Deferred()
47 class Helper(object):
48- """helper class"""
49- def push(self, *args):
50- """callback"""
51- d.callback(args)
52+ """Helper class."""
53+ def push(self, event, **kwargs):
54+ """Callback."""
55+ d.callback((event, kwargs))
56 receiver = Helper()
57 hasher = hash_queue._Hasher(queue, mark, receiver)
58 hasher.start()
59@@ -140,23 +140,22 @@
60 fh.write("foobar")
61 queue.put((testfile, "mdid"))
62
63- def check_info(args):
64+ def check_info((event, kwargs)):
65 """check the info pushed by the hasher"""
66 # pylint: disable-msg=W0612
67 hasher.stop()
68 hasher.join(timeout=5)
69- event, path, hash, crc, size, stat = args
70 self.assertEqual(event, "HQ_HASH_NEW")
71 # calculate what we should receive
72 realh = content_hash_factory()
73 realh.hash_object.update("foobar")
74 should_be = realh.content_hash()
75 curr_stat = os.stat(testfile)
76- self.assertEquals(should_be, hash)
77+ self.assertEquals(should_be, kwargs['hash'])
78 for attr in ('st_mode', 'st_ino', 'st_dev', 'st_nlink', 'st_uid',
79 'st_gid', 'st_size', 'st_ctime', 'st_mtime'):
80 self.assertEquals(getattr(curr_stat, attr),
81- getattr(stat, attr))
82+ getattr(kwargs['stat'], attr))
83
84
85 d.addCallback(check_info)
86@@ -164,16 +163,16 @@
87 return d
88
89 def test_called_back_error(self):
90- """Tests that the hasher signals error when no file."""
91+ """Test that the hasher signals error when no file."""
92 # create the hasher
93 mark = object()
94 queue = hash_queue.UniqueQueue()
95 d = defer.Deferred()
96 class Helper(object):
97- """helper class"""
98- def push(self, *args):
99- """callback"""
100- d.callback(args)
101+ """Helper class."""
102+ def push(self, event, **kwargs):
103+ """Callback."""
104+ d.callback((event, kwargs))
105 receiver = Helper()
106 hasher = hash_queue._Hasher(queue, mark, receiver)
107 hasher.start()
108@@ -181,13 +180,12 @@
109 # send what to hash
110 queue.put(("not_to_be_found", "foo"))
111
112- def check_info(args):
113- """check the info pushed by the hasher"""
114+ def check_info((event, kwargs)):
115+ """Check the info pushed by the hasher."""
116 hasher.stop()
117 hasher.join(timeout=5)
118- event, mdid = args
119 self.assertEqual(event, "HQ_HASH_ERROR")
120- self.assertEqual(mdid, "foo")
121+ self.assertEqual(kwargs['mdid'], "foo")
122
123
124 d.addCallback(check_info)
125@@ -205,8 +203,9 @@
126 tfile = os.path.join(self.test_dir, "tfile"+str(i))
127 with open(tfile, "w") as fh:
128 fh.write("supercalifragilistico"+str(i))
129- should_be.append(("HQ_HASH_NEW", tfile, hasher.content_hash(),
130- crc32(text), len(text), os.stat(tfile)))
131+ d = dict(path=tfile, hash=hasher.content_hash(),
132+ crc32=crc32(text), size=len(text), stat=os.stat(tfile))
133+ should_be.append(("HQ_HASH_NEW", d))
134
135 # create the hasher
136 mark = object()
137@@ -214,13 +213,13 @@
138 d = defer.Deferred()
139
140 class Helper(object):
141- """helper class"""
142+ """Helper class."""
143 # class-closure, cannot use self, pylint: disable-msg=E0213
144 def __init__(innerself):
145 innerself.store = []
146- def push(innerself, *args):
147- """callback"""
148- innerself.store.append(args)
149+ def push(innerself, event, **kwargs):
150+ """Callback."""
151+ innerself.store.append((event, kwargs))
152 if len(innerself.store) == 10:
153 hasher.stop()
154 hasher.join(timeout=5)
155@@ -255,8 +254,8 @@
156 d = defer.Deferred()
157
158 class Helper(object):
159- """helper class"""
160- def push(self, event, path, newhash, crc, size, stat):
161+ """Helper class."""
162+ def push(self, event, path, hash, crc32, size, stat):
163 """callback"""
164 hasher.stop()
165 hasher.join(timeout=5)
166@@ -264,7 +263,7 @@
167 d.errback(Exception("envent is not HQ_HASH_NEW"))
168 elif path != testfile:
169 d.errback(Exception("path is not the original one"))
170- elif newhash != testhash:
171+ elif hash != testhash:
172 d.errback(Exception("the hashes are different!"))
173 else:
174 d.callback(True)
175@@ -297,14 +296,14 @@
176 testcase.BaseTwistedTestCase.tearDown(self)
177
178 def test_called_back_ok(self):
179- """Tests that the hasher produces correct info."""
180+ """Test that the hasher produces correct info."""
181 # create the hasher
182 d = defer.Deferred()
183 class Helper(object):
184- """helper class"""
185- def push(self, *args):
186- """callback"""
187- d.callback(args)
188+ """Helper class."""
189+ def push(self, event, **kwargs):
190+ """Callback."""
191+ d.callback((event, kwargs))
192 receiver = Helper()
193 hq = hash_queue.HashQueue(receiver)
194 self.addCleanup(hq.shutdown)
195@@ -315,34 +314,33 @@
196 fh.write("foobar")
197 hq.insert(testfile, "mdid")
198
199- def check_info(args):
200+ def check_info((event, kwargs)):
201 """check the info pushed by the hasher"""
202 # pylint: disable-msg=W0612
203- event, path, hash, crc, size, stat = args
204 self.assertEqual(event, "HQ_HASH_NEW")
205 # calculate what we should receive
206 realh = content_hash_factory()
207 realh.hash_object.update("foobar")
208 should_be = realh.content_hash()
209 curr_stat = os.stat(testfile)
210- self.assertEquals(should_be, hash)
211+ self.assertEquals(should_be, kwargs['hash'])
212 for attr in ('st_mode', 'st_ino', 'st_dev', 'st_nlink', 'st_uid',
213 'st_gid', 'st_size', 'st_ctime', 'st_mtime'):
214 self.assertEquals(getattr(curr_stat, attr),
215- getattr(stat, attr))
216+ getattr(kwargs['stat'], attr))
217
218 d.addCallback(check_info)
219 return d
220
221 def test_called_back_error(self):
222- """Tests that the hasher generates an error when no file."""
223+ """Test that the hasher generates an error when no file."""
224 # create the hasher
225 d = defer.Deferred()
226 class Helper(object):
227- """helper class"""
228- def push(self, *args):
229- """callback"""
230- d.callback(args)
231+ """Helper class."""
232+ def push(self, event, **kwargs):
233+ """Callback."""
234+ d.callback((event, kwargs))
235 receiver = Helper()
236 hq = hash_queue.HashQueue(receiver)
237 self.addCleanup(hq.shutdown)
238@@ -350,11 +348,10 @@
239 # send what to hash
240 hq.insert("not_to_be_found", "foo")
241
242- def check_info(args):
243- """check the info pushed by the hasher"""
244- event, mdid = args
245+ def check_info((event, kwargs)):
246+ """Check the info pushed by the hasher."""
247 self.assertEqual(event, "HQ_HASH_ERROR")
248- self.assertEqual(mdid, "foo")
249+ self.assertEqual(kwargs['mdid'], "foo")
250
251 d.addCallback(check_info)
252 return d
253@@ -367,9 +364,9 @@
254 open(tfile2, "w").close()
255
256 class C(object):
257- """bogus"""
258- def push(self, *a):
259- """none"""
260+ """Bogus."""
261+ def push(self, e, **k):
262+ """None."""
263 hq = hash_queue.HashQueue(C())
264 self.addCleanup(hq.shutdown)
265
266@@ -377,6 +374,7 @@
267 original_hash = hash_queue._Hasher._hash
268
269 def f(*a):
270+ """Fake _hash."""
271 event.wait()
272 return "foo"
273 hash_queue._Hasher._hash = f
274@@ -409,18 +407,19 @@
275 tfile = os.path.join(self.test_dir, "tfile"+str(i))
276 with open(tfile, "w") as fh:
277 fh.write("supercalifragilistico"+str(i))
278- should_be.append(("HQ_HASH_NEW", tfile, hasher.content_hash(),
279- crc32(text), len(text), os.stat(tfile)))
280+ d = dict(path=tfile, hash=hasher.content_hash(),
281+ crc32=crc32(text), size=len(text), stat=os.stat(tfile))
282+ should_be.append(("HQ_HASH_NEW", d))
283
284 d = defer.Deferred()
285 class Helper(object):
286- """helper class"""
287+ """Helper class."""
288 # class-closure, cannot use self, pylint: disable-msg=E0213
289 def __init__(innerself):
290 innerself.store = []
291- def push(innerself, *args):
292- """callback"""
293- innerself.store.append(args)
294+ def push(innerself, event, **kwargs):
295+ """Callback."""
296+ innerself.store.append((event, kwargs))
297 if len(innerself.store) == 10:
298 if innerself.store[:-1] == should_be[:-1]:
299 d.callback(True)
300@@ -448,24 +447,22 @@
301 tfile = os.path.join(self.test_dir, "tfile"+str(i))
302 with open(tfile, "w") as fh:
303 fh.write("supercalifragilistico"+str(i))
304- should_be.append(("HQ_HASH_NEW", tfile, hasher.content_hash(),
305- crc32(text), len(text), os.stat(tfile)))
306+ d = dict(path=tfile, hash=hasher.content_hash(),
307+ crc32=crc32(text), size=len(text), stat=os.stat(tfile))
308+ should_be.append(("HQ_HASH_NEW", d))
309
310 d = defer.Deferred()
311 class Helper(object):
312- """helper class"""
313+ """Helper class."""
314 # class-closure, cannot use self, pylint: disable-msg=E0213
315 def __init__(innerself):
316 innerself.store = []
317- def push(innerself, *args):
318- """callback"""
319- innerself.store.append(args)
320+ def push(innerself, event, **kwargs):
321+ """Callback."""
322+ innerself.store.append((event, kwargs))
323 if len(innerself.store) == 10:
324- if len(innerself.store[:-1]) == len(should_be[:-1]):
325- stored = set(innerself.store[:-1])
326- expected = set(should_be[:-1])
327- if len(stored.symmetric_difference(expected)) == 0:
328- d.callback(True)
329+ if innerself.store == should_be:
330+ d.callback(True)
331 else:
332 d.errback(Exception("are different!"))
333
334@@ -512,14 +509,14 @@
335 d = defer.Deferred()
336
337 class Helper(object):
338- """helper class"""
339- def push(self, event, path, newhash, crc, size, stat):
340- """callback"""
341+ """Helper class."""
342+ def push(self, event, path, hash, crc32, size, stat):
343+ """Callback."""
344 if event != "HQ_HASH_NEW":
345 d.errback(Exception("envent is not HQ_HASH_NEW"))
346 elif path != testfile:
347 d.errback(Exception("path is not the original one"))
348- elif newhash != testhash:
349+ elif hash != testhash:
350 d.errback(Exception("the hashes are different!"))
351 else:
352 d.callback(True)
353@@ -570,9 +567,9 @@
354 def test_shutdown(self):
355 """Test that the HashQueue shutdown """
356 class Helper(object):
357- """helper class"""
358- def push(self, *args):
359- """callback"""
360+ """Helper class."""
361+ def push(self, event, **kwargs):
362+ """Callback."""
363 receiver = Helper()
364 hq = hash_queue.HashQueue(receiver)
365 hq.shutdown()
366@@ -590,9 +587,9 @@
367 fh.write(testinfo)
368
369 class Helper(object):
370- """helper class"""
371- def push(self, *args):
372- """callback"""
373+ """Helper class."""
374+ def push(self, event, **kwargs):
375+ """Callback."""
376 receiver = Helper()
377 hq = hash_queue.HashQueue(receiver)
378 self.addCleanup(hq.shutdown)
379@@ -612,9 +609,9 @@
380 def test_insert_post_shutdown(self):
381 """test inserting a path after the shutdown"""
382 class Helper(object):
383- """helper class"""
384- def push(self, *args):
385- """callback"""
386+ """Helper class."""
387+ def push(self, event, **kwargs):
388+ """Callback."""
389 receiver = Helper()
390 hq = hash_queue.HashQueue(receiver)
391 hq.shutdown()
392
393=== modified file 'tests/syncdaemon/test_localrescan.py'
394--- tests/syncdaemon/test_localrescan.py 2011-02-03 17:43:37 +0000
395+++ tests/syncdaemon/test_localrescan.py 2011-02-04 22:28:56 +0000
396@@ -45,9 +45,9 @@
397 def __init__(self):
398 self.pushed = []
399
400- def push(self, event, *args):
401+ def push(self, event, **kwargs):
402 """Store stuff as pushed."""
403- self.pushed.append((event,) + args)
404+ self.pushed.append((event, kwargs))
405
406 def _fake(self, *a, **k):
407 """fake"""
408@@ -1019,7 +1019,7 @@
409 def check(_):
410 """check"""
411 self.assertEqual(self.eq.pushed, [
412- ('LR_SCAN_ERROR', "mdid", False),
413+ ('LR_SCAN_ERROR', dict(mdid="mdid", udfmode=False)),
414 ])
415
416 self.deferred.addCallback(check)
417@@ -1036,7 +1036,7 @@
418 def check(_):
419 """check"""
420 self.assertEqual(self.eq.pushed, [
421- ('LR_SCAN_ERROR', "mdid", True),
422+ ('LR_SCAN_ERROR', dict(mdid="mdid", udfmode=True)),
423 ])
424
425 self.deferred.addCallback(check)
426
427=== modified file 'ubuntuone/syncdaemon/hash_queue.py'
428--- ubuntuone/syncdaemon/hash_queue.py 2010-12-20 12:28:01 +0000
429+++ ubuntuone/syncdaemon/hash_queue.py 2011-02-04 22:28:56 +0000
430@@ -76,7 +76,8 @@
431 except (IOError, OSError), e:
432 m = "Hasher: hash error %s (path %r mdid %s)"
433 self.logger.debug(m, e, path, mdid)
434- reactor.callFromThread(self.eq.push, "HQ_HASH_ERROR", mdid)
435+ reactor.callFromThread(self.eq.push,
436+ "HQ_HASH_ERROR", mdid=mdid)
437 except StopHashing, e:
438 self.logger.debug(str(e))
439 else:
440@@ -85,8 +86,9 @@
441 " crc=%s size=%d st_ino=%d st_size=%d"
442 " st_mtime=%r", path, hashdata,crc, size,
443 stat.st_ino, stat.st_size, stat.st_mtime)
444- reactor.callFromThread(self.eq.push,
445- "HQ_HASH_NEW", path, *result)
446+ reactor.callFromThread(self.eq.push, "HQ_HASH_NEW", path=path,
447+ hash=hashdata, crc32=crc,
448+ size=size, stat=stat)
449 finally:
450 with self.mutex:
451 self.hashing = None
452
453=== modified file 'ubuntuone/syncdaemon/local_rescan.py'
454--- ubuntuone/syncdaemon/local_rescan.py 2011-01-14 14:04:37 +0000
455+++ ubuntuone/syncdaemon/local_rescan.py 2011-02-04 22:28:56 +0000
456@@ -223,7 +223,7 @@
457
458 def _send_scan_error(self, mdid, udfmode):
459 """Sends the scan error event."""
460- self.eq.push("LR_SCAN_ERROR", mdid, udfmode)
461+ self.eq.push("LR_SCAN_ERROR", mdid=mdid, udfmode=udfmode)
462
463 def _queue_scan(self):
464 """If there's a scan in progress, queue the new one for later."""

Subscribers

People subscribed via source and target branches