Dmedia

Merge lp:~jderose/dmedia/rank into lp:dmedia

  1. rank
  2. Merge into trunk
Proposed by Jason Gerard DeRose
Status: Merged
Merged at revision: 733
Proposed branch: lp:~jderose/dmedia/rank
Merge into: lp:dmedia
Diff against target: 533 lines (+476/-3)
4 files modified
dmedia/metastore.py (+3/-3)
dmedia/tests/test_metastore.py (+80/-0)
dmedia/tests/test_views.py (+369/-0)
dmedia/views.py (+24/-0)
To merge this branch: bzr merge lp:~jderose/dmedia/rank
Reviewer Review Type Date Requested Status
James Raymond Approve
Review via email: mp+189448@code.launchpad.net

Description of the change

For background, see:
https://bugs.launchpad.net/dmedia/+bug/1235538

Changes include:

* Added new file/rank view (map) function, and detailed unit test for the same

* Changed MetaStore.inter_fragile() to use file/rank view instead of file/fragile

* Added missing unit test for MetaStore.iter_fragile()

To post a comment you must log in.
Revision history for this message
James Raymond (jamesmr) :
review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'dmedia/metastore.py'
2--- dmedia/metastore.py 2013-09-30 04:26:36 +0000
3+++ dmedia/metastore.py 2013-10-05 03:41:38 +0000
4@@ -874,13 +874,13 @@
5 """
6 Yield doc for each fragile file.
7 """
8- for copies in range(3):
9- result = self.db.view('file', 'fragile', key=copies, update_seq=True)
10+ for rank in range(6):
11+ result = self.db.view('file', 'rank', key=rank, update_seq=True)
12 update_seq = result.get('update_seq')
13 ids = [row['id'] for row in result['rows']]
14 del result # result might be quite large, free some memory
15 random.shuffle(ids)
16- log.info('%d files with copies=%d', len(ids), copies)
17+ log.info('vigilance: %d files at rank=%d', len(ids), rank)
18 for _id in ids:
19 yield self.db.get(_id)
20 if not monitor:
21
22=== modified file 'dmedia/tests/test_metastore.py'
23--- dmedia/tests/test_metastore.py 2013-09-24 00:37:23 +0000
24+++ dmedia/tests/test_metastore.py 2013-10-05 03:41:38 +0000
25@@ -3116,6 +3116,86 @@
26 fs2.verify(_id)
27 fs3.verify(_id)
28
29+ def test_iter_fragile(self):
30+ db = util.get_db(self.env, True)
31+ ms = metastore.MetaStore(db)
32+
33+ # Test when no files are in the library:
34+ self.assertEqual(list(ms.iter_fragile()), [])
35+
36+ # Create rank=(0 through 6) test data:
37+ ids = tuple(random_file_id() for i in range(7))
38+ stores = tuple(random_id() for i in range(3))
39+ docs = [
40+ {
41+ '_id': ids[0],
42+ 'type': 'dmedia/file',
43+ 'origin': 'user',
44+ 'stored': {},
45+ },
46+ {
47+ '_id': ids[1],
48+ 'type': 'dmedia/file',
49+ 'origin': 'user',
50+ 'stored': {
51+ stores[0]: {'copies': 0},
52+ },
53+ },
54+ {
55+ '_id': ids[2],
56+ 'type': 'dmedia/file',
57+ 'origin': 'user',
58+ 'stored': {
59+ stores[0]: {'copies': 1},
60+ },
61+ },
62+ {
63+ '_id': ids[3],
64+ 'type': 'dmedia/file',
65+ 'origin': 'user',
66+ 'stored': {
67+ stores[0]: {'copies': 1},
68+ stores[1]: {'copies': 0},
69+ },
70+ },
71+ {
72+ '_id': ids[4],
73+ 'type': 'dmedia/file',
74+ 'origin': 'user',
75+ 'stored': {
76+ stores[0]: {'copies': 1},
77+ stores[1]: {'copies': 1},
78+ },
79+ },
80+ {
81+ '_id': ids[5],
82+ 'type': 'dmedia/file',
83+ 'origin': 'user',
84+ 'stored': {
85+ stores[0]: {'copies': 1},
86+ stores[1]: {'copies': 1},
87+ stores[2]: {'copies': 0},
88+ },
89+ },
90+ {
91+ '_id': ids[6],
92+ 'type': 'dmedia/file',
93+ 'origin': 'user',
94+ 'stored': {
95+ stores[0]: {'copies': 1},
96+ stores[1]: {'copies': 1},
97+ stores[2]: {'copies': 1},
98+ },
99+ },
100+ ]
101+ db.save_many(docs)
102+
103+ # We should get docs[0:6]:
104+ self.assertEqual(list(ms.iter_fragile()), docs[0:-1])
105+
106+ # Docs should not be changed:
107+ self.assertEqual(db.get_many(ids), docs)
108+
109 def test_iter_actionable_fragile(self):
110 db = util.get_db(self.env, True)
111 ms = metastore.MetaStore(db)
112
113=== modified file 'dmedia/tests/test_views.py'
114--- dmedia/tests/test_views.py 2013-07-02 07:34:11 +0000
115+++ dmedia/tests/test_views.py 2013-10-05 03:41:38 +0000
116@@ -884,6 +884,375 @@
117 {'rows': [], 'offset': 0, 'total_rows': 1},
118 )
119
120+ def test_rank(self):
121+ ####################################################################
122+ # 1st, test sort order and other properties on a collection of docs:
123+ db = Database('foo', self.env)
124+ db.put(None)
125+ design = self.build_view('rank')
126+ db.save(design)
127+
128+ # Test when empty
129+ self.assertEqual(
130+ db.view('file', 'rank'),
131+ {'rows': [], 'offset': 0, 'total_rows': 0},
132+ )
133+
134+ # Create rank=(0 through 6) test data:
135+ ids = tuple(random_file_id() for i in range(7))
136+ stores = tuple(random_id() for i in range(3))
137+ docs = [
138+ {
139+ '_id': ids[0],
140+ 'type': 'dmedia/file',
141+ 'origin': 'user',
142+ 'stored': {},
143+ },
144+ {
145+ '_id': ids[1],
146+ 'type': 'dmedia/file',
147+ 'origin': 'user',
148+ 'stored': {
149+ stores[0]: {'copies': 0},
150+ },
151+ },
152+ {
153+ '_id': ids[2],
154+ 'type': 'dmedia/file',
155+ 'origin': 'user',
156+ 'stored': {
157+ stores[0]: {'copies': 1},
158+ },
159+ },
160+ {
161+ '_id': ids[3],
162+ 'type': 'dmedia/file',
163+ 'origin': 'user',
164+ 'stored': {
165+ stores[0]: {'copies': 1},
166+ stores[1]: {'copies': 0},
167+ },
168+ },
169+ {
170+ '_id': ids[4],
171+ 'type': 'dmedia/file',
172+ 'origin': 'user',
173+ 'stored': {
174+ stores[0]: {'copies': 1},
175+ stores[1]: {'copies': 1},
176+ },
177+ },
178+ {
179+ '_id': ids[5],
180+ 'type': 'dmedia/file',
181+ 'origin': 'user',
182+ 'stored': {
183+ stores[0]: {'copies': 1},
184+ stores[1]: {'copies': 1},
185+ stores[2]: {'copies': 0},
186+ },
187+ },
188+ {
189+ '_id': ids[6],
190+ 'type': 'dmedia/file',
191+ 'origin': 'user',
192+ 'stored': {
193+ stores[0]: {'copies': 1},
194+ stores[1]: {'copies': 1},
195+ stores[2]: {'copies': 1},
196+ },
197+ },
198+ ]
199+ db.save_many(docs)
200+
201+ # Test sorting, overall properties:
202+ self.assertEqual(
203+ db.view('file', 'rank'),
204+ {
205+ 'offset': 0,
206+ 'total_rows': 7,
207+ 'rows': [
208+ {'key': 0, 'id': ids[0], 'value': None},
209+ {'key': 1, 'id': ids[1], 'value': None},
210+ {'key': 2, 'id': ids[2], 'value': None},
211+ {'key': 3, 'id': ids[3], 'value': None},
212+ {'key': 4, 'id': ids[4], 'value': None},
213+ {'key': 5, 'id': ids[5], 'value': None},
214+ {'key': 6, 'id': ids[6], 'value': None},
215+ ],
216+ },
217+ )
218+ self.assertEqual(
219+ db.view('file', 'rank', descending=True),
220+ {
221+ 'offset': 0,
222+ 'total_rows': 7,
223+ 'rows': [
224+ {'key': 6, 'id': ids[6], 'value': None},
225+ {'key': 5, 'id': ids[5], 'value': None},
226+ {'key': 4, 'id': ids[4], 'value': None},
227+ {'key': 3, 'id': ids[3], 'value': None},
228+ {'key': 2, 'id': ids[2], 'value': None},
229+ {'key': 1, 'id': ids[1], 'value': None},
230+ {'key': 0, 'id': ids[0], 'value': None},
231+ ],
232+ },
233+ )
234+
235+ ###################################################
236+ # 2nd, drill down on the minutia with a single doc:
237+ db = Database('bar', self.env)
238+ db.put(None)
239+ design = self.build_view('rank')
240+ db.save(design)
241+
242+ # Test when empty
243+ self.assertEqual(
244+ db.view('file', 'rank'),
245+ {'rows': [], 'offset': 0, 'total_rows': 0},
246+ )
247+
248+ # rank should be 0 when doc['stored'] is missing, empty, or wrong type:
249+ _id = random_file_id()
250+ store_id = random_id()
251+ doc = {
252+ '_id': _id,
253+ 'type': 'dmedia/file',
254+ 'origin': 'user',
255+ }
256+ db.save(doc)
257+ self.assertEqual(
258+ db.view('file', 'rank'),
259+ {
260+ 'offset': 0,
261+ 'total_rows': 1,
262+ 'rows': [
263+ {'key': 0, 'id': _id, 'value': None},
264+ ],
265+ },
266+ )
267+ doc['stored'] = {}
268+ db.save(doc)
269+ self.assertEqual(
270+ db.view('file', 'rank'),
271+ {
272+ 'offset': 0,
273+ 'total_rows': 1,
274+ 'rows': [
275+ {'key': 0, 'id': _id, 'value': None},
276+ ],
277+ },
278+ )
279+ doc['stored'] = ['hello', 'world']
280+ db.save(doc)
281+ self.assertEqual(
282+ db.view('file', 'rank'),
283+ {
284+ 'offset': 0,
285+ 'total_rows': 1,
286+ 'rows': [
287+ {'key': 0, 'id': _id, 'value': None},
288+ ],
289+ },
290+ )
291+ doc['stored'] = 'helloworld'
292+ db.save(doc)
293+ self.assertEqual(
294+ db.view('file', 'rank'),
295+ {
296+ 'offset': 0,
297+ 'total_rows': 1,
298+ 'rows': [
299+ {'key': 0, 'id': _id, 'value': None},
300+ ],
301+ },
302+ )
303+ doc['stored'] = None
304+ db.save(doc)
305+ self.assertEqual(
306+ db.view('file', 'rank'),
307+ {
308+ 'offset': 0,
309+ 'total_rows': 1,
310+ 'rows': [
311+ {'key': 0, 'id': _id, 'value': None},
312+ ],
313+ },
314+ )
315+ doc['stored'] = 17
316+ db.save(doc)
317+ self.assertEqual(
318+ db.view('file', 'rank'),
319+ {
320+ 'offset': 0,
321+ 'total_rows': 1,
322+ 'rows': [
323+ {'key': 0, 'id': _id, 'value': None},
324+ ],
325+ },
326+ )
327+
328+ # Test with one location, broken doc['stored'][STORE_id]:
329+ doc['stored'] = {random_id(): 'blah blah broken'}
330+ db.save(doc)
331+ self.assertEqual(
332+ db.view('file', 'rank'),
333+ {
334+ 'offset': 0,
335+ 'total_rows': 1,
336+ 'rows': [
337+ {'key': 1, 'id': _id, 'value': None},
338+ ],
339+ },
340+ )
341+
342+ # Test with one location, broken doc['stored'][STORE_id]['copies']:
343+ doc['stored'] = {random_id(): {'copies': '2'}}
344+ db.save(doc)
345+ self.assertEqual(
346+ db.view('file', 'rank'),
347+ {
348+ 'offset': 0,
349+ 'total_rows': 1,
350+ 'rows': [
351+ {'key': 1, 'id': _id, 'value': None},
352+ ],
353+ },
354+ )
355+ doc['stored'] = {random_id(): {'copies': 'foo bar'}}
356+ db.save(doc)
357+ self.assertEqual(
358+ db.view('file', 'rank'),
359+ {
360+ 'offset': 0,
361+ 'total_rows': 1,
362+ 'rows': [
363+ {'key': 1, 'id': _id, 'value': None},
364+ ],
365+ },
366+ )
367+ doc['stored'] = {random_id(): {'copies': -3}}
368+ db.save(doc)
369+ self.assertEqual(
370+ db.view('file', 'rank'),
371+ {
372+ 'offset': 0,
373+ 'total_rows': 1,
374+ 'rows': [
375+ {'key': 1, 'id': _id, 'value': None},
376+ ],
377+ },
378+ )
379+
380+ # Test with locations=1, durability=2:
381+ doc['stored'] = {random_id(): {'copies': 2}}
382+ db.save(doc)
383+ self.assertEqual(
384+ db.view('file', 'rank'),
385+ {
386+ 'offset': 0,
387+ 'total_rows': 1,
388+ 'rows': [
389+ {'key': 3, 'id': _id, 'value': None},
390+ ],
391+ },
392+ )
393+
394+ # Test with locations=2, durability=0:
395+ doc['stored'] = {
396+ random_id(): {'copies': 0},
397+ random_id(): {'copies': 0},
398+ }
399+ db.save(doc)
400+ self.assertEqual(
401+ db.view('file', 'rank'),
402+ {
403+ 'offset': 0,
404+ 'total_rows': 1,
405+ 'rows': [
406+ {'key': 2, 'id': _id, 'value': None},
407+ ],
408+ },
409+ )
410+
411+ # Test with locations=2, durability=1:
412+ doc['stored'] = {
413+ random_id(): {'copies': 1},
414+ random_id(): {'copies': 0},
415+ }
416+ db.save(doc)
417+ self.assertEqual(
418+ db.view('file', 'rank'),
419+ {
420+ 'offset': 0,
421+ 'total_rows': 1,
422+ 'rows': [
423+ {'key': 3, 'id': _id, 'value': None},
424+ ],
425+ },
426+ )
427+
428+ # Test that locations is clamped at 3:
429+ doc['stored'] = {
430+ random_id(): {'copies': 0},
431+ random_id(): {'copies': 0},
432+ random_id(): {'copies': 0},
433+ random_id(): {'copies': 0},
434+ }
435+ db.save(doc)
436+ self.assertEqual(
437+ db.view('file', 'rank'),
438+ {
439+ 'offset': 0,
440+ 'total_rows': 1,
441+ 'rows': [
442+ {'key': 3, 'id': _id, 'value': None},
443+ ],
444+ },
445+ )
446+
447+ # Test that durability is clamped at 3:
448+ doc['stored'] = {random_id(): {'copies': 17}}
449+ db.save(doc)
450+ self.assertEqual(
451+ db.view('file', 'rank'),
452+ {
453+ 'offset': 0,
454+ 'total_rows': 1,
455+ 'rows': [
456+ {'key': 4, 'id': _id, 'value': None},
457+ ],
458+ },
459+ )
460+
461+ # Make sure doc['type'] is checked:
462+ doc['type'] = 'dmedia/guile'
463+ db.save(doc)
464+ self.assertEqual(
465+ db.view('file', 'rank'),
466+ {'rows': [], 'offset': 0, 'total_rows': 0},
467+ )
468+
469+ # Make sure doc['orign'] is checked:
470+ doc['type'] = 'dmedia/file'
471+ db.save(doc)
472+ self.assertEqual(
473+ db.view('file', 'rank'),
474+ {
475+ 'offset': 0,
476+ 'total_rows': 1,
477+ 'rows': [
478+ {'key': 4, 'id': _id, 'value': None},
479+ ],
480+ },
481+ )
482+ doc['origin'] = 'render'
483+ db.save(doc)
484+ self.assertEqual(
485+ db.view('file', 'rank'),
486+ {'rows': [], 'offset': 0, 'total_rows': 0},
487+ )
488+
489 def test_fragile(self):
490 db = Database('foo', self.env)
491 db.put(None)
492
493=== modified file 'dmedia/views.py'
494--- dmedia/views.py 2013-07-29 00:22:01 +0000
495+++ dmedia/views.py 2013-10-05 03:41:38 +0000
496@@ -123,6 +123,29 @@
497 }
498 """
499
500+file_rank = """
501+function(doc) {
502+ if (doc.type == 'dmedia/file' && doc.origin == 'user') {
503+ if (typeof doc.stored != 'object' || isArray(doc.stored)) {
504+ emit(0, null);
505+ return;
506+ }
507+ var locations = 0;
508+ var durability = 0;
509+ var key, copies;
510+ for (key in doc.stored) {
511+ locations += 1;
512+ copies = doc.stored[key].copies;
513+ if (typeof copies == 'number' && copies > 0) {
514+ durability += copies;
515+ }
516+ }
517+ var rank = Math.min(3, locations) + Math.min(3, durability);
518+ emit(rank, null);
519+ }
520+}
521+"""
522+
523 file_fragile = """
524 function(doc) {
525 if (doc.type == 'dmedia/file' && doc.origin == 'user') {
526@@ -279,6 +302,7 @@
527 'stored': {'map': file_stored, 'reduce': _stats},
528 'nonzero': {'map': file_nonzero},
529 'copies': {'map': file_copies},
530+ 'rank': {'map': file_rank},
531 'fragile': {'map': file_fragile},
532 'never-verified': {'map': file_never_verified},
533 'last-verified': {'map': file_last_verified},

Subscribers

People subscribed via source and target branches