Merge lp:~jderose/microfiber/replication into lp:microfiber

Proposed by Jason Gerard DeRose
Status: Merged
Merged at revision: 140
Proposed branch: lp:~jderose/microfiber/replication
Merge into: lp:microfiber
Diff against target: 795 lines (+526/-225)
2 files modified
microfiber.py (+28/-10)
test_microfiber.py (+498/-215)
To merge this branch: bzr merge lp:~jderose/microfiber/replication
Reviewer Review Type Date Requested Status
David Jordan Approve
Review via email: mp+123019@code.launchpad.net

Description of the change

* Changed Server.push() signature from:

  Server.push(name, env, **kw)

 To:

  Server.push(local_db, remote_db, remote_env, **kw)

* Changed push_replication() signature from:

  push_replication(name, env, **kw)

 To:

  push_replication(local_db, remote_db, remote_env, **kw)

* Added pull_replication(local_db, remote_db, remote_env, **kw) function

* Added Server.pull(local_db, remote_db, remote_env, **kw) method

To post a comment you must log in.
Revision history for this message
David Jordan (dmj726) wrote :

Pull sounds like the right thing here. Approved.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'microfiber.py'
2--- microfiber.py 2012-08-16 19:24:20 +0000
3+++ microfiber.py 2012-09-06 07:09:35 +0000
4@@ -271,16 +271,30 @@
5 return peer
6
7
8-def push_replication(name, env, **kw):
9+def push_replication(local_db, remote_db, remote_env, **kw):
10 """
11 Build the object to POST for push replication.
12-
13- For details on what keyword arguments you might want to use, see:
14-
15- http://wiki.apache.org/couchdb/Replication
16- """
17- peer = replication_peer(name, env)
18- return replication_body(name, peer, **kw)
19+
20+ For details on what keyword arguments you might want to use, see:
21+
22+ http://wiki.apache.org/couchdb/Replication
23+ """
24+ source = local_db
25+ target = replication_peer(remote_db, remote_env)
26+ return replication_body(source, target, **kw)
27+
28+
29+def pull_replication(local_db, remote_db, remote_env, **kw):
30+ """
31+ Build the object to POST for pull replication.
32+
33+ For details on what keyword arguments you might want to use, see:
34+
35+ http://wiki.apache.org/couchdb/Replication
36+ """
37+ source = replication_peer(remote_db, remote_env)
38+ target = local_db
39+ return replication_body(source, target, **kw)
40
41
42 def id_slice_iter(rows, size=25):
43@@ -756,8 +770,12 @@
44 db.ensure()
45 return db
46
47- def push(self, name, env, **kw):
48- obj = push_replication(name, env, **kw)
49+ def push(self, local_db, remote_db, remote_env, **kw):
50+ obj = push_replication(local_db, remote_db, remote_env, **kw)
51+ return self.post(obj, '_replicate')
52+
53+ def pull(self, local_db, remote_db, remote_env, **kw):
54+ obj = pull_replication(local_db, remote_db, remote_env, **kw)
55 return self.post(obj, '_replicate')
56
57
58
59=== modified file 'test_microfiber.py'
60--- test_microfiber.py 2012-08-16 03:43:29 +0000
61+++ test_microfiber.py 2012-09-06 07:09:35 +0000
62@@ -87,6 +87,10 @@
63 )
64
65
66+def random_dbname():
67+ return 'db-' + microfiber.random_id().lower()
68+
69+
70 def random_oauth():
71 return dict(
72 (k, random_id())
73@@ -438,191 +442,432 @@
74 )
75
76 def test_push_replication(self):
77- url = 'http://' + random_id().lower() + ':5984/'
78- name = 'db-' + random_id().lower()
79-
80- # Test with no auth
81- env = {'url': url}
82- self.assertEqual(
83- microfiber.push_replication(name, env),
84- {
85- 'source': name,
86- 'target': {
87- 'url': url + name,
88- },
89- }
90- )
91- self.assertEqual(
92- microfiber.push_replication(name, env, cancel=True),
93- {
94- 'source': name,
95- 'target': {
96- 'url': url + name,
97- },
98- 'cancel': True,
99- }
100- )
101- self.assertEqual(
102- microfiber.push_replication(name, env, continuous=True),
103- {
104- 'source': name,
105- 'target': {
106- 'url': url + name,
107- },
108- 'continuous': True,
109- }
110- )
111- self.assertEqual(
112- microfiber.push_replication(name, env, continuous=True, cancel=True),
113- {
114- 'source': name,
115- 'target': {
116- 'url': url + name,
117- },
118- 'continuous': True,
119- 'cancel': True,
120- }
121- )
122-
123- # Test with OAuth
124- tokens = random_oauth()
125- env = {'url': url, 'oauth': deepcopy(tokens)}
126- self.assertEqual(
127- microfiber.push_replication(name, env),
128- {
129- 'source': name,
130- 'target': {
131- 'url': url + name,
132- 'auth': {'oauth': tokens},
133- },
134- }
135- )
136- self.assertEqual(
137- microfiber.push_replication(name, env, cancel=True),
138- {
139- 'source': name,
140- 'target': {
141- 'url': url + name,
142- 'auth': {'oauth': tokens},
143- },
144- 'cancel': True,
145- }
146- )
147- self.assertEqual(
148- microfiber.push_replication(name, env, continuous=True),
149- {
150- 'source': name,
151- 'target': {
152- 'url': url + name,
153- 'auth': {'oauth': tokens},
154- },
155- 'continuous': True,
156- }
157- )
158- self.assertEqual(
159- microfiber.push_replication(name, env, continuous=True, cancel=True),
160- {
161- 'source': name,
162- 'target': {
163- 'url': url + name,
164- 'auth': {'oauth': tokens},
165- },
166- 'continuous': True,
167- 'cancel': True,
168- }
169- )
170-
171- # Test with basic HTTP auth
172- basic = random_basic()
173- headers = microfiber._basic_auth_header(basic)
174- env = {'url': url, 'basic': basic}
175- self.assertEqual(
176- microfiber.push_replication(name, env),
177- {
178- 'source': name,
179- 'target': {
180- 'url': url + name,
181- 'headers': headers,
182- },
183- }
184- )
185- self.assertEqual(
186- microfiber.push_replication(name, env, cancel=True),
187- {
188- 'source': name,
189- 'target': {
190- 'url': url + name,
191- 'headers': headers,
192- },
193- 'cancel': True,
194- }
195- )
196- self.assertEqual(
197- microfiber.push_replication(name, env, continuous=True),
198- {
199- 'source': name,
200- 'target': {
201- 'url': url + name,
202- 'headers': headers,
203- },
204- 'continuous': True,
205- }
206- )
207- self.assertEqual(
208- microfiber.push_replication(name, env, continuous=True, cancel=True),
209- {
210- 'source': name,
211- 'target': {
212- 'url': url + name,
213- 'headers': headers,
214- },
215- 'continuous': True,
216- 'cancel': True,
217- }
218- )
219-
220- # Test that OAuth takes precedence over basic auth
221- env['oauth'] = deepcopy(tokens)
222- self.assertEqual(
223- microfiber.push_replication(name, env),
224- {
225- 'source': name,
226- 'target': {
227- 'url': url + name,
228- 'auth': {'oauth': tokens},
229- },
230- }
231- )
232- self.assertEqual(
233- microfiber.push_replication(name, env, cancel=True),
234- {
235- 'source': name,
236- 'target': {
237- 'url': url + name,
238- 'auth': {'oauth': tokens},
239- },
240- 'cancel': True,
241- }
242- )
243- self.assertEqual(
244- microfiber.push_replication(name, env, continuous=True),
245- {
246- 'source': name,
247- 'target': {
248- 'url': url + name,
249- 'auth': {'oauth': tokens},
250- },
251- 'continuous': True,
252- }
253- )
254- self.assertEqual(
255- microfiber.push_replication(name, env, continuous=True, cancel=True),
256- {
257- 'source': name,
258- 'target': {
259- 'url': url + name,
260- 'auth': {'oauth': tokens},
261- },
262+ local_db = random_dbname()
263+ remote_db = random_dbname()
264+ remote_url = 'http://' + random_id().lower() + ':5984/'
265+
266+ # Test with no auth
267+ remote_env = {'url': remote_url}
268+ self.assertEqual(
269+ microfiber.push_replication(local_db, remote_db, remote_env),
270+ {
271+ 'source': local_db,
272+ 'target': {
273+ 'url': remote_url + remote_db,
274+ },
275+ }
276+ )
277+ self.assertEqual(
278+ microfiber.push_replication(local_db, remote_db, remote_env,
279+ cancel=True
280+ ),
281+ {
282+ 'source': local_db,
283+ 'target': {
284+ 'url': remote_url + remote_db,
285+ },
286+ 'cancel': True,
287+ }
288+ )
289+ self.assertEqual(
290+ microfiber.push_replication(local_db, remote_db, remote_env,
291+ continuous=True
292+ ),
293+ {
294+ 'source': local_db,
295+ 'target': {
296+ 'url': remote_url + remote_db,
297+ },
298+ 'continuous': True,
299+ }
300+ )
301+ self.assertEqual(
302+ microfiber.push_replication(local_db, remote_db, remote_env,
303+ continuous=True, cancel=True
304+ ),
305+ {
306+ 'source': local_db,
307+ 'target': {
308+ 'url': remote_url + remote_db,
309+ },
310+ 'continuous': True,
311+ 'cancel': True,
312+ }
313+ )
314+
315+ # Test with OAuth
316+ tokens = random_oauth()
317+ remote_env = {'url': remote_url, 'oauth': deepcopy(tokens)}
318+ self.assertEqual(
319+ microfiber.push_replication(local_db, remote_db, remote_env),
320+ {
321+ 'source': local_db,
322+ 'target': {
323+ 'url': remote_url + remote_db,
324+ 'auth': {'oauth': tokens},
325+ },
326+ }
327+ )
328+ self.assertEqual(
329+ microfiber.push_replication(local_db, remote_db, remote_env,
330+ cancel=True
331+ ),
332+ {
333+ 'source': local_db,
334+ 'target': {
335+ 'url': remote_url + remote_db,
336+ 'auth': {'oauth': tokens},
337+ },
338+ 'cancel': True,
339+ }
340+ )
341+ self.assertEqual(
342+ microfiber.push_replication(local_db, remote_db, remote_env,
343+ continuous=True
344+ ),
345+ {
346+ 'source': local_db,
347+ 'target': {
348+ 'url': remote_url + remote_db,
349+ 'auth': {'oauth': tokens},
350+ },
351+ 'continuous': True,
352+ }
353+ )
354+ self.assertEqual(
355+ microfiber.push_replication(local_db, remote_db, remote_env,
356+ continuous=True, cancel=True
357+ ),
358+ {
359+ 'source': local_db,
360+ 'target': {
361+ 'url': remote_url + remote_db,
362+ 'auth': {'oauth': tokens},
363+ },
364+ 'continuous': True,
365+ 'cancel': True,
366+ }
367+ )
368+
369+ # Test with basic HTTP auth
370+ basic = random_basic()
371+ headers = microfiber._basic_auth_header(basic)
372+ remote_env = {'url': remote_url, 'basic': basic}
373+ self.assertEqual(
374+ microfiber.push_replication(local_db, remote_db, remote_env),
375+ {
376+ 'source': local_db,
377+ 'target': {
378+ 'url': remote_url + remote_db,
379+ 'headers': headers,
380+ },
381+ }
382+ )
383+ self.assertEqual(
384+ microfiber.push_replication(local_db, remote_db, remote_env,
385+ cancel=True
386+ ),
387+ {
388+ 'source': local_db,
389+ 'target': {
390+ 'url': remote_url + remote_db,
391+ 'headers': headers,
392+ },
393+ 'cancel': True,
394+ }
395+ )
396+ self.assertEqual(
397+ microfiber.push_replication(local_db, remote_db, remote_env,
398+ continuous=True
399+ ),
400+ {
401+ 'source': local_db,
402+ 'target': {
403+ 'url': remote_url + remote_db,
404+ 'headers': headers,
405+ },
406+ 'continuous': True,
407+ }
408+ )
409+ self.assertEqual(
410+ microfiber.push_replication(local_db, remote_db, remote_env,
411+ continuous=True, cancel=True
412+ ),
413+ {
414+ 'source': local_db,
415+ 'target': {
416+ 'url': remote_url + remote_db,
417+ 'headers': headers,
418+ },
419+ 'continuous': True,
420+ 'cancel': True,
421+ }
422+ )
423+
424+ # Test that OAuth takes precedence over basic auth
425+ remote_env['oauth'] = deepcopy(tokens)
426+ self.assertEqual(
427+ microfiber.push_replication(local_db, remote_db, remote_env),
428+ {
429+ 'source': local_db,
430+ 'target': {
431+ 'url': remote_url + remote_db,
432+ 'auth': {'oauth': tokens},
433+ },
434+ }
435+ )
436+ self.assertEqual(
437+ microfiber.push_replication(local_db, remote_db, remote_env,
438+ cancel=True
439+ ),
440+ {
441+ 'source': local_db,
442+ 'target': {
443+ 'url': remote_url + remote_db,
444+ 'auth': {'oauth': tokens},
445+ },
446+ 'cancel': True,
447+ }
448+ )
449+ self.assertEqual(
450+ microfiber.push_replication(local_db, remote_db, remote_env,
451+ continuous=True
452+ ),
453+ {
454+ 'source': local_db,
455+ 'target': {
456+ 'url': remote_url + remote_db,
457+ 'auth': {'oauth': tokens},
458+ },
459+ 'continuous': True,
460+ }
461+ )
462+ self.assertEqual(
463+ microfiber.push_replication(local_db, remote_db, remote_env,
464+ continuous=True, cancel=True
465+ ),
466+ {
467+ 'source': local_db,
468+ 'target': {
469+ 'url': remote_url + remote_db,
470+ 'auth': {'oauth': tokens},
471+ },
472+ 'continuous': True,
473+ 'cancel': True,
474+ }
475+ )
476+
477+ def test_pull_replication(self):
478+ local_db = 'db-' + random_id().lower()
479+ remote_db = 'db-' + random_id().lower()
480+ remote_url = 'http://' + random_id().lower() + ':5984/'
481+
482+ # Test with no auth
483+ remote_env = {'url': remote_url}
484+ self.assertEqual(
485+ microfiber.pull_replication(local_db, remote_db, remote_env),
486+ {
487+ 'source': {
488+ 'url': remote_url + remote_db,
489+ },
490+ 'target': local_db,
491+ }
492+ )
493+ self.assertEqual(
494+ microfiber.pull_replication(local_db, remote_db, remote_env,
495+ cancel=True
496+ ),
497+ {
498+ 'source': {
499+ 'url': remote_url + remote_db,
500+ },
501+ 'target': local_db,
502+ 'cancel': True,
503+ }
504+ )
505+ self.assertEqual(
506+ microfiber.pull_replication(local_db, remote_db, remote_env,
507+ continuous=True
508+ ),
509+ {
510+ 'source': {
511+ 'url': remote_url + remote_db,
512+ },
513+ 'target': local_db,
514+ 'continuous': True,
515+ }
516+ )
517+ self.assertEqual(
518+ microfiber.pull_replication(local_db, remote_db, remote_env,
519+ continuous=True, cancel=True
520+ ),
521+ {
522+ 'source': {
523+ 'url': remote_url + remote_db,
524+ },
525+ 'target': local_db,
526+ 'continuous': True,
527+ 'cancel': True,
528+ }
529+ )
530+
531+ # Test with OAuth
532+ tokens = random_oauth()
533+ remote_env = {'url': remote_url, 'oauth': deepcopy(tokens)}
534+ self.assertEqual(
535+ microfiber.pull_replication(local_db, remote_db, remote_env),
536+ {
537+ 'source': {
538+ 'url': remote_url + remote_db,
539+ 'auth': {'oauth': tokens},
540+ },
541+ 'target': local_db,
542+ }
543+ )
544+ self.assertEqual(
545+ microfiber.pull_replication(local_db, remote_db, remote_env,
546+ cancel=True
547+ ),
548+ {
549+ 'source': {
550+ 'url': remote_url + remote_db,
551+ 'auth': {'oauth': tokens},
552+ },
553+ 'target': local_db,
554+ 'cancel': True,
555+ }
556+ )
557+ self.assertEqual(
558+ microfiber.pull_replication(local_db, remote_db, remote_env,
559+ continuous=True
560+ ),
561+ {
562+ 'source': {
563+ 'url': remote_url + remote_db,
564+ 'auth': {'oauth': tokens},
565+ },
566+ 'target': local_db,
567+ 'continuous': True,
568+ }
569+ )
570+ self.assertEqual(
571+ microfiber.pull_replication(local_db, remote_db, remote_env,
572+ continuous=True, cancel=True
573+ ),
574+ {
575+ 'source': {
576+ 'url': remote_url + remote_db,
577+ 'auth': {'oauth': tokens},
578+ },
579+ 'target': local_db,
580+ 'continuous': True,
581+ 'cancel': True,
582+ }
583+ )
584+
585+ # Test with basic HTTP auth
586+ basic = random_basic()
587+ headers = microfiber._basic_auth_header(basic)
588+ remote_env = {'url': remote_url, 'basic': basic}
589+ self.assertEqual(
590+ microfiber.pull_replication(local_db, remote_db, remote_env),
591+ {
592+ 'source': {
593+ 'url': remote_url + remote_db,
594+ 'headers': headers,
595+ },
596+ 'target': local_db,
597+ }
598+ )
599+ self.assertEqual(
600+ microfiber.pull_replication(local_db, remote_db, remote_env,
601+ cancel=True
602+ ),
603+ {
604+ 'source': {
605+ 'url': remote_url + remote_db,
606+ 'headers': headers,
607+ },
608+ 'target': local_db,
609+ 'cancel': True,
610+ }
611+ )
612+ self.assertEqual(
613+ microfiber.pull_replication(local_db, remote_db, remote_env,
614+ continuous=True
615+ ),
616+ {
617+ 'source': {
618+ 'url': remote_url + remote_db,
619+ 'headers': headers,
620+ },
621+ 'target': local_db,
622+ 'continuous': True,
623+ }
624+ )
625+ self.assertEqual(
626+ microfiber.pull_replication(local_db, remote_db, remote_env,
627+ continuous=True, cancel=True
628+ ),
629+ {
630+ 'source': {
631+ 'url': remote_url + remote_db,
632+ 'headers': headers,
633+ },
634+ 'target': local_db,
635+ 'continuous': True,
636+ 'cancel': True,
637+ }
638+ )
639+
640+ # Test that OAuth takes precedence over basic auth
641+ remote_env['oauth'] = deepcopy(tokens)
642+ self.assertEqual(
643+ microfiber.pull_replication(local_db, remote_db, remote_env),
644+ {
645+ 'source': {
646+ 'url': remote_url + remote_db,
647+ 'auth': {'oauth': tokens},
648+ },
649+ 'target': local_db,
650+ }
651+ )
652+ self.assertEqual(
653+ microfiber.pull_replication(local_db, remote_db, remote_env,
654+ cancel=True
655+ ),
656+ {
657+ 'source': {
658+ 'url': remote_url + remote_db,
659+ 'auth': {'oauth': tokens},
660+ },
661+ 'target': local_db,
662+ 'cancel': True,
663+ }
664+ )
665+ self.assertEqual(
666+ microfiber.pull_replication(local_db, remote_db, remote_env,
667+ continuous=True
668+ ),
669+ {
670+ 'source': {
671+ 'url': remote_url + remote_db,
672+ 'auth': {'oauth': tokens},
673+ },
674+ 'target': local_db,
675+ 'continuous': True,
676+ }
677+ )
678+ self.assertEqual(
679+ microfiber.pull_replication(local_db, remote_db, remote_env,
680+ continuous=True, cancel=True
681+ ),
682+ {
683+ 'source': {
684+ 'url': remote_url + remote_db,
685+ 'auth': {'oauth': tokens},
686+ },
687+ 'target': local_db,
688 'continuous': True,
689 'cancel': True,
690 }
691@@ -986,36 +1231,74 @@
692 def test_push(self):
693 s1 = microfiber.Server(self.env1)
694 s2 = microfiber.Server(self.env2)
695-
696- # Create databases
697- self.assertEqual(s1.put(None, 'foo'), {'ok': True})
698- self.assertEqual(s2.put(None, 'foo'), {'ok': True})
699-
700- # Start continuous s1 => s2 replication of foo
701- result = s1.push('foo', self.env2, continuous=True)
702- self.assertEqual(set(result), set(['_local_id', 'ok']))
703- self.assertIs(result['ok'], True)
704-
705- # Save docs in s1, make sure they show up in s2
706- docs1 = [{'_id': test_id()} for i in range(100)]
707- for doc in docs1:
708- doc['_rev'] = s1.post(doc, 'foo')['rev']
709- time.sleep(1)
710- for doc in docs1:
711- self.assertEqual(s2.get('foo', doc['_id']), doc)
712-
713- # Start continuous s2 => s1 replication of foo
714- result = s2.push('foo', self.env1, continuous=True)
715- self.assertEqual(set(result), set(['_local_id', 'ok']))
716- self.assertIs(result['ok'], True)
717-
718- # Save docs in s2, make sure they show up in s1
719- docs2 = [{'_id': test_id(), 'two': True} for i in range(100)]
720- for doc in docs2:
721- doc['_rev'] = s2.post(doc, 'foo')['rev']
722- time.sleep(1)
723- for doc in docs2:
724- self.assertEqual(s1.get('foo', doc['_id']), doc)
725+ name1 = random_dbname()
726+ name2 = random_dbname()
727+
728+ # Create databases
729+ self.assertEqual(s1.put(None, name1), {'ok': True})
730+ self.assertEqual(s2.put(None, name2), {'ok': True})
731+
732+ # Start continuous s1.name1 -> s2.name2 push replication
733+ result = s1.push(name1, name2, self.env2, continuous=True)
734+ self.assertEqual(set(result), set(['_local_id', 'ok']))
735+ self.assertIs(result['ok'], True)
736+
737+ # Save docs in s1.name1, make sure they show up in s2.name2
738+ docs1 = [{'_id': test_id()} for i in range(100)]
739+ for doc in docs1:
740+ doc['_rev'] = s1.post(doc, name1)['rev']
741+ time.sleep(1)
742+ for doc in docs1:
743+ self.assertEqual(s2.get(name2, doc['_id']), doc)
744+
745+ # Start continuous s2.name2 -> s1.name1 push replication
746+ result = s2.push(name2, name1, self.env1, continuous=True)
747+ self.assertEqual(set(result), set(['_local_id', 'ok']))
748+ self.assertIs(result['ok'], True)
749+
750+ # Save docs in s2.name2, make sure they show up in s1.name1
751+ docs2 = [{'_id': test_id(), 'two': True} for i in range(100)]
752+ for doc in docs2:
753+ doc['_rev'] = s2.post(doc, name2)['rev']
754+ time.sleep(1)
755+ for doc in docs2:
756+ self.assertEqual(s1.get(name1, doc['_id']), doc)
757+
758+ def test_pull(self):
759+ s1 = microfiber.Server(self.env1)
760+ s2 = microfiber.Server(self.env2)
761+ name1 = random_dbname()
762+ name2 = random_dbname()
763+
764+ # Create databases
765+ self.assertEqual(s1.put(None, name1), {'ok': True})
766+ self.assertEqual(s2.put(None, name2), {'ok': True})
767+
768+ # Start continuous s1.name1 <- s2.name2 pull replication
769+ result = s1.pull(name1, name2, self.env2, continuous=True)
770+ self.assertEqual(set(result), set(['_local_id', 'ok']))
771+ self.assertIs(result['ok'], True)
772+
773+ # Save docs in s2.name2, make sure they show up in s1.name1
774+ docs1 = [{'_id': test_id()} for i in range(100)]
775+ for doc in docs1:
776+ doc['_rev'] = s2.post(doc, name2)['rev']
777+ time.sleep(1)
778+ for doc in docs1:
779+ self.assertEqual(s1.get(name1, doc['_id']), doc)
780+
781+ # Start continuous s2.name2 <- s1.name1 pull replication
782+ result = s2.pull(name2, name1, self.env1, continuous=True)
783+ self.assertEqual(set(result), set(['_local_id', 'ok']))
784+ self.assertIs(result['ok'], True)
785+
786+ # Save docs in s1.name1, make sure they show up in s2.name2
787+ docs2 = [{'_id': test_id(), 'two': True} for i in range(100)]
788+ for doc in docs2:
789+ doc['_rev'] = s1.post(doc, name1)['rev']
790+ time.sleep(1)
791+ for doc in docs2:
792+ self.assertEqual(s2.get(name2, doc['_id']), doc)
793
794
795 class LiveTestCase(TestCase):

Subscribers

People subscribed via source and target branches