Microfiber

Merge lp:~jderose/microfiber/thread-safe into lp:microfiber

  1. thread-safe
  2. Merge into trunk
Proposed by Jason Gerard DeRose
Status: Merged
Merged at revision: 109
Proposed branch: lp:~jderose/microfiber/thread-safe
Merge into: lp:microfiber
Diff against target: 177 lines (+37/-4)
4 files modified
benchmark_microfiber.py (+3/-1)
benchmark_pycouchdb.py (+4/-0)
microfiber.py (+9/-2)
test_microfiber.py (+21/-1)
To merge this branch: bzr merge lp:~jderose/microfiber/thread-safe
Reviewer Review Type Date Requested Status
Jason Gerard DeRose Approve
Review via email: mp+104670@code.launchpad.net

Description of the change

Changes:

 * New CouchBase.Conn attribute stores the connection class (either HTTPConnection or HTTPSConnection)

 * New CouchBase._threadlocal attribute is a threading.local instance

 * CocuhBase.conn is now a property, which lazily creates the connection instance the first time it's needed in the current thread

 * A few tweaks to benchmark_microfiber.py that seemingly make the results more consistent

Note that there is no measurable performance difference between the new thread-safe code and the old code.

To post a comment you must log in.
lp:~jderose/microfiber/thread-safe updated
112. By Jason Gerard DeRose

Updated pycouchdb benchmark

Revision history for this message
Jason Gerard DeRose (jderose) wrote :

I'm gonna self-approve as we need this in a hurry for some features we're going to be using at UDS.

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'benchmark_microfiber.py'
2--- benchmark_microfiber.py 2012-01-23 10:46:06 +0000
3+++ benchmark_microfiber.py 2012-05-04 05:44:17 +0000
4@@ -4,6 +4,7 @@
5 import platform
6 import json
7 import optparse
8+from subprocess import check_call
9
10 from usercouch.misc import TempCouch
11 import microfiber
12@@ -26,9 +27,10 @@
13 auth = ('oauth' if options.oauth else 'basic')
14 env = tmpcouch.bootstrap(auth)
15 print('\nenv = {!r}\n'.format(env))
16-time.sleep(2) # Let CouchDB settle a moment
17 db = microfiber.Database(name, env)
18 db.put(None)
19+time.sleep(3) # Let CouchDB settle a moment
20+check_call(['/bin/sync']) # Flush any pending IO so test is more consistent
21
22 master = dict(
23 ('a' * i, 'b' * i) for i in range(1, keys)
24
25=== modified file 'benchmark_pycouchdb.py'
26--- benchmark_pycouchdb.py 2011-03-27 23:14:39 +0000
27+++ benchmark_pycouchdb.py 2012-05-04 05:44:17 +0000
28@@ -6,6 +6,7 @@
29 import time
30 import platform
31 import json
32+from subprocess import check_call
33
34 import couchdb
35
36@@ -20,6 +21,9 @@
37 pass
38 db = s.create(name)
39
40+time.sleep(3) # Let CouchDB settle a moment
41+check_call(['/bin/sync']) # Flush any pending IO so test is more consistent
42+
43
44 def random_id():
45 return b32encode(os.urandom(15))
46
47=== modified file 'microfiber.py'
48--- microfiber.py 2012-04-24 03:53:02 +0000
49+++ microfiber.py 2012-05-04 05:44:17 +0000
50@@ -48,6 +48,7 @@
51 import hmac
52 from urllib.parse import urlparse, urlencode, quote_plus
53 from http.client import HTTPConnection, HTTPSConnection, BadStatusLine
54+import threading
55
56
57 __all__ = (
58@@ -379,8 +380,14 @@
59 self.url = self._full_url(self.basepath)
60 self._oauth = self.env.get('oauth')
61 self._basic = self.env.get('basic')
62- klass = (HTTPConnection if t.scheme == 'http' else HTTPSConnection)
63- self.conn = klass(t.netloc)
64+ self.Conn = (HTTPConnection if t.scheme == 'http' else HTTPSConnection)
65+ self._threadlocal = threading.local()
66+
67+ @property
68+ def conn(self):
69+ if not hasattr(self._threadlocal, 'conn'):
70+ self._threadlocal.conn = self.Conn(self.netloc)
71+ return self._threadlocal.conn
72
73 def _full_url(self, path):
74 return ''.join([self.scheme, '://', self.netloc, path])
75
76=== modified file 'test_microfiber.py'
77--- test_microfiber.py 2011-12-21 05:33:29 +0000
78+++ test_microfiber.py 2012-05-04 05:44:17 +0000
79@@ -37,6 +37,7 @@
80 from hashlib import md5
81 from urllib.parse import urlparse, urlencode
82 from http.client import HTTPConnection, HTTPSConnection
83+import threading
84
85 try:
86 import usercouch.misc
87@@ -309,6 +310,7 @@
88 self.assertEqual(inst.url, 'https://localhost:5984/couch/')
89 self.assertEqual(inst.basepath, '/couch/')
90 self.assertIsInstance(inst.conn, HTTPSConnection)
91+ self.assertIs(inst.Conn, HTTPSConnection)
92 self.assertIsNone(inst._oauth)
93 self.assertIsNone(inst._basic)
94
95@@ -316,39 +318,53 @@
96 self.assertEqual(inst.url, 'http://localhost:5984/')
97 self.assertEqual(inst.basepath, '/')
98 self.assertIsInstance(inst.conn, HTTPConnection)
99+ self.assertIs(inst.Conn, HTTPConnection)
100 self.assertIsNone(inst._oauth)
101 self.assertIsNone(inst._basic)
102
103 inst = self.klass('http://localhost:5001/')
104 self.assertEqual(inst.url, 'http://localhost:5001/')
105 self.assertIsInstance(inst.conn, HTTPConnection)
106+ self.assertIs(inst.Conn, HTTPConnection)
107 self.assertIsNone(inst._oauth)
108 self.assertIsNone(inst._basic)
109
110 inst = self.klass('http://localhost:5002')
111 self.assertEqual(inst.url, 'http://localhost:5002/')
112 self.assertIsInstance(inst.conn, HTTPConnection)
113+ self.assertIs(inst.Conn, HTTPConnection)
114 self.assertIsNone(inst._oauth)
115 self.assertIsNone(inst._basic)
116
117 inst = self.klass('https://localhost:5003/')
118 self.assertEqual(inst.url, 'https://localhost:5003/')
119 self.assertIsInstance(inst.conn, HTTPSConnection)
120+ self.assertIs(inst.Conn, HTTPSConnection)
121 self.assertIsNone(inst._oauth)
122 self.assertIsNone(inst._basic)
123
124 inst = self.klass('https://localhost:5004')
125 self.assertEqual(inst.url, 'https://localhost:5004/')
126 self.assertIsInstance(inst.conn, HTTPSConnection)
127+ self.assertIs(inst.Conn, HTTPSConnection)
128 self.assertIsNone(inst._oauth)
129 self.assertIsNone(inst._basic)
130
131 inst = self.klass({'oauth': 'foo'})
132 self.assertEqual(inst._oauth, 'foo')
133-
134+
135 inst = self.klass({'basic': 'bar'})
136 self.assertEqual(inst._basic, 'bar')
137
138+ def test_conn(self):
139+ inst = microfiber.CouchBase()
140+ self.assertIsInstance(inst._threadlocal, threading.local)
141+ value = random_id()
142+ inst._threadlocal.conn = value
143+ self.assertEqual(inst.conn, value)
144+ delattr(inst._threadlocal, 'conn')
145+ self.assertIsInstance(inst.conn, HTTPConnection)
146+
147 def test_full_url(self):
148 inst = self.klass('https://localhost:5003/')
149 self.assertEqual(
150@@ -379,23 +395,27 @@
151 self.assertEqual(inst.url, 'http://localhost:5984/')
152 self.assertEqual(inst.basepath, '/')
153 self.assertIsInstance(inst.conn, HTTPConnection)
154+ self.assertIs(inst.Conn, HTTPConnection)
155 self.assertNotIsInstance(inst.conn, HTTPSConnection)
156
157 inst = self.klass('https://localhost:6000')
158 self.assertEqual(inst.url, 'https://localhost:6000/')
159 self.assertEqual(inst.basepath, '/')
160 self.assertIsInstance(inst.conn, HTTPSConnection)
161+ self.assertIs(inst.Conn, HTTPSConnection)
162
163 inst = self.klass('http://example.com/foo')
164 self.assertEqual(inst.url, 'http://example.com/foo/')
165 self.assertEqual(inst.basepath, '/foo/')
166 self.assertIsInstance(inst.conn, HTTPConnection)
167+ self.assertIs(inst.Conn, HTTPConnection)
168 self.assertNotIsInstance(inst.conn, HTTPSConnection)
169
170 inst = self.klass('https://example.com/bar')
171 self.assertEqual(inst.url, 'https://example.com/bar/')
172 self.assertEqual(inst.basepath, '/bar/')
173 self.assertIsInstance(inst.conn, HTTPSConnection)
174+ self.assertIs(inst.Conn, HTTPSConnection)
175
176 inst = self.klass({'oauth': 'bar'})
177 self.assertEqual(inst._oauth, 'bar')

Subscribers

People subscribed via source and target branches