Merge lp:~cthier/swift/drybins into lp:~hudson-openstack/swift/trunk
- drybins
- Merge into trunk
Status: | Merged | ||||
---|---|---|---|---|---|
Approved by: | gholt | ||||
Approved revision: | 68 | ||||
Merged at revision: | 67 | ||||
Proposed branch: | lp:~cthier/swift/drybins | ||||
Merge into: | lp:~hudson-openstack/swift/trunk | ||||
Diff against target: |
1152 lines (+241/-320) 26 files modified
bin/swift-account-auditor (+1/-30) bin/swift-account-reaper (+1/-30) bin/swift-account-replicator (+8/-25) bin/swift-container-auditor (+1/-30) bin/swift-container-replicator (+8/-24) bin/swift-container-updater (+1/-23) bin/swift-object-auditor (+1/-27) bin/swift-object-replicator (+5/-44) bin/swift-object-updater (+1/-24) etc/proxy-server.conf-sample (+2/-2) swift/account/auditor.py (+5/-4) swift/account/reaper.py (+9/-7) swift/account/replicator.py (+26/-0) swift/common/daemon.py (+60/-0) swift/common/db_replicator.py (+7/-6) swift/common/utils.py (+2/-0) swift/container/auditor.py (+5/-3) swift/container/replicator.py (+25/-0) swift/container/updater.py (+7/-5) swift/obj/auditor.py (+8/-6) swift/obj/replicator.py (+34/-8) swift/obj/updater.py (+6/-4) test/unit/common/test_db_replicator.py (+2/-2) test/unit/container/test_updater.py (+7/-7) test/unit/obj/test_replicator.py (+3/-3) test/unit/obj/test_updater.py (+6/-6) |
||||
To merge this branch: | bzr merge lp:~cthier/swift/drybins | ||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
gholt (community) | Approve | ||
Review via email: mp+34256@code.launchpad.net |
Commit message
Description of the change
Refactored bin files to be more DRY
Chuck Thier (cthier) wrote : | # |
gholt (gholt) wrote : | # |
swift/account/
swift/account/
swift/common/
Chuck Thier (cthier) wrote : | # |
> swift/account/
> swift/account/
> swift/common/
oops... thanks for catching that, should be fixed now
Jay Pipes (jaypipes) wrote : | # |
Hi! You may want to use the existing daemon Python library instead of rewriting that stuff ;)
http://
Also, a litte nit: the standard way of passing options for daemonizing a process is not to pass "once" as a CLI arg, but to use CLI options, such as --nodaemon
Chuck Thier (cthier) wrote : | # |
> Hi! You may want to use the existing daemon Python library instead of
> rewriting that stuff ;)
>
> http://
>
> Also, a litte nit: the standard way of passing options for daemonizing a
> process is not to pass "once" as a CLI arg, but to use CLI options, such as
> --nodaemon
Hey Jay, The main purpose of this was to refactor what we have currently to be more DRY. I'm not against moving to a standard lib for doing the daemon stuff, but don't have time to evaulate them. And as to the CLI args, I would like for those to change eventually as well, but that requires other changes as well, and was trying to keep the changes as minimal as possible.
Preview Diff
1 | === modified file 'bin/swift-account-auditor' | |||
2 | --- bin/swift-account-auditor 2010-08-24 13:41:58 +0000 | |||
3 | +++ bin/swift-account-auditor 2010-09-01 15:57:44 +0000 | |||
4 | @@ -14,10 +14,7 @@ | |||
5 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
6 | 15 | # limitations under the License. | 15 | # limitations under the License. |
7 | 16 | 16 | ||
8 | 17 | import os | ||
9 | 18 | import signal | ||
10 | 19 | import sys | 17 | import sys |
11 | 20 | from ConfigParser import ConfigParser | ||
12 | 21 | 18 | ||
13 | 22 | from swift.account.auditor import AccountAuditor | 19 | from swift.account.auditor import AccountAuditor |
14 | 23 | from swift.common import utils | 20 | from swift.common import utils |
15 | @@ -26,32 +23,6 @@ | |||
16 | 26 | if len(sys.argv) < 2: | 23 | if len(sys.argv) < 2: |
17 | 27 | print "Usage: swift-account-auditor CONFIG_FILE [once]" | 24 | print "Usage: swift-account-auditor CONFIG_FILE [once]" |
18 | 28 | sys.exit() | 25 | sys.exit() |
19 | 29 | |||
20 | 30 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' |
21 | 31 | |||
22 | 32 | conf = utils.readconf(sys.argv[1], 'account-auditor') | 27 | conf = utils.readconf(sys.argv[1], 'account-auditor') |
48 | 33 | logger = utils.get_logger(conf) | 28 | auditor = AccountAuditor(conf).run(once) |
24 | 34 | # log uncaught exceptions | ||
25 | 35 | sys.excepthook = lambda *exc_info: \ | ||
26 | 36 | logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) | ||
27 | 37 | sys.stdout = sys.stderr = utils.LoggerFileObject(logger) | ||
28 | 38 | |||
29 | 39 | utils.drop_privileges(conf.get('user', 'swift')) | ||
30 | 40 | |||
31 | 41 | try: | ||
32 | 42 | os.setsid() | ||
33 | 43 | except OSError: | ||
34 | 44 | pass | ||
35 | 45 | |||
36 | 46 | def kill_children(*args): | ||
37 | 47 | signal.signal(signal.SIGTERM, signal.SIG_IGN) | ||
38 | 48 | os.killpg(0, signal.SIGTERM) | ||
39 | 49 | sys.exit() | ||
40 | 50 | |||
41 | 51 | signal.signal(signal.SIGTERM, kill_children) | ||
42 | 52 | |||
43 | 53 | auditor = AccountAuditor(conf) | ||
44 | 54 | if once: | ||
45 | 55 | auditor.audit_once() | ||
46 | 56 | else: | ||
47 | 57 | auditor.audit_forever() | ||
49 | 58 | 29 | ||
50 | === modified file 'bin/swift-account-reaper' | |||
51 | --- bin/swift-account-reaper 2010-08-24 13:41:58 +0000 | |||
52 | +++ bin/swift-account-reaper 2010-09-01 15:57:44 +0000 | |||
53 | @@ -14,10 +14,7 @@ | |||
54 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
55 | 15 | # limitations under the License. | 15 | # limitations under the License. |
56 | 16 | 16 | ||
57 | 17 | import os | ||
58 | 18 | import signal | ||
59 | 19 | import sys | 17 | import sys |
60 | 20 | from ConfigParser import ConfigParser | ||
61 | 21 | 18 | ||
62 | 22 | from swift.account.reaper import AccountReaper | 19 | from swift.account.reaper import AccountReaper |
63 | 23 | from swift.common import utils | 20 | from swift.common import utils |
64 | @@ -26,32 +23,6 @@ | |||
65 | 26 | if len(sys.argv) < 2: | 23 | if len(sys.argv) < 2: |
66 | 27 | print "Usage: account-reaper CONFIG_FILE [once]" | 24 | print "Usage: account-reaper CONFIG_FILE [once]" |
67 | 28 | sys.exit() | 25 | sys.exit() |
68 | 29 | |||
69 | 30 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' |
70 | 31 | |||
71 | 32 | conf = utils.readconf(sys.argv[1], 'account-reaper') | 27 | conf = utils.readconf(sys.argv[1], 'account-reaper') |
97 | 33 | logger = utils.get_logger(conf) | 28 | reaper = AccountReaper(conf).run(once) |
73 | 34 | # log uncaught exceptions | ||
74 | 35 | sys.excepthook = lambda *exc_info: \ | ||
75 | 36 | logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) | ||
76 | 37 | sys.stdout = sys.stderr = utils.LoggerFileObject(logger) | ||
77 | 38 | |||
78 | 39 | utils.drop_privileges(conf.get('user', 'swift')) | ||
79 | 40 | |||
80 | 41 | try: | ||
81 | 42 | os.setsid() | ||
82 | 43 | except OSError: | ||
83 | 44 | pass | ||
84 | 45 | |||
85 | 46 | def kill_children(*args): | ||
86 | 47 | signal.signal(signal.SIGTERM, signal.SIG_IGN) | ||
87 | 48 | os.killpg(0, signal.SIGTERM) | ||
88 | 49 | sys.exit() | ||
89 | 50 | |||
90 | 51 | signal.signal(signal.SIGTERM, kill_children) | ||
91 | 52 | |||
92 | 53 | reaper = AccountReaper(conf) | ||
93 | 54 | if once: | ||
94 | 55 | reaper.reap_once() | ||
95 | 56 | else: | ||
96 | 57 | reaper.reap_forever() | ||
98 | 58 | 29 | ||
99 | === modified file 'bin/swift-account-replicator' | |||
100 | --- bin/swift-account-replicator 2010-08-24 13:41:58 +0000 | |||
101 | +++ bin/swift-account-replicator 2010-09-01 15:57:44 +0000 | |||
102 | @@ -15,31 +15,14 @@ | |||
103 | 15 | # limitations under the License. | 15 | # limitations under the License. |
104 | 16 | 16 | ||
105 | 17 | import sys | 17 | import sys |
118 | 18 | from ConfigParser import ConfigParser | 18 | |
119 | 19 | import getopt | 19 | from swift.common import utils |
120 | 20 | 20 | from swift.account.replicator import AccountReplicator | |
109 | 21 | from swift.account import server as account_server | ||
110 | 22 | from swift.common import db, db_replicator, utils | ||
111 | 23 | |||
112 | 24 | class AccountReplicator(db_replicator.Replicator): | ||
113 | 25 | server_type = 'account' | ||
114 | 26 | ring_file = 'account.ring.gz' | ||
115 | 27 | brokerclass = db.AccountBroker | ||
116 | 28 | datadir = account_server.DATADIR | ||
117 | 29 | default_port = 6002 | ||
121 | 30 | 21 | ||
122 | 31 | if __name__ == '__main__': | 22 | if __name__ == '__main__': |
130 | 32 | optlist, args = getopt.getopt(sys.argv[1:], '', ['once']) | 23 | if len(sys.argv) < 2: |
131 | 33 | 24 | print "Usage: swift-account-replicator CONFIG_FILE [once]" | |
132 | 34 | if not args: | 25 | sys.exit(1) |
133 | 35 | print "Usage: swift-account-replicator <--once> CONFIG_FILE [once]" | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' |
127 | 36 | sys.exit() | ||
128 | 37 | |||
129 | 38 | once = len(args) > 1 and args[1] == 'once' | ||
134 | 39 | conf = utils.readconf(sys.argv[1], 'account-replicator') | 27 | conf = utils.readconf(sys.argv[1], 'account-replicator') |
141 | 40 | utils.drop_privileges(conf.get('user', 'swift')) | 28 | AccountReplicator(conf).run(once) |
136 | 41 | if once or '--once' in [opt[0] for opt in optlist]: | ||
137 | 42 | AccountReplicator(conf).replicate_once() | ||
138 | 43 | else: | ||
139 | 44 | AccountReplicator(conf).replicate_forever() | ||
140 | 45 | |||
142 | 46 | 29 | ||
143 | === modified file 'bin/swift-container-auditor' | |||
144 | --- bin/swift-container-auditor 2010-08-24 13:41:58 +0000 | |||
145 | +++ bin/swift-container-auditor 2010-09-01 15:57:44 +0000 | |||
146 | @@ -14,10 +14,7 @@ | |||
147 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
148 | 15 | # limitations under the License. | 15 | # limitations under the License. |
149 | 16 | 16 | ||
150 | 17 | import os | ||
151 | 18 | import signal | ||
152 | 19 | import sys | 17 | import sys |
153 | 20 | from ConfigParser import ConfigParser | ||
154 | 21 | 18 | ||
155 | 22 | from swift.container.auditor import ContainerAuditor | 19 | from swift.container.auditor import ContainerAuditor |
156 | 23 | from swift.common import utils | 20 | from swift.common import utils |
157 | @@ -26,32 +23,6 @@ | |||
158 | 26 | if len(sys.argv) < 2: | 23 | if len(sys.argv) < 2: |
159 | 27 | print "Usage: swift-container-auditor CONFIG_FILE [once]" | 24 | print "Usage: swift-container-auditor CONFIG_FILE [once]" |
160 | 28 | sys.exit() | 25 | sys.exit() |
161 | 29 | |||
162 | 30 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' |
163 | 31 | |||
164 | 32 | conf = utils.readconf(sys.argv[1], 'container-auditor') | 27 | conf = utils.readconf(sys.argv[1], 'container-auditor') |
190 | 33 | logger = utils.get_logger(conf) | 28 | ContainerAuditor(conf).run(once) |
166 | 34 | # log uncaught exceptions | ||
167 | 35 | sys.excepthook = lambda *exc_info: \ | ||
168 | 36 | logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) | ||
169 | 37 | sys.stdout = sys.stderr = utils.LoggerFileObject(logger) | ||
170 | 38 | |||
171 | 39 | utils.drop_privileges(conf.get('user', 'swift')) | ||
172 | 40 | |||
173 | 41 | try: | ||
174 | 42 | os.setsid() | ||
175 | 43 | except OSError: | ||
176 | 44 | pass | ||
177 | 45 | |||
178 | 46 | def kill_children(*args): | ||
179 | 47 | signal.signal(signal.SIGTERM, signal.SIG_IGN) | ||
180 | 48 | os.killpg(0, signal.SIGTERM) | ||
181 | 49 | sys.exit() | ||
182 | 50 | |||
183 | 51 | signal.signal(signal.SIGTERM, kill_children) | ||
184 | 52 | |||
185 | 53 | auditor = ContainerAuditor(conf) | ||
186 | 54 | if once: | ||
187 | 55 | auditor.audit_once() | ||
188 | 56 | else: | ||
189 | 57 | auditor.audit_forever() | ||
191 | 58 | 29 | ||
192 | === modified file 'bin/swift-container-replicator' | |||
193 | --- bin/swift-container-replicator 2010-08-24 13:41:58 +0000 | |||
194 | +++ bin/swift-container-replicator 2010-09-01 15:57:44 +0000 | |||
195 | @@ -15,31 +15,15 @@ | |||
196 | 15 | # limitations under the License. | 15 | # limitations under the License. |
197 | 16 | 16 | ||
198 | 17 | import sys | 17 | import sys |
211 | 18 | from ConfigParser import ConfigParser | 18 | |
212 | 19 | import getopt | 19 | from swift.common import db, utils |
213 | 20 | 20 | from swift.container.replicator import ContainerReplicator | |
202 | 21 | from swift.container import server as container_server | ||
203 | 22 | from swift.common import db, db_replicator, utils | ||
204 | 23 | |||
205 | 24 | class ContainerReplicator(db_replicator.Replicator): | ||
206 | 25 | server_type = 'container' | ||
207 | 26 | ring_file = 'container.ring.gz' | ||
208 | 27 | brokerclass = db.ContainerBroker | ||
209 | 28 | datadir = container_server.DATADIR | ||
210 | 29 | default_port = 6001 | ||
214 | 30 | 21 | ||
215 | 31 | if __name__ == '__main__': | 22 | if __name__ == '__main__': |
220 | 32 | optlist, args = getopt.getopt(sys.argv[1:], '', ['once']) | 23 | if len(sys.argv) < 2: |
221 | 33 | 24 | print "Usage: swift-container-replicator CONFIG_FILE [once]" | |
218 | 34 | if not args: | ||
219 | 35 | print "Usage: swift-container-replicator <--once> CONFIG_FILE [once]" | ||
222 | 36 | sys.exit(1) | 25 | sys.exit(1) |
231 | 37 | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | |
232 | 38 | once = len(args) > 1 and args[1] == 'once' | 27 | conf = utils.readconf(sys.argv[1], 'container-replicator') |
233 | 39 | conf = utils.readconf(args[0], 'container-replicator') | 28 | ContainerReplicator(conf).run(once) |
226 | 40 | utils.drop_privileges(conf.get('user', 'swift')) | ||
227 | 41 | if once or '--once' in [opt[0] for opt in optlist]: | ||
228 | 42 | ContainerReplicator(conf).replicate_once() | ||
229 | 43 | else: | ||
230 | 44 | ContainerReplicator(conf).replicate_forever() | ||
234 | 45 | 29 | ||
235 | 46 | 30 | ||
236 | === modified file 'bin/swift-container-updater' | |||
237 | --- bin/swift-container-updater 2010-08-24 14:55:20 +0000 | |||
238 | +++ bin/swift-container-updater 2010-09-01 15:57:44 +0000 | |||
239 | @@ -14,10 +14,7 @@ | |||
240 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
241 | 15 | # limitations under the License. | 15 | # limitations under the License. |
242 | 16 | 16 | ||
243 | 17 | import os | ||
244 | 18 | import signal | ||
245 | 19 | import sys | 17 | import sys |
246 | 20 | from ConfigParser import ConfigParser | ||
247 | 21 | 18 | ||
248 | 22 | from swift.container.updater import ContainerUpdater | 19 | from swift.container.updater import ContainerUpdater |
249 | 23 | from swift.common import utils | 20 | from swift.common import utils |
250 | @@ -26,25 +23,6 @@ | |||
251 | 26 | if len(sys.argv) < 2: | 23 | if len(sys.argv) < 2: |
252 | 27 | print "Usage: swift-container-updater CONFIG_FILE [once]" | 24 | print "Usage: swift-container-updater CONFIG_FILE [once]" |
253 | 28 | sys.exit() | 25 | sys.exit() |
254 | 29 | |||
255 | 30 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' |
256 | 31 | conf = utils.readconf(sys.argv[1], 'container-updater') | 27 | conf = utils.readconf(sys.argv[1], 'container-updater') |
276 | 32 | utils.drop_privileges(conf.get('user', 'swift')) | 28 | ContainerUpdater(conf).run(once) |
258 | 33 | |||
259 | 34 | try: | ||
260 | 35 | os.setsid() | ||
261 | 36 | except OSError: | ||
262 | 37 | pass | ||
263 | 38 | |||
264 | 39 | def kill_children(*args): | ||
265 | 40 | signal.signal(signal.SIGTERM, signal.SIG_IGN) | ||
266 | 41 | os.killpg(0, signal.SIGTERM) | ||
267 | 42 | sys.exit() | ||
268 | 43 | |||
269 | 44 | signal.signal(signal.SIGTERM, kill_children) | ||
270 | 45 | |||
271 | 46 | updater = ContainerUpdater(conf) | ||
272 | 47 | if once: | ||
273 | 48 | updater.update_once_single_threaded() | ||
274 | 49 | else: | ||
275 | 50 | updater.update_forever() | ||
277 | 51 | 29 | ||
278 | === modified file 'bin/swift-object-auditor' | |||
279 | --- bin/swift-object-auditor 2010-08-24 13:41:58 +0000 | |||
280 | +++ bin/swift-object-auditor 2010-09-01 15:57:44 +0000 | |||
281 | @@ -14,8 +14,6 @@ | |||
282 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
283 | 15 | # limitations under the License. | 15 | # limitations under the License. |
284 | 16 | 16 | ||
285 | 17 | import os | ||
286 | 18 | import signal | ||
287 | 19 | import sys | 17 | import sys |
288 | 20 | 18 | ||
289 | 21 | from swift.obj.auditor import ObjectAuditor | 19 | from swift.obj.auditor import ObjectAuditor |
290 | @@ -28,28 +26,4 @@ | |||
291 | 28 | 26 | ||
292 | 29 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | 27 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' |
293 | 30 | conf = utils.readconf(sys.argv[1], 'object-auditor') | 28 | conf = utils.readconf(sys.argv[1], 'object-auditor') |
319 | 31 | logger = utils.get_logger(conf) | 29 | ObjectAuditor(conf).run(once) |
295 | 32 | # log uncaught exceptions | ||
296 | 33 | sys.excepthook = lambda *exc_info: \ | ||
297 | 34 | logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) | ||
298 | 35 | sys.stdout = sys.stderr = utils.LoggerFileObject(logger) | ||
299 | 36 | |||
300 | 37 | utils.drop_privileges(conf.get('user', 'swift')) | ||
301 | 38 | |||
302 | 39 | try: | ||
303 | 40 | os.setsid() | ||
304 | 41 | except OSError: | ||
305 | 42 | pass | ||
306 | 43 | |||
307 | 44 | def kill_children(*args): | ||
308 | 45 | signal.signal(signal.SIGTERM, signal.SIG_IGN) | ||
309 | 46 | os.killpg(0, signal.SIGTERM) | ||
310 | 47 | sys.exit() | ||
311 | 48 | |||
312 | 49 | signal.signal(signal.SIGTERM, kill_children) | ||
313 | 50 | |||
314 | 51 | auditor = ObjectAuditor(conf) | ||
315 | 52 | if once: | ||
316 | 53 | auditor.audit_once() | ||
317 | 54 | else: | ||
318 | 55 | auditor.audit_forever() | ||
320 | 56 | 30 | ||
321 | === modified file 'bin/swift-object-replicator' | |||
322 | --- bin/swift-object-replicator 2010-08-24 13:41:58 +0000 | |||
323 | +++ bin/swift-object-replicator 2010-09-01 15:57:44 +0000 | |||
324 | @@ -15,54 +15,15 @@ | |||
325 | 15 | # limitations under the License. | 15 | # limitations under the License. |
326 | 16 | 16 | ||
327 | 17 | import sys | 17 | import sys |
328 | 18 | import logging | ||
329 | 19 | import time | ||
330 | 20 | |||
331 | 21 | from eventlet import sleep, hubs | ||
332 | 22 | hubs.use_hub('poll') | ||
333 | 23 | 18 | ||
334 | 24 | from swift.obj.replicator import ObjectReplicator | 19 | from swift.obj.replicator import ObjectReplicator |
339 | 25 | from swift.common.utils import get_logger, drop_privileges, LoggerFileObject, \ | 20 | from swift.common import utils |
336 | 26 | readconf | ||
337 | 27 | |||
338 | 28 | TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes')) | ||
340 | 29 | 21 | ||
341 | 30 | if __name__ == '__main__': | 22 | if __name__ == '__main__': |
342 | 31 | if len(sys.argv) < 2: | 23 | if len(sys.argv) < 2: |
343 | 32 | print "Usage: swift-object-replicator CONFIG_FILE [once]" | 24 | print "Usage: swift-object-replicator CONFIG_FILE [once]" |
344 | 33 | sys.exit() | 25 | sys.exit() |
380 | 34 | conf = readconf(sys.argv[1], "object-replicator") | 26 | conf = utils.readconf(sys.argv[1], "object-replicator") |
381 | 35 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | 27 | once = (len(sys.argv) > 2 and sys.argv[2] == 'once') or \ |
382 | 36 | logger = get_logger(conf) | 28 | conf.get('daemonize', 'true') not in utils.TRUE_VALUES |
383 | 37 | # log uncaught exceptions | 29 | ObjectReplicator(conf).run(once) |
349 | 38 | sys.excepthook = lambda *exc_info: \ | ||
350 | 39 | logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) | ||
351 | 40 | sys.stdout = sys.stderr = LoggerFileObject(logger) | ||
352 | 41 | drop_privileges(conf.get('user', 'swift')) | ||
353 | 42 | if not once and conf.get('daemonize', 'true') in TRUE_VALUES: | ||
354 | 43 | logger.info("Starting object replicator in daemon mode.") | ||
355 | 44 | # Run the replicator continually | ||
356 | 45 | while True: | ||
357 | 46 | start = time.time() | ||
358 | 47 | logger.info("Starting object replication pass.") | ||
359 | 48 | # Run the replicator | ||
360 | 49 | replicator = ObjectReplicator(conf, logger) | ||
361 | 50 | replicator.run() | ||
362 | 51 | total = (time.time() - start)/60 | ||
363 | 52 | # Reload the config | ||
364 | 53 | logger.info("Object replication complete. (%.02f minutes)" % total) | ||
365 | 54 | conf = read_configs(sys.argv[1]) | ||
366 | 55 | if conf.get('daemonize', 'true') not in TRUE_VALUES: | ||
367 | 56 | # Stop running | ||
368 | 57 | logger.info("Daemon mode turned off in config, stopping.") | ||
369 | 58 | break | ||
370 | 59 | logger.debug('Replication sleeping for %s seconds.' % | ||
371 | 60 | conf['run_pause']) | ||
372 | 61 | sleep(int(conf['run_pause'])) | ||
373 | 62 | else: | ||
374 | 63 | start = time.time() | ||
375 | 64 | logger.info("Running object replicator in script mode.") | ||
376 | 65 | replicator = ObjectReplicator(conf, logger) | ||
377 | 66 | replicator.run() | ||
378 | 67 | total = (time.time() - start)/60 | ||
379 | 68 | logger.info("Object replication complete. (%.02f minutes)" % total) | ||
384 | 69 | 30 | ||
385 | === modified file 'bin/swift-object-updater' | |||
386 | --- bin/swift-object-updater 2010-08-24 13:41:58 +0000 | |||
387 | +++ bin/swift-object-updater 2010-09-01 15:57:44 +0000 | |||
388 | @@ -14,8 +14,6 @@ | |||
389 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
390 | 15 | # limitations under the License. | 15 | # limitations under the License. |
391 | 16 | 16 | ||
392 | 17 | import os | ||
393 | 18 | import signal | ||
394 | 19 | import sys | 17 | import sys |
395 | 20 | 18 | ||
396 | 21 | from swift.obj.updater import ObjectUpdater | 19 | from swift.obj.updater import ObjectUpdater |
397 | @@ -25,27 +23,6 @@ | |||
398 | 25 | if len(sys.argv) < 2: | 23 | if len(sys.argv) < 2: |
399 | 26 | print "Usage: swift-object-updater CONFIG_FILE [once]" | 24 | print "Usage: swift-object-updater CONFIG_FILE [once]" |
400 | 27 | sys.exit(1) | 25 | sys.exit(1) |
401 | 28 | |||
402 | 29 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' |
403 | 30 | |||
404 | 31 | conf = utils.readconf(sys.argv[1], 'object-updater') | 27 | conf = utils.readconf(sys.argv[1], 'object-updater') |
425 | 32 | utils.drop_privileges(conf.get('user', 'swift')) | 28 | ObjectUpdater(conf).run(once) |
406 | 33 | |||
407 | 34 | try: | ||
408 | 35 | os.setsid() | ||
409 | 36 | except OSError: | ||
410 | 37 | pass | ||
411 | 38 | |||
412 | 39 | def kill_children(*args): | ||
413 | 40 | signal.signal(signal.SIGTERM, signal.SIG_IGN) | ||
414 | 41 | os.killpg(0, signal.SIGTERM) | ||
415 | 42 | sys.exit() | ||
416 | 43 | |||
417 | 44 | signal.signal(signal.SIGTERM, kill_children) | ||
418 | 45 | |||
419 | 46 | updater = ObjectUpdater(conf) | ||
420 | 47 | if once: | ||
421 | 48 | updater.update_once_single_threaded() | ||
422 | 49 | else: | ||
423 | 50 | updater.update_forever() | ||
424 | 51 | |||
426 | 52 | 29 | ||
427 | === modified file 'etc/proxy-server.conf-sample' | |||
428 | --- etc/proxy-server.conf-sample 2010-08-24 13:58:32 +0000 | |||
429 | +++ etc/proxy-server.conf-sample 2010-09-01 15:57:44 +0000 | |||
430 | @@ -8,9 +8,9 @@ | |||
431 | 8 | # key_file = /etc/swift/proxy.key | 8 | # key_file = /etc/swift/proxy.key |
432 | 9 | 9 | ||
433 | 10 | [pipeline:main] | 10 | [pipeline:main] |
435 | 11 | pipeline = healthcheck cache auth proxy | 11 | pipeline = healthcheck cache auth proxy-server |
436 | 12 | 12 | ||
438 | 13 | [app:proxy] | 13 | [app:proxy-server] |
439 | 14 | use = egg:swift#proxy | 14 | use = egg:swift#proxy |
440 | 15 | # log_name = proxy-server | 15 | # log_name = proxy-server |
441 | 16 | # log_facility = LOG_LOCAL0 | 16 | # log_facility = LOG_LOCAL0 |
442 | 17 | 17 | ||
443 | === modified file 'swift/account/auditor.py' | |||
444 | --- swift/account/auditor.py 2010-08-20 00:42:38 +0000 | |||
445 | +++ swift/account/auditor.py 2010-09-01 15:57:44 +0000 | |||
446 | @@ -26,16 +26,18 @@ | |||
447 | 26 | from swift.common.exceptions import ConnectionTimeout | 26 | from swift.common.exceptions import ConnectionTimeout |
448 | 27 | from swift.common.ring import Ring | 27 | from swift.common.ring import Ring |
449 | 28 | from swift.common.utils import get_logger | 28 | from swift.common.utils import get_logger |
450 | 29 | from swift.common.daemon import Daemon | ||
451 | 29 | 30 | ||
452 | 30 | 31 | ||
453 | 31 | class AuditException(Exception): | 32 | class AuditException(Exception): |
454 | 32 | pass | 33 | pass |
455 | 33 | 34 | ||
456 | 34 | 35 | ||
458 | 35 | class AccountAuditor(object): | 36 | class AccountAuditor(Daemon): |
459 | 36 | """Audit accounts.""" | 37 | """Audit accounts.""" |
460 | 37 | 38 | ||
461 | 38 | def __init__(self, conf): | 39 | def __init__(self, conf): |
462 | 40 | self.conf = conf | ||
463 | 39 | self.logger = get_logger(conf, 'account-auditor') | 41 | self.logger = get_logger(conf, 'account-auditor') |
464 | 40 | self.devices = conf.get('devices', '/srv/node') | 42 | self.devices = conf.get('devices', '/srv/node') |
465 | 41 | self.mount_check = conf.get('mount_check', 'true').lower() in \ | 43 | self.mount_check = conf.get('mount_check', 'true').lower() in \ |
466 | @@ -60,12 +62,11 @@ | |||
467 | 60 | """ | 62 | """ |
468 | 61 | if not self.container_ring: | 63 | if not self.container_ring: |
469 | 62 | self.logger.debug( | 64 | self.logger.debug( |
470 | 63 | |||
471 | 64 | 'Loading container ring from %s' % self.container_ring_path) | 65 | 'Loading container ring from %s' % self.container_ring_path) |
472 | 65 | self.container_ring = Ring(self.container_ring_path) | 66 | self.container_ring = Ring(self.container_ring_path) |
473 | 66 | return self.container_ring | 67 | return self.container_ring |
474 | 67 | 68 | ||
476 | 68 | def audit_forever(self): # pragma: no cover | 69 | def run_forever(self): # pragma: no cover |
477 | 69 | """Run the account audit until stopped.""" | 70 | """Run the account audit until stopped.""" |
478 | 70 | reported = time.time() | 71 | reported = time.time() |
479 | 71 | time.sleep(random() * self.interval) | 72 | time.sleep(random() * self.interval) |
480 | @@ -92,7 +93,7 @@ | |||
481 | 92 | if elapsed < self.interval: | 93 | if elapsed < self.interval: |
482 | 93 | time.sleep(self.interval - elapsed) | 94 | time.sleep(self.interval - elapsed) |
483 | 94 | 95 | ||
485 | 95 | def audit_once(self): | 96 | def run_once(self): |
486 | 96 | """Run the account audit once.""" | 97 | """Run the account audit once.""" |
487 | 97 | self.logger.info('Begin account audit "once" mode') | 98 | self.logger.info('Begin account audit "once" mode') |
488 | 98 | begin = time.time() | 99 | begin = time.time() |
489 | 99 | 100 | ||
490 | === modified file 'swift/account/reaper.py' | |||
491 | --- swift/account/reaper.py 2010-08-20 00:42:38 +0000 | |||
492 | +++ swift/account/reaper.py 2010-09-01 15:57:44 +0000 | |||
493 | @@ -27,9 +27,10 @@ | |||
494 | 27 | direct_delete_container, direct_delete_object, direct_get_container | 27 | direct_delete_container, direct_delete_object, direct_get_container |
495 | 28 | from swift.common.ring import Ring | 28 | from swift.common.ring import Ring |
496 | 29 | from swift.common.utils import get_logger, whataremyips | 29 | from swift.common.utils import get_logger, whataremyips |
500 | 30 | 30 | from swift.common.daemon import Daemon | |
501 | 31 | 31 | ||
502 | 32 | class AccountReaper(object): | 32 | |
503 | 33 | class AccountReaper(Daemon): | ||
504 | 33 | """ | 34 | """ |
505 | 34 | Removes data from status=DELETED accounts. These are accounts that have | 35 | Removes data from status=DELETED accounts. These are accounts that have |
506 | 35 | been asked to be removed by the reseller via services | 36 | been asked to be removed by the reseller via services |
507 | @@ -51,6 +52,7 @@ | |||
508 | 51 | """ | 52 | """ |
509 | 52 | 53 | ||
510 | 53 | def __init__(self, conf): | 54 | def __init__(self, conf): |
511 | 55 | self.conf = conf | ||
512 | 54 | self.logger = get_logger(conf) | 56 | self.logger = get_logger(conf) |
513 | 55 | self.devices = conf.get('devices', '/srv/node') | 57 | self.devices = conf.get('devices', '/srv/node') |
514 | 56 | self.mount_check = conf.get('mount_check', 'true').lower() in \ | 58 | self.mount_check = conf.get('mount_check', 'true').lower() in \ |
515 | @@ -95,7 +97,7 @@ | |||
516 | 95 | self.object_ring = Ring(self.object_ring_path) | 97 | self.object_ring = Ring(self.object_ring_path) |
517 | 96 | return self.object_ring | 98 | return self.object_ring |
518 | 97 | 99 | ||
520 | 98 | def reap_forever(self): | 100 | def run_forever(self): |
521 | 99 | """ | 101 | """ |
522 | 100 | Main entry point when running the reaper in its normal daemon mode. | 102 | Main entry point when running the reaper in its normal daemon mode. |
523 | 101 | This repeatedly calls :func:`reap_once` no quicker than the | 103 | This repeatedly calls :func:`reap_once` no quicker than the |
524 | @@ -105,16 +107,16 @@ | |||
525 | 105 | sleep(random.random() * self.interval) | 107 | sleep(random.random() * self.interval) |
526 | 106 | while True: | 108 | while True: |
527 | 107 | begin = time() | 109 | begin = time() |
529 | 108 | self.reap_once() | 110 | self.run_once() |
530 | 109 | elapsed = time() - begin | 111 | elapsed = time() - begin |
531 | 110 | if elapsed < self.interval: | 112 | if elapsed < self.interval: |
532 | 111 | sleep(self.interval - elapsed) | 113 | sleep(self.interval - elapsed) |
533 | 112 | 114 | ||
535 | 113 | def reap_once(self): | 115 | def run_once(self): |
536 | 114 | """ | 116 | """ |
537 | 115 | Main entry point when running the reaper in 'once' mode, where it will | 117 | Main entry point when running the reaper in 'once' mode, where it will |
538 | 116 | do a single pass over all accounts on the server. This is called | 118 | do a single pass over all accounts on the server. This is called |
540 | 117 | repeatedly by :func:`reap_forever`. This will call :func:`reap_device` | 119 | repeatedly by :func:`run_forever`. This will call :func:`reap_device` |
541 | 118 | once for each device on the server. | 120 | once for each device on the server. |
542 | 119 | """ | 121 | """ |
543 | 120 | self.logger.debug('Begin devices pass: %s' % self.devices) | 122 | self.logger.debug('Begin devices pass: %s' % self.devices) |
544 | 121 | 123 | ||
545 | === added file 'swift/account/replicator.py' | |||
546 | --- swift/account/replicator.py 1970-01-01 00:00:00 +0000 | |||
547 | +++ swift/account/replicator.py 2010-09-01 15:57:44 +0000 | |||
548 | @@ -0,0 +1,26 @@ | |||
549 | 1 | # Copyright (c) 2010 OpenStack, LLC. | ||
550 | 2 | # | ||
551 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
552 | 4 | # you may not use this file except in compliance with the License. | ||
553 | 5 | # You may obtain a copy of the License at | ||
554 | 6 | # | ||
555 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
556 | 8 | # | ||
557 | 9 | # Unless required by applicable law or agreed to in writing, software | ||
558 | 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
559 | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
560 | 12 | # implied. | ||
561 | 13 | # See the License for the specific language governing permissions and | ||
562 | 14 | # limitations under the License. | ||
563 | 15 | |||
564 | 16 | from swift.account import server as account_server | ||
565 | 17 | from swift.common import db, db_replicator | ||
566 | 18 | |||
567 | 19 | |||
568 | 20 | class AccountReplicator(db_replicator.Replicator): | ||
569 | 21 | server_type = 'account' | ||
570 | 22 | ring_file = 'account.ring.gz' | ||
571 | 23 | brokerclass = db.AccountBroker | ||
572 | 24 | datadir = account_server.DATADIR | ||
573 | 25 | default_port = 6002 | ||
574 | 26 | |||
575 | 0 | 27 | ||
576 | === added file 'swift/common/daemon.py' | |||
577 | --- swift/common/daemon.py 1970-01-01 00:00:00 +0000 | |||
578 | +++ swift/common/daemon.py 2010-09-01 15:57:44 +0000 | |||
579 | @@ -0,0 +1,60 @@ | |||
580 | 1 | # Copyright (c) 2010 OpenStack, LLC. | ||
581 | 2 | # | ||
582 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
583 | 4 | # you may not use this file except in compliance with the License. | ||
584 | 5 | # You may obtain a copy of the License at | ||
585 | 6 | # | ||
586 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
587 | 8 | # | ||
588 | 9 | # Unless required by applicable law or agreed to in writing, software | ||
589 | 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
590 | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
591 | 12 | # implied. | ||
592 | 13 | # See the License for the specific language governing permissions and | ||
593 | 14 | # limitations under the License. | ||
594 | 15 | |||
595 | 16 | import os | ||
596 | 17 | import sys | ||
597 | 18 | import signal | ||
598 | 19 | from swift.common import utils | ||
599 | 20 | |||
600 | 21 | class Daemon(object): | ||
601 | 22 | """Daemon base class""" | ||
602 | 23 | |||
603 | 24 | def __init__(self, conf): | ||
604 | 25 | self.conf = conf | ||
605 | 26 | self.logger = utils.get_logger(conf, 'swift-daemon') | ||
606 | 27 | |||
607 | 28 | def run_once(self): | ||
608 | 29 | """Override this to run the script once""" | ||
609 | 30 | raise NotImplementedError('run_once not implemented') | ||
610 | 31 | |||
611 | 32 | def run_forever(self): | ||
612 | 33 | """Override this to run forever""" | ||
613 | 34 | raise NotImplementedError('run_forever not implemented') | ||
614 | 35 | |||
615 | 36 | def run(self, once=False): | ||
616 | 37 | """Run the daemon""" | ||
617 | 38 | # log uncaught exceptions | ||
618 | 39 | sys.excepthook = lambda *exc_info: \ | ||
619 | 40 | self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) | ||
620 | 41 | sys.stdout = sys.stderr = utils.LoggerFileObject(self.logger) | ||
621 | 42 | |||
622 | 43 | utils.drop_privileges(self.conf.get('user', 'swift')) | ||
623 | 44 | |||
624 | 45 | try: | ||
625 | 46 | os.setsid() | ||
626 | 47 | except OSError: | ||
627 | 48 | pass | ||
628 | 49 | |||
629 | 50 | def kill_children(*args): | ||
630 | 51 | signal.signal(signal.SIGTERM, signal.SIG_IGN) | ||
631 | 52 | os.killpg(0, signal.SIGTERM) | ||
632 | 53 | sys.exit() | ||
633 | 54 | |||
634 | 55 | signal.signal(signal.SIGTERM, kill_children) | ||
635 | 56 | |||
636 | 57 | if once: | ||
637 | 58 | self.run_once() | ||
638 | 59 | else: | ||
639 | 60 | self.run_forever() | ||
640 | 0 | 61 | ||
641 | === modified file 'swift/common/db_replicator.py' | |||
642 | --- swift/common/db_replicator.py 2010-08-20 00:50:12 +0000 | |||
643 | +++ swift/common/db_replicator.py 2010-09-01 15:57:44 +0000 | |||
644 | @@ -33,6 +33,7 @@ | |||
645 | 33 | from swift.common import ring | 33 | from swift.common import ring |
646 | 34 | from swift.common.bufferedhttp import BufferedHTTPConnection | 34 | from swift.common.bufferedhttp import BufferedHTTPConnection |
647 | 35 | from swift.common.exceptions import DriveNotMounted, ConnectionTimeout | 35 | from swift.common.exceptions import DriveNotMounted, ConnectionTimeout |
648 | 36 | from swift.common.daemon import Daemon | ||
649 | 36 | 37 | ||
650 | 37 | 38 | ||
651 | 38 | def quarantine_db(object_file, server_type): | 39 | def quarantine_db(object_file, server_type): |
652 | @@ -84,14 +85,14 @@ | |||
653 | 84 | return None | 85 | return None |
654 | 85 | 86 | ||
655 | 86 | 87 | ||
657 | 87 | class Replicator(object): | 88 | class Replicator(Daemon): |
658 | 88 | """ | 89 | """ |
659 | 89 | Implements the logic for directing db replication. | 90 | Implements the logic for directing db replication. |
660 | 90 | """ | 91 | """ |
661 | 91 | 92 | ||
662 | 92 | def __init__(self, conf): | 93 | def __init__(self, conf): |
665 | 93 | self.logger = \ | 94 | self.conf = conf |
666 | 94 | get_logger(conf) | 95 | self.logger = get_logger(conf) |
667 | 95 | # log uncaught exceptions | 96 | # log uncaught exceptions |
668 | 96 | sys.excepthook = lambda * exc_info: \ | 97 | sys.excepthook = lambda * exc_info: \ |
669 | 97 | self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) | 98 | self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) |
670 | @@ -396,7 +397,7 @@ | |||
671 | 396 | except StopIteration: | 397 | except StopIteration: |
672 | 397 | its.remove(it) | 398 | its.remove(it) |
673 | 398 | 399 | ||
675 | 399 | def replicate_once(self): | 400 | def run_once(self): |
676 | 400 | """Run a replication pass once.""" | 401 | """Run a replication pass once.""" |
677 | 401 | self._zero_stats() | 402 | self._zero_stats() |
678 | 402 | dirs = [] | 403 | dirs = [] |
679 | @@ -425,13 +426,13 @@ | |||
680 | 425 | self.logger.info('Replication run OVER') | 426 | self.logger.info('Replication run OVER') |
681 | 426 | self._report_stats() | 427 | self._report_stats() |
682 | 427 | 428 | ||
684 | 428 | def replicate_forever(self): | 429 | def run_forever(self): |
685 | 429 | """ | 430 | """ |
686 | 430 | Replicate dbs under the given root in an infinite loop. | 431 | Replicate dbs under the given root in an infinite loop. |
687 | 431 | """ | 432 | """ |
688 | 432 | while True: | 433 | while True: |
689 | 433 | try: | 434 | try: |
691 | 434 | self.replicate_once() | 435 | self.run_once() |
692 | 435 | except: | 436 | except: |
693 | 436 | self.logger.exception('ERROR trying to replicate') | 437 | self.logger.exception('ERROR trying to replicate') |
694 | 437 | sleep(self.run_pause) | 438 | sleep(self.run_pause) |
695 | 438 | 439 | ||
696 | === modified file 'swift/common/utils.py' | |||
697 | --- swift/common/utils.py 2010-08-24 13:41:58 +0000 | |||
698 | +++ swift/common/utils.py 2010-09-01 15:57:44 +0000 | |||
699 | @@ -55,6 +55,8 @@ | |||
700 | 55 | # will end up with would also require knowing this suffix. | 55 | # will end up with would also require knowing this suffix. |
701 | 56 | HASH_PATH_SUFFIX = os.environ.get('SWIFT_HASH_PATH_SUFFIX', 'endcap') | 56 | HASH_PATH_SUFFIX = os.environ.get('SWIFT_HASH_PATH_SUFFIX', 'endcap') |
702 | 57 | 57 | ||
703 | 58 | # Used when reading config values | ||
704 | 59 | TRUE_VALUES = set(('true', '1', 'yes', 'True', 'Yes')) | ||
705 | 58 | 60 | ||
706 | 59 | def load_libc_function(func_name): | 61 | def load_libc_function(func_name): |
707 | 60 | """ | 62 | """ |
708 | 61 | 63 | ||
709 | === modified file 'swift/container/auditor.py' | |||
710 | --- swift/container/auditor.py 2010-08-20 00:42:38 +0000 | |||
711 | +++ swift/container/auditor.py 2010-09-01 15:57:44 +0000 | |||
712 | @@ -27,16 +27,18 @@ | |||
713 | 27 | from swift.common.exceptions import ConnectionTimeout | 27 | from swift.common.exceptions import ConnectionTimeout |
714 | 28 | from swift.common.ring import Ring | 28 | from swift.common.ring import Ring |
715 | 29 | from swift.common.utils import get_logger | 29 | from swift.common.utils import get_logger |
716 | 30 | from swift.common.daemon import Daemon | ||
717 | 30 | 31 | ||
718 | 31 | 32 | ||
719 | 32 | class AuditException(Exception): | 33 | class AuditException(Exception): |
720 | 33 | pass | 34 | pass |
721 | 34 | 35 | ||
722 | 35 | 36 | ||
724 | 36 | class ContainerAuditor(object): | 37 | class ContainerAuditor(Daemon): |
725 | 37 | """Audit containers.""" | 38 | """Audit containers.""" |
726 | 38 | 39 | ||
727 | 39 | def __init__(self, conf): | 40 | def __init__(self, conf): |
728 | 41 | self.conf = conf | ||
729 | 40 | self.logger = get_logger(conf) | 42 | self.logger = get_logger(conf) |
730 | 41 | self.devices = conf.get('devices', '/srv/node') | 43 | self.devices = conf.get('devices', '/srv/node') |
731 | 42 | self.mount_check = conf.get('mount_check', 'true').lower() in \ | 44 | self.mount_check = conf.get('mount_check', 'true').lower() in \ |
732 | @@ -81,7 +83,7 @@ | |||
733 | 81 | self.object_ring = Ring(self.object_ring_path) | 83 | self.object_ring = Ring(self.object_ring_path) |
734 | 82 | return self.object_ring | 84 | return self.object_ring |
735 | 83 | 85 | ||
737 | 84 | def audit_forever(self): # pragma: no cover | 86 | def run_forever(self): # pragma: no cover |
738 | 85 | """Run the container audit until stopped.""" | 87 | """Run the container audit until stopped.""" |
739 | 86 | reported = time.time() | 88 | reported = time.time() |
740 | 87 | time.sleep(random() * self.interval) | 89 | time.sleep(random() * self.interval) |
741 | @@ -114,7 +116,7 @@ | |||
742 | 114 | if elapsed < self.interval: | 116 | if elapsed < self.interval: |
743 | 115 | time.sleep(self.interval - elapsed) | 117 | time.sleep(self.interval - elapsed) |
744 | 116 | 118 | ||
746 | 117 | def audit_once(self): | 119 | def run_once(self): |
747 | 118 | """Run the container audit once.""" | 120 | """Run the container audit once.""" |
748 | 119 | self.logger.info('Begin container audit "once" mode') | 121 | self.logger.info('Begin container audit "once" mode') |
749 | 120 | begin = time.time() | 122 | begin = time.time() |
750 | 121 | 123 | ||
751 | === added file 'swift/container/replicator.py' | |||
752 | --- swift/container/replicator.py 1970-01-01 00:00:00 +0000 | |||
753 | +++ swift/container/replicator.py 2010-09-01 15:57:44 +0000 | |||
754 | @@ -0,0 +1,25 @@ | |||
755 | 1 | # Copyright (c) 2010 OpenStack, LLC. | ||
756 | 2 | # | ||
757 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); | ||
758 | 4 | # you may not use this file except in compliance with the License. | ||
759 | 5 | # You may obtain a copy of the License at | ||
760 | 6 | # | ||
761 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 | ||
762 | 8 | # | ||
763 | 9 | # Unless required by applicable law or agreed to in writing, software | ||
764 | 10 | # distributed under the License is distributed on an "AS IS" BASIS, | ||
765 | 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or | ||
766 | 12 | # implied. | ||
767 | 13 | # See the License for the specific language governing permissions and | ||
768 | 14 | # limitations under the License. | ||
769 | 15 | |||
770 | 16 | from swift.container import server as container_server | ||
771 | 17 | from swift.common import db, db_replicator | ||
772 | 18 | |||
773 | 19 | class ContainerReplicator(db_replicator.Replicator): | ||
774 | 20 | server_type = 'container' | ||
775 | 21 | ring_file = 'container.ring.gz' | ||
776 | 22 | brokerclass = db.ContainerBroker | ||
777 | 23 | datadir = container_server.DATADIR | ||
778 | 24 | default_port = 6001 | ||
779 | 25 | |||
780 | 0 | 26 | ||
781 | === modified file 'swift/container/updater.py' | |||
782 | --- swift/container/updater.py 2010-08-20 00:42:38 +0000 | |||
783 | +++ swift/container/updater.py 2010-09-01 15:57:44 +0000 | |||
784 | @@ -28,12 +28,14 @@ | |||
785 | 28 | from swift.common.exceptions import ConnectionTimeout | 28 | from swift.common.exceptions import ConnectionTimeout |
786 | 29 | from swift.common.ring import Ring | 29 | from swift.common.ring import Ring |
787 | 30 | from swift.common.utils import get_logger, whataremyips | 30 | from swift.common.utils import get_logger, whataremyips |
791 | 31 | 31 | from swift.common.daemon import Daemon | |
792 | 32 | 32 | ||
793 | 33 | class ContainerUpdater(object): | 33 | |
794 | 34 | class ContainerUpdater(Daemon): | ||
795 | 34 | """Update container information in account listings.""" | 35 | """Update container information in account listings.""" |
796 | 35 | 36 | ||
797 | 36 | def __init__(self, conf): | 37 | def __init__(self, conf): |
798 | 38 | self.conf = conf | ||
799 | 37 | self.logger = get_logger(conf, 'container-updater') | 39 | self.logger = get_logger(conf, 'container-updater') |
800 | 38 | self.devices = conf.get('devices', '/srv/node') | 40 | self.devices = conf.get('devices', '/srv/node') |
801 | 39 | self.mount_check = conf.get('mount_check', 'true').lower() in \ | 41 | self.mount_check = conf.get('mount_check', 'true').lower() in \ |
802 | @@ -78,7 +80,7 @@ | |||
803 | 78 | shuffle(paths) | 80 | shuffle(paths) |
804 | 79 | return paths | 81 | return paths |
805 | 80 | 82 | ||
807 | 81 | def update_forever(self): # pragma: no cover | 83 | def run_forever(self): # pragma: no cover |
808 | 82 | """ | 84 | """ |
809 | 83 | Run the updator continuously. | 85 | Run the updator continuously. |
810 | 84 | """ | 86 | """ |
811 | @@ -118,7 +120,7 @@ | |||
812 | 118 | if elapsed < self.interval: | 120 | if elapsed < self.interval: |
813 | 119 | time.sleep(self.interval - elapsed) | 121 | time.sleep(self.interval - elapsed) |
814 | 120 | 122 | ||
816 | 121 | def update_once_single_threaded(self): | 123 | def run_once(self): |
817 | 122 | """ | 124 | """ |
818 | 123 | Run the updater once. | 125 | Run the updater once. |
819 | 124 | """ | 126 | """ |
820 | 125 | 127 | ||
821 | === modified file 'swift/obj/auditor.py' | |||
822 | --- swift/obj/auditor.py 2010-08-20 00:42:38 +0000 | |||
823 | +++ swift/obj/auditor.py 2010-09-01 15:57:44 +0000 | |||
824 | @@ -28,13 +28,15 @@ | |||
825 | 28 | from swift.common.ring import Ring | 28 | from swift.common.ring import Ring |
826 | 29 | from swift.common.utils import get_logger, renamer | 29 | from swift.common.utils import get_logger, renamer |
827 | 30 | from swift.common.exceptions import AuditException | 30 | from swift.common.exceptions import AuditException |
831 | 31 | 31 | from swift.common.daemon import Daemon | |
832 | 32 | 32 | ||
833 | 33 | class ObjectAuditor(object): | 33 | |
834 | 34 | class ObjectAuditor(Daemon): | ||
835 | 34 | """Audit objects.""" | 35 | """Audit objects.""" |
836 | 35 | 36 | ||
837 | 36 | def __init__(self, conf): | 37 | def __init__(self, conf): |
839 | 37 | self.logger = get_logger(conf) | 38 | self.conf = conf |
840 | 39 | self.logger = get_logger(conf, 'object-auditor') | ||
841 | 38 | self.devices = conf.get('devices', '/srv/node') | 40 | self.devices = conf.get('devices', '/srv/node') |
842 | 39 | self.mount_check = conf.get('mount_check', 'true').lower() in \ | 41 | self.mount_check = conf.get('mount_check', 'true').lower() in \ |
843 | 40 | ('true', 't', '1', 'on', 'yes', 'y') | 42 | ('true', 't', '1', 'on', 'yes', 'y') |
844 | @@ -63,7 +65,7 @@ | |||
845 | 63 | self.container_ring = Ring(self.container_ring_path) | 65 | self.container_ring = Ring(self.container_ring_path) |
846 | 64 | return self.container_ring | 66 | return self.container_ring |
847 | 65 | 67 | ||
849 | 66 | def audit_forever(self): # pragma: no cover | 68 | def run_forever(self): # pragma: no cover |
850 | 67 | """Run the object audit until stopped.""" | 69 | """Run the object audit until stopped.""" |
851 | 68 | reported = time.time() | 70 | reported = time.time() |
852 | 69 | time.sleep(random() * self.interval) | 71 | time.sleep(random() * self.interval) |
853 | @@ -97,7 +99,7 @@ | |||
854 | 97 | if elapsed < self.interval: | 99 | if elapsed < self.interval: |
855 | 98 | time.sleep(self.interval - elapsed) | 100 | time.sleep(self.interval - elapsed) |
856 | 99 | 101 | ||
858 | 100 | def audit_once(self): | 102 | def run_once(self): |
859 | 101 | """Run the object audit once.""" | 103 | """Run the object audit once.""" |
860 | 102 | self.logger.info('Begin object audit "once" mode') | 104 | self.logger.info('Begin object audit "once" mode') |
861 | 103 | begin = time.time() | 105 | begin = time.time() |
862 | 104 | 106 | ||
863 | === modified file 'swift/obj/replicator.py' | |||
864 | --- swift/obj/replicator.py 2010-08-24 18:27:38 +0000 | |||
865 | +++ swift/obj/replicator.py 2010-09-01 15:57:44 +0000 | |||
866 | @@ -24,15 +24,17 @@ | |||
867 | 24 | import cPickle as pickle | 24 | import cPickle as pickle |
868 | 25 | 25 | ||
869 | 26 | import eventlet | 26 | import eventlet |
871 | 27 | from eventlet import GreenPool, tpool, Timeout, sleep | 27 | from eventlet import GreenPool, tpool, Timeout, sleep, hubs |
872 | 28 | from eventlet.green import subprocess | 28 | from eventlet.green import subprocess |
873 | 29 | from eventlet.support.greenlets import GreenletExit | 29 | from eventlet.support.greenlets import GreenletExit |
874 | 30 | 30 | ||
875 | 31 | from swift.common.ring import Ring | 31 | from swift.common.ring import Ring |
876 | 32 | from swift.common.utils import whataremyips, unlink_older_than, lock_path, \ | 32 | from swift.common.utils import whataremyips, unlink_older_than, lock_path, \ |
878 | 33 | renamer, compute_eta | 33 | renamer, compute_eta, get_logger |
879 | 34 | from swift.common.bufferedhttp import http_connect | 34 | from swift.common.bufferedhttp import http_connect |
880 | 35 | from swift.common.daemon import Daemon | ||
881 | 35 | 36 | ||
882 | 37 | hubs.use_hub('poll') | ||
883 | 36 | 38 | ||
884 | 37 | PICKLE_PROTOCOL = 2 | 39 | PICKLE_PROTOCOL = 2 |
885 | 38 | ONE_WEEK = 604800 | 40 | ONE_WEEK = 604800 |
886 | @@ -190,22 +192,22 @@ | |||
887 | 190 | return hashed, hashes | 192 | return hashed, hashes |
888 | 191 | 193 | ||
889 | 192 | 194 | ||
891 | 193 | class ObjectReplicator(object): | 195 | class ObjectReplicator(Daemon): |
892 | 194 | """ | 196 | """ |
893 | 195 | Replicate objects. | 197 | Replicate objects. |
894 | 196 | 198 | ||
895 | 197 | Encapsulates most logic and data needed by the object replication process. | 199 | Encapsulates most logic and data needed by the object replication process. |
898 | 198 | Each call to .run() performs one replication pass. It's up to the caller | 200 | Each call to .replicate() performs one replication pass. It's up to the |
899 | 199 | to do this in a loop. | 201 | caller to do this in a loop. |
900 | 200 | """ | 202 | """ |
901 | 201 | 203 | ||
903 | 202 | def __init__(self, conf, logger): | 204 | def __init__(self, conf): |
904 | 203 | """ | 205 | """ |
905 | 204 | :param conf: configuration object obtained from ConfigParser | 206 | :param conf: configuration object obtained from ConfigParser |
906 | 205 | :param logger: logging object | 207 | :param logger: logging object |
907 | 206 | """ | 208 | """ |
908 | 207 | self.conf = conf | 209 | self.conf = conf |
910 | 208 | self.logger = logger | 210 | self.logger = get_logger(conf, 'object-replicator') |
911 | 209 | self.devices_dir = conf.get('devices', '/srv/node') | 211 | self.devices_dir = conf.get('devices', '/srv/node') |
912 | 210 | self.mount_check = conf.get('mount_check', 'true').lower() in \ | 212 | self.mount_check = conf.get('mount_check', 'true').lower() in \ |
913 | 211 | ('true', 't', '1', 'on', 'yes', 'y') | 213 | ('true', 't', '1', 'on', 'yes', 'y') |
914 | @@ -221,6 +223,7 @@ | |||
915 | 221 | self.next_check = time.time() + self.ring_check_interval | 223 | self.next_check = time.time() + self.ring_check_interval |
916 | 222 | self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7)) | 224 | self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7)) |
917 | 223 | self.partition_times = [] | 225 | self.partition_times = [] |
918 | 226 | self.run_pause = int(conf.get('run_pause', 30)) | ||
919 | 224 | 227 | ||
920 | 225 | def _rsync(self, args): | 228 | def _rsync(self, args): |
921 | 226 | """ | 229 | """ |
922 | @@ -450,7 +453,7 @@ | |||
923 | 450 | eventlet.sleep(300) | 453 | eventlet.sleep(300) |
924 | 451 | self.stats_line() | 454 | self.stats_line() |
925 | 452 | 455 | ||
927 | 453 | def run(self): | 456 | def replicate(self): |
928 | 454 | """Run a replication pass""" | 457 | """Run a replication pass""" |
929 | 455 | self.start = time.time() | 458 | self.start = time.time() |
930 | 456 | self.suffix_count = 0 | 459 | self.suffix_count = 0 |
931 | @@ -506,3 +509,26 @@ | |||
932 | 506 | self.kill_coros() | 509 | self.kill_coros() |
933 | 507 | self.stats_line() | 510 | self.stats_line() |
934 | 508 | stats.kill() | 511 | stats.kill() |
935 | 512 | |||
936 | 513 | def run_once(self): | ||
937 | 514 | start = time.time() | ||
938 | 515 | self.logger.info("Running object replicator in script mode.") | ||
939 | 516 | self.replicate() | ||
940 | 517 | total = (time.time() - start)/60 | ||
941 | 518 | self.logger.info( | ||
942 | 519 | "Object replication complete. (%.02f minutes)" % total) | ||
943 | 520 | |||
944 | 521 | def run_forever(self): | ||
945 | 522 | self.logger.info("Starting object replicator in daemon mode.") | ||
946 | 523 | # Run the replicator continually | ||
947 | 524 | while True: | ||
948 | 525 | start = time.time() | ||
949 | 526 | self.logger.info("Starting object replication pass.") | ||
950 | 527 | # Run the replicator | ||
951 | 528 | self.replicate() | ||
952 | 529 | total = (time.time() - start)/60 | ||
953 | 530 | self.logger.info( | ||
954 | 531 | "Object replication complete. (%.02f minutes)" % total) | ||
955 | 532 | self.logger.debug('Replication sleeping for %s seconds.' % | ||
956 | 533 | self.run_pause) | ||
957 | 534 | sleep(self.run_pause) | ||
958 | 509 | 535 | ||
959 | === modified file 'swift/obj/updater.py' | |||
960 | --- swift/obj/updater.py 2010-08-20 00:42:38 +0000 | |||
961 | +++ swift/obj/updater.py 2010-09-01 15:57:44 +0000 | |||
962 | @@ -26,14 +26,16 @@ | |||
963 | 26 | from swift.common.exceptions import ConnectionTimeout | 26 | from swift.common.exceptions import ConnectionTimeout |
964 | 27 | from swift.common.ring import Ring | 27 | from swift.common.ring import Ring |
965 | 28 | from swift.common.utils import get_logger, renamer | 28 | from swift.common.utils import get_logger, renamer |
966 | 29 | from swift.common.daemon import Daemon | ||
967 | 29 | from swift.obj.server import ASYNCDIR | 30 | from swift.obj.server import ASYNCDIR |
968 | 30 | 31 | ||
969 | 31 | 32 | ||
971 | 32 | class ObjectUpdater(object): | 33 | class ObjectUpdater(Daemon): |
972 | 33 | """Update object information in container listings.""" | 34 | """Update object information in container listings.""" |
973 | 34 | 35 | ||
974 | 35 | def __init__(self, conf): | 36 | def __init__(self, conf): |
976 | 36 | self.logger = get_logger(conf) | 37 | self.conf = conf |
977 | 38 | self.logger = get_logger(conf, 'object-updater') | ||
978 | 37 | self.devices = conf.get('devices', '/srv/node') | 39 | self.devices = conf.get('devices', '/srv/node') |
979 | 38 | self.mount_check = conf.get('mount_check', 'true').lower() in \ | 40 | self.mount_check = conf.get('mount_check', 'true').lower() in \ |
980 | 39 | ('true', 't', '1', 'on', 'yes', 'y') | 41 | ('true', 't', '1', 'on', 'yes', 'y') |
981 | @@ -56,7 +58,7 @@ | |||
982 | 56 | self.container_ring = Ring(self.container_ring_path) | 58 | self.container_ring = Ring(self.container_ring_path) |
983 | 57 | return self.container_ring | 59 | return self.container_ring |
984 | 58 | 60 | ||
986 | 59 | def update_forever(self): # pragma: no cover | 61 | def run_forever(self): # pragma: no cover |
987 | 60 | """Run the updater continuously.""" | 62 | """Run the updater continuously.""" |
988 | 61 | time.sleep(random() * self.interval) | 63 | time.sleep(random() * self.interval) |
989 | 62 | while True: | 64 | while True: |
990 | @@ -95,7 +97,7 @@ | |||
991 | 95 | if elapsed < self.interval: | 97 | if elapsed < self.interval: |
992 | 96 | time.sleep(self.interval - elapsed) | 98 | time.sleep(self.interval - elapsed) |
993 | 97 | 99 | ||
995 | 98 | def update_once_single_threaded(self): | 100 | def run_once(self): |
996 | 99 | """Run the updater once""" | 101 | """Run the updater once""" |
997 | 100 | self.logger.info('Begin object update single threaded sweep') | 102 | self.logger.info('Begin object update single threaded sweep') |
998 | 101 | begin = time.time() | 103 | begin = time.time() |
999 | 102 | 104 | ||
1000 | === modified file 'test/unit/common/test_db_replicator.py' | |||
1001 | --- test/unit/common/test_db_replicator.py 2010-08-20 00:50:12 +0000 | |||
1002 | +++ test/unit/common/test_db_replicator.py 2010-09-01 15:57:44 +0000 | |||
1003 | @@ -170,9 +170,9 @@ | |||
1004 | 170 | {'id': 'a', 'point': -1, 'max_row': 10, 'hash': 'd'}, | 170 | {'id': 'a', 'point': -1, 'max_row': 10, 'hash': 'd'}, |
1005 | 171 | FakeBroker(), -1)), False) | 171 | FakeBroker(), -1)), False) |
1006 | 172 | 172 | ||
1008 | 173 | def test_replicate_once(self): | 173 | def test_run_once(self): |
1009 | 174 | replicator = TestReplicator({}) | 174 | replicator = TestReplicator({}) |
1011 | 175 | replicator.replicate_once() | 175 | replicator.run_once() |
1012 | 176 | 176 | ||
1013 | 177 | def test_usync(self): | 177 | def test_usync(self): |
1014 | 178 | fake_http = ReplHttp() | 178 | fake_http = ReplHttp() |
1015 | 179 | 179 | ||
1016 | === modified file 'test/unit/container/test_updater.py' | |||
1017 | --- test/unit/container/test_updater.py 2010-08-20 00:42:38 +0000 | |||
1018 | +++ test/unit/container/test_updater.py 2010-09-01 15:57:44 +0000 | |||
1019 | @@ -77,7 +77,7 @@ | |||
1020 | 77 | self.assertEquals(cu.node_timeout, 5) | 77 | self.assertEquals(cu.node_timeout, 5) |
1021 | 78 | self.assert_(cu.get_account_ring() is not None) | 78 | self.assert_(cu.get_account_ring() is not None) |
1022 | 79 | 79 | ||
1024 | 80 | def test_update_once_single_threaded(self): | 80 | def test_run_once(self): |
1025 | 81 | cu = container_updater.ContainerUpdater({ | 81 | cu = container_updater.ContainerUpdater({ |
1026 | 82 | 'devices': self.devices_dir, | 82 | 'devices': self.devices_dir, |
1027 | 83 | 'mount_check': 'false', | 83 | 'mount_check': 'false', |
1028 | @@ -86,17 +86,17 @@ | |||
1029 | 86 | 'concurrency': '1', | 86 | 'concurrency': '1', |
1030 | 87 | 'node_timeout': '15', | 87 | 'node_timeout': '15', |
1031 | 88 | }) | 88 | }) |
1033 | 89 | cu.update_once_single_threaded() | 89 | cu.run_once() |
1034 | 90 | containers_dir = os.path.join(self.sda1, container_server.DATADIR) | 90 | containers_dir = os.path.join(self.sda1, container_server.DATADIR) |
1035 | 91 | os.mkdir(containers_dir) | 91 | os.mkdir(containers_dir) |
1037 | 92 | cu.update_once_single_threaded() | 92 | cu.run_once() |
1038 | 93 | self.assert_(os.path.exists(containers_dir)) | 93 | self.assert_(os.path.exists(containers_dir)) |
1039 | 94 | subdir = os.path.join(containers_dir, 'subdir') | 94 | subdir = os.path.join(containers_dir, 'subdir') |
1040 | 95 | os.mkdir(subdir) | 95 | os.mkdir(subdir) |
1041 | 96 | cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a', | 96 | cb = ContainerBroker(os.path.join(subdir, 'hash.db'), account='a', |
1042 | 97 | container='c') | 97 | container='c') |
1043 | 98 | cb.initialize(normalize_timestamp(1)) | 98 | cb.initialize(normalize_timestamp(1)) |
1045 | 99 | cu.update_once_single_threaded() | 99 | cu.run_once() |
1046 | 100 | info = cb.get_info() | 100 | info = cb.get_info() |
1047 | 101 | self.assertEquals(info['object_count'], 0) | 101 | self.assertEquals(info['object_count'], 0) |
1048 | 102 | self.assertEquals(info['bytes_used'], 0) | 102 | self.assertEquals(info['bytes_used'], 0) |
1049 | @@ -105,7 +105,7 @@ | |||
1050 | 105 | 105 | ||
1051 | 106 | cb.put_object('o', normalize_timestamp(2), 3, 'text/plain', | 106 | cb.put_object('o', normalize_timestamp(2), 3, 'text/plain', |
1052 | 107 | '68b329da9893e34099c7d8ad5cb9c940') | 107 | '68b329da9893e34099c7d8ad5cb9c940') |
1054 | 108 | cu.update_once_single_threaded() | 108 | cu.run_once() |
1055 | 109 | info = cb.get_info() | 109 | info = cb.get_info() |
1056 | 110 | self.assertEquals(info['object_count'], 1) | 110 | self.assertEquals(info['object_count'], 1) |
1057 | 111 | self.assertEquals(info['bytes_used'], 3) | 111 | self.assertEquals(info['bytes_used'], 3) |
1058 | @@ -148,7 +148,7 @@ | |||
1059 | 148 | for dev in cu.get_account_ring().devs: | 148 | for dev in cu.get_account_ring().devs: |
1060 | 149 | if dev is not None: | 149 | if dev is not None: |
1061 | 150 | dev['port'] = bindsock.getsockname()[1] | 150 | dev['port'] = bindsock.getsockname()[1] |
1063 | 151 | cu.update_once_single_threaded() | 151 | cu.run_once() |
1064 | 152 | for event in spawned.wait(): | 152 | for event in spawned.wait(): |
1065 | 153 | err = event.wait() | 153 | err = event.wait() |
1066 | 154 | if err: | 154 | if err: |
1067 | @@ -202,7 +202,7 @@ | |||
1068 | 202 | for dev in cu.get_account_ring().devs: | 202 | for dev in cu.get_account_ring().devs: |
1069 | 203 | if dev is not None: | 203 | if dev is not None: |
1070 | 204 | dev['port'] = bindsock.getsockname()[1] | 204 | dev['port'] = bindsock.getsockname()[1] |
1072 | 205 | cu.update_once_single_threaded() | 205 | cu.run_once() |
1073 | 206 | for event in spawned.wait(): | 206 | for event in spawned.wait(): |
1074 | 207 | err = event.wait() | 207 | err = event.wait() |
1075 | 208 | if err: | 208 | if err: |
1076 | 209 | 209 | ||
1077 | === modified file 'test/unit/obj/test_replicator.py' | |||
1078 | --- test/unit/obj/test_replicator.py 2010-07-19 16:25:18 +0000 | |||
1079 | +++ test/unit/obj/test_replicator.py 2010-09-01 15:57:44 +0000 | |||
1080 | @@ -105,7 +105,7 @@ | |||
1081 | 105 | swift_dir=self.testdir, devices=self.devices, mount_check='false', | 105 | swift_dir=self.testdir, devices=self.devices, mount_check='false', |
1082 | 106 | timeout='300', stats_interval='1') | 106 | timeout='300', stats_interval='1') |
1083 | 107 | self.replicator = object_replicator.ObjectReplicator( | 107 | self.replicator = object_replicator.ObjectReplicator( |
1085 | 108 | self.conf, null_logger) | 108 | self.conf) |
1086 | 109 | 109 | ||
1087 | 110 | # def test_check_ring(self): | 110 | # def test_check_ring(self): |
1088 | 111 | # self.replicator.collect_jobs('sda', 0, self.ring) | 111 | # self.replicator.collect_jobs('sda', 0, self.ring) |
1089 | @@ -184,11 +184,11 @@ | |||
1090 | 184 | 184 | ||
1091 | 185 | def test_run(self): | 185 | def test_run(self): |
1092 | 186 | with _mock_process([(0,'')]*100): | 186 | with _mock_process([(0,'')]*100): |
1094 | 187 | self.replicator.run() | 187 | self.replicator.replicate() |
1095 | 188 | 188 | ||
1096 | 189 | def test_run_withlog(self): | 189 | def test_run_withlog(self): |
1097 | 190 | with _mock_process([(0,"stuff in log")]*100): | 190 | with _mock_process([(0,"stuff in log")]*100): |
1099 | 191 | self.replicator.run() | 191 | self.replicator.replicate() |
1100 | 192 | 192 | ||
1101 | 193 | if __name__ == '__main__': | 193 | if __name__ == '__main__': |
1102 | 194 | unittest.main() | 194 | unittest.main() |
1103 | 195 | 195 | ||
1104 | === modified file 'test/unit/obj/test_updater.py' | |||
1105 | --- test/unit/obj/test_updater.py 2010-08-26 16:03:08 +0000 | |||
1106 | +++ test/unit/obj/test_updater.py 2010-09-01 15:57:44 +0000 | |||
1107 | @@ -69,7 +69,7 @@ | |||
1108 | 69 | self.assertEquals(cu.node_timeout, 5) | 69 | self.assertEquals(cu.node_timeout, 5) |
1109 | 70 | self.assert_(cu.get_container_ring() is not None) | 70 | self.assert_(cu.get_container_ring() is not None) |
1110 | 71 | 71 | ||
1112 | 72 | def test_update_once_single_threaded(self): | 72 | def test_run_once(self): |
1113 | 73 | cu = object_updater.ObjectUpdater({ | 73 | cu = object_updater.ObjectUpdater({ |
1114 | 74 | 'devices': self.devices_dir, | 74 | 'devices': self.devices_dir, |
1115 | 75 | 'mount_check': 'false', | 75 | 'mount_check': 'false', |
1116 | @@ -78,15 +78,15 @@ | |||
1117 | 78 | 'concurrency': '1', | 78 | 'concurrency': '1', |
1118 | 79 | 'node_timeout': '15', | 79 | 'node_timeout': '15', |
1119 | 80 | }) | 80 | }) |
1121 | 81 | cu.update_once_single_threaded() | 81 | cu.run_once() |
1122 | 82 | async_dir = os.path.join(self.sda1, object_server.ASYNCDIR) | 82 | async_dir = os.path.join(self.sda1, object_server.ASYNCDIR) |
1123 | 83 | os.mkdir(async_dir) | 83 | os.mkdir(async_dir) |
1125 | 84 | cu.update_once_single_threaded() | 84 | cu.run_once() |
1126 | 85 | self.assert_(os.path.exists(async_dir)) | 85 | self.assert_(os.path.exists(async_dir)) |
1127 | 86 | 86 | ||
1128 | 87 | odd_dir = os.path.join(async_dir, 'not really supposed to be here') | 87 | odd_dir = os.path.join(async_dir, 'not really supposed to be here') |
1129 | 88 | os.mkdir(odd_dir) | 88 | os.mkdir(odd_dir) |
1131 | 89 | cu.update_once_single_threaded() | 89 | cu.run_once() |
1132 | 90 | self.assert_(os.path.exists(async_dir)) | 90 | self.assert_(os.path.exists(async_dir)) |
1133 | 91 | self.assert_(not os.path.exists(odd_dir)) | 91 | self.assert_(not os.path.exists(odd_dir)) |
1134 | 92 | 92 | ||
1135 | @@ -98,7 +98,7 @@ | |||
1136 | 98 | pickle.dump({'op': 'PUT', 'account': 'a', 'container': 'c', 'obj': 'o', | 98 | pickle.dump({'op': 'PUT', 'account': 'a', 'container': 'c', 'obj': 'o', |
1137 | 99 | 'headers': {'X-Container-Timestamp': normalize_timestamp(0)}}, | 99 | 'headers': {'X-Container-Timestamp': normalize_timestamp(0)}}, |
1138 | 100 | open(op_path, 'wb')) | 100 | open(op_path, 'wb')) |
1140 | 101 | cu.update_once_single_threaded() | 101 | cu.run_once() |
1141 | 102 | self.assert_(os.path.exists(op_path)) | 102 | self.assert_(os.path.exists(op_path)) |
1142 | 103 | 103 | ||
1143 | 104 | bindsock = listen(('127.0.0.1', 0)) | 104 | bindsock = listen(('127.0.0.1', 0)) |
1144 | @@ -140,7 +140,7 @@ | |||
1145 | 140 | for dev in cu.get_container_ring().devs: | 140 | for dev in cu.get_container_ring().devs: |
1146 | 141 | if dev is not None: | 141 | if dev is not None: |
1147 | 142 | dev['port'] = bindsock.getsockname()[1] | 142 | dev['port'] = bindsock.getsockname()[1] |
1149 | 143 | cu.update_once_single_threaded() | 143 | cu.run_once() |
1150 | 144 | err = event.wait() | 144 | err = event.wait() |
1151 | 145 | if err: | 145 | if err: |
1152 | 146 | raise err | 146 | raise err |
unit tests, functest, and probe tests all ran on my vm