Merge lp:~clay-gerrard/swift/run_daemon into lp:~hudson-openstack/swift/trunk
- run_daemon
- Merge into trunk
Status: | Merged | ||||||||
---|---|---|---|---|---|---|---|---|---|
Approved by: | John Dickinson | ||||||||
Approved revision: | 134 | ||||||||
Merged at revision: | 132 | ||||||||
Proposed branch: | lp:~clay-gerrard/swift/run_daemon | ||||||||
Merge into: | lp:~hudson-openstack/swift/trunk | ||||||||
Diff against target: |
1287 lines (+653/-224) 26 files modified
bin/swift-account-auditor (+4/-9) bin/swift-account-reaper (+4/-9) bin/swift-account-replicator (+4/-9) bin/swift-account-server (+3/-6) bin/swift-account-stats-logger (+7/-8) bin/swift-auth-server (+3/-5) bin/swift-container-auditor (+4/-9) bin/swift-container-replicator (+4/-10) bin/swift-container-server (+3/-5) bin/swift-container-updater (+4/-9) bin/swift-log-stats-collector (+7/-8) bin/swift-log-uploader (+17/-7) bin/swift-object-auditor (+4/-10) bin/swift-object-replicator (+4/-10) bin/swift-object-server (+3/-5) bin/swift-object-updater (+4/-9) bin/swift-proxy-server (+3/-5) swift/common/daemon.py (+41/-15) swift/common/db_replicator.py (+0/-4) swift/common/utils.py (+102/-12) swift/common/wsgi.py (+51/-37) test/unit/__init__.py (+15/-0) test/unit/common/test_daemon.py (+83/-4) test/unit/common/test_utils.py (+175/-4) test/unit/common/test_wsgi.py (+103/-1) test/unit/stats/test_log_processor.py (+1/-14) |
||||||||
To merge this branch: | bzr merge lp:~clay-gerrard/swift/run_daemon | ||||||||
Related bugs: |
|
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
John Dickinson | Approve | ||
gholt (community) | Approve | ||
Review via email: mp+40678@code.launchpad.net |
Commit message
updated daemonize process, added option for servers/daemons to log to console
Description of the change
added helper/util to parse command line args
removed some duplicated code in server/daemon bin scripts
more standardized python/linux daemonization procedures
fixed lp:666957 "devauth server creates auth.db with the wrong
privileges"
new run_daemon helper based on run_wsgi simplifies daemon
launching/testing
new - all servers/daemons support verbose option when started
interactivity which will log to the console
fixed lp:667839 "can't start servers with relative paths to configs"
added tests
gholt (gholt) : | # |
Chuck Thier (cthier) wrote : | # |
The only question I have is why the daemonize function ended up in swift.common.utils rather than swift.common.
clayg (clay-gerrard) wrote : | # |
thanks!
heh, funny story. When redbo and I talked about it - the plan was to expand utils.drop_
tl;dr - daemonize is dumb, run_daemon can call drop_privileges and capture_stdio same as run_wsgi; good catch - I'll remove it and rework the tests.
John Dickinson (notmyname) wrote : | # |
run_daemon looks good, and it all works well with the stats processing code
however, the unit tests seem to print something out that needs to be removed (see test_utils.py:180)
clayg (clay-gerrard) wrote : | # |
awesome, works great on nose 0.11.4 - it's way worse on 0.11.1 :P
clayg (clay-gerrard) wrote : | # |
John, gholt - if you guys could pull down the latest changes and see if the unittest output got cleaned up?
clayg (clay-gerrard) wrote : | # |
after discussion with redbo and gholt - we decided to pull out the stdio redirection from common.
John Dickinson (notmyname) wrote : | # |
works now! yay!
Preview Diff
1 | === modified file 'bin/swift-account-auditor' | |||
2 | --- bin/swift-account-auditor 2010-08-31 23:12:59 +0000 | |||
3 | +++ bin/swift-account-auditor 2010-11-19 22:38:08 +0000 | |||
4 | @@ -14,15 +14,10 @@ | |||
5 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
6 | 15 | # limitations under the License. | 15 | # limitations under the License. |
7 | 16 | 16 | ||
8 | 17 | import sys | ||
9 | 18 | |||
10 | 19 | from swift.account.auditor import AccountAuditor | 17 | from swift.account.auditor import AccountAuditor |
12 | 20 | from swift.common import utils | 18 | from swift.common.utils import parse_options |
13 | 19 | from swift.common.daemon import run_daemon | ||
14 | 21 | 20 | ||
15 | 22 | if __name__ == '__main__': | 21 | if __name__ == '__main__': |
22 | 23 | if len(sys.argv) < 2: | 22 | conf_file, options = parse_options(once=True) |
23 | 24 | print "Usage: swift-account-auditor CONFIG_FILE [once]" | 23 | run_daemon(AccountAuditor, conf_file, **options) |
18 | 25 | sys.exit() | ||
19 | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | ||
20 | 27 | conf = utils.readconf(sys.argv[1], 'account-auditor') | ||
21 | 28 | auditor = AccountAuditor(conf).run(once) | ||
24 | 29 | 24 | ||
25 | === modified file 'bin/swift-account-reaper' | |||
26 | --- bin/swift-account-reaper 2010-08-31 23:12:59 +0000 | |||
27 | +++ bin/swift-account-reaper 2010-11-19 22:38:08 +0000 | |||
28 | @@ -14,15 +14,10 @@ | |||
29 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
30 | 15 | # limitations under the License. | 15 | # limitations under the License. |
31 | 16 | 16 | ||
32 | 17 | import sys | ||
33 | 18 | |||
34 | 19 | from swift.account.reaper import AccountReaper | 17 | from swift.account.reaper import AccountReaper |
36 | 20 | from swift.common import utils | 18 | from swift.common.utils import parse_options |
37 | 19 | from swift.common.daemon import run_daemon | ||
38 | 21 | 20 | ||
39 | 22 | if __name__ == '__main__': | 21 | if __name__ == '__main__': |
46 | 23 | if len(sys.argv) < 2: | 22 | conf_file, options = parse_options(once=True) |
47 | 24 | print "Usage: account-reaper CONFIG_FILE [once]" | 23 | run_daemon(AccountReaper, conf_file, **options) |
42 | 25 | sys.exit() | ||
43 | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | ||
44 | 27 | conf = utils.readconf(sys.argv[1], 'account-reaper') | ||
45 | 28 | reaper = AccountReaper(conf).run(once) | ||
48 | 29 | 24 | ||
49 | === modified file 'bin/swift-account-replicator' | |||
50 | --- bin/swift-account-replicator 2010-08-31 23:12:59 +0000 | |||
51 | +++ bin/swift-account-replicator 2010-11-19 22:38:08 +0000 | |||
52 | @@ -14,15 +14,10 @@ | |||
53 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
54 | 15 | # limitations under the License. | 15 | # limitations under the License. |
55 | 16 | 16 | ||
56 | 17 | import sys | ||
57 | 18 | |||
58 | 19 | from swift.common import utils | ||
59 | 20 | from swift.account.replicator import AccountReplicator | 17 | from swift.account.replicator import AccountReplicator |
60 | 18 | from swift.common.utils import parse_options | ||
61 | 19 | from swift.common.daemon import run_daemon | ||
62 | 21 | 20 | ||
63 | 22 | if __name__ == '__main__': | 21 | if __name__ == '__main__': |
70 | 23 | if len(sys.argv) < 2: | 22 | conf_file, options = parse_options(once=True) |
71 | 24 | print "Usage: swift-account-replicator CONFIG_FILE [once]" | 23 | run_daemon(AccountReplicator, conf_file, **options) |
66 | 25 | sys.exit(1) | ||
67 | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | ||
68 | 27 | conf = utils.readconf(sys.argv[1], 'account-replicator') | ||
69 | 28 | AccountReplicator(conf).run(once) | ||
72 | 29 | 24 | ||
73 | === modified file 'bin/swift-account-server' | |||
74 | --- bin/swift-account-server 2010-08-24 14:04:44 +0000 | |||
75 | +++ bin/swift-account-server 2010-11-19 22:38:08 +0000 | |||
76 | @@ -14,12 +14,9 @@ | |||
77 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
78 | 15 | # limitations under the License. | 15 | # limitations under the License. |
79 | 16 | 16 | ||
82 | 17 | import sys | 17 | from swift.common.utils import parse_options |
81 | 18 | |||
83 | 19 | from swift.common.wsgi import run_wsgi | 18 | from swift.common.wsgi import run_wsgi |
84 | 20 | 19 | ||
85 | 21 | if __name__ == '__main__': | 20 | if __name__ == '__main__': |
90 | 22 | if len(sys.argv) != 2: | 21 | conf_file, options = parse_options() |
91 | 23 | sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) | 22 | run_wsgi(conf_file, 'account-server', default_port=6002, **options) |
88 | 24 | run_wsgi(sys.argv[1], 'account-server', default_port=6002) | ||
89 | 25 | |||
92 | 26 | 23 | ||
93 | === modified file 'bin/swift-account-stats-logger' | |||
94 | --- bin/swift-account-stats-logger 2010-09-13 16:11:12 +0000 | |||
95 | +++ bin/swift-account-stats-logger 2010-11-19 22:38:08 +0000 | |||
96 | @@ -14,14 +14,13 @@ | |||
97 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
98 | 15 | # limitations under the License. | 15 | # limitations under the License. |
99 | 16 | 16 | ||
100 | 17 | import sys | ||
101 | 18 | |||
102 | 19 | from swift.stats.account_stats import AccountStat | 17 | from swift.stats.account_stats import AccountStat |
104 | 20 | from swift.common import utils | 18 | from swift.common.utils import parse_options |
105 | 19 | from swift.common.daemon import run_daemon | ||
106 | 21 | 20 | ||
107 | 22 | if __name__ == '__main__': | 21 | if __name__ == '__main__': |
113 | 23 | if len(sys.argv) < 2: | 22 | conf_file, options = parse_options() |
114 | 24 | print "Usage: swift-account-stats-logger CONFIG_FILE" | 23 | # currently AccountStat only supports run_once |
115 | 25 | sys.exit() | 24 | options['once'] = True |
116 | 26 | stats_conf = utils.readconf(sys.argv[1], 'log-processor-stats') | 25 | run_daemon(AccountStat, conf_file, section_name='log-processor-stats', |
117 | 27 | stats = AccountStat(stats_conf).run(once=True) | 26 | **options) |
118 | 28 | 27 | ||
119 | === modified file 'bin/swift-auth-server' | |||
120 | --- bin/swift-auth-server 2010-08-24 14:08:16 +0000 | |||
121 | +++ bin/swift-auth-server 2010-11-19 22:38:08 +0000 | |||
122 | @@ -14,11 +14,9 @@ | |||
123 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
124 | 15 | # limitations under the License. | 15 | # limitations under the License. |
125 | 16 | 16 | ||
128 | 17 | import sys | 17 | from swift.common.utils import parse_options |
127 | 18 | |||
129 | 19 | from swift.common.wsgi import run_wsgi | 18 | from swift.common.wsgi import run_wsgi |
130 | 20 | 19 | ||
131 | 21 | if __name__ == '__main__': | 20 | if __name__ == '__main__': |
135 | 22 | if len(sys.argv) != 2: | 21 | conf_file, options = parse_options() |
136 | 23 | sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) | 22 | run_wsgi(conf_file, 'auth-server', default_port=11000, **options) |
134 | 24 | run_wsgi(sys.argv[1], 'auth-server', default_port=11000) | ||
137 | 25 | 23 | ||
138 | === modified file 'bin/swift-container-auditor' | |||
139 | --- bin/swift-container-auditor 2010-08-31 23:12:59 +0000 | |||
140 | +++ bin/swift-container-auditor 2010-11-19 22:38:08 +0000 | |||
141 | @@ -14,15 +14,10 @@ | |||
142 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
143 | 15 | # limitations under the License. | 15 | # limitations under the License. |
144 | 16 | 16 | ||
145 | 17 | import sys | ||
146 | 18 | |||
147 | 19 | from swift.container.auditor import ContainerAuditor | 17 | from swift.container.auditor import ContainerAuditor |
149 | 20 | from swift.common import utils | 18 | from swift.common.utils import parse_options |
150 | 19 | from swift.common.daemon import run_daemon | ||
151 | 21 | 20 | ||
152 | 22 | if __name__ == '__main__': | 21 | if __name__ == '__main__': |
159 | 23 | if len(sys.argv) < 2: | 22 | conf_file, options = parse_options(once=True) |
160 | 24 | print "Usage: swift-container-auditor CONFIG_FILE [once]" | 23 | run_daemon(ContainerAuditor, conf_file, **options) |
155 | 25 | sys.exit() | ||
156 | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | ||
157 | 27 | conf = utils.readconf(sys.argv[1], 'container-auditor') | ||
158 | 28 | ContainerAuditor(conf).run(once) | ||
161 | 29 | 24 | ||
162 | === modified file 'bin/swift-container-replicator' | |||
163 | --- bin/swift-container-replicator 2010-08-31 23:12:59 +0000 | |||
164 | +++ bin/swift-container-replicator 2010-11-19 22:38:08 +0000 | |||
165 | @@ -14,16 +14,10 @@ | |||
166 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
167 | 15 | # limitations under the License. | 15 | # limitations under the License. |
168 | 16 | 16 | ||
169 | 17 | import sys | ||
170 | 18 | |||
171 | 19 | from swift.common import db, utils | ||
172 | 20 | from swift.container.replicator import ContainerReplicator | 17 | from swift.container.replicator import ContainerReplicator |
173 | 18 | from swift.common.utils import parse_options | ||
174 | 19 | from swift.common.daemon import run_daemon | ||
175 | 21 | 20 | ||
176 | 22 | if __name__ == '__main__': | 21 | if __name__ == '__main__': |
184 | 23 | if len(sys.argv) < 2: | 22 | conf_file, options = parse_options(once=True) |
185 | 24 | print "Usage: swift-container-replicator CONFIG_FILE [once]" | 23 | run_daemon(ContainerReplicator, conf_file, **options) |
179 | 25 | sys.exit(1) | ||
180 | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | ||
181 | 27 | conf = utils.readconf(sys.argv[1], 'container-replicator') | ||
182 | 28 | ContainerReplicator(conf).run(once) | ||
183 | 29 | |||
186 | 30 | 24 | ||
187 | === modified file 'bin/swift-container-server' | |||
188 | --- bin/swift-container-server 2010-08-24 14:04:44 +0000 | |||
189 | +++ bin/swift-container-server 2010-11-19 22:38:08 +0000 | |||
190 | @@ -14,11 +14,9 @@ | |||
191 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
192 | 15 | # limitations under the License. | 15 | # limitations under the License. |
193 | 16 | 16 | ||
196 | 17 | import sys | 17 | from swift.common.utils import parse_options |
195 | 18 | |||
197 | 19 | from swift.common.wsgi import run_wsgi | 18 | from swift.common.wsgi import run_wsgi |
198 | 20 | 19 | ||
199 | 21 | if __name__ == '__main__': | 20 | if __name__ == '__main__': |
203 | 22 | if len(sys.argv) != 2: | 21 | conf_file, options = parse_options() |
204 | 23 | sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) | 22 | run_wsgi(conf_file, 'container-server', default_port=6001, **options) |
202 | 24 | run_wsgi(sys.argv[1], 'container-server', default_port=6001) | ||
205 | 25 | 23 | ||
206 | === modified file 'bin/swift-container-updater' | |||
207 | --- bin/swift-container-updater 2010-08-31 23:12:59 +0000 | |||
208 | +++ bin/swift-container-updater 2010-11-19 22:38:08 +0000 | |||
209 | @@ -14,15 +14,10 @@ | |||
210 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
211 | 15 | # limitations under the License. | 15 | # limitations under the License. |
212 | 16 | 16 | ||
213 | 17 | import sys | ||
214 | 18 | |||
215 | 19 | from swift.container.updater import ContainerUpdater | 17 | from swift.container.updater import ContainerUpdater |
217 | 20 | from swift.common import utils | 18 | from swift.common.utils import parse_options |
218 | 19 | from swift.common.daemon import run_daemon | ||
219 | 21 | 20 | ||
220 | 22 | if __name__ == '__main__': | 21 | if __name__ == '__main__': |
227 | 23 | if len(sys.argv) < 2: | 22 | conf_file, options = parse_options(once=True) |
228 | 24 | print "Usage: swift-container-updater CONFIG_FILE [once]" | 23 | run_daemon(ContainerUpdater, conf_file, **options) |
223 | 25 | sys.exit() | ||
224 | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | ||
225 | 27 | conf = utils.readconf(sys.argv[1], 'container-updater') | ||
226 | 28 | ContainerUpdater(conf).run(once) | ||
229 | 29 | 24 | ||
230 | === modified file 'bin/swift-log-stats-collector' | |||
231 | --- bin/swift-log-stats-collector 2010-09-22 14:32:40 +0000 | |||
232 | +++ bin/swift-log-stats-collector 2010-11-19 22:38:08 +0000 | |||
233 | @@ -14,14 +14,13 @@ | |||
234 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
235 | 15 | # limitations under the License. | 15 | # limitations under the License. |
236 | 16 | 16 | ||
237 | 17 | import sys | ||
238 | 18 | |||
239 | 19 | from swift.stats.log_processor import LogProcessorDaemon | 17 | from swift.stats.log_processor import LogProcessorDaemon |
241 | 20 | from swift.common import utils | 18 | from swift.common.utils import parse_options |
242 | 19 | from swift.common.daemon import run_daemon | ||
243 | 21 | 20 | ||
244 | 22 | if __name__ == '__main__': | 21 | if __name__ == '__main__': |
250 | 23 | if len(sys.argv) < 2: | 22 | conf_file, options = parse_options() |
251 | 24 | print "Usage: swift-log-stats-collector CONFIG_FILE" | 23 | # currently the LogProcessorDaemon only supports run_once |
252 | 25 | sys.exit() | 24 | options['once'] = True |
253 | 26 | conf = utils.readconf(sys.argv[1], log_name='log-stats-collector') | 25 | run_daemon(LogProcessorDaemon, conf_file, section_name=None, |
254 | 27 | stats = LogProcessorDaemon(conf).run(once=True) | 26 | log_name='log-stats-collector', **options) |
255 | 28 | 27 | ||
256 | === modified file 'bin/swift-log-uploader' | |||
257 | --- bin/swift-log-uploader 2010-09-10 20:08:06 +0000 | |||
258 | +++ bin/swift-log-uploader 2010-11-19 22:38:08 +0000 | |||
259 | @@ -17,15 +17,25 @@ | |||
260 | 17 | import sys | 17 | import sys |
261 | 18 | 18 | ||
262 | 19 | from swift.stats.log_uploader import LogUploader | 19 | from swift.stats.log_uploader import LogUploader |
263 | 20 | from swift.common.utils import parse_options | ||
264 | 20 | from swift.common import utils | 21 | from swift.common import utils |
265 | 21 | 22 | ||
266 | 22 | if __name__ == '__main__': | 23 | if __name__ == '__main__': |
272 | 23 | if len(sys.argv) < 3: | 24 | conf_file, options = parse_options(usage="Usage: %prog CONFIG_FILE PLUGIN") |
273 | 24 | print "Usage: swift-log-uploader CONFIG_FILE plugin" | 25 | try: |
274 | 25 | sys.exit() | 26 | plugin = options['extra_args'][0] |
275 | 26 | uploader_conf = utils.readconf(sys.argv[1], 'log-processor') | 27 | except IndexError: |
276 | 27 | plugin = sys.argv[2] | 28 | print "Error: missing plugin name" |
277 | 29 | sys.exit(1) | ||
278 | 30 | |||
279 | 31 | uploader_conf = utils.readconf(conf_file, 'log-processor') | ||
280 | 28 | section_name = 'log-processor-%s' % plugin | 32 | section_name = 'log-processor-%s' % plugin |
282 | 29 | plugin_conf = utils.readconf(sys.argv[1], section_name) | 33 | plugin_conf = utils.readconf(conf_file, section_name) |
283 | 30 | uploader_conf.update(plugin_conf) | 34 | uploader_conf.update(plugin_conf) |
285 | 31 | uploader = LogUploader(uploader_conf, plugin).run(once=True) | 35 | |
286 | 36 | # pre-configure logger | ||
287 | 37 | logger = utils.get_logger(uploader_conf, plugin, | ||
288 | 38 | log_to_console=options.get('verbose', False)) | ||
289 | 39 | # currently LogUploader only supports run_once | ||
290 | 40 | options['once'] = True | ||
291 | 41 | uploader = LogUploader(uploader_conf, plugin).run(**options) | ||
292 | 32 | 42 | ||
293 | === modified file 'bin/swift-object-auditor' | |||
294 | --- bin/swift-object-auditor 2010-08-31 23:12:59 +0000 | |||
295 | +++ bin/swift-object-auditor 2010-11-19 22:38:08 +0000 | |||
296 | @@ -14,16 +14,10 @@ | |||
297 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
298 | 15 | # limitations under the License. | 15 | # limitations under the License. |
299 | 16 | 16 | ||
300 | 17 | import sys | ||
301 | 18 | |||
302 | 19 | from swift.obj.auditor import ObjectAuditor | 17 | from swift.obj.auditor import ObjectAuditor |
304 | 20 | from swift.common import utils | 18 | from swift.common.utils import parse_options |
305 | 19 | from swift.common.daemon import run_daemon | ||
306 | 21 | 20 | ||
307 | 22 | if __name__ == '__main__': | 21 | if __name__ == '__main__': |
315 | 23 | if len(sys.argv) < 2: | 22 | conf_file, options = parse_options(once=True) |
316 | 24 | print "Usage: swift-object-auditor CONFIG_FILE [once]" | 23 | run_daemon(ObjectAuditor, conf_file, **options) |
310 | 25 | sys.exit() | ||
311 | 26 | |||
312 | 27 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | ||
313 | 28 | conf = utils.readconf(sys.argv[1], 'object-auditor') | ||
314 | 29 | ObjectAuditor(conf).run(once) | ||
317 | 30 | 24 | ||
318 | === modified file 'bin/swift-object-replicator' | |||
319 | --- bin/swift-object-replicator 2010-08-31 23:12:59 +0000 | |||
320 | +++ bin/swift-object-replicator 2010-11-19 22:38:08 +0000 | |||
321 | @@ -14,16 +14,10 @@ | |||
322 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
323 | 15 | # limitations under the License. | 15 | # limitations under the License. |
324 | 16 | 16 | ||
325 | 17 | import sys | ||
326 | 18 | |||
327 | 19 | from swift.obj.replicator import ObjectReplicator | 17 | from swift.obj.replicator import ObjectReplicator |
329 | 20 | from swift.common import utils | 18 | from swift.common.utils import parse_options |
330 | 19 | from swift.common.daemon import run_daemon | ||
331 | 21 | 20 | ||
332 | 22 | if __name__ == '__main__': | 21 | if __name__ == '__main__': |
340 | 23 | if len(sys.argv) < 2: | 22 | conf_file, options = parse_options(once=True) |
341 | 24 | print "Usage: swift-object-replicator CONFIG_FILE [once]" | 23 | run_daemon(ObjectReplicator, conf_file, **options) |
335 | 25 | sys.exit() | ||
336 | 26 | conf = utils.readconf(sys.argv[1], "object-replicator") | ||
337 | 27 | once = (len(sys.argv) > 2 and sys.argv[2] == 'once') or \ | ||
338 | 28 | conf.get('daemonize', 'true') not in utils.TRUE_VALUES | ||
339 | 29 | ObjectReplicator(conf).run(once) | ||
342 | 30 | 24 | ||
343 | === modified file 'bin/swift-object-server' | |||
344 | --- bin/swift-object-server 2010-08-24 14:04:44 +0000 | |||
345 | +++ bin/swift-object-server 2010-11-19 22:38:08 +0000 | |||
346 | @@ -14,11 +14,9 @@ | |||
347 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
348 | 15 | # limitations under the License. | 15 | # limitations under the License. |
349 | 16 | 16 | ||
352 | 17 | import sys | 17 | from swift.common.utils import parse_options |
351 | 18 | |||
353 | 19 | from swift.common.wsgi import run_wsgi | 18 | from swift.common.wsgi import run_wsgi |
354 | 20 | 19 | ||
355 | 21 | if __name__ == '__main__': | 20 | if __name__ == '__main__': |
359 | 22 | if len(sys.argv) != 2: | 21 | conf_file, options = parse_options() |
360 | 23 | sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) | 22 | run_wsgi(conf_file, 'object-server', default_port=6000, **options) |
358 | 24 | run_wsgi(sys.argv[1], 'object-server', default_port=6000) | ||
361 | 25 | 23 | ||
362 | === modified file 'bin/swift-object-updater' | |||
363 | --- bin/swift-object-updater 2010-08-31 23:12:59 +0000 | |||
364 | +++ bin/swift-object-updater 2010-11-19 22:38:08 +0000 | |||
365 | @@ -14,15 +14,10 @@ | |||
366 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
367 | 15 | # limitations under the License. | 15 | # limitations under the License. |
368 | 16 | 16 | ||
369 | 17 | import sys | ||
370 | 18 | |||
371 | 19 | from swift.obj.updater import ObjectUpdater | 17 | from swift.obj.updater import ObjectUpdater |
373 | 20 | from swift.common import utils | 18 | from swift.common.utils import parse_options |
374 | 19 | from swift.common.daemon import run_daemon | ||
375 | 21 | 20 | ||
376 | 22 | if __name__ == '__main__': | 21 | if __name__ == '__main__': |
383 | 23 | if len(sys.argv) < 2: | 22 | conf_file, options = parse_options(once=True) |
384 | 24 | print "Usage: swift-object-updater CONFIG_FILE [once]" | 23 | run_daemon(ObjectUpdater, conf_file, **options) |
379 | 25 | sys.exit(1) | ||
380 | 26 | once = len(sys.argv) > 2 and sys.argv[2] == 'once' | ||
381 | 27 | conf = utils.readconf(sys.argv[1], 'object-updater') | ||
382 | 28 | ObjectUpdater(conf).run(once) | ||
385 | 29 | 24 | ||
386 | === modified file 'bin/swift-proxy-server' | |||
387 | --- bin/swift-proxy-server 2010-08-24 14:04:44 +0000 | |||
388 | +++ bin/swift-proxy-server 2010-11-19 22:38:08 +0000 | |||
389 | @@ -14,11 +14,9 @@ | |||
390 | 14 | # See the License for the specific language governing permissions and | 14 | # See the License for the specific language governing permissions and |
391 | 15 | # limitations under the License. | 15 | # limitations under the License. |
392 | 16 | 16 | ||
395 | 17 | import sys | 17 | from swift.common.utils import parse_options |
394 | 18 | |||
396 | 19 | from swift.common.wsgi import run_wsgi | 18 | from swift.common.wsgi import run_wsgi |
397 | 20 | 19 | ||
398 | 21 | if __name__ == '__main__': | 20 | if __name__ == '__main__': |
402 | 22 | if len(sys.argv) != 2: | 21 | conf_file, options = parse_options() |
403 | 23 | sys.exit("Usage: %s CONFIG_FILE" % sys.argv[0]) | 22 | run_wsgi(conf_file, 'proxy-server', default_port=8080, **options) |
401 | 24 | run_wsgi(sys.argv[1], 'proxy-server', default_port=8080) | ||
404 | 25 | 23 | ||
405 | === modified file 'swift/common/daemon.py' | |||
406 | --- swift/common/daemon.py 2010-10-15 19:28:38 +0000 | |||
407 | +++ swift/common/daemon.py 2010-11-19 22:38:08 +0000 | |||
408 | @@ -16,6 +16,7 @@ | |||
409 | 16 | import os | 16 | import os |
410 | 17 | import sys | 17 | import sys |
411 | 18 | import signal | 18 | import signal |
412 | 19 | from re import sub | ||
413 | 19 | from swift.common import utils | 20 | from swift.common import utils |
414 | 20 | 21 | ||
415 | 21 | 22 | ||
416 | @@ -34,23 +35,11 @@ | |||
417 | 34 | """Override this to run forever""" | 35 | """Override this to run forever""" |
418 | 35 | raise NotImplementedError('run_forever not implemented') | 36 | raise NotImplementedError('run_forever not implemented') |
419 | 36 | 37 | ||
421 | 37 | def run(self, once=False, capture_stdout=True, capture_stderr=True): | 38 | def run(self, once=False, **kwargs): |
422 | 38 | """Run the daemon""" | 39 | """Run the daemon""" |
431 | 39 | # log uncaught exceptions | 40 | utils.validate_configuration() |
432 | 40 | sys.excepthook = lambda *exc_info: \ | 41 | utils.capture_stdio(self.logger, **kwargs) |
425 | 41 | self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) | ||
426 | 42 | if capture_stdout: | ||
427 | 43 | sys.stdout = utils.LoggerFileObject(self.logger) | ||
428 | 44 | if capture_stderr: | ||
429 | 45 | sys.stderr = utils.LoggerFileObject(self.logger) | ||
430 | 46 | |||
433 | 47 | utils.drop_privileges(self.conf.get('user', 'swift')) | 42 | utils.drop_privileges(self.conf.get('user', 'swift')) |
434 | 48 | utils.validate_configuration() | ||
435 | 49 | |||
436 | 50 | try: | ||
437 | 51 | os.setsid() | ||
438 | 52 | except OSError: | ||
439 | 53 | pass | ||
440 | 54 | 43 | ||
441 | 55 | def kill_children(*args): | 44 | def kill_children(*args): |
442 | 56 | signal.signal(signal.SIGTERM, signal.SIG_IGN) | 45 | signal.signal(signal.SIGTERM, signal.SIG_IGN) |
443 | @@ -63,3 +52,40 @@ | |||
444 | 63 | self.run_once() | 52 | self.run_once() |
445 | 64 | else: | 53 | else: |
446 | 65 | self.run_forever() | 54 | self.run_forever() |
447 | 55 | |||
448 | 56 | |||
449 | 57 | def run_daemon(klass, conf_file, section_name='', | ||
450 | 58 | once=False, **kwargs): | ||
451 | 59 | """ | ||
452 | 60 | Loads settings from conf, then instantiates daemon "klass" and runs the | ||
453 | 61 | daemon with the specified once kwarg. The section_name will be derived | ||
454 | 62 | from the daemon "klass" if not provided (e.g. ObjectReplicator => | ||
455 | 63 | object-replicator). | ||
456 | 64 | |||
457 | 65 | :param klass: Class to instantiate, subclass of common.daemon.Daemon | ||
458 | 66 | :param conf_file: Path to configuration file | ||
459 | 67 | :param section_name: Section name from conf file to load config from | ||
460 | 68 | :param once: Passed to daemon run method | ||
461 | 69 | """ | ||
462 | 70 | # very often the config section_name is based on the class name | ||
463 | 71 | # the None singleton will be passed through to readconf as is | ||
464 | 72 | if section_name is '': | ||
465 | 73 | section_name = sub(r'([a-z])([A-Z])', r'\1-\2', | ||
466 | 74 | klass.__name__).lower() | ||
467 | 75 | conf = utils.readconf(conf_file, section_name, | ||
468 | 76 | log_name=kwargs.get('log_name')) | ||
469 | 77 | |||
470 | 78 | # once on command line (i.e. daemonize=false) will over-ride config | ||
471 | 79 | once = once or conf.get('daemonize', 'true') not in utils.TRUE_VALUES | ||
472 | 80 | |||
473 | 81 | # pre-configure logger | ||
474 | 82 | if 'logger' in kwargs: | ||
475 | 83 | logger = kwargs.pop('logger') | ||
476 | 84 | else: | ||
477 | 85 | logger = utils.get_logger(conf, conf.get('log_name', section_name), | ||
478 | 86 | log_to_console=kwargs.pop('verbose', False)) | ||
479 | 87 | try: | ||
480 | 88 | klass(conf).run(once=once, **kwargs) | ||
481 | 89 | except KeyboardInterrupt: | ||
482 | 90 | logger.info('User quit') | ||
483 | 91 | logger.info('Exited') | ||
484 | 66 | 92 | ||
485 | === modified file 'swift/common/db_replicator.py' | |||
486 | --- swift/common/db_replicator.py 2010-09-01 15:56:37 +0000 | |||
487 | +++ swift/common/db_replicator.py 2010-11-19 22:38:08 +0000 | |||
488 | @@ -93,10 +93,6 @@ | |||
489 | 93 | def __init__(self, conf): | 93 | def __init__(self, conf): |
490 | 94 | self.conf = conf | 94 | self.conf = conf |
491 | 95 | self.logger = get_logger(conf) | 95 | self.logger = get_logger(conf) |
492 | 96 | # log uncaught exceptions | ||
493 | 97 | sys.excepthook = lambda * exc_info: \ | ||
494 | 98 | self.logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) | ||
495 | 99 | sys.stdout = sys.stderr = LoggerFileObject(self.logger) | ||
496 | 100 | self.root = conf.get('devices', '/srv/node') | 96 | self.root = conf.get('devices', '/srv/node') |
497 | 101 | self.mount_check = conf.get('mount_check', 'true').lower() in \ | 97 | self.mount_check = conf.get('mount_check', 'true').lower() in \ |
498 | 102 | ('true', 't', '1', 'on', 'yes', 'y') | 98 | ('true', 't', '1', 'on', 'yes', 'y') |
499 | 103 | 99 | ||
500 | === modified file 'swift/common/utils.py' | |||
501 | --- swift/common/utils.py 2010-11-16 19:52:18 +0000 | |||
502 | +++ swift/common/utils.py 2010-11-19 22:38:08 +0000 | |||
503 | @@ -31,6 +31,7 @@ | |||
504 | 31 | import ctypes.util | 31 | import ctypes.util |
505 | 32 | import struct | 32 | import struct |
506 | 33 | from ConfigParser import ConfigParser, NoSectionError, NoOptionError | 33 | from ConfigParser import ConfigParser, NoSectionError, NoOptionError |
507 | 34 | from optparse import OptionParser | ||
508 | 34 | from tempfile import mkstemp | 35 | from tempfile import mkstemp |
509 | 35 | import cPickle as pickle | 36 | import cPickle as pickle |
510 | 36 | 37 | ||
511 | @@ -283,17 +284,6 @@ | |||
512 | 283 | return self | 284 | return self |
513 | 284 | 285 | ||
514 | 285 | 286 | ||
515 | 286 | def drop_privileges(user): | ||
516 | 287 | """ | ||
517 | 288 | Sets the userid of the current process | ||
518 | 289 | |||
519 | 290 | :param user: User id to change privileges to | ||
520 | 291 | """ | ||
521 | 292 | user = pwd.getpwnam(user) | ||
522 | 293 | os.setgid(user[3]) | ||
523 | 294 | os.setuid(user[2]) | ||
524 | 295 | |||
525 | 296 | |||
526 | 297 | class NamedLogger(object): | 287 | class NamedLogger(object): |
527 | 298 | """Cheesy version of the LoggerAdapter available in Python 3""" | 288 | """Cheesy version of the LoggerAdapter available in Python 3""" |
528 | 299 | 289 | ||
529 | @@ -343,7 +333,7 @@ | |||
530 | 343 | call('%s %s: %s' % (self.server, msg, emsg), *args) | 333 | call('%s %s: %s' % (self.server, msg, emsg), *args) |
531 | 344 | 334 | ||
532 | 345 | 335 | ||
534 | 346 | def get_logger(conf, name=None): | 336 | def get_logger(conf, name=None, log_to_console=False): |
535 | 347 | """ | 337 | """ |
536 | 348 | Get the current system logger using config settings. | 338 | Get the current system logger using config settings. |
537 | 349 | 339 | ||
538 | @@ -355,11 +345,18 @@ | |||
539 | 355 | 345 | ||
540 | 356 | :param conf: Configuration dict to read settings from | 346 | :param conf: Configuration dict to read settings from |
541 | 357 | :param name: Name of the logger | 347 | :param name: Name of the logger |
542 | 348 | :param log_to_console: Add handler which writes to console on stderr | ||
543 | 358 | """ | 349 | """ |
544 | 359 | root_logger = logging.getLogger() | 350 | root_logger = logging.getLogger() |
545 | 360 | if hasattr(get_logger, 'handler') and get_logger.handler: | 351 | if hasattr(get_logger, 'handler') and get_logger.handler: |
546 | 361 | root_logger.removeHandler(get_logger.handler) | 352 | root_logger.removeHandler(get_logger.handler) |
547 | 362 | get_logger.handler = None | 353 | get_logger.handler = None |
548 | 354 | if log_to_console: | ||
549 | 355 | # check if a previous call to get_logger already added a console logger | ||
550 | 356 | if hasattr(get_logger, 'console') and get_logger.console: | ||
551 | 357 | root_logger.removeHandler(get_logger.console) | ||
552 | 358 | get_logger.console = logging.StreamHandler(sys.__stderr__) | ||
553 | 359 | root_logger.addHandler(get_logger.console) | ||
554 | 363 | if conf is None: | 360 | if conf is None: |
555 | 364 | root_logger.setLevel(logging.INFO) | 361 | root_logger.setLevel(logging.INFO) |
556 | 365 | return NamedLogger(root_logger, name) | 362 | return NamedLogger(root_logger, name) |
557 | @@ -375,6 +372,99 @@ | |||
558 | 375 | return NamedLogger(root_logger, name) | 372 | return NamedLogger(root_logger, name) |
559 | 376 | 373 | ||
560 | 377 | 374 | ||
561 | 375 | def drop_privileges(user): | ||
562 | 376 | """ | ||
563 | 377 | Sets the userid/groupid of the current process, get session leader, etc. | ||
564 | 378 | |||
565 | 379 | :param user: User name to change privileges to | ||
566 | 380 | """ | ||
567 | 381 | user = pwd.getpwnam(user) | ||
568 | 382 | os.setgid(user[3]) | ||
569 | 383 | os.setuid(user[2]) | ||
570 | 384 | try: | ||
571 | 385 | os.setsid() | ||
572 | 386 | except OSError: | ||
573 | 387 | pass | ||
574 | 388 | os.chdir('/') # in case you need to rmdir on where you started the daemon | ||
575 | 389 | os.umask(0) # ensure files are created with the correct privileges | ||
576 | 390 | |||
577 | 391 | |||
578 | 392 | def capture_stdio(logger, **kwargs): | ||
579 | 393 | """ | ||
580 | 394 | Log unhandled exceptions, close stdio, capture stdout and stderr. | ||
581 | 395 | |||
582 | 396 | param logger: Logger object to use | ||
583 | 397 | """ | ||
584 | 398 | # log uncaught exceptions | ||
585 | 399 | sys.excepthook = lambda * exc_info: \ | ||
586 | 400 | logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) | ||
587 | 401 | |||
588 | 402 | # collect stdio file desc not in use for logging | ||
589 | 403 | stdio_fds = [0, 1, 2] | ||
590 | 404 | if hasattr(get_logger, 'console'): | ||
591 | 405 | stdio_fds.remove(get_logger.console.stream.fileno()) | ||
592 | 406 | |||
593 | 407 | with open(os.devnull, 'r+b') as nullfile: | ||
594 | 408 | # close stdio (excludes fds open for logging) | ||
595 | 409 | for desc in stdio_fds: | ||
596 | 410 | try: | ||
597 | 411 | os.dup2(nullfile.fileno(), desc) | ||
598 | 412 | except OSError: | ||
599 | 413 | pass | ||
600 | 414 | |||
601 | 415 | # redirect stdio | ||
602 | 416 | if kwargs.pop('capture_stdout', True): | ||
603 | 417 | sys.stdout = LoggerFileObject(logger) | ||
604 | 418 | if kwargs.pop('capture_stderr', True): | ||
605 | 419 | sys.stderr = LoggerFileObject(logger) | ||
606 | 420 | |||
607 | 421 | |||
608 | 422 | def parse_options(usage="%prog CONFIG [options]", once=False, test_args=None): | ||
609 | 423 | """ | ||
610 | 424 | Parse standard swift server/daemon options with optparse.OptionParser. | ||
611 | 425 | |||
612 | 426 | :param usage: String describing usage | ||
613 | 427 | :param once: Boolean indicating the "once" option is available | ||
614 | 428 | :param test_args: Override sys.argv; used in testing | ||
615 | 429 | |||
616 | 430 | :returns : Tuple of (config, options); config is an absolute path to the | ||
617 | 431 | config file, options is the parser options as a dictionary. | ||
618 | 432 | |||
619 | 433 | :raises SystemExit: First arg (CONFIG) is required, file must exist | ||
620 | 434 | """ | ||
621 | 435 | parser = OptionParser(usage) | ||
622 | 436 | parser.add_option("-v", "--verbose", default=False, action="store_true", | ||
623 | 437 | help="log to console") | ||
624 | 438 | if once: | ||
625 | 439 | parser.add_option("-o", "--once", default=False, action="store_true", | ||
626 | 440 | help="only run one pass of daemon") | ||
627 | 441 | |||
628 | 442 | # if test_args is None, optparse will use sys.argv[:1] | ||
629 | 443 | options, args = parser.parse_args(args=test_args) | ||
630 | 444 | |||
631 | 445 | if not args: | ||
632 | 446 | parser.print_usage() | ||
633 | 447 | print "Error: missing config file argument" | ||
634 | 448 | sys.exit(1) | ||
635 | 449 | config = os.path.abspath(args.pop(0)) | ||
636 | 450 | if not os.path.exists(config): | ||
637 | 451 | parser.print_usage() | ||
638 | 452 | print "Error: unable to locate %s" % config | ||
639 | 453 | sys.exit(1) | ||
640 | 454 | |||
641 | 455 | extra_args = [] | ||
642 | 456 | # if any named options appear in remaining args, set the option to True | ||
643 | 457 | for arg in args: | ||
644 | 458 | if arg in options.__dict__: | ||
645 | 459 | setattr(options, arg, True) | ||
646 | 460 | else: | ||
647 | 461 | extra_args.append(arg) | ||
648 | 462 | |||
649 | 463 | options = vars(options) | ||
650 | 464 | options['extra_args'] = extra_args | ||
651 | 465 | return config, options | ||
652 | 466 | |||
653 | 467 | |||
654 | 378 | def whataremyips(): | 468 | def whataremyips(): |
655 | 379 | """ | 469 | """ |
656 | 380 | Get the machine's ip addresses using ifconfig | 470 | Get the machine's ip addresses using ifconfig |
657 | 381 | 471 | ||
658 | === modified file 'swift/common/wsgi.py' | |||
659 | --- swift/common/wsgi.py 2010-10-15 19:28:38 +0000 | |||
660 | +++ swift/common/wsgi.py 2010-11-19 22:38:08 +0000 | |||
661 | @@ -34,7 +34,7 @@ | |||
662 | 34 | from eventlet.green import socket, ssl | 34 | from eventlet.green import socket, ssl |
663 | 35 | 35 | ||
664 | 36 | from swift.common.utils import get_logger, drop_privileges, \ | 36 | from swift.common.utils import get_logger, drop_privileges, \ |
666 | 37 | validate_configuration, LoggerFileObject, NullLogger | 37 | validate_configuration, capture_stdio, NullLogger |
667 | 38 | 38 | ||
668 | 39 | 39 | ||
669 | 40 | def monkey_patch_mimetools(): | 40 | def monkey_patch_mimetools(): |
670 | @@ -56,41 +56,17 @@ | |||
671 | 56 | 56 | ||
672 | 57 | mimetools.Message.parsetype = parsetype | 57 | mimetools.Message.parsetype = parsetype |
673 | 58 | 58 | ||
707 | 59 | 59 | def get_socket(conf, default_port=8080): | |
708 | 60 | # We might be able to pull pieces of this out to test, but right now it seems | 60 | """Bind socket to bind ip:port in conf |
709 | 61 | # like more work than it's worth. | 61 | |
710 | 62 | def run_wsgi(conf_file, app_section, *args, **kwargs): # pragma: no cover | 62 | :param conf: Configuration dict to read settings from |
711 | 63 | """ | 63 | :param default_port: port to use if not specified in conf |
712 | 64 | Loads common settings from conf, then instantiates app and runs | 64 | |
713 | 65 | the server using the specified number of workers. | 65 | :returns : a socket object as returned from socket.listen or ssl.wrap_socket |
714 | 66 | 66 | if conf specifies cert_file | |
715 | 67 | :param conf_file: Path to paste.deploy style configuration file | 67 | """ |
683 | 68 | :param app_section: App name from conf file to load config from | ||
684 | 69 | """ | ||
685 | 70 | |||
686 | 71 | try: | ||
687 | 72 | conf = appconfig('config:%s' % conf_file, name=app_section) | ||
688 | 73 | log_name = conf.get('log_name', app_section) | ||
689 | 74 | app = loadapp('config:%s' % conf_file, | ||
690 | 75 | global_conf={'log_name': log_name}) | ||
691 | 76 | except Exception, e: | ||
692 | 77 | print "Error trying to load config %s: %s" % (conf_file, e) | ||
693 | 78 | return | ||
694 | 79 | if 'logger' in kwargs: | ||
695 | 80 | logger = kwargs['logger'] | ||
696 | 81 | else: | ||
697 | 82 | logger = get_logger(conf, log_name) | ||
698 | 83 | # log uncaught exceptions | ||
699 | 84 | sys.excepthook = lambda * exc_info: \ | ||
700 | 85 | logger.critical('UNCAUGHT EXCEPTION', exc_info=exc_info) | ||
701 | 86 | sys.stdout = sys.stderr = LoggerFileObject(logger) | ||
702 | 87 | |||
703 | 88 | try: | ||
704 | 89 | os.setsid() | ||
705 | 90 | except OSError: | ||
706 | 91 | no_cover = True # pass | ||
716 | 92 | bind_addr = (conf.get('bind_ip', '0.0.0.0'), | 68 | bind_addr = (conf.get('bind_ip', '0.0.0.0'), |
718 | 93 | int(conf.get('bind_port', kwargs.get('default_port', 8080)))) | 69 | int(conf.get('bind_port', default_port))) |
719 | 94 | sock = None | 70 | sock = None |
720 | 95 | retry_until = time.time() + 30 | 71 | retry_until = time.time() + 30 |
721 | 96 | while not sock and time.time() < retry_until: | 72 | while not sock and time.time() < retry_until: |
722 | @@ -110,9 +86,43 @@ | |||
723 | 110 | # in my experience, sockets can hang around forever without keepalive | 86 | # in my experience, sockets can hang around forever without keepalive |
724 | 111 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) | 87 | sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) |
725 | 112 | sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) | 88 | sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, 600) |
727 | 113 | worker_count = int(conf.get('workers', '1')) | 89 | return sock |
728 | 90 | |||
729 | 91 | |||
730 | 92 | # TODO: pull pieces of this out to test | ||
731 | 93 | def run_wsgi(conf_file, app_section, *args, **kwargs): | ||
732 | 94 | """ | ||
733 | 95 | Loads common settings from conf, then instantiates app and runs | ||
734 | 96 | the server using the specified number of workers. | ||
735 | 97 | |||
736 | 98 | :param conf_file: Path to paste.deploy style configuration file | ||
737 | 99 | :param app_section: App name from conf file to load config from | ||
738 | 100 | """ | ||
739 | 101 | |||
740 | 102 | try: | ||
741 | 103 | conf = appconfig('config:%s' % conf_file, name=app_section) | ||
742 | 104 | except Exception, e: | ||
743 | 105 | print "Error trying to load config %s: %s" % (conf_file, e) | ||
744 | 106 | return | ||
745 | 107 | validate_configuration() | ||
746 | 108 | |||
747 | 109 | # pre-configure logger | ||
748 | 110 | log_name = conf.get('log_name', app_section) | ||
749 | 111 | if 'logger' in kwargs: | ||
750 | 112 | logger = kwargs.pop('logger') | ||
751 | 113 | else: | ||
752 | 114 | logger = get_logger(conf, log_name, | ||
753 | 115 | log_to_console=kwargs.pop('verbose', False)) | ||
754 | 116 | |||
755 | 117 | # redirect errors to logger and close stdio | ||
756 | 118 | capture_stdio(logger) | ||
757 | 119 | # bind to address and port | ||
758 | 120 | sock = get_socket(conf, default_port=kwargs.get('default_port', 8080)) | ||
759 | 121 | # remaining tasks should not require elevated privileges | ||
760 | 114 | drop_privileges(conf.get('user', 'swift')) | 122 | drop_privileges(conf.get('user', 'swift')) |
762 | 115 | validate_configuration() | 123 | |
763 | 124 | # finally after binding to ports and privilege drop, run app __init__ code | ||
764 | 125 | app = loadapp('config:%s' % conf_file, global_conf={'log_name': log_name}) | ||
765 | 116 | 126 | ||
766 | 117 | def run_server(): | 127 | def run_server(): |
767 | 118 | wsgi.HttpProtocol.default_request_version = "HTTP/1.0" | 128 | wsgi.HttpProtocol.default_request_version = "HTTP/1.0" |
768 | @@ -127,6 +137,7 @@ | |||
769 | 127 | raise | 137 | raise |
770 | 128 | pool.waitall() | 138 | pool.waitall() |
771 | 129 | 139 | ||
772 | 140 | worker_count = int(conf.get('workers', '1')) | ||
773 | 130 | # Useful for profiling [no forks]. | 141 | # Useful for profiling [no forks]. |
774 | 131 | if worker_count == 0: | 142 | if worker_count == 0: |
775 | 132 | run_server() | 143 | run_server() |
776 | @@ -169,6 +180,9 @@ | |||
777 | 169 | except OSError, err: | 180 | except OSError, err: |
778 | 170 | if err.errno not in (errno.EINTR, errno.ECHILD): | 181 | if err.errno not in (errno.EINTR, errno.ECHILD): |
779 | 171 | raise | 182 | raise |
780 | 183 | except KeyboardInterrupt: | ||
781 | 184 | logger.info('User quit') | ||
782 | 185 | break | ||
783 | 172 | greenio.shutdown_safe(sock) | 186 | greenio.shutdown_safe(sock) |
784 | 173 | sock.close() | 187 | sock.close() |
785 | 174 | logger.info('Exited') | 188 | logger.info('Exited') |
786 | 175 | 189 | ||
787 | === modified file 'test/unit/__init__.py' | |||
788 | --- test/unit/__init__.py 2010-07-29 20:06:01 +0000 | |||
789 | +++ test/unit/__init__.py 2010-11-19 22:38:08 +0000 | |||
790 | @@ -1,5 +1,8 @@ | |||
791 | 1 | """ Swift tests """ | 1 | """ Swift tests """ |
792 | 2 | 2 | ||
793 | 3 | import os | ||
794 | 4 | from contextlib import contextmanager | ||
795 | 5 | from tempfile import NamedTemporaryFile | ||
796 | 3 | from eventlet.green import socket | 6 | from eventlet.green import socket |
797 | 4 | 7 | ||
798 | 5 | 8 | ||
799 | @@ -23,6 +26,18 @@ | |||
800 | 23 | rv.connect(hostport) | 26 | rv.connect(hostport) |
801 | 24 | return rv | 27 | return rv |
802 | 25 | 28 | ||
803 | 29 | |||
804 | 30 | @contextmanager | ||
805 | 31 | def tmpfile(content): | ||
806 | 32 | with NamedTemporaryFile('w', delete=False) as f: | ||
807 | 33 | file_name = f.name | ||
808 | 34 | f.write(str(content)) | ||
809 | 35 | try: | ||
810 | 36 | yield file_name | ||
811 | 37 | finally: | ||
812 | 38 | os.unlink(file_name) | ||
813 | 39 | |||
814 | 40 | |||
815 | 26 | class MockTrue(object): | 41 | class MockTrue(object): |
816 | 27 | """ | 42 | """ |
817 | 28 | Instances of MockTrue evaluate like True | 43 | Instances of MockTrue evaluate like True |
818 | 29 | 44 | ||
819 | === modified file 'test/unit/common/test_daemon.py' | |||
820 | --- test/unit/common/test_daemon.py 2010-10-07 15:23:17 +0000 | |||
821 | +++ test/unit/common/test_daemon.py 2010-11-19 22:38:08 +0000 | |||
822 | @@ -13,16 +13,95 @@ | |||
823 | 13 | # See the License for the specific language governing permissions and | 13 | # See the License for the specific language governing permissions and |
824 | 14 | # limitations under the License. | 14 | # limitations under the License. |
825 | 15 | 15 | ||
827 | 16 | # TODO: Tests | 16 | # TODO: Test kill_children signal handlers |
828 | 17 | 17 | ||
829 | 18 | import unittest | 18 | import unittest |
831 | 19 | from swift.common import daemon | 19 | from getpass import getuser |
832 | 20 | import logging | ||
833 | 21 | from StringIO import StringIO | ||
834 | 22 | from test.unit import tmpfile | ||
835 | 23 | |||
836 | 24 | from swift.common import daemon, utils | ||
837 | 25 | |||
838 | 26 | |||
839 | 27 | class MyDaemon(daemon.Daemon): | ||
840 | 28 | |||
841 | 29 | def __init__(self, conf): | ||
842 | 30 | self.conf = conf | ||
843 | 31 | self.logger = utils.get_logger(None) | ||
844 | 32 | MyDaemon.forever_called = False | ||
845 | 33 | MyDaemon.once_called = False | ||
846 | 34 | |||
847 | 35 | def run_forever(self): | ||
848 | 36 | MyDaemon.forever_called = True | ||
849 | 37 | |||
850 | 38 | def run_once(self): | ||
851 | 39 | MyDaemon.once_called = True | ||
852 | 40 | |||
853 | 41 | def run_raise(self): | ||
854 | 42 | raise OSError | ||
855 | 43 | |||
856 | 44 | def run_quit(self): | ||
857 | 45 | raise KeyboardInterrupt | ||
858 | 20 | 46 | ||
859 | 21 | 47 | ||
860 | 22 | class TestDaemon(unittest.TestCase): | 48 | class TestDaemon(unittest.TestCase): |
861 | 23 | 49 | ||
864 | 24 | def test_placeholder(self): | 50 | def test_create(self): |
865 | 25 | pass | 51 | d = daemon.Daemon({}) |
866 | 52 | self.assertEquals(d.conf, {}) | ||
867 | 53 | self.assert_(isinstance(d.logger, utils.NamedLogger)) | ||
868 | 54 | |||
869 | 55 | def test_stubs(self): | ||
870 | 56 | d = daemon.Daemon({}) | ||
871 | 57 | self.assertRaises(NotImplementedError, d.run_once) | ||
872 | 58 | self.assertRaises(NotImplementedError, d.run_forever) | ||
873 | 59 | |||
874 | 60 | |||
875 | 61 | class TestRunDaemon(unittest.TestCase): | ||
876 | 62 | |||
877 | 63 | def setUp(self): | ||
878 | 64 | utils.HASH_PATH_SUFFIX = 'endcap' | ||
879 | 65 | utils.drop_privileges = lambda *args: None | ||
880 | 66 | utils.capture_stdio = lambda *args: None | ||
881 | 67 | |||
882 | 68 | def tearDown(self): | ||
883 | 69 | reload(utils) | ||
884 | 70 | |||
885 | 71 | def test_run(self): | ||
886 | 72 | d = MyDaemon({}) | ||
887 | 73 | self.assertFalse(MyDaemon.forever_called) | ||
888 | 74 | self.assertFalse(MyDaemon.once_called) | ||
889 | 75 | # test default | ||
890 | 76 | d.run() | ||
891 | 77 | self.assertEquals(d.forever_called, True) | ||
892 | 78 | # test once | ||
893 | 79 | d.run(once=True) | ||
894 | 80 | self.assertEquals(d.once_called, True) | ||
895 | 81 | |||
896 | 82 | def test_run_daemon(self): | ||
897 | 83 | sample_conf = """[my-daemon] | ||
898 | 84 | user = %s | ||
899 | 85 | """ % getuser() | ||
900 | 86 | with tmpfile(sample_conf) as conf_file: | ||
901 | 87 | daemon.run_daemon(MyDaemon, conf_file) | ||
902 | 88 | self.assertEquals(MyDaemon.forever_called, True) | ||
903 | 89 | daemon.run_daemon(MyDaemon, conf_file, once=True) | ||
904 | 90 | self.assertEquals(MyDaemon.once_called, True) | ||
905 | 91 | |||
906 | 92 | # test raise in daemon code | ||
907 | 93 | MyDaemon.run_once = MyDaemon.run_raise | ||
908 | 94 | self.assertRaises(OSError, daemon.run_daemon, MyDaemon, | ||
909 | 95 | conf_file, once=True) | ||
910 | 96 | |||
911 | 97 | # test user quit | ||
912 | 98 | MyDaemon.run_forever = MyDaemon.run_quit | ||
913 | 99 | sio = StringIO() | ||
914 | 100 | logger = logging.getLogger() | ||
915 | 101 | logger.addHandler(logging.StreamHandler(sio)) | ||
916 | 102 | logger = utils.get_logger(None, 'server') | ||
917 | 103 | daemon.run_daemon(MyDaemon, conf_file, logger=logger) | ||
918 | 104 | self.assert_('user quit' in sio.getvalue().lower()) | ||
919 | 26 | 105 | ||
920 | 27 | 106 | ||
921 | 28 | if __name__ == '__main__': | 107 | if __name__ == '__main__': |
922 | 29 | 108 | ||
923 | === modified file 'test/unit/common/test_utils.py' | |||
924 | --- test/unit/common/test_utils.py 2010-11-01 21:47:48 +0000 | |||
925 | +++ test/unit/common/test_utils.py 2010-11-19 22:38:08 +0000 | |||
926 | @@ -25,12 +25,55 @@ | |||
927 | 25 | from getpass import getuser | 25 | from getpass import getuser |
928 | 26 | from shutil import rmtree | 26 | from shutil import rmtree |
929 | 27 | from StringIO import StringIO | 27 | from StringIO import StringIO |
930 | 28 | from functools import partial | ||
931 | 29 | from tempfile import NamedTemporaryFile | ||
932 | 28 | 30 | ||
933 | 29 | from eventlet import sleep | 31 | from eventlet import sleep |
934 | 30 | 32 | ||
935 | 31 | from swift.common import utils | 33 | from swift.common import utils |
936 | 32 | 34 | ||
937 | 33 | 35 | ||
938 | 36 | class MockOs(): | ||
939 | 37 | def __init__(self, pass_funcs=[], called_funcs=[], raise_funcs=[]): | ||
940 | 38 | self.closed_fds = [] | ||
941 | 39 | for func in pass_funcs: | ||
942 | 40 | setattr(self, func, self.pass_func) | ||
943 | 41 | self.called_funcs = {} | ||
944 | 42 | for func in called_funcs: | ||
945 | 43 | c_func = partial(self.called_func, func) | ||
946 | 44 | setattr(self, func, c_func) | ||
947 | 45 | for func in raise_funcs: | ||
948 | 46 | r_func = partial(self.raise_func, func) | ||
949 | 47 | setattr(self, func, r_func) | ||
950 | 48 | |||
951 | 49 | def pass_func(self, *args, **kwargs): | ||
952 | 50 | pass | ||
953 | 51 | |||
954 | 52 | chdir = setsid = setgid = setuid = umask = pass_func | ||
955 | 53 | |||
956 | 54 | def called_func(self, name, *args, **kwargs): | ||
957 | 55 | self.called_funcs[name] = True | ||
958 | 56 | |||
959 | 57 | def raise_func(self, name, *args, **kwargs): | ||
960 | 58 | self.called_funcs[name] = True | ||
961 | 59 | raise OSError() | ||
962 | 60 | |||
963 | 61 | def dup2(self, source, target): | ||
964 | 62 | self.closed_fds.append(target) | ||
965 | 63 | |||
966 | 64 | def __getattr__(self, name): | ||
967 | 65 | # I only over-ride portions of the os module | ||
968 | 66 | try: | ||
969 | 67 | return object.__getattr__(self, name) | ||
970 | 68 | except AttributeError: | ||
971 | 69 | return getattr(os, name) | ||
972 | 70 | |||
973 | 71 | |||
974 | 72 | class MockSys(): | ||
975 | 73 | |||
976 | 74 | __stderr__ = sys.__stderr__ | ||
977 | 75 | |||
978 | 76 | |||
979 | 34 | class TestUtils(unittest.TestCase): | 77 | class TestUtils(unittest.TestCase): |
980 | 35 | """ Tests for swift.common.utils """ | 78 | """ Tests for swift.common.utils """ |
981 | 36 | 79 | ||
982 | @@ -182,10 +225,63 @@ | |||
983 | 182 | self.assertRaises(IOError, lfo.readline, 1024) | 225 | self.assertRaises(IOError, lfo.readline, 1024) |
984 | 183 | lfo.tell() | 226 | lfo.tell() |
985 | 184 | 227 | ||
990 | 185 | def test_drop_privileges(self): | 228 | def test_parse_options(self): |
991 | 186 | # Note that this doesn't really drop privileges as it just sets them to | 229 | # use mkstemp to get a file that is definately on disk |
992 | 187 | # what they already are; but it exercises the code at least. | 230 | with NamedTemporaryFile() as f: |
993 | 188 | utils.drop_privileges(getuser()) | 231 | conf_file = f.name |
994 | 232 | conf, options = utils.parse_options(test_args=[conf_file]) | ||
995 | 233 | self.assertEquals(conf, conf_file) | ||
996 | 234 | # assert defaults | ||
997 | 235 | self.assertEquals(options['verbose'], False) | ||
998 | 236 | self.assert_('once' not in options) | ||
999 | 237 | # assert verbose as option | ||
1000 | 238 | conf, options = utils.parse_options(test_args=[conf_file, '-v']) | ||
1001 | 239 | self.assertEquals(options['verbose'], True) | ||
1002 | 240 | # check once option | ||
1003 | 241 | conf, options = utils.parse_options(test_args=[conf_file], | ||
1004 | 242 | once=True) | ||
1005 | 243 | self.assertEquals(options['once'], False) | ||
1006 | 244 | test_args = [conf_file, '--once'] | ||
1007 | 245 | conf, options = utils.parse_options(test_args=test_args, once=True) | ||
1008 | 246 | self.assertEquals(options['once'], True) | ||
1009 | 247 | # check options as arg parsing | ||
1010 | 248 | test_args = [conf_file, 'once', 'plugin_name', 'verbose'] | ||
1011 | 249 | conf, options = utils.parse_options(test_args=test_args, once=True) | ||
1012 | 250 | self.assertEquals(options['verbose'], True) | ||
1013 | 251 | self.assertEquals(options['once'], True) | ||
1014 | 252 | self.assertEquals(options['extra_args'], ['plugin_name']) | ||
1015 | 253 | |||
1016 | 254 | def test_parse_options_errors(self): | ||
1017 | 255 | orig_stdout = sys.stdout | ||
1018 | 256 | orig_stderr = sys.stderr | ||
1019 | 257 | stdo = StringIO() | ||
1020 | 258 | stde = StringIO() | ||
1021 | 259 | utils.sys.stdout = stdo | ||
1022 | 260 | utils.sys.stderr = stde | ||
1023 | 261 | err_msg = """Usage: test usage | ||
1024 | 262 | |||
1025 | 263 | Error: missing config file argument | ||
1026 | 264 | """ | ||
1027 | 265 | test_args = [] | ||
1028 | 266 | self.assertRaises(SystemExit, utils.parse_options, 'test usage', True, | ||
1029 | 267 | test_args) | ||
1030 | 268 | self.assertEquals(stdo.getvalue(), err_msg) | ||
1031 | 269 | |||
1032 | 270 | # verify conf file must exist, context manager will delete temp file | ||
1033 | 271 | with NamedTemporaryFile() as f: | ||
1034 | 272 | conf_file = f.name | ||
1035 | 273 | err_msg += """Usage: test usage | ||
1036 | 274 | |||
1037 | 275 | Error: unable to locate %s | ||
1038 | 276 | """ % conf_file | ||
1039 | 277 | test_args = [conf_file] | ||
1040 | 278 | self.assertRaises(SystemExit, utils.parse_options, 'test usage', True, | ||
1041 | 279 | test_args) | ||
1042 | 280 | self.assertEquals(stdo.getvalue(), err_msg) | ||
1043 | 281 | |||
1044 | 282 | # reset stdio | ||
1045 | 283 | utils.sys.stdout = orig_stdout | ||
1046 | 284 | utils.sys.stderr = orig_stderr | ||
1047 | 189 | 285 | ||
1048 | 190 | def test_NamedLogger(self): | 286 | def test_NamedLogger(self): |
1049 | 191 | sio = StringIO() | 287 | sio = StringIO() |
1050 | @@ -275,5 +371,80 @@ | |||
1051 | 275 | self.assertEquals(result, expected) | 371 | self.assertEquals(result, expected) |
1052 | 276 | os.unlink('/tmp/test') | 372 | os.unlink('/tmp/test') |
1053 | 277 | 373 | ||
1054 | 374 | def test_drop_privileges(self): | ||
1055 | 375 | user = getuser() | ||
1056 | 376 | # over-ride os with mock | ||
1057 | 377 | required_func_calls = ('setgid', 'setuid', 'setsid', 'chdir', 'umask') | ||
1058 | 378 | utils.os = MockOs(called_funcs=required_func_calls) | ||
1059 | 379 | # exercise the code | ||
1060 | 380 | utils.drop_privileges(user) | ||
1061 | 381 | for func in required_func_calls: | ||
1062 | 382 | self.assert_(utils.os.called_funcs[func]) | ||
1063 | 383 | |||
1064 | 384 | # reset; test same args, OSError trying to get session leader | ||
1065 | 385 | utils.os = MockOs(called_funcs=required_func_calls, | ||
1066 | 386 | raise_funcs=('setsid',)) | ||
1067 | 387 | for func in required_func_calls: | ||
1068 | 388 | self.assertFalse(utils.os.called_funcs.get(func, False)) | ||
1069 | 389 | utils.drop_privileges(user) | ||
1070 | 390 | for func in required_func_calls: | ||
1071 | 391 | self.assert_(utils.os.called_funcs[func]) | ||
1072 | 392 | |||
1073 | 393 | def test_capture_stdio(self): | ||
1074 | 394 | # stubs | ||
1075 | 395 | logger = utils.get_logger(None, 'dummy') | ||
1076 | 396 | |||
1077 | 397 | # mock utils system modules | ||
1078 | 398 | utils.sys = MockSys() | ||
1079 | 399 | utils.os = MockOs() | ||
1080 | 400 | |||
1081 | 401 | # basic test | ||
1082 | 402 | utils.capture_stdio(logger) | ||
1083 | 403 | self.assert_(utils.sys.excepthook is not None) | ||
1084 | 404 | self.assertEquals(utils.os.closed_fds, [0, 1, 2]) | ||
1085 | 405 | self.assert_(utils.sys.stdout is not None) | ||
1086 | 406 | self.assert_(utils.sys.stderr is not None) | ||
1087 | 407 | |||
1088 | 408 | # reset; test same args, but exc when trying to close stdio | ||
1089 | 409 | utils.os = MockOs(raise_funcs=('dup2',)) | ||
1090 | 410 | utils.sys = MockSys() | ||
1091 | 411 | |||
1092 | 412 | # test unable to close stdio | ||
1093 | 413 | utils.capture_stdio(logger) | ||
1094 | 414 | self.assert_(utils.sys.excepthook is not None) | ||
1095 | 415 | self.assertEquals(utils.os.closed_fds, []) | ||
1096 | 416 | self.assert_(utils.sys.stdout is not None) | ||
1097 | 417 | self.assert_(utils.sys.stderr is not None) | ||
1098 | 418 | |||
1099 | 419 | # reset; test some other args | ||
1100 | 420 | logger = utils.get_logger(None, log_to_console=True) | ||
1101 | 421 | utils.os = MockOs() | ||
1102 | 422 | utils.sys = MockSys() | ||
1103 | 423 | |||
1104 | 424 | # test console log | ||
1105 | 425 | utils.capture_stdio(logger, capture_stdout=False, | ||
1106 | 426 | capture_stderr=False) | ||
1107 | 427 | self.assert_(utils.sys.excepthook is not None) | ||
1108 | 428 | # when logging to console, stderr remains open | ||
1109 | 429 | self.assertEquals(utils.os.closed_fds, [0, 1]) | ||
1110 | 430 | logger.logger.removeHandler(utils.get_logger.console) | ||
1111 | 431 | # stdio not captured | ||
1112 | 432 | self.assertFalse(hasattr(utils.sys, 'stdout')) | ||
1113 | 433 | self.assertFalse(hasattr(utils.sys, 'stderr')) | ||
1114 | 434 | |||
1115 | 435 | def test_get_logger_console(self): | ||
1116 | 436 | reload(utils) # reset get_logger attrs | ||
1117 | 437 | logger = utils.get_logger(None) | ||
1118 | 438 | self.assertFalse(hasattr(utils.get_logger, 'console')) | ||
1119 | 439 | logger = utils.get_logger(None, log_to_console=True) | ||
1120 | 440 | self.assert_(hasattr(utils.get_logger, 'console')) | ||
1121 | 441 | self.assert_(isinstance(utils.get_logger.console, | ||
1122 | 442 | logging.StreamHandler)) | ||
1123 | 443 | # make sure you can't have two console handlers | ||
1124 | 444 | old_handler = utils.get_logger.console | ||
1125 | 445 | logger = utils.get_logger(None, log_to_console=True) | ||
1126 | 446 | self.assertNotEquals(utils.get_logger.console, old_handler) | ||
1127 | 447 | logger.logger.removeHandler(utils.get_logger.console) | ||
1128 | 448 | |||
1129 | 278 | if __name__ == '__main__': | 449 | if __name__ == '__main__': |
1130 | 279 | unittest.main() | 450 | unittest.main() |
1131 | 280 | 451 | ||
1132 | === modified file 'test/unit/common/test_wsgi.py' | |||
1133 | --- test/unit/common/test_wsgi.py 2010-07-19 16:25:18 +0000 | |||
1134 | +++ test/unit/common/test_wsgi.py 2010-11-19 22:38:08 +0000 | |||
1135 | @@ -25,12 +25,12 @@ | |||
1136 | 25 | from getpass import getuser | 25 | from getpass import getuser |
1137 | 26 | from shutil import rmtree | 26 | from shutil import rmtree |
1138 | 27 | from StringIO import StringIO | 27 | from StringIO import StringIO |
1139 | 28 | from collections import defaultdict | ||
1140 | 28 | 29 | ||
1141 | 29 | from eventlet import sleep | 30 | from eventlet import sleep |
1142 | 30 | 31 | ||
1143 | 31 | from swift.common import wsgi | 32 | from swift.common import wsgi |
1144 | 32 | 33 | ||
1145 | 33 | |||
1146 | 34 | class TestWSGI(unittest.TestCase): | 34 | class TestWSGI(unittest.TestCase): |
1147 | 35 | """ Tests for swift.common.wsgi """ | 35 | """ Tests for swift.common.wsgi """ |
1148 | 36 | 36 | ||
1149 | @@ -72,5 +72,107 @@ | |||
1150 | 72 | sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') | 72 | sio = StringIO('Content-Type: text/html; charset=ISO-8859-4') |
1151 | 73 | self.assertEquals(mimetools.Message(sio).subtype, 'html') | 73 | self.assertEquals(mimetools.Message(sio).subtype, 'html') |
1152 | 74 | 74 | ||
1153 | 75 | def test_get_socket(self): | ||
1154 | 76 | # stubs | ||
1155 | 77 | conf = {} | ||
1156 | 78 | ssl_conf = { | ||
1157 | 79 | 'cert_file': '', | ||
1158 | 80 | 'key_file': '', | ||
1159 | 81 | } | ||
1160 | 82 | |||
1161 | 83 | # mocks | ||
1162 | 84 | class MockSocket(): | ||
1163 | 85 | def __init__(self): | ||
1164 | 86 | self.opts = defaultdict(dict) | ||
1165 | 87 | |||
1166 | 88 | def setsockopt(self, level, optname, value): | ||
1167 | 89 | self.opts[level][optname] = value | ||
1168 | 90 | |||
1169 | 91 | def mock_listen(*args, **kwargs): | ||
1170 | 92 | return MockSocket() | ||
1171 | 93 | |||
1172 | 94 | class MockSsl(): | ||
1173 | 95 | def __init__(self): | ||
1174 | 96 | self.wrap_socket_called = [] | ||
1175 | 97 | |||
1176 | 98 | def wrap_socket(self, sock, **kwargs): | ||
1177 | 99 | self.wrap_socket_called.append(kwargs) | ||
1178 | 100 | return sock | ||
1179 | 101 | |||
1180 | 102 | # patch | ||
1181 | 103 | old_listen = wsgi.listen | ||
1182 | 104 | old_ssl = wsgi.ssl | ||
1183 | 105 | try: | ||
1184 | 106 | wsgi.listen = mock_listen | ||
1185 | 107 | wsgi.ssl = MockSsl() | ||
1186 | 108 | # test | ||
1187 | 109 | sock = wsgi.get_socket(conf) | ||
1188 | 110 | # assert | ||
1189 | 111 | self.assert_(isinstance(sock, MockSocket)) | ||
1190 | 112 | expected_socket_opts = { | ||
1191 | 113 | socket.SOL_SOCKET: { | ||
1192 | 114 | socket.SO_REUSEADDR: 1, | ||
1193 | 115 | socket.SO_KEEPALIVE: 1, | ||
1194 | 116 | }, | ||
1195 | 117 | socket.IPPROTO_TCP: { | ||
1196 | 118 | socket.TCP_KEEPIDLE: 600, | ||
1197 | 119 | }, | ||
1198 | 120 | } | ||
1199 | 121 | self.assertEquals(sock.opts, expected_socket_opts) | ||
1200 | 122 | # test ssl | ||
1201 | 123 | sock = wsgi.get_socket(ssl_conf) | ||
1202 | 124 | expected_kwargs = { | ||
1203 | 125 | 'certfile': '', | ||
1204 | 126 | 'keyfile': '', | ||
1205 | 127 | } | ||
1206 | 128 | self.assertEquals(wsgi.ssl.wrap_socket_called, [expected_kwargs]) | ||
1207 | 129 | finally: | ||
1208 | 130 | wsgi.listen = old_listen | ||
1209 | 131 | wsgi.ssl = old_ssl | ||
1210 | 132 | |||
1211 | 133 | def test_address_in_use(self): | ||
1212 | 134 | # stubs | ||
1213 | 135 | conf = {} | ||
1214 | 136 | |||
1215 | 137 | # mocks | ||
1216 | 138 | def mock_listen(*args, **kwargs): | ||
1217 | 139 | raise socket.error(errno.EADDRINUSE) | ||
1218 | 140 | |||
1219 | 141 | def value_error_listen(*args, **kwargs): | ||
1220 | 142 | raise ValueError('fake') | ||
1221 | 143 | |||
1222 | 144 | def mock_sleep(*args): | ||
1223 | 145 | pass | ||
1224 | 146 | |||
1225 | 147 | class MockTime(): | ||
1226 | 148 | """Fast clock advances 10 seconds after every call to time | ||
1227 | 149 | """ | ||
1228 | 150 | def __init__(self): | ||
1229 | 151 | self.current_time = old_time.time() | ||
1230 | 152 | |||
1231 | 153 | def time(self, *args, **kwargs): | ||
1232 | 154 | rv = self.current_time | ||
1233 | 155 | # advance for next call | ||
1234 | 156 | self.current_time += 10 | ||
1235 | 157 | return rv | ||
1236 | 158 | |||
1237 | 159 | old_listen = wsgi.listen | ||
1238 | 160 | old_sleep = wsgi.sleep | ||
1239 | 161 | old_time = wsgi.time | ||
1240 | 162 | try: | ||
1241 | 163 | wsgi.listen = mock_listen | ||
1242 | 164 | wsgi.sleep = mock_sleep | ||
1243 | 165 | wsgi.time = MockTime() | ||
1244 | 166 | # test error | ||
1245 | 167 | self.assertRaises(Exception, wsgi.get_socket, conf) | ||
1246 | 168 | # different error | ||
1247 | 169 | wsgi.listen = value_error_listen | ||
1248 | 170 | self.assertRaises(ValueError, wsgi.get_socket, conf) | ||
1249 | 171 | finally: | ||
1250 | 172 | wsgi.listen = old_listen | ||
1251 | 173 | wsgi.sleep = old_sleep | ||
1252 | 174 | wsgi.time = old_time | ||
1253 | 175 | |||
1254 | 176 | |||
1255 | 75 | if __name__ == '__main__': | 177 | if __name__ == '__main__': |
1256 | 76 | unittest.main() | 178 | unittest.main() |
1257 | 77 | 179 | ||
1258 | === modified file 'test/unit/stats/test_log_processor.py' | |||
1259 | --- test/unit/stats/test_log_processor.py 2010-11-17 15:36:21 +0000 | |||
1260 | +++ test/unit/stats/test_log_processor.py 2010-11-19 22:38:08 +0000 | |||
1261 | @@ -14,25 +14,12 @@ | |||
1262 | 14 | # limitations under the License. | 14 | # limitations under the License. |
1263 | 15 | 15 | ||
1264 | 16 | import unittest | 16 | import unittest |
1268 | 17 | import os | 17 | from test.unit import tmpfile |
1266 | 18 | from contextlib import contextmanager | ||
1267 | 19 | from tempfile import NamedTemporaryFile | ||
1269 | 20 | 18 | ||
1270 | 21 | from swift.common import internal_proxy | 19 | from swift.common import internal_proxy |
1271 | 22 | from swift.stats import log_processor | 20 | from swift.stats import log_processor |
1272 | 23 | 21 | ||
1273 | 24 | 22 | ||
1274 | 25 | @contextmanager | ||
1275 | 26 | def tmpfile(content): | ||
1276 | 27 | with NamedTemporaryFile('w', delete=False) as f: | ||
1277 | 28 | file_name = f.name | ||
1278 | 29 | f.write(str(content)) | ||
1279 | 30 | try: | ||
1280 | 31 | yield file_name | ||
1281 | 32 | finally: | ||
1282 | 33 | os.unlink(file_name) | ||
1283 | 34 | |||
1284 | 35 | |||
1285 | 36 | class FakeUploadApp(object): | 23 | class FakeUploadApp(object): |
1286 | 37 | def __init__(self, *args, **kwargs): | 24 | def __init__(self, *args, **kwargs): |
1287 | 38 | pass | 25 | pass |
This one's a wee bit hairy so another should take a peek as well.