Merge lp:~jderose/dmedia/simple-default into lp:dmedia
- simple-default
- Merge into trunk
Status: | Merged |
---|---|
Merged at revision: | 524 |
Proposed branch: | lp:~jderose/dmedia/simple-default |
Merge into: | lp:dmedia |
Diff against target: |
668 lines (+233/-250) 8 files modified
debian/dmedia.postinst (+0/-10) dmedia-cli (+3/-3) dmedia-service (+6/-14) dmedia/core.py (+66/-36) dmedia/tests/base.py (+5/-0) dmedia/tests/test_core.py (+118/-186) dmedia/tests/test_util.py (+26/-1) dmedia/util.py (+9/-0) |
To merge this branch: | bzr merge lp:~jderose/dmedia/simple-default |
Related bugs: |
Reviewer | Review Type | Date Requested | Status |
---|---|---|---|
James Raymond | Approve | ||
Review via email: mp+136073@code.launchpad.net |
Commit message
Description of the change
For background, see this bug:
https:/
Changes include:
* Removed debian/
* Reworked Core so it always has a default file store, doesn't use the shared store in /home/.dmedia anymore
* Moved the private store from ~/.dmedia to ~/.local/
* Removed SetDefaultStore() DBus method
* Added Core.purge_store() implementation and test
* Added PurgeStore() DBus method
* Added quick and dirty migration functionality for the default store changes, which:
1) If ~/.dmedia exists, it is move to ~/.local/
2) If /home/.dmedia exists, all files there are moved (or copied if needed) into ~/.local/
3) Finally, calls Core.purge_store() to remove any references to copies in /home/.dmedia
James Raymond (jamesmr) : | # |
Preview Diff
1 | === removed file 'debian/dmedia.postinst' | |||
2 | --- debian/dmedia.postinst 2012-07-26 21:05:25 +0000 | |||
3 | +++ debian/dmedia.postinst 1970-01-01 00:00:00 +0000 | |||
4 | @@ -1,10 +0,0 @@ | |||
5 | 1 | #!/bin/sh -e | ||
6 | 2 | |||
7 | 3 | case $1 in | ||
8 | 4 | configure) | ||
9 | 5 | # Initalize shared file-store in /home | ||
10 | 6 | /usr/lib/dmedia/init-filestore /home | ||
11 | 7 | esac | ||
12 | 8 | |||
13 | 9 | |||
14 | 10 | #DEBHELPER# | ||
15 | 11 | 0 | ||
16 | === modified file 'dmedia-cli' | |||
17 | --- dmedia-cli 2012-11-01 11:06:53 +0000 | |||
18 | +++ dmedia-cli 2012-11-26 00:24:26 +0000 | |||
19 | @@ -134,10 +134,10 @@ | |||
20 | 134 | return [path.abspath(directory)] | 134 | return [path.abspath(directory)] |
21 | 135 | 135 | ||
22 | 136 | 136 | ||
25 | 137 | class SetDefaultStore(_Method): | 137 | class PurgeStore(_Method): |
26 | 138 | "Set default store to 'private', 'shared', or 'none'" | 138 | 'Purge references to a store' |
27 | 139 | 139 | ||
29 | 140 | args = ['value'] | 140 | args = ['store_id'] |
30 | 141 | 141 | ||
31 | 142 | 142 | ||
32 | 143 | class Resolve(_Method): | 143 | class Resolve(_Method): |
33 | 144 | 144 | ||
34 | === modified file 'dmedia-service' | |||
35 | --- dmedia-service 2012-11-25 07:25:50 +0000 | |||
36 | +++ dmedia-service 2012-11-26 00:24:26 +0000 | |||
37 | @@ -181,11 +181,6 @@ | |||
38 | 181 | self.snapshot_queue_in.put(dbname) | 181 | self.snapshot_queue_in.put(dbname) |
39 | 182 | return True | 182 | return True |
40 | 183 | 183 | ||
41 | 184 | @dbus.service.method(IFACE, in_signature='as', out_signature='') | ||
42 | 185 | def SnapshotMany(self, names): | ||
43 | 186 | for dbname in names: | ||
44 | 187 | self.Snapshot(dbname) | ||
45 | 188 | |||
46 | 189 | @dbus.service.method(IFACE, in_signature='', out_signature='') | 184 | @dbus.service.method(IFACE, in_signature='', out_signature='') |
47 | 190 | def SnapshotAll(self): | 185 | def SnapshotAll(self): |
48 | 191 | log.info('Dmedia.SnapshotAll()') | 186 | log.info('Dmedia.SnapshotAll()') |
49 | @@ -209,9 +204,7 @@ | |||
50 | 209 | log.info('Starting CouchDB took %.3f', time.time() - start) | 204 | log.info('Starting CouchDB took %.3f', time.time() - start) |
51 | 210 | self.core = Core(env) | 205 | self.core = Core(env) |
52 | 211 | self.core.load_identity(self.couch.machine, self.couch.user) | 206 | self.core.load_identity(self.couch.machine, self.couch.user) |
56 | 212 | self.core.init_default_store() | 207 | self.core.load_default_filestore(self.couch.basedir) |
54 | 213 | if self.core.local.get('default_store') is None: | ||
55 | 214 | self.core.set_default_store('shared') | ||
57 | 215 | self.env_s = dumps(self.core.env, pretty=True) | 208 | self.env_s = dumps(self.core.env, pretty=True) |
58 | 216 | log.info('Finished core startup in %.3f', time.time() - start) | 209 | log.info('Finished core startup in %.3f', time.time() - start) |
59 | 217 | GObject.timeout_add(250, self.on_idle1) | 210 | GObject.timeout_add(250, self.on_idle1) |
60 | @@ -468,12 +461,11 @@ | |||
61 | 468 | self.core.create_filestore(parentdir) | 461 | self.core.create_filestore(parentdir) |
62 | 469 | return self.LocalDmedia() | 462 | return self.LocalDmedia() |
63 | 470 | 463 | ||
70 | 471 | @dbus.service.method(IFACE, in_signature='s', out_signature='s') | 464 | @dbus.service.method(IFACE, in_signature='s', out_signature='') |
71 | 472 | def SetDefaultStore(self, value): | 465 | def PurgeStore(self, store_id): |
72 | 473 | value = str(value) | 466 | store_id = str(store_id) |
73 | 474 | log.info('Dmedia.SetDefaultStore(%r)', value) | 467 | log.info('Dmedia.PurgeStore(%r)', store_id) |
74 | 475 | self.core.set_default_store(value) | 468 | start_thread(self.core.purge_store, store_id) |
69 | 476 | return self.LocalDmedia() | ||
75 | 477 | 469 | ||
76 | 478 | @dbus.service.method(IFACE, in_signature='s', out_signature='s') | 470 | @dbus.service.method(IFACE, in_signature='s', out_signature='s') |
77 | 479 | def Resolve(self, _id): | 471 | def Resolve(self, _id): |
78 | 480 | 472 | ||
79 | === modified file 'dmedia/core.py' | |||
80 | --- dmedia/core.py 2012-11-18 11:57:51 +0000 | |||
81 | +++ dmedia/core.py 2012-11-26 00:24:26 +0000 | |||
82 | @@ -42,7 +42,7 @@ | |||
83 | 42 | from base64 import b64encode | 42 | from base64 import b64encode |
84 | 43 | 43 | ||
85 | 44 | from microfiber import Server, Database, NotFound, Conflict, BulkConflict | 44 | from microfiber import Server, Database, NotFound, Conflict, BulkConflict |
87 | 45 | from filestore import FileStore, check_root_hash, check_id | 45 | from filestore import FileStore, check_root_hash, check_id, DOTNAME |
88 | 46 | 46 | ||
89 | 47 | import dmedia | 47 | import dmedia |
90 | 48 | from dmedia.parallel import start_thread, start_process | 48 | from dmedia.parallel import start_thread, start_process |
91 | @@ -54,8 +54,10 @@ | |||
92 | 54 | 54 | ||
93 | 55 | log = logging.getLogger() | 55 | log = logging.getLogger() |
94 | 56 | LOCAL_ID = '_local/dmedia' | 56 | LOCAL_ID = '_local/dmedia' |
95 | 57 | HOME = path.abspath(os.environ['HOME']) | ||
96 | 58 | if not path.isdir(HOME): | ||
97 | 59 | raise Exception('$HOME is not a directory: {!r}'.format(HOME)) | ||
98 | 57 | SHARED = '/home' | 60 | SHARED = '/home' |
99 | 58 | PRIVATE = path.abspath(os.environ['HOME']) | ||
100 | 59 | 61 | ||
101 | 60 | 62 | ||
102 | 61 | def start_httpd(couch_env, ssl_config): | 63 | def start_httpd(couch_env, ssl_config): |
103 | @@ -185,18 +187,37 @@ | |||
104 | 185 | log.exception('Error updating project stats for %r', project_id) | 187 | log.exception('Error updating project stats for %r', project_id) |
105 | 186 | 188 | ||
106 | 187 | 189 | ||
107 | 190 | def migrate_shared(srcdir, dstdir): | ||
108 | 191 | try: | ||
109 | 192 | count = 0 | ||
110 | 193 | src = FileStore(srcdir) | ||
111 | 194 | dst = FileStore(dstdir) | ||
112 | 195 | log.info('Migrating files from %r to %r', src, dst) | ||
113 | 196 | for st in src: | ||
114 | 197 | if dst.exists(st.id): | ||
115 | 198 | continue | ||
116 | 199 | log.info('Migrating %s %s', st.id, st.size) | ||
117 | 200 | try: | ||
118 | 201 | os.rename(st.name, dst.path(st.id)) | ||
119 | 202 | except OSError: | ||
120 | 203 | src.copy(st.id, dst) | ||
121 | 204 | src.remove(st.id) | ||
122 | 205 | count += 1 | ||
123 | 206 | log.info('Migrating %d files from %r to %r', count, src, dst) | ||
124 | 207 | except Exception: | ||
125 | 208 | log.exception('Error migrating files from shared FileStore') | ||
126 | 209 | return count | ||
127 | 210 | |||
128 | 211 | |||
129 | 188 | class Core: | 212 | class Core: |
131 | 189 | def __init__(self, env, private=None, shared=None): | 213 | def __init__(self, env): |
132 | 190 | self.env = env | 214 | self.env = env |
133 | 191 | self._private = (PRIVATE if private is None else private) | ||
134 | 192 | self._shared = (SHARED if shared is None else shared) | ||
135 | 193 | self.db = util.get_db(env, init=True) | 215 | self.db = util.get_db(env, init=True) |
136 | 194 | self.server = self.db.server() | 216 | self.server = self.db.server() |
137 | 195 | self.ms = MetaStore(self.db) | 217 | self.ms = MetaStore(self.db) |
138 | 196 | self.stores = LocalStores() | 218 | self.stores = LocalStores() |
139 | 197 | self.queue = Queue() | 219 | self.queue = Queue() |
140 | 198 | self.thread = None | 220 | self.thread = None |
141 | 199 | self.default = None | ||
142 | 200 | try: | 221 | try: |
143 | 201 | self.local = self.db.get(LOCAL_ID) | 222 | self.local = self.db.get(LOCAL_ID) |
144 | 202 | except NotFound: | 223 | except NotFound: |
145 | @@ -230,23 +251,18 @@ | |||
146 | 230 | self.local['user_id'] = user['_id'] | 251 | self.local['user_id'] = user['_id'] |
147 | 231 | self.save_local() | 252 | self.save_local() |
148 | 232 | 253 | ||
162 | 233 | def init_default_store(self): | 254 | def load_default_filestore(self, parentdir): |
163 | 234 | value = self.local.get('default_store') | 255 | if util.isfilestore(HOME) and not util.isfilestore(parentdir): |
164 | 235 | if value not in ('private', 'shared'): | 256 | src = FileStore(HOME) |
165 | 236 | log.info('no default FileStore') | 257 | src.init_dirs() |
166 | 237 | self.default = None | 258 | dstdir = path.join(parentdir, DOTNAME) |
167 | 238 | self._sync_stores() | 259 | log.info('Moving %r to %r', src.basedir, dstdir) |
168 | 239 | return | 260 | os.rename(src.basedir, dstdir) |
169 | 240 | if value == 'shared' and not util.isfilestore(self._shared): | 261 | self.parentdir = parentdir |
157 | 241 | log.warning('Switching to private, no shared FileStore at %r', self._shared) | ||
158 | 242 | value = 'private' | ||
159 | 243 | self.local['default_store'] = value | ||
160 | 244 | self.db.save(self.local) | ||
161 | 245 | parentdir = (self._private if value == 'private' else self._shared) | ||
170 | 246 | (fs, doc) = util.init_filestore(parentdir) | 262 | (fs, doc) = util.init_filestore(parentdir) |
173 | 247 | self.default = fs | 263 | log.info('Default FileStore %s at %r', doc['_id'], parentdir) |
172 | 248 | log.info('Connecting default FileStore %r at %r', fs.id, fs.parentdir) | ||
174 | 249 | self._add_filestore(fs, doc) | 264 | self._add_filestore(fs, doc) |
175 | 265 | return fs | ||
176 | 250 | 266 | ||
177 | 251 | def _sync_stores(self): | 267 | def _sync_stores(self): |
178 | 252 | self.local['stores'] = self.stores.local_stores() | 268 | self.local['stores'] = self.stores.local_stores() |
179 | @@ -270,6 +286,13 @@ | |||
180 | 270 | self._sync_stores() | 286 | self._sync_stores() |
181 | 271 | 287 | ||
182 | 272 | def _background_worker(self): | 288 | def _background_worker(self): |
183 | 289 | if util.isfilestore(SHARED): | ||
184 | 290 | log.info('Running migration...') | ||
185 | 291 | process = start_process(migrate_shared, SHARED, self.parentdir) | ||
186 | 292 | process.join() | ||
187 | 293 | store_id = util.get_filestore_id(SHARED) | ||
188 | 294 | if store_id is not None: | ||
189 | 295 | self.purge_store(store_id) | ||
190 | 273 | log.info('Background worker listing to queue...') | 296 | log.info('Background worker listing to queue...') |
191 | 274 | while True: | 297 | while True: |
192 | 275 | try: | 298 | try: |
193 | @@ -327,19 +350,6 @@ | |||
194 | 327 | def update_project(self, project_id): | 350 | def update_project(self, project_id): |
195 | 328 | update_project(self.db, project_id) | 351 | update_project(self.db, project_id) |
196 | 329 | 352 | ||
197 | 330 | def set_default_store(self, value): | ||
198 | 331 | if value not in ('private', 'shared', 'none'): | ||
199 | 332 | raise ValueError( | ||
200 | 333 | "need 'private', 'shared', or 'none'; got {!r}".format(value) | ||
201 | 334 | ) | ||
202 | 335 | if self.local.get('default_store') != value: | ||
203 | 336 | self.local['default_store'] = value | ||
204 | 337 | self.db.save(self.local) | ||
205 | 338 | if self.default is not None: | ||
206 | 339 | self.disconnect_filestore(self.default.parentdir, self.default.id) | ||
207 | 340 | self.default = None | ||
208 | 341 | self.init_default_store() | ||
209 | 342 | |||
210 | 343 | def create_filestore(self, parentdir): | 353 | def create_filestore(self, parentdir): |
211 | 344 | """ | 354 | """ |
212 | 345 | Create a new file-store in *parentdir*. | 355 | Create a new file-store in *parentdir*. |
213 | @@ -412,16 +422,36 @@ | |||
214 | 412 | * ``doc['partial'][store_id]`` | 422 | * ``doc['partial'][store_id]`` |
215 | 413 | 423 | ||
216 | 414 | Some scenarios in which you might want to do this: | 424 | Some scenarios in which you might want to do this: |
218 | 415 | 425 | ||
219 | 416 | 1. The HDD was run over by a bus, the data is gone. We need to | 426 | 1. The HDD was run over by a bus, the data is gone. We need to |
220 | 417 | embrace reality, the sooner the better. | 427 | embrace reality, the sooner the better. |
221 | 418 | 428 | ||
222 | 419 | 2. We're going to format or otherwise repurpose an HDD. Ideally, we | 429 | 2. We're going to format or otherwise repurpose an HDD. Ideally, we |
223 | 420 | would have called `Core2.downgrade_store()` first. | 430 | would have called `Core2.downgrade_store()` first. |
225 | 421 | 431 | ||
226 | 422 | Note that this method makes sense for remote cloud stores as well as for | 432 | Note that this method makes sense for remote cloud stores as well as for |
227 | 423 | local file-stores | 433 | local file-stores |
228 | 424 | """ | 434 | """ |
229 | 435 | log.info('Purging store %s', store_id) | ||
230 | 436 | ids = [] | ||
231 | 437 | while True: | ||
232 | 438 | rows = self.db.view('file', 'stored', | ||
233 | 439 | key=store_id, | ||
234 | 440 | include_docs=True, | ||
235 | 441 | limit=25, | ||
236 | 442 | )['rows'] | ||
237 | 443 | if not rows: | ||
238 | 444 | break | ||
239 | 445 | ids.extend(r['id'] for r in rows) | ||
240 | 446 | docs = [r['doc'] for r in rows] | ||
241 | 447 | for doc in docs: | ||
242 | 448 | del doc['stored'][store_id] | ||
243 | 449 | try: | ||
244 | 450 | self.db.save_many(docs) | ||
245 | 451 | except BulkConflict: | ||
246 | 452 | log.exception('Conflict purging %s', store_id) | ||
247 | 453 | log.info('Purged %d references to %s', len(ids), store_id) | ||
248 | 454 | return ids | ||
249 | 425 | 455 | ||
250 | 426 | def stat(self, _id): | 456 | def stat(self, _id): |
251 | 427 | doc = self.db.get(_id) | 457 | doc = self.db.get(_id) |
252 | 428 | 458 | ||
253 | === modified file 'dmedia/tests/base.py' | |||
254 | --- dmedia/tests/base.py 2012-05-06 22:10:00 +0000 | |||
255 | +++ dmedia/tests/base.py 2012-11-26 00:24:26 +0000 | |||
256 | @@ -33,6 +33,7 @@ | |||
257 | 33 | from random import SystemRandom | 33 | from random import SystemRandom |
258 | 34 | from zipfile import ZipFile | 34 | from zipfile import ZipFile |
259 | 35 | 35 | ||
260 | 36 | import filestore | ||
261 | 36 | from filestore import File, Leaf, ContentHash, Batch, Hasher, LEAF_SIZE | 37 | from filestore import File, Leaf, ContentHash, Batch, Hasher, LEAF_SIZE |
262 | 37 | from filestore import scandir | 38 | from filestore import scandir |
263 | 38 | from microfiber import random_id | 39 | from microfiber import random_id |
264 | @@ -42,6 +43,10 @@ | |||
265 | 42 | random = SystemRandom() | 43 | random = SystemRandom() |
266 | 43 | 44 | ||
267 | 44 | 45 | ||
268 | 46 | def random_file_id(): | ||
269 | 47 | return random_id(filestore.DIGEST_BYTES) | ||
270 | 48 | |||
271 | 49 | |||
272 | 45 | class DummyQueue(object): | 50 | class DummyQueue(object): |
273 | 46 | def __init__(self): | 51 | def __init__(self): |
274 | 47 | self.items = [] | 52 | self.items = [] |
275 | 48 | 53 | ||
276 | === modified file 'dmedia/tests/test_core.py' | |||
277 | --- dmedia/tests/test_core.py 2012-10-31 20:42:27 +0000 | |||
278 | +++ dmedia/tests/test_core.py 2012-11-26 00:24:26 +0000 | |||
279 | @@ -41,7 +41,7 @@ | |||
280 | 41 | from dmedia import util, core | 41 | from dmedia import util, core |
281 | 42 | 42 | ||
282 | 43 | from .couch import CouchCase | 43 | from .couch import CouchCase |
284 | 44 | from .base import TempDir | 44 | from .base import TempDir, random_file_id |
285 | 45 | 45 | ||
286 | 46 | 46 | ||
287 | 47 | class TestFunctions(TestCase): | 47 | class TestFunctions(TestCase): |
288 | @@ -64,6 +64,32 @@ | |||
289 | 64 | } | 64 | } |
290 | 65 | ) | 65 | ) |
291 | 66 | 66 | ||
292 | 67 | def test_migrate_shared(self): | ||
293 | 68 | tmp1 = TempDir() | ||
294 | 69 | tmp2 = TempDir() | ||
295 | 70 | src = filestore.FileStore(tmp1.dir) | ||
296 | 71 | dst = filestore.FileStore(tmp2.dir) | ||
297 | 72 | st_list = [] | ||
298 | 73 | for i in range(10): | ||
299 | 74 | (file, ch) = tmp1.random_file() | ||
300 | 75 | os.rename(file.name, src.path(ch.id)) | ||
301 | 76 | st = src.stat(ch.id) | ||
302 | 77 | assert st.size == ch.file_size | ||
303 | 78 | st_list.append(st) | ||
304 | 79 | st_list.sort(key=lambda st: st.id) | ||
305 | 80 | self.assertEqual(list(src), st_list) | ||
306 | 81 | self.assertEqual(list(dst), []) | ||
307 | 82 | self.assertEqual(core.migrate_shared(tmp1.dir, tmp2.dir), 10) | ||
308 | 83 | self.assertEqual(list(src), []) | ||
309 | 84 | self.assertEqual( | ||
310 | 85 | [st.id for st in dst], | ||
311 | 86 | [st.id for st in st_list] | ||
312 | 87 | ) | ||
313 | 88 | for st in st_list: | ||
314 | 89 | ch = dst.verify(st.id) | ||
315 | 90 | self.assertEqual(st.size, ch.file_size) | ||
316 | 91 | self.assertEqual(dst.stat(st.id).mtime, st.mtime) | ||
317 | 92 | |||
318 | 67 | 93 | ||
319 | 68 | class TestCouchFunctions(CouchCase): | 94 | class TestCouchFunctions(CouchCase): |
320 | 69 | def test_projects_iter(self): | 95 | def test_projects_iter(self): |
321 | @@ -135,191 +161,6 @@ | |||
322 | 135 | } | 161 | } |
323 | 136 | ) | 162 | ) |
324 | 137 | 163 | ||
325 | 138 | def test_init_default_store(self): | ||
326 | 139 | private = TempDir() | ||
327 | 140 | shared = TempDir() | ||
328 | 141 | machine_id = random_id() | ||
329 | 142 | |||
330 | 143 | # Test when default_store is missing | ||
331 | 144 | inst = core.Core(self.env, private.dir, shared.dir) | ||
332 | 145 | self.assertEqual(inst._private, private.dir) | ||
333 | 146 | self.assertEqual(inst._shared, shared.dir) | ||
334 | 147 | inst.init_default_store() | ||
335 | 148 | self.assertIsNone(inst.default) | ||
336 | 149 | self.assertEqual(inst.local, | ||
337 | 150 | { | ||
338 | 151 | '_id': '_local/dmedia', | ||
339 | 152 | 'stores': {}, | ||
340 | 153 | } | ||
341 | 154 | ) | ||
342 | 155 | |||
343 | 156 | # Test when default_store is 'private' | ||
344 | 157 | inst = core.Core(self.env, private.dir, shared.dir) | ||
345 | 158 | self.assertEqual(inst._private, private.dir) | ||
346 | 159 | self.assertEqual(inst._shared, shared.dir) | ||
347 | 160 | inst.local['default_store'] = 'private' | ||
348 | 161 | self.assertFalse(util.isfilestore(private.dir)) | ||
349 | 162 | inst.init_default_store() | ||
350 | 163 | self.assertEqual( | ||
351 | 164 | set(inst.local['stores']), | ||
352 | 165 | set([private.dir]) | ||
353 | 166 | ) | ||
354 | 167 | store_id = inst.local['stores'][private.dir]['id'] | ||
355 | 168 | (fs1, doc) = util.get_filestore(private.dir, store_id) | ||
356 | 169 | self.assertEqual(inst.local, | ||
357 | 170 | { | ||
358 | 171 | '_id': '_local/dmedia', | ||
359 | 172 | '_rev': '0-1', | ||
360 | 173 | 'default_store': 'private', | ||
361 | 174 | 'stores': { | ||
362 | 175 | fs1.parentdir: { | ||
363 | 176 | 'id': fs1.id, | ||
364 | 177 | 'copies': fs1.copies, | ||
365 | 178 | } | ||
366 | 179 | } | ||
367 | 180 | } | ||
368 | 181 | ) | ||
369 | 182 | |||
370 | 183 | # Again test when default_store is 'private' to make sure local isn't | ||
371 | 184 | # updated needlessly | ||
372 | 185 | inst = core.Core(self.env, private.dir, shared.dir) | ||
373 | 186 | self.assertEqual(inst._private, private.dir) | ||
374 | 187 | self.assertEqual(inst._shared, shared.dir) | ||
375 | 188 | inst.init_default_store() | ||
376 | 189 | self.assertEqual(inst.local, | ||
377 | 190 | { | ||
378 | 191 | '_id': '_local/dmedia', | ||
379 | 192 | '_rev': '0-1', | ||
380 | 193 | 'default_store': 'private', | ||
381 | 194 | 'stores': { | ||
382 | 195 | fs1.parentdir: { | ||
383 | 196 | 'id': fs1.id, | ||
384 | 197 | 'copies': fs1.copies, | ||
385 | 198 | } | ||
386 | 199 | } | ||
387 | 200 | } | ||
388 | 201 | ) | ||
389 | 202 | |||
390 | 203 | # Test when default_store is 'shared' (which we're assuming exists) | ||
391 | 204 | self.assertFalse(util.isfilestore(shared.dir)) | ||
392 | 205 | (fs2, doc) = util.init_filestore(shared.dir) | ||
393 | 206 | inst = core.Core(self.env, private.dir, shared.dir) | ||
394 | 207 | self.assertEqual(inst._private, private.dir) | ||
395 | 208 | self.assertEqual(inst._shared, shared.dir) | ||
396 | 209 | inst.local['default_store'] = 'shared' | ||
397 | 210 | inst.init_default_store() | ||
398 | 211 | self.assertEqual(inst.local, | ||
399 | 212 | { | ||
400 | 213 | '_id': '_local/dmedia', | ||
401 | 214 | '_rev': '0-2', | ||
402 | 215 | 'default_store': 'shared', | ||
403 | 216 | 'stores': { | ||
404 | 217 | fs2.parentdir: { | ||
405 | 218 | 'id': fs2.id, | ||
406 | 219 | 'copies': fs2.copies, | ||
407 | 220 | } | ||
408 | 221 | } | ||
409 | 222 | } | ||
410 | 223 | ) | ||
411 | 224 | |||
412 | 225 | # Test when default_store is 'shared' and it doesn't exist, in which | ||
413 | 226 | # case it should automatically switch to 'private' instead | ||
414 | 227 | nope = TempDir() | ||
415 | 228 | inst = core.Core(self.env, private.dir, nope.dir) | ||
416 | 229 | self.assertEqual(inst._private, private.dir) | ||
417 | 230 | self.assertEqual(inst._shared, nope.dir) | ||
418 | 231 | inst.local['default_store'] = 'shared' | ||
419 | 232 | inst.init_default_store() | ||
420 | 233 | self.assertEqual(inst.local, | ||
421 | 234 | { | ||
422 | 235 | '_id': '_local/dmedia', | ||
423 | 236 | '_rev': '0-4', | ||
424 | 237 | 'default_store': 'private', | ||
425 | 238 | 'stores': { | ||
426 | 239 | fs1.parentdir: { | ||
427 | 240 | 'id': fs1.id, | ||
428 | 241 | 'copies': fs1.copies, | ||
429 | 242 | } | ||
430 | 243 | } | ||
431 | 244 | } | ||
432 | 245 | ) | ||
433 | 246 | |||
434 | 247 | def test_set_default_store(self): | ||
435 | 248 | private = TempDir() | ||
436 | 249 | shared = TempDir() | ||
437 | 250 | (fs1, doc1) = util.init_filestore(private.dir) | ||
438 | 251 | (fs2, doc2) = util.init_filestore(shared.dir) | ||
439 | 252 | inst = core.Core(self.env, private.dir, shared.dir) | ||
440 | 253 | self.assertEqual(inst._private, private.dir) | ||
441 | 254 | self.assertEqual(inst._shared, shared.dir) | ||
442 | 255 | |||
443 | 256 | with self.assertRaises(ValueError) as cm: | ||
444 | 257 | inst.set_default_store('foobar') | ||
445 | 258 | self.assertEqual( | ||
446 | 259 | str(cm.exception), | ||
447 | 260 | "need 'private', 'shared', or 'none'; got 'foobar'" | ||
448 | 261 | ) | ||
449 | 262 | self.assertEqual(inst.local, | ||
450 | 263 | { | ||
451 | 264 | '_id': '_local/dmedia', | ||
452 | 265 | 'stores': {}, | ||
453 | 266 | } | ||
454 | 267 | ) | ||
455 | 268 | |||
456 | 269 | # Test with 'private' | ||
457 | 270 | inst.set_default_store('private') | ||
458 | 271 | self.assertEqual(inst.local, | ||
459 | 272 | { | ||
460 | 273 | '_id': '_local/dmedia', | ||
461 | 274 | '_rev': '0-2', | ||
462 | 275 | 'default_store': 'private', | ||
463 | 276 | 'stores': { | ||
464 | 277 | fs1.parentdir: {'id': fs1.id, 'copies': 1}, | ||
465 | 278 | }, | ||
466 | 279 | } | ||
467 | 280 | ) | ||
468 | 281 | |||
469 | 282 | # Test with 'shared' | ||
470 | 283 | inst.set_default_store('shared') | ||
471 | 284 | self.assertEqual(inst.local, | ||
472 | 285 | { | ||
473 | 286 | '_id': '_local/dmedia', | ||
474 | 287 | '_rev': '0-5', | ||
475 | 288 | 'default_store': 'shared', | ||
476 | 289 | 'stores': { | ||
477 | 290 | fs2.parentdir: {'id': fs2.id, 'copies': 1}, | ||
478 | 291 | }, | ||
479 | 292 | } | ||
480 | 293 | ) | ||
481 | 294 | |||
482 | 295 | # Test with 'none' | ||
483 | 296 | inst.set_default_store('none') | ||
484 | 297 | self.assertEqual(inst.local, | ||
485 | 298 | { | ||
486 | 299 | '_id': '_local/dmedia', | ||
487 | 300 | '_rev': '0-7', | ||
488 | 301 | 'default_store': 'none', | ||
489 | 302 | 'stores': {}, | ||
490 | 303 | } | ||
491 | 304 | ) | ||
492 | 305 | |||
493 | 306 | # Test with 'shared' when it doesn't exist | ||
494 | 307 | nope = TempDir() | ||
495 | 308 | inst = core.Core(self.env, private.dir, nope.dir) | ||
496 | 309 | self.assertEqual(inst._private, private.dir) | ||
497 | 310 | self.assertEqual(inst._shared, nope.dir) | ||
498 | 311 | inst.set_default_store('shared') | ||
499 | 312 | self.assertEqual(inst.local, | ||
500 | 313 | { | ||
501 | 314 | '_id': '_local/dmedia', | ||
502 | 315 | '_rev': '0-10', | ||
503 | 316 | 'default_store': 'private', | ||
504 | 317 | 'stores': { | ||
505 | 318 | fs1.parentdir: {'id': fs1.id, 'copies': 1}, | ||
506 | 319 | }, | ||
507 | 320 | } | ||
508 | 321 | ) | ||
509 | 322 | |||
510 | 323 | def test_create_filestore(self): | 164 | def test_create_filestore(self): |
511 | 324 | inst = core.Core(self.env) | 165 | inst = core.Core(self.env) |
512 | 325 | 166 | ||
513 | @@ -502,6 +343,97 @@ | |||
514 | 502 | inst.disconnect_filestore(fs1.parentdir, fs1.id) | 343 | inst.disconnect_filestore(fs1.parentdir, fs1.id) |
515 | 503 | self.assertEqual(str(cm.exception), repr(fs1.parentdir)) | 344 | self.assertEqual(str(cm.exception), repr(fs1.parentdir)) |
516 | 504 | 345 | ||
517 | 346 | def test_purge_store(self): | ||
518 | 347 | store_id1 = random_id() | ||
519 | 348 | store_id2 = random_id() | ||
520 | 349 | store_id3 = random_id() | ||
521 | 350 | inst = core.Core(self.env) | ||
522 | 351 | db = inst.db | ||
523 | 352 | |||
524 | 353 | # Test when empty | ||
525 | 354 | self.assertEqual(inst.purge_store(store_id1), []) | ||
526 | 355 | |||
527 | 356 | docs = [ | ||
528 | 357 | { | ||
529 | 358 | '_id': random_file_id(), | ||
530 | 359 | 'type': 'dmedia/file', | ||
531 | 360 | 'bytes': 1776, | ||
532 | 361 | 'stored': { | ||
533 | 362 | store_id1: { | ||
534 | 363 | 'copies': 1, | ||
535 | 364 | 'mtime': 1234567890, | ||
536 | 365 | }, | ||
537 | 366 | store_id2: { | ||
538 | 367 | 'copies': 2, | ||
539 | 368 | 'mtime': 1234567891, | ||
540 | 369 | }, | ||
541 | 370 | }, | ||
542 | 371 | } | ||
543 | 372 | for i in range(533) | ||
544 | 373 | ] | ||
545 | 374 | ids = [doc['_id'] for doc in docs] | ||
546 | 375 | ids.sort() | ||
547 | 376 | db.save_many(docs) | ||
548 | 377 | |||
549 | 378 | # Test when store isn't present | ||
550 | 379 | self.assertEqual(inst.purge_store(store_id3), []) | ||
551 | 380 | for doc in docs: | ||
552 | 381 | self.assertEqual(db.get(doc['_id']), doc) | ||
553 | 382 | |||
554 | 383 | # Purge one of the stores, make sure the other remains | ||
555 | 384 | self.assertEqual(inst.purge_store(store_id1), ids) | ||
556 | 385 | for doc in db.get_many(ids): | ||
557 | 386 | _id = doc['_id'] | ||
558 | 387 | rev = doc.pop('_rev') | ||
559 | 388 | self.assertTrue(rev.startswith('2-')) | ||
560 | 389 | self.assertEqual( | ||
561 | 390 | doc, | ||
562 | 391 | { | ||
563 | 392 | '_id': _id, | ||
564 | 393 | 'type': 'dmedia/file', | ||
565 | 394 | 'bytes': 1776, | ||
566 | 395 | 'stored': { | ||
567 | 396 | store_id2: { | ||
568 | 397 | 'copies': 2, | ||
569 | 398 | 'mtime': 1234567891, | ||
570 | 399 | }, | ||
571 | 400 | }, | ||
572 | 401 | } | ||
573 | 402 | ) | ||
574 | 403 | |||
575 | 404 | # Purge the other store | ||
576 | 405 | self.assertEqual(inst.purge_store(store_id2), ids) | ||
577 | 406 | for doc in db.get_many(ids): | ||
578 | 407 | _id = doc['_id'] | ||
579 | 408 | rev = doc.pop('_rev') | ||
580 | 409 | self.assertTrue(rev.startswith('3-')) | ||
581 | 410 | self.assertEqual( | ||
582 | 411 | doc, | ||
583 | 412 | { | ||
584 | 413 | '_id': _id, | ||
585 | 414 | 'type': 'dmedia/file', | ||
586 | 415 | 'bytes': 1776, | ||
587 | 416 | 'stored': {}, | ||
588 | 417 | } | ||
589 | 418 | ) | ||
590 | 419 | |||
591 | 420 | # Purge both again, make sure no doc changes result: | ||
592 | 421 | self.assertEqual(inst.purge_store(store_id1), []) | ||
593 | 422 | self.assertEqual(inst.purge_store(store_id2), []) | ||
594 | 423 | for doc in db.get_many(ids): | ||
595 | 424 | _id = doc['_id'] | ||
596 | 425 | rev = doc.pop('_rev') | ||
597 | 426 | self.assertTrue(rev.startswith('3-')) | ||
598 | 427 | self.assertEqual( | ||
599 | 428 | doc, | ||
600 | 429 | { | ||
601 | 430 | '_id': _id, | ||
602 | 431 | 'type': 'dmedia/file', | ||
603 | 432 | 'bytes': 1776, | ||
604 | 433 | 'stored': {}, | ||
605 | 434 | } | ||
606 | 435 | ) | ||
607 | 436 | |||
608 | 505 | def test_update_atime(self): | 437 | def test_update_atime(self): |
609 | 506 | inst = core.Core(self.env) | 438 | inst = core.Core(self.env) |
610 | 507 | _id = random_id() | 439 | _id = random_id() |
611 | 508 | 440 | ||
612 | === modified file 'dmedia/tests/test_util.py' | |||
613 | --- dmedia/tests/test_util.py 2012-07-09 14:45:38 +0000 | |||
614 | +++ dmedia/tests/test_util.py 2012-11-26 00:24:26 +0000 | |||
615 | @@ -51,7 +51,32 @@ | |||
616 | 51 | tmp.makedirs('.dmedia') | 51 | tmp.makedirs('.dmedia') |
617 | 52 | self.assertTrue(util.isfilestore(tmp.dir)) | 52 | self.assertTrue(util.isfilestore(tmp.dir)) |
618 | 53 | 53 | ||
620 | 54 | def test_getfilestore(self): | 54 | def test_get_filestore_id(self): |
621 | 55 | tmp = TempDir() | ||
622 | 56 | doc = schema.create_filestore(1) | ||
623 | 57 | _id = doc['_id'] | ||
624 | 58 | |||
625 | 59 | # Test when .dmedia/ doesn't exist | ||
626 | 60 | self.assertIsNone(util.get_filestore_id(tmp.dir)) | ||
627 | 61 | |||
628 | 62 | # Test when .dmedia/ exists, but store.json doesn't: | ||
629 | 63 | tmp.makedirs('.dmedia') | ||
630 | 64 | self.assertIsNone(util.get_filestore_id(tmp.dir)) | ||
631 | 65 | |||
632 | 66 | # Test when .dmedia/store.json exists | ||
633 | 67 | store = tmp.join('.dmedia', 'store.json') | ||
634 | 68 | json.dump(doc, open(store, 'w')) | ||
635 | 69 | self.assertEqual(util.get_filestore_id(tmp.dir), _id) | ||
636 | 70 | |||
637 | 71 | # Test with bad JSON | ||
638 | 72 | open(store, 'wb').write(b'bad JSON, no cookie for you') | ||
639 | 73 | self.assertIsNone(util.get_filestore_id(tmp.dir)) | ||
640 | 74 | |||
641 | 75 | # Test with non-dict | ||
642 | 76 | json.dump('hello', open(store, 'w')) | ||
643 | 77 | self.assertIsNone(util.get_filestore_id(tmp.dir)) | ||
644 | 78 | |||
645 | 79 | def test_get_filestore(self): | ||
646 | 55 | tmp = TempDir() | 80 | tmp = TempDir() |
647 | 56 | doc = schema.create_filestore(1) | 81 | doc = schema.create_filestore(1) |
648 | 57 | 82 | ||
649 | 58 | 83 | ||
650 | === modified file 'dmedia/util.py' | |||
651 | --- dmedia/util.py 2012-10-19 00:11:46 +0000 | |||
652 | +++ dmedia/util.py 2012-11-26 00:24:26 +0000 | |||
653 | @@ -43,6 +43,15 @@ | |||
654 | 43 | return path.isdir(path.join(parentdir, DOTNAME)) | 43 | return path.isdir(path.join(parentdir, DOTNAME)) |
655 | 44 | 44 | ||
656 | 45 | 45 | ||
657 | 46 | def get_filestore_id(parentdir): | ||
658 | 47 | store = path.join(parentdir, DOTNAME, 'store.json') | ||
659 | 48 | try: | ||
660 | 49 | doc = json.load(open(store, 'r')) | ||
661 | 50 | return doc['_id'] | ||
662 | 51 | except Exception: | ||
663 | 52 | pass | ||
664 | 53 | |||
665 | 54 | |||
666 | 46 | def get_filestore(parentdir, store_id, copies=None): | 55 | def get_filestore(parentdir, store_id, copies=None): |
667 | 47 | store = path.join(parentdir, DOTNAME, 'store.json') | 56 | store = path.join(parentdir, DOTNAME, 'store.json') |
668 | 48 | doc = json.load(open(store, 'r')) | 57 | doc = json.load(open(store, 'r')) |