Merge lp:~facundo/magicicada-server/update-client-1440 into lp:magicicada-server

Proposed by Facundo Batista on 2018-03-20
Status: Merged
Approved by: Natalia Bidart on 2018-04-03
Approved revision: 84
Merged at revision: 84
Proposed branch: lp:~facundo/magicicada-server/update-client-1440
Merge into: lp:magicicada-server
Diff against target: 454 lines (+69/-106)
8 files modified
Makefile.db (+3/-3)
config-manager.txt (+1/-1)
magicicada/server/integtests/test_action_queue.py (+0/-47)
magicicada/server/integtests/test_sync.py (+48/-43)
magicicada/server/server.py (+1/-1)
magicicada/server/testing/aq_helpers.py (+13/-9)
magicicada/server/tests/test_shutdown.py (+1/-1)
magicicada/settings/__init__.py (+2/-1)
To merge this branch: bzr merge lp:~facundo/magicicada-server/update-client-1440
Reviewer Review Type Date Requested Status
Natalia Bidart 2018-03-20 Approve on 2018-04-03
Review via email: mp+341765@code.launchpad.net

Commit message

Updated magicicada client to r1440.

Description of the change

Updated magicicada client to r1440.

Had to remove everything related to DNSs rotation for server selection, and update the way the connection info is passed to the client.

Not related to new client, I also needed to change a little something about Django config: send the complain about "no importable local config" to stderr, so it can be avoided when trying to print settings in Makefile.db

Included several aesthetic changes as I need to do those for `make test` to run properly... in the next branch I'll pin the flake8 revision in the venv so no more of these "spurious fixes" bother us.

To post a comment you must log in.
Natalia Bidart (nataliabidart) wrote :

Looks good!

review: Approve

Preview Diff

[H/L] Next/Prev Comment, [J/K] Next/Prev File, [N/P] Next/Prev Hunk
1=== modified file 'Makefile.db'
2--- Makefile.db 2016-03-08 00:32:28 +0000
3+++ Makefile.db 2018-03-20 19:58:45 +0000
4@@ -1,7 +1,7 @@
5 SHELL_ENV = DJANGO_SETTINGS_MODULE=$(DJANGO_SETTINGS_MODULE) PYTHONPATH=$(PYTHONPATH)
6-PGNAME = $(shell $(SHELL_ENV) $(PYTHON) -c "from django.conf import settings; print settings.DATABASES['default']['NAME']")
7-PGHOST = $(shell $(SHELL_ENV) $(PYTHON) -c "from django.conf import settings; print settings.DATABASES['default']['HOST']")
8-PGUSER = $(shell $(SHELL_ENV) $(PYTHON) -c "from django.conf import settings; print settings.DATABASES['default']['USER']")
9+PGNAME = $(shell $(SHELL_ENV) $(PYTHON) -c "from django.conf import settings; print settings.DATABASES['default']['NAME']" 2> /dev/null)
10+PGHOST = $(shell $(SHELL_ENV) $(PYTHON) -c "from django.conf import settings; print settings.DATABASES['default']['HOST']" 2> /dev/null)
11+PGUSER = $(shell $(SHELL_ENV) $(PYTHON) -c "from django.conf import settings; print settings.DATABASES['default']['USER']" 2> /dev/null)
12 DATA_DIR = $(PGHOST)/data
13 LOG_FILE = $(PGHOST)/postgresql.log
14 CONF_FILE = $(PGHOST)/postgresql.conf
15
16=== modified file 'config-manager.txt'
17--- config-manager.txt 2017-04-22 21:07:36 +0000
18+++ config-manager.txt 2018-03-20 19:58:45 +0000
19@@ -25,5 +25,5 @@
20 ./.sourcecode/dirspec ~ubuntuone-control-tower/dirspec/trunk;revno=14
21 ./.sourcecode/django ~ubuntuone-pqm-team/django/stable;revno=17
22 ./.sourcecode/u1sync ~facundo/u1sync/opensourcing;revno=10
23-./.sourcecode/magicicada-client ~chicharreros/magicicada-client/trunk;revno=1433
24+./.sourcecode/magicicada-client ~chicharreros/magicicada-client/trunk;revno=1440
25 ./.sourcecode/magicicada-protocol ~chicharreros/magicicada-protocol/trunk;revno=169
26
27=== modified file 'magicicada/server/integtests/test_action_queue.py'
28--- magicicada/server/integtests/test_action_queue.py 2016-06-07 19:06:06 +0000
29+++ magicicada/server/integtests/test_action_queue.py 2018-03-20 19:58:45 +0000
30@@ -35,13 +35,11 @@
31 ActionQueue,
32 Download,
33 Upload,
34- dns_client,
35 )
36 from ubuntuone.syncdaemon.marker import MDMarker as Marker
37
38 from magicicada.server.testing.aq_helpers import (
39 FakeFailure,
40- FakeResolver,
41 NO_CONTENT_HASH,
42 NoCloseStringIO,
43 TestBase,
44@@ -958,48 +956,3 @@
45 d.addBoth(lambda _: self.main.wait_for_nirvana())
46 d.addBoth(lambda _: self.eq.push('SYS_USER_DISCONNECT'))
47 return d
48-
49-
50-class SRVLookupTest(TestWithDatabase):
51- """ Test for SRV lookup in the ActionQueue. """
52-
53- @defer.inlineCallbacks
54- def setUp(self):
55- """
56- Replace the resolver with a FakeResolver
57- """
58- yield super(SRVLookupTest, self).setUp()
59- dns_client.theResolver = FakeResolver()
60-
61- @defer.inlineCallbacks
62- def tearDown(self):
63- """
64- By setting the resolver to None, it will be recreated next time a name
65- lookup is done.
66- """
67- yield super(SRVLookupTest, self).tearDown()
68- dns_client.theResolver = None
69-
70- def test_SRV_lookup_dev(self):
71- """Test the srv lookup in development mode (localhost:<randomport>)."""
72-
73- def checkResult(result):
74- """ Verify that we are correclty doing the lookup """
75- host, port = result
76- self.assertEqual(host, self.aq.host)
77-
78- d = self.aq._lookup_srv()
79- d.addCallback(checkResult)
80- return d
81-
82- def test_SRV_lookup_prod(self):
83- """ test the srv lookup using a fake resolver. """
84- def checkResult(result):
85- """ Verify that we are correclty doing the lookup """
86- host, port = result
87- self.assertTrue(host in ['fs-1.server.com', 'fs-0.server.com'],
88- host)
89- self.assertEqual(port, 443)
90- self.aq.dns_srv = '_http._tcp.fs.server.com'
91- d = self.aq._lookup_srv()
92- d.addCallback(checkResult)
93
94=== modified file 'magicicada/server/integtests/test_sync.py'
95--- magicicada/server/integtests/test_sync.py 2016-11-07 01:37:46 +0000
96+++ magicicada/server/integtests/test_sync.py 2018-03-20 19:58:45 +0000
97@@ -113,9 +113,14 @@
98 self.share_source_dir = self.mktemp("source/share")
99
100 self.patch(hash_queue, "HASHQUEUE_DELAY", 0.1)
101+ connection_info = [{
102+ 'host': "localhost",
103+ 'port': self.ssl_port,
104+ 'use_ssl': True,
105+ 'disable_ssl_verify': True,
106+ }]
107 self.main = Main(root_dir, shares_dir, data_dir, partials_dir,
108- "localhost", self.ssl_port, dns_srv=None, ssl=True,
109- disable_ssl_verify=True)
110+ connection_info=connection_info)
111 self.addCleanup(self.main.shutdown)
112
113 self.eq = self.main.event_q
114@@ -972,8 +977,8 @@
115
116 # check locally and in the server
117 # it should only be our file in the directory, with content ok
118- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
119- self.assertEqual(l, ["foo"])
120+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
121+ self.assertEqual(files, ["foo"])
122 self.assertEqual(open(foo_path).read(), ":)")
123 yield self.compare_server()
124
125@@ -1130,8 +1135,8 @@
126 yield self.main.wait_for_nirvana(.5)
127
128 # check that everything is ok: it should only be our file in the dir...
129- l = os.listdir(dir2)
130- self.assertEqual(l, ["test_file"])
131+ files = os.listdir(dir2)
132+ self.assertEqual(files, ["test_file"])
133
134 # ... and our file should have the correct content
135 f = open(filepath2)
136@@ -1172,8 +1177,8 @@
137 def local_check(_):
138 """Check that everything is ok"""
139 # it should only be our file in the directory
140- l = os.listdir(dir2)
141- self.assertEqual(l, ["test_file"])
142+ files = os.listdir(dir2)
143+ self.assertEqual(files, ["test_file"])
144
145 # our file should have the correct content
146 f = open(filepath2)
147@@ -1233,8 +1238,8 @@
148
149 # wait dust to settle, check locally and in the server
150 yield self.main.wait_for_nirvana(.5)
151- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
152- self.assertEqual(l, ["test_file"])
153+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
154+ self.assertEqual(files, ["test_file"])
155 yield self.compare_server()
156
157 @defer.inlineCallbacks
158@@ -1317,8 +1322,8 @@
159 yield self.main.wait_for_nirvana(0.5)
160
161 # check that everything is ok
162- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
163- self.assertEqual(l, ["test_file"])
164+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
165+ self.assertEqual(files, ["test_file"])
166 with open(self.root_dir + "/test_file") as f:
167 content = f.read()
168 self.assertEqual(content, ":)")
169@@ -1474,8 +1479,8 @@
170
171 def local_check(_):
172 """check that everything is in sync"""
173- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
174- self.assertEqual(l, ["test_file"])
175+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
176+ self.assertEqual(files, ["test_file"])
177
178 final_data = open(self.root_dir + '/test_file').read()
179 self.assertEqual(data, final_data)
180@@ -1503,8 +1508,8 @@
181
182 def content_check(_):
183 """Check that everything is in sync."""
184- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
185- self.assertEqual(l, ["test_dir"])
186+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
187+ self.assertEqual(files, ["test_dir"])
188 self.assertTrue(stat.S_ISDIR(
189 os.stat(self.root_dir + '/test_dir')[stat.ST_MODE]))
190
191@@ -1564,8 +1569,8 @@
192 yield self.main.wait_for_nirvana(.5)
193
194 # it should only be our file in the directory
195- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
196- self.assertEqual(l, ["test_file"])
197+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
198+ self.assertEqual(files, ["test_file"])
199
200 # our file should have the full content
201 filesize = os.stat(self.root_dir + "/test_file").st_size
202@@ -1610,8 +1615,8 @@
203 yield self.main.wait_for_nirvana(.5)
204
205 # we should have both files...
206- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
207- self.assertEqual(sorted(l), ["test_file", "test_file.u1conflict"])
208+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
209+ self.assertEqual(sorted(files), ["test_file", "test_file.u1conflict"])
210
211 # ...with the respective content
212 sanefile = open(test_file).read()
213@@ -1631,8 +1636,8 @@
214 yield self.main.wait_for_nirvana(.5)
215
216 # finally, only one file with right content
217- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
218- self.assertEqual(sorted(l), ["test_file"])
219+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
220+ self.assertEqual(sorted(files), ["test_file"])
221 sanefile = open(test_file).read()
222 self.assertEqual(sanefile, data_local)
223 yield self.compare_server()
224@@ -1647,8 +1652,8 @@
225 yield self.main.wait_for_nirvana(.5)
226
227 # finally, only one file with right content
228- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
229- self.assertEqual(sorted(l), ["test_file", "test_file_new"])
230+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
231+ self.assertEqual(sorted(files), ["test_file", "test_file_new"])
232 srvfile = open(test_file).read()
233 self.assertEqual(srvfile, data_server)
234 newfile = open(test_file + "_new").read()
235@@ -1703,8 +1708,8 @@
236
237 def no_conflict_check(_):
238 """Check the absence of conflict, and the deletion of the node."""
239- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
240- self.assertEqual(l, [])
241+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
242+ self.assertEqual(files, [])
243
244 d = self.get_client()
245 d.addCallback(self.create_and_check)
246@@ -1740,8 +1745,8 @@
247 self.update_state(None)
248 yield self.main.wait_for_nirvana(.5)
249
250- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
251- self.assertEqual(sorted(l), [self.dir_name_conflict])
252+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
253+ self.assertEqual(sorted(files), [self.dir_name_conflict])
254
255
256 class TestConflict2(TestServerBase):
257@@ -1757,7 +1762,7 @@
258 os.mkdir(self.root_dir + "/test_dir/test_dir")
259 try:
260 yield self.client.unlink(request.ROOT, req.new_id)
261- except:
262+ except Exception:
263 # ignore if the unlink fails
264 pass
265 yield self.main.wait_for_nirvana(0.5)
266@@ -1840,8 +1845,8 @@
267 yield self.wait_for_nirvana(.5)
268
269 # no conflicts, and the server with correct data
270- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
271- self.assertEqual(l, ['testfile'])
272+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
273+ self.assertEqual(files, ['testfile'])
274 self.assertEqual(open(fname).read(), "other content")
275 yield self.check()
276
277@@ -1879,8 +1884,8 @@
278 play_hash()
279
280 # check that we don't have conflict, and all is the same with server
281- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
282- self.assertEqual(l, ['tfile'])
283+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
284+ self.assertEqual(files, ['tfile'])
285 self.assertEqual(open(fname).read(), fcontent)
286 yield self.check()
287
288@@ -1916,8 +1921,8 @@
289 yield self.wait_for_nirvana(.5)
290
291 # check that we don't have conflict, and all is the same with server
292- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
293- self.assertEqual(l, ['tfile'])
294+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
295+ self.assertEqual(files, ['tfile'])
296 self.assertEqual(open(fname).read(), fcontent)
297 yield self.check()
298
299@@ -1958,8 +1963,8 @@
300 yield self.wait_for_nirvana(.5)
301
302 # check that we don't have conflict, and all is the same with server
303- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
304- self.assertEqual(sorted(l), ['tfile'])
305+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
306+ self.assertEqual(sorted(files), ['tfile'])
307 self.assertEqual(open(fname).read(), 'foo')
308 yield self.check()
309
310@@ -2301,8 +2306,8 @@
311 self.aq.get_delta(request.ROOT, 0)
312 yield self.main.wait_for_nirvana(0)
313
314- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
315- self.assertEqual(l, ["tfile"])
316+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
317+ self.assertEqual(files, ["tfile"])
318 yield self.check()
319
320 @defer.inlineCallbacks
321@@ -2494,8 +2499,8 @@
322 yield wait_download
323
324 # check the node is conflicted, both files with corerct content
325- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
326- self.assertEqual(sorted(l), ["test_file", "test_file.u1conflict"])
327+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
328+ self.assertEqual(sorted(files), ["test_file", "test_file.u1conflict"])
329 with open(filepath) as f:
330 self.assertEqual(f.read(), ":)")
331 with open(filepath + ".u1conflict") as f:
332@@ -2520,8 +2525,8 @@
333 yield self.main.wait_for_nirvana(0.5)
334
335 # check that everything is ok
336- l = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
337- self.assertEqual(l, ["test_file"])
338+ files = filter_symlinks(self.root_dir, os.listdir(self.root_dir))
339+ self.assertEqual(files, ["test_file"])
340 with open(filepath) as f:
341 self.assertEqual(f.read(), ":)")
342
343
344=== modified file 'magicicada/server/server.py'
345--- magicicada/server/server.py 2017-04-22 21:07:36 +0000
346+++ magicicada/server/server.py 2018-03-20 19:58:45 +0000
347@@ -1698,7 +1698,7 @@
348 self._log_exception(exc)
349 yield self._send_protocol_error(failure)
350 yield self.done()
351- except:
352+ except Exception:
353 yield self.internal_error(Failure())
354
355 def _processMessage(self, message):
356
357=== modified file 'magicicada/server/testing/aq_helpers.py'
358--- magicicada/server/testing/aq_helpers.py 2017-04-22 21:07:36 +0000
359+++ magicicada/server/testing/aq_helpers.py 2018-03-20 19:58:45 +0000
360@@ -165,8 +165,7 @@
361 class ReallyFakeMain(main.Main):
362 """This main is so fake, it breaks nearly everything."""
363
364- def __init__(self, port, root_dir, data_dir, partials_dir,
365- dns_srv=None):
366+ def __init__(self, port, root_dir, data_dir, partials_dir):
367 self.root_dir = root_dir
368 self.shares_dir = os.path.join(os.path.dirname(root_dir), 'shares_dir')
369 self.shares_dir_link = os.path.join(root_dir, 'shares_link')
370@@ -186,8 +185,13 @@
371 self.event_q.subscribe(self.vm)
372 self.fs.register_eq(self.event_q)
373 self.sync = Sync(self)
374- self.action_q = ActionQueue(self.event_q, self, '127.0.0.1', port,
375- dns_srv, disable_ssl_verify=True)
376+ connection_info = [{
377+ 'host': '127.0.0.1',
378+ 'port': port,
379+ 'use_ssl': False,
380+ 'disable_ssl_verify': True,
381+ }]
382+ self.action_q = ActionQueue(self.event_q, self, connection_info)
383 self.state_manager = main.StateManager(self, handshake_timeout=30)
384 self.state_manager.connection.waiting_timeout = .1
385 self.vm.init_root()
386@@ -353,7 +357,6 @@
387 }
388 user = self.make_user(username=username, password=password)
389 self.storage_users[username] = user
390- self.dns_srv = None
391
392 # override and cleanup user config
393 self.old_get_config_files = main.config.get_config_files
394@@ -442,8 +445,7 @@
395 root_dir = self.mktemp('fake_root_dir')
396 data_dir = self.mktemp('fake_data_dir')
397 partials = self.mktemp('partials_dir')
398- self.main = ReallyFakeMain(self.port, root_dir,
399- data_dir, partials, self.dns_srv)
400+ self.main = ReallyFakeMain(self.port, root_dir, data_dir, partials)
401 self.state = self.main.state_manager
402 self.eq = self.main.event_q
403 self.listener = ReallyAttentiveListener()
404@@ -765,7 +767,8 @@
405 """Nukes the method"""
406 self.old = getattr(self.obj, self.meth)
407 if func is None:
408- def func(*args, **kwargs): return None
409+ def func(*args, **kwargs):
410+ return None
411 setattr(self.obj, self.meth, func)
412 return info
413
414@@ -808,7 +811,8 @@
415 """Nukes the method"""
416 self.old = getattr(self.aq.client, self.meth)
417 if func is None:
418- def func(*args, **kwargs): return defer.Deferred
419+ def func(*args, **kwargs):
420+ return defer.Deferred
421 setattr(self.aq.client, self.meth, func)
422 return info
423
424
425=== modified file 'magicicada/server/tests/test_shutdown.py'
426--- magicicada/server/tests/test_shutdown.py 2016-06-03 00:35:53 +0000
427+++ magicicada/server/tests/test_shutdown.py 2018-03-20 19:58:45 +0000
428@@ -99,7 +99,7 @@
429 try:
430 yield client.put_content(request.ROOT, mkfile_req.new_id,
431 empty_hash, "fake_hash", 1234, 1000, None)
432- except:
433+ except Exception:
434 client.transport.loseConnection()
435
436 ujobs = self.usr0.get_uploadjobs(node_id=mkfile_req.new_id)
437
438=== modified file 'magicicada/settings/__init__.py'
439--- magicicada/settings/__init__.py 2017-06-28 01:37:05 +0000
440+++ magicicada/settings/__init__.py 2018-03-20 19:58:45 +0000
441@@ -32,6 +32,7 @@
442 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
443 import logging
444 import os
445+import sys
446
447 from psycopg2.extensions import ISOLATION_LEVEL_REPEATABLE_READ
448
449@@ -264,4 +265,4 @@
450 try:
451 from magicicada.settings.local import * # noqa
452 except ImportError as err:
453- print("ERROR importing local settings:", err)
454+ print("ERROR importing local settings:", err, file=sys.stderr)

Subscribers

People subscribed via source and target branches

to all changes: