Merge lp:~facundo/ubuntuone-client/configurable-two-values into lp:ubuntuone-client

[Facundo Batista]

Use "simultaneous transfers" and "maximum payload size" from configuration (LP: #767466).

Use "simultaneous transfers" and "maximum payload size" from configuration.

"simult_transfers" moved from a code constant to a value from configuration, and "max_payload_size" is set in the protocol also from the configuration.

Added configuration for these two values, with sensible defaults; also, methods added to get and set these values.

Tests included for everything.

[Lucio Torre (lucio.torre)]

+1

review: Approve

[lp:~facundo/ubuntuone-client/configurable-two-values]

965. By Facundo Batista on 2011-04-20

Fixes using new protocol.

[Guillermo Gonzalez (verterok)]

+1

review: Approve

=== modified file 'data/syncdaemon.conf'
--- data/syncdaemon.conf	2011-04-06 13:50:33 +0000
+++ data/syncdaemon.conf	2011-04-20 21:52:54 +0000
@@ -91,6 +91,15 @@
 use_trash.help = Send the files and folders to Trash folder instead of
     removing them permanently.
 
+simult_transfers.default = 10
+simult_transfers.parser = int
+simult_transfers.help = Quantity of Uploads and Downloads that will be run
+    in parallel.
+
+max_payload_size.default = 1400
+max_payload_size.parser = int
+max_payload_size.help = The maximum payload for messages sent to the server.
+
 
 [notifications]
 show_all_notifications.default = True

=== modified file 'tests/platform/linux/eventlog/test_zg_listener.py'
--- tests/platform/linux/eventlog/test_zg_listener.py	2011-03-22 18:16:28 +0000
+++ tests/platform/linux/eventlog/test_zg_listener.py	2011-04-20 21:52:54 +0000
@@ -74,6 +74,7 @@
 from tests.syncdaemon.test_action_queue import (
     ConnectedBaseTestCase,
     FakeSemaphore,
+    TestingProtocol,
 )
 
 VOLUME = uuid.UUID('12345678-1234-1234-1234-123456789abc')
@@ -606,7 +607,8 @@
         cmd.handle_success(request)
 
         # create a request and fill it with succesful information
-        request = client.PutContent(self.action_queue.client, self.share_id,
+        aq_client = TestingProtocol()
+        request = client.PutContent(aq_client, self.share_id,
                                     'node', 'prvhash', 'newhash', 'crc32',
                                     'size', 'deflated', 'fd')
         request.new_generation = 13
@@ -695,7 +697,8 @@
         self.fsm.set_node_id(path, "a_node_id")
 
         # create a request and fill it with succesful information
-        request = client.PutContent(self.action_queue.client, self.share_id,
+        aq_client = TestingProtocol()
+        request = client.PutContent(aq_client, self.share_id,
                                     'node', 'prvhash', 'newhash', 'crc32',
                                     'size', 'deflated', 'fd')
         request.new_generation = 13

=== modified file 'tests/syncdaemon/test_action_queue.py'
--- tests/syncdaemon/test_action_queue.py	2011-04-12 19:46:23 +0000
+++ tests/syncdaemon/test_action_queue.py	2011-04-20 21:52:54 +0000
@@ -55,12 +55,12 @@
     protocol_pb2,
     request,
 )
-from ubuntuone.syncdaemon import states, interfaces
+from ubuntuone.syncdaemon import states, interfaces, config
 from ubuntuone.syncdaemon import action_queue
 from ubuntuone.syncdaemon.action_queue import (
     ActionQueue, ActionQueueCommand, ChangePublicAccess, CreateUDF,
     DeleteVolume, Download, ListVolumes, ActionQueueProtocol, ListShares,
-    RequestQueue, UploadProgressWrapper, Upload, SIMULT_TRANSFERS,
+    RequestQueue, UploadProgressWrapper, Upload,
     CreateShare, DeleteShare, GetPublicFiles, GetDelta, GetDeltaFromScratch,
     TRANSFER_PROGRESS_THRESHOLD, Unlink, Move, MakeFile, MakeDir, DeltaList,
     ZipQueue, DeferredMap, ThrottlingStorageClient, PathLockingTree,
@@ -199,6 +199,8 @@
 class TestingProtocol(ActionQueue.protocol):
     """Protocol for testing."""
 
+    max_payload_size = 65536
+
     def connectionMade(self):
         """connectionMade."""
         ActionQueue.protocol.connectionMade(self)
@@ -722,6 +724,13 @@
         self.rq.unqueue(cmd2)
         self.assertEqual(len(self.rq), 0)
 
+    def test_init_simult_transfers(self):
+        """Configure the transfers semaphore according to config."""
+        user_config = config.get_user_config()
+        user_config.set_simult_transfers(12345)
+        rq = RequestQueue(action_queue=None)
+        self.assertEqual(rq.transfers_semaphore.tokens, 12345)
+
 
 class TestDeferredMap(TwistedTestCase):
     """Test the deferred map."""
@@ -3239,7 +3248,8 @@
     def test_handle_success_push_event(self):
         """Test AQ_UPLOAD_FINISHED is pushed on success."""
         # create a request and fill it with succesful information
-        request = client.PutContent(self.action_queue.client, VOLUME, 'node',
+        aq_client = TestingProtocol()
+        request = client.PutContent(aq_client, VOLUME, 'node',
                                     'prvhash', 'newhash', 'crc32', 'size',
                                     'deflated', 'fd')
         request.new_generation = 13
@@ -4940,6 +4950,13 @@
         # check also the log
         self.assertTrue(self.handler.check_debug('Ping! rtt: 1.123 segs'))
 
+    def test_init_max_payload_size(self):
+        """Configure max_payload_size on init according to config."""
+        user_config = config.get_user_config()
+        user_config.set_max_payload_size(12345)
+        aqp = ActionQueueProtocol()
+        self.assertEqual(aqp.max_payload_size, 12345)
+
 
 class CommandCycleTestCase(BasicTestCase):
     """Test the command behaviour on run, retry, stop, etc.
@@ -5363,7 +5380,8 @@
         # transfer semaphore in _start
         transfers_semaphore = self.queue.transfers_semaphore
         semaphores = []
-        for i in xrange(SIMULT_TRANSFERS):
+        user_config = config.get_user_config()
+        for i in xrange(user_config.get_simult_transfers()):
             s = transfers_semaphore.acquire()
             s.addCallback(semaphores.append)
 

=== modified file 'tests/syncdaemon/test_config.py'
--- tests/syncdaemon/test_config.py	2011-04-13 19:23:22 +0000
+++ tests/syncdaemon/test_config.py	2011-04-20 21:52:54 +0000
@@ -428,6 +428,45 @@
         branch_config = os.path.join(rootdir, "data", config.CONFIG_FILE)
         self.assertIn(branch_config, config_files)
 
+    def test_get_simult_transfers(self):
+        """Get simult transfers."""
+        conf_file = os.path.join(self.test_root, 'test_load_config.conf')
+        with open(conf_file, 'w') as fh:
+            fh.write('[__main__]\n')
+            fh.write('simult_transfers = 12345\n')
+        conf = config._Config(conf_file)
+        self.assertEqual(conf.get_simult_transfers(), 12345)
+
+    def test_set_simult_transfers(self):
+        """Set simult transfers."""
+        conf_file = os.path.join(self.test_root, 'test_load_config.conf')
+        with open(conf_file, 'w') as fh:
+            fh.write('[__main__]\n')
+            fh.write('simult_transfers = 12345\n')
+        conf = config._Config(conf_file)
+        conf.set_simult_transfers(666)
+        self.assertEqual(conf.get_simult_transfers(), 666)
+
+    def test_get_max_payload_size(self):
+        """Get the maximum payload size."""
+        conf_file = os.path.join(self.test_root, 'test_load_config.conf')
+        with open(conf_file, 'w') as fh:
+            fh.write('[__main__]\n')
+            fh.write('max_payload_size = 12345\n')
+        conf = config._Config(conf_file)
+        self.assertEqual(conf.get_max_payload_size(), 12345)
+
+    def test_set_max_payload_size(self):
+        """Set the maximum payload size."""
+        conf_file = os.path.join(self.test_root, 'test_load_config.conf')
+        with open(conf_file, 'w') as fh:
+            fh.write('[__main__]\n')
+            fh.write('max_payload_size = 12345\n')
+        conf = config._Config(conf_file)
+        conf.set_max_payload_size(666)
+        self.assertEqual(conf.get_max_payload_size(), 666)
+
+
 class ConfigglueParsersTests(BaseTwistedTestCase):
     """Tests for our custom configglue parsers"""
 
@@ -603,3 +642,14 @@
         self.cp.parse_all()
         self.assertIn(r'\A\.~lock\..*#\Z',
                       self.cp.get('__main__', 'ignore').value)
+
+    def test_simult_transfers(self):
+        """Test default configuration for simultaneous transfers."""
+        self.cp.parse_all()
+        self.assertEqual(self.cp.get('__main__', 'simult_transfers').value, 10)
+
+    def test_max_payload_size(self):
+        """Test default configuration for maximum payload size."""
+        self.cp.parse_all()
+        configured = self.cp.get('__main__', 'max_payload_size').value
+        self.assertEqual(configured, 1400)

=== modified file 'ubuntuone/syncdaemon/action_queue.py'
--- ubuntuone/syncdaemon/action_queue.py	2011-04-18 20:46:44 +0000
+++ ubuntuone/syncdaemon/action_queue.py	2011-04-20 21:52:54 +0000
@@ -57,6 +57,7 @@
 from ubuntuone.storageprotocol.context import get_ssl_context
 from ubuntuone.syncdaemon.interfaces import IActionQueue, IMarker
 from ubuntuone.syncdaemon.logger import mklog, TRACE
+from ubuntuone.syncdaemon import config
 
 logger = logging.getLogger("ubuntuone.SyncDaemon.ActionQueue")
 
@@ -69,9 +70,6 @@
 # progress threshold to emit a download/upload progress event: 64Kb
 TRANSFER_PROGRESS_THRESHOLD = 64*1024*1024
 
-# no more than this uploads are executed simultaneously
-SIMULT_TRANSFERS = 10
-
 def passit(func):
     """Pass the value on for the next deferred, while calling func with it."""
 
@@ -302,6 +300,11 @@
     _looping_ping = None
     _ping_delay = 600  # 10 minutes
 
+    def __init__(self):
+        LoggingStorageClient.__init__(self)
+        user_config = config.get_user_config()
+        self.max_payload_size = user_config.get_max_payload_size()
+
     def connectionMade(self):
         """A new connection was made."""
         self.log.info('Connection made.')
@@ -458,9 +461,13 @@
         self.waiting = []
         self.hashed_waiting = {}
         self.active = False
-        self.transfers_semaphore = defer.DeferredSemaphore(SIMULT_TRANSFERS)
         self.active_deferred = defer.Deferred()
 
+        # transfers semaphore
+        user_config = config.get_user_config()
+        simult_transfers = user_config.get_simult_transfers()
+        self.transfers_semaphore = defer.DeferredSemaphore(simult_transfers)
+
     def __len__(self):
         """Return the length of the waiting queue."""
         return len(self.waiting)

=== modified file 'ubuntuone/syncdaemon/config.py'
--- ubuntuone/syncdaemon/config.py	2011-04-05 19:14:18 +0000
+++ ubuntuone/syncdaemon/config.py	2011-04-20 21:52:54 +0000
@@ -374,6 +374,26 @@
     def set_use_trash(self, enabled):
         self.set(MAIN, 'use_trash', str(enabled))
 
+    @requires_section(MAIN)
+    def get_simult_transfers(self):
+        """Get the simultaneous transfers value."""
+        return self.get_parsed(MAIN, 'simult_transfers')
+
+    @requires_section(MAIN)
+    def set_simult_transfers(self, value):
+        """Set the simultaneous transfers value."""
+        self.set(MAIN, 'simult_transfers', str(value))
+
+    @requires_section(MAIN)
+    def get_max_payload_size(self):
+        """Get the maximum payload size."""
+        return self.get_parsed(MAIN, 'max_payload_size')
+
+    @requires_section(MAIN)
+    def set_max_payload_size(self, value):
+        """Set the maximum payload size."""
+        self.set(MAIN, 'max_payload_size', str(value))
+
     @requires_section(NOTIFICATIONS)
     def set_show_all_notifications(self, enabled):
         self.set(NOTIFICATIONS, 'show_all_notifications', str(enabled))