Merge lp:~mthaddon/charms/trusty/swift-storage/workers-cpu-scaling into lp:~openstack-charmers-archive/charms/trusty/swift-storage/next

[Tom Haddon]

Make the number of workers for account, container and object servers a multiplier of the number of CPUs, and default to a sensible setting per recommendations in http://docs.openstack.org/developer/swift/deployment_guide.html#general-service-tuning. Also, add in a tunable option for object server threads_per_disk and tune default to recommendations from http://docs.openstack.org/developer/swift/deployment_guide.html#object-server-configuration.

[James Page (james-page)]

Hi Tom

Thanks for the MP - looks like a good feature to have.

Right now this branch introduces a test failure:

======================================================================
FAIL: test_swift_storage_server_context (unit_tests.test_swift_storage_context.SwiftStorageContextTests)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/home/jamespage/src/charms/review/swift-storage/unit_tests/test_swift_storage_context.py", line 71, in test_swift_storage_server_context
    self.assertEquals(ex, result)
AssertionError: {'container_server_port': '502', 'local_ip': '10.0.0.5', 'account_server_port': [truncated]... != {'local_ip': '10.0.0.5', 'workers': 4, 'object_server_threads_per_disk': 4, 'con [truncated]...
  {'account_server_port': '500',
   'container_server_port': '502',
   'local_ip': '10.0.0.5',
- 'object_server_port': '501'}
? ^

+ 'object_server_port': '501',
? ^

+ 'object_server_threads_per_disk': 4,
+ 'workers': 4}

Please could you fix up - you can run units tests by using 'make test'. You will need to patch out psutils.NUM_CPU's as well in order to make the test deterministic.

Cheers

James

review: Needs Fixing

[lp:~mthaddon/charms/trusty/swift-storage/workers-cpu-scaling]

32. By Tom Haddon on 2014-06-18

Unit tests for the new worker multiplier and threads per disk config values

[Tom Haddon (mthaddon)]

I've updated this to fix the tests. I've converted the "workers" variable to a string in both the unit tests and the SwiftStorageServerContext class, as that seems to match what we have for the other values.

=== modified file 'config.yaml'
--- config.yaml	2012-12-19 23:09:13 +0000
+++ config.yaml	2014-06-18 11:34:26 +0000
@@ -49,4 +49,17 @@
     default: 6002
     type: int
     description: Listening port of the swift-account-server.
-
+  worker-multiplier:
+    default: 1
+    type: int
+    description: |
+      The CPU multiplier to use when configuring worker processes for the
+      account, container and object server processes.
+  object-server-threads-per-disk:
+    default: 4
+    type: int
+    description: |
+      Size of the per-disk thread pool used for performing disk I/O. 0 means
+      to not use a per-disk thread pool. It is recommended to keep this value 
+      small, as large values can result in high read latencies due to large 
+      queue depths. A good starting point is 4 threads per disk.

=== modified file 'hooks/swift_storage_context.py'
--- hooks/swift_storage_context.py	2013-08-16 20:38:32 +0000
+++ hooks/swift_storage_context.py	2014-06-18 11:34:26 +0000
@@ -61,10 +61,15 @@
     interfaces = []
 
     def __call__(self):
+        import psutil
+        multiplier = int(config('worker-multiplier')) or 1
         ctxt = {
             'local_ip': unit_private_ip(),
             'account_server_port': config('account-server-port'),
             'container_server_port': config('container-server-port'),
             'object_server_port': config('object-server-port'),
+            'workers': str(psutil.NUM_CPUS * multiplier),
+            'object_server_threads_per_disk': config(
+                'object-server-threads-per-disk'),
         }
         return ctxt

=== modified file 'hooks/swift_storage_utils.py'
--- hooks/swift_storage_utils.py	2014-05-23 18:35:14 +0000
+++ hooks/swift_storage_utils.py	2014-06-18 11:34:26 +0000
@@ -48,7 +48,7 @@
 
 PACKAGES = [
     'swift', 'swift-account', 'swift-container', 'swift-object',
-    'xfsprogs', 'gdisk', 'lvm2', 'python-jinja2',
+    'xfsprogs', 'gdisk', 'lvm2', 'python-jinja2', 'python-psutil',
 ]
 
 TEMPLATES = 'templates/'

=== modified file 'templates/account-server.conf'
--- templates/account-server.conf	2013-09-30 13:29:32 +0000
+++ templates/account-server.conf	2014-06-18 11:34:26 +0000
@@ -1,7 +1,7 @@
 [DEFAULT]
 bind_ip = 0.0.0.0
 bind_port = {{ account_server_port }}
-workers = 2
+workers = {{ workers }}
 
 [pipeline:main]
 pipeline = recon account-server

=== modified file 'templates/container-server.conf'
--- templates/container-server.conf	2014-04-02 12:49:56 +0000
+++ templates/container-server.conf	2014-06-18 11:34:26 +0000
@@ -1,7 +1,7 @@
 [DEFAULT]
 bind_ip = 0.0.0.0
 bind_port = {{ container_server_port }}
-workers = 2
+workers = {{ workers }}
 
 [pipeline:main]
 pipeline = recon container-server

=== modified file 'templates/object-server.conf'
--- templates/object-server.conf	2013-09-30 13:29:32 +0000
+++ templates/object-server.conf	2014-06-18 11:34:26 +0000
@@ -1,7 +1,7 @@
 [DEFAULT]
 bind_ip = 0.0.0.0
 bind_port = {{ object_server_port }}
-workers = 2
+workers = {{ workers }}
 
 [pipeline:main]
 pipeline = recon object-server
@@ -12,6 +12,7 @@
 
 [app:object-server]
 use = egg:swift#object
+threads_per_disk = {{ object_server_threads_per_disk }}
 
 [object-replicator]
 

=== modified file 'unit_tests/test_swift_storage_context.py'
--- unit_tests/test_swift_storage_context.py	2013-09-27 16:33:06 +0000
+++ unit_tests/test_swift_storage_context.py	2014-06-18 11:34:26 +0000
@@ -56,16 +56,22 @@
             _file.write.assert_called_with('RSYNC_ENABLE=true\n')
 
     def test_swift_storage_server_context(self):
+        import psutil
         self.unit_private_ip.return_value = '10.0.0.5'
         self.test_config.set('account-server-port', '500')
         self.test_config.set('object-server-port', '501')
         self.test_config.set('container-server-port', '502')
+        self.test_config.set('object-server-threads-per-disk', '3')
+        self.test_config.set('worker-multiplier', '3')
+        num_workers = psutil.NUM_CPUS * 3 
         ctxt = swift_context.SwiftStorageServerContext()
         result = ctxt()
         ex = {
             'container_server_port': '502',
             'object_server_port': '501',
             'account_server_port': '500',
-            'local_ip': '10.0.0.5'
+            'local_ip': '10.0.0.5',
+            'object_server_threads_per_disk': '3',
+            'workers': str(num_workers),
         }
         self.assertEquals(ex, result)