Merge lp:~stub/charms/trusty/cassandra/ensure-thrift into lp:~stub/charms/trusty/cassandra/spike

[Stuart Bishop]

Fix Bug #1523546 by supporting Cassandra 2.2.

Note that Cassandra 2.1 is still the release series recommended for production use.

[lp:~stub/charms/trusty/cassandra/ensure-thrift]

370. By Stuart Bishop on 2015-12-08

Turn on Cassandra 2.2 tests

371. By Stuart Bishop on 2015-12-09

Cassandra 2.2 is now recommended upstream, so switch it to the default

372. By Stuart Bishop on 2015-12-09

Cassandra 3.0 support, switch to OpenJDK 8

373. By Stuart Bishop on 2015-12-09

Rollup unlanded features, resolving conflicts

374. By Stuart Bishop on 2015-12-09

Merge rsync-port

375. By Stuart Bishop on 2015-12-09

Merge noauthentication

376. By Stuart Bishop on 2015-12-09

Fix 3.0 test

377. By Stuart Bishop on 2015-12-09

Fix defaults in config.yaml

378. By Stuart Bishop on 2016-01-15

delint

379. By Stuart Bishop on 2016-01-15

Merge trunk

Unmerged revisions

379. By Stuart Bishop on 2016-01-15

Merge trunk

378. By Stuart Bishop on 2016-01-15

delint

377. By Stuart Bishop on 2015-12-09

Fix defaults in config.yaml

376. By Stuart Bishop on 2015-12-09

Fix 3.0 test

375. By Stuart Bishop on 2015-12-09

Merge noauthentication

374. By Stuart Bishop on 2015-12-09

Merge rsync-port

373. By Stuart Bishop on 2015-12-09

Rollup unlanded features, resolving conflicts

372. By Stuart Bishop on 2015-12-09

Cassandra 3.0 support, switch to OpenJDK 8

371. By Stuart Bishop on 2015-12-09

Cassandra 2.2 is now recommended upstream, so switch it to the default

370. By Stuart Bishop on 2015-12-08

Turn on Cassandra 2.2 tests

=== modified file 'Makefile'
--- Makefile	2015-08-19 09:57:40 +0000
+++ Makefile	2015-12-08 10:18:39 +0000
@@ -70,12 +70,18 @@
 	AMULET_TIMEOUT=5400 \
 	$(NOSETESTS) tests.test_integration:Test1UnitDeployment 2>&1 | ts
 	
-20test: unittest Test21Deployment
+20test: unittest Test20Deployment
 Test20Deployment: deps
 	date
 	AMULET_TIMEOUT=5400 \
 	$(NOSETESTS) tests.test_integration:Test20Deployment 2>&1 | ts
-	
+
+22test: unittest Test22Deployment
+Test22Deployment: deps
+	date
+	AMULET_TIMEOUT=5400 \
+	$(NOSETESTS) tests.test_integration:Test22Deployment 2>&1 | ts
+		
 3test: unittest Test3UnitDeployment
 Test3UnitDeployment: deps
 	date

=== modified file 'README.md'
--- README.md	2015-10-20 09:41:37 +0000
+++ README.md	2015-12-08 10:18:39 +0000
@@ -13,7 +13,7 @@
 
 # Editions
 
-This charm supports Apache Cassandra 2.0, Apache Cassandra 2.1, and
+This charm supports Apache Cassandra 2.0, 2.1 & 2.2, and
 Datastax Enterprise 4.8. The default is Apache Cassandra 2.1.
 
 To use Apache Cassandra 2.0, specify the Apache Cassandra 2.0 archive source

=== modified file 'hooks/helpers.py'
--- hooks/helpers.py	2015-09-11 05:07:47 +0000
+++ hooks/helpers.py	2015-12-08 10:18:39 +0000
@@ -16,6 +16,7 @@
 import configparser
 from contextlib import contextmanager
 from datetime import timedelta
+from distutils.version import LooseVersion
 import errno
 from functools import wraps
 import io
@@ -303,10 +304,20 @@
 
 def get_cassandra_version():
     if get_cassandra_edition() == 'dse':
-        return '2.1' if get_package_version('dse-full') else None
+        dse_ver = get_package_version('dse-full')
+        if not dse_ver:
+            return None
+        elif LooseVersion(dse_ver) >= LooseVersion('4.7'):
+            return '2.1'
+        else:
+            return '2.0'
     return get_package_version('cassandra')
 
 
+def has_cassandra_version(minimum_ver):
+    return LooseVersion(get_cassandra_version()) >= LooseVersion(minimum_ver)
+
+
 def get_cassandra_config_dir():
     if get_cassandra_edition() == 'dse':
         return '/etc/dse/cassandra'
@@ -523,13 +534,21 @@
         hookenv.log('Creating SUPERUSER {}'.format(username))
     else:
         hookenv.log('Creating user {}'.format(username))
-    query(session,
-          'INSERT INTO system_auth.users (name, super) VALUES (%s, %s)',
-          ConsistencyLevel.ALL, (username, superuser))
-    query(session,
-          'INSERT INTO system_auth.credentials (username, salted_hash) '
-          'VALUES (%s, %s)',
-          ConsistencyLevel.ALL, (username, encrypted_password))
+    if has_cassandra_version('2.2'):
+        query(session,
+              'INSERT INTO system_auth.roles '
+              '(role, can_login, is_superuser, salted_hash) '
+              'VALUES (%s, TRUE, %s, %s)',
+              ConsistencyLevel.ALL,
+              (username, superuser, encrypted_password))
+    else:
+        query(session,
+              'INSERT INTO system_auth.users (name, super) VALUES (%s, %s)',
+              ConsistencyLevel.ALL, (username, superuser))
+        query(session,
+              'INSERT INTO system_auth.credentials (username, salted_hash) '
+              'VALUES (%s, %s)',
+              ConsistencyLevel.ALL, (username, encrypted_password))
 
 
 @logged
@@ -714,6 +733,11 @@
     # with the system_auth keyspace replication settings.
     cassandra_yaml['endpoint_snitch'] = 'GossipingPropertyFileSnitch'
 
+    # Per Bug #1523546 and CASSANDRA-9319, Thrift is disabled by default in
+    # Cassandra 2.2. Ensure it is enabled if rpc_port is non-zero.
+    if int(config['rpc_port']) > 0:
+        cassandra_yaml['start_rpc'] = True
+
     cassandra_yaml.update(overrides)
 
     write_cassandra_yaml(cassandra_yaml)

=== modified file 'tests/test_helpers.py'
--- tests/test_helpers.py	2015-09-04 11:12:27 +0000
+++ tests/test_helpers.py	2015-12-08 10:18:39 +0000
@@ -731,8 +731,10 @@
                           sentinel.consistency, sentinel.args)
         self.assertEqual(session.execute.call_count, 4)
 
+    @patch('helpers.get_cassandra_version')
     @patch('helpers.query')
-    def test_ensure_user(self, query):
+    def test_ensure_user(self, query, ver):
+        ver.return_value = '2.1'
         helpers.ensure_user(sentinel.session,
                             sentinel.username, sentinel.pwhash,
                             superuser=sentinel.supflag)
@@ -746,6 +748,21 @@
                  ConsistencyLevel.ALL,
                  (sentinel.username, sentinel.pwhash))])
 
+    @patch('helpers.get_cassandra_version')
+    @patch('helpers.query')
+    def test_ensure_user_22(self, query, ver):
+        ver.return_value = '2.2'
+        helpers.ensure_user(sentinel.session,
+                            sentinel.username, sentinel.pwhash,
+                            superuser=sentinel.supflag)
+        query.assert_called_once_with(sentinel.session,
+                                      'INSERT INTO system_auth.roles (role, '
+                                      'can_login, is_superuser, salted_hash) '
+                                      'VALUES (%s, TRUE, %s, %s)',
+                                      ConsistencyLevel.ALL,
+                                      (sentinel.username, sentinel.supflag,
+                                       sentinel.pwhash))
+
     @patch('helpers.ensure_user')
     @patch('helpers.encrypt_password')
     @patch('helpers.nodetool')
@@ -992,6 +1009,79 @@
                 stream_throughput_outbound_megabits_per_sec: 200
                 tombstone_warn_threshold: 1000
                 tombstone_failure_threshold: 100000
+                start_rpc: true
+                ''')
+            self.maxDiff = None
+            self.assertEqual(yaml.safe_load(new_config),
+                             yaml.safe_load(expected_config))
+
+            # Confirm we can use an explicit cluster_name too.
+            write_file.reset_mock()
+            hookenv.config()['cluster_name'] = 'fubar'
+            helpers.configure_cassandra_yaml()
+            new_config = write_file.call_args[0][1]
+            self.assertEqual(yaml.safe_load(new_config)['cluster_name'],
+                             'fubar')
+
+    @patch('helpers.get_cassandra_version')
+    @patch('helpers.get_cassandra_yaml_file')
+    @patch('helpers.get_seed_ips')
+    @patch('charmhelpers.core.host.write_file')
+    def test_configure_cassandra_yaml_22(self, write_file, seed_ips, yaml_file,
+                                         get_cassandra_version):
+        get_cassandra_version.return_value = '2.0'
+        hookenv.config().update(dict(num_tokens=128,
+                                     cluster_name='test_cluster_name',
+                                     partitioner='test_partitioner'))
+
+        seed_ips.return_value = ['10.20.0.1', '10.20.0.2', '10.20.0.3']
+
+        existing_config = '''
+            seed_provider:
+                - class_name: blah.SimpleSeedProvider
+                  parameters:
+                      - seeds: 127.0.0.1  # Comma separated list.
+            start_rpc: false  # Defaults to False starting 2.2
+            '''
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            yaml_config = os.path.join(tmpdir, 'c.yaml')
+            yaml_file.return_value = yaml_config
+            with open(yaml_config, 'w', encoding='UTF-8') as f:
+                f.write(existing_config)
+
+            helpers.configure_cassandra_yaml()
+
+            self.assertEqual(write_file.call_count, 2)
+            new_config = write_file.call_args[0][1]
+
+            expected_config = dedent('''\
+                start_rpc: true
+                cluster_name: test_cluster_name
+                authenticator: PasswordAuthenticator
+                num_tokens: 128
+                partitioner: test_partitioner
+                listen_address: 10.20.0.1
+                rpc_address: 0.0.0.0
+                rpc_port: 9160
+                native_transport_port: 9042
+                storage_port: 7000
+                ssl_storage_port: 7001
+                authorizer: AllowAllAuthorizer
+                seed_provider:
+                    - class_name: blah.SimpleSeedProvider
+                      parameters:
+                        # No whitespace in seeds is important.
+                        - seeds: '10.20.0.1,10.20.0.2,10.20.0.3'
+                endpoint_snitch: GossipingPropertyFileSnitch
+                data_file_directories:
+                    - /var/lib/cassandra/data
+                commitlog_directory: /var/lib/cassandra/commitlog
+                saved_caches_directory: /var/lib/cassandra/saved_caches
+                compaction_throughput_mb_per_sec: 16
+                stream_throughput_outbound_megabits_per_sec: 200
+                tombstone_warn_threshold: 1000
+                tombstone_failure_threshold: 100000
                 ''')
             self.maxDiff = None
             self.assertEqual(yaml.safe_load(new_config),
@@ -1044,6 +1134,7 @@
                 listen_address: 10.20.0.1
                 rpc_address: 0.0.0.0
                 broadcast_rpc_address: 10.30.0.1
+                start_rpc: true
                 rpc_port: 9160
                 native_transport_port: 9042
                 storage_port: 7000

=== modified file 'tests/test_integration.py'
--- tests/test_integration.py	2015-11-13 05:35:42 +0000
+++ tests/test_integration.py	2015-12-08 10:18:39 +0000
@@ -556,6 +556,17 @@
         install_keys=yaml.safe_dump([None]))
 
 
+class Test22Deployment(Test1UnitDeployment):
+    """Tests run on a single node Apache Cassandra 2.2 cluster.
+    """
+    rf = 1
+    test_config = dict(
+        edition='community',
+        install_sources=yaml.safe_dump([
+            'deb http://www.apache.org/dist/cassandra/debian 22x main']),
+        install_keys=yaml.safe_dump([None]))
+
+
 # Bug #1417097 means we need to monkey patch Amulet for now.
 real_juju = amulet.helpers.juju