Merge lp:~thumper/python-jujuclient/jes-cache-file into lp:python-jujuclient

[Tim Penhey]

Teach python-jujuclient about the cache.yaml file for cached environment information.

With the jes feature flag, we now don't store environment details in .jenv files, but in a single cache.yaml file.

This branch teaches python-jujuclient how to get the required information out of it.

Tests all pass.

[Tim Penhey (thumper)]

I have not been able to run the tests for some reason, and I am unfamiliar with tox:

$ JUJU_TEST_ENV="mary2" tox -e py27
Traceback (most recent call last):
  File "/usr/bin/tox", line 9, in <module>
    load_entry_point('tox==1.6.0', 'console_scripts', 'tox')()
  File "/usr/lib/python2.7/dist-packages/tox/_cmdline.py", line 25, in main
    config = parseconfig(args, 'tox')
  File "/usr/lib/python2.7/dist-packages/tox/_config.py", line 44, in parseconfig
    parseini(config, inipath)
  File "/usr/lib/python2.7/dist-packages/tox/_config.py", line 236, in __init__
    config)
  File "/usr/lib/python2.7/dist-packages/tox/_config.py", line 292, in _makeenvconfig
    vc.setenv = reader.getdict(section, 'setenv')
  File "/usr/lib/python2.7/dist-packages/tox/_config.py", line 419, in getdict
    s = self.getdefault(section, name, None)
  File "/usr/lib/python2.7/dist-packages/tox/_config.py", line 511, in getdefault
    x = self._replace(x)
  File "/usr/lib/python2.7/dist-packages/tox/_config.py", line 618, in _replace
    return rexpattern.sub(replace_func, x)
  File "/usr/lib/python2.7/dist-packages/tox/_config.py", line 609, in _replace_match_no_quote
    return self._replace_match(match, quote=False)
  File "/usr/lib/python2.7/dist-packages/tox/_config.py", line 604, in _replace_match
    return handler(match, quote)
  File "/usr/lib/python2.7/dist-packages/tox/_config.py", line 541, in _replace_env
    (envkey, envkey))
tox.ConfigError: ConfigError: substitution env:'JUJU_TEST_ENV:"test"': unkown environment variable 'JUJU_TEST_ENV:"test"'

[Kapil Thangavelu (hazmat)]

talked with tim on irc, think he worked through this

[Tim Penhey (thumper)]

Managed to get the tests running, but they fail, even for trunk.

https://bugs.launchpad.net/python-jujuclient/+bug/1465084

[Tim Van Steenburgh (tvansteenburgh)]

LGTM, thanks thumper.

review: Approve

=== modified file '.bzrignore'
--- .bzrignore	2015-07-09 20:46:45 +0000
+++ .bzrignore	2015-07-17 05:40:13 +0000
@@ -13,3 +13,4 @@
 .tox
 .coverage
 juju-backup-20150709-202028.tar.gz
+__pycache__

=== modified file 'jujuclient.py'
--- jujuclient.py	2015-07-16 22:19:20 +0000
+++ jujuclient.py	2015-07-17 05:40:13 +0000
@@ -207,16 +207,48 @@
         return s
 
     def parse_env(self, env_name):
-        import yaml
         jhome = os.path.expanduser(
             os.environ.get('JUJU_HOME', '~/.juju'))
+
+        # Look in the cache file first.
+        cache_file = os.path.join(jhome, 'environments', 'cache.yaml')
         jenv = os.path.join(jhome, 'environments', '%s.jenv' % env_name)
+
+        if os.path.exists(cache_file):
+            try:
+                return jhome, self.environment_from_cache(env_name, cache_file)
+            except EnvironmentNotBootstrapped:
+                pass
+                # Fall through to getting the info from the jenv
         if not os.path.exists(jenv):
             raise EnvironmentNotBootstrapped(env_name)
-
+        return jhome, self.environment_from_jenv(jenv)
+
+    def environment_from_cache(self, env_name, cache_filename):
+        import yaml
+        with open(cache_filename) as fh:
+            data = yaml.safe_load(fh.read())
+            try:
+                # environment holds:
+                #   user, env-uuid, server-uuid
+                environment = data['environment'][env_name]
+                server = data['server-data'][environment['server-uuid']]
+                return {
+                    'user': environment['user'],
+                    'password': server['identities'][environment['user']],
+                    'environ-uuid': environment['env-uuid'],
+                    'server-uuid': environment['server-uuid'],
+                    'state-servers': server['api-endpoints'],
+                    'ca-cert': server['ca-cert'],
+                }
+            except KeyError:
+                raise EnvironmentNotBootstrapped(env_name)
+
+    def environment_from_jenv(self, jenv):
+        import yaml
         with open(jenv) as fh:
             data = yaml.safe_load(fh.read())
-            return jhome, data
+            return data
 
     @staticmethod
     def split_host_port(server):

=== modified file 'test_jujuclient.py'
--- test_jujuclient.py	2015-07-16 22:19:20 +0000
+++ test_jujuclient.py	2015-07-17 05:40:13 +0000
@@ -1,3 +1,4 @@
+import copy
 import errno
 import mock
 import os
@@ -15,7 +16,10 @@
     # Watch wrappers
     WaitForNoMachines,
     # Facades
-    Actions, Annotations, Backups, Charms, HA, KeyManager, UserManager)
+    Actions, Annotations, Backups, Charms, HA, KeyManager, UserManager,
+    # Exceptions
+    EnvironmentNotBootstrapped,
+)
 
 
 ENV_NAME = os.environ.get("JUJU_TEST_ENV")
@@ -24,6 +28,16 @@
     raise ValueError("No Testing Environment Defined.")
 
 
+SAMPLE_CONFIG = {
+    'user': 'tester',
+    'password': 'sekrit',
+    'environ-uuid': 'some-uuid',
+    'server-uuid': 'server-uuid',
+    'state-servers': ['localhost:12345'],
+    'ca-cert': 'test-cert',
+}
+
+
 def reset(env):
     status = env.status()
     env.destroy_machines(
@@ -57,6 +71,11 @@
 
 class ClientConnectorTest(unittest.TestCase):
 
+    def mkdir(self):
+        d = tempfile.mkdtemp()
+        self.addCleanup(shutil.rmtree, d)
+        return d
+
     @mock.patch('jujuclient.websocket')
     def test_connect_socket(self, websocket):
         address = "wss://abc:17070"
@@ -104,6 +123,87 @@
             ValueError, Connector.split_host_port, 'I am not an ip/port combo'
         )
 
+    def test_parse_env_missing(self):
+        temp_juju_home = self.mkdir()
+        with mock.patch.dict('os.environ', {'JUJU_HOME': temp_juju_home}):
+            connector = Connector()
+            self.assertRaises(
+                EnvironmentNotBootstrapped,
+                connector.parse_env,
+                'missing')
+
+    def test_parse_env_jenv(self):
+        temp_juju_home = self.mkdir()
+        self.write_jenv(temp_juju_home, 'test-env', SAMPLE_CONFIG)
+        with mock.patch.dict('os.environ', {'JUJU_HOME': temp_juju_home}):
+            connector = Connector()
+            home, env = connector.parse_env('test-env')
+            self.assertEqual(home, temp_juju_home)
+            self.assertEqual(env, SAMPLE_CONFIG)
+
+    def test_parse_cache_file(self):
+        temp_juju_home = self.mkdir()
+        self.write_cache_file(temp_juju_home, 'test-env', SAMPLE_CONFIG)
+        with mock.patch.dict('os.environ', {'JUJU_HOME': temp_juju_home}):
+            connector = Connector()
+            home, env = connector.parse_env('test-env')
+            self.assertEqual(home, temp_juju_home)
+            self.assertEqual(env, SAMPLE_CONFIG)
+
+    def test_parse_cache_file_missing_env(self):
+        """Create a valid cache file, but look for an environment that isn't there.
+        """
+        temp_juju_home = self.mkdir()
+        self.write_cache_file(temp_juju_home, 'test-env', SAMPLE_CONFIG)
+        with mock.patch.dict('os.environ', {'JUJU_HOME': temp_juju_home}):
+            connector = Connector()
+            self.assertRaises(
+                EnvironmentNotBootstrapped,
+                connector.parse_env,
+                'missing')
+
+    def test_parse_env_cache_file_first(self):
+        """The cache file has priority over a jenv file."""
+        temp_juju_home = self.mkdir()
+        content = copy.deepcopy(SAMPLE_CONFIG)
+        self.write_jenv(temp_juju_home, 'test-env', content)
+        # Now change the password.
+        content['password'] = 'new password'
+        self.write_cache_file(temp_juju_home, 'test-env', content)
+        with mock.patch.dict('os.environ', {'JUJU_HOME': temp_juju_home}):
+            connector = Connector()
+            home, env = connector.parse_env('test-env')
+            self.assertEqual(home, temp_juju_home)
+            self.assertEqual(env, content)
+
+    def write_jenv(self, juju_home, env_name, content):
+        env_dir = os.path.join(juju_home, 'environments')
+        if not os.path.exists(env_dir):
+            os.mkdir(env_dir)
+        jenv = os.path.join(env_dir, '%s.jenv' % env_name)
+        with open(jenv, 'w') as f:
+            yaml.dump(content, f, default_flow_style=False)
+
+    def write_cache_file(self, juju_home, env_name, content):
+        env_dir = os.path.join(juju_home, 'environments')
+        if not os.path.exists(env_dir):
+            os.mkdir(env_dir)
+        filename = os.path.join(env_dir, 'cache.yaml')
+        cache_content = {
+            'environment': {
+                env_name: {'env-uuid': content['environ-uuid'],
+                           'server-uuid': content['server-uuid'],
+                           'user': content['user']}},
+            'server-data': {
+                content['server-uuid']: {
+                    'api-endpoints': content['state-servers'],
+                    'ca-cert': content['ca-cert'],
+                    'identities': {content['user']: content['password']}}},
+            # Explicitly don't care about 'server-user' here.
+            }
+        with open(filename, 'w') as f:
+            yaml.dump(cache_content, f, default_flow_style=False)
+
 
 class KeyManagerTest(unittest.TestCase):
     def setUp(self):